├── .ci ├── update_windows │ ├── update.py │ ├── update_comfyui.bat │ └── update_comfyui_stable.bat ├── windows_base_files │ ├── README_VERY_IMPORTANT.txt │ ├── run_cpu.bat │ ├── run_nvidia_gpu.bat │ └── run_nvidia_gpu_fast_fp16_accumulation.bat └── windows_nightly_base_files │ └── run_nvidia_gpu_fast.bat ├── .gitattributes ├── .github ├── ISSUE_TEMPLATE │ ├── bug-report.yml │ ├── config.yml │ ├── feature-request.yml │ └── user-support.yml └── workflows │ ├── pullrequest-ci-run.yml │ ├── ruff.yml │ ├── stable-release.yml │ ├── stale-issues.yml │ ├── test-build.yml │ ├── test-ci.yml │ ├── test-launch.yml │ ├── test-unit.yml │ ├── update-api-stubs.yml │ ├── update-version.yml │ ├── windows_release_dependencies.yml │ ├── windows_release_nightly_pytorch.yml │ └── windows_release_package.yml ├── .gitignore ├── CODEOWNERS ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── api_server ├── __init__.py ├── routes │ ├── __init__.py │ └── internal │ │ ├── README.md │ │ ├── __init__.py │ │ └── internal_routes.py ├── services │ ├── __init__.py │ └── terminal_service.py └── utils │ └── file_operations.py ├── app ├── __init__.py ├── app_settings.py ├── custom_node_manager.py ├── frontend_management.py ├── logger.py ├── model_manager.py └── user_manager.py ├── comfy ├── checkpoint_pickle.py ├── cldm │ ├── cldm.py │ ├── control_types.py │ ├── dit_embedder.py │ └── mmdit.py ├── cli_args.py ├── clip_config_bigg.json ├── clip_model.py ├── clip_vision.py ├── clip_vision_config_g.json ├── clip_vision_config_h.json ├── clip_vision_config_vitl.json ├── clip_vision_config_vitl_336.json ├── clip_vision_config_vitl_336_llava.json ├── clip_vision_siglip_384.json ├── clip_vision_siglip_512.json ├── comfy_types │ ├── README.md │ ├── __init__.py │ ├── examples │ │ ├── example_nodes.py │ │ ├── input_options.png │ │ ├── input_types.png │ │ └── required_hint.png │ └── node_typing.py ├── conds.py ├── controlnet.py ├── diffusers_convert.py ├── diffusers_load.py ├── extra_samplers │ └── uni_pc.py ├── float.py ├── gligen.py ├── hooks.py ├── image_encoders │ ├── dino2.py │ └── dino2_giant.json ├── k_diffusion │ ├── deis.py │ ├── sampling.py │ └── utils.py ├── latent_formats.py ├── ldm │ ├── ace │ │ ├── attention.py │ │ ├── lyric_encoder.py │ │ ├── model.py │ │ └── vae │ │ │ ├── autoencoder_dc.py │ │ │ ├── music_dcae_pipeline.py │ │ │ ├── music_log_mel.py │ │ │ └── music_vocoder.py │ ├── audio │ │ ├── autoencoder.py │ │ ├── dit.py │ │ └── embedders.py │ ├── aura │ │ └── mmdit.py │ ├── cascade │ │ ├── common.py │ │ ├── controlnet.py │ │ ├── stage_a.py │ │ ├── stage_b.py │ │ ├── stage_c.py │ │ └── stage_c_coder.py │ ├── chroma │ │ ├── layers.py │ │ └── model.py │ ├── common_dit.py │ ├── cosmos │ │ ├── blocks.py │ │ ├── cosmos_tokenizer │ │ │ ├── layers3d.py │ │ │ ├── patching.py │ │ │ └── utils.py │ │ ├── model.py │ │ ├── position_embedding.py │ │ └── vae.py │ ├── flux │ │ ├── controlnet.py │ │ ├── layers.py │ │ ├── math.py │ │ ├── model.py │ │ └── redux.py │ ├── genmo │ │ ├── joint_model │ │ │ ├── asymm_models_joint.py │ │ │ ├── layers.py │ │ │ ├── rope_mixed.py │ │ │ ├── temporal_rope.py │ │ │ └── utils.py │ │ └── vae │ │ │ └── model.py │ ├── hidream │ │ └── model.py │ ├── hunyuan3d │ │ ├── model.py │ │ └── vae.py │ ├── hunyuan_video │ │ └── model.py │ ├── hydit │ │ ├── attn_layers.py │ │ ├── controlnet.py │ │ ├── models.py │ │ ├── poolers.py │ │ └── posemb_layers.py │ ├── lightricks │ │ ├── model.py │ │ ├── symmetric_patchifier.py │ │ └── vae │ │ │ ├── causal_conv3d.py │ │ │ ├── causal_video_autoencoder.py │ │ │ ├── conv_nd_factory.py │ │ │ ├── dual_conv3d.py │ │ │ └── pixel_norm.py │ ├── lumina │ │ └── model.py │ ├── models │ │ └── autoencoder.py │ ├── modules │ │ ├── attention.py │ │ ├── diffusionmodules │ │ │ ├── __init__.py │ │ │ ├── mmdit.py │ │ │ ├── model.py │ │ │ ├── openaimodel.py │ │ │ ├── upscaling.py │ │ │ └── util.py │ │ ├── distributions │ │ │ ├── __init__.py │ │ │ └── distributions.py │ │ ├── ema.py │ │ ├── encoders │ │ │ ├── __init__.py │ │ │ └── noise_aug_modules.py │ │ ├── sub_quadratic_attention.py │ │ └── temporal_ae.py │ ├── pixart │ │ ├── blocks.py │ │ └── pixartms.py │ ├── util.py │ └── wan │ │ ├── model.py │ │ └── vae.py ├── lora.py ├── lora_convert.py ├── model_base.py ├── model_detection.py ├── model_management.py ├── model_patcher.py ├── model_sampling.py ├── ops.py ├── options.py ├── patcher_extension.py ├── rmsnorm.py ├── sample.py ├── sampler_helpers.py ├── samplers.py ├── sd.py ├── sd1_clip.py ├── sd1_clip_config.json ├── sd1_tokenizer │ ├── merges.txt │ ├── special_tokens_map.json │ ├── tokenizer_config.json │ └── vocab.json ├── sdxl_clip.py ├── supported_models.py ├── supported_models_base.py ├── t2i_adapter │ └── adapter.py ├── taesd │ └── taesd.py ├── text_encoders │ ├── ace.py │ ├── ace_lyrics_tokenizer │ │ └── vocab.json │ ├── ace_text_cleaners.py │ ├── aura_t5.py │ ├── bert.py │ ├── cosmos.py │ ├── flux.py │ ├── genmo.py │ ├── hidream.py │ ├── hunyuan_video.py │ ├── hydit.py │ ├── hydit_clip.json │ ├── hydit_clip_tokenizer │ │ ├── special_tokens_map.json │ │ ├── tokenizer_config.json │ │ └── vocab.txt │ ├── llama.py │ ├── llama_tokenizer │ │ ├── tokenizer.json │ │ └── tokenizer_config.json │ ├── long_clipl.py │ ├── lt.py │ ├── lumina2.py │ ├── mt5_config_xl.json │ ├── pixart_t5.py │ ├── sa_t5.py │ ├── sd2_clip.py │ ├── sd2_clip_config.json │ ├── sd3_clip.py │ ├── spiece_tokenizer.py │ ├── t5.py │ ├── t5_config_base.json │ ├── t5_config_xxl.json │ ├── t5_old_config_xxl.json │ ├── t5_pile_config_xl.json │ ├── t5_pile_tokenizer │ │ └── tokenizer.model │ ├── t5_tokenizer │ │ ├── special_tokens_map.json │ │ ├── tokenizer.json │ │ └── tokenizer_config.json │ ├── umt5_config_base.json │ ├── umt5_config_xxl.json │ └── wan.py ├── utils.py └── weight_adapter │ ├── __init__.py │ ├── base.py │ ├── boft.py │ ├── glora.py │ ├── loha.py │ ├── lokr.py │ ├── lora.py │ └── oft.py ├── comfy_api ├── input │ ├── __init__.py │ ├── basic_types.py │ └── video_types.py ├── input_impl │ ├── __init__.py │ └── video_types.py ├── torch_helpers │ ├── __init__.py │ └── torch_compile.py └── util │ ├── __init__.py │ └── video_types.py ├── comfy_api_nodes ├── README.md ├── __init__.py ├── apinode_utils.py ├── apis │ ├── PixverseController.py │ ├── PixverseDto.py │ ├── __init__.py │ ├── bfl_api.py │ ├── client.py │ ├── luma_api.py │ ├── pixverse_api.py │ ├── recraft_api.py │ ├── request_logger.py │ ├── rodin_api.py │ ├── stability_api.py │ └── tripo_api.py ├── canary.py ├── mapper_utils.py ├── nodes_bfl.py ├── nodes_gemini.py ├── nodes_ideogram.py ├── nodes_kling.py ├── nodes_luma.py ├── nodes_minimax.py ├── nodes_openai.py ├── nodes_pika.py ├── nodes_pixverse.py ├── nodes_recraft.py ├── nodes_rodin.py ├── nodes_runway.py ├── nodes_stability.py ├── nodes_tripo.py ├── nodes_veo2.py ├── redocly-dev.yaml ├── redocly.yaml └── util │ ├── __init__.py │ └── validation_utils.py ├── comfy_execution ├── caching.py ├── graph.py ├── graph_utils.py └── validation.py ├── comfy_extras ├── chainner_models │ └── model_loading.py ├── nodes_ace.py ├── nodes_advanced_samplers.py ├── nodes_align_your_steps.py ├── nodes_apg.py ├── nodes_attention_multiply.py ├── nodes_audio.py ├── nodes_camera_trajectory.py ├── nodes_canny.py ├── nodes_cfg.py ├── nodes_clip_sdxl.py ├── nodes_compositing.py ├── nodes_cond.py ├── nodes_controlnet.py ├── nodes_cosmos.py ├── nodes_custom_sampler.py ├── nodes_differential_diffusion.py ├── nodes_flux.py ├── nodes_freelunch.py ├── nodes_fresca.py ├── nodes_gits.py ├── nodes_hidream.py ├── nodes_hooks.py ├── nodes_hunyuan.py ├── nodes_hunyuan3d.py ├── nodes_hypernetwork.py ├── nodes_hypertile.py ├── nodes_images.py ├── nodes_ip2p.py ├── nodes_latent.py ├── nodes_load_3d.py ├── nodes_lora_extract.py ├── nodes_lotus.py ├── nodes_lt.py ├── nodes_lumina2.py ├── nodes_mahiro.py ├── nodes_mask.py ├── nodes_mochi.py ├── nodes_model_advanced.py ├── nodes_model_downscale.py ├── nodes_model_merging.py ├── nodes_model_merging_model_specific.py ├── nodes_morphology.py ├── nodes_optimalsteps.py ├── nodes_pag.py ├── nodes_perpneg.py ├── nodes_photomaker.py ├── nodes_pixart.py ├── nodes_post_processing.py ├── nodes_preview_any.py ├── nodes_primitive.py ├── nodes_rebatch.py ├── nodes_sag.py ├── nodes_sd3.py ├── nodes_sdupscale.py ├── nodes_slg.py ├── nodes_stable3d.py ├── nodes_stable_cascade.py ├── nodes_string.py ├── nodes_tomesd.py ├── nodes_torch_compile.py ├── nodes_upscale_model.py ├── nodes_video.py ├── nodes_video_model.py ├── nodes_wan.py └── nodes_webcam.py ├── comfyui_version.py ├── cuda_malloc.py ├── custom_nodes ├── example_node.py.example └── websocket_image_save.py ├── execution.py ├── extra_model_paths.yaml.example ├── folder_paths.py ├── hook_breaker_ac10a0.py ├── input └── example.png ├── latent_preview.py ├── main.py ├── models ├── checkpoints │ └── put_checkpoints_here ├── clip │ └── put_clip_or_text_encoder_models_here ├── clip_vision │ └── put_clip_vision_models_here ├── configs │ ├── anything_v3.yaml │ ├── v1-inference.yaml │ ├── v1-inference_clip_skip_2.yaml │ ├── v1-inference_clip_skip_2_fp16.yaml │ ├── v1-inference_fp16.yaml │ ├── v1-inpainting-inference.yaml │ ├── v2-inference-v.yaml │ ├── v2-inference-v_fp32.yaml │ ├── v2-inference.yaml │ ├── v2-inference_fp32.yaml │ └── v2-inpainting-inference.yaml ├── controlnet │ └── put_controlnets_and_t2i_here ├── diffusers │ └── put_diffusers_models_here ├── diffusion_models │ └── put_diffusion_model_files_here ├── embeddings │ └── put_embeddings_or_textual_inversion_concepts_here ├── gligen │ └── put_gligen_models_here ├── hypernetworks │ └── put_hypernetworks_here ├── loras │ └── put_loras_here ├── photomaker │ └── put_photomaker_models_here ├── style_models │ └── put_t2i_style_model_here ├── text_encoders │ └── put_text_encoder_files_here ├── unet │ └── put_unet_files_here ├── upscale_models │ └── put_esrgan_and_other_upscale_models_here ├── vae │ └── put_vae_here └── vae_approx │ └── put_taesd_encoder_pth_and_taesd_decoder_pth_here ├── new_updater.py ├── node_helpers.py ├── nodes.py ├── notebooks └── comfyui_colab.ipynb ├── output └── _output_images_will_be_put_here ├── pyproject.toml ├── pytest.ini ├── requirements.txt ├── script_examples ├── basic_api_example.py ├── websockets_api_example.py └── websockets_api_example_ws_images.py ├── server.py ├── tests-unit ├── README.md ├── app_test │ ├── __init__.py │ ├── custom_node_manager_test.py │ ├── frontend_manager_test.py │ └── model_manager_test.py ├── comfy_api_nodes_test │ └── mapper_utils_test.py ├── comfy_api_test │ ├── input_impl_test.py │ └── video_types_test.py ├── comfy_extras_test │ ├── __init__.py │ └── image_stitch_test.py ├── comfy_test │ └── folder_path_test.py ├── execution_test │ └── validate_node_input_test.py ├── folder_paths_test │ ├── __init__.py │ └── filter_by_content_types_test.py ├── prompt_server_test │ ├── __init__.py │ └── user_manager_test.py ├── requirements.txt ├── server │ └── utils │ │ └── file_operations_test.py └── utils │ ├── extra_config_test.py │ └── json_util_test.py ├── tests ├── README.md ├── __init__.py ├── compare │ ├── conftest.py │ └── test_quality.py ├── conftest.py └── inference │ ├── __init__.py │ ├── extra_model_paths.yaml │ ├── graphs │ └── default_graph_sdxl1_0.json │ ├── test_execution.py │ ├── test_inference.py │ └── testing_nodes │ └── testing-pack │ ├── __init__.py │ ├── conditions.py │ ├── flow_control.py │ ├── specific_tests.py │ ├── stubs.py │ ├── tools.py │ └── util.py └── utils ├── __init__.py ├── extra_config.py └── json_util.py /.ci/update_windows/update_comfyui.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | ..\python_embeded\python.exe .\update.py ..\ComfyUI\ 3 | if exist update_new.py ( 4 | move /y update_new.py update.py 5 | echo Running updater again since it got updated. 6 | ..\python_embeded\python.exe .\update.py ..\ComfyUI\ --skip_self_update 7 | ) 8 | if "%~1"=="" pause 9 | -------------------------------------------------------------------------------- /.ci/update_windows/update_comfyui_stable.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | ..\python_embeded\python.exe .\update.py ..\ComfyUI\ --stable 3 | if exist update_new.py ( 4 | move /y update_new.py update.py 5 | echo Running updater again since it got updated. 6 | ..\python_embeded\python.exe .\update.py ..\ComfyUI\ --skip_self_update --stable 7 | ) 8 | if "%~1"=="" pause 9 | -------------------------------------------------------------------------------- /.ci/windows_base_files/README_VERY_IMPORTANT.txt: -------------------------------------------------------------------------------- 1 | HOW TO RUN: 2 | 3 | if you have a NVIDIA gpu: 4 | 5 | run_nvidia_gpu.bat 6 | 7 | 8 | 9 | To run it in slow CPU mode: 10 | 11 | run_cpu.bat 12 | 13 | 14 | 15 | IF YOU GET A RED ERROR IN THE UI MAKE SURE YOU HAVE A MODEL/CHECKPOINT IN: ComfyUI\models\checkpoints 16 | 17 | You can download the stable diffusion 1.5 one from: https://huggingface.co/Comfy-Org/stable-diffusion-v1-5-archive/blob/main/v1-5-pruned-emaonly-fp16.safetensors 18 | 19 | 20 | RECOMMENDED WAY TO UPDATE: 21 | To update the ComfyUI code: update\update_comfyui.bat 22 | 23 | 24 | 25 | To update ComfyUI with the python dependencies, note that you should ONLY run this if you have issues with python dependencies. 26 | update\update_comfyui_and_python_dependencies.bat 27 | 28 | 29 | TO SHARE MODELS BETWEEN COMFYUI AND ANOTHER UI: 30 | In the ComfyUI directory you will find a file: extra_model_paths.yaml.example 31 | Rename this file to: extra_model_paths.yaml and edit it with your favorite text editor. 32 | -------------------------------------------------------------------------------- /.ci/windows_base_files/run_cpu.bat: -------------------------------------------------------------------------------- 1 | .\python_embeded\python.exe -s ComfyUI\main.py --cpu --windows-standalone-build 2 | pause 3 | -------------------------------------------------------------------------------- /.ci/windows_base_files/run_nvidia_gpu.bat: -------------------------------------------------------------------------------- 1 | .\python_embeded\python.exe -s ComfyUI\main.py --windows-standalone-build 2 | pause 3 | -------------------------------------------------------------------------------- /.ci/windows_base_files/run_nvidia_gpu_fast_fp16_accumulation.bat: -------------------------------------------------------------------------------- 1 | .\python_embeded\python.exe -s ComfyUI\main.py --windows-standalone-build --fast fp16_accumulation 2 | pause 3 | -------------------------------------------------------------------------------- /.ci/windows_nightly_base_files/run_nvidia_gpu_fast.bat: -------------------------------------------------------------------------------- 1 | .\python_embeded\python.exe -s ComfyUI\main.py --windows-standalone-build --fast 2 | pause 3 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | /web/assets/** linguist-generated 2 | /web/** linguist-vendored 3 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug-report.yml: -------------------------------------------------------------------------------- 1 | name: Bug Report 2 | description: "Something is broken inside of ComfyUI. (Do not use this if you're just having issues and need help, or if the issue relates to a custom node)" 3 | labels: ["Potential Bug"] 4 | body: 5 | - type: markdown 6 | attributes: 7 | value: | 8 | Before submitting a **Bug Report**, please ensure the following: 9 | 10 | - **1:** You are running the latest version of ComfyUI. 11 | - **2:** You have looked at the existing bug reports and made sure this isn't already reported. 12 | - **3:** You confirmed that the bug is not caused by a custom node. You can disable all custom nodes by passing 13 | `--disable-all-custom-nodes` command line argument. 14 | - **4:** This is an actual bug in ComfyUI, not just a support question. A bug is when you can specify exact 15 | steps to replicate what went wrong and others will be able to repeat your steps and see the same issue happen. 16 | 17 | If unsure, ask on the [ComfyUI Matrix Space](https://app.element.io/#/room/%23comfyui_space%3Amatrix.org) or the [Comfy Org Discord](https://discord.gg/comfyorg) first. 18 | - type: checkboxes 19 | id: custom-nodes-test 20 | attributes: 21 | label: Custom Node Testing 22 | description: Please confirm you have tried to reproduce the issue with all custom nodes disabled. 23 | options: 24 | - label: I have tried disabling custom nodes and the issue persists (see [how to disable custom nodes](https://docs.comfy.org/troubleshooting/custom-node-issues#step-1%3A-test-with-all-custom-nodes-disabled) if you need help) 25 | required: true 26 | - type: textarea 27 | attributes: 28 | label: Expected Behavior 29 | description: "What you expected to happen." 30 | validations: 31 | required: true 32 | - type: textarea 33 | attributes: 34 | label: Actual Behavior 35 | description: "What actually happened. Please include a screenshot of the issue if possible." 36 | validations: 37 | required: true 38 | - type: textarea 39 | attributes: 40 | label: Steps to Reproduce 41 | description: "Describe how to reproduce the issue. Please be sure to attach a workflow JSON or PNG, ideally one that doesn't require custom nodes to test. If the bug open happens when certain custom nodes are used, most likely that custom node is what has the bug rather than ComfyUI, in which case it should be reported to the node's author." 42 | validations: 43 | required: true 44 | - type: textarea 45 | attributes: 46 | label: Debug Logs 47 | description: "Please copy the output from your terminal logs here." 48 | render: powershell 49 | validations: 50 | required: true 51 | - type: textarea 52 | attributes: 53 | label: Other 54 | description: "Any other additional information you think might be helpful." 55 | validations: 56 | required: false 57 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: true 2 | contact_links: 3 | - name: ComfyUI Frontend Issues 4 | url: https://github.com/Comfy-Org/ComfyUI_frontend/issues 5 | about: Issues related to the ComfyUI frontend (display issues, user interaction bugs), please go to the frontend repo to file the issue 6 | - name: ComfyUI Matrix Space 7 | url: https://app.element.io/#/room/%23comfyui_space%3Amatrix.org 8 | about: The ComfyUI Matrix Space is available for support and general discussion related to ComfyUI (Matrix is like Discord but open source). 9 | - name: Comfy Org Discord 10 | url: https://discord.gg/comfyorg 11 | about: The Comfy Org Discord is available for support and general discussion related to ComfyUI. 12 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature-request.yml: -------------------------------------------------------------------------------- 1 | name: Feature Request 2 | description: "You have an idea for something new you would like to see added to ComfyUI's core." 3 | labels: [ "Feature" ] 4 | body: 5 | - type: markdown 6 | attributes: 7 | value: | 8 | Before submitting a **Feature Request**, please ensure the following: 9 | 10 | **1:** You are running the latest version of ComfyUI. 11 | **2:** You have looked to make sure there is not already a feature that does what you need, and there is not already a Feature Request listed for the same idea. 12 | **3:** This is something that makes sense to add to ComfyUI Core, and wouldn't make more sense as a custom node. 13 | 14 | If unsure, ask on the [ComfyUI Matrix Space](https://app.element.io/#/room/%23comfyui_space%3Amatrix.org) or the [Comfy Org Discord](https://discord.gg/comfyorg) first. 15 | - type: textarea 16 | attributes: 17 | label: Feature Idea 18 | description: "Describe the feature you want to see." 19 | validations: 20 | required: true 21 | - type: textarea 22 | attributes: 23 | label: Existing Solutions 24 | description: "Please search through available custom nodes / extensions to see if there are existing custom solutions for this. If so, please link the options you found here as a reference." 25 | validations: 26 | required: false 27 | - type: textarea 28 | attributes: 29 | label: Other 30 | description: "Any other additional information you think might be helpful." 31 | validations: 32 | required: false 33 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/user-support.yml: -------------------------------------------------------------------------------- 1 | name: User Support 2 | description: "Use this if you need help with something, or you're experiencing an issue." 3 | labels: [ "User Support" ] 4 | body: 5 | - type: markdown 6 | attributes: 7 | value: | 8 | Before submitting a **User Report** issue, please ensure the following: 9 | 10 | **1:** You are running the latest version of ComfyUI. 11 | **2:** You have made an effort to find public answers to your question before asking here. In other words, you googled it first, and scrolled through recent help topics. 12 | 13 | If unsure, ask on the [ComfyUI Matrix Space](https://app.element.io/#/room/%23comfyui_space%3Amatrix.org) or the [Comfy Org Discord](https://discord.gg/comfyorg) first. 14 | - type: checkboxes 15 | id: custom-nodes-test 16 | attributes: 17 | label: Custom Node Testing 18 | description: Please confirm you have tried to reproduce the issue with all custom nodes disabled. 19 | options: 20 | - label: I have tried disabling custom nodes and the issue persists (see [how to disable custom nodes](https://docs.comfy.org/troubleshooting/custom-node-issues#step-1%3A-test-with-all-custom-nodes-disabled) if you need help) 21 | required: true 22 | - type: textarea 23 | attributes: 24 | label: Your question 25 | description: "Post your question here. Please be as detailed as possible." 26 | validations: 27 | required: true 28 | - type: textarea 29 | attributes: 30 | label: Logs 31 | description: "If your question relates to an issue you're experiencing, please go to `Server` -> `Logs` -> potentially set `View Type` to `Debug` as well, then copypaste all the text into here." 32 | render: powershell 33 | validations: 34 | required: false 35 | - type: textarea 36 | attributes: 37 | label: Other 38 | description: "Any other additional information you think might be helpful." 39 | validations: 40 | required: false 41 | -------------------------------------------------------------------------------- /.github/workflows/pullrequest-ci-run.yml: -------------------------------------------------------------------------------- 1 | # This is the GitHub Workflow that drives full-GPU-enabled tests of pull requests to ComfyUI, when the 'Run-CI-Test' label is added 2 | # Results are reported as checkmarks on the commits, as well as onto https://ci.comfy.org/ 3 | name: Pull Request CI Workflow Runs 4 | on: 5 | pull_request_target: 6 | types: [labeled] 7 | 8 | jobs: 9 | pr-test-stable: 10 | if: ${{ github.event.label.name == 'Run-CI-Test' }} 11 | strategy: 12 | fail-fast: false 13 | matrix: 14 | os: [macos, linux, windows] 15 | python_version: ["3.9", "3.10", "3.11", "3.12"] 16 | cuda_version: ["12.1"] 17 | torch_version: ["stable"] 18 | include: 19 | - os: macos 20 | runner_label: [self-hosted, macOS] 21 | flags: "--use-pytorch-cross-attention" 22 | - os: linux 23 | runner_label: [self-hosted, Linux] 24 | flags: "" 25 | - os: windows 26 | runner_label: [self-hosted, Windows] 27 | flags: "" 28 | runs-on: ${{ matrix.runner_label }} 29 | steps: 30 | - name: Test Workflows 31 | uses: comfy-org/comfy-action@main 32 | with: 33 | os: ${{ matrix.os }} 34 | python_version: ${{ matrix.python_version }} 35 | torch_version: ${{ matrix.torch_version }} 36 | google_credentials: ${{ secrets.GCS_SERVICE_ACCOUNT_JSON }} 37 | comfyui_flags: ${{ matrix.flags }} 38 | use_prior_commit: 'true' 39 | comment: 40 | if: ${{ github.event.label.name == 'Run-CI-Test' }} 41 | runs-on: ubuntu-latest 42 | permissions: 43 | pull-requests: write 44 | steps: 45 | - uses: actions/github-script@v6 46 | with: 47 | script: | 48 | github.rest.issues.createComment({ 49 | issue_number: context.issue.number, 50 | owner: context.repo.owner, 51 | repo: context.repo.repo, 52 | body: '(Automated Bot Message) CI Tests are running, you can view the results at https://ci.comfy.org/?branch=${{ github.event.pull_request.number }}%2Fmerge' 53 | }) 54 | -------------------------------------------------------------------------------- /.github/workflows/ruff.yml: -------------------------------------------------------------------------------- 1 | name: Python Linting 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | ruff: 7 | name: Run Ruff 8 | runs-on: ubuntu-latest 9 | 10 | steps: 11 | - name: Checkout repository 12 | uses: actions/checkout@v4 13 | 14 | - name: Set up Python 15 | uses: actions/setup-python@v2 16 | with: 17 | python-version: 3.x 18 | 19 | - name: Install Ruff 20 | run: pip install ruff 21 | 22 | - name: Run Ruff 23 | run: ruff check . 24 | -------------------------------------------------------------------------------- /.github/workflows/stale-issues.yml: -------------------------------------------------------------------------------- 1 | name: 'Close stale issues' 2 | on: 3 | schedule: 4 | # Run daily at 430 am PT 5 | - cron: '30 11 * * *' 6 | permissions: 7 | issues: write 8 | 9 | jobs: 10 | stale: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/stale@v9 14 | with: 15 | stale-issue-message: "This issue is being marked stale because it has not had any activity for 30 days. Reply below within 7 days if your issue still isn't solved, and it will be left open. Otherwise, the issue will be closed automatically." 16 | days-before-stale: 30 17 | days-before-close: 7 18 | stale-issue-label: 'Stale' 19 | only-labels: 'User Support' 20 | exempt-all-assignees: true 21 | exempt-all-milestones: true 22 | -------------------------------------------------------------------------------- /.github/workflows/test-build.yml: -------------------------------------------------------------------------------- 1 | name: Build package 2 | 3 | # 4 | # This workflow is a test of the python package build. 5 | # Install Python dependencies across different Python versions. 6 | # 7 | 8 | on: 9 | push: 10 | paths: 11 | - "requirements.txt" 12 | - ".github/workflows/test-build.yml" 13 | 14 | jobs: 15 | build: 16 | name: Build Test 17 | runs-on: ubuntu-latest 18 | strategy: 19 | fail-fast: false 20 | matrix: 21 | python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] 22 | steps: 23 | - uses: actions/checkout@v4 24 | - name: Set up Python ${{ matrix.python-version }} 25 | uses: actions/setup-python@v4 26 | with: 27 | python-version: ${{ matrix.python-version }} 28 | - name: Install dependencies 29 | run: | 30 | python -m pip install --upgrade pip 31 | pip install -r requirements.txt 32 | -------------------------------------------------------------------------------- /.github/workflows/test-launch.yml: -------------------------------------------------------------------------------- 1 | name: Test server launches without errors 2 | 3 | on: 4 | push: 5 | branches: [ main, master ] 6 | pull_request: 7 | branches: [ main, master ] 8 | 9 | jobs: 10 | test: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - name: Checkout ComfyUI 14 | uses: actions/checkout@v4 15 | with: 16 | repository: "comfyanonymous/ComfyUI" 17 | path: "ComfyUI" 18 | - uses: actions/setup-python@v4 19 | with: 20 | python-version: '3.10' 21 | - name: Install requirements 22 | run: | 23 | python -m pip install --upgrade pip 24 | pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu 25 | pip install -r requirements.txt 26 | pip install wait-for-it 27 | working-directory: ComfyUI 28 | - name: Start ComfyUI server 29 | run: | 30 | python main.py --cpu 2>&1 | tee console_output.log & 31 | wait-for-it --service 127.0.0.1:8188 -t 30 32 | working-directory: ComfyUI 33 | - name: Check for unhandled exceptions in server log 34 | run: | 35 | if grep -qE "Exception|Error" console_output.log; then 36 | echo "Unhandled exception/error found in server log." 37 | exit 1 38 | fi 39 | working-directory: ComfyUI 40 | - uses: actions/upload-artifact@v4 41 | if: always() 42 | with: 43 | name: console-output 44 | path: ComfyUI/console_output.log 45 | retention-days: 30 46 | -------------------------------------------------------------------------------- /.github/workflows/test-unit.yml: -------------------------------------------------------------------------------- 1 | name: Unit Tests 2 | 3 | on: 4 | push: 5 | branches: [ main, master ] 6 | pull_request: 7 | branches: [ main, master ] 8 | 9 | jobs: 10 | test: 11 | strategy: 12 | matrix: 13 | os: [ubuntu-latest, windows-latest, macos-latest] 14 | runs-on: ${{ matrix.os }} 15 | continue-on-error: true 16 | steps: 17 | - uses: actions/checkout@v4 18 | - name: Set up Python 19 | uses: actions/setup-python@v4 20 | with: 21 | python-version: '3.12' 22 | - name: Install requirements 23 | run: | 24 | python -m pip install --upgrade pip 25 | pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu 26 | pip install -r requirements.txt 27 | - name: Run Unit Tests 28 | run: | 29 | pip install -r tests-unit/requirements.txt 30 | python -m pytest tests-unit 31 | -------------------------------------------------------------------------------- /.github/workflows/update-api-stubs.yml: -------------------------------------------------------------------------------- 1 | name: Generate Pydantic Stubs from api.comfy.org 2 | 3 | on: 4 | schedule: 5 | - cron: '0 0 * * 1' 6 | workflow_dispatch: 7 | 8 | jobs: 9 | generate-models: 10 | runs-on: ubuntu-latest 11 | 12 | steps: 13 | - name: Checkout repository 14 | uses: actions/checkout@v4 15 | 16 | - name: Set up Python 17 | uses: actions/setup-python@v4 18 | with: 19 | python-version: '3.10' 20 | 21 | - name: Install dependencies 22 | run: | 23 | python -m pip install --upgrade pip 24 | pip install 'datamodel-code-generator[http]' 25 | npm install @redocly/cli 26 | 27 | - name: Download OpenAPI spec 28 | run: | 29 | curl -o openapi.yaml https://api.comfy.org/openapi 30 | 31 | - name: Filter OpenAPI spec with Redocly 32 | run: | 33 | npx @redocly/cli bundle openapi.yaml --output filtered-openapi.yaml --config comfy_api_nodes/redocly.yaml --remove-unused-components 34 | 35 | - name: Generate API models 36 | run: | 37 | datamodel-codegen --use-subclass-enum --input filtered-openapi.yaml --output comfy_api_nodes/apis --output-model-type pydantic_v2.BaseModel 38 | 39 | - name: Check for changes 40 | id: git-check 41 | run: | 42 | git diff --exit-code comfy_api_nodes/apis || echo "changes=true" >> $GITHUB_OUTPUT 43 | 44 | - name: Create Pull Request 45 | if: steps.git-check.outputs.changes == 'true' 46 | uses: peter-evans/create-pull-request@v5 47 | with: 48 | commit-message: 'chore: update API models from OpenAPI spec' 49 | title: 'Update API models from api.comfy.org' 50 | body: | 51 | This PR updates the API models based on the latest api.comfy.org OpenAPI specification. 52 | 53 | Generated automatically by the a Github workflow. 54 | branch: update-api-stubs 55 | delete-branch: true 56 | base: master 57 | -------------------------------------------------------------------------------- /.github/workflows/update-version.yml: -------------------------------------------------------------------------------- 1 | name: Update Version File 2 | 3 | on: 4 | pull_request: 5 | paths: 6 | - "pyproject.toml" 7 | branches: 8 | - master 9 | 10 | jobs: 11 | update-version: 12 | runs-on: ubuntu-latest 13 | # Don't run on fork PRs 14 | if: github.event.pull_request.head.repo.full_name == github.repository 15 | permissions: 16 | pull-requests: write 17 | contents: write 18 | 19 | steps: 20 | - name: Checkout repository 21 | uses: actions/checkout@v4 22 | 23 | - name: Set up Python 24 | uses: actions/setup-python@v4 25 | with: 26 | python-version: "3.11" 27 | 28 | - name: Install dependencies 29 | run: | 30 | python -m pip install --upgrade pip 31 | 32 | - name: Update comfyui_version.py 33 | run: | 34 | # Read version from pyproject.toml and update comfyui_version.py 35 | python -c ' 36 | import tomllib 37 | 38 | # Read version from pyproject.toml 39 | with open("pyproject.toml", "rb") as f: 40 | config = tomllib.load(f) 41 | version = config["project"]["version"] 42 | 43 | # Write version to comfyui_version.py 44 | with open("comfyui_version.py", "w") as f: 45 | f.write("# This file is automatically generated by the build process when version is\n") 46 | f.write("# updated in pyproject.toml.\n") 47 | f.write(f"__version__ = \"{version}\"\n") 48 | ' 49 | 50 | - name: Commit changes 51 | run: | 52 | git config --local user.name "github-actions" 53 | git config --local user.email "github-actions@github.com" 54 | git fetch origin ${{ github.head_ref }} 55 | git checkout -B ${{ github.head_ref }} origin/${{ github.head_ref }} 56 | git add comfyui_version.py 57 | git diff --quiet && git diff --staged --quiet || git commit -m "chore: Update comfyui_version.py to match pyproject.toml" 58 | git push origin HEAD:${{ github.head_ref }} 59 | -------------------------------------------------------------------------------- /.github/workflows/windows_release_dependencies.yml: -------------------------------------------------------------------------------- 1 | name: "Windows Release dependencies" 2 | 3 | on: 4 | workflow_dispatch: 5 | inputs: 6 | xformers: 7 | description: 'xformers version' 8 | required: false 9 | type: string 10 | default: "" 11 | extra_dependencies: 12 | description: 'extra dependencies' 13 | required: false 14 | type: string 15 | default: "" 16 | cu: 17 | description: 'cuda version' 18 | required: true 19 | type: string 20 | default: "128" 21 | 22 | python_minor: 23 | description: 'python minor version' 24 | required: true 25 | type: string 26 | default: "12" 27 | 28 | python_patch: 29 | description: 'python patch version' 30 | required: true 31 | type: string 32 | default: "10" 33 | # push: 34 | # branches: 35 | # - master 36 | 37 | jobs: 38 | build_dependencies: 39 | runs-on: windows-latest 40 | steps: 41 | - uses: actions/checkout@v4 42 | - uses: actions/setup-python@v5 43 | with: 44 | python-version: 3.${{ inputs.python_minor }}.${{ inputs.python_patch }} 45 | 46 | - shell: bash 47 | run: | 48 | echo "@echo off 49 | call update_comfyui.bat nopause 50 | echo - 51 | echo This will try to update pytorch and all python dependencies. 52 | echo - 53 | echo If you just want to update normally, close this and run update_comfyui.bat instead. 54 | echo - 55 | pause 56 | ..\python_embeded\python.exe -s -m pip install --upgrade torch torchvision torchaudio ${{ inputs.xformers }} --extra-index-url https://download.pytorch.org/whl/cu${{ inputs.cu }} -r ../ComfyUI/requirements.txt pygit2 57 | pause" > update_comfyui_and_python_dependencies.bat 58 | 59 | python -m pip wheel --no-cache-dir torch torchvision torchaudio ${{ inputs.xformers }} ${{ inputs.extra_dependencies }} --extra-index-url https://download.pytorch.org/whl/cu${{ inputs.cu }} -r requirements.txt pygit2 -w ./temp_wheel_dir 60 | python -m pip install --no-cache-dir ./temp_wheel_dir/* 61 | echo installed basic 62 | ls -lah temp_wheel_dir 63 | mv temp_wheel_dir cu${{ inputs.cu }}_python_deps 64 | tar cf cu${{ inputs.cu }}_python_deps.tar cu${{ inputs.cu }}_python_deps 65 | 66 | - uses: actions/cache/save@v4 67 | with: 68 | path: | 69 | cu${{ inputs.cu }}_python_deps.tar 70 | update_comfyui_and_python_dependencies.bat 71 | key: ${{ runner.os }}-build-cu${{ inputs.cu }}-${{ inputs.python_minor }} 72 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/ 2 | *.py[cod] 3 | /output/ 4 | /input/ 5 | !/input/example.png 6 | /models/ 7 | /temp/ 8 | /custom_nodes/ 9 | !custom_nodes/example_node.py.example 10 | extra_model_paths.yaml 11 | /.vs 12 | .vscode/ 13 | .idea/ 14 | venv/ 15 | .venv/ 16 | /web/extensions/* 17 | !/web/extensions/logging.js.example 18 | !/web/extensions/core/ 19 | /tests-ui/data/object_info.json 20 | /user/ 21 | *.log 22 | web_custom_versions/ 23 | .DS_Store 24 | openapi.yaml 25 | filtered-openapi.yaml 26 | uv.lock 27 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | # Admins 2 | * @comfyanonymous 3 | 4 | # Note: Github teams syntax cannot be used here as the repo is not owned by Comfy-Org. 5 | # Inlined the team members for now. 6 | 7 | # Maintainers 8 | *.md @yoland68 @robinjhuang @webfiltered @pythongosssss @ltdrdata @Kosinkadink @christian-byrne 9 | /tests/ @yoland68 @robinjhuang @webfiltered @pythongosssss @ltdrdata @Kosinkadink @christian-byrne 10 | /tests-unit/ @yoland68 @robinjhuang @webfiltered @pythongosssss @ltdrdata @Kosinkadink @christian-byrne 11 | /notebooks/ @yoland68 @robinjhuang @webfiltered @pythongosssss @ltdrdata @Kosinkadink @christian-byrne 12 | /script_examples/ @yoland68 @robinjhuang @webfiltered @pythongosssss @ltdrdata @Kosinkadink @christian-byrne 13 | /.github/ @yoland68 @robinjhuang @webfiltered @pythongosssss @ltdrdata @Kosinkadink @christian-byrne 14 | /requirements.txt @yoland68 @robinjhuang @webfiltered @pythongosssss @ltdrdata @Kosinkadink @christian-byrne 15 | /pyproject.toml @yoland68 @robinjhuang @webfiltered @pythongosssss @ltdrdata @Kosinkadink @christian-byrne 16 | 17 | # Python web server 18 | /api_server/ @yoland68 @robinjhuang @webfiltered @pythongosssss @ltdrdata @christian-byrne 19 | /app/ @yoland68 @robinjhuang @webfiltered @pythongosssss @ltdrdata @christian-byrne 20 | /utils/ @yoland68 @robinjhuang @webfiltered @pythongosssss @ltdrdata @christian-byrne 21 | 22 | # Node developers 23 | /comfy_extras/ @yoland68 @robinjhuang @pythongosssss @ltdrdata @Kosinkadink @webfiltered @christian-byrne 24 | /comfy/comfy_types/ @yoland68 @robinjhuang @pythongosssss @ltdrdata @Kosinkadink @webfiltered @christian-byrne 25 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to ComfyUI 2 | 3 | Welcome, and thank you for your interest in contributing to ComfyUI! 4 | 5 | There are several ways in which you can contribute, beyond writing code. The goal of this document is to provide a high-level overview of how you can get involved. 6 | 7 | ## Asking Questions 8 | 9 | Have a question? Instead of opening an issue, please ask on [Discord](https://comfy.org/discord) or [Matrix](https://app.element.io/#/room/%23comfyui_space%3Amatrix.org) channels. Our team and the community will help you. 10 | 11 | ## Providing Feedback 12 | 13 | Your comments and feedback are welcome, and the development team is available via a handful of different channels. 14 | 15 | See the `#bug-report`, `#feature-request` and `#feedback` channels on Discord. 16 | 17 | ## Reporting Issues 18 | 19 | Have you identified a reproducible problem in ComfyUI? Do you have a feature request? We want to hear about it! Here's how you can report your issue as effectively as possible. 20 | 21 | 22 | ### Look For an Existing Issue 23 | 24 | Before you create a new issue, please do a search in [open issues](https://github.com/comfyanonymous/ComfyUI/issues) to see if the issue or feature request has already been filed. 25 | 26 | If you find your issue already exists, make relevant comments and add your [reaction](https://github.com/blog/2119-add-reactions-to-pull-requests-issues-and-comments). Use a reaction in place of a "+1" comment: 27 | 28 | * 👍 - upvote 29 | * 👎 - downvote 30 | 31 | If you cannot find an existing issue that describes your bug or feature, create a new issue. We have an issue template in place to organize new issues. 32 | 33 | 34 | ### Creating Pull Requests 35 | 36 | * Please refer to the article on [creating pull requests](https://github.com/comfyanonymous/ComfyUI/wiki/How-to-Contribute-Code) and contributing to this project. 37 | 38 | 39 | ## Thank You 40 | 41 | Your contributions to open source, large or small, make great projects like this possible. Thank you for taking the time to contribute. 42 | -------------------------------------------------------------------------------- /api_server/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/comfyanonymous/ComfyUI/312d511630db4907c3bed04dee297b28f61941a8/api_server/__init__.py -------------------------------------------------------------------------------- /api_server/routes/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/comfyanonymous/ComfyUI/312d511630db4907c3bed04dee297b28f61941a8/api_server/routes/__init__.py -------------------------------------------------------------------------------- /api_server/routes/internal/README.md: -------------------------------------------------------------------------------- 1 | # ComfyUI Internal Routes 2 | 3 | All routes under the `/internal` path are designated for **internal use by ComfyUI only**. These routes are not intended for use by external applications may change at any time without notice. 4 | -------------------------------------------------------------------------------- /api_server/routes/internal/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/comfyanonymous/ComfyUI/312d511630db4907c3bed04dee297b28f61941a8/api_server/routes/internal/__init__.py -------------------------------------------------------------------------------- /api_server/services/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/comfyanonymous/ComfyUI/312d511630db4907c3bed04dee297b28f61941a8/api_server/services/__init__.py -------------------------------------------------------------------------------- /api_server/services/terminal_service.py: -------------------------------------------------------------------------------- 1 | from app.logger import on_flush 2 | import os 3 | import shutil 4 | 5 | 6 | class TerminalService: 7 | def __init__(self, server): 8 | self.server = server 9 | self.cols = None 10 | self.rows = None 11 | self.subscriptions = set() 12 | on_flush(self.send_messages) 13 | 14 | def get_terminal_size(self): 15 | try: 16 | size = os.get_terminal_size() 17 | return (size.columns, size.lines) 18 | except OSError: 19 | try: 20 | size = shutil.get_terminal_size() 21 | return (size.columns, size.lines) 22 | except OSError: 23 | return (80, 24) # fallback to 80x24 24 | 25 | def update_size(self): 26 | columns, lines = self.get_terminal_size() 27 | changed = False 28 | 29 | if columns != self.cols: 30 | self.cols = columns 31 | changed = True 32 | 33 | if lines != self.rows: 34 | self.rows = lines 35 | changed = True 36 | 37 | if changed: 38 | return {"cols": self.cols, "rows": self.rows} 39 | 40 | return None 41 | 42 | def subscribe(self, client_id): 43 | self.subscriptions.add(client_id) 44 | 45 | def unsubscribe(self, client_id): 46 | self.subscriptions.discard(client_id) 47 | 48 | def send_messages(self, entries): 49 | if not len(entries) or not len(self.subscriptions): 50 | return 51 | 52 | new_size = self.update_size() 53 | 54 | for client_id in self.subscriptions.copy(): # prevent: Set changed size during iteration 55 | if client_id not in self.server.sockets: 56 | # Automatically unsub if the socket has disconnected 57 | self.unsubscribe(client_id) 58 | continue 59 | 60 | self.server.send_sync("logs", {"entries": entries, "size": new_size}, client_id) 61 | -------------------------------------------------------------------------------- /api_server/utils/file_operations.py: -------------------------------------------------------------------------------- 1 | import os 2 | from typing import List, Union, TypedDict, Literal 3 | from typing_extensions import TypeGuard 4 | class FileInfo(TypedDict): 5 | name: str 6 | path: str 7 | type: Literal["file"] 8 | size: int 9 | 10 | class DirectoryInfo(TypedDict): 11 | name: str 12 | path: str 13 | type: Literal["directory"] 14 | 15 | FileSystemItem = Union[FileInfo, DirectoryInfo] 16 | 17 | def is_file_info(item: FileSystemItem) -> TypeGuard[FileInfo]: 18 | return item["type"] == "file" 19 | 20 | class FileSystemOperations: 21 | @staticmethod 22 | def walk_directory(directory: str) -> List[FileSystemItem]: 23 | file_list: List[FileSystemItem] = [] 24 | for root, dirs, files in os.walk(directory): 25 | for name in files: 26 | file_path = os.path.join(root, name) 27 | relative_path = os.path.relpath(file_path, directory) 28 | file_list.append({ 29 | "name": name, 30 | "path": relative_path, 31 | "type": "file", 32 | "size": os.path.getsize(file_path) 33 | }) 34 | for name in dirs: 35 | dir_path = os.path.join(root, name) 36 | relative_path = os.path.relpath(dir_path, directory) 37 | file_list.append({ 38 | "name": name, 39 | "path": relative_path, 40 | "type": "directory" 41 | }) 42 | return file_list 43 | -------------------------------------------------------------------------------- /app/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/comfyanonymous/ComfyUI/312d511630db4907c3bed04dee297b28f61941a8/app/__init__.py -------------------------------------------------------------------------------- /app/app_settings.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | from aiohttp import web 4 | import logging 5 | 6 | 7 | class AppSettings(): 8 | def __init__(self, user_manager): 9 | self.user_manager = user_manager 10 | 11 | def get_settings(self, request): 12 | try: 13 | file = self.user_manager.get_request_user_filepath( 14 | request, 15 | "comfy.settings.json" 16 | ) 17 | except KeyError as e: 18 | logging.error("User settings not found.") 19 | raise web.HTTPUnauthorized() from e 20 | if os.path.isfile(file): 21 | try: 22 | with open(file) as f: 23 | return json.load(f) 24 | except: 25 | logging.error(f"The user settings file is corrupted: {file}") 26 | return {} 27 | else: 28 | return {} 29 | 30 | def save_settings(self, request, settings): 31 | file = self.user_manager.get_request_user_filepath( 32 | request, "comfy.settings.json") 33 | with open(file, "w") as f: 34 | f.write(json.dumps(settings, indent=4)) 35 | 36 | def add_routes(self, routes): 37 | @routes.get("/settings") 38 | async def get_settings(request): 39 | return web.json_response(self.get_settings(request)) 40 | 41 | @routes.get("/settings/{id}") 42 | async def get_setting(request): 43 | value = None 44 | settings = self.get_settings(request) 45 | setting_id = request.match_info.get("id", None) 46 | if setting_id and setting_id in settings: 47 | value = settings[setting_id] 48 | return web.json_response(value) 49 | 50 | @routes.post("/settings") 51 | async def post_settings(request): 52 | settings = self.get_settings(request) 53 | new_settings = await request.json() 54 | self.save_settings(request, {**settings, **new_settings}) 55 | return web.Response(status=200) 56 | 57 | @routes.post("/settings/{id}") 58 | async def post_setting(request): 59 | setting_id = request.match_info.get("id", None) 60 | if not setting_id: 61 | return web.Response(status=400) 62 | settings = self.get_settings(request) 63 | settings[setting_id] = await request.json() 64 | self.save_settings(request, settings) 65 | return web.Response(status=200) 66 | -------------------------------------------------------------------------------- /comfy/checkpoint_pickle.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | 3 | load = pickle.load 4 | 5 | class Empty: 6 | pass 7 | 8 | class Unpickler(pickle.Unpickler): 9 | def find_class(self, module, name): 10 | #TODO: safe unpickle 11 | if module.startswith("pytorch_lightning"): 12 | return Empty 13 | return super().find_class(module, name) 14 | -------------------------------------------------------------------------------- /comfy/cldm/control_types.py: -------------------------------------------------------------------------------- 1 | UNION_CONTROLNET_TYPES = { 2 | "openpose": 0, 3 | "depth": 1, 4 | "hed/pidi/scribble/ted": 2, 5 | "canny/lineart/anime_lineart/mlsd": 3, 6 | "normal": 4, 7 | "segment": 5, 8 | "tile": 6, 9 | "repaint": 7, 10 | } 11 | -------------------------------------------------------------------------------- /comfy/cldm/mmdit.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from typing import Optional 3 | import comfy.ldm.modules.diffusionmodules.mmdit 4 | 5 | class ControlNet(comfy.ldm.modules.diffusionmodules.mmdit.MMDiT): 6 | def __init__( 7 | self, 8 | num_blocks = None, 9 | control_latent_channels = None, 10 | dtype = None, 11 | device = None, 12 | operations = None, 13 | **kwargs, 14 | ): 15 | super().__init__(dtype=dtype, device=device, operations=operations, final_layer=False, num_blocks=num_blocks, **kwargs) 16 | # controlnet_blocks 17 | self.controlnet_blocks = torch.nn.ModuleList([]) 18 | for _ in range(len(self.joint_blocks)): 19 | self.controlnet_blocks.append(operations.Linear(self.hidden_size, self.hidden_size, device=device, dtype=dtype)) 20 | 21 | if control_latent_channels is None: 22 | control_latent_channels = self.in_channels 23 | 24 | self.pos_embed_input = comfy.ldm.modules.diffusionmodules.mmdit.PatchEmbed( 25 | None, 26 | self.patch_size, 27 | control_latent_channels, 28 | self.hidden_size, 29 | bias=True, 30 | strict_img_size=False, 31 | dtype=dtype, 32 | device=device, 33 | operations=operations 34 | ) 35 | 36 | def forward( 37 | self, 38 | x: torch.Tensor, 39 | timesteps: torch.Tensor, 40 | y: Optional[torch.Tensor] = None, 41 | context: Optional[torch.Tensor] = None, 42 | hint = None, 43 | ) -> torch.Tensor: 44 | 45 | #weird sd3 controlnet specific stuff 46 | y = torch.zeros_like(y) 47 | 48 | if self.context_processor is not None: 49 | context = self.context_processor(context) 50 | 51 | hw = x.shape[-2:] 52 | x = self.x_embedder(x) + self.cropped_pos_embed(hw, device=x.device).to(dtype=x.dtype, device=x.device) 53 | x += self.pos_embed_input(hint) 54 | 55 | c = self.t_embedder(timesteps, dtype=x.dtype) 56 | if y is not None and self.y_embedder is not None: 57 | y = self.y_embedder(y) 58 | c = c + y 59 | 60 | if context is not None: 61 | context = self.context_embedder(context) 62 | 63 | output = [] 64 | 65 | blocks = len(self.joint_blocks) 66 | for i in range(blocks): 67 | context, x = self.joint_blocks[i]( 68 | context, 69 | x, 70 | c=c, 71 | use_checkpoint=self.use_checkpoint, 72 | ) 73 | 74 | out = self.controlnet_blocks[i](x) 75 | count = self.depth // blocks 76 | if i == blocks - 1: 77 | count -= 1 78 | for j in range(count): 79 | output.append(out) 80 | 81 | return {"output": output} 82 | -------------------------------------------------------------------------------- /comfy/clip_config_bigg.json: -------------------------------------------------------------------------------- 1 | { 2 | "architectures": [ 3 | "CLIPTextModel" 4 | ], 5 | "attention_dropout": 0.0, 6 | "bos_token_id": 0, 7 | "dropout": 0.0, 8 | "eos_token_id": 49407, 9 | "hidden_act": "gelu", 10 | "hidden_size": 1280, 11 | "initializer_factor": 1.0, 12 | "initializer_range": 0.02, 13 | "intermediate_size": 5120, 14 | "layer_norm_eps": 1e-05, 15 | "max_position_embeddings": 77, 16 | "model_type": "clip_text_model", 17 | "num_attention_heads": 20, 18 | "num_hidden_layers": 32, 19 | "pad_token_id": 1, 20 | "projection_dim": 1280, 21 | "torch_dtype": "float32", 22 | "vocab_size": 49408 23 | } 24 | -------------------------------------------------------------------------------- /comfy/clip_vision_config_g.json: -------------------------------------------------------------------------------- 1 | { 2 | "attention_dropout": 0.0, 3 | "dropout": 0.0, 4 | "hidden_act": "gelu", 5 | "hidden_size": 1664, 6 | "image_size": 224, 7 | "initializer_factor": 1.0, 8 | "initializer_range": 0.02, 9 | "intermediate_size": 8192, 10 | "layer_norm_eps": 1e-05, 11 | "model_type": "clip_vision_model", 12 | "num_attention_heads": 16, 13 | "num_channels": 3, 14 | "num_hidden_layers": 48, 15 | "patch_size": 14, 16 | "projection_dim": 1280, 17 | "torch_dtype": "float32" 18 | } 19 | -------------------------------------------------------------------------------- /comfy/clip_vision_config_h.json: -------------------------------------------------------------------------------- 1 | { 2 | "attention_dropout": 0.0, 3 | "dropout": 0.0, 4 | "hidden_act": "gelu", 5 | "hidden_size": 1280, 6 | "image_size": 224, 7 | "initializer_factor": 1.0, 8 | "initializer_range": 0.02, 9 | "intermediate_size": 5120, 10 | "layer_norm_eps": 1e-05, 11 | "model_type": "clip_vision_model", 12 | "num_attention_heads": 16, 13 | "num_channels": 3, 14 | "num_hidden_layers": 32, 15 | "patch_size": 14, 16 | "projection_dim": 1024, 17 | "torch_dtype": "float32" 18 | } 19 | -------------------------------------------------------------------------------- /comfy/clip_vision_config_vitl.json: -------------------------------------------------------------------------------- 1 | { 2 | "attention_dropout": 0.0, 3 | "dropout": 0.0, 4 | "hidden_act": "quick_gelu", 5 | "hidden_size": 1024, 6 | "image_size": 224, 7 | "initializer_factor": 1.0, 8 | "initializer_range": 0.02, 9 | "intermediate_size": 4096, 10 | "layer_norm_eps": 1e-05, 11 | "model_type": "clip_vision_model", 12 | "num_attention_heads": 16, 13 | "num_channels": 3, 14 | "num_hidden_layers": 24, 15 | "patch_size": 14, 16 | "projection_dim": 768, 17 | "torch_dtype": "float32" 18 | } 19 | -------------------------------------------------------------------------------- /comfy/clip_vision_config_vitl_336.json: -------------------------------------------------------------------------------- 1 | { 2 | "attention_dropout": 0.0, 3 | "dropout": 0.0, 4 | "hidden_act": "quick_gelu", 5 | "hidden_size": 1024, 6 | "image_size": 336, 7 | "initializer_factor": 1.0, 8 | "initializer_range": 0.02, 9 | "intermediate_size": 4096, 10 | "layer_norm_eps": 1e-5, 11 | "model_type": "clip_vision_model", 12 | "num_attention_heads": 16, 13 | "num_channels": 3, 14 | "num_hidden_layers": 24, 15 | "patch_size": 14, 16 | "projection_dim": 768, 17 | "torch_dtype": "float32" 18 | } 19 | -------------------------------------------------------------------------------- /comfy/clip_vision_config_vitl_336_llava.json: -------------------------------------------------------------------------------- 1 | { 2 | "attention_dropout": 0.0, 3 | "dropout": 0.0, 4 | "hidden_act": "quick_gelu", 5 | "hidden_size": 1024, 6 | "image_size": 336, 7 | "initializer_factor": 1.0, 8 | "initializer_range": 0.02, 9 | "intermediate_size": 4096, 10 | "layer_norm_eps": 1e-5, 11 | "model_type": "clip_vision_model", 12 | "num_attention_heads": 16, 13 | "num_channels": 3, 14 | "num_hidden_layers": 24, 15 | "patch_size": 14, 16 | "projection_dim": 768, 17 | "projector_type": "llava3", 18 | "torch_dtype": "float32" 19 | } 20 | -------------------------------------------------------------------------------- /comfy/clip_vision_siglip_384.json: -------------------------------------------------------------------------------- 1 | { 2 | "num_channels": 3, 3 | "hidden_act": "gelu_pytorch_tanh", 4 | "hidden_size": 1152, 5 | "image_size": 384, 6 | "intermediate_size": 4304, 7 | "model_type": "siglip_vision_model", 8 | "num_attention_heads": 16, 9 | "num_hidden_layers": 27, 10 | "patch_size": 14, 11 | "image_mean": [0.5, 0.5, 0.5], 12 | "image_std": [0.5, 0.5, 0.5] 13 | } 14 | -------------------------------------------------------------------------------- /comfy/clip_vision_siglip_512.json: -------------------------------------------------------------------------------- 1 | { 2 | "num_channels": 3, 3 | "hidden_act": "gelu_pytorch_tanh", 4 | "hidden_size": 1152, 5 | "image_size": 512, 6 | "intermediate_size": 4304, 7 | "model_type": "siglip_vision_model", 8 | "num_attention_heads": 16, 9 | "num_hidden_layers": 27, 10 | "patch_size": 16, 11 | "image_mean": [0.5, 0.5, 0.5], 12 | "image_std": [0.5, 0.5, 0.5] 13 | } 14 | -------------------------------------------------------------------------------- /comfy/comfy_types/README.md: -------------------------------------------------------------------------------- 1 | # Comfy Typing 2 | ## Type hinting for ComfyUI Node development 3 | 4 | This module provides type hinting and concrete convenience types for node developers. 5 | If cloned to the custom_nodes directory of ComfyUI, types can be imported using: 6 | 7 | ```python 8 | from comfy.comfy_types import IO, ComfyNodeABC, CheckLazyMixin 9 | 10 | class ExampleNode(ComfyNodeABC): 11 | @classmethod 12 | def INPUT_TYPES(s) -> InputTypeDict: 13 | return {"required": {}} 14 | ``` 15 | 16 | Full example is in [examples/example_nodes.py](examples/example_nodes.py). 17 | 18 | # Types 19 | A few primary types are documented below. More complete information is available via the docstrings on each type. 20 | 21 | ## `IO` 22 | 23 | A string enum of built-in and a few custom data types. Includes the following special types and their requisite plumbing: 24 | 25 | - `ANY`: `"*"` 26 | - `NUMBER`: `"FLOAT,INT"` 27 | - `PRIMITIVE`: `"STRING,FLOAT,INT,BOOLEAN"` 28 | 29 | ## `ComfyNodeABC` 30 | 31 | An abstract base class for nodes, offering type-hinting / autocomplete, and somewhat-alright docstrings. 32 | 33 | ### Type hinting for `INPUT_TYPES` 34 | 35 | ![INPUT_TYPES auto-completion in Visual Studio Code](examples/input_types.png) 36 | 37 | ### `INPUT_TYPES` return dict 38 | 39 | ![INPUT_TYPES return value type hinting in Visual Studio Code](examples/required_hint.png) 40 | 41 | ### Options for individual inputs 42 | 43 | ![INPUT_TYPES return value option auto-completion in Visual Studio Code](examples/input_options.png) 44 | -------------------------------------------------------------------------------- /comfy/comfy_types/__init__.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from typing import Callable, Protocol, TypedDict, Optional, List 3 | from .node_typing import IO, InputTypeDict, ComfyNodeABC, CheckLazyMixin, FileLocator 4 | 5 | 6 | class UnetApplyFunction(Protocol): 7 | """Function signature protocol on comfy.model_base.BaseModel.apply_model""" 8 | 9 | def __call__(self, x: torch.Tensor, t: torch.Tensor, **kwargs) -> torch.Tensor: 10 | pass 11 | 12 | 13 | class UnetApplyConds(TypedDict): 14 | """Optional conditions for unet apply function.""" 15 | 16 | c_concat: Optional[torch.Tensor] 17 | c_crossattn: Optional[torch.Tensor] 18 | control: Optional[torch.Tensor] 19 | transformer_options: Optional[dict] 20 | 21 | 22 | class UnetParams(TypedDict): 23 | # Tensor of shape [B, C, H, W] 24 | input: torch.Tensor 25 | # Tensor of shape [B] 26 | timestep: torch.Tensor 27 | c: UnetApplyConds 28 | # List of [0, 1], [0], [1], ... 29 | # 0 means conditional, 1 means conditional unconditional 30 | cond_or_uncond: List[int] 31 | 32 | 33 | UnetWrapperFunction = Callable[[UnetApplyFunction, UnetParams], torch.Tensor] 34 | 35 | 36 | __all__ = [ 37 | "UnetWrapperFunction", 38 | UnetApplyConds.__name__, 39 | UnetParams.__name__, 40 | UnetApplyFunction.__name__, 41 | IO.__name__, 42 | InputTypeDict.__name__, 43 | ComfyNodeABC.__name__, 44 | CheckLazyMixin.__name__, 45 | FileLocator.__name__, 46 | ] 47 | -------------------------------------------------------------------------------- /comfy/comfy_types/examples/example_nodes.py: -------------------------------------------------------------------------------- 1 | from comfy.comfy_types import IO, ComfyNodeABC, InputTypeDict 2 | from inspect import cleandoc 3 | 4 | 5 | class ExampleNode(ComfyNodeABC): 6 | """An example node that just adds 1 to an input integer. 7 | 8 | * Requires a modern IDE to provide any benefit (detail: an IDE configured with analysis paths etc). 9 | * This node is intended as an example for developers only. 10 | """ 11 | 12 | DESCRIPTION = cleandoc(__doc__) 13 | CATEGORY = "examples" 14 | 15 | @classmethod 16 | def INPUT_TYPES(s) -> InputTypeDict: 17 | return { 18 | "required": { 19 | "input_int": (IO.INT, {"defaultInput": True}), 20 | } 21 | } 22 | 23 | RETURN_TYPES = (IO.INT,) 24 | RETURN_NAMES = ("input_plus_one",) 25 | FUNCTION = "execute" 26 | 27 | def execute(self, input_int: int): 28 | return (input_int + 1,) 29 | -------------------------------------------------------------------------------- /comfy/comfy_types/examples/input_options.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/comfyanonymous/ComfyUI/312d511630db4907c3bed04dee297b28f61941a8/comfy/comfy_types/examples/input_options.png -------------------------------------------------------------------------------- /comfy/comfy_types/examples/input_types.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/comfyanonymous/ComfyUI/312d511630db4907c3bed04dee297b28f61941a8/comfy/comfy_types/examples/input_types.png -------------------------------------------------------------------------------- /comfy/comfy_types/examples/required_hint.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/comfyanonymous/ComfyUI/312d511630db4907c3bed04dee297b28f61941a8/comfy/comfy_types/examples/required_hint.png -------------------------------------------------------------------------------- /comfy/diffusers_load.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import comfy.sd 4 | 5 | def first_file(path, filenames): 6 | for f in filenames: 7 | p = os.path.join(path, f) 8 | if os.path.exists(p): 9 | return p 10 | return None 11 | 12 | def load_diffusers(model_path, output_vae=True, output_clip=True, embedding_directory=None): 13 | diffusion_model_names = ["diffusion_pytorch_model.fp16.safetensors", "diffusion_pytorch_model.safetensors", "diffusion_pytorch_model.fp16.bin", "diffusion_pytorch_model.bin"] 14 | unet_path = first_file(os.path.join(model_path, "unet"), diffusion_model_names) 15 | vae_path = first_file(os.path.join(model_path, "vae"), diffusion_model_names) 16 | 17 | text_encoder_model_names = ["model.fp16.safetensors", "model.safetensors", "pytorch_model.fp16.bin", "pytorch_model.bin"] 18 | text_encoder1_path = first_file(os.path.join(model_path, "text_encoder"), text_encoder_model_names) 19 | text_encoder2_path = first_file(os.path.join(model_path, "text_encoder_2"), text_encoder_model_names) 20 | 21 | text_encoder_paths = [text_encoder1_path] 22 | if text_encoder2_path is not None: 23 | text_encoder_paths.append(text_encoder2_path) 24 | 25 | unet = comfy.sd.load_diffusion_model(unet_path) 26 | 27 | clip = None 28 | if output_clip: 29 | clip = comfy.sd.load_clip(text_encoder_paths, embedding_directory=embedding_directory) 30 | 31 | vae = None 32 | if output_vae: 33 | sd = comfy.utils.load_torch_file(vae_path) 34 | vae = comfy.sd.VAE(sd=sd) 35 | 36 | return (unet, clip, vae) 37 | -------------------------------------------------------------------------------- /comfy/float.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | def calc_mantissa(abs_x, exponent, normal_mask, MANTISSA_BITS, EXPONENT_BIAS, generator=None): 4 | mantissa_scaled = torch.where( 5 | normal_mask, 6 | (abs_x / (2.0 ** (exponent - EXPONENT_BIAS)) - 1.0) * (2**MANTISSA_BITS), 7 | (abs_x / (2.0 ** (-EXPONENT_BIAS + 1 - MANTISSA_BITS))) 8 | ) 9 | 10 | mantissa_scaled += torch.rand(mantissa_scaled.size(), dtype=mantissa_scaled.dtype, layout=mantissa_scaled.layout, device=mantissa_scaled.device, generator=generator) 11 | return mantissa_scaled.floor() / (2**MANTISSA_BITS) 12 | 13 | #Not 100% sure about this 14 | def manual_stochastic_round_to_float8(x, dtype, generator=None): 15 | if dtype == torch.float8_e4m3fn: 16 | EXPONENT_BITS, MANTISSA_BITS, EXPONENT_BIAS = 4, 3, 7 17 | elif dtype == torch.float8_e5m2: 18 | EXPONENT_BITS, MANTISSA_BITS, EXPONENT_BIAS = 5, 2, 15 19 | else: 20 | raise ValueError("Unsupported dtype") 21 | 22 | x = x.half() 23 | sign = torch.sign(x) 24 | abs_x = x.abs() 25 | sign = torch.where(abs_x == 0, 0, sign) 26 | 27 | # Combine exponent calculation and clamping 28 | exponent = torch.clamp( 29 | torch.floor(torch.log2(abs_x)) + EXPONENT_BIAS, 30 | 0, 2**EXPONENT_BITS - 1 31 | ) 32 | 33 | # Combine mantissa calculation and rounding 34 | normal_mask = ~(exponent == 0) 35 | 36 | abs_x[:] = calc_mantissa(abs_x, exponent, normal_mask, MANTISSA_BITS, EXPONENT_BIAS, generator=generator) 37 | 38 | sign *= torch.where( 39 | normal_mask, 40 | (2.0 ** (exponent - EXPONENT_BIAS)) * (1.0 + abs_x), 41 | (2.0 ** (-EXPONENT_BIAS + 1)) * abs_x 42 | ) 43 | 44 | inf = torch.finfo(dtype) 45 | torch.clamp(sign, min=inf.min, max=inf.max, out=sign) 46 | return sign 47 | 48 | 49 | 50 | def stochastic_rounding(value, dtype, seed=0): 51 | if dtype == torch.float32: 52 | return value.to(dtype=torch.float32) 53 | if dtype == torch.float16: 54 | return value.to(dtype=torch.float16) 55 | if dtype == torch.bfloat16: 56 | return value.to(dtype=torch.bfloat16) 57 | if dtype == torch.float8_e4m3fn or dtype == torch.float8_e5m2: 58 | generator = torch.Generator(device=value.device) 59 | generator.manual_seed(seed) 60 | output = torch.empty_like(value, dtype=dtype) 61 | num_slices = max(1, (value.numel() / (4096 * 4096))) 62 | slice_size = max(1, round(value.shape[0] / num_slices)) 63 | for i in range(0, value.shape[0], slice_size): 64 | output[i:i+slice_size].copy_(manual_stochastic_round_to_float8(value[i:i+slice_size], dtype, generator=generator)) 65 | return output 66 | 67 | return value.to(dtype=dtype) 68 | -------------------------------------------------------------------------------- /comfy/image_encoders/dino2_giant.json: -------------------------------------------------------------------------------- 1 | { 2 | "attention_probs_dropout_prob": 0.0, 3 | "drop_path_rate": 0.0, 4 | "hidden_act": "gelu", 5 | "hidden_dropout_prob": 0.0, 6 | "hidden_size": 1536, 7 | "image_size": 518, 8 | "initializer_range": 0.02, 9 | "layer_norm_eps": 1e-06, 10 | "layerscale_value": 1.0, 11 | "mlp_ratio": 4, 12 | "model_type": "dinov2", 13 | "num_attention_heads": 24, 14 | "num_channels": 3, 15 | "num_hidden_layers": 40, 16 | "patch_size": 14, 17 | "qkv_bias": true, 18 | "use_swiglu_ffn": true, 19 | "image_mean": [0.485, 0.456, 0.406], 20 | "image_std": [0.229, 0.224, 0.225] 21 | } 22 | -------------------------------------------------------------------------------- /comfy/ldm/common_dit.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import comfy.rmsnorm 3 | 4 | 5 | def pad_to_patch_size(img, patch_size=(2, 2), padding_mode="circular"): 6 | if padding_mode == "circular" and (torch.jit.is_tracing() or torch.jit.is_scripting()): 7 | padding_mode = "reflect" 8 | 9 | pad = () 10 | for i in range(img.ndim - 2): 11 | pad = (0, (patch_size[i] - img.shape[i + 2] % patch_size[i]) % patch_size[i]) + pad 12 | 13 | return torch.nn.functional.pad(img, pad, mode=padding_mode) 14 | 15 | 16 | rms_norm = comfy.rmsnorm.rms_norm 17 | -------------------------------------------------------------------------------- /comfy/ldm/flux/math.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from einops import rearrange 3 | from torch import Tensor 4 | 5 | from comfy.ldm.modules.attention import optimized_attention 6 | import comfy.model_management 7 | 8 | 9 | def attention(q: Tensor, k: Tensor, v: Tensor, pe: Tensor, mask=None) -> Tensor: 10 | q_shape = q.shape 11 | k_shape = k.shape 12 | 13 | if pe is not None: 14 | q = q.to(dtype=pe.dtype).reshape(*q.shape[:-1], -1, 1, 2) 15 | k = k.to(dtype=pe.dtype).reshape(*k.shape[:-1], -1, 1, 2) 16 | q = (pe[..., 0] * q[..., 0] + pe[..., 1] * q[..., 1]).reshape(*q_shape).type_as(v) 17 | k = (pe[..., 0] * k[..., 0] + pe[..., 1] * k[..., 1]).reshape(*k_shape).type_as(v) 18 | 19 | heads = q.shape[1] 20 | x = optimized_attention(q, k, v, heads, skip_reshape=True, mask=mask) 21 | return x 22 | 23 | 24 | def rope(pos: Tensor, dim: int, theta: int) -> Tensor: 25 | assert dim % 2 == 0 26 | if comfy.model_management.is_device_mps(pos.device) or comfy.model_management.is_intel_xpu() or comfy.model_management.is_directml_enabled(): 27 | device = torch.device("cpu") 28 | else: 29 | device = pos.device 30 | 31 | scale = torch.linspace(0, (dim - 2) / dim, steps=dim//2, dtype=torch.float64, device=device) 32 | omega = 1.0 / (theta**scale) 33 | out = torch.einsum("...n,d->...nd", pos.to(dtype=torch.float32, device=device), omega) 34 | out = torch.stack([torch.cos(out), -torch.sin(out), torch.sin(out), torch.cos(out)], dim=-1) 35 | out = rearrange(out, "b n d (i j) -> b n d i j", i=2, j=2) 36 | return out.to(dtype=torch.float32, device=pos.device) 37 | 38 | 39 | def apply_rope(xq: Tensor, xk: Tensor, freqs_cis: Tensor): 40 | xq_ = xq.to(dtype=freqs_cis.dtype).reshape(*xq.shape[:-1], -1, 1, 2) 41 | xk_ = xk.to(dtype=freqs_cis.dtype).reshape(*xk.shape[:-1], -1, 1, 2) 42 | xq_out = freqs_cis[..., 0] * xq_[..., 0] + freqs_cis[..., 1] * xq_[..., 1] 43 | xk_out = freqs_cis[..., 0] * xk_[..., 0] + freqs_cis[..., 1] * xk_[..., 1] 44 | return xq_out.reshape(*xq.shape).type_as(xq), xk_out.reshape(*xk.shape).type_as(xk) 45 | 46 | -------------------------------------------------------------------------------- /comfy/ldm/flux/redux.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import comfy.ops 3 | 4 | ops = comfy.ops.manual_cast 5 | 6 | class ReduxImageEncoder(torch.nn.Module): 7 | def __init__( 8 | self, 9 | redux_dim: int = 1152, 10 | txt_in_features: int = 4096, 11 | device=None, 12 | dtype=None, 13 | ) -> None: 14 | super().__init__() 15 | 16 | self.redux_dim = redux_dim 17 | self.device = device 18 | self.dtype = dtype 19 | 20 | self.redux_up = ops.Linear(redux_dim, txt_in_features * 3, dtype=dtype) 21 | self.redux_down = ops.Linear(txt_in_features * 3, txt_in_features, dtype=dtype) 22 | 23 | def forward(self, sigclip_embeds) -> torch.Tensor: 24 | projected_x = self.redux_down(torch.nn.functional.silu(self.redux_up(sigclip_embeds))) 25 | return projected_x 26 | -------------------------------------------------------------------------------- /comfy/ldm/genmo/joint_model/temporal_rope.py: -------------------------------------------------------------------------------- 1 | #original code from https://github.com/genmoai/models under apache 2.0 license 2 | 3 | # Based on Llama3 Implementation. 4 | import torch 5 | 6 | 7 | def apply_rotary_emb_qk_real( 8 | xqk: torch.Tensor, 9 | freqs_cos: torch.Tensor, 10 | freqs_sin: torch.Tensor, 11 | ) -> torch.Tensor: 12 | """ 13 | Apply rotary embeddings to input tensors using the given frequency tensor without complex numbers. 14 | 15 | Args: 16 | xqk (torch.Tensor): Query and/or Key tensors to apply rotary embeddings. Shape: (B, S, *, num_heads, D) 17 | Can be either just query or just key, or both stacked along some batch or * dim. 18 | freqs_cos (torch.Tensor): Precomputed cosine frequency tensor. 19 | freqs_sin (torch.Tensor): Precomputed sine frequency tensor. 20 | 21 | Returns: 22 | torch.Tensor: The input tensor with rotary embeddings applied. 23 | """ 24 | # Split the last dimension into even and odd parts 25 | xqk_even = xqk[..., 0::2] 26 | xqk_odd = xqk[..., 1::2] 27 | 28 | # Apply rotation 29 | cos_part = (xqk_even * freqs_cos - xqk_odd * freqs_sin).type_as(xqk) 30 | sin_part = (xqk_even * freqs_sin + xqk_odd * freqs_cos).type_as(xqk) 31 | 32 | # Interleave the results back into the original shape 33 | out = torch.stack([cos_part, sin_part], dim=-1).flatten(-2) 34 | return out 35 | -------------------------------------------------------------------------------- /comfy/ldm/hydit/poolers.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from comfy.ldm.modules.attention import optimized_attention 4 | import comfy.ops 5 | 6 | class AttentionPool(nn.Module): 7 | def __init__(self, spacial_dim: int, embed_dim: int, num_heads: int, output_dim: int = None, dtype=None, device=None, operations=None): 8 | super().__init__() 9 | self.positional_embedding = nn.Parameter(torch.empty(spacial_dim + 1, embed_dim, dtype=dtype, device=device)) 10 | self.k_proj = operations.Linear(embed_dim, embed_dim, dtype=dtype, device=device) 11 | self.q_proj = operations.Linear(embed_dim, embed_dim, dtype=dtype, device=device) 12 | self.v_proj = operations.Linear(embed_dim, embed_dim, dtype=dtype, device=device) 13 | self.c_proj = operations.Linear(embed_dim, output_dim or embed_dim, dtype=dtype, device=device) 14 | self.num_heads = num_heads 15 | self.embed_dim = embed_dim 16 | 17 | def forward(self, x): 18 | x = x[:,:self.positional_embedding.shape[0] - 1] 19 | x = x.permute(1, 0, 2) # NLC -> LNC 20 | x = torch.cat([x.mean(dim=0, keepdim=True), x], dim=0) # (L+1)NC 21 | x = x + comfy.ops.cast_to_input(self.positional_embedding[:, None, :], x) # (L+1)NC 22 | 23 | q = self.q_proj(x[:1]) 24 | k = self.k_proj(x) 25 | v = self.v_proj(x) 26 | 27 | batch_size = q.shape[1] 28 | head_dim = self.embed_dim // self.num_heads 29 | q = q.view(1, batch_size * self.num_heads, head_dim).transpose(0, 1).view(batch_size, self.num_heads, -1, head_dim) 30 | k = k.view(k.shape[0], batch_size * self.num_heads, head_dim).transpose(0, 1).view(batch_size, self.num_heads, -1, head_dim) 31 | v = v.view(v.shape[0], batch_size * self.num_heads, head_dim).transpose(0, 1).view(batch_size, self.num_heads, -1, head_dim) 32 | 33 | attn_output = optimized_attention(q, k, v, self.num_heads, skip_reshape=True).transpose(0, 1) 34 | 35 | attn_output = self.c_proj(attn_output) 36 | return attn_output.squeeze(0) 37 | -------------------------------------------------------------------------------- /comfy/ldm/lightricks/vae/causal_conv3d.py: -------------------------------------------------------------------------------- 1 | from typing import Tuple, Union 2 | 3 | import torch 4 | import torch.nn as nn 5 | import comfy.ops 6 | ops = comfy.ops.disable_weight_init 7 | 8 | 9 | class CausalConv3d(nn.Module): 10 | def __init__( 11 | self, 12 | in_channels, 13 | out_channels, 14 | kernel_size: int = 3, 15 | stride: Union[int, Tuple[int]] = 1, 16 | dilation: int = 1, 17 | groups: int = 1, 18 | spatial_padding_mode: str = "zeros", 19 | **kwargs, 20 | ): 21 | super().__init__() 22 | 23 | self.in_channels = in_channels 24 | self.out_channels = out_channels 25 | 26 | kernel_size = (kernel_size, kernel_size, kernel_size) 27 | self.time_kernel_size = kernel_size[0] 28 | 29 | dilation = (dilation, 1, 1) 30 | 31 | height_pad = kernel_size[1] // 2 32 | width_pad = kernel_size[2] // 2 33 | padding = (0, height_pad, width_pad) 34 | 35 | self.conv = ops.Conv3d( 36 | in_channels, 37 | out_channels, 38 | kernel_size, 39 | stride=stride, 40 | dilation=dilation, 41 | padding=padding, 42 | padding_mode=spatial_padding_mode, 43 | groups=groups, 44 | ) 45 | 46 | def forward(self, x, causal: bool = True): 47 | if causal: 48 | first_frame_pad = x[:, :, :1, :, :].repeat( 49 | (1, 1, self.time_kernel_size - 1, 1, 1) 50 | ) 51 | x = torch.concatenate((first_frame_pad, x), dim=2) 52 | else: 53 | first_frame_pad = x[:, :, :1, :, :].repeat( 54 | (1, 1, (self.time_kernel_size - 1) // 2, 1, 1) 55 | ) 56 | last_frame_pad = x[:, :, -1:, :, :].repeat( 57 | (1, 1, (self.time_kernel_size - 1) // 2, 1, 1) 58 | ) 59 | x = torch.concatenate((first_frame_pad, x, last_frame_pad), dim=2) 60 | x = self.conv(x) 61 | return x 62 | 63 | @property 64 | def weight(self): 65 | return self.conv.weight 66 | -------------------------------------------------------------------------------- /comfy/ldm/lightricks/vae/pixel_norm.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch import nn 3 | 4 | 5 | class PixelNorm(nn.Module): 6 | def __init__(self, dim=1, eps=1e-8): 7 | super(PixelNorm, self).__init__() 8 | self.dim = dim 9 | self.eps = eps 10 | 11 | def forward(self, x): 12 | return x / torch.sqrt(torch.mean(x**2, dim=self.dim, keepdim=True) + self.eps) 13 | -------------------------------------------------------------------------------- /comfy/ldm/modules/diffusionmodules/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/comfyanonymous/ComfyUI/312d511630db4907c3bed04dee297b28f61941a8/comfy/ldm/modules/diffusionmodules/__init__.py -------------------------------------------------------------------------------- /comfy/ldm/modules/distributions/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/comfyanonymous/ComfyUI/312d511630db4907c3bed04dee297b28f61941a8/comfy/ldm/modules/distributions/__init__.py -------------------------------------------------------------------------------- /comfy/ldm/modules/encoders/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/comfyanonymous/ComfyUI/312d511630db4907c3bed04dee297b28f61941a8/comfy/ldm/modules/encoders/__init__.py -------------------------------------------------------------------------------- /comfy/ldm/modules/encoders/noise_aug_modules.py: -------------------------------------------------------------------------------- 1 | from ..diffusionmodules.upscaling import ImageConcatWithNoiseAugmentation 2 | from ..diffusionmodules.openaimodel import Timestep 3 | import torch 4 | 5 | class CLIPEmbeddingNoiseAugmentation(ImageConcatWithNoiseAugmentation): 6 | def __init__(self, *args, clip_stats_path=None, timestep_dim=256, **kwargs): 7 | super().__init__(*args, **kwargs) 8 | if clip_stats_path is None: 9 | clip_mean, clip_std = torch.zeros(timestep_dim), torch.ones(timestep_dim) 10 | else: 11 | clip_mean, clip_std = torch.load(clip_stats_path, map_location="cpu") 12 | self.register_buffer("data_mean", clip_mean[None, :], persistent=False) 13 | self.register_buffer("data_std", clip_std[None, :], persistent=False) 14 | self.time_embed = Timestep(timestep_dim) 15 | 16 | def scale(self, x): 17 | # re-normalize to centered mean and unit variance 18 | x = (x - self.data_mean.to(x.device)) * 1. / self.data_std.to(x.device) 19 | return x 20 | 21 | def unscale(self, x): 22 | # back to original data stats 23 | x = (x * self.data_std.to(x.device)) + self.data_mean.to(x.device) 24 | return x 25 | 26 | def forward(self, x, noise_level=None, seed=None): 27 | if noise_level is None: 28 | noise_level = torch.randint(0, self.max_noise_level, (x.shape[0],), device=x.device).long() 29 | else: 30 | assert isinstance(noise_level, torch.Tensor) 31 | x = self.scale(x) 32 | z = self.q_sample(x, noise_level, seed=seed) 33 | z = self.unscale(z) 34 | noise_level = self.time_embed(noise_level) 35 | return z, noise_level 36 | -------------------------------------------------------------------------------- /comfy/lora_convert.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import comfy.utils 3 | 4 | 5 | def convert_lora_bfl_control(sd): #BFL loras for Flux 6 | sd_out = {} 7 | for k in sd: 8 | k_to = "diffusion_model.{}".format(k.replace(".lora_B.bias", ".diff_b").replace("_norm.scale", "_norm.scale.set_weight")) 9 | sd_out[k_to] = sd[k] 10 | 11 | sd_out["diffusion_model.img_in.reshape_weight"] = torch.tensor([sd["img_in.lora_B.weight"].shape[0], sd["img_in.lora_A.weight"].shape[1]]) 12 | return sd_out 13 | 14 | 15 | def convert_lora_wan_fun(sd): #Wan Fun loras 16 | return comfy.utils.state_dict_prefix_replace(sd, {"lora_unet__": "lora_unet_"}) 17 | 18 | 19 | def convert_lora(sd): 20 | if "img_in.lora_A.weight" in sd and "single_blocks.0.norm.key_norm.scale" in sd: 21 | return convert_lora_bfl_control(sd) 22 | if "lora_unet__blocks_0_cross_attn_k.lora_down.weight" in sd: 23 | return convert_lora_wan_fun(sd) 24 | return sd 25 | -------------------------------------------------------------------------------- /comfy/options.py: -------------------------------------------------------------------------------- 1 | 2 | args_parsing = False 3 | 4 | def enable_args_parsing(enable=True): 5 | global args_parsing 6 | args_parsing = enable 7 | -------------------------------------------------------------------------------- /comfy/rmsnorm.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import comfy.model_management 3 | import numbers 4 | 5 | RMSNorm = None 6 | 7 | try: 8 | rms_norm_torch = torch.nn.functional.rms_norm 9 | RMSNorm = torch.nn.RMSNorm 10 | except: 11 | rms_norm_torch = None 12 | 13 | 14 | def rms_norm(x, weight=None, eps=1e-6): 15 | if rms_norm_torch is not None and not (torch.jit.is_tracing() or torch.jit.is_scripting()): 16 | if weight is None: 17 | return rms_norm_torch(x, (x.shape[-1],), eps=eps) 18 | else: 19 | return rms_norm_torch(x, weight.shape, weight=comfy.model_management.cast_to(weight, dtype=x.dtype, device=x.device), eps=eps) 20 | else: 21 | r = x * torch.rsqrt(torch.mean(x**2, dim=-1, keepdim=True) + eps) 22 | if weight is None: 23 | return r 24 | else: 25 | return r * comfy.model_management.cast_to(weight, dtype=x.dtype, device=x.device) 26 | 27 | 28 | if RMSNorm is None: 29 | class RMSNorm(torch.nn.Module): 30 | def __init__( 31 | self, 32 | normalized_shape, 33 | eps=1e-6, 34 | elementwise_affine=True, 35 | device=None, 36 | dtype=None, 37 | ): 38 | factory_kwargs = {"device": device, "dtype": dtype} 39 | super().__init__() 40 | if isinstance(normalized_shape, numbers.Integral): 41 | # mypy error: incompatible types in assignment 42 | normalized_shape = (normalized_shape,) # type: ignore[assignment] 43 | self.normalized_shape = tuple(normalized_shape) # type: ignore[arg-type] 44 | self.eps = eps 45 | self.elementwise_affine = elementwise_affine 46 | if self.elementwise_affine: 47 | self.weight = torch.nn.Parameter( 48 | torch.empty(self.normalized_shape, **factory_kwargs) 49 | ) 50 | else: 51 | self.register_parameter("weight", None) 52 | self.bias = None 53 | 54 | def forward(self, x): 55 | return rms_norm(x, self.weight, self.eps) 56 | -------------------------------------------------------------------------------- /comfy/sd1_clip_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "_name_or_path": "openai/clip-vit-large-patch14", 3 | "architectures": [ 4 | "CLIPTextModel" 5 | ], 6 | "attention_dropout": 0.0, 7 | "bos_token_id": 0, 8 | "dropout": 0.0, 9 | "eos_token_id": 49407, 10 | "hidden_act": "quick_gelu", 11 | "hidden_size": 768, 12 | "initializer_factor": 1.0, 13 | "initializer_range": 0.02, 14 | "intermediate_size": 3072, 15 | "layer_norm_eps": 1e-05, 16 | "max_position_embeddings": 77, 17 | "model_type": "clip_text_model", 18 | "num_attention_heads": 12, 19 | "num_hidden_layers": 12, 20 | "pad_token_id": 1, 21 | "projection_dim": 768, 22 | "torch_dtype": "float32", 23 | "transformers_version": "4.24.0", 24 | "vocab_size": 49408 25 | } 26 | -------------------------------------------------------------------------------- /comfy/sd1_tokenizer/special_tokens_map.json: -------------------------------------------------------------------------------- 1 | { 2 | "bos_token": { 3 | "content": "<|startoftext|>", 4 | "lstrip": false, 5 | "normalized": true, 6 | "rstrip": false, 7 | "single_word": false 8 | }, 9 | "eos_token": { 10 | "content": "<|endoftext|>", 11 | "lstrip": false, 12 | "normalized": true, 13 | "rstrip": false, 14 | "single_word": false 15 | }, 16 | "pad_token": "<|endoftext|>", 17 | "unk_token": { 18 | "content": "<|endoftext|>", 19 | "lstrip": false, 20 | "normalized": true, 21 | "rstrip": false, 22 | "single_word": false 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /comfy/sd1_tokenizer/tokenizer_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "add_prefix_space": false, 3 | "bos_token": { 4 | "__type": "AddedToken", 5 | "content": "<|startoftext|>", 6 | "lstrip": false, 7 | "normalized": true, 8 | "rstrip": false, 9 | "single_word": false 10 | }, 11 | "do_lower_case": true, 12 | "eos_token": { 13 | "__type": "AddedToken", 14 | "content": "<|endoftext|>", 15 | "lstrip": false, 16 | "normalized": true, 17 | "rstrip": false, 18 | "single_word": false 19 | }, 20 | "errors": "replace", 21 | "model_max_length": 77, 22 | "name_or_path": "openai/clip-vit-large-patch14", 23 | "pad_token": "<|endoftext|>", 24 | "special_tokens_map_file": "./special_tokens_map.json", 25 | "tokenizer_class": "CLIPTokenizer", 26 | "unk_token": { 27 | "__type": "AddedToken", 28 | "content": "<|endoftext|>", 29 | "lstrip": false, 30 | "normalized": true, 31 | "rstrip": false, 32 | "single_word": false 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /comfy/text_encoders/aura_t5.py: -------------------------------------------------------------------------------- 1 | from comfy import sd1_clip 2 | from .spiece_tokenizer import SPieceTokenizer 3 | import comfy.text_encoders.t5 4 | import os 5 | 6 | class PT5XlModel(sd1_clip.SDClipModel): 7 | def __init__(self, device="cpu", layer="last", layer_idx=None, dtype=None, model_options={}): 8 | textmodel_json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "t5_pile_config_xl.json") 9 | super().__init__(device=device, layer=layer, layer_idx=layer_idx, textmodel_json_config=textmodel_json_config, dtype=dtype, special_tokens={"end": 2, "pad": 1}, model_class=comfy.text_encoders.t5.T5, enable_attention_masks=True, zero_out_masked=True, model_options=model_options) 10 | 11 | class PT5XlTokenizer(sd1_clip.SDTokenizer): 12 | def __init__(self, embedding_directory=None, tokenizer_data={}): 13 | tokenizer_path = os.path.join(os.path.join(os.path.dirname(os.path.realpath(__file__)), "t5_pile_tokenizer"), "tokenizer.model") 14 | super().__init__(tokenizer_path, pad_with_end=False, embedding_size=2048, embedding_key='pile_t5xl', tokenizer_class=SPieceTokenizer, has_start_token=False, pad_to_max_length=False, max_length=99999999, min_length=256, pad_token=1, tokenizer_data=tokenizer_data) 15 | 16 | class AuraT5Tokenizer(sd1_clip.SD1Tokenizer): 17 | def __init__(self, embedding_directory=None, tokenizer_data={}): 18 | super().__init__(embedding_directory=embedding_directory, tokenizer_data=tokenizer_data, clip_name="pile_t5xl", tokenizer=PT5XlTokenizer) 19 | 20 | class AuraT5Model(sd1_clip.SD1ClipModel): 21 | def __init__(self, device="cpu", dtype=None, model_options={}, **kwargs): 22 | super().__init__(device=device, dtype=dtype, model_options=model_options, name="pile_t5xl", clip_model=PT5XlModel, **kwargs) 23 | -------------------------------------------------------------------------------- /comfy/text_encoders/cosmos.py: -------------------------------------------------------------------------------- 1 | from comfy import sd1_clip 2 | import comfy.text_encoders.t5 3 | import os 4 | from transformers import T5TokenizerFast 5 | 6 | 7 | class T5XXLModel(sd1_clip.SDClipModel): 8 | def __init__(self, device="cpu", layer="last", layer_idx=None, dtype=None, attention_mask=True, model_options={}): 9 | textmodel_json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "t5_old_config_xxl.json") 10 | t5xxl_scaled_fp8 = model_options.get("t5xxl_scaled_fp8", None) 11 | if t5xxl_scaled_fp8 is not None: 12 | model_options = model_options.copy() 13 | model_options["scaled_fp8"] = t5xxl_scaled_fp8 14 | 15 | super().__init__(device=device, layer=layer, layer_idx=layer_idx, textmodel_json_config=textmodel_json_config, dtype=dtype, special_tokens={"end": 1, "pad": 0}, model_class=comfy.text_encoders.t5.T5, enable_attention_masks=attention_mask, return_attention_masks=attention_mask, zero_out_masked=attention_mask, model_options=model_options) 16 | 17 | class CosmosT5XXL(sd1_clip.SD1ClipModel): 18 | def __init__(self, device="cpu", dtype=None, model_options={}): 19 | super().__init__(device=device, dtype=dtype, name="t5xxl", clip_model=T5XXLModel, model_options=model_options) 20 | 21 | 22 | class T5XXLTokenizer(sd1_clip.SDTokenizer): 23 | def __init__(self, embedding_directory=None, tokenizer_data={}): 24 | tokenizer_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "t5_tokenizer") 25 | super().__init__(tokenizer_path, embedding_directory=embedding_directory, pad_with_end=False, embedding_size=1024, embedding_key='t5xxl', tokenizer_class=T5TokenizerFast, has_start_token=False, pad_to_max_length=False, max_length=99999999, min_length=512, tokenizer_data=tokenizer_data) 26 | 27 | 28 | class CosmosT5Tokenizer(sd1_clip.SD1Tokenizer): 29 | def __init__(self, embedding_directory=None, tokenizer_data={}): 30 | super().__init__(embedding_directory=embedding_directory, tokenizer_data=tokenizer_data, clip_name="t5xxl", tokenizer=T5XXLTokenizer) 31 | 32 | 33 | def te(dtype_t5=None, t5xxl_scaled_fp8=None): 34 | class CosmosTEModel_(CosmosT5XXL): 35 | def __init__(self, device="cpu", dtype=None, model_options={}): 36 | if t5xxl_scaled_fp8 is not None and "t5xxl_scaled_fp8" not in model_options: 37 | model_options = model_options.copy() 38 | model_options["t5xxl_scaled_fp8"] = t5xxl_scaled_fp8 39 | if dtype is None: 40 | dtype = dtype_t5 41 | super().__init__(device=device, dtype=dtype, model_options=model_options) 42 | return CosmosTEModel_ 43 | -------------------------------------------------------------------------------- /comfy/text_encoders/genmo.py: -------------------------------------------------------------------------------- 1 | from comfy import sd1_clip 2 | import comfy.text_encoders.sd3_clip 3 | import os 4 | from transformers import T5TokenizerFast 5 | 6 | 7 | class T5XXLModel(comfy.text_encoders.sd3_clip.T5XXLModel): 8 | def __init__(self, **kwargs): 9 | kwargs["attention_mask"] = True 10 | super().__init__(**kwargs) 11 | 12 | 13 | class MochiT5XXL(sd1_clip.SD1ClipModel): 14 | def __init__(self, device="cpu", dtype=None, model_options={}): 15 | super().__init__(device=device, dtype=dtype, name="t5xxl", clip_model=T5XXLModel, model_options=model_options) 16 | 17 | 18 | class T5XXLTokenizer(sd1_clip.SDTokenizer): 19 | def __init__(self, embedding_directory=None, tokenizer_data={}): 20 | tokenizer_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "t5_tokenizer") 21 | super().__init__(tokenizer_path, embedding_directory=embedding_directory, pad_with_end=False, embedding_size=4096, embedding_key='t5xxl', tokenizer_class=T5TokenizerFast, has_start_token=False, pad_to_max_length=False, max_length=99999999, min_length=256, tokenizer_data=tokenizer_data) 22 | 23 | 24 | class MochiT5Tokenizer(sd1_clip.SD1Tokenizer): 25 | def __init__(self, embedding_directory=None, tokenizer_data={}): 26 | super().__init__(embedding_directory=embedding_directory, tokenizer_data=tokenizer_data, clip_name="t5xxl", tokenizer=T5XXLTokenizer) 27 | 28 | 29 | def mochi_te(dtype_t5=None, t5xxl_scaled_fp8=None): 30 | class MochiTEModel_(MochiT5XXL): 31 | def __init__(self, device="cpu", dtype=None, model_options={}): 32 | if t5xxl_scaled_fp8 is not None and "t5xxl_scaled_fp8" not in model_options: 33 | model_options = model_options.copy() 34 | model_options["t5xxl_scaled_fp8"] = t5xxl_scaled_fp8 35 | if dtype is None: 36 | dtype = dtype_t5 37 | super().__init__(device=device, dtype=dtype, model_options=model_options) 38 | return MochiTEModel_ 39 | -------------------------------------------------------------------------------- /comfy/text_encoders/hydit_clip.json: -------------------------------------------------------------------------------- 1 | { 2 | "_name_or_path": "hfl/chinese-roberta-wwm-ext-large", 3 | "architectures": [ 4 | "BertModel" 5 | ], 6 | "attention_probs_dropout_prob": 0.1, 7 | "bos_token_id": 0, 8 | "classifier_dropout": null, 9 | "directionality": "bidi", 10 | "eos_token_id": 2, 11 | "hidden_act": "gelu", 12 | "hidden_dropout_prob": 0.1, 13 | "hidden_size": 1024, 14 | "initializer_range": 0.02, 15 | "intermediate_size": 4096, 16 | "layer_norm_eps": 1e-12, 17 | "max_position_embeddings": 512, 18 | "model_type": "bert", 19 | "num_attention_heads": 16, 20 | "num_hidden_layers": 24, 21 | "output_past": true, 22 | "pad_token_id": 0, 23 | "pooler_fc_size": 768, 24 | "pooler_num_attention_heads": 12, 25 | "pooler_num_fc_layers": 3, 26 | "pooler_size_per_head": 128, 27 | "pooler_type": "first_token_transform", 28 | "position_embedding_type": "absolute", 29 | "torch_dtype": "float32", 30 | "transformers_version": "4.22.1", 31 | "type_vocab_size": 2, 32 | "use_cache": true, 33 | "vocab_size": 47020 34 | } 35 | 36 | -------------------------------------------------------------------------------- /comfy/text_encoders/hydit_clip_tokenizer/special_tokens_map.json: -------------------------------------------------------------------------------- 1 | { 2 | "cls_token": "[CLS]", 3 | "mask_token": "[MASK]", 4 | "pad_token": "[PAD]", 5 | "sep_token": "[SEP]", 6 | "unk_token": "[UNK]" 7 | } 8 | -------------------------------------------------------------------------------- /comfy/text_encoders/hydit_clip_tokenizer/tokenizer_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "cls_token": "[CLS]", 3 | "do_basic_tokenize": true, 4 | "do_lower_case": true, 5 | "mask_token": "[MASK]", 6 | "name_or_path": "hfl/chinese-roberta-wwm-ext", 7 | "never_split": null, 8 | "pad_token": "[PAD]", 9 | "sep_token": "[SEP]", 10 | "special_tokens_map_file": "/home/chenweifeng/.cache/huggingface/hub/models--hfl--chinese-roberta-wwm-ext/snapshots/5c58d0b8ec1d9014354d691c538661bf00bfdb44/special_tokens_map.json", 11 | "strip_accents": null, 12 | "tokenize_chinese_chars": true, 13 | "tokenizer_class": "BertTokenizer", 14 | "unk_token": "[UNK]", 15 | "model_max_length": 77 16 | } 17 | -------------------------------------------------------------------------------- /comfy/text_encoders/long_clipl.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | def model_options_long_clip(sd, tokenizer_data, model_options): 4 | w = sd.get("clip_l.text_model.embeddings.position_embedding.weight", None) 5 | if w is None: 6 | w = sd.get("clip_g.text_model.embeddings.position_embedding.weight", None) 7 | else: 8 | model_name = "clip_g" 9 | 10 | if w is None: 11 | w = sd.get("text_model.embeddings.position_embedding.weight", None) 12 | if w is not None: 13 | if "text_model.encoder.layers.30.mlp.fc1.weight" in sd: 14 | model_name = "clip_g" 15 | elif "text_model.encoder.layers.1.mlp.fc1.weight" in sd: 16 | model_name = "clip_l" 17 | else: 18 | model_name = "clip_l" 19 | 20 | if w is not None: 21 | tokenizer_data = tokenizer_data.copy() 22 | model_options = model_options.copy() 23 | model_config = model_options.get("model_config", {}) 24 | model_config["max_position_embeddings"] = w.shape[0] 25 | model_options["{}_model_config".format(model_name)] = model_config 26 | tokenizer_data["{}_max_length".format(model_name)] = w.shape[0] 27 | return tokenizer_data, model_options 28 | -------------------------------------------------------------------------------- /comfy/text_encoders/lt.py: -------------------------------------------------------------------------------- 1 | from comfy import sd1_clip 2 | import os 3 | from transformers import T5TokenizerFast 4 | import comfy.text_encoders.genmo 5 | 6 | class T5XXLTokenizer(sd1_clip.SDTokenizer): 7 | def __init__(self, embedding_directory=None, tokenizer_data={}): 8 | tokenizer_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "t5_tokenizer") 9 | super().__init__(tokenizer_path, embedding_directory=embedding_directory, pad_with_end=False, embedding_size=4096, embedding_key='t5xxl', tokenizer_class=T5TokenizerFast, has_start_token=False, pad_to_max_length=False, max_length=99999999, min_length=128, tokenizer_data=tokenizer_data) #pad to 128? 10 | 11 | 12 | class LTXVT5Tokenizer(sd1_clip.SD1Tokenizer): 13 | def __init__(self, embedding_directory=None, tokenizer_data={}): 14 | super().__init__(embedding_directory=embedding_directory, tokenizer_data=tokenizer_data, clip_name="t5xxl", tokenizer=T5XXLTokenizer) 15 | 16 | 17 | def ltxv_te(*args, **kwargs): 18 | return comfy.text_encoders.genmo.mochi_te(*args, **kwargs) 19 | -------------------------------------------------------------------------------- /comfy/text_encoders/lumina2.py: -------------------------------------------------------------------------------- 1 | from comfy import sd1_clip 2 | from .spiece_tokenizer import SPieceTokenizer 3 | import comfy.text_encoders.llama 4 | 5 | 6 | class Gemma2BTokenizer(sd1_clip.SDTokenizer): 7 | def __init__(self, embedding_directory=None, tokenizer_data={}): 8 | tokenizer = tokenizer_data.get("spiece_model", None) 9 | super().__init__(tokenizer, pad_with_end=False, embedding_size=2304, embedding_key='gemma2_2b', tokenizer_class=SPieceTokenizer, has_end_token=False, pad_to_max_length=False, max_length=99999999, min_length=1, tokenizer_args={"add_bos": True, "add_eos": False}, tokenizer_data=tokenizer_data) 10 | 11 | def state_dict(self): 12 | return {"spiece_model": self.tokenizer.serialize_model()} 13 | 14 | 15 | class LuminaTokenizer(sd1_clip.SD1Tokenizer): 16 | def __init__(self, embedding_directory=None, tokenizer_data={}): 17 | super().__init__(embedding_directory=embedding_directory, tokenizer_data=tokenizer_data, name="gemma2_2b", tokenizer=Gemma2BTokenizer) 18 | 19 | 20 | class Gemma2_2BModel(sd1_clip.SDClipModel): 21 | def __init__(self, device="cpu", layer="hidden", layer_idx=-2, dtype=None, attention_mask=True, model_options={}): 22 | super().__init__(device=device, layer=layer, layer_idx=layer_idx, textmodel_json_config={}, dtype=dtype, special_tokens={"start": 2, "pad": 0}, layer_norm_hidden_state=False, model_class=comfy.text_encoders.llama.Gemma2_2B, enable_attention_masks=attention_mask, return_attention_masks=attention_mask, model_options=model_options) 23 | 24 | 25 | class LuminaModel(sd1_clip.SD1ClipModel): 26 | def __init__(self, device="cpu", dtype=None, model_options={}): 27 | super().__init__(device=device, dtype=dtype, name="gemma2_2b", clip_model=Gemma2_2BModel, model_options=model_options) 28 | 29 | 30 | def te(dtype_llama=None, llama_scaled_fp8=None): 31 | class LuminaTEModel_(LuminaModel): 32 | def __init__(self, device="cpu", dtype=None, model_options={}): 33 | if llama_scaled_fp8 is not None and "scaled_fp8" not in model_options: 34 | model_options = model_options.copy() 35 | model_options["scaled_fp8"] = llama_scaled_fp8 36 | if dtype_llama is not None: 37 | dtype = dtype_llama 38 | super().__init__(device=device, dtype=dtype, model_options=model_options) 39 | return LuminaTEModel_ 40 | -------------------------------------------------------------------------------- /comfy/text_encoders/mt5_config_xl.json: -------------------------------------------------------------------------------- 1 | { 2 | "d_ff": 5120, 3 | "d_kv": 64, 4 | "d_model": 2048, 5 | "decoder_start_token_id": 0, 6 | "dropout_rate": 0.1, 7 | "eos_token_id": 1, 8 | "dense_act_fn": "gelu_pytorch_tanh", 9 | "initializer_factor": 1.0, 10 | "is_encoder_decoder": true, 11 | "is_gated_act": true, 12 | "layer_norm_epsilon": 1e-06, 13 | "model_type": "mt5", 14 | "num_decoder_layers": 24, 15 | "num_heads": 32, 16 | "num_layers": 24, 17 | "output_past": true, 18 | "pad_token_id": 0, 19 | "relative_attention_num_buckets": 32, 20 | "tie_word_embeddings": false, 21 | "vocab_size": 250112 22 | } 23 | -------------------------------------------------------------------------------- /comfy/text_encoders/pixart_t5.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from comfy import sd1_clip 4 | import comfy.text_encoders.t5 5 | import comfy.text_encoders.sd3_clip 6 | from comfy.sd1_clip import gen_empty_tokens 7 | 8 | from transformers import T5TokenizerFast 9 | 10 | class T5XXLModel(comfy.text_encoders.sd3_clip.T5XXLModel): 11 | def __init__(self, **kwargs): 12 | super().__init__(**kwargs) 13 | 14 | def gen_empty_tokens(self, special_tokens, *args, **kwargs): 15 | # PixArt expects the negative to be all pad tokens 16 | special_tokens = special_tokens.copy() 17 | special_tokens.pop("end") 18 | return gen_empty_tokens(special_tokens, *args, **kwargs) 19 | 20 | class PixArtT5XXL(sd1_clip.SD1ClipModel): 21 | def __init__(self, device="cpu", dtype=None, model_options={}): 22 | super().__init__(device=device, dtype=dtype, name="t5xxl", clip_model=T5XXLModel, model_options=model_options) 23 | 24 | class T5XXLTokenizer(sd1_clip.SDTokenizer): 25 | def __init__(self, embedding_directory=None, tokenizer_data={}): 26 | tokenizer_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "t5_tokenizer") 27 | super().__init__(tokenizer_path, embedding_directory=embedding_directory, pad_with_end=False, embedding_size=4096, embedding_key='t5xxl', tokenizer_class=T5TokenizerFast, has_start_token=False, pad_to_max_length=False, max_length=99999999, min_length=1, tokenizer_data=tokenizer_data) # no padding 28 | 29 | class PixArtTokenizer(sd1_clip.SD1Tokenizer): 30 | def __init__(self, embedding_directory=None, tokenizer_data={}): 31 | super().__init__(embedding_directory=embedding_directory, tokenizer_data=tokenizer_data, clip_name="t5xxl", tokenizer=T5XXLTokenizer) 32 | 33 | def pixart_te(dtype_t5=None, t5xxl_scaled_fp8=None): 34 | class PixArtTEModel_(PixArtT5XXL): 35 | def __init__(self, device="cpu", dtype=None, model_options={}): 36 | if t5xxl_scaled_fp8 is not None and "t5xxl_scaled_fp8" not in model_options: 37 | model_options = model_options.copy() 38 | model_options["t5xxl_scaled_fp8"] = t5xxl_scaled_fp8 39 | if dtype is None: 40 | dtype = dtype_t5 41 | super().__init__(device=device, dtype=dtype, model_options=model_options) 42 | return PixArtTEModel_ 43 | -------------------------------------------------------------------------------- /comfy/text_encoders/sa_t5.py: -------------------------------------------------------------------------------- 1 | from comfy import sd1_clip 2 | from transformers import T5TokenizerFast 3 | import comfy.text_encoders.t5 4 | import os 5 | 6 | class T5BaseModel(sd1_clip.SDClipModel): 7 | def __init__(self, device="cpu", layer="last", layer_idx=None, dtype=None, model_options={}): 8 | textmodel_json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "t5_config_base.json") 9 | super().__init__(device=device, layer=layer, layer_idx=layer_idx, textmodel_json_config=textmodel_json_config, dtype=dtype, model_options=model_options, special_tokens={"end": 1, "pad": 0}, model_class=comfy.text_encoders.t5.T5, enable_attention_masks=True, zero_out_masked=True) 10 | 11 | class T5BaseTokenizer(sd1_clip.SDTokenizer): 12 | def __init__(self, embedding_directory=None, tokenizer_data={}): 13 | tokenizer_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "t5_tokenizer") 14 | super().__init__(tokenizer_path, pad_with_end=False, embedding_size=768, embedding_key='t5base', tokenizer_class=T5TokenizerFast, has_start_token=False, pad_to_max_length=False, max_length=99999999, min_length=128, tokenizer_data=tokenizer_data) 15 | 16 | class SAT5Tokenizer(sd1_clip.SD1Tokenizer): 17 | def __init__(self, embedding_directory=None, tokenizer_data={}): 18 | super().__init__(embedding_directory=embedding_directory, tokenizer_data=tokenizer_data, clip_name="t5base", tokenizer=T5BaseTokenizer) 19 | 20 | class SAT5Model(sd1_clip.SD1ClipModel): 21 | def __init__(self, device="cpu", dtype=None, model_options={}, **kwargs): 22 | super().__init__(device=device, dtype=dtype, model_options=model_options, name="t5base", clip_model=T5BaseModel, **kwargs) 23 | -------------------------------------------------------------------------------- /comfy/text_encoders/sd2_clip.py: -------------------------------------------------------------------------------- 1 | from comfy import sd1_clip 2 | import os 3 | 4 | class SD2ClipHModel(sd1_clip.SDClipModel): 5 | def __init__(self, arch="ViT-H-14", device="cpu", max_length=77, freeze=True, layer="penultimate", layer_idx=None, dtype=None, model_options={}): 6 | if layer == "penultimate": 7 | layer="hidden" 8 | layer_idx=-2 9 | 10 | textmodel_json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "sd2_clip_config.json") 11 | super().__init__(device=device, freeze=freeze, layer=layer, layer_idx=layer_idx, textmodel_json_config=textmodel_json_config, dtype=dtype, special_tokens={"start": 49406, "end": 49407, "pad": 0}, return_projected_pooled=True, model_options=model_options) 12 | 13 | class SD2ClipHTokenizer(sd1_clip.SDTokenizer): 14 | def __init__(self, tokenizer_path=None, embedding_directory=None, tokenizer_data={}): 15 | super().__init__(tokenizer_path, pad_with_end=False, embedding_directory=embedding_directory, embedding_size=1024, embedding_key='clip_h', tokenizer_data=tokenizer_data) 16 | 17 | class SD2Tokenizer(sd1_clip.SD1Tokenizer): 18 | def __init__(self, embedding_directory=None, tokenizer_data={}): 19 | super().__init__(embedding_directory=embedding_directory, tokenizer_data=tokenizer_data, clip_name="h", tokenizer=SD2ClipHTokenizer) 20 | 21 | class SD2ClipModel(sd1_clip.SD1ClipModel): 22 | def __init__(self, device="cpu", dtype=None, model_options={}, **kwargs): 23 | super().__init__(device=device, dtype=dtype, model_options=model_options, clip_name="h", clip_model=SD2ClipHModel, **kwargs) 24 | -------------------------------------------------------------------------------- /comfy/text_encoders/sd2_clip_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "architectures": [ 3 | "CLIPTextModel" 4 | ], 5 | "attention_dropout": 0.0, 6 | "bos_token_id": 0, 7 | "dropout": 0.0, 8 | "eos_token_id": 49407, 9 | "hidden_act": "gelu", 10 | "hidden_size": 1024, 11 | "initializer_factor": 1.0, 12 | "initializer_range": 0.02, 13 | "intermediate_size": 4096, 14 | "layer_norm_eps": 1e-05, 15 | "max_position_embeddings": 77, 16 | "model_type": "clip_text_model", 17 | "num_attention_heads": 16, 18 | "num_hidden_layers": 24, 19 | "pad_token_id": 1, 20 | "projection_dim": 1024, 21 | "torch_dtype": "float32", 22 | "vocab_size": 49408 23 | } 24 | -------------------------------------------------------------------------------- /comfy/text_encoders/spiece_tokenizer.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import os 3 | 4 | class SPieceTokenizer: 5 | @staticmethod 6 | def from_pretrained(path, **kwargs): 7 | return SPieceTokenizer(path, **kwargs) 8 | 9 | def __init__(self, tokenizer_path, add_bos=False, add_eos=True): 10 | self.add_bos = add_bos 11 | self.add_eos = add_eos 12 | import sentencepiece 13 | if torch.is_tensor(tokenizer_path): 14 | tokenizer_path = tokenizer_path.numpy().tobytes() 15 | 16 | if isinstance(tokenizer_path, bytes): 17 | self.tokenizer = sentencepiece.SentencePieceProcessor(model_proto=tokenizer_path, add_bos=self.add_bos, add_eos=self.add_eos) 18 | else: 19 | if not os.path.isfile(tokenizer_path): 20 | raise ValueError("invalid tokenizer") 21 | self.tokenizer = sentencepiece.SentencePieceProcessor(model_file=tokenizer_path, add_bos=self.add_bos, add_eos=self.add_eos) 22 | 23 | def get_vocab(self): 24 | out = {} 25 | for i in range(self.tokenizer.get_piece_size()): 26 | out[self.tokenizer.id_to_piece(i)] = i 27 | return out 28 | 29 | def __call__(self, string): 30 | out = self.tokenizer.encode(string) 31 | return {"input_ids": out} 32 | 33 | def serialize_model(self): 34 | return torch.ByteTensor(list(self.tokenizer.serialized_model_proto())) 35 | -------------------------------------------------------------------------------- /comfy/text_encoders/t5_config_base.json: -------------------------------------------------------------------------------- 1 | { 2 | "d_ff": 3072, 3 | "d_kv": 64, 4 | "d_model": 768, 5 | "decoder_start_token_id": 0, 6 | "dropout_rate": 0.1, 7 | "eos_token_id": 1, 8 | "dense_act_fn": "relu", 9 | "initializer_factor": 1.0, 10 | "is_encoder_decoder": true, 11 | "is_gated_act": false, 12 | "layer_norm_epsilon": 1e-06, 13 | "model_type": "t5", 14 | "num_decoder_layers": 12, 15 | "num_heads": 12, 16 | "num_layers": 12, 17 | "output_past": true, 18 | "pad_token_id": 0, 19 | "relative_attention_num_buckets": 32, 20 | "tie_word_embeddings": false, 21 | "vocab_size": 32128 22 | } 23 | -------------------------------------------------------------------------------- /comfy/text_encoders/t5_config_xxl.json: -------------------------------------------------------------------------------- 1 | { 2 | "d_ff": 10240, 3 | "d_kv": 64, 4 | "d_model": 4096, 5 | "decoder_start_token_id": 0, 6 | "dropout_rate": 0.1, 7 | "eos_token_id": 1, 8 | "dense_act_fn": "gelu_pytorch_tanh", 9 | "initializer_factor": 1.0, 10 | "is_encoder_decoder": true, 11 | "is_gated_act": true, 12 | "layer_norm_epsilon": 1e-06, 13 | "model_type": "t5", 14 | "num_decoder_layers": 24, 15 | "num_heads": 64, 16 | "num_layers": 24, 17 | "output_past": true, 18 | "pad_token_id": 0, 19 | "relative_attention_num_buckets": 32, 20 | "tie_word_embeddings": false, 21 | "vocab_size": 32128 22 | } 23 | -------------------------------------------------------------------------------- /comfy/text_encoders/t5_old_config_xxl.json: -------------------------------------------------------------------------------- 1 | { 2 | "d_ff": 65536, 3 | "d_kv": 128, 4 | "d_model": 1024, 5 | "decoder_start_token_id": 0, 6 | "dropout_rate": 0.1, 7 | "eos_token_id": 1, 8 | "dense_act_fn": "relu", 9 | "initializer_factor": 1.0, 10 | "is_encoder_decoder": true, 11 | "is_gated_act": false, 12 | "layer_norm_epsilon": 1e-06, 13 | "model_type": "t5", 14 | "num_decoder_layers": 24, 15 | "num_heads": 128, 16 | "num_layers": 24, 17 | "output_past": true, 18 | "pad_token_id": 0, 19 | "relative_attention_num_buckets": 32, 20 | "tie_word_embeddings": false, 21 | "vocab_size": 32128 22 | } 23 | -------------------------------------------------------------------------------- /comfy/text_encoders/t5_pile_config_xl.json: -------------------------------------------------------------------------------- 1 | { 2 | "d_ff": 5120, 3 | "d_kv": 64, 4 | "d_model": 2048, 5 | "decoder_start_token_id": 0, 6 | "dropout_rate": 0.1, 7 | "eos_token_id": 2, 8 | "dense_act_fn": "gelu_pytorch_tanh", 9 | "initializer_factor": 1.0, 10 | "is_encoder_decoder": true, 11 | "is_gated_act": true, 12 | "layer_norm_epsilon": 1e-06, 13 | "model_type": "umt5", 14 | "num_decoder_layers": 24, 15 | "num_heads": 32, 16 | "num_layers": 24, 17 | "output_past": true, 18 | "pad_token_id": 1, 19 | "relative_attention_num_buckets": 32, 20 | "tie_word_embeddings": false, 21 | "vocab_size": 32128 22 | } 23 | -------------------------------------------------------------------------------- /comfy/text_encoders/t5_pile_tokenizer/tokenizer.model: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/comfyanonymous/ComfyUI/312d511630db4907c3bed04dee297b28f61941a8/comfy/text_encoders/t5_pile_tokenizer/tokenizer.model -------------------------------------------------------------------------------- /comfy/text_encoders/umt5_config_base.json: -------------------------------------------------------------------------------- 1 | { 2 | "d_ff": 2048, 3 | "d_kv": 64, 4 | "d_model": 768, 5 | "decoder_start_token_id": 0, 6 | "dropout_rate": 0.1, 7 | "eos_token_id": 1, 8 | "dense_act_fn": "gelu_pytorch_tanh", 9 | "initializer_factor": 1.0, 10 | "is_encoder_decoder": true, 11 | "is_gated_act": true, 12 | "layer_norm_epsilon": 1e-06, 13 | "model_type": "umt5", 14 | "num_decoder_layers": 12, 15 | "num_heads": 12, 16 | "num_layers": 12, 17 | "output_past": true, 18 | "pad_token_id": 0, 19 | "relative_attention_num_buckets": 32, 20 | "tie_word_embeddings": false, 21 | "vocab_size": 256384 22 | } 23 | -------------------------------------------------------------------------------- /comfy/text_encoders/umt5_config_xxl.json: -------------------------------------------------------------------------------- 1 | { 2 | "d_ff": 10240, 3 | "d_kv": 64, 4 | "d_model": 4096, 5 | "decoder_start_token_id": 0, 6 | "dropout_rate": 0.1, 7 | "eos_token_id": 1, 8 | "dense_act_fn": "gelu_pytorch_tanh", 9 | "initializer_factor": 1.0, 10 | "is_encoder_decoder": true, 11 | "is_gated_act": true, 12 | "layer_norm_epsilon": 1e-06, 13 | "model_type": "umt5", 14 | "num_decoder_layers": 24, 15 | "num_heads": 64, 16 | "num_layers": 24, 17 | "output_past": true, 18 | "pad_token_id": 0, 19 | "relative_attention_num_buckets": 32, 20 | "tie_word_embeddings": false, 21 | "vocab_size": 256384 22 | } 23 | -------------------------------------------------------------------------------- /comfy/text_encoders/wan.py: -------------------------------------------------------------------------------- 1 | from comfy import sd1_clip 2 | from .spiece_tokenizer import SPieceTokenizer 3 | import comfy.text_encoders.t5 4 | import os 5 | 6 | class UMT5XXlModel(sd1_clip.SDClipModel): 7 | def __init__(self, device="cpu", layer="last", layer_idx=None, dtype=None, model_options={}): 8 | textmodel_json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "umt5_config_xxl.json") 9 | super().__init__(device=device, layer=layer, layer_idx=layer_idx, textmodel_json_config=textmodel_json_config, dtype=dtype, special_tokens={"end": 1, "pad": 0}, model_class=comfy.text_encoders.t5.T5, enable_attention_masks=True, zero_out_masked=True, model_options=model_options) 10 | 11 | class UMT5XXlTokenizer(sd1_clip.SDTokenizer): 12 | def __init__(self, embedding_directory=None, tokenizer_data={}): 13 | tokenizer = tokenizer_data.get("spiece_model", None) 14 | super().__init__(tokenizer, pad_with_end=False, embedding_size=4096, embedding_key='umt5xxl', tokenizer_class=SPieceTokenizer, has_start_token=False, pad_to_max_length=False, max_length=99999999, min_length=512, pad_token=0, tokenizer_data=tokenizer_data) 15 | 16 | def state_dict(self): 17 | return {"spiece_model": self.tokenizer.serialize_model()} 18 | 19 | 20 | class WanT5Tokenizer(sd1_clip.SD1Tokenizer): 21 | def __init__(self, embedding_directory=None, tokenizer_data={}): 22 | super().__init__(embedding_directory=embedding_directory, tokenizer_data=tokenizer_data, clip_name="umt5xxl", tokenizer=UMT5XXlTokenizer) 23 | 24 | class WanT5Model(sd1_clip.SD1ClipModel): 25 | def __init__(self, device="cpu", dtype=None, model_options={}, **kwargs): 26 | super().__init__(device=device, dtype=dtype, model_options=model_options, name="umt5xxl", clip_model=UMT5XXlModel, **kwargs) 27 | 28 | def te(dtype_t5=None, t5xxl_scaled_fp8=None): 29 | class WanTEModel(WanT5Model): 30 | def __init__(self, device="cpu", dtype=None, model_options={}): 31 | if t5xxl_scaled_fp8 is not None and "scaled_fp8" not in model_options: 32 | model_options = model_options.copy() 33 | model_options["scaled_fp8"] = t5xxl_scaled_fp8 34 | if dtype_t5 is not None: 35 | dtype = dtype_t5 36 | super().__init__(device=device, dtype=dtype, model_options=model_options) 37 | return WanTEModel 38 | -------------------------------------------------------------------------------- /comfy/weight_adapter/__init__.py: -------------------------------------------------------------------------------- 1 | from .base import WeightAdapterBase 2 | from .lora import LoRAAdapter 3 | from .loha import LoHaAdapter 4 | from .lokr import LoKrAdapter 5 | from .glora import GLoRAAdapter 6 | from .oft import OFTAdapter 7 | from .boft import BOFTAdapter 8 | 9 | 10 | adapters: list[type[WeightAdapterBase]] = [ 11 | LoRAAdapter, 12 | LoHaAdapter, 13 | LoKrAdapter, 14 | GLoRAAdapter, 15 | OFTAdapter, 16 | BOFTAdapter, 17 | ] 18 | -------------------------------------------------------------------------------- /comfy_api/input/__init__.py: -------------------------------------------------------------------------------- 1 | from .basic_types import ImageInput, AudioInput 2 | from .video_types import VideoInput 3 | 4 | __all__ = [ 5 | "ImageInput", 6 | "AudioInput", 7 | "VideoInput", 8 | ] 9 | -------------------------------------------------------------------------------- /comfy_api/input/basic_types.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from typing import TypedDict 3 | 4 | ImageInput = torch.Tensor 5 | """ 6 | An image in format [B, H, W, C] where B is the batch size, C is the number of channels, 7 | """ 8 | 9 | class AudioInput(TypedDict): 10 | """ 11 | TypedDict representing audio input. 12 | """ 13 | 14 | waveform: torch.Tensor 15 | """ 16 | Tensor in the format [B, C, T] where B is the batch size, C is the number of channels, 17 | """ 18 | 19 | sample_rate: int 20 | 21 | -------------------------------------------------------------------------------- /comfy_api/input/video_types.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | from abc import ABC, abstractmethod 3 | from typing import Optional 4 | from comfy_api.util import VideoContainer, VideoCodec, VideoComponents 5 | 6 | class VideoInput(ABC): 7 | """ 8 | Abstract base class for video input types. 9 | """ 10 | 11 | @abstractmethod 12 | def get_components(self) -> VideoComponents: 13 | """ 14 | Abstract method to get the video components (images, audio, and frame rate). 15 | 16 | Returns: 17 | VideoComponents containing images, audio, and frame rate 18 | """ 19 | pass 20 | 21 | @abstractmethod 22 | def save_to( 23 | self, 24 | path: str, 25 | format: VideoContainer = VideoContainer.AUTO, 26 | codec: VideoCodec = VideoCodec.AUTO, 27 | metadata: Optional[dict] = None 28 | ): 29 | """ 30 | Abstract method to save the video input to a file. 31 | """ 32 | pass 33 | 34 | # Provide a default implementation, but subclasses can provide optimized versions 35 | # if possible. 36 | def get_dimensions(self) -> tuple[int, int]: 37 | """ 38 | Returns the dimensions of the video input. 39 | 40 | Returns: 41 | Tuple of (width, height) 42 | """ 43 | components = self.get_components() 44 | return components.images.shape[2], components.images.shape[1] 45 | 46 | def get_duration(self) -> float: 47 | """ 48 | Returns the duration of the video in seconds. 49 | 50 | Returns: 51 | Duration in seconds 52 | """ 53 | components = self.get_components() 54 | frame_count = components.images.shape[0] 55 | return float(frame_count / components.frame_rate) 56 | -------------------------------------------------------------------------------- /comfy_api/input_impl/__init__.py: -------------------------------------------------------------------------------- 1 | from .video_types import VideoFromFile, VideoFromComponents 2 | 3 | __all__ = [ 4 | # Implementations 5 | "VideoFromFile", 6 | "VideoFromComponents", 7 | ] 8 | -------------------------------------------------------------------------------- /comfy_api/torch_helpers/__init__.py: -------------------------------------------------------------------------------- 1 | from .torch_compile import set_torch_compile_wrapper 2 | 3 | __all__ = [ 4 | "set_torch_compile_wrapper", 5 | ] 6 | -------------------------------------------------------------------------------- /comfy_api/util/__init__.py: -------------------------------------------------------------------------------- 1 | from .video_types import VideoContainer, VideoCodec, VideoComponents 2 | 3 | __all__ = [ 4 | # Utility Types 5 | "VideoContainer", 6 | "VideoCodec", 7 | "VideoComponents", 8 | ] 9 | -------------------------------------------------------------------------------- /comfy_api/util/video_types.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | from dataclasses import dataclass 3 | from enum import Enum 4 | from fractions import Fraction 5 | from typing import Optional 6 | from comfy_api.input import ImageInput, AudioInput 7 | 8 | class VideoCodec(str, Enum): 9 | AUTO = "auto" 10 | H264 = "h264" 11 | 12 | @classmethod 13 | def as_input(cls) -> list[str]: 14 | """ 15 | Returns a list of codec names that can be used as node input. 16 | """ 17 | return [member.value for member in cls] 18 | 19 | class VideoContainer(str, Enum): 20 | AUTO = "auto" 21 | MP4 = "mp4" 22 | 23 | @classmethod 24 | def as_input(cls) -> list[str]: 25 | """ 26 | Returns a list of container names that can be used as node input. 27 | """ 28 | return [member.value for member in cls] 29 | 30 | @classmethod 31 | def get_extension(cls, value) -> str: 32 | """ 33 | Returns the file extension for the container. 34 | """ 35 | if isinstance(value, str): 36 | value = cls(value) 37 | if value == VideoContainer.MP4 or value == VideoContainer.AUTO: 38 | return "mp4" 39 | return "" 40 | 41 | @dataclass 42 | class VideoComponents: 43 | """ 44 | Dataclass representing the components of a video. 45 | """ 46 | 47 | images: ImageInput 48 | frame_rate: Fraction 49 | audio: Optional[AudioInput] = None 50 | metadata: Optional[dict] = None 51 | 52 | -------------------------------------------------------------------------------- /comfy_api_nodes/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/comfyanonymous/ComfyUI/312d511630db4907c3bed04dee297b28f61941a8/comfy_api_nodes/__init__.py -------------------------------------------------------------------------------- /comfy_api_nodes/apis/PixverseController.py: -------------------------------------------------------------------------------- 1 | # generated by datamodel-codegen: 2 | # filename: filtered-openapi.yaml 3 | # timestamp: 2025-04-29T23:44:54+00:00 4 | 5 | from __future__ import annotations 6 | 7 | from typing import Optional 8 | 9 | from pydantic import BaseModel 10 | 11 | from . import PixverseDto 12 | 13 | 14 | class ResponseData(BaseModel): 15 | ErrCode: Optional[int] = None 16 | ErrMsg: Optional[str] = None 17 | Resp: Optional[PixverseDto.V2OpenAPII2VResp] = None 18 | -------------------------------------------------------------------------------- /comfy_api_nodes/apis/PixverseDto.py: -------------------------------------------------------------------------------- 1 | # generated by datamodel-codegen: 2 | # filename: filtered-openapi.yaml 3 | # timestamp: 2025-04-29T23:44:54+00:00 4 | 5 | from __future__ import annotations 6 | 7 | from typing import Optional 8 | 9 | from pydantic import BaseModel, Field 10 | 11 | 12 | class V2OpenAPII2VResp(BaseModel): 13 | video_id: Optional[int] = Field(None, description='Video_id') 14 | 15 | 16 | class V2OpenAPIT2VReq(BaseModel): 17 | aspect_ratio: str = Field( 18 | ..., description='Aspect ratio (16:9, 4:3, 1:1, 3:4, 9:16)', examples=['16:9'] 19 | ) 20 | duration: int = Field( 21 | ..., 22 | description='Video duration (5, 8 seconds, --model=v3.5 only allows 5,8; --quality=1080p does not support 8s)', 23 | examples=[5], 24 | ) 25 | model: str = Field( 26 | ..., description='Model version (only supports v3.5)', examples=['v3.5'] 27 | ) 28 | motion_mode: Optional[str] = Field( 29 | 'normal', 30 | description='Motion mode (normal, fast, --fast only available when duration=5; --quality=1080p does not support fast)', 31 | examples=['normal'], 32 | ) 33 | negative_prompt: Optional[str] = Field( 34 | None, description='Negative prompt\n', max_length=2048 35 | ) 36 | prompt: str = Field(..., description='Prompt', max_length=2048) 37 | quality: str = Field( 38 | ..., 39 | description='Video quality ("360p"(Turbo model), "540p", "720p", "1080p")', 40 | examples=['540p'], 41 | ) 42 | seed: Optional[int] = Field(None, description='Random seed, range: 0 - 2147483647') 43 | style: Optional[str] = Field( 44 | None, 45 | description='Style (effective when model=v3.5, "anime", "3d_animation", "clay", "comic", "cyberpunk") Do not include style parameter unless needed', 46 | examples=['anime'], 47 | ) 48 | template_id: Optional[int] = Field( 49 | None, 50 | description='Template ID (template_id must be activated before use)', 51 | examples=[302325299692608], 52 | ) 53 | water_mark: Optional[bool] = Field( 54 | False, 55 | description='Watermark (true: add watermark, false: no watermark)', 56 | examples=[False], 57 | ) 58 | -------------------------------------------------------------------------------- /comfy_api_nodes/apis/rodin_api.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from enum import Enum 4 | from typing import Optional, List 5 | from pydantic import BaseModel, Field 6 | 7 | 8 | class Rodin3DGenerateRequest(BaseModel): 9 | seed: int = Field(..., description="seed_") 10 | tier: str = Field(..., description="Tier of generation.") 11 | material: str = Field(..., description="The material type.") 12 | quality: str = Field(..., description="The generation quality of the mesh.") 13 | mesh_mode: str = Field(..., description="It controls the type of faces of generated models.") 14 | 15 | class GenerateJobsData(BaseModel): 16 | uuids: List[str] = Field(..., description="str LIST") 17 | subscription_key: str = Field(..., description="subscription key") 18 | 19 | class Rodin3DGenerateResponse(BaseModel): 20 | message: Optional[str] = Field(None, description="Return message.") 21 | prompt: Optional[str] = Field(None, description="Generated Prompt from image.") 22 | submit_time: Optional[str] = Field(None, description="Submit Time") 23 | uuid: Optional[str] = Field(None, description="Task str") 24 | jobs: Optional[GenerateJobsData] = Field(None, description="Details of jobs") 25 | 26 | class JobStatus(str, Enum): 27 | """ 28 | Status for jobs 29 | """ 30 | Done = "Done" 31 | Failed = "Failed" 32 | Generating = "Generating" 33 | Waiting = "Waiting" 34 | 35 | class Rodin3DCheckStatusRequest(BaseModel): 36 | subscription_key: str = Field(..., description="subscription from generate endpoint") 37 | 38 | class JobItem(BaseModel): 39 | uuid: str = Field(..., description="uuid") 40 | status: JobStatus = Field(...,description="Status Currently") 41 | 42 | class Rodin3DCheckStatusResponse(BaseModel): 43 | jobs: List[JobItem] = Field(..., description="Job status List") 44 | 45 | class Rodin3DDownloadRequest(BaseModel): 46 | task_uuid: str = Field(..., description="Task str") 47 | 48 | class RodinResourceItem(BaseModel): 49 | url: str = Field(..., description="Download Url") 50 | name: str = Field(..., description="File name with ext") 51 | 52 | class Rodin3DDownloadResponse(BaseModel): 53 | list: List[RodinResourceItem] = Field(..., description="Source List") 54 | 55 | 56 | 57 | 58 | -------------------------------------------------------------------------------- /comfy_api_nodes/canary.py: -------------------------------------------------------------------------------- 1 | import av 2 | 3 | ver = av.__version__.split(".") 4 | if int(ver[0]) < 14: 5 | raise Exception("INSTALL NEW VERSION OF PYAV TO USE API NODES.") 6 | 7 | if int(ver[0]) == 14 and int(ver[1]) < 2: 8 | raise Exception("INSTALL NEW VERSION OF PYAV TO USE API NODES.") 9 | 10 | NODE_CLASS_MAPPINGS = {} 11 | -------------------------------------------------------------------------------- /comfy_api_nodes/redocly-dev.yaml: -------------------------------------------------------------------------------- 1 | # This file is used to filter the Comfy Org OpenAPI spec for schemas related to API Nodes. 2 | # This is used for development purposes to generate stubs for unreleased API endpoints. 3 | apis: 4 | filter: 5 | root: openapi.yaml 6 | decorators: 7 | filter-in: 8 | property: tags 9 | value: ['API Nodes'] 10 | matchStrategy: all 11 | -------------------------------------------------------------------------------- /comfy_api_nodes/redocly.yaml: -------------------------------------------------------------------------------- 1 | # This file is used to filter the Comfy Org OpenAPI spec for schemas related to API Nodes. 2 | 3 | apis: 4 | filter: 5 | root: openapi.yaml 6 | decorators: 7 | filter-in: 8 | property: tags 9 | value: ['API Nodes', 'Released'] 10 | matchStrategy: all 11 | -------------------------------------------------------------------------------- /comfy_api_nodes/util/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/comfyanonymous/ComfyUI/312d511630db4907c3bed04dee297b28f61941a8/comfy_api_nodes/util/__init__.py -------------------------------------------------------------------------------- /comfy_execution/validation.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | 4 | def validate_node_input( 5 | received_type: str, input_type: str, strict: bool = False 6 | ) -> bool: 7 | """ 8 | received_type and input_type are both strings of the form "T1,T2,...". 9 | 10 | If strict is True, the input_type must contain the received_type. 11 | For example, if received_type is "STRING" and input_type is "STRING,INT", 12 | this will return True. But if received_type is "STRING,INT" and input_type is 13 | "INT", this will return False. 14 | 15 | If strict is False, the input_type must have overlap with the received_type. 16 | For example, if received_type is "STRING,BOOLEAN" and input_type is "STRING,INT", 17 | this will return True. 18 | 19 | Supports pre-union type extension behaviour of ``__ne__`` overrides. 20 | """ 21 | # If the types are exactly the same, we can return immediately 22 | # Use pre-union behaviour: inverse of `__ne__` 23 | if not received_type != input_type: 24 | return True 25 | 26 | # Not equal, and not strings 27 | if not isinstance(received_type, str) or not isinstance(input_type, str): 28 | return False 29 | 30 | # Split the type strings into sets for comparison 31 | received_types = set(t.strip() for t in received_type.split(",")) 32 | input_types = set(t.strip() for t in input_type.split(",")) 33 | 34 | if strict: 35 | # In strict mode, all received types must be in the input types 36 | return received_types.issubset(input_types) 37 | else: 38 | # In non-strict mode, there must be at least one type in common 39 | return len(received_types.intersection(input_types)) > 0 40 | -------------------------------------------------------------------------------- /comfy_extras/chainner_models/model_loading.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from spandrel import ModelLoader 3 | 4 | def load_state_dict(state_dict): 5 | logging.warning("comfy_extras.chainner_models is deprecated and has been replaced by the spandrel library.") 6 | return ModelLoader().load_from_state_dict(state_dict).eval() 7 | -------------------------------------------------------------------------------- /comfy_extras/nodes_ace.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import comfy.model_management 3 | import node_helpers 4 | 5 | class TextEncodeAceStepAudio: 6 | @classmethod 7 | def INPUT_TYPES(s): 8 | return {"required": { 9 | "clip": ("CLIP", ), 10 | "tags": ("STRING", {"multiline": True, "dynamicPrompts": True}), 11 | "lyrics": ("STRING", {"multiline": True, "dynamicPrompts": True}), 12 | "lyrics_strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}), 13 | }} 14 | RETURN_TYPES = ("CONDITIONING",) 15 | FUNCTION = "encode" 16 | 17 | CATEGORY = "conditioning" 18 | 19 | def encode(self, clip, tags, lyrics, lyrics_strength): 20 | tokens = clip.tokenize(tags, lyrics=lyrics) 21 | conditioning = clip.encode_from_tokens_scheduled(tokens) 22 | conditioning = node_helpers.conditioning_set_values(conditioning, {"lyrics_strength": lyrics_strength}) 23 | return (conditioning, ) 24 | 25 | 26 | class EmptyAceStepLatentAudio: 27 | def __init__(self): 28 | self.device = comfy.model_management.intermediate_device() 29 | 30 | @classmethod 31 | def INPUT_TYPES(s): 32 | return {"required": {"seconds": ("FLOAT", {"default": 120.0, "min": 1.0, "max": 1000.0, "step": 0.1}), 33 | "batch_size": ("INT", {"default": 1, "min": 1, "max": 4096, "tooltip": "The number of latent images in the batch."}), 34 | }} 35 | RETURN_TYPES = ("LATENT",) 36 | FUNCTION = "generate" 37 | 38 | CATEGORY = "latent/audio" 39 | 40 | def generate(self, seconds, batch_size): 41 | length = int(seconds * 44100 / 512 / 8) 42 | latent = torch.zeros([batch_size, 8, 16, length], device=self.device) 43 | return ({"samples": latent, "type": "audio"}, ) 44 | 45 | 46 | NODE_CLASS_MAPPINGS = { 47 | "TextEncodeAceStepAudio": TextEncodeAceStepAudio, 48 | "EmptyAceStepLatentAudio": EmptyAceStepLatentAudio, 49 | } 50 | -------------------------------------------------------------------------------- /comfy_extras/nodes_align_your_steps.py: -------------------------------------------------------------------------------- 1 | #from: https://research.nvidia.com/labs/toronto-ai/AlignYourSteps/howto.html 2 | import numpy as np 3 | import torch 4 | 5 | def loglinear_interp(t_steps, num_steps): 6 | """ 7 | Performs log-linear interpolation of a given array of decreasing numbers. 8 | """ 9 | xs = np.linspace(0, 1, len(t_steps)) 10 | ys = np.log(t_steps[::-1]) 11 | 12 | new_xs = np.linspace(0, 1, num_steps) 13 | new_ys = np.interp(new_xs, xs, ys) 14 | 15 | interped_ys = np.exp(new_ys)[::-1].copy() 16 | return interped_ys 17 | 18 | NOISE_LEVELS = {"SD1": [14.6146412293, 6.4745760956, 3.8636745985, 2.6946151520, 1.8841921177, 1.3943805092, 0.9642583904, 0.6523686016, 0.3977456272, 0.1515232662, 0.0291671582], 19 | "SDXL":[14.6146412293, 6.3184485287, 3.7681790315, 2.1811480769, 1.3405244945, 0.8620721141, 0.5550693289, 0.3798540708, 0.2332364134, 0.1114188177, 0.0291671582], 20 | "SVD": [700.00, 54.5, 15.886, 7.977, 4.248, 1.789, 0.981, 0.403, 0.173, 0.034, 0.002]} 21 | 22 | class AlignYourStepsScheduler: 23 | @classmethod 24 | def INPUT_TYPES(s): 25 | return {"required": 26 | {"model_type": (["SD1", "SDXL", "SVD"], ), 27 | "steps": ("INT", {"default": 10, "min": 1, "max": 10000}), 28 | "denoise": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01}), 29 | } 30 | } 31 | RETURN_TYPES = ("SIGMAS",) 32 | CATEGORY = "sampling/custom_sampling/schedulers" 33 | 34 | FUNCTION = "get_sigmas" 35 | 36 | def get_sigmas(self, model_type, steps, denoise): 37 | total_steps = steps 38 | if denoise < 1.0: 39 | if denoise <= 0.0: 40 | return (torch.FloatTensor([]),) 41 | total_steps = round(steps * denoise) 42 | 43 | sigmas = NOISE_LEVELS[model_type][:] 44 | if (steps + 1) != len(sigmas): 45 | sigmas = loglinear_interp(sigmas, steps + 1) 46 | 47 | sigmas = sigmas[-(total_steps + 1):] 48 | sigmas[-1] = 0 49 | return (torch.FloatTensor(sigmas), ) 50 | 51 | NODE_CLASS_MAPPINGS = { 52 | "AlignYourStepsScheduler": AlignYourStepsScheduler, 53 | } 54 | -------------------------------------------------------------------------------- /comfy_extras/nodes_canny.py: -------------------------------------------------------------------------------- 1 | from kornia.filters import canny 2 | import comfy.model_management 3 | 4 | 5 | class Canny: 6 | @classmethod 7 | def INPUT_TYPES(s): 8 | return {"required": {"image": ("IMAGE",), 9 | "low_threshold": ("FLOAT", {"default": 0.4, "min": 0.01, "max": 0.99, "step": 0.01}), 10 | "high_threshold": ("FLOAT", {"default": 0.8, "min": 0.01, "max": 0.99, "step": 0.01}) 11 | }} 12 | 13 | RETURN_TYPES = ("IMAGE",) 14 | FUNCTION = "detect_edge" 15 | 16 | CATEGORY = "image/preprocessors" 17 | 18 | def detect_edge(self, image, low_threshold, high_threshold): 19 | output = canny(image.to(comfy.model_management.get_torch_device()).movedim(-1, 1), low_threshold, high_threshold) 20 | img_out = output[1].to(comfy.model_management.intermediate_device()).repeat(1, 3, 1, 1).movedim(1, -1) 21 | return (img_out,) 22 | 23 | NODE_CLASS_MAPPINGS = { 24 | "Canny": Canny, 25 | } 26 | -------------------------------------------------------------------------------- /comfy_extras/nodes_cfg.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | # https://github.com/WeichenFan/CFG-Zero-star 4 | def optimized_scale(positive, negative): 5 | positive_flat = positive.reshape(positive.shape[0], -1) 6 | negative_flat = negative.reshape(negative.shape[0], -1) 7 | 8 | # Calculate dot production 9 | dot_product = torch.sum(positive_flat * negative_flat, dim=1, keepdim=True) 10 | 11 | # Squared norm of uncondition 12 | squared_norm = torch.sum(negative_flat ** 2, dim=1, keepdim=True) + 1e-8 13 | 14 | # st_star = v_cond^T * v_uncond / ||v_uncond||^2 15 | st_star = dot_product / squared_norm 16 | 17 | return st_star.reshape([positive.shape[0]] + [1] * (positive.ndim - 1)) 18 | 19 | class CFGZeroStar: 20 | @classmethod 21 | def INPUT_TYPES(s): 22 | return {"required": {"model": ("MODEL",), 23 | }} 24 | RETURN_TYPES = ("MODEL",) 25 | RETURN_NAMES = ("patched_model",) 26 | FUNCTION = "patch" 27 | CATEGORY = "advanced/guidance" 28 | 29 | def patch(self, model): 30 | m = model.clone() 31 | def cfg_zero_star(args): 32 | guidance_scale = args['cond_scale'] 33 | x = args['input'] 34 | cond_p = args['cond_denoised'] 35 | uncond_p = args['uncond_denoised'] 36 | out = args["denoised"] 37 | alpha = optimized_scale(x - cond_p, x - uncond_p) 38 | 39 | return out + uncond_p * (alpha - 1.0) + guidance_scale * uncond_p * (1.0 - alpha) 40 | m.set_model_sampler_post_cfg_function(cfg_zero_star) 41 | return (m, ) 42 | 43 | NODE_CLASS_MAPPINGS = { 44 | "CFGZeroStar": CFGZeroStar 45 | } 46 | -------------------------------------------------------------------------------- /comfy_extras/nodes_clip_sdxl.py: -------------------------------------------------------------------------------- 1 | from nodes import MAX_RESOLUTION 2 | 3 | class CLIPTextEncodeSDXLRefiner: 4 | @classmethod 5 | def INPUT_TYPES(s): 6 | return {"required": { 7 | "ascore": ("FLOAT", {"default": 6.0, "min": 0.0, "max": 1000.0, "step": 0.01}), 8 | "width": ("INT", {"default": 1024.0, "min": 0, "max": MAX_RESOLUTION}), 9 | "height": ("INT", {"default": 1024.0, "min": 0, "max": MAX_RESOLUTION}), 10 | "text": ("STRING", {"multiline": True, "dynamicPrompts": True}), "clip": ("CLIP", ), 11 | }} 12 | RETURN_TYPES = ("CONDITIONING",) 13 | FUNCTION = "encode" 14 | 15 | CATEGORY = "advanced/conditioning" 16 | 17 | def encode(self, clip, ascore, width, height, text): 18 | tokens = clip.tokenize(text) 19 | return (clip.encode_from_tokens_scheduled(tokens, add_dict={"aesthetic_score": ascore, "width": width, "height": height}), ) 20 | 21 | class CLIPTextEncodeSDXL: 22 | @classmethod 23 | def INPUT_TYPES(s): 24 | return {"required": { 25 | "clip": ("CLIP", ), 26 | "width": ("INT", {"default": 1024.0, "min": 0, "max": MAX_RESOLUTION}), 27 | "height": ("INT", {"default": 1024.0, "min": 0, "max": MAX_RESOLUTION}), 28 | "crop_w": ("INT", {"default": 0, "min": 0, "max": MAX_RESOLUTION}), 29 | "crop_h": ("INT", {"default": 0, "min": 0, "max": MAX_RESOLUTION}), 30 | "target_width": ("INT", {"default": 1024.0, "min": 0, "max": MAX_RESOLUTION}), 31 | "target_height": ("INT", {"default": 1024.0, "min": 0, "max": MAX_RESOLUTION}), 32 | "text_g": ("STRING", {"multiline": True, "dynamicPrompts": True}), 33 | "text_l": ("STRING", {"multiline": True, "dynamicPrompts": True}), 34 | }} 35 | RETURN_TYPES = ("CONDITIONING",) 36 | FUNCTION = "encode" 37 | 38 | CATEGORY = "advanced/conditioning" 39 | 40 | def encode(self, clip, width, height, crop_w, crop_h, target_width, target_height, text_g, text_l): 41 | tokens = clip.tokenize(text_g) 42 | tokens["l"] = clip.tokenize(text_l)["l"] 43 | if len(tokens["l"]) != len(tokens["g"]): 44 | empty = clip.tokenize("") 45 | while len(tokens["l"]) < len(tokens["g"]): 46 | tokens["l"] += empty["l"] 47 | while len(tokens["l"]) > len(tokens["g"]): 48 | tokens["g"] += empty["g"] 49 | return (clip.encode_from_tokens_scheduled(tokens, add_dict={"width": width, "height": height, "crop_w": crop_w, "crop_h": crop_h, "target_width": target_width, "target_height": target_height}), ) 50 | 51 | NODE_CLASS_MAPPINGS = { 52 | "CLIPTextEncodeSDXLRefiner": CLIPTextEncodeSDXLRefiner, 53 | "CLIPTextEncodeSDXL": CLIPTextEncodeSDXL, 54 | } 55 | -------------------------------------------------------------------------------- /comfy_extras/nodes_cond.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | class CLIPTextEncodeControlnet: 4 | @classmethod 5 | def INPUT_TYPES(s): 6 | return {"required": {"clip": ("CLIP", ), "conditioning": ("CONDITIONING", ), "text": ("STRING", {"multiline": True, "dynamicPrompts": True})}} 7 | RETURN_TYPES = ("CONDITIONING",) 8 | FUNCTION = "encode" 9 | 10 | CATEGORY = "_for_testing/conditioning" 11 | 12 | def encode(self, clip, conditioning, text): 13 | tokens = clip.tokenize(text) 14 | cond, pooled = clip.encode_from_tokens(tokens, return_pooled=True) 15 | c = [] 16 | for t in conditioning: 17 | n = [t[0], t[1].copy()] 18 | n[1]['cross_attn_controlnet'] = cond 19 | n[1]['pooled_output_controlnet'] = pooled 20 | c.append(n) 21 | return (c, ) 22 | 23 | class T5TokenizerOptions: 24 | @classmethod 25 | def INPUT_TYPES(s): 26 | return { 27 | "required": { 28 | "clip": ("CLIP", ), 29 | "min_padding": ("INT", {"default": 0, "min": 0, "max": 10000, "step": 1}), 30 | "min_length": ("INT", {"default": 0, "min": 0, "max": 10000, "step": 1}), 31 | } 32 | } 33 | 34 | CATEGORY = "_for_testing/conditioning" 35 | RETURN_TYPES = ("CLIP",) 36 | FUNCTION = "set_options" 37 | 38 | def set_options(self, clip, min_padding, min_length): 39 | clip = clip.clone() 40 | for t5_type in ["t5xxl", "pile_t5xl", "t5base", "mt5xl", "umt5xxl"]: 41 | clip.set_tokenizer_option("{}_min_padding".format(t5_type), min_padding) 42 | clip.set_tokenizer_option("{}_min_length".format(t5_type), min_length) 43 | 44 | return (clip, ) 45 | 46 | NODE_CLASS_MAPPINGS = { 47 | "CLIPTextEncodeControlnet": CLIPTextEncodeControlnet, 48 | "T5TokenizerOptions": T5TokenizerOptions, 49 | } 50 | -------------------------------------------------------------------------------- /comfy_extras/nodes_controlnet.py: -------------------------------------------------------------------------------- 1 | from comfy.cldm.control_types import UNION_CONTROLNET_TYPES 2 | import nodes 3 | import comfy.utils 4 | 5 | class SetUnionControlNetType: 6 | @classmethod 7 | def INPUT_TYPES(s): 8 | return {"required": {"control_net": ("CONTROL_NET", ), 9 | "type": (["auto"] + list(UNION_CONTROLNET_TYPES.keys()),) 10 | }} 11 | 12 | CATEGORY = "conditioning/controlnet" 13 | RETURN_TYPES = ("CONTROL_NET",) 14 | 15 | FUNCTION = "set_controlnet_type" 16 | 17 | def set_controlnet_type(self, control_net, type): 18 | control_net = control_net.copy() 19 | type_number = UNION_CONTROLNET_TYPES.get(type, -1) 20 | if type_number >= 0: 21 | control_net.set_extra_arg("control_type", [type_number]) 22 | else: 23 | control_net.set_extra_arg("control_type", []) 24 | 25 | return (control_net,) 26 | 27 | class ControlNetInpaintingAliMamaApply(nodes.ControlNetApplyAdvanced): 28 | @classmethod 29 | def INPUT_TYPES(s): 30 | return {"required": {"positive": ("CONDITIONING", ), 31 | "negative": ("CONDITIONING", ), 32 | "control_net": ("CONTROL_NET", ), 33 | "vae": ("VAE", ), 34 | "image": ("IMAGE", ), 35 | "mask": ("MASK", ), 36 | "strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}), 37 | "start_percent": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001}), 38 | "end_percent": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001}) 39 | }} 40 | 41 | FUNCTION = "apply_inpaint_controlnet" 42 | 43 | CATEGORY = "conditioning/controlnet" 44 | 45 | def apply_inpaint_controlnet(self, positive, negative, control_net, vae, image, mask, strength, start_percent, end_percent): 46 | extra_concat = [] 47 | if control_net.concat_mask: 48 | mask = 1.0 - mask.reshape((-1, 1, mask.shape[-2], mask.shape[-1])) 49 | mask_apply = comfy.utils.common_upscale(mask, image.shape[2], image.shape[1], "bilinear", "center").round() 50 | image = image * mask_apply.movedim(1, -1).repeat(1, 1, 1, image.shape[3]) 51 | extra_concat = [mask] 52 | 53 | return self.apply_controlnet(positive, negative, control_net, image, strength, start_percent, end_percent, vae=vae, extra_concat=extra_concat) 54 | 55 | 56 | 57 | NODE_CLASS_MAPPINGS = { 58 | "SetUnionControlNetType": SetUnionControlNetType, 59 | "ControlNetInpaintingAliMamaApply": ControlNetInpaintingAliMamaApply, 60 | } 61 | -------------------------------------------------------------------------------- /comfy_extras/nodes_differential_diffusion.py: -------------------------------------------------------------------------------- 1 | # code adapted from https://github.com/exx8/differential-diffusion 2 | 3 | import torch 4 | 5 | class DifferentialDiffusion(): 6 | @classmethod 7 | def INPUT_TYPES(s): 8 | return {"required": {"model": ("MODEL", ), 9 | }} 10 | RETURN_TYPES = ("MODEL",) 11 | FUNCTION = "apply" 12 | CATEGORY = "_for_testing" 13 | INIT = False 14 | 15 | def apply(self, model): 16 | model = model.clone() 17 | model.set_model_denoise_mask_function(self.forward) 18 | return (model,) 19 | 20 | def forward(self, sigma: torch.Tensor, denoise_mask: torch.Tensor, extra_options: dict): 21 | model = extra_options["model"] 22 | step_sigmas = extra_options["sigmas"] 23 | sigma_to = model.inner_model.model_sampling.sigma_min 24 | if step_sigmas[-1] > sigma_to: 25 | sigma_to = step_sigmas[-1] 26 | sigma_from = step_sigmas[0] 27 | 28 | ts_from = model.inner_model.model_sampling.timestep(sigma_from) 29 | ts_to = model.inner_model.model_sampling.timestep(sigma_to) 30 | current_ts = model.inner_model.model_sampling.timestep(sigma[0]) 31 | 32 | threshold = (current_ts - ts_to) / (ts_from - ts_to) 33 | 34 | return (denoise_mask >= threshold).to(denoise_mask.dtype) 35 | 36 | 37 | NODE_CLASS_MAPPINGS = { 38 | "DifferentialDiffusion": DifferentialDiffusion, 39 | } 40 | NODE_DISPLAY_NAME_MAPPINGS = { 41 | "DifferentialDiffusion": "Differential Diffusion", 42 | } 43 | -------------------------------------------------------------------------------- /comfy_extras/nodes_flux.py: -------------------------------------------------------------------------------- 1 | import node_helpers 2 | 3 | class CLIPTextEncodeFlux: 4 | @classmethod 5 | def INPUT_TYPES(s): 6 | return {"required": { 7 | "clip": ("CLIP", ), 8 | "clip_l": ("STRING", {"multiline": True, "dynamicPrompts": True}), 9 | "t5xxl": ("STRING", {"multiline": True, "dynamicPrompts": True}), 10 | "guidance": ("FLOAT", {"default": 3.5, "min": 0.0, "max": 100.0, "step": 0.1}), 11 | }} 12 | RETURN_TYPES = ("CONDITIONING",) 13 | FUNCTION = "encode" 14 | 15 | CATEGORY = "advanced/conditioning/flux" 16 | 17 | def encode(self, clip, clip_l, t5xxl, guidance): 18 | tokens = clip.tokenize(clip_l) 19 | tokens["t5xxl"] = clip.tokenize(t5xxl)["t5xxl"] 20 | 21 | return (clip.encode_from_tokens_scheduled(tokens, add_dict={"guidance": guidance}), ) 22 | 23 | class FluxGuidance: 24 | @classmethod 25 | def INPUT_TYPES(s): 26 | return {"required": { 27 | "conditioning": ("CONDITIONING", ), 28 | "guidance": ("FLOAT", {"default": 3.5, "min": 0.0, "max": 100.0, "step": 0.1}), 29 | }} 30 | 31 | RETURN_TYPES = ("CONDITIONING",) 32 | FUNCTION = "append" 33 | 34 | CATEGORY = "advanced/conditioning/flux" 35 | 36 | def append(self, conditioning, guidance): 37 | c = node_helpers.conditioning_set_values(conditioning, {"guidance": guidance}) 38 | return (c, ) 39 | 40 | 41 | class FluxDisableGuidance: 42 | @classmethod 43 | def INPUT_TYPES(s): 44 | return {"required": { 45 | "conditioning": ("CONDITIONING", ), 46 | }} 47 | 48 | RETURN_TYPES = ("CONDITIONING",) 49 | FUNCTION = "append" 50 | 51 | CATEGORY = "advanced/conditioning/flux" 52 | DESCRIPTION = "This node completely disables the guidance embed on Flux and Flux like models" 53 | 54 | def append(self, conditioning): 55 | c = node_helpers.conditioning_set_values(conditioning, {"guidance": None}) 56 | return (c, ) 57 | 58 | 59 | NODE_CLASS_MAPPINGS = { 60 | "CLIPTextEncodeFlux": CLIPTextEncodeFlux, 61 | "FluxGuidance": FluxGuidance, 62 | "FluxDisableGuidance": FluxDisableGuidance, 63 | } 64 | -------------------------------------------------------------------------------- /comfy_extras/nodes_hidream.py: -------------------------------------------------------------------------------- 1 | import folder_paths 2 | import comfy.sd 3 | import comfy.model_management 4 | 5 | 6 | class QuadrupleCLIPLoader: 7 | @classmethod 8 | def INPUT_TYPES(s): 9 | return {"required": { "clip_name1": (folder_paths.get_filename_list("text_encoders"), ), 10 | "clip_name2": (folder_paths.get_filename_list("text_encoders"), ), 11 | "clip_name3": (folder_paths.get_filename_list("text_encoders"), ), 12 | "clip_name4": (folder_paths.get_filename_list("text_encoders"), ) 13 | }} 14 | RETURN_TYPES = ("CLIP",) 15 | FUNCTION = "load_clip" 16 | 17 | CATEGORY = "advanced/loaders" 18 | 19 | DESCRIPTION = "[Recipes]\n\nhidream: long clip-l, long clip-g, t5xxl, llama_8b_3.1_instruct" 20 | 21 | def load_clip(self, clip_name1, clip_name2, clip_name3, clip_name4): 22 | clip_path1 = folder_paths.get_full_path_or_raise("text_encoders", clip_name1) 23 | clip_path2 = folder_paths.get_full_path_or_raise("text_encoders", clip_name2) 24 | clip_path3 = folder_paths.get_full_path_or_raise("text_encoders", clip_name3) 25 | clip_path4 = folder_paths.get_full_path_or_raise("text_encoders", clip_name4) 26 | clip = comfy.sd.load_clip(ckpt_paths=[clip_path1, clip_path2, clip_path3, clip_path4], embedding_directory=folder_paths.get_folder_paths("embeddings")) 27 | return (clip,) 28 | 29 | class CLIPTextEncodeHiDream: 30 | @classmethod 31 | def INPUT_TYPES(s): 32 | return {"required": { 33 | "clip": ("CLIP", ), 34 | "clip_l": ("STRING", {"multiline": True, "dynamicPrompts": True}), 35 | "clip_g": ("STRING", {"multiline": True, "dynamicPrompts": True}), 36 | "t5xxl": ("STRING", {"multiline": True, "dynamicPrompts": True}), 37 | "llama": ("STRING", {"multiline": True, "dynamicPrompts": True}) 38 | }} 39 | RETURN_TYPES = ("CONDITIONING",) 40 | FUNCTION = "encode" 41 | 42 | CATEGORY = "advanced/conditioning" 43 | 44 | def encode(self, clip, clip_l, clip_g, t5xxl, llama): 45 | 46 | tokens = clip.tokenize(clip_g) 47 | tokens["l"] = clip.tokenize(clip_l)["l"] 48 | tokens["t5xxl"] = clip.tokenize(t5xxl)["t5xxl"] 49 | tokens["llama"] = clip.tokenize(llama)["llama"] 50 | return (clip.encode_from_tokens_scheduled(tokens), ) 51 | 52 | NODE_CLASS_MAPPINGS = { 53 | "QuadrupleCLIPLoader": QuadrupleCLIPLoader, 54 | "CLIPTextEncodeHiDream": CLIPTextEncodeHiDream, 55 | } 56 | -------------------------------------------------------------------------------- /comfy_extras/nodes_ip2p.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | class InstructPixToPixConditioning: 4 | @classmethod 5 | def INPUT_TYPES(s): 6 | return {"required": {"positive": ("CONDITIONING", ), 7 | "negative": ("CONDITIONING", ), 8 | "vae": ("VAE", ), 9 | "pixels": ("IMAGE", ), 10 | }} 11 | 12 | RETURN_TYPES = ("CONDITIONING","CONDITIONING","LATENT") 13 | RETURN_NAMES = ("positive", "negative", "latent") 14 | FUNCTION = "encode" 15 | 16 | CATEGORY = "conditioning/instructpix2pix" 17 | 18 | def encode(self, positive, negative, pixels, vae): 19 | x = (pixels.shape[1] // 8) * 8 20 | y = (pixels.shape[2] // 8) * 8 21 | 22 | if pixels.shape[1] != x or pixels.shape[2] != y: 23 | x_offset = (pixels.shape[1] % 8) // 2 24 | y_offset = (pixels.shape[2] % 8) // 2 25 | pixels = pixels[:,x_offset:x + x_offset, y_offset:y + y_offset,:] 26 | 27 | concat_latent = vae.encode(pixels) 28 | 29 | out_latent = {} 30 | out_latent["samples"] = torch.zeros_like(concat_latent) 31 | 32 | out = [] 33 | for conditioning in [positive, negative]: 34 | c = [] 35 | for t in conditioning: 36 | d = t[1].copy() 37 | d["concat_latent_image"] = concat_latent 38 | n = [t[0], d] 39 | c.append(n) 40 | out.append(c) 41 | return (out[0], out[1], out_latent) 42 | 43 | NODE_CLASS_MAPPINGS = { 44 | "InstructPixToPixConditioning": InstructPixToPixConditioning, 45 | } 46 | -------------------------------------------------------------------------------- /comfy_extras/nodes_mahiro.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.functional as F 3 | 4 | class Mahiro: 5 | @classmethod 6 | def INPUT_TYPES(s): 7 | return {"required": {"model": ("MODEL",), 8 | }} 9 | RETURN_TYPES = ("MODEL",) 10 | RETURN_NAMES = ("patched_model",) 11 | FUNCTION = "patch" 12 | CATEGORY = "_for_testing" 13 | DESCRIPTION = "Modify the guidance to scale more on the 'direction' of the positive prompt rather than the difference between the negative prompt." 14 | def patch(self, model): 15 | m = model.clone() 16 | def mahiro_normd(args): 17 | scale: float = args['cond_scale'] 18 | cond_p: torch.Tensor = args['cond_denoised'] 19 | uncond_p: torch.Tensor = args['uncond_denoised'] 20 | #naive leap 21 | leap = cond_p * scale 22 | #sim with uncond leap 23 | u_leap = uncond_p * scale 24 | cfg = args["denoised"] 25 | merge = (leap + cfg) / 2 26 | normu = torch.sqrt(u_leap.abs()) * u_leap.sign() 27 | normm = torch.sqrt(merge.abs()) * merge.sign() 28 | sim = F.cosine_similarity(normu, normm).mean() 29 | simsc = 2 * (sim+1) 30 | wm = (simsc*cfg + (4-simsc)*leap) / 4 31 | return wm 32 | m.set_model_sampler_post_cfg_function(mahiro_normd) 33 | return (m, ) 34 | 35 | NODE_CLASS_MAPPINGS = { 36 | "Mahiro": Mahiro 37 | } 38 | 39 | NODE_DISPLAY_NAME_MAPPINGS = { 40 | "Mahiro": "Mahiro is so cute that she deserves a better guidance function!! (。・ω・。)", 41 | } 42 | -------------------------------------------------------------------------------- /comfy_extras/nodes_mochi.py: -------------------------------------------------------------------------------- 1 | import nodes 2 | import torch 3 | import comfy.model_management 4 | 5 | class EmptyMochiLatentVideo: 6 | @classmethod 7 | def INPUT_TYPES(s): 8 | return {"required": { "width": ("INT", {"default": 848, "min": 16, "max": nodes.MAX_RESOLUTION, "step": 16}), 9 | "height": ("INT", {"default": 480, "min": 16, "max": nodes.MAX_RESOLUTION, "step": 16}), 10 | "length": ("INT", {"default": 25, "min": 7, "max": nodes.MAX_RESOLUTION, "step": 6}), 11 | "batch_size": ("INT", {"default": 1, "min": 1, "max": 4096})}} 12 | RETURN_TYPES = ("LATENT",) 13 | FUNCTION = "generate" 14 | 15 | CATEGORY = "latent/video" 16 | 17 | def generate(self, width, height, length, batch_size=1): 18 | latent = torch.zeros([batch_size, 12, ((length - 1) // 6) + 1, height // 8, width // 8], device=comfy.model_management.intermediate_device()) 19 | return ({"samples":latent}, ) 20 | 21 | NODE_CLASS_MAPPINGS = { 22 | "EmptyMochiLatentVideo": EmptyMochiLatentVideo, 23 | } 24 | -------------------------------------------------------------------------------- /comfy_extras/nodes_model_downscale.py: -------------------------------------------------------------------------------- 1 | import comfy.utils 2 | 3 | class PatchModelAddDownscale: 4 | upscale_methods = ["bicubic", "nearest-exact", "bilinear", "area", "bislerp"] 5 | @classmethod 6 | def INPUT_TYPES(s): 7 | return {"required": { "model": ("MODEL",), 8 | "block_number": ("INT", {"default": 3, "min": 1, "max": 32, "step": 1}), 9 | "downscale_factor": ("FLOAT", {"default": 2.0, "min": 0.1, "max": 9.0, "step": 0.001}), 10 | "start_percent": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001}), 11 | "end_percent": ("FLOAT", {"default": 0.35, "min": 0.0, "max": 1.0, "step": 0.001}), 12 | "downscale_after_skip": ("BOOLEAN", {"default": True}), 13 | "downscale_method": (s.upscale_methods,), 14 | "upscale_method": (s.upscale_methods,), 15 | }} 16 | RETURN_TYPES = ("MODEL",) 17 | FUNCTION = "patch" 18 | 19 | CATEGORY = "model_patches/unet" 20 | 21 | def patch(self, model, block_number, downscale_factor, start_percent, end_percent, downscale_after_skip, downscale_method, upscale_method): 22 | model_sampling = model.get_model_object("model_sampling") 23 | sigma_start = model_sampling.percent_to_sigma(start_percent) 24 | sigma_end = model_sampling.percent_to_sigma(end_percent) 25 | 26 | def input_block_patch(h, transformer_options): 27 | if transformer_options["block"][1] == block_number: 28 | sigma = transformer_options["sigmas"][0].item() 29 | if sigma <= sigma_start and sigma >= sigma_end: 30 | h = comfy.utils.common_upscale(h, round(h.shape[-1] * (1.0 / downscale_factor)), round(h.shape[-2] * (1.0 / downscale_factor)), downscale_method, "disabled") 31 | return h 32 | 33 | def output_block_patch(h, hsp, transformer_options): 34 | if h.shape[2] != hsp.shape[2]: 35 | h = comfy.utils.common_upscale(h, hsp.shape[-1], hsp.shape[-2], upscale_method, "disabled") 36 | return h, hsp 37 | 38 | m = model.clone() 39 | if downscale_after_skip: 40 | m.set_model_input_block_patch_after_skip(input_block_patch) 41 | else: 42 | m.set_model_input_block_patch(input_block_patch) 43 | m.set_model_output_block_patch(output_block_patch) 44 | return (m, ) 45 | 46 | NODE_CLASS_MAPPINGS = { 47 | "PatchModelAddDownscale": PatchModelAddDownscale, 48 | } 49 | 50 | NODE_DISPLAY_NAME_MAPPINGS = { 51 | # Sampling 52 | "PatchModelAddDownscale": "PatchModelAddDownscale (Kohya Deep Shrink)", 53 | } 54 | -------------------------------------------------------------------------------- /comfy_extras/nodes_optimalsteps.py: -------------------------------------------------------------------------------- 1 | # from https://github.com/bebebe666/OptimalSteps 2 | 3 | 4 | import numpy as np 5 | import torch 6 | 7 | def loglinear_interp(t_steps, num_steps): 8 | """ 9 | Performs log-linear interpolation of a given array of decreasing numbers. 10 | """ 11 | xs = np.linspace(0, 1, len(t_steps)) 12 | ys = np.log(t_steps[::-1]) 13 | 14 | new_xs = np.linspace(0, 1, num_steps) 15 | new_ys = np.interp(new_xs, xs, ys) 16 | 17 | interped_ys = np.exp(new_ys)[::-1].copy() 18 | return interped_ys 19 | 20 | 21 | NOISE_LEVELS = {"FLUX": [0.9968, 0.9886, 0.9819, 0.975, 0.966, 0.9471, 0.9158, 0.8287, 0.5512, 0.2808, 0.001], 22 | "Wan":[1.0, 0.997, 0.995, 0.993, 0.991, 0.989, 0.987, 0.985, 0.98, 0.975, 0.973, 0.968, 0.96, 0.946, 0.927, 0.902, 0.864, 0.776, 0.539, 0.208, 0.001], 23 | "Chroma": [0.992, 0.99, 0.988, 0.985, 0.982, 0.978, 0.973, 0.968, 0.961, 0.953, 0.943, 0.931, 0.917, 0.9, 0.881, 0.858, 0.832, 0.802, 0.769, 0.731, 0.69, 0.646, 0.599, 0.55, 0.501, 0.451, 0.402, 0.355, 0.311, 0.27, 0.232, 0.199, 0.169, 0.143, 0.12, 0.101, 0.084, 0.07, 0.058, 0.048, 0.001], 24 | } 25 | 26 | class OptimalStepsScheduler: 27 | @classmethod 28 | def INPUT_TYPES(s): 29 | return {"required": 30 | {"model_type": (["FLUX", "Wan", "Chroma"], ), 31 | "steps": ("INT", {"default": 20, "min": 3, "max": 1000}), 32 | "denoise": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01}), 33 | } 34 | } 35 | RETURN_TYPES = ("SIGMAS",) 36 | CATEGORY = "sampling/custom_sampling/schedulers" 37 | 38 | FUNCTION = "get_sigmas" 39 | 40 | def get_sigmas(self, model_type, steps, denoise): 41 | total_steps = steps 42 | if denoise < 1.0: 43 | if denoise <= 0.0: 44 | return (torch.FloatTensor([]),) 45 | total_steps = round(steps * denoise) 46 | 47 | sigmas = NOISE_LEVELS[model_type][:] 48 | if (steps + 1) != len(sigmas): 49 | sigmas = loglinear_interp(sigmas, steps + 1) 50 | 51 | sigmas = sigmas[-(total_steps + 1):] 52 | sigmas[-1] = 0 53 | return (torch.FloatTensor(sigmas), ) 54 | 55 | NODE_CLASS_MAPPINGS = { 56 | "OptimalStepsScheduler": OptimalStepsScheduler, 57 | } 58 | -------------------------------------------------------------------------------- /comfy_extras/nodes_pag.py: -------------------------------------------------------------------------------- 1 | #Modified/simplified version of the node from: https://github.com/pamparamm/sd-perturbed-attention 2 | #If you want the one with more options see the above repo. 3 | 4 | #My modified one here is more basic but has less chances of breaking with ComfyUI updates. 5 | 6 | import comfy.model_patcher 7 | import comfy.samplers 8 | 9 | class PerturbedAttentionGuidance: 10 | @classmethod 11 | def INPUT_TYPES(s): 12 | return { 13 | "required": { 14 | "model": ("MODEL",), 15 | "scale": ("FLOAT", {"default": 3.0, "min": 0.0, "max": 100.0, "step": 0.01, "round": 0.01}), 16 | } 17 | } 18 | 19 | RETURN_TYPES = ("MODEL",) 20 | FUNCTION = "patch" 21 | 22 | CATEGORY = "model_patches/unet" 23 | 24 | def patch(self, model, scale): 25 | unet_block = "middle" 26 | unet_block_id = 0 27 | m = model.clone() 28 | 29 | def perturbed_attention(q, k, v, extra_options, mask=None): 30 | return v 31 | 32 | def post_cfg_function(args): 33 | model = args["model"] 34 | cond_pred = args["cond_denoised"] 35 | cond = args["cond"] 36 | cfg_result = args["denoised"] 37 | sigma = args["sigma"] 38 | model_options = args["model_options"].copy() 39 | x = args["input"] 40 | 41 | if scale == 0: 42 | return cfg_result 43 | 44 | # Replace Self-attention with PAG 45 | model_options = comfy.model_patcher.set_model_options_patch_replace(model_options, perturbed_attention, "attn1", unet_block, unet_block_id) 46 | (pag,) = comfy.samplers.calc_cond_batch(model, [cond], x, sigma, model_options) 47 | 48 | return cfg_result + (cond_pred - pag) * scale 49 | 50 | m.set_model_sampler_post_cfg_function(post_cfg_function) 51 | 52 | return (m,) 53 | 54 | NODE_CLASS_MAPPINGS = { 55 | "PerturbedAttentionGuidance": PerturbedAttentionGuidance, 56 | } 57 | -------------------------------------------------------------------------------- /comfy_extras/nodes_pixart.py: -------------------------------------------------------------------------------- 1 | from nodes import MAX_RESOLUTION 2 | 3 | class CLIPTextEncodePixArtAlpha: 4 | @classmethod 5 | def INPUT_TYPES(s): 6 | return {"required": { 7 | "width": ("INT", {"default": 1024.0, "min": 0, "max": MAX_RESOLUTION}), 8 | "height": ("INT", {"default": 1024.0, "min": 0, "max": MAX_RESOLUTION}), 9 | # "aspect_ratio": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}), 10 | "text": ("STRING", {"multiline": True, "dynamicPrompts": True}), "clip": ("CLIP", ), 11 | }} 12 | 13 | RETURN_TYPES = ("CONDITIONING",) 14 | FUNCTION = "encode" 15 | CATEGORY = "advanced/conditioning" 16 | DESCRIPTION = "Encodes text and sets the resolution conditioning for PixArt Alpha. Does not apply to PixArt Sigma." 17 | 18 | def encode(self, clip, width, height, text): 19 | tokens = clip.tokenize(text) 20 | return (clip.encode_from_tokens_scheduled(tokens, add_dict={"width": width, "height": height}),) 21 | 22 | NODE_CLASS_MAPPINGS = { 23 | "CLIPTextEncodePixArtAlpha": CLIPTextEncodePixArtAlpha, 24 | } 25 | -------------------------------------------------------------------------------- /comfy_extras/nodes_preview_any.py: -------------------------------------------------------------------------------- 1 | import json 2 | from comfy.comfy_types.node_typing import IO 3 | 4 | # Preview Any - original implement from 5 | # https://github.com/rgthree/rgthree-comfy/blob/main/py/display_any.py 6 | # upstream requested in https://github.com/Kosinkadink/rfcs/blob/main/rfcs/0000-corenodes.md#preview-nodes 7 | class PreviewAny(): 8 | @classmethod 9 | def INPUT_TYPES(cls): 10 | return { 11 | "required": {"source": (IO.ANY, {})}, 12 | } 13 | 14 | RETURN_TYPES = () 15 | FUNCTION = "main" 16 | OUTPUT_NODE = True 17 | 18 | CATEGORY = "utils" 19 | 20 | def main(self, source=None): 21 | value = 'None' 22 | if isinstance(source, str): 23 | value = source 24 | elif isinstance(source, (int, float, bool)): 25 | value = str(source) 26 | elif source is not None: 27 | try: 28 | value = json.dumps(source) 29 | except Exception: 30 | try: 31 | value = str(source) 32 | except Exception: 33 | value = 'source exists, but could not be serialized.' 34 | 35 | return {"ui": {"text": (value,)}} 36 | 37 | NODE_CLASS_MAPPINGS = { 38 | "PreviewAny": PreviewAny, 39 | } 40 | 41 | NODE_DISPLAY_NAME_MAPPINGS = { 42 | "PreviewAny": "Preview Any", 43 | } 44 | -------------------------------------------------------------------------------- /comfy_extras/nodes_primitive.py: -------------------------------------------------------------------------------- 1 | # Primitive nodes that are evaluated at backend. 2 | from __future__ import annotations 3 | 4 | import sys 5 | 6 | from comfy.comfy_types.node_typing import ComfyNodeABC, InputTypeDict, IO 7 | 8 | 9 | class String(ComfyNodeABC): 10 | @classmethod 11 | def INPUT_TYPES(cls) -> InputTypeDict: 12 | return { 13 | "required": {"value": (IO.STRING, {})}, 14 | } 15 | 16 | RETURN_TYPES = (IO.STRING,) 17 | FUNCTION = "execute" 18 | CATEGORY = "utils/primitive" 19 | 20 | def execute(self, value: str) -> tuple[str]: 21 | return (value,) 22 | 23 | 24 | class StringMultiline(ComfyNodeABC): 25 | @classmethod 26 | def INPUT_TYPES(cls) -> InputTypeDict: 27 | return { 28 | "required": {"value": (IO.STRING, {"multiline": True,},)}, 29 | } 30 | 31 | RETURN_TYPES = (IO.STRING,) 32 | FUNCTION = "execute" 33 | CATEGORY = "utils/primitive" 34 | 35 | def execute(self, value: str) -> tuple[str]: 36 | return (value,) 37 | 38 | 39 | class Int(ComfyNodeABC): 40 | @classmethod 41 | def INPUT_TYPES(cls) -> InputTypeDict: 42 | return { 43 | "required": {"value": (IO.INT, {"min": -sys.maxsize, "max": sys.maxsize, "control_after_generate": True})}, 44 | } 45 | 46 | RETURN_TYPES = (IO.INT,) 47 | FUNCTION = "execute" 48 | CATEGORY = "utils/primitive" 49 | 50 | def execute(self, value: int) -> tuple[int]: 51 | return (value,) 52 | 53 | 54 | class Float(ComfyNodeABC): 55 | @classmethod 56 | def INPUT_TYPES(cls) -> InputTypeDict: 57 | return { 58 | "required": {"value": (IO.FLOAT, {"min": -sys.maxsize, "max": sys.maxsize})}, 59 | } 60 | 61 | RETURN_TYPES = (IO.FLOAT,) 62 | FUNCTION = "execute" 63 | CATEGORY = "utils/primitive" 64 | 65 | def execute(self, value: float) -> tuple[float]: 66 | return (value,) 67 | 68 | 69 | class Boolean(ComfyNodeABC): 70 | @classmethod 71 | def INPUT_TYPES(cls) -> InputTypeDict: 72 | return { 73 | "required": {"value": (IO.BOOLEAN, {})}, 74 | } 75 | 76 | RETURN_TYPES = (IO.BOOLEAN,) 77 | FUNCTION = "execute" 78 | CATEGORY = "utils/primitive" 79 | 80 | def execute(self, value: bool) -> tuple[bool]: 81 | return (value,) 82 | 83 | 84 | NODE_CLASS_MAPPINGS = { 85 | "PrimitiveString": String, 86 | "PrimitiveStringMultiline": StringMultiline, 87 | "PrimitiveInt": Int, 88 | "PrimitiveFloat": Float, 89 | "PrimitiveBoolean": Boolean, 90 | } 91 | 92 | NODE_DISPLAY_NAME_MAPPINGS = { 93 | "PrimitiveString": "String", 94 | "PrimitiveStringMultiline": "String (Multiline)", 95 | "PrimitiveInt": "Int", 96 | "PrimitiveFloat": "Float", 97 | "PrimitiveBoolean": "Boolean", 98 | } 99 | -------------------------------------------------------------------------------- /comfy_extras/nodes_sdupscale.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import comfy.utils 3 | 4 | class SD_4XUpscale_Conditioning: 5 | @classmethod 6 | def INPUT_TYPES(s): 7 | return {"required": { "images": ("IMAGE",), 8 | "positive": ("CONDITIONING",), 9 | "negative": ("CONDITIONING",), 10 | "scale_ratio": ("FLOAT", {"default": 4.0, "min": 0.0, "max": 10.0, "step": 0.01}), 11 | "noise_augmentation": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001}), 12 | }} 13 | RETURN_TYPES = ("CONDITIONING", "CONDITIONING", "LATENT") 14 | RETURN_NAMES = ("positive", "negative", "latent") 15 | 16 | FUNCTION = "encode" 17 | 18 | CATEGORY = "conditioning/upscale_diffusion" 19 | 20 | def encode(self, images, positive, negative, scale_ratio, noise_augmentation): 21 | width = max(1, round(images.shape[-2] * scale_ratio)) 22 | height = max(1, round(images.shape[-3] * scale_ratio)) 23 | 24 | pixels = comfy.utils.common_upscale((images.movedim(-1,1) * 2.0) - 1.0, width // 4, height // 4, "bilinear", "center") 25 | 26 | out_cp = [] 27 | out_cn = [] 28 | 29 | for t in positive: 30 | n = [t[0], t[1].copy()] 31 | n[1]['concat_image'] = pixels 32 | n[1]['noise_augmentation'] = noise_augmentation 33 | out_cp.append(n) 34 | 35 | for t in negative: 36 | n = [t[0], t[1].copy()] 37 | n[1]['concat_image'] = pixels 38 | n[1]['noise_augmentation'] = noise_augmentation 39 | out_cn.append(n) 40 | 41 | latent = torch.zeros([images.shape[0], 4, height // 4, width // 4]) 42 | return (out_cp, out_cn, {"samples":latent}) 43 | 44 | NODE_CLASS_MAPPINGS = { 45 | "SD_4XUpscale_Conditioning": SD_4XUpscale_Conditioning, 46 | } 47 | -------------------------------------------------------------------------------- /comfy_extras/nodes_torch_compile.py: -------------------------------------------------------------------------------- 1 | from comfy_api.torch_helpers import set_torch_compile_wrapper 2 | 3 | 4 | class TorchCompileModel: 5 | @classmethod 6 | def INPUT_TYPES(s): 7 | return {"required": { "model": ("MODEL",), 8 | "backend": (["inductor", "cudagraphs"],), 9 | }} 10 | RETURN_TYPES = ("MODEL",) 11 | FUNCTION = "patch" 12 | 13 | CATEGORY = "_for_testing" 14 | EXPERIMENTAL = True 15 | 16 | def patch(self, model, backend): 17 | m = model.clone() 18 | set_torch_compile_wrapper(model=m, backend=backend) 19 | return (m, ) 20 | 21 | NODE_CLASS_MAPPINGS = { 22 | "TorchCompileModel": TorchCompileModel, 23 | } 24 | -------------------------------------------------------------------------------- /comfy_extras/nodes_webcam.py: -------------------------------------------------------------------------------- 1 | import nodes 2 | import folder_paths 3 | 4 | MAX_RESOLUTION = nodes.MAX_RESOLUTION 5 | 6 | 7 | class WebcamCapture(nodes.LoadImage): 8 | @classmethod 9 | def INPUT_TYPES(s): 10 | return { 11 | "required": { 12 | "image": ("WEBCAM", {}), 13 | "width": ("INT", {"default": 0, "min": 0, "max": MAX_RESOLUTION, "step": 1}), 14 | "height": ("INT", {"default": 0, "min": 0, "max": MAX_RESOLUTION, "step": 1}), 15 | "capture_on_queue": ("BOOLEAN", {"default": True}), 16 | } 17 | } 18 | RETURN_TYPES = ("IMAGE",) 19 | FUNCTION = "load_capture" 20 | 21 | CATEGORY = "image" 22 | 23 | def load_capture(self, image, **kwargs): 24 | return super().load_image(folder_paths.get_annotated_filepath(image)) 25 | 26 | 27 | NODE_CLASS_MAPPINGS = { 28 | "WebcamCapture": WebcamCapture, 29 | } 30 | 31 | NODE_DISPLAY_NAME_MAPPINGS = { 32 | "WebcamCapture": "Webcam Capture", 33 | } 34 | -------------------------------------------------------------------------------- /comfyui_version.py: -------------------------------------------------------------------------------- 1 | # This file is automatically generated by the build process when version is 2 | # updated in pyproject.toml. 3 | __version__ = "0.3.39" 4 | -------------------------------------------------------------------------------- /custom_nodes/websocket_image_save.py: -------------------------------------------------------------------------------- 1 | from PIL import Image 2 | import numpy as np 3 | import comfy.utils 4 | import time 5 | 6 | #You can use this node to save full size images through the websocket, the 7 | #images will be sent in exactly the same format as the image previews: as 8 | #binary images on the websocket with a 8 byte header indicating the type 9 | #of binary message (first 4 bytes) and the image format (next 4 bytes). 10 | 11 | #Note that no metadata will be put in the images saved with this node. 12 | 13 | class SaveImageWebsocket: 14 | @classmethod 15 | def INPUT_TYPES(s): 16 | return {"required": 17 | {"images": ("IMAGE", ),} 18 | } 19 | 20 | RETURN_TYPES = () 21 | FUNCTION = "save_images" 22 | 23 | OUTPUT_NODE = True 24 | 25 | CATEGORY = "api/image" 26 | 27 | def save_images(self, images): 28 | pbar = comfy.utils.ProgressBar(images.shape[0]) 29 | step = 0 30 | for image in images: 31 | i = 255. * image.cpu().numpy() 32 | img = Image.fromarray(np.clip(i, 0, 255).astype(np.uint8)) 33 | pbar.update_absolute(step, images.shape[0], ("PNG", img, None)) 34 | step += 1 35 | 36 | return {} 37 | 38 | @classmethod 39 | def IS_CHANGED(s, images): 40 | return time.time() 41 | 42 | NODE_CLASS_MAPPINGS = { 43 | "SaveImageWebsocket": SaveImageWebsocket, 44 | } 45 | -------------------------------------------------------------------------------- /extra_model_paths.yaml.example: -------------------------------------------------------------------------------- 1 | #Rename this to extra_model_paths.yaml and ComfyUI will load it 2 | 3 | 4 | #config for a1111 ui 5 | #all you have to do is change the base_path to where yours is installed 6 | a111: 7 | base_path: path/to/stable-diffusion-webui/ 8 | 9 | checkpoints: models/Stable-diffusion 10 | configs: models/Stable-diffusion 11 | vae: models/VAE 12 | loras: | 13 | models/Lora 14 | models/LyCORIS 15 | upscale_models: | 16 | models/ESRGAN 17 | models/RealESRGAN 18 | models/SwinIR 19 | embeddings: embeddings 20 | hypernetworks: models/hypernetworks 21 | controlnet: models/ControlNet 22 | 23 | #config for comfyui 24 | #your base path should be either an existing comfy install or a central folder where you store all of your models, loras, etc. 25 | 26 | #comfyui: 27 | # base_path: path/to/comfyui/ 28 | # # You can use is_default to mark that these folders should be listed first, and used as the default dirs for eg downloads 29 | # #is_default: true 30 | # checkpoints: models/checkpoints/ 31 | # clip: models/clip/ 32 | # clip_vision: models/clip_vision/ 33 | # configs: models/configs/ 34 | # controlnet: models/controlnet/ 35 | # diffusion_models: | 36 | # models/diffusion_models 37 | # models/unet 38 | # embeddings: models/embeddings/ 39 | # loras: models/loras/ 40 | # upscale_models: models/upscale_models/ 41 | # vae: models/vae/ 42 | 43 | #other_ui: 44 | # base_path: path/to/ui 45 | # checkpoints: models/checkpoints 46 | # gligen: models/gligen 47 | # custom_nodes: path/custom_nodes 48 | -------------------------------------------------------------------------------- /hook_breaker_ac10a0.py: -------------------------------------------------------------------------------- 1 | # Prevent custom nodes from hooking anything important 2 | import comfy.model_management 3 | 4 | HOOK_BREAK = [(comfy.model_management, "cast_to")] 5 | 6 | 7 | SAVED_FUNCTIONS = [] 8 | 9 | 10 | def save_functions(): 11 | for f in HOOK_BREAK: 12 | SAVED_FUNCTIONS.append((f[0], f[1], getattr(f[0], f[1]))) 13 | 14 | 15 | def restore_functions(): 16 | for f in SAVED_FUNCTIONS: 17 | setattr(f[0], f[1], f[2]) 18 | -------------------------------------------------------------------------------- /input/example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/comfyanonymous/ComfyUI/312d511630db4907c3bed04dee297b28f61941a8/input/example.png -------------------------------------------------------------------------------- /models/checkpoints/put_checkpoints_here: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/comfyanonymous/ComfyUI/312d511630db4907c3bed04dee297b28f61941a8/models/checkpoints/put_checkpoints_here -------------------------------------------------------------------------------- /models/clip/put_clip_or_text_encoder_models_here: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/comfyanonymous/ComfyUI/312d511630db4907c3bed04dee297b28f61941a8/models/clip/put_clip_or_text_encoder_models_here -------------------------------------------------------------------------------- /models/clip_vision/put_clip_vision_models_here: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/comfyanonymous/ComfyUI/312d511630db4907c3bed04dee297b28f61941a8/models/clip_vision/put_clip_vision_models_here -------------------------------------------------------------------------------- /models/configs/anything_v3.yaml: -------------------------------------------------------------------------------- 1 | model: 2 | base_learning_rate: 1.0e-04 3 | target: ldm.models.diffusion.ddpm.LatentDiffusion 4 | params: 5 | linear_start: 0.00085 6 | linear_end: 0.0120 7 | num_timesteps_cond: 1 8 | log_every_t: 200 9 | timesteps: 1000 10 | first_stage_key: "jpg" 11 | cond_stage_key: "txt" 12 | image_size: 64 13 | channels: 4 14 | cond_stage_trainable: false # Note: different from the one we trained before 15 | conditioning_key: crossattn 16 | monitor: val/loss_simple_ema 17 | scale_factor: 0.18215 18 | use_ema: False 19 | 20 | scheduler_config: # 10000 warmup steps 21 | target: ldm.lr_scheduler.LambdaLinearScheduler 22 | params: 23 | warm_up_steps: [ 10000 ] 24 | cycle_lengths: [ 10000000000000 ] # incredibly large number to prevent corner cases 25 | f_start: [ 1.e-6 ] 26 | f_max: [ 1. ] 27 | f_min: [ 1. ] 28 | 29 | unet_config: 30 | target: ldm.modules.diffusionmodules.openaimodel.UNetModel 31 | params: 32 | image_size: 32 # unused 33 | in_channels: 4 34 | out_channels: 4 35 | model_channels: 320 36 | attention_resolutions: [ 4, 2, 1 ] 37 | num_res_blocks: 2 38 | channel_mult: [ 1, 2, 4, 4 ] 39 | num_heads: 8 40 | use_spatial_transformer: True 41 | transformer_depth: 1 42 | context_dim: 768 43 | use_checkpoint: True 44 | legacy: False 45 | 46 | first_stage_config: 47 | target: ldm.models.autoencoder.AutoencoderKL 48 | params: 49 | embed_dim: 4 50 | monitor: val/rec_loss 51 | ddconfig: 52 | double_z: true 53 | z_channels: 4 54 | resolution: 256 55 | in_channels: 3 56 | out_ch: 3 57 | ch: 128 58 | ch_mult: 59 | - 1 60 | - 2 61 | - 4 62 | - 4 63 | num_res_blocks: 2 64 | attn_resolutions: [] 65 | dropout: 0.0 66 | lossconfig: 67 | target: torch.nn.Identity 68 | 69 | cond_stage_config: 70 | target: ldm.modules.encoders.modules.FrozenCLIPEmbedder 71 | params: 72 | layer: "hidden" 73 | layer_idx: -2 74 | -------------------------------------------------------------------------------- /models/configs/v1-inference.yaml: -------------------------------------------------------------------------------- 1 | model: 2 | base_learning_rate: 1.0e-04 3 | target: ldm.models.diffusion.ddpm.LatentDiffusion 4 | params: 5 | linear_start: 0.00085 6 | linear_end: 0.0120 7 | num_timesteps_cond: 1 8 | log_every_t: 200 9 | timesteps: 1000 10 | first_stage_key: "jpg" 11 | cond_stage_key: "txt" 12 | image_size: 64 13 | channels: 4 14 | cond_stage_trainable: false # Note: different from the one we trained before 15 | conditioning_key: crossattn 16 | monitor: val/loss_simple_ema 17 | scale_factor: 0.18215 18 | use_ema: False 19 | 20 | scheduler_config: # 10000 warmup steps 21 | target: ldm.lr_scheduler.LambdaLinearScheduler 22 | params: 23 | warm_up_steps: [ 10000 ] 24 | cycle_lengths: [ 10000000000000 ] # incredibly large number to prevent corner cases 25 | f_start: [ 1.e-6 ] 26 | f_max: [ 1. ] 27 | f_min: [ 1. ] 28 | 29 | unet_config: 30 | target: ldm.modules.diffusionmodules.openaimodel.UNetModel 31 | params: 32 | image_size: 32 # unused 33 | in_channels: 4 34 | out_channels: 4 35 | model_channels: 320 36 | attention_resolutions: [ 4, 2, 1 ] 37 | num_res_blocks: 2 38 | channel_mult: [ 1, 2, 4, 4 ] 39 | num_heads: 8 40 | use_spatial_transformer: True 41 | transformer_depth: 1 42 | context_dim: 768 43 | use_checkpoint: True 44 | legacy: False 45 | 46 | first_stage_config: 47 | target: ldm.models.autoencoder.AutoencoderKL 48 | params: 49 | embed_dim: 4 50 | monitor: val/rec_loss 51 | ddconfig: 52 | double_z: true 53 | z_channels: 4 54 | resolution: 256 55 | in_channels: 3 56 | out_ch: 3 57 | ch: 128 58 | ch_mult: 59 | - 1 60 | - 2 61 | - 4 62 | - 4 63 | num_res_blocks: 2 64 | attn_resolutions: [] 65 | dropout: 0.0 66 | lossconfig: 67 | target: torch.nn.Identity 68 | 69 | cond_stage_config: 70 | target: ldm.modules.encoders.modules.FrozenCLIPEmbedder 71 | -------------------------------------------------------------------------------- /models/configs/v1-inference_clip_skip_2.yaml: -------------------------------------------------------------------------------- 1 | model: 2 | base_learning_rate: 1.0e-04 3 | target: ldm.models.diffusion.ddpm.LatentDiffusion 4 | params: 5 | linear_start: 0.00085 6 | linear_end: 0.0120 7 | num_timesteps_cond: 1 8 | log_every_t: 200 9 | timesteps: 1000 10 | first_stage_key: "jpg" 11 | cond_stage_key: "txt" 12 | image_size: 64 13 | channels: 4 14 | cond_stage_trainable: false # Note: different from the one we trained before 15 | conditioning_key: crossattn 16 | monitor: val/loss_simple_ema 17 | scale_factor: 0.18215 18 | use_ema: False 19 | 20 | scheduler_config: # 10000 warmup steps 21 | target: ldm.lr_scheduler.LambdaLinearScheduler 22 | params: 23 | warm_up_steps: [ 10000 ] 24 | cycle_lengths: [ 10000000000000 ] # incredibly large number to prevent corner cases 25 | f_start: [ 1.e-6 ] 26 | f_max: [ 1. ] 27 | f_min: [ 1. ] 28 | 29 | unet_config: 30 | target: ldm.modules.diffusionmodules.openaimodel.UNetModel 31 | params: 32 | image_size: 32 # unused 33 | in_channels: 4 34 | out_channels: 4 35 | model_channels: 320 36 | attention_resolutions: [ 4, 2, 1 ] 37 | num_res_blocks: 2 38 | channel_mult: [ 1, 2, 4, 4 ] 39 | num_heads: 8 40 | use_spatial_transformer: True 41 | transformer_depth: 1 42 | context_dim: 768 43 | use_checkpoint: True 44 | legacy: False 45 | 46 | first_stage_config: 47 | target: ldm.models.autoencoder.AutoencoderKL 48 | params: 49 | embed_dim: 4 50 | monitor: val/rec_loss 51 | ddconfig: 52 | double_z: true 53 | z_channels: 4 54 | resolution: 256 55 | in_channels: 3 56 | out_ch: 3 57 | ch: 128 58 | ch_mult: 59 | - 1 60 | - 2 61 | - 4 62 | - 4 63 | num_res_blocks: 2 64 | attn_resolutions: [] 65 | dropout: 0.0 66 | lossconfig: 67 | target: torch.nn.Identity 68 | 69 | cond_stage_config: 70 | target: ldm.modules.encoders.modules.FrozenCLIPEmbedder 71 | params: 72 | layer: "hidden" 73 | layer_idx: -2 74 | -------------------------------------------------------------------------------- /models/configs/v1-inference_clip_skip_2_fp16.yaml: -------------------------------------------------------------------------------- 1 | model: 2 | base_learning_rate: 1.0e-04 3 | target: ldm.models.diffusion.ddpm.LatentDiffusion 4 | params: 5 | linear_start: 0.00085 6 | linear_end: 0.0120 7 | num_timesteps_cond: 1 8 | log_every_t: 200 9 | timesteps: 1000 10 | first_stage_key: "jpg" 11 | cond_stage_key: "txt" 12 | image_size: 64 13 | channels: 4 14 | cond_stage_trainable: false # Note: different from the one we trained before 15 | conditioning_key: crossattn 16 | monitor: val/loss_simple_ema 17 | scale_factor: 0.18215 18 | use_ema: False 19 | 20 | scheduler_config: # 10000 warmup steps 21 | target: ldm.lr_scheduler.LambdaLinearScheduler 22 | params: 23 | warm_up_steps: [ 10000 ] 24 | cycle_lengths: [ 10000000000000 ] # incredibly large number to prevent corner cases 25 | f_start: [ 1.e-6 ] 26 | f_max: [ 1. ] 27 | f_min: [ 1. ] 28 | 29 | unet_config: 30 | target: ldm.modules.diffusionmodules.openaimodel.UNetModel 31 | params: 32 | use_fp16: True 33 | image_size: 32 # unused 34 | in_channels: 4 35 | out_channels: 4 36 | model_channels: 320 37 | attention_resolutions: [ 4, 2, 1 ] 38 | num_res_blocks: 2 39 | channel_mult: [ 1, 2, 4, 4 ] 40 | num_heads: 8 41 | use_spatial_transformer: True 42 | transformer_depth: 1 43 | context_dim: 768 44 | use_checkpoint: True 45 | legacy: False 46 | 47 | first_stage_config: 48 | target: ldm.models.autoencoder.AutoencoderKL 49 | params: 50 | embed_dim: 4 51 | monitor: val/rec_loss 52 | ddconfig: 53 | double_z: true 54 | z_channels: 4 55 | resolution: 256 56 | in_channels: 3 57 | out_ch: 3 58 | ch: 128 59 | ch_mult: 60 | - 1 61 | - 2 62 | - 4 63 | - 4 64 | num_res_blocks: 2 65 | attn_resolutions: [] 66 | dropout: 0.0 67 | lossconfig: 68 | target: torch.nn.Identity 69 | 70 | cond_stage_config: 71 | target: ldm.modules.encoders.modules.FrozenCLIPEmbedder 72 | params: 73 | layer: "hidden" 74 | layer_idx: -2 75 | -------------------------------------------------------------------------------- /models/configs/v1-inference_fp16.yaml: -------------------------------------------------------------------------------- 1 | model: 2 | base_learning_rate: 1.0e-04 3 | target: ldm.models.diffusion.ddpm.LatentDiffusion 4 | params: 5 | linear_start: 0.00085 6 | linear_end: 0.0120 7 | num_timesteps_cond: 1 8 | log_every_t: 200 9 | timesteps: 1000 10 | first_stage_key: "jpg" 11 | cond_stage_key: "txt" 12 | image_size: 64 13 | channels: 4 14 | cond_stage_trainable: false # Note: different from the one we trained before 15 | conditioning_key: crossattn 16 | monitor: val/loss_simple_ema 17 | scale_factor: 0.18215 18 | use_ema: False 19 | 20 | scheduler_config: # 10000 warmup steps 21 | target: ldm.lr_scheduler.LambdaLinearScheduler 22 | params: 23 | warm_up_steps: [ 10000 ] 24 | cycle_lengths: [ 10000000000000 ] # incredibly large number to prevent corner cases 25 | f_start: [ 1.e-6 ] 26 | f_max: [ 1. ] 27 | f_min: [ 1. ] 28 | 29 | unet_config: 30 | target: ldm.modules.diffusionmodules.openaimodel.UNetModel 31 | params: 32 | use_fp16: True 33 | image_size: 32 # unused 34 | in_channels: 4 35 | out_channels: 4 36 | model_channels: 320 37 | attention_resolutions: [ 4, 2, 1 ] 38 | num_res_blocks: 2 39 | channel_mult: [ 1, 2, 4, 4 ] 40 | num_heads: 8 41 | use_spatial_transformer: True 42 | transformer_depth: 1 43 | context_dim: 768 44 | use_checkpoint: True 45 | legacy: False 46 | 47 | first_stage_config: 48 | target: ldm.models.autoencoder.AutoencoderKL 49 | params: 50 | embed_dim: 4 51 | monitor: val/rec_loss 52 | ddconfig: 53 | double_z: true 54 | z_channels: 4 55 | resolution: 256 56 | in_channels: 3 57 | out_ch: 3 58 | ch: 128 59 | ch_mult: 60 | - 1 61 | - 2 62 | - 4 63 | - 4 64 | num_res_blocks: 2 65 | attn_resolutions: [] 66 | dropout: 0.0 67 | lossconfig: 68 | target: torch.nn.Identity 69 | 70 | cond_stage_config: 71 | target: ldm.modules.encoders.modules.FrozenCLIPEmbedder 72 | -------------------------------------------------------------------------------- /models/configs/v1-inpainting-inference.yaml: -------------------------------------------------------------------------------- 1 | model: 2 | base_learning_rate: 7.5e-05 3 | target: ldm.models.diffusion.ddpm.LatentInpaintDiffusion 4 | params: 5 | linear_start: 0.00085 6 | linear_end: 0.0120 7 | num_timesteps_cond: 1 8 | log_every_t: 200 9 | timesteps: 1000 10 | first_stage_key: "jpg" 11 | cond_stage_key: "txt" 12 | image_size: 64 13 | channels: 4 14 | cond_stage_trainable: false # Note: different from the one we trained before 15 | conditioning_key: hybrid # important 16 | monitor: val/loss_simple_ema 17 | scale_factor: 0.18215 18 | finetune_keys: null 19 | 20 | scheduler_config: # 10000 warmup steps 21 | target: ldm.lr_scheduler.LambdaLinearScheduler 22 | params: 23 | warm_up_steps: [ 2500 ] # NOTE for resuming. use 10000 if starting from scratch 24 | cycle_lengths: [ 10000000000000 ] # incredibly large number to prevent corner cases 25 | f_start: [ 1.e-6 ] 26 | f_max: [ 1. ] 27 | f_min: [ 1. ] 28 | 29 | unet_config: 30 | target: ldm.modules.diffusionmodules.openaimodel.UNetModel 31 | params: 32 | image_size: 32 # unused 33 | in_channels: 9 # 4 data + 4 downscaled image + 1 mask 34 | out_channels: 4 35 | model_channels: 320 36 | attention_resolutions: [ 4, 2, 1 ] 37 | num_res_blocks: 2 38 | channel_mult: [ 1, 2, 4, 4 ] 39 | num_heads: 8 40 | use_spatial_transformer: True 41 | transformer_depth: 1 42 | context_dim: 768 43 | use_checkpoint: True 44 | legacy: False 45 | 46 | first_stage_config: 47 | target: ldm.models.autoencoder.AutoencoderKL 48 | params: 49 | embed_dim: 4 50 | monitor: val/rec_loss 51 | ddconfig: 52 | double_z: true 53 | z_channels: 4 54 | resolution: 256 55 | in_channels: 3 56 | out_ch: 3 57 | ch: 128 58 | ch_mult: 59 | - 1 60 | - 2 61 | - 4 62 | - 4 63 | num_res_blocks: 2 64 | attn_resolutions: [] 65 | dropout: 0.0 66 | lossconfig: 67 | target: torch.nn.Identity 68 | 69 | cond_stage_config: 70 | target: ldm.modules.encoders.modules.FrozenCLIPEmbedder 71 | 72 | -------------------------------------------------------------------------------- /models/configs/v2-inference-v.yaml: -------------------------------------------------------------------------------- 1 | model: 2 | base_learning_rate: 1.0e-4 3 | target: ldm.models.diffusion.ddpm.LatentDiffusion 4 | params: 5 | parameterization: "v" 6 | linear_start: 0.00085 7 | linear_end: 0.0120 8 | num_timesteps_cond: 1 9 | log_every_t: 200 10 | timesteps: 1000 11 | first_stage_key: "jpg" 12 | cond_stage_key: "txt" 13 | image_size: 64 14 | channels: 4 15 | cond_stage_trainable: false 16 | conditioning_key: crossattn 17 | monitor: val/loss_simple_ema 18 | scale_factor: 0.18215 19 | use_ema: False # we set this to false because this is an inference only config 20 | 21 | unet_config: 22 | target: ldm.modules.diffusionmodules.openaimodel.UNetModel 23 | params: 24 | use_checkpoint: True 25 | use_fp16: True 26 | image_size: 32 # unused 27 | in_channels: 4 28 | out_channels: 4 29 | model_channels: 320 30 | attention_resolutions: [ 4, 2, 1 ] 31 | num_res_blocks: 2 32 | channel_mult: [ 1, 2, 4, 4 ] 33 | num_head_channels: 64 # need to fix for flash-attn 34 | use_spatial_transformer: True 35 | use_linear_in_transformer: True 36 | transformer_depth: 1 37 | context_dim: 1024 38 | legacy: False 39 | 40 | first_stage_config: 41 | target: ldm.models.autoencoder.AutoencoderKL 42 | params: 43 | embed_dim: 4 44 | monitor: val/rec_loss 45 | ddconfig: 46 | #attn_type: "vanilla-xformers" 47 | double_z: true 48 | z_channels: 4 49 | resolution: 256 50 | in_channels: 3 51 | out_ch: 3 52 | ch: 128 53 | ch_mult: 54 | - 1 55 | - 2 56 | - 4 57 | - 4 58 | num_res_blocks: 2 59 | attn_resolutions: [] 60 | dropout: 0.0 61 | lossconfig: 62 | target: torch.nn.Identity 63 | 64 | cond_stage_config: 65 | target: ldm.modules.encoders.modules.FrozenOpenCLIPEmbedder 66 | params: 67 | freeze: True 68 | layer: "penultimate" 69 | -------------------------------------------------------------------------------- /models/configs/v2-inference-v_fp32.yaml: -------------------------------------------------------------------------------- 1 | model: 2 | base_learning_rate: 1.0e-4 3 | target: ldm.models.diffusion.ddpm.LatentDiffusion 4 | params: 5 | parameterization: "v" 6 | linear_start: 0.00085 7 | linear_end: 0.0120 8 | num_timesteps_cond: 1 9 | log_every_t: 200 10 | timesteps: 1000 11 | first_stage_key: "jpg" 12 | cond_stage_key: "txt" 13 | image_size: 64 14 | channels: 4 15 | cond_stage_trainable: false 16 | conditioning_key: crossattn 17 | monitor: val/loss_simple_ema 18 | scale_factor: 0.18215 19 | use_ema: False # we set this to false because this is an inference only config 20 | 21 | unet_config: 22 | target: ldm.modules.diffusionmodules.openaimodel.UNetModel 23 | params: 24 | use_checkpoint: True 25 | use_fp16: False 26 | image_size: 32 # unused 27 | in_channels: 4 28 | out_channels: 4 29 | model_channels: 320 30 | attention_resolutions: [ 4, 2, 1 ] 31 | num_res_blocks: 2 32 | channel_mult: [ 1, 2, 4, 4 ] 33 | num_head_channels: 64 # need to fix for flash-attn 34 | use_spatial_transformer: True 35 | use_linear_in_transformer: True 36 | transformer_depth: 1 37 | context_dim: 1024 38 | legacy: False 39 | 40 | first_stage_config: 41 | target: ldm.models.autoencoder.AutoencoderKL 42 | params: 43 | embed_dim: 4 44 | monitor: val/rec_loss 45 | ddconfig: 46 | #attn_type: "vanilla-xformers" 47 | double_z: true 48 | z_channels: 4 49 | resolution: 256 50 | in_channels: 3 51 | out_ch: 3 52 | ch: 128 53 | ch_mult: 54 | - 1 55 | - 2 56 | - 4 57 | - 4 58 | num_res_blocks: 2 59 | attn_resolutions: [] 60 | dropout: 0.0 61 | lossconfig: 62 | target: torch.nn.Identity 63 | 64 | cond_stage_config: 65 | target: ldm.modules.encoders.modules.FrozenOpenCLIPEmbedder 66 | params: 67 | freeze: True 68 | layer: "penultimate" 69 | -------------------------------------------------------------------------------- /models/configs/v2-inference.yaml: -------------------------------------------------------------------------------- 1 | model: 2 | base_learning_rate: 1.0e-4 3 | target: ldm.models.diffusion.ddpm.LatentDiffusion 4 | params: 5 | linear_start: 0.00085 6 | linear_end: 0.0120 7 | num_timesteps_cond: 1 8 | log_every_t: 200 9 | timesteps: 1000 10 | first_stage_key: "jpg" 11 | cond_stage_key: "txt" 12 | image_size: 64 13 | channels: 4 14 | cond_stage_trainable: false 15 | conditioning_key: crossattn 16 | monitor: val/loss_simple_ema 17 | scale_factor: 0.18215 18 | use_ema: False # we set this to false because this is an inference only config 19 | 20 | unet_config: 21 | target: ldm.modules.diffusionmodules.openaimodel.UNetModel 22 | params: 23 | use_checkpoint: True 24 | use_fp16: True 25 | image_size: 32 # unused 26 | in_channels: 4 27 | out_channels: 4 28 | model_channels: 320 29 | attention_resolutions: [ 4, 2, 1 ] 30 | num_res_blocks: 2 31 | channel_mult: [ 1, 2, 4, 4 ] 32 | num_head_channels: 64 # need to fix for flash-attn 33 | use_spatial_transformer: True 34 | use_linear_in_transformer: True 35 | transformer_depth: 1 36 | context_dim: 1024 37 | legacy: False 38 | 39 | first_stage_config: 40 | target: ldm.models.autoencoder.AutoencoderKL 41 | params: 42 | embed_dim: 4 43 | monitor: val/rec_loss 44 | ddconfig: 45 | #attn_type: "vanilla-xformers" 46 | double_z: true 47 | z_channels: 4 48 | resolution: 256 49 | in_channels: 3 50 | out_ch: 3 51 | ch: 128 52 | ch_mult: 53 | - 1 54 | - 2 55 | - 4 56 | - 4 57 | num_res_blocks: 2 58 | attn_resolutions: [] 59 | dropout: 0.0 60 | lossconfig: 61 | target: torch.nn.Identity 62 | 63 | cond_stage_config: 64 | target: ldm.modules.encoders.modules.FrozenOpenCLIPEmbedder 65 | params: 66 | freeze: True 67 | layer: "penultimate" 68 | -------------------------------------------------------------------------------- /models/configs/v2-inference_fp32.yaml: -------------------------------------------------------------------------------- 1 | model: 2 | base_learning_rate: 1.0e-4 3 | target: ldm.models.diffusion.ddpm.LatentDiffusion 4 | params: 5 | linear_start: 0.00085 6 | linear_end: 0.0120 7 | num_timesteps_cond: 1 8 | log_every_t: 200 9 | timesteps: 1000 10 | first_stage_key: "jpg" 11 | cond_stage_key: "txt" 12 | image_size: 64 13 | channels: 4 14 | cond_stage_trainable: false 15 | conditioning_key: crossattn 16 | monitor: val/loss_simple_ema 17 | scale_factor: 0.18215 18 | use_ema: False # we set this to false because this is an inference only config 19 | 20 | unet_config: 21 | target: ldm.modules.diffusionmodules.openaimodel.UNetModel 22 | params: 23 | use_checkpoint: True 24 | use_fp16: False 25 | image_size: 32 # unused 26 | in_channels: 4 27 | out_channels: 4 28 | model_channels: 320 29 | attention_resolutions: [ 4, 2, 1 ] 30 | num_res_blocks: 2 31 | channel_mult: [ 1, 2, 4, 4 ] 32 | num_head_channels: 64 # need to fix for flash-attn 33 | use_spatial_transformer: True 34 | use_linear_in_transformer: True 35 | transformer_depth: 1 36 | context_dim: 1024 37 | legacy: False 38 | 39 | first_stage_config: 40 | target: ldm.models.autoencoder.AutoencoderKL 41 | params: 42 | embed_dim: 4 43 | monitor: val/rec_loss 44 | ddconfig: 45 | #attn_type: "vanilla-xformers" 46 | double_z: true 47 | z_channels: 4 48 | resolution: 256 49 | in_channels: 3 50 | out_ch: 3 51 | ch: 128 52 | ch_mult: 53 | - 1 54 | - 2 55 | - 4 56 | - 4 57 | num_res_blocks: 2 58 | attn_resolutions: [] 59 | dropout: 0.0 60 | lossconfig: 61 | target: torch.nn.Identity 62 | 63 | cond_stage_config: 64 | target: ldm.modules.encoders.modules.FrozenOpenCLIPEmbedder 65 | params: 66 | freeze: True 67 | layer: "penultimate" 68 | -------------------------------------------------------------------------------- /models/controlnet/put_controlnets_and_t2i_here: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/comfyanonymous/ComfyUI/312d511630db4907c3bed04dee297b28f61941a8/models/controlnet/put_controlnets_and_t2i_here -------------------------------------------------------------------------------- /models/diffusers/put_diffusers_models_here: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/comfyanonymous/ComfyUI/312d511630db4907c3bed04dee297b28f61941a8/models/diffusers/put_diffusers_models_here -------------------------------------------------------------------------------- /models/diffusion_models/put_diffusion_model_files_here: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/comfyanonymous/ComfyUI/312d511630db4907c3bed04dee297b28f61941a8/models/diffusion_models/put_diffusion_model_files_here -------------------------------------------------------------------------------- /models/embeddings/put_embeddings_or_textual_inversion_concepts_here: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/comfyanonymous/ComfyUI/312d511630db4907c3bed04dee297b28f61941a8/models/embeddings/put_embeddings_or_textual_inversion_concepts_here -------------------------------------------------------------------------------- /models/gligen/put_gligen_models_here: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/comfyanonymous/ComfyUI/312d511630db4907c3bed04dee297b28f61941a8/models/gligen/put_gligen_models_here -------------------------------------------------------------------------------- /models/hypernetworks/put_hypernetworks_here: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/comfyanonymous/ComfyUI/312d511630db4907c3bed04dee297b28f61941a8/models/hypernetworks/put_hypernetworks_here -------------------------------------------------------------------------------- /models/loras/put_loras_here: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/comfyanonymous/ComfyUI/312d511630db4907c3bed04dee297b28f61941a8/models/loras/put_loras_here -------------------------------------------------------------------------------- /models/photomaker/put_photomaker_models_here: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/comfyanonymous/ComfyUI/312d511630db4907c3bed04dee297b28f61941a8/models/photomaker/put_photomaker_models_here -------------------------------------------------------------------------------- /models/style_models/put_t2i_style_model_here: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/comfyanonymous/ComfyUI/312d511630db4907c3bed04dee297b28f61941a8/models/style_models/put_t2i_style_model_here -------------------------------------------------------------------------------- /models/text_encoders/put_text_encoder_files_here: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/comfyanonymous/ComfyUI/312d511630db4907c3bed04dee297b28f61941a8/models/text_encoders/put_text_encoder_files_here -------------------------------------------------------------------------------- /models/unet/put_unet_files_here: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/comfyanonymous/ComfyUI/312d511630db4907c3bed04dee297b28f61941a8/models/unet/put_unet_files_here -------------------------------------------------------------------------------- /models/upscale_models/put_esrgan_and_other_upscale_models_here: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/comfyanonymous/ComfyUI/312d511630db4907c3bed04dee297b28f61941a8/models/upscale_models/put_esrgan_and_other_upscale_models_here -------------------------------------------------------------------------------- /models/vae/put_vae_here: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/comfyanonymous/ComfyUI/312d511630db4907c3bed04dee297b28f61941a8/models/vae/put_vae_here -------------------------------------------------------------------------------- /models/vae_approx/put_taesd_encoder_pth_and_taesd_decoder_pth_here: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/comfyanonymous/ComfyUI/312d511630db4907c3bed04dee297b28f61941a8/models/vae_approx/put_taesd_encoder_pth_and_taesd_decoder_pth_here -------------------------------------------------------------------------------- /new_updater.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shutil 3 | 4 | base_path = os.path.dirname(os.path.realpath(__file__)) 5 | 6 | 7 | def update_windows_updater(): 8 | top_path = os.path.dirname(base_path) 9 | updater_path = os.path.join(base_path, ".ci/update_windows/update.py") 10 | bat_path = os.path.join(base_path, ".ci/update_windows/update_comfyui.bat") 11 | 12 | dest_updater_path = os.path.join(top_path, "update/update.py") 13 | dest_bat_path = os.path.join(top_path, "update/update_comfyui.bat") 14 | dest_bat_deps_path = os.path.join(top_path, "update/update_comfyui_and_python_dependencies.bat") 15 | 16 | try: 17 | with open(dest_bat_path, 'rb') as f: 18 | contents = f.read() 19 | except: 20 | return 21 | 22 | if not contents.startswith(b"..\\python_embeded\\python.exe .\\update.py"): 23 | return 24 | 25 | shutil.copy(updater_path, dest_updater_path) 26 | try: 27 | with open(dest_bat_deps_path, 'rb') as f: 28 | contents = f.read() 29 | contents = contents.replace(b'..\\python_embeded\\python.exe .\\update.py ..\\ComfyUI\\', b'call update_comfyui.bat nopause') 30 | with open(dest_bat_deps_path, 'wb') as f: 31 | f.write(contents) 32 | except: 33 | pass 34 | shutil.copy(bat_path, dest_bat_path) 35 | print("Updated the windows standalone package updater.") # noqa: T201 36 | -------------------------------------------------------------------------------- /node_helpers.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | import torch 3 | 4 | from comfy.cli_args import args 5 | 6 | from PIL import ImageFile, UnidentifiedImageError 7 | 8 | def conditioning_set_values(conditioning, values={}, append=False): 9 | c = [] 10 | for t in conditioning: 11 | n = [t[0], t[1].copy()] 12 | for k in values: 13 | val = values[k] 14 | if append: 15 | old_val = n[1].get(k, None) 16 | if old_val is not None: 17 | val = old_val + val 18 | 19 | n[1][k] = val 20 | c.append(n) 21 | 22 | return c 23 | 24 | def pillow(fn, arg): 25 | prev_value = None 26 | try: 27 | x = fn(arg) 28 | except (OSError, UnidentifiedImageError, ValueError): #PIL issues #4472 and #2445, also fixes ComfyUI issue #3416 29 | prev_value = ImageFile.LOAD_TRUNCATED_IMAGES 30 | ImageFile.LOAD_TRUNCATED_IMAGES = True 31 | x = fn(arg) 32 | finally: 33 | if prev_value is not None: 34 | ImageFile.LOAD_TRUNCATED_IMAGES = prev_value 35 | return x 36 | 37 | def hasher(): 38 | hashfuncs = { 39 | "md5": hashlib.md5, 40 | "sha1": hashlib.sha1, 41 | "sha256": hashlib.sha256, 42 | "sha512": hashlib.sha512 43 | } 44 | return hashfuncs[args.default_hashing_function] 45 | 46 | def string_to_torch_dtype(string): 47 | if string == "fp32": 48 | return torch.float32 49 | if string == "fp16": 50 | return torch.float16 51 | if string == "bf16": 52 | return torch.bfloat16 53 | 54 | def image_alpha_fix(destination, source): 55 | if destination.shape[-1] < source.shape[-1]: 56 | source = source[...,:destination.shape[-1]] 57 | elif destination.shape[-1] > source.shape[-1]: 58 | destination = torch.nn.functional.pad(destination, (0, 1)) 59 | destination[..., -1] = 1.0 60 | return destination, source 61 | -------------------------------------------------------------------------------- /output/_output_images_will_be_put_here: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/comfyanonymous/ComfyUI/312d511630db4907c3bed04dee297b28f61941a8/output/_output_images_will_be_put_here -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "ComfyUI" 3 | version = "0.3.39" 4 | readme = "README.md" 5 | license = { file = "LICENSE" } 6 | requires-python = ">=3.9" 7 | 8 | [project.urls] 9 | homepage = "https://www.comfy.org/" 10 | repository = "https://github.com/comfyanonymous/ComfyUI" 11 | documentation = "https://docs.comfy.org/" 12 | 13 | [tool.ruff] 14 | lint.select = [ 15 | "N805", # invalid-first-argument-name-for-method 16 | "S307", # suspicious-eval-usage 17 | "S102", # exec 18 | "T", # print-usage 19 | "W", 20 | # The "F" series in Ruff stands for "Pyflakes" rules, which catch various Python syntax errors and undefined names. 21 | # See all rules here: https://docs.astral.sh/ruff/rules/#pyflakes-f 22 | "F", 23 | ] 24 | exclude = ["*.ipynb"] 25 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | markers = 3 | inference: mark as inference test (deselect with '-m "not inference"') 4 | execution: mark as execution test (deselect with '-m "not execution"') 5 | testpaths = 6 | tests 7 | tests-unit 8 | addopts = -s 9 | pythonpath = . 10 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | comfyui-frontend-package==1.21.6 2 | comfyui-workflow-templates==0.1.25 3 | comfyui-embedded-docs==0.2.0 4 | torch 5 | torchsde 6 | torchvision 7 | torchaudio 8 | numpy>=1.25.0 9 | einops 10 | transformers>=4.28.1 11 | tokenizers>=0.13.3 12 | sentencepiece 13 | safetensors>=0.4.2 14 | aiohttp>=3.11.8 15 | yarl>=1.18.0 16 | pyyaml 17 | Pillow 18 | scipy 19 | tqdm 20 | psutil 21 | 22 | #non essential dependencies: 23 | kornia>=0.7.1 24 | spandrel 25 | soundfile 26 | av>=14.2.0 27 | pydantic~=2.0 28 | -------------------------------------------------------------------------------- /tests-unit/README.md: -------------------------------------------------------------------------------- 1 | # Pytest Unit Tests 2 | 3 | ## Install test dependencies 4 | 5 | `pip install -r tests-unit/requirements.txt` 6 | 7 | ## Run tests 8 | `pytest tests-unit/` 9 | -------------------------------------------------------------------------------- /tests-unit/app_test/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/comfyanonymous/ComfyUI/312d511630db4907c3bed04dee297b28f61941a8/tests-unit/app_test/__init__.py -------------------------------------------------------------------------------- /tests-unit/app_test/model_manager_test.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import base64 3 | import json 4 | import struct 5 | from io import BytesIO 6 | from PIL import Image 7 | from aiohttp import web 8 | from unittest.mock import patch 9 | from app.model_manager import ModelFileManager 10 | 11 | pytestmark = ( 12 | pytest.mark.asyncio 13 | ) # This applies the asyncio mark to all test functions in the module 14 | 15 | @pytest.fixture 16 | def model_manager(): 17 | return ModelFileManager() 18 | 19 | @pytest.fixture 20 | def app(model_manager): 21 | app = web.Application() 22 | routes = web.RouteTableDef() 23 | model_manager.add_routes(routes) 24 | app.add_routes(routes) 25 | return app 26 | 27 | async def test_get_model_preview_safetensors(aiohttp_client, app, tmp_path): 28 | img = Image.new('RGB', (100, 100), 'white') 29 | img_byte_arr = BytesIO() 30 | img.save(img_byte_arr, format='PNG') 31 | img_byte_arr.seek(0) 32 | img_b64 = base64.b64encode(img_byte_arr.getvalue()).decode('utf-8') 33 | 34 | safetensors_file = tmp_path / "test_model.safetensors" 35 | header_bytes = json.dumps({ 36 | "__metadata__": { 37 | "ssmd_cover_images": json.dumps([img_b64]) 38 | } 39 | }).encode('utf-8') 40 | length_bytes = struct.pack('=7.8.0 2 | pytest-aiohttp 3 | pytest-asyncio 4 | -------------------------------------------------------------------------------- /tests-unit/server/utils/file_operations_test.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from typing import List 3 | from api_server.utils.file_operations import FileSystemOperations, FileSystemItem, is_file_info 4 | 5 | @pytest.fixture 6 | def temp_directory(tmp_path): 7 | # Create a temporary directory structure 8 | dir1 = tmp_path / "dir1" 9 | dir2 = tmp_path / "dir2" 10 | dir1.mkdir() 11 | dir2.mkdir() 12 | (dir1 / "file1.txt").write_text("content1") 13 | (dir2 / "file2.txt").write_text("content2") 14 | (tmp_path / "file3.txt").write_text("content3") 15 | return tmp_path 16 | 17 | def test_walk_directory(temp_directory): 18 | result: List[FileSystemItem] = FileSystemOperations.walk_directory(str(temp_directory)) 19 | 20 | assert len(result) == 5 # 2 directories and 3 files 21 | 22 | files = [item for item in result if item['type'] == 'file'] 23 | dirs = [item for item in result if item['type'] == 'directory'] 24 | 25 | assert len(files) == 3 26 | assert len(dirs) == 2 27 | 28 | file_names = {file['name'] for file in files} 29 | assert file_names == {'file1.txt', 'file2.txt', 'file3.txt'} 30 | 31 | dir_names = {dir['name'] for dir in dirs} 32 | assert dir_names == {'dir1', 'dir2'} 33 | 34 | def test_walk_directory_empty(tmp_path): 35 | result = FileSystemOperations.walk_directory(str(tmp_path)) 36 | assert len(result) == 0 37 | 38 | def test_walk_directory_file_size(temp_directory): 39 | result: List[FileSystemItem] = FileSystemOperations.walk_directory(str(temp_directory)) 40 | files = [item for item in result if is_file_info(item)] 41 | for file in files: 42 | assert file['size'] > 0 # Assuming all files have some content 43 | -------------------------------------------------------------------------------- /tests-unit/utils/json_util_test.py: -------------------------------------------------------------------------------- 1 | from utils.json_util import merge_json_recursive 2 | 3 | 4 | def test_merge_simple_dicts(): 5 | base = {"a": 1, "b": 2} 6 | update = {"b": 3, "c": 4} 7 | expected = {"a": 1, "b": 3, "c": 4} 8 | assert merge_json_recursive(base, update) == expected 9 | 10 | 11 | def test_merge_nested_dicts(): 12 | base = {"a": {"x": 1, "y": 2}, "b": 3} 13 | update = {"a": {"y": 4, "z": 5}} 14 | expected = {"a": {"x": 1, "y": 4, "z": 5}, "b": 3} 15 | assert merge_json_recursive(base, update) == expected 16 | 17 | 18 | def test_merge_lists(): 19 | base = {"a": [1, 2], "b": 3} 20 | update = {"a": [3, 4]} 21 | expected = {"a": [1, 2, 3, 4], "b": 3} 22 | assert merge_json_recursive(base, update) == expected 23 | 24 | 25 | def test_merge_nested_lists(): 26 | base = {"a": {"x": [1, 2]}} 27 | update = {"a": {"x": [3, 4]}} 28 | expected = {"a": {"x": [1, 2, 3, 4]}} 29 | assert merge_json_recursive(base, update) == expected 30 | 31 | 32 | def test_merge_mixed_types(): 33 | base = {"a": [1, 2], "b": {"x": 1}} 34 | update = {"a": [3], "b": {"y": 2}} 35 | expected = {"a": [1, 2, 3], "b": {"x": 1, "y": 2}} 36 | assert merge_json_recursive(base, update) == expected 37 | 38 | 39 | def test_merge_overwrite_non_dict(): 40 | base = {"a": 1} 41 | update = {"a": {"x": 2}} 42 | expected = {"a": {"x": 2}} 43 | assert merge_json_recursive(base, update) == expected 44 | 45 | 46 | def test_merge_empty_dicts(): 47 | base = {} 48 | update = {"a": 1} 49 | expected = {"a": 1} 50 | assert merge_json_recursive(base, update) == expected 51 | 52 | 53 | def test_merge_none_values(): 54 | base = {"a": None} 55 | update = {"a": {"x": 1}} 56 | expected = {"a": {"x": 1}} 57 | assert merge_json_recursive(base, update) == expected 58 | 59 | 60 | def test_merge_different_types(): 61 | base = {"a": [1, 2]} 62 | update = {"a": "string"} 63 | expected = {"a": "string"} 64 | assert merge_json_recursive(base, update) == expected 65 | 66 | 67 | def test_merge_complex_nested(): 68 | base = {"a": [1, 2], "b": {"x": [3, 4], "y": {"p": 1}}} 69 | update = {"a": [5], "b": {"x": [6], "y": {"q": 2}}} 70 | expected = {"a": [1, 2, 5], "b": {"x": [3, 4, 6], "y": {"p": 1, "q": 2}}} 71 | assert merge_json_recursive(base, update) == expected 72 | -------------------------------------------------------------------------------- /tests/README.md: -------------------------------------------------------------------------------- 1 | # Automated Testing 2 | 3 | ## Running tests locally 4 | 5 | Additional requirements for running tests: 6 | ``` 7 | pip install pytest 8 | pip install websocket-client==1.6.1 9 | opencv-python==4.6.0.66 10 | scikit-image==0.21.0 11 | ``` 12 | Run inference tests: 13 | ``` 14 | pytest tests/inference 15 | ``` 16 | 17 | ## Quality regression test 18 | Compares images in 2 directories to ensure they are the same 19 | 20 | 1) Run an inference test to save a directory of "ground truth" images 21 | ``` 22 | pytest tests/inference --output_dir tests/inference/baseline 23 | ``` 24 | 2) Make code edits 25 | 26 | 3) Run inference and quality comparison tests 27 | ``` 28 | pytest 29 | ``` -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/comfyanonymous/ComfyUI/312d511630db4907c3bed04dee297b28f61941a8/tests/__init__.py -------------------------------------------------------------------------------- /tests/compare/conftest.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pytest 3 | 4 | # Command line arguments for pytest 5 | def pytest_addoption(parser): 6 | parser.addoption('--baseline_dir', action="store", default='tests/inference/baseline', help='Directory for ground-truth images') 7 | parser.addoption('--test_dir', action="store", default='tests/inference/samples', help='Directory for images to test') 8 | parser.addoption('--metrics_file', action="store", default='tests/metrics.md', help='Output file for metrics') 9 | parser.addoption('--img_output_dir', action="store", default='tests/compare/samples', help='Output directory for diff metric images') 10 | 11 | # This initializes args at the beginning of the test session 12 | @pytest.fixture(scope="session", autouse=True) 13 | def args_pytest(pytestconfig): 14 | args = {} 15 | args['baseline_dir'] = pytestconfig.getoption('baseline_dir') 16 | args['test_dir'] = pytestconfig.getoption('test_dir') 17 | args['metrics_file'] = pytestconfig.getoption('metrics_file') 18 | args['img_output_dir'] = pytestconfig.getoption('img_output_dir') 19 | 20 | # Initialize metrics file 21 | with open(args['metrics_file'], 'a') as f: 22 | # if file is empty, write header 23 | if os.stat(args['metrics_file']).st_size == 0: 24 | f.write("| date | run | file | status | value | \n") 25 | f.write("| --- | --- | --- | --- | --- | \n") 26 | 27 | return args 28 | 29 | 30 | def gather_file_basenames(directory: str): 31 | files = [] 32 | for file in os.listdir(directory): 33 | if file.endswith(".png"): 34 | files.append(file) 35 | return files 36 | 37 | # Creates the list of baseline file names to use as a fixture 38 | def pytest_generate_tests(metafunc): 39 | if "baseline_fname" in metafunc.fixturenames: 40 | baseline_fnames = gather_file_basenames(metafunc.config.getoption("baseline_dir")) 41 | metafunc.parametrize("baseline_fname", baseline_fnames) 42 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pytest 3 | 4 | # Command line arguments for pytest 5 | def pytest_addoption(parser): 6 | parser.addoption('--output_dir', action="store", default='tests/inference/samples', help='Output directory for generated images') 7 | parser.addoption("--listen", type=str, default="127.0.0.1", metavar="IP", nargs="?", const="0.0.0.0", help="Specify the IP address to listen on (default: 127.0.0.1). If --listen is provided without an argument, it defaults to 0.0.0.0. (listens on all)") 8 | parser.addoption("--port", type=int, default=8188, help="Set the listen port.") 9 | 10 | # This initializes args at the beginning of the test session 11 | @pytest.fixture(scope="session", autouse=True) 12 | def args_pytest(pytestconfig): 13 | args = {} 14 | args['output_dir'] = pytestconfig.getoption('output_dir') 15 | args['listen'] = pytestconfig.getoption('listen') 16 | args['port'] = pytestconfig.getoption('port') 17 | 18 | os.makedirs(args['output_dir'], exist_ok=True) 19 | 20 | return args 21 | 22 | def pytest_collection_modifyitems(items): 23 | # Modifies items so tests run in the correct order 24 | 25 | LAST_TESTS = ['test_quality'] 26 | 27 | # Move the last items to the end 28 | last_items = [] 29 | for test_name in LAST_TESTS: 30 | for item in items.copy(): 31 | print(item.module.__name__, item) # noqa: T201 32 | if item.module.__name__ == test_name: 33 | last_items.append(item) 34 | items.remove(item) 35 | 36 | items.extend(last_items) 37 | -------------------------------------------------------------------------------- /tests/inference/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/comfyanonymous/ComfyUI/312d511630db4907c3bed04dee297b28f61941a8/tests/inference/__init__.py -------------------------------------------------------------------------------- /tests/inference/extra_model_paths.yaml: -------------------------------------------------------------------------------- 1 | # Config for testing nodes 2 | testing: 3 | custom_nodes: tests/inference/testing_nodes 4 | 5 | -------------------------------------------------------------------------------- /tests/inference/testing_nodes/testing-pack/__init__.py: -------------------------------------------------------------------------------- 1 | from .specific_tests import TEST_NODE_CLASS_MAPPINGS, TEST_NODE_DISPLAY_NAME_MAPPINGS 2 | from .flow_control import FLOW_CONTROL_NODE_CLASS_MAPPINGS, FLOW_CONTROL_NODE_DISPLAY_NAME_MAPPINGS 3 | from .util import UTILITY_NODE_CLASS_MAPPINGS, UTILITY_NODE_DISPLAY_NAME_MAPPINGS 4 | from .conditions import CONDITION_NODE_CLASS_MAPPINGS, CONDITION_NODE_DISPLAY_NAME_MAPPINGS 5 | from .stubs import TEST_STUB_NODE_CLASS_MAPPINGS, TEST_STUB_NODE_DISPLAY_NAME_MAPPINGS 6 | 7 | # NODE_CLASS_MAPPINGS = GENERAL_NODE_CLASS_MAPPINGS.update(COMPONENT_NODE_CLASS_MAPPINGS) 8 | # NODE_DISPLAY_NAME_MAPPINGS = GENERAL_NODE_DISPLAY_NAME_MAPPINGS.update(COMPONENT_NODE_DISPLAY_NAME_MAPPINGS) 9 | 10 | NODE_CLASS_MAPPINGS = {} 11 | NODE_CLASS_MAPPINGS.update(TEST_NODE_CLASS_MAPPINGS) 12 | NODE_CLASS_MAPPINGS.update(FLOW_CONTROL_NODE_CLASS_MAPPINGS) 13 | NODE_CLASS_MAPPINGS.update(UTILITY_NODE_CLASS_MAPPINGS) 14 | NODE_CLASS_MAPPINGS.update(CONDITION_NODE_CLASS_MAPPINGS) 15 | NODE_CLASS_MAPPINGS.update(TEST_STUB_NODE_CLASS_MAPPINGS) 16 | 17 | NODE_DISPLAY_NAME_MAPPINGS = {} 18 | NODE_DISPLAY_NAME_MAPPINGS.update(TEST_NODE_DISPLAY_NAME_MAPPINGS) 19 | NODE_DISPLAY_NAME_MAPPINGS.update(FLOW_CONTROL_NODE_DISPLAY_NAME_MAPPINGS) 20 | NODE_DISPLAY_NAME_MAPPINGS.update(UTILITY_NODE_DISPLAY_NAME_MAPPINGS) 21 | NODE_DISPLAY_NAME_MAPPINGS.update(CONDITION_NODE_DISPLAY_NAME_MAPPINGS) 22 | NODE_DISPLAY_NAME_MAPPINGS.update(TEST_STUB_NODE_DISPLAY_NAME_MAPPINGS) 23 | 24 | -------------------------------------------------------------------------------- /tests/inference/testing_nodes/testing-pack/tools.py: -------------------------------------------------------------------------------- 1 | 2 | def MakeSmartType(t): 3 | if isinstance(t, str): 4 | return SmartType(t) 5 | return t 6 | 7 | class SmartType(str): 8 | def __ne__(self, other): 9 | if self == "*" or other == "*": 10 | return False 11 | selfset = set(self.split(',')) 12 | otherset = set(other.split(',')) 13 | return not selfset.issubset(otherset) 14 | 15 | def VariantSupport(): 16 | def decorator(cls): 17 | if hasattr(cls, "INPUT_TYPES"): 18 | old_input_types = getattr(cls, "INPUT_TYPES") 19 | def new_input_types(*args, **kwargs): 20 | types = old_input_types(*args, **kwargs) 21 | for category in ["required", "optional"]: 22 | if category not in types: 23 | continue 24 | for key, value in types[category].items(): 25 | if isinstance(value, tuple): 26 | types[category][key] = (MakeSmartType(value[0]),) + value[1:] 27 | return types 28 | setattr(cls, "INPUT_TYPES", new_input_types) 29 | if hasattr(cls, "RETURN_TYPES"): 30 | old_return_types = cls.RETURN_TYPES 31 | setattr(cls, "RETURN_TYPES", tuple(MakeSmartType(x) for x in old_return_types)) 32 | if hasattr(cls, "VALIDATE_INPUTS"): 33 | # Reflection is used to determine what the function signature is, so we can't just change the function signature 34 | raise NotImplementedError("VariantSupport does not support VALIDATE_INPUTS yet") 35 | else: 36 | def validate_inputs(input_types): 37 | inputs = cls.INPUT_TYPES() 38 | for key, value in input_types.items(): 39 | if isinstance(value, SmartType): 40 | continue 41 | if "required" in inputs and key in inputs["required"]: 42 | expected_type = inputs["required"][key][0] 43 | elif "optional" in inputs and key in inputs["optional"]: 44 | expected_type = inputs["optional"][key][0] 45 | else: 46 | expected_type = None 47 | if expected_type is not None and MakeSmartType(value) != expected_type: 48 | return f"Invalid type of {key}: {value} (expected {expected_type})" 49 | return True 50 | setattr(cls, "VALIDATE_INPUTS", validate_inputs) 51 | return cls 52 | return decorator 53 | 54 | -------------------------------------------------------------------------------- /utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/comfyanonymous/ComfyUI/312d511630db4907c3bed04dee297b28f61941a8/utils/__init__.py -------------------------------------------------------------------------------- /utils/extra_config.py: -------------------------------------------------------------------------------- 1 | import os 2 | import yaml 3 | import folder_paths 4 | import logging 5 | 6 | def load_extra_path_config(yaml_path): 7 | with open(yaml_path, 'r', encoding='utf-8') as stream: 8 | config = yaml.safe_load(stream) 9 | yaml_dir = os.path.dirname(os.path.abspath(yaml_path)) 10 | for c in config: 11 | conf = config[c] 12 | if conf is None: 13 | continue 14 | base_path = None 15 | if "base_path" in conf: 16 | base_path = conf.pop("base_path") 17 | base_path = os.path.expandvars(os.path.expanduser(base_path)) 18 | if not os.path.isabs(base_path): 19 | base_path = os.path.abspath(os.path.join(yaml_dir, base_path)) 20 | is_default = False 21 | if "is_default" in conf: 22 | is_default = conf.pop("is_default") 23 | for x in conf: 24 | for y in conf[x].split("\n"): 25 | if len(y) == 0: 26 | continue 27 | full_path = y 28 | if base_path: 29 | full_path = os.path.join(base_path, full_path) 30 | elif not os.path.isabs(full_path): 31 | full_path = os.path.abspath(os.path.join(yaml_dir, y)) 32 | normalized_path = os.path.normpath(full_path) 33 | logging.info("Adding extra search path {} {}".format(x, normalized_path)) 34 | folder_paths.add_model_folder_path(x, normalized_path, is_default) 35 | -------------------------------------------------------------------------------- /utils/json_util.py: -------------------------------------------------------------------------------- 1 | def merge_json_recursive(base, update): 2 | """Recursively merge two JSON-like objects. 3 | - Dictionaries are merged recursively 4 | - Lists are concatenated 5 | - Other types are overwritten by the update value 6 | 7 | Args: 8 | base: Base JSON-like object 9 | update: Update JSON-like object to merge into base 10 | 11 | Returns: 12 | Merged JSON-like object 13 | """ 14 | if not isinstance(base, dict) or not isinstance(update, dict): 15 | if isinstance(base, list) and isinstance(update, list): 16 | return base + update 17 | return update 18 | 19 | merged = base.copy() 20 | for key, value in update.items(): 21 | if key in merged: 22 | merged[key] = merge_json_recursive(merged[key], value) 23 | else: 24 | merged[key] = value 25 | 26 | return merged 27 | --------------------------------------------------------------------------------