├── .gitignore ├── LICENSE ├── README.md ├── blender_templates └── multicn_depth+seg.blend ├── multicn.py └── seg.py /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Jin Liu 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Blender-ControlNet 2 | 3 | Using Multiple ControlNet in Blender. 4 | 5 | ## Required 6 | 7 | - [AUTOMATIC1111/stable-diffusion-webui](https://github.com/AUTOMATIC1111/stable-diffusion-webui) 8 | - [Mikubill/sd-webui-controlnet](https://github.com/Mikubill/sd-webui-controlnet) 9 | 10 | ## Usage 11 | 12 | ### 1. Start A1111 in API mode. 13 | 14 | First, of course, is to run web ui with `--api` commandline argument 15 | 16 | - example in your "webui-user.bat": `set COMMANDLINE_ARGS=--api` 17 | 18 | ### 2. Install Mikubill/sd-webui-controlnet extension 19 | 20 | You have to install the `Mikubill/sd-webui-controlnet` extension in A1111 and download the ControlNet models. 21 | Please refer to the installation instructions from [Mikubill/sd-webui-controlnet](https://github.com/Mikubill/sd-webui-controlnet). 22 | 23 | **Notes** 24 | 25 | - In new version of Mikubill/sd-webui-controlnet, you need to enable `Allow other script to control this extension` in settings for API access. 26 | - To enable Multi ControlNet, change `Multi ControlNet: Max models amount (requires restart)` in the settings. Note that you will need to restart the WebUI for changes to take effect. 27 | 28 | ### 3. Copy and paste the `multicn.py` code into your blender Scripting pane. 29 | 30 | ### 4. How to use the script? 31 | 32 | Basically, the script utilizes Blender Compositor to generate the required maps and then sends them to AUTOMATIC1111. 33 | 34 | To generate the desired output, you need to make adjustments to either the code or Blender Compositor nodes before pressing `F12`. To simplify this process, I have provided a basic [Blender template](./blender_templates/multicn_depth%2Bseg.blend) that sends depth and segmentation maps to ControlNet. 35 | 36 | Here is a brief [tutorial](https://twitter.com/Songzi39590361/status/1632706795365072897) on how to modify to suit @toyxyz3's rig if you wish to send openpose/depth/canny maps. 37 | 38 | **Notes** 39 | 40 | - Make sure you have the right name for `controlnet_model`, hash does matter. You can get your local installed ControlNet models list from [here](http://localhost:7860/docs#/default/model_list_controlnet_model_list_get). 41 | 42 | ### 5. Hit "Run Script" 43 | 44 | Before you hit "Run Script", here are the parameters that you may want to modify in the scripts: 45 | 46 | ``` 47 | # specify your images output folder 48 | IMAGE_FOLDER = "//sd_results" 49 | 50 | # if you don't want to send your maps to AI, set this option to False 51 | is_using_ai = True 52 | 53 | # which maps are you going to send to AI 54 | is_send_canny = False 55 | is_send_depth = False 56 | is_send_bone = True 57 | is_send_seg = False 58 | 59 | # prepare data for API 60 | params = { 61 | "prompt": "a room", 62 | "negative_prompt": "(worst quality:2), (low quality:2), (normal quality:2), lowres, normal quality", 63 | "width": get_output_width(scene), 64 | "height": get_output_height(scene), 65 | "sampler_index": "DPM++ SDE Karras", 66 | "sampler_name": "", 67 | "batch_size": 1, 68 | "n_iter": 1, 69 | "steps": 20, 70 | "cfg_scale": 7, 71 | "seed": -1, 72 | "subseed": -1, 73 | "subseed_strength": 0, 74 | "restore_faces": False, 75 | "enable_hr": False, 76 | "hr_scale": 1.5, 77 | "hr_upscaler": "R-ESRGAN General WDN 4xV3", 78 | "denoising_strength": 0.5, 79 | "hr_second_pass_steps": 10, 80 | "hr_resize_x": 0, 81 | "hr_resize_y": 0, 82 | "firstphase_width": 0, 83 | "firstphase_height": 0, 84 | "override_settings": {"CLIP_stop_at_last_layers": 2}, 85 | "override_settings_restore_afterwards": True, 86 | "alwayson_scripts": {"controlnet": {"args": []}}, 87 | } 88 | 89 | canny_cn_units = { 90 | "mask": "", 91 | "module": "none", 92 | "model": "diff_control_sd15_canny_fp16 [ea6e3b9c]", 93 | "weight": 1.2, 94 | "resize_mode": "Scale to Fit (Inner Fit)", 95 | "lowvram": False, 96 | "processor_res": 64, 97 | "threshold_a": 64, 98 | "threshold_b": 64, 99 | "guidance": 1, 100 | "guidance_start": 0.19, 101 | "guidance_end": 1, 102 | "guessmode": False, 103 | } 104 | 105 | depth_cn_units = { 106 | "mask": "", 107 | "module": "none", 108 | "model": "diff_control_sd15_depth_fp16 [978ef0a1]", 109 | "weight": 1.2, 110 | "resize_mode": "Scale to Fit (Inner Fit)", 111 | "lowvram": False, 112 | "processor_res": 64, 113 | "threshold_a": 64, 114 | "threshold_b": 64, 115 | "guidance": 1, 116 | "guidance_start": 0.19, 117 | "guidance_end": 1, 118 | "guessmode": False, 119 | } 120 | 121 | bone_cn_units = { 122 | "mask": "", 123 | "module": "none", 124 | "model": "diff_control_sd15_openpose_fp16 [1723948e]", 125 | "weight": 1.1, 126 | "resize_mode": "Scale to Fit (Inner Fit)", 127 | "lowvram": False, 128 | "processor_res": 64, 129 | "threshold_a": 64, 130 | "threshold_b": 64, 131 | "guidance": 1, 132 | "guidance_start": 0, 133 | "guidance_end": 1, 134 | "guessmode": False, 135 | } 136 | 137 | seg_cn_units = { 138 | "mask": "", 139 | "module": "none", 140 | "model": "diff_control_sd15_seg_fp16 [a1e85e27]", 141 | "weight": 1, 142 | "resize_mode": "Scale to Fit (Inner Fit)", 143 | "lowvram": False, 144 | "processor_res": 64, 145 | "threshold_a": 64, 146 | "threshold_b": 64, 147 | "guidance": 1, 148 | "guidance_start": 0, 149 | "guidance_end": 1, 150 | "guessmode": False, 151 | } 152 | ``` 153 | 154 | ### 6. Hit **F12**, wait for it... 155 | 156 | ## Bonus 157 | 158 | To create 150 ControlNet segmentation colors materials, run `seg.py`. Check out this [tweet](https://twitter.com/Songzi39590361/status/1631190450710409216) for instructions. 159 | 160 | ## Todo 161 | 162 | - [ ] animation support 163 | -------------------------------------------------------------------------------- /blender_templates/multicn_depth+seg.blend: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coolzilj/Blender-ControlNet/3e6f2c0812893876e3ca7a37a3a082230b1a6c6f/blender_templates/multicn_depth+seg.blend -------------------------------------------------------------------------------- /multicn.py: -------------------------------------------------------------------------------- 1 | from bpy.app.handlers import persistent 2 | import bpy 3 | import os 4 | import shutil 5 | import time 6 | import tempfile 7 | import base64 8 | import requests 9 | 10 | # specify your images output folder 11 | IMAGE_FOLDER = "//sd_results" 12 | 13 | # if you don't want to send your maps to AI, set this option to False 14 | is_using_ai = True 15 | 16 | # which maps are you going to send to AI 17 | is_send_canny = False 18 | is_send_depth = False 19 | is_send_bone = True 20 | is_send_seg = False 21 | 22 | 23 | @persistent 24 | def render_complete_handler(scene): 25 | is_img_ready = bpy.data.images["Render Result"].has_data 26 | 27 | if is_img_ready: 28 | if is_using_ai: 29 | send_to_api(scene) 30 | else: 31 | print("Rendered image is not ready.") 32 | 33 | 34 | def send_to_api(scene): 35 | # prepare filenames 36 | frame_num = f"{bpy.context.scene.frame_current}".zfill(4) 37 | comp_output_canny_filename = "canny" + frame_num + ".png" 38 | comp_output_depth_filename = "depth" + frame_num + ".png" 39 | comp_output_bone_filename = "bone" + frame_num + ".png" 40 | comp_output_seg_filename = "seg" + frame_num + ".png" 41 | 42 | timestamp = int(time.time()) 43 | before_output_canny_filename = f"{timestamp}-1-canny-before.png" 44 | before_output_depth_filename = f"{timestamp}-1-depth-before.png" 45 | before_output_bone_filename = f"{timestamp}-1-bone-before.png" 46 | before_output_seg_filename = f"{timestamp}-1-seg-before.png" 47 | after_output_filename_prefix = f"{timestamp}-2-after" 48 | 49 | # check if comp output images exists 50 | if is_send_canny: 51 | if not os.path.exists(get_asset_path(comp_output_canny_filename)): 52 | return print("Couldn't find the canny image.") 53 | else: 54 | os.rename( 55 | get_asset_path(comp_output_canny_filename), 56 | get_asset_path(before_output_canny_filename), 57 | ) 58 | if is_send_depth: 59 | if not os.path.exists(get_asset_path(comp_output_depth_filename)): 60 | return print("Couldn't find the depth image.") 61 | else: 62 | os.rename( 63 | get_asset_path(comp_output_depth_filename), 64 | get_asset_path(before_output_depth_filename), 65 | ) 66 | if is_send_bone: 67 | if not os.path.exists(get_asset_path(comp_output_bone_filename)): 68 | return print("Couldn't find the bone image.") 69 | else: 70 | os.rename( 71 | get_asset_path(comp_output_bone_filename), 72 | get_asset_path(before_output_bone_filename), 73 | ) 74 | if is_send_seg: 75 | if not os.path.exists(get_asset_path(comp_output_seg_filename)): 76 | return print("Couldn't find the seg image.") 77 | else: 78 | os.rename( 79 | get_asset_path(comp_output_seg_filename), 80 | get_asset_path(before_output_seg_filename), 81 | ) 82 | 83 | # prepare data for API 84 | params = { 85 | "prompt": "a room", 86 | "negative_prompt": "(worst quality:2), (low quality:2), (normal quality:2), lowres, normal quality", 87 | "width": get_output_width(scene), 88 | "height": get_output_height(scene), 89 | "sampler_index": "DPM++ SDE Karras", 90 | "sampler_name": "", 91 | "batch_size": 1, 92 | "n_iter": 1, 93 | "steps": 20, 94 | "cfg_scale": 7, 95 | "seed": -1, 96 | "subseed": -1, 97 | "subseed_strength": 0, 98 | "restore_faces": False, 99 | "enable_hr": False, 100 | "hr_scale": 1.5, 101 | "hr_upscaler": "R-ESRGAN General WDN 4xV3", 102 | "denoising_strength": 0.5, 103 | "hr_second_pass_steps": 10, 104 | "hr_resize_x": 0, 105 | "hr_resize_y": 0, 106 | "firstphase_width": 0, 107 | "firstphase_height": 0, 108 | "override_settings": {"CLIP_stop_at_last_layers": 2}, 109 | "override_settings_restore_afterwards": True, 110 | "alwayson_scripts": {"controlnet": {"args": []}}, 111 | } 112 | 113 | # if is_send_canny is True: 114 | if is_send_canny: 115 | canny_cn_units = { 116 | "mask": "", 117 | "module": "none", 118 | "model": "diff_control_sd15_canny_fp16 [ea6e3b9c]", 119 | "weight": 1.2, 120 | "resize_mode": "Scale to Fit (Inner Fit)", 121 | "lowvram": False, 122 | "processor_res": 64, 123 | "threshold_a": 64, 124 | "threshold_b": 64, 125 | "guidance": 1, 126 | "guidance_start": 0.19, 127 | "guidance_end": 1, 128 | "guessmode": False, 129 | } 130 | with open(get_asset_path(before_output_canny_filename), "rb") as canny_file: 131 | canny_cn_units["input_image"] = base64.b64encode(canny_file.read()).decode() 132 | params["alwayson_scripts"]["controlnet"]["args"].append(canny_cn_units) 133 | 134 | # if is_send_depth is True: 135 | if is_send_depth: 136 | depth_cn_units = { 137 | "mask": "", 138 | "module": "none", 139 | "model": "diff_control_sd15_depth_fp16 [978ef0a1]", 140 | "weight": 1.2, 141 | "resize_mode": "Scale to Fit (Inner Fit)", 142 | "lowvram": False, 143 | "processor_res": 64, 144 | "threshold_a": 64, 145 | "threshold_b": 64, 146 | "guidance": 1, 147 | "guidance_start": 0.19, 148 | "guidance_end": 1, 149 | "guessmode": False, 150 | } 151 | with open(get_asset_path(before_output_depth_filename), "rb") as depth_file: 152 | depth_cn_units["input_image"] = base64.b64encode(depth_file.read()).decode() 153 | params["alwayson_scripts"]["controlnet"]["args"].append(depth_cn_units) 154 | 155 | # if is_send_bone is True: 156 | if is_send_bone: 157 | bone_cn_units = { 158 | "mask": "", 159 | "module": "none", 160 | "model": "diff_control_sd15_openpose_fp16 [1723948e]", 161 | "weight": 1.1, 162 | "resize_mode": "Scale to Fit (Inner Fit)", 163 | "lowvram": False, 164 | "processor_res": 64, 165 | "threshold_a": 64, 166 | "threshold_b": 64, 167 | "guidance": 1, 168 | "guidance_start": 0, 169 | "guidance_end": 1, 170 | "guessmode": False, 171 | } 172 | with open(get_asset_path(before_output_bone_filename), "rb") as bone_file: 173 | bone_cn_units["input_image"] = base64.b64encode(bone_file.read()).decode() 174 | params["alwayson_scripts"]["controlnet"]["args"].append(bone_cn_units) 175 | 176 | # if is_send_seg is True: 177 | if is_send_seg: 178 | seg_cn_units = { 179 | "mask": "", 180 | "module": "none", 181 | "model": "diff_control_sd15_seg_fp16 [a1e85e27]", 182 | "weight": 1, 183 | "resize_mode": "Scale to Fit (Inner Fit)", 184 | "lowvram": False, 185 | "processor_res": 64, 186 | "threshold_a": 64, 187 | "threshold_b": 64, 188 | "guidance": 1, 189 | "guidance_start": 0, 190 | "guidance_end": 1, 191 | "guessmode": False, 192 | } 193 | with open(get_asset_path(before_output_seg_filename), "rb") as seg_file: 194 | seg_cn_units["input_image"] = base64.b64encode(seg_file.read()).decode() 195 | params["alwayson_scripts"]["controlnet"]["args"].append(seg_cn_units) 196 | 197 | # send to API 198 | output_file = actually_send_to_api(params, after_output_filename_prefix) 199 | 200 | # if we got a successful image created, load it into the scene 201 | if output_file: 202 | new_output_file = None 203 | 204 | # save the after image 205 | new_output_file = save_after_image( 206 | scene, after_output_filename_prefix, output_file 207 | ) 208 | 209 | # if we saved a new output image, use it 210 | if new_output_file: 211 | output_file = new_output_file 212 | 213 | # load the image into image editor 214 | try: 215 | img = bpy.data.images.load(output_file, check_existing=False) 216 | for window in bpy.data.window_managers["WinMan"].windows: 217 | for area in window.screen.areas: 218 | if area.type == "IMAGE_EDITOR": 219 | area.spaces.active.image = img 220 | except: 221 | return print("Couldn't load the image.") 222 | 223 | return True 224 | else: 225 | return False 226 | 227 | 228 | def actually_send_to_api(params, filename_prefix): 229 | # create headers 230 | headers = { 231 | "User-Agent": "Blender/" + bpy.app.version_string, 232 | "Accept": "*/*", 233 | "Accept-Encoding": "gzip, deflate, br", 234 | } 235 | 236 | # prepare server url 237 | server_url = "http://localhost:7860" + "/sdapi/v1/txt2img" 238 | 239 | # send API request 240 | try: 241 | response = requests.post(server_url, json=params, headers=headers, timeout=1000) 242 | except requests.exceptions.ConnectionError: 243 | return print(f"The Automatic1111 server couldn't be found.") 244 | except requests.exceptions.MissingSchema: 245 | return print(f"The url for your Automatic1111 server is invalid.") 246 | except requests.exceptions.ReadTimeout: 247 | return print("The Automatic1111 server timed out.") 248 | 249 | # handle the response 250 | if response.status_code == 200: 251 | return handle_api_success(response, filename_prefix) 252 | else: 253 | return handle_api_error(response) 254 | 255 | 256 | def handle_api_success(response, filename_prefix): 257 | try: 258 | response_obj = response.json() 259 | base64_img = response_obj["images"][0] 260 | except: 261 | print("Automatic1111 response content: ") 262 | print(response.content) 263 | return print("Received an unexpected response from the Automatic1111 server.") 264 | 265 | # create a temp file 266 | try: 267 | output_file = create_temp_file(filename_prefix + "-") 268 | except: 269 | return print("Couldn't create a temp file to save image.") 270 | 271 | # decode base64 image 272 | try: 273 | img_binary = base64.b64decode(base64_img.replace("data:image/png;base64,", "")) 274 | except: 275 | return print("Couldn't decode base64 image.") 276 | 277 | # save the image to the temp file 278 | try: 279 | with open(output_file, "wb") as file: 280 | file.write(img_binary) 281 | except: 282 | return print("Couldn't write to temp file.") 283 | 284 | # return the temp file 285 | return output_file 286 | 287 | 288 | def handle_api_error(response): 289 | if response.status_code == 404: 290 | import json 291 | 292 | try: 293 | response_obj = response.json() 294 | if response_obj.get("detail") and response_obj["detail"] == "Not Found": 295 | return print( 296 | "It looks like the Automatic1111 server is running, but it's not in API mode." 297 | ) 298 | elif ( 299 | response_obj.get("detail") 300 | and response_obj["detail"] == "Sampler not found" 301 | ): 302 | return print("The sampler you selected is not available.") 303 | else: 304 | return print( 305 | f"An error occurred in the Automatic1111 server. Full server response: {json.dumps(response_obj)}" 306 | ) 307 | except: 308 | return print( 309 | "It looks like the Automatic1111 server is running, but it's not in API mode." 310 | ) 311 | else: 312 | print(response.content) 313 | return print("An error occurred in the Automatic1111 server.") 314 | 315 | 316 | def create_temp_file(prefix, suffix=".png"): 317 | return tempfile.NamedTemporaryFile(prefix=prefix, suffix=suffix).name 318 | 319 | 320 | def save_after_image(scene, filename_prefix, img_file): 321 | filename = f"{filename_prefix}.png" 322 | full_path_and_filename = os.path.join( 323 | os.path.abspath(bpy.path.abspath(IMAGE_FOLDER)), filename 324 | ) 325 | try: 326 | copy_file(img_file, full_path_and_filename) 327 | return full_path_and_filename 328 | except: 329 | return print( 330 | f"Couldn't save 'after' image to {bpy.path.abspath(full_path_and_filename)}" 331 | ) 332 | 333 | 334 | def get_absolute_path(path): 335 | return os.path.abspath(bpy.path.abspath(path)) 336 | 337 | 338 | def get_asset_path(filename): 339 | return os.path.join(get_absolute_path(IMAGE_FOLDER), filename) 340 | 341 | 342 | def get_output_width(scene): 343 | return round(scene.render.resolution_x * scene.render.resolution_percentage / 100) 344 | 345 | 346 | def get_output_height(scene): 347 | return round(scene.render.resolution_y * scene.render.resolution_percentage / 100) 348 | 349 | 350 | def copy_file(src, dest): 351 | shutil.copy2(src, dest) 352 | 353 | 354 | bpy.app.handlers.render_complete.clear() 355 | bpy.app.handlers.render_complete.append(render_complete_handler) 356 | -------------------------------------------------------------------------------- /seg.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import os 3 | import bpy 4 | import math 5 | 6 | 7 | def newMaterial(id): 8 | mat = bpy.data.materials.get(id) 9 | 10 | if mat is None: 11 | mat = bpy.data.materials.new(name=id) 12 | 13 | mat.use_nodes = True 14 | 15 | if mat.node_tree: 16 | mat.node_tree.links.clear() 17 | mat.node_tree.nodes.clear() 18 | 19 | return mat 20 | 21 | 22 | def newShader(id, type, r, g, b): 23 | mat = newMaterial(id) 24 | 25 | nodes = mat.node_tree.nodes 26 | links = mat.node_tree.links 27 | output = nodes.new(type='ShaderNodeOutputMaterial') 28 | 29 | if type == "diffuse": 30 | shader = nodes.new(type='ShaderNodeBsdfDiffuse') 31 | nodes["Diffuse BSDF"].inputs[0].default_value = (r, g, b, 1) 32 | 33 | elif type == "emission": 34 | shader = nodes.new(type='ShaderNodeEmission') 35 | nodes["Emission"].inputs[0].default_value = (r, g, b, 1) 36 | nodes["Emission"].inputs[1].default_value = 1 37 | 38 | elif type == "glossy": 39 | shader = nodes.new(type='ShaderNodeBsdfGlossy') 40 | nodes["Glossy BSDF"].inputs[0].default_value = (r, g, b, 1) 41 | nodes["Glossy BSDF"].inputs[1].default_value = 0 42 | 43 | links.new(shader.outputs[0], output.inputs[0]) 44 | 45 | return mat 46 | 47 | 48 | def to_blender_color(c): 49 | c = min(max(0, c), 255) / 255 50 | return c / 12.92 if c < 0.04045 else math.pow((c + 0.055) / 1.055, 2.4) 51 | 52 | 53 | filename = os.path.abspath(bpy.path.abspath("//color_coding_semantic_segmentation_classes.csv")) 54 | with open(filename) as csv_file: 55 | csv_reader = csv.reader(csv_file, delimiter=',') 56 | line_count = 0 57 | for row in csv_reader: 58 | if line_count == 0: 59 | print(f'Column names are {", ".join(row)}') 60 | line_count += 1 61 | else: 62 | print(f'\t Creating shader {row[8]}: {row[5]}') 63 | r,g,b = row[5].strip("()").split(",") 64 | newShader(row[8], "emission", to_blender_color(float(r)), to_blender_color(float(g)), to_blender_color(float(b))) 65 | line_count += 1 66 | print(f'Processed {line_count} lines.') 67 | --------------------------------------------------------------------------------