├── .gitignore ├── README.md ├── __init__.py ├── blender_manifest.toml ├── condition_setup.py ├── diffusedtexture ├── diffusers_utils.py ├── img_parallel.py ├── img_parasequential.py ├── img_sequential.py ├── latent_parallel.py ├── process_operations.py ├── process_utils.py └── uv_pass.py ├── documentation ├── DOCUMENTATION.md └── WORKFLOW.md ├── images ├── download.png ├── elephant.gif ├── install.png ├── process │ ├── cameras_16.png │ ├── cameras_4.png │ └── cameras_9.png ├── rabbit.gif └── usage.gif ├── object_ops.py ├── operators.py ├── panel.py ├── properties.py ├── render_setup.py ├── requirements.txt ├── scene_backup.py └── utils.py /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | *__pycache__ 3 | .__pycache__ 4 | *.py[cod] 5 | *$py.class 6 | 7 | # conda env 8 | .conda/ 9 | 10 | # scratchbook code snippets for prototyping 11 | scratchbook/ 12 | 13 | # Distribution / packaging 14 | .Python build/ 15 | develop-eggs/ 16 | dist/ 17 | downloads/ 18 | eggs/ 19 | .eggs/ 20 | lib/ 21 | lib64/ 22 | parts/ 23 | sdist/ 24 | var/ 25 | wheels/ 26 | *.whl 27 | *.egg-info/ 28 | .installed.cfg 29 | *.egg 30 | *.manifest 31 | *.spec 32 | *.zip 33 | 34 | # Log files 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | *.log 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .coverage 43 | .coverage.* 44 | .cache 45 | .pytest_cache/ 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | .hypothesis/ 50 | 51 | # Translations 52 | *.mo 53 | *.pot 54 | 55 | # PyBuilder 56 | target/ 57 | 58 | # Jupyter Notebook 59 | .ipynb_checkpoints 60 | 61 | # IPython 62 | profile_default/ 63 | ipython_config.py 64 | 65 | # pyenv 66 | .python-version 67 | 68 | # pyflow 69 | __pypackages__/ 70 | 71 | # Environment 72 | .env 73 | .venv 74 | env/ 75 | venv/ 76 | ENV/ 77 | 78 | # If you are using PyCharm # 79 | .idea/**/workspace.xml 80 | .idea/**/tasks.xml 81 | .idea/dictionaries 82 | .idea/**/dataSources/ 83 | .idea/**/dataSources.ids 84 | .idea/**/dataSources.xml 85 | .idea/**/dataSources.local.xml 86 | .idea/**/sqlDataSources.xml 87 | .idea/**/dynamic.xml 88 | .idea/**/uiDesigner.xml 89 | .idea/**/gradle.xml 90 | .idea/**/libraries 91 | *.iws /out/ 92 | 93 | # Sublime Text 94 | *.tmlanguage.cache 95 | *.tmPreferences.cache 96 | *.stTheme.cache 97 | *.sublime-workspace 98 | *.sublime-project 99 | 100 | # sftp configuration file 101 | sftp-config.json 102 | 103 | # Package control specific files Package 104 | Control.last-run 105 | Control.ca-list 106 | Control.ca-bundle 107 | Control.system-ca-bundle 108 | GitHub.sublime-settings 109 | 110 | # Visual Studio Code # 111 | .vscode 112 | .vscode/* 113 | !.vscode/settings.json 114 | !.vscode/tasks.json 115 | !.vscode/launch.json 116 | !.vscode/extensions.json 117 | .history 118 | 119 | # Dont commit data by accident! 120 | *.exr 121 | *.blend -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # DiffusedTexture: AI-Powered Texture Generation for Blender 2 | 3 | DiffusedTexture is a Blender add-on that uses Stable Diffusion to create textures directly on 3D meshes. 4 | 5 | ![General Usage](https://github.com/FrederikHasecke/diffused-texture-addon/blob/master/images/usage.gif) 6 | 7 | ## Table of Contents 8 | - [DiffusedTexture: AI-Powered Texture Generation for Blender](#diffusedtexture-ai-powered-texture-generation-for-blender) 9 | - [Table of Contents](#table-of-contents) 10 | - [Examples](#examples) 11 | - [Features](#features) 12 | - [Installation (Windows)](#installation-windows) 13 | - [Installation (Linux)](#installation-linux) 14 | - [Setup](#setup) 15 | - [Blender Configuration](#blender-configuration) 16 | - [Usage](#usage) 17 | - [Additional Options](#additional-options) 18 | - [Troubleshooting](#troubleshooting) 19 | - [**Roadmap**](#roadmap) 20 | - [**Acknowledgements**](#acknowledgements) 21 | 22 | ## Examples 23 | ![https://www.cgtrader.com/free-3d-print-models/miniatures/other/elephant-natural-history-museum-1](https://github.com/FrederikHasecke/diffused-texture-addon/blob/master/images/elephant.gif) 24 | ![https://graphics.stanford.edu/data/3Dscanrep/](https://github.com/FrederikHasecke/diffused-texture-addon/blob/master/images/rabbit.gif) 25 | 26 | 27 | ## Features 28 | - **AI-Driven Texture Creation:** Generate diffuse textures directly on 3D models 29 | - **Modes for Different Workflows:** 30 | - **Text2Image Parallel**: Create textures from text prompts, ensuring global consistency. 31 | - **Image2Image Parallel**: Generates textures from input textures, applied parallel across all views. 32 | - **Image2Image Sequential**: Sequentially adjusts textures across views, great for refinement. 33 | - **LoRA Integration**: Uses LoRA conditioning for specific styles. 34 | - **IPAdapter Integration**: Fit specific styles or objects with images for enhanced flexibility and control. 35 | 36 | ## Installation (Windows) 37 | 0. Download [7-Zip](https://7-zip.de/download.html) 38 | 1. Download all .tar files of the [latest release](https://github.com/FrederikHasecke/diffused-texture-addon/releases/latest) 39 | 2. Untar the file `diffused_texture_addon-0.0.4-windows_x64.7z.001` this will automatically untar the other `.7z` files 40 | >**WARNING:** _DO NOT_ unzip the resulting `diffused_texture_addon-0.0.4-windows_x64.zip` 41 | 3. If you did not already do so: **__You need to "Allow Online Access" under "System" in the Preferences.__** 42 | 4. Install the `diffused_texture_addon-0.0.4-windows_x64.zip` file in Blender as an Add-On. 43 | - `Edit` -> `Preferences...` -> Sidebar `Add-ons` -> Top right corner dropdown menu -> `Install from Disk...` 44 | 45 | ![Installatíon](https://github.com/FrederikHasecke/diffused-texture-addon/blob/master/images/install.png) 46 | - If neccessary, provide a custom `HuggingFace Cache Path` to install and/or load the checkpoints, else the default path is choosen. 47 | 48 | ![Download](https://github.com/FrederikHasecke/diffused-texture-addon/blob/master/images/download.png) 49 | - Download necessary models (~10.6 GB total): 50 | - **Tip:** Open Blender's system console (`Window > Toggle System Console`) __BEFORE__ starting the download to monitor download progress. 51 | 52 | ## Installation (Linux) 53 | Download and install the [latest release](https://github.com/FrederikHasecke/diffused-texture-addon/releases/latest) `diffused_texture_addon-0.0.4-linux_x64.zip` file in Blender as an Add-On. 54 | - `Edit` -> `Preferences...` -> Sidebar `Add-ons` -> Top right corner dropdown menu -> `Install from Disk...` 55 | 56 | ![Installatíon](https://github.com/FrederikHasecke/diffused-texture-addon/blob/master/images/install.png) 57 | - If neccessary, provide a custom `HuggingFace Cache Path` to install and/or load the checkpoints, else the default path is choosen. 58 | 59 | ![Download](https://github.com/FrederikHasecke/diffused-texture-addon/blob/master/images/download.png) 60 | - Download necessary models (~10.6 GB total): 61 | - **Tip:** Open Blender's system console (`Window > Toggle System Console`) __BEFORE__ starting the download to monitor download progress. 62 | 63 | ## Setup 64 | ### Blender Configuration 65 | 1. Enable CUDA or OPTIX in Blender if using an NVIDIA GPU. 66 | - Go to `Edit > Preferences > System` and configure GPU settings. 67 | - **Note:** Requires a modern NVIDIA GPU with at least 4GB(-ish) VRAM, 9 and 16 camera parallel runs will require more VRAM. 68 | 69 | ## Usage 70 | 71 | 1. **Load a 3D Model**: 72 | - Import or create a `.blend` file containing the 3D model. 73 | 2. **UV Unwrap the Model**: 74 | - Apply a UV map (`Smart UV Project` works well). 75 | 3. **Access the Add-On**: 76 | - Open the `DiffusedTexture` panel in the N-panel (right-hand sidebar). 77 | 4. **Set Up Texture Generation**: 78 | - **Prompt & Negative Prompt**: Describe the desired texture/object and what to avoid. 79 | - **Guidance Scale**: Adjust creativity vs. fidelity. 80 | - **Denoise Strength**: Default to `1.0` for `Text2Image`. 81 | 5. **Adjust Advanced Options**: 82 | - **Mesh Complexity**: 83 | - `Low`: Depth ControlNet only. 84 | - `Medium`: Adds Canny ControlNet. 85 | - `High`: Adds Normalmap ControlNet for maximum detail. 86 | - **Cameras**: Use more viewpoints for better texture blending. 87 | - **Texture & Render Resolution**: Ensure render resolution is at least 2x texture resolution. 88 | 6. **Generate Texture**: 89 | - Click `Start Texture Generation`. Monitor progress in the system console. 90 | 91 | 92 | ### Additional Options 93 | - **LoRA Models**: Add one or multiple LoRA models to match specific results. 94 | - **IPAdapter**: Supply the desired "look" as an image instead of a text prompt. 95 | 96 | ## Troubleshooting 97 | - **Add-On Not Visible**: Ensure it’s enabled in `Edit > Preferences > Add-ons`. 98 | - **Blender Freezes**: Open the system console to track progress during long tasks. 99 | - **Permission Issues**: Specify a valid output path. 100 | - **Out of GPU Memory**: 101 | - Reduce camera count. 102 | - Close other GPU-intensive applications. 103 | - **Crashes**: Restart Blender or your PC if crashes persist. 104 | 105 | ## **Roadmap** 106 | - **Performance Enhancements**: 107 | - Multi-threaded execution to prevent UI freezing. 108 | - **Checkpoint Flexibility**: 109 | - Allow external Stable Diffusion checkpoints for varied outputs. 110 | - **Masking Support**: 111 | - Apply textures to specific areas of the mesh. 112 | - **Multi-Mesh Workflow**: 113 | - Simultaneously texture multiple objects. 114 | 115 | ## **Acknowledgements** 116 | - Inspired by [Dream Textures](https://github.com/carson-katri/dream-textures) by [Carson Katri](https://github.com/carson-katri). 117 | - Powered by: 118 | - [Stable Diffusion](https://arxiv.org/pdf/2112.10752) 119 | - [HuggingFace Diffusers](https://huggingface.co/docs/diffusers/index) 120 | - [ControlNet](https://arxiv.org/pdf/2302.05543) 121 | - [IPAdapter](https://arxiv.org/pdf/2308.06721) 122 | - Influenced by research in [TEXTure](https://arxiv.org/pdf/2302.01721), [Text2Tex](https://arxiv.org/pdf/2303.11396), [Paint3D](https://arxiv.org/pdf/2312.13913), [MatAtlas](https://arxiv.org/pdf/2404.02899) and [EucliDreamer](https://arxiv.org/pdf/2404.10279). 123 | -------------------------------------------------------------------------------- /__init__.py: -------------------------------------------------------------------------------- 1 | import bpy 2 | import os 3 | import sys 4 | 5 | from .properties import register_properties, unregister_properties 6 | from .operators import OBJECT_OT_GenerateTexture, OBJECT_OT_SelectPipette 7 | from .panel import ( 8 | OBJECT_PT_MainPanel, 9 | OBJECT_OT_OpenNewInputImage, 10 | OBJECT_PT_IPAdapterPanel, 11 | OBJECT_OT_OpenNewIPAdapterImage, 12 | OBJECT_PT_LoRAPanel, 13 | OBJECT_PT_AdvancedPanel, 14 | ) 15 | 16 | 17 | class MockUpScene: 18 | """ 19 | A mockup class to simulate the scene properties for pipeline testing. 20 | """ 21 | 22 | def __init__(self): 23 | self.num_loras = 0 24 | self.use_ipadapter = True 25 | self.ipadapter_strength = 0.5 26 | self.mesh_complexity = "HIGH" 27 | self.depth_controlnet_strength = 1.0 28 | self.canny_controlnet_strength = 1.0 29 | self.normal_controlnet_strength = 1.0 30 | self.sd_version = "sd15" 31 | self.checkpoint_path = "runwayml/stable-diffusion-v1-5" 32 | self.canny_controlnet_path = "lllyasviel/sd-controlnet-canny" 33 | self.normal_controlnet_path = "lllyasviel/sd-controlnet-normal" 34 | self.depth_controlnet_path = "lllyasviel/sd-controlnet-depth" 35 | self.controlnet_type = "MULTIPLE" 36 | 37 | 38 | class InstallModelsOperator(bpy.types.Operator): 39 | """Operator to install necessary models""" 40 | 41 | bl_idname = "diffused_texture_addon.install_models" 42 | bl_label = "Install Models" 43 | bl_description = "Install the necessary models for DiffusedTexture" 44 | bl_options = {"REGISTER", "INTERNAL"} 45 | 46 | def execute(self, context): 47 | # Retrieve the HuggingFace cache path from preferences 48 | prefs = bpy.context.preferences.addons[__package__].preferences 49 | hf_cache_path = prefs.hf_cache_path 50 | 51 | # Set environment variable if path is provided 52 | if hf_cache_path: 53 | os.environ["HF_HOME"] = hf_cache_path 54 | 55 | # If blender is set to offline this will not work, but lets not sneak always on onto the user 56 | if not bpy.app.online_access: 57 | os.environ["HF_HUB_OFFLINE"] = "1" 58 | 59 | # Logic for model installation 60 | try: 61 | 62 | # Import after setting HF_HOME 63 | import diffusers 64 | from .diffusedtexture.diffusers_utils import create_pipeline 65 | 66 | # Safely create the cache directory if it does not exist 67 | if hf_cache_path: 68 | os.makedirs(hf_cache_path, exist_ok=True) 69 | 70 | # Create the pipeline 71 | mockup_scene = MockUpScene() 72 | 73 | pipe = create_pipeline(mockup_scene) 74 | del pipe # Clean up to avoid memory issues 75 | 76 | self.report({"INFO"}, f"Models installed successfully in {hf_cache_path}.") 77 | except Exception as e: 78 | self.report({"ERROR"}, f"Failed to install models: {str(e)}") 79 | return {"CANCELLED"} 80 | 81 | return {"FINISHED"} 82 | 83 | 84 | class DiffuseTexPreferences(bpy.types.AddonPreferences): 85 | # bl_idname = __package__ 86 | bl_idname = __package__ 87 | 88 | # Path setting for HuggingFace cache directory 89 | hf_cache_path: bpy.props.StringProperty( 90 | name="HuggingFace Cache Path", 91 | description="Path to a custom HuggingFace cache directory", 92 | subtype="DIR_PATH", 93 | default="", 94 | ) 95 | 96 | def draw(self, context): 97 | layout = self.layout 98 | 99 | # Add a text block to explain that the user needs to explicitly allow online access 100 | box = layout.box() 101 | row = box.row() 102 | row.label(text="Please ensure that Blender is allowed to access the internet") 103 | row = box.row() 104 | row.label(text="in order to install models. Do so in:") 105 | row = box.row() 106 | row.label(text="Preferences > System > Network > Allow Online Access.") 107 | 108 | # HuggingFace Cache Path setting 109 | layout.prop(self, "hf_cache_path", text="HuggingFace Cache Path") 110 | 111 | # make the Install Models button unavailable if the "online access" is disabled 112 | if not bpy.app.online_access: 113 | layout.label( 114 | text="Online access is disabled. Enable it in Preferences > System > Network > Allow Online Access." 115 | ) 116 | 117 | row = layout.row() 118 | row.enabled = False 119 | row.operator( 120 | InstallModelsOperator.bl_idname, text="Install Models", icon="IMPORT" 121 | ) 122 | 123 | else: 124 | # Button to execute the model installation function 125 | layout.operator( 126 | InstallModelsOperator.bl_idname, text="Install Models", icon="IMPORT" 127 | ) 128 | 129 | 130 | classes = [ 131 | DiffuseTexPreferences, 132 | InstallModelsOperator, 133 | OBJECT_OT_GenerateTexture, 134 | OBJECT_OT_SelectPipette, 135 | OBJECT_PT_MainPanel, 136 | OBJECT_OT_OpenNewInputImage, 137 | OBJECT_PT_AdvancedPanel, 138 | OBJECT_PT_IPAdapterPanel, 139 | OBJECT_OT_OpenNewIPAdapterImage, 140 | OBJECT_PT_LoRAPanel, 141 | ] 142 | 143 | 144 | def register(): 145 | for cls in classes: 146 | bpy.utils.register_class(cls) 147 | register_properties() 148 | 149 | 150 | def unregister(): 151 | unregister_properties() 152 | for cls in classes: 153 | bpy.utils.unregister_class(cls) 154 | 155 | 156 | if __name__ == "__main__": 157 | register() 158 | -------------------------------------------------------------------------------- /blender_manifest.toml: -------------------------------------------------------------------------------- 1 | schema_version = "1.0.0" 2 | 3 | id = "diffused_texture_addon" 4 | version = "0.0.4" 5 | name = "DiffusedTexture" 6 | tagline = "Generate Diffuse Textures on Meshes with Stable Diffusion" 7 | maintainer = "Frederik Hasecke " 8 | type = "add-on" 9 | 10 | website = "https://github.com/FrederikHasecke/diffused-texture-addon/" 11 | tags = ["Material", "Paint"] 12 | 13 | blender_version_min = "4.2.0" 14 | license = ["SPDX:GPL-3.0-or-later"] 15 | 16 | # ✅ Specify the supported platforms, but do not define per-platform lists 17 | platforms = ["windows-x64", "linux-x64"] 18 | 19 | # ✅ Flat list of wheels - Blender will split them automatically 20 | wheels = [ 21 | # Common wheels (cross-platform) 22 | "./wheels/certifi-2024.12.14-py3-none-any.whl", 23 | "./wheels/colorama-0.4.6-py2.py3-none-any.whl", 24 | "./wheels/filelock-3.16.1-py3-none-any.whl", 25 | "./wheels/fsspec-2024.12.0-py3-none-any.whl", 26 | "./wheels/idna-3.10-py3-none-any.whl", 27 | "./wheels/importlib_metadata-8.5.0-py3-none-any.whl", 28 | "./wheels/jinja2-3.1.5-py3-none-any.whl", 29 | "./wheels/mpmath-1.3.0-py3-none-any.whl", 30 | "./wheels/networkx-3.4.2-py3-none-any.whl", 31 | "./wheels/packaging-24.2-py3-none-any.whl", 32 | "./wheels/peft-0.14.0-py3-none-any.whl", 33 | "./wheels/requests-2.32.3-py3-none-any.whl", 34 | "./wheels/sympy-1.13.1-py3-none-any.whl", 35 | "./wheels/tqdm-4.67.1-py3-none-any.whl", 36 | "./wheels/typing_extensions-4.12.2-py3-none-any.whl", 37 | "./wheels/urllib3-2.3.0-py3-none-any.whl", 38 | "./wheels/zipp-3.21.0-py3-none-any.whl", 39 | "./wheels/huggingface_hub-0.27.1-py3-none-any.whl", 40 | "./wheels/accelerate-1.3.0-py3-none-any.whl", 41 | "./wheels/transformers-4.48.0-py3-none-any.whl", 42 | "./wheels/diffusers-0.32.2-py3-none-any.whl", 43 | 44 | # ✅ Windows-specific wheels 45 | "./wheels/scipy-1.15.1-cp311-cp311-win_amd64.whl", 46 | "./wheels/charset_normalizer-3.4.1-cp311-cp311-win_amd64.whl", 47 | "./wheels/MarkupSafe-3.0.2-cp311-cp311-win_amd64.whl", 48 | "./wheels/PyYAML-6.0.2-cp311-cp311-win_amd64.whl", 49 | "./wheels/psutil-6.1.1-cp311-abi3-win_amd64.whl", 50 | "./wheels/regex-2024.11.6-cp311-cp311-win_amd64.whl", 51 | "./wheels/pillow-11.1.0-cp311-cp311-win_amd64.whl", 52 | "./wheels/opencv_python_headless-4.8.1.78-cp311-abi3-win_amd64.whl", 53 | "./wheels/torch-2.5.1+cu118-cp311-cp311-win_amd64.whl", 54 | "./wheels/tokenizers-0.21.0-cp311-abi3-win_amd64.whl", 55 | "./wheels/safetensors-0.5.2-cp311-abi3-win_amd64.whl", 56 | 57 | # ✅ Linux-specific wheels 58 | "./wheels/scipy-1.15.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", 59 | "./wheels/charset_normalizer-3.4.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", 60 | "./wheels/MarkupSafe-3.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", 61 | "./wheels/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", 62 | "./wheels/psutil-6.1.1-cp311-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", 63 | "./wheels/regex-2024.11.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", 64 | "./wheels/pillow-11.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", 65 | "./wheels/opencv_python_headless-4.8.1.78-cp311-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", 66 | "./wheels/torch-2.5.1-cp311-cp311-manylinux1_x86_64.whl", 67 | "./wheels/tokenizers-0.21.0-cp311-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", 68 | "./wheels/safetensors-0.5.2-cp311-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" 69 | ] 70 | -------------------------------------------------------------------------------- /condition_setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | import bpy 3 | import numpy as np 4 | from .diffusedtexture.process_utils import blendercs_to_ccs 5 | 6 | 7 | def bpy_img_to_numpy(img_path): 8 | """ 9 | Load an image as a Blender image and converts it to a NumPy array. 10 | 11 | :param img_path: The path to the image. 12 | :return: A NumPy array representation of the image. 13 | """ 14 | 15 | # Load image using Blender's bpy 16 | bpy.data.images.load(img_path) 17 | 18 | # Get the file name from the path 19 | img_file_name = os.path.basename(img_path) 20 | 21 | # Access the image by name after loading 22 | img_bpy = bpy.data.images.get(img_file_name) 23 | 24 | # Get image dimensions 25 | width, height = img_bpy.size 26 | 27 | # Determine the number of channels 28 | num_channels = len(img_bpy.pixels) // (width * height) 29 | 30 | # Convert the flat pixel array to a NumPy array 31 | pixels = np.array(img_bpy.pixels[:], dtype=np.float32) 32 | 33 | # Reshape the array to match the image's dimensions and channels 34 | image_array = pixels.reshape((height, width, num_channels)) 35 | 36 | # FLIP vertically 37 | image_array = np.flipud(image_array) 38 | 39 | return image_array 40 | 41 | 42 | def create_depth_condition(depth_image_path): 43 | 44 | depth_array = bpy_img_to_numpy(depth_image_path)[..., 0] 45 | 46 | # Replace large values with NaN (assuming 1e10 represents invalid depth) 47 | depth_array[depth_array >= 1e10] = np.nan 48 | 49 | # Invert the depth values so that closer objects have higher values 50 | depth_array = np.nanmax(depth_array) - depth_array 51 | 52 | # Normalize the depth array to range [0, 1] 53 | depth_array -= np.nanmin(depth_array) 54 | depth_array /= np.nanmax(depth_array) 55 | 56 | # Add a small margin to the background 57 | depth_array += 10 / 255.0 # Approximately 0.039 58 | 59 | # Replace NaN values with 0 (background) 60 | depth_array[np.isnan(depth_array)] = 0 61 | 62 | # Normalize again to ensure values are within [0, 1] 63 | depth_array /= np.nanmax(depth_array) 64 | 65 | # Scale to [0, 255] and convert to uint8 66 | depth_array = (depth_array * 255).astype(np.uint8) 67 | 68 | # stack on third axis to get a rgb png 69 | depth_array = np.stack((depth_array, depth_array, depth_array), axis=-1) 70 | 71 | return depth_array 72 | 73 | 74 | def create_normal_condition(normal_img_path, camera_obj): 75 | normal_array = bpy_img_to_numpy(normal_img_path) 76 | 77 | normal_array = normal_array[..., :3] 78 | 79 | # Get image dimensions 80 | image_size = normal_array.shape[:2] 81 | 82 | # Flatten the normal array for transformation 83 | normal_pc = normal_array.reshape((-1, 3)) 84 | 85 | # Rotate the normal vectors to the camera space without translating 86 | normal_pc = blendercs_to_ccs( 87 | points_bcs=normal_pc, camera=camera_obj, rotation_only=True 88 | ) 89 | 90 | # Map normalized values to the [0, 1] range for RGB display 91 | red_channel = ((normal_pc[:, 0] + 1) / 2).reshape(image_size) # Normal X 92 | green_channel = ((normal_pc[:, 1] + 1) / 2).reshape(image_size) # Normal Y 93 | blue_channel = ((normal_pc[:, 2] + 1) / 2).reshape(image_size) # Normal Z 94 | 95 | # Adjust to shapenet colors 96 | blue_channel = 1 - blue_channel 97 | green_channel = 1 - green_channel 98 | 99 | # Stack channels into a single image 100 | normal_image = np.stack((red_channel, green_channel, blue_channel), axis=-1) 101 | normal_image = np.clip(normal_image, 0, 1) 102 | 103 | # Convert to uint8 for display 104 | normal_image *= 255.0 105 | normal_image = normal_image.astype(np.uint8) 106 | 107 | return normal_image 108 | 109 | 110 | def create_similar_angle_image(normal_array, position_array, camera_obj): 111 | """ 112 | Create an image where each pixel's intensity represents how aligned the normal vector at 113 | that point is with the direction vector from the point to the camera. 114 | 115 | :param normal_array: NumPy array of shape (height, width, 3) containing normal vectors. 116 | :param position_array: NumPy array of shape (height, width, 3) containing positions in global coordinates. 117 | :param camera_obj: Blender camera object to get the camera position in global coordinates. 118 | 119 | :return: A NumPy array (height, width) with values ranging from 0 to 1, where 1 means perfect alignment. 120 | """ 121 | 122 | # Extract camera position in global coordinates 123 | camera_position = np.array(camera_obj.matrix_world.to_translation()) 124 | 125 | # Calculate direction vectors from each point to the camera 126 | direction_to_camera = position_array - camera_position[None, None, :] 127 | 128 | # Normalize the normal vectors and direction vectors 129 | normal_array_normalized = normal_array / np.linalg.norm( 130 | normal_array, axis=2, keepdims=True 131 | ) 132 | direction_to_camera_normalized = direction_to_camera / np.linalg.norm( 133 | direction_to_camera, axis=2, keepdims=True 134 | ) 135 | 136 | # Compute the dot product between the normalized vectors 137 | alignment = np.sum(normal_array_normalized * direction_to_camera_normalized, axis=2) 138 | 139 | # Ensure values are in range -1 to 1; clip them just in case due to floating-point errors 140 | alignment = np.clip(alignment, -1.0, 1.0) 141 | 142 | # and invert 143 | similar_angle_image = -1 * alignment 144 | 145 | similar_angle_image[np.isnan(similar_angle_image)] = 0 146 | 147 | # Return the final similarity image 148 | return similar_angle_image 149 | -------------------------------------------------------------------------------- /diffusedtexture/diffusers_utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from PIL import Image 4 | 5 | from ..utils import image_to_numpy 6 | 7 | 8 | def get_controlnet_config(scene): 9 | import torch 10 | from diffusers import ControlNetModel, ControlNetUnionModel 11 | 12 | if scene.controlnet_type == "UNION": 13 | # https://github.com/xinsir6/ControlNetPlus/blob/main/promax/controlnet_union_test_inpainting.py 14 | # 0 -- openpose 15 | # 1 -- depth 16 | # 2 -- hed/pidi/scribble/ted 17 | # 3 -- canny/lineart/anime_lineart/mlsd 18 | # 4 -- normal 19 | # 5 -- segment 20 | # 6 -- tile 21 | # 7 -- repaint 22 | 23 | controlnet_config = { 24 | "LOW": { 25 | "union_control": True, 26 | "control_mode": [1], 27 | "controlnets": ControlNetUnionModel.from_pretrained( 28 | scene.controlnet_union_path, torch_dtype=torch.float16 29 | ), 30 | "conditioning_scale": scene.union_controlnet_strength, 31 | "inputs": [ 32 | # None, 33 | "depth", 34 | # None, None, None, None, None, None 35 | ], 36 | }, 37 | "MEDIUM": { 38 | "union_control": True, 39 | "control_mode": [1, 3], 40 | "controlnets": ControlNetUnionModel.from_pretrained( 41 | scene.controlnet_union_path, torch_dtype=torch.float16 42 | ), 43 | "conditioning_scale": scene.union_controlnet_strength, 44 | "inputs": [ 45 | # None, 46 | "depth", 47 | # None, 48 | "canny", 49 | # None, 50 | # None, 51 | # None, 52 | # None 53 | ], 54 | }, 55 | "HIGH": { 56 | "union_control": True, 57 | "control_mode": [1, 3, 4], 58 | "controlnets": ControlNetUnionModel.from_pretrained( 59 | scene.controlnet_union_path, torch_dtype=torch.float16 60 | ), 61 | "conditioning_scale": scene.union_controlnet_strength, 62 | "inputs": [ 63 | # None, 64 | "depth", 65 | # None, 66 | "canny", 67 | "normal", 68 | # None, 69 | # None, 70 | # None, 71 | ], 72 | }, 73 | } 74 | 75 | return controlnet_config 76 | 77 | # Create a dictionary to map model complexities to their corresponding controlnet weights and inputs 78 | controlnet_config = { 79 | "LOW": { 80 | "conditioning_scale": [scene.depth_controlnet_strength], # 0.8, 81 | "controlnets": [ 82 | ControlNetModel.from_pretrained( 83 | scene.depth_controlnet_path, torch_dtype=torch.float16 84 | ), 85 | ], 86 | "inputs": ["depth"], 87 | }, 88 | "MEDIUM": { 89 | "conditioning_scale": [ 90 | scene.depth_controlnet_strength, 91 | scene.canny_controlnet_strength, 92 | ], # [0.7, 0.8], 93 | "controlnets": [ 94 | ControlNetModel.from_pretrained( 95 | scene.depth_controlnet_path, torch_dtype=torch.float16 96 | ), 97 | ControlNetModel.from_pretrained( 98 | scene.canny_controlnet_path, torch_dtype=torch.float16 99 | ), 100 | ], 101 | "inputs": ["depth", "canny"], 102 | }, 103 | "HIGH": { 104 | "conditioning_scale": [ 105 | scene.depth_controlnet_strength, 106 | scene.canny_controlnet_strength, 107 | scene.normal_controlnet_strength, 108 | ], # [1.0, 0.9, 0.9], 109 | "controlnets": [ 110 | ControlNetModel.from_pretrained( 111 | scene.depth_controlnet_path, torch_dtype=torch.float16 112 | ), 113 | ControlNetModel.from_pretrained( 114 | scene.canny_controlnet_path, torch_dtype=torch.float16 115 | ), 116 | ControlNetModel.from_pretrained( 117 | scene.normal_controlnet_path, torch_dtype=torch.float16 118 | ), 119 | ], 120 | "inputs": ["depth", "canny", "normal"], 121 | }, 122 | } 123 | return controlnet_config 124 | 125 | 126 | def create_pipeline(scene): 127 | 128 | # re-import if hf_home was re-set 129 | import torch 130 | from diffusers import ( 131 | StableDiffusionControlNetInpaintPipeline, 132 | StableDiffusionXLControlNetInpaintPipeline, 133 | StableDiffusionXLControlNetUnionInpaintPipeline, 134 | ) 135 | 136 | controlnet_config = get_controlnet_config(scene) 137 | 138 | if scene.sd_version == "sd15": 139 | 140 | # Load the model from a checkpoint if provided as safe tensor or checkpoint 141 | if str(scene.checkpoint_path).endswith(".safetensors"): 142 | pipe = StableDiffusionControlNetInpaintPipeline.from_single_file( 143 | scene.checkpoint_path, 144 | use_safetensors=True, 145 | torch_dtype=torch.float16, 146 | variant="fp16", 147 | safety_checker=None, 148 | ) 149 | 150 | # check if the checkpoint_path ends with .ckpt, .pt, .pth, .bin or any other extension, then load the model from the checkpoint 151 | elif str(scene.checkpoint_path).endswith((".ckpt", ".pt", ".pth", ".bin")): 152 | try: 153 | pipe = StableDiffusionControlNetInpaintPipeline.from_single_file( 154 | scene.checkpoint_path, 155 | torch_dtype=torch.float16, 156 | variant="fp16", 157 | safety_checker=None, 158 | ) 159 | except: 160 | # untested so raise verbose error 161 | raise ValueError( 162 | "Invalid checkpoint path provided. Please provide a valid checkpoint path" 163 | ) 164 | 165 | else: 166 | pipe = StableDiffusionControlNetInpaintPipeline.from_pretrained( 167 | scene.checkpoint_path, 168 | controlnet=controlnet_config[scene.mesh_complexity]["controlnets"], 169 | torch_dtype=torch.float16, 170 | use_safetensors=True, 171 | safety_checker=None, 172 | ) 173 | 174 | elif scene.sd_version == "sdxl": 175 | 176 | if str(scene.checkpoint_path).endswith(".safetensors"): 177 | 178 | if scene.controlnet_type == "UNION": 179 | pipe = StableDiffusionXLControlNetUnionInpaintPipeline.from_single_file( 180 | scene.checkpoint_path, 181 | use_safetensors=True, 182 | torch_dtype=torch.float16, 183 | variant="fp16", 184 | safety_checker=None, 185 | ) 186 | 187 | else: 188 | 189 | pipe = StableDiffusionXLControlNetInpaintPipeline.from_single_file( 190 | scene.checkpoint_path, 191 | use_safetensors=True, 192 | torch_dtype=torch.float16, 193 | variant="fp16", 194 | safety_checker=None, 195 | ) 196 | 197 | elif str(scene.checkpoint_path).endswith((".ckpt", ".pt", ".pth", ".bin")): 198 | try: 199 | if scene.controlnet_type == "UNION": 200 | pipe = StableDiffusionXLControlNetUnionInpaintPipeline.from_single_file( 201 | scene.checkpoint_path, 202 | torch_dtype=torch.float16, 203 | variant="fp16", 204 | safety_checker=None, 205 | ) 206 | else: 207 | pipe = StableDiffusionXLControlNetInpaintPipeline.from_single_file( 208 | scene.checkpoint_path, 209 | torch_dtype=torch.float16, 210 | variant="fp16", 211 | safety_checker=None, 212 | ) 213 | except: 214 | # untested so raise verbose error 215 | raise ValueError( 216 | "Invalid checkpoint path provided. Please provide a valid checkpoint path" 217 | ) 218 | 219 | else: 220 | if scene.controlnet_type == "UNION": 221 | pipe = StableDiffusionXLControlNetUnionInpaintPipeline.from_pretrained( 222 | scene.checkpoint_path, 223 | controlnet=controlnet_config[scene.mesh_complexity]["controlnets"], 224 | torch_dtype=torch.float16, 225 | use_safetensors=True, 226 | safety_checker=None, 227 | ) 228 | else: 229 | pipe = StableDiffusionXLControlNetInpaintPipeline.from_pretrained( 230 | scene.checkpoint_path, 231 | controlnet=controlnet_config[scene.mesh_complexity]["controlnets"], 232 | torch_dtype=torch.float16, 233 | use_safetensors=True, 234 | safety_checker=None, 235 | ) 236 | else: 237 | raise ValueError("Invalid SD Version, can only be 'sd15' or 'sdxl'") 238 | 239 | if scene.num_loras > 0: 240 | 241 | for lora in scene.lora_models: 242 | 243 | # Extract the directory (everything but the file name) 244 | file_path = os.path.dirname(lora.path) 245 | 246 | # Extract the file name (just the file name, including extension) 247 | file_name = os.path.basename(lora.path) 248 | 249 | # Load the LoRA weights using the extracted path and file name 250 | pipe.load_lora_weights(file_path, weight_name=file_name) 251 | 252 | pipe.fuse_lora(lora_scale=lora.strength) 253 | 254 | if scene.use_ipadapter: 255 | if scene.sd_version == "sd15": 256 | pipe.load_ip_adapter( 257 | "h94/IP-Adapter", subfolder="models", weight_name="ip-adapter_sd15.bin" 258 | ) 259 | elif scene.sd_version == "sdxl": 260 | pipe.load_ip_adapter( 261 | "h94/IP-Adapter", 262 | subfolder="sdxl_models", 263 | weight_name="ip-adapter_sdxl.bin", 264 | ) 265 | else: 266 | raise ValueError("Invalid SD Version, can only be 'sd15' or 'sdxl'") 267 | 268 | pipe.set_ip_adapter_scale(scene.ipadapter_strength) 269 | 270 | pipe.to("cuda") 271 | pipe.enable_model_cpu_offload() 272 | 273 | return pipe 274 | 275 | 276 | def infer_pipeline( 277 | pipe, 278 | scene, 279 | input_image, 280 | uv_mask, 281 | canny_img, 282 | normal_img, 283 | depth_img, 284 | strength=1.0, 285 | guidance_scale=7.5, 286 | num_inference_steps=None, 287 | denoising_start=None, 288 | denoising_end=None, 289 | output_type="pil", 290 | ): 291 | 292 | controlnet_config = get_controlnet_config(scene) 293 | 294 | # run the pipeline 295 | control_images = [] 296 | 297 | for entry in controlnet_config[scene.mesh_complexity]["inputs"]: 298 | if entry == "depth": 299 | control_images.append(Image.fromarray(depth_img)) 300 | elif entry == "canny": 301 | control_images.append(Image.fromarray(canny_img)) 302 | elif entry == "normal": 303 | control_images.append(Image.fromarray(normal_img)) 304 | # elif entry is None: 305 | # control_images.append(0) 306 | 307 | ip_adapter_image = None 308 | if scene.use_ipadapter: 309 | ip_adapter_image = image_to_numpy(scene.ipadapter_image) 310 | 311 | if num_inference_steps is None: 312 | num_inference_steps = scene.num_inference_steps 313 | 314 | if scene.controlnet_type == "UNION": 315 | output = pipe( 316 | prompt=scene.my_prompt, 317 | negative_prompt=scene.my_negative_prompt, 318 | image=input_image, 319 | mask_image=Image.fromarray(uv_mask), 320 | control_image=control_images, 321 | control_mode=controlnet_config[scene.mesh_complexity]["control_mode"], 322 | ip_adapter_image=ip_adapter_image, 323 | num_images_per_prompt=1, 324 | controlnet_conditioning_scale=controlnet_config[scene.mesh_complexity][ 325 | "conditioning_scale" 326 | ], 327 | num_inference_steps=num_inference_steps, 328 | denoising_start=denoising_start, 329 | denoising_end=denoising_end, 330 | strength=strength, 331 | guidance_scale=guidance_scale, 332 | output_type=output_type, 333 | ).images 334 | 335 | else: 336 | output = pipe( 337 | prompt=scene.my_prompt, 338 | negative_prompt=scene.my_negative_prompt, 339 | image=input_image, 340 | mask_image=Image.fromarray(uv_mask), 341 | control_image=control_images, 342 | ip_adapter_image=ip_adapter_image, 343 | num_images_per_prompt=1, 344 | controlnet_conditioning_scale=controlnet_config[scene.mesh_complexity][ 345 | "conditioning_scale" 346 | ], 347 | num_inference_steps=num_inference_steps, 348 | denoising_start=denoising_start, 349 | denoising_end=denoising_end, 350 | strength=strength, 351 | guidance_scale=guidance_scale, 352 | output_type=output_type, 353 | ).images 354 | 355 | return output 356 | -------------------------------------------------------------------------------- /diffusedtexture/img_parallel.py: -------------------------------------------------------------------------------- 1 | import math 2 | import numpy as np 3 | from PIL import Image 4 | 5 | from .diffusers_utils import ( 6 | create_pipeline, 7 | infer_pipeline, 8 | ) 9 | from .process_operations import ( 10 | process_uv_texture, 11 | generate_multiple_views, 12 | assemble_multiview_grid, 13 | create_input_image_grid, 14 | delete_render_folders, 15 | ) 16 | 17 | 18 | def img_parallel(scene, max_size, texture=None): 19 | """Run the first pass for texture generation.""" 20 | 21 | multiview_images, render_img_folders = generate_multiple_views( 22 | scene=scene, 23 | max_size=max_size, 24 | suffix="img_parallel", 25 | render_resolution=int(scene.render_resolution), 26 | ) 27 | 28 | # sd_resolution = 512 if scene.sd_version == "sd15" else 1024 29 | 30 | if scene.custom_sd_resolution: 31 | sd_resolution = scene.custom_sd_resolution 32 | else: 33 | sd_resolution = 512 if scene.sd_version == "sd15" else 1024 34 | 35 | _, resized_multiview_grids = assemble_multiview_grid( 36 | multiview_images, 37 | render_resolution=int(scene.render_resolution), 38 | sd_resolution=sd_resolution, 39 | ) 40 | 41 | if texture is not None: 42 | # Flip texture vertically (blender y 0 is down, opencv y 0 is up) 43 | texture = texture[::-1] 44 | 45 | input_image_sd = create_input_image_grid( 46 | texture, 47 | resized_multiview_grids["uv_grid"], 48 | resized_multiview_grids["uv_grid"], 49 | ) 50 | else: 51 | input_image_sd = ( 52 | 255 * np.ones_like(resized_multiview_grids["canny_grid"]) 53 | ).astype(np.uint8) 54 | 55 | pipe = create_pipeline(scene) 56 | output_grid = infer_pipeline( 57 | pipe, 58 | scene, 59 | Image.fromarray(input_image_sd), 60 | resized_multiview_grids["content_mask"], 61 | resized_multiview_grids["canny_grid"], 62 | resized_multiview_grids["normal_grid"], 63 | resized_multiview_grids["depth_grid"], 64 | strength=scene.denoise_strength, 65 | guidance_scale=scene.guidance_scale, 66 | )[0] 67 | 68 | output_grid = np.array(output_grid) 69 | 70 | filled_uv_texture = process_uv_texture( 71 | scene=scene, 72 | uv_images=multiview_images["uv"], 73 | facing_images=multiview_images["facing"], 74 | output_grid=output_grid, 75 | target_resolution=int(scene.texture_resolution), 76 | render_resolution=int(scene.render_resolution), 77 | facing_percentile=0.5, 78 | ) 79 | 80 | # delete all rendering folders 81 | delete_render_folders(render_img_folders) 82 | 83 | return filled_uv_texture 84 | -------------------------------------------------------------------------------- /diffusedtexture/img_parasequential.py: -------------------------------------------------------------------------------- 1 | # TODO: Combine parallel and sequential to create a sequential approach that works on a subset of the total number of cameras 2 | -------------------------------------------------------------------------------- /diffusedtexture/img_sequential.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | from PIL import Image 4 | 5 | import cv2 6 | from .diffusers_utils import create_pipeline, infer_pipeline 7 | from .process_operations import ( 8 | generate_multiple_views, 9 | create_input_image, 10 | create_content_mask, 11 | delete_render_folders, 12 | ) 13 | 14 | 15 | # Helper Functions 16 | def prepare_texture(texture, texture_resolution): 17 | """Prepare the texture by flipping and scaling.""" 18 | if texture is None: 19 | return ( 20 | np.ones((texture_resolution, texture_resolution, 3), dtype=np.uint8) * 255 21 | ) 22 | texture = texture[::-1] 23 | if texture.shape[0] != texture_resolution: 24 | texture = cv2.resize( 25 | texture[..., :3], 26 | (texture_resolution, texture_resolution), 27 | interpolation=cv2.INTER_LANCZOS4, 28 | ) 29 | return np.copy(texture)[..., :3] 30 | 31 | 32 | def prepare_uv_coordinates(uv_image, texture_resolution, target_resolution=None): 33 | """Prepare UV coordinates scaled to the texture resolution.""" 34 | 35 | if target_resolution is not None: 36 | uv_image = cv2.resize( 37 | uv_image, 38 | (target_resolution, target_resolution), 39 | interpolation=cv2.INTER_NEAREST, 40 | ) 41 | 42 | uv_image = uv_image[..., :2].reshape(-1, 2) 43 | 44 | uv_coords = np.round(uv_image * (texture_resolution - 1)).astype(np.int32) 45 | uv_coords[:, 1] = texture_resolution - 1 - uv_coords[:, 1] # Flip v-axis 46 | uv_coords %= texture_resolution 47 | return uv_coords 48 | 49 | 50 | def update_content_mask( 51 | content_mask_texture, pixel_status, facing_texture, max_angle_status 52 | ): 53 | """Update the content mask based on pixel status and facing angles.""" 54 | content_mask_texture[pixel_status == 2] = 0 # Exclude fixed pixels 55 | content_mask_texture[(pixel_status == 1) & (max_angle_status > facing_texture)] = 0 56 | return content_mask_texture 57 | 58 | 59 | def generate_inputs_for_inference( 60 | input_render_resolution, 61 | content_mask_texture, 62 | multiview_images, 63 | uv_coordinates_sd, 64 | scene, 65 | i, 66 | ): 67 | """Generate inputs for Stable Diffusion inference.""" 68 | 69 | # sd_resolution = 512 if scene.sd_version == "sd15" else 1024 70 | 71 | if scene.custom_sd_resolution: 72 | sd_resolution = scene.custom_sd_resolution 73 | else: 74 | sd_resolution = 512 if scene.sd_version == "sd15" else 1024 75 | 76 | input_image_sd = cv2.resize( 77 | input_render_resolution, 78 | (sd_resolution, sd_resolution), 79 | interpolation=cv2.INTER_LINEAR, 80 | ) 81 | canny_img = cv2.Canny( 82 | cv2.resize( 83 | multiview_images["normal"][i].astype(np.uint8), 84 | (sd_resolution, sd_resolution), 85 | ), 86 | 100, 87 | 200, 88 | ) 89 | canny_img = np.stack([canny_img] * 3, axis=-1) 90 | normal_img = cv2.resize( 91 | multiview_images["normal"][i], 92 | (sd_resolution, sd_resolution), 93 | interpolation=cv2.INTER_LINEAR, 94 | ) 95 | depth_img = cv2.resize( 96 | multiview_images["depth"][i], 97 | (sd_resolution, sd_resolution), 98 | interpolation=cv2.INTER_LINEAR, 99 | ) 100 | 101 | content_mask_sd = content_mask_texture_to_render_sd( 102 | content_mask_texture, uv_coordinates_sd, scene, sd_resolution 103 | ) 104 | 105 | return input_image_sd, content_mask_sd, canny_img, normal_img, depth_img 106 | 107 | 108 | def save_debug_images( 109 | scene, 110 | i, 111 | input_image_sd, 112 | content_mask_render_sd, 113 | content_mask_texture, 114 | canny_img, 115 | normal_img, 116 | depth_img, 117 | output, 118 | ): 119 | """Save intermediate debugging images.""" 120 | output_path = scene.output_path 121 | cv2.imwrite( 122 | f"{output_path}/input_image_sd_{i}.png", 123 | cv2.cvtColor(input_image_sd, cv2.COLOR_RGB2BGR), 124 | ) 125 | cv2.imwrite(f"{output_path}/content_mask_render_sd_{i}.png", content_mask_render_sd) 126 | cv2.imwrite(f"{output_path}/content_mask_texture_{i}.png", content_mask_texture) 127 | cv2.imwrite( 128 | f"{output_path}/canny_{i}.png", cv2.cvtColor(canny_img, cv2.COLOR_RGB2BGR) 129 | ) 130 | cv2.imwrite( 131 | f"{output_path}/normal_{i}.png", cv2.cvtColor(normal_img, cv2.COLOR_RGB2BGR) 132 | ) 133 | cv2.imwrite( 134 | f"{output_path}/depth_{i}.png", cv2.cvtColor(depth_img, cv2.COLOR_RGB2BGR) 135 | ) 136 | cv2.imwrite( 137 | f"{output_path}/output_{i}.png", cv2.cvtColor(output, cv2.COLOR_RGB2BGR) 138 | ) 139 | 140 | 141 | def blend_output( 142 | input_render_resolution, output, content_mask_render, render_resolution 143 | ): 144 | """Blend output with input using feathered overlay.""" 145 | overlay_mask = cv2.resize( 146 | content_mask_render, 147 | (render_resolution, render_resolution), 148 | interpolation=cv2.INTER_LINEAR, 149 | ) 150 | # overlay_mask = cv2.erode(overlay_mask, np.ones((3, 3)), iterations=2) 151 | # overlay_mask = cv2.blur(overlay_mask, (9, 9)) 152 | overlay_mask = cv2.blur(overlay_mask, (3, 3)) 153 | overlay_mask = np.stack([overlay_mask] * 3, axis=-1).astype(np.float32) / 255 154 | 155 | # blended = (1 - overlay_mask) * input_render_resolution + overlay_mask * output 156 | blended = (1 - overlay_mask) * input_render_resolution + overlay_mask * output 157 | 158 | return np.clip(blended, 0, 255).astype(np.uint8) 159 | 160 | 161 | def content_mask_texture_to_render_sd( 162 | content_mask_texture, uv_coordinates_sd, scene, target_resolution 163 | ): 164 | """Convert content mask from texture resolution to SD resolution.""" 165 | 166 | content_mask_sd = content_mask_texture[ 167 | uv_coordinates_sd[:, 1], uv_coordinates_sd[:, 0] 168 | ] 169 | 170 | input_resolution = np.sqrt(len(uv_coordinates_sd)).astype(np.int32) 171 | 172 | content_mask_sd = cv2.resize( 173 | content_mask_sd.reshape(input_resolution, input_resolution), 174 | (target_resolution, target_resolution), 175 | interpolation=cv2.INTER_NEAREST, 176 | ) 177 | 178 | content_mask_sd = content_mask_sd.reshape( 179 | target_resolution, target_resolution 180 | ).astype(np.uint8) 181 | 182 | return content_mask_sd 183 | 184 | 185 | # Main Function 186 | def img_sequential(scene, max_size, texture): 187 | """Run the third pass for sequential texture refinement.""" 188 | texture_resolution = int(scene.texture_resolution) 189 | render_resolution = int(scene.render_resolution) 190 | num_cameras = int(scene.num_cameras) 191 | 192 | final_texture = prepare_texture(texture, texture_resolution) 193 | multiview_images, render_img_folders = generate_multiple_views( 194 | scene, max_size, suffix="img_sequential", render_resolution=render_resolution 195 | ) 196 | 197 | pixel_status = np.zeros((texture_resolution, texture_resolution)) 198 | max_angle_status = np.zeros((texture_resolution, texture_resolution)) 199 | 200 | pipe = create_pipeline(scene) 201 | 202 | for i in range(num_cameras): 203 | _, input_render_resolution, _ = create_input_image( 204 | final_texture, 205 | multiview_images["uv"][i], 206 | render_resolution, 207 | texture_resolution, 208 | ) 209 | 210 | content_mask_fullsize = create_content_mask(multiview_images["uv"][i]) 211 | uv_image = multiview_images["uv"][i][..., :2] 212 | uv_image[content_mask_fullsize == 0] = 0 213 | 214 | uv_coordinates_tex = prepare_uv_coordinates( 215 | uv_image, texture_resolution, texture_resolution 216 | ) 217 | facing_render = cv2.resize( 218 | multiview_images["facing"][i], 219 | (texture_resolution, texture_resolution), 220 | interpolation=cv2.INTER_LINEAR, 221 | ).flatten() 222 | 223 | facing_texture = np.zeros((texture_resolution, texture_resolution)) 224 | 225 | facing_texture[uv_coordinates_tex[:, 1], uv_coordinates_tex[:, 0]] = ( 226 | facing_render 227 | ) 228 | 229 | content_mask_tex_size = cv2.resize( 230 | content_mask_fullsize, 231 | (texture_resolution, texture_resolution), 232 | interpolation=cv2.INTER_NEAREST, 233 | ) 234 | 235 | content_mask_texture = np.zeros((texture_resolution, texture_resolution)) 236 | content_mask_texture[uv_coordinates_tex[:, 1], uv_coordinates_tex[:, 0]] = ( 237 | content_mask_tex_size.reshape(-1) 238 | ) 239 | 240 | content_mask_texture = cv2.resize( 241 | content_mask_texture, 242 | (texture_resolution, texture_resolution), 243 | interpolation=cv2.INTER_NEAREST, 244 | ) 245 | 246 | content_mask_texture = update_content_mask( 247 | content_mask_texture, pixel_status, facing_texture, max_angle_status 248 | ) 249 | 250 | max_angle_status[content_mask_texture > 0] = facing_texture[ 251 | content_mask_texture > 0 252 | ] 253 | pixel_status[content_mask_texture > 0] = np.clip( 254 | pixel_status[content_mask_texture > 0] + 1, 0, 2 255 | ) 256 | 257 | uv_coordinates_sd = prepare_uv_coordinates( 258 | uv_image, texture_resolution, 512 if scene.sd_version == "sd15" else 1024 259 | ) 260 | 261 | input_image_sd, content_mask_render_sd, canny_img, normal_img, depth_img = ( 262 | generate_inputs_for_inference( 263 | input_render_resolution, 264 | content_mask_texture, 265 | multiview_images, 266 | uv_coordinates_sd, 267 | scene, 268 | i, 269 | ) 270 | ) 271 | 272 | output = infer_pipeline( 273 | pipe, 274 | scene, 275 | Image.fromarray(input_image_sd), 276 | content_mask_render_sd, 277 | canny_img, 278 | normal_img, 279 | depth_img, 280 | strength=scene.denoise_strength, 281 | guidance_scale=scene.guidance_scale, 282 | )[0] 283 | output = cv2.resize( 284 | np.array(output)[..., :3], 285 | (render_resolution, render_resolution), 286 | interpolation=cv2.INTER_LINEAR, 287 | ) 288 | 289 | save_debug_images( 290 | scene, 291 | i, 292 | input_image_sd, 293 | content_mask_render_sd, 294 | content_mask_texture, 295 | canny_img, 296 | normal_img, 297 | depth_img, 298 | output, 299 | ) 300 | 301 | uv_coordinates_render = prepare_uv_coordinates( 302 | uv_image, texture_resolution, render_resolution 303 | ) 304 | content_mask_render = content_mask_texture_to_render_sd( 305 | content_mask_texture, uv_coordinates_render, scene, render_resolution 306 | ) 307 | 308 | # TODO: Implement feathered overlay 309 | 310 | if i > 0: 311 | output = blend_output( 312 | input_render_resolution, output, content_mask_render, render_resolution 313 | ) 314 | 315 | uv_coordinates_render = prepare_uv_coordinates(uv_image, texture_resolution) 316 | final_texture[uv_coordinates_render[:, 1], uv_coordinates_render[:, 0], :] = ( 317 | output.reshape(-1, 3) 318 | ) 319 | 320 | delete_render_folders(render_img_folders) 321 | return final_texture 322 | -------------------------------------------------------------------------------- /diffusedtexture/latent_parallel.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module contains the latent_parallel function, which is used to generate a texture from a scene using the latent parallel method. 3 | The latent parallel method uses a pipeline to denoise the input image at multiple stages, mixing latents inbetween each stage. 4 | The final output is a texture that is generated from the mixed latents. The texture is then returned as a numpy array. 5 | 6 | It does not work with the current version of the Diffuser's Library, as pipelines with denoising_start and denoising_end parameters 7 | do not support ControlNets yet. I will need to update the pipeline to support ControlNets in order to use this method. 8 | """ 9 | 10 | import numpy as np 11 | 12 | from .diffusers_utils import ( 13 | create_pipeline, 14 | infer_pipeline, 15 | ) 16 | from .process_operations import ( 17 | latent_mixing_parallel, 18 | process_uv_texture, 19 | generate_multiple_views, 20 | assemble_multiview_grid, 21 | create_input_image_grid, 22 | delete_render_folders, 23 | ) 24 | 25 | 26 | def latent_parallel(scene, max_size, texture=None): 27 | multiview_images, render_img_folders = generate_multiple_views( 28 | scene=scene, 29 | max_size=max_size, 30 | suffix="img_parallel", 31 | render_resolution=int(scene.render_resolution), 32 | ) 33 | 34 | if scene.custom_sd_resolution: 35 | sd_resolution = scene.custom_sd_resolution 36 | else: 37 | sd_resolution = 512 if scene.sd_version == "sd15" else 1024 38 | 39 | latent_resolution = sd_resolution // 16 40 | 41 | _, resized_multiview_grids = assemble_multiview_grid( 42 | multiview_images, 43 | render_resolution=int(scene.render_resolution), 44 | sd_resolution=sd_resolution, 45 | latent_resolution=latent_resolution, 46 | ) 47 | 48 | if texture is not None: 49 | # Flip texture vertically (blender y 0 is down, opencv y 0 is up) 50 | texture = texture[::-1] 51 | 52 | input_image_sd = create_input_image_grid( 53 | texture, multiview_images["uv_grid"], resized_multiview_grids["uv_grid"] 54 | ) 55 | else: 56 | input_image_sd = ( 57 | 255 * np.ones_like(resized_multiview_grids["canny_grid"]) 58 | ).astype(np.uint8) 59 | 60 | latents = None 61 | pipe = create_pipeline(scene) 62 | 63 | # add latent mixing inbetween at multiple times throughout the denoising process 64 | for denoising_start, denoising_end in [ 65 | (0.0, 0.25), 66 | (0.25, 0.5), 67 | (0.5, 0.75), 68 | (0.75, 1.0), 69 | ]: 70 | 71 | if latents is None: 72 | latents = infer_pipeline( 73 | pipe, 74 | scene, 75 | input_image=input_image_sd, 76 | uv_mask=resized_multiview_grids["content_mask"], 77 | canny_img=resized_multiview_grids["canny_grid"], 78 | normal_img=resized_multiview_grids["normal_grid"], 79 | depth_img=resized_multiview_grids["depth_grid"], 80 | strength=scene.denoise_strength, 81 | num_inference_steps=int(0.25 * scene.num_inference_steps), 82 | guidance_scale=scene.guidance_scale, 83 | denoising_start=denoising_start, 84 | denoising_end=denoising_end, 85 | output_type="latent", 86 | ) 87 | 88 | else: 89 | 90 | # Mix latents via normal face direction 91 | latents = latent_mixing_parallel( 92 | scene=scene, 93 | uv_list=multiview_images["uv"], 94 | facing_list=multiview_images["facing"], 95 | latents=latents, 96 | ) 97 | 98 | if denoising_end == 1.0: 99 | output_grid = infer_pipeline( 100 | pipe, 101 | scene, 102 | input_image=latents, 103 | uv_mask=resized_multiview_grids["content_mask"], 104 | canny_img=resized_multiview_grids["canny_grid"], 105 | normal_img=resized_multiview_grids["normal_grid"], 106 | depth_img=resized_multiview_grids["depth_grid"], 107 | strength=scene.denoise_strength, 108 | num_inference_steps=int(0.25 * scene.num_inference_steps), 109 | guidance_scale=scene.guidance_scale, 110 | denoising_start=denoising_start, 111 | denoising_end=denoising_end, 112 | output_type="PIL", 113 | )[0] 114 | output_grid = np.array(output_grid) 115 | 116 | else: 117 | latents = infer_pipeline( 118 | pipe, 119 | scene, 120 | input_image=latents, 121 | uv_mask=resized_multiview_grids["content_mask"], 122 | canny_img=resized_multiview_grids["canny_grid"], 123 | normal_img=resized_multiview_grids["normal_grid"], 124 | depth_img=resized_multiview_grids["depth_grid"], 125 | strength=scene.denoise_strength, 126 | num_inference_steps=int(0.25 * scene.num_inference_steps), 127 | guidance_scale=scene.guidance_scale, 128 | denoising_start=denoising_start, 129 | denoising_end=denoising_end, 130 | output_type="latent", 131 | ) 132 | 133 | # Save output_grid as image 134 | filled_uv_texture = process_uv_texture( 135 | scene=scene, 136 | uv_images=multiview_images["uv"], 137 | facing_images=multiview_images["facing"], 138 | output_grid=output_grid, 139 | target_resolution=int(scene.texture_resolution), 140 | render_resolution=int(scene.render_resolution), 141 | facing_percentile=0.5, 142 | ) 143 | 144 | # delete all rendering folders 145 | delete_render_folders(render_img_folders) 146 | 147 | return filled_uv_texture 148 | -------------------------------------------------------------------------------- /diffusedtexture/process_operations.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shutil 3 | import math 4 | 5 | import bpy 6 | import mathutils 7 | import cv2 8 | import numpy as np 9 | import torch 10 | from scipy.spatial.transform import Rotation 11 | from pathlib import Path 12 | 13 | from ..condition_setup import ( 14 | bpy_img_to_numpy, 15 | create_depth_condition, 16 | create_normal_condition, 17 | create_similar_angle_image, 18 | ) 19 | 20 | from ..render_setup import ( 21 | setup_render_settings, 22 | create_cameras_on_sphere, 23 | create_cameras_on_one_ring, 24 | ) 25 | 26 | 27 | def delete_render_folders(render_img_folders): 28 | """ 29 | Deletes all rendering folders and their contents if they exist. 30 | 31 | :param render_img_folders: List of folder paths to delete. 32 | """ 33 | for render_folder in render_img_folders: 34 | # Check if the folder exists 35 | if os.path.exists(render_folder) and os.path.isdir(render_folder): 36 | # Delete the folder and all its contents 37 | shutil.rmtree(render_folder) 38 | print(f"Deleted folder: {render_folder}") 39 | else: 40 | print(f"Folder not found or not a directory: {render_folder}") 41 | 42 | 43 | def latent_mixing_parallel( 44 | scene, uv_list, facing_list, latents, mixing_mode="weighted" 45 | ): 46 | """ 47 | Perform latent mixing using the provided latents and return the mixed latents. 48 | 49 | Args: 50 | scene (bpy.types.Scene): The Blender scene object. 51 | uv_list (list): List of UV images corresponding to different camera views. 52 | facing_list (list): List of facing images corresponding to different camera views. 53 | latents (numpy.ndarray): Numpy array of all latents in one image corresponding to different camera views. 54 | """ 55 | 56 | # remove the batch dimension 57 | latents = latents.squeeze(0) 58 | 59 | # support "mean", "weighted" and "max" mixing 60 | if mixing_mode not in ["mean", "weighted", "max"]: 61 | raise ValueError("Invalid mixing mode, can only be 'mean', 'weighted' or 'max'") 62 | 63 | # save the device of the latents 64 | device = latents.device 65 | 66 | # move the latents to the cpu 67 | latents = latents.cpu().numpy() 68 | 69 | # latents shape is torch.Size([4, 64, 64]) for sd15 and torch.Size([4, 128, 128]) for sdxl 70 | 71 | # get the target resolution of a latent image 72 | latent_resolution = ( 73 | 64 if scene.sd_version == "sd15" else 128 74 | ) # We cant use the size of the latents since they are multiple ones together 75 | 76 | # change the latents from torch.Size([4, 64, 64]) to (64, 64, 4) 77 | latents = np.moveaxis(latents, 0, -1) 78 | 79 | # split the latents into the different views 80 | num_cameras = len(uv_list) 81 | latent_images = [] 82 | for cam_index in range(num_cameras): 83 | # Calculate the position in the grid 84 | row = int((cam_index // int(math.sqrt(num_cameras))) * latent_resolution) 85 | col = int((cam_index % int(math.sqrt(num_cameras))) * latent_resolution) 86 | 87 | output_chunk = latents[ 88 | row : row + latent_resolution, col : col + latent_resolution 89 | ] 90 | 91 | latent_images.append(output_chunk) 92 | 93 | # create a num_camera*latent_res*latent_res*-1 mixed latents array (one channel for each grid img) 94 | mixed_latents = np.zeros( 95 | ( 96 | num_cameras, 97 | latent_resolution, 98 | latent_resolution, 99 | 4, 100 | ), 101 | dtype=np.float32, 102 | ) 103 | 104 | if mixing_mode == "mean": 105 | weights_latents = np.ones( 106 | (num_cameras, latent_resolution, latent_resolution), dtype=np.float32 107 | ) 108 | weights_latents = weights_latents / num_cameras # all weights are equal 109 | 110 | else: 111 | # create a num_camera*latent_res*latent_res mixed latents array (one for each grid img) 112 | weights_latents = np.zeros( 113 | ( 114 | num_cameras, 115 | latent_resolution, 116 | latent_resolution, 117 | ), 118 | dtype=np.float32, 119 | ) 120 | 121 | for cam_index in range(num_cameras): 122 | # Calculate the position in the grid 123 | row = int((cam_index // int(math.sqrt(num_cameras))) * latent_resolution) 124 | col = int((cam_index % int(math.sqrt(num_cameras))) * latent_resolution) 125 | 126 | # load the uv image 127 | uv_image = uv_list[cam_index] 128 | uv_image = uv_image[..., :2] # Keep only u and v 129 | 130 | # resize the uv_image to the latent_resolution 131 | uv_image = cv2.resize( 132 | uv_image, 133 | (latent_resolution, latent_resolution), 134 | interpolation=cv2.INTER_NEAREST, 135 | ) 136 | 137 | content_mask = np.zeros((latent_resolution, latent_resolution)) 138 | content_mask[np.sum(uv_image, axis=-1) > 0] = 255 139 | content_mask = content_mask.astype(np.uint8) 140 | 141 | uv_image[content_mask == 0] = 0 142 | 143 | # resize the uv values to 0-511 144 | uv_coordinates = ( 145 | (uv_image * int(latent_resolution - 1)).astype(np.uint16).reshape(-1, 2) 146 | ) 147 | 148 | # the uvs are meant to start from the bottom left corner, so we flip the y axis (v axis) 149 | uv_coordinates[:, 1] = int(latent_resolution - 1) - uv_coordinates[:, 1] 150 | 151 | # in case we have uv coordinates beyond the texture 152 | uv_coordinates = uv_coordinates % int(latent_resolution) 153 | 154 | mixed_latents[cam_index, uv_coordinates[:, 1], uv_coordinates[:, 0], ...] = ( 155 | latent_images[cam_index].reshape(-1, 4) 156 | ) 157 | 158 | # adjust the facing weight to the chosen percentile 159 | cur_facing_image = facing_list[cam_index] 160 | 161 | # resize the facing image to the latent_resolution 162 | cur_facing_image = cv2.resize( 163 | cur_facing_image, 164 | (latent_resolution, latent_resolution), 165 | interpolation=cv2.INTER_LINEAR, 166 | ) 167 | 168 | if mixing_mode != "mean": 169 | weights_latents[ 170 | cam_index, uv_coordinates[:, 1], uv_coordinates[:, 0], ... 171 | ] = cur_facing_image.reshape( 172 | -1, 173 | ) 174 | 175 | if mixing_mode == "max": 176 | max_latents = np.argmax(weights_latents, axis=0) 177 | 178 | weighted_latents_texture = mixed_latents[ 179 | max_latents, np.arange(latent_resolution), np.arange(latent_resolution) 180 | ] 181 | 182 | # reshape the weighted latents to the original shape 183 | weighted_latents_texture = weighted_latents_texture.reshape( 184 | latent_resolution, latent_resolution, -1 185 | ) 186 | 187 | else: 188 | # multiply the texture channels by the point at factor 189 | weighted_latents_texture = ( 190 | np.stack( 191 | ( 192 | weights_latents, 193 | weights_latents, 194 | weights_latents, 195 | weights_latents, 196 | ), 197 | axis=-1, 198 | ) 199 | * mixed_latents 200 | ) 201 | 202 | weighted_latents_texture = np.sum(weighted_latents_texture, axis=0) / np.stack( 203 | ( 204 | np.sum(weights_latents, axis=0), 205 | np.sum(weights_latents, axis=0), 206 | np.sum(weights_latents, axis=0), 207 | np.sum(weights_latents, axis=0), 208 | ), 209 | axis=-1, 210 | ) 211 | 212 | # project the weighted_latents_texture to the original views and stitch them back together 213 | weighted_latents = np.copy(latents) 214 | 215 | for cam_index in range(num_cameras): 216 | # Calculate the position in the grid 217 | row = int((cam_index // int(math.sqrt(num_cameras))) * latent_resolution) 218 | col = int((cam_index % int(math.sqrt(num_cameras))) * latent_resolution) 219 | 220 | # load the uv image 221 | uv_image = uv_list[cam_index] 222 | uv_image = uv_image[..., :2] # Keep only u and v 223 | 224 | # resize the uv_image to the latent_resolution 225 | uv_image = cv2.resize( 226 | uv_image, 227 | (latent_resolution, latent_resolution), 228 | interpolation=cv2.INTER_NEAREST, 229 | ) 230 | 231 | content_mask = np.zeros((latent_resolution, latent_resolution)) 232 | content_mask[np.sum(uv_image, axis=-1) > 0] = 255 233 | content_mask = content_mask.astype(np.uint8) 234 | 235 | uv_image[content_mask == 0] = 0 236 | 237 | # resize the uv values to 0 .. latent_res-1 238 | uv_coordinates = ( 239 | (uv_image * int(latent_resolution - 1)).astype(np.uint16).reshape(-1, 2) 240 | ) 241 | 242 | # the uvs are meant to start from the bottom left corner, so we flip the y axis (v axis) 243 | uv_coordinates[:, 1] = int(latent_resolution - 1) - uv_coordinates[:, 1] 244 | 245 | # in case we have uv coordinates beyond the texture 246 | uv_coordinates = uv_coordinates % int(latent_resolution) 247 | 248 | # only replace the parts that are in the content mask 249 | cams_latent_image = latent_images[cam_index].copy() 250 | 251 | cams_weighted_latents_texture = weighted_latents_texture[ 252 | uv_coordinates[:, 1], uv_coordinates[:, 0], ... 253 | ].reshape(latent_resolution, latent_resolution, 4) 254 | 255 | cams_latent_image[content_mask == 255] = cams_weighted_latents_texture[ 256 | content_mask == 255 257 | ] 258 | 259 | weighted_latents[ 260 | row : row + latent_resolution, col : col + latent_resolution 261 | ] = cams_latent_image 262 | 263 | # change the latents from torch.Size([4, 64, 64]) to (64, 64, 4) 264 | weighted_latents = np.moveaxis(weighted_latents, -1, 0) 265 | 266 | # move the latents back to the device 267 | weighted_latents = torch.tensor(weighted_latents, device=device) 268 | 269 | # add the batch dimension back 270 | weighted_latents = weighted_latents.unsqueeze(0) 271 | 272 | return weighted_latents 273 | 274 | 275 | def process_uv_texture( 276 | scene, 277 | uv_images, 278 | facing_images, 279 | output_grid, 280 | target_resolution=512, 281 | render_resolution=2048, 282 | facing_percentile=1.0, 283 | ): 284 | """ 285 | Processes the UV texture from multiple images and applies the inpainting on missing pixels. 286 | 287 | :param scene: Blender scene object, used to get the output path. 288 | :param uv_images: List of UV images corresponding to different camera views. 289 | :param output_grid: Output grid image containing the combined render. 290 | :param target_resolution: Resolution for each UV grid image. 291 | :param render_resolution: Resolution of each render in the output_grid. 292 | :return: Inpainted UV texture. 293 | """ 294 | 295 | num_cameras = len(uv_images) 296 | 297 | # Convert the output grid to a NumPy array and save it for debugging 298 | output_grid = np.array(output_grid) 299 | 300 | if scene.custom_sd_resolution: 301 | sd_resolution = scene.custom_sd_resolution 302 | else: 303 | sd_resolution = 512 if scene.sd_version == "sd15" else 1024 304 | 305 | # Resize output_grid to render resolution 306 | output_grid = cv2.resize( 307 | output_grid, 308 | ( 309 | int( 310 | (output_grid.shape[0] * render_resolution / sd_resolution), 311 | ), 312 | int( 313 | (output_grid.shape[0] * render_resolution / sd_resolution), 314 | ), 315 | ), 316 | interpolation=cv2.INTER_LANCZOS4, 317 | ) 318 | 319 | resized_tiles = [] 320 | for cam_index in range(num_cameras): 321 | # Calculate the position in the grid 322 | row = int((cam_index // int(math.sqrt(num_cameras))) * render_resolution) 323 | col = int((cam_index % int(math.sqrt(num_cameras))) * render_resolution) 324 | 325 | output_chunk = output_grid[ 326 | row : row + render_resolution, col : col + render_resolution 327 | ] 328 | 329 | resized_tiles.append(output_chunk) 330 | 331 | # create a 16x512x512x3 uv map (one for each grid img) 332 | uv_texture_first_pass = np.zeros( 333 | (num_cameras, target_resolution, target_resolution, 3), dtype=np.float32 334 | ) 335 | 336 | # create a 16x512x512x3 uv map (one for each grid img) 337 | uv_texture_first_pass_weight = np.zeros( 338 | (num_cameras, target_resolution, target_resolution), dtype=np.float32 339 | ) 340 | 341 | for cam_index in range(num_cameras): 342 | # Calculate the position in the grid 343 | row = int((cam_index // int(math.sqrt(num_cameras))) * render_resolution) 344 | col = int((cam_index % int(math.sqrt(num_cameras))) * render_resolution) 345 | 346 | # load the uv image 347 | uv_image = uv_images[cam_index] 348 | uv_image = uv_image[..., :2] # Keep only u and v 349 | 350 | content_mask = np.zeros((render_resolution, render_resolution)) 351 | content_mask[np.sum(uv_image, axis=-1) > 0] = 255 352 | content_mask = content_mask.astype(np.uint8) 353 | 354 | uv_image[content_mask == 0] = 0 355 | 356 | # resize the uv values to 0-511 357 | uv_coordinates = ( 358 | (uv_image * int(target_resolution - 1)).astype(np.uint16).reshape(-1, 2) 359 | ) 360 | 361 | # the uvs are meant to start from the bottom left corner, so we flip the y axis (v axis) 362 | uv_coordinates[:, 1] = int(target_resolution - 1) - uv_coordinates[:, 1] 363 | 364 | # in case we have uv coordinates beyond the texture 365 | uv_coordinates = uv_coordinates % int(target_resolution) 366 | 367 | uv_texture_first_pass[ 368 | cam_index, uv_coordinates[:, 1], uv_coordinates[:, 0], ... 369 | ] = resized_tiles[cam_index].reshape(-1, 3) 370 | 371 | # adjust the facing weight to the chosen percentile 372 | cur_facing_image = facing_images[cam_index] 373 | 374 | # goes from 0 to 1, we cut of the bottom 1.0-facing_percentile and stretch the rest 0 to 1 375 | cur_facing_image = cur_facing_image * ( 376 | 1.0 + facing_percentile 377 | ) # 0..1 now 0..1.2 378 | cur_facing_image = cur_facing_image - facing_percentile # 0..1.2 now -0.2..1.0 379 | cur_facing_image[cur_facing_image < 0] = 0 380 | 381 | uv_texture_first_pass_weight[ 382 | cam_index, uv_coordinates[:, 1], uv_coordinates[:, 0], ... 383 | ] = cur_facing_image.reshape( 384 | -1, 385 | ) 386 | 387 | # multiply the texture channels by the point at factor 388 | weighted_tex = ( 389 | np.stack( 390 | ( 391 | uv_texture_first_pass_weight, 392 | uv_texture_first_pass_weight, 393 | uv_texture_first_pass_weight, 394 | ), 395 | axis=-1, 396 | ) 397 | * uv_texture_first_pass 398 | ) 399 | 400 | weighted_tex = np.sum(weighted_tex, axis=0) / np.stack( 401 | ( 402 | np.sum(uv_texture_first_pass_weight, axis=0), 403 | np.sum(uv_texture_first_pass_weight, axis=0), 404 | np.sum(uv_texture_first_pass_weight, axis=0), 405 | ), 406 | axis=-1, 407 | ) 408 | 409 | # make the unpainted mask 0-255 uint8 410 | unpainted_mask = np.zeros((target_resolution, target_resolution)) 411 | unpainted_mask[np.sum(uv_texture_first_pass_weight, axis=0) <= 0.1] = 255 412 | 413 | # inpaint all the missing pixels 414 | filled_uv_texture = cv2.inpaint( 415 | weighted_tex.astype(np.uint8), 416 | unpainted_mask.astype(np.uint8), 417 | inpaintRadius=3, 418 | flags=cv2.INPAINT_TELEA, 419 | ) 420 | 421 | return filled_uv_texture 422 | 423 | 424 | def load_image(base_path, index, frame_number, prefix=""): 425 | """Helper function to load an image file as a numpy array.""" 426 | file_path = Path(base_path) / f"{prefix}camera_{index:02d}_{frame_number:04d}.exr" 427 | return bpy_img_to_numpy(str(file_path))[..., :3] 428 | 429 | 430 | def load_depth_image(base_path, index, frame_number): 431 | """Helper function to load and process a depth image.""" 432 | file_path = Path(base_path) / f"depth_camera_{index:02d}_{frame_number:04d}.exr" 433 | return create_depth_condition(str(file_path))[..., :3] 434 | 435 | 436 | def load_normal_image(base_path, index, frame_number, camera): 437 | """Helper function to load and process a normal image.""" 438 | file_path = Path(base_path) / f"normal_camera_{index:02d}_{frame_number:04d}.exr" 439 | return create_normal_condition(str(file_path), camera)[..., :3] 440 | 441 | 442 | def rotate_cameras_around_origin(angle_degrees): 443 | """ 444 | Rotates all cameras in the current Blender scene around the world origin by the specified angle. 445 | 446 | :param angle_degrees: The angle in degrees to rotate the cameras. 447 | """ 448 | # Convert angle to radians 449 | angle_radians = math.radians(angle_degrees) 450 | 451 | # Create a rotation matrix for rotation around the Z-axis 452 | rotation_matrix = mathutils.Matrix.Rotation(angle_radians, 4, "Z") 453 | 454 | # Iterate through all objects in the scene 455 | for obj in bpy.data.objects: 456 | if obj.type == "CAMERA": 457 | # Apply the rotation matrix to the object's location 458 | obj.location = rotation_matrix @ obj.location 459 | 460 | # Rotate the camera's orientation as well 461 | obj.rotation_euler.rotate(rotation_matrix) 462 | 463 | 464 | def generate_multiple_views( 465 | scene, max_size, suffix, render_resolution=2048, offset_additional=0 466 | ): 467 | """ 468 | Generates texture maps for a 3D model using multiple views and saves outputs for depth, normal, UV, position, and image maps. 469 | 470 | :param scene: The Blender scene object. 471 | :param max_size: Maximum bounding box size of the model. 472 | :param create_cameras_on_one_ring: Function to create cameras on a single ring around the object. 473 | :param create_cameras_on_sphere: Function to create cameras in a spherical arrangement. 474 | :param setup_render_settings: Function to set up render settings and node outputs. 475 | :return: A dictionary of rendered images with keys: 'depth', 'normal', 'uv', 'position', 'image', 'facing'. 476 | """ 477 | 478 | # Set parameters 479 | num_cameras = int(scene.num_cameras) 480 | 481 | # Create cameras based on the number specified in the scene 482 | if num_cameras == 4: 483 | cameras = create_cameras_on_one_ring( 484 | num_cameras=num_cameras, 485 | max_size=max_size, 486 | name_prefix=f"Camera_{suffix}", 487 | ) 488 | elif num_cameras in [9, 16]: 489 | cameras = create_cameras_on_sphere( 490 | num_cameras=num_cameras, 491 | max_size=max_size, 492 | name_prefix=f"Camera_{suffix}", 493 | ) 494 | else: 495 | raise ValueError("Only 4, 9, or 16 cameras are supported.") 496 | 497 | # To have different viewpoints between modes 498 | if offset_additional > 0: 499 | rotate_cameras_around_origin(offset_additional) 500 | 501 | # Set up render nodes and paths 502 | output_path = Path(scene.output_path) 503 | output_nodes = setup_render_settings( 504 | scene, resolution=(render_resolution, render_resolution) 505 | ) 506 | output_dirs = ["depth", "normal", "uv", "position", "img"] 507 | render_img_folders = [] 508 | for output_type in output_dirs: 509 | output_nodes[output_type].base_path = str( 510 | output_path / f"first_pass_{output_type}" 511 | ) 512 | os.makedirs(output_nodes[output_type].base_path, exist_ok=True) 513 | 514 | render_img_folders.append(str(output_nodes[output_type].base_path)) 515 | 516 | render_img_folders.append(str(output_path / "RenderOutput")) 517 | 518 | # Update to make new cameras available 519 | bpy.context.view_layer.update() 520 | 521 | # Initialize lists for loaded images 522 | ( 523 | depth_images, 524 | normal_images, 525 | uv_images, 526 | position_images, 527 | img_images, 528 | facing_images, 529 | ) = ([] for _ in range(6)) 530 | frame_number = scene.frame_current 531 | 532 | # Render and load images for each camera 533 | for i, camera in enumerate(cameras): 534 | scene.camera = camera # Set active camera 535 | 536 | # Set file paths for each render pass 537 | for pass_type in output_dirs: 538 | output_nodes[pass_type].file_slots[0].path = f"{pass_type}_camera_{i:02d}_" 539 | 540 | bpy.context.view_layer.update() 541 | bpy.ops.render.render(write_still=True) # Render and save images 542 | 543 | # Load UV image 544 | uv_images.append( 545 | load_image(output_nodes["uv"].base_path, i, frame_number, "uv_") 546 | ) 547 | 548 | # Load depth image with processing 549 | depth_images.append( 550 | load_depth_image(output_nodes["depth"].base_path, i, frame_number) 551 | ) 552 | 553 | # Load position image 554 | position_images.append( 555 | load_image(output_nodes["position"].base_path, i, frame_number, "position_") 556 | ) 557 | 558 | # Load normal image with processing 559 | normal_images.append( 560 | load_normal_image(output_nodes["normal"].base_path, i, frame_number, camera) 561 | ) 562 | 563 | # Create a facing ratio image to show alignment between normals and camera direction 564 | facing_img = create_similar_angle_image( 565 | load_image(output_nodes["normal"].base_path, i, frame_number, "normal_")[ 566 | ..., :3 567 | ], 568 | position_images[-1][..., :3], 569 | camera, 570 | ) 571 | facing_images.append(facing_img) 572 | 573 | return { 574 | "depth": depth_images, 575 | "normal": normal_images, 576 | "uv": uv_images, 577 | "position": position_images, 578 | "image": img_images, 579 | "facing": facing_images, 580 | }, render_img_folders 581 | 582 | 583 | def assemble_multiview_grid( 584 | multiview_images, render_resolution=2048, sd_resolution=512, latent_resolution=None 585 | ): 586 | """ 587 | Assembles images from multiple views into a structured grid, applies a mask, and resizes the outputs. 588 | 589 | :param multiview_images: Dictionary containing lists of images for 'depth', 'normal', 'facing', and 'uv'. 590 | :param render_resolution: Resolution for rendering each camera view before scaling. 591 | :param target_resolution: Target resolution for the SD images. 592 | :return: A dictionary containing assembled grids for 'depth', 'normal', 'uv', 'facing', and 'content mask'. 593 | """ 594 | 595 | num_cameras = len(multiview_images["depth"]) 596 | grid_size = int(math.sqrt(num_cameras)) # Assuming a square grid 597 | 598 | # Initialize empty arrays for each type of grid 599 | grids = initialize_grids(grid_size, render_resolution) 600 | 601 | # Populate the grids with multiview images 602 | for i, (depth_img, normal_img, facing_img, uv_img) in enumerate( 603 | zip( 604 | multiview_images["depth"], 605 | multiview_images["normal"], 606 | multiview_images["facing"], 607 | multiview_images["uv"], 608 | ) 609 | ): 610 | row, col = compute_grid_position(i, grid_size, render_resolution) 611 | populate_grids( 612 | grids, 613 | depth_img, 614 | normal_img, 615 | facing_img, 616 | uv_img, 617 | row, 618 | col, 619 | render_resolution, 620 | ) 621 | 622 | # Generate content mask and input image 623 | grids["content_mask"] = create_content_mask(grids["uv_grid"]) 624 | 625 | # Resize grids to target resolution for SD model input 626 | resized_grids = resize_grids(grids, render_resolution, sd_resolution) 627 | resized_grids["content_mask"] = create_content_mask(resized_grids["uv_grid"]) 628 | 629 | if latent_resolution is not None: 630 | latent_grids = resize_grids( 631 | grids, render_resolution, latent_resolution, interpolation=cv2.INTER_LINEAR 632 | ) 633 | # Add the latent resolution grids to the resized grids with "latent_" prefix 634 | for key, grid in latent_grids.items(): 635 | resized_grids[f"latent_{key}"] = grid 636 | 637 | # Create the canny for the resized grids 638 | resized_grids["canny_grid"] = np.stack( 639 | ( 640 | cv2.Canny(resized_grids["normal_grid"].astype(np.uint8), 100, 200), 641 | cv2.Canny(resized_grids["normal_grid"].astype(np.uint8), 100, 200), 642 | cv2.Canny(resized_grids["normal_grid"].astype(np.uint8), 100, 200), 643 | ), 644 | axis=-1, 645 | ) 646 | 647 | return grids, resized_grids 648 | 649 | 650 | def initialize_grids(grid_size, render_resolution): 651 | """Initialize blank grids for each map type.""" 652 | grid_shape = (grid_size * render_resolution, grid_size * render_resolution) 653 | return { 654 | "depth_grid": np.zeros((*grid_shape, 3), dtype=np.uint8), 655 | "normal_grid": np.zeros((*grid_shape, 3), dtype=np.uint8), 656 | "facing_grid": np.zeros(grid_shape, dtype=np.uint8), 657 | "uv_grid": np.zeros((*grid_shape, 3), dtype=np.float32), 658 | } 659 | 660 | 661 | def compute_grid_position(index, grid_size, render_resolution): 662 | """Compute row and column position in the grid based on index.""" 663 | row = (index // grid_size) * render_resolution 664 | col = (index % grid_size) * render_resolution 665 | return row, col 666 | 667 | 668 | def populate_grids( 669 | grids, depth_img, normal_img, facing_img, uv_img, row, col, render_resolution 670 | ): 671 | """Populate each grid with the corresponding multiview images.""" 672 | grids["depth_grid"][ 673 | row : row + render_resolution, col : col + render_resolution 674 | ] = depth_img 675 | grids["normal_grid"][ 676 | row : row + render_resolution, col : col + render_resolution 677 | ] = normal_img 678 | grids["facing_grid"][ 679 | row : row + render_resolution, col : col + render_resolution 680 | ] = (255 * facing_img).astype(np.uint8) 681 | grids["uv_grid"][ 682 | row : row + render_resolution, col : col + render_resolution 683 | ] = uv_img 684 | 685 | 686 | def create_content_mask(uv_img): 687 | """Generate a content mask from the UV image.""" 688 | content_mask = np.zeros(uv_img.shape[:2], dtype=np.uint8) 689 | content_mask[np.sum(uv_img, axis=-1) > 0] = 255 690 | return cv2.dilate(content_mask, np.ones((10, 10), np.uint8), iterations=3) 691 | 692 | 693 | def resize_grids( 694 | grids, render_resolution, sd_resolution, interpolation=cv2.INTER_NEAREST 695 | ): 696 | """Resize grids to target resolution for Stable Diffusion model.""" 697 | 698 | scale_factor = sd_resolution / render_resolution 699 | resized_grids = {} 700 | for key, grid in grids.items(): 701 | 702 | height, width = grid.shape[:2] 703 | 704 | interpolation = interpolation if "mask" in key else cv2.INTER_LINEAR 705 | resized_grids[key] = cv2.resize( 706 | grid, 707 | (int(scale_factor * height), int(scale_factor * width)), 708 | interpolation=interpolation, 709 | ) 710 | return resized_grids 711 | 712 | 713 | def save_multiview_grids(multiview_grids, output_folder, file_format="png", prefix=""): 714 | """ 715 | Save each grid in multiview_grids to the specified output folder. 716 | 717 | :param multiview_grids: Dictionary where keys are grid names and values are images (numpy arrays). 718 | :param output_folder: Path to the output folder where images will be saved. 719 | :param file_format: File format for the saved images (default is 'png'). 720 | """ 721 | os.makedirs(output_folder, exist_ok=True) # Ensure output directory exists 722 | 723 | for key, image in multiview_grids.items(): 724 | file_path = os.path.join(output_folder, f"{prefix}{key}.{file_format}") 725 | 726 | if image.dtype == np.float32: 727 | image = 255 * image 728 | image = image.astype(np.uint8) 729 | 730 | # Convert to BGR format if needed for OpenCV (assumes input is RGB) 731 | if image.ndim == 3 and image.shape[2] == 3: 732 | image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR) 733 | 734 | # Write the image to disk 735 | cv2.imwrite(file_path, image) 736 | 737 | print(f"Saved {key} grid to {file_path}") 738 | 739 | 740 | def create_input_image(texture, uv, render_resolution, texture_resolution): 741 | """ 742 | Project a texture onto renderings using UV coordinates, then resize the projected texture 743 | images to match the resolution of the resized UV rendering images. 744 | 745 | :param texture: Input texture image (H, W, C) with texture resolution as dimensions. 746 | :param uv: Original UV coordinates (H, W, 2) corresponding to the renderings, values in range [0, 1]. 747 | :param input_texture_res: Target Resolution for the input 748 | :param render_resolution: render_resolution 749 | :param texture_resolution: texture_resolution 750 | :return: The projected and resized texture as 3 image arrays. 751 | """ 752 | 753 | input_texture_res = texture.shape[0] 754 | 755 | # Scale UV coordinates to texture coordinates 756 | uv_scaled = uv[..., :2] 757 | uv_scaled = (uv_scaled * (input_texture_res - 1)).astype(np.int32) 758 | uv_scaled[..., 1] = int(input_texture_res - 1) - uv_scaled[..., 1] 759 | 760 | # Reshape UV coordinates for indexing 761 | uv_coordinates = uv_scaled[..., :2].reshape(-1, 2) 762 | uv_coordinates = uv_coordinates % input_texture_res # Handle wrap-around 763 | 764 | # Ensure texture has three channels 765 | texture = texture[..., :3] 766 | 767 | # Project texture using the UV coordinates 768 | projected_texture = texture[uv_coordinates[:, 1], uv_coordinates[:, 0], :3].reshape( 769 | (uv.shape[0], uv.shape[1], 3) 770 | ) 771 | 772 | # Resize to match the resized UV rendering dimensions 773 | input_texture_resolution = cv2.resize( 774 | projected_texture, 775 | (input_texture_res, input_texture_res), 776 | interpolation=cv2.INTER_LINEAR, 777 | ) 778 | 779 | input_render_resolution = cv2.resize( 780 | projected_texture, 781 | (render_resolution, render_resolution), 782 | interpolation=cv2.INTER_LINEAR, 783 | ) 784 | 785 | target_texture_resolution = cv2.resize( 786 | projected_texture, 787 | (texture_resolution, texture_resolution), 788 | interpolation=cv2.INTER_LINEAR, 789 | ) 790 | 791 | return ( 792 | input_texture_resolution, 793 | input_render_resolution, 794 | target_texture_resolution, 795 | ) 796 | 797 | 798 | def create_input_image_grid(texture, uv_grid, target_size_grid): 799 | """ 800 | Project a texture onto renderings using UV coordinates, then resize the projected texture 801 | images to match the resolution of the resized UV rendering images. 802 | 803 | :param texture: Input texture image (H, W, C) with texture resolution as dimensions. 804 | :param uv: Original UV coordinates (H, W, 2) corresponding to the renderings, values in range [0, 1]. 805 | :return: The projected texture onto the grid. 806 | """ 807 | 808 | input_texture_res = texture.shape[0] 809 | 810 | target_texture_res = target_size_grid.shape[0] 811 | 812 | # Scale UV coordinates to texture coordinates 813 | uv_scaled = uv_grid[..., :2] 814 | uv_scaled = (uv_scaled * (input_texture_res - 1)).astype(np.int32) 815 | uv_scaled[..., 1] = int(input_texture_res - 1) - uv_scaled[..., 1] 816 | 817 | # Reshape UV coordinates for indexing 818 | uv_coordinates = uv_scaled[..., :2].reshape(-1, 2) 819 | uv_coordinates = uv_coordinates % input_texture_res # Handle wrap-around 820 | 821 | # Ensure texture has three channels 822 | texture = texture[..., :3] 823 | 824 | # Project texture using the UV coordinates 825 | projected_texture_grid = texture[ 826 | uv_coordinates[:, 1], uv_coordinates[:, 0], :3 827 | ].reshape((uv_grid.shape[0], uv_grid.shape[1], 3)) 828 | 829 | # resize to target_texture_res size 830 | projected_texture_grid = cv2.resize( 831 | projected_texture_grid[..., :3], 832 | (int(target_texture_res), int(target_texture_res)), 833 | interpolation=cv2.INTER_LANCZOS4, 834 | ) 835 | 836 | return projected_texture_grid 837 | -------------------------------------------------------------------------------- /diffusedtexture/process_utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def blendercs_to_ccs(points_bcs, camera, rotation_only=False): 5 | """ 6 | Converts a point cloud from the Blender coordinate system to the camera coordinate system. 7 | """ 8 | # Extract camera rotation in world space 9 | camera_rotation = np.array(camera.matrix_world.to_quaternion().to_matrix()).T 10 | 11 | # Apply the rotation to align normals with the camera’s view 12 | if rotation_only: 13 | point_3d_cam = np.dot(camera_rotation, points_bcs.T).T 14 | else: 15 | # Translate points to the camera's coordinate system 16 | camera_position = np.array(camera.matrix_world.to_translation()).reshape((3,)) 17 | points_bcs = points_bcs - camera_position 18 | point_3d_cam = np.dot(camera_rotation, points_bcs.T).T 19 | 20 | # Convert to camera coordinate system by inverting the Z-axis 21 | R_blender_to_cv = np.array([[1, 0, 0], [0, -1, 0], [0, 0, -1]]) 22 | point_3d_cam = np.dot(R_blender_to_cv, point_3d_cam.T).T 23 | 24 | return point_3d_cam 25 | -------------------------------------------------------------------------------- /diffusedtexture/uv_pass.py: -------------------------------------------------------------------------------- 1 | import os 2 | import bpy 3 | import cv2 4 | import numpy as np 5 | from pathlib import Path 6 | 7 | from ..condition_setup import ( 8 | bpy_img_to_numpy, 9 | ) 10 | from .diffusers_utils import ( 11 | create_uv_pass_pipeline, 12 | infer_uv_pass_pipeline, 13 | ) 14 | 15 | 16 | def export_uv_layout(obj_name, export_path, uv_map_name=None, size=(1024, 1024)): 17 | """ 18 | Export the UV layout of a given mesh object and specified UV map to an image file. 19 | 20 | :param obj_name: Name of the object in Blender. 21 | :param export_path: File path where the UV layout should be saved. 22 | :param uv_map_name: Name of the UV map to use (default is the active UV map). 23 | :param size: Resolution of the UV layout image (default is 1024x1024). 24 | """ 25 | # Get the object 26 | obj = bpy.data.objects.get(obj_name) 27 | if obj is None or obj.type != "MESH": 28 | print(f"Object {obj_name} not found or is not a mesh.") 29 | return 30 | 31 | # Set the object as active 32 | bpy.context.view_layer.objects.active = obj 33 | 34 | # Ensure it is in object mode 35 | bpy.ops.object.mode_set(mode="OBJECT") 36 | 37 | # Get the UV map layers 38 | uv_layers = obj.data.uv_layers 39 | if not uv_layers: 40 | print(f"No UV maps found for object {obj_name}.") 41 | return 42 | 43 | # Find or set the active UV map 44 | if uv_map_name: 45 | uv_layer = uv_layers.get(uv_map_name) 46 | if uv_layer is None: 47 | print( 48 | f"UV map {uv_map_name} not found on object {obj_name}. Using active UV map." 49 | ) 50 | uv_layer = obj.data.uv_layers.active 51 | else: 52 | uv_layers.active = uv_layer # Set the selected UV map as active 53 | else: 54 | uv_layer = uv_layers.active # Use active UV map if none is provided 55 | 56 | if uv_layer is None: 57 | print(f"No active UV map found for object {obj_name}.") 58 | return 59 | 60 | # Set UV export settings 61 | bpy.ops.uv.export_layout( 62 | filepath=export_path, 63 | size=size, 64 | opacity=1.0, # Adjust opacity of the lines (0.0 to 1.0) 65 | export_all=False, # Export only active UV map 66 | ) 67 | 68 | 69 | def uv_pass(scene, texture_input): 70 | """Run the UV space refinement pass.""" 71 | 72 | obj_name = scene.my_mesh_object 73 | uv_map_name = scene.my_uv_map 74 | 75 | # uv output path 76 | uv_map_path = Path(scene.output_path) # Use Path for OS-independent path handling 77 | uv_map_path = str(uv_map_path / "uv_map_layout.png") 78 | 79 | export_uv_layout( 80 | obj_name, 81 | uv_map_path, 82 | uv_map_name=uv_map_name, 83 | size=(int(scene.texture_resolution), int(scene.texture_resolution)), 84 | ) 85 | 86 | # load the uv layout 87 | uv_layout_image = bpy_img_to_numpy(str(uv_map_path)) 88 | 89 | uv_layout_image = uv_layout_image / np.max(uv_layout_image) 90 | uv_layout_image = uv_layout_image * 255 91 | uv_layout_image = uv_layout_image.astype(np.uint8) 92 | 93 | # Texture mask based on the alpha channel (3rd channel in RGBA image) 94 | mask = uv_layout_image[..., 3] 95 | 96 | # Apply dilation to the mask 97 | mask = cv2.dilate(mask, np.ones((3, 3), np.uint8), iterations=2) 98 | 99 | # Stack the mask into 3 channels (to match the RGB format) 100 | mask = np.stack((mask, mask, mask), axis=-1) 101 | mask = mask.astype(np.uint8) 102 | 103 | # Create the Canny edge detection mask 104 | canny = 255 * np.ones( 105 | (int(scene.texture_resolution), int(scene.texture_resolution)) 106 | ) 107 | 108 | # Only process areas where the alpha channel is 255 109 | canny[uv_layout_image[..., 3] >= 1] = uv_layout_image[..., 0][ 110 | uv_layout_image[..., 3] >= 1 111 | ] 112 | 113 | canny[canny < 64] = 0 114 | canny[canny >= 192] = 255 115 | canny = 255 - canny 116 | 117 | # Stack the canny result into 3 channels (RGB) 118 | canny = np.stack((canny, canny, canny), axis=-1) 119 | canny = canny.astype(np.uint8) 120 | 121 | # create the pipe 122 | pipe = create_uv_pass_pipeline(scene) 123 | 124 | texture_input = texture_input[..., :3] 125 | 126 | # create the base of the final texture 127 | texture_input = cv2.resize( 128 | texture_input, 129 | (int(scene.texture_resolution), int(scene.texture_resolution)), 130 | interpolation=cv2.INTER_LANCZOS4, 131 | ) 132 | 133 | output_image = infer_uv_pass_pipeline( 134 | pipe, 135 | scene, 136 | texture_input, 137 | mask, 138 | canny, 139 | strength=scene.denoise_strength, 140 | ) 141 | 142 | # remove alpha 143 | output_image = np.array(output_image)[..., :3] 144 | 145 | cv2.imwrite( 146 | os.path.join(scene.output_path, f"texture_input_uv_pass.png"), 147 | cv2.cvtColor(texture_input, cv2.COLOR_RGB2BGR), 148 | ) 149 | cv2.imwrite( 150 | os.path.join(scene.output_path, f"mask_uv_pass.png"), 151 | cv2.cvtColor(mask, cv2.COLOR_RGB2BGR), 152 | ) 153 | cv2.imwrite( 154 | os.path.join(scene.output_path, f"canny_uv_pass.png"), 155 | cv2.cvtColor(canny, cv2.COLOR_RGB2BGR), 156 | ) 157 | cv2.imwrite( 158 | os.path.join(scene.output_path, f"output_image_uv_pass.png"), 159 | cv2.cvtColor(output_image, cv2.COLOR_RGB2BGR), 160 | ) 161 | 162 | return output_image 163 | -------------------------------------------------------------------------------- /documentation/DOCUMENTATION.md: -------------------------------------------------------------------------------- 1 | # How Does It Work? 2 | 3 | ## Overview 4 | DiffusedTexture generates textures for 3D models by leveraging Stable Diffusion with ControlNets. The process involves generating viewpoints from multiple cameras, creating control images, and blending results into a cohesive texture. The workflow adapts to different modes: Text2Image, Image2Image Parallel, and Image2Image Sequential. 5 | 6 | --- 7 | 8 | ## Cameras 9 | The number of cameras determines the viewpoints used to generate the texture. These viewpoints ensure adequate coverage of the model's surface, capturing details from various angles. 10 | 11 | ### **4 Cameras:** Minimal coverage for quick texture generation; best for simple models or initial textures which are further refined. 12 | ![4 Cameras](https://github.com/FrederikHasecke/diffused-texture-addon/blob/master/images/process/cameras_4.png) 13 | 14 | ### **9 Cameras:** Balanced detail and coverage; suitable for moderately complex models. 15 | ![9 Cameras](https://github.com/FrederikHasecke/diffused-texture-addon/blob/master/images/process/cameras_9.png) 16 | 17 | ### **16 Cameras:** High detail and coverage; ideal for highly detailed models, especially in a second or third pass of an already textures object. 18 | ![16 Cameras](https://github.com/FrederikHasecke/diffused-texture-addon/blob/master/images/process/cameras_16.png) 19 | 20 | --- 21 | 22 | ## Processes 23 | 24 | ### **Parallel Processing on Images** 25 | This mode generates textures by running Stable Diffusion on multiple viewpoints in parallel. 26 | 27 | #### Steps: 28 | 1. **Multiple Viewpoints**: 29 | - The camera viewpoints are laid out in a grid (e.g., 2x2, 3x3, or 4x4). 30 | 2. **Grid Input**: 31 | - The input to Stable Diffusion is either a supplied texture or a fully white image, when generating a texture from text only. 32 | 3. **Control Images**: 33 | - ControlNets use depth, canny (derived from normals), and surface normal images in the same grid structure to guide the texture generation. 34 | 4. **Resolution Setup**: 35 | - *Render Resolution*: The resolution used for camera views and projections. Should be at least 2x the Texture Resolution to avoid striping artifacts. 36 | - *Texture Resolution*: Final resolution of the generated texture. 37 | - *Stable Diffusion Resolution*: If you want more or less resolution in the Stable Diffusion process, you can adjust this setting. Can easily lead to out of memory errors. 38 | 5. **Projection**: 39 | - The texture contributions are weighted by the perpendicularity of the surface to the corresponing camera, ensuring smooth blending between viewpoints. 40 | 41 | --- 42 | 43 | ### **Sequential Processing on Images** 44 | This mode processes each viewpoint individually, refining the texture step-by-step for greater control over the final result. 45 | 46 | #### Steps: 47 | 1. **Single View at a Time**: 48 | - Instead of processing all viewpoints in parallel, this mode handles one camera view at a time. 49 | 2. **Input Masking**: 50 | - Only faces close to perpendicular to the camera are provided as a mask to Stable Diffusion. 51 | 3. **Reprojection**: 52 | - After each view is processed, the resulting texture is reprojected onto the model, contributing to the next iteration. 53 | 54 | #### Advantages: 55 | - Greater control over texture alignment and details. 56 | - Allows finer adjustments compared to parallel modes. 57 | 58 | #### Disadvantages: 59 | - Slower than parallel processing. 60 | - Requires more manual intervention. 61 | - May lead to visible seams if not handled carefully. 62 | 63 | --- 64 | 65 | ## Parameters and Considerations 66 | 67 | ### Cameras 68 | - **Impact**: The number of cameras affects texture quality and detail and runtime. 69 | - **Recommendation**: Use more cameras for complex models, but balance against VRAM limitations. 70 | 71 | ### Resolutions 72 | - **Render Resolution**: Should be at least 2x the Texture Resolution to prevent artifacts. 73 | - **Texture Resolution**: Determines the final quality of the generated texture. 74 | 75 | ### ControlNets 76 | - Control images (depth, canny, normal) guide the texture generation process. 77 | - **Complexity Settings**: 78 | - *Low*: Depth ControlNet only (creative but less accurate). 79 | - *Medium*: Depth and Canny ControlNets (balanced results). 80 | - *High*: Depth, Canny, and Normal ControlNets (precise but less creative). 81 | 82 | -------------------------------------------------------------------------------- /documentation/WORKFLOW.md: -------------------------------------------------------------------------------- 1 | # Recommended Workflow 2 | 3 | ## First Pass: Global Consistency 4 | - Use the `Parallel Processing on Images` mode with 4 or 9 cameras to create one or more variations of what you want to achieve as a texture. 5 | - If your description does not yield the results that you want to reach, try to use IPAdapter and provide an image of what you want. 6 | 7 | ## Second Pass: Local Refinement 8 | - Use the `Parallel Processing on Images` mode to refine the texture with more cameras. 9 | - Use the `ControlNets` to guide the texture generation process, more complex settings can be used for better results if the model is complex, if you want more creative results, use the `Low` setting. 10 | - Repeat the process on the resulting texture to refine it further if needed. 11 | 12 | ## Third Pass: Fine-Tuning 13 | - Use the `Sequential Processing on Images` mode to fine-tune the texture. This mode allows each viewpoint to be processed individually, refining the texture step-by-step for greater control over the final result. 14 | - Repetition of the process on the resulting texture to refine it further if needed. 15 | 16 | -------------------------------------------------------------------------------- /images/download.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FrederikHasecke/diffused-texture-addon/089bdc36d8a716ab116fa4a0cd1d3a3a5a942c82/images/download.png -------------------------------------------------------------------------------- /images/elephant.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FrederikHasecke/diffused-texture-addon/089bdc36d8a716ab116fa4a0cd1d3a3a5a942c82/images/elephant.gif -------------------------------------------------------------------------------- /images/install.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FrederikHasecke/diffused-texture-addon/089bdc36d8a716ab116fa4a0cd1d3a3a5a942c82/images/install.png -------------------------------------------------------------------------------- /images/process/cameras_16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FrederikHasecke/diffused-texture-addon/089bdc36d8a716ab116fa4a0cd1d3a3a5a942c82/images/process/cameras_16.png -------------------------------------------------------------------------------- /images/process/cameras_4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FrederikHasecke/diffused-texture-addon/089bdc36d8a716ab116fa4a0cd1d3a3a5a942c82/images/process/cameras_4.png -------------------------------------------------------------------------------- /images/process/cameras_9.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FrederikHasecke/diffused-texture-addon/089bdc36d8a716ab116fa4a0cd1d3a3a5a942c82/images/process/cameras_9.png -------------------------------------------------------------------------------- /images/rabbit.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FrederikHasecke/diffused-texture-addon/089bdc36d8a716ab116fa4a0cd1d3a3a5a942c82/images/rabbit.gif -------------------------------------------------------------------------------- /images/usage.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FrederikHasecke/diffused-texture-addon/089bdc36d8a716ab116fa4a0cd1d3a3a5a942c82/images/usage.gif -------------------------------------------------------------------------------- /object_ops.py: -------------------------------------------------------------------------------- 1 | import bpy 2 | import mathutils 3 | 4 | 5 | def move_object_to_origin(obj): 6 | """Move the object to the world origin.""" 7 | obj.location = (0, 0, 0) 8 | 9 | 10 | def calculate_mesh_midpoint(obj): 11 | """ 12 | Calculate the midpoint of the mesh, move the object's origin to it, 13 | and return the maximum size of the mesh in any dimension. 14 | 15 | :param obj: The object whose mesh midpoint and size are calculated. 16 | :return: The maximum size of the mesh (float). 17 | """ 18 | # Calculate local coordinates in world space 19 | local_coords = [obj.matrix_world @ vert.co for vert in obj.data.vertices] 20 | 21 | # Determine the minimum and maximum coordinates 22 | min_coord = mathutils.Vector(map(min, zip(*local_coords))) 23 | max_coord = mathutils.Vector(map(max, zip(*local_coords))) 24 | 25 | # Calculate the midpoint 26 | midpoint = (min_coord + max_coord) / 2 27 | 28 | # Set the origin to the calculated midpoint 29 | bpy.context.view_layer.objects.active = obj 30 | bpy.ops.object.origin_set(type="ORIGIN_GEOMETRY", center="BOUNDS") 31 | 32 | # Move the object to align its origin with the world origin 33 | obj.location -= midpoint 34 | 35 | # Calculate the maximum size in any dimension 36 | size_vector = max_coord - min_coord 37 | max_size = max(size_vector) 38 | 39 | return max_size 40 | -------------------------------------------------------------------------------- /operators.py: -------------------------------------------------------------------------------- 1 | import os 2 | import threading 3 | 4 | import bpy 5 | import copy 6 | import traceback 7 | import datetime 8 | import numpy as np 9 | from pathlib import Path 10 | 11 | from .diffusedtexture import img_parallel, img_sequential, latent_parallel 12 | 13 | from .object_ops import move_object_to_origin, calculate_mesh_midpoint 14 | from .scene_backup import clean_scene, clean_object 15 | from .utils import image_to_numpy 16 | 17 | 18 | class OBJECT_OT_GenerateTexture(bpy.types.Operator): 19 | bl_idname = "object.generate_texture" 20 | bl_label = "Generate Texture" 21 | 22 | def execute(self, context): 23 | scene = context.scene 24 | output_path = Path(scene.output_path) 25 | 26 | # Retrieve the HuggingFace cache path from preferences 27 | prefs = bpy.context.preferences.addons[__package__].preferences 28 | hf_cache_path = prefs.hf_cache_path 29 | 30 | # Set environment variable if path is provided 31 | if hf_cache_path: 32 | os.environ["HF_HOME"] = hf_cache_path 33 | 34 | # Explizitly prevent online access if set by user 35 | if not bpy.app.online_access: 36 | os.environ["HF_HUB_OFFLINE"] = "1" 37 | 38 | # # import affter setting the HF home 39 | # from .diffusedtexture import img_parallel 40 | 41 | selected_mesh_name = scene.my_mesh_object 42 | selected_object = bpy.data.objects.get(selected_mesh_name) 43 | 44 | # Ensure the output path exists 45 | if not output_path: 46 | self.report({"ERROR"}, "Output path is not set.") 47 | return {"CANCELLED"} 48 | 49 | if not output_path.exists(): 50 | self.report({"ERROR"}, "Output path does not exist.") 51 | return {"CANCELLED"} 52 | 53 | # Save a backup of the current .blend file 54 | backup_file = output_path / "scene_backup.blend" 55 | bpy.ops.wm.save_as_mainfile(filepath=str(backup_file)) 56 | 57 | try: 58 | 59 | # Start progress indicator using context.window_manager 60 | wm = context.window_manager 61 | wm.progress_begin(0, 100) 62 | 63 | # Clean the scene, removing all other objects 64 | clean_scene(scene) 65 | clean_object(scene) 66 | 67 | # Move object to world origin and calculate midpoint 68 | max_size = calculate_mesh_midpoint(selected_object) 69 | move_object_to_origin(selected_object) 70 | 71 | # Execute texture passes based on user selection 72 | if scene.operation_mode == "PARALLEL_IMG": 73 | 74 | if scene.input_texture_path: 75 | # texture_input = self.load_texture(str(input_texture_path)) 76 | texture_input = image_to_numpy(scene.input_texture_path) 77 | else: 78 | texture_input = None 79 | 80 | texture_output = img_parallel.img_parallel( 81 | scene, 1.25 * max_size, texture_input 82 | ) 83 | 84 | # flip along the v axis 85 | texture_output = texture_output[::-1] 86 | 87 | output_file_name = str( 88 | output_path 89 | / ( 90 | f"PARALLEL_IMG_" 91 | + datetime.datetime.now().strftime(r"%y-%m-%d_%H-%M-%S") 92 | + ".png" 93 | ) 94 | ) 95 | 96 | # Save texture 97 | self.save_texture(texture_output, output_file_name) 98 | 99 | elif scene.operation_mode == "SEQUENTIAL_IMG": 100 | 101 | if scene.input_texture_path: 102 | texture_input = image_to_numpy(scene.input_texture_path) 103 | else: 104 | texture_input = None 105 | 106 | texture_output = img_sequential.img_sequential( 107 | scene, 1.25 * max_size, texture_input 108 | ) 109 | # flip along the v axis 110 | texture_output = texture_output[::-1] 111 | 112 | output_file_name = str( 113 | output_path 114 | / ( 115 | f"SEQUENTIAL_IMG_" 116 | + datetime.datetime.now().strftime(r"%y-%m-%d_%H-%M-%S") 117 | + ".png" 118 | ) 119 | ) 120 | 121 | # Save texture 122 | self.save_texture(texture_output, output_file_name) 123 | 124 | # elif scene.operation_mode == "PARALLEL_LATENT": 125 | 126 | # if scene.input_texture_path: 127 | # # texture_input = self.load_texture(str(input_texture_path)) 128 | # texture_input = image_to_numpy(scene.input_texture_path) 129 | # else: 130 | # texture_input = None 131 | 132 | # texture_output = latent_parallel.latent_parallel( 133 | # scene=scene, max_size=1.25 * max_size, texture=texture_input 134 | # ) 135 | 136 | # # flip along the v axis 137 | # texture_output = texture_output[::-1] 138 | 139 | # output_file_name = str( 140 | # output_path 141 | # / ( 142 | # f"PARALLEL_LATENT_" 143 | # + datetime.datetime.now().strftime(r"%y-%m-%d_%H-%M-%S") 144 | # + ".png" 145 | # ) 146 | # ) 147 | 148 | # # Save texture 149 | # self.save_texture(texture_output, output_file_name) 150 | 151 | # TODO: At one point revisit this part. Limit to texture parts and leave texture patch borders out 152 | # elif scene.operation_mode == "TEXTURE2TEXTURE_ENHANCEMENT": 153 | 154 | # texture_input = self.load_texture(str(input_texture_path)) 155 | 156 | # texture_uv_pass = uv_pass.uv_pass(scene, texture_input) 157 | # texture_final = copy.deepcopy(texture_uv_pass) 158 | 159 | # # Save texture_final as texture_uv_pass 160 | # self.save_texture(texture_final, str(output_path / "uv_pass.png")) 161 | # self.save_texture(texture_final, str(output_path / "final_texture.png")) 162 | 163 | else: 164 | raise NotImplementedError( 165 | "Only the two modes 'PARALLEL_IMG' and 'SEQUENTIAL_IMG' are implemented" 166 | ) 167 | 168 | # Process complete 169 | wm.progress_end() 170 | 171 | except Exception as e: 172 | # Capture and format the stack trace 173 | error_message = "".join( 174 | traceback.format_exception(None, e, e.__traceback__) 175 | ) 176 | 177 | # Report the error to the user in Blender's interface 178 | self.report( 179 | {"ERROR"}, f"An error occurred: {str(e)}\nDetails:\n{error_message}" 180 | ) 181 | 182 | # End the progress indicator 183 | wm.progress_end() 184 | 185 | # Optionally, print the error message to the console for detailed inspection 186 | print(error_message) 187 | 188 | return {"CANCELLED"} 189 | 190 | finally: 191 | 192 | # Restore the original scene by reloading the backup file 193 | bpy.ops.wm.open_mainfile(filepath=str(backup_file)) 194 | 195 | # Select the new object since we reloaded 196 | selected_object = bpy.data.objects.get(selected_mesh_name) 197 | 198 | # Assign the texture_final to the object 199 | self.assign_texture_to_object( 200 | selected_object, str(output_path / output_file_name) 201 | ) 202 | 203 | return {"FINISHED"} 204 | 205 | def save_texture(self, texture, filepath): 206 | """ 207 | Save a numpy array as an image texture in Blender. 208 | 209 | :param texture: numpy array representing the texture (shape: [height, width, channels]). 210 | The array should be in the range [0, 255] for integer values. 211 | :param path: The path where the texture will be saved. 212 | """ 213 | 214 | (height, width) = texture.shape[:2] 215 | 216 | # Ensure the numpy array is in float format and normalize if necessary 217 | if texture.dtype == np.uint8: 218 | texture = texture.astype(np.float32) / 255.0 219 | 220 | # Handle grayscale textures (add alpha channel if needed) 221 | if texture.shape[2] == 1: # Grayscale 222 | texture = np.repeat(texture, 4, axis=2) 223 | texture[:, :, 3] = 1.0 # Set alpha to 1 for grayscale 224 | 225 | elif texture.shape[2] == 3: # RGB 226 | alpha_channel = np.ones((height, width, 1), dtype=np.float32) 227 | texture = np.concatenate( 228 | (texture, alpha_channel), axis=2 229 | ) # Add alpha channel 230 | 231 | # Flatten the numpy array to a list 232 | flattened_texture = texture.flatten() 233 | 234 | # Create a new image in Blender with the provided dimensions 235 | image = bpy.data.images.new( 236 | name="SavedTexture", width=width, height=height, alpha=True 237 | ) 238 | 239 | # Update the image's pixel data with the flattened texture 240 | image.pixels = flattened_texture 241 | 242 | # Save the image to the specified path 243 | image.filepath_raw = filepath 244 | image.file_format = "PNG" # Set to PNG or any other format you prefer 245 | image.save() 246 | 247 | def load_texture(self, filepath): 248 | """Load a texture from a file and return it as a numpy array.""" 249 | if os.path.exists(filepath): 250 | # Load the texture using Blender's image system 251 | image = bpy.data.images.load(filepath) 252 | 253 | # Get image dimensions 254 | width, height = image.size 255 | 256 | # Extract the pixel data (Blender stores it in RGBA format, float [0, 1]) 257 | pixels = np.array(image.pixels[:], dtype=np.float32) 258 | 259 | # Reshape the flattened pixel data into (height, width, 4) array 260 | pixels = pixels.reshape((height, width, 4)) 261 | 262 | # Convert the pixel values from float [0, 1] to [0, 255] uint8 263 | pixels = (pixels * 255).astype(np.uint8) 264 | 265 | return pixels 266 | else: 267 | raise FileNotFoundError(f"Texture file {filepath} not found.") 268 | 269 | def no_material_on(self, mesh): 270 | if 1 > len(mesh.materials): 271 | return True 272 | 273 | for i in range(len(mesh.materials)): 274 | if mesh.materials[i] is not None: 275 | return False 276 | return True 277 | 278 | def assign_texture_to_object(self, obj, texture_filepath): 279 | """ 280 | Assign the final texture to the object's material using the texture filepath. 281 | 282 | :param obj: The object to assign the texture to. 283 | :param texture_filepath: Path to the texture file (PNG or other format). 284 | """ 285 | # Check if the object has an existing material 286 | if obj.data.materials: 287 | material = obj.data.materials[0] 288 | else: 289 | # Create a new material if none exists 290 | material = bpy.data.materials.new(name="GeneratedMaterial") 291 | obj.data.materials.append(material) 292 | 293 | # Enable 'Use Nodes' for the material 294 | material.use_nodes = True 295 | nodes = material.node_tree.nodes 296 | 297 | # Find the Principled BSDF node or add one if it doesn't exist 298 | bsdf = nodes.get("Principled BSDF") 299 | if bsdf is None: 300 | bsdf = nodes.new(type="ShaderNodeBsdfPrincipled") 301 | nodes["Material Output"].location = (400, 0) 302 | 303 | # TODO: check if some image node is already connected to the diffuse Input of the BSDF, only create a new node if not present, else replace the image 304 | 305 | # Create a new image texture node 306 | texture_node = nodes.new(type="ShaderNodeTexImage") 307 | 308 | # Load the texture file as an image 309 | try: 310 | texture_image = bpy.data.images.load(texture_filepath) 311 | texture_node.image = texture_image 312 | except RuntimeError as e: 313 | self.report({"ERROR"}, f"Could not load texture: {e}") 314 | return 315 | 316 | # Link the texture node to the Base Color input of the Principled BSDF node 317 | material.node_tree.links.new( 318 | bsdf.inputs["Base Color"], texture_node.outputs["Color"] 319 | ) 320 | 321 | # Set the location of nodes for better layout 322 | texture_node.location = (-300, 0) 323 | bsdf.location = (0, 0) 324 | 325 | 326 | class OBJECT_OT_SelectPipette(bpy.types.Operator): 327 | bl_idname = "object.select_pipette" 328 | bl_label = "Select Object with Pipette" 329 | 330 | def execute(self, context): 331 | context.scene.my_mesh_object = context.object.name 332 | return {"FINISHED"} 333 | -------------------------------------------------------------------------------- /panel.py: -------------------------------------------------------------------------------- 1 | import bpy 2 | 3 | 4 | class OBJECT_PT_MainPanel(bpy.types.Panel): 5 | bl_label = "DiffusedTexture" 6 | bl_idname = "OBJECT_PT_diffused_texture_panel" 7 | bl_space_type = "VIEW_3D" 8 | bl_region_type = "UI" 9 | bl_category = "DiffusedTexture" 10 | bl_order = 0 # Main panel order 11 | 12 | def draw(self, context): 13 | layout = self.layout 14 | scene = context.scene 15 | 16 | # Object Selection 17 | row = layout.row(align=True) 18 | row.prop(scene, "my_mesh_object", text="Mesh Object") 19 | row.operator("object.select_pipette", text="", icon="VIS_SEL_11") 20 | 21 | # UV Map Selection 22 | layout.prop(scene, "my_uv_map", text="UV Map") 23 | 24 | box_sd = layout.box() 25 | box_sd.label(text="Stable Diffusion Options") 26 | 27 | # Prompt Text Field 28 | box_sd.prop(scene, "my_prompt", text="Prompt") 29 | 30 | # Negative Prompt Text Field 31 | box_sd.prop(scene, "my_negative_prompt", text="Negative Prompt") 32 | 33 | # Denoise 34 | box_sd.prop(scene, "denoise_strength", text="Denoise Strength") 35 | 36 | # Num Inference Steps 37 | box_sd.prop(scene, "num_inference_steps", text="Number of Inference Steps") 38 | 39 | # guidance_scale 40 | box_sd.prop(scene, "guidance_scale", text="Guidance Scale") 41 | 42 | box_dt = layout.box() 43 | box_dt.label(text="DiffusedTexture Options") 44 | 45 | # operation_mode Dropdown 46 | box_dt.prop(scene, "operation_mode", text="Operation Mode") 47 | 48 | # Mesh Complexity Dropdown 49 | box_dt.prop(scene, "mesh_complexity", text="Mesh Complexity") 50 | 51 | # Num Cameras Dropdow 52 | box_dt.prop( 53 | scene, 54 | "num_cameras", 55 | text="Cameras", 56 | ) 57 | 58 | # Warning for Many Cameras 59 | if scene.num_cameras == "16": 60 | box_dt.label( 61 | text="Warning: long freeze, might produce OUT OF MEMORY error", 62 | icon="ERROR", 63 | ) 64 | 65 | # Texture Resolution Dropdown 66 | box_dt.prop(scene, "texture_resolution", text="Texture Resolution") 67 | 68 | # Texture Resolution Dropdown 69 | box_dt.prop(scene, "render_resolution", text="Render Resolution") 70 | 71 | # Warning for low render Resolution 72 | if int(scene.render_resolution) <= int(scene.texture_resolution): 73 | layout.label( 74 | text="Render Resolution should be at least 2x Texture Resolution to prevent 'banding artifacts' in the texture", 75 | icon="ERROR", 76 | ) 77 | 78 | # Output Directory Path 79 | box_dt.prop(scene, "output_path", text="Output Path") 80 | if scene.output_path.startswith("//"): 81 | absolute_path = bpy.path.abspath(scene.output_path) 82 | box_dt.label(text=f"Absolute Path: {absolute_path}", icon="FILE_FOLDER") 83 | 84 | # Warning for missing output path 85 | if scene.output_path == "": 86 | box_dt.label( 87 | text="Warning: No Output Path Given!", 88 | icon="ERROR", 89 | ) 90 | 91 | # Seed Input Field 92 | box_dt.prop(scene, "texture_seed", text="Seed") 93 | 94 | # Text for the user to select the input texture 95 | layout.label(text="Input Texture") 96 | 97 | # Input Image Preview and Selection 98 | row = layout.row() 99 | row.template_ID_preview(scene, "input_texture_path", rows=2, cols=6) 100 | 101 | # Button to open the file browser and load a new image 102 | layout.operator( 103 | "image.open_new_input_image", 104 | text="Open New Input Texture", 105 | icon="IMAGE_DATA", 106 | ) 107 | 108 | # Button to execute the texture generation function 109 | 110 | # Disable the button if the output path is not specified 111 | if not bool(scene.output_path.strip()): 112 | layout.label( 113 | text="Please specify an output path to start texture generation.", 114 | icon="ERROR", 115 | ) 116 | row = layout.row() 117 | row.scale_y = 2.0 118 | row.operator( 119 | "object.generate_texture", 120 | text="Start Texture Generation", 121 | icon="SHADERFX", 122 | ) 123 | row.enabled = False 124 | else: 125 | # Enable the button if the output path is specified but give a warning 126 | if not bpy.app.online_access: 127 | box = layout.box() 128 | box.label(text="Online access is disabled.", icon="ERROR") 129 | box.label( 130 | text="If you don't have the models installed, please enable it in" 131 | ) 132 | box.label(text="Preferences > System > Network > Allow Online Access,") 133 | box.label(text="else the texture generation will fail.") 134 | row = layout.row() 135 | row.scale_y = 2.0 136 | row.operator( 137 | "object.generate_texture", 138 | text="Start Texture Generation", 139 | icon="SHADERFX", 140 | ) 141 | else: 142 | # Enable the button if the output path is specified 143 | row = layout.row() 144 | row.scale_y = 2.0 145 | row.operator( 146 | "object.generate_texture", 147 | text="Start Texture Generation", 148 | icon="SHADERFX", 149 | ) 150 | 151 | 152 | class OBJECT_OT_OpenNewInputImage(bpy.types.Operator): 153 | """Operator to open a new image for the input texture""" 154 | 155 | bl_idname = "image.open_new_input_image" 156 | bl_label = "Open New Input Image" 157 | 158 | filepath: bpy.props.StringProperty(subtype="FILE_PATH") 159 | 160 | def execute(self, context): 161 | # Load the new image using the provided filepath 162 | image = bpy.data.images.load(self.filepath) 163 | context.scene.input_texture_path = image 164 | return {"FINISHED"} 165 | 166 | def invoke(self, context, event): 167 | context.window_manager.fileselect_add(self) 168 | return {"RUNNING_MODAL"} 169 | 170 | 171 | class OBJECT_PT_LoRAPanel(bpy.types.Panel): 172 | bl_label = "LoRA Models" 173 | bl_idname = "OBJECT_PT_lora_panel" 174 | bl_space_type = "VIEW_3D" 175 | bl_region_type = "UI" 176 | bl_category = "DiffusedTexture" 177 | bl_options = {"DEFAULT_CLOSED"} 178 | bl_order = 4 179 | 180 | def draw(self, context): 181 | layout = self.layout 182 | scene = context.scene 183 | 184 | layout.prop(scene, "num_loras", text="Number of LoRAs") 185 | 186 | for i in range(scene.num_loras): 187 | lora_box = layout.box() 188 | lora_box.label(text=f"LoRA Model {i+1}") 189 | lora = scene.lora_models[i] 190 | lora_box.prop(lora, "path", text="Path LoRA") 191 | 192 | if lora.path.startswith("//"): 193 | absolute_path = bpy.path.abspath(lora.path) 194 | lora_box.label( 195 | text=f"Absolute Path: {absolute_path}", icon="FILE_FOLDER" 196 | ) 197 | 198 | lora_box.prop(lora, "strength", text="Strength LoRA") 199 | 200 | 201 | class OBJECT_PT_IPAdapterPanel(bpy.types.Panel): 202 | bl_label = "IPAdapter" 203 | bl_idname = "OBJECT_PT_ipadapter_panel" 204 | bl_space_type = "VIEW_3D" 205 | bl_region_type = "UI" 206 | bl_category = "DiffusedTexture" 207 | bl_options = {"DEFAULT_CLOSED"} 208 | bl_order = 2 209 | 210 | def draw(self, context): 211 | layout = self.layout 212 | scene = context.scene 213 | 214 | # IPAdapter Activation Checkbox 215 | layout.prop(scene, "use_ipadapter", text="Activate IPAdapter") 216 | 217 | # IPAdapter Image Preview and Selection 218 | row = layout.row() 219 | 220 | # disable the IPAdapter image selection if the IPAdapter is not activated 221 | row.enabled = scene.use_ipadapter 222 | row.template_ID_preview(scene, "ipadapter_image", rows=2, cols=6) 223 | 224 | row = layout.row() 225 | row.enabled = scene.use_ipadapter 226 | # Button to open the file browser and load a new image 227 | row.operator( 228 | "image.open_new_ipadapter_image", 229 | text="Open New IPAdapter Image", 230 | icon="IMAGE_DATA", 231 | ) 232 | 233 | # IPAdapter Strength Slider 234 | row = layout.row() 235 | row.enabled = scene.use_ipadapter 236 | row.prop(scene, "ipadapter_strength", text="Strength IPAdapter") 237 | 238 | 239 | class OBJECT_OT_OpenNewIPAdapterImage(bpy.types.Operator): 240 | """Operator to open a new image for IPAdapter""" 241 | 242 | bl_idname = "image.open_new_ipadapter_image" 243 | bl_label = "Open New IPAdapter Image" 244 | bl_order = 3 245 | 246 | filepath: bpy.props.StringProperty(subtype="FILE_PATH") 247 | 248 | def execute(self, context): 249 | # Load the new image using the provided filepath 250 | image = bpy.data.images.load(self.filepath) 251 | context.scene.ipadapter_image = image 252 | return {"FINISHED"} 253 | 254 | def invoke(self, context, event): 255 | context.window_manager.fileselect_add(self) 256 | return {"RUNNING_MODAL"} 257 | 258 | 259 | class OBJECT_PT_AdvancedPanel(bpy.types.Panel): 260 | """Advanced Settings Panel""" 261 | 262 | bl_label = "Advanced Settings" 263 | bl_idname = "OBJECT_PT_advanced_panel" 264 | bl_space_type = "VIEW_3D" 265 | bl_region_type = "UI" 266 | bl_category = "DiffusedTexture" 267 | bl_options = {"DEFAULT_CLOSED"} 268 | bl_order = 1 269 | 270 | def draw(self, context): 271 | layout = self.layout 272 | scene = context.scene 273 | 274 | box = layout.box() 275 | 276 | # dropdown menu for the sd model (sd15 or sdxl so far) 277 | box.prop(scene, "sd_version", text="Stable Diffusion Version:") 278 | 279 | # custom SD checkpoints 280 | box.prop(scene, "checkpoint_path", text="Checkpoint") 281 | 282 | # custom SD resolution 283 | box.prop(scene, "custom_sd_resolution", text="Custom SD Resolution") 284 | 285 | # warning for the user to not go too high with the resolution, especially for the parallel operation, 286 | # since the resolution will be multiplied by sqrt(num of cameras) 287 | if scene.custom_sd_resolution: 288 | box.label( 289 | text="Warning: High resolutions can lead to memory issues and long processing times.", 290 | icon="ERROR", 291 | ) 292 | box.label( 293 | text="The resolution will be multiplied by the square root of the number of cameras.", 294 | ) 295 | box.label( 296 | text="It is recommended to keep the resolution below 1024.", 297 | ) 298 | 299 | # TODO If the user selected SDXL, offer a dropdown menu for ControlNets, to switch between the default (multiple paths) and the Controlnet Union (single path) 300 | if scene.sd_version == "sdxl": 301 | box.label(text="ControlNet Mode:") 302 | box.prop(scene, "controlnet_type", text="") 303 | 304 | # If ControlNet Union is selected, show tick-boxes instead of paths 305 | if scene.controlnet_type == "UNION": 306 | box.label(text="ControlNet Union Inputs:") 307 | box.prop(scene, "controlnet_union_path", text="ControlNet Union Path") 308 | 309 | box.label(text="ControlNet Strength:") 310 | box.prop( 311 | scene, "union_controlnet_strength", text="Union Control Strength" 312 | ) 313 | 314 | # If Multiple ControlNets are selected, show the paths and strengths 315 | else: 316 | box.label(text="ControlNet Checkpoints:") 317 | box.label(text="Change the `Mesh Complexity` to enable more options.") 318 | box.prop(scene, "depth_controlnet_path", text="Depth Path") 319 | 320 | # Disable the canny_controlnet_path option if the mesh complexity is set to LOW 321 | row = box.row() 322 | row.enabled = scene.mesh_complexity != "LOW" 323 | row.prop(scene, "canny_controlnet_path", text="Canny Path") 324 | 325 | # Disable the normal_controlnet_path option if the mesh complexity is set to LOW or MID 326 | row = box.row() 327 | row.enabled = scene.mesh_complexity == "HIGH" 328 | row.prop(scene, "normal_controlnet_path", text="Normal Path") 329 | 330 | box.label(text="ControlNet Strengths:") 331 | box.prop(scene, "depth_controlnet_strength", text="Depth Strength") 332 | 333 | # Disable the canny_controlnet_strength option if the mesh complexity is set to LOW 334 | row = box.row() 335 | row.enabled = scene.mesh_complexity != "LOW" 336 | row.prop(scene, "canny_controlnet_strength", text="Canny Strength") 337 | 338 | # Disable the normal_controlnet_strength option if the mesh complexity is set to LOW or MID 339 | row = box.row() 340 | row.enabled = scene.mesh_complexity == "HIGH" 341 | row.prop(scene, "normal_controlnet_strength", text="Normal Strength") 342 | 343 | else: 344 | # Add advanced settings 345 | box.label(text="ControlNet Checkpoints:") 346 | box.prop(scene, "depth_controlnet_path", text="Depth Path") 347 | 348 | # disable the canny_controlnet_path option if the mesh complexity is set to LOW 349 | row = box.row() 350 | row.enabled = scene.mesh_complexity != "LOW" 351 | row.prop(scene, "canny_controlnet_path", text="Canny Path") 352 | 353 | # disable the normal_controlnet_path option if the mesh complexity is set to LOW or MID 354 | row = box.row() 355 | row.enabled = scene.mesh_complexity == "HIGH" 356 | row.prop(scene, "normal_controlnet_path", text="Normal Path") 357 | 358 | box.label(text="ControlNet Strengths:") 359 | box.prop(scene, "depth_controlnet_strength", text="Depth Strength") 360 | 361 | # disable the canny_controlnet_strength option if the mesh complexity is set to LOW 362 | row = box.row() 363 | row.enabled = scene.mesh_complexity != "LOW" 364 | row.prop(scene, "canny_controlnet_strength", text="Canny Strength") 365 | 366 | # disable the normal_controlnet_strength option if the mesh complexity is set to LOW or MID 367 | row = box.row() 368 | row.enabled = scene.mesh_complexity == "HIGH" 369 | row.prop(scene, "normal_controlnet_strength", text="Normal Strength") 370 | -------------------------------------------------------------------------------- /properties.py: -------------------------------------------------------------------------------- 1 | import os 2 | import bpy 3 | from bpy.props import ( 4 | StringProperty, 5 | FloatProperty, 6 | BoolProperty, 7 | IntProperty, 8 | CollectionProperty, 9 | EnumProperty, 10 | ) 11 | from .utils import update_uv_maps, get_mesh_objects 12 | 13 | 14 | class LoRAModel(bpy.types.PropertyGroup): 15 | path: StringProperty( 16 | name="LoRA Path", description="Path to the LoRA model file", subtype="FILE_PATH" 17 | ) 18 | strength: FloatProperty( 19 | name="Strength LoRA", 20 | description="Strength of the LoRA model", 21 | default=1.0, 22 | min=0.0, 23 | max=2.0, 24 | ) 25 | 26 | 27 | def update_paths(self, context): 28 | if context.scene.sd_version == "sd15": 29 | context.scene.checkpoint_path = "runwayml/stable-diffusion-v1-5" 30 | context.scene.canny_controlnet_path = "lllyasviel/sd-controlnet-canny" 31 | context.scene.normal_controlnet_path = "lllyasviel/sd-controlnet-normal" 32 | context.scene.depth_controlnet_path = "lllyasviel/sd-controlnet-depth" 33 | elif context.scene.sd_version == "sdxl": 34 | context.scene.checkpoint_path = "stabilityai/stable-diffusion-xl-base-1.0" 35 | 36 | # Set the default controlnet paths for Stable Diffusion XL 37 | context.scene.canny_controlnet_path = "diffusers/controlnet-canny-sdxl-1.0" 38 | context.scene.normal_controlnet_path = ( 39 | "xinsir/controlnet-union-sdxl-1.0" # TODO: Change this to a normal only CN 40 | ) 41 | context.scene.depth_controlnet_path = "diffusers/controlnet-depth-sdxl-1.0" 42 | context.scene.controlnet_union_path = "xinsir/controlnet-union-sdxl-1.0" 43 | 44 | 45 | def update_loras(self, context): 46 | scene = context.scene 47 | num_loras = scene.num_loras 48 | lora_models = scene.lora_models 49 | 50 | while len(lora_models) < num_loras: 51 | lora_models.add() 52 | 53 | while len(lora_models) > num_loras: 54 | lora_models.remove(len(lora_models) - 1) 55 | 56 | 57 | def update_ipadapter_image(self, context): 58 | """Ensure the selected image from the preview window is set in scene.ipadapter_image.""" 59 | image = context.scene.ipadapter_image 60 | if image: 61 | image_data = bpy.data.images.get(image.name) 62 | 63 | # Only set the image if it's not already correctly set to prevent recursion 64 | if image_data != context.scene.ipadapter_image: 65 | context.scene.ipadapter_image = image_data 66 | 67 | 68 | def update_input_image(self, context): 69 | """Ensure the selected image from the preview window is set in scene.input_image.""" 70 | image = context.scene.input_texture_path 71 | if image: 72 | image_data = bpy.data.images.get(image.name) 73 | 74 | # Only set the image if it's not already correctly set to prevent recursion 75 | if image_data != context.scene.input_texture_path: 76 | context.scene.input_texture_path = image_data 77 | 78 | 79 | def update_output_path(self, context): 80 | if self.output_path.startswith("//"): 81 | self.output_path = bpy.path.abspath(self.output_path) 82 | 83 | 84 | def register_properties(): 85 | 86 | try: 87 | bpy.utils.register_class(LoRAModel) 88 | except Exception as e: 89 | print( 90 | f"Warning: {LoRAModel.__name__} was not registered or failed to register. {e}" 91 | ) 92 | 93 | bpy.types.Scene.my_mesh_object = EnumProperty( 94 | name="Mesh Object", 95 | items=get_mesh_objects, 96 | description="Select the mesh object you want to use texturize.", 97 | ) 98 | 99 | bpy.types.Scene.my_uv_map = EnumProperty( 100 | name="UV Map", 101 | items=update_uv_maps, 102 | description="Select the UV map you want to use for the final texture.", 103 | ) 104 | 105 | bpy.types.Scene.my_prompt = StringProperty( 106 | name="Prompt", description="Define what the object should be" 107 | ) 108 | 109 | bpy.types.Scene.my_negative_prompt = StringProperty( 110 | name="Negative Prompt", description="Define what the object should NOT be" 111 | ) 112 | 113 | bpy.types.Scene.guidance_scale = FloatProperty( 114 | name="Guidance Scale", 115 | description="A higher guidance scale value encourages the model to generate images closely linked to the text `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`.", 116 | default=10.0, 117 | min=0.0, 118 | max=30.0, # Ensure this is a float value for finer control 119 | ) 120 | 121 | bpy.types.Scene.operation_mode = EnumProperty( 122 | name="Operation Mode", 123 | description="The complexity, polycount and detail of the selected mesh.", 124 | items=[ 125 | ( 126 | "PARALLEL_IMG", 127 | "Parallel Processing on Images", 128 | "Generate textures by merging images in parallel.", 129 | ), 130 | ( 131 | "SEQUENTIAL_IMG", 132 | "Sequential Processing on Images", 133 | "Generate textures by merging images sequentially.", 134 | ), 135 | # ( 136 | # "PARALLEL_LATENT", 137 | # "Parallel Processing on Latents", 138 | # "Generate textures by merging latents in parallel.", 139 | # ), 140 | # ( 141 | # "SEQUENTIAL_LATENT", 142 | # "Sequential Processing on Latents", 143 | # "Generate textures by repeatedly merging latents sequentially.", 144 | # ) 145 | # # ( 146 | # # "TEXTURE2TEXTURE_ENHANCEMENT", 147 | # # "Texture2Texture Enhancement", 148 | # # "Enhance textures using input textures.", 149 | # # ), 150 | ], 151 | # update=update_operation_mode, # Add the update function 152 | ) 153 | 154 | bpy.types.Scene.num_inference_steps = IntProperty( 155 | name="Number of Inference Steps", 156 | description="Number of inference steps to run the model for", 157 | default=50, 158 | min=1, 159 | ) 160 | 161 | bpy.types.Scene.denoise_strength = FloatProperty( 162 | name="Denoise Strength", 163 | description="Strength of denoise for Stable Diffusion", 164 | default=1.0, 165 | min=0.0, 166 | max=1.0, # Ensure this is a float value for finer control 167 | ) 168 | 169 | bpy.types.Scene.texture_resolution = EnumProperty( 170 | name="Texture Resolution", 171 | description="The final texture resolution of the selected mesh object.", 172 | items=[ 173 | ("512", "512x512", ""), 174 | ("1024", "1024x1024", ""), 175 | ("2048", "2048x2048", ""), 176 | ("4096", "4096x4096", ""), 177 | ], 178 | default="1024", 179 | ) 180 | 181 | bpy.types.Scene.render_resolution = EnumProperty( 182 | name="Render Resolution", 183 | description="The Render resolution used in Stable Diffusion.", 184 | items=[ 185 | ("1024", "1024x1024", ""), 186 | ("2048", "2048x2048", ""), 187 | ("4096", "4096x4096", ""), 188 | ("8192", "8192x8192", ""), 189 | ], 190 | default="2048", 191 | ) 192 | 193 | bpy.types.Scene.output_path = StringProperty( 194 | name="Output Path", 195 | description="Directory to store the resulting texture and temporary files", 196 | subtype="DIR_PATH", 197 | default="", 198 | update=update_output_path, 199 | ) 200 | 201 | bpy.types.Scene.input_texture_path = bpy.props.PointerProperty( 202 | type=bpy.types.Image, 203 | name="Input Texture", 204 | description="Select an image to use as input texture", 205 | update=update_input_image, # Attach the update callback 206 | ) 207 | 208 | bpy.types.Scene.mesh_complexity = EnumProperty( 209 | name="Mesh Complexity", 210 | description="How complex is the mesh.", 211 | items=[("LOW", "Low", ""), ("MEDIUM", "Medium", ""), ("HIGH", "High", "")], 212 | ) 213 | 214 | bpy.types.Scene.num_cameras = EnumProperty( 215 | name="Cameras", 216 | description="Number of camera viewpoints. 4 Cameras for a quick process, 16 for more details.", 217 | items=[ 218 | ("4", "4 Camera Viewpoints", ""), 219 | ("9", "9 Camera Viewpoints", ""), 220 | ("16", "16 Camera Viewpoints", ""), 221 | ], 222 | ) 223 | 224 | bpy.types.Scene.texture_seed = IntProperty( 225 | name="Seed", 226 | description="Seed for randomization to ensure repeatable results", 227 | default=0, 228 | min=0, 229 | ) 230 | 231 | # bpy.types.Scene.checkpoint_path = StringProperty( 232 | # name="Checkpoint Path", 233 | # description="Optional path to the Stable Diffusion base model checkpoint", 234 | # subtype="FILE_PATH", 235 | # ) 236 | 237 | bpy.types.Scene.num_loras = IntProperty( 238 | name="Number of LoRAs", 239 | description="Number of additional LoRA models to use", 240 | default=0, 241 | min=0, 242 | update=update_loras, 243 | ) 244 | 245 | bpy.types.Scene.lora_models = CollectionProperty(type=LoRAModel) 246 | 247 | # IPAdapter-specific properties 248 | bpy.types.Scene.use_ipadapter = bpy.props.BoolProperty( 249 | name="Use IPAdapter", 250 | description="Activate IPAdapter for texture generation", 251 | default=False, 252 | ) 253 | 254 | bpy.types.Scene.ipadapter_image = bpy.props.PointerProperty( 255 | type=bpy.types.Image, 256 | name="IPAdapter Image", 257 | description="Select an image to use for IPAdapter", 258 | update=update_ipadapter_image, # Attach the update callback 259 | ) 260 | 261 | bpy.types.Scene.ipadapter_strength = bpy.props.FloatProperty( 262 | name="IPAdapter Strength", 263 | description="This method controls the amount of text or image conditioning to apply to the model. A value of 1.0 means the model is only conditioned on the image prompt. Lowering this value encourages the model to produce more diverse images, but they may not be as aligned with the image prompt. Typically, a value of 0.5 achieves a good balance between the two prompt types and produces good results.", 264 | default=0.5, 265 | min=0.0, 266 | soft_max=1.0, 267 | ) 268 | 269 | # Advanced settings 270 | bpy.types.Scene.sd_version = EnumProperty( 271 | name="Stable Diffusion Version", 272 | description="Select the version of Stable Diffusion to use", 273 | items=[ 274 | ("sd15", "Stable Diffusion 1.5", "Use Stable Diffusion 1.5 models"), 275 | ("sdxl", "Stable Diffusion XL", "Use Stable Diffusion XL models"), 276 | ], 277 | default="sd15", 278 | update=update_paths, 279 | ) 280 | 281 | bpy.types.Scene.checkpoint_path = StringProperty( 282 | name="Checkpoint Path", 283 | description="Optional path to the Stable Diffusion base model checkpoint", 284 | subtype="FILE_PATH", 285 | default="runwayml/stable-diffusion-v1-5", 286 | ) 287 | 288 | bpy.types.Scene.custom_sd_resolution = IntProperty( 289 | name="Custom SD Resolution", 290 | description="Custom resolution for Stable Diffusion", 291 | default=0, 292 | min=0, 293 | ) 294 | 295 | # Add a dropdown menu to switch between ControlNet Union or multiple ControlNets (only available if SDXL is used) 296 | bpy.types.Scene.controlnet_type = EnumProperty( 297 | name="ControlNet Type", 298 | description="Select the type of ControlNet to use", 299 | items=[ 300 | ("MULTIPLE", "Multiple ControlNets", "Use multiple normal ControlNets"), 301 | ("UNION", "ControlNet Union", "Use the ControlNet Union model"), 302 | ], 303 | default="MULTIPLE", 304 | ) 305 | 306 | # Boolean properties for enabling/disabling ControlNet Union inputs 307 | bpy.types.Scene.controlnet_union_path = StringProperty( 308 | name="ControlNet Union Path", 309 | description="Optional path to the ControlNet Union checkpoint", 310 | subtype="FILE_PATH", 311 | default="xinsir/controlnet-union-sdxl-1.0", 312 | ) 313 | bpy.types.Scene.canny_controlnet_path = StringProperty( 314 | name="Canny ControlNet Path", 315 | description="Optional path to the Canny ControlNet checkpoint", 316 | subtype="FILE_PATH", 317 | default="lllyasviel/sd-controlnet-canny", 318 | ) 319 | 320 | bpy.types.Scene.normal_controlnet_path = StringProperty( 321 | name="Normal ControlNet Path", 322 | description="Optional path to the Normal ControlNet checkpoint", 323 | subtype="FILE_PATH", 324 | default="lllyasviel/sd-controlnet-normal", 325 | ) 326 | 327 | bpy.types.Scene.depth_controlnet_path = StringProperty( 328 | name="Depth ControlNet Path", 329 | description="Optional path to the Depth ControlNet checkpoint", 330 | subtype="FILE_PATH", 331 | default="lllyasviel/sd-controlnet-depth", 332 | ) 333 | 334 | bpy.types.Scene.canny_controlnet_strength = FloatProperty( 335 | name="Canny ControlNet Strength", 336 | description="Strength of the Canny ControlNet", 337 | default=0.9, 338 | min=0.0, 339 | max=1.0, 340 | ) 341 | 342 | bpy.types.Scene.normal_controlnet_strength = FloatProperty( 343 | name="Normal ControlNet Strength", 344 | description="Strength of the Normal ControlNet", 345 | default=0.9, 346 | min=0.0, 347 | max=1.0, 348 | ) 349 | 350 | bpy.types.Scene.depth_controlnet_strength = FloatProperty( 351 | name="Depth ControlNet Strength", 352 | description="Strength of the Depth ControlNet", 353 | default=1.0, 354 | min=0.0, 355 | max=1.0, 356 | ) 357 | 358 | bpy.types.Scene.union_controlnet_strength = FloatProperty( 359 | name="Union ControlNet Strength", 360 | description="Strength of ControlNet Union", 361 | default=1.0, 362 | min=0.0, 363 | max=1.0, 364 | ) 365 | 366 | 367 | def unregister_properties(): 368 | 369 | bpy.utils.unregister_class(LoRAModel) 370 | del bpy.types.Scene.num_loras 371 | del bpy.types.Scene.lora_models 372 | del bpy.types.Scene.use_ipadapter 373 | del bpy.types.Scene.ipadapter_image 374 | del bpy.types.Scene.ipadapter_strength 375 | del bpy.types.Scene.my_mesh_object 376 | del bpy.types.Scene.my_uv_map 377 | del bpy.types.Scene.my_prompt 378 | del bpy.types.Scene.my_negative_prompt 379 | del bpy.types.Scene.guidance_scale 380 | del bpy.types.Scene.mesh_complexity 381 | del bpy.types.Scene.texture_resolution 382 | del bpy.types.Scene.render_resolution 383 | del bpy.types.Scene.operation_mode 384 | del bpy.types.Scene.denoise_strength 385 | del bpy.types.Scene.output_path 386 | del bpy.types.Scene.texture_seed 387 | del bpy.types.Scene.num_inference_steps 388 | del bpy.types.Scene.num_cameras 389 | del bpy.types.Scene.input_texture_path 390 | del bpy.types.Scene.sd_version 391 | del bpy.types.Scene.checkpoint_path 392 | del bpy.types.Scene.canny_controlnet_path 393 | del bpy.types.Scene.normal_controlnet_path 394 | del bpy.types.Scene.depth_controlnet_path 395 | del bpy.types.Scene.canny_controlnet_strength 396 | del bpy.types.Scene.normal_controlnet_strength 397 | del bpy.types.Scene.depth_controlnet_strength 398 | del bpy.types.Scene.union_controlnet_strength 399 | del bpy.types.Scene.custom_sd_resolution 400 | del bpy.types.Scene.controlnet_type 401 | -------------------------------------------------------------------------------- /render_setup.py: -------------------------------------------------------------------------------- 1 | import bpy 2 | import math 3 | import mathutils 4 | 5 | 6 | def create_cameras_on_one_ring( 7 | num_cameras=16, max_size=1, name_prefix="Camera", fov=22.5 8 | ): 9 | """ 10 | Create n cameras evenly distributed on a ring around the world origin (Z-axis). 11 | 12 | :param max_size: The maximum size of the object to be captured by the cameras. 13 | :param name_prefix: Prefix for naming the cameras. 14 | :param fov: Field of view for the cameras in degrees. 15 | :return: List of created camera objects. 16 | """ 17 | cameras = [] 18 | 19 | # Convert FOV from degrees to radians 20 | fov_rad = math.radians(fov) 21 | 22 | # Calculate the distance from the origin such that the FOV covers max_size 23 | radius = (max_size * 0.5) / math.tan(fov_rad * 0.5) 24 | 25 | angle_offset = math.pi / num_cameras # Offset for the ring cameras 26 | 27 | # Set the vertical offset for the rings (small elevation above/below the object) 28 | elevation = radius * 0.25 # Adjust this value to control the elevation 29 | 30 | # Loop to create the ring (around Z-axis), offset by half an angle 31 | for i in range(num_cameras): 32 | theta = (2 * math.pi / num_cameras) * i + angle_offset 33 | 34 | # Position for the upper ring (XZ-plane) 35 | x = radius * math.cos(theta) 36 | y = radius * math.sin(theta) 37 | location = mathutils.Vector((x, y, elevation)) 38 | 39 | # Create upper ring camera 40 | bpy.ops.object.camera_add(location=location) 41 | camera = bpy.context.object 42 | camera.name = f"{name_prefix}_{i+1}" 43 | 44 | # Set the camera's FOV 45 | camera.data.lens_unit = "FOV" 46 | camera.data.angle = fov_rad 47 | 48 | # Point the camera at the origin 49 | direction_upper = camera.location - mathutils.Vector((0, 0, 0)) 50 | rot_quat_upper = direction_upper.to_track_quat("Z", "Y") 51 | camera.rotation_euler = rot_quat_upper.to_euler() 52 | 53 | cameras.append(camera) 54 | 55 | return cameras 56 | 57 | 58 | def create_cameras_on_two_rings( 59 | num_cameras=16, max_size=1, name_prefix="Camera", fov=22.5 60 | ): 61 | """ 62 | Create 16 cameras evenly distributed on two rings around the world origin (Z-axis). 63 | Each ring will have 8 cameras, with the upper and lower rings' cameras placed in the gaps of each other. 64 | 65 | :param max_size: The maximum size of the object to be captured by the cameras. 66 | :param name_prefix: Prefix for naming the cameras. 67 | :param fov: Field of view for the cameras in degrees. 68 | :return: List of created camera objects. 69 | """ 70 | cameras = [] 71 | 72 | # Convert FOV from degrees to radians 73 | fov_rad = math.radians(fov) 74 | 75 | # Calculate the distance from the origin such that the FOV covers max_size 76 | radius = (max_size * 0.5) / math.tan(fov_rad * 0.5) 77 | 78 | num_cameras_per_ring = num_cameras // 2 79 | angle_offset = math.pi / num_cameras_per_ring # Offset for the upper ring cameras 80 | 81 | # Set the vertical offset for the rings (small elevation above/below the object) 82 | elevation_upper = radius * 0.5 # Adjust this value to control the elevation 83 | elevation_lower = -radius * 0.5 84 | 85 | # Loop to create the lower ring (around Z-axis) 86 | for i in range(num_cameras_per_ring): 87 | theta = (2 * math.pi / num_cameras_per_ring) * i 88 | 89 | # Position for the lower ring (XZ-plane) 90 | x = radius * math.cos(theta) 91 | y = radius * math.sin(theta) 92 | location_lower = mathutils.Vector((x, y, elevation_lower)) 93 | 94 | # Create lower ring camera 95 | bpy.ops.object.camera_add(location=location_lower) 96 | camera_lower = bpy.context.object 97 | camera_lower.name = f"{name_prefix}_LowerRing_{i+1}" 98 | 99 | # Set the camera's FOV 100 | camera_lower.data.lens_unit = "FOV" 101 | camera_lower.data.angle = fov_rad 102 | 103 | # Point the camera at the origin 104 | direction_lower = camera_lower.location - mathutils.Vector((0, 0, 0)) 105 | rot_quat_lower = direction_lower.to_track_quat("Z", "Y") 106 | camera_lower.rotation_euler = rot_quat_lower.to_euler() 107 | 108 | cameras.append(camera_lower) 109 | 110 | # Loop to create the upper ring (around Z-axis), offset by half an angle 111 | for i in range(num_cameras_per_ring): 112 | theta = (2 * math.pi / num_cameras_per_ring) * i + angle_offset 113 | 114 | # Position for the upper ring (XZ-plane) 115 | x = radius * math.cos(theta) 116 | y = radius * math.sin(theta) 117 | location_upper = mathutils.Vector((x, y, elevation_upper)) 118 | 119 | # Create upper ring camera 120 | bpy.ops.object.camera_add(location=location_upper) 121 | camera_upper = bpy.context.object 122 | camera_upper.name = f"{name_prefix}_UpperRing_{i+1}" 123 | 124 | # Set the camera's FOV 125 | camera_upper.data.lens_unit = "FOV" 126 | camera_upper.data.angle = fov_rad 127 | 128 | # Point the camera at the origin 129 | direction_upper = camera_upper.location - mathutils.Vector((0, 0, 0)) 130 | rot_quat_upper = direction_upper.to_track_quat("Z", "Y") 131 | camera_upper.rotation_euler = rot_quat_upper.to_euler() 132 | 133 | cameras.append(camera_upper) 134 | 135 | return cameras 136 | 137 | 138 | def create_cameras_on_sphere( 139 | num_cameras=16, max_size=1, name_prefix="Camera", fov=22.5 140 | ): 141 | """ 142 | Create cameras evenly distributed on a sphere around the world origin, 143 | with each camera positioned such that it perfectly frames an object of size 144 | max_size using a field of view of fov. 145 | 146 | :param num_cameras: Number of cameras to create. 147 | :param max_size: The maximum size of the object to be captured by the cameras. 148 | :param name_prefix: Prefix for naming the cameras. 149 | :param offset: Offset to change views slightly. 150 | :param fov: Field of view for the cameras in degrees. 151 | :return: List of created camera objects. 152 | """ 153 | 154 | cameras = [] 155 | phi = math.pi * (3.0 - math.sqrt(5.0)) # Golden angle in radians 156 | 157 | # Convert FOV from degrees to radians 158 | fov_rad = math.radians(fov) 159 | 160 | # Calculate the distance from the origin such that the FOV covers max_size 161 | radius = (max_size * 0.5) / math.tan(fov_rad * 0.5) 162 | 163 | for i in range(num_cameras): 164 | y = 1 - (i / float(num_cameras - 1)) * 2 # y goes from 1 to -1 165 | radius_at_y = math.sqrt(1 - y * y) # Radius at y 166 | theta = (phi) * i # Golden angle increment 167 | 168 | x = math.cos(theta) * radius_at_y 169 | z = math.sin(theta) * radius_at_y 170 | location = mathutils.Vector((x, y, z)) * radius 171 | 172 | # if offset: 173 | # # Swap coordinates: x -> y, y -> z, z -> x 174 | # location = mathutils.Vector((location.y, location.z, location.x)) 175 | 176 | # Create camera 177 | bpy.ops.object.camera_add(location=location) 178 | camera = bpy.context.object 179 | camera.name = f"{name_prefix}_{i+1}" 180 | 181 | # Set the camera's FOV 182 | camera.data.lens_unit = "FOV" 183 | camera.data.angle = fov_rad 184 | 185 | # Point the camera at the origin 186 | direction = camera.location - mathutils.Vector((0, 0, 0)) 187 | rot_quat = direction.to_track_quat("Z", "Y") 188 | camera.rotation_euler = rot_quat.to_euler() 189 | 190 | cameras.append(camera) 191 | 192 | return cameras 193 | 194 | 195 | def setup_render_settings(scene, resolution=(512, 512)): 196 | """ 197 | Configure render settings, including enabling specific passes and setting up the node tree. 198 | 199 | :param scene: The scene to configure. 200 | :param resolution: Tuple specifying the render resolution (width, height). 201 | :return: A dictionary containing references to the output nodes for each pass. 202 | """ 203 | 204 | # Enable Cycles (Eevee does not offer UV output) 205 | scene.render.engine = "CYCLES" 206 | 207 | # Attempt to enable GPU support with preference order: OPTIX, CUDA, OPENCL, CPU 208 | preferences = bpy.context.preferences.addons["cycles"].preferences 209 | try: 210 | preferences.compute_device_type = "OPTIX" 211 | print("Using OPTIX for rendering.") 212 | except Exception as e_optix: 213 | print(f"OPTIX failed: {e_optix}") 214 | try: 215 | preferences.compute_device_type = "CUDA" 216 | print("Using CUDA for rendering.") 217 | except: 218 | raise SystemError("You need an NVidia GPU for this Addon!") 219 | 220 | # Set rendering samples and noise threshold 221 | scene.cycles.samples = 1 # Reduce to 1 sample for no anti-aliasing in Cycles 222 | scene.cycles.use_denoising = False 223 | scene.cycles.use_light_tree = False 224 | scene.cycles.max_bounces = 1 225 | scene.cycles.diffuse_bounces = 1 226 | scene.cycles.glossy_bounces = 0 227 | scene.cycles.transmission_bounces = 0 228 | scene.cycles.volume_bounces = 0 229 | scene.cycles.transparent_max_bounces = 0 230 | 231 | # Set filter size to minimum (0.01 to disable most filtering) 232 | scene.render.filter_size = 0.01 233 | 234 | # Enable transparent background 235 | scene.render.film_transparent = True 236 | 237 | # Set the render resolution 238 | scene.render.resolution_x, scene.render.resolution_y = resolution 239 | 240 | # put render resolution scale to 100% 241 | scene.render.resolution_percentage = 100 242 | 243 | # Prevent interpolation for the UV, depth, and normal outputs 244 | scene.render.image_settings.file_format = "OPEN_EXR" 245 | scene.render.image_settings.color_depth = "32" # Ensure high precision 246 | 247 | # Ensure the scene uses nodes 248 | scene.use_nodes = True 249 | 250 | # Clear existing nodes 251 | if scene.node_tree: 252 | scene.node_tree.nodes.clear() 253 | 254 | # Create a new node tree 255 | tree = scene.node_tree 256 | links = tree.links 257 | 258 | # Create render layers node 259 | render_layers = tree.nodes.new("CompositorNodeRLayers") 260 | 261 | # Enable necessary passes 262 | scene.view_layers["ViewLayer"].use_pass_z = True 263 | scene.view_layers["ViewLayer"].use_pass_normal = True 264 | scene.view_layers["ViewLayer"].use_pass_uv = True 265 | scene.view_layers["ViewLayer"].use_pass_position = True 266 | 267 | # scene.world.light_settings.use_ambient_occlusion = True # turn AO on 268 | # scene.world.light_settings.ao_factor = 1.0 269 | scene.view_layers["ViewLayer"].use_pass_ambient_occlusion = True # Enable AO pass 270 | 271 | # output path for the render 272 | scene.render.filepath = scene.output_path + "RenderOutput/render_" 273 | 274 | # Create output nodes for each pass 275 | output_nodes = {} 276 | 277 | # Depth pass 278 | depth_output = tree.nodes.new("CompositorNodeOutputFile") 279 | depth_output.label = "Depth Output" 280 | depth_output.name = "DepthOutput" 281 | depth_output.base_path = "" # Set the base path in the calling function if needed 282 | depth_output.file_slots[0].path = "depth_" 283 | links.new(render_layers.outputs["Depth"], depth_output.inputs[0]) 284 | output_nodes["depth"] = depth_output 285 | 286 | # Normal pass 287 | normal_output = tree.nodes.new("CompositorNodeOutputFile") 288 | normal_output.label = "Normal Output" 289 | normal_output.name = "NormalOutput" 290 | normal_output.base_path = "" 291 | normal_output.file_slots[0].path = "normal_" 292 | links.new(render_layers.outputs["Normal"], normal_output.inputs[0]) 293 | output_nodes["normal"] = normal_output 294 | 295 | # UV pass 296 | uv_output = tree.nodes.new("CompositorNodeOutputFile") 297 | uv_output.label = "UV Output" 298 | uv_output.name = "UVOutput" 299 | uv_output.base_path = "" 300 | uv_output.file_slots[0].path = "uv_" 301 | links.new(render_layers.outputs["UV"], uv_output.inputs[0]) 302 | output_nodes["uv"] = uv_output 303 | 304 | # Position pass 305 | position_output = tree.nodes.new("CompositorNodeOutputFile") 306 | position_output.label = "Position Output" 307 | position_output.name = "PositionOutput" 308 | position_output.base_path = "" 309 | position_output.file_slots[0].path = "position_" 310 | links.new(render_layers.outputs["Position"], position_output.inputs[0]) 311 | output_nodes["position"] = position_output 312 | 313 | # Ambient Occlusion pass 314 | img_output = tree.nodes.new("CompositorNodeOutputFile") 315 | img_output.label = "Image Output" 316 | img_output.name = "ImageOutput" 317 | img_output.base_path = "" 318 | img_output.file_slots[0].path = "img_" 319 | links.new(render_layers.outputs["Image"], img_output.inputs[0]) 320 | output_nodes["img"] = img_output 321 | 322 | return output_nodes 323 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | scipy 2 | opencv-python-headless==4.8.1.78 3 | --extra-index-url https://download.pytorch.org/whl/cu118 4 | torch>=2.0 5 | accelerate 6 | diffusers 7 | transformers 8 | peft -------------------------------------------------------------------------------- /scene_backup.py: -------------------------------------------------------------------------------- 1 | # scene_backup.py 2 | import bpy 3 | import copy 4 | 5 | 6 | class SceneBackup: 7 | def __init__(self): 8 | self.original_scene_data = None 9 | 10 | def save_scene_state(self): 11 | """Save the current state of the scene.""" 12 | self.original_scene_data = copy.deepcopy(bpy.context.scene) 13 | 14 | def restore_scene_state(self): 15 | """Restore the scene to its saved state.""" 16 | if not self.original_scene_data: 17 | print("No scene data to restore.") 18 | return 19 | 20 | # Clear current scene objects 21 | for obj in bpy.context.scene.objects: 22 | bpy.data.objects.remove(obj, do_unlink=True) 23 | 24 | # Restore original scene objects 25 | for obj in self.original_scene_data.objects: 26 | bpy.context.scene.collection.objects.link(obj) 27 | 28 | print("Scene restored to its original state.") 29 | 30 | 31 | def clean_scene(scene): 32 | """ 33 | Remove all objects from the scene except the selected mesh object. 34 | """ 35 | selected_object_name = scene.my_mesh_object 36 | selected_object = bpy.data.objects.get(selected_object_name) 37 | 38 | if not selected_object: 39 | raise ValueError(f"Object '{selected_object_name}' not found in the scene.") 40 | 41 | # List of objects to remove 42 | objects_to_remove = [] 43 | 44 | # Collect all objects in the scene and collections except the selected one 45 | for obj in bpy.context.scene.objects: 46 | if obj.name != selected_object_name: 47 | objects_to_remove.append(obj) 48 | 49 | # Also check other collections 50 | for collection in bpy.data.collections: 51 | for obj in collection.objects: 52 | if obj.name != selected_object_name and obj not in objects_to_remove: 53 | objects_to_remove.append(obj) 54 | 55 | # Remove the collected objects 56 | for obj in objects_to_remove: 57 | bpy.data.objects.remove(obj, do_unlink=True) 58 | 59 | print( 60 | f"Scene cleaned: All objects except '{selected_object_name}' have been removed." 61 | ) 62 | 63 | 64 | def clean_object(scene): 65 | """ 66 | Remove all materials from the selected mesh object. 67 | """ 68 | # Get the selected object's name and retrieve the object 69 | selected_object_name = scene.my_mesh_object 70 | selected_object = bpy.data.objects.get(selected_object_name) 71 | selected_object.data.materials.clear() 72 | -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- 1 | import bpy 2 | import numpy as np 3 | 4 | 5 | def update_uv_maps(self, context): 6 | """ 7 | Update the list of available UV maps for the selected mesh object. 8 | This function is used in the panel to populate the UV map dropdown. 9 | """ 10 | obj = bpy.data.objects.get(self.my_mesh_object) 11 | if obj and obj.type == "MESH": 12 | uv_layers = obj.data.uv_layers.keys() 13 | return [(uv, uv, "") for uv in uv_layers] 14 | else: 15 | return [("None", "None", "")] 16 | 17 | 18 | def get_mesh_objects(self, context): 19 | """ 20 | Get all mesh objects in the current Blender scene. 21 | This function is used to populate the mesh object dropdown in the panel. 22 | """ 23 | return [(obj.name, obj.name, "") for obj in bpy.data.objects if obj.type == "MESH"] 24 | 25 | 26 | def update_image_list(self, context): 27 | """ 28 | Update the list of available images in the Blender file. 29 | This function is used to populate the image selection dropdown in the panel. 30 | """ 31 | images = [(img.name, img.name, "") for img in bpy.data.images] 32 | if not images: 33 | images.append(("None", "None", "No images available")) 34 | return images 35 | 36 | 37 | def apply_texture_to_uv_map(obj, uv_map_name, image_name): 38 | """ 39 | Apply the selected image to the specified UV map of the selected object. 40 | This is used when the user selects an image in the panel and it needs to be 41 | applied to the object's UV map. 42 | """ 43 | image = bpy.data.images.get(image_name) 44 | if image: 45 | for uv_layer in obj.data.uv_layers: 46 | if uv_layer.name == uv_map_name: 47 | # Assuming the object has a material and texture slots 48 | material = obj.material_slots[0].material 49 | texture_slot = material.texture_paint_slots[0] 50 | texture_slot.texture.image = image 51 | break 52 | 53 | 54 | def image_to_numpy(image): 55 | """ 56 | Convert a Blender image object to a NumPy array. 57 | 58 | :param image: Blender image object 59 | :return: NumPy array representing the image (height, width, channels) 60 | """ 61 | # Get image dimensions 62 | width, height = image.size 63 | 64 | # Get pixel data (Blender stores pixels in a flat array as RGBA values in [0, 1]) 65 | pixels = np.array(image.pixels[:], dtype=np.float32) 66 | 67 | # Reshape to (height, width, 4) since Blender stores data in RGBA format 68 | pixels = pixels.reshape((height, width, 4)) 69 | 70 | # Discard the alpha channel 71 | pixels = pixels[:, :, :3] # Keep only RGB channels 72 | 73 | # Convert the pixel values from [0, 1] to [0, 255] for typical image use 74 | pixels = (pixels * 255).astype(np.uint8) 75 | 76 | return pixels 77 | --------------------------------------------------------------------------------