├── README.md ├── diffusers_image_outpaint_gradio_jupyter.ipynb └── diffusers_image_outpaint_jupyter.ipynb /README.md: -------------------------------------------------------------------------------- 1 | 🐣 Please follow me for new updates https://twitter.com/camenduru
2 | 🔥 Please join our discord server https://discord.gg/k5BwmmvJJU
3 | 🥳 Please join my patreon community https://patreon.com/camenduru
4 | 5 | ### 🍊 Jupyter Notebook 6 | 7 | | Notebook | Info 8 | | --- | --- | 9 | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/diffusers-image-outpaint-jupyter/blob/main/diffusers_image_outpaint_jupyter.ipynb) | diffusers_image_outpaint_jupyter 10 | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/diffusers-image-outpaint-jupyter/blob/main/diffusers_image_outpaint_gradio_jupyter.ipynb) | diffusers_image_outpaint_gradio_jupyter 11 | 12 | ### 🧬 Code 13 | https://github.com/lllyasviel/ControlNet-v1-1-nightly
14 | https://huggingface.co/spaces/fffiloni/diffusers-image-outpaint/blob/main/app.py
15 | 16 | ### 🖼 Output 17 | ![jmwknt](https://github.com/user-attachments/assets/a1d9b593-a90b-4ea2-87f5-91f018b23d95) 18 | ![oqij2v](https://github.com/user-attachments/assets/6469a926-56a2-491f-9d09-c85465e4f027) 19 | 20 | ### 🏢 Sponsor 21 | https://runpod.io 22 | -------------------------------------------------------------------------------- /diffusers_image_outpaint_gradio_jupyter.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "view-in-github" 7 | }, 8 | "source": [ 9 | "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/diffusers-image-outpaint-jupyter/blob/main/diffusers_image_outpaint_gradio_jupyter.ipynb)" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "id": "VjYy0F2gZIPR" 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "%cd /content\n", 21 | "!git clone -b dev https://github.com/camenduru/diffusers-image-outpaint-hf\n", 22 | "%cd /content/diffusers-image-outpaint-hf\n", 23 | "\n", 24 | "!pip install gradio==4.42.0 gradio-imageslider transformers accelerate diffusers fastapi==0.112.4\n", 25 | "\n", 26 | "!python app.py" 27 | ] 28 | } 29 | ], 30 | "metadata": { 31 | "accelerator": "GPU", 32 | "colab": { 33 | "gpuType": "T4", 34 | "provenance": [] 35 | }, 36 | "kernelspec": { 37 | "display_name": "Python 3", 38 | "name": "python3" 39 | }, 40 | "language_info": { 41 | "name": "python" 42 | } 43 | }, 44 | "nbformat": 4, 45 | "nbformat_minor": 0 46 | } 47 | -------------------------------------------------------------------------------- /diffusers_image_outpaint_jupyter.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "view-in-github" 7 | }, 8 | "source": [ 9 | "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/diffusers-image-outpaint-jupyter/blob/main/diffusers_image_outpaint_jupyter.ipynb)" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "id": "VjYy0F2gZIPR" 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "%cd /content\n", 21 | "!git clone -b dev https://github.com/camenduru/diffusers-image-outpaint-hf\n", 22 | "%cd /content/diffusers-image-outpaint-hf\n", 23 | "!pip install transformers accelerate diffusers\n", 24 | "\n", 25 | "!apt -y install -qq aria2\n", 26 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/outpaint/raw/main/lightning/model_index.json -d /content/model/lightning -o model_index.json\n", 27 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/outpaint/raw/main/lightning/scheduler/scheduler_config.json -d /content/model/lightning/scheduler -o scheduler_config.json\n", 28 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/outpaint/raw/main/lightning/text_encoder/config.json -d /content/model/lightning/text_encoder -o config.json\n", 29 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/outpaint/resolve/main/lightning/text_encoder/model.fp16.safetensors -d /content/model/lightning/text_encoder -o model.fp16.safetensors\n", 30 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/outpaint/raw/main/lightning/text_encoder_2/config.json -d /content/model/lightning/text_encoder_2 -o config.json\n", 31 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/outpaint/resolve/main/lightning/text_encoder_2/model.fp16.safetensors -d /content/model/lightning/text_encoder_2 -o model.fp16.safetensors\n", 32 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/outpaint/raw/main/lightning/tokenizer/merges.txt -d /content/model/lightning/tokenizer -o merges.txt\n", 33 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/outpaint/raw/main/lightning/tokenizer/special_tokens_map.json -d /content/model/lightning/tokenizer -o special_tokens_map.json\n", 34 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/outpaint/raw/main/lightning/tokenizer/tokenizer_config.json -d /content/model/lightning/tokenizer -o tokenizer_config.json\n", 35 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/outpaint/raw/main/lightning/tokenizer/vocab.json -d /content/model/lightning/tokenizer -o vocab.json\n", 36 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/outpaint/raw/main/lightning/tokenizer_2/merges.txt -d /content/model/lightning/tokenizer_2 -o merges.txt\n", 37 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/outpaint/raw/main/lightning/tokenizer_2/special_tokens_map.json -d /content/model/lightning/tokenizer_2 -o special_tokens_map.json\n", 38 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/outpaint/raw/main/lightning/tokenizer_2/tokenizer_config.json -d /content/model/lightning/tokenizer_2 -o tokenizer_config.json\n", 39 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/outpaint/raw/main/lightning/tokenizer_2/vocab.json -d /content/model/lightning/tokenizer_2 -o vocab.json\n", 40 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/outpaint/raw/main/lightning/unet/config.json -d /content/model/lightning/unet -o config.json\n", 41 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/outpaint/resolve/main/lightning/unet/diffusion_pytorch_model.fp16.safetensors -d /content/model/lightning/unet -o diffusion_pytorch_model.fp16.safetensors\n", 42 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/outpaint/raw/main/lightning/unet/diffusion_pytorch_model.safetensors.index.json -d /content/model/lightning/unet -o diffusion_pytorch_model.safetensors.index.json\n", 43 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/outpaint/raw/main/lightning/vae/config.json -d /content/model/lightning/vae -o config.json\n", 44 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/outpaint/resolve/main/lightning/vae/diffusion_pytorch_model.fp16.safetensors -d /content/model/lightning/vae -o diffusion_pytorch_model.fp16.safetensors\n", 45 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/outpaint/raw/main/vae-fix/config.json -d /content/model/vae-fix -o config.json\n", 46 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/outpaint/resolve/main/vae-fix/diffusion_pytorch_model.safetensors -d /content/model/vae-fix -o diffusion_pytorch_model.safetensors\n", 47 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/outpaint/raw/main/union/config_promax.json -d /content/model/union -o config_promax.json\n", 48 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/outpaint/resolve/main/union/diffusion_pytorch_model_promax.safetensors -d /content/model/union -o diffusion_pytorch_model_promax.safetensors\n", 49 | "\n", 50 | "import torch\n", 51 | "from diffusers import AutoencoderKL\n", 52 | "from diffusers.models.model_loading_utils import load_state_dict\n", 53 | "from controlnet_union import ControlNetModel_Union\n", 54 | "from pipeline_fill_sd_xl import StableDiffusionXLFillPipeline\n", 55 | "from PIL import Image, ImageDraw\n", 56 | "\n", 57 | "config = ControlNetModel_Union.load_config(\"/content/model/union/config_promax.json\")\n", 58 | "controlnet_model = ControlNetModel_Union.from_config(config)\n", 59 | "state_dict = load_state_dict(\"/content/model/union/diffusion_pytorch_model_promax.safetensors\")\n", 60 | "model, _, _, _, _ = ControlNetModel_Union._load_pretrained_model(controlnet_model, state_dict, \"/content/model/union/diffusion_pytorch_model_promax.safetensors\", \"/content/model/union\")\n", 61 | "model.to(device=\"cuda\", dtype=torch.float16)\n", 62 | "vae = AutoencoderKL.from_pretrained(\"/content/model/vae-fix\", torch_dtype=torch.float16).to(\"cuda\")\n", 63 | "pipe = StableDiffusionXLFillPipeline.from_pretrained(\"/content/model/lightning\", torch_dtype=torch.float16, vae=vae, controlnet=model, variant=\"fp16\").to(\"cuda\")\n", 64 | "\n", 65 | "def infer(image, width, height, overlap_width, num_inference_steps, prompt_input=None):\n", 66 | " source = image\n", 67 | " target_size = (width, height)\n", 68 | " overlap = overlap_width\n", 69 | "\n", 70 | " if source.width < target_size[0] and source.height < target_size[1]:\n", 71 | " scale_factor = min(target_size[0] / source.width, target_size[1] / source.height)\n", 72 | " new_width = int(source.width * scale_factor)\n", 73 | " new_height = int(source.height * scale_factor)\n", 74 | " source = source.resize((new_width, new_height), Image.LANCZOS)\n", 75 | "\n", 76 | " if source.width > target_size[0] or source.height > target_size[1]:\n", 77 | " scale_factor = min(target_size[0] / source.width, target_size[1] / source.height)\n", 78 | " new_width = int(source.width * scale_factor)\n", 79 | " new_height = int(source.height * scale_factor)\n", 80 | " source = source.resize((new_width, new_height), Image.LANCZOS)\n", 81 | "\n", 82 | " margin_x = (target_size[0] - source.width) // 2\n", 83 | " margin_y = (target_size[1] - source.height) // 2\n", 84 | "\n", 85 | " background = Image.new('RGB', target_size, (255, 255, 255))\n", 86 | " background.paste(source, (margin_x, margin_y))\n", 87 | "\n", 88 | " mask = Image.new('L', target_size, 255)\n", 89 | " mask_draw = ImageDraw.Draw(mask)\n", 90 | " mask_draw.rectangle([\n", 91 | " (margin_x + overlap, margin_y + overlap),\n", 92 | " (margin_x + source.width - overlap, margin_y + source.height - overlap)\n", 93 | " ], fill=0)\n", 94 | "\n", 95 | " cnet_image = background.copy()\n", 96 | " cnet_image.paste(0, (0, 0), mask)\n", 97 | "\n", 98 | " final_prompt = \"high quality\"\n", 99 | " if prompt_input and prompt_input.strip():\n", 100 | " final_prompt += \", \" + prompt_input\n", 101 | "\n", 102 | " (\n", 103 | " prompt_embeds,\n", 104 | " negative_prompt_embeds,\n", 105 | " pooled_prompt_embeds,\n", 106 | " negative_pooled_prompt_embeds,\n", 107 | " ) = pipe.encode_prompt(final_prompt, \"cuda\", True)\n", 108 | "\n", 109 | " results = []\n", 110 | " \n", 111 | " for image in pipe(\n", 112 | " prompt_embeds=prompt_embeds,\n", 113 | " negative_prompt_embeds=negative_prompt_embeds,\n", 114 | " pooled_prompt_embeds=pooled_prompt_embeds,\n", 115 | " negative_pooled_prompt_embeds=negative_pooled_prompt_embeds,\n", 116 | " image=cnet_image,\n", 117 | " num_inference_steps=num_inference_steps\n", 118 | " ):\n", 119 | " results.append((cnet_image, image))\n", 120 | "\n", 121 | " image = image.convert(\"RGBA\")\n", 122 | " cnet_image.paste(image, (0, 0), mask)\n", 123 | "\n", 124 | " results.append((background, cnet_image))\n", 125 | "\n", 126 | " return results" 127 | ] 128 | }, 129 | { 130 | "cell_type": "code", 131 | "execution_count": null, 132 | "metadata": {}, 133 | "outputs": [], 134 | "source": [ 135 | "input_image = Image.open(\"/content/diffusers-image-outpaint-hf/examples/example_3.jpg\")\n", 136 | "width = 1280\n", 137 | "height = 720\n", 138 | "overlap_width = 42\n", 139 | "num_inference_steps = 8\n", 140 | "prompt_input = None\n", 141 | "output_image = infer(input_image, height, width, overlap_width, num_inference_steps, None)\n", 142 | "output_image[num_inference_steps+1][1]" 143 | ] 144 | } 145 | ], 146 | "metadata": { 147 | "accelerator": "GPU", 148 | "colab": { 149 | "gpuType": "T4", 150 | "provenance": [] 151 | }, 152 | "kernelspec": { 153 | "display_name": "Python 3", 154 | "name": "python3" 155 | }, 156 | "language_info": { 157 | "name": "python" 158 | } 159 | }, 160 | "nbformat": 4, 161 | "nbformat_minor": 0 162 | } 163 | --------------------------------------------------------------------------------