├── .github └── FUNDING.yml ├── LICENSE ├── README.md ├── ap's_360_panorama.ipynb ├── ap's_animate_image.ipynb ├── camenduru's_3d_photo_inpainting.ipynb ├── camenduru's_codeformer.ipynb ├── camenduru's_compare.ipynb ├── camenduru's_diffusers_to_sd.ipynb ├── camenduru's_discord_diffusers.ipynb ├── camenduru's_flax_to_pt_converter.ipynb ├── camenduru's_gfpgan.ipynb ├── camenduru's_hand_detect_crop.ipynb ├── camenduru's_html_to_datasets.ipynb ├── camenduru's_instant_ngp.ipynb ├── camenduru's_karlo.ipynb ├── camenduru's_latent_diffusion.ipynb ├── camenduru's_latent_diffusion_latex.ipynb ├── camenduru's_latent_diffusion_one_cell.ipynb ├── camenduru's_latent_upscale.ipynb ├── camenduru's_lora.ipynb ├── camenduru's_peft_finetune_transformers.ipynb ├── camenduru's_peft_finetune_transformers_int8.ipynb ├── camenduru's_replace_vae.ipynb ├── camenduru's_sd_dreambooth_training.ipynb ├── camenduru's_sd_image_variations.ipynb ├── camenduru's_stable_diffusion.ipynb ├── camenduru's_stable_diffusion_csv.ipynb ├── camenduru's_stable_diffusion_discord.ipynb ├── camenduru's_stable_diffusion_flax.ipynb ├── camenduru's_stable_diffusion_tile.ipynb ├── camenduru's_stable_diffusion_ui_gradio.ipynb ├── camenduru's_stable_diffusion_ui_gradio_drive.ipynb ├── camenduru's_stable_diffusion_vanilla.ipynb ├── camenduru's_stable_diffusion_video.ipynb ├── camenduru's_tensor_rt.ipynb ├── camenduru's_tensorrt_diffusers.ipynb ├── camenduru's_text2light.ipynb ├── camenduru's_torrent_to_git.ipynb ├── camenduru's_tortoise.ipynb ├── camenduru's_tpu_diffusers.ipynb ├── camenduru's_unet_traced.ipynb ├── camenduru's_waifu_diffusion.ipynb ├── camenduru's_watermark_remover.ipynb ├── camenduru's_webui_to_diffusers_embeddings.ipynb ├── camenduru's_whisper.ipynb ├── camenduru's_whisper_ja_video.ipynb ├── ddPn08's_tensor-rt.ipynb ├── hysts's_lora.ipynb ├── kjsman's_stable_diffusion_pytorch.ipynb ├── multimodalart's_dreambooth.ipynb └── sam_witteveen's_dolly_lora_gptj6b_alpaca.ipynb /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] 4 | patreon: camenduru 5 | open_collective: # Replace with a single Open Collective username 6 | ko_fi: camenduru 7 | tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel 8 | community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry 9 | liberapay: # Replace with a single Liberapay username 10 | issuehunt: # Replace with a single IssueHunt username 11 | otechie: # Replace with a single Otechie username 12 | lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry 13 | custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] 14 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | This is free and unencumbered software released into the public domain. 2 | 3 | Anyone is free to copy, modify, publish, use, compile, sell, or 4 | distribute this software, either in source code form or as a compiled 5 | binary, for any purpose, commercial or non-commercial, and by any 6 | means. 7 | 8 | In jurisdictions that recognize copyright laws, the author or authors 9 | of this software dedicate any and all copyright interest in the 10 | software to the public domain. We make this dedication for the benefit 11 | of the public at large and to the detriment of our heirs and 12 | successors. We intend this dedication to be an overt act of 13 | relinquishment in perpetuity of all present and future rights to this 14 | software under copyright law. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 19 | IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 | OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 | ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | OTHER DEALINGS IN THE SOFTWARE. 23 | 24 | For more information, please refer to 25 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 🐣 Please follow me for new updates https://twitter.com/camenduru
2 | 🔥 Please join our discord server https://discord.gg/k5BwmmvJJU
3 | 🥳 Please join my patreon community https://patreon.com/camenduru
4 | 5 | # 6 | - [Deep Unsupervised Learning using Nonequilibrium Thermodynamics](https://arxiv.org/abs/1503.03585) 7 | - https://github.com/Sohl-Dickstein/Diffusion-Probabilistic-Models 8 | - [Denoising Diffusion Probabilistic Models](https://arxiv.org/abs/2102.09672) 9 | - https://github.com/hojonathanho/diffusion 10 | - https://github.com/lucidrains/denoising-diffusion-pytorch 11 | - [Denoising Diffusion Implicit Models](https://arxiv.org/abs/2010.02502) 12 | - https://github.com/ermongroup/ddim 13 | - [Improved Denoising Diffusion Probabilistic Models](https://arxiv.org/abs/2102.09672) 14 | - https://github.com/openai/improved-diffusion 15 | - [Diffusion Models Beat GANs on Image Synthesis](https://arxiv.org/abs/2105.05233) 16 | - https://github.com/openai/guided-diffusion 17 | - [High-Resolution Image Synthesis with Latent Diffusion Models](https://arxiv.org/abs/2112.10752) 18 | - https://github.com/CompVis/stable-diffusion 19 | - [Text-Guided Synthesis of Artistic Images with Retrieval-Augmented Diffusion Models](https://arxiv.org/abs/2207.13038) 20 | - https://github.com/compvis/latent-diffusion 21 | # 22 | - [U-Net: Convolutional Networks for Biomedical Image Segmentation](https://arxiv.org/abs/1505.04597) 23 | - https://github.com/milesial/Pytorch-UNet 24 | - [Auto-Encoding Variational Bayes](https://arxiv.org/abs/1312.6114) 25 | - https://github.com/AntixK/PyTorch-VAE 26 | - [Neural Discrete Representation Learning](https://arxiv.org/abs/1711.00937) 27 | - https://github.com/deepmind/sonnet/blob/v2/sonnet/src/nets/vqvae.py 28 | - [Learning Transferable Visual Models From Natural Language Supervision](https://arxiv.org/abs/2103.00020) 29 | - https://github.com/openai/CLIP 30 | - [Score-Based Generative Modeling through Stochastic Differential Equations](https://arxiv.org/abs/2011.13456) 31 | - https://github.com/yang-song/score_sde 32 | # 33 | - [labml.ai Deep Learning Paper Implementations](https://nn.labml.ai/index.html) 34 | - https://github.com/labmlai/annotated_deep_learning_paper_implementations 35 | - https://github.com/acids-ircam/diffusion_models 36 | - https://yang-song.net/blog/2021/score/ 37 | -------------------------------------------------------------------------------- /ap's_360_panorama.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "view-in-github" 7 | }, 8 | "source": [ 9 | "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/notebooks/blob/main/ap's_360_panorama.ipynb)" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": {}, 16 | "outputs": [], 17 | "source": [ 18 | "#step 1\n", 19 | "!pip install diffusers==0.19.1 transformers accelerate streamlit streamlit_pannellum" 20 | ] 21 | }, 22 | { 23 | "cell_type": "code", 24 | "execution_count": null, 25 | "metadata": {}, 26 | "outputs": [], 27 | "source": [ 28 | "#step2\n", 29 | "from diffusers import StableDiffusionLDM3DPipeline\n", 30 | "from PIL import Image\n", 31 | "from typing import Optional\n", 32 | "from torch import Tensor\n", 33 | "from torch.nn import functional as F\n", 34 | "from torch.nn import Conv2d\n", 35 | "from torch.nn.modules.utils import _pair\n", 36 | "\n", 37 | "# Load the pipeline\n", 38 | "pipe = StableDiffusionLDM3DPipeline.from_pretrained(\"Intel/ldm3d-pano\")\n", 39 | "pipe.to(\"cuda\")\n", 40 | "\n", 41 | "# Function to override _conv_forward method\n", 42 | "def asymmetricConv2DConvForward(self, input: Tensor, weight: Tensor, bias: Optional[Tensor]):\n", 43 | " paddingX = (self._reversed_padding_repeated_twice[0], self._reversed_padding_repeated_twice[1], 0, 0)\n", 44 | " paddingY = (0, 0, self._reversed_padding_repeated_twice[2], self._reversed_padding_repeated_twice[3])\n", 45 | " working = F.pad(input, paddingX, mode='circular')\n", 46 | " working = F.pad(working, paddingY, mode='constant')\n", 47 | " return F.conv2d(working, weight, bias, self.stride, _pair(0), self.dilation, self.groups)\n", 48 | "\n", 49 | "# Patch the Conv2d layers\n", 50 | "targets = [pipe.vae, pipe.text_encoder, pipe.unet]\n", 51 | "for target in targets:\n", 52 | " for module in target.modules():\n", 53 | " if isinstance(module, Conv2d):\n", 54 | " module._conv_forward = asymmetricConv2DConvForward.__get__(module, Conv2d)" 55 | ] 56 | }, 57 | { 58 | "cell_type": "code", 59 | "execution_count": null, 60 | "metadata": {}, 61 | "outputs": [], 62 | "source": [ 63 | "# Define the prompt and filename base\n", 64 | "prompt = \"The living room\"\n", 65 | "name = \"jungle_pano\"\n", 66 | "\n", 67 | "# Generate the images\n", 68 | "output = pipe(prompt, width=1024, height=512, guidance_scale=7.0, num_inference_steps=50)\n", 69 | "\n", 70 | "# Extract the RGB and depth images\n", 71 | "rgb_image, depth_image = output.rgb, output.depth\n", 72 | "\n", 73 | "# Save images to files\n", 74 | "rgb_image[0].save(name+\"_ldm3d_rgb.jpg\")\n", 75 | "depth_image[0].save(name+\"_ldd3d_depth.png\")\n", 76 | "\n", 77 | "# Display images\n", 78 | "rgb_image[0].show()\n", 79 | "depth_image[0].show()\n", 80 | "\n", 81 | "#step3\n", 82 | "from PIL import Image\n", 83 | "from IPython.display import display\n", 84 | "\n", 85 | "# File names from the saved images\n", 86 | "rgb_filename = name + \"_ldm3d_rgb.jpg\"\n", 87 | "\n", 88 | "# Open the saved images\n", 89 | "rgb_saved_image = Image.open(rgb_filename)\n", 90 | "\n", 91 | "# Display the images\n", 92 | "display(rgb_saved_image)" 93 | ] 94 | }, 95 | { 96 | "cell_type": "code", 97 | "execution_count": null, 98 | "metadata": {}, 99 | "outputs": [], 100 | "source": [ 101 | "# upscale step 1\n", 102 | "!pip install -q realesrgan==0.3.0\n", 103 | "!wget https://huggingface.co/camenduru/sd-t2i-360panoimage/resolve/main/RealESRGAN_x2plus.pth?download=true -O /content/RealESRGAN_x2plus.pth" 104 | ] 105 | }, 106 | { 107 | "cell_type": "code", 108 | "execution_count": null, 109 | "metadata": {}, 110 | "outputs": [], 111 | "source": [ 112 | "# upscale step 2\n", 113 | "from PIL import Image\n", 114 | "rgb_saved_image = Image.open('/content/jungle_pano_ldm3d_rgb.jpg')\n", 115 | "\n", 116 | "from basicsr.archs.rrdbnet_arch import RRDBNet\n", 117 | "from realesrgan import RealESRGANer\n", 118 | "\n", 119 | "sr_model = RRDBNet(num_in_ch=3, num_out_ch=3, num_feat=64, num_block=23, num_grow_ch=32,scale=2)\n", 120 | "upsampler = RealESRGANer(scale=2, model_path='/content/RealESRGAN_x2plus.pth', dni_weight=None, model=sr_model, tile=384, tile_pad=20, pre_pad=20, half=False, device='cuda',)\n", 121 | "\n", 122 | "import numpy as np\n", 123 | "output_img = rgb_saved_image.resize((1024 * 2, 512 * 2))\n", 124 | "w = output_img.size[0]\n", 125 | "blend_extend = 10\n", 126 | "outscale = 2\n", 127 | "output_img = np.array(output_img)\n", 128 | "output_img = np.concatenate([output_img, output_img[:, :blend_extend, :]], axis=1)\n", 129 | "output_img, _ = upsampler.enhance(output_img, outscale=outscale)\n", 130 | "output_img = Image.fromarray(output_img[:, :w * outscale, :])\n", 131 | "output_img" 132 | ] 133 | } 134 | ], 135 | "metadata": { 136 | "language_info": { 137 | "name": "python" 138 | } 139 | }, 140 | "nbformat": 4, 141 | "nbformat_minor": 2 142 | } 143 | -------------------------------------------------------------------------------- /ap's_animate_image.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "view-in-github" 7 | }, 8 | "source": [ 9 | "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/notebooks/blob/main/ap's_animate_image.ipynb)" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": {}, 16 | "outputs": [], 17 | "source": [ 18 | "!pip install -q diffusers accelerate transformers peft gradio \n", 19 | "!pip install -q https://download.pytorch.org/whl/cu121/xformers-0.0.22.post7-cp310-cp310-manylinux2014_x86_64.whl\n", 20 | "\n", 21 | "import gradio as gr\n", 22 | "import torch\n", 23 | "from diffusers import AnimateDiffPipeline, MotionAdapter, DDIMScheduler\n", 24 | "from diffusers.utils import export_to_gif\n", 25 | "import random\n", 26 | "\n", 27 | "def generate_gif(image, animation_type):\n", 28 | " # Load the motion adapter\n", 29 | " adapter = MotionAdapter.from_pretrained(\"guoyww/animatediff-motion-adapter-v1-5-2\", torch_dtype=torch.float16)\n", 30 | "\n", 31 | " # Load SD 1.5 based finetuned model\n", 32 | " model_id = \"SG161222/Realistic_Vision_V6.0_B1_noVAE\"\n", 33 | " pipe = AnimateDiffPipeline.from_pretrained(model_id, motion_adapter=adapter, torch_dtype=torch.float16)\n", 34 | "\n", 35 | " # Scheduler setup\n", 36 | " scheduler = DDIMScheduler(\n", 37 | " clip_sample=False,\n", 38 | " beta_start=0.00085,\n", 39 | " beta_end=0.012,\n", 40 | " beta_schedule=\"linear\",\n", 41 | " timestep_spacing=\"trailing\",\n", 42 | " steps_offset=1\n", 43 | " )\n", 44 | " pipe.scheduler = scheduler\n", 45 | "\n", 46 | " # Enable memory savings\n", 47 | " pipe.enable_vae_slicing()\n", 48 | " pipe.enable_model_cpu_offload()\n", 49 | "\n", 50 | " # Load ip_adapter\n", 51 | " pipe.load_ip_adapter(\"h94/IP-Adapter\", subfolder=\"models\", weight_name=\"ip-adapter_sd15.bin\")\n", 52 | "\n", 53 | " # Load the selected motion adapter\n", 54 | " pipe.load_lora_weights(f\"guoyww/animatediff-motion-lora-{animation_type}\", adapter_name=animation_type)\n", 55 | "\n", 56 | " # Generate a random seed\n", 57 | " seed = random.randint(0, 2**32 - 1)\n", 58 | " prompt = \"best quality, high quality, trending on artstation\"\n", 59 | "\n", 60 | " # Set adapter weights for the selected adapter\n", 61 | " adapter_weight = [0.75]\n", 62 | "\n", 63 | " pipe.set_adapters([animation_type], adapter_weights=adapter_weight)\n", 64 | "\n", 65 | " # Generate GIF\n", 66 | " output = pipe(\n", 67 | " prompt=prompt,\n", 68 | " num_frames=16,\n", 69 | " guidance_scale=7.5,\n", 70 | " num_inference_steps=30,\n", 71 | " ip_adapter_image=image,\n", 72 | " generator=torch.Generator(\"cpu\").manual_seed(seed),\n", 73 | " )\n", 74 | " frames = output.frames[0]\n", 75 | "\n", 76 | " gif_path = \"output_animation.gif\"\n", 77 | " export_to_gif(frames, gif_path)\n", 78 | " return gif_path\n", 79 | "\n", 80 | "# Gradio interface\n", 81 | "iface = gr.Interface(\n", 82 | " fn=generate_gif,\n", 83 | " inputs=[gr.Image(type=\"pil\"), gr.Radio([\"zoom-out\", \"tilt-up\", \"pan-left\"])],\n", 84 | " outputs=gr.Image(type=\"pil\", label=\"Generated GIF\"),\n", 85 | " title=\"AnimateDiff + IP Adapter Demo\",\n", 86 | " description=\"Upload an image and select an motion module type to generate a GIF!\"\n", 87 | ")\n", 88 | "\n", 89 | "iface.launch(debug=True,share=True,inline=False)" 90 | ] 91 | } 92 | ], 93 | "metadata": { 94 | "language_info": { 95 | "name": "python" 96 | } 97 | }, 98 | "nbformat": 4, 99 | "nbformat_minor": 2 100 | } 101 | -------------------------------------------------------------------------------- /camenduru's_3d_photo_inpainting.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "view-in-github" 7 | }, 8 | "source": [ 9 | "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/notebooks/blob/main/camenduru's_3d_photo_inpainting.ipynb)" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "id": "5o-EIMeaghU0", 17 | "vscode": { 18 | "languageId": "python" 19 | } 20 | }, 21 | "outputs": [], 22 | "source": [ 23 | "!pip install -q torch==1.13.1+cu116 torchvision==0.14.1+cu116 torchaudio==0.13.1 torchtext==0.14.1 torchdata==0.5.1 --extra-index-url https://download.pytorch.org/whl/cu116 -U\n", 24 | "\n", 25 | "from google.colab import drive\n", 26 | "drive.mount('/content/gdrive')\n", 27 | "\n", 28 | "from IPython.display import clear_output, HTML\n", 29 | "from base64 import b64encode\n", 30 | "\n", 31 | "!pip install vispy transforms3d imageio==2.4.1 networkx==2.3 pyyaml==5.4.1\n", 32 | " \n", 33 | "!git lfs install\n", 34 | "\n", 35 | "!git clone https://huggingface.co/camenduru/3dphoto\n", 36 | "!git clone https://github.com/camenduru/3d-photo-inpainting\n", 37 | "%cd /content/3d-photo-inpainting\n", 38 | "\n", 39 | "!git clone https://github.com/compphoto/BoostingMonocularDepth.git\n", 40 | "!mkdir -p /content/3d-photo-inpainting/BoostingMonocularDepth/pix2pix/checkpoints/mergemodel\n", 41 | "!mkdir /content/3d-photo-inpainting/checkpoints\n", 42 | "!mkdir /content/3d-photo-inpainting/image\n", 43 | "!mkdir /content/3d-photo-inpainting/BoostingMonocularDepth/outputs\n", 44 | "\n", 45 | "!mv /content/3dphoto/color-model.pth /content/3d-photo-inpainting/checkpoints\n", 46 | "!mv /content/3dphoto/depth-model.pth /content/3d-photo-inpainting/checkpoints\n", 47 | "!mv /content/3dphoto/edge-model.pth /content/3d-photo-inpainting/checkpoints\n", 48 | "!mv /content/3dphoto/latest_net_G.pth /content/3d-photo-inpainting/BoostingMonocularDepth/pix2pix/checkpoints/mergemodel\n", 49 | "!mv /content/3dphoto/midas-model1.pt /content/3d-photo-inpainting/MiDaS/model.pt\n", 50 | "!mv /content/3dphoto/midas-model2.pt /content/3d-photo-inpainting/BoostingMonocularDepth/midas/model.pt\n", 51 | "\n", 52 | "!rm /content/3d-photo-inpainting/image/moon.jpg\n", 53 | "!rm /content/3d-photo-inpainting/video/moon_circle.mp4\n", 54 | "!rm /content/3d-photo-inpainting/video/moon_dolly-zoom-in.mp4\n", 55 | "!rm /content/3d-photo-inpainting/video/moon_swing.mp4\n", 56 | "!rm /content/3d-photo-inpainting/video/moon_zoom-in.mp4\n", 57 | "!rm /content/3d-photo-inpainting/depth/moon.npy\n", 58 | "!rm /content/3d-photo-inpainting/depth/moon.png\n", 59 | "\n", 60 | "clear_output()" 61 | ] 62 | }, 63 | { 64 | "cell_type": "code", 65 | "execution_count": null, 66 | "metadata": { 67 | "id": "n-Kl6fo1oows", 68 | "vscode": { 69 | "languageId": "python" 70 | } 71 | }, 72 | "outputs": [], 73 | "source": [ 74 | "%cd image\n", 75 | "from google.colab import files\n", 76 | "uploaded = files.upload()\n", 77 | "for fn in uploaded.keys():\n", 78 | " print('User uploaded file \"{name}\" with length {length} bytes'.format(\n", 79 | " name=fn, length=len(uploaded[fn])))\n", 80 | "%cd .." 81 | ] 82 | }, 83 | { 84 | "cell_type": "code", 85 | "execution_count": null, 86 | "metadata": { 87 | "id": "i5-MWEjfBjYx", 88 | "vscode": { 89 | "languageId": "python" 90 | } 91 | }, 92 | "outputs": [], 93 | "source": [ 94 | "!python main.py --config argument.yml" 95 | ] 96 | }, 97 | { 98 | "cell_type": "code", 99 | "execution_count": null, 100 | "metadata": { 101 | "id": "ZAkKO6Cw2iDx", 102 | "vscode": { 103 | "languageId": "python" 104 | } 105 | }, 106 | "outputs": [], 107 | "source": [ 108 | "mp4 = open(f'/content/3d-photo-inpainting/video/VIDEO.mp4','rb').read()\n", 109 | "data_url = \"data:video/mp4;base64,\" + b64encode(mp4).decode()\n", 110 | "HTML(f'')" 111 | ] 112 | }, 113 | { 114 | "cell_type": "code", 115 | "execution_count": 9, 116 | "metadata": { 117 | "id": "uGpcUorKObOc", 118 | "vscode": { 119 | "languageId": "python" 120 | } 121 | }, 122 | "outputs": [], 123 | "source": [ 124 | "!cp -r /content/3d-photo-inpainting/video /content/gdrive/MyDrive/AI/3DPhoto" 125 | ] 126 | } 127 | ], 128 | "metadata": { 129 | "accelerator": "GPU", 130 | "colab": { 131 | "collapsed_sections": [], 132 | "provenance": [] 133 | }, 134 | "kernelspec": { 135 | "display_name": "Python 3", 136 | "name": "python3" 137 | } 138 | }, 139 | "nbformat": 4, 140 | "nbformat_minor": 0 141 | } 142 | -------------------------------------------------------------------------------- /camenduru's_codeformer.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "view-in-github" 7 | }, 8 | "source": [ 9 | "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/notebooks/blob/main/camenduru's_codeformer.ipynb)" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "id": "j47auBtDwHj4" 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "!pip install -q torch==1.13.1+cu116 torchvision==0.14.1+cu116 torchaudio==0.13.1 torchtext==0.14.1 torchdata==0.5.1 --extra-index-url https://download.pytorch.org/whl/cu116 -U\n", 21 | "!git clone https://huggingface.co/spaces/sczhou/CodeFormer\n", 22 | "%cd CodeFormer\n", 23 | "!pip install -qq -r requirements.txt\n", 24 | "!pip install -qq gradio" 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "execution_count": null, 30 | "metadata": { 31 | "id": "QhBs6EKxwiiO" 32 | }, 33 | "outputs": [], 34 | "source": [ 35 | "import sys\n", 36 | "sys.path.append('CodeFormer')\n", 37 | "import os\n", 38 | "import cv2\n", 39 | "import torch\n", 40 | "import torch.nn.functional as F\n", 41 | "import gradio as gr\n", 42 | "\n", 43 | "from torchvision.transforms.functional import normalize\n", 44 | "\n", 45 | "from basicsr.utils import imwrite, img2tensor, tensor2img\n", 46 | "from basicsr.utils.download_util import load_file_from_url\n", 47 | "from facelib.utils.face_restoration_helper import FaceRestoreHelper\n", 48 | "from basicsr.archs.rrdbnet_arch import RRDBNet\n", 49 | "from basicsr.utils.realesrgan_utils import RealESRGANer\n", 50 | "\n", 51 | "from basicsr.utils.registry import ARCH_REGISTRY\n", 52 | "\n", 53 | "\n", 54 | "os.system(\"pip freeze\")\n", 55 | "\n", 56 | "pretrain_model_url = {\n", 57 | " 'codeformer': 'https://github.com/sczhou/CodeFormer/releases/download/v0.1.0/codeformer.pth',\n", 58 | " 'detection': 'https://github.com/sczhou/CodeFormer/releases/download/v0.1.0/detection_Resnet50_Final.pth',\n", 59 | " 'parsing': 'https://github.com/sczhou/CodeFormer/releases/download/v0.1.0/parsing_parsenet.pth',\n", 60 | " 'realesrgan': 'https://github.com/sczhou/CodeFormer/releases/download/v0.1.0/RealESRGAN_x2plus.pth'\n", 61 | "}\n", 62 | "# download weights\n", 63 | "if not os.path.exists('CodeFormer/weights/CodeFormer/codeformer.pth'):\n", 64 | " load_file_from_url(url=pretrain_model_url['codeformer'], model_dir='CodeFormer/weights/CodeFormer', progress=True, file_name=None)\n", 65 | "if not os.path.exists('CodeFormer/weights/facelib/detection_Resnet50_Final.pth'):\n", 66 | " load_file_from_url(url=pretrain_model_url['detection'], model_dir='CodeFormer/weights/facelib', progress=True, file_name=None)\n", 67 | "if not os.path.exists('CodeFormer/weights/facelib/parsing_parsenet.pth'):\n", 68 | " load_file_from_url(url=pretrain_model_url['parsing'], model_dir='CodeFormer/weights/facelib', progress=True, file_name=None)\n", 69 | "if not os.path.exists('CodeFormer/weights/realesrgan/RealESRGAN_x2plus.pth'):\n", 70 | " load_file_from_url(url=pretrain_model_url['realesrgan'], model_dir='CodeFormer/weights/realesrgan', progress=True, file_name=None)\n", 71 | "\n", 72 | "# download images\n", 73 | "torch.hub.download_url_to_file(\n", 74 | " 'https://replicate.com/api/models/sczhou/codeformer/files/fa3fe3d1-76b0-4ca8-ac0d-0a925cb0ff54/06.png',\n", 75 | " '01.png')\n", 76 | "torch.hub.download_url_to_file(\n", 77 | " 'https://replicate.com/api/models/sczhou/codeformer/files/a1daba8e-af14-4b00-86a4-69cec9619b53/04.jpg',\n", 78 | " '02.jpg')\n", 79 | "torch.hub.download_url_to_file(\n", 80 | " 'https://replicate.com/api/models/sczhou/codeformer/files/542d64f9-1712-4de7-85f7-3863009a7c3d/03.jpg',\n", 81 | " '03.jpg')\n", 82 | "torch.hub.download_url_to_file(\n", 83 | " 'https://replicate.com/api/models/sczhou/codeformer/files/a11098b0-a18a-4c02-a19a-9a7045d68426/010.jpg',\n", 84 | " '04.jpg')\n", 85 | "torch.hub.download_url_to_file(\n", 86 | " 'https://replicate.com/api/models/sczhou/codeformer/files/7cf19c2c-e0cf-4712-9af8-cf5bdbb8d0ee/012.jpg',\n", 87 | " '05.jpg')\n", 88 | "\n", 89 | "def imread(img_path):\n", 90 | " img = cv2.imread(img_path)\n", 91 | " img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n", 92 | " return img\n", 93 | "\n", 94 | "# set enhancer with RealESRGAN\n", 95 | "def set_realesrgan():\n", 96 | " half = True if torch.cuda.is_available() else False\n", 97 | " model = RRDBNet(\n", 98 | " num_in_ch=3,\n", 99 | " num_out_ch=3,\n", 100 | " num_feat=64,\n", 101 | " num_block=23,\n", 102 | " num_grow_ch=32,\n", 103 | " scale=2,\n", 104 | " )\n", 105 | " upsampler = RealESRGANer(\n", 106 | " scale=2,\n", 107 | " model_path=\"CodeFormer/weights/realesrgan/RealESRGAN_x2plus.pth\",\n", 108 | " model=model,\n", 109 | " tile=400,\n", 110 | " tile_pad=40,\n", 111 | " pre_pad=0,\n", 112 | " half=half,\n", 113 | " )\n", 114 | " return upsampler\n", 115 | "\n", 116 | "upsampler = set_realesrgan()\n", 117 | "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n", 118 | "codeformer_net = ARCH_REGISTRY.get(\"CodeFormer\")(\n", 119 | " dim_embd=512,\n", 120 | " codebook_size=1024,\n", 121 | " n_head=8,\n", 122 | " n_layers=9,\n", 123 | " connect_list=[\"32\", \"64\", \"128\", \"256\"],\n", 124 | ").to(device)\n", 125 | "ckpt_path = \"CodeFormer/weights/CodeFormer/codeformer.pth\"\n", 126 | "checkpoint = torch.load(ckpt_path)[\"params_ema\"]\n", 127 | "codeformer_net.load_state_dict(checkpoint)\n", 128 | "codeformer_net.eval()\n", 129 | "\n", 130 | "os.makedirs('output', exist_ok=True)\n", 131 | "\n", 132 | "def inference(image, background_enhance, face_upsample, upscale, codeformer_fidelity):\n", 133 | " \"\"\"Run a single prediction on the model\"\"\"\n", 134 | " # take the default setting for the demo\n", 135 | " has_aligned = False\n", 136 | " only_center_face = False\n", 137 | " draw_box = False\n", 138 | " detection_model = \"retinaface_resnet50\"\n", 139 | "\n", 140 | " face_helper = FaceRestoreHelper(\n", 141 | " upscale,\n", 142 | " face_size=512,\n", 143 | " crop_ratio=(1, 1),\n", 144 | " det_model=detection_model,\n", 145 | " save_ext=\"png\",\n", 146 | " use_parse=True,\n", 147 | " device=device,\n", 148 | " )\n", 149 | " bg_upsampler = upsampler if background_enhance else None\n", 150 | " face_upsampler = upsampler if face_upsample else None\n", 151 | "\n", 152 | " img = cv2.imread(str(image), cv2.IMREAD_COLOR)\n", 153 | "\n", 154 | " if has_aligned:\n", 155 | " # the input faces are already cropped and aligned\n", 156 | " img = cv2.resize(img, (512, 512), interpolation=cv2.INTER_LINEAR)\n", 157 | " face_helper.cropped_faces = [img]\n", 158 | " else:\n", 159 | " face_helper.read_image(img)\n", 160 | " # get face landmarks for each face\n", 161 | " num_det_faces = face_helper.get_face_landmarks_5(\n", 162 | " only_center_face=only_center_face, resize=640, eye_dist_threshold=5\n", 163 | " )\n", 164 | " print(f\"\\tdetect {num_det_faces} faces\")\n", 165 | " # align and warp each face\n", 166 | " face_helper.align_warp_face()\n", 167 | "\n", 168 | " # face restoration for each cropped face\n", 169 | " for idx, cropped_face in enumerate(face_helper.cropped_faces):\n", 170 | " # prepare data\n", 171 | " cropped_face_t = img2tensor(\n", 172 | " cropped_face / 255.0, bgr2rgb=True, float32=True\n", 173 | " )\n", 174 | " normalize(cropped_face_t, (0.5, 0.5, 0.5), (0.5, 0.5, 0.5), inplace=True)\n", 175 | " cropped_face_t = cropped_face_t.unsqueeze(0).to(device)\n", 176 | "\n", 177 | " try:\n", 178 | " with torch.no_grad():\n", 179 | " output = codeformer_net(\n", 180 | " cropped_face_t, w=codeformer_fidelity, adain=True\n", 181 | " )[0]\n", 182 | " restored_face = tensor2img(output, rgb2bgr=True, min_max=(-1, 1))\n", 183 | " del output\n", 184 | " torch.cuda.empty_cache()\n", 185 | " except Exception as error:\n", 186 | " print(f\"\\tFailed inference for CodeFormer: {error}\")\n", 187 | " restored_face = tensor2img(\n", 188 | " cropped_face_t, rgb2bgr=True, min_max=(-1, 1)\n", 189 | " )\n", 190 | "\n", 191 | " restored_face = restored_face.astype(\"uint8\")\n", 192 | " face_helper.add_restored_face(restored_face)\n", 193 | "\n", 194 | " # paste_back\n", 195 | " if not has_aligned:\n", 196 | " # upsample the background\n", 197 | " if bg_upsampler is not None:\n", 198 | " # Now only support RealESRGAN for upsampling background\n", 199 | " bg_img = bg_upsampler.enhance(img, outscale=upscale)[0]\n", 200 | " else:\n", 201 | " bg_img = None\n", 202 | " face_helper.get_inverse_affine(None)\n", 203 | " # paste each restored face to the input image\n", 204 | " if face_upsample and face_upsampler is not None:\n", 205 | " restored_img = face_helper.paste_faces_to_input_image(\n", 206 | " upsample_img=bg_img,\n", 207 | " draw_box=draw_box,\n", 208 | " face_upsampler=face_upsampler,\n", 209 | " )\n", 210 | " else:\n", 211 | " restored_img = face_helper.paste_faces_to_input_image(\n", 212 | " upsample_img=bg_img, draw_box=draw_box\n", 213 | " )\n", 214 | "\n", 215 | " # save restored img\n", 216 | " save_path = f'output/out.png'\n", 217 | " imwrite(restored_img, str(save_path))\n", 218 | "\n", 219 | " restored_img = cv2.cvtColor(restored_img, cv2.COLOR_BGR2RGB)\n", 220 | " return restored_img\n", 221 | "\n", 222 | "\n", 223 | "gr.Interface(\n", 224 | " inference, [\n", 225 | " gr.inputs.Image(type=\"filepath\", label=\"Input\"),\n", 226 | " gr.inputs.Checkbox(default=True, label=\"Background_Enhance\"),\n", 227 | " gr.inputs.Checkbox(default=True, label=\"Face_Upsample\"),\n", 228 | " gr.inputs.Number(default=2, label=\"Rescaling_Factor\"),\n", 229 | " gr.Slider(0, 1, value=0.5, step=0.01, label='Codeformer_Fidelity: 0 for better quality, 1 for better identity')\n", 230 | " ], [\n", 231 | " gr.outputs.Image(type=\"numpy\", label=\"Output\"),\n", 232 | " ]\n", 233 | " ).launch(inline=False, debug=True)" 234 | ] 235 | } 236 | ], 237 | "metadata": { 238 | "colab": { 239 | "collapsed_sections": [], 240 | "provenance": [] 241 | }, 242 | "kernelspec": { 243 | "display_name": "Python 3", 244 | "name": "python3" 245 | }, 246 | "language_info": { 247 | "name": "python" 248 | } 249 | }, 250 | "nbformat": 4, 251 | "nbformat_minor": 0 252 | } 253 | -------------------------------------------------------------------------------- /camenduru's_compare.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "view-in-github" 7 | }, 8 | "source": [ 9 | "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/notebooks/blob/main/camenduru's_compare.ipynb)" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "id": "rbyb3ZDDyMoE" 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "!pip install -q torch==1.13.1+cu116 torchvision==0.14.1+cu116 torchaudio==0.13.1 torchtext==0.14.1 torchdata==0.5.1 --extra-index-url https://download.pytorch.org/whl/cu116 -U\n", 21 | "!pip install safetensors pytorch_lightning" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": null, 27 | "metadata": { 28 | "id": "pe33yPgLyZHZ" 29 | }, 30 | "outputs": [], 31 | "source": [ 32 | "# from https://huggingface.co/JosephusCheung/ASimilarityCalculatior\n", 33 | "\n", 34 | "from safetensors.torch import load_file\n", 35 | "import sys\n", 36 | "import torch\n", 37 | "from pathlib import Path\n", 38 | "import torch.nn as nn\n", 39 | "import torch.nn.functional as F\n", 40 | "\n", 41 | "def cal_cross_attn(to_q, to_k, to_v, rand_input):\n", 42 | " hidden_dim, embed_dim = to_q.shape\n", 43 | " attn_to_q = nn.Linear(hidden_dim, embed_dim, bias=False)\n", 44 | " attn_to_k = nn.Linear(hidden_dim, embed_dim, bias=False)\n", 45 | " attn_to_v = nn.Linear(hidden_dim, embed_dim, bias=False)\n", 46 | " attn_to_q.load_state_dict({\"weight\": to_q})\n", 47 | " attn_to_k.load_state_dict({\"weight\": to_k})\n", 48 | " attn_to_v.load_state_dict({\"weight\": to_v})\n", 49 | " \n", 50 | " return torch.einsum(\n", 51 | " \"ik, jk -> ik\", \n", 52 | " F.softmax(torch.einsum(\"ij, kj -> ik\", attn_to_q(rand_input), attn_to_k(rand_input)), dim=-1),\n", 53 | " attn_to_v(rand_input)\n", 54 | " )\n", 55 | "\n", 56 | "def model_hash(filename):\n", 57 | " try:\n", 58 | " with open(filename, \"rb\") as file:\n", 59 | " import hashlib\n", 60 | " m = hashlib.sha256()\n", 61 | "\n", 62 | " file.seek(0x100000)\n", 63 | " m.update(file.read(0x10000))\n", 64 | " return m.hexdigest()[0:8]\n", 65 | " except FileNotFoundError:\n", 66 | " return 'NOFILE'\n", 67 | " \n", 68 | "def load_model(path):\n", 69 | " if path.suffix == \".safetensors\":\n", 70 | " return load_file(path, device=\"cpu\")\n", 71 | " else:\n", 72 | " ckpt = torch.load(path, map_location=\"cpu\")\n", 73 | " return ckpt[\"state_dict\"] if \"state_dict\" in ckpt else ckpt\n", 74 | " \n", 75 | "def eval(model, n, input):\n", 76 | " qk = f\"model.diffusion_model.output_blocks.{n}.1.transformer_blocks.0.attn1.to_q.weight\"\n", 77 | " uk = f\"model.diffusion_model.output_blocks.{n}.1.transformer_blocks.0.attn1.to_k.weight\"\n", 78 | " vk = f\"model.diffusion_model.output_blocks.{n}.1.transformer_blocks.0.attn1.to_v.weight\"\n", 79 | " atoq, atok, atov = model[qk], model[uk], model[vk]\n", 80 | "\n", 81 | " attn = cal_cross_attn(atoq, atok, atov, input)\n", 82 | " return attn\n", 83 | "\n", 84 | "def main():\n", 85 | " file1 = Path(sys.argv[1])\n", 86 | " files = sys.argv[2:]\n", 87 | " \n", 88 | " seed = 114514\n", 89 | " torch.manual_seed(seed)\n", 90 | " print(f\"seed: {seed}\") \n", 91 | " \n", 92 | " model_a = load_model(file1)\n", 93 | " \n", 94 | " print()\n", 95 | " print(f\"base: {file1.name} [{model_hash(file1)}]\")\n", 96 | " print()\n", 97 | "\n", 98 | " map_attn_a = {}\n", 99 | " map_rand_input = {}\n", 100 | " for n in range(3, 11):\n", 101 | " hidden_dim, embed_dim = model_a[f\"model.diffusion_model.output_blocks.{n}.1.transformer_blocks.0.attn1.to_q.weight\"].shape\n", 102 | " rand_input = torch.randn([embed_dim, hidden_dim])\n", 103 | "\n", 104 | " map_attn_a[n] = eval(model_a, n, rand_input)\n", 105 | " map_rand_input[n] = rand_input\n", 106 | " \n", 107 | " del model_a\n", 108 | " \n", 109 | " for file2 in files:\n", 110 | " file2 = Path(file2)\n", 111 | " model_b = load_model(file2)\n", 112 | " \n", 113 | " sims = []\n", 114 | " for n in range(3, 11):\n", 115 | " attn_a = map_attn_a[n]\n", 116 | " attn_b = eval(model_b, n, map_rand_input[n])\n", 117 | " \n", 118 | " sim = torch.mean(torch.cosine_similarity(attn_a, attn_b))\n", 119 | " sims.append(sim)\n", 120 | " \n", 121 | " print(f\"{file2} [{model_hash(file2)}] - {torch.mean(torch.stack(sims)) * 1e2:.2f}%\")\n", 122 | " \n", 123 | "if __name__ == \"__main__\":\n", 124 | " main()" 125 | ] 126 | }, 127 | { 128 | "cell_type": "code", 129 | "execution_count": null, 130 | "metadata": {}, 131 | "outputs": [], 132 | "source": [ 133 | "!wget https://huggingface.co/ckpt/anything-v3.0/resolve/main/Anything-V3.0-pruned.safetensors\n", 134 | "!wget https://huggingface.co/ckpt/anything-v4.0/resolve/main/anything-v4.5-pruned.safetensors" 135 | ] 136 | }, 137 | { 138 | "cell_type": "code", 139 | "execution_count": null, 140 | "metadata": {}, 141 | "outputs": [], 142 | "source": [ 143 | "!python test.py \"anything-v4.5-pruned.safetensors\" \"Anything-V3.0-pruned.safetensors\"" 144 | ] 145 | } 146 | ], 147 | "metadata": { 148 | "colab": { 149 | "collapsed_sections": [], 150 | "provenance": [] 151 | }, 152 | "kernelspec": { 153 | "display_name": "Python 3", 154 | "name": "python3" 155 | }, 156 | "language_info": { 157 | "name": "python" 158 | } 159 | }, 160 | "nbformat": 4, 161 | "nbformat_minor": 0 162 | } 163 | -------------------------------------------------------------------------------- /camenduru's_diffusers_to_sd.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "view-in-github" 7 | }, 8 | "source": [ 9 | "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/notebooks/blob/main/camenduru's_diffusers_to_sd.ipynb)" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "id": "W_hvBD84lggc" 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "!pip install -q torch==1.13.1+cu116 torchvision==0.14.1+cu116 torchaudio==0.13.1 torchtext==0.14.1 torchdata==0.5.1 --extra-index-url https://download.pytorch.org/whl/cu116 -U\n", 21 | "!pip -qq install huggingface-hub\n", 22 | "!wget -q https://gist.github.com/jachiam/8a5c0b607e38fcc585168b90c686eb05/raw/2af0a9c0237ed98b863a75e1db21d7ed5541094f/convert_diffusers_to_sd.py\n", 23 | "!git lfs install\n", 24 | "from huggingface_hub import notebook_login\n", 25 | "!git config --global credential.helper store\n", 26 | "notebook_login()" 27 | ] 28 | }, 29 | { 30 | "cell_type": "code", 31 | "execution_count": null, 32 | "metadata": { 33 | "id": "V8DWrvEhlJNT" 34 | }, 35 | "outputs": [], 36 | "source": [ 37 | "old_model_name = \"adambrody\" #@param {type:\"string\"}\n", 38 | "new_model_name = \"camenduru\" #@param {type:\"string\"}\n", 39 | "!git clone https://huggingface.co/camenduru/$old_model_name\n", 40 | "\n", 41 | "old_model_name_ckpt = f\"/content/{old_model_name}_ckpt\"\n", 42 | "!mkdir $old_model_name_ckpt\n", 43 | "\n", 44 | "ckpt_file = f\"{old_model_name_ckpt}/{old_model_name}.ckpt\"\n", 45 | "!python convert_diffusers_to_sd.py --model_path /content/$old_model_name --checkpoint_path $ckpt_file \n", 46 | "\n", 47 | "from slugify import slugify\n", 48 | "from huggingface_hub import HfApi, HfFolder, CommitOperationAdd\n", 49 | "from huggingface_hub import create_repo\n", 50 | "from IPython.display import display_markdown\n", 51 | "api = HfApi()\n", 52 | "your_username = api.whoami()[\"name\"]\n", 53 | "repo_id = f\"{your_username}/{slugify(new_model_name)}\"\n", 54 | "with open(HfFolder.path_token, 'r') as fin: hf_token = fin.read();\n", 55 | "\n", 56 | "create_repo(repo_id,private=True, token=hf_token)\n", 57 | "api.create_commit(repo_id=repo_id,operations=[],commit_message=f\"{old_model_name} to {new_model_name}\",token=hf_token)\n", 58 | "api.upload_folder(folder_path=old_model_name_ckpt,path_in_repo=\"\",repo_id=repo_id,token=hf_token)\n", 59 | "\n", 60 | "display_markdown(f'https://huggingface.co/{repo_id}', raw=True)" 61 | ] 62 | } 63 | ], 64 | "metadata": { 65 | "colab": { 66 | "provenance": [] 67 | }, 68 | "kernelspec": { 69 | "display_name": "Python 3", 70 | "name": "python3" 71 | }, 72 | "language_info": { 73 | "name": "python" 74 | } 75 | }, 76 | "nbformat": 4, 77 | "nbformat_minor": 0 78 | } 79 | -------------------------------------------------------------------------------- /camenduru's_discord_diffusers.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "view-in-github" 7 | }, 8 | "source": [ 9 | "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/notebooks/blob/main/camenduru's_discord_diffusers.ipynb)" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "id": "wR009yNE2Gko" 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "!pip install -q torch==1.13.1+cu116 torchvision==0.14.1+cu116 torchaudio==0.13.1 torchtext==0.14.1 torchdata==0.5.1 --extra-index-url https://download.pytorch.org/whl/cu116 -U\n", 21 | "!pip install -q https://github.com/camenduru/stable-diffusion-webui-colab/releases/download/0.0.15/xformers-0.0.15.dev0+189828c.d20221207-cp38-cp38-linux_x86_64.whl\n", 22 | "!pip install -U -q git+https://github.com/camenduru/diffusers transformers" 23 | ] 24 | }, 25 | { 26 | "cell_type": "code", 27 | "execution_count": null, 28 | "metadata": { 29 | "id": "wvUpT9bGvwdN" 30 | }, 31 | "outputs": [], 32 | "source": [ 33 | "import torch, os, gc, requests, json\n", 34 | "from diffusers import StableDiffusionPipeline\n", 35 | "from PIL.PngImagePlugin import PngInfo\n", 36 | "\n", 37 | "metadata = PngInfo()\n", 38 | "\n", 39 | "def closestNumber(n, m):\n", 40 | " q = int(n / m)\n", 41 | " n1 = m * q\n", 42 | " if (n * m) > 0:\n", 43 | " n2 = m * (q + 1)\n", 44 | " else:\n", 45 | " n2 = m * (q - 1)\n", 46 | " if abs(n - n1) < abs(n - n2):\n", 47 | " return n1\n", 48 | " return n2\n", 49 | "\n", 50 | "is_tile = False #@param {type: 'boolean'}\n", 51 | "if is_tile:\n", 52 | " def patch_conv(cls):\n", 53 | " init = cls.__init__\n", 54 | " def __init__(self, *args, **kwargs):\n", 55 | " return init(self, *args, **kwargs, padding_mode=\"circular\")\n", 56 | " cls.__init__ = __init__\n", 57 | " patch_conv(torch.nn.Conv2d)\n", 58 | "\n", 59 | "model_hash = \"none\" #@param {type: 'string'}\n", 60 | "max_files = 500 #@param {type: 'string'}\n", 61 | "root_folder = \"images\" #@param {type: 'string'}\n", 62 | "model_name = \"hakurei/waifu-diffusion\" #@param [\"nitrosocke/mo-di-diffusion\", \"hakurei/waifu-diffusion\"] {allow-input: true}\n", 63 | "pipe = StableDiffusionPipeline.from_pretrained(model_name, safety_checker=None).to(\"cuda\")\n", 64 | "\n", 65 | "if os.path.exists(f\"{root_folder}\") == False:\n", 66 | " os.mkdir(f\"{root_folder}\")\n", 67 | "image_folder = max([int(f) for f in os.listdir(f\"{root_folder}\")], default=0)\n", 68 | "if os.path.exists(f\"{root_folder}/{image_folder:04}\") == False:\n", 69 | " os.mkdir(f\"{root_folder}/{image_folder:04}\")\n", 70 | "name = max([int(f[: f.index(\".\")]) for f in os.listdir(f\"{root_folder}/{image_folder:04}\")],default=0,)\n", 71 | "\n", 72 | "def generate(discord_token, discord_channel_id, by, num_inference_steps, guidance_scale, sampler, width, height, prompt, image_folder, name):\n", 73 | " width = closestNumber(width, 8)\n", 74 | " height = closestNumber(height, 8)\n", 75 | " metadata.add_text(\"Prompt\", f\"{prompt}\")\n", 76 | " metadata.add_text(\"by\", f\"{by}\")\n", 77 | " gc.collect()\n", 78 | " torch.cuda.empty_cache()\n", 79 | " image = pipe([prompt], height=height, width=width, num_inference_steps=num_inference_steps, guidance_scale=guidance_scale).images[0]\n", 80 | " initial_seed = torch.cuda.initial_seed()\n", 81 | " image.save(f\"{root_folder}/{image_folder:04}/{name:04}.png\", pnginfo=metadata)\n", 82 | " files = {f\"{image_folder:04}_{name:04}.png\": open(f\"{root_folder}/{image_folder:04}/{name:04}.png\", \"rb\").read()}\n", 83 | " payload = {\"content\": f\"{prompt}\\nSteps: {num_inference_steps}, Sampler: {sampler}, CFG scale: {guidance_scale}, Seed: {initial_seed}, Size: {width}x{height}, Model hash: {model_hash}, Model name: {model_name}\"}\n", 84 | " requests.post(f\"https://discord.com/api/v9/channels/{discord_channel_id}/messages\", data=payload, headers={\"authorization\": f\"Bot {discord_token}\"}, files=files)\n", 85 | "\n", 86 | "while True:\n", 87 | " if name < max_files:\n", 88 | " with open(\"prompts.json\", \"r\") as file:\n", 89 | " prompts = file.readlines()\n", 90 | " for prompt in prompts:\n", 91 | " d = json.loads(prompt)\n", 92 | " name += 1\n", 93 | " generate(d[\"discord_token\"], d[\"discord_channel_id\"], d[\"by\"], d[\"num_inference_steps\"], d[\"guidance_scale\"], d[\"sampler\"], d[\"width\"], d[\"height\"], d[\"prompt\"], image_folder, name)\n", 94 | " else:\n", 95 | " image_folder += 1\n", 96 | " if os.path.exists(f\"{root_folder}/{image_folder:04}\") == False:\n", 97 | " os.mkdir(f\"{root_folder}/{image_folder:04}\")\n", 98 | " name = 0" 99 | ] 100 | } 101 | ], 102 | "metadata": { 103 | "accelerator": "GPU", 104 | "colab": { 105 | "collapsed_sections": [], 106 | "provenance": [] 107 | }, 108 | "gpuClass": "standard", 109 | "kernelspec": { 110 | "display_name": "Python 3", 111 | "name": "python3" 112 | }, 113 | "language_info": { 114 | "name": "python" 115 | } 116 | }, 117 | "nbformat": 4, 118 | "nbformat_minor": 0 119 | } 120 | -------------------------------------------------------------------------------- /camenduru's_gfpgan.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "view-in-github" 7 | }, 8 | "source": [ 9 | "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/notebooks/blob/main/camenduru's_gfpgan.ipynb)" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "id": "rbyb3ZDDyMoE" 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "!pip install -q torch==1.13.1+cu116 torchvision==0.14.1+cu116 torchaudio==0.13.1 torchtext==0.14.1 torchdata==0.5.1 --extra-index-url https://download.pytorch.org/whl/cu116 -U\n", 21 | "!git clone https://huggingface.co/spaces/Xintao/GFPGAN\n", 22 | "%cd GFPGAN\n", 23 | "!pip install -qq -r requirements.txt\n", 24 | "!pip install -qq gradio" 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "execution_count": null, 30 | "metadata": { 31 | "id": "pe33yPgLyZHZ" 32 | }, 33 | "outputs": [], 34 | "source": [ 35 | "import os\n", 36 | "\n", 37 | "import cv2\n", 38 | "import gradio as gr\n", 39 | "import torch\n", 40 | "from basicsr.archs.srvgg_arch import SRVGGNetCompact\n", 41 | "from gfpgan.utils import GFPGANer\n", 42 | "from realesrgan.utils import RealESRGANer\n", 43 | "\n", 44 | "os.system(\"pip freeze\")\n", 45 | "os.system(\"wget https://github.com/xinntao/Real-ESRGAN/releases/download/v0.2.5.0/realesr-general-x4v3.pth -P .\")\n", 46 | "os.system(\"wget https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.2.pth -P .\")\n", 47 | "os.system(\"wget https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.3.pth -P .\")\n", 48 | "\n", 49 | "torch.hub.download_url_to_file(\n", 50 | " 'https://upload.wikimedia.org/wikipedia/commons/thumb/a/ab/Abraham_Lincoln_O-77_matte_collodion_print.jpg/1024px-Abraham_Lincoln_O-77_matte_collodion_print.jpg',\n", 51 | " 'lincoln.jpg')\n", 52 | "torch.hub.download_url_to_file(\n", 53 | " 'https://user-images.githubusercontent.com/17445847/187400315-87a90ac9-d231-45d6-b377-38702bd1838f.jpg',\n", 54 | " 'AI-generate.jpg')\n", 55 | "torch.hub.download_url_to_file(\n", 56 | " 'https://user-images.githubusercontent.com/17445847/187400981-8a58f7a4-ef61-42d9-af80-bc6234cef860.jpg',\n", 57 | " 'Blake_Lively.jpg')\n", 58 | "torch.hub.download_url_to_file(\n", 59 | " 'https://user-images.githubusercontent.com/17445847/187401133-8a3bf269-5b4d-4432-b2f0-6d26ee1d3307.png',\n", 60 | " '10045.png')\n", 61 | "\n", 62 | "# background enhancer with RealESRGAN\n", 63 | "model = SRVGGNetCompact(num_in_ch=3, num_out_ch=3, num_feat=64, num_conv=32, upscale=4, act_type='prelu')\n", 64 | "model_path = 'realesr-general-x4v3.pth'\n", 65 | "half = True if torch.cuda.is_available() else False\n", 66 | "upsampler = RealESRGANer(scale=4, model_path=model_path, model=model, tile=0, tile_pad=10, pre_pad=0, half=half)\n", 67 | "\n", 68 | "# Use GFPGAN for face enhancement\n", 69 | "face_enhancer_v3 = GFPGANer(\n", 70 | " model_path='GFPGANv1.3.pth', upscale=2, arch='clean', channel_multiplier=2, bg_upsampler=upsampler)\n", 71 | "face_enhancer_v2 = GFPGANer(\n", 72 | " model_path='GFPGANv1.2.pth', upscale=2, arch='clean', channel_multiplier=2, bg_upsampler=upsampler)\n", 73 | "os.makedirs('output', exist_ok=True)\n", 74 | "\n", 75 | "\n", 76 | "def inference(img, version, scale):\n", 77 | " print(img, version, scale)\n", 78 | " try:\n", 79 | " img = cv2.imread(img, cv2.IMREAD_UNCHANGED)\n", 80 | " if len(img.shape) == 3 and img.shape[2] == 4:\n", 81 | " img_mode = 'RGBA'\n", 82 | " else:\n", 83 | " img_mode = None\n", 84 | "\n", 85 | " h, w = img.shape[0:2]\n", 86 | " if h < 300:\n", 87 | " img = cv2.resize(img, (w * 2, h * 2), interpolation=cv2.INTER_LANCZOS4)\n", 88 | "\n", 89 | " if version == 'v1.2':\n", 90 | " face_enhancer = face_enhancer_v2\n", 91 | " else:\n", 92 | " face_enhancer = face_enhancer_v3\n", 93 | " try:\n", 94 | " _, _, output = face_enhancer.enhance(img, has_aligned=False, only_center_face=False, paste_back=True)\n", 95 | " except RuntimeError as error:\n", 96 | " print('Error', error)\n", 97 | " else:\n", 98 | " extension = 'png'\n", 99 | "\n", 100 | " try:\n", 101 | " if scale != 2:\n", 102 | " interpolation = cv2.INTER_AREA if scale < 2 else cv2.INTER_LANCZOS4\n", 103 | " h, w = img.shape[0:2]\n", 104 | " output = cv2.resize(output, (int(w * scale / 2), int(h * scale / 2)), interpolation=interpolation)\n", 105 | " except Exception as error:\n", 106 | " print('wrong scale input.', error)\n", 107 | " if img_mode == 'RGBA': # RGBA images should be saved in png format\n", 108 | " extension = 'png'\n", 109 | " else:\n", 110 | " extension = 'jpg'\n", 111 | " save_path = f'output/out.{extension}'\n", 112 | " cv2.imwrite(save_path, output)\n", 113 | "\n", 114 | " output = cv2.cvtColor(output, cv2.COLOR_BGR2RGB)\n", 115 | " return output, save_path\n", 116 | " except Exception as error:\n", 117 | " print('global exception', error)\n", 118 | " return None, None\n", 119 | "\n", 120 | "gr.Interface(\n", 121 | " inference, [\n", 122 | " gr.inputs.Image(type=\"filepath\", label=\"Input\"),\n", 123 | " gr.inputs.Radio(['v1.2', 'v1.3'], type=\"value\", default='v1.3', label='GFPGAN version'),\n", 124 | " gr.inputs.Number(label=\"Rescaling factor\", default=2)\n", 125 | " ], [\n", 126 | " gr.outputs.Image(type=\"numpy\", label=\"Output (The whole image)\"),\n", 127 | " gr.outputs.File(label=\"Download the output image\")\n", 128 | " ]).launch(inline=False, debug=True)" 129 | ] 130 | } 131 | ], 132 | "metadata": { 133 | "colab": { 134 | "collapsed_sections": [], 135 | "provenance": [] 136 | }, 137 | "kernelspec": { 138 | "display_name": "Python 3", 139 | "name": "python3" 140 | }, 141 | "language_info": { 142 | "name": "python" 143 | } 144 | }, 145 | "nbformat": 4, 146 | "nbformat_minor": 0 147 | } 148 | -------------------------------------------------------------------------------- /camenduru's_hand_detect_crop.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "provenance": [] 7 | }, 8 | "kernelspec": { 9 | "name": "python3", 10 | "display_name": "Python 3" 11 | }, 12 | "language_info": { 13 | "name": "python" 14 | } 15 | }, 16 | "cells": [ 17 | { 18 | "cell_type": "markdown", 19 | "metadata": { 20 | "id": "view-in-github" 21 | }, 22 | "source": [ 23 | "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/notebooks/blob/main/camenduru's_hand_detect_crop.ipynb)" 24 | ] 25 | }, 26 | { 27 | "cell_type": "code", 28 | "execution_count": null, 29 | "metadata": { 30 | "id": "w8jYWxI3iwmY" 31 | }, 32 | "outputs": [], 33 | "source": [ 34 | "import cv2\n", 35 | "import time\n", 36 | "import os, gc\n", 37 | "import mediapipe as mp\n", 38 | "import gradio as gr\n", 39 | "from threading import Thread\n", 40 | "import math\n", 41 | "from google.colab.patches import cv2_imshow\n", 42 | "from PIL import Image\n", 43 | "import numpy as np\n", 44 | "example_flag = False\n", 45 | "\n", 46 | "# https://huggingface.co/spaces/datasciencedojo/Finger-Counting-Right-Hand/blob/main/app.py modified \n", 47 | "class handDetector():\n", 48 | " def __init__(self, mode=True, modelComplexity=1, maxHands=2, detectionCon=0.5, trackCon=0.5):\n", 49 | " self.mode = mode\n", 50 | " self.maxHands = maxHands\n", 51 | " self.detectionCon = detectionCon\n", 52 | " self.modelComplex = modelComplexity\n", 53 | " self.trackCon = trackCon\n", 54 | " self.mpHands = mp.solutions.hands\n", 55 | " self.hands = self.mpHands.Hands(self.mode, self.maxHands,self.modelComplex,self.detectionCon, self.trackCon)\n", 56 | " self.mpDraw = mp.solutions.drawing_utils\n", 57 | "\n", 58 | " def findHands(self, img, draw=True, flipType=True):\n", 59 | " \"\"\"\n", 60 | " Finds hands in a BGR image.\n", 61 | " :param img: Image to find the hands in.\n", 62 | " :param draw: Flag to draw the output on the image.\n", 63 | " :return: Image with or without drawings\n", 64 | " \"\"\"\n", 65 | " imgRGB = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n", 66 | " #cv2.imshow('test',imgRGB)\n", 67 | " self.results = self.hands.process(imgRGB)\n", 68 | " allHands = []\n", 69 | " h, w, c = img.shape\n", 70 | " if self.results.multi_hand_landmarks:\n", 71 | " for handType, handLms in zip(self.results.multi_handedness, self.results.multi_hand_landmarks):\n", 72 | " myHand = {}\n", 73 | " ## lmList\n", 74 | " mylmList = []\n", 75 | " xList = []\n", 76 | " yList = []\n", 77 | " for id, lm in enumerate(handLms.landmark):\n", 78 | " px, py, pz = int(lm.x * w), int(lm.y * h), int(lm.z * w)\n", 79 | " mylmList.append([px, py, pz])\n", 80 | " xList.append(px)\n", 81 | " yList.append(py)\n", 82 | "\n", 83 | " ## bbox\n", 84 | " xmin, xmax = min(xList), max(xList)\n", 85 | " ymin, ymax = min(yList), max(yList)\n", 86 | " boxW, boxH = xmax - xmin, ymax - ymin\n", 87 | " bbox = xmin, ymin, boxW, boxH\n", 88 | " cx, cy = bbox[0] + (bbox[2] // 2), \\\n", 89 | " bbox[1] + (bbox[3] // 2)\n", 90 | "\n", 91 | " myHand[\"lmList\"] = mylmList\n", 92 | " myHand[\"bbox\"] = bbox\n", 93 | " myHand[\"center\"] = (cx, cy)\n", 94 | "\n", 95 | " if flipType:\n", 96 | " if handType.classification[0].label == \"Right\":\n", 97 | " myHand[\"type\"] = \"Left\"\n", 98 | " else:\n", 99 | " myHand[\"type\"] = \"Right\"\n", 100 | " else:\n", 101 | " myHand[\"type\"] = handType.classification[0].label\n", 102 | " allHands.append(myHand)\n", 103 | "\n", 104 | " if draw:\n", 105 | " # self.mpDraw.draw_landmarks(img, handLms,self.mpHands.HAND_CONNECTIONS)\n", 106 | " # cv2.rectangle(img, (bbox[0] - 20, bbox[1] - 20), (bbox[0] + bbox[2] + 20, bbox[1] + bbox[3] + 20), (255, 0, 255), 10)\n", 107 | "\n", 108 | " center_x = bbox[0] + bbox[2] // 2\n", 109 | " center_y = bbox[1] + bbox[3] // 2\n", 110 | " center = (center_x, center_y)\n", 111 | " # cv2.circle(img, center, radius=5, color=(255, 0, 255), thickness=10)\n", 112 | " length = math.sqrt(bbox[2]**2 + bbox[3]**2)\n", 113 | " x = int(center_x - int(length//2))\n", 114 | " y = int(center_y - int(length//2))\n", 115 | "\n", 116 | " if x < 0:\n", 117 | " x = 0\n", 118 | " if y < 0:\n", 119 | " y = 0\n", 120 | " if x + int(length) > img.shape[1]:\n", 121 | " x = img.shape[1] - int(length)\n", 122 | " if y + int(length) > img.shape[0]:\n", 123 | " y = img.shape[0] - int(length)\n", 124 | " # cv2.rectangle(img, (x, y), (x+int(length), y+int(length)), (0, 0, 255), 10)\n", 125 | "\n", 126 | " width, height = int(length), int(length)\n", 127 | " x = center_x - (width // 2)\n", 128 | " y = center_y - (height // 2)\n", 129 | " \n", 130 | " if x < 0:\n", 131 | " x = 0\n", 132 | " if y < 0:\n", 133 | " y = 0\n", 134 | " if x + width > img.shape[1]:\n", 135 | " x = img.shape[1] - width\n", 136 | " if y + height > img.shape[0]:\n", 137 | " y = img.shape[0] - height\n", 138 | " img = img[y:y+height, x:x+width]\n", 139 | " img = cv2.resize(img, (512, 512))\n", 140 | "\n", 141 | " if draw:\n", 142 | " return allHands, img\n", 143 | " else:\n", 144 | " return allHands\n", 145 | "\n", 146 | "def multi(image_path):\n", 147 | " img = cv2.imread(image_path) \n", 148 | " image_size = img.shape[:2]\n", 149 | " if image_size == (512, 512):\n", 150 | " print(\"512x512\")\n", 151 | " return img\n", 152 | " else:\n", 153 | " detector = handDetector(detectionCon=0.75)\n", 154 | " allhands, img = detector.findHands(img, 1)\n", 155 | " del detector\n", 156 | " gc.collect()\n", 157 | " cv2.imwrite(image_path, img[:,:,::-1]) \n", 158 | " return img\n", 159 | "\n", 160 | "def get_image_paths(folder_path):\n", 161 | " image_paths = []\n", 162 | " for dirpath, dirnames, filenames in os.walk(folder_path):\n", 163 | " for filename in filenames:\n", 164 | " if filename.endswith(('.jpg', '.png', '.jpeg')):\n", 165 | " file_path = os.path.join(dirpath, filename)\n", 166 | " image_paths.append(file_path)\n", 167 | " return image_paths\n", 168 | "\n", 169 | "root_folder_path = '/content/images'\n", 170 | "all_image_paths = get_image_paths(root_folder_path)\n", 171 | "\n", 172 | "for image_path in all_image_paths:\n", 173 | " multi(image_path)\n", 174 | " print(image_path)" 175 | ] 176 | } 177 | ] 178 | } -------------------------------------------------------------------------------- /camenduru's_html_to_datasets.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "view-in-github" 7 | }, 8 | "source": [ 9 | "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/notebooks/blob/main/camenduru's_html_to_datasets.ipynb)" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "id": "ExL6QTs80tn4" 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "!pip install -q torch==1.13.1+cu116 torchvision==0.14.1+cu116 torchaudio==0.13.1 torchtext==0.14.1 torchdata==0.5.1 --extra-index-url https://download.pytorch.org/whl/cu116 -U\n", 21 | "!pip install datasets bs4" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": null, 27 | "metadata": { 28 | "id": "dYNuukd4AOP7" 29 | }, 30 | "outputs": [], 31 | "source": [ 32 | "from huggingface_hub import notebook_login\n", 33 | "!git config --global credential.helper store\n", 34 | "notebook_login()" 35 | ] 36 | }, 37 | { 38 | "cell_type": "code", 39 | "execution_count": null, 40 | "metadata": { 41 | "id": "juJqLu-Z7FAA" 42 | }, 43 | "outputs": [], 44 | "source": [ 45 | "!mkdir plushies" 46 | ] 47 | }, 48 | { 49 | "cell_type": "code", 50 | "execution_count": null, 51 | "metadata": { 52 | "id": "uyAtj6x_3nz7" 53 | }, 54 | "outputs": [], 55 | "source": [ 56 | "import urllib.request\n", 57 | "from bs4 import BeautifulSoup\n", 58 | "\n", 59 | "with open('/content/plushies.txt') as html:\n", 60 | " content = html.read()\n", 61 | "\n", 62 | "soup = BeautifulSoup(content)\n", 63 | "for imgtag in soup.find_all('img'):\n", 64 | " url=imgtag['src']\n", 65 | " name = url.split('/')[-1]\n", 66 | " headers={'user-agent': 'Mozilla/5.0'}\n", 67 | " r=requests.get(url, headers=headers)\n", 68 | " with open(f\"/content/plushies/{name}\", 'wb') as f:\n", 69 | " f.write(r.content)" 70 | ] 71 | }, 72 | { 73 | "cell_type": "code", 74 | "execution_count": null, 75 | "metadata": { 76 | "id": "EhxZkJzt-i3W" 77 | }, 78 | "outputs": [], 79 | "source": [ 80 | "from datasets import load_dataset, Dataset, Image\n", 81 | "\n", 82 | "with open('/content/plushies.txt') as html:\n", 83 | " content = html.read()\n", 84 | "\n", 85 | "texts = []\n", 86 | "images = []\n", 87 | "soup = BeautifulSoup(content)\n", 88 | "for imgtag in soup.find_all('img'):\n", 89 | " texts.append(imgtag['alt'])\n", 90 | " images.append(f\"/content/plushies/{imgtag['src'].split('/')[-1]}\")\n", 91 | " \n", 92 | "ds = Dataset.from_dict({\"image\": images, \"text\": texts})\n", 93 | "ds = ds.cast_column(\"image\", Image())" 94 | ] 95 | }, 96 | { 97 | "cell_type": "code", 98 | "execution_count": null, 99 | "metadata": { 100 | "id": "Q62zPmku_zO7" 101 | }, 102 | "outputs": [], 103 | "source": [ 104 | "ds.push_to_hub(\"camenduru/plushies\")" 105 | ] 106 | }, 107 | { 108 | "cell_type": "code", 109 | "execution_count": null, 110 | "metadata": { 111 | "id": "A_O_uz3P_1V3" 112 | }, 113 | "outputs": [], 114 | "source": [ 115 | "ds = load_dataset(\"camenduru/plushies\")" 116 | ] 117 | }, 118 | { 119 | "cell_type": "code", 120 | "execution_count": null, 121 | "metadata": { 122 | "id": "RwEOGAnh0lGz" 123 | }, 124 | "outputs": [], 125 | "source": [ 126 | "ds.shape" 127 | ] 128 | } 129 | ], 130 | "metadata": { 131 | "colab": { 132 | "provenance": [] 133 | }, 134 | "kernelspec": { 135 | "display_name": "Python 3", 136 | "name": "python3" 137 | }, 138 | "language_info": { 139 | "name": "python" 140 | } 141 | }, 142 | "nbformat": 4, 143 | "nbformat_minor": 0 144 | } 145 | -------------------------------------------------------------------------------- /camenduru's_instant_ngp.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "view-in-github" 7 | }, 8 | "source": [ 9 | "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/notebooks/blob/main/camenduru's_instant_ngp.ipynb)" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "id": "z6vdkVf8_kqk" 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "!pip install -q torch==1.13.1+cu116 torchvision==0.14.1+cu116 torchaudio==0.13.1 torchtext==0.14.1 torchdata==0.5.1 --extra-index-url https://download.pytorch.org/whl/cu116 -U\n", 21 | "# !git lfs install\n", 22 | "# !git clone https://huggingface.co/camenduru/instant-ngp\n", 23 | "\n", 24 | "!wget https://huggingface.co/camenduru/instant-ngp/resolve/main/instant-ngp.zip\n", 25 | "!unzip /content/instant-ngp.zip -d instant-ngp\n", 26 | "\n", 27 | "!pip install commentjson\n", 28 | "%cd /content/instant-ngp\n", 29 | "import os" 30 | ] 31 | }, 32 | { 33 | "cell_type": "code", 34 | "execution_count": null, 35 | "metadata": {}, 36 | "outputs": [], 37 | "source": [ 38 | "scene_path = \"/content/instant-ngp/data/nerf/fox\" #@param {type:\"string\"}\n", 39 | "if not os.path.isdir(scene_path):\n", 40 | " raise NotADirectoryError(scene_path)\n", 41 | "\n", 42 | "train_steps = 2000 #@param {type:\"integer\"}\n", 43 | "snapshot_path = os.path.join(scene_path, f\"{train_steps}.msgpack\")\n", 44 | "!python ./scripts/run.py --scene {scene_path} --mode nerf --n_steps {train_steps} --save_snapshot {snapshot_path}" 45 | ] 46 | } 47 | ], 48 | "metadata": { 49 | "accelerator": "GPU", 50 | "colab": { 51 | "provenance": [] 52 | }, 53 | "gpuClass": "standard", 54 | "kernelspec": { 55 | "display_name": "Python 3", 56 | "name": "python3" 57 | }, 58 | "language_info": { 59 | "name": "python" 60 | } 61 | }, 62 | "nbformat": 4, 63 | "nbformat_minor": 0 64 | } 65 | -------------------------------------------------------------------------------- /camenduru's_karlo.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "view-in-github" 7 | }, 8 | "source": [ 9 | "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/notebooks/blob/main/camenduru's_karlo.ipynb)" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "id": "2_4SeDFVbOF1" 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "!pip install -q torch==1.13.1+cu116 torchvision==0.14.1+cu116 torchaudio==0.13.1 torchtext==0.14.1 torchdata==0.5.1 --extra-index-url https://download.pytorch.org/whl/cu116 -U\n", 21 | "!pip install diffusers transformers accelerate\n", 22 | "import torch\n", 23 | "from diffusers import UnCLIPPipeline\n", 24 | "pipe = UnCLIPPipeline.from_pretrained(\"kakaobrain/karlo-v1-alpha\", torch_dtype=torch.float16).to(\"cuda\")" 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "execution_count": null, 30 | "metadata": { 31 | "id": "7jm2YHTjfk6i" 32 | }, 33 | "outputs": [], 34 | "source": [ 35 | "images = pipe(\"mermaid\", num_images_per_prompt=1).images\n", 36 | "for image in images:\n", 37 | " display(image)" 38 | ] 39 | } 40 | ], 41 | "metadata": { 42 | "accelerator": "GPU", 43 | "colab": { 44 | "provenance": [] 45 | }, 46 | "gpuClass": "standard", 47 | "kernelspec": { 48 | "display_name": "Python 3", 49 | "name": "python3" 50 | }, 51 | "language_info": { 52 | "name": "python" 53 | } 54 | }, 55 | "nbformat": 4, 56 | "nbformat_minor": 0 57 | } 58 | -------------------------------------------------------------------------------- /camenduru's_latent_upscale.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "view-in-github" 7 | }, 8 | "source": [ 9 | "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/notebooks/blob/main/camenduru's_latent_upscale.ipynb)" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "id": "ZsJUxc0aRsAf" 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "!pip install -q torch==1.13.1+cu116 torchvision==0.14.1+cu116 torchaudio==0.13.1 torchtext==0.14.1 torchdata==0.5.1 --extra-index-url https://download.pytorch.org/whl/cu116 -U\n", 21 | "!pip install -q git+https://github.com/huggingface/diffusers.git \n", 22 | "!pip install -q transformers accelerate safetensors xformers==0.0.16 triton==2.0.0\n", 23 | "\n", 24 | "from diffusers import StableDiffusionLatentUpscalePipeline, StableDiffusionPipeline\n", 25 | "import torch\n", 26 | "\n", 27 | "pipeline = StableDiffusionPipeline.from_pretrained(\"ckpt/sd15\", torch_dtype=torch.float16, safety_checker=None).to(\"cuda\")\n", 28 | "pipeline.enable_xformers_memory_efficient_attention()\n", 29 | "upscaler = StableDiffusionLatentUpscalePipeline.from_pretrained(\"stabilityai/sd-x2-latent-upscaler\", torch_dtype=torch.float16).to(\"cuda\")\n", 30 | "upscaler.enable_xformers_memory_efficient_attention()" 31 | ] 32 | }, 33 | { 34 | "cell_type": "code", 35 | "execution_count": null, 36 | "metadata": {}, 37 | "outputs": [], 38 | "source": [ 39 | "prompt = \"duck\"\n", 40 | "generator = torch.manual_seed(420)\n", 41 | "\n", 42 | "latents = pipeline(prompt, generator=generator, output_type=\"latent\").images\n", 43 | "high = upscaler(prompt, generator=generator, image=latents, num_inference_steps=50, guidance_scale=0).images[0]\n", 44 | "\n", 45 | "with torch.no_grad():\n", 46 | " low = pipeline.decode_latents(latents)[0]\n", 47 | "low = pipeline.numpy_to_pil(low)[0]" 48 | ] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "execution_count": null, 53 | "metadata": {}, 54 | "outputs": [], 55 | "source": [ 56 | "display(low)" 57 | ] 58 | }, 59 | { 60 | "cell_type": "code", 61 | "execution_count": null, 62 | "metadata": {}, 63 | "outputs": [], 64 | "source": [ 65 | "display(high)" 66 | ] 67 | } 68 | ], 69 | "metadata": { 70 | "accelerator": "GPU", 71 | "colab": { 72 | "collapsed_sections": [], 73 | "provenance": [] 74 | }, 75 | "gpuClass": "standard", 76 | "kernelspec": { 77 | "display_name": "Python 3 (ipykernel)", 78 | "language": "python", 79 | "name": "python3" 80 | }, 81 | "language_info": { 82 | "codemirror_mode": { 83 | "name": "ipython", 84 | "version": 3 85 | }, 86 | "file_extension": ".py", 87 | "mimetype": "text/x-python", 88 | "name": "python", 89 | "nbconvert_exporter": "python", 90 | "pygments_lexer": "ipython3", 91 | "version": "3.9.9" 92 | } 93 | }, 94 | "nbformat": 4, 95 | "nbformat_minor": 0 96 | } 97 | -------------------------------------------------------------------------------- /camenduru's_lora.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "view-in-github" 7 | }, 8 | "source": [ 9 | "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/notebooks/blob/main/camenduru's_lora.ipynb)" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": {}, 16 | "outputs": [], 17 | "source": [ 18 | "!pip install -q torch==1.13.1+cu116 torchvision==0.14.1+cu116 torchaudio==0.13.1 torchtext==0.14.1 torchdata==0.5.1 --extra-index-url https://download.pytorch.org/whl/cu116 -U\n", 19 | "from google.colab import drive\n", 20 | "drive.mount('/content/drive')" 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": null, 26 | "metadata": { 27 | "id": "2exmZTOBs-RS" 28 | }, 29 | "outputs": [], 30 | "source": [ 31 | "!git clone -b v0.4.4 https://github.com/camenduru/sd-scripts\n", 32 | "!sed -i -e 's/requests==2.28.2/# requests==2.28.2/' /content/sd-scripts/requirements.txt\n", 33 | "%cd /content/sd-scripts\n", 34 | "!pip install -r requirements.txt\n", 35 | "!pip install -q xformers==0.0.16\n", 36 | "!pip install -q triton==2.0.0\n", 37 | "!git clone https://huggingface.co/JosephusCheung/ACertainty /content/ACertainty" 38 | ] 39 | }, 40 | { 41 | "cell_type": "code", 42 | "execution_count": null, 43 | "metadata": {}, 44 | "outputs": [], 45 | "source": [ 46 | "!git -C /content/sd-scripts pull" 47 | ] 48 | }, 49 | { 50 | "cell_type": "code", 51 | "execution_count": null, 52 | "metadata": { 53 | "id": "Q2LwDcv6uHLJ" 54 | }, 55 | "outputs": [], 56 | "source": [ 57 | "!python -u finetune/tag_images_by_wd14_tagger.py /content/drive/MyDrive/AI/training/parkminyoung --repo_id SmilingWolf/wd-v1-4-convnext-tagger-v2 --model_dir wd14_tagger_model --thresh 0.35 --batch_size 1 --caption_extension .txt\n", 58 | "!python -u finetune/merge_dd_tags_to_metadata.py /content/drive/MyDrive/AI/training/parkminyoung /content/drive/MyDrive/AI/training/parkminyoung/parkminyoung.json --caption_extension .txt\n", 59 | "!python -u finetune/prepare_buckets_latents.py /content/drive/MyDrive/AI/training/parkminyoung /content/drive/MyDrive/AI/training/parkminyoung/parkminyoung.json /content/drive/MyDrive/AI/training/parkminyoung/parkminyoung-latents.json /content/ACertainty --batch_size 1 --max_resolution 512,512 --min_bucket_reso 256 --max_bucket_reso 1024 --bucket_reso_steps 64 --mixed_precision no\n", 60 | "!python -u train_network.py --pretrained_model_name_or_path /content/ACertainty --train_data_dir /content/drive/MyDrive/AI/training/parkminyoung --in_json /content/drive/MyDrive/AI/training/parkminyoung/parkminyoung-latents.json --output_dir /content/trained --xformers --max_train_steps 1600 --use_8bit_adam --network_module networks.lora" 61 | ] 62 | }, 63 | { 64 | "cell_type": "code", 65 | "execution_count": null, 66 | "metadata": {}, 67 | "outputs": [], 68 | "source": [ 69 | "!python -u finetune/tag_images_by_wd14_tagger.py /content/drive/MyDrive/AI/training/parkminyoung --repo_id SmilingWolf/wd-v1-4-convnext-tagger-v2 --model_dir wd14_tagger_model --thresh 0.35 --batch_size 1 --caption_extension .txt\n", 70 | "!python -u finetune/merge_dd_tags_to_metadata.py /content/drive/MyDrive/AI/training/parkminyoung /content/drive/MyDrive/AI/training/parkminyoung/parkminyoung.json --caption_extension .txt\n", 71 | "!python -u finetune/prepare_buckets_latents.py /content/drive/MyDrive/AI/training/parkminyoung /content/drive/MyDrive/AI/training/parkminyoung/parkminyoung.json /content/drive/MyDrive/AI/training/parkminyoung/parkminyoung-latents.json /content/ACertainty/ACertainty.ckpt --batch_size 1 --max_resolution 512,512 --min_bucket_reso 256 --max_bucket_reso 1024 --bucket_reso_steps 64 --mixed_precision no\n", 72 | "!python -u train_network.py --pretrained_model_name_or_path /content/ACertainty/ACertainty.ckpt --train_data_dir /content/drive/MyDrive/AI/training/parkminyoung --in_json /content/drive/MyDrive/AI/training/parkminyoung/parkminyoung-latents.json --output_dir /content/trained --xformers --max_train_steps 1600 --use_8bit_adam --network_module networks.lora" 73 | ] 74 | }, 75 | { 76 | "cell_type": "code", 77 | "execution_count": null, 78 | "metadata": { 79 | "id": "MOWZYJU1y4lL" 80 | }, 81 | "outputs": [], 82 | "source": [ 83 | "from diffusers import StableDiffusionPipeline\n", 84 | "pipe = StableDiffusionPipeline.from_pretrained(\"/content/ACertainty\", safety_checker=None).to(\"cuda\")\n", 85 | "pipe.enable_xformers_memory_efficient_attention()\n", 86 | "image = pipe(\"duck\", num_inference_steps=25, guidance_scale=7).images[0]\n", 87 | "display(image)" 88 | ] 89 | }, 90 | { 91 | "cell_type": "code", 92 | "execution_count": null, 93 | "metadata": { 94 | "id": "n4IR-E6rz7oi" 95 | }, 96 | "outputs": [], 97 | "source": [ 98 | "!rm /content/drive/MyDrive/AI/training/parkminyoung/*.txt\n", 99 | "!rm /content/drive/MyDrive/AI/training/parkminyoung/*.npz\n", 100 | "!rm /content/drive/MyDrive/AI/training/parkminyoung/parkminyoung.json\n", 101 | "!rm /content/drive/MyDrive/AI/training/parkminyoung/parkminyoung-latents.json" 102 | ] 103 | } 104 | ], 105 | "metadata": { 106 | "accelerator": "GPU", 107 | "colab": { 108 | "provenance": [] 109 | }, 110 | "gpuClass": "standard", 111 | "kernelspec": { 112 | "display_name": "Python 3", 113 | "name": "python3" 114 | }, 115 | "language_info": { 116 | "name": "python" 117 | } 118 | }, 119 | "nbformat": 4, 120 | "nbformat_minor": 0 121 | } 122 | -------------------------------------------------------------------------------- /camenduru's_peft_finetune_transformers.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "view-in-github" 7 | }, 8 | "source": [ 9 | "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/notebooks/blob/main/camenduru's_peft_finetune_transformers.ipynb)" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "id": "ZsJUxc0aRsAf" 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "# https://github.com/huggingface/peft example modified\n", 21 | "\n", 22 | "!pip install -q bitsandbytes datasets accelerate loralib\n", 23 | "!pip install -q git+https://github.com/huggingface/transformers.git@main \n", 24 | "!pip install -q git+https://github.com/huggingface/peft.git" 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "execution_count": null, 30 | "metadata": {}, 31 | "outputs": [], 32 | "source": [ 33 | "import os\n", 34 | "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"\n", 35 | "import torch\n", 36 | "import torch.nn as nn\n", 37 | "import bitsandbytes as bnb\n", 38 | "from transformers import AutoTokenizer, AutoConfig, AutoModelForCausalLM\n", 39 | "\n", 40 | "model = AutoModelForCausalLM.from_pretrained(\"facebook/opt-6.7b\", load_in_8bit=True, device_map='auto',)\n", 41 | "\n", 42 | "tokenizer = AutoTokenizer.from_pretrained(\"facebook/opt-6.7b\")" 43 | ] 44 | }, 45 | { 46 | "cell_type": "code", 47 | "execution_count": null, 48 | "metadata": {}, 49 | "outputs": [], 50 | "source": [ 51 | "for param in model.parameters():\n", 52 | " param.requires_grad = False # freeze the model - train adapters later\n", 53 | " if param.ndim == 1:\n", 54 | " # cast the small parameters (e.g. layernorm) to fp32 for stability\n", 55 | " param.data = param.data.to(torch.float32)\n", 56 | "\n", 57 | "model.gradient_checkpointing_enable() # reduce number of stored activations\n", 58 | "model.enable_input_require_grads()\n", 59 | "\n", 60 | "class CastOutputToFloat(nn.Sequential):\n", 61 | " def forward(self, x): return super().forward(x).to(torch.float32)\n", 62 | "model.lm_head = CastOutputToFloat(model.lm_head)" 63 | ] 64 | }, 65 | { 66 | "cell_type": "code", 67 | "execution_count": null, 68 | "metadata": {}, 69 | "outputs": [], 70 | "source": [ 71 | "def print_trainable_parameters(model):\n", 72 | " \"\"\"\n", 73 | " Prints the number of trainable parameters in the model.\n", 74 | " \"\"\"\n", 75 | " trainable_params = 0\n", 76 | " all_param = 0\n", 77 | " for _, param in model.named_parameters():\n", 78 | " all_param += param.numel()\n", 79 | " if param.requires_grad:\n", 80 | " trainable_params += param.numel()\n", 81 | " print(\n", 82 | " f\"trainable params: {trainable_params} || all params: {all_param} || trainable%: {100 * trainable_params / all_param}\"\n", 83 | " )" 84 | ] 85 | }, 86 | { 87 | "cell_type": "code", 88 | "execution_count": null, 89 | "metadata": {}, 90 | "outputs": [], 91 | "source": [ 92 | "from peft import LoraConfig, get_peft_model \n", 93 | "\n", 94 | "config = LoraConfig(\n", 95 | " r=16,\n", 96 | " lora_alpha=32,\n", 97 | " target_modules=[\"q_proj\", \"v_proj\"],\n", 98 | " lora_dropout=0.05,\n", 99 | " bias=\"none\",\n", 100 | " task_type=\"CAUSAL_LM\"\n", 101 | ")\n", 102 | "\n", 103 | "model = get_peft_model(model, config)\n", 104 | "print_trainable_parameters(model)" 105 | ] 106 | }, 107 | { 108 | "cell_type": "code", 109 | "execution_count": null, 110 | "metadata": {}, 111 | "outputs": [], 112 | "source": [ 113 | "import transformers\n", 114 | "from datasets import load_dataset\n", 115 | "data = load_dataset(\"Abirate/english_quotes\")\n", 116 | "data = data.map(lambda samples: tokenizer(samples['quote']), batched=True)\n", 117 | "\n", 118 | "trainer = transformers.Trainer(\n", 119 | " model=model, \n", 120 | " train_dataset=data['train'],\n", 121 | " args=transformers.TrainingArguments(\n", 122 | " per_device_train_batch_size=4, \n", 123 | " gradient_accumulation_steps=4,\n", 124 | " warmup_steps=100, \n", 125 | " max_steps=200, \n", 126 | " learning_rate=2e-4, \n", 127 | " fp16=True,\n", 128 | " logging_steps=1, \n", 129 | " output_dir='outputs'\n", 130 | " ),\n", 131 | " data_collator=transformers.DataCollatorForLanguageModeling(tokenizer, mlm=False)\n", 132 | ")\n", 133 | "model.config.use_cache = False # silence the warnings. Please re-enable for inference!\n", 134 | "trainer.train()" 135 | ] 136 | }, 137 | { 138 | "cell_type": "code", 139 | "execution_count": null, 140 | "metadata": {}, 141 | "outputs": [], 142 | "source": [ 143 | "from huggingface_hub import notebook_login\n", 144 | "notebook_login()" 145 | ] 146 | }, 147 | { 148 | "cell_type": "code", 149 | "execution_count": null, 150 | "metadata": {}, 151 | "outputs": [], 152 | "source": [ 153 | "model.push_to_hub(\"ybelkada/opt-6.7b-lora\", use_auth_token=True)" 154 | ] 155 | }, 156 | { 157 | "cell_type": "code", 158 | "execution_count": null, 159 | "metadata": {}, 160 | "outputs": [], 161 | "source": [ 162 | "import torch\n", 163 | "from peft import PeftModel, PeftConfig\n", 164 | "from transformers import AutoModelForCausalLM, AutoTokenizer\n", 165 | "\n", 166 | "peft_model_id = \"ybelkada/opt-6.7b-lora\"\n", 167 | "config = PeftConfig.from_pretrained(peft_model_id)\n", 168 | "model = AutoModelForCausalLM.from_pretrained(config.base_model_name_or_path, return_dict=True, load_in_8bit=True, device_map='auto')\n", 169 | "tokenizer = AutoTokenizer.from_pretrained(config.base_model_name_or_path)\n", 170 | "\n", 171 | "# Load the Lora model\n", 172 | "model = PeftModel.from_pretrained(model, peft_model_id)" 173 | ] 174 | }, 175 | { 176 | "cell_type": "code", 177 | "execution_count": null, 178 | "metadata": {}, 179 | "outputs": [], 180 | "source": [ 181 | "batch = tokenizer(\"Two things are infinite: \", return_tensors='pt')\n", 182 | "\n", 183 | "with torch.cuda.amp.autocast():\n", 184 | " output_tokens = model.generate(**batch, max_new_tokens=50)\n", 185 | "\n", 186 | "print('\\n\\n', tokenizer.decode(output_tokens[0], skip_special_tokens=True))" 187 | ] 188 | } 189 | ], 190 | "metadata": { 191 | "accelerator": "GPU", 192 | "colab": { 193 | "collapsed_sections": [], 194 | "provenance": [] 195 | }, 196 | "gpuClass": "standard", 197 | "kernelspec": { 198 | "display_name": "Python 3 (ipykernel)", 199 | "language": "python", 200 | "name": "python3" 201 | }, 202 | "language_info": { 203 | "codemirror_mode": { 204 | "name": "ipython", 205 | "version": 3 206 | }, 207 | "file_extension": ".py", 208 | "mimetype": "text/x-python", 209 | "name": "python", 210 | "nbconvert_exporter": "python", 211 | "pygments_lexer": "ipython3", 212 | "version": "3.9.9" 213 | } 214 | }, 215 | "nbformat": 4, 216 | "nbformat_minor": 0 217 | } 218 | -------------------------------------------------------------------------------- /camenduru's_peft_finetune_transformers_int8.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "view-in-github" 7 | }, 8 | "source": [ 9 | "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/notebooks/blob/main/camenduru's_peft_finetune_transformers_int8.ipynb)" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "id": "ZsJUxc0aRsAf" 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "# https://github.com/huggingface/peft example modified\n", 21 | "\n", 22 | "!pip install -q bitsandbytes datasets accelerate loralib\n", 23 | "!pip install -q git+https://github.com/huggingface/transformers.git@main \n", 24 | "!pip install -q git+https://github.com/huggingface/peft.git" 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "execution_count": null, 30 | "metadata": {}, 31 | "outputs": [], 32 | "source": [ 33 | "import os\n", 34 | "\n", 35 | "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n", 36 | "import torch\n", 37 | "import torch.nn as nn\n", 38 | "import bitsandbytes as bnb\n", 39 | "from transformers import AutoTokenizer, AutoConfig, AutoModelForCausalLM\n", 40 | "\n", 41 | "model = AutoModelForCausalLM.from_pretrained(\n", 42 | " \"facebook/opt-6.7b\",\n", 43 | " load_in_8bit=True,\n", 44 | " device_map=\"auto\",\n", 45 | ")\n", 46 | "\n", 47 | "tokenizer = AutoTokenizer.from_pretrained(\"facebook/opt-6.7b\")" 48 | ] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "execution_count": null, 53 | "metadata": {}, 54 | "outputs": [], 55 | "source": [ 56 | "from peft import prepare_model_for_int8_training\n", 57 | "\n", 58 | "model = prepare_model_for_int8_training(model)" 59 | ] 60 | }, 61 | { 62 | "cell_type": "code", 63 | "execution_count": null, 64 | "metadata": {}, 65 | "outputs": [], 66 | "source": [ 67 | "def print_trainable_parameters(model):\n", 68 | " \"\"\"\n", 69 | " Prints the number of trainable parameters in the model.\n", 70 | " \"\"\"\n", 71 | " trainable_params = 0\n", 72 | " all_param = 0\n", 73 | " for _, param in model.named_parameters():\n", 74 | " all_param += param.numel()\n", 75 | " if param.requires_grad:\n", 76 | " trainable_params += param.numel()\n", 77 | " print(\n", 78 | " f\"trainable params: {trainable_params} || all params: {all_param} || trainable%: {100 * trainable_params / all_param}\"\n", 79 | " )" 80 | ] 81 | }, 82 | { 83 | "cell_type": "code", 84 | "execution_count": null, 85 | "metadata": {}, 86 | "outputs": [], 87 | "source": [ 88 | "from peft import LoraConfig, get_peft_model\n", 89 | "\n", 90 | "config = LoraConfig(\n", 91 | " r=16, lora_alpha=32, target_modules=[\"q_proj\", \"v_proj\"], lora_dropout=0.05, bias=\"none\", task_type=\"CAUSAL_LM\"\n", 92 | ")\n", 93 | "\n", 94 | "model = get_peft_model(model, config)\n", 95 | "print_trainable_parameters(model)" 96 | ] 97 | }, 98 | { 99 | "cell_type": "code", 100 | "execution_count": null, 101 | "metadata": {}, 102 | "outputs": [], 103 | "source": [ 104 | "import transformers\n", 105 | "from datasets import load_dataset\n", 106 | "\n", 107 | "data = load_dataset(\"Abirate/english_quotes\")\n", 108 | "data = data.map(lambda samples: tokenizer(samples[\"quote\"]), batched=True)\n", 109 | "\n", 110 | "trainer = transformers.Trainer(\n", 111 | " model=model,\n", 112 | " train_dataset=data[\"train\"],\n", 113 | " args=transformers.TrainingArguments(\n", 114 | " per_device_train_batch_size=4,\n", 115 | " gradient_accumulation_steps=4,\n", 116 | " warmup_steps=100,\n", 117 | " max_steps=200,\n", 118 | " learning_rate=2e-4,\n", 119 | " fp16=True,\n", 120 | " logging_steps=1,\n", 121 | " output_dir=\"outputs\",\n", 122 | " ),\n", 123 | " data_collator=transformers.DataCollatorForLanguageModeling(tokenizer, mlm=False),\n", 124 | ")\n", 125 | "model.config.use_cache = False # silence the warnings. Please re-enable for inference!\n", 126 | "trainer.train()" 127 | ] 128 | }, 129 | { 130 | "cell_type": "code", 131 | "execution_count": null, 132 | "metadata": {}, 133 | "outputs": [], 134 | "source": [ 135 | "from huggingface_hub import notebook_login\n", 136 | "notebook_login()" 137 | ] 138 | }, 139 | { 140 | "cell_type": "code", 141 | "execution_count": null, 142 | "metadata": {}, 143 | "outputs": [], 144 | "source": [ 145 | "model.push_to_hub(\"ybelkada/opt-6.7b-lora\", use_auth_token=True)" 146 | ] 147 | }, 148 | { 149 | "cell_type": "code", 150 | "execution_count": null, 151 | "metadata": {}, 152 | "outputs": [], 153 | "source": [ 154 | "import torch\n", 155 | "from peft import PeftModel, PeftConfig\n", 156 | "from transformers import AutoModelForCausalLM, AutoTokenizer\n", 157 | "\n", 158 | "peft_model_id = \"ybelkada/opt-6.7b-lora\"\n", 159 | "config = PeftConfig.from_pretrained(peft_model_id)\n", 160 | "model = AutoModelForCausalLM.from_pretrained(\n", 161 | " config.base_model_name_or_path, return_dict=True, load_in_8bit=True, device_map=\"auto\"\n", 162 | ")\n", 163 | "tokenizer = AutoTokenizer.from_pretrained(config.base_model_name_or_path)\n", 164 | "\n", 165 | "# Load the Lora model\n", 166 | "model = PeftModel.from_pretrained(model, peft_model_id)" 167 | ] 168 | }, 169 | { 170 | "cell_type": "code", 171 | "execution_count": null, 172 | "metadata": {}, 173 | "outputs": [], 174 | "source": [ 175 | "batch = tokenizer(\"Two things are infinite: \", return_tensors=\"pt\")\n", 176 | "\n", 177 | "with torch.cuda.amp.autocast():\n", 178 | " output_tokens = model.generate(**batch, max_new_tokens=50)\n", 179 | "\n", 180 | "print(\"\\n\\n\", tokenizer.decode(output_tokens[0], skip_special_tokens=True))" 181 | ] 182 | } 183 | ], 184 | "metadata": { 185 | "accelerator": "GPU", 186 | "colab": { 187 | "collapsed_sections": [], 188 | "provenance": [] 189 | }, 190 | "gpuClass": "standard", 191 | "kernelspec": { 192 | "display_name": "Python 3 (ipykernel)", 193 | "language": "python", 194 | "name": "python3" 195 | }, 196 | "language_info": { 197 | "codemirror_mode": { 198 | "name": "ipython", 199 | "version": 3 200 | }, 201 | "file_extension": ".py", 202 | "mimetype": "text/x-python", 203 | "name": "python", 204 | "nbconvert_exporter": "python", 205 | "pygments_lexer": "ipython3", 206 | "version": "3.9.9" 207 | } 208 | }, 209 | "nbformat": 4, 210 | "nbformat_minor": 0 211 | } 212 | -------------------------------------------------------------------------------- /camenduru's_replace_vae.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "view-in-github" 7 | }, 8 | "source": [ 9 | "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/notebooks/blob/main/camenduru's_replace_vae.ipynb)" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "id": "-yAt44bHJ2FA" 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "!pip install -q torch==1.13.1+cu116 torchvision==0.14.1+cu116 torchaudio==0.13.1 torchtext==0.14.1 torchdata==0.5.1 --extra-index-url https://download.pytorch.org/whl/cu116 -U\n", 21 | "!pip install huggingface_hub pytorch_lightning" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": null, 27 | "metadata": { 28 | "id": "M_2AkptCHdoN" 29 | }, 30 | "outputs": [], 31 | "source": [ 32 | "# kohya_ss さん、記事ありがとうございます。❤ ❤ https://note.com/kohya_ss/n/nf5893a2e719c \n", 33 | "\n", 34 | "import torch\n", 35 | "\n", 36 | "def merge_vae(ckpt, vae, output):\n", 37 | " print(f\"load checkpoint: {ckpt}\")\n", 38 | " model = torch.load(ckpt, map_location=\"cpu\")\n", 39 | " if \"state_dict\" in model:\n", 40 | " sd = model[\"state_dict\"]\n", 41 | " else:\n", 42 | " sd = model\n", 43 | "\n", 44 | " full_model = False\n", 45 | "\n", 46 | " print(f\"load VAE: {vae}\")\n", 47 | " vae_model = torch.load(vae, map_location=\"cpu\")\n", 48 | " vae_sd = vae_model['state_dict']\n", 49 | "\n", 50 | " for vae_key in vae_sd:\n", 51 | " if vae_key.startswith(\"first_stage_model.\"):\n", 52 | " full_model = True\n", 53 | " break\n", 54 | "\n", 55 | " for vae_key in vae_sd:\n", 56 | " sd_key = vae_key\n", 57 | " if full_model:\n", 58 | " if not sd_key.startswith(\"first_stage_model.\"):\n", 59 | " continue\n", 60 | " else:\n", 61 | " if sd_key not in sd:\n", 62 | " sd_key = \"first_stage_model.\" + sd_key\n", 63 | " if sd_key not in sd:\n", 64 | " print(f\"key not exists in model: {vae_key}\")\n", 65 | " continue\n", 66 | " sd[sd_key] = vae_sd[vae_key]\n", 67 | "\n", 68 | " print(f\"saving checkpoint to: {output}\")\n", 69 | " torch.save(model, output)\n", 70 | "\n", 71 | "!wget https://huggingface.co/andite/anything-v4.0/resolve/main/anything-v4.5-pruned.ckpt\n", 72 | "!wget https://huggingface.co/andite/anything-v4.0/resolve/main/anything-v4.0.vae.pt\n", 73 | "!mkdir /content/swapped\n", 74 | "merge_vae(\"/content/anything-v4.5-pruned.ckpt\", \"/content/anything-v4.0.vae.pt\", \"/content/swapped/anything-v4.5-vae-swapped.ckpt\")" 75 | ] 76 | }, 77 | { 78 | "cell_type": "code", 79 | "execution_count": null, 80 | "metadata": { 81 | "id": "OWB0IWajJB1r" 82 | }, 83 | "outputs": [], 84 | "source": [ 85 | "import os\n", 86 | "from huggingface_hub import create_repo, upload_folder\n", 87 | "\n", 88 | "hf_token=\"\"\n", 89 | "repo_id = \"ckpt/anything-v4.5-vae-swapped\"\n", 90 | "path_in_repo = \"\"\n", 91 | "create_repo(repo_id, private=True, token=hf_token)\n", 92 | "upload_folder(folder_path=\"/content/swapped\", path_in_repo=path_in_repo, repo_id=repo_id, commit_message=f\"anything-v4.5-vae-swapped\", token=hf_token)" 93 | ] 94 | } 95 | ], 96 | "metadata": { 97 | "colab": { 98 | "provenance": [] 99 | }, 100 | "kernelspec": { 101 | "display_name": "Python 3", 102 | "name": "python3" 103 | }, 104 | "language_info": { 105 | "name": "python" 106 | } 107 | }, 108 | "nbformat": 4, 109 | "nbformat_minor": 0 110 | } 111 | -------------------------------------------------------------------------------- /camenduru's_sd_image_variations.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "view-in-github" 7 | }, 8 | "source": [ 9 | "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/notebooks/blob/main/camenduru's_sd_image_variations.ipynb)" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "id": "L6mOBiSq8idf" 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "!pip install -q torch==1.13.1+cu116 torchvision==0.14.1+cu116 torchaudio==0.13.1 torchtext==0.14.1 torchdata==0.5.1 --extra-index-url https://download.pytorch.org/whl/cu116 -U\n", 21 | "!git clone https://huggingface.co/spaces/lambdalabs/stable-diffusion-image-variations\n", 22 | "%cd stable-diffusion-image-variations/\n", 23 | "!pip install -qq -r requirements.txt\n", 24 | "!pip install -qq gradio" 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "execution_count": null, 30 | "metadata": { 31 | "id": "Fe3bzAto8uX5" 32 | }, 33 | "outputs": [], 34 | "source": [ 35 | "import gradio as gr\n", 36 | "import torch\n", 37 | "from PIL import Image\n", 38 | "\n", 39 | "from lambda_diffusers import StableDiffusionImageEmbedPipeline\n", 40 | "\n", 41 | "def main(\n", 42 | " input_im,\n", 43 | " scale=3.0,\n", 44 | " n_samples=4,\n", 45 | " steps=25,\n", 46 | " seed=0,\n", 47 | " ):\n", 48 | " generator = torch.Generator(device=device).manual_seed(int(seed))\n", 49 | "\n", 50 | " images_list = pipe(\n", 51 | " n_samples*[input_im],\n", 52 | " guidance_scale=scale,\n", 53 | " num_inference_steps=steps,\n", 54 | " generator=generator,\n", 55 | " )\n", 56 | "\n", 57 | " images = []\n", 58 | " for i, image in enumerate(images_list[\"sample\"]):\n", 59 | " # if(images_list[\"nsfw_content_detected\"][i]):\n", 60 | " # safe_image = Image.open(r\"unsafe.png\")\n", 61 | " # images.append(safe_image)\n", 62 | " # else:\n", 63 | " images.append(image)\n", 64 | " return images\n", 65 | "\n", 66 | "device = \"cuda\"\n", 67 | "pipe = StableDiffusionImageEmbedPipeline.from_pretrained(\n", 68 | " \"lambdalabs/sd-image-variations-diffusers\",\n", 69 | " revision=\"273115e88df42350019ef4d628265b8c29ef4af5\",\n", 70 | " )\n", 71 | "pipe = pipe.to(device)\n", 72 | "pipe.safety_checker = lambda images, clip_input: (images, False)\n", 73 | "\n", 74 | "inputs = [\n", 75 | " gr.Image(),\n", 76 | " gr.Slider(0, 25, value=3, step=1, label=\"Guidance scale\"),\n", 77 | " gr.Slider(1, 4, value=1, step=1, label=\"Number images\"),\n", 78 | " gr.Slider(5, 50, value=25, step=5, label=\"Steps\"),\n", 79 | " gr.Number(0, labal=\"Seed\", precision=0)\n", 80 | "]\n", 81 | "output = gr.Gallery(label=\"Generated variations\")\n", 82 | "output.style(grid=2)\n", 83 | "\n", 84 | "demo = gr.Interface(\n", 85 | " fn=main,\n", 86 | " inputs=inputs,\n", 87 | " outputs=output,\n", 88 | " )\n", 89 | "demo.launch(inline=False, debug=True)\n" 90 | ] 91 | } 92 | ], 93 | "metadata": { 94 | "accelerator": "GPU", 95 | "colab": { 96 | "collapsed_sections": [], 97 | "machine_shape": "hm", 98 | "provenance": [] 99 | }, 100 | "gpuClass": "standard", 101 | "kernelspec": { 102 | "display_name": "Python 3", 103 | "name": "python3" 104 | }, 105 | "language_info": { 106 | "name": "python" 107 | } 108 | }, 109 | "nbformat": 4, 110 | "nbformat_minor": 0 111 | } 112 | -------------------------------------------------------------------------------- /camenduru's_stable_diffusion.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "view-in-github" 7 | }, 8 | "source": [ 9 | "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/notebooks/blob/main/camenduru's_stable_diffusion.ipynb)" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "id": "F1WTBlIF4SyA" 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "!pip install -q torch==1.13.1+cu116 torchvision==0.14.1+cu116 torchaudio==0.13.1 torchtext==0.14.1 torchdata==0.5.1 --extra-index-url https://download.pytorch.org/whl/cu116 -U\n", 21 | "from google.colab import drive\n", 22 | "drive.mount('/content/gdrive')\n", 23 | "\n", 24 | "!pip install -qq git+https://github.com/huggingface/diffusers.git\n", 25 | "!pip install -qq transformers ftfy\n", 26 | "\n", 27 | "import torch, os, gc\n", 28 | "from diffusers import StableDiffusionPipeline\n", 29 | "from torch import autocast\n", 30 | "from IPython.display import clear_output\n", 31 | "\n", 32 | "batch_idx = 0\n", 33 | "outputs_path = \"/content/gdrive/MyDrive/AI/StableDiffusion\"\n", 34 | "clear_output()" 35 | ] 36 | }, 37 | { 38 | "cell_type": "code", 39 | "execution_count": null, 40 | "metadata": { 41 | "id": "yXHMNznPdlkX" 42 | }, 43 | "outputs": [], 44 | "source": [ 45 | "is_tile = False #@param {type:'boolean'}\n", 46 | "if(is_tile):\n", 47 | "\tdef patch_conv(cls):\n", 48 | "\t\tinit = cls.__init__\n", 49 | "\t\tdef __init__(self, *args, **kwargs):\n", 50 | "\t\t\treturn init(self, *args, **kwargs, padding_mode='circular')\n", 51 | "\t\tcls.__init__ = __init__\n", 52 | "\tpatch_conv(torch.nn.Conv2d)\n", 53 | " \n", 54 | "pipe = StableDiffusionPipeline.from_pretrained(\"CompVis/stable-diffusion-v1-4\", revision=\"fp16\", torch_dtype=torch.float16, use_auth_token=\"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\").to(\"cuda\")\n", 55 | "pipe.safety_checker = lambda images, clip_input: (images, False)\n", 56 | "clear_output()" 57 | ] 58 | }, 59 | { 60 | "cell_type": "code", 61 | "execution_count": null, 62 | "metadata": { 63 | "id": "nVBs8Pai52gS" 64 | }, 65 | "outputs": [], 66 | "source": [ 67 | "from PIL.PngImagePlugin import PngInfo\n", 68 | "metadata = PngInfo()\n", 69 | " \n", 70 | "images_path_name = \"000\" #@param {type:'string'}\n", 71 | "num_batch_images = 1000 \n", 72 | "height = 512 #@param {type:'integer'}\n", 73 | "width = 512 #@param {type:'integer'}\n", 74 | "\n", 75 | "path = os.path.join(outputs_path, images_path_name)\n", 76 | "\n", 77 | "if os.path.exists(path) == False:\n", 78 | " os.mkdir(path)\n", 79 | "\n", 80 | "prompt = 'cat' #@param {type:'string'}\n", 81 | "metadata.add_text(\"Prompt\", f\"{prompt}\")\n", 82 | "by = 'camenduru' #@param {type:'string'}\n", 83 | "metadata.add_text(\"by\", f\"{by}\")\n", 84 | "\n", 85 | "for i in range(num_batch_images):\n", 86 | " gc.collect()\n", 87 | " torch.cuda.empty_cache()\n", 88 | " with autocast(\"cuda\"):\n", 89 | " image = pipe(prompt, height=height, width=width, num_inference_steps=50, eta=0.0, guidance_scale=7.5)[\"sample\"][0]\n", 90 | "\n", 91 | " image.save(f\"{path}/{images_path_name}_{batch_idx}.png\", pnginfo=metadata)\n", 92 | " batch_idx += 1" 93 | ] 94 | } 95 | ], 96 | "metadata": { 97 | "accelerator": "GPU", 98 | "colab": { 99 | "collapsed_sections": [], 100 | "provenance": [] 101 | }, 102 | "gpuClass": "standard", 103 | "kernelspec": { 104 | "display_name": "Python 3", 105 | "name": "python3" 106 | }, 107 | "language_info": { 108 | "name": "python" 109 | } 110 | }, 111 | "nbformat": 4, 112 | "nbformat_minor": 0 113 | } 114 | -------------------------------------------------------------------------------- /camenduru's_stable_diffusion_discord.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "view-in-github" 7 | }, 8 | "source": [ 9 | "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/notebooks/blob/main/camenduru's_stable_diffusion_discord.ipynb)" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "id": "F1WTBlIF4SyA" 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "!pip install -q torch==1.13.1+cu116 torchvision==0.14.1+cu116 torchaudio==0.13.1 torchtext==0.14.1 torchdata==0.5.1 --extra-index-url https://download.pytorch.org/whl/cu116 -U\n", 21 | "!pip install -qq git+https://github.com/huggingface/diffusers.git\n", 22 | "!pip install -qq transformers ftfy\n", 23 | "\n", 24 | "import torch, os, gc, requests, random\n", 25 | "from diffusers import StableDiffusionPipeline\n", 26 | "from torch import autocast\n", 27 | "from IPython.display import clear_output\n", 28 | "clear_output()" 29 | ] 30 | }, 31 | { 32 | "cell_type": "code", 33 | "execution_count": null, 34 | "metadata": { 35 | "id": "MBkgalN4dHV9" 36 | }, 37 | "outputs": [], 38 | "source": [ 39 | "is_tile = False #@param {type: 'boolean'}\n", 40 | "if(is_tile):\n", 41 | "\tdef patch_conv(cls):\n", 42 | "\t\tinit = cls.__init__\n", 43 | "\t\tdef __init__(self, *args, **kwargs):\n", 44 | "\t\t\treturn init(self, *args, **kwargs, padding_mode='circular')\n", 45 | "\t\tcls.__init__ = __init__\n", 46 | "\tpatch_conv(torch.nn.Conv2d)\n", 47 | " \n", 48 | "pipe = StableDiffusionPipeline.from_pretrained(\"CompVis/stable-diffusion-v1-4\", revision=\"fp16\", torch_dtype=torch.float16, use_auth_token=\"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\").to(\"cuda\")\n", 49 | "pipe.safety_checker = lambda images, clip_input: (images, False)\n", 50 | "clear_output()" 51 | ] 52 | }, 53 | { 54 | "cell_type": "code", 55 | "execution_count": null, 56 | "metadata": { 57 | "id": "7tOGrS5lA6-D" 58 | }, 59 | "outputs": [], 60 | "source": [ 61 | "token = '' #@param {type: 'string'}\n", 62 | "header = {\"authorization\": f\"Bot {token}\"}\n", 63 | "by = 'camenduru' #@param {type: 'string'}\n", 64 | "\n", 65 | "from PIL.PngImagePlugin import PngInfo\n", 66 | "metadata = PngInfo()\n", 67 | " \n", 68 | "height = 512 #@param {type: 'integer'}\n", 69 | "width = 512 #@param {type: 'integer'}\n", 70 | "\n", 71 | "def generate(prompt):\n", 72 | " metadata.add_text(\"Prompt\", f\"{prompt}\")\n", 73 | " metadata.add_text(\"by\", f\"{by}\")\n", 74 | " gc.collect()\n", 75 | " torch.cuda.empty_cache()\n", 76 | " with autocast(\"cuda\"):\n", 77 | " image = pipe(prompt, height=height, width=width, num_inference_steps=50, eta=0.0, guidance_scale=7.5)[\"sample\"][0]\n", 78 | " random_name = random.randint(0,10000000000)\n", 79 | " image.save(f\"{random_name}.png\", pnginfo=metadata)\n", 80 | " files = {f\"{random_name}.png\" : open(f\"{random_name}.png\", \"rb\").read()}\n", 81 | " payload = {\"content\":f\"{prompt}\"}\n", 82 | " channel_id = 0 #@param {type: 'integer'}\n", 83 | " r = requests.post(f\"https://discord.com/api/v9/channels/{channel_id}/messages\", data=payload, headers=header, files=files).text\n", 84 | " os.remove(f\"{random_name}.png\")\n", 85 | " clear_output()\n", 86 | "\n", 87 | "is_from_prompts_txt = True #@param {type: 'boolean'}\n", 88 | "prompts_txt = 'prompts.txt' #@param {type: 'string'}\n", 89 | "if(is_from_prompts_txt):\n", 90 | " while True:\n", 91 | " with open(f'{prompts_txt}', \"r\") as file:\n", 92 | " prompts = file.readlines()\n", 93 | " for prompt in prompts:\n", 94 | " generate(prompt)\n", 95 | "else:\n", 96 | " while True:\n", 97 | " prompt = '' #@param {type: 'string'}\n", 98 | " generate(prompt)" 99 | ] 100 | } 101 | ], 102 | "metadata": { 103 | "accelerator": "GPU", 104 | "colab": { 105 | "collapsed_sections": [], 106 | "provenance": [] 107 | }, 108 | "gpuClass": "standard", 109 | "kernelspec": { 110 | "display_name": "Python 3", 111 | "name": "python3" 112 | }, 113 | "language_info": { 114 | "name": "python" 115 | } 116 | }, 117 | "nbformat": 4, 118 | "nbformat_minor": 0 119 | } 120 | -------------------------------------------------------------------------------- /camenduru's_stable_diffusion_flax.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "view-in-github" 7 | }, 8 | "source": [ 9 | "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/notebooks/blob/main/camenduru's_stable_diffusion_flax.ipynb)" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "id": "BH7CrFTy81w9" 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "!pip install -q torch==1.13.1+cu116 torchvision==0.14.1+cu116 torchaudio==0.13.1 torchtext==0.14.1 torchdata==0.5.1 --extra-index-url https://download.pytorch.org/whl/cu116 -U\n", 21 | "from google.colab import drive\n", 22 | "drive.mount('/content/gdrive')\n", 23 | "\n", 24 | "!pip install git+https://github.com/huggingface/diffusers.git\n", 25 | "!pip install --upgrade jax jaxlib \n", 26 | "\n", 27 | "import jax.tools.colab_tpu\n", 28 | "jax.tools.colab_tpu.setup_tpu('tpu_driver_20221011')\n", 29 | "\n", 30 | "!pip install flax transformers ftfy\n", 31 | "jax.devices()\n", 32 | "\n", 33 | "import jax\n", 34 | "import numpy as np\n", 35 | "from flax.jax_utils import replicate\n", 36 | "from flax.training.common_utils import shard\n", 37 | "\n", 38 | "import os, gc, requests, subprocess, random\n", 39 | "from diffusers import FlaxStableDiffusionPipeline\n", 40 | "\n", 41 | "from IPython.display import clear_output\n", 42 | "clear_output()" 43 | ] 44 | }, 45 | { 46 | "cell_type": "code", 47 | "execution_count": null, 48 | "metadata": { 49 | "id": "c8WiZyYL9R3V" 50 | }, 51 | "outputs": [], 52 | "source": [ 53 | "from huggingface_hub import notebook_login\n", 54 | "!git config --global credential.helper store\n", 55 | "notebook_login()" 56 | ] 57 | }, 58 | { 59 | "cell_type": "code", 60 | "execution_count": 9, 61 | "metadata": { 62 | "id": "NgFF2Rop9Z4H" 63 | }, 64 | "outputs": [], 65 | "source": [ 66 | "pipe, params = FlaxStableDiffusionPipeline.from_pretrained(\"CompVis/stable-diffusion-v1-4\", revision=\"bf16\", dtype=jax.numpy.bfloat16, safety_checker=None)\n", 67 | "params = replicate(params)\n", 68 | "clear_output()" 69 | ] 70 | }, 71 | { 72 | "cell_type": "code", 73 | "execution_count": null, 74 | "metadata": { 75 | "id": "78Bcq6JH9rXL" 76 | }, 77 | "outputs": [], 78 | "source": [ 79 | "from PIL import Image\n", 80 | "def image_grid(imgs, rows, cols):\n", 81 | " w,h = imgs[0].size\n", 82 | " grid = Image.new('RGB', size=(cols*w, rows*h))\n", 83 | " for i, img in enumerate(imgs): grid.paste(img, box=(i%cols*w, i//cols*h))\n", 84 | " return grid\n", 85 | "\n", 86 | "token = '' #@param {type: 'string'}\n", 87 | "channel_id = 0 #@param {type: 'integer'}\n", 88 | "header = {\"authorization\": f\"Bot {token}\"}\n", 89 | "by = 'camenduru' #@param {type: 'string'}\n", 90 | "\n", 91 | "root_folder = '/content/gdrive/MyDrive/AI/StableDiffusion' #@param {type: 'string'}\n", 92 | "image_folder = '000' #@param {type: 'string'}\n", 93 | "if os.path.exists(f\"{root_folder}/{image_folder}\") == False:\n", 94 | " os.mkdir(f\"{root_folder}/{image_folder}\")\n", 95 | "name = max([int(f[:f.index('.')]) for f in os.listdir(f\"{root_folder}/{image_folder}\")], default=0)\n", 96 | "\n", 97 | "from PIL.PngImagePlugin import PngInfo\n", 98 | "metadata = PngInfo()\n", 99 | " \n", 100 | "height = 448 #@param {type: 'integer'}\n", 101 | "width = 832 #@param {type: 'integer'}\n", 102 | "\n", 103 | "def generate(prompt, name):\n", 104 | " metadata.add_text(\"Prompt\", f\"{prompt}\")\n", 105 | " metadata.add_text(\"by\", f\"{by}\")\n", 106 | " gc.collect()\n", 107 | " real_seed = random.randint(0, 2147483647)\n", 108 | " prng_seed = jax.random.PRNGKey(real_seed)\n", 109 | " num_samples = jax.device_count()\n", 110 | " prompt_n = num_samples * [prompt]\n", 111 | " prompt_ids = pipe.prepare_inputs(prompt_n)\n", 112 | " prng_seed = jax.random.split(prng_seed, jax.device_count())\n", 113 | " prompt_ids = shard(prompt_ids)\n", 114 | " images = pipe(prompt_ids, params, prng_seed, num_inference_steps=50, height=height, width=width, guidance_scale=7.5, jit=True).images\n", 115 | " images = pipe.numpy_to_pil(np.asarray(images.reshape((num_samples,) + images.shape[-3:])))\n", 116 | " image = image_grid(images, 2, 4)\n", 117 | " image.save(f\"{root_folder}/{image_folder}/{name:04}.png\", pnginfo=metadata)\n", 118 | " files = {f\"{image_folder}_{name:04}.png\" : open(f\"{root_folder}/{image_folder}/{name:04}.png\", \"rb\").read()}\n", 119 | " payload = {\"content\":f\"{prompt}\"}\n", 120 | " r = requests.post(f\"https://discord.com/api/v9/channels/{channel_id}/messages\", data=payload, headers=header, files=files).text\n", 121 | " clear_output()\n", 122 | "\n", 123 | "max_files = 100 #@param {type: 'integer'}\n", 124 | "is_from_prompts_txt = False #@param {type: 'boolean'}\n", 125 | "prompts_txt = 'prompts.txt' #@param {type: 'string'}\n", 126 | "if(is_from_prompts_txt):\n", 127 | " while name < max_files:\n", 128 | " with open(f'{prompts_txt}', \"r\") as file:\n", 129 | " prompts = file.readlines()\n", 130 | " for prompt in prompts:\n", 131 | " name += 1\n", 132 | " generate(prompt, name)\n", 133 | "else:\n", 134 | " while name < max_files:\n", 135 | " prompt = 'panda by Mike Winkelmann Beeple, ultra-detailed pen and ink illustration' #@param {type: 'string'}\n", 136 | " name += 1\n", 137 | " generate(prompt, name)\n" 138 | ] 139 | } 140 | ], 141 | "metadata": { 142 | "accelerator": "TPU", 143 | "colab": { 144 | "provenance": [] 145 | }, 146 | "kernelspec": { 147 | "display_name": "Python 3", 148 | "name": "python3" 149 | }, 150 | "language_info": { 151 | "name": "python" 152 | } 153 | }, 154 | "nbformat": 4, 155 | "nbformat_minor": 0 156 | } 157 | -------------------------------------------------------------------------------- /camenduru's_stable_diffusion_ui_gradio.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "view-in-github" 7 | }, 8 | "source": [ 9 | "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/notebooks/blob/main/camenduru's_stable_diffusion_ui_gradio.ipynb)" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "id": "SaAJk33ppFw1" 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "!pip install -q torch==1.13.1+cu116 torchvision==0.14.1+cu116 torchaudio==0.13.1 torchtext==0.14.1 torchdata==0.5.1 --extra-index-url https://download.pytorch.org/whl/cu116 -U\n", 21 | "!git clone https://github.com/camenduru/stable-diffusion-webui\n", 22 | "!wget -q https://gist.githubusercontent.com/camenduru/9ec5f8141db9902e375967e93250860f/raw/c1a03eb447548adbef1858c0e69d3567a390d2f4/run_n_times.py -O /content/stable-diffusion-webui/scripts/run_n_times.py\n", 23 | "%cd stable-diffusion-webui\n", 24 | "\n", 25 | "!sed -i -e 's/numpy==1.23.3/numpy==1.21.6/g' requirements_versions.txt\n", 26 | "!sed -i -e 's/fairscale==0.4.9/fairscale==0.4.6/g' requirements_versions.txt\n", 27 | "\n", 28 | "!wget https://huggingface.co/camenduru/sd14/resolve/main/sd-v1-4.ckpt -O /content/stable-diffusion-webui/models/Stable-diffusion/sd-v1-4.ckpt\n", 29 | "!wget https://huggingface.co/hakurei/waifu-diffusion-v1-3/resolve/main/wd-v1-3-float32.ckpt -O /content/stable-diffusion-webui/models/Stable-diffusion/wd-v1-3-float32.ckpt\n", 30 | "\n", 31 | "!python launch.py --share --opt-split-attention" 32 | ] 33 | }, 34 | { 35 | "cell_type": "code", 36 | "execution_count": null, 37 | "metadata": { 38 | "id": "R3xWyVKLY2o_" 39 | }, 40 | "outputs": [], 41 | "source": [ 42 | "!zip -r /content/outputs.zip outputs/" 43 | ] 44 | } 45 | ], 46 | "metadata": { 47 | "accelerator": "GPU", 48 | "colab": { 49 | "collapsed_sections": [], 50 | "private_outputs": true, 51 | "provenance": [] 52 | }, 53 | "gpuClass": "standard", 54 | "kernelspec": { 55 | "display_name": "Python 3", 56 | "name": "python3" 57 | }, 58 | "language_info": { 59 | "name": "python" 60 | } 61 | }, 62 | "nbformat": 4, 63 | "nbformat_minor": 0 64 | } 65 | -------------------------------------------------------------------------------- /camenduru's_stable_diffusion_ui_gradio_drive.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "view-in-github" 7 | }, 8 | "source": [ 9 | "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/notebooks/blob/main/camenduru's_stable_diffusion_ui_gradio_drive.ipynb)" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "id": "SaAJk33ppFw1" 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "!pip install -q torch==1.13.1+cu116 torchvision==0.14.1+cu116 torchaudio==0.13.1 torchtext==0.14.1 torchdata==0.5.1 --extra-index-url https://download.pytorch.org/whl/cu116 -U\n", 21 | "import os\n", 22 | "from google.colab import drive\n", 23 | "drive.mount('/content/gdrive')\n", 24 | "\n", 25 | "!pip install -q https://github.com/camenduru/stable-diffusion-webui-colab/releases/download/0.0.15/xformers-0.0.15.dev0+189828c.d20221207-cp38-cp38-linux_x86_64.whl\n", 26 | "\n", 27 | "!git clone -b v1.6 https://github.com/camenduru/stable-diffusion-webui\n", 28 | "!wget -q https://gist.githubusercontent.com/camenduru/9ec5f8141db9902e375967e93250860f/raw/b5c741676c5514105b9a1ea7dd438ca83802f16f/run_n_times.py -O /content/stable-diffusion-webui/scripts/run_n_times.py\n", 29 | "!wget -q https://gist.githubusercontent.com/camenduru/125b2992422ca294d15a392ad0e35d7d/raw/09493801313438128b4268bfa86f81ac781e7a60/post_to_discord.py -O /content/stable-diffusion-webui/scripts/post_to_discord.py\n", 30 | "%cd stable-diffusion-webui\n", 31 | "\n", 32 | "!sed -i -e 's/numpy==1.23.3/numpy==1.21.6/g' requirements_versions.txt\n", 33 | "!sed -i -e 's/fairscale==0.4.9/fairscale==0.4.6/g' requirements_versions.txt\n", 34 | "!sed -i -e 's/ tensorflow==2.10.0 tensorflow-io==0.27.0//g' launch.py\n", 35 | "!sed -i -e 's/ and deepdanbooru//g' launch.py\n", 36 | "\n", 37 | "!sed -i -e 's/infotexts=None/infotexts=None, fullfn=None/g' modules/processing.py\n", 38 | "!sed -i -e 's/self.eta = p.eta/self.eta = p.eta\\n self.fullfn = fullfn/g' modules/processing.py\n", 39 | "!sed -i -e 's/images.save_image/fullfn = images.save_image/g' modules/processing.py\n", 40 | "!sed -i -e 's/infotexts=infotexts/infotexts=infotexts, fullfn=fullfn/g' modules/processing.py\n", 41 | "\n", 42 | "if not (os.path.exists(\"/content/gdrive/MyDrive/AI\")):\n", 43 | " os.system(\"mkdir /content/gdrive/MyDrive/AI\")\n", 44 | "if not (os.path.exists(\"/content/gdrive/MyDrive/AI/StableDiffusion\")):\n", 45 | " os.system(\"mkdir /content/gdrive/MyDrive/AI/StableDiffusion\")\n", 46 | "if not (os.path.exists(\"/content/gdrive/MyDrive/AI/StableDiffusion/models\")):\n", 47 | " os.system(\"mkdir /content/gdrive/MyDrive/AI/StableDiffusion/models\")\n", 48 | "if not (os.path.exists(\"/content/gdrive/MyDrive/AI/StableDiffusion/models/embeddings\")):\n", 49 | " os.system(\"mkdir /content/gdrive/MyDrive/AI/StableDiffusion/models/embeddings\")" 50 | ] 51 | }, 52 | { 53 | "cell_type": "code", 54 | "execution_count": null, 55 | "metadata": { 56 | "id": "sTG8IisM4mKN" 57 | }, 58 | "outputs": [], 59 | "source": [ 60 | "#@markdown Which ai model you want? You should select at least one model.\n", 61 | "stable = False #@param {type:\"boolean\"}\n", 62 | "waifu = False #@param {type:\"boolean\"}\n", 63 | "hentai = False #@param {type:\"boolean\"}\n", 64 | "furry = False #@param {type:\"boolean\"}\n", 65 | "anime = True #@param {type:\"boolean\"}\n", 66 | "anime_sfw = False #@param {type:\"boolean\"}\n", 67 | "\n", 68 | "if(stable):\n", 69 | " if not (os.path.isfile(\"/content/stable-diffusion-webui/models/Stable-diffusion/sd-v1-4.ckpt\")):\n", 70 | " os.system(\"wget https://huggingface.co/camenduru/sd14/resolve/main/sd-v1-4.ckpt -O /content/stable-diffusion-webui/models/Stable-diffusion/sd-v1-4.ckpt\")\n", 71 | "if(waifu):\n", 72 | " if not (os.path.isfile(\"/content/stable-diffusion-webui/models/Stable-diffusion/wd-v1-3-float32.ckpt\")):\n", 73 | " os.system(\"wget https://huggingface.co/hakurei/waifu-diffusion-v1-3/resolve/main/wd-v1-3-float32.ckpt -O /content/stable-diffusion-webui/models/Stable-diffusion/wd-v1-3-float32.ckpt\")\n", 74 | "if(hentai):\n", 75 | " if not (os.path.isfile(\"/content/stable-diffusion-webui/models/Stable-diffusion/1412p.ckpt\")):\n", 76 | " os.system(\"wget https://lilpotat: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@huggingface.co/lilpotat/1412p/resolve/main/1412p.ckpt -O /content/stable-diffusion-webui/models/Stable-diffusion/1412p.ckpt\")\n", 77 | "if(furry):\n", 78 | " if not (os.path.isfile(\"/content/stable-diffusion-webui/models/Stable-diffusion/hd.ckpt\")):\n", 79 | " os.system(\"wget https://lilpotat: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@huggingface.co/lilpotat/y-e18/resolve/main/y-e18.ckpt -O /content/stable-diffusion-webui/models/Stable-diffusion/y-e18.ckpt\")\n", 80 | "if(anime):\n", 81 | " if not (os.path.isfile(\"/content/stable-diffusion-webui/models/Stable-diffusion/anime.ckpt\")):\n", 82 | " os.system(\"wget https://lilpotat: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@huggingface.co/lilpotat/ani/resolve/main/afp.ckpt -O /content/stable-diffusion-webui/models/Stable-diffusion/anime.ckpt\")\n", 83 | " os.system(\"wget https://gitlab.com/HappyLilPotat/anime/-/raw/master/anime.vae.pt -O /content/stable-diffusion-webui/models/Stable-diffusion/anime.vae.pt\")\n", 84 | " if not (os.path.exists(\"/content/stable-diffusion-webui/models/hypernetworks\")):\n", 85 | " os.system(\"mkdir /content/stable-diffusion-webui/models/hypernetworks\")\n", 86 | " os.system(\"wget https://gitlab.com/HappyLilPotat/anime/-/raw/master/1.pt -O /content/stable-diffusion-webui/models/hypernetworks/1.pt\")\n", 87 | " os.system(\"wget https://gitlab.com/HappyLilPotat/anime/-/raw/master/2.pt -O /content/stable-diffusion-webui/models/hypernetworks/2.pt\")\n", 88 | " os.system(\"wget https://gitlab.com/HappyLilPotat/anime/-/raw/master/3.pt -O /content/stable-diffusion-webui/models/hypernetworks/3.pt\")\n", 89 | " os.system(\"wget https://gitlab.com/HappyLilPotat/anime/-/raw/master/4.pt -O /content/stable-diffusion-webui/models/hypernetworks/4.pt\")\n", 90 | " os.system(\"wget https://gitlab.com/HappyLilPotat/anime/-/raw/master/5.pt -O /content/stable-diffusion-webui/models/hypernetworks/5.pt\")\n", 91 | " os.system(\"wget https://gitlab.com/HappyLilPotat/anime/-/raw/master/6.pt -O /content/stable-diffusion-webui/models/hypernetworks/6.pt\")\n", 92 | " os.system(\"wget https://gitlab.com/HappyLilPotat/anime/-/raw/master/7.pt -O /content/stable-diffusion-webui/models/hypernetworks/7.pt\")\n", 93 | " os.system(\"wget https://gitlab.com/HappyLilPotat/anime/-/raw/master/8.pt -O /content/stable-diffusion-webui/models/hypernetworks/8.pt\")\n", 94 | " os.system(\"wget https://gitlab.com/HappyLilPotat/anime/-/raw/master/9.pt -O /content/stable-diffusion-webui/models/hypernetworks/9.pt\")\n", 95 | " os.system(\"wget https://gitlab.com/HappyLilPotat/anime/-/raw/master/10.pt -O /content/stable-diffusion-webui/models/hypernetworks/10.pt\")\n", 96 | " os.system(\"wget https://gitlab.com/HappyLilPotat/anime/-/raw/master/11.pt -O /content/stable-diffusion-webui/models/hypernetworks/11.pt\")\n", 97 | " os.system(\"wget https://gitlab.com/HappyLilPotat/anime/-/raw/master/12.pt -O /content/stable-diffusion-webui/models/hypernetworks/12.pt\")\n", 98 | "if(anime_sfw):\n", 99 | " if not (os.path.isfile(\"/content/stable-diffusion-webui/models/Stable-diffusion/anime_sfw.ckpt\")):\n", 100 | " os.system(\"wget https://lilpotat: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@huggingface.co/lilpotat/anis/resolve/main/asfp.ckpt -O /content/stable-diffusion-webui/models/Stable-diffusion/anime_sfw.ckpt\")" 101 | ] 102 | }, 103 | { 104 | "cell_type": "code", 105 | "execution_count": null, 106 | "metadata": { 107 | "id": "INlfgmpdw3Q1" 108 | }, 109 | "outputs": [], 110 | "source": [ 111 | "!python launch.py --xformers --disable-safe-unpickle --deepdanbooru --share --styles-file /content/gdrive/MyDrive/AI/StableDiffusion/models/styles.csv --ui-settings-file /content/gdrive/MyDrive/AI/StableDiffusion/models/config.json --ui-config-file /content/gdrive/MyDrive/AI/StableDiffusion/models/ui-config.json --embeddings-dir /content/gdrive/MyDrive/AI/StableDiffusion/models/embeddings" 112 | ] 113 | } 114 | ], 115 | "metadata": { 116 | "accelerator": "GPU", 117 | "colab": { 118 | "collapsed_sections": [], 119 | "private_outputs": true, 120 | "provenance": [] 121 | }, 122 | "gpuClass": "standard", 123 | "kernelspec": { 124 | "display_name": "Python 3", 125 | "name": "python3" 126 | }, 127 | "language_info": { 128 | "name": "python" 129 | } 130 | }, 131 | "nbformat": 4, 132 | "nbformat_minor": 0 133 | } 134 | -------------------------------------------------------------------------------- /camenduru's_stable_diffusion_video.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "view-in-github" 7 | }, 8 | "source": [ 9 | "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/notebooks/blob/main/camenduru's_stable_diffusion_video.ipynb)" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "id": "F1WTBlIF4SyA" 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "!pip install -q torch==1.13.1+cu116 torchvision==0.14.1+cu116 torchaudio==0.13.1 torchtext==0.14.1 torchdata==0.5.1 --extra-index-url https://download.pytorch.org/whl/cu116 -U\n", 21 | "from google.colab import drive\n", 22 | "drive.mount('/content/gdrive')\n", 23 | "\n", 24 | "!git clone https://github.com/rokibulislaam/colab-ffmpeg-cuda.git\n", 25 | "!cp -r ./colab-ffmpeg-cuda/bin/. /usr/bin/\n", 26 | "\n", 27 | "!pip install -q https://github.com/camenduru/stable-diffusion-webui-colab/releases/download/0.0.15/xformers-0.0.15.dev0+189828c.d20221207-cp38-cp38-linux_x86_64.whl\n", 28 | "!pip install -q git+https://github.com/camenduru/diffusers.git\n", 29 | "!pip install -q transformers ftfy\n", 30 | "\n", 31 | "import torch, os, gc, requests, subprocess\n", 32 | "from diffusers import StableDiffusionPipeline\n", 33 | "from torch import autocast\n", 34 | "from IPython.display import clear_output, HTML\n", 35 | "from base64 import b64encode\n", 36 | "clear_output()" 37 | ] 38 | }, 39 | { 40 | "cell_type": "code", 41 | "execution_count": null, 42 | "metadata": { 43 | "id": "xIar6IEfmdvA" 44 | }, 45 | "outputs": [], 46 | "source": [ 47 | "from huggingface_hub import notebook_login\n", 48 | "!git config --global credential.helper store\n", 49 | "notebook_login()" 50 | ] 51 | }, 52 | { 53 | "cell_type": "code", 54 | "execution_count": null, 55 | "metadata": { 56 | "id": "Xi6SOT9mZ6-b" 57 | }, 58 | "outputs": [], 59 | "source": [ 60 | "is_tile = False #@param {type: 'boolean'}\n", 61 | "if(is_tile):\n", 62 | "\tdef patch_conv(cls):\n", 63 | "\t\tinit = cls.__init__\n", 64 | "\t\tdef __init__(self, *args, **kwargs):\n", 65 | "\t\t\treturn init(self, *args, **kwargs, padding_mode='circular')\n", 66 | "\t\tcls.__init__ = __init__\n", 67 | "\tpatch_conv(torch.nn.Conv2d)\n", 68 | " \n", 69 | "pipe = StableDiffusionPipeline.from_pretrained(\"CompVis/stable-diffusion-v1-4\", revision=\"fp16\", torch_dtype=torch.float16, use_auth_token=True).to(\"cuda\")\n", 70 | "pipe.safety_checker = lambda images, clip_input: (images, False)\n", 71 | "clear_output()" 72 | ] 73 | }, 74 | { 75 | "cell_type": "code", 76 | "execution_count": null, 77 | "metadata": { 78 | "id": "7tOGrS5lA6-D" 79 | }, 80 | "outputs": [], 81 | "source": [ 82 | "token = '' #@param {type: 'string'}\n", 83 | "header = {\"authorization\": f\"Bot {token}\"}\n", 84 | "by = 'camenduru' #@param {type: 'string'}\n", 85 | "\n", 86 | "root_folder = '/content/gdrive/MyDrive/AI/StableDiffusion' #@param {type: 'string'}\n", 87 | "image_folder = '000' #@param {type: 'string'}\n", 88 | "if os.path.exists(f\"{root_folder}/{image_folder}\") == False:\n", 89 | " os.mkdir(f\"{root_folder}/{image_folder}\")\n", 90 | "name = max([int(f[:f.index('.')]) for f in os.listdir(f\"{root_folder}/{image_folder}\")], default=0)\n", 91 | "\n", 92 | "from PIL.PngImagePlugin import PngInfo\n", 93 | "metadata = PngInfo()\n", 94 | " \n", 95 | "height = 448 #@param {type: 'integer'}\n", 96 | "width = 832 #@param {type: 'integer'}\n", 97 | "\n", 98 | "def generate(prompt, name):\n", 99 | " metadata.add_text(\"Prompt\", f\"{prompt}\")\n", 100 | " metadata.add_text(\"by\", f\"{by}\")\n", 101 | " gc.collect()\n", 102 | " torch.cuda.empty_cache()\n", 103 | " with autocast(\"cuda\"):\n", 104 | " image = pipe(prompt, height=height, width=width, num_inference_steps=50, eta=0.0, guidance_scale=7.5)[\"sample\"][0]\n", 105 | " image.save(f\"{root_folder}/{image_folder}/{name:04}.png\", pnginfo=metadata)\n", 106 | " files = {f\"{image_folder}_{name:04}.png\" : open(f\"{root_folder}/{image_folder}/{name:04}.png\", \"rb\").read()}\n", 107 | " payload = {\"content\":f\"{prompt}\"}\n", 108 | " channel_id = 0 #@param {type: 'integer'}\n", 109 | " r = requests.post(f\"https://discord.com/api/v9/channels/{channel_id}/messages\", data=payload, headers=header, files=files).text\n", 110 | " clear_output()\n", 111 | "\n", 112 | "max_files = 100 #@param {type: 'integer'}\n", 113 | "is_from_prompts_txt = False #@param {type: 'boolean'}\n", 114 | "prompts_txt = 'prompts.txt' #@param {type: 'string'}\n", 115 | "if(is_from_prompts_txt):\n", 116 | " while name < max_files:\n", 117 | " with open(f'{prompts_txt}', \"r\") as file:\n", 118 | " prompts = file.readlines()\n", 119 | " for prompt in prompts:\n", 120 | " name += 1\n", 121 | " generate(prompt, name)\n", 122 | "else:\n", 123 | " while name < max_files:\n", 124 | " prompt = '' #@param {type: 'string'}\n", 125 | " name += 1\n", 126 | " generate(prompt, name)\n", 127 | "\n", 128 | "video_folder = 'videos' #@param {type: 'string'}\n", 129 | "if os.path.exists(f\"{root_folder}/{video_folder}\") == False:\n", 130 | " os.mkdir(f\"{root_folder}/{video_folder}\")\n", 131 | "\n", 132 | "fps = 1/3\n", 133 | "scale = '854:480' #@param {type: 'string'}\n", 134 | "cmd = ['ffmpeg','-y','-vcodec','png','-r',f'{fps}','-hwaccel','cuda','-hwaccel_output_format','cuda','-i',f'{root_folder}/{image_folder}/%04d.png','-vf',f'scale={scale}','-c:v','h264_nvenc','-b:v','100M',f'{root_folder}/{video_folder}/{image_folder}.mp4']\n", 135 | "process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)\n", 136 | "stdout, stderr = process.communicate()\n", 137 | "if process.returncode != 0:\n", 138 | " print(stderr)\n", 139 | " raise RuntimeError(stderr)\n", 140 | "\n", 141 | "mp4 = open(f'{root_folder}/{video_folder}/{image_folder}.mp4','rb').read()\n", 142 | "data_url = \"data:video/mp4;base64,\" + b64encode(mp4).decode()\n", 143 | "HTML(f'')" 144 | ] 145 | } 146 | ], 147 | "metadata": { 148 | "accelerator": "GPU", 149 | "colab": { 150 | "collapsed_sections": [], 151 | "provenance": [] 152 | }, 153 | "gpuClass": "standard", 154 | "kernelspec": { 155 | "display_name": "Python 3", 156 | "name": "python3" 157 | }, 158 | "language_info": { 159 | "name": "python" 160 | } 161 | }, 162 | "nbformat": 4, 163 | "nbformat_minor": 0 164 | } 165 | -------------------------------------------------------------------------------- /camenduru's_tensor_rt.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "view-in-github" 7 | }, 8 | "source": [ 9 | "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/notebooks/blob/main/camenduru's_tensor_rt.ipynb)" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "id": "n_Kgy0tjscUC" 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "!pip install -q torch==1.13.1+cu116 torchvision==0.14.1+cu116 torchaudio==0.13.1 torchtext==0.14.1 torchdata==0.5.1 --extra-index-url https://download.pytorch.org/whl/cu116 -U\n", 21 | "!sudo apt-get update\n", 22 | "!sudo apt-get install tensorrt\n", 23 | "!sudo apt-get install python3-libnvinfer-dev\n", 24 | "!sudo apt-get install uff-converter-tf graphsurgeon-tf onnx-graphsurgeon\n", 25 | "\n", 26 | "!pip install pycuda nvidia-tensorrt==8.4.1.5" 27 | ] 28 | }, 29 | { 30 | "cell_type": "code", 31 | "execution_count": null, 32 | "metadata": { 33 | "id": "Ila5z9MlnSBP" 34 | }, 35 | "outputs": [], 36 | "source": [ 37 | "!nvcc --version" 38 | ] 39 | }, 40 | { 41 | "cell_type": "code", 42 | "execution_count": null, 43 | "metadata": { 44 | "id": "fo0Ur35ynh2i" 45 | }, 46 | "outputs": [], 47 | "source": [ 48 | "!cat /etc/issue\n", 49 | "import tensorflow as tf\n", 50 | "print(tf.__version__)" 51 | ] 52 | }, 53 | { 54 | "cell_type": "code", 55 | "execution_count": null, 56 | "metadata": { 57 | "id": "f88d_bomtWSV" 58 | }, 59 | "outputs": [], 60 | "source": [ 61 | "!dpkg -l | grep libcudnn" 62 | ] 63 | }, 64 | { 65 | "cell_type": "code", 66 | "execution_count": null, 67 | "metadata": { 68 | "id": "ALLJj6pWshmB" 69 | }, 70 | "outputs": [], 71 | "source": [ 72 | "!dpkg -l | grep TensorRT" 73 | ] 74 | }, 75 | { 76 | "cell_type": "code", 77 | "execution_count": null, 78 | "metadata": { 79 | "id": "0d-3kTlqtn69" 80 | }, 81 | "outputs": [], 82 | "source": [ 83 | "!pip show pycuda" 84 | ] 85 | }, 86 | { 87 | "cell_type": "code", 88 | "execution_count": null, 89 | "metadata": { 90 | "id": "kh8bh8BUyHZs" 91 | }, 92 | "outputs": [], 93 | "source": [ 94 | "!pip show nvidia-tensorrt" 95 | ] 96 | }, 97 | { 98 | "cell_type": "code", 99 | "execution_count": null, 100 | "metadata": { 101 | "id": "4QUYkcqvuI-1" 102 | }, 103 | "outputs": [], 104 | "source": [ 105 | "import pycuda.autoinit\n", 106 | "import pycuda.driver as drv\n", 107 | "import numpy\n", 108 | "\n", 109 | "from pycuda.compiler import SourceModule\n", 110 | "mod = SourceModule(\"\"\"\n", 111 | "__global__ void multiply_them(float *dest, float *a, float *b)\n", 112 | "{\n", 113 | " const int i = threadIdx.x;\n", 114 | " dest[i] = a[i] * b[i];\n", 115 | "}\n", 116 | "\"\"\")\n", 117 | "\n", 118 | "multiply_them = mod.get_function(\"multiply_them\")\n", 119 | "\n", 120 | "a = numpy.random.randn(400).astype(numpy.float32)\n", 121 | "b = numpy.random.randn(400).astype(numpy.float32)\n", 122 | "\n", 123 | "dest = numpy.zeros_like(a)\n", 124 | "multiply_them(\n", 125 | " drv.Out(dest), drv.In(a), drv.In(b),\n", 126 | " block=(400,1,1), grid=(1,1))\n", 127 | "\n", 128 | "print(dest-a*b)" 129 | ] 130 | }, 131 | { 132 | "cell_type": "code", 133 | "execution_count": null, 134 | "metadata": { 135 | "id": "cXQrJfbuvexJ" 136 | }, 137 | "outputs": [], 138 | "source": [ 139 | "!git clone https://github.com/stochasticai/x-stable-diffusion" 140 | ] 141 | }, 142 | { 143 | "cell_type": "code", 144 | "execution_count": null, 145 | "metadata": { 146 | "id": "clVZdSPwvyA7" 147 | }, 148 | "outputs": [], 149 | "source": [ 150 | "!pip install transformers diffusers scipy uvicorn pydantic fastapi pycuda huggingface_hub" 151 | ] 152 | }, 153 | { 154 | "cell_type": "code", 155 | "execution_count": null, 156 | "metadata": { 157 | "id": "Q6K4XTBvxLCu" 158 | }, 159 | "outputs": [], 160 | "source": [ 161 | "!git lfs install " 162 | ] 163 | }, 164 | { 165 | "cell_type": "code", 166 | "execution_count": null, 167 | "metadata": { 168 | "id": "Z-h_7WmtxBde" 169 | }, 170 | "outputs": [], 171 | "source": [ 172 | "!git clone -b onnx https://huggingface.co/runwayml/stable-diffusion-v1-5" 173 | ] 174 | }, 175 | { 176 | "cell_type": "code", 177 | "execution_count": null, 178 | "metadata": { 179 | "id": "cP88SJhEx6wC" 180 | }, 181 | "outputs": [], 182 | "source": [ 183 | "!python /content/x-stable-diffusion/TensorRT/convert_unet_to_tensorrt.py --onnx_unet_path /content/stable-diffusion-v1-4/unet/model.onnx --save_path /content/unet.engine" 184 | ] 185 | }, 186 | { 187 | "cell_type": "code", 188 | "execution_count": null, 189 | "metadata": { 190 | "id": "Cn7WnI0782lI" 191 | }, 192 | "outputs": [], 193 | "source": [ 194 | "!wget https://huggingface.co/kamalkraj/stable-diffusion-v1-4-onnx/resolve/main/models.tar.gz" 195 | ] 196 | }, 197 | { 198 | "cell_type": "code", 199 | "execution_count": null, 200 | "metadata": { 201 | "id": "Wzyk03Oc9Aut" 202 | }, 203 | "outputs": [], 204 | "source": [ 205 | "!tar -xf models.tar.gz" 206 | ] 207 | }, 208 | { 209 | "cell_type": "code", 210 | "execution_count": null, 211 | "metadata": { 212 | "id": "HyFuIaxw8zlC" 213 | }, 214 | "outputs": [], 215 | "source": [ 216 | "!python /content/x-stable-diffusion/TensorRT/convert_unet_to_tensorrt.py" 217 | ] 218 | }, 219 | { 220 | "cell_type": "code", 221 | "execution_count": null, 222 | "metadata": { 223 | "id": "Kj_ROHBS8oY5" 224 | }, 225 | "outputs": [], 226 | "source": [ 227 | "!python /content/x-stable-diffusion/TensorRT/demo.py" 228 | ] 229 | }, 230 | { 231 | "cell_type": "code", 232 | "execution_count": null, 233 | "metadata": { 234 | "id": "_46dgIhkFAtN" 235 | }, 236 | "outputs": [], 237 | "source": [ 238 | "!git clone https://github.com/huggingface/diffusers" 239 | ] 240 | }, 241 | { 242 | "cell_type": "code", 243 | "execution_count": null, 244 | "metadata": { 245 | "id": "RTUU55euFLe2" 246 | }, 247 | "outputs": [], 248 | "source": [ 249 | "!wget https://huggingface.co/Linaqruf/anything-v3.0/resolve/main/Anything-V3.0-pruned-fp16.ckpt" 250 | ] 251 | }, 252 | { 253 | "cell_type": "code", 254 | "execution_count": null, 255 | "metadata": { 256 | "id": "3TOmTwYPFQy-" 257 | }, 258 | "outputs": [], 259 | "source": [ 260 | "!python /content/diffusers/scripts/convert_stable_diffusion_checkpoint_to_onnx.py --model_path Anything-V3.0-pruned-fp16.ckpt --output_path /content/onnx --fp16" 261 | ] 262 | }, 263 | { 264 | "cell_type": "code", 265 | "execution_count": null, 266 | "metadata": { 267 | "id": "BTOZpTXWFoPT" 268 | }, 269 | "outputs": [], 270 | "source": [ 271 | "!pip install onnx" 272 | ] 273 | }, 274 | { 275 | "cell_type": "code", 276 | "execution_count": null, 277 | "metadata": { 278 | "id": "ZEYf_Ym5GWxF" 279 | }, 280 | "outputs": [], 281 | "source": [ 282 | "!git clone -b onnx https://huggingface.co/CompVis/stable-diffusion-v1-4" 283 | ] 284 | }, 285 | { 286 | "cell_type": "code", 287 | "execution_count": null, 288 | "metadata": { 289 | "id": "C7QL5fx0SXdF" 290 | }, 291 | "outputs": [], 292 | "source": [ 293 | "!pip install huggingface-hub" 294 | ] 295 | }, 296 | { 297 | "cell_type": "code", 298 | "execution_count": 29, 299 | "metadata": { 300 | "id": "CIT7Ij8ASWMk" 301 | }, 302 | "outputs": [], 303 | "source": [ 304 | "from huggingface_hub import create_repo, create_branch, upload_folder, upload_file" 305 | ] 306 | }, 307 | { 308 | "cell_type": "code", 309 | "execution_count": null, 310 | "metadata": { 311 | "id": "NsXeEyz9SAxP" 312 | }, 313 | "outputs": [], 314 | "source": [ 315 | "create_repo(\"embed/sd14e\", token=\"token\") \n", 316 | "upload_file(path_or_fileobj=\"/content/unet.engine\", path_in_repo=\"unet.engine\", repo_id=\"embed/sd14e\", commit_message=f\"unet.engine\", token=\"token\")" 317 | ] 318 | }, 319 | { 320 | "cell_type": "code", 321 | "execution_count": 34, 322 | "metadata": { 323 | "id": "JofpxS9OUWxE" 324 | }, 325 | "outputs": [], 326 | "source": [ 327 | "!mv /content/stable-diffusion-v1-4 /content/stable-diffusion-v1-4-old" 328 | ] 329 | } 330 | ], 331 | "metadata": { 332 | "accelerator": "GPU", 333 | "colab": { 334 | "provenance": [] 335 | }, 336 | "gpuClass": "standard", 337 | "kernelspec": { 338 | "display_name": "Python 3", 339 | "name": "python3" 340 | }, 341 | "language_info": { 342 | "name": "python" 343 | } 344 | }, 345 | "nbformat": 4, 346 | "nbformat_minor": 0 347 | } 348 | -------------------------------------------------------------------------------- /camenduru's_tensorrt_diffusers.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "view-in-github" 7 | }, 8 | "source": [ 9 | "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/notebooks/blob/main/camenduru's_tensorrt_diffusers.ipynb)" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": {}, 16 | "outputs": [], 17 | "source": [ 18 | "%cd /content\n", 19 | "!mkdir /content/external\n", 20 | "%cd /content/external\n", 21 | "!git clone https://github.com/pybind/pybind11\n", 22 | "!wget https://www.python.org/ftp/python/3.10.12/Python-3.10.12.tgz\n", 23 | "!tar -xvf Python-3.10.12.tgz\n", 24 | "!mkdir -p /content/external/python3.10/include\n", 25 | "!cp -r /content/external/Python-3.10.12/Include/* /content/external/python3.10/include\n", 26 | "!wget http://http.us.debian.org/debian/pool/main/p/python3.10/libpython3.10-dev_3.10.13-1_amd64.deb\n", 27 | "!ar x libpython3.10-dev*.deb\n", 28 | "!mkdir /content/external/debian\n", 29 | "!tar -xf data.tar.xz -C /content/external/debian\n", 30 | "!cp /content/external/debian/usr/include/x86_64-linux-gnu/python3.10/pyconfig.h /content/external/python3.10/include\n", 31 | "%cd /content/TensorRT/python\n", 32 | "!TENSORRT_MODULE=tensorrt PYTHON_MAJOR_VERSION=3 PYTHON_MINOR_VERSION=10 TARGET_ARCHITECTURE=x86_64 ./build.sh" 33 | ] 34 | }, 35 | { 36 | "cell_type": "code", 37 | "execution_count": null, 38 | "metadata": {}, 39 | "outputs": [], 40 | "source": [ 41 | "%cd /content\n", 42 | "!git clone -b release/9.1 --recursive https://github.com/NVIDIA/TensorRT\n", 43 | "%cd /content/TensorRT\n", 44 | "\n", 45 | "%cd /content\n", 46 | "!wget https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/secure/9.1.0/tars/tensorrt-9.1.0.4.linux.x86_64-gnu.cuda-11.8.tar.gz\n", 47 | "!mkdir trt\n", 48 | "%cd /content/trt\n", 49 | "!tar -xvzf /content/tensorrt-9.1.0.4.linux.x86_64-gnu.cuda-11.8.tar.gz\n", 50 | "\n", 51 | "%cd /content/TensorRT\n", 52 | "!mkdir -p /content/TensorRT/build\n", 53 | "%cd /content/TensorRT/build\n", 54 | "!cmake .. -DTRT_LIB_DIR=/content/trt/TensorRT-9.1.0.4/lib -DTRT_OUT_DIR=/content/TensorRT/build/out -DCUDNN_ROOT_DIR=/usr/local/cuda -DGPU_ARCHS=\"75\"\n", 55 | "!make -j$(nproc)\n", 56 | "# !/content/TensorRT/build/out/trtexec -h" 57 | ] 58 | }, 59 | { 60 | "cell_type": "code", 61 | "execution_count": null, 62 | "metadata": {}, 63 | "outputs": [], 64 | "source": [ 65 | "# !pip install -q diffusers transformers accelerate colored cuda-python ftfy nvtx controlnet_aux\n", 66 | "# !pip install -q --extra-index-url https://pypi.ngc.nvidia.com \"onnx\" \"onnxruntime\" \"onnx-graphsurgeon\" \"polygraphy\"\n", 67 | "!pip install -q diffusers==0.19.3 transformers==4.31.0 accelerate colored cuda-python ftfy nvtx controlnet_aux==0.0.6 onnx==1.14.0 onnxruntime==1.15.1 -U\n", 68 | "!pip install -q --extra-index-url https://pypi.ngc.nvidia.com \"onnx-graphsurgeon>=0.3.27\" \"polygraphy\" -U\n", 69 | "!pip install -q https://huggingface.co/tensorrt/build/resolve/main/tensorrt-9.1.0.post11.dev4-cp310-none-linux_x86_64.whl -U" 70 | ] 71 | }, 72 | { 73 | "cell_type": "code", 74 | "execution_count": null, 75 | "metadata": {}, 76 | "outputs": [], 77 | "source": [ 78 | "%env LD_LIBRARY_PATH=/content/trt/TensorRT-9.1.0.4/lib:/usr/lib64-nvidia\n", 79 | "!ln -s /content/trt/TensorRT-9.1.0.4/lib/* /usr/lib\n", 80 | "# !echo $LD_LIBRARY_PATH\n", 81 | "%cd /content/TensorRT/demo/Diffusion\n", 82 | "# !LD_PRELOAD=/content/trt/TensorRT-9.1.0.4/lib/libnvinfer.so.9 python demo_txt2img_xl.py -h\n", 83 | "# !LD_PRELOAD=/content/trt/TensorRT-9.1.0.4/lib/libnvinfer.so.9 python demo_txt2img_xl.py \"a beautiful photograph of Mt. Fuji during cherry blossom\"\n", 84 | "!LD_PRELOAD=/content/trt/TensorRT-9.1.0.4/lib/libnvinfer.so.9 python demo_txt2img.py \"a beautiful photograph of Mt. Fuji during cherry blossom\" # sd1.5" 85 | ] 86 | }, 87 | { 88 | "cell_type": "code", 89 | "execution_count": null, 90 | "metadata": { 91 | "id": "n_Kgy0tjscUC" 92 | }, 93 | "outputs": [], 94 | "source": [ 95 | "# https://github.com/NVIDIA/TensorRT/blob/release/8.6/demo/experimental/HuggingFace-Diffusers/TensorRT-diffusers-txt2img.ipynb\n", 96 | "# https://github.com/huggingface/diffusers/tree/main/examples/community#tensorrt-text2image-stable-diffusion-pipeline\n", 97 | "# https://github.com/huggingface/diffusers/tree/main/examples/community#tensorrt-image2image-stable-diffusion-pipeline\n", 98 | "# https://github.com/huggingface/diffusers/tree/main/examples/community#tensorrt-inpainting-stable-diffusion-pipeline\n", 99 | "\n", 100 | "# https://github.com/huggingface/diffusers/blob/main/examples/community/stable_diffusion_tensorrt_txt2img.py\n", 101 | "# https://github.com/huggingface/diffusers/blob/main/examples/community/stable_diffusion_tensorrt_img2img.py\n", 102 | "# https://github.com/huggingface/diffusers/blob/main/examples/community/stable_diffusion_tensorrt_inpaint.py\n", 103 | "# https://github.com/huggingface/diffusers/blob/main/examples/community/run_tensorrt_controlnet.py\n", 104 | "\n", 105 | "!pip install -q tensorrt diffusers transformers accelerate\n", 106 | "!pip install -q --extra-index-url https://pypi.ngc.nvidia.com \"onnx-graphsurgeon\" \"onnxruntime\" \"polygraphy\"" 107 | ] 108 | }, 109 | { 110 | "cell_type": "code", 111 | "execution_count": null, 112 | "metadata": {}, 113 | "outputs": [], 114 | "source": [ 115 | "# !pip install -q https://huggingface.co/tensorrt/build/resolve/main/tensorrt-9.1.0.post11.dev4-cp310-none-linux_x86_64.whl\n", 116 | "# %cd /content\n", 117 | "# !wget https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/secure/9.1.0/tars/tensorrt-9.1.0.4.linux.x86_64-gnu.cuda-11.8.tar.gz\n", 118 | "# !mkdir trt\n", 119 | "# %cd /content/trt\n", 120 | "# !tar -xvzf /content/tensorrt-9.1.0.4.linux.x86_64-gnu.cuda-11.8.tar.gz\n", 121 | "# !ln -s /content/trt/TensorRT-9.1.0.4/lib/* /usr/lib\n", 122 | "# !pip install -q diffusers transformers accelerate colored cuda-python ftfy nvtx controlnet_aux onnx onnxruntime\n", 123 | "# !pip install -q --extra-index-url https://pypi.ngc.nvidia.com \"onnx-graphsurgeon\" \"polygraphy\"\n", 124 | "\n", 125 | "import torch\n", 126 | "from diffusers.pipelines.stable_diffusion import StableDiffusionPipeline\n", 127 | "from diffusers import DDIMScheduler\n", 128 | "\n", 129 | "scheduler = DDIMScheduler.from_pretrained(\"stabilityai/stable-diffusion-2-1\", subfolder=\"scheduler\")\n", 130 | "pipe_trt = StableDiffusionPipeline.from_pretrained(\"stabilityai/stable-diffusion-2-1\", custom_pipeline=\"stable_diffusion_tensorrt_txt2img\", revision='fp16',\n", 131 | " torch_dtype=torch.float16, scheduler=scheduler, image_height=512, image_width=512)\n", 132 | "\n", 133 | "pipe_trt.set_cached_folder(\"stabilityai/stable-diffusion-2-1\", revision='fp16')\n", 134 | "\n", 135 | "pipe_trt = pipe_trt.to(\"cuda\")" 136 | ] 137 | }, 138 | { 139 | "cell_type": "code", 140 | "execution_count": null, 141 | "metadata": {}, 142 | "outputs": [], 143 | "source": [ 144 | "prompt = \"a beautiful photograph of Mt. Fuji during cherry blossom\"\n", 145 | "\n", 146 | "# warm up runs to stabilize performance benchmarking\n", 147 | "num_warm_up_steps=5\n", 148 | "for _ in range(num_warm_up_steps):\n", 149 | " _ = pipe_trt(prompt)\n", 150 | "\n", 151 | "image = pipe_trt(prompt).images[0]\n", 152 | "display(image)" 153 | ] 154 | } 155 | ], 156 | "metadata": { 157 | "accelerator": "GPU", 158 | "colab": { 159 | "provenance": [] 160 | }, 161 | "gpuClass": "standard", 162 | "kernelspec": { 163 | "display_name": "Python 3", 164 | "name": "python3" 165 | }, 166 | "language_info": { 167 | "name": "python" 168 | } 169 | }, 170 | "nbformat": 4, 171 | "nbformat_minor": 0 172 | } 173 | -------------------------------------------------------------------------------- /camenduru's_text2light.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "view-in-github" 7 | }, 8 | "source": [ 9 | "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/notebooks/blob/main/camenduru's_text2light.ipynb)" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "id": "W7adzSv7ghtk" 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "!pip install -q torch==1.13.1+cu116 torchvision==0.14.1+cu116 torchaudio==0.13.1 torchtext==0.14.1 torchdata==0.5.1 --extra-index-url https://download.pytorch.org/whl/cu116 -U\n", 21 | "from google.colab import drive\n", 22 | "drive.mount('/content/gdrive')\n", 23 | "\n", 24 | "!pip install faiss-cpu tensorboardx ftfy einops transformers streamlit omegaconf==2.0 pudb==2019.2 pytorch-lightning==1.6.0\n", 25 | "\n", 26 | "!git clone https://github.com/FrozenBurning/Text2Light\n", 27 | "%cd /content/Text2Light\n", 28 | "\n", 29 | "!git lfs install\n", 30 | "!git clone https://huggingface.co/camenduru/text2light models" 31 | ] 32 | }, 33 | { 34 | "cell_type": "code", 35 | "execution_count": null, 36 | "metadata": { 37 | "id": "MyZ2c6OeyaTI" 38 | }, 39 | "outputs": [], 40 | "source": [ 41 | "import os\n", 42 | "os.environ[\"OPENCV_IO_ENABLE_OPENEXR\"]=\"1\"\n", 43 | "\n", 44 | "!python text2light.py -rg /content/models/global_sampler_clip -rl /content/models/local_sampler_outdoor --outdir /content/gdrive/MyDrive/AI/Text2Light/outputs/001 --text \"beach blue and white cloudy sky during daytime\" --clip clip_emb.npy --sritmo /content/models/sritmo.pth --sr_factor 4" 45 | ] 46 | } 47 | ], 48 | "metadata": { 49 | "accelerator": "GPU", 50 | "colab": { 51 | "provenance": [] 52 | }, 53 | "kernelspec": { 54 | "display_name": "Python 3", 55 | "name": "python3" 56 | }, 57 | "language_info": { 58 | "name": "python" 59 | } 60 | }, 61 | "nbformat": 4, 62 | "nbformat_minor": 0 63 | } 64 | -------------------------------------------------------------------------------- /camenduru's_tortoise.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "view-in-github" 7 | }, 8 | "source": [ 9 | "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/notebooks/blob/main/camenduru's_tortoise.ipynb)" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "id": "8XK7FDkwryA5" 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "!pip install -q torch==1.13.1+cu116 torchvision==0.14.1+cu116 torchaudio==0.13.1 torchtext==0.14.1 torchdata==0.5.1 --extra-index-url https://download.pytorch.org/whl/cu116 -U\n", 21 | "!git clone https://github.com/neonbjb/tortoise-tts.git\n", 22 | "%cd tortoise-tts\n", 23 | "!pip3 install -r requirements.txt\n", 24 | "!python3 setup.py install\n", 25 | "\n", 26 | "!python3 tortoise/read.py --voice=tom --textfile=/content/text.txt --preset=high_quality --output_path=.\n", 27 | "\n", 28 | "import IPython\n", 29 | "IPython.display.Audio('tom/combined.wav')" 30 | ] 31 | } 32 | ], 33 | "metadata": { 34 | "accelerator": "GPU", 35 | "colab": { 36 | "provenance": [] 37 | }, 38 | "kernelspec": { 39 | "display_name": "Python 3", 40 | "name": "python3" 41 | }, 42 | "language_info": { 43 | "name": "python" 44 | } 45 | }, 46 | "nbformat": 4, 47 | "nbformat_minor": 0 48 | } 49 | -------------------------------------------------------------------------------- /camenduru's_tpu_diffusers.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "view-in-github" 7 | }, 8 | "source": [ 9 | "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/notebooks/blob/main/camenduru's_tpu_diffusers.ipynb)" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "cellView": "form", 17 | "id": "ymcDoPQn41U4" 18 | }, 19 | "outputs": [], 20 | "source": [ 21 | "!pip install -q torch==1.13.1+cu116 torchvision==0.14.1+cu116 torchaudio==0.13.1 torchtext==0.14.1 torchdata==0.5.1 --extra-index-url https://download.pytorch.org/whl/cu116 -U\n", 22 | "save_to = \"here\" #@param [\"here\", \"discord\", \"gdrive\"]\n", 23 | "if save_to == \"gdrive\":\n", 24 | " from google.colab import drive\n", 25 | " drive.mount('/content/gdrive')\n", 26 | "\n", 27 | "!pip install -U flax diffusers transformers piexif fold_to_ascii ftfy" 28 | ] 29 | }, 30 | { 31 | "cell_type": "code", 32 | "execution_count": null, 33 | "metadata": { 34 | "cellView": "form", 35 | "id": "VhsXOJ4BNBRK" 36 | }, 37 | "outputs": [], 38 | "source": [ 39 | "import jax.tools.colab_tpu\n", 40 | "jax.tools.colab_tpu.setup_tpu('tpu_driver_nightly')\n", 41 | "\n", 42 | "import jax, random, os, gc, requests, json, piexif, shutil\n", 43 | "import numpy as np\n", 44 | "from flax.jax_utils import replicate\n", 45 | "from flax.training.common_utils import shard\n", 46 | "from diffusers import FlaxStableDiffusionPipeline\n", 47 | "from PIL import Image\n", 48 | "from PIL.PngImagePlugin import PngInfo\n", 49 | "from fold_to_ascii import fold\n", 50 | "metadata = PngInfo()\n", 51 | "\n", 52 | "discord_token = \"token\" #@param {type: 'string'}\n", 53 | "discord_channel_id = 0 #@param {type: 'integer'}\n", 54 | "prompt = \"duck\" #@param {type: 'string'}\n", 55 | "negative_prompt = \"red\" #@param {type: 'string'}\n", 56 | "width = 512 #@param {type: 'integer'}\n", 57 | "height = 512 #@param {type: 'integer'}\n", 58 | "guidance_scale = 7.5 #@param {type: 'number'}\n", 59 | "num_inference_steps = 50 #@param {type: 'integer'}\n", 60 | "suffix = \"jpg\" #@param [\"jpg\", \"png\"]\n", 61 | "by = \"camenduru\" #@param {type: 'string'}\n", 62 | "template = {\n", 63 | " \"discord_token\": discord_token,\n", 64 | " \"discord_channel_id\": discord_channel_id,\n", 65 | " \"by\": by,\n", 66 | " \"num_inference_steps\": num_inference_steps,\n", 67 | " \"guidance_scale\": guidance_scale,\n", 68 | " \"sampler\": \"PLMS\",\n", 69 | " \"width\": width,\n", 70 | " \"height\": height,\n", 71 | " \"prompt\": prompt,\n", 72 | " \"negative_prompt\": negative_prompt,\n", 73 | " \"suffix\": suffix\n", 74 | " }\n", 75 | "with open(\"template.json\", \"w\") as outfile:\n", 76 | " outfile.write(json.dumps(template))\n", 77 | "\n", 78 | "def closestNumber(n, m):\n", 79 | " q = int(n / m)\n", 80 | " n1 = m * q\n", 81 | " if (n * m) > 0:\n", 82 | " n2 = m * (q + 1)\n", 83 | " else:\n", 84 | " n2 = m * (q - 1)\n", 85 | " if abs(n - n1) < abs(n - n2):\n", 86 | " return n1\n", 87 | " return n2\n", 88 | "\n", 89 | "def image_grid(imgs, rows, cols):\n", 90 | " w,h = imgs[0].size\n", 91 | " grid = Image.new('RGB', size=(cols*w, rows*h))\n", 92 | " for i, img in enumerate(imgs): grid.paste(img, box=(i%cols*w, i//cols*h))\n", 93 | " return grid\n", 94 | "\n", 95 | "folder_max_files = 500 #@param {type: 'integer'}\n", 96 | "root_folder = 'ai_images' #@param {type: 'string'}\n", 97 | "\n", 98 | "if os.path.exists(f\"{root_folder}\") == False:\n", 99 | " os.mkdir(f\"{root_folder}\")\n", 100 | "image_folder = max([int(f) for f in os.listdir(f\"{root_folder}\")], default=0)\n", 101 | "if os.path.exists(f\"{root_folder}/{image_folder:04}\") == False:\n", 102 | " os.mkdir(f\"{root_folder}/{image_folder:04}\")\n", 103 | "name = max([int(f[: f.index(\".\")]) for f in os.listdir(f\"{root_folder}/{image_folder:04}\")],default=0,)\n", 104 | "\n", 105 | "model_folder = \"flax/mo-di-diffusion\" #@param [\"flax/Stable_Diffusion_PaperCut_Model\", \"flax/redshift-diffusion\", \"flax/sd15\", \"flax/sd15-non-ema\", \"flax/midjourney-v4-diffusion\", \"flax/StudioGhibli\", \"flax/Tron-Legacy-diffusion\", \"flax/Cyberpunk-Anime-Diffusion\", \"flax/waifu-diffusion\", \"flax/Arcane-Diffusion\", \"flax/classic-anim-diffusion\", \"flax/mo-di-diffusion\", \"camenduru/plushies\", \"lilpotat/f2\", \"lilpotat/a3\", \"lilpotat/rbm\"]\n", 106 | "if model_folder == \"lilpotat/f2\" or model_folder ==\"lilpotat/a3\" or model_folder == \"lilpotat/rbm\":\n", 107 | " revision = \"flax\"\n", 108 | "else:\n", 109 | " revision = \"main\"\n", 110 | "pipe, params = FlaxStableDiffusionPipeline.from_pretrained(model_folder, revision=revision, from_pt=False, dtype=jax.numpy.bfloat16, safety_checker=None)\n", 111 | "params = replicate(params)\n", 112 | "\n", 113 | "def generate(discord_token, discord_channel_id, discord_user, by, num_inference_steps, guidance_scale, sampler, width, height, prompt, negative_prompt, suffix, image_folder, name):\n", 114 | " width = closestNumber(width, 8)\n", 115 | " height = closestNumber(height, 8)\n", 116 | " metadata.add_text(\"Prompt\", f\"{prompt}\")\n", 117 | " metadata.add_text(\"by\", f\"{by}\")\n", 118 | " gc.collect()\n", 119 | " real_seed = random.randint(0, 2147483647)\n", 120 | " prng_seed = jax.random.PRNGKey(real_seed)\n", 121 | " prng_seed = jax.random.split(prng_seed, jax.device_count())\n", 122 | " num_samples = jax.device_count()\n", 123 | " prompt_n = num_samples * [prompt]\n", 124 | " prompt_ids = pipe.prepare_inputs(prompt_n)\n", 125 | " prompt_ids = shard(prompt_ids)\n", 126 | " images = pipe(prompt_ids, params, prng_seed, num_inference_steps=num_inference_steps, height=height, width=width, guidance_scale=guidance_scale, jit=True).images\n", 127 | " images = pipe.numpy_to_pil(np.asarray(images.reshape((num_samples,) + images.shape[-3:])))\n", 128 | " image = image_grid(images, 2, 4)\n", 129 | " if(suffix == 'png'):\n", 130 | " image.save(f\"{root_folder}/{image_folder:04}/{name:04}.{suffix}\", pnginfo=metadata)\n", 131 | " else:\n", 132 | " zeroth_ifd = {piexif.ImageIFD.ImageDescription: f\"{fold(prompt)}\", piexif.ImageIFD.Make: f\"{fold(by)}\", piexif.ImageIFD.Model: f\"{model_folder}\"}\n", 133 | " exif_dict = {\"0th\": zeroth_ifd}\n", 134 | " exif_bytes = piexif.dump(exif_dict)\n", 135 | " image.save(f\"{root_folder}/{image_folder:04}/{name:04}.{suffix}\", \"JPEG\", quality=70, exif=exif_bytes)\n", 136 | " files = {f\"{image_folder:04}_{name:04}.{suffix}\": open(f\"{root_folder}/{image_folder:04}/{name:04}.{suffix}\", \"rb\").read()}\n", 137 | " if save_to == \"gdrive\":\n", 138 | " shutil.copyfile(f\"{root_folder}/{image_folder:04}/{name:04}.{suffix}\", f\"/content/gdrive/{root_folder}/{image_folder:04}/{name:04}.{suffix}\")\n", 139 | " os.remove(f\"{root_folder}/{image_folder:04}/{name:04}.{suffix}\")\n", 140 | " elif save_to == \"discord\":\n", 141 | " payload = {\"content\": f\"{prompt}\\nSteps: {num_inference_steps}, Sampler: {sampler}, CFG scale: {guidance_scale}, Seed: {real_seed}, Size: {width}x{height}, Model folder: {model_folder} - {discord_user}\"}\n", 142 | " requests.post(f\"https://discord.com/api/v9/channels/{discord_channel_id}/messages\", data=payload, headers={\"authorization\": f\"Bot {discord_token}\"}, files=files)\n", 143 | " os.remove(f\"{root_folder}/{image_folder:04}/{name:04}.{suffix}\")" 144 | ] 145 | }, 146 | { 147 | "cell_type": "code", 148 | "execution_count": null, 149 | "metadata": { 150 | "cellView": "form", 151 | "id": "DDUev8TOMxJ4" 152 | }, 153 | "outputs": [], 154 | "source": [ 155 | "is_loop = False #@param {type:\"boolean\"}\n", 156 | "\n", 157 | "if is_loop:\n", 158 | " while True:\n", 159 | " if name < folder_max_files:\n", 160 | " with open(\"template.json\", \"r\") as file:\n", 161 | " prompts = file.readlines()\n", 162 | " for prompt in prompts:\n", 163 | " d = json.loads(prompt)\n", 164 | " name += 1\n", 165 | " generate(d[\"discord_token\"], d[\"discord_channel_id\"], \"camenduru\", d[\"by\"], d[\"num_inference_steps\"], d[\"guidance_scale\"], d[\"sampler\"], d[\"width\"], d[\"height\"], d[\"prompt\"], d[\"negative_prompt\"], d[\"suffix\"], image_folder, name)\n", 166 | " else:\n", 167 | " image_folder += 1\n", 168 | " if os.path.exists(f\"{root_folder}/{image_folder:04}\") == False:\n", 169 | " os.mkdir(f\"{root_folder}/{image_folder:04}\")\n", 170 | " name = 0\n", 171 | "else:\n", 172 | " if name < folder_max_files:\n", 173 | " with open(\"template.json\", \"r\") as file:\n", 174 | " prompts = file.readlines()\n", 175 | " for prompt in prompts:\n", 176 | " d = json.loads(prompt)\n", 177 | " name += 1\n", 178 | " generate(d[\"discord_token\"], d[\"discord_channel_id\"], \"camenduru\", d[\"by\"], d[\"num_inference_steps\"], d[\"guidance_scale\"], d[\"sampler\"], d[\"width\"], d[\"height\"], d[\"prompt\"], d[\"negative_prompt\"], d[\"suffix\"], image_folder, name)\n", 179 | " else:\n", 180 | " image_folder += 1\n", 181 | " if os.path.exists(f\"{root_folder}/{image_folder:04}\") == False:\n", 182 | " os.mkdir(f\"{root_folder}/{image_folder:04}\")\n", 183 | " name = 0" 184 | ] 185 | } 186 | ], 187 | "metadata": { 188 | "accelerator": "TPU", 189 | "colab": { 190 | "collapsed_sections": [], 191 | "provenance": [] 192 | }, 193 | "gpuClass": "standard", 194 | "kernelspec": { 195 | "display_name": "Python 3", 196 | "name": "python3" 197 | }, 198 | "language_info": { 199 | "name": "python" 200 | } 201 | }, 202 | "nbformat": 4, 203 | "nbformat_minor": 0 204 | } 205 | -------------------------------------------------------------------------------- /camenduru's_unet_traced.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "view-in-github" 7 | }, 8 | "source": [ 9 | "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/notebooks/blob/main/camenduru's_unet_traced.ipynb)" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "id": "dxxnQJw15FCx" 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "!pip install -q torch==1.13.1+cu116 torchvision==0.14.1+cu116 torchaudio==0.13.1 torchtext==0.14.1 torchdata==0.5.1 --extra-index-url https://download.pytorch.org/whl/cu116 -U\n", 21 | "!pip install diffusers transformers accelerate\n", 22 | "!pip install -q https://github.com/camenduru/stable-diffusion-webui-colab/releases/download/0.0.15/xformers-0.0.15.dev0+189828c.d20221207-cp38-cp38-linux_x86_64.whl" 23 | ] 24 | }, 25 | { 26 | "cell_type": "code", 27 | "execution_count": null, 28 | "metadata": { 29 | "id": "KwssL-ep4wFr" 30 | }, 31 | "outputs": [], 32 | "source": [ 33 | "import time\n", 34 | "import torch\n", 35 | "from diffusers import StableDiffusionPipeline\n", 36 | "import functools\n", 37 | "\n", 38 | "# torch disable grad\n", 39 | "torch.set_grad_enabled(False)\n", 40 | "\n", 41 | "# load inputs\n", 42 | "def generate_inputs():\n", 43 | " sample = torch.randn(2, 4, 64, 64).half().cuda()\n", 44 | " timestep = torch.rand(1).half().cuda() * 999\n", 45 | " encoder_hidden_states = torch.randn(2, 77, 768).half().cuda()\n", 46 | " return sample, timestep, encoder_hidden_states\n", 47 | "\n", 48 | "\n", 49 | "pipe = StableDiffusionPipeline.from_pretrained(\"runwayml/stable-diffusion-v1-5\", torch_dtype=torch.float16).to(\"cuda\")\n", 50 | "unet = pipe.unet\n", 51 | "unet.eval()\n", 52 | "unet.to(memory_format=torch.channels_last) # use channels_last memory format\n", 53 | "unet.forward = functools.partial(unet.forward, return_dict=False) # set return_dict=False as default\n", 54 | "\n", 55 | "# warmup\n", 56 | "for _ in range(3):\n", 57 | " with torch.inference_mode():\n", 58 | " inputs = generate_inputs()\n", 59 | " orig_output = unet(*inputs)\n", 60 | "\n", 61 | "# trace\n", 62 | "print(\"tracing..\")\n", 63 | "unet_traced = torch.jit.trace(unet, inputs)\n", 64 | "unet_traced.eval()\n", 65 | "print(\"done tracing\")\n", 66 | "\n", 67 | "# save the model\n", 68 | "unet_traced.save(\"unet_traced.pt\")" 69 | ] 70 | }, 71 | { 72 | "cell_type": "code", 73 | "execution_count": null, 74 | "metadata": { 75 | "id": "ZWDyn6Mo4tom" 76 | }, 77 | "outputs": [], 78 | "source": [ 79 | "from diffusers import StableDiffusionPipeline\n", 80 | "import torch\n", 81 | "from dataclasses import dataclass\n", 82 | "\n", 83 | "@dataclass\n", 84 | "class UNet2DConditionOutput:\n", 85 | " sample: torch.FloatTensor\n", 86 | "\n", 87 | "pipe = StableDiffusionPipeline.from_pretrained(\"runwayml/stable-diffusion-v1-5\", torch_dtype=torch.float16, safety_checker=None, custom_pipeline=\"interpolate_stable_diffusion\").to(\"cuda\")\n", 88 | "unet_traced = torch.jit.load(\"unet_traced.pt\")\n", 89 | "\n", 90 | "class TracedUNet(torch.nn.Module):\n", 91 | " def __init__(self):\n", 92 | " super().__init__()\n", 93 | " self.in_channels = pipe.unet.in_channels\n", 94 | " self.device = pipe.unet.device\n", 95 | " self.dtype = torch.float16\n", 96 | "\n", 97 | " def forward(self, latent_model_input, t, encoder_hidden_states):\n", 98 | " sample = unet_traced(latent_model_input, t, encoder_hidden_states)[0]\n", 99 | " return UNet2DConditionOutput(sample=sample)\n", 100 | "\n", 101 | "pipe.unet = TracedUNet()\n", 102 | "pipe.to(\"cuda\")" 103 | ] 104 | }, 105 | { 106 | "cell_type": "code", 107 | "execution_count": null, 108 | "metadata": { 109 | "id": "7DnlDKJO647f" 110 | }, 111 | "outputs": [], 112 | "source": [ 113 | "with torch.inference_mode():\n", 114 | " generator = torch.Generator(\"cuda\").manual_seed(1)\n", 115 | " image = pipe(\"duck\", generator=generator).images[0]\n", 116 | " display(image)" 117 | ] 118 | }, 119 | { 120 | "cell_type": "code", 121 | "execution_count": null, 122 | "metadata": { 123 | "id": "k-Eml3ODR2BY" 124 | }, 125 | "outputs": [], 126 | "source": [ 127 | "pipe2 = StableDiffusionPipeline.from_pretrained(\"runwayml/stable-diffusion-v1-5\", torch_dtype=torch.float16, safety_checker=None,).to(\"cuda\")\n", 128 | "pipe2.enable_xformers_memory_efficient_attention()" 129 | ] 130 | }, 131 | { 132 | "cell_type": "code", 133 | "execution_count": null, 134 | "metadata": { 135 | "id": "ricKjbd9R8C1" 136 | }, 137 | "outputs": [], 138 | "source": [ 139 | "with torch.autocast(\"cuda\"):\n", 140 | " generator = torch.Generator(\"cuda\").manual_seed(1)\n", 141 | " image = pipe2(\"duck\", generator=generator).images[0]\n", 142 | " display(image)" 143 | ] 144 | } 145 | ], 146 | "metadata": { 147 | "accelerator": "GPU", 148 | "colab": { 149 | "provenance": [] 150 | }, 151 | "gpuClass": "standard", 152 | "kernelspec": { 153 | "display_name": "Python 3", 154 | "name": "python3" 155 | }, 156 | "language_info": { 157 | "name": "python" 158 | } 159 | }, 160 | "nbformat": 4, 161 | "nbformat_minor": 0 162 | } 163 | -------------------------------------------------------------------------------- /camenduru's_waifu_diffusion.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "view-in-github" 7 | }, 8 | "source": [ 9 | "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/notebooks/blob/main/camenduru's_waifu_diffusion.ipynb)" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "id": "GlFhGnMdz0YY" 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "!pip install -q torch==1.13.1+cu116 torchvision==0.14.1+cu116 torchaudio==0.13.1 torchtext==0.14.1 torchdata==0.5.1 --extra-index-url https://download.pytorch.org/whl/cu116 -U\n", 21 | "from google.colab import drive\n", 22 | "drive.mount('/content/gdrive')\n", 23 | "\n", 24 | "!pip install -qq git+https://github.com/huggingface/diffusers.git\n", 25 | "!pip install -qq transformers ftfy datasets\n", 26 | "\n", 27 | "import torch, os, gc\n", 28 | "from diffusers import StableDiffusionPipeline\n", 29 | "from torch import autocast\n", 30 | "from IPython.display import clear_output\n", 31 | "\n", 32 | "batch_idx = 0\n", 33 | "outputs_path = \"/content/gdrive/MyDrive/AI/WaifuDiffusion\"\n", 34 | "clear_output()" 35 | ] 36 | }, 37 | { 38 | "cell_type": "code", 39 | "execution_count": null, 40 | "metadata": { 41 | "id": "KF30OI0Wb91P" 42 | }, 43 | "outputs": [], 44 | "source": [ 45 | "is_tile = False #@param {type:'boolean'}\n", 46 | "if(is_tile):\n", 47 | "\tdef patch_conv(cls):\n", 48 | "\t\tinit = cls.__init__\n", 49 | "\t\tdef __init__(self, *args, **kwargs):\n", 50 | "\t\t\treturn init(self, *args, **kwargs, padding_mode='circular')\n", 51 | "\t\tcls.__init__ = __init__\n", 52 | "\tpatch_conv(torch.nn.Conv2d)\n", 53 | " \n", 54 | "pipe = StableDiffusionPipeline.from_pretrained(\"hakurei/waifu-diffusion\", revision=\"fp16\", torch_dtype=torch.float16).to(\"cuda\")\n", 55 | "pipe.safety_checker = lambda images, clip_input: (images, False)\n", 56 | "clear_output()" 57 | ] 58 | }, 59 | { 60 | "cell_type": "code", 61 | "execution_count": null, 62 | "metadata": { 63 | "id": "7tOGrS5lA6-D" 64 | }, 65 | "outputs": [], 66 | "source": [ 67 | "from PIL.PngImagePlugin import PngInfo\n", 68 | "metadata = PngInfo()\n", 69 | " \n", 70 | "images_path_name = \"000\" #@param {type:'string'}\n", 71 | "num_batch_images = 1000 \n", 72 | "height = 512 #@param {type:'integer'}\n", 73 | "width = 512 #@param {type:'integer'}\n", 74 | "\n", 75 | "path = os.path.join(outputs_path, images_path_name)\n", 76 | "\n", 77 | "if os.path.exists(path) == False:\n", 78 | " os.mkdir(path)\n", 79 | "\n", 80 | "prompt = 'manga' #@param {type:'string'}\n", 81 | "metadata.add_text(\"Prompt\", f\"{prompt}\")\n", 82 | "by = 'camenduru' #@param {type:'string'}\n", 83 | "metadata.add_text(\"by\", f\"{by}\")\n", 84 | "\n", 85 | "for i in range(num_batch_images):\n", 86 | " gc.collect()\n", 87 | " torch.cuda.empty_cache()\n", 88 | " with autocast(\"cuda\"):\n", 89 | " image = pipe(prompt, height=height, width=width, num_inference_steps=50, eta=0.0, guidance_scale=7.5)[\"sample\"][0]\n", 90 | "\n", 91 | " image.save(f\"{path}/{images_path_name}_{batch_idx}.png\", pnginfo=metadata)\n", 92 | " batch_idx += 1" 93 | ] 94 | } 95 | ], 96 | "metadata": { 97 | "accelerator": "GPU", 98 | "colab": { 99 | "collapsed_sections": [], 100 | "provenance": [] 101 | }, 102 | "gpuClass": "standard", 103 | "kernelspec": { 104 | "display_name": "Python 3", 105 | "name": "python3" 106 | }, 107 | "language_info": { 108 | "name": "python" 109 | } 110 | }, 111 | "nbformat": 4, 112 | "nbformat_minor": 0 113 | } 114 | -------------------------------------------------------------------------------- /camenduru's_watermark_remover.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "view-in-github" 7 | }, 8 | "source": [ 9 | "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/notebooks/blob/main/camenduru's_watermark_remover.ipynb)" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "id": "VSNbyJrXkQXb" 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "!pip install -q torch==1.13.1+cu116 torchvision==0.14.1+cu116 torchaudio==0.13.1 torchtext==0.14.1 torchdata==0.5.1 --extra-index-url https://download.pytorch.org/whl/cu116 -U\n", 21 | "!pip install -q gradio==3.24.1 pytorch-lightning==1.8.3.post0 hydra-core==1.1.0 webdataset kornia==0.5.0\n", 22 | "!git clone -b dev https://github.com/camenduru/lama\n", 23 | "%cd /content/lama\n", 24 | "\n", 25 | "!mkdir /content/lama/big-lama\n", 26 | "!mkdir /content/lama/big-lama/models\n", 27 | "!mkdir /content/lama/masks\n", 28 | "!wget https://huggingface.co/camenduru/big-lama/resolve/main/big-lama/models/best.ckpt -O /content/lama/big-lama/models/best.ckpt\n", 29 | "!wget https://huggingface.co/camenduru/big-lama/raw/main/big-lama/config.yaml -O /content/lama/big-lama/config.yaml\n", 30 | "!wget https://huggingface.co/camenduru/big-lama/resolve/main/big-lama/mask.png -O /content/lama/masks/test_mask.png\n", 31 | "\n", 32 | "!mkdir /content/lama/indir\n", 33 | "!mkdir /content/lama/outdir\n", 34 | "!wget https://user-images.githubusercontent.com/54370274/230710342-a4411c86-298d-4622-b5f1-4f823c8e9a06.png -O /content/lama/indir/image.png\n", 35 | "!wget https://huggingface.co/camenduru/big-lama/resolve/main/big-lama/image_mask.png -O /content/lama/indir/image_mask.png" 36 | ] 37 | }, 38 | { 39 | "cell_type": "code", 40 | "execution_count": null, 41 | "metadata": { 42 | "id": "bKPfV3zvFJ8h" 43 | }, 44 | "outputs": [], 45 | "source": [ 46 | "%cd /content/lama\n", 47 | "!python predict.py model.path=/content/lama/big-lama indir=/content/lama/indir outdir=/content/lama/outdir" 48 | ] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "execution_count": null, 53 | "metadata": {}, 54 | "outputs": [], 55 | "source": [ 56 | "#https://huggingface.co/spaces/fffiloni/lama-video-watermark-remover/blob/main/app.py modified\n", 57 | "\n", 58 | "%cd /content/lama\n", 59 | "\n", 60 | "from moviepy.editor import *\n", 61 | "import os, cv2\n", 62 | "import gradio as gr\n", 63 | "from PIL import Image, ImageOps\n", 64 | "from moviepy.editor import *\n", 65 | "\n", 66 | "def get_frames(video_in):\n", 67 | " frames = []\n", 68 | " clip = VideoFileClip(video_in)\n", 69 | " if clip.fps > 30:\n", 70 | " print(\"vide rate is over 30, resetting to 30\")\n", 71 | " clip_resized = clip.resize(height=256)\n", 72 | " clip_resized.write_videofile(\"video_resized.mp4\", fps=30)\n", 73 | " else:\n", 74 | " print(\"video rate is OK\")\n", 75 | " clip_resized = clip.resize(height=256)\n", 76 | " clip_resized.write_videofile(\"video_resized.mp4\", fps=clip.fps)\n", 77 | " print(\"video resized to 512 height\")\n", 78 | " cap=cv2.VideoCapture(\"video_resized.mp4\")\n", 79 | " fps=cap.get(cv2.CAP_PROP_FPS)\n", 80 | " print(\"video fps: \" + str(fps))\n", 81 | " i=0\n", 82 | " while(cap.isOpened()):\n", 83 | " ret, frame = cap.read()\n", 84 | " if ret == False:\n", 85 | " break\n", 86 | " cv2.imwrite('in'+str(i)+'.jpg',frame)\n", 87 | " frames.append('in'+str(i)+'.jpg')\n", 88 | " i+=1\n", 89 | " cap.release()\n", 90 | " cv2.destroyAllWindows()\n", 91 | " print(\"broke the video into frames\")\n", 92 | " return frames, fps\n", 93 | "\n", 94 | "def create_video(frames, fps, type):\n", 95 | " print(\"building video result\")\n", 96 | " clip = ImageSequenceClip(frames, fps=fps)\n", 97 | " clip.write_videofile(type + \"_result.mp4\", fps=fps)\n", 98 | " return type + \"_result.mp4\"\n", 99 | "\n", 100 | "def lama(img):\n", 101 | " print(img)\n", 102 | " img = Image.open(img)\n", 103 | " imageio.imwrite(f\"/content/lama/indir/image.png\", img)\n", 104 | " os.system('python predict.py model.path=/content/lama/big-lama indir=/content/lama/indir outdir=/content/lama/outdir device=cuda')\n", 105 | " return f\"/content/lama/outdir/image_mask.png\"\n", 106 | "\n", 107 | "def infer(video_in):\n", 108 | " break_vid = get_frames(video_in)\n", 109 | " frames_list= break_vid[0]\n", 110 | " fps = break_vid[1]\n", 111 | " n_frame = len(frames_list)\n", 112 | " if n_frame >= len(frames_list):\n", 113 | " print(\"video is shorter than the cut value\")\n", 114 | " n_frame = len(frames_list)\n", 115 | " result_frames = []\n", 116 | " print(\"set stop frames to: \" + str(n_frame))\n", 117 | " for i in frames_list[0:int(n_frame)]:\n", 118 | " lama_frame = lama(i)\n", 119 | " lama_frame = Image.open(lama_frame)\n", 120 | " imageio.imwrite(f\"out-{i}\", lama_frame)\n", 121 | " result_frames.append(f\"out-{i}\")\n", 122 | " print(\"frame \" + i + \"/\" + str(n_frame) + \": done;\")\n", 123 | " final_vid = create_video(result_frames, fps, \"cleaned\")\n", 124 | " files = [final_vid]\n", 125 | " return final_vid\n", 126 | "\n", 127 | "with gr.Blocks() as demo:\n", 128 | " with gr.Column(elem_id=\"col-container\"):\n", 129 | " with gr.Row():\n", 130 | " with gr.Column():\n", 131 | " inputs = gr.Video(label=\"Input\", source=\"upload\", type=\"filepath\")\n", 132 | " with gr.Column():\n", 133 | " outputs = gr.Video(label=\"output\")\n", 134 | " submit_btn = gr.Button(\"Remove Watermark\")\n", 135 | " submit_btn.click(infer, inputs=[inputs], outputs=[outputs])\n", 136 | "# demo.queue().launch(debug=True, share=True, inline=False)\n", 137 | "demo.queue().launch(debug=True)" 138 | ] 139 | } 140 | ], 141 | "metadata": { 142 | "accelerator": "GPU", 143 | "colab": { 144 | "provenance": [] 145 | }, 146 | "gpuClass": "standard", 147 | "kernelspec": { 148 | "display_name": "Python 3", 149 | "name": "python3" 150 | }, 151 | "language_info": { 152 | "name": "python" 153 | } 154 | }, 155 | "nbformat": 4, 156 | "nbformat_minor": 0 157 | } 158 | -------------------------------------------------------------------------------- /camenduru's_webui_to_diffusers_embeddings.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "view-in-github" 7 | }, 8 | "source": [ 9 | "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/notebooks/blob/main/camenduru's_webui_to_diffusers_embeddings.ipynb)" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "id": "oRljt9Kfhh5r" 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "!pip install -q torch==1.13.1+cu116 torchvision==0.14.1+cu116 torchaudio==0.13.1 torchtext==0.14.1 torchdata==0.5.1 --extra-index-url https://download.pytorch.org/whl/cu116 -U\n", 21 | "!pip install -q diffusers transformers ftfy accelerate\n", 22 | "!pip install -q https://github.com/camenduru/stable-diffusion-webui-colab/releases/download/0.0.15/xformers-0.0.15.dev0+189828c.d20221207-cp38-cp38-linux_x86_64.whl\n", 23 | "\n", 24 | "import os, torch\n", 25 | "from PIL import Image\n", 26 | "from diffusers import StableDiffusionPipeline\n", 27 | "from transformers import CLIPTextModel, CLIPTokenizer\n", 28 | "\n", 29 | "model_path = \"lilpotat/a3\"\n", 30 | "tokenizer = CLIPTokenizer.from_pretrained(model_path, subfolder=\"tokenizer\")\n", 31 | "text_encoder = CLIPTextModel.from_pretrained(model_path, subfolder=\"text_encoder\")\n", 32 | "\n", 33 | "def load_learned_embed_in_clip(learned_embeds_path, text_encoder, tokenizer, token=None, type=0):\n", 34 | " # filename, file_extension = os.path.splitext(learned_embeds_path)\n", 35 | " loaded_learned_embeds = torch.load(learned_embeds_path, map_location=\"cpu\")\n", 36 | " # if (file_extension == \".pt\"):\n", 37 | " if(type == 0):\n", 38 | " trained_token = list(loaded_learned_embeds.keys())[0]\n", 39 | " embeds = loaded_learned_embeds[trained_token]\n", 40 | " elif(type == 1):\n", 41 | " string_to_token = loaded_learned_embeds['string_to_token']\n", 42 | " string_to_param = loaded_learned_embeds['string_to_param']\n", 43 | " trained_token = list(string_to_token.keys())[0]\n", 44 | " embeds = string_to_param[trained_token]\n", 45 | " embeds = embeds.detach()\n", 46 | " embeds = embeds[1]\n", 47 | " elif(type == 2):\n", 48 | " embeds = loaded_learned_embeds[0]\n", 49 | " dtype = text_encoder.get_input_embeddings().weight.dtype\n", 50 | " embeds.to(dtype)\n", 51 | " token = token if token is not None else trained_token\n", 52 | " num_added_tokens = tokenizer.add_tokens(token)\n", 53 | " if num_added_tokens == 0:\n", 54 | " raise ValueError(f\"The tokenizer already contains the token {token}.\")\n", 55 | " text_encoder.resize_token_embeddings(len(tokenizer))\n", 56 | " token_id = tokenizer.convert_tokens_to_ids(token)\n", 57 | " text_encoder.get_input_embeddings().weight.data[token_id] = embeds" 58 | ] 59 | }, 60 | { 61 | "cell_type": "code", 62 | "execution_count": null, 63 | "metadata": { 64 | "id": "WulKgO4aSK1L" 65 | }, 66 | "outputs": [], 67 | "source": [ 68 | "!wget https://huggingface.co/datasets/Nerfgun3/bad_prompt/resolve/main/bad_prompt_version2.pt\n", 69 | "!wget https://github.com/MushroomFleet/djz-Aesthetic-Embeddings/raw/main/djz-CyberCityV0.pt\n", 70 | "!wget https://gitgud.io/viper1/stable-diffusion-embeddings/-/raw/master/embeddings/botan-50000.pt\n", 71 | "!wget https://raw.githubusercontent.com/hlky/sd-embeddings/main/anya/anya.pt\n", 72 | "!wget https://huggingface.co/datasets/Nerfgun3/bad_prompt/resolve/main/bad_prompt.pt\n", 73 | "!wget https://huggingface.co/sd-concepts-library/midjourney-style/resolve/main/learned_embeds.bin" 74 | ] 75 | }, 76 | { 77 | "cell_type": "code", 78 | "execution_count": null, 79 | "metadata": { 80 | "id": "vHP8f5gWMnlc" 81 | }, 82 | "outputs": [], 83 | "source": [ 84 | "# load_learned_embed_in_clip(\"/content/bad_prompt.pt\", text_encoder, tokenizer, \"bad_prompt\", type=1)\n", 85 | "# load_learned_embed_in_clip(\"/content/djz-CyberCityV0.pt\", text_encoder, tokenizer, \"CyberCityV0\", type=2)\n", 86 | "# load_learned_embed_in_clip(\"/content/botan-50000.pt\", text_encoder, tokenizer, \"botan-50000\", type=2)\n", 87 | "load_learned_embed_in_clip(\"/content/learned_embeds.bin\", text_encoder, tokenizer, type=0)\n", 88 | "# load_learned_embed_in_clip(\"/content/4tnght.pt\", text_encoder, tokenizer, \"<4tNGHT>\", type=0)" 89 | ] 90 | }, 91 | { 92 | "cell_type": "code", 93 | "execution_count": null, 94 | "metadata": { 95 | "id": "SUu2dkv5DOe3" 96 | }, 97 | "outputs": [], 98 | "source": [ 99 | "pipe = StableDiffusionPipeline.from_pretrained(model_path, text_encoder=text_encoder, tokenizer=tokenizer, safety_checker=None).to(\"cuda\")\n", 100 | "pipe.enable_xformers_memory_efficient_attention()" 101 | ] 102 | }, 103 | { 104 | "cell_type": "code", 105 | "execution_count": null, 106 | "metadata": { 107 | "id": "wj-Bz_5vhdcO" 108 | }, 109 | "outputs": [], 110 | "source": [ 111 | "# with torch.autocast(\"cuda\"):\n", 112 | "# image = pipe(\" house\").images[0]\n", 113 | "# display(image)\n", 114 | "\n", 115 | "# with torch.autocast(\"cuda\"):\n", 116 | "# image = pipe(\"<4tNGHT>\").images[0]\n", 117 | "# display(image)\n", 118 | "\n", 119 | "# with torch.autocast(\"cuda\"):\n", 120 | "# image = pipe(\"girl waiving\", negative_prompt=\"bad_prompt\").images[0]\n", 121 | "# display(image)\n", 122 | " \n", 123 | "generator = torch.cuda.manual_seed(10)\n", 124 | "with torch.autocast(\"cuda\"):\n", 125 | " image = pipe(\"cute girl waving to camera \", negative_prompt=\"\", generator=generator).images[0]\n", 126 | "real_seed = torch.cuda.initial_seed()\n", 127 | "display(image)\n", 128 | "print(real_seed)" 129 | ] 130 | }, 131 | { 132 | "cell_type": "code", 133 | "execution_count": null, 134 | "metadata": { 135 | "colab": { 136 | "base_uri": "https://localhost:8080/" 137 | }, 138 | "id": "vjtrVr7ZECFn", 139 | "outputId": "3f7b250d-41b5-4413-c79f-5a068242ac3d" 140 | }, 141 | "outputs": [ 142 | { 143 | "name": "stdout", 144 | "output_type": "stream", 145 | "text": [ 146 | "torch.Size([768])\n" 147 | ] 148 | } 149 | ], 150 | "source": [ 151 | "def test_learned_embed_in_clip(learned_embeds_path):\n", 152 | " loaded_learned_embeds = torch.load(learned_embeds_path, map_location=\"cpu\")\n", 153 | " trained_token = list(loaded_learned_embeds.keys())[0]\n", 154 | " embeds = loaded_learned_embeds[trained_token]\n", 155 | " # print(loaded_learned_embeds)\n", 156 | " print(embeds.shape)\n", 157 | "\n", 158 | "# test_learned_embed_in_clip(\"/content/anya.pt\")\n", 159 | "# test_learned_embed_in_clip(\"/content/botan-50000.pt\")\n", 160 | "# test_learned_embed_in_clip(\"/content/djz-CyberCityV0.pt\")\n", 161 | "test_learned_embed_in_clip(\"/content/learned_embeds.bin\")\n", 162 | "# test_learned_embed_in_clip(\"/content/bad_prompt_version2.pt\")\n", 163 | "# test_learned_embed_in_clip(\"/content/4tnght.pt\")" 164 | ] 165 | } 166 | ], 167 | "metadata": { 168 | "accelerator": "GPU", 169 | "colab": { 170 | "provenance": [] 171 | }, 172 | "gpuClass": "standard", 173 | "kernelspec": { 174 | "display_name": "Python 3", 175 | "name": "python3" 176 | }, 177 | "language_info": { 178 | "name": "python" 179 | } 180 | }, 181 | "nbformat": 4, 182 | "nbformat_minor": 0 183 | } 184 | -------------------------------------------------------------------------------- /camenduru's_whisper.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "view-in-github" 7 | }, 8 | "source": [ 9 | "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/notebooks/blob/main/camenduru's_whisper.ipynb)" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "id": "ZsJUxc0aRsAf" 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "!pip install -q torch==1.13.1+cu116 torchvision==0.14.1+cu116 torchaudio==0.13.1 torchtext==0.14.1 torchdata==0.5.1 --extra-index-url https://download.pytorch.org/whl/cu116 -U\n", 21 | "from google.colab import drive\n", 22 | "drive.mount('/content/drive')\n", 23 | "\n", 24 | "!pip install git+https://github.com/openai/whisper.git -q\n", 25 | "\n", 26 | "import whisper\n", 27 | "\n", 28 | "model = whisper.load_model(\"large\")\n", 29 | "result = model.transcribe(\"input.mp3\")\n", 30 | "print(result[\"text\"])" 31 | ] 32 | } 33 | ], 34 | "metadata": { 35 | "accelerator": "GPU", 36 | "colab": { 37 | "collapsed_sections": [], 38 | "provenance": [] 39 | }, 40 | "gpuClass": "standard", 41 | "kernelspec": { 42 | "display_name": "Python 3 (ipykernel)", 43 | "language": "python", 44 | "name": "python3" 45 | }, 46 | "language_info": { 47 | "codemirror_mode": { 48 | "name": "ipython", 49 | "version": 3 50 | }, 51 | "file_extension": ".py", 52 | "mimetype": "text/x-python", 53 | "name": "python", 54 | "nbconvert_exporter": "python", 55 | "pygments_lexer": "ipython3", 56 | "version": "3.9.9" 57 | } 58 | }, 59 | "nbformat": 4, 60 | "nbformat_minor": 0 61 | } 62 | -------------------------------------------------------------------------------- /camenduru's_whisper_ja_video.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "view-in-github" 7 | }, 8 | "source": [ 9 | "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/notebooks/blob/main/camenduru's_whisper_ja_video.ipynb)" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "vscode": { 17 | "languageId": "plaintext" 18 | } 19 | }, 20 | "outputs": [], 21 | "source": [ 22 | "!pip install -q torch==1.13.1+cu116 torchvision==0.14.1+cu116 torchaudio==0.13.1 torchtext==0.14.1 torchdata==0.5.1 --extra-index-url https://download.pytorch.org/whl/cu116 -U\n", 23 | "!pip install transformers gradio yt_dlp pytube git+https://github.com/openai/whisper.git -q" 24 | ] 25 | }, 26 | { 27 | "cell_type": "code", 28 | "execution_count": null, 29 | "metadata": { 30 | "vscode": { 31 | "languageId": "plaintext" 32 | } 33 | }, 34 | "outputs": [], 35 | "source": [ 36 | "# https://huggingface.co/spaces/vumichien/whisper-speaker-diarization/blob/main/app.py modifiyed\n", 37 | "\n", 38 | "from transformers import pipeline\n", 39 | "from pytube import YouTube\n", 40 | "\n", 41 | "pipe = pipeline(\n", 42 | " task=\"automatic-speech-recognition\",\n", 43 | " model=\"vumichien/whisper-medium-jp\",\n", 44 | " chunk_length_s=30,\n", 45 | " device=0,\n", 46 | ")\n", 47 | "\n", 48 | "pipe.model.config.forced_decoder_ids = pipe.tokenizer.get_decoder_prompt_ids(language=\"ja\", task=\"transcribe\")\n", 49 | "\n", 50 | "def yt_transcribe(yt_url):\n", 51 | " yt = YouTube(yt_url)\n", 52 | " stream = yt.streams.filter(only_audio=True)[0]\n", 53 | " stream.download(filename=\"audio.mp3\")\n", 54 | " text = pipe(\"audio.mp3\")[\"text\"]\n", 55 | " return text\n", 56 | "\n", 57 | "print(yt_transcribe(\"https://www.youtube.com/watch?v=ulziqjIMDKQ\"))" 58 | ] 59 | }, 60 | { 61 | "cell_type": "code", 62 | "execution_count": null, 63 | "metadata": { 64 | "vscode": { 65 | "languageId": "plaintext" 66 | } 67 | }, 68 | "outputs": [], 69 | "source": [ 70 | "from yt_dlp import YoutubeDL\n", 71 | "#with YoutubeDL({'overwrites':True, 'format':'bestvideo[ext=mp4]+bestaudio[ext=m4a]/mp4', 'outtmpl':'/content/video'}) as ydl:\n", 72 | "with YoutubeDL({'overwrites':True, 'format':'bestaudio[ext=m4a]', 'outtmpl':'/content/audio.mp4'}) as ydl:\n", 73 | " ydl.download(\"https://www.youtube.com/watch?v=ulziqjIMDKQ\")" 74 | ] 75 | }, 76 | { 77 | "cell_type": "code", 78 | "execution_count": null, 79 | "metadata": { 80 | "vscode": { 81 | "languageId": "plaintext" 82 | } 83 | }, 84 | "outputs": [], 85 | "source": [ 86 | "from transformers import pipeline\n", 87 | "import gradio as gr\n", 88 | "\n", 89 | "pipe = pipeline(model=\"vumichien/whisper-medium-jp\", chunk_length_s=30, device=0)\n", 90 | "text = pipe(\"/content/audio.mp4\")[\"text\"]\n", 91 | "print(text)" 92 | ] 93 | }, 94 | { 95 | "cell_type": "code", 96 | "execution_count": null, 97 | "metadata": { 98 | "vscode": { 99 | "languageId": "plaintext" 100 | } 101 | }, 102 | "outputs": [], 103 | "source": [ 104 | "import whisper\n", 105 | "\n", 106 | "model = whisper.load_model(\"large\")\n", 107 | "result = model.transcribe(\"/content/audio.mp4\")\n", 108 | "print(result[\"text\"])" 109 | ] 110 | } 111 | ], 112 | "metadata": { 113 | "accelerator": "GPU", 114 | "colab": { 115 | "provenance": [] 116 | }, 117 | "gpuClass": "standard", 118 | "kernelspec": { 119 | "display_name": "Python 3", 120 | "name": "python3" 121 | }, 122 | "language_info": { 123 | "name": "python" 124 | } 125 | }, 126 | "nbformat": 4, 127 | "nbformat_minor": 0 128 | } 129 | -------------------------------------------------------------------------------- /ddPn08's_tensor-rt.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "view-in-github" 7 | }, 8 | "source": [ 9 | "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/notebooks/blob/main/ddPn08's_tensor-rt.ipynb)" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": {}, 16 | "outputs": [], 17 | "source": [ 18 | "!pip install -q torch==1.13.1+cu116 torchvision==0.14.1+cu116 torchaudio==0.13.1 torchtext==0.14.1 torchdata==0.5.1 --extra-index-url https://download.pytorch.org/whl/cu116 -U\n", 19 | "# https://github.com/ddPn08/tensorrt-diffusion-colab modified\n", 20 | "import os\n", 21 | "\n", 22 | "!apt update && apt install software-properties-common -y && add-apt-repository --yes ppa:deadsnakes/ppa\n", 23 | "!apt update && apt install tensorrt tensorrt-dev tensorrt-libs git-lfs -y\n", 24 | "!pip install tensorrt==8.5.3.1" 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "execution_count": null, 30 | "metadata": {}, 31 | "outputs": [], 32 | "source": [ 33 | "!git clone --single-branch https://github.com/camenduru/TensorRT && \\\n", 34 | " cd TensorRT && \\\n", 35 | " git submodule update --init --recursive\n", 36 | "\n", 37 | "!cd TensorRT && mkdir -p build && \\\n", 38 | " cd build && \\\n", 39 | " cmake /content/TensorRT -DTRT_OUT_DIR=$PWD/out && \\\n", 40 | " cd plugin && \\\n", 41 | " make -j$(nproc)" 42 | ] 43 | }, 44 | { 45 | "cell_type": "code", 46 | "execution_count": null, 47 | "metadata": {}, 48 | "outputs": [], 49 | "source": [ 50 | "from huggingface_hub import create_repo, upload_folder\n", 51 | "hf_token = 'token_here' # @param{type:\"string\"}\n", 52 | "repo_id = 'camenduru/libnvinfer_plugin.so.8.5.3_colab' # @param{type:\"string\"}\n", 53 | "create_repo(repo_id, private=True, token=hf_token)\n", 54 | "upload_folder(folder_path='/content/TensorRT/build/out', path_in_repo='', repo_id=repo_id, commit_message='content', token=hf_token)" 55 | ] 56 | }, 57 | { 58 | "cell_type": "code", 59 | "execution_count": null, 60 | "metadata": {}, 61 | "outputs": [], 62 | "source": [ 63 | "TRT_OSSPATH = os.path.abspath(\"./TensorRT\")\n", 64 | "script_filepath = os.path.join(TRT_OSSPATH, \"demo\", \"Diffusion\", \"build.py\")\n", 65 | "!curl -o {script_filepath} -L https://raw.githubusercontent.com/camenduru/tensorrt-diffusion-colab/main/scripts/build.py\n", 66 | "!pip install --upgrade pip\n", 67 | "!cd TensorRT/demo/Diffusion && pip install -r requirements.txt && pip install numpy==1.21.6" 68 | ] 69 | }, 70 | { 71 | "cell_type": "code", 72 | "execution_count": null, 73 | "metadata": { 74 | "id": "OEp_KcG-_yeA" 75 | }, 76 | "outputs": [], 77 | "source": [ 78 | "!git clone https://huggingface.co/trrt/anything-v4.5-vae-swapped /content/model" 79 | ] 80 | }, 81 | { 82 | "cell_type": "code", 83 | "execution_count": null, 84 | "metadata": { 85 | "id": "SSE-qQcmw20m" 86 | }, 87 | "outputs": [], 88 | "source": [ 89 | "unet_pretrained_model_id = \"ckpt/anything-v4.5-vae-swapped\" # @param{type:\"string\"}\n", 90 | "vae_pretrained_model_id = \"ckpt/anything-v4.5-vae-swapped\" # @param{type:\"string\"}\n", 91 | "clip_pretrained_model_id = \"openai/clip-vit-large-patch14\" # @param{type:\"string\"}\n", 92 | "\n", 93 | "os.environ[\"UNET_PRETRAINED_MODEL_ID\"] = unet_pretrained_model_id\n", 94 | "os.environ[\"VAE_PRETRAINED_MODEL_ID\"] = vae_pretrained_model_id\n", 95 | "os.environ[\"CLIP_PRETRAINED_MODEL_ID\"] = clip_pretrained_model_id\n", 96 | "\n", 97 | "engine_dir = \"/content/model/engine\" # @param{type:\"string\"}\n", 98 | "onnx_dir = \"/content/model/onnx\" # @param{type:\"string\"}\n", 99 | "\n", 100 | "os.makedirs(engine_dir, exist_ok=True)\n", 101 | "os.makedirs(onnx_dir, exist_ok=True)\n", 102 | "\n", 103 | "hf_token = \"\" # @param{type:\"string\"}\n", 104 | "denoising_prec = \"fp32\" # @param [\"fp32\", \"fp16\"]\n", 105 | "scheduler = \"LMSD\" # @param [\"LMSD\", \"DPM\"]\n", 106 | "height = 512 # @param{type:\"slider\", min:256, max:1024, step:64}\n", 107 | "width = 512 # @param{type:\"slider\", min:256, max:1024, step:64}\n", 108 | "\n", 109 | "onnx_opset = 16 # @param{type:\"integer\"}\n", 110 | "force_onnx_export = False # @param{type:\"boolean\"}\n", 111 | "force_onnx_optimize = False # @param{type:\"boolean\"}\n", 112 | "onnx_minimal_optimization = False # @param{type:\"boolean\"}\n", 113 | "\n", 114 | "force_engine_build = False # @param{type:\"boolean\"}\n", 115 | "build_static_batch = False # @param{type:\"boolean\"}\n", 116 | "build_dynamic_shape = False # @param{type:\"boolean\"}\n", 117 | "build_preview_features = False # @param{type:\"boolean\"}\n", 118 | "\n", 119 | "def make_args(d):\n", 120 | " arguments = []\n", 121 | " for k, v in d.items():\n", 122 | " k = k.replace(\"_\", \"-\")\n", 123 | " if type(v) == bool:\n", 124 | " arguments.append(f\"--{k}\" if v else \"\")\n", 125 | " elif type(v) == str and v:\n", 126 | " arguments.extend([f\"--{k}\", f\"{v}\"])\n", 127 | " elif v:\n", 128 | " arguments.extend([f\"--{k}\", f\"{v}\"])\n", 129 | " return \" \".join(arguments)\n", 130 | "\n", 131 | "args = make_args({\n", 132 | " \"engine-dir\": engine_dir,\n", 133 | " \"onnx-dir\": onnx_dir,\n", 134 | " \"hf-token\": hf_token,\n", 135 | " \"denoising-prec\": denoising_prec,\n", 136 | " \"scheduler\": scheduler,\n", 137 | " \"height\": height,\n", 138 | " \"width\": width,\n", 139 | " \"onnx-opset\": onnx_opset,\n", 140 | " \"force-onnx-export\": force_onnx_export,\n", 141 | " \"force-onnx-optimize\": force_onnx_optimize,\n", 142 | " \"onnx-minimal-optimization\": onnx_minimal_optimization,\n", 143 | " \"force-engine-build\": force_engine_build,\n", 144 | " \"build-static-batch\": build_static_batch,\n", 145 | " \"build-dynamic-shape\": build_dynamic_shape,\n", 146 | " \"build-preview-features\": build_preview_features\n", 147 | "})\n", 148 | "\n", 149 | "! cd /content/TensorRT/demo/Diffusion && LD_PRELOAD=\"/content/TensorRT/build/out/libnvinfer_plugin.so.8.5.3\" python {script_filepath} {args}" 150 | ] 151 | }, 152 | { 153 | "cell_type": "code", 154 | "execution_count": null, 155 | "metadata": {}, 156 | "outputs": [], 157 | "source": [ 158 | "from huggingface_hub import create_repo, upload_folder\n", 159 | "hf_token = 'token_here' # @param{type:\"string\"}\n", 160 | "repo_id = 'trrt/anything-v4.5-vae-swapped' # @param{type:\"string\"}\n", 161 | "create_repo(repo_id, private=True, token=hf_token)\n", 162 | "upload_folder(folder_path='/content/model', path_in_repo='', repo_id=repo_id, commit_message='content', token=hf_token)" 163 | ] 164 | }, 165 | { 166 | "cell_type": "code", 167 | "execution_count": null, 168 | "metadata": { 169 | "id": "RHR74sk6bszi" 170 | }, 171 | "outputs": [], 172 | "source": [ 173 | "engine_dir = \"/content/model/engine\" # @param{type:\"string\"}\n", 174 | "output_dir = \"/content/outputs\" # @param{type:\"string\"}\n", 175 | "\n", 176 | "os.makedirs(output_dir, exist_ok=True)\n", 177 | "\n", 178 | "prompt = \"masterpiece, best quality, 1girl\" # @param{type:\"string\"}\n", 179 | "negative_prompt = \"\" # @param{type:\"string\"}\n", 180 | "repeat_prompt = 1 # @param [1, 2, 4, 8, 16] {type:\"raw\"}\n", 181 | "height = 512 # @param{type:\"slider\", min:256, max:1024, step:64}\n", 182 | "width = 512 # @param{type:\"slider\", min:256, max:1024, step:64}\n", 183 | "denoising_steps = 50 # @param{type:\"integer\"}\n", 184 | "denoising_prec = \"fp32\" # @param [\"fp32\", \"fp16\"]\n", 185 | "scheduler = \"LMSD\" # @param [\"LMSD\", \"DPM\"]\n", 186 | "seed = \"1\" # @param{type:\"string\"}\n", 187 | "\n", 188 | "num_warmup_runs = 0 # @param{type:\"integer\"}\n", 189 | "\n", 190 | "def make_args(d):\n", 191 | " arguments = []\n", 192 | " for k, v in d.items():\n", 193 | " k = k.replace(\"_\", \"-\")\n", 194 | " if type(v) == bool:\n", 195 | " arguments.append(f\"--{k}\" if v else \"\")\n", 196 | " elif type(v) == str and v:\n", 197 | " arguments.extend([f\"--{k}\", f\"{v}\"])\n", 198 | " elif v:\n", 199 | " arguments.extend([f\"--{k}\", f\"{v}\"])\n", 200 | " return \" \".join(arguments)\n", 201 | "\n", 202 | "args = make_args({\n", 203 | " \"engine-dir\": engine_dir,\n", 204 | " \"output-dir\": output_dir,\n", 205 | " \"negative-prompt\": negative_prompt,\n", 206 | " \"repeat-prompt\": repeat_prompt,\n", 207 | " \"height\": height,\n", 208 | " \"width\": height,\n", 209 | " \"denoising-steps\": denoising_steps,\n", 210 | " \"denoising-prec\": denoising_prec,\n", 211 | " \"scheduler\": scheduler,\n", 212 | " \"seed\": seed,\n", 213 | " \"num-warmup-runs\": num_warmup_runs\n", 214 | "})\n", 215 | "\n", 216 | "!cd /content/TensorRT/demo/Diffusion && LD_PRELOAD=\"/content/TensorRT/build/out/libnvinfer_plugin.so.8.5.3\" python demo-diffusion.py \"{prompt}\" {args}" 217 | ] 218 | }, 219 | { 220 | "cell_type": "code", 221 | "execution_count": null, 222 | "metadata": {}, 223 | "outputs": [], 224 | "source": [ 225 | "!curl -s https://ngrok-agent.s3.amazonaws.com/ngrok.asc | sudo tee /etc/apt/trusted.gpg.d/ngrok.asc >/dev/null && echo \"deb https://ngrok-agent.s3.amazonaws.com buster main\" | sudo tee /etc/apt/sources.list.d/ngrok.list && sudo apt update && sudo apt install ngrok" 226 | ] 227 | }, 228 | { 229 | "cell_type": "code", 230 | "execution_count": null, 231 | "metadata": {}, 232 | "outputs": [], 233 | "source": [ 234 | "!ngrok config add-authtoken token_here" 235 | ] 236 | }, 237 | { 238 | "cell_type": "code", 239 | "execution_count": null, 240 | "metadata": {}, 241 | "outputs": [], 242 | "source": [ 243 | "!pip install colab-xterm" 244 | ] 245 | }, 246 | { 247 | "cell_type": "code", 248 | "execution_count": null, 249 | "metadata": {}, 250 | "outputs": [], 251 | "source": [ 252 | "%load_ext colabxterm\n", 253 | "%xterm" 254 | ] 255 | }, 256 | { 257 | "cell_type": "code", 258 | "execution_count": null, 259 | "metadata": {}, 260 | "outputs": [], 261 | "source": [ 262 | "# %env CUDA_MODULE_LOADING=LAZY" 263 | ] 264 | }, 265 | { 266 | "cell_type": "code", 267 | "execution_count": null, 268 | "metadata": {}, 269 | "outputs": [], 270 | "source": [ 271 | "!npm install -g pnpm" 272 | ] 273 | }, 274 | { 275 | "cell_type": "code", 276 | "execution_count": null, 277 | "metadata": {}, 278 | "outputs": [], 279 | "source": [ 280 | "!git clone https://github.com/camenduru/Lsmith.git" 281 | ] 282 | }, 283 | { 284 | "cell_type": "code", 285 | "execution_count": null, 286 | "metadata": {}, 287 | "outputs": [], 288 | "source": [ 289 | "!cd /content/Lsmith/frontend && pnpm i && pnpm build --out-dir ../dist" 290 | ] 291 | }, 292 | { 293 | "cell_type": "code", 294 | "execution_count": null, 295 | "metadata": {}, 296 | "outputs": [], 297 | "source": [ 298 | "!LD_PRELOAD=\"/content/TensorRT/build/out/libnvinfer_plugin.so.8.5.3\" python -m uvicorn modules.main:app --host 0.0.0.0 --port 7860 " 299 | ] 300 | } 301 | ], 302 | "metadata": { 303 | "accelerator": "GPU", 304 | "colab": { 305 | "provenance": [] 306 | }, 307 | "gpuClass": "standard", 308 | "kernelspec": { 309 | "display_name": "Python 3", 310 | "name": "python3" 311 | }, 312 | "language_info": { 313 | "name": "python" 314 | } 315 | }, 316 | "nbformat": 4, 317 | "nbformat_minor": 0 318 | } 319 | -------------------------------------------------------------------------------- /hysts's_lora.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "view-in-github" 7 | }, 8 | "source": [ 9 | "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/notebooks/blob/main/hysts's_lora.ipynb)" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "id": "u15OvQkxH7WN" 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "!pip install -q torch==1.13.1+cu116 torchvision==0.14.1+cu116 torchaudio==0.13.1 torchtext==0.14.1 torchdata==0.5.1 --extra-index-url https://download.pytorch.org/whl/cu116 -U\n", 21 | "!git clone https://huggingface.co/spaces/hysts/LoRA-SD-training\n", 22 | "\n", 23 | "!pip install -r /content/LoRA-SD-training/requirements.txt\n", 24 | "!pip install gradio\n", 25 | "!pip install -q https://github.com/camenduru/stable-diffusion-webui-colab/releases/download/0.0.15/xformers-0.0.15.dev0+189828c.d20221207-cp38-cp38-linux_x86_64.whl\n", 26 | "!git clone https://github.com/cloneofsimo/lora /content/LoRA-SD-training/lora\n", 27 | "\n", 28 | "import os\n", 29 | "os.system(f\"sed -i -e 's/share=False/share=True/g' /content/LoRA-SD-training/app.py\")\n", 30 | "\n", 31 | "%cd /content/LoRA-SD-training\n", 32 | "!python app.py" 33 | ] 34 | } 35 | ], 36 | "metadata": { 37 | "accelerator": "GPU", 38 | "colab": { 39 | "provenance": [] 40 | }, 41 | "gpuClass": "standard", 42 | "kernelspec": { 43 | "display_name": "Python 3", 44 | "name": "python3" 45 | }, 46 | "language_info": { 47 | "name": "python" 48 | } 49 | }, 50 | "nbformat": 4, 51 | "nbformat_minor": 0 52 | } 53 | -------------------------------------------------------------------------------- /multimodalart's_dreambooth.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "view-in-github" 7 | }, 8 | "source": [ 9 | "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/notebooks/blob/main/multimodalart's_dreambooth.ipynb)" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "id": "pPrggpwgVdHM" 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "!pip install -q torch==1.13.1+cu116 torchvision==0.14.1+cu116 torchaudio==0.13.1 torchtext==0.14.1 torchdata==0.5.1 --extra-index-url https://download.pytorch.org/whl/cu116 -U\n", 21 | "!git lfs install\n", 22 | "!git clone https://huggingface.co/spaces/multimodalart/dreambooth-training\n", 23 | "\n", 24 | "import os\n", 25 | "os.system(f\"sed -i -e 's/debug=True/debug=True, share=True/g' /content/dreambooth-training/app.py\")\n", 26 | "os.system(f\"sed -i -e '/apolinario/d' /content/dreambooth-training/requirements.txt\")\n", 27 | "\n", 28 | "os.system(f\"sed -i -e '/torch/d' /content/dreambooth-training/requirements.txt\")\n", 29 | "os.system(f\"sed -i -e '/torchvision/d' /content/dreambooth-training/requirements.txt\")\n", 30 | "\n", 31 | "!pip install -r /content/dreambooth-training/requirements.txt\n", 32 | "!pip install gradio\n", 33 | "!pip install -q https://github.com/camenduru/stable-diffusion-webui-colab/releases/download/0.0.15/xformers-0.0.15.dev0+189828c.d20221207-cp38-cp38-linux_x86_64.whl\n", 34 | "\n", 35 | "#!wget https://huggingface.co/spaces/multimodalart/dreambooth-training/resolve/main/mix.zip -O /content/dreambooth-training/mix.zip\n", 36 | "\n", 37 | "%cd /content/dreambooth-training\n", 38 | "!python app.py" 39 | ] 40 | } 41 | ], 42 | "metadata": { 43 | "colab": { 44 | "provenance": [] 45 | }, 46 | "kernelspec": { 47 | "display_name": "Python 3", 48 | "name": "python3" 49 | }, 50 | "language_info": { 51 | "name": "python" 52 | } 53 | }, 54 | "nbformat": 4, 55 | "nbformat_minor": 0 56 | } 57 | -------------------------------------------------------------------------------- /sam_witteveen's_dolly_lora_gptj6b_alpaca.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "view-in-github" 7 | }, 8 | "source": [ 9 | "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/notebooks/blob/main/sam_witteveen's_dolly_lora_gptj6b_alpaca.ipynb)" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "id": "hDcH7FqJQssL" 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "!pip install -q torch==1.13.1+cu116 torchvision==0.14.1+cu116 torchaudio==0.13.1 torchtext==0.14.1 torchdata==0.5.1 --extra-index-url https://download.pytorch.org/whl/cu116 -U\n", 21 | "# https://huggingface.co/samwit/dolly-lora/tree/main\n", 22 | "# https://www.databricks.com/blog/2023/03/24/hello-dolly-democratizing-magic-chatgpt-open-models.html\n", 23 | "# https://github.com/gururise/AlpacaDataCleaned\n", 24 | "\n", 25 | "!pip -q install datasets loralib sentencepiece\n", 26 | "!pip -q install git+https://github.com/huggingface/transformers\n", 27 | "!pip -q install git+https://github.com/huggingface/peft.git\n", 28 | "!pip -q install bitsandbytes" 29 | ] 30 | }, 31 | { 32 | "cell_type": "code", 33 | "execution_count": null, 34 | "metadata": { 35 | "id": "LAiWUGrCQ8qr" 36 | }, 37 | "outputs": [], 38 | "source": [ 39 | "from transformers import AutoTokenizer, AutoModelForCausalLM\n", 40 | "tokenizer = AutoTokenizer.from_pretrained(\"EleutherAI/gpt-j-6B\")\n", 41 | "tokenizer.pad_token = tokenizer.eos_token\n", 42 | "tokenizer.pad_token_id = tokenizer.eos_token_id" 43 | ] 44 | }, 45 | { 46 | "cell_type": "code", 47 | "execution_count": null, 48 | "metadata": { 49 | "id": "tvDnB1n9RqpX" 50 | }, 51 | "outputs": [], 52 | "source": [ 53 | "from peft import PeftModel\n", 54 | "from transformers import AutoTokenizer, GPTJForCausalLM, GenerationConfig\n", 55 | "\n", 56 | "model = GPTJForCausalLM.from_pretrained(\n", 57 | " \"EleutherAI/gpt-j-6B\",\n", 58 | " load_in_8bit=True,\n", 59 | " device_map=\"auto\",\n", 60 | ")\n", 61 | "model = PeftModel.from_pretrained(model, \"samwit/dolly-lora\")\n" 62 | ] 63 | }, 64 | { 65 | "cell_type": "code", 66 | "execution_count": null, 67 | "metadata": { 68 | "id": "9M6HnRDLTXqG" 69 | }, 70 | "outputs": [], 71 | "source": [ 72 | "%%time\n", 73 | "PROMPT = \"\"\"Below is an instruction that describes a task. Write a response that appropriately completes the request.\n", 74 | "### Instruction:\n", 75 | "What are the differences between alpacas and sheep?\n", 76 | "### Response:\"\"\"\n", 77 | "\n", 78 | "inputs = tokenizer(\n", 79 | " PROMPT,\n", 80 | " return_tensors=\"pt\",\n", 81 | ")\n", 82 | "input_ids = inputs[\"input_ids\"].cuda()\n", 83 | "\n", 84 | "generation_config = GenerationConfig(\n", 85 | " temperature=0.6,\n", 86 | " top_p=0.95,\n", 87 | " repetition_penalty=1.2,\n", 88 | ")\n", 89 | "\n", 90 | "print(\"Generating...\")\n", 91 | "generation_output = model.generate(\n", 92 | " input_ids=input_ids,\n", 93 | " generation_config=generation_config,\n", 94 | " return_dict_in_generate=True,\n", 95 | " output_scores=True,\n", 96 | " max_new_tokens=128,\n", 97 | " pad_token_id = 0,\n", 98 | " eos_token_id = 50256\n", 99 | ")\n", 100 | "\n", 101 | "for s in generation_output.sequences:\n", 102 | " print(tokenizer.decode(s))" 103 | ] 104 | } 105 | ], 106 | "metadata": { 107 | "accelerator": "GPU", 108 | "colab": { 109 | "provenance": [] 110 | }, 111 | "gpuClass": "premium", 112 | "kernelspec": { 113 | "display_name": "Python 3", 114 | "name": "python3" 115 | }, 116 | "language_info": { 117 | "name": "python" 118 | } 119 | }, 120 | "nbformat": 4, 121 | "nbformat_minor": 0 122 | } 123 | --------------------------------------------------------------------------------