├── README.md ├── environment.yml ├── .gitignore └── inference_playground.ipynb /README.md: -------------------------------------------------------------------------------- 1 | # vtoonify-in-studio-lab 2 | 3 | [![Open In SageMaker Studio Lab](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/machinelearnear/vtoonify-in-studio-lab/blob/main/inference_playground.ipynb) 4 | [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/machinelearnear/vtoonify-in-studio-lab/blob/main/inference_playground.ipynb) -------------------------------------------------------------------------------- /environment.yml: -------------------------------------------------------------------------------- 1 | # To use: 2 | # $ conda env create -f environment.yml 3 | # $ conda activate 4 | name: machinelearnear-vtoonify-env 5 | channels: 6 | - conda-forge 7 | - defaults 8 | dependencies: 9 | - _libgcc_mutex=0.1=conda_forge 10 | - ca-certificates=2022.2.1=h06a4308_0 11 | - certifi=2021.10.8=py38h06a4308_2 12 | - faiss=1.7.1=py38h7b17aaf_0_cpu 13 | - libedit=3.1.20191231=he28a2e2_2 14 | - libfaiss=1.7.1=hb573701_0_cpu 15 | - libfaiss-avx2=1.7.1=h1234567_0_cpu 16 | - libffi=3.2.1=he1b5a44_1007 17 | - libgcc-ng=9.3.0=h2828fa1_19 18 | - libstdcxx-ng=9.3.0=h6de172a_19 19 | - matplotlib-base=3.3.4=py38h62a2d02_0 20 | - pillow=8.3.1=py38h2c7a002_0 21 | - pip=21.1.3=pyhd8ed1ab_0 22 | - python=3.8.3=cpython_he5300dc_0 23 | - python-lmdb=1.2.1=py38h2531618_1 24 | - setuptools=49.6.0=py38h578d9bd_3 25 | - scikit-image=0.18.1=py38ha9443f7_0 26 | - nb_conda_kernels 27 | - ipykernel 28 | - ipywidgets 29 | - gh 30 | - opencv 31 | - pip: 32 | - torch==1.7.1 33 | - torchvision==0.8.2 34 | - matplotlib==3.4.2 35 | - ninja==1.10.2 36 | - numpy==1.21.0 37 | - scipy==1.7.0 38 | - tqdm==4.61.2 39 | - wget==3.2 -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | -------------------------------------------------------------------------------- /inference_playground.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "1683dd62-1001-4be2-ab5a-d89965adf311", 6 | "metadata": {}, 7 | "source": [ 8 | "[![Open In SageMaker Studio Lab](https://studiolab.sagemaker.aws/studiolab.svg)](https://studiolab.sagemaker.aws/import/github/machinelearnear/vtoonify-in-studio-lab/blob/main/inference_playground.ipynb)\n", 9 | "[![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/machinelearnear/vtoonify-in-studio-lab/blob/main/inference_playground.ipynb)" 10 | ] 11 | }, 12 | { 13 | "cell_type": "markdown", 14 | "id": "d9089e70-f655-43d6-a210-fb749dc2f74b", 15 | "metadata": {}, 16 | "source": [ 17 | "# Run `VToonify`\n", 18 | "- Original: https://github.com/williamyang1991/VToonify\n", 19 | "- Modified from [pixel2style2pixel](https://github.com/eladrich/pixel2style2pixel/blob/master/notebooks/inference_playground.ipynb)" 20 | ] 21 | }, 22 | { 23 | "cell_type": "markdown", 24 | "id": "1e088059-045c-49c6-9c7c-d522984a50ce", 25 | "metadata": {}, 26 | "source": [ 27 | "## Setup environment" 28 | ] 29 | }, 30 | { 31 | "cell_type": "code", 32 | "execution_count": 1, 33 | "id": "542f4d8c-7bb1-4375-a6b0-262455b54a7f", 34 | "metadata": {}, 35 | "outputs": [], 36 | "source": [ 37 | "import os\n", 38 | "from os.path import exists as path_exists" 39 | ] 40 | }, 41 | { 42 | "cell_type": "code", 43 | "execution_count": 2, 44 | "id": "bf8a6a6b-c5e5-4458-816c-fb0d4780ec7c", 45 | "metadata": {}, 46 | "outputs": [], 47 | "source": [ 48 | "os.environ['CUDA_VISIBLE_DEVICES'] = \"0\"\n", 49 | "CODE_DIR = 'VToonify'\n", 50 | "device = 'cuda'" 51 | ] 52 | }, 53 | { 54 | "cell_type": "code", 55 | "execution_count": 3, 56 | "id": "74a2a1ff-e79e-4e3f-b255-3d926a873939", 57 | "metadata": {}, 58 | "outputs": [], 59 | "source": [ 60 | "if not path_exists(CODE_DIR):\n", 61 | " !git clone https://github.com/williamyang1991/VToonify.git $CODE_DIR" 62 | ] 63 | }, 64 | { 65 | "cell_type": "code", 66 | "execution_count": 4, 67 | "id": "8d1ee642-8927-4332-b6a1-50b232ca347c", 68 | "metadata": {}, 69 | "outputs": [], 70 | "source": [ 71 | "if not path_exists('ninja-linux.zip'):\n", 72 | " !wget https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-linux.zip\n", 73 | " !sudo unzip ninja-linux.zip -d /usr/local/bin/\n", 74 | " !sudo update-alternatives --install /usr/bin/ninja ninja /usr/local/bin/ninja 1 --force \n", 75 | "\n", 76 | "try:\n", 77 | " import wget\n", 78 | "except ImportError:\n", 79 | " !pip install wget" 80 | ] 81 | }, 82 | { 83 | "cell_type": "code", 84 | "execution_count": 5, 85 | "id": "c7cad44f-25ab-48bd-b9c1-99c1362fbb29", 86 | "metadata": {}, 87 | "outputs": [], 88 | "source": [ 89 | "os.chdir(f'./{CODE_DIR}')\n", 90 | "MODEL_DIR = os.path.join(os.path.dirname(os.getcwd()), CODE_DIR, 'checkpoint')\n", 91 | "DATA_DIR = os.path.join(os.path.dirname(os.getcwd()), CODE_DIR, 'data')\n", 92 | "OUT_DIR = os.path.join(os.path.dirname(os.getcwd()), CODE_DIR, 'output')" 93 | ] 94 | }, 95 | { 96 | "cell_type": "markdown", 97 | "id": "54151247-e1bd-4a6a-9d18-cf8cefc5878f", 98 | "metadata": {}, 99 | "source": [ 100 | "### Import libraries" 101 | ] 102 | }, 103 | { 104 | "cell_type": "code", 105 | "execution_count": 6, 106 | "id": "fd24618c-3045-45c9-8e0d-fd056ccce98a", 107 | "metadata": {}, 108 | "outputs": [ 109 | { 110 | "name": "stdout", 111 | "output_type": "stream", 112 | "text": [ 113 | "Collecting cmake==3.21.0\n", 114 | " Using cached cmake-3.21.0-py2.py3-none-manylinux_2_5_x86_64.manylinux1_x86_64.whl (19.8 MB)\n", 115 | "Installing collected packages: cmake\n", 116 | "Successfully installed cmake-3.21.0\n", 117 | "Collecting dlib==19.21.0\n", 118 | " Using cached dlib-19.21.0.tar.gz (3.2 MB)\n", 119 | "Building wheels for collected packages: dlib\n", 120 | " Building wheel for dlib (setup.py) ... \u001b[?25ldone\n", 121 | "\u001b[?25h Created wheel for dlib: filename=dlib-19.21.0-cp38-cp38-linux_x86_64.whl size=4063007 sha256=7dc975dcad0627f6d619d3e31af696f96804e8a11829fc7a22dad84c02036cb4\n", 122 | " Stored in directory: /home/studio-lab-user/.cache/pip/wheels/3a/4e/b6/77346839e430150a62d9b46bf7e0a37181fe01fd07d5d452a7\n", 123 | "Successfully built dlib\n", 124 | "Installing collected packages: dlib\n", 125 | "Successfully installed dlib-19.21.0\n" 126 | ] 127 | } 128 | ], 129 | "source": [ 130 | "try:\n", 131 | " import dlib\n", 132 | "except ImportError:\n", 133 | " !pip install cmake==3.21.0\n", 134 | " !pip install dlib==19.21.0" 135 | ] 136 | }, 137 | { 138 | "cell_type": "markdown", 139 | "id": "35263d7f-9eb9-4e23-a440-c2596f7d496f", 140 | "metadata": {}, 141 | "source": [ 142 | "### Mount Drive and copy files" 143 | ] 144 | }, 145 | { 146 | "cell_type": "code", 147 | "execution_count": null, 148 | "id": "6c9b76a2-febc-4c34-ade6-7acd03f4cafe", 149 | "metadata": {}, 150 | "outputs": [], 151 | "source": [ 152 | "from google.colab import drive\n", 153 | "from pathlib import Path\n", 154 | "import os\n", 155 | "\n", 156 | "ROOT_FOLDER = \"vtoonify\" # @param {type:\"string\"}\n", 157 | "PROJECT_FOLDER = \"checkpoint\" # @param {type:\"string\"}\n", 158 | "MOUNTED_PATH = Path('/content/drive')\n", 159 | "MYDRIVE_PATH = MOUNTED_PATH / \"MyDrive\"\n", 160 | "PROJECT_PATH = MYDRIVE_PATH / ROOT_FOLDER / PROJECT_FOLDER\n", 161 | "\n", 162 | "drive.mount(str(MOUNTED_PATH))\n", 163 | "\n", 164 | "os.makedirs(PROJECT_PATH, exist_ok=True)" 165 | ] 166 | }, 167 | { 168 | "cell_type": "markdown", 169 | "id": "6af736ea-dee0-454c-bbce-9949217990e5", 170 | "metadata": {}, 171 | "source": [ 172 | "Your checkpoint folder should contain the `encoder.pt/faceparsing.pth` files and the `vtoonify_t_{xyz}` checkpoints " 173 | ] 174 | }, 175 | { 176 | "cell_type": "code", 177 | "execution_count": null, 178 | "id": "fff3b49e-37a9-435a-a78c-2dbf8020ab57", 179 | "metadata": {}, 180 | "outputs": [], 181 | "source": [ 182 | "!cp -r $PROJECT_PATH ." 183 | ] 184 | }, 185 | { 186 | "cell_type": "markdown", 187 | "id": "04553477-61f7-48b1-91b0-12fcb4f2115d", 188 | "metadata": { 189 | "tags": [] 190 | }, 191 | "source": [ 192 | "## Style Transfer with `VToonify-D`" 193 | ] 194 | }, 195 | { 196 | "cell_type": "markdown", 197 | "id": "8a1ecf48-9924-4b99-ae52-911e272c35e0", 198 | "metadata": {}, 199 | "source": [ 200 | "Transfer a default cartoon style onto a default face image `./data/077436.jpg`:\n", 201 | "\n", 202 | "`python style_transfer.py --scale_image`\n", 203 | "\n", 204 | "The results are saved in the folder `./output/`, where `077436_input.jpg` is the rescaled input image to fit VToonify (this image can serve as the input without `--scale_image`) and `077436_vtoonify_d.jpg` is the result." 205 | ] 206 | }, 207 | { 208 | "cell_type": "markdown", 209 | "id": "48138851-29fc-43ff-8dae-2deddbfbfdb2", 210 | "metadata": {}, 211 | "source": [ 212 | "Specify the content image and the model, control the style with the following options:\n", 213 | "\n", 214 | "- `--content`: path to the target face image or video\n", 215 | "- `--style_id`: the index of the style image (find the mapping between index and the style image here).\n", 216 | "- `--style_degree` (default: 0.5): adjust the degree of style.\n", 217 | "- `--color_transfer` (default: False): perform color transfer if loading a VToonify-Dsdc model.\n", 218 | "- `--ckpt`: path of the VToonify-D model. By default, a VToonify-Dsd trained on cartoon style is loaded.\n", 219 | "- `--exstyle_path`: path of the extrinsic style code. By default, codes in the same directory as --ckpt are loaded.\n", 220 | "- `--scale_image`: rescale the input image/video to fit VToonify (highly recommend).\n", 221 | "- `--padding` (default: 200, 200, 200, 200): left, right, top, bottom paddings to the eye center.\n", 222 | "Here is an example of arcane style transfer:" 223 | ] 224 | }, 225 | { 226 | "cell_type": "markdown", 227 | "id": "ebd66e96-eb3c-4813-bb0e-5f3ba7aa5146", 228 | "metadata": {}, 229 | "source": [ 230 | "**Style transfer for images**\n", 231 | "```sh\n", 232 | "python style_transfer.py --content ./data/038648.jpg \\\n", 233 | " --scale_image --style_id 77 --style_degree 0.5 \\\n", 234 | " --ckpt ./checkpoint/vtoonify_d_arcane/vtoonify_s_d.pt \\\n", 235 | " --padding 600 600 600 600 # use large padding to avoid cropping the image\n", 236 | "```\n", 237 | "\n", 238 | "Specify `--video` to perform video toonification:\n", 239 | "\n", 240 | "```sh\n", 241 | "python style_transfer.py --scale_image --content ./data/YOUR_VIDEO.mp4 --video\n", 242 | "```\n", 243 | "\n", 244 | "The above style control options (`--style_id`, `--style_degree`, `--color_transfer`) also work for videos." 245 | ] 246 | }, 247 | { 248 | "cell_type": "code", 249 | "execution_count": null, 250 | "id": "fa24169a-c3b9-4c2b-b630-a5db35cf2dfa", 251 | "metadata": {}, 252 | "outputs": [], 253 | "source": [ 254 | "# cartoon026: balanced \n", 255 | "# cartoon299: big eyes \n", 256 | "# arcane000: for female \n", 257 | "# arcane077: for male \n", 258 | "# pixar052: \n", 259 | "# caricature039: big mouth \n", 260 | "# caricature068: balanced\n", 261 | "# style_type = \"cartoon026\" # [\"cartoon026\", \"cartoon299\", \"arcane000\", \"arcane077\", \"pixar052\", \"caricature039\", \"caricature068\"]" 262 | ] 263 | }, 264 | { 265 | "cell_type": "code", 266 | "execution_count": null, 267 | "id": "4c640fc1-4dfa-4a71-b5da-d430d3bd87b1", 268 | "metadata": {}, 269 | "outputs": [], 270 | "source": [ 271 | "# python style_transfer.py --scale_image --content ./data/YOUR_VIDEO.mp4 --video" 272 | ] 273 | }, 274 | { 275 | "cell_type": "markdown", 276 | "id": "1b0bc873-78ba-4fe4-98be-ff9fdddc33a7", 277 | "metadata": {}, 278 | "source": [ 279 | "## Style Transfer with `VToonify-T`" 280 | ] 281 | }, 282 | { 283 | "cell_type": "markdown", 284 | "id": "e1182e0a-f6e3-4c12-95ae-33414da4a55d", 285 | "metadata": {}, 286 | "source": [ 287 | "Specify `--backbone` as ''toonify'' to load and use a VToonify-T model.\n", 288 | "\n", 289 | "```sh\n", 290 | "python style_transfer.py --content ./data/038648.jpg \\\n", 291 | " --scale_image --backbone toonify \\\n", 292 | " --ckpt ./checkpoint/vtoonify_t_arcane/vtoonify.pt \\\n", 293 | " --padding 600 600 600 600 # use large padding to avoid cropping the image\n", 294 | "```\n", 295 | "\n", 296 | "In VToonify-T, `--style_id`, `--style_degree`, `--color_transfer`, `--exstyle_path` are not used.\n", 297 | "\n", 298 | "As with VToonify-D, specify `--video` to perform video toonification." 299 | ] 300 | }, 301 | { 302 | "cell_type": "code", 303 | "execution_count": null, 304 | "id": "1ae4e8d9-bbb3-4f03-b130-0f66d5330aa4", 305 | "metadata": {}, 306 | "outputs": [], 307 | "source": [ 308 | "style_type = \"arcane\" # [\"cartoon\", \"caricature\", \"arcane\", \"comic\", \"pixar\"]" 309 | ] 310 | }, 311 | { 312 | "cell_type": "code", 313 | "execution_count": null, 314 | "id": "09a7000f-5d23-4874-b75c-1c67d339c2b8", 315 | "metadata": {}, 316 | "outputs": [], 317 | "source": [ 318 | "!python style_transfer.py --content ./data/input.mp4 \\\n", 319 | " --scale_image --backbone toonify \\\n", 320 | " --ckpt ./checkpoint/vtoonify_t_arcane/vtoonify.pt \\\n", 321 | " --padding 600 600 600 600 # use large padding to avoid cropping the image" 322 | ] 323 | } 324 | ], 325 | "metadata": { 326 | "kernelspec": { 327 | "display_name": "machinelearnear-vtoonify-env:Python", 328 | "language": "python", 329 | "name": "conda-env-machinelearnear-vtoonify-env-py" 330 | }, 331 | "language_info": { 332 | "codemirror_mode": { 333 | "name": "ipython", 334 | "version": 3 335 | }, 336 | "file_extension": ".py", 337 | "mimetype": "text/x-python", 338 | "name": "python", 339 | "nbconvert_exporter": "python", 340 | "pygments_lexer": "ipython3", 341 | "version": "3.8.3" 342 | } 343 | }, 344 | "nbformat": 4, 345 | "nbformat_minor": 5 346 | } 347 | --------------------------------------------------------------------------------