├── .gitignore ├── .idea ├── .gitignore ├── inspectionProfiles │ ├── Project_Default.xml │ └── profiles_settings.xml ├── misc.xml ├── modules.xml ├── sd-webui-mov2mov.iml └── vcs.xml ├── CHANGELOG.md ├── CHANGELOG_CN.md ├── LICENSE ├── README.md ├── README_CN.md ├── ebsynth ├── __init__.py ├── _ebsynth.py ├── ebsynth.dll └── ebsynth_generate.py ├── images ├── 1.png ├── 2.jpg ├── alipay.png ├── img.png └── wechat.png ├── install.py ├── javascript └── m2m_ui.js ├── requirements.txt ├── scripts ├── m2m_config.py ├── m2m_hook.py ├── m2m_ui.py ├── m2m_ui_common.py ├── m2m_util.py ├── module_ui_extensions.py ├── mov2mov.py └── movie_editor.py ├── style.css └── tests ├── ebsynth ├── __init__.py ├── ebsynth_generate_test.py └── ebsynth_test.py └── utils.py /.gitignore: -------------------------------------------------------------------------------- 1 | ### macOS template 2 | # General 3 | .DS_Store 4 | .AppleDouble 5 | .LSOverride 6 | 7 | # Icon must end with two \r 8 | Icon 9 | 10 | # Thumbnails 11 | ._* 12 | 13 | # Files that might appear in the root of a volume 14 | .DocumentRevisions-V100 15 | .fseventsd 16 | .Spotlight-V100 17 | .TemporaryItems 18 | .Trashes 19 | .VolumeIcon.icns 20 | .com.apple.timemachine.donotpresent 21 | 22 | # Directories potentially created on remote AFP share 23 | .AppleDB 24 | .AppleDesktop 25 | Network Trash Folder 26 | Temporary Items 27 | .apdisk 28 | 29 | ### Python template 30 | # Byte-compiled / optimized / DLL files 31 | __pycache__/ 32 | *.py[cod] 33 | *$py.class 34 | 35 | # C extensions 36 | *.so 37 | 38 | # Distribution / packaging 39 | .Python 40 | build/ 41 | develop-eggs/ 42 | dist/ 43 | downloads/ 44 | eggs/ 45 | .eggs/ 46 | lib/ 47 | lib64/ 48 | parts/ 49 | sdist/ 50 | var/ 51 | wheels/ 52 | share/python-wheels/ 53 | *.egg-info/ 54 | .installed.cfg 55 | *.egg 56 | MANIFEST 57 | 58 | # PyInstaller 59 | # Usually these files are written by a python script from a template 60 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 61 | *.manifest 62 | *.spec 63 | 64 | # Installer logs 65 | pip-log.txt 66 | pip-delete-this-directory.txt 67 | 68 | # Unit test / coverage reports 69 | htmlcov/ 70 | .tox/ 71 | .nox/ 72 | .coverage 73 | .coverage.* 74 | .cache 75 | nosetests.xml 76 | coverage.xml 77 | *.cover 78 | *.py,cover 79 | .hypothesis/ 80 | .pytest_cache/ 81 | cover/ 82 | 83 | # Translations 84 | *.mo 85 | *.pot 86 | 87 | # Django stuff: 88 | *.log 89 | local_settings.py 90 | db.sqlite3 91 | db.sqlite3-journal 92 | 93 | # Flask stuff: 94 | instance/ 95 | .webassets-cache 96 | 97 | # Scrapy stuff: 98 | .scrapy 99 | 100 | # Sphinx documentation 101 | docs/_build/ 102 | 103 | # PyBuilder 104 | .pybuilder/ 105 | target/ 106 | 107 | # Jupyter Notebook 108 | .ipynb_checkpoints 109 | 110 | # IPython 111 | profile_default/ 112 | ipython_config.py 113 | 114 | # pyenv 115 | # For a library or package, you might want to ignore these files since the code is 116 | # intended to run in multiple environments; otherwise, check them in: 117 | # .python-version 118 | 119 | # pipenv 120 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 121 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 122 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 123 | # install all needed dependencies. 124 | #Pipfile.lock 125 | 126 | # poetry 127 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 128 | # This is especially recommended for binary packages to ensure reproducibility, and is more 129 | # commonly ignored for libraries. 130 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 131 | #poetry.lock 132 | 133 | # pdm 134 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 135 | #pdm.lock 136 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 137 | # in version control. 138 | # https://pdm.fming.dev/#use-with-ide 139 | .pdm.toml 140 | 141 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 142 | __pypackages__/ 143 | 144 | # Celery stuff 145 | celerybeat-schedule 146 | celerybeat.pid 147 | 148 | # SageMath parsed files 149 | *.sage.py 150 | 151 | # Environments 152 | .env 153 | .venv 154 | env/ 155 | venv/ 156 | ENV/ 157 | env.bak/ 158 | venv.bak/ 159 | 160 | # Spyder project settings 161 | .spyderproject 162 | .spyproject 163 | 164 | # Rope project settings 165 | .ropeproject 166 | 167 | # mkdocs documentation 168 | /site 169 | 170 | # mypy 171 | .mypy_cache/ 172 | .dmypy.json 173 | dmypy.json 174 | 175 | # Pyre type checker 176 | .pyre/ 177 | 178 | # pytype static type analyzer 179 | .pytype/ 180 | 181 | # Cython debug symbols 182 | cython_debug/ 183 | 184 | # PyCharm 185 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 186 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 187 | # and can be added to the global gitignore or merged into this file. For a more nuclear 188 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 189 | #.idea/ 190 | 191 | -------------------------------------------------------------------------------- /.idea/.gitignore: -------------------------------------------------------------------------------- 1 | # 默认忽略的文件 2 | /shelf/ 3 | /workspace.xml 4 | # 基于编辑器的 HTTP 客户端请求 5 | /httpRequests/ 6 | # Datasource local storage ignored files 7 | /dataSources/ 8 | /dataSources.local.xml 9 | -------------------------------------------------------------------------------- /.idea/inspectionProfiles/Project_Default.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 12 | -------------------------------------------------------------------------------- /.idea/inspectionProfiles/profiles_settings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 6 | -------------------------------------------------------------------------------- /.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/sd-webui-mov2mov.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 12 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ### 2023/9/30 2 | 1. automatic video fps parsing 3 | 2. Video editing features. 4 | 1. Customizable selection of keyframes or automatic generation of keyframes. 5 | 2. Backtrack keyframe tag. 6 | 3. automatically synthesize video based on keyframes via Ezsynth(https://github.com/Trentonom0r3/Ezsynth). 7 | 4. Currently, only the Windows system is supported. If your system does not support it, you can close this tab. 8 | 9 | ### 2023/9/24 10 | 1. Move the tab behind img2img. 11 | 2. Fix the issue of video synthesis failure on the Mac system. 12 | 3. Fix the problem of refiner not taking effect 13 | 14 | ### 2023/9/23 15 | 1. Fixed the issue where the tab is not displayed in the sd1.6 version. 16 | 2. Inference of video width and height. 17 | 3. Support for Refiner. 18 | 4. Temporarily removed modnet functionality. 19 | 5. Temporarily removed the function to add prompts frame by frame (ps: I believe there's a better approach, will add in the next version). 20 | 6. Changed video synthesis from ffmpeg to imageio. 21 | -------------------------------------------------------------------------------- /CHANGELOG_CN.md: -------------------------------------------------------------------------------- 1 | ### 2023/9/30 2 | 1. 自动解析视频的fps 3 | 2. 视频编辑功能. 4 | 1. 可以自定义选择关键帧或者自动生成关键帧 5 | 2. 反推关键帧tag 6 | 3. 通过Ezsynth(https://github.com/Trentonom0r3/Ezsynth)自动根据关键帧合成视频. 7 | 4. 目前只有windows系统可以使用,如果您系统不支持,可以关闭该选项卡. 8 | 9 | ### 2023/9/26 10 | 1. 编辑关键帧 11 | 2. 自动反推关键帧 12 | 13 | ### 2023/9/24 14 | 1. 移动选项卡至img2img后面 15 | 2. 修复mac系统下,视频合成失败的问题 16 | 3. 修复refiner不生效的问题 17 | 18 | 19 | ### 2023/9/23 20 | 21 | 1. 修复sd1.6版本选项卡不显示的问题. 22 | 2. 推理视频宽高 23 | 3. 支持Refiner 24 | 4. 暂时移除modnet功能. 25 | 5. 暂时移除逐帧添加prompt功能(ps:我觉得有更好的方式,下个版本添加) 26 | 6. 修改合成视频的ffmpeg为imageio -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Scholar0 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [English](README.md) | [中文简体](README_CN.md) 2 | 3 | ## Mov2mov This is the Mov2mov plugin for Automatic1111/stable-diffusion-webui. 4 | 5 | ![img.png](images/2.jpg) 6 | ![img1.png](images/1.png) 7 | 8 | 9 | 10 | Features: 11 | - Directly process frames from videos 12 | - Package into a video after processing 13 | - Video Editing(beta) 14 | - Dramatically reduce video flicker by keyframe compositing! 15 | - You can customize the keyframe selection or auto-generate keyframes. 16 | - Backpropel keyframe tag 17 | - Currently only available for windows, if your system does not support, you can turn off this tab. 18 | 19 | Also, mov2mov will work better with the [bg-mask](https://github.com/Scholar01/sd-webui-bg-mask) plugin 😃 20 | 21 | # Table of Contents 22 | 23 | 24 | - [Table of Contents](#table-of-contents) 25 | - [Usage Regulations](#usage-regulations) 26 | - [Installation](#installation) 27 | - [Change Log](#change-log) 28 | - [Instructions](#instructions) 29 | - [Thanks](#thanks) 30 | ## Usage Regulations 31 | 32 | 1. Please resolve the authorization issues of the video source on your own. Any problems caused by using unauthorized videos for conversion must be borne by the user. It has nothing to do with mov2mov! 33 | 2. Any video made with mov2mov and published on video platforms must clearly specify the source of the video used for conversion in the description. For example, if you use someone else's video and convert it through AI, you must provide a clear link to the original video; if you use your own video, you must also state this in the description. 34 | 3. All copyright issues caused by the input source must be borne by the user. Note that many videos explicitly state that they cannot be reproduced or copied! 35 | 4. Please strictly comply with national laws and regulations to ensure that the content is legal and compliant. Any legal responsibility caused by using this plugin must be borne by the user. It has nothing to do with mov2mov! 36 | 37 | ## Installation 38 | 39 | 1. Open the Extensions tab. 40 | 2. Click on Install from URL. 41 | 3. Enter the URL for the extension's git repository. 42 | 4. Click Install. 43 | 5. Restart WebUI. 44 | 45 | 46 | 47 | ## Change Log 48 | 49 | [Change Log](CHANGELOG.md) 50 | 51 | 52 | 53 | ## Instructions 54 | 55 | - Video tutorials: 56 | - [https://www.bilibili.com/video/BV1Mo4y1a7DF](https://www.bilibili.com/video/BV1Mo4y1a7DF) 57 | - [https://www.bilibili.com/video/BV1rY4y1C7Q5](https://www.bilibili.com/video/BV1rY4y1C7Q5) 58 | - QQ channel: [https://pd.qq.com/s/akxpjjsgd](https://pd.qq.com/s/akxpjjsgd) 59 | - Discord: [https://discord.gg/hUzF3kQKFW](https://discord.gg/hUzF3kQKFW) 60 | 61 | ## Thanks 62 | 63 | - modnet-entry: [https://github.com/RimoChan/modnet-entry](https://github.com/RimoChan/modnet-entry) 64 | - MODNet: [https://github.com/ZHKKKe/MODNet](https://github.com/ZHKKKe/MODNet) 65 | - Ezsynth: [https://github.com/Trentonom0r3/Ezsynth](https://github.com/Trentonom0r3/Ezsynth) -------------------------------------------------------------------------------- /README_CN.md: -------------------------------------------------------------------------------- 1 | [English](README.md) | [中文简体](README_CN.md) 2 | 3 | # Mov2mov 适用于Automatic1111/stable-diffusion-webui 的 Mov2mov 插件。 4 | 5 | ![img.png](images/2.jpg) 6 | ![img1.png](images/1.png) 7 | 8 | 9 | 功能： 10 | - 直接从视频逐帧处理 11 | - 处理完成后打包成视频 12 | - 视频编辑(beta) 13 | - 通过关键帧合成的方式,大幅度减少视频闪烁! 14 | - 可以自定义选择关键帧或者自动生成关键帧 15 | - 反推关键帧tag 16 | - 目前只有windows系统可以使用,如果您系统不支持,可以关闭该选项卡. 17 | 18 | 19 | 另外,mov2mov与[bg-mask](https://github.com/Scholar01/sd-webui-bg-mask)插件一起工作会更好😃 20 | 21 | # 目录 22 | 23 | - [Mov2mov 适用于Automatic1111/stable-diffusion-webui 的 Mov2mov 插件。](#mov2mov-适用于automatic1111stable-diffusion-webui-的-mov2mov-插件) 24 | - [目录](#目录) 25 | - [使用规约](#使用规约) 26 | - [安装方法](#安装方法) 27 | - [更新日志](#更新日志) 28 | - [说明](#说明) 29 | - [感谢](#感谢) 30 | 31 | ## 使用规约 32 | 33 | 1. 请自行解决视频来源的授权问题，任何由于使用非授权视频进行转换造成的问题，需自行承担全部责任和一切后果，于mov2mov无关！ 34 | 2. 任何发布到视频平台的基于mov2mov制作的视频，都必须要在简介中明确指明用于转换的视频来源。例如：使用他人发布的视频，通过ai进行转换的，必须要给出明确的原视频链接；若使用的是自己/自己的视频，也必须在简介加以说明。 35 | 3. 由输入源造成的侵权问题需自行承担全部责任和一切后果。注意，很多视频明确指出不可转载，复制！ 36 | 4. 请严格遵守国家相关法律法规，确保内容合法合规。任何由于使用本插件造成的法律责任，需自行承担全部责任和一切后果，于mov2mov无关！ 37 | 38 | ## 安装方法 39 | 40 | 1. 打开扩展（Extension）标签。 41 | 2. 点击从网址安装（Install from URL） 42 | 3. 在扩展的 git 仓库网址（URL for extension's git repository）处输入 43 | 4. 点击安装（Install） 44 | 5. 重启 WebUI 45 | 46 | 47 | ## 更新日志 48 | 49 | [更新日志](CHANGELOG_CN.md) 50 | 51 | ## 说明 52 | 53 | - 视频教程: 54 | - [https://www.bilibili.com/video/BV1Mo4y1a7DF](https://www.bilibili.com/video/BV1Mo4y1a7DF) 55 | - [https://www.bilibili.com/video/BV1rY4y1C7Q5](https://www.bilibili.com/video/BV1rY4y1C7Q5) 56 | - QQ频道: [https://pd.qq.com/s/akxpjjsgd](https://pd.qq.com/s/akxpjjsgd) 57 | - Discord: [https://discord.gg/hUzF3kQKFW](https://discord.gg/hUzF3kQKFW) 58 | 59 | ## 感谢 60 | 61 | - modnet-entry: [https://github.com/RimoChan/modnet-entry](https://github.com/RimoChan/modnet-entry) 62 | - MODNet: [https://github.com/ZHKKKe/MODNet](https://github.com/ZHKKKe/MODNet) 63 | - Ezsynth: [https://github.com/Trentonom0r3/Ezsynth](https://github.com/Trentonom0r3/Ezsynth) 64 | -------------------------------------------------------------------------------- /ebsynth/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from .ebsynth_generate import EbsynthGenerate, Keyframe, Sequence, EbSynthTask 3 | 4 | AFTER_DETAILER = "ADetailer" 5 | 6 | __all__ = [ 7 | "EbsynthGenerate", "Keyframe", "Sequence", "EbSynthTask" 8 | ] 9 | -------------------------------------------------------------------------------- /ebsynth/_ebsynth.py: -------------------------------------------------------------------------------- 1 | # fork for Ezsynth(https://github.com/Trentonom0r3/Ezsynth) 2 | 3 | import sys 4 | from ctypes import * 5 | from pathlib import Path 6 | 7 | import cv2 8 | import numpy as np 9 | 10 | libebsynth = None 11 | cached_buffer = {} 12 | 13 | EBSYNTH_BACKEND_CPU = 0x0001 14 | EBSYNTH_BACKEND_CUDA = 0x0002 15 | EBSYNTH_BACKEND_AUTO = 0x0000 16 | EBSYNTH_MAX_STYLE_CHANNELS = 8 17 | EBSYNTH_MAX_GUIDE_CHANNELS = 24 18 | EBSYNTH_VOTEMODE_PLAIN = 0x0001 # weight = 1 19 | EBSYNTH_VOTEMODE_WEIGHTED = 0x0002 # weight = 1/(1+error) 20 | 21 | 22 | def _normalize_img_shape(img): 23 | img_len = len(img.shape) 24 | if img_len == 2: 25 | sh, sw = img.shape 26 | sc = 0 27 | elif img_len == 3: 28 | sh, sw, sc = img.shape 29 | 30 | if sc == 0: 31 | sc = 1 32 | img = img[..., np.newaxis] 33 | return img 34 | 35 | 36 | def run(img_style, guides, 37 | patch_size=5, 38 | num_pyramid_levels=-1, 39 | num_search_vote_iters=6, 40 | num_patch_match_iters=4, 41 | stop_threshold=5, 42 | uniformity_weight=3500.0, 43 | extraPass3x3=False, 44 | ): 45 | if patch_size < 3: 46 | raise ValueError("patch_size is too small") 47 | if patch_size % 2 == 0: 48 | raise ValueError("patch_size must be an odd number") 49 | if len(guides) == 0: 50 | raise ValueError("at least one guide must be specified") 51 | 52 | global libebsynth 53 | if libebsynth is None: 54 | if sys.platform[0:3] == 'win': 55 | libebsynth_path = str(Path(__file__).parent / 'ebsynth.dll') 56 | libebsynth = CDLL(libebsynth_path) 57 | else: 58 | # todo: implement for linux 59 | pass 60 | 61 | if libebsynth is not None: 62 | libebsynth.ebsynthRun.argtypes = ( \ 63 | c_int, 64 | c_int, 65 | c_int, 66 | c_int, 67 | c_int, 68 | c_void_p, 69 | c_void_p, 70 | c_int, 71 | c_int, 72 | c_void_p, 73 | c_void_p, 74 | POINTER(c_float), 75 | POINTER(c_float), 76 | c_float, 77 | c_int, 78 | c_int, 79 | c_int, 80 | POINTER(c_int), 81 | POINTER(c_int), 82 | POINTER(c_int), 83 | c_int, 84 | c_void_p, 85 | c_void_p 86 | ) 87 | 88 | if libebsynth is None: 89 | return img_style 90 | 91 | img_style = _normalize_img_shape(img_style) 92 | sh, sw, sc = img_style.shape 93 | t_h, t_w, t_c = 0, 0, 0 94 | 95 | if sc > EBSYNTH_MAX_STYLE_CHANNELS: 96 | raise ValueError(f"error: too many style channels {sc}, maximum number is {EBSYNTH_MAX_STYLE_CHANNELS}") 97 | 98 | guides_source = [] 99 | guides_target = [] 100 | guides_weights = [] 101 | 102 | for i in range(len(guides)): 103 | source_guide, target_guide, guide_weight = guides[i] 104 | source_guide = _normalize_img_shape(source_guide) 105 | target_guide = _normalize_img_shape(target_guide) 106 | s_h, s_w, s_c = source_guide.shape 107 | nt_h, nt_w, nt_c = target_guide.shape 108 | 109 | if s_h != sh or s_w != sw: 110 | raise ValueError("guide source and style resolution must match style resolution.") 111 | 112 | if t_c == 0: 113 | t_h, t_w, t_c = nt_h, nt_w, nt_c 114 | elif nt_h != t_h or nt_w != t_w: 115 | raise ValueError("guides target resolutions must be equal") 116 | 117 | if s_c != nt_c: 118 | raise ValueError("guide source and target channels must match exactly.") 119 | 120 | guides_source.append(source_guide) 121 | guides_target.append(target_guide) 122 | 123 | guides_weights += [guide_weight / s_c] * s_c 124 | 125 | guides_source = np.concatenate(guides_source, axis=-1) 126 | guides_target = np.concatenate(guides_target, axis=-1) 127 | guides_weights = (c_float * len(guides_weights))(*guides_weights) 128 | 129 | styleWeight = 1.0 130 | style_weights = [styleWeight / sc for i in range(sc)] 131 | style_weights = (c_float * sc)(*style_weights) 132 | 133 | maxPyramidLevels = 0 134 | for level in range(32, -1, -1): 135 | if min(min(sh, t_h) * pow(2.0, -level), \ 136 | min(sw, t_w) * pow(2.0, -level)) >= (2 * patch_size + 1): 137 | maxPyramidLevels = level + 1 138 | break 139 | 140 | if num_pyramid_levels == -1: 141 | num_pyramid_levels = maxPyramidLevels 142 | num_pyramid_levels = min(num_pyramid_levels, maxPyramidLevels) 143 | 144 | num_search_vote_iters_per_level = (c_int * num_pyramid_levels)(*[num_search_vote_iters] * num_pyramid_levels) 145 | num_patch_match_iters_per_level = (c_int * num_pyramid_levels)(*[num_patch_match_iters] * num_pyramid_levels) 146 | stop_threshold_per_level = (c_int * num_pyramid_levels)(*[stop_threshold] * num_pyramid_levels) 147 | 148 | buffer = cached_buffer.get((t_h, t_w, sc), None) 149 | if buffer is None: 150 | buffer = create_string_buffer(t_h * t_w * sc) 151 | cached_buffer[(t_h, t_w, sc)] = buffer 152 | 153 | libebsynth.ebsynthRun(EBSYNTH_BACKEND_AUTO, # backend 154 | sc, # numStyleChannels 155 | guides_source.shape[-1], # numGuideChannels 156 | sw, # sourceWidth 157 | sh, # sourceHeight 158 | img_style.tobytes(), 159 | # sourceStyleData (width * height * numStyleChannels) bytes, scan-line order 160 | guides_source.tobytes(), 161 | # sourceGuideData (width * height * numGuideChannels) bytes, scan-line order 162 | t_w, # targetWidth 163 | t_h, # targetHeight 164 | guides_target.tobytes(), 165 | # targetGuideData (width * height * numGuideChannels) bytes, scan-line order 166 | None, 167 | # targetModulationData (width * height * numGuideChannels) bytes, scan-line order; pass NULL to switch off the modulation 168 | style_weights, # styleWeights (numStyleChannels) floats 169 | guides_weights, # guideWeights (numGuideChannels) floats 170 | uniformity_weight, 171 | # uniformityWeight reasonable values are between 500-15000, 3500 is a good default 172 | patch_size, # patchSize odd sizes only, use 5 for 5x5 patch, 7 for 7x7, etc. 173 | EBSYNTH_VOTEMODE_WEIGHTED, # voteMode use VOTEMODE_WEIGHTED for sharper result 174 | num_pyramid_levels, # numPyramidLevels 175 | 176 | num_search_vote_iters_per_level, 177 | # numSearchVoteItersPerLevel how many search/vote iters to perform at each level (array of ints, coarse first, fine last) 178 | num_patch_match_iters_per_level, 179 | # numPatchMatchItersPerLevel how many Patch-Match iters to perform at each level (array of ints, coarse first, fine last) 180 | stop_threshold_per_level, 181 | # stopThresholdPerLevel stop improving pixel when its change since last iteration falls under this threshold 182 | 1 if extraPass3x3 else 0, 183 | # extraPass3x3 perform additional polishing pass with 3x3 patches at the finest level, use 0 to disable 184 | None, # outputNnfData (width * height * 2) ints, scan-line order; pass NULL to ignore 185 | buffer # outputImageData (width * height * numStyleChannels) bytes, scan-line order 186 | ) 187 | 188 | return np.frombuffer(buffer, dtype=np.uint8).reshape((t_h, t_w, sc)).copy() 189 | 190 | 191 | # transfer color from source to target 192 | def color_transfer(img_source, img_target): 193 | guides = [(cv2.cvtColor(img_source, cv2.COLOR_BGR2GRAY), 194 | cv2.cvtColor(img_target, cv2.COLOR_BGR2GRAY), 195 | 1)] 196 | h, w, c = img_source.shape 197 | result = [] 198 | for i in range(c): 199 | result += [ 200 | run(img_source[..., i:i + 1], guides=guides, 201 | patch_size=11, 202 | num_pyramid_levels=40, 203 | num_search_vote_iters=6, 204 | num_patch_match_iters=4, 205 | stop_threshold=5, 206 | uniformity_weight=500.0, 207 | extraPass3x3=True, 208 | ) 209 | 210 | ] 211 | return np.concatenate(result, axis=-1) 212 | 213 | 214 | def task(img_style, guides): 215 | return run(img_style, 216 | guides, 217 | patch_size=5, 218 | num_pyramid_levels=6, 219 | num_search_vote_iters=12, 220 | num_patch_match_iters=6, 221 | uniformity_weight=3500.0, 222 | extraPass3x3=False 223 | ) 224 | -------------------------------------------------------------------------------- /ebsynth/ebsynth.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Scholar01/sd-webui-mov2mov/4bb1fbb8bc848d47a3f41711e0839a5c6e5845f5/ebsynth/ebsynth.dll -------------------------------------------------------------------------------- /ebsynth/ebsynth_generate.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import numpy as np 3 | from dataclasses import dataclass, field 4 | 5 | 6 | @dataclass 7 | class Keyframe: 8 | num: int 9 | image: np.ndarray = field(repr=False) 10 | prompt: str = field(repr=False) 11 | 12 | 13 | @dataclass 14 | class Sequence: 15 | start: int 16 | keyframe: Keyframe 17 | end: int 18 | # 当前序列的所有帧 19 | frames: dict[int, np.ndarray] = field(default_factory=dict, repr=False) 20 | # 序列生成的所有帧 21 | generate_frames: dict[int, np.ndarray] = field(default_factory=dict, repr=False) 22 | 23 | 24 | @dataclass 25 | class EbSynthTask: 26 | style: np.ndarray = field(repr=False) 27 | source: np.ndarray = field(repr=False) 28 | target: np.ndarray = field(repr=False) 29 | frame_num: int 30 | key_frame_num: int 31 | weight: float = field(default=1.0, repr=False) 32 | 33 | 34 | class EbsynthGenerate: 35 | def __init__(self, keyframes: list[Keyframe], frames: list[np.ndarray], fps: int): 36 | self.keyframes = keyframes 37 | self.frames = frames 38 | self.fps = fps 39 | self.sequences = [] 40 | self.setup_sequences() 41 | 42 | def setup_sequences(self): 43 | """ 44 | 初始化序列,在这个阶段,frame_num对应的帧就已经处理好了,在后面使用不需要再处理frame-1了 45 | """ 46 | self.sequences.clear() 47 | all_frames = len(self.frames) 48 | left_frame = 1 49 | for i, keyframe in enumerate(self.keyframes): 50 | right_frame = self.keyframes[i + 1].num if i + 1 < len(self.keyframes) else all_frames 51 | frames = {} 52 | for frame_num in range(left_frame, right_frame + 1): 53 | frames[frame_num] = self.frames[frame_num - 1] 54 | sequence = Sequence(left_frame, keyframe, right_frame, frames) 55 | self.sequences.append(sequence) 56 | left_frame = keyframe.num 57 | return self.sequences 58 | 59 | def get_tasks(self, weight: float = 4.0) -> list[EbSynthTask]: 60 | tasks = [] 61 | for i, sequence in enumerate(self.sequences): 62 | frames = sequence.frames.items() 63 | source = sequence.frames[sequence.keyframe.num] 64 | style = sequence.keyframe.image 65 | for frame_num, frame in frames: 66 | target = frame 67 | task = EbSynthTask(style, source, target, frame_num, sequence.keyframe.num, weight) 68 | tasks.append(task) 69 | return tasks 70 | 71 | def append_generate_frames(self, key_frames_num, frame_num, generate_frames): 72 | """ 73 | 74 | Args: 75 | key_frames_num: 用于定位sequence 76 | frame_num: key 77 | generate_frames: value 78 | 79 | Returns: 80 | 81 | """ 82 | for sequence in self.sequences: 83 | if sequence.keyframe.num == key_frames_num: 84 | sequence.generate_frames[frame_num] = generate_frames 85 | break 86 | else: 87 | raise ValueError(f'not found key frame num {key_frames_num}') 88 | 89 | def merge_sequences(self): 90 | # 存储合并后的结果 91 | merged_frames = [] 92 | border = 1 93 | for i in range(len(self.sequences)): 94 | current_seq = self.sequences[i] 95 | next_seq = self.sequences[i + 1] if i + 1 < len(self.sequences) else None 96 | 97 | # 如果存在下一个序列 98 | if next_seq: 99 | # 获取两个序列的帧交集 100 | common_frames_nums = set(current_seq.frames.keys()).intersection( 101 | set(range(next_seq.start + border, next_seq.end)) if i > 0 else set( 102 | range(next_seq.start, next_seq.end))) 103 | 104 | for j, frame_num in enumerate(common_frames_nums): 105 | # 从两个序列中获取帧并合并 106 | frame1 = current_seq.generate_frames[frame_num] 107 | frame2 = next_seq.generate_frames[frame_num] 108 | 109 | weight = float(j) / float(len(common_frames_nums)) 110 | merged_frame = cv2.addWeighted(frame1, 1 - weight, frame2, weight, 0) 111 | merged_frames.append((frame_num, merged_frame)) 112 | 113 | # 如果没有下一个序列 114 | else: 115 | # 添加与前一序列的差集帧到结果中 116 | if i > 0: 117 | prev_seq = self.sequences[i - 1] 118 | difference_frames_nums = set(current_seq.frames.keys()) - set(prev_seq.frames.keys()) 119 | else: 120 | difference_frames_nums = set(current_seq.frames.keys()) 121 | 122 | for frame_num in difference_frames_nums: 123 | merged_frames.append((frame_num, current_seq.generate_frames[frame_num])) 124 | 125 | # group_merged_frames = groupby(lambda x: x[0], merged_frames) 126 | # merged_frames.clear() 127 | # # 取出value长度大于1的元素 128 | # for key, value in group_merged_frames.items(): 129 | # if len(value) > 1: 130 | # # 将value中的所有元素合并 131 | # merged_frame = value[0][1] 132 | # for i in range(1, len(value)): 133 | # merged_frame = cv2.addWeighted(merged_frame, weight, value[i][1], 1 - weight, 0) 134 | # merged_frames.append((key, merged_frame)) 135 | # else: 136 | # merged_frames.append((key, value[0][1])) 137 | result = [] 138 | for i, frame in sorted(merged_frames, key=lambda x: x[0]): 139 | result.append(frame) 140 | 141 | return result 142 | -------------------------------------------------------------------------------- /images/1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Scholar01/sd-webui-mov2mov/4bb1fbb8bc848d47a3f41711e0839a5c6e5845f5/images/1.png -------------------------------------------------------------------------------- /images/2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Scholar01/sd-webui-mov2mov/4bb1fbb8bc848d47a3f41711e0839a5c6e5845f5/images/2.jpg -------------------------------------------------------------------------------- /images/alipay.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Scholar01/sd-webui-mov2mov/4bb1fbb8bc848d47a3f41711e0839a5c6e5845f5/images/alipay.png -------------------------------------------------------------------------------- /images/img.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Scholar01/sd-webui-mov2mov/4bb1fbb8bc848d47a3f41711e0839a5c6e5845f5/images/img.png -------------------------------------------------------------------------------- /images/wechat.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Scholar01/sd-webui-mov2mov/4bb1fbb8bc848d47a3f41711e0839a5c6e5845f5/images/wechat.png -------------------------------------------------------------------------------- /install.py: -------------------------------------------------------------------------------- 1 | import os 2 | import platform 3 | import launch 4 | 5 | if not launch.is_installed("cv2"): 6 | print('Installing requirements for Mov2mov') 7 | launch.run_pip("install opencv-python", "requirements for opencv") 8 | 9 | if platform.system() == 'Windows': 10 | if not launch.is_installed('imageio'): 11 | print('Installing requirements for Mov2mov') 12 | launch.run_pip("install imageio", "requirements for imageio") 13 | if not launch.is_installed('imageio-ffmpeg'): 14 | print('Installing requirements for Mov2mov') 15 | launch.run_pip("install imageio-ffmpeg", "requirements for imageio-ffmpeg") 16 | else: 17 | if not launch.is_installed('ffmpeg'): 18 | print('Installing requirements for Mov2mov') 19 | launch.run_pip("install ffmpeg", "requirements for ffmpeg") 20 | -------------------------------------------------------------------------------- /javascript/m2m_ui.js: -------------------------------------------------------------------------------- 1 | function submit_mov2mov() { 2 | 3 | showSubmitButtons('mov2mov', false) 4 | showResultVideo('mov2mov', false) 5 | 6 | var id = randomId(); 7 | localSet("mov2mov_task_id", id); 8 | 9 | requestProgress(id, gradioApp().getElementById('mov2mov_gallery_container'), gradioApp().getElementById('mov2mov_gallery'), function () { 10 | showSubmitButtons('mov2mov', true) 11 | showResultVideo('mov2mov', true) 12 | localRemove("mov2mov_task_id"); 13 | 14 | }) 15 | 16 | var res = create_submit_args(arguments) 17 | res[0] = id 18 | res[1] = 2 19 | return res 20 | } 21 | 22 | function showResultVideo(tabname, show) { 23 | gradioApp().getElementById(tabname + '_video').style.display = show ? "block" : "none" 24 | gradioApp().getElementById(tabname + '_gallery').style.display = show ? "none" : "block" 25 | 26 | } 27 | 28 | 29 | function showModnetModels() { 30 | var check = arguments[0] 31 | gradioApp().getElementById('mov2mov_modnet_model').style.display = check ? "block" : "none" 32 | gradioApp().getElementById('mov2mov_merge_background').style.display = check ? "block" : "none" 33 | return [] 34 | } 35 | 36 | function switchModnetMode() { 37 | let mode = arguments[0] 38 | 39 | if (mode === 'Clear' || mode === 'Origin' || mode === 'Green' || mode === 'Image') { 40 | gradioApp().getElementById('modnet_background_movie').style.display = "none" 41 | gradioApp().getElementById('modnet_background_image').style.display = "block" 42 | } else { 43 | gradioApp().getElementById('modnet_background_movie').style.display = "block" 44 | gradioApp().getElementById('modnet_background_image').style.display = "none" 45 | } 46 | 47 | return [] 48 | } 49 | 50 | 51 | function copy_from(type) { 52 | return [] 53 | } 54 | 55 | 56 | function currentMov2movSourceResolution(w, h, scaleBy) { 57 | var video = gradioApp().querySelector('#mov2mov_mov video'); 58 | 59 | // 检查视频元素是否存在并且已加载 60 | if (video && video.videoWidth && video.videoHeight) { 61 | return [video.videoWidth, video.videoHeight, scaleBy]; 62 | } 63 | return [0, 0, scaleBy]; 64 | } -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | opencv-python 2 | imageio 3 | imageio-ffmpeg -------------------------------------------------------------------------------- /scripts/m2m_config.py: -------------------------------------------------------------------------------- 1 | mov2mov_outpath_samples = 'outputs/mov2mov-images' 2 | mov2mov_output_dir = 'outputs/mov2mov-videos' 3 | -------------------------------------------------------------------------------- /scripts/m2m_hook.py: -------------------------------------------------------------------------------- 1 | from collections import defaultdict 2 | 3 | def patch(key, obj, field, replacement): 4 | """Replaces a function in a module or a class. 5 | 6 | Also stores the original function in this module, possible to be retrieved via original(key, obj, field). 7 | If the function is already replaced by this caller (key), an exception is raised -- use undo() before that. 8 | 9 | Arguments: 10 | key: identifying information for who is doing the replacement. You can use __name__. 11 | obj: the module or the class 12 | field: name of the function as a string 13 | replacement: the new function 14 | 15 | Returns: 16 | the original function 17 | """ 18 | 19 | patch_key = (obj, field) 20 | if patch_key in originals[key]: 21 | raise RuntimeError(f"patch for {field} is already applied") 22 | 23 | original_func = getattr(obj, field) 24 | originals[key][patch_key] = original_func 25 | 26 | setattr(obj, field, replacement) 27 | 28 | return original_func 29 | 30 | 31 | def undo(key, obj, field): 32 | """Undoes the peplacement by the patch(). 33 | 34 | If the function is not replaced, raises an exception. 35 | 36 | Arguments: 37 | key: identifying information for who is doing the replacement. You can use __name__. 38 | obj: the module or the class 39 | field: name of the function as a string 40 | 41 | Returns: 42 | Always None 43 | """ 44 | 45 | patch_key = (obj, field) 46 | 47 | if patch_key not in originals[key]: 48 | raise RuntimeError(f"there is no patch for {field} to undo") 49 | 50 | original_func = originals[key].pop(patch_key) 51 | setattr(obj, field, original_func) 52 | 53 | return None 54 | 55 | 56 | def original(key, obj, field): 57 | """Returns the original function for the patch created by the patch() function""" 58 | patch_key = (obj, field) 59 | 60 | return originals[key].get(patch_key, None) 61 | 62 | 63 | originals = defaultdict(dict) 64 | -------------------------------------------------------------------------------- /scripts/m2m_ui.py: -------------------------------------------------------------------------------- 1 | from contextlib import ExitStack 2 | 3 | import gradio as gr 4 | 5 | from modules import ( 6 | script_callbacks, 7 | scripts, 8 | shared, 9 | ui_toprow, 10 | ) 11 | from modules.call_queue import wrap_gradio_gpu_call 12 | from modules.shared import opts 13 | from modules.ui import ( 14 | create_output_panel, 15 | create_override_settings_dropdown, 16 | ordered_ui_categories, 17 | resize_from_to_html, 18 | switch_values_symbol, 19 | detect_image_size_symbol, 20 | ) 21 | from modules.ui_components import ( 22 | FormGroup, 23 | FormHTML, 24 | FormRow, 25 | ResizeHandleRow, 26 | ToolButton, 27 | ) 28 | from scripts import m2m_hook as patches 29 | from scripts import mov2mov 30 | from scripts.m2m_config import mov2mov_outpath_samples, mov2mov_output_dir 31 | from scripts.mov2mov import scripts_mov2mov 32 | from scripts.movie_editor import MovieEditor 33 | from scripts.m2m_ui_common import create_output_panel 34 | 35 | id_part = "mov2mov" 36 | 37 | 38 | def on_ui_settings(): 39 | section = ("mov2mov", "Mov2Mov") 40 | shared.opts.add_option( 41 | "mov2mov_outpath_samples", 42 | shared.OptionInfo( 43 | mov2mov_outpath_samples, "Mov2Mov output path for image", section=section 44 | ), 45 | ) 46 | shared.opts.add_option( 47 | "mov2mov_output_dir", 48 | shared.OptionInfo( 49 | mov2mov_output_dir, "Mov2Mov output path for video", section=section 50 | ), 51 | ) 52 | 53 | 54 | img2img_toprow: gr.Row = None 55 | 56 | 57 | def on_ui_tabs(): 58 | """ 59 | 60 | 构造ui 61 | """ 62 | scripts.scripts_current = scripts_mov2mov 63 | scripts_mov2mov.initialize_scripts(is_img2img=True) 64 | with gr.TabItem( 65 | "mov2mov", id=f"tab_{id_part}", elem_id=f"tab_{id_part}" 66 | ) as mov2mov_interface: 67 | toprow = ui_toprow.Toprow( 68 | is_img2img=True, is_compact=shared.opts.compact_prompt_box, id_part=id_part 69 | ) 70 | dummy_component = gr.Label(visible=False) 71 | 72 | extra_tabs = gr.Tabs( 73 | elem_id="txt2img_extra_tabs", elem_classes=["extra-networks"] 74 | ) 75 | extra_tabs.__enter__() 76 | 77 | with gr.Tab( 78 | "Generation", id=f"{id_part}_generation" 79 | ) as mov2mov_generation_tab, ResizeHandleRow(equal_height=False): 80 | 81 | with ExitStack() as stack: 82 | stack.enter_context( 83 | gr.Column(variant="compact", elem_id=f"{id_part}_settings") 84 | ) 85 | 86 | for category in ordered_ui_categories(): 87 | 88 | if category == "prompt": 89 | toprow.create_inline_toprow_prompts() 90 | 91 | if category == "image": 92 | init_mov = gr.Video( 93 | label="Video for mov2mov", 94 | elem_id=f"{id_part}_mov", 95 | show_label=False, 96 | source="upload", 97 | ) 98 | 99 | with FormRow(): 100 | resize_mode = gr.Radio( 101 | label="Resize mode", 102 | elem_id="resize_mode", 103 | choices=[ 104 | "Just resize", 105 | "Crop and resize", 106 | "Resize and fill", 107 | "Just resize (latent upscale)", 108 | ], 109 | type="index", 110 | value="Just resize", 111 | ) 112 | 113 | scripts_mov2mov.prepare_ui() 114 | 115 | elif category == "dimensions": 116 | with FormRow(): 117 | with gr.Column(elem_id=f"{id_part}_column_size", scale=4): 118 | selected_scale_tab = gr.Number(value=0, visible=False) 119 | with gr.Tabs(elem_id=f"{id_part}_tabs_resize"): 120 | with gr.Tab( 121 | label="Resize to", 122 | id="to", 123 | elem_id=f"{id_part}_tab_resize_to", 124 | ) as tab_scale_to: 125 | with FormRow(): 126 | with gr.Column( 127 | elem_id=f"{id_part}_column_size", 128 | scale=4, 129 | ): 130 | width = gr.Slider( 131 | minimum=64, 132 | maximum=2048, 133 | step=8, 134 | label="Width", 135 | value=512, 136 | elem_id=f"{id_part}_width", 137 | ) 138 | height = gr.Slider( 139 | minimum=64, 140 | maximum=2048, 141 | step=8, 142 | label="Height", 143 | value=512, 144 | elem_id=f"{id_part}_height", 145 | ) 146 | 147 | with gr.Column( 148 | elem_id=f"{id_part}_dimensions_row", 149 | scale=1, 150 | elem_classes="dimensions-tools", 151 | ): 152 | res_switch_btn = ToolButton( 153 | value=switch_values_symbol, 154 | elem_id=f"{id_part}_res_switch_btn", 155 | tooltip="Switch width/height", 156 | ) 157 | detect_image_size_btn = ToolButton( 158 | value=detect_image_size_symbol, 159 | elem_id=f"{id_part}_detect_image_size_btn", 160 | tooltip="Auto detect size from img2img", 161 | ) 162 | 163 | with gr.Tab( 164 | label="Resize by", 165 | id="by", 166 | elem_id=f"{id_part}_tab_resize_by", 167 | ) as tab_scale_by: 168 | scale_by = gr.Slider( 169 | minimum=0.05, 170 | maximum=4.0, 171 | step=0.05, 172 | label="Scale", 173 | value=1.0, 174 | elem_id=f"{id_part}_scale", 175 | ) 176 | 177 | with FormRow(): 178 | scale_by_html = FormHTML( 179 | resize_from_to_html(0, 0, 0.0), 180 | elem_id=f"{id_part}_scale_resolution_preview", 181 | ) 182 | gr.Slider( 183 | label="Unused", 184 | elem_id=f"{id_part}_unused_scale_by_slider", 185 | ) 186 | button_update_resize_to = gr.Button( 187 | visible=False, 188 | elem_id=f"{id_part}_update_resize_to", 189 | ) 190 | 191 | on_change_args = dict( 192 | fn=resize_from_to_html, 193 | _js="currentMov2movSourceResolution", 194 | inputs=[ 195 | dummy_component, 196 | dummy_component, 197 | scale_by, 198 | ], 199 | outputs=scale_by_html, 200 | show_progress=False, 201 | ) 202 | 203 | scale_by.release(**on_change_args) 204 | button_update_resize_to.click(**on_change_args) 205 | 206 | tab_scale_to.select( 207 | fn=lambda: 0, 208 | inputs=[], 209 | outputs=[selected_scale_tab], 210 | ) 211 | tab_scale_by.select( 212 | fn=lambda: 1, 213 | inputs=[], 214 | outputs=[selected_scale_tab], 215 | ) 216 | 217 | elif category == "denoising": 218 | denoising_strength = gr.Slider( 219 | minimum=0.0, 220 | maximum=1.0, 221 | step=0.01, 222 | label="Denoising strength", 223 | value=0.75, 224 | elem_id=f"{id_part}_denoising_strength", 225 | ) 226 | noise_multiplier = gr.Slider( 227 | minimum=0, 228 | maximum=1.5, 229 | step=0.01, 230 | label="Noise multiplier", 231 | elem_id=f"{id_part}_noise_multiplier", 232 | value=1, 233 | ) 234 | with gr.Row(elem_id=f"{id_part}_frames_setting"): 235 | movie_frames = gr.Slider( 236 | minimum=10, 237 | maximum=60, 238 | step=1, 239 | label="Movie FPS", 240 | elem_id=f"{id_part}_movie_frames", 241 | value=30, 242 | ) 243 | max_frames = gr.Number( 244 | label="Max FPS", 245 | value=-1, 246 | elem_id=f"{id_part}_max_frames", 247 | ) 248 | 249 | elif category == "cfg": 250 | with gr.Row(): 251 | cfg_scale = gr.Slider( 252 | minimum=1.0, 253 | maximum=30.0, 254 | step=0.5, 255 | label="CFG Scale", 256 | value=7.0, 257 | elem_id=f"{id_part}_cfg_scale", 258 | ) 259 | image_cfg_scale = gr.Slider( 260 | minimum=0, 261 | maximum=3.0, 262 | step=0.05, 263 | label="Image CFG Scale", 264 | value=1.5, 265 | elem_id=f"{id_part}_image_cfg_scale", 266 | visible=False, 267 | ) 268 | 269 | elif category == "checkboxes": 270 | with FormRow(elem_classes="checkboxes-row", variant="compact"): 271 | pass 272 | 273 | elif category == "accordions": 274 | with gr.Row( 275 | elem_id=f"{id_part}_accordions", elem_classes="accordions" 276 | ): 277 | scripts_mov2mov.setup_ui_for_section(category) 278 | 279 | elif category == "override_settings": 280 | with FormRow(elem_id=f"{id_part}_override_settings_row") as row: 281 | override_settings = create_override_settings_dropdown( 282 | id_part, row 283 | ) 284 | 285 | elif category == "scripts": 286 | editor = MovieEditor(id_part, init_mov, movie_frames) 287 | editor.render() 288 | with FormGroup(elem_id=f"{id_part}_script_container"): 289 | custom_inputs = scripts_mov2mov.setup_ui() 290 | 291 | if category not in {"accordions"}: 292 | scripts_mov2mov.setup_ui_for_section(category) 293 | 294 | output_panel = create_output_panel(id_part, opts.mov2mov_output_dir) 295 | mov2mov_args = dict( 296 | fn=wrap_gradio_gpu_call(mov2mov.mov2mov, extra_outputs=[None, "", ""]), 297 | _js="submit_mov2mov", 298 | inputs=[ 299 | dummy_component, 300 | dummy_component, 301 | toprow.prompt, 302 | toprow.negative_prompt, 303 | toprow.ui_styles.dropdown, 304 | init_mov, 305 | cfg_scale, 306 | image_cfg_scale, 307 | denoising_strength, 308 | selected_scale_tab, 309 | height, 310 | width, 311 | scale_by, 312 | resize_mode, 313 | override_settings, 314 | # refiner 315 | # enable_refiner, refiner_checkpoint, refiner_switch_at, 316 | # mov2mov params 317 | noise_multiplier, 318 | movie_frames, 319 | max_frames, 320 | # editor 321 | editor.gr_enable_movie_editor, 322 | editor.gr_df, 323 | editor.gr_eb_weight, 324 | ] 325 | + custom_inputs, 326 | outputs=[ 327 | 328 | output_panel.video, 329 | output_panel.infotext, 330 | output_panel.html_log, 331 | ], 332 | show_progress=False, 333 | ) 334 | 335 | toprow.prompt.submit(**mov2mov_args) 336 | toprow.submit.click(**mov2mov_args) 337 | 338 | res_switch_btn.click( 339 | fn=None, 340 | _js="function(){switchWidthHeight('mov2mov')}", 341 | inputs=None, 342 | outputs=None, 343 | show_progress=False, 344 | ) 345 | detect_image_size_btn.click( 346 | fn=lambda w, h, _: (w or gr.update(), h or gr.update()), 347 | _js="currentMov2movSourceResolution", 348 | inputs=[dummy_component, dummy_component, dummy_component], 349 | outputs=[width, height], 350 | show_progress=False, 351 | ) 352 | 353 | extra_tabs.__exit__() 354 | scripts.scripts_current = None 355 | 356 | return [(mov2mov_interface, "mov2mov", f"{id_part}_tabs")] 357 | 358 | 359 | def block_context_init(self, *args, **kwargs): 360 | origin_block_context_init(self, *args, **kwargs) 361 | 362 | if self.elem_id == "tab_img2img": 363 | self.parent.__enter__() 364 | on_ui_tabs() 365 | self.parent.__exit__() 366 | 367 | 368 | def on_app_reload(): 369 | global origin_block_context_init 370 | if origin_block_context_init: 371 | patches.undo(__name__, obj=gr.blocks.BlockContext, field="__init__") 372 | origin_block_context_init = None 373 | 374 | 375 | origin_block_context_init = patches.patch( 376 | __name__, 377 | obj=gr.blocks.BlockContext, 378 | field="__init__", 379 | replacement=block_context_init, 380 | ) 381 | script_callbacks.on_before_reload(on_app_reload) 382 | script_callbacks.on_ui_settings(on_ui_settings) 383 | # script_callbacks.on_ui_tabs(on_ui_tabs) 384 | -------------------------------------------------------------------------------- /scripts/m2m_ui_common.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | import os 3 | import shutil 4 | import gradio as gr 5 | 6 | from modules import call_queue, shared, ui_tempdir, util 7 | from modules.ui_common import plaintext_to_html, update_generation_info 8 | from modules.ui_components import ToolButton 9 | import modules 10 | 11 | import modules.infotext_utils as parameters_copypaste 12 | from scripts import mov2mov 13 | 14 | 15 | folder_symbol = "\U0001f4c2" # 📂 16 | refresh_symbol = "\U0001f504" # 🔄 17 | 18 | 19 | @dataclasses.dataclass 20 | class OutputPanel: 21 | gallery = None 22 | video = None 23 | generation_info = None 24 | infotext = None 25 | html_log = None 26 | button_upscale = None 27 | 28 | def create_output_panel(tabname, outdir, toprow=None): 29 | res = OutputPanel() 30 | 31 | def open_folder(f, images=None, index=None): 32 | if shared.cmd_opts.hide_ui_dir_config: 33 | return 34 | 35 | try: 36 | if 'Sub' in shared.opts.open_dir_button_choice: 37 | image_dir = os.path.split(images[index]["name"].rsplit('?', 1)[0])[0] 38 | if 'temp' in shared.opts.open_dir_button_choice or not ui_tempdir.is_gradio_temp_path(image_dir): 39 | f = image_dir 40 | except Exception: 41 | pass 42 | 43 | util.open_folder(f) 44 | 45 | with gr.Column(elem_id=f"{tabname}_results"): 46 | if toprow: 47 | toprow.create_inline_toprow_image() 48 | 49 | with gr.Column(variant='panel', elem_id=f"{tabname}_results_panel"): 50 | with gr.Group(elem_id=f"{tabname}_gallery_container"): 51 | res.gallery = gr.Gallery(label='Output', show_label=False, elem_id=f"{tabname}_gallery", columns=4, preview=True, height=shared.opts.gallery_height or None) 52 | res.video = gr.Video(label='Output', show_label=False, elem_id=f"{tabname}_video", height=shared.opts.gallery_height or None) 53 | 54 | with gr.Row(elem_id=f"image_buttons_{tabname}", elem_classes="image-buttons"): 55 | open_folder_button = ToolButton(folder_symbol, elem_id=f'{tabname}_open_folder', visible=not shared.cmd_opts.hide_ui_dir_config, tooltip="Open images output directory.") 56 | 57 | if tabname != "extras": 58 | save = ToolButton('💾', elem_id=f'save_{tabname}', tooltip=f"Save the image to a dedicated directory ({shared.opts.outdir_save}).") 59 | save_zip = ToolButton('🗃️', elem_id=f'save_zip_{tabname}', tooltip=f"Save zip archive with images to a dedicated directory ({shared.opts.outdir_save})") 60 | 61 | buttons = { 62 | 'img2img': ToolButton('🖼️', elem_id=f'{tabname}_send_to_img2img', tooltip="Send image and generation parameters to img2img tab."), 63 | 'inpaint': ToolButton('🎨️', elem_id=f'{tabname}_send_to_inpaint', tooltip="Send image and generation parameters to img2img inpaint tab."), 64 | 'extras': ToolButton('📐', elem_id=f'{tabname}_send_to_extras', tooltip="Send image and generation parameters to extras tab.") 65 | } 66 | 67 | if tabname == 'txt2img': 68 | res.button_upscale = ToolButton('✨', elem_id=f'{tabname}_upscale', tooltip="Create an upscaled version of the current image using hires fix settings.") 69 | 70 | open_folder_button.click( 71 | fn=lambda images, index: open_folder(shared.opts.outdir_samples or outdir, images, index), 72 | _js="(y, w) => [y, selected_gallery_index()]", 73 | inputs=[ 74 | res.gallery, 75 | open_folder_button, # placeholder for index 76 | ], 77 | outputs=[], 78 | ) 79 | 80 | if tabname != "extras": 81 | download_files = gr.File(None, file_count="multiple", interactive=False, show_label=False, visible=False, elem_id=f'download_files_{tabname}') 82 | 83 | with gr.Group(): 84 | res.infotext = gr.HTML(elem_id=f'html_info_{tabname}', elem_classes="infotext") 85 | res.html_log = gr.HTML(elem_id=f'html_log_{tabname}', elem_classes="html-log") 86 | 87 | res.generation_info = gr.Textbox(visible=False, elem_id=f'generation_info_{tabname}') 88 | if tabname == 'txt2img' or tabname == 'img2img': 89 | generation_info_button = gr.Button(visible=False, elem_id=f"{tabname}_generation_info_button") 90 | generation_info_button.click( 91 | fn=update_generation_info, 92 | _js="function(x, y, z){ return [x, y, selected_gallery_index()] }", 93 | inputs=[res.generation_info, res.infotext, res.infotext], 94 | outputs=[res.infotext, res.infotext], 95 | show_progress=False, 96 | ) 97 | 98 | save.click( 99 | fn=call_queue.wrap_gradio_call_no_job(save_video), 100 | _js="(x, y, z, w) => [x, y, false, selected_gallery_index()]", 101 | inputs=[ 102 | res.video, 103 | ], 104 | outputs=[ 105 | download_files, 106 | res.html_log, 107 | ], 108 | show_progress=False, 109 | ) 110 | 111 | save_zip.click( 112 | fn=call_queue.wrap_gradio_call_no_job(save_video), 113 | _js="(x, y, z, w) => [x, y, true, selected_gallery_index()]", 114 | inputs=[ 115 | res.video, 116 | ], 117 | outputs=[ 118 | download_files, 119 | res.html_log, 120 | ] 121 | ) 122 | 123 | else: 124 | res.generation_info = gr.HTML(elem_id=f'html_info_x_{tabname}') 125 | res.infotext = gr.HTML(elem_id=f'html_info_{tabname}', elem_classes="infotext") 126 | res.html_log = gr.HTML(elem_id=f'html_log_{tabname}') 127 | 128 | paste_field_names = [] 129 | if tabname == "txt2img": 130 | paste_field_names = modules.scripts.scripts_txt2img.paste_field_names 131 | elif tabname == "img2img": 132 | paste_field_names = modules.scripts.scripts_img2img.paste_field_names 133 | elif tabname == "mov2mov": 134 | paste_field_names = mov2mov.scripts_mov2mov.paste_field_names 135 | 136 | for paste_tabname, paste_button in buttons.items(): 137 | parameters_copypaste.register_paste_params_button(parameters_copypaste.ParamBinding( 138 | paste_button=paste_button, tabname=paste_tabname, source_tabname="txt2img" if tabname == "txt2img" else "img2img" if tabname == "img2img" else "mov2mov", source_image_component=res.gallery, 139 | paste_field_names=paste_field_names 140 | )) 141 | 142 | return res 143 | 144 | 145 | def save_video(video): 146 | path = "logs/movies" 147 | if not os.path.exists(path): 148 | os.makedirs(path, exist_ok=True) 149 | index = len([path for path in os.listdir(path) if path.endswith(".mp4")]) + 1 150 | video_path = os.path.join(path, str(index).zfill(5) + ".mp4") 151 | shutil.copyfile(video, video_path) 152 | filename = os.path.relpath(video_path, path) 153 | return gr.File.update(value=video_path, visible=True), plaintext_to_html( 154 | f"Saved: {filename}" 155 | ) 156 | -------------------------------------------------------------------------------- /scripts/m2m_util.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import platform 3 | import cv2 4 | import numpy 5 | import imageio 6 | 7 | 8 | def calc_video_w_h(video_path): 9 | cap = cv2.VideoCapture(video_path) 10 | 11 | if not cap.isOpened(): 12 | raise ValueError("Can't open video file") 13 | 14 | width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) 15 | height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) 16 | 17 | cap.release() 18 | 19 | return width, height 20 | 21 | 22 | def get_mov_frame_count(file): 23 | if file is None: 24 | return None 25 | cap = cv2.VideoCapture(file) 26 | 27 | if not cap.isOpened(): 28 | return None 29 | 30 | frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) 31 | cap.release() 32 | return frames 33 | 34 | 35 | def get_mov_fps(file): 36 | if file is None: 37 | return None 38 | cap = cv2.VideoCapture(file) 39 | 40 | if not cap.isOpened(): 41 | return None 42 | 43 | fps = cap.get(cv2.CAP_PROP_FPS) 44 | cap.release() 45 | return fps 46 | 47 | 48 | def get_mov_all_images(file, frames, rgb=False): 49 | if file is None: 50 | return None 51 | cap = cv2.VideoCapture(file) 52 | 53 | if not cap.isOpened(): 54 | return None 55 | 56 | fps = cap.get(cv2.CAP_PROP_FPS) 57 | if frames > fps: 58 | print('Waring: The set number of frames is greater than the number of video frames') 59 | frames = int(fps) 60 | 61 | skip = fps // frames 62 | count = 1 63 | fs = 1 64 | image_list = [] 65 | while (True): 66 | flag, frame = cap.read() 67 | if not flag: 68 | break 69 | else: 70 | if fs % skip == 0: 71 | if rgb: 72 | frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) 73 | image_list.append(frame) 74 | count += 1 75 | fs += 1 76 | cap.release() 77 | return image_list 78 | 79 | 80 | def images_to_video(images, frames, out_path): 81 | if platform.system() == 'Windows': 82 | # Use imageio with the 'libx264' codec on Windows 83 | return images_to_video_imageio(images, frames, out_path, 'libx264') 84 | elif platform.system() == 'Darwin': 85 | # Use cv2 with the 'avc1' codec on Mac 86 | return images_to_video_cv2(images, frames, out_path, 'avc1') 87 | else: 88 | # Use cv2 with the 'mp4v' codec on other operating systems as it's the most widely supported 89 | return images_to_video_cv2(images, frames, out_path, 'mp4v') 90 | 91 | 92 | def images_to_video_imageio(images, frames, out_path, codec): 93 | # 判断out_path是否存在,不存在则创建 94 | if not os.path.exists(os.path.dirname(out_path)): 95 | os.makedirs(os.path.dirname(out_path), exist_ok=True) 96 | 97 | with imageio.v2.get_writer(out_path, format='ffmpeg', mode='I', fps=frames, codec=codec) as writer: 98 | for img in images: 99 | writer.append_data(numpy.asarray(img)) 100 | return out_path 101 | 102 | 103 | def images_to_video_cv2(images, frames, out_path, codec): 104 | if len(images) <= 0: 105 | return None 106 | # 判断out_path是否存在,不存在则创建 107 | if not os.path.exists(os.path.dirname(out_path)): 108 | os.makedirs(os.path.dirname(out_path), exist_ok=True) 109 | 110 | fourcc = cv2.VideoWriter_fourcc(*codec) 111 | if len(images) > 0: 112 | img = images[0] 113 | img_width, img_height = img.size 114 | w = img_width 115 | h = img_height 116 | video = cv2.VideoWriter(out_path, fourcc, frames, (w, h)) 117 | for image in images: 118 | img = cv2.cvtColor(numpy.asarray(image), cv2.COLOR_RGB2BGR) 119 | video.write(img) 120 | video.release() 121 | return out_path 122 | -------------------------------------------------------------------------------- /scripts/module_ui_extensions.py: -------------------------------------------------------------------------------- 1 | import gradio 2 | from modules import script_callbacks, ui_components 3 | from scripts import m2m_hook as patches 4 | 5 | 6 | elem_ids = [] 7 | 8 | 9 | def fix_elem_id(component, **kwargs): 10 | if "elem_id" not in kwargs: 11 | return None 12 | elem_id = kwargs["elem_id"] 13 | if not elem_id: 14 | return None 15 | if elem_id not in elem_ids: 16 | elem_ids.append(elem_id) 17 | else: 18 | elem_id = elem_id + "_" + str(elem_ids.count(elem_id)) 19 | elem_ids.append(elem_id) 20 | 21 | return elem_id 22 | 23 | 24 | def IOComponent_init(self, *args, **kwargs): 25 | elem_id = fix_elem_id(self, **kwargs) 26 | if elem_id: 27 | kwargs.pop("elem_id") 28 | res = original_IOComponent_init(self, elem_id=elem_id, *args, **kwargs) 29 | else: 30 | res = original_IOComponent_init(self, *args, **kwargs) 31 | return res 32 | 33 | 34 | def InputAccordion_init(self, *args, **kwargs): 35 | elem_id = fix_elem_id(self, **kwargs) 36 | if elem_id: 37 | kwargs.pop("elem_id") 38 | res = original_InputAccordion_init(self, elem_id=elem_id, *args, **kwargs) 39 | else: 40 | res = original_InputAccordion_init(self, *args, **kwargs) 41 | return res 42 | 43 | 44 | original_IOComponent_init = patches.patch( 45 | __name__, 46 | obj=gradio.components.IOComponent, 47 | field="__init__", 48 | replacement=IOComponent_init, 49 | ) 50 | 51 | original_InputAccordion_init = patches.patch( 52 | __name__, 53 | obj=ui_components.InputAccordion, 54 | field="__init__", 55 | replacement=InputAccordion_init, 56 | ) 57 | -------------------------------------------------------------------------------- /scripts/mov2mov.py: -------------------------------------------------------------------------------- 1 | from contextlib import closing 2 | import os.path 3 | import platform 4 | import time 5 | import gradio as gr 6 | import PIL.Image 7 | from tqdm import tqdm 8 | 9 | from ebsynth.ebsynth_generate import EbsynthGenerate 10 | import modules 11 | 12 | import cv2 13 | import numpy as np 14 | import pandas 15 | from PIL import Image 16 | from modules import shared, processing 17 | from modules.infotext_utils import create_override_settings_dict 18 | from modules.processing import ( 19 | StableDiffusionProcessingImg2Img, 20 | process_images, 21 | ) 22 | from modules.shared import state 23 | import modules.scripts as scripts 24 | 25 | from modules.ui_common import plaintext_to_html 26 | from scripts.m2m_util import calc_video_w_h, get_mov_all_images, images_to_video 27 | from scripts.m2m_config import mov2mov_output_dir 28 | import modules 29 | from ebsynth import Keyframe 30 | from modules.processing import Processed 31 | from modules.shared import opts 32 | 33 | scripts_mov2mov = scripts.ScriptRunner() 34 | 35 | 36 | def check_data_frame(df: pandas.DataFrame): 37 | # 删除df的frame值为0的行 38 | df = df[df["frame"] > 0] 39 | 40 | # 判断df是否为空 41 | if len(df) <= 0: 42 | return False 43 | 44 | return True 45 | 46 | 47 | def save_video(images, fps, extension=".mp4"): 48 | if not os.path.exists( 49 | shared.opts.data.get("mov2mov_output_dir", mov2mov_output_dir) 50 | ): 51 | os.makedirs( 52 | shared.opts.data.get("mov2mov_output_dir", mov2mov_output_dir), 53 | exist_ok=True, 54 | ) 55 | 56 | r_f = extension 57 | 58 | print(f"Start generating {r_f} file") 59 | 60 | video = images_to_video( 61 | images, 62 | fps, 63 | os.path.join( 64 | shared.opts.data.get("mov2mov_output_dir", mov2mov_output_dir), 65 | str(int(time.time())) + r_f, 66 | ), 67 | ) 68 | print(f"The generation is complete, the directory::{video}") 69 | 70 | return video 71 | 72 | 73 | def process_mov2mov(p, mov_file, movie_frames, max_frames, resize_mode, w, h, args): 74 | processing.fix_seed(p) 75 | images = get_mov_all_images(mov_file, movie_frames) 76 | if not images: 77 | print("Failed to parse the video, please check") 78 | return 79 | 80 | print(f"The video conversion is completed, images:{len(images)}") 81 | if max_frames == -1 or max_frames > len(images): 82 | max_frames = len(images) 83 | 84 | max_frames = int(max_frames) 85 | 86 | p.do_not_save_grid = True 87 | state.job_count = max_frames # * p.n_iter 88 | generate_images = [] 89 | for i, image in enumerate(images): 90 | if i >= max_frames: 91 | break 92 | 93 | state.job = f"{i + 1} out of {max_frames}" 94 | if state.skipped: 95 | state.skipped = False 96 | 97 | if state.interrupted: 98 | break 99 | 100 | # 存一张底图 101 | img = Image.fromarray(cv2.cvtColor(image, cv2.COLOR_BGR2RGB), "RGB") 102 | 103 | p.init_images = [img] * p.batch_size 104 | proc = scripts_mov2mov.run(p, *args) 105 | if proc is None: 106 | print(f"current progress: {i + 1}/{max_frames}") 107 | processed = process_images(p) 108 | # 只取第一张 109 | gen_image = processed.images[0] 110 | generate_images.append(gen_image) 111 | 112 | video = save_video(generate_images, movie_frames) 113 | 114 | return video 115 | 116 | 117 | def process_keyframes(p, mov_file, fps, df, args): 118 | processing.fix_seed(p) 119 | images = get_mov_all_images(mov_file, fps) 120 | if not images: 121 | print("Failed to parse the video, please check") 122 | return 123 | 124 | # 通过宽高,缩放模式,预处理图片 125 | images = [PIL.Image.fromarray(image) for image in images] 126 | images = [ 127 | modules.images.resize_image(p.resize_mode, image, p.width, p.height) 128 | for image in images 129 | ] 130 | images = [np.asarray(image) for image in images] 131 | 132 | default_prompt = p.prompt 133 | max_frames = len(df) 134 | 135 | p.do_not_save_grid = True 136 | state.job_count = max_frames # * p.n_iter 137 | generate_images = [] 138 | 139 | for i, row in df.iterrows(): 140 | p.prompt = default_prompt + row["prompt"] 141 | frame = images[row["frame"] - 1] 142 | 143 | state.job = f"{i + 1} out of {max_frames}" 144 | if state.skipped: 145 | state.skipped = False 146 | 147 | if state.interrupted: 148 | break 149 | 150 | img = Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB), "RGB") 151 | p.init_images = [img] 152 | proc = scripts_mov2mov.run(p, *args) 153 | if proc is None: 154 | print(f"current progress: {i + 1}/{max_frames}") 155 | processed = process_images(p) 156 | gen_image = processed.images[0] 157 | 158 | if gen_image.height != p.height or gen_image.width != p.width: 159 | print( 160 | f"Warning: The generated image size is inconsistent with the original image size, " 161 | f"please check the configuration parameters" 162 | ) 163 | gen_image = gen_image.resize((p.width, p.height)) 164 | 165 | keyframe = Keyframe(row["frame"], np.asarray(gen_image), row["prompt"]) 166 | generate_images.append(keyframe) 167 | 168 | # 由于生成图片可能会产生像素偏差,这里再对齐一次宽高 169 | images = [PIL.Image.fromarray(image) for image in images] 170 | images = [ 171 | ( 172 | image.resize(p.width, p.height) 173 | if image.width != p.width or image.height != p.height 174 | else image 175 | ) 176 | for image in images 177 | ] 178 | images = [np.asarray(image) for image in images] 179 | 180 | return generate_images, images 181 | 182 | 183 | def process_mov2mov_ebsynth(p, eb_generate, weight=4.0): 184 | from ebsynth._ebsynth import task as EbsyncthRun 185 | 186 | tasks = eb_generate.get_tasks(weight) 187 | tasks_len = len(tasks) 188 | state.job_count = tasks_len # * p.n_iter 189 | 190 | for i, task in tqdm(enumerate(tasks)): 191 | 192 | state.job = f"{i + 1} out of {tasks_len}" 193 | if state.skipped: 194 | state.skipped = False 195 | 196 | if state.interrupted: 197 | break 198 | 199 | result = EbsyncthRun(task.style, [(task.source, task.target, task.weight)]) 200 | eb_generate.append_generate_frames(task.key_frame_num, task.frame_num, result) 201 | state.nextjob() 202 | 203 | print(f"Start merge frames") 204 | result = eb_generate.merge_sequences() 205 | video = save_video(result, eb_generate.fps) 206 | return video 207 | 208 | 209 | def mov2mov( 210 | id_task: str, 211 | request: gr.Request, 212 | tab_index: int, 213 | prompt, 214 | negative_prompt, 215 | prompt_styles, 216 | mov_file, 217 | cfg_scale, 218 | image_cfg_scale, 219 | denoising_strength, 220 | selected_scale_tab, 221 | height, 222 | width, 223 | scale_by, 224 | resize_mode, 225 | override_settings_texts, 226 | # refiner 227 | # enable_refiner, refiner_checkpoint, refiner_switch_at, 228 | # mov2mov params 229 | noise_multiplier, 230 | movie_frames, 231 | max_frames, 232 | # editor 233 | enable_movie_editor, 234 | df: pandas.DataFrame, 235 | eb_weight, 236 | *args, 237 | ): 238 | if not mov_file: 239 | raise Exception("Error！ Please add a video file!") 240 | 241 | if selected_scale_tab == 1: 242 | width, height = calc_video_w_h(mov_file) 243 | width = int(width * scale_by) 244 | height = int(height * scale_by) 245 | 246 | override_settings = create_override_settings_dict(override_settings_texts) 247 | assert 0.0 <= denoising_strength <= 1.0, "can only work with strength in [0.0, 1.0]" 248 | mask_blur = 4 249 | inpainting_fill = 1 250 | inpaint_full_res = False 251 | inpaint_full_res_padding = 32 252 | inpainting_mask_invert = 0 253 | 254 | p = StableDiffusionProcessingImg2Img( 255 | sd_model=shared.sd_model, 256 | outpath_samples=shared.opts.data.get("mov2mov_output_dir", mov2mov_output_dir), 257 | outpath_grids=shared.opts.data.get("mov2mov_output_dir", mov2mov_output_dir), 258 | prompt=prompt, 259 | negative_prompt=negative_prompt, 260 | styles=prompt_styles, 261 | batch_size=1, 262 | n_iter=1, 263 | cfg_scale=cfg_scale, 264 | width=width, 265 | height=height, 266 | init_images=[None], 267 | mask=None, 268 | mask_blur=mask_blur, 269 | inpainting_fill=inpainting_fill, 270 | resize_mode=resize_mode, 271 | denoising_strength=denoising_strength, 272 | image_cfg_scale=image_cfg_scale, 273 | inpaint_full_res=inpaint_full_res, 274 | inpaint_full_res_padding=inpaint_full_res_padding, 275 | inpainting_mask_invert=inpainting_mask_invert, 276 | override_settings=override_settings, 277 | initial_noise_multiplier=noise_multiplier, 278 | ) 279 | 280 | p.scripts = modules.scripts.scripts_img2img 281 | p.script_args = args 282 | 283 | p.user = request.username 284 | 285 | if shared.opts.enable_console_prompts: 286 | print(f"\nmov2mov: {prompt}", file=shared.progress_print_out) 287 | 288 | with closing(p): 289 | if not enable_movie_editor: 290 | print(f"\nStart parsing the number of mov frames") 291 | generate_video = process_mov2mov( 292 | p, mov_file, movie_frames, max_frames, resize_mode, width, height, args 293 | ) 294 | processed = Processed(p, [], p.seed, "") 295 | else: 296 | # editor 297 | if platform.system() != "Windows": 298 | raise Exception( 299 | "The Movie Editor is currently only supported on Windows" 300 | ) 301 | 302 | # check df no frame 303 | if not check_data_frame(df): 304 | raise Exception("Please add a frame in the Movie Editor or disable it") 305 | 306 | # sort df for index 307 | df = df.sort_values(by="frame").reset_index(drop=True) 308 | 309 | # generate keyframes 310 | print(f"Start generate keyframes") 311 | keyframes, frames = process_keyframes(p, mov_file, movie_frames, df, args) 312 | eb_generate = EbsynthGenerate(keyframes, frames, movie_frames) 313 | print(f"\nStart generate frames") 314 | 315 | generate_video = process_mov2mov_ebsynth(p, eb_generate, weight=eb_weight) 316 | 317 | processed = Processed(p, [], p.seed, "") 318 | 319 | shared.total_tqdm.clear() 320 | 321 | generation_info_js = processed.js() 322 | if opts.samples_log_stdout: 323 | print(generation_info_js) 324 | 325 | if opts.do_not_show_images: 326 | processed.images = [] 327 | 328 | return ( 329 | generate_video, 330 | generation_info_js, 331 | plaintext_to_html(processed.info), 332 | plaintext_to_html(processed.comments, classname="comments"), 333 | ) 334 | -------------------------------------------------------------------------------- /scripts/movie_editor.py: -------------------------------------------------------------------------------- 1 | import platform 2 | 3 | import gradio as gr 4 | import pandas 5 | from PIL import Image 6 | from tqdm import tqdm 7 | 8 | from modules import shared, deepbooru 9 | from modules.ui_components import InputAccordion, ToolButton 10 | from scripts import m2m_util 11 | 12 | 13 | class MovieEditor: 14 | def __init__(self, id_part, gr_movie: gr.Video, gr_fps: gr.Slider): 15 | self.gr_eb_weight = None 16 | self.gr_df = None 17 | self.gr_keyframe = None 18 | self.gr_frame_number = None 19 | self.gr_frame_image = None 20 | self.gr_movie = gr_movie 21 | self.gr_fps = gr_fps 22 | self.gr_enable_movie_editor = None 23 | 24 | self.is_windows = platform.system() == "Windows" 25 | self.id_part = id_part 26 | self.frames = [] 27 | self.frame_count = 0 28 | self.selected_data_frame = -1 29 | 30 | def render(self): 31 | id_part = self.id_part 32 | with InputAccordion( 33 | True, label="Movie Editor", elem_id=f"{id_part}_editor_accordion" 34 | ): 35 | gr.HTML( 36 | "
" 37 | "This feature is in beta version!!!
" 38 | "It only supports Windows!!!
" 39 | "Make sure you have installed the ControlNet and IP-Adapter models." 40 | "
" 41 | ) 42 | 43 | self.gr_enable_movie_editor = gr.Checkbox( 44 | label="Enable Movie Editor", 45 | elem_id=f"{id_part}_editor_enable", 46 | ) 47 | 48 | self.gr_frame_image = gr.Image( 49 | label="Frame", 50 | elem_id=f"{id_part}_video_frame", 51 | source="upload", 52 | visible=False, 53 | height=480, 54 | ) 55 | 56 | # key frame tabs 57 | with gr.Tabs(elem_id=f"{id_part}_keyframe_tabs"): 58 | with gr.TabItem("Custom", id=f"{id_part}_keyframe_tab_custom"): 59 | with gr.Row(): 60 | self.gr_frame_number = gr.Slider( 61 | label="Frame number", 62 | elem_id=f"{id_part}_video_frame_number", 63 | step=1, 64 | maximum=0, 65 | minimum=0, 66 | ) 67 | 68 | with gr.TabItem("Auto", elem_id=f"{id_part}_keyframe_tab_auto"): 69 | with gr.Row(): 70 | key_frame_interval = gr.Slider( 71 | label="Key frame interval", 72 | elem_id=f"{id_part}_video_keyframe_interval", 73 | step=1, 74 | maximum=100, 75 | minimum=0, 76 | value=2, 77 | ) 78 | key_frame_interval_generate = ToolButton( 79 | "♺", 80 | elem_id=f"{id_part}_video_editor_auto_keyframe", 81 | visible=True, 82 | tooltip="generate", 83 | ) 84 | 85 | with gr.Row(elem_id=f"{id_part}_keyframe_custom_container"): 86 | add_keyframe = ToolButton( 87 | "✚", 88 | elem_id=f"{id_part}_video_editor_add_keyframe", 89 | visible=True, 90 | tooltip="Add keyframe", 91 | ) 92 | remove_keyframe = ToolButton( 93 | "✖", 94 | elem_id=f"{id_part}_video_editor_remove_keyframe", 95 | visible=True, 96 | tooltip="Remove selected keyframe", 97 | ) 98 | 99 | clear_keyframe = ToolButton( 100 | "🗑", 101 | elem_id=f"{id_part}_video_editor_clear_keyframe", 102 | visible=True, 103 | tooltip="Clear keyframe", 104 | ) 105 | 106 | with gr.Row(): 107 | data_frame = gr.Dataframe( 108 | headers=["id", "frame", "prompt"], 109 | datatype=["number", "number", "str"], 110 | row_count=1, 111 | col_count=(3, "fixed"), 112 | max_rows=None, 113 | height=480, 114 | elem_id=f"{id_part}_video_editor_custom_data_frame", 115 | ) 116 | self.gr_df = data_frame 117 | 118 | with gr.Row(): 119 | interrogate = gr.Button( 120 | value="Clip Interrogate Keyframe", 121 | size="sm", 122 | elem_id=f"{id_part}_video_editor_interrogate", 123 | ) 124 | deepbooru = gr.Button( 125 | value="Deepbooru Keyframe", 126 | size="sm", 127 | elem_id=f"{id_part}_video_editor_deepbooru", 128 | ) 129 | 130 | with gr.Row(): 131 | self.gr_eb_weight = gr.Slider( 132 | label="EbSynth weight", 133 | elem_id=f"{id_part}_video_eb_weight", 134 | step=0.1, 135 | maximum=10, 136 | minimum=0, 137 | value=4.0, 138 | ) 139 | 140 | self.gr_movie.change( 141 | fn=self.movie_change, 142 | inputs=[self.gr_movie], 143 | outputs=[self.gr_frame_image, self.gr_frame_number, self.gr_fps], 144 | show_progress=True, 145 | ) 146 | 147 | self.gr_frame_number.change( 148 | fn=self.movie_frame_change, 149 | inputs=[self.gr_movie, self.gr_frame_number], 150 | outputs=[self.gr_frame_image], 151 | show_progress=True, 152 | ) 153 | 154 | self.gr_fps.change( 155 | fn=self.fps_change, 156 | inputs=[self.gr_movie, self.gr_fps], 157 | outputs=[self.gr_frame_image, self.gr_frame_number], 158 | show_progress=True, 159 | ) 160 | 161 | data_frame.select(self.data_frame_select, data_frame, self.gr_frame_number) 162 | 163 | add_keyframe.click( 164 | fn=self.add_keyframe_click, 165 | inputs=[data_frame, self.gr_frame_number], 166 | outputs=[data_frame], 167 | show_progress=False, 168 | ) 169 | remove_keyframe.click( 170 | fn=self.remove_keyframe_click, 171 | inputs=[data_frame], 172 | outputs=[data_frame], 173 | show_progress=False, 174 | ) 175 | 176 | clear_keyframe.click( 177 | fn=lambda df: df.drop(df.index, inplace=True), 178 | inputs=[data_frame], 179 | outputs=[data_frame], 180 | show_progress=False, 181 | ) 182 | 183 | key_frame_interval_generate.click( 184 | fn=self.key_frame_interval_generate_click, 185 | inputs=[data_frame, key_frame_interval], 186 | outputs=[data_frame], 187 | show_progress=True, 188 | ) 189 | 190 | interrogate.click( 191 | fn=self.interrogate_keyframe, 192 | inputs=[data_frame], 193 | outputs=[data_frame], 194 | show_progress=True, 195 | ) 196 | 197 | deepbooru.click( 198 | fn=self.deepbooru_keyframe, 199 | inputs=[data_frame], 200 | outputs=[data_frame], 201 | show_progress=True, 202 | ) 203 | 204 | def interrogate_keyframe(self, data_frame: pandas.DataFrame): 205 | """ 206 | Interrogate key frame 207 | """ 208 | bar = tqdm(total=len(data_frame)) 209 | for index, row in data_frame.iterrows(): 210 | if row["frame"] <= 0: 211 | continue 212 | bar.set_description(f'Interrogate key frame {row["frame"]}') 213 | frame = row["frame"] - 1 214 | image = self.frames[frame] 215 | image = Image.fromarray(image) 216 | prompt = shared.interrogator.interrogate(image.convert("RGB")) 217 | data_frame.at[index, "prompt"] = prompt 218 | bar.update(1) 219 | 220 | return data_frame 221 | 222 | def deepbooru_keyframe(self, data_frame: pandas.DataFrame): 223 | """ 224 | Deepbooru key frame 225 | 226 | """ 227 | bar = tqdm(total=len(data_frame)) 228 | for index, row in data_frame.iterrows(): 229 | if row["frame"] <= 0: 230 | continue 231 | bar.set_description(f'Interrogate key frame {row["frame"]}') 232 | frame = row["frame"] - 1 233 | image = self.frames[frame] 234 | image = Image.fromarray(image) 235 | prompt = deepbooru.model.tag(image) 236 | data_frame.at[index, "prompt"] = prompt 237 | bar.update(1) 238 | 239 | return data_frame 240 | 241 | def data_frame_select(self, event: gr.SelectData, data_frame: pandas.DataFrame): 242 | row, col = event.index 243 | self.selected_data_frame = row 244 | row = data_frame.iloc[row] 245 | frame = row["frame"] 246 | if 0 < frame <= self.frame_count: 247 | return int(frame) 248 | else: 249 | return 0 250 | 251 | def add_keyframe_click(self, data_frame: pandas.DataFrame, gr_frame_number: int): 252 | """ 253 | Add a key frame to the data frame 254 | """ 255 | if gr_frame_number < 1: 256 | return data_frame 257 | 258 | data_frame = data_frame[data_frame["frame"] > 0] 259 | 260 | if gr_frame_number in data_frame["frame"].values: 261 | return data_frame 262 | 263 | row = {"id": len(data_frame), "frame": gr_frame_number, "prompt": ""} 264 | data_frame.loc[len(data_frame)] = row 265 | 266 | data_frame = data_frame.sort_values(by="frame").reset_index(drop=True) 267 | 268 | data_frame["id"] = range(len(data_frame)) 269 | 270 | return data_frame 271 | 272 | def remove_keyframe_click(self, data_frame: pandas.DataFrame): 273 | """ 274 | Remove the selected key frame 275 | """ 276 | if self.selected_data_frame < 0: 277 | return data_frame 278 | 279 | data_frame = data_frame.drop(self.selected_data_frame) 280 | 281 | data_frame = data_frame.sort_values(by="frame").reset_index(drop=True) 282 | 283 | data_frame["id"] = range(len(data_frame)) 284 | 285 | return data_frame 286 | 287 | def key_frame_interval_generate_click( 288 | self, data_frame: pandas.DataFrame, key_frame_interval: int 289 | ): 290 | if key_frame_interval < 1: 291 | return data_frame 292 | 293 | # 按照key_frame_interval的间隔添加关键帧 294 | for i in range(0, self.frame_count, key_frame_interval): 295 | data_frame = self.add_keyframe_click(data_frame, i + 1) 296 | 297 | # 添加最后一帧 298 | data_frame = self.add_keyframe_click(data_frame, self.frame_count) 299 | 300 | return data_frame 301 | 302 | def movie_change(self, movie_path): 303 | if not movie_path: 304 | return ( 305 | gr.Image.update(visible=False), 306 | gr.Slider.update(maximum=0, minimum=0), 307 | gr.Slider.update(), 308 | ) 309 | fps = m2m_util.get_mov_fps(movie_path) 310 | self.frames = m2m_util.get_mov_all_images(movie_path, fps, True) 311 | 312 | self.frame_count = len(self.frames) 313 | return ( 314 | gr.Image.update(visible=True), 315 | gr.Slider.update(maximum=self.frame_count, minimum=0, value=0), 316 | gr.Slider.update(maximum=fps, minimum=0, value=fps), 317 | ) 318 | 319 | def movie_frame_change(self, movie_path, frame_number): 320 | if not movie_path: 321 | return gr.Image.update(visible=False) 322 | 323 | if frame_number <= 0: 324 | return gr.Image.update( 325 | visible=True, label=f"Frame: {frame_number}", value=None 326 | ) 327 | 328 | return gr.Image.update( 329 | visible=True, 330 | label=f"Frame: {frame_number}", 331 | value=self.frames[frame_number - 1], 332 | ) 333 | 334 | def fps_change(self, movie_path, fps): 335 | if not movie_path: 336 | return gr.Image.update(visible=False), gr.Slider.update( 337 | maximum=0, minimum=0 338 | ) 339 | 340 | self.frames = m2m_util.get_mov_all_images(movie_path, fps, True) 341 | self.frame_count = len(self.frames) 342 | return ( 343 | gr.Image.update(visible=True), 344 | gr.Slider.update(maximum=self.frame_count, minimum=0, value=0), 345 | ) 346 | -------------------------------------------------------------------------------- /style.css: -------------------------------------------------------------------------------- 1 | #mov2mov_interrupt, #mov2mov_skip { 2 | position: absolute; 3 | width: 50%; 4 | height: 100%; 5 | background: #b4c0cc; 6 | display: none; 7 | } 8 | 9 | 10 | 11 | #mov2mov_unused_scale_by_slider{ 12 | visibility: hidden; 13 | width: 0.5em; 14 | max-width: 0.5em; 15 | min-width: 0.5em; 16 | } 17 | 18 | #modnet_background_movie { 19 | display: none; 20 | } 21 | 22 | 23 | #mov2mov_video { 24 | display: none; 25 | } 26 | 27 | #mov2mov_generate { 28 | min-height: 4.5em; 29 | } 30 | 31 | @media screen and (min-width: 2500px) { 32 | #mov2mov_gallery { 33 | min-height: 768px; 34 | } 35 | } 36 | 37 | #mov2mov_gallery img { 38 | object-fit: scale-down; 39 | } 40 | 41 | #mov2mov_tools { 42 | gap: 0.4em; 43 | } 44 | 45 | #mov2mov_actions_column { 46 | margin: 0.35rem 0.75rem 0.35rem 0; 47 | } 48 | 49 | 50 | #mov2mov_actions_column { 51 | gap: 0; 52 | margin-right: .75rem; 53 | } 54 | 55 | 56 | #mov2mov_styles_row { 57 | gap: 0.25em; 58 | margin-top: 0.3em; 59 | } 60 | 61 | #mov2mov_styles_row > button { 62 | margin: 0; 63 | } 64 | 65 | #mov2mov_styles { 66 | padding: 0; 67 | } 68 | 69 | #mov2mov_styles > label > div { 70 | min-height: 3.2em; 71 | } 72 | 73 | #mov2mov_extra_networks .search { 74 | display: inline-block; 75 | max-width: 16em; 76 | margin: 0.3em; 77 | align-self: center; 78 | } 79 | 80 | #mov2mov_extra_view { 81 | width: auto; 82 | } 83 | 84 | #mov2mov_preview { 85 | position: absolute; 86 | width: 320px; 87 | left: 0; 88 | right: 0; 89 | margin-left: auto; 90 | margin-right: auto; 91 | margin-top: 34px; 92 | z-index: 100; 93 | border: none; 94 | border-top-left-radius: 0; 95 | border-top-right-radius: 0; 96 | } 97 | 98 | @media screen and (min-width: 768px) { 99 | #mov2mov_preview { 100 | position: absolute; 101 | } 102 | } 103 | 104 | @media screen and (max-width: 767px) { 105 | #mov2mov_preview { 106 | position: relative; 107 | } 108 | } 109 | 110 | #mov2mov_preview div.left-0.top-0 { 111 | display: none; 112 | } 113 | 114 | 115 | #mov2mov_interrupt, #mov2mov_skip { 116 | position: absolute; 117 | width: 50%; 118 | height: 100%; 119 | background: #b4c0cc; 120 | display: none; 121 | } 122 | 123 | #mov2mov_interrupt { 124 | left: 0; 125 | border-radius: 0.5rem 0 0 0.5rem; 126 | } 127 | 128 | #mov2mov_skip { 129 | right: 0; 130 | border-radius: 0 0.5rem 0.5rem 0; 131 | } 132 | 133 | #mov2mov_mov video { 134 | height: 480px; 135 | max-height: 480px; 136 | min-height: 480px; 137 | } 138 | 139 | #mov2mov_video video { 140 | height: 480px; 141 | max-height: 480px; 142 | min-height: 480px; 143 | } 144 | 145 | #mov2mov_video_frame img { 146 | height: 480px; 147 | max-height: 480px; 148 | min-height: 480px; 149 | } 150 | 151 | 152 | #mov2mov_checkboxes { 153 | margin-bottom: 0.5em; 154 | margin-left: 0em; 155 | } 156 | 157 | #mov2mov_checkboxes > div { 158 | flex: 0; 159 | white-space: nowrap; 160 | min-width: auto; 161 | } 162 | 163 | #mov2mov_extra_view { 164 | width: auto; 165 | } 166 | 167 | /* dataframe */ 168 | #mov2mov_video_editor_custom_data_frame button { 169 | display: none; 170 | } 171 | 172 | #mov2mov_keyframe_custom_container>div { 173 | display: flex; 174 | justify-content: center; 175 | /*增加间距*/ 176 | gap: 1.5em; 177 | } -------------------------------------------------------------------------------- /tests/ebsynth/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Scholar01/sd-webui-mov2mov/4bb1fbb8bc848d47a3f41711e0839a5c6e5845f5/tests/ebsynth/__init__.py -------------------------------------------------------------------------------- /tests/ebsynth/ebsynth_generate_test.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import numpy as np 4 | import unittest 5 | import cv2 6 | 7 | import importlib 8 | 9 | utils = importlib.import_module("extensions.sd-webui-mov2mov.tests.utils", "utils") 10 | utils.setup_test_env() 11 | 12 | from ebsynth.ebsynth_generate import EbsynthGenerate, Keyframe, Sequence 13 | 14 | 15 | class EbsynthGenerateTestCase(unittest.TestCase): 16 | def get_image(self, folder, name): 17 | return cv2.imread(os.path.join(os.path.dirname(__file__), 'images', folder, name)) 18 | 19 | def test_sequences(self): 20 | # 模拟100帧的视频 21 | keyframes = [ 22 | Keyframe(1, None, None), 23 | Keyframe(10, None, None), 24 | Keyframe(20, None, None), 25 | Keyframe(30, None, None), 26 | 27 | ] 28 | frames = [np.zeros((100, 100, 3))] * 40 29 | 30 | eb_generate = EbsynthGenerate(keyframes, frames, 24) 31 | eb_generate.setup_sequences() 32 | self.assertEqual(len(eb_generate.sequences), 4) 33 | self.assertEqual(eb_generate.sequences[0].start, 1) 34 | self.assertEqual(eb_generate.sequences[0].keyframe.num, 1) 35 | self.assertEqual(eb_generate.sequences[0].end, 10) 36 | 37 | self.assertEqual(eb_generate.sequences[1].start, 1) 38 | self.assertEqual(eb_generate.sequences[1].keyframe.num, 10) 39 | self.assertEqual(eb_generate.sequences[1].end, 20) 40 | 41 | self.assertEqual(eb_generate.sequences[2].start, 10) 42 | self.assertEqual(eb_generate.sequences[2].keyframe.num, 20) 43 | self.assertEqual(eb_generate.sequences[2].end, 30) 44 | 45 | self.assertEqual(eb_generate.sequences[3].start, 20) 46 | self.assertEqual(eb_generate.sequences[3].keyframe.num, 30) 47 | self.assertEqual(eb_generate.sequences[3].end, 40) 48 | 49 | self.assertEqual(list(eb_generate.sequences[0].frames.keys()), [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) 50 | self.assertEqual(list(eb_generate.sequences[1].frames.keys()), [i for i in range(1, 21)]) 51 | self.assertEqual(list(eb_generate.sequences[2].frames.keys()), [i for i in range(10, 31)]) 52 | self.assertEqual(list(eb_generate.sequences[3].frames.keys()), [i for i in range(20, 41)]) 53 | 54 | keyframes = [ 55 | Keyframe(1, None, None), 56 | Keyframe(3, None, None), 57 | Keyframe(5, None, None), 58 | ] 59 | frames = [np.zeros((100, 100, 3))] * 10 60 | 61 | eb_generate = EbsynthGenerate(keyframes, frames, 24) 62 | eb_generate.setup_sequences() 63 | 64 | self.assertEqual(len(eb_generate.sequences), 3) 65 | self.assertEqual(eb_generate.sequences[0].start, 1) 66 | self.assertEqual(eb_generate.sequences[0].keyframe.num, 1) 67 | self.assertEqual(eb_generate.sequences[0].end, 3) 68 | 69 | self.assertEqual(eb_generate.sequences[1].start, 1) 70 | self.assertEqual(eb_generate.sequences[1].keyframe.num, 3) 71 | self.assertEqual(eb_generate.sequences[1].end, 5) 72 | 73 | self.assertEqual(eb_generate.sequences[2].start, 3) 74 | self.assertEqual(eb_generate.sequences[2].keyframe.num, 5) 75 | self.assertEqual(eb_generate.sequences[2].end, 10) 76 | 77 | keyframes = [ 78 | Keyframe(1, None, None), 79 | Keyframe(3, None, None), 80 | Keyframe(5, None, None), 81 | ] 82 | frames = [np.zeros((100, 100, 3))] * 5 83 | 84 | eb_generate = EbsynthGenerate(keyframes, frames, 24) 85 | eb_generate.setup_sequences() 86 | 87 | self.assertEqual(len(eb_generate.sequences), 3) 88 | self.assertEqual(eb_generate.sequences[0].start, 1) 89 | self.assertEqual(eb_generate.sequences[0].keyframe.num, 1) 90 | self.assertEqual(eb_generate.sequences[0].end, 3) 91 | 92 | self.assertEqual(eb_generate.sequences[1].start, 1) 93 | self.assertEqual(eb_generate.sequences[1].keyframe.num, 3) 94 | self.assertEqual(eb_generate.sequences[1].end, 5) 95 | 96 | self.assertEqual(eb_generate.sequences[2].start, 3) 97 | self.assertEqual(eb_generate.sequences[2].keyframe.num, 5) 98 | self.assertEqual(eb_generate.sequences[2].end, 5) 99 | 100 | def test_get_guides(self): 101 | keyframes = [Keyframe(i, self.get_image('keys', f'{i:04d}.png'), '') for i in range(1, 72, 10)] 102 | frames = [self.get_image('video', f'{i:04d}.png') for i in range(0, 73)] 103 | 104 | eb_generate = EbsynthGenerate(keyframes, frames, 24) 105 | tasks = eb_generate.get_tasks(4.0) 106 | num = 0 107 | for sequence in eb_generate.sequences: 108 | for i in range(sequence.start, sequence.end + 1): 109 | self.assertEqual(tasks[num].frame_num, i) 110 | num += 1 111 | 112 | 113 | if __name__ == '__main__': 114 | unittest.main() 115 | -------------------------------------------------------------------------------- /tests/ebsynth/ebsynth_test.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | import os 3 | import unittest 4 | 5 | import cv2 6 | 7 | utils = importlib.import_module("extensions.sd-webui-mov2mov.tests.utils", "utils") 8 | utils.setup_test_env() 9 | 10 | from ebsynth.ebsynth_generate import EbsynthGenerate, Keyframe, Sequence 11 | from ebsynth._ebsynth import task as EbsyncthRun 12 | from scripts import m2m_util 13 | 14 | 15 | class MyTestCase(unittest.TestCase): 16 | def get_image(self, folder, name): 17 | return cv2.imread(os.path.join(os.path.dirname(__file__), 'images', folder, name)) 18 | 19 | def setUp(self) -> None: 20 | self.keyframes = [Keyframe(i, self.get_image('keys', f'{i:04d}.png'), '') for i in range(1, 72, 10)] 21 | frames = [self.get_image('video', f'{i:04d}.png') for i in range(0, 72)] 22 | 23 | self.eb_generate = EbsynthGenerate(self.keyframes, frames, 24) 24 | 25 | def test_keyframes(self): 26 | for i, sequence in enumerate(self.eb_generate.sequences): 27 | self.assertEqual(sequence.keyframe.num, self.keyframes[i].num) 28 | self.assertTrue((sequence.keyframe.image == self.keyframes[i].image).all()) 29 | 30 | def test_task(self): 31 | """ 32 | 测试生成的任务是否正确 33 | 34 | Returns: 35 | 36 | """ 37 | 38 | tasks = self.eb_generate.get_tasks(4.0) 39 | for task in tasks: 40 | result = EbsyncthRun(task.style, [(task.source, task.target, task.weight)]) 41 | dir_name = os.path.join(os.path.dirname(__file__), 'images', 'test', f'out_{task.key_frame_num}') 42 | if not os.path.exists(dir_name): 43 | os.mkdir(dir_name) 44 | cv2.imwrite(os.path.join(dir_name, f'{task.frame_num:04d}.png'), result) 45 | 46 | def test_merge(self): 47 | """ 48 | 测试merge是否正确 49 | 50 | """ 51 | 52 | def get_sequence(keyframe_num): 53 | for sequence in self.eb_generate.sequences: 54 | if sequence.keyframe.num == keyframe_num: 55 | return sequence 56 | else: 57 | raise ValueError(f'not found key frame num {keyframe_num}') 58 | 59 | # 模拟结果 60 | test_dir = os.path.join(os.path.dirname(__file__), 'images', 'test') 61 | 62 | # 获取out_{keyframe}文件夹 63 | for keyframe in self.keyframes: 64 | out_dir = os.path.join(test_dir, f'out_{keyframe.num:04d}') 65 | # 获取out_{keyframe}文件夹下的所有文件,并且按照 {i:04d}.png 的顺序添加到eb_generate.generate_frames 66 | sequence = get_sequence(keyframe.num) 67 | for i in range(sequence.start, sequence.end + 1): 68 | self.eb_generate.append_generate_frames(keyframe.num, i, 69 | cv2.imread(os.path.join(out_dir, f'{i:04d}.png'))) 70 | # 测试merge 71 | result = self.eb_generate.merge_sequences(0.4) 72 | 73 | if not os.path.exists(os.path.join(test_dir, 'merge_1')): 74 | os.mkdir(os.path.join(test_dir, 'merge_1')) 75 | 76 | frames = [] 77 | 78 | for i, frame in enumerate(result): 79 | if frame is not None: 80 | cv2.imwrite(os.path.join(test_dir, 'merge_1', f'{i:04d}.png'), frame) 81 | frames.append(frame) 82 | m2m_util.images_to_video(frames, self.eb_generate.fps, 83 | os.path.join(test_dir, 'merge_1', f'm.mp4')) 84 | 85 | 86 | if __name__ == '__main__': 87 | unittest.main() 88 | -------------------------------------------------------------------------------- /tests/utils.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | import os 4 | 5 | 6 | def setup_test_env(): 7 | ext_root = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) 8 | if ext_root not in sys.path: 9 | sys.path.append(ext_root) 10 | --------------------------------------------------------------------------------