├── .gitignore ├── demo4.mp4 ├── demo4.png ├── output.mp4 ├── .dockerignore ├── README.md ├── script └── download-weights ├── config.yaml ├── predict.py └── cog.yaml /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | .cog 3 | magic-animate 4 | pretrained_models 5 | -------------------------------------------------------------------------------- /demo4.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lucataco/cog-MagicAnimate/HEAD/demo4.mp4 -------------------------------------------------------------------------------- /demo4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lucataco/cog-MagicAnimate/HEAD/demo4.png -------------------------------------------------------------------------------- /output.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lucataco/cog-MagicAnimate/HEAD/output.mp4 -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | # The .dockerignore file excludes files from the container build process. 2 | # 3 | # https://docs.docker.com/engine/reference/builder/#dockerignore-file 4 | 5 | # Exclude Git files 6 | .git 7 | .github 8 | .gitignore 9 | 10 | # Exclude Python cache files 11 | __pycache__ 12 | .mypy_cache 13 | .pytest_cache 14 | .ruff_cache 15 | 16 | # Exclude Python virtual environment 17 | /venv 18 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # magic-research / magic-animate Cog model 2 | 3 | This is an implementation of [magic-research / magic-animate](https://github.com/magic-research/magic-animate) as a [Cog](https://github.com/replicate/cog) model. 4 | 5 | ## Development 6 | 7 | Follow the [model pushing guide](https://replicate.com/docs/guides/push-a-model) to push your own model to [Replicate](https://replicate.com). 8 | 9 | ## Basic Usage 10 | 11 | Download weights first 12 | 13 | cog run script/download-weights 14 | 15 | Then for predictions, 16 | 17 | cog predict -i image=@demo4.png -i video=@demo4.mp4 18 | 19 | -------------------------------------------------------------------------------- /script/download-weights: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os 4 | import sys 5 | 6 | MODEL_CACHE = "pretrained_models" 7 | 8 | if not os.path.exists(MODEL_CACHE): 9 | os.makedirs(MODEL_CACHE) 10 | 11 | os.system("git lfs clone --branch fp16 https://huggingface.co/runwayml/stable-diffusion-v1-5/ pretrained_models/stable-diffusion-v1-5/") 12 | os.system("git lfs clone https://huggingface.co/stabilityai/sd-vae-ft-mse pretrained_models/sd-vae-ft-mse") 13 | os.system("git lfs clone https://huggingface.co/zcxu-eric/MagicAnimate pretrained_models/MagicAnimate") 14 | 15 | os.system("git clone https://github.com/magic-research/magic-animate.git") -------------------------------------------------------------------------------- /config.yaml: -------------------------------------------------------------------------------- 1 | I: '0' 2 | L: '16' 3 | S: '1' 4 | clip: '0' 5 | fusion_blocks: midup 6 | guidance_scale: '7.5' 7 | inference_config: configs/inference/inference.yaml 8 | invert_video: 'false' 9 | max_length: 'null' 10 | motion_module: pretrained_models/MagicAnimate/temporal_attention/temporal_attention.ckpt 11 | offset: '0' 12 | pretrained_appearance_encoder_path: pretrained_models/MagicAnimate/appearance_encoder 13 | pretrained_controlnet_path: pretrained_models/MagicAnimate/densepose_controlnet 14 | pretrained_model_path: pretrained_models/stable-diffusion-v1-5 15 | pretrained_vae_path: pretrained_models/sd-vae-ft-mse 16 | save_individual_videos: 'false' 17 | savename: 'null' 18 | seed: '[1]' 19 | size: '512' 20 | source_image: 21 | - inputs/applications/source_image/demo4.png 22 | steps: '25' 23 | video_path: 24 | - inputs/applications/driving/densepose/demo4.mp4 25 | video_type: condition 26 | -------------------------------------------------------------------------------- /predict.py: -------------------------------------------------------------------------------- 1 | # Prediction interface for Cog ⚙️ 2 | # https://github.com/replicate/cog/blob/main/docs/python.md 3 | 4 | from cog import BasePredictor, Input, Path 5 | import os 6 | import sys 7 | import re 8 | sys.path.extend(['/src/magic-animate']) 9 | 10 | class Predictor(BasePredictor): 11 | def setup(self) -> None: 12 | """Load the model into memory to make running multiple predictions efficient""" 13 | # link pretrained_models to /src/magic-animate/pretrained_models 14 | # os.system("ln -s /src/pretrained_models /src/magic-animate/pretrained_models") 15 | 16 | def predict( 17 | self, 18 | image: Path = Input(description="Input image"), 19 | video: Path = Input(description="Input motion video"), 20 | num_inference_steps: int = Input( 21 | description="Number of denoising steps", ge=1, le=200, default=25 22 | ), 23 | guidance_scale: float = Input( 24 | description="Scale for classifier-free guidance", ge=1, le=50, default=7.5 25 | ), 26 | seed: int = Input( 27 | description="Random seed. Leave blank to randomize the seed", default=None 28 | ), 29 | ) -> Path: 30 | """Run a single prediction on the model""" 31 | if seed is None: 32 | seed = int.from_bytes(os.urandom(2), "big") 33 | print(f"Using seed: {seed}") 34 | 35 | # system change directory to /src/magic-animate 36 | os.chdir("/src/magic-animate") 37 | 38 | # Clean up past runs (just in case) 39 | output_dir = "samples/" 40 | os.system("rm -rf " + output_dir) 41 | os.system("mkdir -p " + output_dir) 42 | 43 | # Create a config.yaml file with the following content: 44 | config_data = f""" 45 | pretrained_model_path: "/src/pretrained_models/stable-diffusion-v1-5" 46 | pretrained_vae_path: "/src/pretrained_models/sd-vae-ft-mse" 47 | pretrained_controlnet_path: "/src/pretrained_models/MagicAnimate/densepose_controlnet" 48 | pretrained_appearance_encoder_path: "/src/pretrained_models/MagicAnimate/appearance_encoder" 49 | pretrained_unet_path: "" 50 | motion_module: "/src/pretrained_models/MagicAnimate/temporal_attention/temporal_attention.ckpt" 51 | savename: null 52 | fusion_blocks: "midup" 53 | seed: [{seed}] 54 | steps: {num_inference_steps} 55 | guidance_scale: {guidance_scale} 56 | source_image: 57 | - "{image}" 58 | video_path: 59 | - "{video}" 60 | inference_config: "configs/inference/inference.yaml" 61 | size: 512 62 | L: 16 63 | S: 1 64 | I: 0 65 | clip: 0 66 | offset: 0 67 | max_length: null 68 | video_type: "condition" 69 | invert_video: false 70 | save_individual_videos: false 71 | """ 72 | with open("configs/config.yaml", "w") as file: 73 | file.write(config_data) 74 | 75 | # run the following: python -m magicanimate.pipelines.animation --config config.yaml 76 | os.system("python -m magicanimate.pipelines.animation --config configs/config.yaml") 77 | 78 | #find a file ending in .mp4 in samples/config-*/video and return it 79 | base_dir = 'samples/' 80 | pattern = re.compile(r'^config-.*$') 81 | for folder in os.listdir(base_dir): 82 | if pattern.match(folder): 83 | video_dir = os.path.join(base_dir, folder, 'videos') 84 | if os.path.exists(video_dir): 85 | for file in os.listdir(video_dir): 86 | if file.endswith('.mp4'): 87 | video_path = os.path.join("/src/magic-animate/", video_dir, file) 88 | print(video_path) 89 | return Path(video_path) 90 | return "No video found" -------------------------------------------------------------------------------- /cog.yaml: -------------------------------------------------------------------------------- 1 | # Configuration for Cog ⚙️ 2 | # Reference: https://github.com/replicate/cog/blob/main/docs/yaml.md 3 | 4 | build: 5 | gpu: true 6 | cuda: "11.8" 7 | system_packages: 8 | - git-lfs 9 | python_version: "3.10" 10 | python_packages: 11 | - "absl-py==1.4.0" 12 | - "accelerate==0.22.0" 13 | - "aiofiles==23.2.1" 14 | - "aiohttp==3.8.5" 15 | - "aiosignal==1.3.1" 16 | - "altair==5.0.1" 17 | - "annotated-types==0.5.0" 18 | - "antlr4-python3-runtime==4.9.3" 19 | - "anyio==3.7.1" 20 | - "async-timeout==4.0.3" 21 | - "attrs==23.1.0" 22 | - "cachetools==5.3.1" 23 | - "certifi==2023.7.22" 24 | - "charset-normalizer==3.2.0" 25 | - "click==8.1.7" 26 | - "cmake==3.27.2" 27 | - "contourpy==1.1.0" 28 | - "cycler==0.11.0" 29 | - "datasets==2.14.4" 30 | - "dill==0.3.7" 31 | - "einops==0.6.1" 32 | - "exceptiongroup==1.1.3" 33 | - "fastapi==0.103.0" 34 | - "ffmpy==0.3.1" 35 | - "filelock==3.12.2" 36 | - "fonttools==4.42.1" 37 | - "frozenlist==1.4.0" 38 | - "fsspec==2023.6.0" 39 | - "google-auth==2.22.0" 40 | - "google-auth-oauthlib==1.0.0" 41 | - "gradio==3.41.2" 42 | - "gradio-client==0.5.0" 43 | - "grpcio==1.57.0" 44 | - "h11==0.14.0" 45 | - "httpcore==0.17.3" 46 | - "httpx==0.24.1" 47 | - "huggingface-hub==0.16.4" 48 | - "idna==3.4" 49 | - "importlib-metadata==6.8.0" 50 | - "importlib-resources==6.0.1" 51 | - "jinja2==3.1.2" 52 | - "joblib==1.3.2" 53 | - "jsonschema==4.19.0" 54 | - "jsonschema-specifications==2023.7.1" 55 | - "kiwisolver==1.4.5" 56 | - "lightning-utilities==0.9.0" 57 | - "lit==16.0.6" 58 | - "markdown==3.4.4" 59 | - "markupsafe==2.1.3" 60 | - "matplotlib==3.7.2" 61 | - "mpmath==1.3.0" 62 | - "multidict==6.0.4" 63 | - "multiprocess==0.70.15" 64 | - "networkx==3.1" 65 | - "numpy==1.24.4" 66 | - "nvidia-cublas-cu11==11.10.3.66" 67 | - "nvidia-cuda-cupti-cu11==11.7.101" 68 | - "nvidia-cuda-nvrtc-cu11==11.7.99" 69 | - "nvidia-cuda-runtime-cu11==11.7.99" 70 | - "nvidia-cudnn-cu11==8.5.0.96" 71 | - "nvidia-cufft-cu11==10.9.0.58" 72 | - "nvidia-curand-cu11==10.2.10.91" 73 | - "nvidia-cusolver-cu11==11.4.0.1" 74 | - "nvidia-cusparse-cu11==11.7.4.91" 75 | - "nvidia-nccl-cu11==2.14.3" 76 | - "nvidia-nvtx-cu11==11.7.91" 77 | - "oauthlib==3.2.2" 78 | - "omegaconf==2.3.0" 79 | - "opencv-python==4.8.0.76" 80 | - "orjson==3.9.5" 81 | - "pandas==2.0.3" 82 | - "pillow==9.5.0" 83 | - "pkgutil-resolve-name==1.3.10" 84 | - "protobuf==4.24.2" 85 | - "psutil==5.9.5" 86 | - "pyarrow==13.0.0" 87 | - "pyasn1==0.5.0" 88 | - "pyasn1-modules==0.3.0" 89 | # - "pydantic==2.3.0" 90 | # - "pydantic-core==2.6.3" 91 | - "pydub==0.25.1" 92 | - "pyparsing==3.0.9" 93 | - "python-multipart==0.0.6" 94 | - "pytorch-lightning==2.0.7" 95 | - "pytz==2023.3" 96 | - "pyyaml==6.0.1" 97 | - "referencing==0.30.2" 98 | - "regex==2023.8.8" 99 | - "requests==2.31.0" 100 | - "requests-oauthlib==1.3.1" 101 | - "rpds-py==0.9.2" 102 | - "rsa==4.9" 103 | - "safetensors==0.3.3" 104 | - "semantic-version==2.10.0" 105 | - "sniffio==1.3.0" 106 | - "starlette==0.27.0" 107 | - "sympy==1.12" 108 | - "tensorboard==2.14.0" 109 | - "tensorboard-data-server==0.7.1" 110 | - "tokenizers==0.13.3" 111 | - "toolz==0.12.0" 112 | - "torchmetrics==1.1.0" 113 | - "tqdm==4.66.1" 114 | - "transformers==4.32.0" 115 | - "triton==2.0.0" 116 | - "tzdata==2023.3" 117 | - "urllib3==1.26.16" 118 | - "uvicorn==0.23.2" 119 | - "websockets==11.0.3" 120 | - "werkzeug==2.3.7" 121 | - "xxhash==3.3.0" 122 | - "yarl==1.9.2" 123 | - "zipp==3.16.2" 124 | - "decord" 125 | - "imageio==2.9.0" 126 | - "imageio-ffmpeg==0.4.3" 127 | - "timm" 128 | - "scipy" 129 | - "scikit-image" 130 | - "av" 131 | - "imgaug" 132 | - "lpips" 133 | - "ffmpeg-python" 134 | - "torch==2.0.1" 135 | - "torchvision==0.15.2" 136 | - "xformers==0.0.22" 137 | - "diffusers==0.21.4" 138 | 139 | run: 140 | # - git clone https://github.com/magic-research/magic-animate.git /magic-animate 141 | 142 | # predict.py defines how predictions are run on your model 143 | predict: "predict.py:Predictor" 144 | --------------------------------------------------------------------------------