├── README.md
└── video_retalking_colab.ipynb
/README.md:
--------------------------------------------------------------------------------
1 | 🐣 Please follow me for new updates https://twitter.com/camenduru
2 | 🔥 Please join our discord server https://discord.gg/k5BwmmvJJU
3 | 🥳 Please join my patreon community https://patreon.com/camenduru
4 |
5 | ## 🦒 Colab
6 |
7 | | Colab | Info
8 | | --- | --- |
9 | [](https://colab.research.google.com/github/camenduru/video-retalking-colab/blob/main/video_retalking_colab.ipynb) | video_retalking_colab
10 |
11 | ## Main Repo
12 | https://github.com/OpenTalker/video-retalking
13 |
14 | ## Paper
15 | https://arxiv.org/abs/2211.14758
16 |
17 | ## Page
18 | https://opentalker.github.io/video-retalking/
19 |
20 | ## Output
21 |
22 | https://github.com/camenduru/video-retalking-colab/assets/54370274/166d176b-d028-41c2-9d8a-6a4fd4677d9e
23 |
24 | https://github.com/camenduru/video-retalking-colab/assets/54370274/f5d9d275-4a61-4540-939c-068d8dd74cb9
25 |
26 | https://github.com/camenduru/video-retalking-colab/assets/54370274/33fefc1a-38d3-4f74-bfc2-0410526a026b
27 |
--------------------------------------------------------------------------------
/video_retalking_colab.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "markdown",
5 | "metadata": {
6 | "id": "view-in-github"
7 | },
8 | "source": [
9 | "[](https://colab.research.google.com/github/camenduru/video-retalking-colab/blob/main/video_retalking_colab.ipynb)"
10 | ]
11 | },
12 | {
13 | "cell_type": "code",
14 | "execution_count": null,
15 | "metadata": {
16 | "id": "VjYy0F2gZIPR"
17 | },
18 | "outputs": [],
19 | "source": [
20 | "%cd /content/\n",
21 | "!git clone https://github.com/camenduru/video-retalking\n",
22 | "\n",
23 | "!apt -y install -qq aria2\n",
24 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/video-retalking/resolve/main/30_net_gen.pth -d /content/video-retalking/checkpoints -o 30_net_gen.pth\n",
25 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/video-retalking/resolve/main/BFM.zip -d /content/video-retalking/checkpoints -o BFM.zip\n",
26 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/video-retalking/resolve/main/DNet.pt -d /content/video-retalking/checkpoints -o DNet.pt\n",
27 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/video-retalking/resolve/main/ENet.pth -d /content/video-retalking/checkpoints -o ENet.pth\n",
28 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/video-retalking/resolve/main/GFPGANv1.3.pth -d /content/video-retalking/checkpoints -o GFPGANv1.3.pth\n",
29 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/video-retalking/resolve/main/GPEN-BFR-512.pth -d /content/video-retalking/checkpoints -o GPEN-BFR-512.pth\n",
30 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/video-retalking/resolve/main/LNet.pth -d /content/video-retalking/checkpoints -o LNet.pth\n",
31 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/video-retalking/resolve/main/ParseNet-latest.pth -d /content/video-retalking/checkpoints -o ParseNet-latest.pth\n",
32 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/video-retalking/resolve/main/RetinaFace-R50.pth -d /content/video-retalking/checkpoints -o RetinaFace-R50.pth\n",
33 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/video-retalking/resolve/main/expression.mat -d /content/video-retalking/checkpoints -o expression.mat\n",
34 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/video-retalking/resolve/main/face3d_pretrain_epoch_20.pth -d /content/video-retalking/checkpoints -o face3d_pretrain_epoch_20.pth\n",
35 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/video-retalking/resolve/main/shape_predictor_68_face_landmarks.dat -d /content/video-retalking/checkpoints -o shape_predictor_68_face_landmarks.dat\n",
36 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/facexlib/resolve/main/detection_Resnet50_Final.pth -d /usr/local/lib/python3.10/dist-packages/facexlib/weights -o detection_Resnet50_Final.pth\n",
37 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/facexlib/resolve/main/s3fd-619a316812.pth -d /root/.cache/torch/hub/checkpoints -o s3fd-619a316812.pth\n",
38 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/facexlib/resolve/main/2DFAN4-cd938726ad.zip -d /root/.cache/torch/hub/checkpoints -o 2DFAN4-cd938726ad.zip\n",
39 | "!unzip -d /content/video-retalking/checkpoints/BFM /content/video-retalking/checkpoints/BFM.zip\n",
40 | "\n",
41 | "!pip install -q basicsr==1.4.2 face-alignment==1.3.4 facexlib==0.2.5 librosa==0.9.2 ninja==1.11.1.1 einops==0.7.0 kornia==0.7.0 -U"
42 | ]
43 | },
44 | {
45 | "cell_type": "code",
46 | "execution_count": null,
47 | "metadata": {},
48 | "outputs": [],
49 | "source": [
50 | "%cd /content/video-retalking\n",
51 | "\n",
52 | "!python inference.py \\\n",
53 | " --face \"/content/video-retalking/examples/face/1.mp4\" \\\n",
54 | " --audio \"/content/video-retalking/examples/audio/1.wav\" \\\n",
55 | " --outfile '/content/output_high_qual.mp4'"
56 | ]
57 | }
58 | ],
59 | "metadata": {
60 | "accelerator": "GPU",
61 | "colab": {
62 | "gpuType": "T4",
63 | "provenance": []
64 | },
65 | "kernelspec": {
66 | "display_name": "Python 3",
67 | "name": "python3"
68 | },
69 | "language_info": {
70 | "name": "python"
71 | }
72 | },
73 | "nbformat": 4,
74 | "nbformat_minor": 0
75 | }
76 |
--------------------------------------------------------------------------------