├── README.md ├── SMPLer_X_colab.ipynb └── SMPLer_X_gradio_jupyter.ipynb /README.md: -------------------------------------------------------------------------------- 1 | 🐣 Please follow me for new updates https://twitter.com/camenduru
2 | 🔥 Please join our discord server https://discord.gg/k5BwmmvJJU
3 | 🥳 Please join my patreon community https://patreon.com/camenduru
4 | 5 | ### 🍊 Jupyter Notebook 6 | 7 | | Notebook | Info 8 | | --- | --- | 9 | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/SMPLer-X-colab/blob/main/SMPLer_X_colab.ipynb) | SMPLer_X_colab 10 | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/SMPLer-X-colab/blob/main/SMPLer_X_gradio_jupyter.ipynb) | SMPLer_X_gradio_jupyter 11 | 12 | ### 🧬 Code 13 | https://github.com/caizhongang/SMPLer-X 14 | 15 | ### 📄 Paper 16 | https://arxiv.org/abs/2309.17448 17 | 18 | ### 🌐 Page 19 | https://caizhongang.github.io/projects/SMPLer-X/ 20 | 21 | ## License 22 | According to the license, we should register at https://smpl.is.tue.mpg.de and https://smpl-x.is.tue.mpg.de 23 | 24 | ### 🖼 Output 25 | 26 | https://huggingface.co/camenduru/SMPLer-X/tree/main/test_video
27 | 28 | inference model: smpler_x_l32
29 | 30 | https://github.com/camenduru/SMPLer-X-colab/assets/54370274/6aa9fefa-fce8-4663-80a1-93a0d1a5aef6 31 | 32 | inference model: smpler_x_h32
33 | 34 | https://github.com/camenduru/SMPLer-X-colab/assets/54370274/5c8366e2-1935-4a10-894d-f48cd7b6db99 35 | 36 | | _ | _ 37 | | --- | --- | 38 | ![Screenshot 2023-10-03 121533](https://github.com/camenduru/SMPLer-X-colab/assets/54370274/4e1349ec-54ff-49bb-a33d-5751c9e9f5af) | ![Screenshot 2023-10-03 121521](https://github.com/camenduru/SMPLer-X-colab/assets/54370274/630afa7c-9c28-4cff-a556-e19de273385d) 39 | -------------------------------------------------------------------------------- /SMPLer_X_colab.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "view-in-github" 7 | }, 8 | "source": [ 9 | "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/SMPLer-X-colab/blob/main/SMPLer_X_colab.ipynb)" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "id": "VjYy0F2gZIPR" 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "%cd /content\n", 21 | "!git clone -b dev https://github.com/camenduru/SMPLer-X\n", 22 | "%cd /content/SMPLer-X\n", 23 | "\n", 24 | "!pip install -q smplx==0.1.28 yacs tensorboardx filterpy chumpy trimesh pyrender json_tricks torchgeometry einops boto3 plyfile timm pyglet mmdet==2.26.0 yapf==0.40.1\n", 25 | "!pip install -q https://github.com/camenduru/wheels/releases/download/colab/mmcv_full-1.7.1-cp310-cp310-linux_x86_64.whl\n", 26 | "\n", 27 | "%cd /content/SMPLer-X/main/transformer_utils\n", 28 | "!pip install -v -e .\n", 29 | "%cd /content/SMPLer-X\n", 30 | "\n", 31 | "!apt -y install -qq aria2\n", 32 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/SMPLer-X/resolve/main/faster_rcnn_r50_fpn_1x_coco_20200130-047c8118.pth -d /content/SMPLer-X/pretrained_models/mmdet -o faster_rcnn_r50_fpn_1x_coco_20200130-047c8118.pth\n", 33 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/SMPLer-X/raw/main/mmdet_faster_rcnn_r50_fpn_coco.py -d /content/SMPLer-X/pretrained_models/mmdet -o mmdet_faster_rcnn_r50_fpn_coco.py\n", 34 | "\n", 35 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/SMPLer-X/resolve/main/smpler_x_s32.pth.tar -d /content/SMPLer-X/pretrained_models -o smpler_x_s32.pth.tar\n", 36 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/SMPLer-X/resolve/main/smpler_x_b32.pth.tar -d /content/SMPLer-X/pretrained_models -o smpler_x_b32.pth.tar\n", 37 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/SMPLer-X/resolve/main/smpler_x_l32.pth.tar -d /content/SMPLer-X/pretrained_models -o smpler_x_l32.pth.tar\n", 38 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/SMPLer-X/resolve/main/smpler_x_h32.pth.tar -d /content/SMPLer-X/pretrained_models -o smpler_x_h32.pth.tar\n", 39 | "\n", 40 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/SMPLer-X/resolve/main/vitpose_small.pth -d /content/SMPLer-X/pretrained_models -o vitpose_small.pth\n", 41 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/SMPLer-X/resolve/main/vitpose_base.pth -d /content/SMPLer-X/pretrained_models -o vitpose_base.pth\n", 42 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/SMPLer-X/resolve/main/vitpose_large.pth -d /content/SMPLer-X/pretrained_models -o vitpose_large.pth\n", 43 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/SMPLer-X/resolve/main/vitpose_huge.pth -d /content/SMPLer-X/pretrained_models -o vitpose_huge.pth\n", 44 | "\n", 45 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/SMPLer-X/resolve/main/SMPL_FEMALE.pkl -d /content/SMPLer-X/common/utils/human_model_files/smpl -o SMPL_FEMALE.pkl\n", 46 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/SMPLer-X/resolve/main/SMPL_MALE.pkl -d /content/SMPLer-X/common/utils/human_model_files/smpl -o SMPL_MALE.pkl\n", 47 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/SMPLer-X/resolve/main/SMPL_NEUTRAL.pkl -d /content/SMPLer-X/common/utils/human_model_files/smpl -o SMPL_NEUTRAL.pkl\n", 48 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/SMPLer-X/resolve/main/MANO_SMPLX_vertex_ids.pkl -d /content/SMPLer-X/common/utils/human_model_files/smplx -o MANO_SMPLX_vertex_ids.pkl\n", 49 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/SMPLer-X/resolve/main/SMPL-X__FLAME_vertex_ids.npy -d /content/SMPLer-X/common/utils/human_model_files/smplx -o SMPL-X__FLAME_vertex_ids.npy\n", 50 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/SMPLer-X/resolve/main/SMPLX_NEUTRAL.npz -d /content/SMPLer-X/common/utils/human_model_files/smplx -o SMPLX_NEUTRAL.npz\n", 51 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/SMPLer-X/resolve/main/SMPLX_NEUTRAL.pkl -d /content/SMPLer-X/common/utils/human_model_files/smplx -o SMPLX_NEUTRAL.pkl\n", 52 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/SMPLer-X/resolve/main/SMPLX_to_J14.pkl -d /content/SMPLer-X/common/utils/human_model_files/smplx -o SMPLX_to_J14.pkl\n", 53 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/SMPLer-X/resolve/main/SMPLX_MALE.npz -d /content/SMPLer-X/common/utils/human_model_files/smplx -o SMPLX_MALE.npz\n", 54 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/SMPLer-X/resolve/main/SMPLX_FEMALE.npz -d /content/SMPLer-X/common/utils/human_model_files/smplx -o SMPLX_FEMALE.npz\n", 55 | "\n", 56 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/camenduru/SMPLer-X/resolve/main/test_video.mp4 -d /content/SMPLer-X/demo/videos -o test_video.mp4\n", 57 | "\n", 58 | "!sed -i '301,304s/^/#/' /usr/local/lib/python3.10/dist-packages/torchgeometry/core/conversions.py\n", 59 | "!sed -i '300a\\ inv_mask_d0_d1 = ~mask_d0_d1\\n inv_mask_d0_nd1 = ~mask_d0_nd1\\n inv_mask_d2 = ~mask_d2\\n mask_c0 = mask_d2 * mask_d0_d1\\n mask_c1 = mask_d2 * inv_mask_d0_d1\\n mask_c2 = inv_mask_d2 * mask_d0_nd1\\n mask_c3 = inv_mask_d2 * inv_mask_d0_nd1' /usr/local/lib/python3.10/dist-packages/torchgeometry/core/conversions.py" 60 | ] 61 | }, 62 | { 63 | "cell_type": "code", 64 | "execution_count": null, 65 | "metadata": {}, 66 | "outputs": [], 67 | "source": [ 68 | "!mkdir -p /content/SMPLer-X/demo/images/test_video\n", 69 | "!mkdir -p /content/SMPLer-X/demo/results/test_video\n", 70 | "%cd /content/SMPLer-X/main\n", 71 | "# !apt -y install -qq slurm-client\n", 72 | "# !sh slurm_inference.sh test_video mp4 24 smpler_x_l32\n", 73 | "!ffmpeg -i /content/SMPLer-X/demo/videos/test_video.mp4 -f image2 -vf fps=24/1 -qscale 0 /content/SMPLer-X/demo/images/test_video/%06d.jpg\n", 74 | "!python inference.py --num_gpus 1 --exp_name output/demo_inference_test_video --pretrained_model smpler_x_h32 --agora_benchmark agora_model --img_path /content/SMPLer-X/demo/images/test_video --start 1 --end 527 --output_folder /content/SMPLer-X/demo/results/test_video --show_verts --show_bbox --save_mesh\n", 75 | "!ffmpeg -y -f image2 -r $24 -i /content/SMPLer-X/demo/results/test_video/img/%06d.jpg -vcodec libx264 -qscale 0 -pix_fmt yuv420p /content/SMPLer-X/demo/results/test_video/test_video.mp4" 76 | ] 77 | } 78 | ], 79 | "metadata": { 80 | "accelerator": "GPU", 81 | "colab": { 82 | "gpuType": "T4", 83 | "provenance": [] 84 | }, 85 | "kernelspec": { 86 | "display_name": "Python 3", 87 | "name": "python3" 88 | }, 89 | "language_info": { 90 | "name": "python" 91 | } 92 | }, 93 | "nbformat": 4, 94 | "nbformat_minor": 0 95 | } 96 | -------------------------------------------------------------------------------- /SMPLer_X_gradio_jupyter.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "view-in-github" 7 | }, 8 | "source": [ 9 | "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/camenduru/SMPLer-X-colab/blob/main/SMPLer_X_gradio_jupyter.ipynb)" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": { 16 | "id": "VjYy0F2gZIPR" 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "%cd /content\n", 21 | "!git clone -b dev https://github.com/camenduru/SMPLer-X-hf\n", 22 | "%cd /content/SMPLer-X-hf\n", 23 | "\n", 24 | "!pip install -q smplx==0.1.28 yacs tensorboardx filterpy chumpy trimesh pyrender json_tricks torchgeometry einops boto3 plyfile timm pyglet mmdet==2.26.0 yapf==0.40.1\n", 25 | "!pip install -q https://github.com/camenduru/wheels/releases/download/colab/mmcv_full-1.7.1-cp310-cp310-linux_x86_64.whl gradio==4.16.0 numpy==1.23\n", 26 | "\n", 27 | "%cd /content/SMPLer-X-hf/main/transformer_utils\n", 28 | "!pip install -v -e .\n", 29 | "%cd /content/SMPLer-X-hf\n", 30 | "\n", 31 | "!apt -y install -qq aria2\n", 32 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/spaces/caizhongang/SMPLer-X/resolve/main/assets/01.mp4 -d /content/SMPLer-X-hf/assets -o 01.mp4\n", 33 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/spaces/caizhongang/SMPLer-X/resolve/main/assets/02.mp4 -d /content/SMPLer-X-hf/assets -o 02.mp4\n", 34 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/spaces/caizhongang/SMPLer-X/resolve/main/assets/03.mp4 -d /content/SMPLer-X-hf/assets -o 03.mp4\n", 35 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/spaces/caizhongang/SMPLer-X/resolve/main/assets/04.mp4 -d /content/SMPLer-X-hf/assets -o 04.mp4\n", 36 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/spaces/caizhongang/SMPLer-X/resolve/main/assets/06.mp4 -d /content/SMPLer-X-hf/assets -o 06.mp4\n", 37 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/spaces/caizhongang/SMPLer-X/resolve/main/assets/07.mp4 -d /content/SMPLer-X-hf/assets -o 07.mp4\n", 38 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/spaces/caizhongang/SMPLer-X/resolve/main/assets/09.mp4 -d /content/SMPLer-X-hf/assets -o 09.mp4\n", 39 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/spaces/caizhongang/SMPLer-X/resolve/main/common/utils/human_model_files/smpl/SMPL_FEMALE.pkl -d /content/SMPLer-X-hf/common/utils/human_model_files/smpl -o SMPL_FEMALE.pkl\n", 40 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/spaces/caizhongang/SMPLer-X/resolve/main/common/utils/human_model_files/smpl/SMPL_MALE.pkl -d /content/SMPLer-X-hf/common/utils/human_model_files/smpl -o SMPL_MALE.pkl\n", 41 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/spaces/caizhongang/SMPLer-X/resolve/main/common/utils/human_model_files/smpl/SMPL_NEUTRAL.pkl -d /content/SMPLer-X-hf/common/utils/human_model_files/smpl -o SMPL_NEUTRAL.pkl\n", 42 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/spaces/caizhongang/SMPLer-X/resolve/main/common/utils/human_model_files/smpl/smpl_uv.npz -d /content/SMPLer-X-hf/common/utils/human_model_files/smpl -o smpl_uv.npz\n", 43 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/spaces/caizhongang/SMPLer-X/resolve/main/common/utils/human_model_files/smplx/MANO_SMPLX_vertex_ids.pkl -d /content/SMPLer-X-hf/common/utils/human_model_files/smplx -o MANO_SMPLX_vertex_ids.pkl\n", 44 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/spaces/caizhongang/SMPLer-X/resolve/main/common/utils/human_model_files/smplx/SMPL-X__FLAME_vertex_ids.npy -d /content/SMPLer-X-hf/common/utils/human_model_files/smplx -o SMPL-X__FLAME_vertex_ids.npy\n", 45 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/spaces/caizhongang/SMPLer-X/resolve/main/common/utils/human_model_files/smplx/SMPLX_FEMALE.npz -d /content/SMPLer-X-hf/common/utils/human_model_files/smplx -o SMPLX_FEMALE.npz\n", 46 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/spaces/caizhongang/SMPLer-X/resolve/main/common/utils/human_model_files/smplx/SMPLX_MALE.npz -d /content/SMPLer-X-hf/common/utils/human_model_files/smplx -o SMPLX_MALE.npz\n", 47 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/spaces/caizhongang/SMPLer-X/resolve/main/common/utils/human_model_files/smplx/SMPLX_NEUTRAL.npz -d /content/SMPLer-X-hf/common/utils/human_model_files/smplx -o SMPLX_NEUTRAL.npz\n", 48 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/spaces/caizhongang/SMPLer-X/resolve/main/common/utils/human_model_files/smplx/SMPLX_NEUTRAL.pkl -d /content/SMPLer-X-hf/common/utils/human_model_files/smplx -o SMPLX_NEUTRAL.pkl\n", 49 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/spaces/caizhongang/SMPLer-X/resolve/main/common/utils/human_model_files/smplx/SMPLX_to_J14.pkl -d /content/SMPLer-X-hf/common/utils/human_model_files/smplx -o SMPLX_to_J14.pkl\n", 50 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/spaces/caizhongang/SMPLer-X/resolve/main/pretrained_models/mmdet/faster_rcnn_r50_fpn_1x_coco_20200130-047c8118.pth -d /content/SMPLer-X-hf/pretrained_models/mmdet -o faster_rcnn_r50_fpn_1x_coco_20200130-047c8118.pth\n", 51 | "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/spaces/caizhongang/SMPLer-X/resolve/main/pretrained_models/smpler_x_h32.pth.tar -d /content/SMPLer-X-hf/pretrained_models -o smpler_x_h32.pth.tar\n", 52 | "\n", 53 | "!python app.py" 54 | ] 55 | } 56 | ], 57 | "metadata": { 58 | "accelerator": "GPU", 59 | "colab": { 60 | "gpuType": "T4", 61 | "provenance": [] 62 | }, 63 | "kernelspec": { 64 | "display_name": "Python 3", 65 | "name": "python3" 66 | }, 67 | "language_info": { 68 | "name": "python" 69 | } 70 | }, 71 | "nbformat": 4, 72 | "nbformat_minor": 0 73 | } 74 | --------------------------------------------------------------------------------