├── .gitignore ├── LICENSE ├── MoRe4D ├── __init__.py ├── data │ ├── bucket_sampler.py │ ├── vae_dataset.py │ └── wan_dataset.py ├── models │ ├── __init__.py │ ├── cache_utils.py │ ├── omnimae.py │ ├── omnivision │ │ ├── models │ │ │ ├── __init__.py │ │ │ ├── heads │ │ │ │ ├── __init__.py │ │ │ │ └── mae_head.py │ │ │ ├── swin_transformer.py │ │ │ └── vision_transformer.py │ │ └── utils │ │ │ ├── checkpoint.py │ │ │ ├── data.py │ │ │ ├── distributed.py │ │ │ ├── generic.py │ │ │ ├── testing.py │ │ │ └── train.py │ ├── trajectory_module.py │ ├── wan_image_encoder.py │ ├── wan_text_encoder.py │ ├── wan_transformer3d.py │ ├── wan_transformer4d.py │ ├── wan_vae.py │ └── wan_xlm_roberta.py ├── pipeline │ ├── __init__.py │ ├── pipeline_wan_fun_control.py │ └── pipeline_wan_fun_inpaint.py └── utils │ ├── __init__.py │ ├── ac_handle.py │ ├── cfg_optimization.py │ ├── discrete_sampler.py │ ├── fm_solvers.py │ ├── fm_solvers_unipc.py │ ├── fp8_optimization.py │ ├── gaussian_splatting.py │ ├── lora_utils.py │ ├── project_utils.py │ └── utils.py ├── README.md ├── config ├── wan2.1 │ ├── wan_civital.yaml │ └── wan_civital_4D_STraG.yaml ├── zero_stage2_config.json ├── zero_stage3_config.json └── zero_stage3_config_cpu_offload.json ├── models └── put_your_models_here.txt ├── requirements.txt ├── scripts ├── 4D_STraG_training │ ├── train_vae.py │ ├── train_vae.sh │ ├── train_wan.py │ └── train_wan.sh ├── 4D_ViSM_training │ ├── train.py │ └── train.sh ├── inference │ ├── infer.py │ ├── infer.sh │ ├── infer_vae.py │ └── infer_vae.sh └── zero_to_bf16.py └── static ├── demo_videos ├── .DS_Store ├── bear │ ├── 4D-STraG_output │ │ ├── .DS_Store │ │ └── bear_render.mp4 │ ├── 4D-ViSM_output │ │ ├── .DS_Store │ │ └── 1.mp4 │ └── Input.png └── camel │ ├── .DS_Store │ ├── 4D-STraG_output │ ├── .DS_Store │ └── camel_render.mp4 │ ├── 4D-ViSM_output │ ├── .DS_Store │ └── 1.mp4 │ └── Input.png └── images ├── dataset.png ├── pipeline.png ├── qualitative_1.png ├── qualitative_2.png ├── qualitative_3.png ├── quantitative.png └── teaser.png /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/LICENSE -------------------------------------------------------------------------------- /MoRe4D/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /MoRe4D/data/bucket_sampler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/MoRe4D/data/bucket_sampler.py -------------------------------------------------------------------------------- /MoRe4D/data/vae_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/MoRe4D/data/vae_dataset.py -------------------------------------------------------------------------------- /MoRe4D/data/wan_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/MoRe4D/data/wan_dataset.py -------------------------------------------------------------------------------- /MoRe4D/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/MoRe4D/models/__init__.py -------------------------------------------------------------------------------- /MoRe4D/models/cache_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/MoRe4D/models/cache_utils.py -------------------------------------------------------------------------------- /MoRe4D/models/omnimae.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/MoRe4D/models/omnimae.py -------------------------------------------------------------------------------- /MoRe4D/models/omnivision/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/MoRe4D/models/omnivision/models/__init__.py -------------------------------------------------------------------------------- /MoRe4D/models/omnivision/models/heads/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | -------------------------------------------------------------------------------- /MoRe4D/models/omnivision/models/heads/mae_head.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/MoRe4D/models/omnivision/models/heads/mae_head.py -------------------------------------------------------------------------------- /MoRe4D/models/omnivision/models/swin_transformer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/MoRe4D/models/omnivision/models/swin_transformer.py -------------------------------------------------------------------------------- /MoRe4D/models/omnivision/models/vision_transformer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/MoRe4D/models/omnivision/models/vision_transformer.py -------------------------------------------------------------------------------- /MoRe4D/models/omnivision/utils/checkpoint.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/MoRe4D/models/omnivision/utils/checkpoint.py -------------------------------------------------------------------------------- /MoRe4D/models/omnivision/utils/data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/MoRe4D/models/omnivision/utils/data.py -------------------------------------------------------------------------------- /MoRe4D/models/omnivision/utils/distributed.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/MoRe4D/models/omnivision/utils/distributed.py -------------------------------------------------------------------------------- /MoRe4D/models/omnivision/utils/generic.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/MoRe4D/models/omnivision/utils/generic.py -------------------------------------------------------------------------------- /MoRe4D/models/omnivision/utils/testing.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/MoRe4D/models/omnivision/utils/testing.py -------------------------------------------------------------------------------- /MoRe4D/models/omnivision/utils/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/MoRe4D/models/omnivision/utils/train.py -------------------------------------------------------------------------------- /MoRe4D/models/trajectory_module.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/MoRe4D/models/trajectory_module.py -------------------------------------------------------------------------------- /MoRe4D/models/wan_image_encoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/MoRe4D/models/wan_image_encoder.py -------------------------------------------------------------------------------- /MoRe4D/models/wan_text_encoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/MoRe4D/models/wan_text_encoder.py -------------------------------------------------------------------------------- /MoRe4D/models/wan_transformer3d.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/MoRe4D/models/wan_transformer3d.py -------------------------------------------------------------------------------- /MoRe4D/models/wan_transformer4d.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/MoRe4D/models/wan_transformer4d.py -------------------------------------------------------------------------------- /MoRe4D/models/wan_vae.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/MoRe4D/models/wan_vae.py -------------------------------------------------------------------------------- /MoRe4D/models/wan_xlm_roberta.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/MoRe4D/models/wan_xlm_roberta.py -------------------------------------------------------------------------------- /MoRe4D/pipeline/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/MoRe4D/pipeline/__init__.py -------------------------------------------------------------------------------- /MoRe4D/pipeline/pipeline_wan_fun_control.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/MoRe4D/pipeline/pipeline_wan_fun_control.py -------------------------------------------------------------------------------- /MoRe4D/pipeline/pipeline_wan_fun_inpaint.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/MoRe4D/pipeline/pipeline_wan_fun_inpaint.py -------------------------------------------------------------------------------- /MoRe4D/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/MoRe4D/utils/__init__.py -------------------------------------------------------------------------------- /MoRe4D/utils/ac_handle.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/MoRe4D/utils/ac_handle.py -------------------------------------------------------------------------------- /MoRe4D/utils/cfg_optimization.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/MoRe4D/utils/cfg_optimization.py -------------------------------------------------------------------------------- /MoRe4D/utils/discrete_sampler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/MoRe4D/utils/discrete_sampler.py -------------------------------------------------------------------------------- /MoRe4D/utils/fm_solvers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/MoRe4D/utils/fm_solvers.py -------------------------------------------------------------------------------- /MoRe4D/utils/fm_solvers_unipc.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/MoRe4D/utils/fm_solvers_unipc.py -------------------------------------------------------------------------------- /MoRe4D/utils/fp8_optimization.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/MoRe4D/utils/fp8_optimization.py -------------------------------------------------------------------------------- /MoRe4D/utils/gaussian_splatting.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/MoRe4D/utils/gaussian_splatting.py -------------------------------------------------------------------------------- /MoRe4D/utils/lora_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/MoRe4D/utils/lora_utils.py -------------------------------------------------------------------------------- /MoRe4D/utils/project_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/MoRe4D/utils/project_utils.py -------------------------------------------------------------------------------- /MoRe4D/utils/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/MoRe4D/utils/utils.py -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/README.md -------------------------------------------------------------------------------- /config/wan2.1/wan_civital.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/config/wan2.1/wan_civital.yaml -------------------------------------------------------------------------------- /config/wan2.1/wan_civital_4D_STraG.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/config/wan2.1/wan_civital_4D_STraG.yaml -------------------------------------------------------------------------------- /config/zero_stage2_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/config/zero_stage2_config.json -------------------------------------------------------------------------------- /config/zero_stage3_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/config/zero_stage3_config.json -------------------------------------------------------------------------------- /config/zero_stage3_config_cpu_offload.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/config/zero_stage3_config_cpu_offload.json -------------------------------------------------------------------------------- /models/put_your_models_here.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/requirements.txt -------------------------------------------------------------------------------- /scripts/4D_STraG_training/train_vae.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/scripts/4D_STraG_training/train_vae.py -------------------------------------------------------------------------------- /scripts/4D_STraG_training/train_vae.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/scripts/4D_STraG_training/train_vae.sh -------------------------------------------------------------------------------- /scripts/4D_STraG_training/train_wan.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/scripts/4D_STraG_training/train_wan.py -------------------------------------------------------------------------------- /scripts/4D_STraG_training/train_wan.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/scripts/4D_STraG_training/train_wan.sh -------------------------------------------------------------------------------- /scripts/4D_ViSM_training/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/scripts/4D_ViSM_training/train.py -------------------------------------------------------------------------------- /scripts/4D_ViSM_training/train.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/scripts/4D_ViSM_training/train.sh -------------------------------------------------------------------------------- /scripts/inference/infer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/scripts/inference/infer.py -------------------------------------------------------------------------------- /scripts/inference/infer.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/scripts/inference/infer.sh -------------------------------------------------------------------------------- /scripts/inference/infer_vae.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/scripts/inference/infer_vae.py -------------------------------------------------------------------------------- /scripts/inference/infer_vae.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/scripts/inference/infer_vae.sh -------------------------------------------------------------------------------- /scripts/zero_to_bf16.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/scripts/zero_to_bf16.py -------------------------------------------------------------------------------- /static/demo_videos/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/static/demo_videos/.DS_Store -------------------------------------------------------------------------------- /static/demo_videos/bear/4D-STraG_output/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/static/demo_videos/bear/4D-STraG_output/.DS_Store -------------------------------------------------------------------------------- /static/demo_videos/bear/4D-STraG_output/bear_render.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/static/demo_videos/bear/4D-STraG_output/bear_render.mp4 -------------------------------------------------------------------------------- /static/demo_videos/bear/4D-ViSM_output/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/static/demo_videos/bear/4D-ViSM_output/.DS_Store -------------------------------------------------------------------------------- /static/demo_videos/bear/4D-ViSM_output/1.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/static/demo_videos/bear/4D-ViSM_output/1.mp4 -------------------------------------------------------------------------------- /static/demo_videos/bear/Input.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/static/demo_videos/bear/Input.png -------------------------------------------------------------------------------- /static/demo_videos/camel/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/static/demo_videos/camel/.DS_Store -------------------------------------------------------------------------------- /static/demo_videos/camel/4D-STraG_output/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/static/demo_videos/camel/4D-STraG_output/.DS_Store -------------------------------------------------------------------------------- /static/demo_videos/camel/4D-STraG_output/camel_render.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/static/demo_videos/camel/4D-STraG_output/camel_render.mp4 -------------------------------------------------------------------------------- /static/demo_videos/camel/4D-ViSM_output/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/static/demo_videos/camel/4D-ViSM_output/.DS_Store -------------------------------------------------------------------------------- /static/demo_videos/camel/4D-ViSM_output/1.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/static/demo_videos/camel/4D-ViSM_output/1.mp4 -------------------------------------------------------------------------------- /static/demo_videos/camel/Input.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/static/demo_videos/camel/Input.png -------------------------------------------------------------------------------- /static/images/dataset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/static/images/dataset.png -------------------------------------------------------------------------------- /static/images/pipeline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/static/images/pipeline.png -------------------------------------------------------------------------------- /static/images/qualitative_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/static/images/qualitative_1.png -------------------------------------------------------------------------------- /static/images/qualitative_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/static/images/qualitative_2.png -------------------------------------------------------------------------------- /static/images/qualitative_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/static/images/qualitative_3.png -------------------------------------------------------------------------------- /static/images/quantitative.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/static/images/quantitative.png -------------------------------------------------------------------------------- /static/images/teaser.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zhangyr2022/MoRe4D/HEAD/static/images/teaser.png --------------------------------------------------------------------------------