├── .gitattributes ├── .gitignore ├── DragAMove-test-batches ├── 00000.gif ├── 00000.pkl ├── 00000.png ├── 00001.gif ├── 00001.pkl ├── 00001.png ├── 00002.gif ├── 00002.pkl ├── 00002.png ├── 00003.gif ├── 00003.pkl ├── 00003.png ├── 00004.gif ├── 00004.pkl ├── 00004.png ├── 00005.gif ├── 00005.pkl ├── 00005.png ├── 00006.gif ├── 00006.pkl ├── 00006.png ├── 00007.gif ├── 00007.pkl ├── 00007.png ├── 00008.gif ├── 00008.pkl ├── 00008.png ├── 00009.gif ├── 00009.pkl ├── 00009.png ├── 00010.gif ├── 00010.pkl ├── 00010.png ├── 00011.gif ├── 00011.pkl ├── 00011.png ├── 00012.gif ├── 00012.pkl ├── 00012.png ├── 00013.gif ├── 00013.pkl ├── 00013.png ├── 00014.gif ├── 00014.pkl ├── 00014.png ├── 00015.gif ├── 00015.pkl ├── 00015.png ├── 00016.gif ├── 00016.pkl ├── 00016.png ├── 00017.gif ├── 00017.pkl ├── 00017.png ├── 00018.gif ├── 00018.pkl ├── 00018.png ├── 00019.gif ├── 00019.pkl ├── 00019.png ├── 00020.gif ├── 00020.pkl ├── 00020.png ├── 00021.gif ├── 00021.pkl ├── 00021.png ├── 00022.gif ├── 00022.pkl ├── 00022.png ├── 00023.gif ├── 00023.pkl ├── 00023.png ├── 00024.gif ├── 00024.pkl ├── 00024.png ├── 00025.gif ├── 00025.pkl ├── 00025.png ├── 00026.gif ├── 00026.pkl ├── 00026.png ├── 00027.gif ├── 00027.pkl ├── 00027.png ├── 00028.gif ├── 00028.pkl ├── 00028.png ├── 00029.gif ├── 00029.pkl ├── 00029.png ├── 00030.gif ├── 00030.pkl ├── 00030.png ├── 00031.gif ├── 00031.pkl ├── 00031.png ├── 00032.gif ├── 00032.pkl ├── 00032.png ├── 00033.gif ├── 00033.pkl ├── 00033.png ├── 00034.gif ├── 00034.pkl ├── 00034.png ├── 00035.gif ├── 00035.pkl ├── 00035.png ├── 00036.gif ├── 00036.pkl ├── 00036.png ├── 00037.gif ├── 00037.pkl ├── 00037.png ├── 00038.gif ├── 00038.pkl ├── 00038.png ├── 00039.gif ├── 00039.pkl ├── 00039.png ├── 00040.gif ├── 00040.pkl ├── 00040.png ├── 00041.gif ├── 00041.pkl ├── 00041.png ├── 00042.gif ├── 00042.pkl ├── 00042.png ├── 00043.gif ├── 00043.pkl ├── 00043.png ├── 00044.gif ├── 00044.pkl ├── 00044.png ├── 00045.gif ├── 00045.pkl ├── 00045.png ├── 00046.gif ├── 00046.pkl ├── 00046.png ├── 00047.gif ├── 00047.pkl ├── 00047.png ├── 00048.gif ├── 00048.pkl ├── 00048.png ├── 00049.gif ├── 00049.pkl ├── 00049.png ├── 00050.gif ├── 00050.pkl ├── 00050.png ├── 00051.gif ├── 00051.pkl ├── 00051.png ├── 00052.gif ├── 00052.pkl ├── 00052.png ├── 00053.gif ├── 00053.pkl ├── 00053.png ├── 00054.gif ├── 00054.pkl ├── 00054.png ├── 00055.gif ├── 00055.pkl ├── 00055.png ├── 00056.gif ├── 00056.pkl ├── 00056.png ├── 00057.gif ├── 00057.pkl ├── 00057.png ├── 00058.gif ├── 00058.pkl ├── 00058.png ├── 00059.gif ├── 00059.pkl ├── 00059.png ├── 00060.gif ├── 00060.pkl ├── 00060.png ├── 00061.gif ├── 00061.pkl ├── 00061.png ├── 00062.gif ├── 00062.pkl ├── 00062.png ├── 00063.gif ├── 00063.pkl ├── 00063.png ├── 00064.gif ├── 00064.pkl ├── 00064.png ├── 00065.gif ├── 00065.pkl ├── 00065.png ├── 00066.gif ├── 00066.pkl ├── 00066.png ├── 00067.gif ├── 00067.pkl ├── 00067.png ├── 00068.gif ├── 00068.pkl ├── 00068.png ├── 00069.gif ├── 00069.pkl ├── 00069.png ├── 00070.gif ├── 00070.pkl ├── 00070.png ├── 00071.gif ├── 00071.pkl ├── 00071.png ├── 00072.gif ├── 00072.pkl ├── 00072.png ├── 00073.gif ├── 00073.pkl ├── 00073.png ├── 00074.gif ├── 00074.pkl ├── 00074.png ├── 00075.gif ├── 00075.pkl ├── 00075.png ├── 00076.gif ├── 00076.pkl ├── 00076.png ├── 00077.gif ├── 00077.pkl ├── 00077.png ├── 00078.gif ├── 00078.pkl ├── 00078.png ├── 00079.gif ├── 00079.pkl ├── 00079.png ├── 00080.gif ├── 00080.pkl ├── 00080.png ├── 00081.gif ├── 00081.pkl ├── 00081.png ├── 00082.gif ├── 00082.pkl ├── 00082.png ├── 00083.gif ├── 00083.pkl ├── 00083.png ├── 00084.gif ├── 00084.pkl ├── 00084.png ├── 00085.gif ├── 00085.pkl ├── 00085.png ├── 00086.gif ├── 00086.pkl ├── 00086.png ├── 00087.gif ├── 00087.pkl ├── 00087.png ├── 00088.gif ├── 00088.pkl ├── 00088.png ├── 00089.gif ├── 00089.pkl ├── 00089.png ├── 00090.gif ├── 00090.pkl ├── 00090.png ├── 00091.gif ├── 00091.pkl ├── 00091.png ├── 00092.gif ├── 00092.pkl ├── 00092.png ├── 00093.gif ├── 00093.pkl ├── 00093.png ├── 00094.gif ├── 00094.pkl ├── 00094.png ├── 00095.gif ├── 00095.pkl ├── 00095.png ├── 00096.gif ├── 00096.pkl ├── 00096.png ├── 00097.gif ├── 00097.pkl ├── 00097.png ├── 00098.gif ├── 00098.pkl ├── 00098.png ├── 00099.gif ├── 00099.pkl └── 00099.png ├── README.md ├── configs └── train-puppet-master.yaml ├── data ├── README.md ├── drag-sampling │ ├── render │ │ ├── light.py │ │ ├── material.py │ │ ├── mesh.py │ │ ├── mlptexture.py │ │ ├── obj.py │ │ ├── regularizer.py │ │ ├── render.py │ │ ├── renderutils │ │ │ ├── __init__.py │ │ │ ├── bsdf.py │ │ │ ├── c_src │ │ │ │ ├── bsdf.cu │ │ │ │ ├── bsdf.h │ │ │ │ ├── common.cpp │ │ │ │ ├── common.h │ │ │ │ ├── cubemap.cu │ │ │ │ ├── cubemap.h │ │ │ │ ├── loss.cu │ │ │ │ ├── loss.h │ │ │ │ ├── mesh.cu │ │ │ │ ├── mesh.h │ │ │ │ ├── normal.cu │ │ │ │ ├── normal.h │ │ │ │ ├── tensor.h │ │ │ │ ├── torch_bindings.cpp │ │ │ │ ├── vec3f.h │ │ │ │ └── vec4f.h │ │ │ ├── loss.py │ │ │ ├── ops.py │ │ │ └── tests │ │ │ │ ├── test_bsdf.py │ │ │ │ ├── test_cubemap.py │ │ │ │ ├── test_loss.py │ │ │ │ ├── test_mesh.py │ │ │ │ └── test_perf.py │ │ ├── texture.py │ │ └── util.py │ └── sample_drags.py ├── examples.zip ├── objaverse-animation-HQ.json ├── objaverse-animation.json ├── rendering │ └── blender_render_animation.py └── svd-cache │ ├── compute_latents.py │ └── extract_embeddings.py ├── dataset.py ├── network_utils.py ├── networks.py ├── requirements.txt ├── train.py └── utils.py /.gitattributes: -------------------------------------------------------------------------------- 1 | *.zip filter=lfs diff=lfs merge=lfs -text 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | package-lock.json 2 | *.DS_Store 3 | __pycache__/ 4 | runs/ -------------------------------------------------------------------------------- /DragAMove-test-batches/00000.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00000.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00000.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00000.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00000.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00000.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00001.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00001.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00001.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00001.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00001.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00001.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00002.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00002.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00002.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00002.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00002.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00002.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00003.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00003.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00003.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00003.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00003.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00003.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00004.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00004.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00004.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00004.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00004.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00004.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00005.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00005.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00005.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00005.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00005.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00005.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00006.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00006.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00006.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00006.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00006.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00006.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00007.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00007.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00007.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00007.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00007.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00007.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00008.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00008.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00008.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00008.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00008.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00008.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00009.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00009.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00009.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00009.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00009.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00009.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00010.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00010.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00010.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00010.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00010.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00010.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00011.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00011.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00011.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00011.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00011.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00011.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00012.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00012.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00012.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00012.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00012.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00012.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00013.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00013.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00013.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00013.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00013.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00013.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00014.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00014.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00014.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00014.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00014.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00014.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00015.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00015.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00015.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00015.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00015.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00015.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00016.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00016.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00016.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00016.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00016.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00016.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00017.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00017.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00017.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00017.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00017.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00017.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00018.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00018.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00018.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00018.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00018.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00018.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00019.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00019.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00019.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00019.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00019.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00019.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00020.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00020.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00020.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00020.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00020.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00020.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00021.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00021.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00021.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00021.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00021.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00021.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00022.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00022.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00022.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00022.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00022.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00022.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00023.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00023.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00023.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00023.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00023.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00023.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00024.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00024.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00024.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00024.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00024.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00024.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00025.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00025.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00025.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00025.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00025.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00025.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00026.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00026.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00026.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00026.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00026.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00026.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00027.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00027.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00027.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00027.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00027.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00027.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00028.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00028.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00028.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00028.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00028.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00028.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00029.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00029.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00029.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00029.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00029.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00029.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00030.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00030.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00030.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00030.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00030.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00030.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00031.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00031.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00031.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00031.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00031.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00031.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00032.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00032.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00032.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00032.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00032.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00032.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00033.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00033.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00033.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00033.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00033.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00033.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00034.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00034.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00034.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00034.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00034.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00034.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00035.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00035.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00035.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00035.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00035.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00035.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00036.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00036.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00036.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00036.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00036.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00036.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00037.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00037.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00037.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00037.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00037.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00037.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00038.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00038.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00038.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00038.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00038.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00038.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00039.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00039.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00039.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00039.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00039.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00039.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00040.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00040.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00040.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00040.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00040.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00040.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00041.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00041.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00041.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00041.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00041.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00041.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00042.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00042.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00042.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00042.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00042.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00042.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00043.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00043.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00043.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00043.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00043.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00043.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00044.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00044.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00044.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00044.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00044.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00044.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00045.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00045.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00045.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00045.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00045.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00045.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00046.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00046.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00046.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00046.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00046.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00046.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00047.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00047.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00047.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00047.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00047.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00047.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00048.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00048.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00048.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00048.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00048.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00048.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00049.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00049.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00049.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00049.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00049.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00049.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00050.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00050.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00050.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00050.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00050.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00050.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00051.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00051.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00051.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00051.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00051.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00051.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00052.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00052.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00052.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00052.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00052.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00052.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00053.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00053.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00053.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00053.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00053.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00053.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00054.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00054.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00054.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00054.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00054.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00054.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00055.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00055.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00055.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00055.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00055.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00055.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00056.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00056.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00056.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00056.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00056.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00056.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00057.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00057.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00057.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00057.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00057.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00057.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00058.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00058.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00058.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00058.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00058.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00058.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00059.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00059.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00059.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00059.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00059.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00059.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00060.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00060.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00060.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00060.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00060.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00060.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00061.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00061.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00061.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00061.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00061.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00061.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00062.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00062.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00062.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00062.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00062.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00062.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00063.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00063.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00063.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00063.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00063.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00063.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00064.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00064.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00064.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00064.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00064.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00064.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00065.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00065.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00065.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00065.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00065.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00065.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00066.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00066.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00066.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00066.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00066.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00066.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00067.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00067.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00067.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00067.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00067.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00067.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00068.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00068.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00068.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00068.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00068.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00068.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00069.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00069.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00069.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00069.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00069.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00069.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00070.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00070.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00070.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00070.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00070.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00070.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00071.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00071.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00071.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00071.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00071.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00071.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00072.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00072.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00072.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00072.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00072.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00072.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00073.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00073.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00073.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00073.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00073.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00073.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00074.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00074.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00074.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00074.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00074.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00074.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00075.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00075.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00075.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00075.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00075.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00075.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00076.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00076.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00076.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00076.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00076.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00076.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00077.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00077.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00077.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00077.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00077.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00077.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00078.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00078.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00078.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00078.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00078.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00078.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00079.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00079.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00079.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00079.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00079.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00079.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00080.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00080.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00080.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00080.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00080.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00080.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00081.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00081.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00081.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00081.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00081.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00081.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00082.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00082.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00082.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00082.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00082.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00082.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00083.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00083.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00083.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00083.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00083.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00083.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00084.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00084.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00084.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00084.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00084.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00084.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00085.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00085.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00085.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00085.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00085.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00085.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00086.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00086.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00086.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00086.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00086.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00086.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00087.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00087.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00087.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00087.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00087.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00087.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00088.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00088.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00088.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00088.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00088.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00088.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00089.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00089.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00089.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00089.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00089.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00089.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00090.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00090.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00090.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00090.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00090.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00090.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00091.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00091.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00091.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00091.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00091.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00091.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00092.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00092.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00092.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00092.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00092.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00092.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00093.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00093.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00093.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00093.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00093.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00093.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00094.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00094.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00094.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00094.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00094.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00094.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00095.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00095.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00095.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00095.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00095.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00095.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00096.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00096.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00096.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00096.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00096.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00096.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00097.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00097.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00097.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00097.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00097.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00097.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00098.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00098.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00098.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00098.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00098.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00098.png -------------------------------------------------------------------------------- /DragAMove-test-batches/00099.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00099.gif -------------------------------------------------------------------------------- /DragAMove-test-batches/00099.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00099.pkl -------------------------------------------------------------------------------- /DragAMove-test-batches/00099.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RuiningLi/puppet-master/f24d70878463cf2a20e2cd805de6846553908cf5/DragAMove-test-batches/00099.png -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Puppet-Master 2 | Official implementation of 'Puppet-Master: Scaling Interactive Video Generation as a Motion Prior for Part-Level Dynamics' 3 | 4 |

5 | [arXiv] 6 | [Demo] 7 | [Project] 8 | [BibTeX] 9 |

10 | 11 | ### News 12 | - **2024-Sept-25** Training script released. 13 | - **2024-Aug-17** Pre-trained checkpoints and demo released on Hugging Face. Check [here](https://huggingface.co/spaces/rayli/Puppet-Master) for demo and [here](https://huggingface.co/spaces/rayli/Puppet-Master/tree/main) for code. 14 | 15 | ### Examples 16 | 17 | ##### Man-Made Objects 18 | ![Man-Made Objects](https://vgg-puppetmaster.github.io/resources/manmade.gif) 19 | 20 | ##### Animals 21 | ![Animals](https://vgg-puppetmaster.github.io/resources/animal.gif) 22 | 23 | ##### Humans 24 | ![Humans](https://vgg-puppetmaster.github.io/resources/human.gif) 25 | 26 | ### Objaverse-Animation & Objaverse-Animation-HQ 27 | See the `data` folder. 28 | 29 | ### Training 30 | We provide a minimum viable training script to demonstrate how to use our dataset to fine-tune Stable Video Diffusion. 31 | 32 | You can use the following command: 33 | ``` 34 | accelerate launch --num_processes 1 --mixed_precision fp16 train.py --config configs/train-puppet-master.yaml 35 | ``` 36 | 37 | To reduce the memory overhead, we cache all the latents and CLIP embeddings of the rendered frames. 38 | 39 | Note this is only a working example. Our final model is trained using a combined dataset of Objaverse-Animation-HQ and [Drag-a-Move](https://github.com/RuiningLi/DragAPart/tree/main/Drag-a-Move). 40 | 41 | ### Inference 42 | We provide an interactive demo [here](https://huggingface.co/spaces/rayli/Puppet-Master). Check it out! 43 | 44 | ### Evaluation 45 | Our evaluation utilizes an unseen test set of [Drag-a-Move](https://github.com/RuiningLi/DragAPart/tree/main/Drag-a-Move), consisting of 100 examples. 46 | The whole test set is provided in `DragAMove-test-batches` folder. 47 | The test examples can be read directly from the `xxxxx.pkl` files and are in the same format as those loaded from `DragVideoDataset` implemented in `dataset.py`. 48 | 49 | ### TODO 50 | - [x] Release pre-trained checkpoint & inference code. 51 | - [x] Release training code. 52 | - [x] Release Objaverse-Animation & Objaverse-Animation rendering script. 53 | 54 | ### Citation 55 | 56 | ``` 57 | @article{li2024puppetmaster, 58 | title = {Puppet-Master: Scaling Interactive Video Generation as a Motion Prior for Part-Level Dynamics}, 59 | author = {Li, Ruining and Zheng, Chuanxia and Rupprecht, Christian and Vedaldi, Andrea}, 60 | journal = {arXiv preprint arXiv:2408.04631}, 61 | year = {2024} 62 | } 63 | ``` 64 | -------------------------------------------------------------------------------- /configs/train-puppet-master.yaml: -------------------------------------------------------------------------------- 1 | pretrained_model_name_or_local_dir: stabilityai/stable-video-diffusion-img2vid 2 | 3 | results_dir: ./runs 4 | num_max_drags: 5 5 | test_dir: ./DragAMove-test-batches 6 | 7 | random_seed: 1024 8 | num_steps: 1500000 9 | global_batch_size: 1 10 | num_workers: 2 11 | log_every: 50 12 | visualize_every: 200 13 | ckpt_every: 50000 14 | vae: "ema" 15 | learning_rate: 1.e-5 16 | 17 | log_sigma_std: 1.6 18 | log_sigma_mean: 0.7 19 | 20 | zero_init: true 21 | 22 | model_args: 23 | cond_dropout_prob: 0.1 24 | drag_token_cross_attn: true 25 | use_modulate: true 26 | pos_embed_dim: 64 27 | drag_embedder_out_channels: [256, 320, 320] 28 | 29 | dataset_args: 30 | latent_dist_roots: /pattern/of/all/precomputed/latents/files/at/the/object/level/*/* 31 | embedding_roots: /pattern/of/all/precomputed/embedding/files/at/the/object/level/*/* 32 | drag_roots: /pattern/of/all/precomputed/drag/samples/at/the/object/level/*/* 33 | num_max_drags: 5 34 | 35 | non_first_frame_weight: 1 36 | weight_increasing: true 37 | enable_gradient_checkpointing: true 38 | gradient_accumulation_steps: 64 39 | max_grad_norm: 1 -------------------------------------------------------------------------------- /data/README.md: -------------------------------------------------------------------------------- 1 | # Objaverse-Animation & Objaverse-Animation-HQ 2 | 3 | ![Dataset](https://vgg-puppetmaster.github.io/resources/data.png) 4 | 5 | ### Filtering 6 | The `objaverse-animation.json` and `objaverse-animation-HQ.json` provide the IDs of the animated 3D models in Objaverse. 7 | 8 | ### Rendering 9 | 1. The rendering script is tested on Linux with Blender 3.2.2: 10 | ``` 11 | wget https://download.blender.org/release/Blender3.2/blender-3.2.2-linux-x64.tar.xz && \ 12 | tar -xf blender-3.2.2-linux-x64.tar.xz && \ 13 | rm blender-3.2.2-linux-x64.tar.xz 14 | ``` 15 | 16 | 2. Download the Objaverse dataset ([instructions](https://objaverse.allenai.org/)). 17 | 18 | 3. Use the following command 19 | ``` 20 | /path/to/blender-3.2.2-linux-x64/blender -noaudio -b -P blender_render_animation.py -- --object_path /path/to/glb --output_dir /path/to/output/directory --only_northern_hemisphere --engine CYCLES --num_renders 12 --max_n_frames 32 --uniform_azimuth --render 21 | ``` 22 | 23 | ### Exporting Meshes 24 | The script also allows you to export a sequence of vertex-aligned .obj meshes for each animated 3D model, which can be useful for other applications: 25 | ``` 26 | /path/to/blender-3.2.2-linux-x64/blender -noaudio -b -P blender_render_animation.py -- --object_path /path/to/glb --output_dir /path/to/output/directory --only_northern_hemisphere --engine CYCLES --max_n_frames 32 --export_mesh 27 | ``` 28 | 29 | ### Sampling drags 30 | 0. Before proceeding, make sure you have the animations rendered and mesh exported. We use the mesh to sample 3D trajectories and use the camera matrices saved from rendering to project the trajectories to the image space. 31 | 32 | 1. Use the following command 33 | ``` 34 | python sample_drags.py --render_root /path/to/the/render/root --save_dir /path/to/the/dir/to/save/the/results --num_renders 12 --render_part_mask --sample_drags --num_samples 20 --visualize 35 | ``` 36 | 37 | Notes: 38 | - The `--render_root` is the `output_dir` specified in blender rendering. 39 | - The `--num_renders` value should be consistent with the one used in blender rendering. 40 | - `--visualize` flag should be disabled for efficiency concern. 41 | 42 | ### Pre-Computing Image Embeddings & VAE Latents 43 | 44 | Please refer to the scripts under `svd-cache` folder. -------------------------------------------------------------------------------- /data/drag-sampling/render/material.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # 3 | # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 4 | # property and proprietary rights in and to this material, related 5 | # documentation and any modifications thereto. Any use, reproduction, 6 | # disclosure or distribution of this material and related documentation 7 | # without an express license agreement from NVIDIA CORPORATION or 8 | # its affiliates is strictly prohibited. 9 | 10 | import os 11 | import numpy as np 12 | import torch 13 | 14 | from . import util 15 | from . import texture 16 | 17 | ###################################################################################### 18 | # Wrapper to make materials behave like a python dict, but register textures as 19 | # torch.nn.Module parameters. 20 | ###################################################################################### 21 | class Material(torch.nn.Module): 22 | def __init__(self, mat_dict): 23 | super(Material, self).__init__() 24 | self.mat_keys = set() 25 | for key in mat_dict.keys(): 26 | self.mat_keys.add(key) 27 | self[key] = mat_dict[key] 28 | 29 | def __contains__(self, key): 30 | return hasattr(self, key) 31 | 32 | def __getitem__(self, key): 33 | return getattr(self, key) 34 | 35 | def __setitem__(self, key, val): 36 | self.mat_keys.add(key) 37 | setattr(self, key, val) 38 | 39 | def __delitem__(self, key): 40 | self.mat_keys.remove(key) 41 | delattr(self, key) 42 | 43 | def keys(self): 44 | return self.mat_keys 45 | 46 | ###################################################################################### 47 | # .mtl material format loading / storing 48 | ###################################################################################### 49 | @torch.no_grad() 50 | def load_mtl(fn, clear_ks=True, device="cuda"): 51 | import re 52 | mtl_path = os.path.dirname(fn) 53 | 54 | # Read file 55 | with open(fn, 'r') as f: 56 | lines = f.readlines() 57 | 58 | # Parse materials 59 | materials = [] 60 | for line in lines: 61 | split_line = re.split(' +|\t+|\n+', line.strip()) 62 | prefix = split_line[0].lower() 63 | data = split_line[1:] 64 | if 'newmtl' in prefix: 65 | material = Material({'name' : data[0]}) 66 | materials += [material] 67 | elif materials: 68 | if 'bsdf' in prefix or 'map_kd' in prefix or 'map_ks' in prefix or 'bump' in prefix: 69 | material[prefix] = data[0] 70 | else: 71 | material[prefix] = torch.tensor(tuple(float(d) for d in data), dtype=torch.float32, device=device) 72 | 73 | # Convert everything to textures. Our code expects 'kd' and 'ks' to be texture maps. So replace constants with 1x1 maps 74 | for mat in materials: 75 | if not 'bsdf' in mat: 76 | mat['bsdf'] = 'pbr' 77 | 78 | if 'map_kd' in mat: 79 | mat['kd'] = texture.load_texture2D(os.path.join(mtl_path, mat['map_kd']), channels=3) 80 | else: 81 | mat['kd'] = texture.Texture2D(mat['kd']) 82 | 83 | if 'map_ks' in mat: 84 | mat['ks'] = texture.load_texture2D(os.path.join(mtl_path, mat['map_ks']), channels=3) 85 | elif 'ks' in mat: 86 | mat['ks'] = texture.Texture2D(mat['ks']) 87 | 88 | if "map_ka" in mat: 89 | mat["ka"] = texture.load_texture2D(os.path.join(mtl_path, mat["map_ka"])) 90 | elif "ka" in mat: 91 | mat["ka"] = texture.Texture2D(mat["ka"]) 92 | 93 | if 'bump' in mat: 94 | mat['normal'] = texture.load_texture2D(os.path.join(mtl_path, mat['bump']), lambda_fn=lambda x: x * 2 - 1, channels=3) 95 | 96 | # Convert Kd from sRGB to linear RGB 97 | # mat['kd'] = texture.srgb_to_rgb(mat['kd']) 98 | 99 | if clear_ks and 'ks' in mat: 100 | # Override ORM occlusion (red) channel by zeros. We hijack this channel 101 | for mip in mat['ks'].getMips(): 102 | mip[..., 0] = 0.0 103 | 104 | return materials 105 | 106 | @torch.no_grad() 107 | def save_mtl(fn, material): 108 | folder = os.path.dirname(fn) 109 | with open(fn, "w") as f: 110 | f.write('newmtl defaultMat\n') 111 | if material is not None: 112 | f.write('bsdf %s\n' % material['bsdf']) 113 | if 'kd' in material.keys(): 114 | f.write('map_Kd texture_kd.png\n') 115 | texture.save_texture2D(os.path.join(folder, 'texture_kd.png'), texture.rgb_to_srgb(material['kd'])) 116 | if 'ks' in material.keys(): 117 | f.write('map_Ks texture_ks.png\n') 118 | texture.save_texture2D(os.path.join(folder, 'texture_ks.png'), material['ks']) 119 | if 'normal' in material.keys(): 120 | f.write('bump texture_n.png\n') 121 | texture.save_texture2D(os.path.join(folder, 'texture_n.png'), material['normal'], lambda_fn=lambda x:(util.safe_normalize(x)+1)*0.5) 122 | else: 123 | f.write('Kd 1 1 1\n') 124 | f.write('Ks 0 0 0\n') 125 | f.write('Ka 0 0 0\n') 126 | f.write('Tf 1 1 1\n') 127 | f.write('Ni 1\n') 128 | f.write('Ns 0\n') 129 | 130 | ###################################################################################### 131 | # Merge multiple materials into a single uber-material 132 | ###################################################################################### 133 | 134 | def _upscale_replicate(x, full_res, tex=""): 135 | x = x.permute(0, 3, 1, 2) 136 | x = torch.nn.functional.pad(x, (0, full_res[1] - x.shape[3], 0, full_res[0] - x.shape[2]), 'replicate') 137 | ret = x.permute(0, 2, 3, 1).contiguous() 138 | return ret 139 | 140 | def merge_materials(materials, texcoords, tfaces, mfaces): 141 | assert len(materials) > 0 142 | for mat in materials: 143 | assert mat['bsdf'] == materials[0]['bsdf'], "All materials must have the same BSDF (uber shader)" 144 | assert ('normal' in mat) is ('normal' in materials[0]), "All materials must have either normal map enabled or disabled" 145 | 146 | uber_material = Material({ 147 | 'name' : 'uber_material', 148 | 'bsdf' : materials[0]['bsdf'], 149 | }) 150 | 151 | textures = ['kd', 'ks', 'normal', 'ka'] 152 | 153 | # Find maximum texture resolution across all materials and textures 154 | max_res = None 155 | for mat in materials: 156 | for tex in textures: 157 | tex_res = np.array(mat[tex].getRes()) if tex in mat else np.array([1, 1]) 158 | max_res = np.maximum(max_res, tex_res) if max_res is not None else tex_res 159 | 160 | # Compute size of compund texture and round up to nearest PoT 161 | full_res = 2**np.ceil(np.log2(max_res * np.array([1, len(materials)]))).astype(np.int) 162 | thresh = 65536 163 | if full_res[1] > thresh: 164 | scale = thresh / full_res[1] 165 | full_res[1] = thresh 166 | max_res[1] = (max_res[1] * scale).astype(np.int) 167 | 168 | # Normalize texture resolution across all materials & combine into a single large texture 169 | for tex in textures: 170 | if tex in materials[0]: 171 | tex_data = torch.cat(tuple(util.scale_img_nhwc(mat[tex].data, tuple(max_res)) for mat in materials), dim=2) # Lay out all textures horizontally, NHWC so dim2 is x 172 | tex_data = _upscale_replicate(tex_data, full_res, tex) 173 | uber_material[tex] = texture.Texture2D(tex_data) 174 | 175 | # Compute scaling values for used / unused texture area 176 | s_coeff = [full_res[0] / max_res[0], full_res[1] / max_res[1]] 177 | 178 | # Recompute texture coordinates to cooincide with new composite texture 179 | new_tverts = {} 180 | new_tverts_data = [] 181 | for fi in range(len(tfaces)): 182 | matIdx = mfaces[fi] 183 | if matIdx is None: 184 | matIdx = 0 185 | for vi in range(3): 186 | ti = tfaces[fi][vi] 187 | if not (ti in new_tverts): 188 | new_tverts[ti] = {} 189 | if not (matIdx in new_tverts[ti]): # create new vertex 190 | if ti >= 0: 191 | new_tverts_data.append([(matIdx + float(texcoords[ti][0]) % 1) / s_coeff[1], float(texcoords[ti][1]) % 1 / s_coeff[0]]) # Offset texture coodrinate (x direction) by material id & scale to local space. Note, texcoords are (u,v) but texture is stored (w,h) so the indexes swap here 192 | else: 193 | new_tverts_data.append([(matIdx + 0.5) / s_coeff[1], 0.5 / s_coeff[0]]) 194 | new_tverts[ti][matIdx] = len(new_tverts_data) - 1 195 | tfaces[fi][vi] = new_tverts[ti][matIdx] # reindex vertex 196 | 197 | return uber_material, new_tverts_data, tfaces 198 | 199 | -------------------------------------------------------------------------------- /data/drag-sampling/render/mlptexture.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # 3 | # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 4 | # property and proprietary rights in and to this material, related 5 | # documentation and any modifications thereto. Any use, reproduction, 6 | # disclosure or distribution of this material and related documentation 7 | # without an express license agreement from NVIDIA CORPORATION or 8 | # its affiliates is strictly prohibited. 9 | 10 | import torch 11 | import tinycudann as tcnn 12 | import numpy as np 13 | 14 | ####################################################################################################################################################### 15 | # Small MLP using PyTorch primitives, internal helper class 16 | ####################################################################################################################################################### 17 | 18 | class _MLP(torch.nn.Module): 19 | def __init__(self, cfg, loss_scale=1.0): 20 | super(_MLP, self).__init__() 21 | self.loss_scale = loss_scale 22 | net = (torch.nn.Linear(cfg['n_input_dims'], cfg['n_neurons'], bias=False), torch.nn.ReLU()) 23 | for i in range(cfg['n_hidden_layers']-1): 24 | net = net + (torch.nn.Linear(cfg['n_neurons'], cfg['n_neurons'], bias=False), torch.nn.ReLU()) 25 | net = net + (torch.nn.Linear(cfg['n_neurons'], cfg['n_output_dims'], bias=False),) 26 | self.net = torch.nn.Sequential(*net).cuda() 27 | 28 | self.net.apply(self._init_weights) 29 | 30 | if self.loss_scale != 1.0: 31 | self.net.register_full_backward_hook(lambda module, grad_i, grad_o: (grad_i[0] * self.loss_scale, )) 32 | 33 | def forward(self, x): 34 | return self.net(x.to(torch.float32)) 35 | 36 | @staticmethod 37 | def _init_weights(m): 38 | if type(m) == torch.nn.Linear: 39 | torch.nn.init.kaiming_uniform_(m.weight, nonlinearity='relu') 40 | if hasattr(m.bias, 'data'): 41 | m.bias.data.fill_(0.0) 42 | 43 | ####################################################################################################################################################### 44 | # Outward visible MLP class 45 | ####################################################################################################################################################### 46 | 47 | class MLPTexture3D(torch.nn.Module): 48 | def __init__(self, AABB, channels = 3, internal_dims = 32, hidden = 2, min_max = None): 49 | super(MLPTexture3D, self).__init__() 50 | 51 | self.channels = channels 52 | self.internal_dims = internal_dims 53 | self.AABB = AABB 54 | self.min_max = min_max 55 | 56 | # Setup positional encoding, see https://github.com/NVlabs/tiny-cuda-nn for details 57 | desired_resolution = 4096 58 | base_grid_resolution = 16 59 | num_levels = 16 60 | per_level_scale = np.exp(np.log(desired_resolution / base_grid_resolution) / (num_levels-1)) 61 | 62 | enc_cfg = { 63 | "otype": "HashGrid", 64 | "n_levels": num_levels, 65 | "n_features_per_level": 2, 66 | "log2_hashmap_size": 19, 67 | "base_resolution": base_grid_resolution, 68 | "per_level_scale" : per_level_scale 69 | } 70 | 71 | gradient_scaling = 128.0 72 | self.encoder = tcnn.Encoding(3, enc_cfg) 73 | self.encoder.register_full_backward_hook(lambda module, grad_i, grad_o: (grad_i[0] / gradient_scaling, )) 74 | 75 | # Setup MLP 76 | mlp_cfg = { 77 | "n_input_dims" : self.encoder.n_output_dims, 78 | "n_output_dims" : self.channels, 79 | "n_hidden_layers" : hidden, 80 | "n_neurons" : self.internal_dims 81 | } 82 | self.net = _MLP(mlp_cfg, gradient_scaling) 83 | print("Encoder output: %d dims" % (self.encoder.n_output_dims)) 84 | 85 | # Sample texture at a given location 86 | def sample(self, texc): 87 | _texc = (texc.view(-1, 3) - self.AABB[0][None, ...]) / (self.AABB[1][None, ...] - self.AABB[0][None, ...]) 88 | _texc = torch.clamp(_texc, min=0, max=1) 89 | 90 | p_enc = self.encoder(_texc.contiguous()) 91 | out = self.net.forward(p_enc) 92 | 93 | # Sigmoid limit and scale to the allowed range 94 | out = torch.sigmoid(out) * (self.min_max[1][None, :] - self.min_max[0][None, :]) + self.min_max[0][None, :] 95 | 96 | return out.view(*texc.shape[:-1], self.channels) # Remap to [n, h, w, c] 97 | 98 | # In-place clamp with no derivative to make sure values are in valid range after training 99 | def clamp_(self): 100 | pass 101 | 102 | def cleanup(self): 103 | tcnn.free_temporary_memory() 104 | -------------------------------------------------------------------------------- /data/drag-sampling/render/regularizer.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # 3 | # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 4 | # property and proprietary rights in and to this material, related 5 | # documentation and any modifications thereto. Any use, reproduction, 6 | # disclosure or distribution of this material and related documentation 7 | # without an express license agreement from NVIDIA CORPORATION or 8 | # its affiliates is strictly prohibited. 9 | 10 | import torch 11 | import nvdiffrast.torch as dr 12 | 13 | from . import util 14 | from . import mesh 15 | 16 | ###################################################################################### 17 | # Computes the image gradient, useful for kd/ks smoothness losses 18 | ###################################################################################### 19 | def image_grad(buf, std=0.01): 20 | t, s = torch.meshgrid(torch.linspace(-1.0 + 1.0 / buf.shape[1], 1.0 - 1.0 / buf.shape[1], buf.shape[1], device="cuda"), 21 | torch.linspace(-1.0 + 1.0 / buf.shape[2], 1.0 - 1.0 / buf.shape[2], buf.shape[2], device="cuda"), 22 | indexing='ij') 23 | tc = torch.normal(mean=0, std=std, size=(buf.shape[0], buf.shape[1], buf.shape[2], 2), device="cuda") + torch.stack((s, t), dim=-1)[None, ...] 24 | tap = dr.texture(buf, tc, filter_mode='linear', boundary_mode='clamp') 25 | return torch.abs(tap[..., :-1] - buf[..., :-1]) * tap[..., -1:] * buf[..., -1:] 26 | 27 | ###################################################################################### 28 | # Computes the avergage edge length of a mesh. 29 | # Rough estimate of the tessellation of a mesh. Can be used e.g. to clamp gradients 30 | ###################################################################################### 31 | def avg_edge_length(v_pos, t_pos_idx): 32 | e_pos_idx = mesh.compute_edges(t_pos_idx) 33 | edge_len = util.length(v_pos[e_pos_idx[:, 0]] - v_pos[e_pos_idx[:, 1]]) 34 | return torch.mean(edge_len) 35 | 36 | ###################################################################################### 37 | # Laplacian regularization using umbrella operator (Fujiwara / Desbrun). 38 | # https://mgarland.org/class/geom04/material/smoothing.pdf 39 | ###################################################################################### 40 | def laplace_regularizer_const(v_pos, t_pos_idx): 41 | term = torch.zeros_like(v_pos) 42 | norm = torch.zeros_like(v_pos[..., 0:1]) 43 | 44 | v0 = v_pos[t_pos_idx[:, 0], :] 45 | v1 = v_pos[t_pos_idx[:, 1], :] 46 | v2 = v_pos[t_pos_idx[:, 2], :] 47 | 48 | term.scatter_add_(0, t_pos_idx[:, 0:1].repeat(1,3), (v1 - v0) + (v2 - v0)) 49 | term.scatter_add_(0, t_pos_idx[:, 1:2].repeat(1,3), (v0 - v1) + (v2 - v1)) 50 | term.scatter_add_(0, t_pos_idx[:, 2:3].repeat(1,3), (v0 - v2) + (v1 - v2)) 51 | 52 | two = torch.ones_like(v0) * 2.0 53 | norm.scatter_add_(0, t_pos_idx[:, 0:1], two) 54 | norm.scatter_add_(0, t_pos_idx[:, 1:2], two) 55 | norm.scatter_add_(0, t_pos_idx[:, 2:3], two) 56 | 57 | term = term / torch.clamp(norm, min=1.0) 58 | 59 | return torch.mean(term**2) 60 | 61 | ###################################################################################### 62 | # Smooth vertex normals 63 | ###################################################################################### 64 | def normal_consistency(v_pos, t_pos_idx): 65 | # Compute face normals 66 | v0 = v_pos[t_pos_idx[:, 0], :] 67 | v1 = v_pos[t_pos_idx[:, 1], :] 68 | v2 = v_pos[t_pos_idx[:, 2], :] 69 | 70 | face_normals = util.safe_normalize(torch.cross(v1 - v0, v2 - v0)) 71 | 72 | tris_per_edge = mesh.compute_edge_to_face_mapping(t_pos_idx) 73 | 74 | # Fetch normals for both faces sharind an edge 75 | n0 = face_normals[tris_per_edge[:, 0], :] 76 | n1 = face_normals[tris_per_edge[:, 1], :] 77 | 78 | # Compute error metric based on normal difference 79 | term = torch.clamp(util.dot(n0, n1), min=-1.0, max=1.0) 80 | term = (1.0 - term) * 0.5 81 | 82 | return torch.mean(torch.abs(term)) 83 | -------------------------------------------------------------------------------- /data/drag-sampling/render/renderutils/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # 3 | # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 4 | # property and proprietary rights in and to this material, related 5 | # documentation and any modifications thereto. Any use, reproduction, 6 | # disclosure or distribution of this material and related documentation 7 | # without an express license agreement from NVIDIA CORPORATION or 8 | # its affiliates is strictly prohibited. 9 | 10 | from .ops import xfm_points, xfm_vectors, image_loss, diffuse_cubemap, specular_cubemap, prepare_shading_normal, lambert, frostbite_diffuse, pbr_specular, pbr_bsdf, _fresnel_shlick, _ndf_ggx, _lambda_ggx, _masking_smith 11 | __all__ = ["xfm_vectors", "xfm_points", "image_loss", "diffuse_cubemap","specular_cubemap", "prepare_shading_normal", "lambert", "frostbite_diffuse", "pbr_specular", "pbr_bsdf", "_fresnel_shlick", "_ndf_ggx", "_lambda_ggx", "_masking_smith", ] 12 | -------------------------------------------------------------------------------- /data/drag-sampling/render/renderutils/bsdf.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # 3 | # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 4 | # property and proprietary rights in and to this material, related 5 | # documentation and any modifications thereto. Any use, reproduction, 6 | # disclosure or distribution of this material and related documentation 7 | # without an express license agreement from NVIDIA CORPORATION or 8 | # its affiliates is strictly prohibited. 9 | 10 | import math 11 | import torch 12 | 13 | NORMAL_THRESHOLD = 0.1 14 | 15 | ################################################################################ 16 | # Vector utility functions 17 | ################################################################################ 18 | 19 | def _dot(x, y): 20 | return torch.sum(x*y, -1, keepdim=True) 21 | 22 | def _reflect(x, n): 23 | return 2*_dot(x, n)*n - x 24 | 25 | def _safe_normalize(x): 26 | return torch.nn.functional.normalize(x, dim = -1) 27 | 28 | def _bend_normal(view_vec, smooth_nrm, geom_nrm, two_sided_shading): 29 | # Swap normal direction for backfacing surfaces 30 | if two_sided_shading: 31 | smooth_nrm = torch.where(_dot(geom_nrm, view_vec) > 0, smooth_nrm, -smooth_nrm) 32 | geom_nrm = torch.where(_dot(geom_nrm, view_vec) > 0, geom_nrm, -geom_nrm) 33 | 34 | t = torch.clamp(_dot(view_vec, smooth_nrm) / NORMAL_THRESHOLD, min=0, max=1) 35 | return torch.lerp(geom_nrm, smooth_nrm, t) 36 | 37 | 38 | def _perturb_normal(perturbed_nrm, smooth_nrm, smooth_tng, opengl): 39 | smooth_bitang = _safe_normalize(torch.cross(smooth_tng, smooth_nrm)) 40 | if opengl: 41 | shading_nrm = smooth_tng * perturbed_nrm[..., 0:1] - smooth_bitang * perturbed_nrm[..., 1:2] + smooth_nrm * torch.clamp(perturbed_nrm[..., 2:3], min=0.0) 42 | else: 43 | shading_nrm = smooth_tng * perturbed_nrm[..., 0:1] + smooth_bitang * perturbed_nrm[..., 1:2] + smooth_nrm * torch.clamp(perturbed_nrm[..., 2:3], min=0.0) 44 | return _safe_normalize(shading_nrm) 45 | 46 | def bsdf_prepare_shading_normal(pos, view_pos, perturbed_nrm, smooth_nrm, smooth_tng, geom_nrm, two_sided_shading, opengl): 47 | smooth_nrm = _safe_normalize(smooth_nrm) 48 | smooth_tng = _safe_normalize(smooth_tng) 49 | view_vec = _safe_normalize(view_pos - pos) 50 | shading_nrm = _perturb_normal(perturbed_nrm, smooth_nrm, smooth_tng, opengl) 51 | return _bend_normal(view_vec, shading_nrm, geom_nrm, two_sided_shading) 52 | 53 | ################################################################################ 54 | # Simple lambertian diffuse BSDF 55 | ################################################################################ 56 | 57 | def bsdf_lambert(nrm, wi): 58 | return torch.clamp(_dot(nrm, wi), min=0.0) / math.pi 59 | 60 | ################################################################################ 61 | # Frostbite diffuse 62 | ################################################################################ 63 | 64 | def bsdf_frostbite(nrm, wi, wo, linearRoughness): 65 | wiDotN = _dot(wi, nrm) 66 | woDotN = _dot(wo, nrm) 67 | 68 | h = _safe_normalize(wo + wi) 69 | wiDotH = _dot(wi, h) 70 | 71 | energyBias = 0.5 * linearRoughness 72 | energyFactor = 1.0 - (0.51 / 1.51) * linearRoughness 73 | f90 = energyBias + 2.0 * wiDotH * wiDotH * linearRoughness 74 | f0 = 1.0 75 | 76 | wiScatter = bsdf_fresnel_shlick(f0, f90, wiDotN) 77 | woScatter = bsdf_fresnel_shlick(f0, f90, woDotN) 78 | res = wiScatter * woScatter * energyFactor 79 | return torch.where((wiDotN > 0.0) & (woDotN > 0.0), res, torch.zeros_like(res)) 80 | 81 | ################################################################################ 82 | # Phong specular, loosely based on mitsuba implementation 83 | ################################################################################ 84 | 85 | def bsdf_phong(nrm, wo, wi, N): 86 | dp_r = torch.clamp(_dot(_reflect(wo, nrm), wi), min=0.0, max=1.0) 87 | dp_l = torch.clamp(_dot(nrm, wi), min=0.0, max=1.0) 88 | return (dp_r ** N) * dp_l * (N + 2) / (2 * math.pi) 89 | 90 | ################################################################################ 91 | # PBR's implementation of GGX specular 92 | ################################################################################ 93 | 94 | specular_epsilon = 1e-4 95 | 96 | def bsdf_fresnel_shlick(f0, f90, cosTheta): 97 | _cosTheta = torch.clamp(cosTheta, min=specular_epsilon, max=1.0 - specular_epsilon) 98 | return f0 + (f90 - f0) * (1.0 - _cosTheta) ** 5.0 99 | 100 | def bsdf_ndf_ggx(alphaSqr, cosTheta): 101 | _cosTheta = torch.clamp(cosTheta, min=specular_epsilon, max=1.0 - specular_epsilon) 102 | d = (_cosTheta * alphaSqr - _cosTheta) * _cosTheta + 1 103 | return alphaSqr / (d * d * math.pi) 104 | 105 | def bsdf_lambda_ggx(alphaSqr, cosTheta): 106 | _cosTheta = torch.clamp(cosTheta, min=specular_epsilon, max=1.0 - specular_epsilon) 107 | cosThetaSqr = _cosTheta * _cosTheta 108 | tanThetaSqr = (1.0 - cosThetaSqr) / cosThetaSqr 109 | res = 0.5 * (torch.sqrt(1 + alphaSqr * tanThetaSqr) - 1.0) 110 | return res 111 | 112 | def bsdf_masking_smith_ggx_correlated(alphaSqr, cosThetaI, cosThetaO): 113 | lambdaI = bsdf_lambda_ggx(alphaSqr, cosThetaI) 114 | lambdaO = bsdf_lambda_ggx(alphaSqr, cosThetaO) 115 | return 1 / (1 + lambdaI + lambdaO) 116 | 117 | def bsdf_pbr_specular(col, nrm, wo, wi, alpha, min_roughness=0.08): 118 | _alpha = torch.clamp(alpha, min=min_roughness*min_roughness, max=1.0) 119 | alphaSqr = _alpha * _alpha 120 | 121 | h = _safe_normalize(wo + wi) 122 | woDotN = _dot(wo, nrm) 123 | wiDotN = _dot(wi, nrm) 124 | woDotH = _dot(wo, h) 125 | nDotH = _dot(nrm, h) 126 | 127 | D = bsdf_ndf_ggx(alphaSqr, nDotH) 128 | G = bsdf_masking_smith_ggx_correlated(alphaSqr, woDotN, wiDotN) 129 | F = bsdf_fresnel_shlick(col, 1, woDotH) 130 | 131 | w = F * D * G * 0.25 / torch.clamp(woDotN, min=specular_epsilon) 132 | 133 | frontfacing = (woDotN > specular_epsilon) & (wiDotN > specular_epsilon) 134 | return torch.where(frontfacing, w, torch.zeros_like(w)) 135 | 136 | def bsdf_pbr(kd, arm, pos, nrm, view_pos, light_pos, min_roughness, BSDF): 137 | wo = _safe_normalize(view_pos - pos) 138 | wi = _safe_normalize(light_pos - pos) 139 | 140 | spec_str = arm[..., 0:1] # x component 141 | roughness = arm[..., 1:2] # y component 142 | metallic = arm[..., 2:3] # z component 143 | ks = (0.04 * (1.0 - metallic) + kd * metallic) * (1 - spec_str) 144 | kd = kd * (1.0 - metallic) 145 | 146 | if BSDF == 0: 147 | diffuse = kd * bsdf_lambert(nrm, wi) 148 | else: 149 | diffuse = kd * bsdf_frostbite(nrm, wi, wo, roughness) 150 | specular = bsdf_pbr_specular(ks, nrm, wo, wi, roughness*roughness, min_roughness=min_roughness) 151 | return diffuse + specular 152 | -------------------------------------------------------------------------------- /data/drag-sampling/render/renderutils/c_src/bsdf.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 | * 4 | * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 5 | * property and proprietary rights in and to this material, related 6 | * documentation and any modifications thereto. Any use, reproduction, 7 | * disclosure or distribution of this material and related documentation 8 | * without an express license agreement from NVIDIA CORPORATION or 9 | * its affiliates is strictly prohibited. 10 | */ 11 | 12 | #pragma once 13 | 14 | #include "common.h" 15 | 16 | struct LambertKernelParams 17 | { 18 | Tensor nrm; 19 | Tensor wi; 20 | Tensor out; 21 | dim3 gridSize; 22 | }; 23 | 24 | struct FrostbiteDiffuseKernelParams 25 | { 26 | Tensor nrm; 27 | Tensor wi; 28 | Tensor wo; 29 | Tensor linearRoughness; 30 | Tensor out; 31 | dim3 gridSize; 32 | }; 33 | 34 | struct FresnelShlickKernelParams 35 | { 36 | Tensor f0; 37 | Tensor f90; 38 | Tensor cosTheta; 39 | Tensor out; 40 | dim3 gridSize; 41 | }; 42 | 43 | struct NdfGGXParams 44 | { 45 | Tensor alphaSqr; 46 | Tensor cosTheta; 47 | Tensor out; 48 | dim3 gridSize; 49 | }; 50 | 51 | struct MaskingSmithParams 52 | { 53 | Tensor alphaSqr; 54 | Tensor cosThetaI; 55 | Tensor cosThetaO; 56 | Tensor out; 57 | dim3 gridSize; 58 | }; 59 | 60 | struct PbrSpecular 61 | { 62 | Tensor col; 63 | Tensor nrm; 64 | Tensor wo; 65 | Tensor wi; 66 | Tensor alpha; 67 | Tensor out; 68 | dim3 gridSize; 69 | float min_roughness; 70 | }; 71 | 72 | struct PbrBSDF 73 | { 74 | Tensor kd; 75 | Tensor arm; 76 | Tensor pos; 77 | Tensor nrm; 78 | Tensor view_pos; 79 | Tensor light_pos; 80 | Tensor out; 81 | dim3 gridSize; 82 | float min_roughness; 83 | int BSDF; 84 | }; 85 | -------------------------------------------------------------------------------- /data/drag-sampling/render/renderutils/c_src/common.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 | * 4 | * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 5 | * property and proprietary rights in and to this material, related 6 | * documentation and any modifications thereto. Any use, reproduction, 7 | * disclosure or distribution of this material and related documentation 8 | * without an express license agreement from NVIDIA CORPORATION or 9 | * its affiliates is strictly prohibited. 10 | */ 11 | 12 | #include 13 | #include 14 | 15 | //------------------------------------------------------------------------ 16 | // Block and grid size calculators for kernel launches. 17 | 18 | dim3 getLaunchBlockSize(int maxWidth, int maxHeight, dim3 dims) 19 | { 20 | int maxThreads = maxWidth * maxHeight; 21 | if (maxThreads <= 1 || (dims.x * dims.y) <= 1) 22 | return dim3(1, 1, 1); // Degenerate. 23 | 24 | // Start from max size. 25 | int bw = maxWidth; 26 | int bh = maxHeight; 27 | 28 | // Optimizations for weirdly sized buffers. 29 | if (dims.x < bw) 30 | { 31 | // Decrease block width to smallest power of two that covers the buffer width. 32 | while ((bw >> 1) >= dims.x) 33 | bw >>= 1; 34 | 35 | // Maximize height. 36 | bh = maxThreads / bw; 37 | if (bh > dims.y) 38 | bh = dims.y; 39 | } 40 | else if (dims.y < bh) 41 | { 42 | // Halve height and double width until fits completely inside buffer vertically. 43 | while (bh > dims.y) 44 | { 45 | bh >>= 1; 46 | if (bw < dims.x) 47 | bw <<= 1; 48 | } 49 | } 50 | 51 | // Done. 52 | return dim3(bw, bh, 1); 53 | } 54 | 55 | // returns the size of a block that can be reduced using horizontal SIMD operations (e.g. __shfl_xor_sync) 56 | dim3 getWarpSize(dim3 blockSize) 57 | { 58 | return dim3( 59 | std::min(blockSize.x, 32u), 60 | std::min(std::max(32u / blockSize.x, 1u), std::min(32u, blockSize.y)), 61 | std::min(std::max(32u / (blockSize.x * blockSize.y), 1u), std::min(32u, blockSize.z)) 62 | ); 63 | } 64 | 65 | dim3 getLaunchGridSize(dim3 blockSize, dim3 dims) 66 | { 67 | dim3 gridSize; 68 | gridSize.x = (dims.x - 1) / blockSize.x + 1; 69 | gridSize.y = (dims.y - 1) / blockSize.y + 1; 70 | gridSize.z = (dims.z - 1) / blockSize.z + 1; 71 | return gridSize; 72 | } 73 | 74 | //------------------------------------------------------------------------ 75 | -------------------------------------------------------------------------------- /data/drag-sampling/render/renderutils/c_src/common.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 | * 4 | * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 5 | * property and proprietary rights in and to this material, related 6 | * documentation and any modifications thereto. Any use, reproduction, 7 | * disclosure or distribution of this material and related documentation 8 | * without an express license agreement from NVIDIA CORPORATION or 9 | * its affiliates is strictly prohibited. 10 | */ 11 | 12 | #pragma once 13 | #include 14 | #include 15 | 16 | #include "vec3f.h" 17 | #include "vec4f.h" 18 | #include "tensor.h" 19 | 20 | dim3 getLaunchBlockSize(int maxWidth, int maxHeight, dim3 dims); 21 | dim3 getLaunchGridSize(dim3 blockSize, dim3 dims); 22 | 23 | #ifdef __CUDACC__ 24 | 25 | #ifdef _MSC_VER 26 | #define M_PI 3.14159265358979323846f 27 | #endif 28 | 29 | __host__ __device__ static inline dim3 getWarpSize(dim3 blockSize) 30 | { 31 | return dim3( 32 | min(blockSize.x, 32u), 33 | min(max(32u / blockSize.x, 1u), min(32u, blockSize.y)), 34 | min(max(32u / (blockSize.x * blockSize.y), 1u), min(32u, blockSize.z)) 35 | ); 36 | } 37 | 38 | __device__ static inline float clamp(float val, float mn, float mx) { return min(max(val, mn), mx); } 39 | #else 40 | dim3 getWarpSize(dim3 blockSize); 41 | #endif -------------------------------------------------------------------------------- /data/drag-sampling/render/renderutils/c_src/cubemap.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 | * 4 | * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 5 | * property and proprietary rights in and to this material, related 6 | * documentation and any modifications thereto. Any use, reproduction, 7 | * disclosure or distribution of this material and related documentation 8 | * without an express license agreement from NVIDIA CORPORATION or 9 | * its affiliates is strictly prohibited. 10 | */ 11 | 12 | #pragma once 13 | 14 | #include "common.h" 15 | 16 | struct DiffuseCubemapKernelParams 17 | { 18 | Tensor cubemap; 19 | Tensor out; 20 | dim3 gridSize; 21 | }; 22 | 23 | struct SpecularCubemapKernelParams 24 | { 25 | Tensor cubemap; 26 | Tensor bounds; 27 | Tensor out; 28 | dim3 gridSize; 29 | float costheta_cutoff; 30 | float roughness; 31 | }; 32 | 33 | struct SpecularBoundsKernelParams 34 | { 35 | float costheta_cutoff; 36 | Tensor out; 37 | dim3 gridSize; 38 | }; 39 | -------------------------------------------------------------------------------- /data/drag-sampling/render/renderutils/c_src/loss.cu: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 | * 4 | * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 5 | * property and proprietary rights in and to this material, related 6 | * documentation and any modifications thereto. Any use, reproduction, 7 | * disclosure or distribution of this material and related documentation 8 | * without an express license agreement from NVIDIA CORPORATION or 9 | * its affiliates is strictly prohibited. 10 | */ 11 | 12 | #include 13 | 14 | #include "common.h" 15 | #include "loss.h" 16 | 17 | //------------------------------------------------------------------------ 18 | // Utils 19 | 20 | __device__ inline float bwdAbs(float x) { return x == 0.0f ? 0.0f : x < 0.0f ? -1.0f : 1.0f; } 21 | 22 | __device__ float warpSum(float val) { 23 | for (int i = 1; i < 32; i *= 2) 24 | val += __shfl_xor_sync(0xFFFFFFFF, val, i); 25 | return val; 26 | } 27 | 28 | //------------------------------------------------------------------------ 29 | // Tonemapping 30 | 31 | __device__ inline float fwdSRGB(float x) 32 | { 33 | return x > 0.0031308f ? powf(max(x, 0.0031308f), 1.0f / 2.4f) * 1.055f - 0.055f : 12.92f * max(x, 0.0f); 34 | } 35 | 36 | __device__ inline void bwdSRGB(float x, float &d_x, float d_out) 37 | { 38 | if (x > 0.0031308f) 39 | d_x += d_out * 0.439583f / powf(x, 0.583333f); 40 | else if (x > 0.0f) 41 | d_x += d_out * 12.92f; 42 | } 43 | 44 | __device__ inline vec3f fwdTonemapLogSRGB(vec3f x) 45 | { 46 | return vec3f(fwdSRGB(logf(x.x + 1.0f)), fwdSRGB(logf(x.y + 1.0f)), fwdSRGB(logf(x.z + 1.0f))); 47 | } 48 | 49 | __device__ inline void bwdTonemapLogSRGB(vec3f x, vec3f& d_x, vec3f d_out) 50 | { 51 | if (x.x > 0.0f && x.x < 65535.0f) 52 | { 53 | bwdSRGB(logf(x.x + 1.0f), d_x.x, d_out.x); 54 | d_x.x *= 1 / (x.x + 1.0f); 55 | } 56 | if (x.y > 0.0f && x.y < 65535.0f) 57 | { 58 | bwdSRGB(logf(x.y + 1.0f), d_x.y, d_out.y); 59 | d_x.y *= 1 / (x.y + 1.0f); 60 | } 61 | if (x.z > 0.0f && x.z < 65535.0f) 62 | { 63 | bwdSRGB(logf(x.z + 1.0f), d_x.z, d_out.z); 64 | d_x.z *= 1 / (x.z + 1.0f); 65 | } 66 | } 67 | 68 | __device__ inline float fwdRELMSE(float img, float target, float eps = 0.1f) 69 | { 70 | return (img - target) * (img - target) / (img * img + target * target + eps); 71 | } 72 | 73 | __device__ inline void bwdRELMSE(float img, float target, float &d_img, float &d_target, float d_out, float eps = 0.1f) 74 | { 75 | float denom = (target * target + img * img + eps); 76 | d_img += d_out * 2 * (img - target) * (target * (target + img) + eps) / (denom * denom); 77 | d_target -= d_out * 2 * (img - target) * (img * (target + img) + eps) / (denom * denom); 78 | } 79 | 80 | __device__ inline float fwdSMAPE(float img, float target, float eps=0.01f) 81 | { 82 | return abs(img - target) / (img + target + eps); 83 | } 84 | 85 | __device__ inline void bwdSMAPE(float img, float target, float& d_img, float& d_target, float d_out, float eps = 0.01f) 86 | { 87 | float denom = (target + img + eps); 88 | d_img += d_out * bwdAbs(img - target) * (2 * target + eps) / (denom * denom); 89 | d_target -= d_out * bwdAbs(img - target) * (2 * img + eps) / (denom * denom); 90 | } 91 | 92 | //------------------------------------------------------------------------ 93 | // Kernels 94 | 95 | __global__ void imgLossFwdKernel(LossKernelParams p) 96 | { 97 | // Calculate pixel position. 98 | unsigned int px = blockIdx.x * blockDim.x + threadIdx.x; 99 | unsigned int py = blockIdx.y * blockDim.y + threadIdx.y; 100 | unsigned int pz = blockIdx.z; 101 | 102 | float floss = 0.0f; 103 | if (px < p.gridSize.x && py < p.gridSize.y && pz < p.gridSize.z) 104 | { 105 | vec3f img = p.img.fetch3(px, py, pz); 106 | vec3f target = p.target.fetch3(px, py, pz); 107 | 108 | img = vec3f(clamp(img.x, 0.0f, 65535.0f), clamp(img.y, 0.0f, 65535.0f), clamp(img.z, 0.0f, 65535.0f)); 109 | target = vec3f(clamp(target.x, 0.0f, 65535.0f), clamp(target.y, 0.0f, 65535.0f), clamp(target.z, 0.0f, 65535.0f)); 110 | 111 | if (p.tonemapper == TONEMAPPER_LOG_SRGB) 112 | { 113 | img = fwdTonemapLogSRGB(img); 114 | target = fwdTonemapLogSRGB(target); 115 | } 116 | 117 | vec3f vloss(0); 118 | if (p.loss == LOSS_MSE) 119 | vloss = (img - target) * (img - target); 120 | else if (p.loss == LOSS_RELMSE) 121 | vloss = vec3f(fwdRELMSE(img.x, target.x), fwdRELMSE(img.y, target.y), fwdRELMSE(img.z, target.z)); 122 | else if (p.loss == LOSS_SMAPE) 123 | vloss = vec3f(fwdSMAPE(img.x, target.x), fwdSMAPE(img.y, target.y), fwdSMAPE(img.z, target.z)); 124 | else 125 | vloss = vec3f(abs(img.x - target.x), abs(img.y - target.y), abs(img.z - target.z)); 126 | 127 | floss = sum(vloss) / 3.0f; 128 | } 129 | 130 | floss = warpSum(floss); 131 | 132 | dim3 warpSize = getWarpSize(blockDim); 133 | if (px < p.gridSize.x && py < p.gridSize.y && pz < p.gridSize.z && threadIdx.x % warpSize.x == 0 && threadIdx.y % warpSize.y == 0 && threadIdx.z % warpSize.z == 0) 134 | p.out.store(px / warpSize.x, py / warpSize.y, pz / warpSize.z, floss); 135 | } 136 | 137 | __global__ void imgLossBwdKernel(LossKernelParams p) 138 | { 139 | // Calculate pixel position. 140 | unsigned int px = blockIdx.x * blockDim.x + threadIdx.x; 141 | unsigned int py = blockIdx.y * blockDim.y + threadIdx.y; 142 | unsigned int pz = blockIdx.z; 143 | 144 | if (px >= p.gridSize.x || py >= p.gridSize.y || pz >= p.gridSize.z) 145 | return; 146 | 147 | dim3 warpSize = getWarpSize(blockDim); 148 | 149 | vec3f _img = p.img.fetch3(px, py, pz); 150 | vec3f _target = p.target.fetch3(px, py, pz); 151 | float d_out = p.out.fetch1(px / warpSize.x, py / warpSize.y, pz / warpSize.z); 152 | 153 | ///////////////////////////////////////////////////////////////////// 154 | // FWD 155 | 156 | vec3f img = _img, target = _target; 157 | if (p.tonemapper == TONEMAPPER_LOG_SRGB) 158 | { 159 | img = fwdTonemapLogSRGB(img); 160 | target = fwdTonemapLogSRGB(target); 161 | } 162 | 163 | ///////////////////////////////////////////////////////////////////// 164 | // BWD 165 | 166 | vec3f d_vloss = vec3f(d_out, d_out, d_out) / 3.0f; 167 | 168 | vec3f d_img(0), d_target(0); 169 | if (p.loss == LOSS_MSE) 170 | { 171 | d_img = vec3f(d_vloss.x * 2 * (img.x - target.x), d_vloss.y * 2 * (img.y - target.y), d_vloss.x * 2 * (img.z - target.z)); 172 | d_target = -d_img; 173 | } 174 | else if (p.loss == LOSS_RELMSE) 175 | { 176 | bwdRELMSE(img.x, target.x, d_img.x, d_target.x, d_vloss.x); 177 | bwdRELMSE(img.y, target.y, d_img.y, d_target.y, d_vloss.y); 178 | bwdRELMSE(img.z, target.z, d_img.z, d_target.z, d_vloss.z); 179 | } 180 | else if (p.loss == LOSS_SMAPE) 181 | { 182 | bwdSMAPE(img.x, target.x, d_img.x, d_target.x, d_vloss.x); 183 | bwdSMAPE(img.y, target.y, d_img.y, d_target.y, d_vloss.y); 184 | bwdSMAPE(img.z, target.z, d_img.z, d_target.z, d_vloss.z); 185 | } 186 | else 187 | { 188 | d_img = d_vloss * vec3f(bwdAbs(img.x - target.x), bwdAbs(img.y - target.y), bwdAbs(img.z - target.z)); 189 | d_target = -d_img; 190 | } 191 | 192 | 193 | if (p.tonemapper == TONEMAPPER_LOG_SRGB) 194 | { 195 | vec3f d__img(0), d__target(0); 196 | bwdTonemapLogSRGB(_img, d__img, d_img); 197 | bwdTonemapLogSRGB(_target, d__target, d_target); 198 | d_img = d__img; d_target = d__target; 199 | } 200 | 201 | if (_img.x <= 0.0f || _img.x >= 65535.0f) d_img.x = 0; 202 | if (_img.y <= 0.0f || _img.y >= 65535.0f) d_img.y = 0; 203 | if (_img.z <= 0.0f || _img.z >= 65535.0f) d_img.z = 0; 204 | if (_target.x <= 0.0f || _target.x >= 65535.0f) d_target.x = 0; 205 | if (_target.y <= 0.0f || _target.y >= 65535.0f) d_target.y = 0; 206 | if (_target.z <= 0.0f || _target.z >= 65535.0f) d_target.z = 0; 207 | 208 | p.img.store_grad(px, py, pz, d_img); 209 | p.target.store_grad(px, py, pz, d_target); 210 | } -------------------------------------------------------------------------------- /data/drag-sampling/render/renderutils/c_src/loss.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 | * 4 | * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 5 | * property and proprietary rights in and to this material, related 6 | * documentation and any modifications thereto. Any use, reproduction, 7 | * disclosure or distribution of this material and related documentation 8 | * without an express license agreement from NVIDIA CORPORATION or 9 | * its affiliates is strictly prohibited. 10 | */ 11 | 12 | #pragma once 13 | 14 | #include "common.h" 15 | 16 | enum TonemapperType 17 | { 18 | TONEMAPPER_NONE = 0, 19 | TONEMAPPER_LOG_SRGB = 1 20 | }; 21 | 22 | enum LossType 23 | { 24 | LOSS_L1 = 0, 25 | LOSS_MSE = 1, 26 | LOSS_RELMSE = 2, 27 | LOSS_SMAPE = 3 28 | }; 29 | 30 | struct LossKernelParams 31 | { 32 | Tensor img; 33 | Tensor target; 34 | Tensor out; 35 | dim3 gridSize; 36 | TonemapperType tonemapper; 37 | LossType loss; 38 | }; 39 | -------------------------------------------------------------------------------- /data/drag-sampling/render/renderutils/c_src/mesh.cu: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 | * 4 | * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 5 | * property and proprietary rights in and to this material, related 6 | * documentation and any modifications thereto. Any use, reproduction, 7 | * disclosure or distribution of this material and related documentation 8 | * without an express license agreement from NVIDIA CORPORATION or 9 | * its affiliates is strictly prohibited. 10 | */ 11 | 12 | #include 13 | #include 14 | 15 | #include "common.h" 16 | #include "mesh.h" 17 | 18 | 19 | //------------------------------------------------------------------------ 20 | // Kernels 21 | 22 | __global__ void xfmPointsFwdKernel(XfmKernelParams p) 23 | { 24 | unsigned int px = blockIdx.x * blockDim.x + threadIdx.x; 25 | unsigned int pz = blockIdx.z * blockDim.z + threadIdx.z; 26 | 27 | __shared__ float mtx[4][4]; 28 | if (threadIdx.x < 16) 29 | mtx[threadIdx.x % 4][threadIdx.x / 4] = p.matrix.fetch(p.matrix.nhwcIndex(pz, threadIdx.x / 4, threadIdx.x % 4, 0)); 30 | __syncthreads(); 31 | 32 | if (px >= p.gridSize.x) 33 | return; 34 | 35 | vec3f pos( 36 | p.points.fetch(p.points.nhwcIndex(pz, px, 0, 0)), 37 | p.points.fetch(p.points.nhwcIndex(pz, px, 1, 0)), 38 | p.points.fetch(p.points.nhwcIndex(pz, px, 2, 0)) 39 | ); 40 | 41 | if (p.isPoints) 42 | { 43 | p.out.store(p.out.nhwcIndex(pz, px, 0, 0), pos.x * mtx[0][0] + pos.y * mtx[1][0] + pos.z * mtx[2][0] + mtx[3][0]); 44 | p.out.store(p.out.nhwcIndex(pz, px, 1, 0), pos.x * mtx[0][1] + pos.y * mtx[1][1] + pos.z * mtx[2][1] + mtx[3][1]); 45 | p.out.store(p.out.nhwcIndex(pz, px, 2, 0), pos.x * mtx[0][2] + pos.y * mtx[1][2] + pos.z * mtx[2][2] + mtx[3][2]); 46 | p.out.store(p.out.nhwcIndex(pz, px, 3, 0), pos.x * mtx[0][3] + pos.y * mtx[1][3] + pos.z * mtx[2][3] + mtx[3][3]); 47 | } 48 | else 49 | { 50 | p.out.store(p.out.nhwcIndex(pz, px, 0, 0), pos.x * mtx[0][0] + pos.y * mtx[1][0] + pos.z * mtx[2][0]); 51 | p.out.store(p.out.nhwcIndex(pz, px, 1, 0), pos.x * mtx[0][1] + pos.y * mtx[1][1] + pos.z * mtx[2][1]); 52 | p.out.store(p.out.nhwcIndex(pz, px, 2, 0), pos.x * mtx[0][2] + pos.y * mtx[1][2] + pos.z * mtx[2][2]); 53 | } 54 | } 55 | 56 | __global__ void xfmPointsBwdKernel(XfmKernelParams p) 57 | { 58 | unsigned int px = blockIdx.x * blockDim.x + threadIdx.x; 59 | unsigned int pz = blockIdx.z * blockDim.z + threadIdx.z; 60 | 61 | __shared__ float mtx[4][4]; 62 | if (threadIdx.x < 16) 63 | mtx[threadIdx.x % 4][threadIdx.x / 4] = p.matrix.fetch(p.matrix.nhwcIndex(pz, threadIdx.x / 4, threadIdx.x % 4, 0)); 64 | __syncthreads(); 65 | 66 | if (px >= p.gridSize.x) 67 | return; 68 | 69 | vec3f pos( 70 | p.points.fetch(p.points.nhwcIndex(pz, px, 0, 0)), 71 | p.points.fetch(p.points.nhwcIndex(pz, px, 1, 0)), 72 | p.points.fetch(p.points.nhwcIndex(pz, px, 2, 0)) 73 | ); 74 | 75 | vec4f d_out( 76 | p.out.fetch(p.out.nhwcIndex(pz, px, 0, 0)), 77 | p.out.fetch(p.out.nhwcIndex(pz, px, 1, 0)), 78 | p.out.fetch(p.out.nhwcIndex(pz, px, 2, 0)), 79 | p.out.fetch(p.out.nhwcIndex(pz, px, 3, 0)) 80 | ); 81 | 82 | if (p.isPoints) 83 | { 84 | p.points.store_grad(p.points.nhwcIndexContinuous(pz, px, 0, 0), d_out.x * mtx[0][0] + d_out.y * mtx[0][1] + d_out.z * mtx[0][2] + d_out.w * mtx[0][3]); 85 | p.points.store_grad(p.points.nhwcIndexContinuous(pz, px, 1, 0), d_out.x * mtx[1][0] + d_out.y * mtx[1][1] + d_out.z * mtx[1][2] + d_out.w * mtx[1][3]); 86 | p.points.store_grad(p.points.nhwcIndexContinuous(pz, px, 2, 0), d_out.x * mtx[2][0] + d_out.y * mtx[2][1] + d_out.z * mtx[2][2] + d_out.w * mtx[2][3]); 87 | } 88 | else 89 | { 90 | p.points.store_grad(p.points.nhwcIndexContinuous(pz, px, 0, 0), d_out.x * mtx[0][0] + d_out.y * mtx[0][1] + d_out.z * mtx[0][2]); 91 | p.points.store_grad(p.points.nhwcIndexContinuous(pz, px, 1, 0), d_out.x * mtx[1][0] + d_out.y * mtx[1][1] + d_out.z * mtx[1][2]); 92 | p.points.store_grad(p.points.nhwcIndexContinuous(pz, px, 2, 0), d_out.x * mtx[2][0] + d_out.y * mtx[2][1] + d_out.z * mtx[2][2]); 93 | } 94 | } -------------------------------------------------------------------------------- /data/drag-sampling/render/renderutils/c_src/mesh.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 | * 4 | * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 5 | * property and proprietary rights in and to this material, related 6 | * documentation and any modifications thereto. Any use, reproduction, 7 | * disclosure or distribution of this material and related documentation 8 | * without an express license agreement from NVIDIA CORPORATION or 9 | * its affiliates is strictly prohibited. 10 | */ 11 | 12 | #pragma once 13 | 14 | #include "common.h" 15 | 16 | struct XfmKernelParams 17 | { 18 | bool isPoints; 19 | Tensor points; 20 | Tensor matrix; 21 | Tensor out; 22 | dim3 gridSize; 23 | }; 24 | -------------------------------------------------------------------------------- /data/drag-sampling/render/renderutils/c_src/normal.cu: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 | * 4 | * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 5 | * property and proprietary rights in and to this material, related 6 | * documentation and any modifications thereto. Any use, reproduction, 7 | * disclosure or distribution of this material and related documentation 8 | * without an express license agreement from NVIDIA CORPORATION or 9 | * its affiliates is strictly prohibited. 10 | */ 11 | 12 | #include "common.h" 13 | #include "normal.h" 14 | 15 | #define NORMAL_THRESHOLD 0.1f 16 | 17 | //------------------------------------------------------------------------ 18 | // Perturb shading normal by tangent frame 19 | 20 | __device__ vec3f fwdPerturbNormal(const vec3f perturbed_nrm, const vec3f smooth_nrm, const vec3f smooth_tng, bool opengl) 21 | { 22 | vec3f _smooth_bitng = cross(smooth_tng, smooth_nrm); 23 | vec3f smooth_bitng = safeNormalize(_smooth_bitng); 24 | vec3f _shading_nrm = smooth_tng * perturbed_nrm.x + (opengl ? -1 : 1) * smooth_bitng * perturbed_nrm.y + smooth_nrm * max(perturbed_nrm.z, 0.0f); 25 | return safeNormalize(_shading_nrm); 26 | } 27 | 28 | __device__ void bwdPerturbNormal(const vec3f perturbed_nrm, const vec3f smooth_nrm, const vec3f smooth_tng, vec3f &d_perturbed_nrm, vec3f &d_smooth_nrm, vec3f &d_smooth_tng, const vec3f d_out, bool opengl) 29 | { 30 | //////////////////////////////////////////////////////////////////////// 31 | // FWD 32 | vec3f _smooth_bitng = cross(smooth_tng, smooth_nrm); 33 | vec3f smooth_bitng = safeNormalize(_smooth_bitng); 34 | vec3f _shading_nrm = smooth_tng * perturbed_nrm.x + (opengl ? -1 : 1) * smooth_bitng * perturbed_nrm.y + smooth_nrm * max(perturbed_nrm.z, 0.0f); 35 | 36 | //////////////////////////////////////////////////////////////////////// 37 | // BWD 38 | vec3f d_shading_nrm(0); 39 | bwdSafeNormalize(_shading_nrm, d_shading_nrm, d_out); 40 | 41 | vec3f d_smooth_bitng(0); 42 | 43 | if (perturbed_nrm.z > 0.0f) 44 | { 45 | d_smooth_nrm += d_shading_nrm * perturbed_nrm.z; 46 | d_perturbed_nrm.z += sum(d_shading_nrm * smooth_nrm); 47 | } 48 | 49 | d_smooth_bitng += (opengl ? -1 : 1) * d_shading_nrm * perturbed_nrm.y; 50 | d_perturbed_nrm.y += (opengl ? -1 : 1) * sum(d_shading_nrm * smooth_bitng); 51 | 52 | d_smooth_tng += d_shading_nrm * perturbed_nrm.x; 53 | d_perturbed_nrm.x += sum(d_shading_nrm * smooth_tng); 54 | 55 | vec3f d__smooth_bitng(0); 56 | bwdSafeNormalize(_smooth_bitng, d__smooth_bitng, d_smooth_bitng); 57 | 58 | bwdCross(smooth_tng, smooth_nrm, d_smooth_tng, d_smooth_nrm, d__smooth_bitng); 59 | } 60 | 61 | //------------------------------------------------------------------------ 62 | #define bent_nrm_eps 0.001f 63 | 64 | __device__ vec3f fwdBendNormal(const vec3f view_vec, const vec3f smooth_nrm, const vec3f geom_nrm) 65 | { 66 | float dp = dot(view_vec, smooth_nrm); 67 | float t = clamp(dp / NORMAL_THRESHOLD, 0.0f, 1.0f); 68 | return geom_nrm * (1.0f - t) + smooth_nrm * t; 69 | } 70 | 71 | __device__ void bwdBendNormal(const vec3f view_vec, const vec3f smooth_nrm, const vec3f geom_nrm, vec3f& d_view_vec, vec3f& d_smooth_nrm, vec3f& d_geom_nrm, const vec3f d_out) 72 | { 73 | //////////////////////////////////////////////////////////////////////// 74 | // FWD 75 | float dp = dot(view_vec, smooth_nrm); 76 | float t = clamp(dp / NORMAL_THRESHOLD, 0.0f, 1.0f); 77 | 78 | //////////////////////////////////////////////////////////////////////// 79 | // BWD 80 | if (dp > NORMAL_THRESHOLD) 81 | d_smooth_nrm += d_out; 82 | else 83 | { 84 | // geom_nrm * (1.0f - t) + smooth_nrm * t; 85 | d_geom_nrm += d_out * (1.0f - t); 86 | d_smooth_nrm += d_out * t; 87 | float d_t = sum(d_out * (smooth_nrm - geom_nrm)); 88 | 89 | float d_dp = dp < 0.0f || dp > NORMAL_THRESHOLD ? 0.0f : d_t / NORMAL_THRESHOLD; 90 | 91 | bwdDot(view_vec, smooth_nrm, d_view_vec, d_smooth_nrm, d_dp); 92 | } 93 | } 94 | 95 | //------------------------------------------------------------------------ 96 | // Kernels 97 | 98 | __global__ void PrepareShadingNormalFwdKernel(PrepareShadingNormalKernelParams p) 99 | { 100 | // Calculate pixel position. 101 | unsigned int px = blockIdx.x * blockDim.x + threadIdx.x; 102 | unsigned int py = blockIdx.y * blockDim.y + threadIdx.y; 103 | unsigned int pz = blockIdx.z; 104 | if (px >= p.gridSize.x || py >= p.gridSize.y || pz >= p.gridSize.z) 105 | return; 106 | 107 | vec3f pos = p.pos.fetch3(px, py, pz); 108 | vec3f view_pos = p.view_pos.fetch3(px, py, pz); 109 | vec3f perturbed_nrm = p.perturbed_nrm.fetch3(px, py, pz); 110 | vec3f _smooth_nrm = p.smooth_nrm.fetch3(px, py, pz); 111 | vec3f _smooth_tng = p.smooth_tng.fetch3(px, py, pz); 112 | vec3f geom_nrm = p.geom_nrm.fetch3(px, py, pz); 113 | 114 | vec3f smooth_nrm = safeNormalize(_smooth_nrm); 115 | vec3f smooth_tng = safeNormalize(_smooth_tng); 116 | vec3f view_vec = safeNormalize(view_pos - pos); 117 | vec3f shading_nrm = fwdPerturbNormal(perturbed_nrm, smooth_nrm, smooth_tng, p.opengl); 118 | 119 | vec3f res; 120 | if (p.two_sided_shading && dot(view_vec, geom_nrm) < 0.0f) 121 | res = fwdBendNormal(view_vec, -shading_nrm, -geom_nrm); 122 | else 123 | res = fwdBendNormal(view_vec, shading_nrm, geom_nrm); 124 | 125 | p.out.store(px, py, pz, res); 126 | } 127 | 128 | __global__ void PrepareShadingNormalBwdKernel(PrepareShadingNormalKernelParams p) 129 | { 130 | // Calculate pixel position. 131 | unsigned int px = blockIdx.x * blockDim.x + threadIdx.x; 132 | unsigned int py = blockIdx.y * blockDim.y + threadIdx.y; 133 | unsigned int pz = blockIdx.z; 134 | if (px >= p.gridSize.x || py >= p.gridSize.y || pz >= p.gridSize.z) 135 | return; 136 | 137 | vec3f pos = p.pos.fetch3(px, py, pz); 138 | vec3f view_pos = p.view_pos.fetch3(px, py, pz); 139 | vec3f perturbed_nrm = p.perturbed_nrm.fetch3(px, py, pz); 140 | vec3f _smooth_nrm = p.smooth_nrm.fetch3(px, py, pz); 141 | vec3f _smooth_tng = p.smooth_tng.fetch3(px, py, pz); 142 | vec3f geom_nrm = p.geom_nrm.fetch3(px, py, pz); 143 | vec3f d_out = p.out.fetch3(px, py, pz); 144 | 145 | /////////////////////////////////////////////////////////////////////////////////////////////////// 146 | // FWD 147 | 148 | vec3f smooth_nrm = safeNormalize(_smooth_nrm); 149 | vec3f smooth_tng = safeNormalize(_smooth_tng); 150 | vec3f _view_vec = view_pos - pos; 151 | vec3f view_vec = safeNormalize(view_pos - pos); 152 | 153 | vec3f shading_nrm = fwdPerturbNormal(perturbed_nrm, smooth_nrm, smooth_tng, p.opengl); 154 | 155 | /////////////////////////////////////////////////////////////////////////////////////////////////// 156 | // BWD 157 | 158 | vec3f d_view_vec(0), d_shading_nrm(0), d_geom_nrm(0); 159 | if (p.two_sided_shading && dot(view_vec, geom_nrm) < 0.0f) 160 | { 161 | bwdBendNormal(view_vec, -shading_nrm, -geom_nrm, d_view_vec, d_shading_nrm, d_geom_nrm, d_out); 162 | d_shading_nrm = -d_shading_nrm; 163 | d_geom_nrm = -d_geom_nrm; 164 | } 165 | else 166 | bwdBendNormal(view_vec, shading_nrm, geom_nrm, d_view_vec, d_shading_nrm, d_geom_nrm, d_out); 167 | 168 | vec3f d_perturbed_nrm(0), d_smooth_nrm(0), d_smooth_tng(0); 169 | bwdPerturbNormal(perturbed_nrm, smooth_nrm, smooth_tng, d_perturbed_nrm, d_smooth_nrm, d_smooth_tng, d_shading_nrm, p.opengl); 170 | 171 | vec3f d__view_vec(0), d__smooth_nrm(0), d__smooth_tng(0); 172 | bwdSafeNormalize(_view_vec, d__view_vec, d_view_vec); 173 | bwdSafeNormalize(_smooth_nrm, d__smooth_nrm, d_smooth_nrm); 174 | bwdSafeNormalize(_smooth_tng, d__smooth_tng, d_smooth_tng); 175 | 176 | p.pos.store_grad(px, py, pz, -d__view_vec); 177 | p.view_pos.store_grad(px, py, pz, d__view_vec); 178 | p.perturbed_nrm.store_grad(px, py, pz, d_perturbed_nrm); 179 | p.smooth_nrm.store_grad(px, py, pz, d__smooth_nrm); 180 | p.smooth_tng.store_grad(px, py, pz, d__smooth_tng); 181 | p.geom_nrm.store_grad(px, py, pz, d_geom_nrm); 182 | } -------------------------------------------------------------------------------- /data/drag-sampling/render/renderutils/c_src/normal.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 | * 4 | * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 5 | * property and proprietary rights in and to this material, related 6 | * documentation and any modifications thereto. Any use, reproduction, 7 | * disclosure or distribution of this material and related documentation 8 | * without an express license agreement from NVIDIA CORPORATION or 9 | * its affiliates is strictly prohibited. 10 | */ 11 | 12 | #pragma once 13 | 14 | #include "common.h" 15 | 16 | struct PrepareShadingNormalKernelParams 17 | { 18 | Tensor pos; 19 | Tensor view_pos; 20 | Tensor perturbed_nrm; 21 | Tensor smooth_nrm; 22 | Tensor smooth_tng; 23 | Tensor geom_nrm; 24 | Tensor out; 25 | dim3 gridSize; 26 | bool two_sided_shading, opengl; 27 | }; 28 | -------------------------------------------------------------------------------- /data/drag-sampling/render/renderutils/c_src/tensor.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 | * 4 | * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 5 | * property and proprietary rights in and to this material, related 6 | * documentation and any modifications thereto. Any use, reproduction, 7 | * disclosure or distribution of this material and related documentation 8 | * without an express license agreement from NVIDIA CORPORATION or 9 | * its affiliates is strictly prohibited. 10 | */ 11 | 12 | #pragma once 13 | #if defined(__CUDACC__) && defined(BFLOAT16) 14 | #include // bfloat16 is float32 compatible with less mantissa bits 15 | #endif 16 | 17 | //--------------------------------------------------------------------------------- 18 | // CUDA-side Tensor class for in/out parameter parsing. Can be float32 or bfloat16 19 | 20 | struct Tensor 21 | { 22 | void* val; 23 | void* d_val; 24 | int dims[4], _dims[4]; 25 | int strides[4]; 26 | bool fp16; 27 | 28 | #if defined(__CUDA__) && !defined(__CUDA_ARCH__) 29 | Tensor() : val(nullptr), d_val(nullptr), fp16(true), dims{ 0, 0, 0, 0 }, _dims{ 0, 0, 0, 0 }, strides{ 0, 0, 0, 0 } {} 30 | #endif 31 | 32 | #ifdef __CUDACC__ 33 | // Helpers to index and read/write a single element 34 | __device__ inline int _nhwcIndex(int n, int h, int w, int c) const { return n * strides[0] + h * strides[1] + w * strides[2] + c * strides[3]; } 35 | __device__ inline int nhwcIndex(int n, int h, int w, int c) const { return (dims[0] == 1 ? 0 : n * strides[0]) + (dims[1] == 1 ? 0 : h * strides[1]) + (dims[2] == 1 ? 0 : w * strides[2]) + (dims[3] == 1 ? 0 : c * strides[3]); } 36 | __device__ inline int nhwcIndexContinuous(int n, int h, int w, int c) const { return ((n * _dims[1] + h) * _dims[2] + w) * _dims[3] + c; } 37 | #ifdef BFLOAT16 38 | __device__ inline float fetch(unsigned int idx) const { return fp16 ? __bfloat162float(((__nv_bfloat16*)val)[idx]) : ((float*)val)[idx]; } 39 | __device__ inline void store(unsigned int idx, float _val) { if (fp16) ((__nv_bfloat16*)val)[idx] = __float2bfloat16(_val); else ((float*)val)[idx] = _val; } 40 | __device__ inline void store_grad(unsigned int idx, float _val) { if (fp16) ((__nv_bfloat16*)d_val)[idx] = __float2bfloat16(_val); else ((float*)d_val)[idx] = _val; } 41 | #else 42 | __device__ inline float fetch(unsigned int idx) const { return ((float*)val)[idx]; } 43 | __device__ inline void store(unsigned int idx, float _val) { ((float*)val)[idx] = _val; } 44 | __device__ inline void store_grad(unsigned int idx, float _val) { ((float*)d_val)[idx] = _val; } 45 | #endif 46 | 47 | ////////////////////////////////////////////////////////////////////////////////////////// 48 | // Fetch, use broadcasting for tensor dimensions of size 1 49 | __device__ inline float fetch1(unsigned int x, unsigned int y, unsigned int z) const 50 | { 51 | return fetch(nhwcIndex(z, y, x, 0)); 52 | } 53 | 54 | __device__ inline vec3f fetch3(unsigned int x, unsigned int y, unsigned int z) const 55 | { 56 | return vec3f( 57 | fetch(nhwcIndex(z, y, x, 0)), 58 | fetch(nhwcIndex(z, y, x, 1)), 59 | fetch(nhwcIndex(z, y, x, 2)) 60 | ); 61 | } 62 | 63 | ///////////////////////////////////////////////////////////////////////////////////////////////////////////// 64 | // Store, no broadcasting here. Assume we output full res gradient and then reduce using torch.sum outside 65 | __device__ inline void store(unsigned int x, unsigned int y, unsigned int z, float _val) 66 | { 67 | store(_nhwcIndex(z, y, x, 0), _val); 68 | } 69 | 70 | __device__ inline void store(unsigned int x, unsigned int y, unsigned int z, vec3f _val) 71 | { 72 | store(_nhwcIndex(z, y, x, 0), _val.x); 73 | store(_nhwcIndex(z, y, x, 1), _val.y); 74 | store(_nhwcIndex(z, y, x, 2), _val.z); 75 | } 76 | 77 | ///////////////////////////////////////////////////////////////////////////////////////////////////////////// 78 | // Store gradient , no broadcasting here. Assume we output full res gradient and then reduce using torch.sum outside 79 | __device__ inline void store_grad(unsigned int x, unsigned int y, unsigned int z, float _val) 80 | { 81 | store_grad(nhwcIndexContinuous(z, y, x, 0), _val); 82 | } 83 | 84 | __device__ inline void store_grad(unsigned int x, unsigned int y, unsigned int z, vec3f _val) 85 | { 86 | store_grad(nhwcIndexContinuous(z, y, x, 0), _val.x); 87 | store_grad(nhwcIndexContinuous(z, y, x, 1), _val.y); 88 | store_grad(nhwcIndexContinuous(z, y, x, 2), _val.z); 89 | } 90 | #endif 91 | 92 | }; 93 | -------------------------------------------------------------------------------- /data/drag-sampling/render/renderutils/c_src/vec3f.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 | * 4 | * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 5 | * property and proprietary rights in and to this material, related 6 | * documentation and any modifications thereto. Any use, reproduction, 7 | * disclosure or distribution of this material and related documentation 8 | * without an express license agreement from NVIDIA CORPORATION or 9 | * its affiliates is strictly prohibited. 10 | */ 11 | 12 | #pragma once 13 | 14 | struct vec3f 15 | { 16 | float x, y, z; 17 | 18 | #ifdef __CUDACC__ 19 | __device__ vec3f() { } 20 | __device__ vec3f(float v) { x = v; y = v; z = v; } 21 | __device__ vec3f(float _x, float _y, float _z) { x = _x; y = _y; z = _z; } 22 | __device__ vec3f(float3 v) { x = v.x; y = v.y; z = v.z; } 23 | 24 | __device__ inline vec3f& operator+=(const vec3f& b) { x += b.x; y += b.y; z += b.z; return *this; } 25 | __device__ inline vec3f& operator-=(const vec3f& b) { x -= b.x; y -= b.y; z -= b.z; return *this; } 26 | __device__ inline vec3f& operator*=(const vec3f& b) { x *= b.x; y *= b.y; z *= b.z; return *this; } 27 | __device__ inline vec3f& operator/=(const vec3f& b) { x /= b.x; y /= b.y; z /= b.z; return *this; } 28 | #endif 29 | }; 30 | 31 | #ifdef __CUDACC__ 32 | __device__ static inline vec3f operator+(const vec3f& a, const vec3f& b) { return vec3f(a.x + b.x, a.y + b.y, a.z + b.z); } 33 | __device__ static inline vec3f operator-(const vec3f& a, const vec3f& b) { return vec3f(a.x - b.x, a.y - b.y, a.z - b.z); } 34 | __device__ static inline vec3f operator*(const vec3f& a, const vec3f& b) { return vec3f(a.x * b.x, a.y * b.y, a.z * b.z); } 35 | __device__ static inline vec3f operator/(const vec3f& a, const vec3f& b) { return vec3f(a.x / b.x, a.y / b.y, a.z / b.z); } 36 | __device__ static inline vec3f operator-(const vec3f& a) { return vec3f(-a.x, -a.y, -a.z); } 37 | 38 | __device__ static inline float sum(vec3f a) 39 | { 40 | return a.x + a.y + a.z; 41 | } 42 | 43 | __device__ static inline vec3f cross(vec3f a, vec3f b) 44 | { 45 | vec3f out; 46 | out.x = a.y * b.z - a.z * b.y; 47 | out.y = a.z * b.x - a.x * b.z; 48 | out.z = a.x * b.y - a.y * b.x; 49 | return out; 50 | } 51 | 52 | __device__ static inline void bwdCross(vec3f a, vec3f b, vec3f &d_a, vec3f &d_b, vec3f d_out) 53 | { 54 | d_a.x += d_out.z * b.y - d_out.y * b.z; 55 | d_a.y += d_out.x * b.z - d_out.z * b.x; 56 | d_a.z += d_out.y * b.x - d_out.x * b.y; 57 | 58 | d_b.x += d_out.y * a.z - d_out.z * a.y; 59 | d_b.y += d_out.z * a.x - d_out.x * a.z; 60 | d_b.z += d_out.x * a.y - d_out.y * a.x; 61 | } 62 | 63 | __device__ static inline float dot(vec3f a, vec3f b) 64 | { 65 | return a.x * b.x + a.y * b.y + a.z * b.z; 66 | } 67 | 68 | __device__ static inline void bwdDot(vec3f a, vec3f b, vec3f& d_a, vec3f& d_b, float d_out) 69 | { 70 | d_a.x += d_out * b.x; d_a.y += d_out * b.y; d_a.z += d_out * b.z; 71 | d_b.x += d_out * a.x; d_b.y += d_out * a.y; d_b.z += d_out * a.z; 72 | } 73 | 74 | __device__ static inline vec3f reflect(vec3f x, vec3f n) 75 | { 76 | return n * 2.0f * dot(n, x) - x; 77 | } 78 | 79 | __device__ static inline void bwdReflect(vec3f x, vec3f n, vec3f& d_x, vec3f& d_n, const vec3f d_out) 80 | { 81 | d_x.x += d_out.x * (2 * n.x * n.x - 1) + d_out.y * (2 * n.x * n.y) + d_out.z * (2 * n.x * n.z); 82 | d_x.y += d_out.x * (2 * n.x * n.y) + d_out.y * (2 * n.y * n.y - 1) + d_out.z * (2 * n.y * n.z); 83 | d_x.z += d_out.x * (2 * n.x * n.z) + d_out.y * (2 * n.y * n.z) + d_out.z * (2 * n.z * n.z - 1); 84 | 85 | d_n.x += d_out.x * (2 * (2 * n.x * x.x + n.y * x.y + n.z * x.z)) + d_out.y * (2 * n.y * x.x) + d_out.z * (2 * n.z * x.x); 86 | d_n.y += d_out.x * (2 * n.x * x.y) + d_out.y * (2 * (n.x * x.x + 2 * n.y * x.y + n.z * x.z)) + d_out.z * (2 * n.z * x.y); 87 | d_n.z += d_out.x * (2 * n.x * x.z) + d_out.y * (2 * n.y * x.z) + d_out.z * (2 * (n.x * x.x + n.y * x.y + 2 * n.z * x.z)); 88 | } 89 | 90 | __device__ static inline vec3f safeNormalize(vec3f v) 91 | { 92 | float l = sqrtf(v.x * v.x + v.y * v.y + v.z * v.z); 93 | return l > 0.0f ? (v / l) : vec3f(0.0f); 94 | } 95 | 96 | __device__ static inline void bwdSafeNormalize(const vec3f v, vec3f& d_v, const vec3f d_out) 97 | { 98 | 99 | float l = sqrtf(v.x * v.x + v.y * v.y + v.z * v.z); 100 | if (l > 0.0f) 101 | { 102 | float fac = 1.0 / powf(v.x * v.x + v.y * v.y + v.z * v.z, 1.5f); 103 | d_v.x += (d_out.x * (v.y * v.y + v.z * v.z) - d_out.y * (v.x * v.y) - d_out.z * (v.x * v.z)) * fac; 104 | d_v.y += (d_out.y * (v.x * v.x + v.z * v.z) - d_out.x * (v.y * v.x) - d_out.z * (v.y * v.z)) * fac; 105 | d_v.z += (d_out.z * (v.x * v.x + v.y * v.y) - d_out.x * (v.z * v.x) - d_out.y * (v.z * v.y)) * fac; 106 | } 107 | } 108 | 109 | #endif -------------------------------------------------------------------------------- /data/drag-sampling/render/renderutils/c_src/vec4f.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 | * 4 | * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 5 | * property and proprietary rights in and to this material, related 6 | * documentation and any modifications thereto. Any use, reproduction, 7 | * disclosure or distribution of this material and related documentation 8 | * without an express license agreement from NVIDIA CORPORATION or 9 | * its affiliates is strictly prohibited. 10 | */ 11 | 12 | #pragma once 13 | 14 | struct vec4f 15 | { 16 | float x, y, z, w; 17 | 18 | #ifdef __CUDACC__ 19 | __device__ vec4f() { } 20 | __device__ vec4f(float v) { x = v; y = v; z = v; w = v; } 21 | __device__ vec4f(float _x, float _y, float _z, float _w) { x = _x; y = _y; z = _z; w = _w; } 22 | __device__ vec4f(float4 v) { x = v.x; y = v.y; z = v.z; w = v.w; } 23 | #endif 24 | }; 25 | 26 | -------------------------------------------------------------------------------- /data/drag-sampling/render/renderutils/loss.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # 3 | # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 4 | # property and proprietary rights in and to this material, related 5 | # documentation and any modifications thereto. Any use, reproduction, 6 | # disclosure or distribution of this material and related documentation 7 | # without an express license agreement from NVIDIA CORPORATION or 8 | # its affiliates is strictly prohibited. 9 | 10 | import torch 11 | 12 | #---------------------------------------------------------------------------- 13 | # HDR image losses 14 | #---------------------------------------------------------------------------- 15 | 16 | def _tonemap_srgb(f): 17 | return torch.where(f > 0.0031308, torch.pow(torch.clamp(f, min=0.0031308), 1.0/2.4)*1.055 - 0.055, 12.92*f) 18 | 19 | def _SMAPE(img, target, eps=0.01): 20 | nom = torch.abs(img - target) 21 | denom = torch.abs(img) + torch.abs(target) + 0.01 22 | return torch.mean(nom / denom) 23 | 24 | def _RELMSE(img, target, eps=0.1): 25 | nom = (img - target) * (img - target) 26 | denom = img * img + target * target + 0.1 27 | return torch.mean(nom / denom) 28 | 29 | def image_loss_fn(img, target, loss, tonemapper): 30 | if tonemapper == 'log_srgb': 31 | img = _tonemap_srgb(torch.log(torch.clamp(img, min=0, max=65535) + 1)) 32 | target = _tonemap_srgb(torch.log(torch.clamp(target, min=0, max=65535) + 1)) 33 | 34 | if loss == 'mse': 35 | return torch.nn.functional.mse_loss(img, target) 36 | elif loss == 'smape': 37 | return _SMAPE(img, target) 38 | elif loss == 'relmse': 39 | return _RELMSE(img, target) 40 | else: 41 | return torch.nn.functional.l1_loss(img, target) 42 | -------------------------------------------------------------------------------- /data/drag-sampling/render/renderutils/tests/test_cubemap.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # 3 | # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 4 | # property and proprietary rights in and to this material, related 5 | # documentation and any modifications thereto. Any use, reproduction, 6 | # disclosure or distribution of this material and related documentation 7 | # without an express license agreement from NVIDIA CORPORATION or 8 | # its affiliates is strictly prohibited. 9 | 10 | import torch 11 | 12 | import os 13 | import sys 14 | sys.path.insert(0, os.path.join(sys.path[0], '../..')) 15 | import renderutils as ru 16 | 17 | RES = 4 18 | DTYPE = torch.float32 19 | 20 | def relative_loss(name, ref, cuda): 21 | ref = ref.float() 22 | cuda = cuda.float() 23 | print(name, torch.max(torch.abs(ref - cuda) / torch.abs(ref + 1e-7)).item()) 24 | 25 | def test_cubemap(): 26 | cubemap_cuda = torch.rand(6, RES, RES, 3, dtype=DTYPE, device='cuda', requires_grad=True) 27 | cubemap_ref = cubemap_cuda.clone().detach().requires_grad_(True) 28 | weights = torch.rand(3, 3, 1, dtype=DTYPE, device='cuda') 29 | target = torch.rand(6, RES, RES, 3, dtype=DTYPE, device='cuda') 30 | 31 | ref = ru.filter_cubemap(cubemap_ref, weights, use_python=True) 32 | ref_loss = torch.nn.MSELoss()(ref, target) 33 | ref_loss.backward() 34 | 35 | cuda = ru.filter_cubemap(cubemap_cuda, weights, use_python=False) 36 | cuda_loss = torch.nn.MSELoss()(cuda, target) 37 | cuda_loss.backward() 38 | 39 | print("-------------------------------------------------------------") 40 | print(" Cubemap:") 41 | print("-------------------------------------------------------------") 42 | 43 | relative_loss("flt:", ref, cuda) 44 | relative_loss("cubemap:", cubemap_ref.grad, cubemap_cuda.grad) 45 | 46 | 47 | test_cubemap() 48 | -------------------------------------------------------------------------------- /data/drag-sampling/render/renderutils/tests/test_loss.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # 3 | # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 4 | # property and proprietary rights in and to this material, related 5 | # documentation and any modifications thereto. Any use, reproduction, 6 | # disclosure or distribution of this material and related documentation 7 | # without an express license agreement from NVIDIA CORPORATION or 8 | # its affiliates is strictly prohibited. 9 | 10 | import torch 11 | 12 | import os 13 | import sys 14 | sys.path.insert(0, os.path.join(sys.path[0], '../..')) 15 | import renderutils as ru 16 | 17 | RES = 8 18 | DTYPE = torch.float32 19 | 20 | def tonemap_srgb(f): 21 | return torch.where(f > 0.0031308, torch.pow(torch.clamp(f, min=0.0031308), 1.0/2.4)*1.055 - 0.055, 12.92*f) 22 | 23 | def l1(output, target): 24 | x = torch.clamp(output, min=0, max=65535) 25 | r = torch.clamp(target, min=0, max=65535) 26 | x = tonemap_srgb(torch.log(x + 1)) 27 | r = tonemap_srgb(torch.log(r + 1)) 28 | return torch.nn.functional.l1_loss(x,r) 29 | 30 | def relative_loss(name, ref, cuda): 31 | ref = ref.float() 32 | cuda = cuda.float() 33 | print(name, torch.max(torch.abs(ref - cuda) / torch.abs(ref + 1e-7)).item()) 34 | 35 | def test_loss(loss, tonemapper): 36 | img_cuda = torch.rand(1, RES, RES, 3, dtype=DTYPE, device='cuda', requires_grad=True) 37 | img_ref = img_cuda.clone().detach().requires_grad_(True) 38 | target_cuda = torch.rand(1, RES, RES, 3, dtype=DTYPE, device='cuda', requires_grad=True) 39 | target_ref = target_cuda.clone().detach().requires_grad_(True) 40 | 41 | ref_loss = ru.image_loss(img_ref, target_ref, loss=loss, tonemapper=tonemapper, use_python=True) 42 | ref_loss.backward() 43 | 44 | cuda_loss = ru.image_loss(img_cuda, target_cuda, loss=loss, tonemapper=tonemapper) 45 | cuda_loss.backward() 46 | 47 | print("-------------------------------------------------------------") 48 | print(" Loss: %s, %s" % (loss, tonemapper)) 49 | print("-------------------------------------------------------------") 50 | 51 | relative_loss("res:", ref_loss, cuda_loss) 52 | relative_loss("img:", img_ref.grad, img_cuda.grad) 53 | relative_loss("target:", target_ref.grad, target_cuda.grad) 54 | 55 | 56 | test_loss('l1', 'none') 57 | test_loss('l1', 'log_srgb') 58 | test_loss('mse', 'log_srgb') 59 | test_loss('smape', 'none') 60 | test_loss('relmse', 'none') 61 | test_loss('mse', 'none') -------------------------------------------------------------------------------- /data/drag-sampling/render/renderutils/tests/test_mesh.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # 3 | # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 4 | # property and proprietary rights in and to this material, related 5 | # documentation and any modifications thereto. Any use, reproduction, 6 | # disclosure or distribution of this material and related documentation 7 | # without an express license agreement from NVIDIA CORPORATION or 8 | # its affiliates is strictly prohibited. 9 | 10 | import torch 11 | 12 | import os 13 | import sys 14 | sys.path.insert(0, os.path.join(sys.path[0], '../..')) 15 | import renderutils as ru 16 | 17 | BATCH = 8 18 | RES = 1024 19 | DTYPE = torch.float32 20 | 21 | torch.manual_seed(0) 22 | 23 | def tonemap_srgb(f): 24 | return torch.where(f > 0.0031308, torch.pow(torch.clamp(f, min=0.0031308), 1.0/2.4)*1.055 - 0.055, 12.92*f) 25 | 26 | def l1(output, target): 27 | x = torch.clamp(output, min=0, max=65535) 28 | r = torch.clamp(target, min=0, max=65535) 29 | x = tonemap_srgb(torch.log(x + 1)) 30 | r = tonemap_srgb(torch.log(r + 1)) 31 | return torch.nn.functional.l1_loss(x,r) 32 | 33 | def relative_loss(name, ref, cuda): 34 | ref = ref.float() 35 | cuda = cuda.float() 36 | print(name, torch.max(torch.abs(ref - cuda) / torch.abs(ref)).item()) 37 | 38 | def test_xfm_points(): 39 | points_cuda = torch.rand(1, RES, 3, dtype=DTYPE, device='cuda', requires_grad=True) 40 | points_ref = points_cuda.clone().detach().requires_grad_(True) 41 | mtx_cuda = torch.rand(BATCH, 4, 4, dtype=DTYPE, device='cuda', requires_grad=False) 42 | mtx_ref = mtx_cuda.clone().detach().requires_grad_(True) 43 | target = torch.rand(BATCH, RES, 4, dtype=DTYPE, device='cuda', requires_grad=True) 44 | 45 | ref_out = ru.xfm_points(points_ref, mtx_ref, use_python=True) 46 | ref_loss = torch.nn.MSELoss()(ref_out, target) 47 | ref_loss.backward() 48 | 49 | cuda_out = ru.xfm_points(points_cuda, mtx_cuda) 50 | cuda_loss = torch.nn.MSELoss()(cuda_out, target) 51 | cuda_loss.backward() 52 | 53 | print("-------------------------------------------------------------") 54 | 55 | relative_loss("res:", ref_out, cuda_out) 56 | relative_loss("points:", points_ref.grad, points_cuda.grad) 57 | 58 | def test_xfm_vectors(): 59 | points_cuda = torch.rand(1, RES, 3, dtype=DTYPE, device='cuda', requires_grad=True) 60 | points_ref = points_cuda.clone().detach().requires_grad_(True) 61 | points_cuda_p = points_cuda.clone().detach().requires_grad_(True) 62 | points_ref_p = points_cuda.clone().detach().requires_grad_(True) 63 | mtx_cuda = torch.rand(BATCH, 4, 4, dtype=DTYPE, device='cuda', requires_grad=False) 64 | mtx_ref = mtx_cuda.clone().detach().requires_grad_(True) 65 | target = torch.rand(BATCH, RES, 4, dtype=DTYPE, device='cuda', requires_grad=True) 66 | 67 | ref_out = ru.xfm_vectors(points_ref.contiguous(), mtx_ref, use_python=True) 68 | ref_loss = torch.nn.MSELoss()(ref_out, target[..., 0:3]) 69 | ref_loss.backward() 70 | 71 | cuda_out = ru.xfm_vectors(points_cuda.contiguous(), mtx_cuda) 72 | cuda_loss = torch.nn.MSELoss()(cuda_out, target[..., 0:3]) 73 | cuda_loss.backward() 74 | 75 | ref_out_p = ru.xfm_points(points_ref_p.contiguous(), mtx_ref, use_python=True) 76 | ref_loss_p = torch.nn.MSELoss()(ref_out_p, target) 77 | ref_loss_p.backward() 78 | 79 | cuda_out_p = ru.xfm_points(points_cuda_p.contiguous(), mtx_cuda) 80 | cuda_loss_p = torch.nn.MSELoss()(cuda_out_p, target) 81 | cuda_loss_p.backward() 82 | 83 | print("-------------------------------------------------------------") 84 | 85 | relative_loss("res:", ref_out, cuda_out) 86 | relative_loss("points:", points_ref.grad, points_cuda.grad) 87 | relative_loss("points_p:", points_ref_p.grad, points_cuda_p.grad) 88 | 89 | test_xfm_points() 90 | test_xfm_vectors() 91 | -------------------------------------------------------------------------------- /data/drag-sampling/render/renderutils/tests/test_perf.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # 3 | # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 4 | # property and proprietary rights in and to this material, related 5 | # documentation and any modifications thereto. Any use, reproduction, 6 | # disclosure or distribution of this material and related documentation 7 | # without an express license agreement from NVIDIA CORPORATION or 8 | # its affiliates is strictly prohibited. 9 | 10 | import torch 11 | 12 | import os 13 | import sys 14 | sys.path.insert(0, os.path.join(sys.path[0], '../..')) 15 | import renderutils as ru 16 | 17 | DTYPE=torch.float32 18 | 19 | def test_bsdf(BATCH, RES, ITR): 20 | kd_cuda = torch.rand(BATCH, RES, RES, 3, dtype=DTYPE, device='cuda', requires_grad=True) 21 | kd_ref = kd_cuda.clone().detach().requires_grad_(True) 22 | arm_cuda = torch.rand(BATCH, RES, RES, 3, dtype=DTYPE, device='cuda', requires_grad=True) 23 | arm_ref = arm_cuda.clone().detach().requires_grad_(True) 24 | pos_cuda = torch.rand(BATCH, RES, RES, 3, dtype=DTYPE, device='cuda', requires_grad=True) 25 | pos_ref = pos_cuda.clone().detach().requires_grad_(True) 26 | nrm_cuda = torch.rand(BATCH, RES, RES, 3, dtype=DTYPE, device='cuda', requires_grad=True) 27 | nrm_ref = nrm_cuda.clone().detach().requires_grad_(True) 28 | view_cuda = torch.rand(BATCH, RES, RES, 3, dtype=DTYPE, device='cuda', requires_grad=True) 29 | view_ref = view_cuda.clone().detach().requires_grad_(True) 30 | light_cuda = torch.rand(BATCH, RES, RES, 3, dtype=DTYPE, device='cuda', requires_grad=True) 31 | light_ref = light_cuda.clone().detach().requires_grad_(True) 32 | target = torch.rand(BATCH, RES, RES, 3, device='cuda') 33 | 34 | start = torch.cuda.Event(enable_timing=True) 35 | end = torch.cuda.Event(enable_timing=True) 36 | 37 | ru.pbr_bsdf(kd_cuda, arm_cuda, pos_cuda, nrm_cuda, view_cuda, light_cuda) 38 | 39 | print("--- Testing: [%d, %d, %d] ---" % (BATCH, RES, RES)) 40 | 41 | start.record() 42 | for i in range(ITR): 43 | ref = ru.pbr_bsdf(kd_ref, arm_ref, pos_ref, nrm_ref, view_ref, light_ref, use_python=True) 44 | end.record() 45 | torch.cuda.synchronize() 46 | print("Pbr BSDF python:", start.elapsed_time(end)) 47 | 48 | start.record() 49 | for i in range(ITR): 50 | cuda = ru.pbr_bsdf(kd_cuda, arm_cuda, pos_cuda, nrm_cuda, view_cuda, light_cuda) 51 | end.record() 52 | torch.cuda.synchronize() 53 | print("Pbr BSDF cuda:", start.elapsed_time(end)) 54 | 55 | test_bsdf(1, 512, 1000) 56 | test_bsdf(16, 512, 1000) 57 | test_bsdf(1, 2048, 1000) 58 | -------------------------------------------------------------------------------- /data/drag-sampling/render/texture.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 2 | # 3 | # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual 4 | # property and proprietary rights in and to this material, related 5 | # documentation and any modifications thereto. Any use, reproduction, 6 | # disclosure or distribution of this material and related documentation 7 | # without an express license agreement from NVIDIA CORPORATION or 8 | # its affiliates is strictly prohibited. 9 | 10 | import os 11 | import numpy as np 12 | import torch 13 | import nvdiffrast.torch as dr 14 | 15 | from . import util 16 | 17 | ###################################################################################### 18 | # Smooth pooling / mip computation with linear gradient upscaling 19 | ###################################################################################### 20 | 21 | class texture2d_mip(torch.autograd.Function): 22 | @staticmethod 23 | def forward(ctx, texture): 24 | return util.avg_pool_nhwc(texture, (2,2)) 25 | 26 | @staticmethod 27 | def backward(ctx, dout): 28 | gy, gx = torch.meshgrid(torch.linspace(0.0 + 0.25 / dout.shape[1], 1.0 - 0.25 / dout.shape[1], dout.shape[1]*2, device="cuda"), 29 | torch.linspace(0.0 + 0.25 / dout.shape[2], 1.0 - 0.25 / dout.shape[2], dout.shape[2]*2, device="cuda"), 30 | indexing='ij') 31 | uv = torch.stack((gx, gy), dim=-1) 32 | return dr.texture(dout * 0.25, uv[None, ...].contiguous(), filter_mode='linear', boundary_mode='clamp') 33 | 34 | ######################################################################################################## 35 | # Simple texture class. A texture can be either 36 | # - A 3D tensor (using auto mipmaps) 37 | # - A list of 3D tensors (full custom mip hierarchy) 38 | ######################################################################################################## 39 | 40 | class Texture2D(torch.nn.Module): 41 | # Initializes a texture from image data. 42 | # Input can be constant value (1D array) or texture (3D array) or mip hierarchy (list of 3d arrays) 43 | def __init__(self, init, min_max=None): 44 | super(Texture2D, self).__init__() 45 | 46 | if isinstance(init, np.ndarray): 47 | init = torch.tensor(init, dtype=torch.float32, device='cuda') 48 | elif isinstance(init, list) and len(init) == 1: 49 | init = init[0] 50 | 51 | if isinstance(init, list): 52 | self.data = list(torch.nn.Parameter(mip.clone().detach(), requires_grad=True) for mip in init) 53 | elif len(init.shape) == 4: 54 | self.data = torch.nn.Parameter(init.clone().detach(), requires_grad=True) 55 | elif len(init.shape) == 3: 56 | self.data = torch.nn.Parameter(init[None, ...].clone().detach(), requires_grad=True) 57 | elif len(init.shape) == 2: 58 | self.data = torch.nn.Parameter(init[None, ..., None].repeat(1, 1, 1, 3).clone().detach(), requires_grad=True) 59 | elif len(init.shape) == 1: 60 | self.data = torch.nn.Parameter(init[None, None, None, :].clone().detach(), requires_grad=True) # Convert constant to 1x1 tensor 61 | else: 62 | assert False, "Invalid texture object" 63 | 64 | self.min_max = min_max 65 | 66 | # Filtered (trilinear) sample texture at a given location 67 | def sample(self, texc, texc_deriv, filter_mode='linear-mipmap-linear'): 68 | if isinstance(self.data, list): 69 | out = dr.texture(self.data[0], texc, texc_deriv, mip=self.data[1:], filter_mode=filter_mode) 70 | else: 71 | if self.data.shape[1] > 1 and self.data.shape[2] > 1: 72 | mips = [self.data] 73 | while mips[-1].shape[1] > 1 and mips[-1].shape[2] > 1: 74 | mips += [texture2d_mip.apply(mips[-1])] 75 | out = dr.texture(mips[0], texc, texc_deriv, mip=mips[1:], filter_mode=filter_mode) 76 | else: 77 | out = dr.texture(self.data, texc, texc_deriv, filter_mode=filter_mode) 78 | return out 79 | 80 | def getRes(self): 81 | return self.getMips()[0].shape[1:3] 82 | 83 | def getChannels(self): 84 | return self.getMips()[0].shape[3] 85 | 86 | def getMips(self): 87 | if isinstance(self.data, list): 88 | return self.data 89 | else: 90 | return [self.data] 91 | 92 | # In-place clamp with no derivative to make sure values are in valid range after training 93 | def clamp_(self): 94 | if self.min_max is not None: 95 | for mip in self.getMips(): 96 | for i in range(mip.shape[-1]): 97 | mip[..., i].clamp_(min=self.min_max[0][i], max=self.min_max[1][i]) 98 | 99 | # In-place clamp with no derivative to make sure values are in valid range after training 100 | def normalize_(self): 101 | with torch.no_grad(): 102 | for mip in self.getMips(): 103 | mip = util.safe_normalize(mip) 104 | 105 | ######################################################################################################## 106 | # Helper function to create a trainable texture from a regular texture. The trainable weights are 107 | # initialized with texture data as an initial guess 108 | ######################################################################################################## 109 | 110 | def create_trainable(init, res=None, auto_mipmaps=True, min_max=None): 111 | with torch.no_grad(): 112 | if isinstance(init, Texture2D): 113 | assert isinstance(init.data, torch.Tensor) 114 | min_max = init.min_max if min_max is None else min_max 115 | init = init.data 116 | elif isinstance(init, np.ndarray): 117 | init = torch.tensor(init, dtype=torch.float32, device='cuda') 118 | 119 | # Pad to NHWC if needed 120 | if len(init.shape) == 1: # Extend constant to NHWC tensor 121 | init = init[None, None, None, :] 122 | elif len(init.shape) == 3: 123 | init = init[None, ...] 124 | 125 | # Scale input to desired resolution. 126 | if res is not None: 127 | init = util.scale_img_nhwc(init, res) 128 | 129 | # Genreate custom mipchain 130 | if not auto_mipmaps: 131 | mip_chain = [init.clone().detach().requires_grad_(True)] 132 | while mip_chain[-1].shape[1] > 1 or mip_chain[-1].shape[2] > 1: 133 | new_size = [max(mip_chain[-1].shape[1] // 2, 1), max(mip_chain[-1].shape[2] // 2, 1)] 134 | mip_chain += [util.scale_img_nhwc(mip_chain[-1], new_size)] 135 | return Texture2D(mip_chain, min_max=min_max) 136 | else: 137 | return Texture2D(init, min_max=min_max) 138 | 139 | ######################################################################################################## 140 | # Convert texture to and from SRGB 141 | ######################################################################################################## 142 | 143 | def srgb_to_rgb(texture): 144 | return Texture2D(list(util.srgb_to_rgb(mip) for mip in texture.getMips())) 145 | 146 | def rgb_to_srgb(texture): 147 | return Texture2D(list(util.rgb_to_srgb(mip) for mip in texture.getMips())) 148 | 149 | ######################################################################################################## 150 | # Utility functions for loading / storing a texture 151 | ######################################################################################################## 152 | 153 | def _load_mip2D(fn, lambda_fn=None, channels=None): 154 | imgdata = torch.tensor(util.load_image(fn), dtype=torch.float32, device='cuda') 155 | if channels is not None: 156 | imgdata = imgdata[..., 0:channels] 157 | if lambda_fn is not None: 158 | imgdata = lambda_fn(imgdata) 159 | return imgdata.detach().clone() 160 | 161 | def load_texture2D(fn, lambda_fn=None, channels=None): 162 | base, ext = os.path.splitext(fn) 163 | if os.path.exists(base + "_0" + ext): 164 | mips = [] 165 | while os.path.exists(base + ("_%d" % len(mips)) + ext): 166 | mips += [_load_mip2D(base + ("_%d" % len(mips)) + ext, lambda_fn, channels)] 167 | return Texture2D(mips) 168 | else: 169 | return Texture2D(_load_mip2D(fn, lambda_fn, channels)) 170 | 171 | def _save_mip2D(fn, mip, mipidx, lambda_fn): 172 | if lambda_fn is not None: 173 | data = lambda_fn(mip).detach().cpu().numpy() 174 | else: 175 | data = mip.detach().cpu().numpy() 176 | 177 | if mipidx is None: 178 | util.save_image(fn, data) 179 | else: 180 | base, ext = os.path.splitext(fn) 181 | util.save_image(base + ("_%d" % mipidx) + ext, data) 182 | 183 | def save_texture2D(fn, tex, lambda_fn=None): 184 | if isinstance(tex.data, list): 185 | for i, mip in enumerate(tex.data): 186 | _save_mip2D(fn, mip[0,...], i, lambda_fn) 187 | else: 188 | _save_mip2D(fn, tex.data[0,...], None, lambda_fn) 189 | -------------------------------------------------------------------------------- /data/examples.zip: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:87906c08154d3cf11a3c09e0d6a94f716e63f8551410dee99eb0c66b2d1359bd 3 | size 988445061 4 | -------------------------------------------------------------------------------- /data/svd-cache/compute_latents.py: -------------------------------------------------------------------------------- 1 | import os 2 | from os import path as osp 3 | from glob import glob 4 | from tqdm import tqdm 5 | import torch 6 | from PIL import Image 7 | import cv2 8 | import argparse 9 | import numpy as np 10 | 11 | from diffusers import StableVideoDiffusionPipeline 12 | from diffusers.image_processor import VaeImageProcessor 13 | 14 | 15 | def convert_rgba_to_rgb_with_white_bg(image_path, background_color=None, background_image_path=None): 16 | """ 17 | Convert an RGBA image to an RGB image, replacing the alpha channel with a white background. 18 | 19 | :param image_path: str - The path to the source image with an RGBA channel. 20 | :return: Image - The converted PIL Image with an RGB channel. 21 | """ 22 | # Read the image with the alpha channel using cv2 23 | image = cv2.imread(image_path, cv2.IMREAD_UNCHANGED) 24 | 25 | # If the image has an alpha channel, process it; otherwise, raise an error 26 | if image is not None and image.shape[2] == 4: 27 | # Extract the alpha channel and create a white background of the same size as the image 28 | alpha_channel = image[:, :, 3] 29 | white_background = np.full(image[:, :, :3].shape, 255, dtype=np.uint8) 30 | 31 | if background_color is None and background_image_path is None: 32 | background = white_background 33 | elif background_color is not None: 34 | background = np.tile(background_color, image[:, :, :1].shape) 35 | background = background.astype(np.uint8) 36 | else: 37 | background = process_image(background_image_path, output_size=image.shape[0]) 38 | 39 | # Use the alpha channel as a mask to blend the image with the white background 40 | front = image[:, :, :3] * (alpha_channel / 255)[:, :, np.newaxis] 41 | back = background * (1 - alpha_channel / 255)[:, :, np.newaxis] 42 | converted_image = np.uint8(front + back) 43 | 44 | # Convert the OpenCV BGR format to PIL RGB format 45 | converted_image_pil = Image.fromarray(converted_image[:, :, ::-1], 'RGB') 46 | return converted_image_pil 47 | else: 48 | return Image.open(image_path).convert("RGB") 49 | 50 | 51 | def pad_image(image, target_size, padding_color=(255, 255, 255)): 52 | """Pad the image to the target size if it is smaller.""" 53 | width, height = image.size 54 | new_width, new_height = target_size, target_size 55 | 56 | # Create a new image with the target size and a black background 57 | new_image = Image.new('RGB', (new_width, new_height), padding_color) 58 | 59 | # Calculate the position to paste the original image on the new image 60 | paste_x = (new_width - width) // 2 61 | paste_y = (new_height - height) // 2 62 | 63 | # Paste the original image on the new image 64 | new_image.paste(image, (paste_x, paste_y)) 65 | 66 | return new_image 67 | 68 | def center_crop(image, size): 69 | """Center crop the image to the given size.""" 70 | width, height = image.size 71 | new_width, new_height = size, size 72 | 73 | left = (width - new_width) / 2 74 | top = (height - new_height) / 2 75 | right = (width + new_width) / 2 76 | bottom = (height + new_height) / 2 77 | 78 | return image.crop((left, top, right, bottom)) 79 | 80 | def process_image(image_path, output_size=256): 81 | """Read an image, pad it if smaller, center crop it to the specified size, and convert to np.uint8 array.""" 82 | # Open the image 83 | with Image.open(image_path) as img: 84 | # Ensure the image is in RGB mode 85 | img = img.convert('RGB') 86 | 87 | # Pad the image if it's smaller than the target size 88 | if img.size[0] < output_size or img.size[1] < output_size: 89 | img = pad_image(img, output_size) 90 | 91 | # Perform the center crop 92 | img_cropped = center_crop(img, output_size) 93 | 94 | # Convert to np.uint8 95 | img_array = np.array(img_cropped, dtype=np.uint8) 96 | 97 | return img_array 98 | 99 | def tensor2vid(video: torch.Tensor, processor: VaeImageProcessor, output_type: str = "np"): 100 | batch_size, channels, num_frames, height, width = video.shape 101 | outputs = [] 102 | for batch_idx in range(batch_size): 103 | batch_vid = video[batch_idx].permute(1, 0, 2, 3) 104 | batch_output = processor.postprocess(batch_vid, output_type) 105 | 106 | outputs.append(batch_output) 107 | 108 | if output_type == "np": 109 | outputs = np.stack(outputs) 110 | 111 | elif output_type == "pt": 112 | outputs = torch.stack(outputs) 113 | 114 | elif not output_type == "pil": 115 | raise ValueError(f"{output_type} does not exist. Please choose one of ['np', 'pt', 'pil']") 116 | 117 | return outputs 118 | 119 | 120 | batch_size = 128 121 | device = "cuda" 122 | 123 | pipe = StableVideoDiffusionPipeline.from_pretrained( 124 | "stabilityai/stable-video-diffusion-img2vid", 125 | torch_dtype=torch.float16, variant="fp16" 126 | ) 127 | pipe.vae.to(dtype=torch.float16, device=device) 128 | 129 | 130 | if __name__ == "__main__": 131 | parser = argparse.ArgumentParser() 132 | parser.add_argument("--num_jobs", type=int, default=1) 133 | parser.add_argument("--job_id", type=int, default=0) 134 | parser.add_argument("--save_dir", type=str, required=True) 135 | parser.add_argument("--dataset_root", type=str, required=True) 136 | args = parser.parse_args() 137 | 138 | all_jobs = [] 139 | dataset_root = args.dataset_root 140 | for category in sorted(os.listdir(dataset_root)): 141 | for obj in tqdm(sorted(os.listdir(osp.join(dataset_root, category)))): 142 | if len(glob(osp.join(dataset_root, category, obj, "rendered.flag"))) == 0: 143 | continue 144 | for action in sorted(os.listdir(osp.join(dataset_root, category, obj))): 145 | if not osp.isdir(osp.join(dataset_root, category, obj, action)): 146 | continue 147 | all_jobs.append((dataset_root, category, obj, action)) 148 | 149 | for dataset_root, category, obj, action in tqdm(all_jobs[args.job_id::args.num_jobs]): 150 | all_image_paths = sorted(glob(osp.join(dataset_root, category, obj, action, "*.png"))) 151 | if len(all_image_paths) == 0: 152 | continue 153 | num_computed_mean = len(glob(osp.join(args.save_dir, category, obj, action, "*_mean.pt"))) 154 | num_computed_std = len(glob(osp.join(args.save_dir, category, obj, action, "*_std.pt"))) 155 | if num_computed_mean == len(all_image_paths) and num_computed_std == len(all_image_paths): 156 | continue 157 | 158 | for i in range(0, len(all_image_paths), batch_size): 159 | image_paths = all_image_paths[i:i+batch_size] 160 | images = [convert_rgba_to_rgb_with_white_bg(image_path) for image_path in image_paths] 161 | with torch.no_grad(): 162 | images = pipe.image_processor.preprocess(images, height=256, width=256).to(device=device, dtype=torch.float16) 163 | latent_dist = pipe.vae.encode(images).latent_dist 164 | 165 | for j, image_path in enumerate(image_paths): 166 | latent_std = latent_dist.std[j].cpu() 167 | latent_mean = latent_dist.mean[j].cpu() 168 | latent_path_std = osp.join(args.save_dir, category, obj, action, image_path.split("/")[-1].replace(".png", "_latents_32_std.pt")) 169 | latent_path_mean = osp.join(args.save_dir, category, obj, action, image_path.split("/")[-1].replace(".png", "_latents_32_mean.pt")) 170 | os.makedirs(osp.dirname(latent_path_std), exist_ok=True) 171 | torch.save(latent_std, latent_path_std) 172 | torch.save(latent_mean, latent_path_mean) 173 | -------------------------------------------------------------------------------- /data/svd-cache/extract_embeddings.py: -------------------------------------------------------------------------------- 1 | import os 2 | from os import path as osp 3 | from glob import glob 4 | from tqdm import tqdm 5 | import torch 6 | from PIL import Image 7 | import cv2 8 | import argparse 9 | import numpy as np 10 | 11 | from diffusers import StableVideoDiffusionPipeline 12 | 13 | def convert_rgba_to_rgb_with_white_bg(image_path, background_color=None, background_image_path=None): 14 | """ 15 | Convert an RGBA image to an RGB image, replacing the alpha channel with a white background. 16 | 17 | :param image_path: str - The path to the source image with an RGBA channel. 18 | :return: Image - The converted PIL Image with an RGB channel. 19 | """ 20 | # Read the image with the alpha channel using cv2 21 | image = cv2.imread(image_path, cv2.IMREAD_UNCHANGED) 22 | 23 | # If the image has an alpha channel, process it; otherwise, raise an error 24 | if image is not None and image.shape[2] == 4: 25 | # Extract the alpha channel and create a white background of the same size as the image 26 | alpha_channel = image[:, :, 3] 27 | white_background = np.full(image[:, :, :3].shape, 255, dtype=np.uint8) 28 | 29 | if background_color is None and background_image_path is None: 30 | background = white_background 31 | elif background_color is not None: 32 | background = np.tile(background_color, image[:, :, :1].shape) 33 | background = background.astype(np.uint8) 34 | else: 35 | background = process_image(background_image_path, output_size=image.shape[0]) 36 | 37 | # Use the alpha channel as a mask to blend the image with the white background 38 | front = image[:, :, :3] * (alpha_channel / 255)[:, :, np.newaxis] 39 | back = background * (1 - alpha_channel / 255)[:, :, np.newaxis] 40 | converted_image = np.uint8(front + back) 41 | 42 | # Convert the OpenCV BGR format to PIL RGB format 43 | converted_image_pil = Image.fromarray(converted_image[:, :, ::-1], 'RGB') 44 | return converted_image_pil 45 | else: 46 | return Image.open(image_path).convert("RGB") 47 | 48 | 49 | def pad_image(image, target_size, padding_color=(255, 255, 255)): 50 | """Pad the image to the target size if it is smaller.""" 51 | width, height = image.size 52 | new_width, new_height = target_size, target_size 53 | 54 | # Create a new image with the target size and a black background 55 | new_image = Image.new('RGB', (new_width, new_height), padding_color) 56 | 57 | # Calculate the position to paste the original image on the new image 58 | paste_x = (new_width - width) // 2 59 | paste_y = (new_height - height) // 2 60 | 61 | # Paste the original image on the new image 62 | new_image.paste(image, (paste_x, paste_y)) 63 | 64 | return new_image 65 | 66 | 67 | def center_crop(image, size): 68 | """Center crop the image to the given size.""" 69 | width, height = image.size 70 | new_width, new_height = size, size 71 | 72 | left = (width - new_width) / 2 73 | top = (height - new_height) / 2 74 | right = (width + new_width) / 2 75 | bottom = (height + new_height) / 2 76 | 77 | return image.crop((left, top, right, bottom)) 78 | 79 | 80 | def process_image(image_path, output_size=256): 81 | """Read an image, pad it if smaller, center crop it to the specified size, and convert to np.uint8 array.""" 82 | # Open the image 83 | with Image.open(image_path) as img: 84 | # Ensure the image is in RGB mode 85 | img = img.convert('RGB') 86 | 87 | # Pad the image if it's smaller than the target size 88 | if img.size[0] < output_size or img.size[1] < output_size: 89 | img = pad_image(img, output_size) 90 | 91 | # Perform the center crop 92 | img_cropped = center_crop(img, output_size) 93 | 94 | # Convert to np.uint8 95 | img_array = np.array(img_cropped, dtype=np.uint8) 96 | 97 | return img_array 98 | 99 | 100 | batch_size = 512 101 | device = "cuda" 102 | 103 | pipe = StableVideoDiffusionPipeline.from_pretrained( 104 | "stabilityai/stable-video-diffusion-img2vid", 105 | torch_dtype=torch.float16, variant="fp16" 106 | ) 107 | pipe.image_encoder = pipe.image_encoder.to(device) 108 | 109 | 110 | if __name__ == "__main__": 111 | parser = argparse.ArgumentParser() 112 | parser.add_argument("--num_jobs", type=int, default=1) 113 | parser.add_argument("--job_id", type=int, default=0) 114 | parser.add_argument("--save_dir", type=str, required=True) 115 | parser.add_argument("--dataset_root", type=str, required=True) 116 | args = parser.parse_args() 117 | 118 | all_jobs = [] 119 | dataset_root = args.dataset_root 120 | for category in sorted(os.listdir(dataset_root)): 121 | for obj in tqdm(sorted(os.listdir(osp.join(dataset_root, category)))): 122 | if len(glob(osp.join(dataset_root, category, obj, "rendered.flag"))) == 0: 123 | continue 124 | for action in sorted(os.listdir(osp.join(dataset_root, category, obj))): 125 | if not osp.isdir(osp.join(dataset_root, category, obj, action)): 126 | continue 127 | all_jobs.append((dataset_root, category, obj, action)) 128 | 129 | for dataset_root, category, obj, action in tqdm(all_jobs[args.job_id::args.num_jobs]): 130 | 131 | all_image_paths = sorted(glob(osp.join(dataset_root, category, obj, action, "*.png"))) 132 | if len(all_image_paths) == 0: 133 | continue 134 | num_computed = len([p for p in glob(osp.join(args.save_dir, category, obj, action, "*.pt")) if "latents" not in p]) 135 | if num_computed == len(all_image_paths): 136 | continue 137 | 138 | for i in range(0, len(all_image_paths), batch_size): 139 | image_paths = all_image_paths[i:i+batch_size] 140 | images = [convert_rgba_to_rgb_with_white_bg(image_path).resize((256, 256), resample=Image.LANCZOS) for image_path in image_paths] 141 | with torch.no_grad(): 142 | embeddings = pipe._encode_image(images, device=device, num_videos_per_prompt=1, do_classifier_free_guidance=False) 143 | for j, image_path in enumerate(image_paths): 144 | embedding = embeddings[j] 145 | embedding_path = osp.join(args.save_dir, category, obj, action, image_path.split("/")[-1].replace(".png", "_embedding.pt")) 146 | os.makedirs(osp.dirname(embedding_path), exist_ok=True) 147 | torch.save(embedding, embedding_path) 148 | -------------------------------------------------------------------------------- /dataset.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import os.path as osp 4 | from glob import glob 5 | import random 6 | from typing import List 7 | 8 | import numpy as np 9 | import torch 10 | from torch.utils.data import Dataset 11 | 12 | 13 | REDUCE_DUPLICATE_DRAGS = True 14 | 15 | 16 | class DragVideoDataset(Dataset): 17 | """ 18 | An example dataset to load the drag samples together with the rendered animations. 19 | Note we pre-compute the latents and embeddings for each frame and save them in latent_dist_roots and embedding_roots, respectively. 20 | """ 21 | def __init__( 22 | self, 23 | latent_dist_roots: List[str], 24 | embedding_roots: List[str], 25 | drag_roots: List[str], 26 | num_max_drags: int = 10, 27 | num_drag_samples: int = 20, 28 | num_views: int = 12, 29 | num_frames: int = 14, 30 | sample_size: int = 256, 31 | ): 32 | """ 33 | Args: 34 | latent_dist_roots: List[str] 35 | Path to the root of the latent distribution. Note the order of the roots must match the order of the drag_roots. 36 | embedding_roots: List[str] 37 | Path to the root of the embedding. Note the order of the roots must match the order of the drag_roots. 38 | drag_roots: List[str] 39 | Path to the root of the drag. Note the order of the roots must match the order of the latent_dist_roots and embedding_roots. 40 | """ 41 | super().__init__() 42 | 43 | self.latent_dist_roots = latent_dist_roots 44 | self.embedding_roots = embedding_roots 45 | self.drag_roots = drag_roots 46 | assert len(latent_dist_roots) == len(embedding_roots) == len(drag_roots), \ 47 | "The length of latent_dist_roots, embedding_roots, and drag_roots must be the same. Got {} {} {}".format( 48 | len(latent_dist_roots), len(embedding_roots), len(drag_roots)) 49 | 50 | self.num_max_drags = num_max_drags 51 | self.num_drag_samples = num_drag_samples 52 | self.num_views = num_views 53 | self.num_frames = num_frames 54 | 55 | self.sample_size = sample_size 56 | 57 | self.obj_action_tuples = [] 58 | for obj_idx, obj_drag_root in enumerate(drag_roots): 59 | for action in os.listdir(obj_drag_root): 60 | self.obj_action_tuples.append((obj_idx, action)) 61 | 62 | def __len__(self): 63 | return len(self.obj_action_tuples) 64 | 65 | def get_batch(self, index): 66 | obj_idx, action = self.obj_action_tuples[index % len(self.obj_action_tuples)] 67 | sample_id = np.random.randint(self.num_drag_samples) 68 | sample_fpath = osp.join(self.drag_roots[obj_idx], action, f"sample_{sample_id:03d}.json") 69 | 70 | with open(sample_fpath, "r") as f: 71 | sample = json.load(f) 72 | assert len(sample["all_tracks"]) == self.num_views 73 | view_id = np.random.randint(self.num_views) 74 | 75 | start_frame, fps = sample["start_frame"], sample["fps"] 76 | drags = sample["all_tracks"][view_id] # num_points, num_frames, 2 77 | if len(drags) > self.num_max_drags: 78 | drags = random.sample(drags, self.num_max_drags) 79 | 80 | # Load latents 81 | latents = [] 82 | all_latents_mean_files = sorted(glob(osp.join( 83 | self.latent_dist_roots[obj_idx], action, f"{view_id:03d}_*_latents_{self.sample_size // 8}_mean.pt"))) 84 | start_frame_id = all_latents_mean_files[start_frame].split("_")[-4] 85 | 86 | cond_latent_fpath = all_latents_mean_files[start_frame] 87 | cond_latent = torch.load(cond_latent_fpath, map_location="cpu") 88 | 89 | for frame_id in range(start_frame, start_frame + self.num_frames * fps, fps): 90 | latent_mean_fpath = all_latents_mean_files[frame_id] 91 | latent_std_fpath = latent_mean_fpath.replace("mean", "std") 92 | latent_mean = torch.load(latent_mean_fpath, map_location="cpu") 93 | latent_std = torch.load(latent_std_fpath, map_location="cpu") 94 | latents.append(latent_mean + latent_std * torch.randn_like(latent_mean)) 95 | latents = torch.stack(latents) 96 | 97 | # Load embedding 98 | embedding_fpath = osp.join( 99 | self.embedding_roots[obj_idx], action, 100 | f"{view_id:03d}_{start_frame_id}_embedding.pt" 101 | ) 102 | embedding = torch.load(embedding_fpath, map_location="cpu") 103 | 104 | # Preprocess drags 105 | # 0. Sanity check 106 | assert all([len(drag_point) == self.num_frames for drag_point in drags]) 107 | assert all([len(coord) == 2 for drag_point in drags for coord in drag_point]) 108 | # 1. Normalize to [0, 1] 109 | if len(drags) == 0: 110 | drags = torch.zeros(self.num_frames, self.num_max_drags, 2) 111 | else: 112 | drags = torch.Tensor(drags).permute(1, 0, 2) # num_frames, num_points, 2 113 | 114 | if REDUCE_DUPLICATE_DRAGS: 115 | removed_drags = [] 116 | # 1. Remove parallel drags 117 | for i in range(drags.shape[1]): 118 | if i in removed_drags: 119 | continue 120 | for j in range(i + 1, drags.shape[1]): 121 | if torch.norm(drags[:, i] - drags[:, j], dim=-1).sum() <= drags.shape[0] * 20: 122 | removed_drags.append(j) 123 | drags = torch.cat([drags[:, i:i+1] for i in range(drags.shape[1]) if i not in removed_drags], dim=1) 124 | 125 | removed_drags = [] 126 | # 2. Calculate the total displacement of each drag 127 | displacement = torch.norm(drags[1:] - drags[:-1], dim=-1).sum(dim=0) 128 | max_drag_idx = displacement.argmax() 129 | for i in range(drags.shape[1]): 130 | if i != max_drag_idx: 131 | if torch.rand(1).item() > displacement[i] / displacement.max() * 2: 132 | removed_drags.append(i) 133 | drags = torch.cat([drags[:, i:i+1] for i in range(drags.shape[1]) if i not in removed_drags], dim=1) 134 | 135 | drags = drags / 512. 136 | drags = torch.cat([drags[0:1].expand_as(drags), drags], dim=-1) 137 | drags = torch.cat([drags, torch.zeros(self.num_frames, self.num_max_drags - drags.shape[1], 4)], dim=1) 138 | 139 | return latents, cond_latent, embedding, drags 140 | 141 | def __getitem__(self, index): 142 | latents, cond_latent, embedding, drags = self.get_batch(index) 143 | return dict( 144 | latents=latents.to(dtype=torch.float32).mul_(0.18215), 145 | cond_latent=cond_latent.to(dtype=torch.float32), 146 | embedding=embedding.to(dtype=torch.float32), 147 | drags=drags.to(dtype=torch.float32), 148 | ) -------------------------------------------------------------------------------- /network_utils.py: -------------------------------------------------------------------------------- 1 | from typing import Tuple 2 | 3 | import torch.nn as nn 4 | import torch.nn.functional as F 5 | import numpy as np 6 | 7 | 8 | class DragEmbedding(nn.Module): 9 | def __init__( 10 | self, 11 | conditioning_embedding_channels: int, # out channel 12 | conditioning_channels: int = 3, 13 | block_out_channels: Tuple[int, ...] = (16, 32, 96), 14 | ): 15 | super().__init__() 16 | 17 | self.conv_in = nn.Conv2d(conditioning_channels, block_out_channels[0], kernel_size=3, padding=1) 18 | 19 | self.blocks = nn.ModuleList([]) 20 | 21 | for i in range(len(block_out_channels) - 1): 22 | channel_in = block_out_channels[i] 23 | channel_out = block_out_channels[i + 1] 24 | self.blocks.append(nn.Conv2d(channel_in, channel_in, kernel_size=3, padding=1)) 25 | self.blocks.append(nn.Conv2d(channel_in, channel_out, kernel_size=3, padding=1)) 26 | 27 | self.conv_out = zero_module( 28 | nn.Conv2d(block_out_channels[-1], conditioning_embedding_channels, kernel_size=3, padding=1) 29 | ) 30 | 31 | def forward(self, conditioning): 32 | conditioning_ndims = len(conditioning.shape) 33 | if conditioning_ndims == 5: 34 | batch_size, num_frames, num_channels, h, w = conditioning.shape 35 | conditioning = conditioning.flatten(0, 1) 36 | 37 | embedding = self.conv_in(conditioning) 38 | embedding = F.silu(embedding) 39 | 40 | for block in self.blocks: 41 | embedding = block(embedding) 42 | embedding = F.silu(embedding) 43 | 44 | embedding = self.conv_out(embedding) 45 | if conditioning_ndims == 5: 46 | embedding = embedding.view(batch_size, num_frames, *embedding.shape[1:]) 47 | 48 | return embedding 49 | 50 | 51 | def zero_module(module): 52 | for p in module.parameters(): 53 | nn.init.zeros_(p) 54 | return module 55 | 56 | 57 | def get_2d_sincos_pos_embed(embed_dim, grid_size, cls_token=False, extra_tokens=0): 58 | """ 59 | grid_size: int of the grid height and width 60 | return: 61 | pos_embed: [grid_size*grid_size, embed_dim] or [1+grid_size*grid_size, embed_dim] (w/ or w/o cls_token) 62 | """ 63 | grid_h = np.arange(grid_size, dtype=np.float32) 64 | grid_w = np.arange(grid_size, dtype=np.float32) 65 | grid = np.meshgrid(grid_w, grid_h) # here w goes first 66 | grid = np.stack(grid, axis=0) 67 | 68 | grid = grid.reshape([2, 1, grid_size, grid_size]) 69 | pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid) 70 | if cls_token and extra_tokens > 0: 71 | pos_embed = np.concatenate([np.zeros([extra_tokens, embed_dim]), pos_embed], axis=0) 72 | return pos_embed 73 | 74 | 75 | def get_2d_sincos_pos_embed_from_grid(embed_dim, grid): 76 | assert embed_dim % 2 == 0 77 | 78 | # use half of dimensions to encode grid_h 79 | emb_h = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[0]) # (H*W, D/2) 80 | emb_w = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[1]) # (H*W, D/2) 81 | 82 | emb = np.concatenate([emb_h, emb_w], axis=1) # (H*W, D) 83 | return emb 84 | 85 | 86 | def get_1d_sincos_pos_embed_from_grid(embed_dim, pos): 87 | """ 88 | embed_dim: output dimension for each position 89 | pos: a list of positions to be encoded: size (M,) 90 | out: (M, D) 91 | """ 92 | assert embed_dim % 2 == 0 93 | omega = np.arange(embed_dim // 2, dtype=np.float64) 94 | omega /= embed_dim / 2. 95 | omega = 1. / 10000**omega # (D/2,) 96 | 97 | pos = pos.reshape(-1) # (M,) 98 | out = np.einsum('m,d->md', pos, omega) # (M, D/2), outer product 99 | 100 | emb_sin = np.sin(out) # (M, D/2) 101 | emb_cos = np.cos(out) # (M, D/2) 102 | 103 | emb = np.concatenate([emb_sin, emb_cos], axis=1) # (M, D) 104 | return emb -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | accelerate==0.31.0 2 | aiofiles==23.2.1 3 | aiohttp==3.9.5 4 | aiosignal==1.3.1 5 | aiostream==0.5.2 6 | altair==5.3.0 7 | annotated-types==0.7.0 8 | antlr4-python3-runtime==4.9.3 9 | anyio==4.4.0 10 | async-timeout==4.0.3 11 | attrs==23.2.0 12 | beautifulsoup4==4.12.3 13 | Brotli==1.1.0 14 | certifi==2024.6.2 15 | charset-normalizer==3.3.2 16 | click==8.1.7 17 | cloudpickle==3.0.0 18 | contourpy==1.2.1 19 | cycler==0.12.1 20 | diffusers @ git+https://github.com/huggingface/diffusers@8e1b7a084addc4711b8d9be2738441dfad680ce0 21 | dnspython==2.6.1 22 | docker-pycreds==0.4.0 23 | einops==0.8.0 24 | email_validator==2.2.0 25 | exceptiongroup==1.2.1 26 | fastapi==0.111.0 27 | fastapi-cli==0.0.4 28 | ffmpy==0.3.2 29 | filelock==3.13.1 30 | fonttools==4.53.0 31 | frozenlist==1.4.1 32 | fsspec==2024.2.0 33 | gdown==5.2.0 34 | gitdb==4.0.11 35 | GitPython==3.1.43 36 | gradio==4.37.2 37 | gradio_client==1.0.2 38 | grpclib==0.4.7 39 | h11==0.14.0 40 | h2==4.1.0 41 | h5py==3.11.0 42 | hpack==4.0.0 43 | httpcore==1.0.5 44 | httptools==0.6.1 45 | httpx==0.27.0 46 | huggingface-hub==0.23.4 47 | hydra-core==1.3.1 48 | hyperframe==6.0.1 49 | idna==3.7 50 | imageio==2.34.1 51 | importlib_metadata==7.1.0 52 | importlib_resources==6.4.0 53 | Jinja2==3.1.3 54 | jsonschema==4.22.0 55 | jsonschema-specifications==2023.12.1 56 | kiwisolver==1.4.5 57 | lazy_loader==0.4 58 | lightning-utilities==0.11.2 59 | markdown-it-py==3.0.0 60 | MarkupSafe==2.1.5 61 | matplotlib==3.9.0 62 | mdurl==0.1.2 63 | modal==0.63.2 64 | mpmath==1.3.0 65 | multidict==6.0.5 66 | mutagen==1.47.0 67 | networkx==3.2.1 68 | numpy==1.26.3 69 | nvidia-cublas-cu11==11.11.3.6 70 | nvidia-cuda-cupti-cu11==11.8.87 71 | nvidia-cuda-nvrtc-cu11==11.8.89 72 | nvidia-cuda-runtime-cu11==11.8.89 73 | nvidia-cudnn-cu11==8.7.0.84 74 | nvidia-cufft-cu11==10.9.0.58 75 | nvidia-curand-cu11==10.3.0.86 76 | nvidia-cusolver-cu11==11.4.1.48 77 | nvidia-cusparse-cu11==11.7.5.86 78 | nvidia-nccl-cu11==2.20.5 79 | nvidia-nvtx-cu11==11.8.86 80 | omegaconf==2.3.0 81 | opencv-python==4.10.0.82 82 | orjson==3.10.5 83 | packaging==24.1 84 | pandas==2.2.2 85 | pillow==10.2.0 86 | platformdirs==4.2.2 87 | protobuf==4.25.3 88 | psutil==5.9.8 89 | pycryptodomex==3.20.0 90 | pydantic==2.7.4 91 | pydantic_core==2.18.4 92 | pydub==0.25.1 93 | Pygments==2.18.0 94 | pyparsing==3.1.2 95 | PySocks==1.7.1 96 | python-dateutil==2.9.0.post0 97 | python-dotenv==1.0.1 98 | python-multipart==0.0.9 99 | pytube==15.0.0 100 | pytube3==9.6.4 101 | pytz==2024.1 102 | PyYAML==6.0.1 103 | referencing==0.35.1 104 | regex==2024.5.15 105 | requests==2.32.3 106 | rich==13.7.1 107 | rpds-py==0.18.1 108 | ruff==0.5.0 109 | safetensors==0.4.3 110 | scikit-image==0.24.0 111 | scipy==1.13.1 112 | segment-anything==1.0 113 | semantic-version==2.10.0 114 | sentry-sdk==2.5.1 115 | setproctitle==1.3.3 116 | shellingham==1.5.4 117 | sigtools==4.0.1 118 | six==1.16.0 119 | smmap==5.0.1 120 | sniffio==1.3.1 121 | soupsieve==2.5 122 | spaces==0.29.3 123 | starlette==0.37.2 124 | submitit==1.5.1 125 | sympy==1.12 126 | synchronicity==0.6.7 127 | tifffile==2024.6.18 128 | tokenizers==0.19.1 129 | toml==0.10.2 130 | tomlkit==0.12.0 131 | toolz==0.12.1 132 | torch==2.3.0+cu118 133 | torchaudio==2.3.1+cu118 134 | torchmetrics==1.4.0.post0 135 | torchvision==0.18.1+cu118 136 | tqdm==4.66.4 137 | transformers==4.41.2 138 | triton==2.3.0 139 | typer==0.12.3 140 | types-certifi==2021.10.8.3 141 | types-toml==0.10.8.20240310 142 | typing_extensions==4.9.0 143 | tzdata==2024.1 144 | ujson==5.10.0 145 | urllib3==2.2.1 146 | uvicorn==0.30.1 147 | uvloop==0.19.0 148 | wandb==0.17.1 149 | watchfiles==0.22.0 150 | websockets==11.0.3 151 | xformers==0.0.26.post1+cu118 152 | yarl==1.9.4 153 | youtube-dl==2021.12.17 154 | yt-dlp==2024.5.27 155 | zipp==3.19.2 156 | -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import numpy as np 3 | import cv2 4 | from PIL import Image 5 | from copy import deepcopy 6 | from tqdm import tqdm 7 | 8 | from diffusers.image_processor import VaeImageProcessor 9 | from diffusers.utils import export_to_gif 10 | 11 | 12 | def tensor2vid(video: torch.Tensor, processor: VaeImageProcessor, output_type: str = "np"): 13 | batch_size = video.shape[0] 14 | outputs = [] 15 | for batch_idx in range(batch_size): 16 | batch_vid = video[batch_idx].permute(1, 0, 2, 3) 17 | batch_output = processor.postprocess(batch_vid, output_type) 18 | 19 | outputs.append(batch_output) 20 | 21 | if output_type == "np": 22 | outputs = np.stack(outputs) 23 | 24 | elif output_type == "pt": 25 | outputs = torch.stack(outputs) 26 | 27 | elif not output_type == "pil": 28 | raise ValueError(f"{output_type} does not exist. Please choose one of ['np', 'pt', 'pil']") 29 | 30 | return outputs 31 | 32 | 33 | def decode_latents_and_save(vae, image_processor, latents, save_path, drags=None): 34 | with torch.no_grad(): 35 | frames = vae.decode(latents.to(torch.float16) / 0.18215, num_frames=14).sample.float() 36 | frames = tensor2vid(frames[None].detach().permute(0, 2, 1, 3, 4), image_processor, output_type="pil")[0] 37 | 38 | # Add drag visualizations. 39 | if drags is not None: 40 | final_video = [] 41 | for fid, frame in enumerate(frames): 42 | frame_np = np.array(frame).copy() 43 | for pid in range(drags.shape[1]): 44 | if (drags[fid, pid] != 0).any(): 45 | frame_np = cv2.circle( 46 | frame_np, 47 | (int(drags[fid, pid, 0] * 256), int(drags[fid, pid, 1] * 256)), 48 | 3, (255, 0, 0), -1 49 | ) 50 | frame_np = cv2.circle( 51 | frame_np, 52 | (int(drags[fid, pid, 2] * 256), int(drags[fid, pid, 3] * 256)), 53 | 3, (0, 255, 0), -1 54 | ) 55 | frame_np = cv2.line( 56 | frame_np, 57 | (int(drags[fid, pid, 0] * 256), int(drags[fid, pid, 1] * 256)), 58 | (int(drags[fid, pid, 2] * 256), int(drags[fid, pid, 3] * 256)), 59 | (0, 0, 255), 60 | 2 61 | ) 62 | final_video.append(Image.fromarray(frame_np)) 63 | else: 64 | final_video = frames 65 | 66 | export_to_gif(final_video, save_path) 67 | 68 | 69 | def sample_from_noise(model, scheduler, cond_latent, cond_embedding, drags, 70 | min_guidance=1.0, max_guidance=3.0, num_inference_steps=25, num_frames=14): 71 | model.eval() 72 | 73 | scheduler_inference = deepcopy(scheduler) 74 | scheduler_inference.set_timesteps(num_inference_steps, device=cond_latent.device) 75 | timesteps = scheduler_inference.timesteps 76 | do_classifier_free_guidance = max_guidance > 1 77 | latents = torch.randn((1, num_frames, 4, 32, 32)).to(cond_latent) * scheduler_inference.init_noise_sigma 78 | guidance_scale = torch.linspace(min_guidance, max_guidance, num_frames).unsqueeze(0).to(cond_latent)[..., None, None, None] 79 | 80 | for i, t in tqdm(enumerate(timesteps)): 81 | latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents 82 | latent_model_input = scheduler_inference.scale_model_input(latent_model_input, t) 83 | 84 | with torch.no_grad(): 85 | noise_pred = model( 86 | latent_model_input, 87 | t, 88 | image_latents=torch.cat([cond_latent, torch.zeros_like(cond_latent)]) if do_classifier_free_guidance else cond_latent, 89 | encoder_hidden_states=torch.cat([cond_embedding, torch.zeros_like(cond_embedding)]) if do_classifier_free_guidance else cond_embedding, 90 | added_time_ids=torch.FloatTensor([[6, 127, 0.02] * 2]).to(cond_latent) if do_classifier_free_guidance else torch.FloatTensor([[6, 127, 0.02]]).to(cond_latent), 91 | drags=torch.cat([drags, torch.zeros_like(drags)]) if do_classifier_free_guidance else drags, 92 | ) 93 | 94 | if do_classifier_free_guidance: 95 | noise_pred_cond, noise_pred_uncond = noise_pred.chunk(2) 96 | noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_cond - noise_pred_uncond) 97 | 98 | latents = scheduler_inference.step(noise_pred, t, latents).prev_sample 99 | 100 | return latents --------------------------------------------------------------------------------