├── GraphMotion ├── __init__.py ├── callback │ ├── __init__.py │ └── progress.py ├── config.py ├── data │ ├── HumanML3D.py │ ├── Humanact12.py │ ├── Kit.py │ ├── Uestc.py │ ├── __init__.py │ ├── a2m │ │ ├── __init__.py │ │ ├── dataset.py │ │ ├── humanact12poses.py │ │ ├── tools.py │ │ ├── uestc.py │ │ └── utils │ │ │ ├── __init__.py │ │ │ ├── misc.py │ │ │ ├── rotation_conversions.py │ │ │ └── tensors.py │ ├── base.py │ ├── get_data.py │ ├── humanml │ │ ├── __init__.py │ │ ├── common │ │ │ ├── quaternion.py │ │ │ └── skeleton.py │ │ ├── data │ │ │ ├── __init__.py │ │ │ └── dataset.py │ │ ├── dataset.py │ │ ├── scripts │ │ │ └── motion_process.py │ │ └── utils │ │ │ ├── __init__.py │ │ │ ├── get_opt.py │ │ │ ├── load_json.py │ │ │ ├── metrics.py │ │ │ ├── paramUtil.py │ │ │ ├── plot_script.py │ │ │ ├── utils.py │ │ │ └── word_vectorizer.py │ ├── sampling │ │ ├── __init__.py │ │ ├── base.py │ │ ├── framerate.py │ │ └── frames.py │ └── utils.py ├── launch │ ├── __init__.py │ ├── blender.py │ ├── prepare.py │ └── tools.py ├── models │ ├── __init__.py │ ├── architectures │ │ ├── __init__.py │ │ ├── actor_vae.py │ │ ├── bert.py │ │ ├── clip.py │ │ ├── denoiser.py │ │ ├── fc.py │ │ ├── gat.py │ │ ├── humanact12_gru.py │ │ ├── t2m_motionenc.py │ │ ├── t2m_textenc.py │ │ ├── tools │ │ │ ├── embeddings.py │ │ │ └── transformer_layers.py │ │ ├── uestc_stgcn.py │ │ ├── vae.py │ │ ├── vision_transformer.py │ │ └── vposert_vae.py │ ├── get_model.py │ ├── losses │ │ ├── GraphMotion.py │ │ ├── __init__.py │ │ ├── actor.py │ │ ├── kl.py │ │ ├── temos.py │ │ ├── tmost.py │ │ └── utils.py │ ├── metrics │ │ ├── __init__.py │ │ ├── compute.py │ │ ├── compute_best.py │ │ ├── compute_worst.py │ │ ├── gru.py │ │ ├── mm.py │ │ ├── mr.py │ │ ├── stgcn.py │ │ ├── tm2t.py │ │ ├── uncond.py │ │ └── utils.py │ ├── modeltype │ │ ├── GraphMotion.py │ │ ├── __init__.py │ │ └── base.py │ ├── operator │ │ ├── __init__.py │ │ ├── adain.py │ │ ├── blocks.py │ │ ├── conv2d_gradfix.py │ │ ├── cross_attention.py │ │ ├── position_encoding.py │ │ ├── position_encoding_layer.py │ │ └── self_attention.py │ └── tools │ │ ├── __init__.py │ │ ├── hessian_penalty.py │ │ └── tools.py ├── render │ ├── __init__.py │ ├── anim.py │ ├── blender │ │ ├── __init__.py │ │ ├── camera.py │ │ ├── data.py │ │ ├── floor.py │ │ ├── joints.py │ │ ├── materials.py │ │ ├── meshes.py │ │ ├── render.py │ │ ├── sampler.py │ │ ├── scene.py │ │ ├── tools.py │ │ └── vertices.py │ ├── renderer.py │ ├── rendermotion.py │ ├── video.py │ └── visualize.py ├── tools │ ├── __init__.py │ ├── geometry.py │ ├── logging.py │ └── runid.py ├── transforms │ ├── __init__.py │ ├── base.py │ ├── feats2smpl.py │ ├── identity.py │ ├── joints2jfeats │ │ ├── __init__.py │ │ ├── base.py │ │ ├── rifke.py │ │ └── tools.py │ ├── joints2rots │ │ ├── config.py │ │ ├── customloss.py │ │ ├── prior.py │ │ └── smplify.py │ ├── rotation2xyz.py │ ├── rots2joints │ │ ├── __init__.py │ │ ├── base.py │ │ └── smplh.py │ ├── rots2rfeats │ │ ├── __init__.py │ │ ├── base.py │ │ └── smplvelp.py │ ├── smpl.py │ └── xyz.py └── utils │ ├── __init__.py │ ├── demo_utils.py │ ├── easyconvert.py │ ├── fixseed.py │ ├── geometry.py │ ├── joints.py │ ├── logger.py │ ├── misc.py │ ├── rotation_conversions.py │ ├── sample_utils.py │ ├── temos_utils.py │ └── tensors.py ├── LICENSE ├── README.md ├── configs ├── assets.yaml ├── base.yaml ├── config_humanml3d.yaml ├── config_vae_humanml3d_action.yaml ├── config_vae_humanml3d_motion.yaml ├── config_vae_humanml3d_specific.yaml ├── modules │ ├── denoiser.yaml │ ├── evaluators.yaml │ ├── motion_vae.yaml │ ├── scheduler.yaml │ └── text_encoder.yaml ├── modules_humanact12 │ ├── denoiser.yaml │ ├── evaluators.yaml │ ├── motion_vae.yaml │ ├── scheduler.yaml │ └── text_encoder.yaml └── modules_novae │ ├── denoiser.yaml │ ├── evaluators.yaml │ ├── motion_vae.yaml │ ├── scheduler.yaml │ └── text_encoder.yaml ├── datasets ├── humanml3d │ ├── new_test_data.json │ └── new_train_data.json └── kit-ml │ ├── new_test_data.json │ └── new_train_data.json ├── demo.py ├── fit.py ├── mld ├── __init__.py ├── callback │ ├── __init__.py │ └── progress.py ├── config.py ├── data │ ├── HumanML3D.py │ ├── Humanact12.py │ ├── Kit.py │ ├── Uestc.py │ ├── __init__.py │ ├── a2m │ │ ├── __init__.py │ │ ├── dataset.py │ │ ├── humanact12poses.py │ │ ├── tools.py │ │ ├── uestc.py │ │ └── utils │ │ │ ├── __init__.py │ │ │ ├── misc.py │ │ │ ├── rotation_conversions.py │ │ │ └── tensors.py │ ├── base.py │ ├── get_data.py │ ├── humanml │ │ ├── __init__.py │ │ ├── common │ │ │ ├── quaternion.py │ │ │ └── skeleton.py │ │ ├── data │ │ │ ├── __init__.py │ │ │ └── dataset.py │ │ ├── dataset.py │ │ ├── scripts │ │ │ └── motion_process.py │ │ └── utils │ │ │ ├── __init__.py │ │ │ ├── get_opt.py │ │ │ ├── load_json.py │ │ │ ├── metrics.py │ │ │ ├── paramUtil.py │ │ │ ├── plot_script.py │ │ │ ├── utils.py │ │ │ └── word_vectorizer.py │ ├── sampling │ │ ├── __init__.py │ │ ├── base.py │ │ ├── framerate.py │ │ └── frames.py │ └── utils.py ├── launch │ ├── __init__.py │ ├── blender.py │ ├── prepare.py │ └── tools.py ├── models │ ├── __init__.py │ ├── architectures │ │ ├── __init__.py │ │ ├── actor_vae.py │ │ ├── fc.py │ │ ├── gat.py │ │ ├── humanact12_gru.py │ │ ├── mld_bert.py │ │ ├── mld_clip.py │ │ ├── mld_denoiser_large.py │ │ ├── mld_vae.py │ │ ├── t2m_motionenc.py │ │ ├── t2m_textenc.py │ │ ├── tools │ │ │ ├── embeddings.py │ │ │ └── transformer_layers.py │ │ ├── uestc_stgcn.py │ │ ├── vision_transformer.py │ │ └── vposert_vae.py │ ├── get_model.py │ ├── losses │ │ ├── __init__.py │ │ ├── actor.py │ │ ├── kl.py │ │ ├── mld.py │ │ ├── temos.py │ │ ├── tmost.py │ │ └── utils.py │ ├── metrics │ │ ├── __init__.py │ │ ├── compute.py │ │ ├── compute_best.py │ │ ├── compute_worst.py │ │ ├── gru.py │ │ ├── mm.py │ │ ├── mr.py │ │ ├── stgcn.py │ │ ├── tm2t.py │ │ ├── uncond.py │ │ └── utils.py │ ├── modeltype │ │ ├── __init__.py │ │ ├── base.py │ │ └── mld.py │ ├── operator │ │ ├── __init__.py │ │ ├── adain.py │ │ ├── blocks.py │ │ ├── conv2d_gradfix.py │ │ ├── cross_attention.py │ │ ├── position_encoding.py │ │ ├── position_encoding_layer.py │ │ └── self_attention.py │ └── tools │ │ ├── __init__.py │ │ ├── hessian_penalty.py │ │ └── tools.py ├── render │ ├── __init__.py │ ├── anim.py │ ├── blender │ │ ├── __init__.py │ │ ├── camera.py │ │ ├── data.py │ │ ├── floor.py │ │ ├── joints.py │ │ ├── materials.py │ │ ├── meshes.py │ │ ├── render.py │ │ ├── sampler.py │ │ ├── scene.py │ │ ├── tools.py │ │ └── vertices.py │ ├── renderer.py │ ├── rendermotion.py │ ├── video.py │ └── visualize.py ├── tools │ ├── __init__.py │ ├── geometry.py │ ├── logging.py │ └── runid.py ├── transforms │ ├── __init__.py │ ├── base.py │ ├── feats2smpl.py │ ├── identity.py │ ├── joints2jfeats │ │ ├── __init__.py │ │ ├── base.py │ │ ├── rifke.py │ │ └── tools.py │ ├── joints2rots │ │ ├── config.py │ │ ├── customloss.py │ │ ├── prior.py │ │ └── smplify.py │ ├── rotation2xyz.py │ ├── rots2joints │ │ ├── __init__.py │ │ ├── base.py │ │ └── smplh.py │ ├── rots2rfeats │ │ ├── __init__.py │ │ ├── base.py │ │ └── smplvelp.py │ ├── smpl.py │ └── xyz.py └── utils │ ├── __init__.py │ ├── demo_utils.py │ ├── easyconvert.py │ ├── fixseed.py │ ├── geometry.py │ ├── joints.py │ ├── logger.py │ ├── misc.py │ ├── rotation_conversions.py │ ├── sample_utils.py │ ├── temos_utils.py │ └── tensors.py ├── pictures ├── fig0.png ├── fig1.png ├── fig2.png ├── fig3.png ├── fig4.png └── fig5.png ├── prepare ├── download_pretrained_models.sh ├── download_smpl_model.sh ├── download_t2m_evaluators.sh ├── merge_smplh_mano.py ├── prepare_bert.sh ├── prepare_clip.sh ├── prepare_kit.sh ├── requirements_render.txt ├── role_graph.py └── smplh.sh ├── render.py ├── requirements.txt ├── scripts ├── fbx_output.py ├── fbx_output_smplx.py ├── fit_motion.sh ├── fit_motion_parallel.sh ├── flops.py ├── plys2npy.py ├── tsne.py ├── visualize_motion.sh └── visualize_motion_parallel.sh ├── test.py └── train.py /GraphMotion/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpthu17/GraphMotion/d56feba6f6d39867b933ca752244fcbbb703a4b8/GraphMotion/__init__.py -------------------------------------------------------------------------------- /GraphMotion/callback/__init__.py: -------------------------------------------------------------------------------- 1 | from .progress import ProgressLogger 2 | -------------------------------------------------------------------------------- /GraphMotion/callback/progress.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from pytorch_lightning import LightningModule, Trainer 4 | from pytorch_lightning.callbacks import Callback 5 | import psutil 6 | 7 | logger = logging.getLogger() 8 | 9 | 10 | class ProgressLogger(Callback): 11 | 12 | def __init__(self, metric_monitor: dict, precision: int = 3): 13 | # Metric to monitor 14 | self.metric_monitor = metric_monitor 15 | self.precision = precision 16 | 17 | def on_train_start(self, trainer: Trainer, pl_module: LightningModule, 18 | **kwargs) -> None: 19 | logger.info("Training started") 20 | 21 | def on_train_end(self, trainer: Trainer, pl_module: LightningModule, 22 | **kwargs) -> None: 23 | logger.info("Training done") 24 | 25 | def on_validation_epoch_end(self, trainer: Trainer, 26 | pl_module: LightningModule, **kwargs) -> None: 27 | if trainer.sanity_checking: 28 | logger.info("Sanity checking ok.") 29 | 30 | def on_train_epoch_end(self, 31 | trainer: Trainer, 32 | pl_module: LightningModule, 33 | padding=False, 34 | **kwargs) -> None: 35 | metric_format = f"{{:.{self.precision}e}}" 36 | line = f"Epoch {trainer.current_epoch}" 37 | if padding: 38 | line = f"{line:>{len('Epoch xxxx')}}" # Right padding 39 | metrics_str = [] 40 | 41 | losses_dict = trainer.callback_metrics 42 | for metric_name, dico_name in self.metric_monitor.items(): 43 | if dico_name in losses_dict: 44 | metric = losses_dict[dico_name].item() 45 | metric = metric_format.format(metric) 46 | metric = f"{metric_name} {metric}" 47 | metrics_str.append(metric) 48 | 49 | if len(metrics_str) == 0: 50 | return 51 | 52 | memory = f"Memory {psutil.virtual_memory().percent}%" 53 | line = line + ": " + " ".join(metrics_str) + " " + memory 54 | logger.info(line) 55 | -------------------------------------------------------------------------------- /GraphMotion/data/Humanact12.py: -------------------------------------------------------------------------------- 1 | from .base import BASEDataModule 2 | from .a2m import HumanAct12Poses 3 | import numpy as np 4 | 5 | 6 | class Humanact12DataModule(BASEDataModule): 7 | 8 | def __init__(self, 9 | cfg, 10 | batch_size, 11 | num_workers, 12 | collate_fn=None, 13 | phase="train", 14 | **kwargs): 15 | super().__init__(batch_size=batch_size, 16 | num_workers=num_workers, 17 | collate_fn=collate_fn) 18 | self.save_hyperparameters(logger=False) 19 | self.name = "HumanAct12" 20 | self.Dataset = HumanAct12Poses 21 | self.cfg = cfg 22 | sample_overrides = { 23 | "num_seq_max": 2, 24 | "split": "test", 25 | "tiny": True, 26 | "progress_bar": False 27 | } 28 | # self._sample_set = self.get_sample_set(overrides=sample_overrides) 29 | # Get additional info of the dataset 30 | self.nfeats = 150 31 | self.njoints = 25 32 | self.nclasses = 12 33 | # self.transforms = self._sample_set.transforms 34 | 35 | # def mm_mode(self, mm_on=True): 36 | # # random select samples for mm 37 | # if mm_on: 38 | # self.is_mm = True 39 | # if self.split == 'train': 40 | # self.name_list = self.test_dataset._train[index] 41 | # else: 42 | # self.name_list = self.test_dataset._test[index] 43 | # self.name_list = self.test_dataset.name_list 44 | # self.mm_list = np.random.choice(self.name_list, 45 | # self.cfg.TEST.MM_NUM_SAMPLES, 46 | # replace=False) 47 | # self.test_dataset.name_list = self.mm_list 48 | # else: 49 | # self.is_mm = False 50 | # self.test_dataset.name_list = self.name_list 51 | -------------------------------------------------------------------------------- /GraphMotion/data/Kit.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | 4 | from GraphMotion.data.humanml.scripts.motion_process import recover_from_ric 5 | 6 | from .base import BASEDataModule 7 | from .humanml.data.dataset import Text2MotionDatasetV2, TextOnlyDataset 8 | from .utils import all_collate 9 | 10 | 11 | class KitDataModule(BASEDataModule): 12 | 13 | def __init__(self, 14 | cfg, 15 | phase='train', 16 | collate_fn=all_collate, 17 | batch_size: int = 32, 18 | num_workers: int = 16, 19 | **kwargs): 20 | super().__init__(batch_size=batch_size, 21 | num_workers=num_workers, 22 | collate_fn=collate_fn) 23 | self.save_hyperparameters(logger=False) 24 | self.name = 'kit' 25 | self.njoints = 21 26 | if phase == 'text_only': 27 | self.Dataset = TextOnlyDataset 28 | else: 29 | self.Dataset = Text2MotionDatasetV2 30 | self.cfg = cfg 31 | 32 | sample_overrides = { 33 | "split": "val", 34 | "tiny": True, 35 | "progress_bar": False 36 | } 37 | self._sample_set = self.get_sample_set(overrides=sample_overrides) 38 | 39 | # Get additional info of the dataset 40 | self.nfeats = self._sample_set.nfeats 41 | # self.transforms = self._sample_set.transforms 42 | 43 | def feats2joints(self, features): 44 | mean = torch.tensor(self.hparams.mean).to(features) 45 | std = torch.tensor(self.hparams.std).to(features) 46 | features = features * std + mean 47 | return recover_from_ric(features, self.njoints) 48 | 49 | def renorm4t2m(self, features): 50 | # renorm to t2m norms for using t2m evaluators 51 | ori_mean = torch.tensor(self.hparams.mean).to(features) 52 | ori_std = torch.tensor(self.hparams.std).to(features) 53 | eval_mean = torch.tensor(self.hparams.mean_eval).to(features) 54 | eval_std = torch.tensor(self.hparams.std_eval).to(features) 55 | features = features * ori_std + ori_mean 56 | features = (features - eval_mean) / eval_std 57 | return features 58 | 59 | def mm_mode(self, mm_on=True): 60 | # random select samples for mm 61 | if mm_on: 62 | self.is_mm = True 63 | self.name_list = self.test_dataset.name_list 64 | self.mm_list = np.random.choice(self.name_list, 65 | self.cfg.TEST.MM_NUM_SAMPLES, 66 | replace=False) 67 | self.test_dataset.name_list = self.mm_list 68 | else: 69 | self.is_mm = False 70 | self.test_dataset.name_list = self.name_list 71 | -------------------------------------------------------------------------------- /GraphMotion/data/Uestc.py: -------------------------------------------------------------------------------- 1 | from .base import BASEDataModule 2 | from .a2m import UESTC 3 | import os 4 | import rich.progress 5 | import pickle as pkl 6 | 7 | 8 | class UestcDataModule(BASEDataModule): 9 | 10 | def __init__(self, 11 | cfg, 12 | batch_size, 13 | num_workers, 14 | collate_fn=None, 15 | method_name="vibe", 16 | phase="train", 17 | **kwargs): 18 | super().__init__(batch_size=batch_size, 19 | num_workers=num_workers, 20 | collate_fn=collate_fn) 21 | self.save_hyperparameters(logger=False) 22 | self.name = "Uestc" 23 | 24 | # if method_name == "vibe": 25 | # vibe_data_path = os.path.join(self.hparams.datapath, 26 | # "vibe_cache_refined.pkl") 27 | # with rich.progress.open( 28 | # vibe_data_path, "rb", 29 | # description="loading uestc vibe data") as f: 30 | # vibe_data = pkl.load(f) 31 | # self.hparams.update({"vibe_data": vibe_data}) 32 | self.Dataset = UESTC 33 | self.cfg = cfg 34 | 35 | # self._sample_set = self.get_sample_set(overrides=sample_overrides) 36 | # Get additional info of the dataset 37 | self.nfeats = 150 38 | self.njoints = 25 39 | self.nclasses = 40 40 | # self.transforms = self._sample_set.transforms 41 | -------------------------------------------------------------------------------- /GraphMotion/data/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpthu17/GraphMotion/d56feba6f6d39867b933ca752244fcbbb703a4b8/GraphMotion/data/__init__.py -------------------------------------------------------------------------------- /GraphMotion/data/a2m/__init__.py: -------------------------------------------------------------------------------- 1 | from .humanact12poses import HumanAct12Poses 2 | from .uestc import UESTC 3 | -------------------------------------------------------------------------------- /GraphMotion/data/a2m/humanact12poses.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pickle as pkl 3 | 4 | import numpy as np 5 | 6 | from .dataset import Dataset 7 | from .utils import rotation_conversions as geometry 8 | import rich.progress 9 | 10 | 11 | class HumanAct12Poses(Dataset): 12 | dataname = "humanact12" 13 | 14 | def __init__(self, datapath="data/HumanAct12Poses", **kargs): 15 | self.datapath = datapath 16 | 17 | super().__init__(**kargs) 18 | 19 | pkldatafilepath = os.path.join(datapath, "humanact12poses.pkl") 20 | with rich.progress.open(pkldatafilepath, "rb", description="loading humanact12 pkl") as f: 21 | data = pkl.load(f) 22 | 23 | self._pose = [x for x in data["poses"]] 24 | self._num_frames_in_video = [p.shape[0] for p in self._pose] 25 | self._joints = [x for x in data["joints3D"]] 26 | 27 | self._actions = [x for x in data["y"]] 28 | 29 | total_num_actions = 12 30 | self.num_classes = total_num_actions 31 | 32 | self._train = list(range(len(self._pose))) 33 | 34 | keep_actions = np.arange(0, total_num_actions) 35 | 36 | self._action_to_label = {x: i for i, x in enumerate(keep_actions)} 37 | self._label_to_action = {i: x for i, x in enumerate(keep_actions)} 38 | 39 | self._action_classes = humanact12_coarse_action_enumerator 40 | 41 | def _load_joints3D(self, ind, frame_ix): 42 | return self._joints[ind][frame_ix] 43 | 44 | def _load_rotvec(self, ind, frame_ix): 45 | pose = self._pose[ind][frame_ix].reshape(-1, 24, 3) 46 | return pose 47 | 48 | 49 | humanact12_coarse_action_enumerator = { 50 | 0: "warm_up", 51 | 1: "walk", 52 | 2: "run", 53 | 3: "jump", 54 | 4: "drink", 55 | 5: "lift_dumbbell", 56 | 6: "sit", 57 | 7: "eat", 58 | 8: "turn steering wheel", 59 | 9: "phone", 60 | 10: "boxing", 61 | 11: "throw", 62 | } 63 | -------------------------------------------------------------------------------- /GraphMotion/data/a2m/tools.py: -------------------------------------------------------------------------------- 1 | import os 2 | import string 3 | 4 | 5 | def parse_info_name(path): 6 | name = os.path.splitext(os.path.split(path)[-1])[0] 7 | info = {} 8 | current_letter = None 9 | for letter in name: 10 | if letter in string.ascii_letters: 11 | info[letter] = [] 12 | current_letter = letter 13 | else: 14 | info[current_letter].append(letter) 15 | for key in info.keys(): 16 | info[key] = "".join(info[key]) 17 | return info 18 | 19 | 20 | -------------------------------------------------------------------------------- /GraphMotion/data/a2m/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpthu17/GraphMotion/d56feba6f6d39867b933ca752244fcbbb703a4b8/GraphMotion/data/a2m/utils/__init__.py -------------------------------------------------------------------------------- /GraphMotion/data/a2m/utils/misc.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | 4 | def to_numpy(tensor): 5 | if torch.is_tensor(tensor): 6 | return tensor.cpu().numpy() 7 | elif type(tensor).__module__ != 'numpy': 8 | raise ValueError("Cannot convert {} to numpy array".format( 9 | type(tensor))) 10 | return tensor 11 | 12 | 13 | def to_torch(ndarray): 14 | if type(ndarray).__module__ == 'numpy': 15 | return torch.from_numpy(ndarray) 16 | elif not torch.is_tensor(ndarray): 17 | raise ValueError("Cannot convert {} to torch tensor".format( 18 | type(ndarray))) 19 | return ndarray 20 | 21 | 22 | def cleanexit(): 23 | import sys 24 | import os 25 | try: 26 | sys.exit(0) 27 | except SystemExit: 28 | os._exit(0) 29 | 30 | -------------------------------------------------------------------------------- /GraphMotion/data/a2m/utils/tensors.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | 4 | def lengths_to_mask(lengths): 5 | max_len = max(lengths) 6 | mask = torch.arange(max_len, device=lengths.device).expand(len(lengths), max_len) < lengths.unsqueeze(1) 7 | return mask 8 | 9 | 10 | def collate_tensors(batch): 11 | dims = batch[0].dim() 12 | max_size = [max([b.size(i) for b in batch]) for i in range(dims)] 13 | size = (len(batch),) + tuple(max_size) 14 | canvas = batch[0].new_zeros(size=size) 15 | for i, b in enumerate(batch): 16 | sub_tensor = canvas[i] 17 | for d in range(dims): 18 | sub_tensor = sub_tensor.narrow(d, 0, b.size(d)) 19 | sub_tensor.add_(b) 20 | return canvas 21 | 22 | 23 | def collate(batch): 24 | databatch = [b[0] for b in batch] 25 | labelbatch = [b[1] for b in batch] 26 | lenbatch = [len(b[0][0][0]) for b in batch] 27 | 28 | databatchTensor = collate_tensors(databatch) 29 | labelbatchTensor = torch.as_tensor(labelbatch) 30 | lenbatchTensor = torch.as_tensor(lenbatch) 31 | 32 | maskbatchTensor = lengths_to_mask(lenbatchTensor) 33 | batch = {"x": databatchTensor, "y": labelbatchTensor, 34 | "mask": maskbatchTensor, "lengths": lenbatchTensor} 35 | return batch 36 | -------------------------------------------------------------------------------- /GraphMotion/data/humanml/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpthu17/GraphMotion/d56feba6f6d39867b933ca752244fcbbb703a4b8/GraphMotion/data/humanml/__init__.py -------------------------------------------------------------------------------- /GraphMotion/data/humanml/data/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpthu17/GraphMotion/d56feba6f6d39867b933ca752244fcbbb703a4b8/GraphMotion/data/humanml/data/__init__.py -------------------------------------------------------------------------------- /GraphMotion/data/humanml/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpthu17/GraphMotion/d56feba6f6d39867b933ca752244fcbbb703a4b8/GraphMotion/data/humanml/utils/__init__.py -------------------------------------------------------------------------------- /GraphMotion/data/humanml/utils/get_opt.py: -------------------------------------------------------------------------------- 1 | import os 2 | from argparse import Namespace 3 | import re 4 | from os.path import join as pjoin 5 | from .word_vectorizer import POS_enumerator 6 | 7 | 8 | def is_float(numStr): 9 | flag = False 10 | numStr = str(numStr).strip().lstrip('-').lstrip('+') # 去除正数(+)、负数(-)符号 11 | try: 12 | reg = re.compile(r'^[-+]?[0-9]+\.[0-9]+$') 13 | res = reg.match(str(numStr)) 14 | if res: 15 | flag = True 16 | except Exception as ex: 17 | print("is_float() - error: " + str(ex)) 18 | return flag 19 | 20 | 21 | def is_number(numStr): 22 | flag = False 23 | numStr = str(numStr).strip().lstrip('-').lstrip('+') # 去除正数(+)、负数(-)符号 24 | if str(numStr).isdigit(): 25 | flag = True 26 | return flag 27 | 28 | 29 | def get_opt(opt_path, device): 30 | opt = Namespace() 31 | opt_dict = vars(opt) 32 | 33 | skip = ('-------------- End ----------------', 34 | '------------ Options -------------', 35 | '\n') 36 | print('Reading', opt_path) 37 | with open(opt_path) as f: 38 | for line in f: 39 | if line.strip() not in skip: 40 | # print(line.strip()) 41 | key, value = line.strip().split(': ') 42 | if value in ('True', 'False'): 43 | opt_dict[key] = bool(value) 44 | elif is_float(value): 45 | opt_dict[key] = float(value) 46 | elif is_number(value): 47 | opt_dict[key] = int(value) 48 | else: 49 | opt_dict[key] = str(value) 50 | 51 | # print(opt) 52 | opt_dict['which_epoch'] = 'latest' 53 | opt.save_root = pjoin(opt.checkpoints_dir, opt.dataset_name, opt.name) 54 | opt.model_dir = pjoin(opt.save_root, 'model') 55 | opt.meta_dir = pjoin(opt.save_root, 'meta') 56 | 57 | if opt.dataset_name == 't2m': 58 | opt.data_root = './dataset/HumanML3D' 59 | opt.motion_dir = pjoin(opt.data_root, 'new_joint_vecs') 60 | opt.text_dir = pjoin(opt.data_root, 'texts') 61 | opt.joints_num = 22 62 | opt.dim_pose = 263 63 | opt.max_motion_length = 196 64 | elif opt.dataset_name == 'kit': 65 | opt.data_root = './dataset/KIT-ML' 66 | opt.motion_dir = pjoin(opt.data_root, 'new_joint_vecs') 67 | opt.text_dir = pjoin(opt.data_root, 'texts') 68 | opt.joints_num = 21 69 | opt.dim_pose = 251 70 | opt.max_motion_length = 196 71 | else: 72 | raise KeyError('Dataset not recognized') 73 | 74 | opt.dim_word = 300 75 | opt.num_classes = 200 // opt.unit_length 76 | opt.dim_pos_ohot = len(POS_enumerator) 77 | opt.is_train = False 78 | opt.is_continue = False 79 | opt.device = device 80 | 81 | return opt 82 | -------------------------------------------------------------------------------- /GraphMotion/data/humanml/utils/load_json.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | 4 | 5 | def save_json(save_path, data): 6 | with open(save_path, "w") as file: 7 | json.dump(data, file) 8 | 9 | 10 | def load_json(file_path): 11 | with open(file_path, "r") as file: 12 | data = json.load(file) 13 | return data 14 | 15 | 16 | def process(graph): 17 | entities, relations = {}, [] 18 | for i in graph["verbs"]: 19 | description = i['description'] 20 | pos = 0 21 | flag = 0 22 | _words, _spans = [], [] 23 | for i in description.split(): 24 | tags, verb = {}, 0 25 | if "[" in i: 26 | _role = i[1:-1] 27 | flag = 1 28 | _spans = [pos] 29 | _words = [] 30 | 31 | elif "]" in i: 32 | _words.append(i[:-1]) 33 | entities[len(entities)] = { 34 | "role": _role, 35 | "spans": _spans, 36 | "words": _words 37 | } 38 | pos += 1 39 | flag = 0 40 | if _role != "V": 41 | tags[len(entities)] = _role 42 | else: 43 | verb = len(entities) 44 | else: 45 | pos += 1 46 | if flag: 47 | _words.append(i) 48 | _spans.append(pos) 49 | 50 | for i in tags: 51 | relations.append((verb, i, tags[i])) 52 | 53 | output = { 54 | "entities": entities, 55 | "relations": relations 56 | } 57 | return output -------------------------------------------------------------------------------- /GraphMotion/data/humanml/utils/paramUtil.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | # Define a kinematic tree for the skeletal struture 4 | kit_kinematic_chain = [[0, 11, 12, 13, 14, 15], [0, 16, 17, 18, 19, 20], [0, 1, 2, 3, 4], [3, 5, 6, 7], [3, 8, 9, 10]] 5 | 6 | kit_raw_offsets = np.array( 7 | [ 8 | [0, 0, 0], 9 | [0, 1, 0], 10 | [0, 1, 0], 11 | [0, 1, 0], 12 | [0, 1, 0], 13 | [1, 0, 0], 14 | [0, -1, 0], 15 | [0, -1, 0], 16 | [-1, 0, 0], 17 | [0, -1, 0], 18 | [0, -1, 0], 19 | [1, 0, 0], 20 | [0, -1, 0], 21 | [0, -1, 0], 22 | [0, 0, 1], 23 | [0, 0, 1], 24 | [-1, 0, 0], 25 | [0, -1, 0], 26 | [0, -1, 0], 27 | [0, 0, 1], 28 | [0, 0, 1] 29 | ] 30 | ) 31 | 32 | t2m_raw_offsets = np.array([[0,0,0], 33 | [1,0,0], 34 | [-1,0,0], 35 | [0,1,0], 36 | [0,-1,0], 37 | [0,-1,0], 38 | [0,1,0], 39 | [0,-1,0], 40 | [0,-1,0], 41 | [0,1,0], 42 | [0,0,1], 43 | [0,0,1], 44 | [0,1,0], 45 | [1,0,0], 46 | [-1,0,0], 47 | [0,0,1], 48 | [0,-1,0], 49 | [0,-1,0], 50 | [0,-1,0], 51 | [0,-1,0], 52 | [0,-1,0], 53 | [0,-1,0]]) 54 | 55 | t2m_kinematic_chain = [[0, 2, 5, 8, 11], [0, 1, 4, 7, 10], [0, 3, 6, 9, 12, 15], [9, 14, 17, 19, 21], [9, 13, 16, 18, 20]] 56 | t2m_left_hand_chain = [[20, 22, 23, 24], [20, 34, 35, 36], [20, 25, 26, 27], [20, 31, 32, 33], [20, 28, 29, 30]] 57 | t2m_right_hand_chain = [[21, 43, 44, 45], [21, 46, 47, 48], [21, 40, 41, 42], [21, 37, 38, 39], [21, 49, 50, 51]] 58 | 59 | 60 | kit_tgt_skel_id = '03950' 61 | 62 | t2m_tgt_skel_id = '000021' 63 | 64 | -------------------------------------------------------------------------------- /GraphMotion/data/humanml/utils/word_vectorizer.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pickle 3 | from os.path import join as pjoin 4 | 5 | POS_enumerator = { 6 | 'VERB': 0, 7 | 'NOUN': 1, 8 | 'DET': 2, 9 | 'ADP': 3, 10 | 'NUM': 4, 11 | 'AUX': 5, 12 | 'PRON': 6, 13 | 'ADJ': 7, 14 | 'ADV': 8, 15 | 'Loc_VIP': 9, 16 | 'Body_VIP': 10, 17 | 'Obj_VIP': 11, 18 | 'Act_VIP': 12, 19 | 'Desc_VIP': 13, 20 | 'OTHER': 14, 21 | } 22 | 23 | Loc_list = ('left', 'right', 'clockwise', 'counterclockwise', 'anticlockwise', 'forward', 'back', 'backward', 24 | 'up', 'down', 'straight', 'curve') 25 | 26 | Body_list = ('arm', 'chin', 'foot', 'feet', 'face', 'hand', 'mouth', 'leg', 'waist', 'eye', 'knee', 'shoulder', 'thigh') 27 | 28 | Obj_List = ('stair', 'dumbbell', 'chair', 'window', 'floor', 'car', 'ball', 'handrail', 'baseball', 'basketball') 29 | 30 | Act_list = ('walk', 'run', 'swing', 'pick', 'bring', 'kick', 'put', 'squat', 'throw', 'hop', 'dance', 'jump', 'turn', 31 | 'stumble', 'dance', 'stop', 'sit', 'lift', 'lower', 'raise', 'wash', 'stand', 'kneel', 'stroll', 32 | 'rub', 'bend', 'balance', 'flap', 'jog', 'shuffle', 'lean', 'rotate', 'spin', 'spread', 'climb') 33 | 34 | Desc_list = ('slowly', 'carefully', 'fast', 'careful', 'slow', 'quickly', 'happy', 'angry', 'sad', 'happily', 35 | 'angrily', 'sadly') 36 | 37 | VIP_dict = { 38 | 'Loc_VIP': Loc_list, 39 | 'Body_VIP': Body_list, 40 | 'Obj_VIP': Obj_List, 41 | 'Act_VIP': Act_list, 42 | 'Desc_VIP': Desc_list, 43 | } 44 | 45 | 46 | class WordVectorizer(object): 47 | def __init__(self, meta_root, prefix): 48 | vectors = np.load(pjoin(meta_root, '%s_data.npy'%prefix)) 49 | words = pickle.load(open(pjoin(meta_root, '%s_words.pkl'%prefix), 'rb')) 50 | word2idx = pickle.load(open(pjoin(meta_root, '%s_idx.pkl'%prefix), 'rb')) 51 | self.word2vec = {w: vectors[word2idx[w]] for w in words} 52 | 53 | def _get_pos_ohot(self, pos): 54 | pos_vec = np.zeros(len(POS_enumerator)) 55 | if pos in POS_enumerator: 56 | pos_vec[POS_enumerator[pos]] = 1 57 | else: 58 | pos_vec[POS_enumerator['OTHER']] = 1 59 | return pos_vec 60 | 61 | def __len__(self): 62 | return len(self.word2vec) 63 | 64 | def __getitem__(self, item): 65 | word, pos = item.split('/') 66 | if word in self.word2vec: 67 | word_vec = self.word2vec[word] 68 | vip_pos = None 69 | for key, values in VIP_dict.items(): 70 | if word in values: 71 | vip_pos = key 72 | break 73 | if vip_pos is not None: 74 | pos_vec = self._get_pos_ohot(vip_pos) 75 | else: 76 | pos_vec = self._get_pos_ohot(pos) 77 | else: 78 | word_vec = self.word2vec['unk'] 79 | pos_vec = self._get_pos_ohot('OTHER') 80 | return word_vec, pos_vec 81 | -------------------------------------------------------------------------------- /GraphMotion/data/sampling/__init__.py: -------------------------------------------------------------------------------- 1 | from .base import FrameSampler 2 | from .framerate import subsample, upsample 3 | -------------------------------------------------------------------------------- /GraphMotion/data/sampling/base.py: -------------------------------------------------------------------------------- 1 | from .frames import get_frameix_from_data_index 2 | 3 | class FrameSampler: 4 | def __init__(self, sampling="conseq", sampling_step=1, request_frames=None,threshold_reject=0.75,max_len=1000,min_len=10): 5 | self.sampling = sampling 6 | 7 | self.sampling_step = sampling_step 8 | self.request_frames = request_frames 9 | self.threshold_reject = threshold_reject 10 | self.max_len = max_len 11 | self.min_len = min_len 12 | 13 | def __call__(self, num_frames): 14 | 15 | return get_frameix_from_data_index(num_frames, 16 | self.request_frames, 17 | self.sampling, 18 | self.sampling_step) 19 | 20 | def accept(self, duration): 21 | # Outputs have original lengths 22 | # Check if it is too long 23 | if self.request_frames is None: 24 | if duration > self.max_len: 25 | return False 26 | elif duration < self.min_len: 27 | return False 28 | else: 29 | # Reject sample if the length is 30 | # too little relative to 31 | # the request frames 32 | min_number = self.threshold_reject * self.request_frames 33 | if duration < min_number: 34 | return False 35 | return True 36 | 37 | def get(self, key, default=None): 38 | return getattr(self, key, default) 39 | 40 | def __getitem__(self, key): 41 | return getattr(self, key) 42 | -------------------------------------------------------------------------------- /GraphMotion/data/sampling/framerate.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | # TODO: use a real subsampler.. 5 | def subsample(num_frames, last_framerate, new_framerate): 6 | step = int(last_framerate / new_framerate) 7 | assert step >= 1 8 | frames = np.arange(0, num_frames, step) 9 | return frames 10 | 11 | 12 | # TODO: use a real upsampler.. 13 | def upsample(motion, last_framerate, new_framerate): 14 | step = int(new_framerate / last_framerate) 15 | assert step >= 1 16 | 17 | # Alpha blending => interpolation 18 | alpha = np.linspace(0, 1, step+1) 19 | last = np.einsum("l,...->l...", 1-alpha, motion[:-1]) 20 | new = np.einsum("l,...->l...", alpha, motion[1:]) 21 | 22 | chuncks = (last + new)[:-1] 23 | output = np.concatenate(chuncks.swapaxes(1, 0)) 24 | # Don't forget the last one 25 | output = np.concatenate((output, motion[[-1]])) 26 | return output 27 | 28 | 29 | if __name__ == "__main__": 30 | motion = np.arange(105) 31 | submotion = motion[subsample(len(motion), 100.0, 12.5)] 32 | newmotion = upsample(submotion, 12.5, 100) 33 | 34 | print(newmotion) 35 | -------------------------------------------------------------------------------- /GraphMotion/data/sampling/frames.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | import numpy as np 4 | from numpy import ndarray as Array 5 | import random 6 | 7 | 8 | def get_frameix_from_data_index(num_frames: int, 9 | request_frames: Optional[int], 10 | sampling: str = "conseq", 11 | sampling_step: int = 1) -> Array: 12 | nframes = num_frames 13 | 14 | if request_frames is None: 15 | frame_ix = np.arange(nframes) 16 | else: 17 | # sampling goal: input: ----------- 11 nframes 18 | # o--o--o--o- 4 ninputs 19 | # 20 | # step number is computed like that: [(11-1)/(4-1)] = 3 21 | # [---][---][---][- 22 | # So step = 3, and we take 0 to step*ninputs+1 with steps 23 | # [o--][o--][o--][o-] 24 | # then we can randomly shift the vector 25 | # -[o--][o--][o--]o 26 | # If there are too much frames required 27 | if request_frames > nframes: 28 | fair = False # True 29 | if fair: 30 | # distills redundancy everywhere 31 | choices = np.random.choice(range(nframes), 32 | request_frames, 33 | replace=True) 34 | frame_ix = sorted(choices) 35 | else: 36 | # adding the last frame until done 37 | ntoadd = max(0, request_frames - nframes) 38 | lastframe = nframes - 1 39 | padding = lastframe * np.ones(ntoadd, dtype=int) 40 | frame_ix = np.concatenate((np.arange(0, nframes), 41 | padding)) 42 | 43 | elif sampling in ["conseq", "random_conseq"]: 44 | step_max = (nframes - 1) // (request_frames - 1) 45 | if sampling == "conseq": 46 | if sampling_step == -1 or sampling_step * (request_frames - 1) >= nframes: 47 | step = step_max 48 | else: 49 | step = sampling_step 50 | elif sampling == "random_conseq": 51 | step = random.randint(1, step_max) 52 | 53 | lastone = step * (request_frames - 1) 54 | shift_max = nframes - lastone - 1 55 | shift = random.randint(0, max(0, shift_max - 1)) 56 | frame_ix = shift + np.arange(0, lastone + 1, step) 57 | 58 | elif sampling == "random": 59 | choices = np.random.choice(range(nframes), 60 | request_frames, 61 | replace=False) 62 | frame_ix = sorted(choices) 63 | 64 | else: 65 | raise ValueError("Sampling not recognized.") 66 | 67 | return frame_ix 68 | -------------------------------------------------------------------------------- /GraphMotion/launch/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpthu17/GraphMotion/d56feba6f6d39867b933ca752244fcbbb703a4b8/GraphMotion/launch/__init__.py -------------------------------------------------------------------------------- /GraphMotion/launch/blender.py: -------------------------------------------------------------------------------- 1 | # Fix blender path 2 | import sys 3 | import os 4 | # local packages 5 | sys.path.append(os.path.expanduser("~/.local/lib/python3.9/site-packages")) 6 | import bpy 7 | import os 8 | from argparse import ArgumentParser 9 | 10 | # Monkey patch argparse such that 11 | # blender / python / hydra parsing works 12 | def parse_args(self, args=None, namespace=None): 13 | if args is not None: 14 | return self.parse_args_bak(args=args, namespace=namespace) 15 | try: 16 | idx = sys.argv.index("--") 17 | args = sys.argv[idx+1:] # the list after '--' 18 | except ValueError as e: # '--' not in the list: 19 | args = [] 20 | return self.parse_args_bak(args=args, namespace=namespace) 21 | 22 | setattr(ArgumentParser, 'parse_args_bak', ArgumentParser.parse_args) 23 | setattr(ArgumentParser, 'parse_args', parse_args) 24 | -------------------------------------------------------------------------------- /GraphMotion/launch/prepare.py: -------------------------------------------------------------------------------- 1 | import os 2 | import warnings 3 | from pathlib import Path 4 | 5 | import hydra 6 | from GraphMotion.tools.runid import generate_id 7 | from omegaconf import OmegaConf 8 | 9 | 10 | # Local paths 11 | def code_path(path=""): 12 | code_dir = hydra.utils.get_original_cwd() 13 | code_dir = Path(code_dir) 14 | return str(code_dir / path) 15 | 16 | 17 | def working_path(path): 18 | return str(Path(os.getcwd()) / path) 19 | 20 | 21 | # fix the id for this run 22 | ID = generate_id() 23 | 24 | 25 | def generate_id(): 26 | return ID 27 | 28 | 29 | def get_last_checkpoint(path, ckpt_name="last.ckpt"): 30 | output_dir = Path(hydra.utils.to_absolute_path(path)) 31 | last_ckpt_path = output_dir / "checkpoints" / ckpt_name 32 | return str(last_ckpt_path) 33 | 34 | 35 | def get_kitname(load_amass_data: bool, load_with_rot: bool): 36 | if not load_amass_data: 37 | return "kit-mmm-xyz" 38 | if load_amass_data and not load_with_rot: 39 | return "kit-amass-xyz" 40 | if load_amass_data and load_with_rot: 41 | return "kit-amass-rot" 42 | 43 | 44 | OmegaConf.register_new_resolver("code_path", code_path) 45 | OmegaConf.register_new_resolver("working_path", working_path) 46 | OmegaConf.register_new_resolver("generate_id", generate_id) 47 | OmegaConf.register_new_resolver("absolute_path", hydra.utils.to_absolute_path) 48 | OmegaConf.register_new_resolver("get_last_checkpoint", get_last_checkpoint) 49 | OmegaConf.register_new_resolver("get_kitname", get_kitname) 50 | 51 | 52 | # Remove warnings 53 | warnings.filterwarnings( 54 | "ignore", ".*Trying to infer the `batch_size` from an ambiguous collection.*" 55 | ) 56 | 57 | warnings.filterwarnings( 58 | "ignore", ".*does not have many workers which may be a bottleneck*" 59 | ) 60 | 61 | warnings.filterwarnings( 62 | "ignore", ".*Our suggested max number of worker in current system is*" 63 | ) 64 | 65 | 66 | # os.environ["HYDRA_FULL_ERROR"] = "1" 67 | os.environ["NUMEXPR_MAX_THREADS"] = "24" 68 | -------------------------------------------------------------------------------- /GraphMotion/launch/tools.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from omegaconf import DictConfig, OmegaConf 3 | import hydra 4 | import os 5 | 6 | 7 | def resolve_cfg_path(cfg: DictConfig): 8 | working_dir = os.getcwd() 9 | cfg.working_dir = working_dir 10 | -------------------------------------------------------------------------------- /GraphMotion/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpthu17/GraphMotion/d56feba6f6d39867b933ca752244fcbbb703a4b8/GraphMotion/models/__init__.py -------------------------------------------------------------------------------- /GraphMotion/models/architectures/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpthu17/GraphMotion/d56feba6f6d39867b933ca752244fcbbb703a4b8/GraphMotion/models/architectures/__init__.py -------------------------------------------------------------------------------- /GraphMotion/models/architectures/t2m_motionenc.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from torch.nn.utils.rnn import pack_padded_sequence 4 | 5 | 6 | class MovementConvEncoder(nn.Module): 7 | def __init__(self, input_size, hidden_size, output_size): 8 | super(MovementConvEncoder, self).__init__() 9 | self.main = nn.Sequential( 10 | nn.Conv1d(input_size, hidden_size, 4, 2, 1), 11 | nn.Dropout(0.2, inplace=True), 12 | nn.LeakyReLU(0.2, inplace=True), 13 | nn.Conv1d(hidden_size, output_size, 4, 2, 1), 14 | nn.Dropout(0.2, inplace=True), 15 | nn.LeakyReLU(0.2, inplace=True), 16 | ) 17 | self.out_net = nn.Linear(output_size, output_size) 18 | # self.main.apply(init_weight) 19 | # self.out_net.apply(init_weight) 20 | 21 | def forward(self, inputs): 22 | inputs = inputs.permute(0, 2, 1) 23 | outputs = self.main(inputs).permute(0, 2, 1) 24 | # print(outputs.shape) 25 | return self.out_net(outputs) 26 | 27 | 28 | class MotionEncoderBiGRUCo(nn.Module): 29 | def __init__(self, input_size, hidden_size, output_size): 30 | super(MotionEncoderBiGRUCo, self).__init__() 31 | 32 | self.input_emb = nn.Linear(input_size, hidden_size) 33 | self.gru = nn.GRU( 34 | hidden_size, hidden_size, batch_first=True, bidirectional=True 35 | ) 36 | self.output_net = nn.Sequential( 37 | nn.Linear(hidden_size * 2, hidden_size), 38 | nn.LayerNorm(hidden_size), 39 | nn.LeakyReLU(0.2, inplace=True), 40 | nn.Linear(hidden_size, output_size), 41 | ) 42 | 43 | # self.input_emb.apply(init_weight) 44 | # self.output_net.apply(init_weight) 45 | self.hidden_size = hidden_size 46 | self.hidden = nn.Parameter( 47 | torch.randn((2, 1, self.hidden_size), requires_grad=True) 48 | ) 49 | 50 | # input(batch_size, seq_len, dim) 51 | def forward(self, inputs, m_lens): 52 | num_samples = inputs.shape[0] 53 | 54 | input_embs = self.input_emb(inputs) 55 | hidden = self.hidden.repeat(1, num_samples, 1) 56 | 57 | cap_lens = m_lens.data.tolist() 58 | emb = pack_padded_sequence(input_embs, cap_lens, batch_first=True) 59 | 60 | gru_seq, gru_last = self.gru(emb, hidden) 61 | 62 | gru_last = torch.cat([gru_last[0], gru_last[1]], dim=-1) 63 | 64 | return self.output_net(gru_last) 65 | -------------------------------------------------------------------------------- /GraphMotion/models/architectures/t2m_textenc.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from torch.nn.utils.rnn import pack_padded_sequence 4 | 5 | 6 | class TextEncoderBiGRUCo(nn.Module): 7 | def __init__(self, word_size, pos_size, hidden_size, output_size): 8 | super(TextEncoderBiGRUCo, self).__init__() 9 | 10 | self.pos_emb = nn.Linear(pos_size, word_size) 11 | self.input_emb = nn.Linear(word_size, hidden_size) 12 | self.gru = nn.GRU( 13 | hidden_size, hidden_size, batch_first=True, bidirectional=True 14 | ) 15 | self.output_net = nn.Sequential( 16 | nn.Linear(hidden_size * 2, hidden_size), 17 | nn.LayerNorm(hidden_size), 18 | nn.LeakyReLU(0.2, inplace=True), 19 | nn.Linear(hidden_size, output_size), 20 | ) 21 | 22 | # self.input_emb.apply(init_weight) 23 | # self.pos_emb.apply(init_weight) 24 | # self.output_net.apply(init_weight) 25 | # self.linear2.apply(init_weight) 26 | # self.batch_size = batch_size 27 | self.hidden_size = hidden_size 28 | self.hidden = nn.Parameter( 29 | torch.randn((2, 1, self.hidden_size), requires_grad=True) 30 | ) 31 | 32 | # input(batch_size, seq_len, dim) 33 | def forward(self, word_embs, pos_onehot, cap_lens): 34 | num_samples = word_embs.shape[0] 35 | 36 | pos_embs = self.pos_emb(pos_onehot) 37 | inputs = word_embs + pos_embs 38 | input_embs = self.input_emb(inputs) 39 | hidden = self.hidden.repeat(1, num_samples, 1) 40 | 41 | cap_lens = cap_lens.data.tolist() 42 | emb = pack_padded_sequence(input_embs, cap_lens, batch_first=True) 43 | 44 | gru_seq, gru_last = self.gru(emb, hidden) 45 | 46 | gru_last = torch.cat([gru_last[0], gru_last[1]], dim=-1) 47 | 48 | return self.output_net(gru_last) 49 | -------------------------------------------------------------------------------- /GraphMotion/models/get_model.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | 3 | 4 | def get_model(cfg, datamodule, phase="train"): 5 | modeltype = cfg.model.model_type 6 | if modeltype == "GraphMotion": 7 | return get_module(cfg, datamodule) 8 | else: 9 | raise ValueError(f"Invalid model type {modeltype}.") 10 | 11 | 12 | def get_module(cfg, datamodule): 13 | modeltype = cfg.model.model_type 14 | model_module = importlib.import_module( 15 | f".modeltype.{cfg.model.model_type}", package="GraphMotion.models") 16 | Model = model_module.__getattribute__(f"{modeltype}") 17 | return Model(cfg=cfg, datamodule=datamodule) 18 | -------------------------------------------------------------------------------- /GraphMotion/models/losses/__init__.py: -------------------------------------------------------------------------------- 1 | from GraphMotion.models.losses.temos import TemosLosses 2 | from GraphMotion.models.losses.tmost import TmostLosses 3 | -------------------------------------------------------------------------------- /GraphMotion/models/losses/kl.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | class KLLoss: 4 | def __init__(self): 5 | pass 6 | 7 | def __call__(self, q, p): 8 | div = torch.distributions.kl_divergence(q, p) 9 | return div.mean() 10 | 11 | def __repr__(self): 12 | return "KLLoss()" 13 | 14 | class KLLossMulti: 15 | def __init__(self): 16 | self.klloss = KLLoss() 17 | 18 | def __call__(self, qlist, plist): 19 | return sum([self.klloss(q, p) 20 | for q, p in zip(qlist, plist)]) 21 | 22 | def __repr__(self): 23 | return "KLLossMulti()" 24 | -------------------------------------------------------------------------------- /GraphMotion/models/metrics/__init__.py: -------------------------------------------------------------------------------- 1 | from .compute import ComputeMetrics 2 | from .mr import MRMetrics 3 | from .tm2t import TM2TMetrics 4 | from .mm import MMMetrics 5 | from .gru import HUMANACTMetrics 6 | from .stgcn import UESTCMetrics 7 | from .uncond import UncondMetrics 8 | -------------------------------------------------------------------------------- /GraphMotion/models/metrics/compute_best.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | import torch 4 | from einops import rearrange 5 | from torch import Tensor 6 | from torchmetrics import Metric 7 | import numpy as np 8 | from .compute import ComputeMetrics, l2_norm, variance 9 | 10 | 11 | class ComputeMetricsBest(ComputeMetrics): 12 | def update(self, jts_text_: List[Tensor], jts_ref_: List[Tensor], lengths: List[List[int]]): 13 | self.count += sum(lengths[0]) 14 | self.count_seq += len(lengths[0]) 15 | 16 | ntrials = len(jts_text_) 17 | metrics = [] 18 | for index in range(ntrials): 19 | jts_text, poses_text, root_text, traj_text = self.transform(jts_text_[index], lengths[index]) 20 | jts_ref, poses_ref, root_ref, traj_ref = self.transform(jts_ref_[index], lengths[index]) 21 | 22 | mets = [] 23 | for i in range(len(lengths[index])): 24 | APE_root = l2_norm(root_text[i], root_ref[i], dim=1).sum() 25 | APE_pose = l2_norm(poses_text[i], poses_ref[i], dim=2).sum(0) 26 | APE_traj = l2_norm(traj_text[i], traj_ref[i], dim=1).sum() 27 | APE_joints = l2_norm(jts_text[i], jts_ref[i], dim=2).sum(0) 28 | 29 | root_sigma_text = variance(root_text[i], lengths[index][i], dim=0) 30 | root_sigma_ref = variance(root_ref[i], lengths[index][i], dim=0) 31 | AVE_root = l2_norm(root_sigma_text, root_sigma_ref, dim=0) 32 | 33 | traj_sigma_text = variance(traj_text[i], lengths[index][i], dim=0) 34 | traj_sigma_ref = variance(traj_ref[i], lengths[index][i], dim=0) 35 | AVE_traj = l2_norm(traj_sigma_text, traj_sigma_ref, dim=0) 36 | 37 | poses_sigma_text = variance(poses_text[i], lengths[index][i], dim=0) 38 | poses_sigma_ref = variance(poses_ref[i], lengths[index][i], dim=0) 39 | AVE_pose = l2_norm(poses_sigma_text, poses_sigma_ref, dim=1) 40 | 41 | jts_sigma_text = variance(jts_text[i], lengths[index][i], dim=0) 42 | jts_sigma_ref = variance(jts_ref[i], lengths[index][i], dim=0) 43 | AVE_joints = l2_norm(jts_sigma_text, jts_sigma_ref, dim=1) 44 | 45 | met = [APE_root, APE_pose, APE_traj, APE_joints, 46 | AVE_root, AVE_pose, AVE_traj, AVE_joints] 47 | mets.append(met) 48 | metrics.append(mets) 49 | 50 | # Quick hacks 51 | mmm = metrics[np.argmin([x[0][0] for x in metrics])] 52 | APE_root, APE_pose, APE_traj, APE_joints, AVE_root, AVE_pose, AVE_traj, AVE_joints = mmm[0] 53 | self.APE_root += APE_root 54 | self.APE_pose += APE_pose 55 | self.APE_traj += APE_traj 56 | self.APE_joints += APE_joints 57 | self.AVE_root += AVE_root 58 | self.AVE_pose += AVE_pose 59 | self.AVE_traj += AVE_traj 60 | self.AVE_joints += AVE_joints 61 | -------------------------------------------------------------------------------- /GraphMotion/models/metrics/compute_worst.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | import torch 4 | from einops import rearrange 5 | from torch import Tensor 6 | from torchmetrics import Metric 7 | import numpy as np 8 | from .compute import ComputeMetrics, l2_norm, variance 9 | 10 | 11 | class ComputeMetricsWorst(ComputeMetrics): 12 | def update(self, jts_text_: List[Tensor], jts_ref_: List[Tensor], lengths: List[List[int]]): 13 | self.count += sum(lengths[0]) 14 | self.count_seq += len(lengths[0]) 15 | 16 | ntrials = len(jts_text_) 17 | metrics = [] 18 | for index in range(ntrials): 19 | jts_text, poses_text, root_text, traj_text = self.transform(jts_text_[index], lengths[index]) 20 | jts_ref, poses_ref, root_ref, traj_ref = self.transform(jts_ref_[index], lengths[index]) 21 | 22 | mets = [] 23 | for i in range(len(lengths[index])): 24 | APE_root = l2_norm(root_text[i], root_ref[i], dim=1).sum() 25 | APE_pose = l2_norm(poses_text[i], poses_ref[i], dim=2).sum(0) 26 | APE_traj = l2_norm(traj_text[i], traj_ref[i], dim=1).sum() 27 | APE_joints = l2_norm(jts_text[i], jts_ref[i], dim=2).sum(0) 28 | 29 | root_sigma_text = variance(root_text[i], lengths[index][i], dim=0) 30 | root_sigma_ref = variance(root_ref[i], lengths[index][i], dim=0) 31 | AVE_root = l2_norm(root_sigma_text, root_sigma_ref, dim=0) 32 | 33 | traj_sigma_text = variance(traj_text[i], lengths[index][i], dim=0) 34 | traj_sigma_ref = variance(traj_ref[i], lengths[index][i], dim=0) 35 | AVE_traj = l2_norm(traj_sigma_text, traj_sigma_ref, dim=0) 36 | 37 | poses_sigma_text = variance(poses_text[i], lengths[index][i], dim=0) 38 | poses_sigma_ref = variance(poses_ref[i], lengths[index][i], dim=0) 39 | AVE_pose = l2_norm(poses_sigma_text, poses_sigma_ref, dim=1) 40 | 41 | jts_sigma_text = variance(jts_text[i], lengths[index][i], dim=0) 42 | jts_sigma_ref = variance(jts_ref[i], lengths[index][i], dim=0) 43 | AVE_joints = l2_norm(jts_sigma_text, jts_sigma_ref, dim=1) 44 | 45 | met = [APE_root, APE_pose, APE_traj, APE_joints, 46 | AVE_root, AVE_pose, AVE_traj, AVE_joints] 47 | mets.append(met) 48 | metrics.append(mets) 49 | 50 | # Quick hacks 51 | mmm = metrics[np.argmax([x[0][0] for x in metrics])] 52 | APE_root, APE_pose, APE_traj, APE_joints, AVE_root, AVE_pose, AVE_traj, AVE_joints = mmm[0] 53 | self.APE_root += APE_root 54 | self.APE_pose += APE_pose 55 | self.APE_traj += APE_traj 56 | self.APE_joints += APE_joints 57 | self.AVE_root += AVE_root 58 | self.AVE_pose += AVE_pose 59 | self.AVE_traj += AVE_traj 60 | self.AVE_joints += AVE_joints 61 | -------------------------------------------------------------------------------- /GraphMotion/models/metrics/mm.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | import torch 4 | from torch import Tensor 5 | from torchmetrics import Metric 6 | from torchmetrics.functional import pairwise_euclidean_distance 7 | 8 | from .utils import * 9 | 10 | 11 | class MMMetrics(Metric): 12 | full_state_update = True 13 | 14 | def __init__(self, mm_num_times=10, dist_sync_on_step=True, stage=0, **kwargs): 15 | super().__init__(dist_sync_on_step=dist_sync_on_step) 16 | 17 | self.name = "MultiModality scores" 18 | 19 | self.mm_num_times = mm_num_times 20 | 21 | self.add_state("count", default=torch.tensor(0), dist_reduce_fx="sum") 22 | self.add_state("count_seq", 23 | default=torch.tensor(0), 24 | dist_reduce_fx="sum") 25 | 26 | self.stage =stage 27 | 28 | if self.stage in [1, 2, 3]: 29 | self.metrics = [f"s{str(self.stage)}_MultiModality"] 30 | self.add_state(f"s{str(self.stage)}_MultiModality", 31 | default=torch.tensor(0.), 32 | dist_reduce_fx="sum") 33 | else: 34 | self.metrics = ["MultiModality"] 35 | self.add_state("MultiModality", 36 | default=torch.tensor(0.), 37 | dist_reduce_fx="sum") 38 | 39 | # chached batches 40 | self.add_state("mm_motion_embeddings", default=[], dist_reduce_fx=None) 41 | 42 | def compute(self, sanity_flag): 43 | count = self.count.item() 44 | count_seq = self.count_seq.item() 45 | 46 | # init metrics 47 | metrics = {metric: getattr(self, metric) for metric in self.metrics} 48 | 49 | # if in sanity check stage then jump 50 | if sanity_flag: 51 | return metrics 52 | 53 | # cat all embeddings 54 | all_mm_motions = torch.cat(self.mm_motion_embeddings, 55 | axis=0).cpu().numpy() 56 | if self.stage in [1, 2, 3]: 57 | metrics[f"s{str(self.stage)}_MultiModality"] = calculate_multimodality_np( 58 | all_mm_motions, self.mm_num_times) 59 | else: 60 | metrics['MultiModality'] = calculate_multimodality_np( 61 | all_mm_motions, self.mm_num_times) 62 | 63 | return {**metrics} 64 | 65 | def update( 66 | self, 67 | mm_motion_embeddings: Tensor, 68 | lengths: List[int], 69 | ): 70 | self.count += sum(lengths) 71 | self.count_seq += len(lengths) 72 | 73 | # store all mm motion embeddings 74 | self.mm_motion_embeddings.append(mm_motion_embeddings) 75 | -------------------------------------------------------------------------------- /GraphMotion/models/modeltype/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpthu17/GraphMotion/d56feba6f6d39867b933ca752244fcbbb703a4b8/GraphMotion/models/modeltype/__init__.py -------------------------------------------------------------------------------- /GraphMotion/models/operator/__init__.py: -------------------------------------------------------------------------------- 1 | from .adain import AdaptiveInstanceNorm1d 2 | from .blocks import ConvBlock, LinearBlock 3 | from .position_encoding_layer import PositionalEncoding 4 | -------------------------------------------------------------------------------- /GraphMotion/models/operator/adain.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | 5 | class AdaptiveInstanceNorm1d(nn.Module): 6 | def __init__(self, num_features, eps=1e-5, momentum=0.1): 7 | super(AdaptiveInstanceNorm1d, self).__init__() 8 | self.num_features = num_features 9 | self.eps = eps 10 | self.momentum = momentum 11 | self.weight = None 12 | self.bias = None 13 | self.register_buffer('running_mean', torch.zeros(num_features)) 14 | self.register_buffer('running_var', torch.ones(num_features)) 15 | 16 | def forward(self, x, direct_weighting=False, no_std=False): 17 | assert self.weight is not None and \ 18 | self.bias is not None, "Please assign AdaIN weight first" 19 | # (bs, nfeats, nframe) <= (nframe, bs, nfeats) 20 | x = x.permute(1,2,0) 21 | 22 | b, c = x.size(0), x.size(1) # batch size & channels 23 | running_mean = self.running_mean.repeat(b) 24 | running_var = self.running_var.repeat(b) 25 | # self.weight = torch.ones_like(self.weight) 26 | 27 | if direct_weighting: 28 | x_reshaped = x.contiguous().view(b * c) 29 | if no_std: 30 | out = x_reshaped + self.bias 31 | else: 32 | out = x_reshaped.mul(self.weight) + self.bias 33 | out = out.view(b, c, *x.size()[2:]) 34 | else: 35 | x_reshaped = x.contiguous().view(1, b * c, *x.size()[2:]) 36 | out = F.batch_norm( 37 | x_reshaped, running_mean, running_var, self.weight, self.bias, 38 | True, self.momentum, self.eps) 39 | out = out.view(b, c, *x.size()[2:]) 40 | 41 | # (nframe, bs, nfeats) <= (bs, nfeats, nframe) 42 | out = out.permute(2,0,1) 43 | return out 44 | 45 | def __repr__(self): 46 | return self.__class__.__name__ + '(' + str(self.num_features) + ')' 47 | 48 | def assign_adain_params(adain_params, model): 49 | # assign the adain_params to the AdaIN layers in model 50 | for m in model.modules(): 51 | if m.__class__.__name__ == "AdaptiveInstanceNorm1d": 52 | mean = adain_params[: , : m.num_features] 53 | std = adain_params[: , m.num_features: 2 * m.num_features] 54 | m.bias = mean.contiguous().view(-1) 55 | m.weight = std.contiguous().view(-1) 56 | if adain_params.size(1) > 2 * m.num_features: 57 | adain_params = adain_params[: , 2 * m.num_features:] 58 | 59 | 60 | def get_num_adain_params(model): 61 | # return the number of AdaIN parameters needed by the model 62 | num_adain_params = 0 63 | for m in model.modules(): 64 | if m.__class__.__name__ == "AdaptiveInstanceNorm1d": 65 | num_adain_params += 2 * m.num_features 66 | return num_adain_params 67 | -------------------------------------------------------------------------------- /GraphMotion/models/operator/position_encoding_layer.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | from torch import nn 4 | 5 | 6 | class PositionalEncoding(nn.Module): 7 | 8 | def __init__(self, d_model, dropout=0.1, max_len=5000, batch_first=False): 9 | super().__init__() 10 | self.batch_first = batch_first 11 | 12 | self.dropout = nn.Dropout(p=dropout) 13 | 14 | pe = torch.zeros(max_len, d_model) 15 | position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1) 16 | div_term = torch.exp(torch.arange( 17 | 0, d_model, 2).float() * (-np.log(10000.0) / d_model)) 18 | pe[:, 0::2] = torch.sin(position * div_term) 19 | pe[:, 1::2] = torch.cos(position * div_term) 20 | pe = pe.unsqueeze(0).transpose(0, 1) 21 | 22 | self.register_buffer("pe", pe) 23 | 24 | def forward(self, x): 25 | # not used in the final model 26 | if self.batch_first: 27 | x = x + self.pe.permute(1, 0, 2)[:, : x.shape[1], :] 28 | else: 29 | x = x + self.pe[: x.shape[0], :] 30 | return self.dropout(x) 31 | -------------------------------------------------------------------------------- /GraphMotion/models/operator/self_attention.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpthu17/GraphMotion/d56feba6f6d39867b933ca752244fcbbb703a4b8/GraphMotion/models/operator/self_attention.py -------------------------------------------------------------------------------- /GraphMotion/models/tools/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpthu17/GraphMotion/d56feba6f6d39867b933ca752244fcbbb703a4b8/GraphMotion/models/tools/__init__.py -------------------------------------------------------------------------------- /GraphMotion/models/tools/tools.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | 3 | def remove_padding(tensors, lengths): 4 | return [tensor[:tensor_length] for tensor, tensor_length in zip(tensors, lengths)] 5 | 6 | class AutoParams(nn.Module): 7 | def __init__(self, **kargs): 8 | try: 9 | for param in self.needed_params: 10 | if param in kargs: 11 | setattr(self, param, kargs[param]) 12 | else: 13 | raise ValueError(f"{param} is needed.") 14 | except : 15 | pass 16 | 17 | try: 18 | for param, default in self.optional_params.items(): 19 | if param in kargs and kargs[param] is not None: 20 | setattr(self, param, kargs[param]) 21 | else: 22 | setattr(self, param, default) 23 | except : 24 | pass 25 | super().__init__() 26 | 27 | 28 | # taken from joeynmt repo 29 | def freeze_params(module: nn.Module) -> None: 30 | """ 31 | Freeze the parameters of this module, 32 | i.e. do not update them during training 33 | 34 | :param module: freeze parameters of this module 35 | """ 36 | for _, p in module.named_parameters(): 37 | p.requires_grad = False 38 | -------------------------------------------------------------------------------- /GraphMotion/render/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpthu17/GraphMotion/d56feba6f6d39867b933ca752244fcbbb703a4b8/GraphMotion/render/__init__.py -------------------------------------------------------------------------------- /GraphMotion/render/blender/__init__.py: -------------------------------------------------------------------------------- 1 | from .render import render 2 | -------------------------------------------------------------------------------- /GraphMotion/render/blender/camera.py: -------------------------------------------------------------------------------- 1 | import bpy 2 | 3 | 4 | class Camera: 5 | def __init__(self, *, first_root, mode, is_mesh): 6 | camera = bpy.data.objects['Camera'] 7 | 8 | ## initial position 9 | camera.location.x = 7.36 10 | camera.location.y = -6.93 11 | if is_mesh: 12 | # camera.location.z = 5.45 13 | camera.location.z = 5.6 14 | else: 15 | camera.location.z = 5.2 16 | 17 | # wider point of view 18 | if mode == "sequence": 19 | if is_mesh: 20 | camera.data.lens = 65 21 | else: 22 | camera.data.lens = 85 23 | elif mode == "frame": 24 | if is_mesh: 25 | camera.data.lens = 130 26 | else: 27 | camera.data.lens = 85 28 | elif mode == "video": 29 | if is_mesh: 30 | camera.data.lens = 110 31 | else: 32 | # avoid cutting person 33 | camera.data.lens = 85 34 | # camera.data.lens = 140 35 | 36 | # camera.location.x += 0.75 37 | 38 | self.mode = mode 39 | self.camera = camera 40 | 41 | self.camera.location.x += first_root[0] 42 | self.camera.location.y += first_root[1] 43 | 44 | self._root = first_root 45 | 46 | def update(self, newroot): 47 | delta_root = newroot - self._root 48 | 49 | self.camera.location.x += delta_root[0] 50 | self.camera.location.y += delta_root[1] 51 | 52 | self._root = newroot 53 | -------------------------------------------------------------------------------- /GraphMotion/render/blender/data.py: -------------------------------------------------------------------------------- 1 | class Data: 2 | def __len__(self): 3 | return self.N 4 | -------------------------------------------------------------------------------- /GraphMotion/render/blender/meshes.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from .materials import body_material 4 | 5 | # green 6 | # GT_SMPL = body_material(0.009, 0.214, 0.029) 7 | GT_SMPL = body_material(0.035, 0.415, 0.122) 8 | 9 | # blue 10 | # GEN_SMPL = body_material(0.022, 0.129, 0.439) 11 | # Blues => cmap(0.87) 12 | # GEN_SMPL = body_material(0.035, 0.322, 0.615) 13 | # Oranges => cmap(0.87) 14 | GEN_SMPL = body_material(0.658, 0.214, 0.0114) 15 | 16 | 17 | class Meshes: 18 | def __init__(self, data, *, gt, mode, faces_path, canonicalize, always_on_floor, oldrender=True, **kwargs): 19 | data = prepare_meshes(data, canonicalize=canonicalize, 20 | always_on_floor=always_on_floor) 21 | 22 | self.faces = np.load(faces_path) 23 | print(faces_path) 24 | self.data = data 25 | self.mode = mode 26 | self.oldrender = oldrender 27 | 28 | self.N = len(data) 29 | self.trajectory = data[:, :, [0, 1]].mean(1) 30 | 31 | if gt: 32 | self.mat = GT_SMPL 33 | else: 34 | self.mat = GEN_SMPL 35 | 36 | def get_sequence_mat(self, frac): 37 | import matplotlib 38 | # cmap = matplotlib.cm.get_cmap('Blues') 39 | cmap = matplotlib.cm.get_cmap('Oranges') 40 | # begin = 0.60 41 | # end = 0.90 42 | begin = 0.50 43 | end = 0.90 44 | rgbcolor = cmap(begin + (end-begin)*frac) 45 | mat = body_material(*rgbcolor, oldrender=self.oldrender) 46 | return mat 47 | 48 | def get_root(self, index): 49 | return self.data[index].mean(0) 50 | 51 | def get_mean_root(self): 52 | return self.data.mean((0, 1)) 53 | 54 | def load_in_blender(self, index, mat): 55 | vertices = self.data[index] 56 | faces = self.faces 57 | name = f"{str(index).zfill(4)}" 58 | 59 | from .tools import load_numpy_vertices_into_blender 60 | load_numpy_vertices_into_blender(vertices, faces, name, mat) 61 | 62 | return name 63 | 64 | def __len__(self): 65 | return self.N 66 | 67 | 68 | def prepare_meshes(data, canonicalize=True, always_on_floor=False): 69 | if canonicalize: 70 | print("No canonicalization for now") 71 | 72 | # fitted mesh do not need fixing axis 73 | # # fix axis 74 | # data[..., 1] = - data[..., 1] 75 | # data[..., 0] = - data[..., 0] 76 | 77 | # Swap axis (gravity=Z instead of Y) 78 | data = data[..., [2, 0, 1]] 79 | 80 | # Remove the floor 81 | data[..., 2] -= data[..., 2].min() 82 | 83 | # Put all the body on the floor 84 | if always_on_floor: 85 | data[..., 2] -= data[..., 2].min(1)[:, None] 86 | 87 | return data 88 | -------------------------------------------------------------------------------- /GraphMotion/render/blender/sampler.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | def get_frameidx(*, mode, nframes, exact_frame, frames_to_keep): 4 | if mode == "sequence": 5 | frameidx = np.linspace(0, nframes - 1, frames_to_keep) 6 | frameidx = np.round(frameidx).astype(int) 7 | frameidx = list(frameidx) 8 | elif mode == "frame": 9 | index_frame = int(exact_frame*nframes) 10 | frameidx = [index_frame] 11 | elif mode == "video": 12 | frameidx = range(0, nframes) 13 | else: 14 | raise ValueError(f"Not support {mode} render mode") 15 | return frameidx 16 | -------------------------------------------------------------------------------- /GraphMotion/render/blender/tools.py: -------------------------------------------------------------------------------- 1 | import bpy 2 | import numpy as np 3 | 4 | 5 | def mesh_detect(data): 6 | # heuristic 7 | if data.shape[1] > 1000: 8 | return True 9 | return False 10 | 11 | 12 | # see this for more explanation 13 | # https://gist.github.com/iyadahmed/7c7c0fae03c40bd87e75dc7059e35377 14 | # This should be solved with new version of blender 15 | class ndarray_pydata(np.ndarray): 16 | def __bool__(self) -> bool: 17 | return len(self) > 0 18 | 19 | 20 | def load_numpy_vertices_into_blender(vertices, faces, name, mat): 21 | mesh = bpy.data.meshes.new(name) 22 | mesh.from_pydata(vertices, [], faces.view(ndarray_pydata)) 23 | mesh.validate() 24 | 25 | obj = bpy.data.objects.new(name, mesh) 26 | bpy.context.scene.collection.objects.link(obj) 27 | 28 | bpy.ops.object.select_all(action='DESELECT') 29 | obj.select_set(True) 30 | obj.active_material = mat 31 | bpy.context.view_layer.objects.active = obj 32 | bpy.ops.object.shade_smooth() 33 | bpy.ops.object.select_all(action='DESELECT') 34 | return True 35 | 36 | 37 | def delete_objs(names): 38 | if not isinstance(names, list): 39 | names = [names] 40 | # bpy.ops.object.mode_set(mode='OBJECT') 41 | bpy.ops.object.select_all(action='DESELECT') 42 | for obj in bpy.context.scene.objects: 43 | for name in names: 44 | if obj.name.startswith(name) or obj.name.endswith(name): 45 | obj.select_set(True) 46 | bpy.ops.object.delete() 47 | bpy.ops.object.select_all(action='DESELECT') 48 | -------------------------------------------------------------------------------- /GraphMotion/render/blender/vertices.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def prepare_vertices(vertices, canonicalize=True): 5 | data = vertices 6 | # Swap axis (gravity=Z instead of Y) 7 | # data = data[..., [2, 0, 1]] 8 | 9 | # Make left/right correct 10 | # data[..., [1]] = -data[..., [1]] 11 | 12 | # Center the first root to the first frame 13 | data -= data[[0], [0], :] 14 | 15 | # Remove the floor 16 | data[..., 2] -= np.min(data[..., 2]) 17 | return data 18 | -------------------------------------------------------------------------------- /GraphMotion/render/video.py: -------------------------------------------------------------------------------- 1 | import moviepy.editor as mp 2 | import moviepy.video.fx.all as vfx 3 | import os 4 | import imageio 5 | 6 | 7 | def mask_png(frames): 8 | for frame in frames: 9 | im = imageio.imread(frame) 10 | im[im[:, :, 3] < 1, :] = 255 11 | imageio.imwrite(frame, im[:, :, 0:3]) 12 | return 13 | 14 | 15 | class Video: 16 | def __init__(self, frame_path: str, fps: float = 12.5, res="high"): 17 | frame_path = str(frame_path) 18 | self.fps = fps 19 | 20 | self._conf = {"codec": "libx264", 21 | "fps": self.fps, 22 | "audio_codec": "aac", 23 | "temp_audiofile": "temp-audio.m4a", 24 | "remove_temp": True} 25 | 26 | if res == "low": 27 | bitrate = "500k" 28 | else: 29 | bitrate = "5000k" 30 | 31 | self._conf = {"bitrate": bitrate, 32 | "fps": self.fps} 33 | 34 | # Load video 35 | # video = mp.VideoFileClip(video1_path, audio=False) 36 | # Load with frames 37 | frames = [os.path.join(frame_path, x) 38 | for x in sorted(os.listdir(frame_path))] 39 | 40 | # mask background white for videos 41 | mask_png(frames) 42 | 43 | video = mp.ImageSequenceClip(frames, fps=fps) 44 | self.video = video 45 | self.duration = video.duration 46 | 47 | def add_text(self, text): 48 | # needs ImageMagick 49 | video_text = mp.TextClip(text, 50 | font='Amiri', 51 | color='white', 52 | method='caption', 53 | align="center", 54 | size=(self.video.w, None), 55 | fontsize=30) 56 | video_text = video_text.on_color(size=(self.video.w, video_text.h + 5), 57 | color=(0, 0, 0), 58 | col_opacity=0.6) 59 | # video_text = video_text.set_pos('bottom') 60 | video_text = video_text.set_pos('top') 61 | 62 | self.video = mp.CompositeVideoClip([self.video, video_text]) 63 | 64 | def save(self, out_path): 65 | out_path = str(out_path) 66 | self.video.subclip(0, self.duration).write_videofile( 67 | out_path, **self._conf) 68 | -------------------------------------------------------------------------------- /GraphMotion/tools/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpthu17/GraphMotion/d56feba6f6d39867b933ca752244fcbbb703a4b8/GraphMotion/tools/__init__.py -------------------------------------------------------------------------------- /GraphMotion/tools/logging.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import tqdm 3 | 4 | 5 | class LevelsFilter(logging.Filter): 6 | def __init__(self, levels): 7 | self.levels = [getattr(logging, level) for level in levels] 8 | 9 | def filter(self, record): 10 | return record.levelno in self.levels 11 | 12 | 13 | class StreamToLogger(object): 14 | """ 15 | Fake file-like stream object that redirects writes to a logger instance. 16 | """ 17 | def __init__(self, logger, level): 18 | self.logger = logger 19 | self.level = level 20 | self.linebuf = '' 21 | 22 | def write(self, buf): 23 | for line in buf.rstrip().splitlines(): 24 | self.logger.log(self.level, line.rstrip()) 25 | 26 | def flush(self): 27 | pass 28 | 29 | 30 | class TqdmLoggingHandler(logging.Handler): 31 | def __init__(self, level=logging.NOTSET): 32 | super().__init__(level) 33 | 34 | def emit(self, record): 35 | try: 36 | msg = self.format(record) 37 | tqdm.tqdm.write(msg) 38 | self.flush() 39 | except Exception: 40 | self.handleError(record) 41 | -------------------------------------------------------------------------------- /GraphMotion/tools/runid.py: -------------------------------------------------------------------------------- 1 | # 2 | """ 3 | runid util. 4 | Taken from wandb.sdk.lib.runid 5 | """ 6 | 7 | import shortuuid # type: ignore 8 | 9 | 10 | def generate_id() -> str: 11 | # ~3t run ids (36**8) 12 | run_gen = shortuuid.ShortUUID(alphabet=list("0123456789abcdefghijklmnopqrstuvwxyz")) 13 | return run_gen.random(8) -------------------------------------------------------------------------------- /GraphMotion/transforms/__init__.py: -------------------------------------------------------------------------------- 1 | from .base import Transform 2 | from .smpl import SMPLTransform 3 | # from .xyz import XYZTransform 4 | -------------------------------------------------------------------------------- /GraphMotion/transforms/base.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass, fields 2 | 3 | 4 | class Transform: 5 | 6 | def collate(self, lst_datastruct): 7 | from GraphMotion.datasets.utils import collate_tensor_with_padding 8 | example = lst_datastruct[0] 9 | 10 | def collate_or_none(key): 11 | if example[key] is None: 12 | return None 13 | key_lst = [x[key] for x in lst_datastruct] 14 | return collate_tensor_with_padding(key_lst) 15 | 16 | kwargs = {key: collate_or_none(key) for key in example.datakeys} 17 | 18 | return self.Datastruct(**kwargs) 19 | 20 | 21 | # Inspired from SMPLX library 22 | # need to define "datakeys" and transforms 23 | @dataclass 24 | class Datastruct: 25 | 26 | def __getitem__(self, key): 27 | return getattr(self, key) 28 | 29 | def __setitem__(self, key, value): 30 | self.__dict__[key] = value 31 | 32 | def get(self, key, default=None): 33 | return getattr(self, key, default) 34 | 35 | def __iter__(self): 36 | return self.keys() 37 | 38 | def keys(self): 39 | keys = [t.name for t in fields(self)] 40 | return iter(keys) 41 | 42 | def values(self): 43 | values = [getattr(self, t.name) for t in fields(self)] 44 | return iter(values) 45 | 46 | def items(self): 47 | data = [(t.name, getattr(self, t.name)) for t in fields(self)] 48 | return iter(data) 49 | 50 | def to(self, *args, **kwargs): 51 | for key in self.datakeys: 52 | if self[key] is not None: 53 | self[key] = self[key].to(*args, **kwargs) 54 | return self 55 | 56 | @property 57 | def device(self): 58 | return self[self.datakeys[0]].device 59 | 60 | def detach(self): 61 | 62 | def detach_or_none(tensor): 63 | if tensor is not None: 64 | return tensor.detach() 65 | return None 66 | 67 | kwargs = {key: detach_or_none(self[key]) for key in self.datakeys} 68 | return self.transforms.Datastruct(**kwargs) 69 | -------------------------------------------------------------------------------- /GraphMotion/transforms/feats2smpl.py: -------------------------------------------------------------------------------- 1 | from os.path import join as pjoin 2 | 3 | import numpy as np 4 | import torch 5 | 6 | import GraphMotion.data.humanml.utils.paramUtil as paramUtil 7 | from GraphMotion.data.humanml.data.dataset import Text2MotionDatasetV2 8 | from GraphMotion.data.humanml.scripts.motion_process import recover_from_ric 9 | from GraphMotion.data.humanml.utils.plot_script import plot_3d_motion 10 | 11 | skeleton = paramUtil.t2m_kinematic_chain 12 | 13 | # convert humanML3d features to skeleton format for rendering 14 | # def feats2joints(motion, data_root = '../datasets/humanml3d'): 15 | # ''' 16 | # input: 263 features 17 | # output: 22 joints? 18 | # ''' 19 | # mean = torch.from_numpy(np.load(pjoin(data_root, 'Mean.npy'))) 20 | # std = torch.from_numpy(np.load(pjoin(data_root, 'Std.npy'))) 21 | 22 | # motion = motion * std + mean 23 | # motion_rec = recover_from_ric(motion, joints_num=22) 24 | # # motion_rec = motion_rec * 1.3 25 | # return motion_rec 26 | 27 | 28 | def main(): 29 | data_root = '../datasets/humanml3d' 30 | feastures_path = 'in.npy' 31 | animation_save_path = 'in.mp4' 32 | 33 | fps = 20 34 | mean = np.load(pjoin(data_root, 'Mean.npy')) 35 | std = np.load(pjoin(data_root, 'Std.npy')) 36 | 37 | motion = np.load(feastures_path) 38 | motion = motion * std + mean 39 | motion_rec = recover_from_ric(torch.tensor(motion), 22).cpu().numpy() 40 | # with open('in_22.npy', 'wb') as f: 41 | # np.save(f,motion_rec) 42 | motion_rec = motion_rec * 1.3 43 | plot_3d_motion(animation_save_path, motion_rec, title='input', fps=fps) 44 | 45 | 46 | if __name__ == '__main__': 47 | main() 48 | -------------------------------------------------------------------------------- /GraphMotion/transforms/identity.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | from torch import Tensor 3 | 4 | from .base import Datastruct, dataclass, Transform 5 | 6 | 7 | class IdentityTransform(Transform): 8 | def __init__(self, **kwargs): 9 | return 10 | 11 | def Datastruct(self, **kwargs): 12 | return IdentityDatastruct(**kwargs) 13 | 14 | def __repr__(self): 15 | return "IdentityTransform()" 16 | 17 | 18 | @dataclass 19 | class IdentityDatastruct(Datastruct): 20 | transforms: IdentityTransform 21 | 22 | features: Optional[Tensor] = None 23 | 24 | def __post_init__(self): 25 | self.datakeys = ["features"] 26 | 27 | def __len__(self): 28 | return len(self.rfeats) 29 | -------------------------------------------------------------------------------- /GraphMotion/transforms/joints2jfeats/__init__.py: -------------------------------------------------------------------------------- 1 | from .base import Joints2Jfeats 2 | from .rifke import Rifke 3 | -------------------------------------------------------------------------------- /GraphMotion/transforms/joints2jfeats/base.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | import torch 4 | from torch import Tensor, nn 5 | from pathlib import Path 6 | 7 | 8 | class Joints2Jfeats(nn.Module): 9 | def __init__(self, path: Optional[str] = None, 10 | normalization: bool = False, 11 | eps: float = 1e-12, 12 | **kwargs) -> None: 13 | if normalization and path is None: 14 | raise TypeError("You should provide a path if normalization is on.") 15 | 16 | super().__init__() 17 | self.normalization = normalization 18 | self.eps = eps 19 | 20 | if normalization: 21 | mean_path = Path(path) / "jfeats_mean.pt" 22 | std_path = Path(path) / "jfeats_std.pt" 23 | self.register_buffer('mean', torch.load(mean_path)) 24 | self.register_buffer('std', torch.load(std_path)) 25 | 26 | def normalize(self, features: Tensor) -> Tensor: 27 | if self.normalization: 28 | features = (features - self.mean)/(self.std + self.eps) 29 | return features 30 | 31 | def unnormalize(self, features: Tensor) -> Tensor: 32 | if self.normalization: 33 | features = features * self.std + self.mean 34 | return features 35 | -------------------------------------------------------------------------------- /GraphMotion/transforms/rots2joints/__init__.py: -------------------------------------------------------------------------------- 1 | from .base import Rots2Joints 2 | from .smplh import SMPLH 3 | -------------------------------------------------------------------------------- /GraphMotion/transforms/rots2joints/base.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | import torch 4 | from torch import Tensor, nn 5 | from pathlib import Path 6 | 7 | 8 | class Rots2Joints(nn.Module): 9 | def __init__(self, path: Optional[str] = None, 10 | normalization: bool = False, 11 | eps: float = 1e-12, 12 | **kwargs) -> None: 13 | if normalization and path is None: 14 | raise TypeError("You should provide a path if normalization is on.") 15 | 16 | super().__init__() 17 | self.normalization = normalization 18 | self.eps = eps 19 | 20 | if normalization: 21 | mean_path = Path(path) / "mean.pt" 22 | std_path = Path(path) / "std.pt" 23 | self.register_buffer('mean', torch.load(mean_path)) 24 | self.register_buffer('std', torch.load(std_path)) 25 | 26 | def normalize(self, features: Tensor) -> Tensor: 27 | if self.normalization: 28 | features = (features - self.mean)/(self.std + self.eps) 29 | return features 30 | 31 | def unnormalize(self, features: Tensor) -> Tensor: 32 | if self.normalization: 33 | features = features * self.std + self.mean 34 | return features 35 | -------------------------------------------------------------------------------- /GraphMotion/transforms/rots2rfeats/__init__.py: -------------------------------------------------------------------------------- 1 | from .base import Rots2Rfeats 2 | from .smplvelp import SMPLVelP 3 | -------------------------------------------------------------------------------- /GraphMotion/transforms/rots2rfeats/base.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | import torch 4 | from torch import Tensor, nn 5 | from pathlib import Path 6 | 7 | 8 | class Rots2Rfeats(nn.Module): 9 | def __init__(self, path: Optional[str] = None, 10 | normalization: bool = False, 11 | eps: float = 1e-12, 12 | **kwargs) -> None: 13 | if normalization and path is None: 14 | raise TypeError("You should provide a path if normalization is on.") 15 | 16 | super().__init__() 17 | self.normalization = normalization 18 | self.eps = eps 19 | 20 | if normalization: 21 | mean_path = Path(path) / "rfeats_mean.pt" 22 | std_path = Path(path) / "rfeats_std.pt" 23 | self.register_buffer('mean', torch.load(mean_path)) 24 | self.register_buffer('std', torch.load(std_path)) 25 | 26 | def normalize(self, features: Tensor) -> Tensor: 27 | if self.normalization: 28 | features = (features - self.mean)/(self.std + self.eps) 29 | return features 30 | 31 | def unnormalize(self, features: Tensor) -> Tensor: 32 | if self.normalization: 33 | features = features * self.std + self.mean 34 | return features 35 | -------------------------------------------------------------------------------- /GraphMotion/transforms/xyz.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | from torch import Tensor 3 | 4 | from .base import Datastruct, dataclass, Transform 5 | from GraphMotion.datasets.utils import collate_tensor_with_padding 6 | 7 | from .joints2jfeats import Joints2Jfeats 8 | 9 | 10 | class XYZTransform(Transform): 11 | 12 | def __init__(self, joints2jfeats: Joints2Jfeats, **kwargs): 13 | self.joints2jfeats = joints2jfeats 14 | 15 | def Datastruct(self, **kwargs): 16 | return XYZDatastruct(_joints2jfeats=self.joints2jfeats, 17 | transforms=self, 18 | **kwargs) 19 | 20 | def __repr__(self): 21 | return "XYZTransform()" 22 | 23 | 24 | @dataclass 25 | class XYZDatastruct(Datastruct): 26 | transforms: XYZTransform 27 | _joints2jfeats: Joints2Jfeats 28 | 29 | features: Optional[Tensor] = None 30 | joints_: Optional[Tensor] = None 31 | jfeats_: Optional[Tensor] = None 32 | 33 | def __post_init__(self): 34 | self.datakeys = ["features", "joints_", "jfeats_"] 35 | # starting point 36 | if self.features is not None and self.jfeats_ is None: 37 | self.jfeats_ = self.features 38 | 39 | @property 40 | def joints(self): 41 | # Cached value 42 | if self.joints_ is not None: 43 | return self.joints_ 44 | 45 | # self.jfeats_ should be defined 46 | assert self.jfeats_ is not None 47 | 48 | self._joints2jfeats.to(self.jfeats.device) 49 | self.joints_ = self._joints2jfeats.inverse(self.jfeats) 50 | return self.joints_ 51 | 52 | @property 53 | def jfeats(self): 54 | # Cached value 55 | if self.jfeats_ is not None: 56 | return self.jfeats_ 57 | 58 | # self.joints_ should be defined 59 | assert self.joints_ is not None 60 | 61 | self._joints2jfeats.to(self.joints.device) 62 | self.jfeats_ = self._joints2jfeats(self.joints) 63 | return self.jfeats_ 64 | 65 | def __len__(self): 66 | return len(self.jfeats) 67 | -------------------------------------------------------------------------------- /GraphMotion/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpthu17/GraphMotion/d56feba6f6d39867b933ca752244fcbbb703a4b8/GraphMotion/utils/__init__.py -------------------------------------------------------------------------------- /GraphMotion/utils/demo_utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | from pathlib import Path 3 | 4 | 5 | # load example data 6 | def load_example_input(txt_path): 7 | file = open(txt_path, "r") 8 | Lines = file.readlines() 9 | count = 0 10 | texts, lens = [], [] 11 | # Strips the newline character 12 | for line in Lines: 13 | count += 1 14 | s = line.strip() 15 | s_l = s.split(" ")[0] 16 | s_t = s[(len(s_l) + 1):] 17 | lens.append(int(s_l)) 18 | texts.append(s_t) 19 | print("Length-{}: {}".format(s_l, s_t)) 20 | return texts, lens 21 | 22 | 23 | # render batch 24 | def render_batch(npy_dir, execute_python="./scripts/visualize_motion.sh", mode="sequence"): 25 | os.system(f"{execute_python} {npy_dir} {mode}") 26 | 27 | 28 | # render 29 | def render(execute_python, npy_path, jointtype, cfg_path): 30 | # execute_python = "/apdcephfs/share_1227775/shingxchen/libs/blender_bpy/blender-2.93.2-linux-x64/blender" 31 | # execute_python = "/apdcephfs/share_1227775/mingzhenzhu/jiangbiao/libs/blender-2.93.2-linux-x64/blender" 32 | export_scripts = "render.py" 33 | 34 | os.system( 35 | f"{execute_python} --background --python {export_scripts} -- --cfg={cfg_path} --npy={npy_path} --joint_type={jointtype}" 36 | ) 37 | 38 | fig_path = Path(str(npy_path).replace(".npy", ".png")) 39 | return fig_path 40 | 41 | 42 | # origin render 43 | # def render(npy_path, jointtype): 44 | # execute_python = '/apdcephfs/share_1227775/shingxchen/libs/blender_bpy/blender-2.93.2-linux-x64/blender' 45 | # export_scripts = 'render.py' 46 | 47 | # os.system(f"{execute_python} --background --python {export_scripts} -- npy={npy_path} jointstype={jointtype}") 48 | 49 | # fig_path = Path(str(npy_path).replace(".npy",".png")) 50 | # return fig_path 51 | 52 | # export fbx with hand params from pkl files 53 | # refer to /apdcephfs/share_1227775/shingxchen/AIMotion/TMOST/scripts/fbx_output_smplx.py 54 | def export_fbx_hand(pkl_path): 55 | input = pkl_path 56 | output = pkl_path.replace(".pkl", ".fbx") 57 | 58 | execute_python = "/apdcephfs/share_1227775/shingxchen/libs/blender_bpy/blender-2.93.2-linux-x64/blender" 59 | export_scripts = "./scripts/fbx_output_smplx.py" 60 | os.system( 61 | f"{execute_python} -noaudio --background --python {export_scripts}\ 62 | --input {input} \ 63 | --output {output}" 64 | ) 65 | 66 | 67 | # export fbx without hand params from pkl files 68 | # refer to /apdcephfs/share_1227775/shingxchen/AIMotion/TMOST/scripts/fbx_output.py 69 | def export_fbx(pkl_path): 70 | input = pkl_path 71 | output = pkl_path.replace(".pkl", ".fbx") 72 | 73 | execute_python = "/apdcephfs/share_1227775/shingxchen/libs/blender_bpy/blender-2.93.2-linux-x64/blender" 74 | export_scripts = "./scripts/fbx_output.py" 75 | os.system( 76 | f"{execute_python} -noaudio --background --python {export_scripts}\ 77 | --input {input} \ 78 | --output {output}" 79 | ) 80 | -------------------------------------------------------------------------------- /GraphMotion/utils/easyconvert.py: -------------------------------------------------------------------------------- 1 | import GraphMotion.utils.geometry as geometry 2 | 3 | 4 | def nfeats_of(rottype): 5 | if rottype in ["rotvec", "axisangle"]: 6 | return 3 7 | elif rottype in ["rotquat", "quaternion"]: 8 | return 4 9 | elif rottype in ["rot6d", "6drot", "rotation6d"]: 10 | return 6 11 | elif rottype in ["rotmat"]: 12 | return 9 13 | else: 14 | return TypeError("This rotation type doesn't have features.") 15 | 16 | 17 | def axis_angle_to(newtype, rotations): 18 | if newtype in ["matrix"]: 19 | rotations = geometry.axis_angle_to_matrix(rotations) 20 | return rotations 21 | elif newtype in ["rotmat"]: 22 | rotations = geometry.axis_angle_to_matrix(rotations) 23 | rotations = matrix_to("rotmat", rotations) 24 | return rotations 25 | elif newtype in ["rot6d", "6drot", "rotation6d"]: 26 | rotations = geometry.axis_angle_to_matrix(rotations) 27 | rotations = matrix_to("rot6d", rotations) 28 | return rotations 29 | elif newtype in ["rotquat", "quaternion"]: 30 | rotations = geometry.axis_angle_to_quaternion(rotations) 31 | return rotations 32 | elif newtype in ["rotvec", "axisangle"]: 33 | return rotations 34 | else: 35 | raise NotImplementedError 36 | 37 | 38 | def matrix_to(newtype, rotations): 39 | if newtype in ["matrix"]: 40 | return rotations 41 | if newtype in ["rotmat"]: 42 | rotations = rotations.reshape((*rotations.shape[:-2], 9)) 43 | return rotations 44 | elif newtype in ["rot6d", "6drot", "rotation6d"]: 45 | rotations = geometry.matrix_to_rotation_6d(rotations) 46 | return rotations 47 | elif newtype in ["rotquat", "quaternion"]: 48 | rotations = geometry.matrix_to_quaternion(rotations) 49 | return rotations 50 | elif newtype in ["rotvec", "axisangle"]: 51 | rotations = geometry.matrix_to_axis_angle(rotations) 52 | return rotations 53 | else: 54 | raise NotImplementedError 55 | 56 | 57 | def to_matrix(oldtype, rotations): 58 | if oldtype in ["matrix"]: 59 | return rotations 60 | if oldtype in ["rotmat"]: 61 | rotations = rotations.reshape((*rotations.shape[:-2], 3, 3)) 62 | return rotations 63 | elif oldtype in ["rot6d", "6drot", "rotation6d"]: 64 | rotations = geometry.rotation_6d_to_matrix(rotations) 65 | return rotations 66 | elif oldtype in ["rotquat", "quaternion"]: 67 | rotations = geometry.quaternion_to_matrix(rotations) 68 | return rotations 69 | elif oldtype in ["rotvec", "axisangle"]: 70 | rotations = geometry.axis_angle_to_matrix(rotations) 71 | return rotations 72 | else: 73 | raise NotImplementedError 74 | -------------------------------------------------------------------------------- /GraphMotion/utils/fixseed.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | import random 4 | 5 | 6 | def fixseed(seed): 7 | random.seed(seed) 8 | np.random.seed(seed) 9 | torch.manual_seed(seed) 10 | 11 | 12 | SEED = 10 13 | EVALSEED = 0 14 | # Provoc warning: not fully functionnal yet 15 | # torch.set_deterministic(True) 16 | torch.backends.cudnn.benchmark = False 17 | 18 | fixseed(SEED) 19 | -------------------------------------------------------------------------------- /GraphMotion/utils/logger.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | import os 3 | import time 4 | import logging 5 | from omegaconf import OmegaConf 6 | from pytorch_lightning.utilities.rank_zero import rank_zero_only 7 | 8 | 9 | def create_logger(cfg, phase='train'): 10 | # root dir set by cfg 11 | root_output_dir = Path(cfg.FOLDER) 12 | # set up logger 13 | if not root_output_dir.exists(): 14 | print('=> creating {}'.format(root_output_dir)) 15 | root_output_dir.mkdir() 16 | 17 | cfg_name = cfg.NAME 18 | model = cfg.model.model_type 19 | cfg_name = os.path.basename(cfg_name).split('.')[0] 20 | 21 | final_output_dir = root_output_dir / model / cfg_name 22 | cfg.FOLDER_EXP = str(final_output_dir) 23 | 24 | time_str = time.strftime('%Y-%m-%d-%H-%M-%S') 25 | 26 | new_dir(cfg, phase, time_str, final_output_dir) 27 | 28 | head = '%(asctime)-15s %(message)s' 29 | logger = config_logger(final_output_dir, time_str, phase, head) 30 | if logger is None: 31 | logger = logging.getLogger() 32 | logger.setLevel(logging.CRITICAL) 33 | logging.basicConfig(format=head) 34 | return logger 35 | 36 | 37 | @rank_zero_only 38 | def config_logger(final_output_dir, time_str, phase, head): 39 | log_file = '{}_{}_{}.log'.format('log', time_str, phase) 40 | final_log_file = final_output_dir / log_file 41 | logging.basicConfig(filename=str(final_log_file)) 42 | logger = logging.getLogger() 43 | logger.setLevel(logging.INFO) 44 | console = logging.StreamHandler() 45 | formatter = logging.Formatter(head) 46 | console.setFormatter(formatter) 47 | logging.getLogger('').addHandler(console) 48 | file_handler = logging.FileHandler(final_log_file, 'w') 49 | file_handler.setFormatter(logging.Formatter(head)) 50 | file_handler.setLevel(logging.INFO) 51 | logging.getLogger('').addHandler(file_handler) 52 | return logger 53 | 54 | 55 | @rank_zero_only 56 | def new_dir(cfg, phase, time_str, final_output_dir): 57 | # new experiment folder 58 | cfg.TIME = str(time_str) 59 | if os.path.exists( 60 | final_output_dir) and cfg.TRAIN.RESUME is None and not cfg.DEBUG: 61 | file_list = sorted(os.listdir(final_output_dir), reverse=True) 62 | for item in file_list: 63 | if item.endswith('.log'): 64 | os.rename(str(final_output_dir), 65 | str(final_output_dir) + '_' + cfg.TIME) 66 | break 67 | final_output_dir.mkdir(parents=True, exist_ok=True) 68 | # write config yaml 69 | config_file = '{}_{}_{}.yaml'.format('config', time_str, phase) 70 | final_config_file = final_output_dir / config_file 71 | OmegaConf.save(config=cfg, f=final_config_file) 72 | -------------------------------------------------------------------------------- /GraphMotion/utils/misc.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | 4 | def to_numpy(tensor): 5 | if torch.is_tensor(tensor): 6 | return tensor.cpu().numpy() 7 | elif type(tensor).__module__ != 'numpy': 8 | raise ValueError("Cannot convert {} to numpy array".format( 9 | type(tensor))) 10 | return tensor 11 | 12 | 13 | def to_torch(ndarray): 14 | if type(ndarray).__module__ == 'numpy': 15 | return torch.from_numpy(ndarray) 16 | elif not torch.is_tensor(ndarray): 17 | raise ValueError("Cannot convert {} to torch tensor".format( 18 | type(ndarray))) 19 | return ndarray 20 | 21 | 22 | def cleanexit(): 23 | import sys 24 | import os 25 | try: 26 | sys.exit(0) 27 | except SystemExit: 28 | os._exit(0) 29 | 30 | -------------------------------------------------------------------------------- /GraphMotion/utils/sample_utils.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from pathlib import Path 3 | logger = logging.getLogger(__name__) 4 | 5 | def cfg_mean_nsamples_resolution(cfg): 6 | if cfg.mean and cfg.number_of_samples > 1: 7 | logger.error("All the samples will be the mean.. cfg.number_of_samples=1 will be forced.") 8 | cfg.number_of_samples = 1 9 | 10 | return cfg.number_of_samples == 1 11 | 12 | 13 | def get_path(sample_path: Path, is_amass: bool, gender: str, split: str, onesample: bool, mean: bool, fact: float): 14 | extra_str = ("_mean" if mean else "") if onesample else "_multi" 15 | fact_str = "" if fact == 1 else f"{fact}_" 16 | gender_str = gender + "_" if is_amass else "" 17 | path = sample_path / f"{fact_str}{gender_str}{split}{extra_str}" 18 | return path 19 | -------------------------------------------------------------------------------- /GraphMotion/utils/tensors.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | 4 | def lengths_to_mask(lengths): 5 | max_len = max(lengths) 6 | mask = torch.arange(max_len, device=lengths.device).expand( 7 | len(lengths), max_len) < lengths.unsqueeze(1) 8 | return mask 9 | 10 | 11 | def collate_tensors(batch): 12 | dims = batch[0].dim() 13 | max_size = [max([b.size(i) for b in batch]) for i in range(dims)] 14 | size = (len(batch),) + tuple(max_size) 15 | canvas = batch[0].new_zeros(size=size) 16 | for i, b in enumerate(batch): 17 | sub_tensor = canvas[i] 18 | for d in range(dims): 19 | sub_tensor = sub_tensor.narrow(d, 0, b.size(d)) 20 | sub_tensor.add_(b) 21 | return canvas 22 | 23 | 24 | def collate(batch): 25 | databatch = [b[0] for b in batch] 26 | labelbatch = [b[1] for b in batch] 27 | lenbatch = [len(b[0][0][0]) for b in batch] 28 | 29 | databatchTensor = collate_tensors(databatch) 30 | labelbatchTensor = torch.as_tensor(labelbatch) 31 | lenbatchTensor = torch.as_tensor(lenbatch) 32 | 33 | maskbatchTensor = lengths_to_mask(lenbatchTensor) 34 | # x - [bs, njoints, nfeats, lengths] 35 | # - nfeats, the representation of a joint 36 | # y - [bs] 37 | # mask - [bs, lengths] 38 | # lengths - [bs] 39 | batch = {"x": databatchTensor, "y": labelbatchTensor, 40 | "mask": maskbatchTensor, 'lengths': lenbatchTensor} 41 | return batch 42 | 43 | 44 | # slow version with padding 45 | def collate_data3d_slow(batch): 46 | batchTensor = {} 47 | for key in batch[0].keys(): 48 | databatch = [b[key] for b in batch] 49 | batchTensor[key] = collate_tensors(databatch) 50 | batch = batchTensor 51 | # theta - [bs, lengths, 85], theta shape (85,) 52 | # - (np.array([1., 0., 0.]), pose(72), shape(10)), axis=0) 53 | # kp_2d - [bs, lengths, njoints, nfeats], nfeats (x,y,weight) 54 | # kp_3d - [bs, lengths, njoints, nfeats], nfeats (x,y,z) 55 | # w_smpl - [bs, lengths] zeros 56 | # w_3d - [bs, lengths] zeros 57 | return batch 58 | 59 | def collate_data3d(batch): 60 | batchTensor = {} 61 | for key in batch[0].keys(): 62 | databatch = [b[key] for b in batch] 63 | if key == "paths": 64 | batchTensor[key] = databatch 65 | else: 66 | batchTensor[key] = torch.stack(databatch,axis=0) 67 | batch = batchTensor 68 | # theta - [bs, lengths, 85], theta shape (85,) 69 | # - (np.array([1., 0., 0.]), pose(72), shape(10)), axis=0) 70 | # kp_2d - [bs, lengths, njoints, nfeats], nfeats (x,y,weight) 71 | # kp_3d - [bs, lengths, njoints, nfeats], nfeats (x,y,z) 72 | # w_smpl - [bs, lengths] zeros 73 | # w_3d - [bs, lengths] zeros 74 | return batch 75 | -------------------------------------------------------------------------------- /configs/assets.yaml: -------------------------------------------------------------------------------- 1 | FOLDER: './experiments' # Experiment files saving path 2 | 3 | TEST: 4 | FOLDER: './results' # Testing files saving path 5 | 6 | DATASET: 7 | SMPL_PATH: './deps/smpl' 8 | TRANSFORM_PATH: './deps/transforms/' 9 | WORD_VERTILIZER_PATH: './deps/glove/' 10 | KIT: 11 | ROOT: './datasets/kit-ml' # KIT directory 12 | SPLIT_ROOT: './datasets/kit-ml' # KIT splits directory 13 | HUMANML3D: 14 | ROOT: './datasets/humanml3d' # HumanML3D directory 15 | SPLIT_ROOT: './datasets/humanml3d' # HumanML3D splits directory 16 | HUMANACT12: 17 | ROOT: ./datasets/HumanAct12Poses 18 | SPLIT_ROOT: ./datasets/HumanAct12Poses 19 | UESTC: 20 | ROOT: ./datasets/uestc 21 | SPLIT_ROOT: ./datasets/uestc 22 | AMASS: 23 | DB_ROOT: /apdcephfs/share_1227775/shingxchen/uicap/data/vibe_db 24 | 25 | model: 26 | bert_path: './deps/distilbert-base-uncased' # bert model path for all text encoders 27 | clip_path: './deps/clip-vit-large-patch14' # bert model path for all text encoders 28 | t2m_path: './deps/t2m/' 29 | 30 | humanact12_rec_path: './deps/actionrecognition' 31 | uestc_rec_path: './deps/actionrecognition' 32 | # Set model path separately for different encoders 33 | # TEXT_ENCODER: 34 | # MODELPATH: './deps/distilbert-base-uncased' # bert model path for text encoder 35 | # TEXT_STYLE_ENCODER: 36 | # MODELPATH: './deps/distilbert-base-uncased' # bert model path for text style encoder 37 | # TEXT_CONTENT_ENCODER: 38 | # MODELPATH: './deps/distilbert-base-uncased' # bert model path for text content encoder 39 | 40 | RENDER: 41 | BLENDER_PATH: '/apdcephfs/share_1227775/mingzhenzhu/jiangbiao/libs/blender-2.93.2-linux-x64/blender' 42 | FACES_PATH: '/apdcephfs/share_1227775/shingxchen/AIMotion/TMOSTData/deps/smplh/smplh.faces' 43 | FOLDER: ./animations 44 | -------------------------------------------------------------------------------- /configs/config_vae_humanml3d_action.yaml: -------------------------------------------------------------------------------- 1 | NAME: VAE_Actions # Experiment name 2 | DEBUG: False # Debug mode 3 | ACCELERATOR: 'gpu' # Devices optioncal: “cpu”, “gpu”, “tpu”, “ipu”, “hpu”, “mps, “auto” 4 | DEVICE: [0,1,2,3] # Index of gpus eg. [0] or [0,1,2,3] 5 | 6 | # Training configuration 7 | TRAIN: 8 | #--------------------------------- 9 | STAGE: vae # stage "vae" or "diffusion", "vae_diffusion" 10 | #--------------------------------- 11 | ABLATION: 12 | SKIP_CONNECT: True 13 | PE_TYPE: GraphMotion 14 | DIFF_PE_TYPE: GraphMotion 15 | DATASETS: ['humanml3d'] # Training datasets 16 | NUM_WORKERS: 11 # Number of workers 17 | BATCH_SIZE: 128 # Size of batches 18 | START_EPOCH: 0 # Start epochMMOTIONENCODER 19 | END_EPOCH: 6000 # End epoch 20 | RESUME: '' # Resume training from this path 21 | PRETRAINED_VAE: '' 22 | OPTIM: 23 | TYPE: AdamW # Optimizer type 24 | LR: 1e-4 # Learning rate 25 | 26 | # Evaluating Configuration 27 | EVAL: 28 | DATASETS: ['humanml3d'] # Evaluating datasets 29 | BATCH_SIZE: 32 # Evaluating Batch size 30 | SPLIT: test 31 | 32 | # Test Configuration 33 | TEST: 34 | CHECKPOINTS: '' # Pretrained model path 35 | DATASETS: ['humanml3d'] # training datasets 36 | SPLIT: test 37 | BATCH_SIZE: 32 # training Batch size 38 | MEAN: False 39 | NUM_SAMPLES: 1 40 | FACT: 1 41 | 42 | # Datasets Configuration 43 | DATASET: 44 | JOINT_TYPE: 'humanml3d' # join type 45 | METRIC: 46 | TYPE: ['TemosMetric', 'TM2TMetrics'] 47 | # Losses Configuration 48 | LOSS: 49 | TYPE: GraphMotion # Losses type 50 | LAMBDA_LATENT: 1.0e-5 # Lambda for latent Losses 51 | LAMBDA_KL: 1.0e-4 # Lambda for kl Losses 52 | LAMBDA_REC: 1.0 # Lambda for reconstruction Losses 53 | LAMBDA_GEN: 1.0 # Lambda for text-motion generation losses 54 | LAMBDA_CROSS: 1.0 # Lambda for reconstruction Losses 55 | LAMBDA_CYCLE: 0.0 # Lambda for cycle Losses 56 | LAMBDA_PRIOR: 0.0 57 | DIST_SYNC_ON_STEP: False # Sync Losses on step when distributed trained 58 | 59 | # Model Configuration 60 | model: 61 | vae: true # whether vae model 62 | model_type: GraphMotion # model type 63 | vae_type: GraphMotion 64 | condition: 'text' 65 | latent_dim: [4, 256] # latent dimension 66 | ff_size: 1024 # 67 | num_layers: 9 # number of layers 68 | num_head: 4 # number of head layers 69 | droupout: 0.1 # dropout rate 70 | activation: gelu # activation type 71 | guidance_scale: 7.5 # 72 | guidance_uncondp: 0.1 # 0.1 0.25 73 | 74 | # Logger configuration 75 | LOGGER: 76 | SACE_CHECKPOINT_EPOCH: 200 77 | LOG_EVERY_STEPS: 1 78 | VAL_EVERY_STEPS: 200 79 | TENSORBOARD: True 80 | WANDB: 81 | PROJECT: null 82 | OFFLINE: False 83 | RESUME_ID: null 84 | -------------------------------------------------------------------------------- /configs/config_vae_humanml3d_motion.yaml: -------------------------------------------------------------------------------- 1 | NAME: VAE_Motions # Experiment name 2 | DEBUG: False # Debug mode 3 | ACCELERATOR: 'gpu' # Devices optioncal: “cpu”, “gpu”, “tpu”, “ipu”, “hpu”, “mps, “auto” 4 | DEVICE: [0,1,2,3] # Index of gpus eg. [0] or [0,1,2,3] 5 | 6 | # Training configuration 7 | TRAIN: 8 | #--------------------------------- 9 | STAGE: vae # stage "vae" or "diffusion", "vae_diffusion" 10 | #--------------------------------- 11 | ABLATION: 12 | SKIP_CONNECT: True 13 | PE_TYPE: GraphMotion 14 | DIFF_PE_TYPE: GraphMotion 15 | DATASETS: ['humanml3d'] # Training datasets 16 | NUM_WORKERS: 11 # Number of workers 17 | BATCH_SIZE: 128 # Size of batches 18 | START_EPOCH: 0 # Start epochMMOTIONENCODER 19 | END_EPOCH: 6000 # End epoch 20 | RESUME: '' # Resume training from this path 21 | PRETRAINED_VAE: '' 22 | OPTIM: 23 | TYPE: AdamW # Optimizer type 24 | LR: 1e-4 # Learning rate 25 | 26 | # Evaluating Configuration 27 | EVAL: 28 | DATASETS: ['humanml3d'] # Evaluating datasets 29 | BATCH_SIZE: 32 # Evaluating Batch size 30 | SPLIT: test 31 | 32 | # Test Configuration 33 | TEST: 34 | CHECKPOINTS: '' # Pretrained model path 35 | DATASETS: ['humanml3d'] # training datasets 36 | SPLIT: test 37 | BATCH_SIZE: 32 # training Batch size 38 | MEAN: False 39 | NUM_SAMPLES: 1 40 | FACT: 1 41 | 42 | # Datasets Configuration 43 | DATASET: 44 | JOINT_TYPE: 'humanml3d' # join type 45 | METRIC: 46 | TYPE: ['TemosMetric', 'TM2TMetrics'] 47 | # Losses Configuration 48 | LOSS: 49 | TYPE: GraphMotion # Losses type 50 | LAMBDA_LATENT: 1.0e-5 # Lambda for latent Losses 51 | LAMBDA_KL: 1.0e-4 # Lambda for kl Losses 52 | LAMBDA_REC: 1.0 # Lambda for reconstruction Losses 53 | LAMBDA_GEN: 1.0 # Lambda for text-motion generation losses 54 | LAMBDA_CROSS: 1.0 # Lambda for reconstruction Losses 55 | LAMBDA_CYCLE: 0.0 # Lambda for cycle Losses 56 | LAMBDA_PRIOR: 0.0 57 | DIST_SYNC_ON_STEP: False # Sync Losses on step when distributed trained 58 | 59 | # Model Configuration 60 | model: 61 | vae: true # whether vae model 62 | model_type: GraphMotion # model type 63 | vae_type: GraphMotion 64 | condition: 'text' 65 | latent_dim: [2, 256] # latent dimension 66 | ff_size: 1024 # 67 | num_layers: 9 # number of layers 68 | num_head: 4 # number of head layers 69 | droupout: 0.1 # dropout rate 70 | activation: gelu # activation type 71 | guidance_scale: 7.5 # 72 | guidance_uncondp: 0.1 # 0.1 0.25 73 | 74 | # Logger configuration 75 | LOGGER: 76 | SACE_CHECKPOINT_EPOCH: 200 77 | LOG_EVERY_STEPS: 1 78 | VAL_EVERY_STEPS: 200 79 | TENSORBOARD: True 80 | WANDB: 81 | PROJECT: null 82 | OFFLINE: False 83 | RESUME_ID: null 84 | -------------------------------------------------------------------------------- /configs/config_vae_humanml3d_specific.yaml: -------------------------------------------------------------------------------- 1 | NAME: VAE_Specifics # Experiment name 2 | DEBUG: False # Debug mode 3 | ACCELERATOR: 'gpu' # Devices optioncal: “cpu”, “gpu”, “tpu”, “ipu”, “hpu”, “mps, “auto” 4 | DEVICE: [0,1,2,3] # Index of gpus eg. [0] or [0,1,2,3] 5 | 6 | # Training configuration 7 | TRAIN: 8 | #--------------------------------- 9 | STAGE: vae # stage "vae" or "diffusion", "vae_diffusion" 10 | #--------------------------------- 11 | ABLATION: 12 | SKIP_CONNECT: True 13 | PE_TYPE: GraphMotion 14 | DIFF_PE_TYPE: GraphMotion 15 | DATASETS: ['humanml3d'] # Training datasets 16 | NUM_WORKERS: 11 # Number of workers 17 | BATCH_SIZE: 128 # Size of batches 18 | START_EPOCH: 0 # Start epochMMOTIONENCODER 19 | END_EPOCH: 6000 # End epoch 20 | RESUME: '' # Resume training from this path 21 | PRETRAINED_VAE: '' 22 | OPTIM: 23 | TYPE: AdamW # Optimizer type 24 | LR: 1e-4 # Learning rate 25 | 26 | # Evaluating Configuration 27 | EVAL: 28 | DATASETS: ['humanml3d'] # Evaluating datasets 29 | BATCH_SIZE: 32 # Evaluating Batch size 30 | SPLIT: test 31 | 32 | # Test Configuration 33 | TEST: 34 | CHECKPOINTS: '' # Pretrained model path 35 | DATASETS: ['humanml3d'] # training datasets 36 | SPLIT: test 37 | BATCH_SIZE: 32 # training Batch size 38 | MEAN: False 39 | NUM_SAMPLES: 1 40 | FACT: 1 41 | 42 | # Datasets Configuration 43 | DATASET: 44 | JOINT_TYPE: 'humanml3d' # join type 45 | METRIC: 46 | TYPE: ['TemosMetric', 'TM2TMetrics'] 47 | # Losses Configuration 48 | LOSS: 49 | TYPE: GraphMotion # Losses type 50 | LAMBDA_LATENT: 1.0e-5 # Lambda for latent Losses 51 | LAMBDA_KL: 1.0e-4 # Lambda for kl Losses 52 | LAMBDA_REC: 1.0 # Lambda for reconstruction Losses 53 | LAMBDA_GEN: 1.0 # Lambda for text-motion generation losses 54 | LAMBDA_CROSS: 1.0 # Lambda for reconstruction Losses 55 | LAMBDA_CYCLE: 0.0 # Lambda for cycle Losses 56 | LAMBDA_PRIOR: 0.0 57 | DIST_SYNC_ON_STEP: False # Sync Losses on step when distributed trained 58 | 59 | # Model Configuration 60 | model: 61 | vae: true # whether vae model 62 | model_type: GraphMotion # model type 63 | vae_type: GraphMotion 64 | condition: 'text' 65 | latent_dim: [8, 256] # latent dimension 66 | ff_size: 1024 # 67 | num_layers: 9 # number of layers 68 | num_head: 4 # number of head layers 69 | droupout: 0.1 # dropout rate 70 | activation: gelu # activation type 71 | guidance_scale: 7.5 # 72 | guidance_uncondp: 0.1 # 0.1 0.25 73 | 74 | # Logger configuration 75 | LOGGER: 76 | SACE_CHECKPOINT_EPOCH: 200 77 | LOG_EVERY_STEPS: 1 78 | VAL_EVERY_STEPS: 200 79 | TENSORBOARD: True 80 | WANDB: 81 | PROJECT: null 82 | OFFLINE: False 83 | RESUME_ID: null 84 | -------------------------------------------------------------------------------- /configs/modules/denoiser.yaml: -------------------------------------------------------------------------------- 1 | denoiser: 2 | target: GraphMotion.models.architectures.denoiser.Denoiser 3 | params: 4 | text_encoded_dim: 768 5 | ff_size: 4096 6 | num_layers: 17 7 | num_heads: 16 8 | dropout: 0.1 9 | normalize_before: False 10 | activation: 'gelu' 11 | flip_sin_to_cos: True 12 | return_intermediate_dec: False 13 | position_embedding: 'learned' 14 | arch: trans_enc 15 | freq_shift: 0 16 | condition: ${model.condition} 17 | latent_dim: [ 1, 1024 ] 18 | guidance_scale: ${model.guidance_scale} 19 | guidance_uncondp: ${model.guidance_uncondp} 20 | nfeats: ${DATASET.NFEATS} 21 | nclasses: ${DATASET.NCLASSES} 22 | ablation: ${TRAIN.ABLATION} 23 | -------------------------------------------------------------------------------- /configs/modules/evaluators.yaml: -------------------------------------------------------------------------------- 1 | t2m_textencoder: 2 | target: GraphMotion.models.architectures.t2m_textenc.TextEncoderBiGRUCo 3 | params: 4 | word_size: 300 5 | pos_size: 15 6 | hidden_size: 512 7 | output_size: 512 8 | 9 | t2m_moveencoder: 10 | target: GraphMotion.models.architectures.t2m_textenc.MovementConvEncoder 11 | params: 12 | hidden_size: 512 13 | output_size: 512 14 | 15 | t2m_motionencoder: 16 | target: GraphMotion.models.architectures.t2m_motionenc.MotionEncoder 17 | params: 18 | input_size: ${model.t2m_moveencoder.output_size} 19 | hidden_size: 1024 20 | output_size: 512 21 | -------------------------------------------------------------------------------- /configs/modules/motion_vae.yaml: -------------------------------------------------------------------------------- 1 | motion_vae: 2 | # Optional: mld_vae, vposert_vae 3 | target: GraphMotion.models.architectures.vae.Vae 4 | params: 5 | arch: 'encoder_decoder' 6 | ff_size: 1024 7 | num_layers: 9 8 | num_heads: 4 9 | dropout: 0.1 10 | normalize_before: false 11 | activation: 'gelu' 12 | position_embedding: 'learned' 13 | latent_dim: ${model.latent_dim} 14 | nfeats: ${DATASET.NFEATS} 15 | ablation: ${TRAIN.ABLATION} 16 | -------------------------------------------------------------------------------- /configs/modules/scheduler.yaml: -------------------------------------------------------------------------------- 1 | scheduler: 2 | target: diffusers.DDIMScheduler 3 | num_inference_timesteps: 50 4 | eta: 0.0 5 | params: 6 | num_train_timesteps: 1000 7 | beta_start: 0.00085 8 | beta_end: 0.012 9 | beta_schedule: 'scaled_linear' # Optional: ['linear', 'scaled_linear', 'squaredcos_cap_v2'] 10 | # variance_type: 'fixed_small' 11 | clip_sample: false # clip sample to -1~1 12 | # below are for ddim 13 | set_alpha_to_one: false 14 | steps_offset: 1 15 | 16 | # scheduler: 17 | # target: diffusers.DDPMScheduler 18 | # num_inference_timesteps: 1000 19 | # eta: 0.0 20 | # params: 21 | # num_train_timesteps: 1000 22 | # beta_start: 0.00085 23 | # beta_end: 0.012 24 | # beta_schedule: 'scaled_linear' # Optional: ['linear', 'scaled_linear', 'squaredcos_cap_v2'] 25 | # variance_type: 'fixed_small' 26 | # clip_sample: true # clip sample to -1~1 27 | # # below are for ddim 28 | # # set_alpha_to_one: false 29 | # # steps_offset: 1 30 | 31 | noise_scheduler: 32 | target: diffusers.DDPMScheduler 33 | params: 34 | num_train_timesteps: 1000 35 | beta_start: 0.00085 36 | beta_end: 0.012 37 | beta_schedule: 'scaled_linear' # Optional: ['linear', 'scaled_linear', 'squaredcos_cap_v2'] 38 | variance_type: 'fixed_small' 39 | clip_sample: false # clip sample to -1~1 40 | # below are for ddim 41 | # set_alpha_to_one: false 42 | # steps_offset: 1 43 | -------------------------------------------------------------------------------- /configs/modules/text_encoder.yaml: -------------------------------------------------------------------------------- 1 | text_encoder: 2 | # Optional: mld_clip, mld_bert 3 | target: GraphMotion.models.architectures.clip.TextEncoder 4 | params: 5 | finetune: false # if false, model weights are frozen 6 | last_hidden_state: false # if true, the last hidden state is used as the text embedding 7 | latent_dim: ${model.latent_dim} 8 | modelpath: ${model.clip_path} 9 | -------------------------------------------------------------------------------- /configs/modules_humanact12/denoiser.yaml: -------------------------------------------------------------------------------- 1 | denoiser: 2 | target: GraphMotion.models.architectures.denoiser.Denoiser 3 | params: 4 | text_encoded_dim: 768 5 | ff_size: 1024 6 | num_layers: 15 7 | num_heads: 4 8 | dropout: 0.1 9 | normalize_before: False 10 | activation: 'gelu' 11 | flip_sin_to_cos: True 12 | return_intermediate_dec: False 13 | position_embedding: 'learned' 14 | arch: trans_enc 15 | freq_shift: 0 16 | condition: ${model.condition} 17 | latent_dim: ${model.latent_dim} 18 | guidance_scale: ${model.guidance_scale} 19 | guidance_uncondp: ${model.guidance_uncondp} 20 | nfeats: ${DATASET.NFEATS} 21 | nclasses: ${DATASET.NCLASSES} 22 | ablation: ${TRAIN.ABLATION} 23 | -------------------------------------------------------------------------------- /configs/modules_humanact12/evaluators.yaml: -------------------------------------------------------------------------------- 1 | t2m_textencoder: 2 | target: GraphMotion.models.architectures.t2m_textenc.TextEncoderBiGRUCo 3 | params: 4 | word_size: 300 5 | pos_size: 15 6 | hidden_size: 512 7 | output_size: 512 8 | 9 | t2m_moveencoder: 10 | target: GraphMotion.models.architectures.t2m_textenc.MovementConvEncoder 11 | params: 12 | hidden_size: 512 13 | output_size: 512 14 | 15 | t2m_motionencoder: 16 | target: GraphMotion.models.architectures.t2m_motionenc.MotionEncoder 17 | params: 18 | input_size: ${model.t2m_moveencoder.output_size} 19 | hidden_size: 1024 20 | output_size: 512 21 | -------------------------------------------------------------------------------- /configs/modules_humanact12/motion_vae.yaml: -------------------------------------------------------------------------------- 1 | motion_vae: 2 | # Optional: mld_vae, vposert_vae 3 | target: GraphMotion.models.architectures.actor_vae.ActorVae 4 | params: 5 | ff_size: 1024 6 | num_layers: 6 7 | num_heads: 4 8 | dropout: 0.1 9 | normalize_before: false 10 | activation: 'gelu' 11 | position_embedding: 'learned' 12 | latent_dim: ${model.latent_dim} 13 | nfeats: ${DATASET.NFEATS} 14 | ablation: ${TRAIN.ABLATION} 15 | -------------------------------------------------------------------------------- /configs/modules_humanact12/scheduler.yaml: -------------------------------------------------------------------------------- 1 | scheduler: 2 | target: diffusers.DDIMScheduler 3 | num_inference_timesteps: 50 4 | eta: 0.0 5 | params: 6 | num_train_timesteps: 1000 7 | beta_start: 0.00085 8 | beta_end: 0.012 9 | beta_schedule: 'scaled_linear' # Optional: ['linear', 'scaled_linear', 'squaredcos_cap_v2'] 10 | # variance_type: 'fixed_small' 11 | clip_sample: false # clip sample to -1~1 12 | # below are for ddim 13 | set_alpha_to_one: false 14 | steps_offset: 1 15 | 16 | # scheduler: 17 | # target: diffusers.DDPMScheduler 18 | # num_inference_timesteps: 1000 19 | # eta: 0.0 20 | # params: 21 | # num_train_timesteps: 1000 22 | # beta_start: 0.00085 23 | # beta_end: 0.012 24 | # beta_schedule: 'scaled_linear' # Optional: ['linear', 'scaled_linear', 'squaredcos_cap_v2'] 25 | # variance_type: 'fixed_small' 26 | # clip_sample: true # clip sample to -1~1 27 | # # below are for ddim 28 | # # set_alpha_to_one: false 29 | # # steps_offset: 1 30 | 31 | noise_scheduler: 32 | target: diffusers.DDPMScheduler 33 | params: 34 | num_train_timesteps: 1000 35 | beta_start: 0.00085 36 | beta_end: 0.012 37 | beta_schedule: 'scaled_linear' # Optional: ['linear', 'scaled_linear', 'squaredcos_cap_v2'] 38 | variance_type: 'fixed_small' 39 | clip_sample: false # clip sample to -1~1 40 | # below are for ddim 41 | # set_alpha_to_one: false 42 | # steps_offset: 1 43 | -------------------------------------------------------------------------------- /configs/modules_humanact12/text_encoder.yaml: -------------------------------------------------------------------------------- 1 | text_encoder: 2 | # Optional: mld_clip, mld_bert 3 | target: GraphMotion.models.architectures.clip.TextEncoder 4 | params: 5 | finetune: false # if false, model weights are frozen 6 | last_hidden_state: false # if true, the last hidden state is used as the text embedding 7 | latent_dim: ${model.latent_dim} 8 | modelpath: ${model.clip_path} 9 | -------------------------------------------------------------------------------- /configs/modules_novae/denoiser.yaml: -------------------------------------------------------------------------------- 1 | denoiser: 2 | target: GraphMotion.models.architectures.denoiser.Denoiser 3 | params: 4 | text_encoded_dim: 768 5 | ff_size: 1024 6 | num_layers: 9 7 | num_heads: 4 8 | dropout: 0.1 9 | normalize_before: False 10 | activation: 'gelu' 11 | flip_sin_to_cos: True 12 | return_intermediate_dec: False 13 | position_embedding: 'learned' 14 | arch: trans_dec 15 | freq_shift: 0 16 | latent_dim: ${model.latent_dim} 17 | guidance_scale: ${model.guidance_scale} 18 | guidance_uncondp: ${model.guidance_uncondp} 19 | nfeats: ${DATASET.NFEATS} 20 | nclasses: ${DATASET.NCLASSES} 21 | ablation: ${TRAIN.ABLATION} 22 | -------------------------------------------------------------------------------- /configs/modules_novae/evaluators.yaml: -------------------------------------------------------------------------------- 1 | t2m_textencoder: 2 | target: GraphMotion.models.architectures.t2m_textenc.TextEncoderBiGRUCo 3 | params: 4 | word_size: 300 5 | pos_size: 15 6 | hidden_size: 512 7 | output_size: 512 8 | 9 | t2m_moveencoder: 10 | target: GraphMotion.models.architectures.t2m_textenc.MovementConvEncoder 11 | params: 12 | hidden_size: 512 13 | output_size: 512 14 | 15 | t2m_motionencoder: 16 | target: GraphMotion.models.architectures.t2m_motionenc.MotionEncoder 17 | params: 18 | input_size: ${model.t2m_moveencoder.output_size} 19 | hidden_size: 1024 20 | output_size: 512 21 | -------------------------------------------------------------------------------- /configs/modules_novae/motion_vae.yaml: -------------------------------------------------------------------------------- 1 | motion_vae: 2 | # Optional: mld_vae, vposert_vae 3 | target: GraphMotion.models.architectures.vae.Vae 4 | params: 5 | arch: 'encoder_decoder' 6 | ff_size: 1024 7 | num_layers: 9 8 | num_heads: 4 9 | dropout: 0.1 10 | normalize_before: false 11 | activation: 'gelu' 12 | position_embedding: 'learned' 13 | latent_dim: ${model.latent_dim} 14 | nfeats: ${DATASET.NFEATS} 15 | ablation: ${TRAIN.ABLATION} 16 | -------------------------------------------------------------------------------- /configs/modules_novae/scheduler.yaml: -------------------------------------------------------------------------------- 1 | scheduler: 2 | target: diffusers.DDPMScheduler 3 | num_inference_timesteps: 1000 4 | eta: 0.0 5 | params: 6 | num_train_timesteps: 1000 7 | beta_start: 0.00085 8 | beta_end: 0.012 9 | beta_schedule: 'scaled_linear' # Optional: ['linear', 'scaled_linear', 'squaredcos_cap_v2'] 10 | variance_type: 'fixed_small' 11 | clip_sample: false # clip sample to -1~1 12 | # below are for ddim 13 | # set_alpha_to_one: false 14 | # steps_offset: 1 15 | 16 | noise_scheduler: 17 | target: diffusers.DDPMScheduler 18 | params: 19 | num_train_timesteps: 1000 20 | beta_start: 0.00085 21 | beta_end: 0.012 22 | beta_schedule: 'scaled_linear' # Optional: ['linear', 'scaled_linear', 'squaredcos_cap_v2'] 23 | variance_type: 'fixed_small' 24 | clip_sample: false # clip sample to -1~1 25 | # below are for ddim 26 | # set_alpha_to_one: false 27 | # steps_offset: 1 28 | -------------------------------------------------------------------------------- /configs/modules_novae/text_encoder.yaml: -------------------------------------------------------------------------------- 1 | text_encoder: 2 | # Optional: mld_clip, mld_bert 3 | target: GraphMotion.models.architectures.clip.TextEncoder 4 | params: 5 | finetune: false # if false, model weights are frozen 6 | last_hidden_state: false # if true, the last hidden state is used as the text embedding 7 | latent_dim: ${model.latent_dim} 8 | modelpath: ${model.clip_path} 9 | -------------------------------------------------------------------------------- /mld/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpthu17/GraphMotion/d56feba6f6d39867b933ca752244fcbbb703a4b8/mld/__init__.py -------------------------------------------------------------------------------- /mld/callback/__init__.py: -------------------------------------------------------------------------------- 1 | from .progress import ProgressLogger 2 | -------------------------------------------------------------------------------- /mld/callback/progress.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from pytorch_lightning import LightningModule, Trainer 4 | from pytorch_lightning.callbacks import Callback 5 | import psutil 6 | 7 | logger = logging.getLogger() 8 | 9 | 10 | class ProgressLogger(Callback): 11 | 12 | def __init__(self, metric_monitor: dict, precision: int = 3): 13 | # Metric to monitor 14 | self.metric_monitor = metric_monitor 15 | self.precision = precision 16 | 17 | def on_train_start(self, trainer: Trainer, pl_module: LightningModule, 18 | **kwargs) -> None: 19 | logger.info("Training started") 20 | 21 | def on_train_end(self, trainer: Trainer, pl_module: LightningModule, 22 | **kwargs) -> None: 23 | logger.info("Training done") 24 | 25 | def on_validation_epoch_end(self, trainer: Trainer, 26 | pl_module: LightningModule, **kwargs) -> None: 27 | if trainer.sanity_checking: 28 | logger.info("Sanity checking ok.") 29 | 30 | def on_train_epoch_end(self, 31 | trainer: Trainer, 32 | pl_module: LightningModule, 33 | padding=False, 34 | **kwargs) -> None: 35 | metric_format = f"{{:.{self.precision}e}}" 36 | line = f"Epoch {trainer.current_epoch}" 37 | if padding: 38 | line = f"{line:>{len('Epoch xxxx')}}" # Right padding 39 | metrics_str = [] 40 | 41 | losses_dict = trainer.callback_metrics 42 | for metric_name, dico_name in self.metric_monitor.items(): 43 | if dico_name in losses_dict: 44 | metric = losses_dict[dico_name].item() 45 | metric = metric_format.format(metric) 46 | metric = f"{metric_name} {metric}" 47 | metrics_str.append(metric) 48 | 49 | if len(metrics_str) == 0: 50 | return 51 | 52 | memory = f"Memory {psutil.virtual_memory().percent}%" 53 | line = line + ": " + " ".join(metrics_str) + " " + memory 54 | logger.info(line) 55 | -------------------------------------------------------------------------------- /mld/data/Humanact12.py: -------------------------------------------------------------------------------- 1 | from .base import BASEDataModule 2 | from .a2m import HumanAct12Poses 3 | import numpy as np 4 | 5 | 6 | class Humanact12DataModule(BASEDataModule): 7 | 8 | def __init__(self, 9 | cfg, 10 | batch_size, 11 | num_workers, 12 | collate_fn=None, 13 | phase="train", 14 | **kwargs): 15 | super().__init__(batch_size=batch_size, 16 | num_workers=num_workers, 17 | collate_fn=collate_fn) 18 | self.save_hyperparameters(logger=False) 19 | self.name = "HumanAct12" 20 | self.Dataset = HumanAct12Poses 21 | self.cfg = cfg 22 | sample_overrides = { 23 | "num_seq_max": 2, 24 | "split": "test", 25 | "tiny": True, 26 | "progress_bar": False 27 | } 28 | # self._sample_set = self.get_sample_set(overrides=sample_overrides) 29 | # Get additional info of the dataset 30 | self.nfeats = 150 31 | self.njoints = 25 32 | self.nclasses = 12 33 | # self.transforms = self._sample_set.transforms 34 | 35 | # def mm_mode(self, mm_on=True): 36 | # # random select samples for mm 37 | # if mm_on: 38 | # self.is_mm = True 39 | # if self.split == 'train': 40 | # self.name_list = self.test_dataset._train[index] 41 | # else: 42 | # self.name_list = self.test_dataset._test[index] 43 | # self.name_list = self.test_dataset.name_list 44 | # self.mm_list = np.random.choice(self.name_list, 45 | # self.cfg.TEST.MM_NUM_SAMPLES, 46 | # replace=False) 47 | # self.test_dataset.name_list = self.mm_list 48 | # else: 49 | # self.is_mm = False 50 | # self.test_dataset.name_list = self.name_list 51 | -------------------------------------------------------------------------------- /mld/data/Kit.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | 4 | from mld.data.humanml.scripts.motion_process import recover_from_ric 5 | 6 | from .base import BASEDataModule 7 | from .humanml.data.dataset import Text2MotionDatasetV2, TextOnlyDataset 8 | from .utils import all_collate 9 | 10 | 11 | class KitDataModule(BASEDataModule): 12 | 13 | def __init__(self, 14 | cfg, 15 | phase='train', 16 | collate_fn=all_collate, 17 | batch_size: int = 32, 18 | num_workers: int = 16, 19 | **kwargs): 20 | super().__init__(batch_size=batch_size, 21 | num_workers=num_workers, 22 | collate_fn=collate_fn) 23 | self.save_hyperparameters(logger=False) 24 | self.name = 'kit' 25 | self.njoints = 21 26 | if phase == 'text_only': 27 | self.Dataset = TextOnlyDataset 28 | else: 29 | self.Dataset = Text2MotionDatasetV2 30 | self.cfg = cfg 31 | 32 | sample_overrides = { 33 | "split": "val", 34 | "tiny": True, 35 | "progress_bar": False 36 | } 37 | self._sample_set = self.get_sample_set(overrides=sample_overrides) 38 | 39 | # Get additional info of the dataset 40 | self.nfeats = self._sample_set.nfeats 41 | # self.transforms = self._sample_set.transforms 42 | 43 | def feats2joints(self, features): 44 | mean = torch.tensor(self.hparams.mean).to(features) 45 | std = torch.tensor(self.hparams.std).to(features) 46 | features = features * std + mean 47 | return recover_from_ric(features, self.njoints) 48 | 49 | def renorm4t2m(self, features): 50 | # renorm to t2m norms for using t2m evaluators 51 | ori_mean = torch.tensor(self.hparams.mean).to(features) 52 | ori_std = torch.tensor(self.hparams.std).to(features) 53 | eval_mean = torch.tensor(self.hparams.mean_eval).to(features) 54 | eval_std = torch.tensor(self.hparams.std_eval).to(features) 55 | features = features * ori_std + ori_mean 56 | features = (features - eval_mean) / eval_std 57 | return features 58 | 59 | def mm_mode(self, mm_on=True): 60 | # random select samples for mm 61 | if mm_on: 62 | self.is_mm = True 63 | self.name_list = self.test_dataset.name_list 64 | self.mm_list = np.random.choice(self.name_list, 65 | self.cfg.TEST.MM_NUM_SAMPLES, 66 | replace=False) 67 | self.test_dataset.name_list = self.mm_list 68 | else: 69 | self.is_mm = False 70 | self.test_dataset.name_list = self.name_list 71 | -------------------------------------------------------------------------------- /mld/data/Uestc.py: -------------------------------------------------------------------------------- 1 | from .base import BASEDataModule 2 | from .a2m import UESTC 3 | import os 4 | import rich.progress 5 | import pickle as pkl 6 | 7 | 8 | class UestcDataModule(BASEDataModule): 9 | 10 | def __init__(self, 11 | cfg, 12 | batch_size, 13 | num_workers, 14 | collate_fn=None, 15 | method_name="vibe", 16 | phase="train", 17 | **kwargs): 18 | super().__init__(batch_size=batch_size, 19 | num_workers=num_workers, 20 | collate_fn=collate_fn) 21 | self.save_hyperparameters(logger=False) 22 | self.name = "Uestc" 23 | 24 | # if method_name == "vibe": 25 | # vibe_data_path = os.path.join(self.hparams.datapath, 26 | # "vibe_cache_refined.pkl") 27 | # with rich.progress.open( 28 | # vibe_data_path, "rb", 29 | # description="loading uestc vibe data") as f: 30 | # vibe_data = pkl.load(f) 31 | # self.hparams.update({"vibe_data": vibe_data}) 32 | self.Dataset = UESTC 33 | self.cfg = cfg 34 | 35 | # self._sample_set = self.get_sample_set(overrides=sample_overrides) 36 | # Get additional info of the dataset 37 | self.nfeats = 150 38 | self.njoints = 25 39 | self.nclasses = 40 40 | # self.transforms = self._sample_set.transforms 41 | -------------------------------------------------------------------------------- /mld/data/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpthu17/GraphMotion/d56feba6f6d39867b933ca752244fcbbb703a4b8/mld/data/__init__.py -------------------------------------------------------------------------------- /mld/data/a2m/__init__.py: -------------------------------------------------------------------------------- 1 | from .humanact12poses import HumanAct12Poses 2 | from .uestc import UESTC 3 | -------------------------------------------------------------------------------- /mld/data/a2m/humanact12poses.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pickle as pkl 3 | 4 | import numpy as np 5 | 6 | from .dataset import Dataset 7 | from .utils import rotation_conversions as geometry 8 | import rich.progress 9 | 10 | 11 | class HumanAct12Poses(Dataset): 12 | dataname = "humanact12" 13 | 14 | def __init__(self, datapath="data/HumanAct12Poses", **kargs): 15 | self.datapath = datapath 16 | 17 | super().__init__(**kargs) 18 | 19 | pkldatafilepath = os.path.join(datapath, "humanact12poses.pkl") 20 | with rich.progress.open(pkldatafilepath, "rb", description="loading humanact12 pkl") as f: 21 | data = pkl.load(f) 22 | 23 | self._pose = [x for x in data["poses"]] 24 | self._num_frames_in_video = [p.shape[0] for p in self._pose] 25 | self._joints = [x for x in data["joints3D"]] 26 | 27 | self._actions = [x for x in data["y"]] 28 | 29 | total_num_actions = 12 30 | self.num_classes = total_num_actions 31 | 32 | self._train = list(range(len(self._pose))) 33 | 34 | keep_actions = np.arange(0, total_num_actions) 35 | 36 | self._action_to_label = {x: i for i, x in enumerate(keep_actions)} 37 | self._label_to_action = {i: x for i, x in enumerate(keep_actions)} 38 | 39 | self._action_classes = humanact12_coarse_action_enumerator 40 | 41 | def _load_joints3D(self, ind, frame_ix): 42 | return self._joints[ind][frame_ix] 43 | 44 | def _load_rotvec(self, ind, frame_ix): 45 | pose = self._pose[ind][frame_ix].reshape(-1, 24, 3) 46 | return pose 47 | 48 | 49 | humanact12_coarse_action_enumerator = { 50 | 0: "warm_up", 51 | 1: "walk", 52 | 2: "run", 53 | 3: "jump", 54 | 4: "drink", 55 | 5: "lift_dumbbell", 56 | 6: "sit", 57 | 7: "eat", 58 | 8: "turn steering wheel", 59 | 9: "phone", 60 | 10: "boxing", 61 | 11: "throw", 62 | } 63 | -------------------------------------------------------------------------------- /mld/data/a2m/tools.py: -------------------------------------------------------------------------------- 1 | import os 2 | import string 3 | 4 | 5 | def parse_info_name(path): 6 | name = os.path.splitext(os.path.split(path)[-1])[0] 7 | info = {} 8 | current_letter = None 9 | for letter in name: 10 | if letter in string.ascii_letters: 11 | info[letter] = [] 12 | current_letter = letter 13 | else: 14 | info[current_letter].append(letter) 15 | for key in info.keys(): 16 | info[key] = "".join(info[key]) 17 | return info 18 | 19 | 20 | -------------------------------------------------------------------------------- /mld/data/a2m/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpthu17/GraphMotion/d56feba6f6d39867b933ca752244fcbbb703a4b8/mld/data/a2m/utils/__init__.py -------------------------------------------------------------------------------- /mld/data/a2m/utils/misc.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | 4 | def to_numpy(tensor): 5 | if torch.is_tensor(tensor): 6 | return tensor.cpu().numpy() 7 | elif type(tensor).__module__ != 'numpy': 8 | raise ValueError("Cannot convert {} to numpy array".format( 9 | type(tensor))) 10 | return tensor 11 | 12 | 13 | def to_torch(ndarray): 14 | if type(ndarray).__module__ == 'numpy': 15 | return torch.from_numpy(ndarray) 16 | elif not torch.is_tensor(ndarray): 17 | raise ValueError("Cannot convert {} to torch tensor".format( 18 | type(ndarray))) 19 | return ndarray 20 | 21 | 22 | def cleanexit(): 23 | import sys 24 | import os 25 | try: 26 | sys.exit(0) 27 | except SystemExit: 28 | os._exit(0) 29 | 30 | -------------------------------------------------------------------------------- /mld/data/a2m/utils/tensors.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | 4 | def lengths_to_mask(lengths): 5 | max_len = max(lengths) 6 | mask = torch.arange(max_len, device=lengths.device).expand(len(lengths), max_len) < lengths.unsqueeze(1) 7 | return mask 8 | 9 | 10 | def collate_tensors(batch): 11 | dims = batch[0].dim() 12 | max_size = [max([b.size(i) for b in batch]) for i in range(dims)] 13 | size = (len(batch),) + tuple(max_size) 14 | canvas = batch[0].new_zeros(size=size) 15 | for i, b in enumerate(batch): 16 | sub_tensor = canvas[i] 17 | for d in range(dims): 18 | sub_tensor = sub_tensor.narrow(d, 0, b.size(d)) 19 | sub_tensor.add_(b) 20 | return canvas 21 | 22 | 23 | def collate(batch): 24 | databatch = [b[0] for b in batch] 25 | labelbatch = [b[1] for b in batch] 26 | lenbatch = [len(b[0][0][0]) for b in batch] 27 | 28 | databatchTensor = collate_tensors(databatch) 29 | labelbatchTensor = torch.as_tensor(labelbatch) 30 | lenbatchTensor = torch.as_tensor(lenbatch) 31 | 32 | maskbatchTensor = lengths_to_mask(lenbatchTensor) 33 | batch = {"x": databatchTensor, "y": labelbatchTensor, 34 | "mask": maskbatchTensor, "lengths": lenbatchTensor} 35 | return batch 36 | -------------------------------------------------------------------------------- /mld/data/humanml/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpthu17/GraphMotion/d56feba6f6d39867b933ca752244fcbbb703a4b8/mld/data/humanml/__init__.py -------------------------------------------------------------------------------- /mld/data/humanml/data/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpthu17/GraphMotion/d56feba6f6d39867b933ca752244fcbbb703a4b8/mld/data/humanml/data/__init__.py -------------------------------------------------------------------------------- /mld/data/humanml/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpthu17/GraphMotion/d56feba6f6d39867b933ca752244fcbbb703a4b8/mld/data/humanml/utils/__init__.py -------------------------------------------------------------------------------- /mld/data/humanml/utils/get_opt.py: -------------------------------------------------------------------------------- 1 | import os 2 | from argparse import Namespace 3 | import re 4 | from os.path import join as pjoin 5 | from .word_vectorizer import POS_enumerator 6 | 7 | 8 | def is_float(numStr): 9 | flag = False 10 | numStr = str(numStr).strip().lstrip('-').lstrip('+') # 去除正数(+)、负数(-)符号 11 | try: 12 | reg = re.compile(r'^[-+]?[0-9]+\.[0-9]+$') 13 | res = reg.match(str(numStr)) 14 | if res: 15 | flag = True 16 | except Exception as ex: 17 | print("is_float() - error: " + str(ex)) 18 | return flag 19 | 20 | 21 | def is_number(numStr): 22 | flag = False 23 | numStr = str(numStr).strip().lstrip('-').lstrip('+') # 去除正数(+)、负数(-)符号 24 | if str(numStr).isdigit(): 25 | flag = True 26 | return flag 27 | 28 | 29 | def get_opt(opt_path, device): 30 | opt = Namespace() 31 | opt_dict = vars(opt) 32 | 33 | skip = ('-------------- End ----------------', 34 | '------------ Options -------------', 35 | '\n') 36 | print('Reading', opt_path) 37 | with open(opt_path) as f: 38 | for line in f: 39 | if line.strip() not in skip: 40 | # print(line.strip()) 41 | key, value = line.strip().split(': ') 42 | if value in ('True', 'False'): 43 | opt_dict[key] = bool(value) 44 | elif is_float(value): 45 | opt_dict[key] = float(value) 46 | elif is_number(value): 47 | opt_dict[key] = int(value) 48 | else: 49 | opt_dict[key] = str(value) 50 | 51 | # print(opt) 52 | opt_dict['which_epoch'] = 'latest' 53 | opt.save_root = pjoin(opt.checkpoints_dir, opt.dataset_name, opt.name) 54 | opt.model_dir = pjoin(opt.save_root, 'model') 55 | opt.meta_dir = pjoin(opt.save_root, 'meta') 56 | 57 | if opt.dataset_name == 't2m': 58 | opt.data_root = './dataset/HumanML3D' 59 | opt.motion_dir = pjoin(opt.data_root, 'new_joint_vecs') 60 | opt.text_dir = pjoin(opt.data_root, 'texts') 61 | opt.joints_num = 22 62 | opt.dim_pose = 263 63 | opt.max_motion_length = 196 64 | elif opt.dataset_name == 'kit': 65 | opt.data_root = './dataset/KIT-ML' 66 | opt.motion_dir = pjoin(opt.data_root, 'new_joint_vecs') 67 | opt.text_dir = pjoin(opt.data_root, 'texts') 68 | opt.joints_num = 21 69 | opt.dim_pose = 251 70 | opt.max_motion_length = 196 71 | else: 72 | raise KeyError('Dataset not recognized') 73 | 74 | opt.dim_word = 300 75 | opt.num_classes = 200 // opt.unit_length 76 | opt.dim_pos_ohot = len(POS_enumerator) 77 | opt.is_train = False 78 | opt.is_continue = False 79 | opt.device = device 80 | 81 | return opt 82 | -------------------------------------------------------------------------------- /mld/data/humanml/utils/load_json.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | 4 | 5 | def save_json(save_path, data): 6 | with open(save_path, "w") as file: 7 | json.dump(data, file) 8 | 9 | 10 | def load_json(file_path): 11 | with open(file_path, "r") as file: 12 | data = json.load(file) 13 | return data 14 | 15 | 16 | def process(graph): 17 | entities, relations = {}, [] 18 | for i in graph["verbs"]: 19 | description = i['description'] 20 | pos = 0 21 | flag = 0 22 | _words, _spans = [], [] 23 | for i in description.split(): 24 | tags, verb = {}, 0 25 | if "[" in i: 26 | _role = i[1:-1] 27 | flag = 1 28 | _spans = [pos] 29 | _words = [] 30 | 31 | elif "]" in i: 32 | _words.append(i[:-1]) 33 | entities[len(entities)] = { 34 | "role": _role, 35 | "spans": _spans, 36 | "words": _words 37 | } 38 | pos += 1 39 | flag = 0 40 | if _role != "V": 41 | tags[len(entities)] = _role 42 | else: 43 | verb = len(entities) 44 | else: 45 | pos += 1 46 | if flag: 47 | _words.append(i) 48 | _spans.append(pos) 49 | 50 | for i in tags: 51 | relations.append((verb, i, tags[i])) 52 | 53 | output = { 54 | "entities": entities, 55 | "relations": relations 56 | } 57 | return output -------------------------------------------------------------------------------- /mld/data/humanml/utils/paramUtil.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | # Define a kinematic tree for the skeletal struture 4 | kit_kinematic_chain = [[0, 11, 12, 13, 14, 15], [0, 16, 17, 18, 19, 20], [0, 1, 2, 3, 4], [3, 5, 6, 7], [3, 8, 9, 10]] 5 | 6 | kit_raw_offsets = np.array( 7 | [ 8 | [0, 0, 0], 9 | [0, 1, 0], 10 | [0, 1, 0], 11 | [0, 1, 0], 12 | [0, 1, 0], 13 | [1, 0, 0], 14 | [0, -1, 0], 15 | [0, -1, 0], 16 | [-1, 0, 0], 17 | [0, -1, 0], 18 | [0, -1, 0], 19 | [1, 0, 0], 20 | [0, -1, 0], 21 | [0, -1, 0], 22 | [0, 0, 1], 23 | [0, 0, 1], 24 | [-1, 0, 0], 25 | [0, -1, 0], 26 | [0, -1, 0], 27 | [0, 0, 1], 28 | [0, 0, 1] 29 | ] 30 | ) 31 | 32 | t2m_raw_offsets = np.array([[0,0,0], 33 | [1,0,0], 34 | [-1,0,0], 35 | [0,1,0], 36 | [0,-1,0], 37 | [0,-1,0], 38 | [0,1,0], 39 | [0,-1,0], 40 | [0,-1,0], 41 | [0,1,0], 42 | [0,0,1], 43 | [0,0,1], 44 | [0,1,0], 45 | [1,0,0], 46 | [-1,0,0], 47 | [0,0,1], 48 | [0,-1,0], 49 | [0,-1,0], 50 | [0,-1,0], 51 | [0,-1,0], 52 | [0,-1,0], 53 | [0,-1,0]]) 54 | 55 | t2m_kinematic_chain = [[0, 2, 5, 8, 11], [0, 1, 4, 7, 10], [0, 3, 6, 9, 12, 15], [9, 14, 17, 19, 21], [9, 13, 16, 18, 20]] 56 | t2m_left_hand_chain = [[20, 22, 23, 24], [20, 34, 35, 36], [20, 25, 26, 27], [20, 31, 32, 33], [20, 28, 29, 30]] 57 | t2m_right_hand_chain = [[21, 43, 44, 45], [21, 46, 47, 48], [21, 40, 41, 42], [21, 37, 38, 39], [21, 49, 50, 51]] 58 | 59 | 60 | kit_tgt_skel_id = '03950' 61 | 62 | t2m_tgt_skel_id = '000021' 63 | 64 | -------------------------------------------------------------------------------- /mld/data/humanml/utils/word_vectorizer.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pickle 3 | from os.path import join as pjoin 4 | 5 | POS_enumerator = { 6 | 'VERB': 0, 7 | 'NOUN': 1, 8 | 'DET': 2, 9 | 'ADP': 3, 10 | 'NUM': 4, 11 | 'AUX': 5, 12 | 'PRON': 6, 13 | 'ADJ': 7, 14 | 'ADV': 8, 15 | 'Loc_VIP': 9, 16 | 'Body_VIP': 10, 17 | 'Obj_VIP': 11, 18 | 'Act_VIP': 12, 19 | 'Desc_VIP': 13, 20 | 'OTHER': 14, 21 | } 22 | 23 | Loc_list = ('left', 'right', 'clockwise', 'counterclockwise', 'anticlockwise', 'forward', 'back', 'backward', 24 | 'up', 'down', 'straight', 'curve') 25 | 26 | Body_list = ('arm', 'chin', 'foot', 'feet', 'face', 'hand', 'mouth', 'leg', 'waist', 'eye', 'knee', 'shoulder', 'thigh') 27 | 28 | Obj_List = ('stair', 'dumbbell', 'chair', 'window', 'floor', 'car', 'ball', 'handrail', 'baseball', 'basketball') 29 | 30 | Act_list = ('walk', 'run', 'swing', 'pick', 'bring', 'kick', 'put', 'squat', 'throw', 'hop', 'dance', 'jump', 'turn', 31 | 'stumble', 'dance', 'stop', 'sit', 'lift', 'lower', 'raise', 'wash', 'stand', 'kneel', 'stroll', 32 | 'rub', 'bend', 'balance', 'flap', 'jog', 'shuffle', 'lean', 'rotate', 'spin', 'spread', 'climb') 33 | 34 | Desc_list = ('slowly', 'carefully', 'fast', 'careful', 'slow', 'quickly', 'happy', 'angry', 'sad', 'happily', 35 | 'angrily', 'sadly') 36 | 37 | VIP_dict = { 38 | 'Loc_VIP': Loc_list, 39 | 'Body_VIP': Body_list, 40 | 'Obj_VIP': Obj_List, 41 | 'Act_VIP': Act_list, 42 | 'Desc_VIP': Desc_list, 43 | } 44 | 45 | 46 | class WordVectorizer(object): 47 | def __init__(self, meta_root, prefix): 48 | vectors = np.load(pjoin(meta_root, '%s_data.npy'%prefix)) 49 | words = pickle.load(open(pjoin(meta_root, '%s_words.pkl'%prefix), 'rb')) 50 | word2idx = pickle.load(open(pjoin(meta_root, '%s_idx.pkl'%prefix), 'rb')) 51 | self.word2vec = {w: vectors[word2idx[w]] for w in words} 52 | 53 | def _get_pos_ohot(self, pos): 54 | pos_vec = np.zeros(len(POS_enumerator)) 55 | if pos in POS_enumerator: 56 | pos_vec[POS_enumerator[pos]] = 1 57 | else: 58 | pos_vec[POS_enumerator['OTHER']] = 1 59 | return pos_vec 60 | 61 | def __len__(self): 62 | return len(self.word2vec) 63 | 64 | def __getitem__(self, item): 65 | word, pos = item.split('/') 66 | if word in self.word2vec: 67 | word_vec = self.word2vec[word] 68 | vip_pos = None 69 | for key, values in VIP_dict.items(): 70 | if word in values: 71 | vip_pos = key 72 | break 73 | if vip_pos is not None: 74 | pos_vec = self._get_pos_ohot(vip_pos) 75 | else: 76 | pos_vec = self._get_pos_ohot(pos) 77 | else: 78 | word_vec = self.word2vec['unk'] 79 | pos_vec = self._get_pos_ohot('OTHER') 80 | return word_vec, pos_vec 81 | -------------------------------------------------------------------------------- /mld/data/sampling/__init__.py: -------------------------------------------------------------------------------- 1 | from .base import FrameSampler 2 | from .framerate import subsample, upsample 3 | -------------------------------------------------------------------------------- /mld/data/sampling/base.py: -------------------------------------------------------------------------------- 1 | from .frames import get_frameix_from_data_index 2 | 3 | class FrameSampler: 4 | def __init__(self, sampling="conseq", sampling_step=1, request_frames=None,threshold_reject=0.75,max_len=1000,min_len=10): 5 | self.sampling = sampling 6 | 7 | self.sampling_step = sampling_step 8 | self.request_frames = request_frames 9 | self.threshold_reject = threshold_reject 10 | self.max_len = max_len 11 | self.min_len = min_len 12 | 13 | def __call__(self, num_frames): 14 | 15 | return get_frameix_from_data_index(num_frames, 16 | self.request_frames, 17 | self.sampling, 18 | self.sampling_step) 19 | 20 | def accept(self, duration): 21 | # Outputs have original lengths 22 | # Check if it is too long 23 | if self.request_frames is None: 24 | if duration > self.max_len: 25 | return False 26 | elif duration < self.min_len: 27 | return False 28 | else: 29 | # Reject sample if the length is 30 | # too little relative to 31 | # the request frames 32 | min_number = self.threshold_reject * self.request_frames 33 | if duration < min_number: 34 | return False 35 | return True 36 | 37 | def get(self, key, default=None): 38 | return getattr(self, key, default) 39 | 40 | def __getitem__(self, key): 41 | return getattr(self, key) 42 | -------------------------------------------------------------------------------- /mld/data/sampling/framerate.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | # TODO: use a real subsampler.. 5 | def subsample(num_frames, last_framerate, new_framerate): 6 | step = int(last_framerate / new_framerate) 7 | assert step >= 1 8 | frames = np.arange(0, num_frames, step) 9 | return frames 10 | 11 | 12 | # TODO: use a real upsampler.. 13 | def upsample(motion, last_framerate, new_framerate): 14 | step = int(new_framerate / last_framerate) 15 | assert step >= 1 16 | 17 | # Alpha blending => interpolation 18 | alpha = np.linspace(0, 1, step+1) 19 | last = np.einsum("l,...->l...", 1-alpha, motion[:-1]) 20 | new = np.einsum("l,...->l...", alpha, motion[1:]) 21 | 22 | chuncks = (last + new)[:-1] 23 | output = np.concatenate(chuncks.swapaxes(1, 0)) 24 | # Don't forget the last one 25 | output = np.concatenate((output, motion[[-1]])) 26 | return output 27 | 28 | 29 | if __name__ == "__main__": 30 | motion = np.arange(105) 31 | submotion = motion[subsample(len(motion), 100.0, 12.5)] 32 | newmotion = upsample(submotion, 12.5, 100) 33 | 34 | print(newmotion) 35 | -------------------------------------------------------------------------------- /mld/data/sampling/frames.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | import numpy as np 4 | from numpy import ndarray as Array 5 | import random 6 | 7 | 8 | def get_frameix_from_data_index(num_frames: int, 9 | request_frames: Optional[int], 10 | sampling: str = "conseq", 11 | sampling_step: int = 1) -> Array: 12 | nframes = num_frames 13 | 14 | if request_frames is None: 15 | frame_ix = np.arange(nframes) 16 | else: 17 | # sampling goal: input: ----------- 11 nframes 18 | # o--o--o--o- 4 ninputs 19 | # 20 | # step number is computed like that: [(11-1)/(4-1)] = 3 21 | # [---][---][---][- 22 | # So step = 3, and we take 0 to step*ninputs+1 with steps 23 | # [o--][o--][o--][o-] 24 | # then we can randomly shift the vector 25 | # -[o--][o--][o--]o 26 | # If there are too much frames required 27 | if request_frames > nframes: 28 | fair = False # True 29 | if fair: 30 | # distills redundancy everywhere 31 | choices = np.random.choice(range(nframes), 32 | request_frames, 33 | replace=True) 34 | frame_ix = sorted(choices) 35 | else: 36 | # adding the last frame until done 37 | ntoadd = max(0, request_frames - nframes) 38 | lastframe = nframes - 1 39 | padding = lastframe * np.ones(ntoadd, dtype=int) 40 | frame_ix = np.concatenate((np.arange(0, nframes), 41 | padding)) 42 | 43 | elif sampling in ["conseq", "random_conseq"]: 44 | step_max = (nframes - 1) // (request_frames - 1) 45 | if sampling == "conseq": 46 | if sampling_step == -1 or sampling_step * (request_frames - 1) >= nframes: 47 | step = step_max 48 | else: 49 | step = sampling_step 50 | elif sampling == "random_conseq": 51 | step = random.randint(1, step_max) 52 | 53 | lastone = step * (request_frames - 1) 54 | shift_max = nframes - lastone - 1 55 | shift = random.randint(0, max(0, shift_max - 1)) 56 | frame_ix = shift + np.arange(0, lastone + 1, step) 57 | 58 | elif sampling == "random": 59 | choices = np.random.choice(range(nframes), 60 | request_frames, 61 | replace=False) 62 | frame_ix = sorted(choices) 63 | 64 | else: 65 | raise ValueError("Sampling not recognized.") 66 | 67 | return frame_ix 68 | -------------------------------------------------------------------------------- /mld/launch/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpthu17/GraphMotion/d56feba6f6d39867b933ca752244fcbbb703a4b8/mld/launch/__init__.py -------------------------------------------------------------------------------- /mld/launch/blender.py: -------------------------------------------------------------------------------- 1 | # Fix blender path 2 | import sys 3 | import os 4 | # local packages 5 | sys.path.append(os.path.expanduser("~/.local/lib/python3.9/site-packages")) 6 | import bpy 7 | import os 8 | from argparse import ArgumentParser 9 | 10 | # Monkey patch argparse such that 11 | # blender / python / hydra parsing works 12 | def parse_args(self, args=None, namespace=None): 13 | if args is not None: 14 | return self.parse_args_bak(args=args, namespace=namespace) 15 | try: 16 | idx = sys.argv.index("--") 17 | args = sys.argv[idx+1:] # the list after '--' 18 | except ValueError as e: # '--' not in the list: 19 | args = [] 20 | return self.parse_args_bak(args=args, namespace=namespace) 21 | 22 | setattr(ArgumentParser, 'parse_args_bak', ArgumentParser.parse_args) 23 | setattr(ArgumentParser, 'parse_args', parse_args) 24 | -------------------------------------------------------------------------------- /mld/launch/prepare.py: -------------------------------------------------------------------------------- 1 | import os 2 | import warnings 3 | from pathlib import Path 4 | 5 | import hydra 6 | from mld.tools.runid import generate_id 7 | from omegaconf import OmegaConf 8 | 9 | 10 | # Local paths 11 | def code_path(path=""): 12 | code_dir = hydra.utils.get_original_cwd() 13 | code_dir = Path(code_dir) 14 | return str(code_dir / path) 15 | 16 | 17 | def working_path(path): 18 | return str(Path(os.getcwd()) / path) 19 | 20 | 21 | # fix the id for this run 22 | ID = generate_id() 23 | 24 | 25 | def generate_id(): 26 | return ID 27 | 28 | 29 | def get_last_checkpoint(path, ckpt_name="last.ckpt"): 30 | output_dir = Path(hydra.utils.to_absolute_path(path)) 31 | last_ckpt_path = output_dir / "checkpoints" / ckpt_name 32 | return str(last_ckpt_path) 33 | 34 | 35 | def get_kitname(load_amass_data: bool, load_with_rot: bool): 36 | if not load_amass_data: 37 | return "kit-mmm-xyz" 38 | if load_amass_data and not load_with_rot: 39 | return "kit-amass-xyz" 40 | if load_amass_data and load_with_rot: 41 | return "kit-amass-rot" 42 | 43 | 44 | OmegaConf.register_new_resolver("code_path", code_path) 45 | OmegaConf.register_new_resolver("working_path", working_path) 46 | OmegaConf.register_new_resolver("generate_id", generate_id) 47 | OmegaConf.register_new_resolver("absolute_path", hydra.utils.to_absolute_path) 48 | OmegaConf.register_new_resolver("get_last_checkpoint", get_last_checkpoint) 49 | OmegaConf.register_new_resolver("get_kitname", get_kitname) 50 | 51 | 52 | # Remove warnings 53 | warnings.filterwarnings( 54 | "ignore", ".*Trying to infer the `batch_size` from an ambiguous collection.*" 55 | ) 56 | 57 | warnings.filterwarnings( 58 | "ignore", ".*does not have many workers which may be a bottleneck*" 59 | ) 60 | 61 | warnings.filterwarnings( 62 | "ignore", ".*Our suggested max number of worker in current system is*" 63 | ) 64 | 65 | 66 | # os.environ["HYDRA_FULL_ERROR"] = "1" 67 | os.environ["NUMEXPR_MAX_THREADS"] = "24" 68 | -------------------------------------------------------------------------------- /mld/launch/tools.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from omegaconf import DictConfig, OmegaConf 3 | import hydra 4 | import os 5 | 6 | 7 | def resolve_cfg_path(cfg: DictConfig): 8 | working_dir = os.getcwd() 9 | cfg.working_dir = working_dir 10 | -------------------------------------------------------------------------------- /mld/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpthu17/GraphMotion/d56feba6f6d39867b933ca752244fcbbb703a4b8/mld/models/__init__.py -------------------------------------------------------------------------------- /mld/models/architectures/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpthu17/GraphMotion/d56feba6f6d39867b933ca752244fcbbb703a4b8/mld/models/architectures/__init__.py -------------------------------------------------------------------------------- /mld/models/architectures/t2m_motionenc.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from torch.nn.utils.rnn import pack_padded_sequence 4 | 5 | 6 | class MovementConvEncoder(nn.Module): 7 | def __init__(self, input_size, hidden_size, output_size): 8 | super(MovementConvEncoder, self).__init__() 9 | self.main = nn.Sequential( 10 | nn.Conv1d(input_size, hidden_size, 4, 2, 1), 11 | nn.Dropout(0.2, inplace=True), 12 | nn.LeakyReLU(0.2, inplace=True), 13 | nn.Conv1d(hidden_size, output_size, 4, 2, 1), 14 | nn.Dropout(0.2, inplace=True), 15 | nn.LeakyReLU(0.2, inplace=True), 16 | ) 17 | self.out_net = nn.Linear(output_size, output_size) 18 | # self.main.apply(init_weight) 19 | # self.out_net.apply(init_weight) 20 | 21 | def forward(self, inputs): 22 | inputs = inputs.permute(0, 2, 1) 23 | outputs = self.main(inputs).permute(0, 2, 1) 24 | # print(outputs.shape) 25 | return self.out_net(outputs) 26 | 27 | 28 | class MotionEncoderBiGRUCo(nn.Module): 29 | def __init__(self, input_size, hidden_size, output_size): 30 | super(MotionEncoderBiGRUCo, self).__init__() 31 | 32 | self.input_emb = nn.Linear(input_size, hidden_size) 33 | self.gru = nn.GRU( 34 | hidden_size, hidden_size, batch_first=True, bidirectional=True 35 | ) 36 | self.output_net = nn.Sequential( 37 | nn.Linear(hidden_size * 2, hidden_size), 38 | nn.LayerNorm(hidden_size), 39 | nn.LeakyReLU(0.2, inplace=True), 40 | nn.Linear(hidden_size, output_size), 41 | ) 42 | 43 | # self.input_emb.apply(init_weight) 44 | # self.output_net.apply(init_weight) 45 | self.hidden_size = hidden_size 46 | self.hidden = nn.Parameter( 47 | torch.randn((2, 1, self.hidden_size), requires_grad=True) 48 | ) 49 | 50 | # input(batch_size, seq_len, dim) 51 | def forward(self, inputs, m_lens): 52 | num_samples = inputs.shape[0] 53 | 54 | input_embs = self.input_emb(inputs) 55 | hidden = self.hidden.repeat(1, num_samples, 1) 56 | 57 | cap_lens = m_lens.data.tolist() 58 | emb = pack_padded_sequence(input_embs, cap_lens, batch_first=True) 59 | 60 | gru_seq, gru_last = self.gru(emb, hidden) 61 | 62 | gru_last = torch.cat([gru_last[0], gru_last[1]], dim=-1) 63 | 64 | return self.output_net(gru_last) 65 | -------------------------------------------------------------------------------- /mld/models/architectures/t2m_textenc.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | from torch.nn.utils.rnn import pack_padded_sequence 4 | 5 | 6 | class TextEncoderBiGRUCo(nn.Module): 7 | def __init__(self, word_size, pos_size, hidden_size, output_size): 8 | super(TextEncoderBiGRUCo, self).__init__() 9 | 10 | self.pos_emb = nn.Linear(pos_size, word_size) 11 | self.input_emb = nn.Linear(word_size, hidden_size) 12 | self.gru = nn.GRU( 13 | hidden_size, hidden_size, batch_first=True, bidirectional=True 14 | ) 15 | self.output_net = nn.Sequential( 16 | nn.Linear(hidden_size * 2, hidden_size), 17 | nn.LayerNorm(hidden_size), 18 | nn.LeakyReLU(0.2, inplace=True), 19 | nn.Linear(hidden_size, output_size), 20 | ) 21 | 22 | # self.input_emb.apply(init_weight) 23 | # self.pos_emb.apply(init_weight) 24 | # self.output_net.apply(init_weight) 25 | # self.linear2.apply(init_weight) 26 | # self.batch_size = batch_size 27 | self.hidden_size = hidden_size 28 | self.hidden = nn.Parameter( 29 | torch.randn((2, 1, self.hidden_size), requires_grad=True) 30 | ) 31 | 32 | # input(batch_size, seq_len, dim) 33 | def forward(self, word_embs, pos_onehot, cap_lens): 34 | num_samples = word_embs.shape[0] 35 | 36 | pos_embs = self.pos_emb(pos_onehot) 37 | inputs = word_embs + pos_embs 38 | input_embs = self.input_emb(inputs) 39 | hidden = self.hidden.repeat(1, num_samples, 1) 40 | 41 | cap_lens = cap_lens.data.tolist() 42 | emb = pack_padded_sequence(input_embs, cap_lens, batch_first=True) 43 | 44 | gru_seq, gru_last = self.gru(emb, hidden) 45 | 46 | gru_last = torch.cat([gru_last[0], gru_last[1]], dim=-1) 47 | 48 | return self.output_net(gru_last) 49 | -------------------------------------------------------------------------------- /mld/models/get_model.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | 3 | 4 | def get_model(cfg, datamodule, phase="train"): 5 | modeltype = cfg.model.model_type 6 | if modeltype == "mld": 7 | return get_module(cfg, datamodule) 8 | else: 9 | raise ValueError(f"Invalid model type {modeltype}.") 10 | 11 | 12 | def get_module(cfg, datamodule): 13 | modeltype = cfg.model.model_type 14 | model_module = importlib.import_module( 15 | f".modeltype.{cfg.model.model_type}", package="mld.models") 16 | Model = model_module.__getattribute__(f"{modeltype.upper()}") 17 | return Model(cfg=cfg, datamodule=datamodule) 18 | -------------------------------------------------------------------------------- /mld/models/losses/__init__.py: -------------------------------------------------------------------------------- 1 | from mld.models.losses.temos import TemosLosses 2 | from mld.models.losses.tmost import TmostLosses 3 | -------------------------------------------------------------------------------- /mld/models/losses/kl.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | class KLLoss: 4 | def __init__(self): 5 | pass 6 | 7 | def __call__(self, q, p): 8 | div = torch.distributions.kl_divergence(q, p) 9 | return div.mean() 10 | 11 | def __repr__(self): 12 | return "KLLoss()" 13 | 14 | class KLLossMulti: 15 | def __init__(self): 16 | self.klloss = KLLoss() 17 | 18 | def __call__(self, qlist, plist): 19 | return sum([self.klloss(q, p) 20 | for q, p in zip(qlist, plist)]) 21 | 22 | def __repr__(self): 23 | return "KLLossMulti()" 24 | -------------------------------------------------------------------------------- /mld/models/metrics/__init__.py: -------------------------------------------------------------------------------- 1 | from .compute import ComputeMetrics 2 | from .mr import MRMetrics 3 | from .tm2t import TM2TMetrics 4 | from .mm import MMMetrics 5 | from .gru import HUMANACTMetrics 6 | from .stgcn import UESTCMetrics 7 | from .uncond import UncondMetrics 8 | -------------------------------------------------------------------------------- /mld/models/metrics/compute_best.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | import torch 4 | from einops import rearrange 5 | from torch import Tensor 6 | from torchmetrics import Metric 7 | import numpy as np 8 | from .compute import ComputeMetrics, l2_norm, variance 9 | 10 | 11 | class ComputeMetricsBest(ComputeMetrics): 12 | def update(self, jts_text_: List[Tensor], jts_ref_: List[Tensor], lengths: List[List[int]]): 13 | self.count += sum(lengths[0]) 14 | self.count_seq += len(lengths[0]) 15 | 16 | ntrials = len(jts_text_) 17 | metrics = [] 18 | for index in range(ntrials): 19 | jts_text, poses_text, root_text, traj_text = self.transform(jts_text_[index], lengths[index]) 20 | jts_ref, poses_ref, root_ref, traj_ref = self.transform(jts_ref_[index], lengths[index]) 21 | 22 | mets = [] 23 | for i in range(len(lengths[index])): 24 | APE_root = l2_norm(root_text[i], root_ref[i], dim=1).sum() 25 | APE_pose = l2_norm(poses_text[i], poses_ref[i], dim=2).sum(0) 26 | APE_traj = l2_norm(traj_text[i], traj_ref[i], dim=1).sum() 27 | APE_joints = l2_norm(jts_text[i], jts_ref[i], dim=2).sum(0) 28 | 29 | root_sigma_text = variance(root_text[i], lengths[index][i], dim=0) 30 | root_sigma_ref = variance(root_ref[i], lengths[index][i], dim=0) 31 | AVE_root = l2_norm(root_sigma_text, root_sigma_ref, dim=0) 32 | 33 | traj_sigma_text = variance(traj_text[i], lengths[index][i], dim=0) 34 | traj_sigma_ref = variance(traj_ref[i], lengths[index][i], dim=0) 35 | AVE_traj = l2_norm(traj_sigma_text, traj_sigma_ref, dim=0) 36 | 37 | poses_sigma_text = variance(poses_text[i], lengths[index][i], dim=0) 38 | poses_sigma_ref = variance(poses_ref[i], lengths[index][i], dim=0) 39 | AVE_pose = l2_norm(poses_sigma_text, poses_sigma_ref, dim=1) 40 | 41 | jts_sigma_text = variance(jts_text[i], lengths[index][i], dim=0) 42 | jts_sigma_ref = variance(jts_ref[i], lengths[index][i], dim=0) 43 | AVE_joints = l2_norm(jts_sigma_text, jts_sigma_ref, dim=1) 44 | 45 | met = [APE_root, APE_pose, APE_traj, APE_joints, 46 | AVE_root, AVE_pose, AVE_traj, AVE_joints] 47 | mets.append(met) 48 | metrics.append(mets) 49 | 50 | # Quick hacks 51 | mmm = metrics[np.argmin([x[0][0] for x in metrics])] 52 | APE_root, APE_pose, APE_traj, APE_joints, AVE_root, AVE_pose, AVE_traj, AVE_joints = mmm[0] 53 | self.APE_root += APE_root 54 | self.APE_pose += APE_pose 55 | self.APE_traj += APE_traj 56 | self.APE_joints += APE_joints 57 | self.AVE_root += AVE_root 58 | self.AVE_pose += AVE_pose 59 | self.AVE_traj += AVE_traj 60 | self.AVE_joints += AVE_joints 61 | -------------------------------------------------------------------------------- /mld/models/metrics/compute_worst.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | import torch 4 | from einops import rearrange 5 | from torch import Tensor 6 | from torchmetrics import Metric 7 | import numpy as np 8 | from .compute import ComputeMetrics, l2_norm, variance 9 | 10 | 11 | class ComputeMetricsWorst(ComputeMetrics): 12 | def update(self, jts_text_: List[Tensor], jts_ref_: List[Tensor], lengths: List[List[int]]): 13 | self.count += sum(lengths[0]) 14 | self.count_seq += len(lengths[0]) 15 | 16 | ntrials = len(jts_text_) 17 | metrics = [] 18 | for index in range(ntrials): 19 | jts_text, poses_text, root_text, traj_text = self.transform(jts_text_[index], lengths[index]) 20 | jts_ref, poses_ref, root_ref, traj_ref = self.transform(jts_ref_[index], lengths[index]) 21 | 22 | mets = [] 23 | for i in range(len(lengths[index])): 24 | APE_root = l2_norm(root_text[i], root_ref[i], dim=1).sum() 25 | APE_pose = l2_norm(poses_text[i], poses_ref[i], dim=2).sum(0) 26 | APE_traj = l2_norm(traj_text[i], traj_ref[i], dim=1).sum() 27 | APE_joints = l2_norm(jts_text[i], jts_ref[i], dim=2).sum(0) 28 | 29 | root_sigma_text = variance(root_text[i], lengths[index][i], dim=0) 30 | root_sigma_ref = variance(root_ref[i], lengths[index][i], dim=0) 31 | AVE_root = l2_norm(root_sigma_text, root_sigma_ref, dim=0) 32 | 33 | traj_sigma_text = variance(traj_text[i], lengths[index][i], dim=0) 34 | traj_sigma_ref = variance(traj_ref[i], lengths[index][i], dim=0) 35 | AVE_traj = l2_norm(traj_sigma_text, traj_sigma_ref, dim=0) 36 | 37 | poses_sigma_text = variance(poses_text[i], lengths[index][i], dim=0) 38 | poses_sigma_ref = variance(poses_ref[i], lengths[index][i], dim=0) 39 | AVE_pose = l2_norm(poses_sigma_text, poses_sigma_ref, dim=1) 40 | 41 | jts_sigma_text = variance(jts_text[i], lengths[index][i], dim=0) 42 | jts_sigma_ref = variance(jts_ref[i], lengths[index][i], dim=0) 43 | AVE_joints = l2_norm(jts_sigma_text, jts_sigma_ref, dim=1) 44 | 45 | met = [APE_root, APE_pose, APE_traj, APE_joints, 46 | AVE_root, AVE_pose, AVE_traj, AVE_joints] 47 | mets.append(met) 48 | metrics.append(mets) 49 | 50 | # Quick hacks 51 | mmm = metrics[np.argmax([x[0][0] for x in metrics])] 52 | APE_root, APE_pose, APE_traj, APE_joints, AVE_root, AVE_pose, AVE_traj, AVE_joints = mmm[0] 53 | self.APE_root += APE_root 54 | self.APE_pose += APE_pose 55 | self.APE_traj += APE_traj 56 | self.APE_joints += APE_joints 57 | self.AVE_root += AVE_root 58 | self.AVE_pose += AVE_pose 59 | self.AVE_traj += AVE_traj 60 | self.AVE_joints += AVE_joints 61 | -------------------------------------------------------------------------------- /mld/models/metrics/mm.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | import torch 4 | from torch import Tensor 5 | from torchmetrics import Metric 6 | from torchmetrics.functional import pairwise_euclidean_distance 7 | 8 | from .utils import * 9 | 10 | 11 | class MMMetrics(Metric): 12 | full_state_update = True 13 | 14 | def __init__(self, mm_num_times=10, dist_sync_on_step=True, stage=0, **kwargs): 15 | super().__init__(dist_sync_on_step=dist_sync_on_step) 16 | 17 | self.name = "MultiModality scores" 18 | 19 | self.mm_num_times = mm_num_times 20 | 21 | self.add_state("count", default=torch.tensor(0), dist_reduce_fx="sum") 22 | self.add_state("count_seq", 23 | default=torch.tensor(0), 24 | dist_reduce_fx="sum") 25 | 26 | self.stage =stage 27 | 28 | if self.stage in [1, 2, 3]: 29 | self.metrics = [f"s{str(self.stage)}_MultiModality"] 30 | self.add_state(f"s{str(self.stage)}_MultiModality", 31 | default=torch.tensor(0.), 32 | dist_reduce_fx="sum") 33 | else: 34 | self.metrics = ["MultiModality"] 35 | self.add_state("MultiModality", 36 | default=torch.tensor(0.), 37 | dist_reduce_fx="sum") 38 | 39 | # chached batches 40 | self.add_state("mm_motion_embeddings", default=[], dist_reduce_fx=None) 41 | 42 | def compute(self, sanity_flag): 43 | count = self.count.item() 44 | count_seq = self.count_seq.item() 45 | 46 | # init metrics 47 | metrics = {metric: getattr(self, metric) for metric in self.metrics} 48 | 49 | # if in sanity check stage then jump 50 | if sanity_flag: 51 | return metrics 52 | 53 | # cat all embeddings 54 | all_mm_motions = torch.cat(self.mm_motion_embeddings, 55 | axis=0).cpu().numpy() 56 | if self.stage in [1, 2, 3]: 57 | metrics[f"s{str(self.stage)}_MultiModality"] = calculate_multimodality_np( 58 | all_mm_motions, self.mm_num_times) 59 | else: 60 | metrics['MultiModality'] = calculate_multimodality_np( 61 | all_mm_motions, self.mm_num_times) 62 | 63 | return {**metrics} 64 | 65 | def update( 66 | self, 67 | mm_motion_embeddings: Tensor, 68 | lengths: List[int], 69 | ): 70 | self.count += sum(lengths) 71 | self.count_seq += len(lengths) 72 | 73 | # store all mm motion embeddings 74 | self.mm_motion_embeddings.append(mm_motion_embeddings) 75 | -------------------------------------------------------------------------------- /mld/models/modeltype/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpthu17/GraphMotion/d56feba6f6d39867b933ca752244fcbbb703a4b8/mld/models/modeltype/__init__.py -------------------------------------------------------------------------------- /mld/models/operator/__init__.py: -------------------------------------------------------------------------------- 1 | from .adain import AdaptiveInstanceNorm1d 2 | from .blocks import ConvBlock, LinearBlock 3 | from .position_encoding_layer import PositionalEncoding 4 | -------------------------------------------------------------------------------- /mld/models/operator/adain.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | 5 | class AdaptiveInstanceNorm1d(nn.Module): 6 | def __init__(self, num_features, eps=1e-5, momentum=0.1): 7 | super(AdaptiveInstanceNorm1d, self).__init__() 8 | self.num_features = num_features 9 | self.eps = eps 10 | self.momentum = momentum 11 | self.weight = None 12 | self.bias = None 13 | self.register_buffer('running_mean', torch.zeros(num_features)) 14 | self.register_buffer('running_var', torch.ones(num_features)) 15 | 16 | def forward(self, x, direct_weighting=False, no_std=False): 17 | assert self.weight is not None and \ 18 | self.bias is not None, "Please assign AdaIN weight first" 19 | # (bs, nfeats, nframe) <= (nframe, bs, nfeats) 20 | x = x.permute(1,2,0) 21 | 22 | b, c = x.size(0), x.size(1) # batch size & channels 23 | running_mean = self.running_mean.repeat(b) 24 | running_var = self.running_var.repeat(b) 25 | # self.weight = torch.ones_like(self.weight) 26 | 27 | if direct_weighting: 28 | x_reshaped = x.contiguous().view(b * c) 29 | if no_std: 30 | out = x_reshaped + self.bias 31 | else: 32 | out = x_reshaped.mul(self.weight) + self.bias 33 | out = out.view(b, c, *x.size()[2:]) 34 | else: 35 | x_reshaped = x.contiguous().view(1, b * c, *x.size()[2:]) 36 | out = F.batch_norm( 37 | x_reshaped, running_mean, running_var, self.weight, self.bias, 38 | True, self.momentum, self.eps) 39 | out = out.view(b, c, *x.size()[2:]) 40 | 41 | # (nframe, bs, nfeats) <= (bs, nfeats, nframe) 42 | out = out.permute(2,0,1) 43 | return out 44 | 45 | def __repr__(self): 46 | return self.__class__.__name__ + '(' + str(self.num_features) + ')' 47 | 48 | def assign_adain_params(adain_params, model): 49 | # assign the adain_params to the AdaIN layers in model 50 | for m in model.modules(): 51 | if m.__class__.__name__ == "AdaptiveInstanceNorm1d": 52 | mean = adain_params[: , : m.num_features] 53 | std = adain_params[: , m.num_features: 2 * m.num_features] 54 | m.bias = mean.contiguous().view(-1) 55 | m.weight = std.contiguous().view(-1) 56 | if adain_params.size(1) > 2 * m.num_features: 57 | adain_params = adain_params[: , 2 * m.num_features:] 58 | 59 | 60 | def get_num_adain_params(model): 61 | # return the number of AdaIN parameters needed by the model 62 | num_adain_params = 0 63 | for m in model.modules(): 64 | if m.__class__.__name__ == "AdaptiveInstanceNorm1d": 65 | num_adain_params += 2 * m.num_features 66 | return num_adain_params 67 | -------------------------------------------------------------------------------- /mld/models/operator/position_encoding_layer.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | from torch import nn 4 | 5 | 6 | class PositionalEncoding(nn.Module): 7 | 8 | def __init__(self, d_model, dropout=0.1, max_len=5000, batch_first=False): 9 | super().__init__() 10 | self.batch_first = batch_first 11 | 12 | self.dropout = nn.Dropout(p=dropout) 13 | 14 | pe = torch.zeros(max_len, d_model) 15 | position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1) 16 | div_term = torch.exp(torch.arange( 17 | 0, d_model, 2).float() * (-np.log(10000.0) / d_model)) 18 | pe[:, 0::2] = torch.sin(position * div_term) 19 | pe[:, 1::2] = torch.cos(position * div_term) 20 | pe = pe.unsqueeze(0).transpose(0, 1) 21 | 22 | self.register_buffer("pe", pe) 23 | 24 | def forward(self, x): 25 | # not used in the final model 26 | if self.batch_first: 27 | x = x + self.pe.permute(1, 0, 2)[:, : x.shape[1], :] 28 | else: 29 | x = x + self.pe[: x.shape[0], :] 30 | return self.dropout(x) 31 | -------------------------------------------------------------------------------- /mld/models/operator/self_attention.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpthu17/GraphMotion/d56feba6f6d39867b933ca752244fcbbb703a4b8/mld/models/operator/self_attention.py -------------------------------------------------------------------------------- /mld/models/tools/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpthu17/GraphMotion/d56feba6f6d39867b933ca752244fcbbb703a4b8/mld/models/tools/__init__.py -------------------------------------------------------------------------------- /mld/models/tools/tools.py: -------------------------------------------------------------------------------- 1 | import torch.nn as nn 2 | 3 | def remove_padding(tensors, lengths): 4 | return [tensor[:tensor_length] for tensor, tensor_length in zip(tensors, lengths)] 5 | 6 | class AutoParams(nn.Module): 7 | def __init__(self, **kargs): 8 | try: 9 | for param in self.needed_params: 10 | if param in kargs: 11 | setattr(self, param, kargs[param]) 12 | else: 13 | raise ValueError(f"{param} is needed.") 14 | except : 15 | pass 16 | 17 | try: 18 | for param, default in self.optional_params.items(): 19 | if param in kargs and kargs[param] is not None: 20 | setattr(self, param, kargs[param]) 21 | else: 22 | setattr(self, param, default) 23 | except : 24 | pass 25 | super().__init__() 26 | 27 | 28 | # taken from joeynmt repo 29 | def freeze_params(module: nn.Module) -> None: 30 | """ 31 | Freeze the parameters of this module, 32 | i.e. do not update them during training 33 | 34 | :param module: freeze parameters of this module 35 | """ 36 | for _, p in module.named_parameters(): 37 | p.requires_grad = False 38 | -------------------------------------------------------------------------------- /mld/render/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpthu17/GraphMotion/d56feba6f6d39867b933ca752244fcbbb703a4b8/mld/render/__init__.py -------------------------------------------------------------------------------- /mld/render/blender/__init__.py: -------------------------------------------------------------------------------- 1 | from .render import render 2 | -------------------------------------------------------------------------------- /mld/render/blender/camera.py: -------------------------------------------------------------------------------- 1 | import bpy 2 | 3 | 4 | class Camera: 5 | def __init__(self, *, first_root, mode, is_mesh): 6 | camera = bpy.data.objects['Camera'] 7 | 8 | ## initial position 9 | camera.location.x = 7.36 10 | camera.location.y = -6.93 11 | if is_mesh: 12 | # camera.location.z = 5.45 13 | camera.location.z = 5.6 14 | else: 15 | camera.location.z = 5.2 16 | 17 | # wider point of view 18 | if mode == "sequence": 19 | if is_mesh: 20 | camera.data.lens = 65 21 | else: 22 | camera.data.lens = 85 23 | elif mode == "frame": 24 | if is_mesh: 25 | camera.data.lens = 130 26 | else: 27 | camera.data.lens = 85 28 | elif mode == "video": 29 | if is_mesh: 30 | camera.data.lens = 110 31 | else: 32 | # avoid cutting person 33 | camera.data.lens = 85 34 | # camera.data.lens = 140 35 | 36 | # camera.location.x += 0.75 37 | 38 | self.mode = mode 39 | self.camera = camera 40 | 41 | self.camera.location.x += first_root[0] 42 | self.camera.location.y += first_root[1] 43 | 44 | self._root = first_root 45 | 46 | def update(self, newroot): 47 | delta_root = newroot - self._root 48 | 49 | self.camera.location.x += delta_root[0] 50 | self.camera.location.y += delta_root[1] 51 | 52 | self._root = newroot 53 | -------------------------------------------------------------------------------- /mld/render/blender/data.py: -------------------------------------------------------------------------------- 1 | class Data: 2 | def __len__(self): 3 | return self.N 4 | -------------------------------------------------------------------------------- /mld/render/blender/meshes.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from .materials import body_material 4 | 5 | # green 6 | # GT_SMPL = body_material(0.009, 0.214, 0.029) 7 | GT_SMPL = body_material(0.035, 0.415, 0.122) 8 | 9 | # blue 10 | # GEN_SMPL = body_material(0.022, 0.129, 0.439) 11 | # Blues => cmap(0.87) 12 | # GEN_SMPL = body_material(0.035, 0.322, 0.615) 13 | # Oranges => cmap(0.87) 14 | GEN_SMPL = body_material(0.658, 0.214, 0.0114) 15 | 16 | 17 | class Meshes: 18 | def __init__(self, data, *, gt, mode, faces_path, canonicalize, always_on_floor, oldrender=True, **kwargs): 19 | data = prepare_meshes(data, canonicalize=canonicalize, 20 | always_on_floor=always_on_floor) 21 | 22 | self.faces = np.load(faces_path) 23 | print(faces_path) 24 | self.data = data 25 | self.mode = mode 26 | self.oldrender = oldrender 27 | 28 | self.N = len(data) 29 | self.trajectory = data[:, :, [0, 1]].mean(1) 30 | 31 | if gt: 32 | self.mat = GT_SMPL 33 | else: 34 | self.mat = GEN_SMPL 35 | 36 | def get_sequence_mat(self, frac): 37 | import matplotlib 38 | # cmap = matplotlib.cm.get_cmap('Blues') 39 | cmap = matplotlib.cm.get_cmap('Oranges') 40 | # begin = 0.60 41 | # end = 0.90 42 | begin = 0.50 43 | end = 0.90 44 | rgbcolor = cmap(begin + (end-begin)*frac) 45 | mat = body_material(*rgbcolor, oldrender=self.oldrender) 46 | return mat 47 | 48 | def get_root(self, index): 49 | return self.data[index].mean(0) 50 | 51 | def get_mean_root(self): 52 | return self.data.mean((0, 1)) 53 | 54 | def load_in_blender(self, index, mat): 55 | vertices = self.data[index] 56 | faces = self.faces 57 | name = f"{str(index).zfill(4)}" 58 | 59 | from .tools import load_numpy_vertices_into_blender 60 | load_numpy_vertices_into_blender(vertices, faces, name, mat) 61 | 62 | return name 63 | 64 | def __len__(self): 65 | return self.N 66 | 67 | 68 | def prepare_meshes(data, canonicalize=True, always_on_floor=False): 69 | if canonicalize: 70 | print("No canonicalization for now") 71 | 72 | # fitted mesh do not need fixing axis 73 | # # fix axis 74 | # data[..., 1] = - data[..., 1] 75 | # data[..., 0] = - data[..., 0] 76 | 77 | # Swap axis (gravity=Z instead of Y) 78 | data = data[..., [2, 0, 1]] 79 | 80 | # Remove the floor 81 | data[..., 2] -= data[..., 2].min() 82 | 83 | # Put all the body on the floor 84 | if always_on_floor: 85 | data[..., 2] -= data[..., 2].min(1)[:, None] 86 | 87 | return data 88 | -------------------------------------------------------------------------------- /mld/render/blender/sampler.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | def get_frameidx(*, mode, nframes, exact_frame, frames_to_keep): 4 | if mode == "sequence": 5 | frameidx = np.linspace(0, nframes - 1, frames_to_keep) 6 | frameidx = np.round(frameidx).astype(int) 7 | frameidx = list(frameidx) 8 | elif mode == "frame": 9 | index_frame = int(exact_frame*nframes) 10 | frameidx = [index_frame] 11 | elif mode == "video": 12 | frameidx = range(0, nframes) 13 | else: 14 | raise ValueError(f"Not support {mode} render mode") 15 | return frameidx 16 | -------------------------------------------------------------------------------- /mld/render/blender/tools.py: -------------------------------------------------------------------------------- 1 | import bpy 2 | import numpy as np 3 | 4 | 5 | def mesh_detect(data): 6 | # heuristic 7 | if data.shape[1] > 1000: 8 | return True 9 | return False 10 | 11 | 12 | # see this for more explanation 13 | # https://gist.github.com/iyadahmed/7c7c0fae03c40bd87e75dc7059e35377 14 | # This should be solved with new version of blender 15 | class ndarray_pydata(np.ndarray): 16 | def __bool__(self) -> bool: 17 | return len(self) > 0 18 | 19 | 20 | def load_numpy_vertices_into_blender(vertices, faces, name, mat): 21 | mesh = bpy.data.meshes.new(name) 22 | mesh.from_pydata(vertices, [], faces.view(ndarray_pydata)) 23 | mesh.validate() 24 | 25 | obj = bpy.data.objects.new(name, mesh) 26 | bpy.context.scene.collection.objects.link(obj) 27 | 28 | bpy.ops.object.select_all(action='DESELECT') 29 | obj.select_set(True) 30 | obj.active_material = mat 31 | bpy.context.view_layer.objects.active = obj 32 | bpy.ops.object.shade_smooth() 33 | bpy.ops.object.select_all(action='DESELECT') 34 | return True 35 | 36 | 37 | def delete_objs(names): 38 | if not isinstance(names, list): 39 | names = [names] 40 | # bpy.ops.object.mode_set(mode='OBJECT') 41 | bpy.ops.object.select_all(action='DESELECT') 42 | for obj in bpy.context.scene.objects: 43 | for name in names: 44 | if obj.name.startswith(name) or obj.name.endswith(name): 45 | obj.select_set(True) 46 | bpy.ops.object.delete() 47 | bpy.ops.object.select_all(action='DESELECT') 48 | -------------------------------------------------------------------------------- /mld/render/blender/vertices.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def prepare_vertices(vertices, canonicalize=True): 5 | data = vertices 6 | # Swap axis (gravity=Z instead of Y) 7 | # data = data[..., [2, 0, 1]] 8 | 9 | # Make left/right correct 10 | # data[..., [1]] = -data[..., [1]] 11 | 12 | # Center the first root to the first frame 13 | data -= data[[0], [0], :] 14 | 15 | # Remove the floor 16 | data[..., 2] -= np.min(data[..., 2]) 17 | return data 18 | -------------------------------------------------------------------------------- /mld/render/video.py: -------------------------------------------------------------------------------- 1 | import moviepy.editor as mp 2 | import moviepy.video.fx.all as vfx 3 | import os 4 | import imageio 5 | 6 | 7 | def mask_png(frames): 8 | for frame in frames: 9 | im = imageio.imread(frame) 10 | im[im[:, :, 3] < 1, :] = 255 11 | imageio.imwrite(frame, im[:, :, 0:3]) 12 | return 13 | 14 | 15 | class Video: 16 | def __init__(self, frame_path: str, fps: float = 12.5, res="high"): 17 | frame_path = str(frame_path) 18 | self.fps = fps 19 | 20 | self._conf = {"codec": "libx264", 21 | "fps": self.fps, 22 | "audio_codec": "aac", 23 | "temp_audiofile": "temp-audio.m4a", 24 | "remove_temp": True} 25 | 26 | if res == "low": 27 | bitrate = "500k" 28 | else: 29 | bitrate = "5000k" 30 | 31 | self._conf = {"bitrate": bitrate, 32 | "fps": self.fps} 33 | 34 | # Load video 35 | # video = mp.VideoFileClip(video1_path, audio=False) 36 | # Load with frames 37 | frames = [os.path.join(frame_path, x) 38 | for x in sorted(os.listdir(frame_path))] 39 | 40 | # mask background white for videos 41 | mask_png(frames) 42 | 43 | video = mp.ImageSequenceClip(frames, fps=fps) 44 | self.video = video 45 | self.duration = video.duration 46 | 47 | def add_text(self, text): 48 | # needs ImageMagick 49 | video_text = mp.TextClip(text, 50 | font='Amiri', 51 | color='white', 52 | method='caption', 53 | align="center", 54 | size=(self.video.w, None), 55 | fontsize=30) 56 | video_text = video_text.on_color(size=(self.video.w, video_text.h + 5), 57 | color=(0, 0, 0), 58 | col_opacity=0.6) 59 | # video_text = video_text.set_pos('bottom') 60 | video_text = video_text.set_pos('top') 61 | 62 | self.video = mp.CompositeVideoClip([self.video, video_text]) 63 | 64 | def save(self, out_path): 65 | out_path = str(out_path) 66 | self.video.subclip(0, self.duration).write_videofile( 67 | out_path, **self._conf) 68 | -------------------------------------------------------------------------------- /mld/tools/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpthu17/GraphMotion/d56feba6f6d39867b933ca752244fcbbb703a4b8/mld/tools/__init__.py -------------------------------------------------------------------------------- /mld/tools/logging.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import tqdm 3 | 4 | 5 | class LevelsFilter(logging.Filter): 6 | def __init__(self, levels): 7 | self.levels = [getattr(logging, level) for level in levels] 8 | 9 | def filter(self, record): 10 | return record.levelno in self.levels 11 | 12 | 13 | class StreamToLogger(object): 14 | """ 15 | Fake file-like stream object that redirects writes to a logger instance. 16 | """ 17 | def __init__(self, logger, level): 18 | self.logger = logger 19 | self.level = level 20 | self.linebuf = '' 21 | 22 | def write(self, buf): 23 | for line in buf.rstrip().splitlines(): 24 | self.logger.log(self.level, line.rstrip()) 25 | 26 | def flush(self): 27 | pass 28 | 29 | 30 | class TqdmLoggingHandler(logging.Handler): 31 | def __init__(self, level=logging.NOTSET): 32 | super().__init__(level) 33 | 34 | def emit(self, record): 35 | try: 36 | msg = self.format(record) 37 | tqdm.tqdm.write(msg) 38 | self.flush() 39 | except Exception: 40 | self.handleError(record) 41 | -------------------------------------------------------------------------------- /mld/tools/runid.py: -------------------------------------------------------------------------------- 1 | # 2 | """ 3 | runid util. 4 | Taken from wandb.sdk.lib.runid 5 | """ 6 | 7 | import shortuuid # type: ignore 8 | 9 | 10 | def generate_id() -> str: 11 | # ~3t run ids (36**8) 12 | run_gen = shortuuid.ShortUUID(alphabet=list("0123456789abcdefghijklmnopqrstuvwxyz")) 13 | return run_gen.random(8) -------------------------------------------------------------------------------- /mld/transforms/__init__.py: -------------------------------------------------------------------------------- 1 | from .base import Transform 2 | from .smpl import SMPLTransform 3 | # from .xyz import XYZTransform 4 | -------------------------------------------------------------------------------- /mld/transforms/base.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass, fields 2 | 3 | 4 | class Transform: 5 | 6 | def collate(self, lst_datastruct): 7 | from mld.datasets.utils import collate_tensor_with_padding 8 | example = lst_datastruct[0] 9 | 10 | def collate_or_none(key): 11 | if example[key] is None: 12 | return None 13 | key_lst = [x[key] for x in lst_datastruct] 14 | return collate_tensor_with_padding(key_lst) 15 | 16 | kwargs = {key: collate_or_none(key) for key in example.datakeys} 17 | 18 | return self.Datastruct(**kwargs) 19 | 20 | 21 | # Inspired from SMPLX library 22 | # need to define "datakeys" and transforms 23 | @dataclass 24 | class Datastruct: 25 | 26 | def __getitem__(self, key): 27 | return getattr(self, key) 28 | 29 | def __setitem__(self, key, value): 30 | self.__dict__[key] = value 31 | 32 | def get(self, key, default=None): 33 | return getattr(self, key, default) 34 | 35 | def __iter__(self): 36 | return self.keys() 37 | 38 | def keys(self): 39 | keys = [t.name for t in fields(self)] 40 | return iter(keys) 41 | 42 | def values(self): 43 | values = [getattr(self, t.name) for t in fields(self)] 44 | return iter(values) 45 | 46 | def items(self): 47 | data = [(t.name, getattr(self, t.name)) for t in fields(self)] 48 | return iter(data) 49 | 50 | def to(self, *args, **kwargs): 51 | for key in self.datakeys: 52 | if self[key] is not None: 53 | self[key] = self[key].to(*args, **kwargs) 54 | return self 55 | 56 | @property 57 | def device(self): 58 | return self[self.datakeys[0]].device 59 | 60 | def detach(self): 61 | 62 | def detach_or_none(tensor): 63 | if tensor is not None: 64 | return tensor.detach() 65 | return None 66 | 67 | kwargs = {key: detach_or_none(self[key]) for key in self.datakeys} 68 | return self.transforms.Datastruct(**kwargs) 69 | -------------------------------------------------------------------------------- /mld/transforms/feats2smpl.py: -------------------------------------------------------------------------------- 1 | from os.path import join as pjoin 2 | 3 | import numpy as np 4 | import torch 5 | 6 | import mld.data.humanml.utils.paramUtil as paramUtil 7 | from mld.data.humanml.data.dataset import Text2MotionDatasetV2 8 | from mld.data.humanml.scripts.motion_process import recover_from_ric 9 | from mld.data.humanml.utils.plot_script import plot_3d_motion 10 | 11 | skeleton = paramUtil.t2m_kinematic_chain 12 | 13 | # convert humanML3d features to skeleton format for rendering 14 | # def feats2joints(motion, data_root = '../datasets/humanml3d'): 15 | # ''' 16 | # input: 263 features 17 | # output: 22 joints? 18 | # ''' 19 | # mean = torch.from_numpy(np.load(pjoin(data_root, 'Mean.npy'))) 20 | # std = torch.from_numpy(np.load(pjoin(data_root, 'Std.npy'))) 21 | 22 | # motion = motion * std + mean 23 | # motion_rec = recover_from_ric(motion, joints_num=22) 24 | # # motion_rec = motion_rec * 1.3 25 | # return motion_rec 26 | 27 | 28 | def main(): 29 | data_root = '../datasets/humanml3d' 30 | feastures_path = 'in.npy' 31 | animation_save_path = 'in.mp4' 32 | 33 | fps = 20 34 | mean = np.load(pjoin(data_root, 'Mean.npy')) 35 | std = np.load(pjoin(data_root, 'Std.npy')) 36 | 37 | motion = np.load(feastures_path) 38 | motion = motion * std + mean 39 | motion_rec = recover_from_ric(torch.tensor(motion), 22).cpu().numpy() 40 | # with open('in_22.npy', 'wb') as f: 41 | # np.save(f,motion_rec) 42 | motion_rec = motion_rec * 1.3 43 | plot_3d_motion(animation_save_path, motion_rec, title='input', fps=fps) 44 | 45 | 46 | if __name__ == '__main__': 47 | main() 48 | -------------------------------------------------------------------------------- /mld/transforms/identity.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | from torch import Tensor 3 | 4 | from .base import Datastruct, dataclass, Transform 5 | 6 | 7 | class IdentityTransform(Transform): 8 | def __init__(self, **kwargs): 9 | return 10 | 11 | def Datastruct(self, **kwargs): 12 | return IdentityDatastruct(**kwargs) 13 | 14 | def __repr__(self): 15 | return "IdentityTransform()" 16 | 17 | 18 | @dataclass 19 | class IdentityDatastruct(Datastruct): 20 | transforms: IdentityTransform 21 | 22 | features: Optional[Tensor] = None 23 | 24 | def __post_init__(self): 25 | self.datakeys = ["features"] 26 | 27 | def __len__(self): 28 | return len(self.rfeats) 29 | -------------------------------------------------------------------------------- /mld/transforms/joints2jfeats/__init__.py: -------------------------------------------------------------------------------- 1 | from .base import Joints2Jfeats 2 | from .rifke import Rifke 3 | -------------------------------------------------------------------------------- /mld/transforms/joints2jfeats/base.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | import torch 4 | from torch import Tensor, nn 5 | from pathlib import Path 6 | 7 | 8 | class Joints2Jfeats(nn.Module): 9 | def __init__(self, path: Optional[str] = None, 10 | normalization: bool = False, 11 | eps: float = 1e-12, 12 | **kwargs) -> None: 13 | if normalization and path is None: 14 | raise TypeError("You should provide a path if normalization is on.") 15 | 16 | super().__init__() 17 | self.normalization = normalization 18 | self.eps = eps 19 | 20 | if normalization: 21 | mean_path = Path(path) / "jfeats_mean.pt" 22 | std_path = Path(path) / "jfeats_std.pt" 23 | self.register_buffer('mean', torch.load(mean_path)) 24 | self.register_buffer('std', torch.load(std_path)) 25 | 26 | def normalize(self, features: Tensor) -> Tensor: 27 | if self.normalization: 28 | features = (features - self.mean)/(self.std + self.eps) 29 | return features 30 | 31 | def unnormalize(self, features: Tensor) -> Tensor: 32 | if self.normalization: 33 | features = features * self.std + self.mean 34 | return features 35 | -------------------------------------------------------------------------------- /mld/transforms/rots2joints/__init__.py: -------------------------------------------------------------------------------- 1 | from .base import Rots2Joints 2 | from .smplh import SMPLH 3 | -------------------------------------------------------------------------------- /mld/transforms/rots2joints/base.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | import torch 4 | from torch import Tensor, nn 5 | from pathlib import Path 6 | 7 | 8 | class Rots2Joints(nn.Module): 9 | def __init__(self, path: Optional[str] = None, 10 | normalization: bool = False, 11 | eps: float = 1e-12, 12 | **kwargs) -> None: 13 | if normalization and path is None: 14 | raise TypeError("You should provide a path if normalization is on.") 15 | 16 | super().__init__() 17 | self.normalization = normalization 18 | self.eps = eps 19 | 20 | if normalization: 21 | mean_path = Path(path) / "mean.pt" 22 | std_path = Path(path) / "std.pt" 23 | self.register_buffer('mean', torch.load(mean_path)) 24 | self.register_buffer('std', torch.load(std_path)) 25 | 26 | def normalize(self, features: Tensor) -> Tensor: 27 | if self.normalization: 28 | features = (features - self.mean)/(self.std + self.eps) 29 | return features 30 | 31 | def unnormalize(self, features: Tensor) -> Tensor: 32 | if self.normalization: 33 | features = features * self.std + self.mean 34 | return features 35 | -------------------------------------------------------------------------------- /mld/transforms/rots2rfeats/__init__.py: -------------------------------------------------------------------------------- 1 | from .base import Rots2Rfeats 2 | from .smplvelp import SMPLVelP 3 | -------------------------------------------------------------------------------- /mld/transforms/rots2rfeats/base.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | import torch 4 | from torch import Tensor, nn 5 | from pathlib import Path 6 | 7 | 8 | class Rots2Rfeats(nn.Module): 9 | def __init__(self, path: Optional[str] = None, 10 | normalization: bool = False, 11 | eps: float = 1e-12, 12 | **kwargs) -> None: 13 | if normalization and path is None: 14 | raise TypeError("You should provide a path if normalization is on.") 15 | 16 | super().__init__() 17 | self.normalization = normalization 18 | self.eps = eps 19 | 20 | if normalization: 21 | mean_path = Path(path) / "rfeats_mean.pt" 22 | std_path = Path(path) / "rfeats_std.pt" 23 | self.register_buffer('mean', torch.load(mean_path)) 24 | self.register_buffer('std', torch.load(std_path)) 25 | 26 | def normalize(self, features: Tensor) -> Tensor: 27 | if self.normalization: 28 | features = (features - self.mean)/(self.std + self.eps) 29 | return features 30 | 31 | def unnormalize(self, features: Tensor) -> Tensor: 32 | if self.normalization: 33 | features = features * self.std + self.mean 34 | return features 35 | -------------------------------------------------------------------------------- /mld/transforms/xyz.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | from torch import Tensor 3 | 4 | from .base import Datastruct, dataclass, Transform 5 | from mld.datasets.utils import collate_tensor_with_padding 6 | 7 | from .joints2jfeats import Joints2Jfeats 8 | 9 | 10 | class XYZTransform(Transform): 11 | 12 | def __init__(self, joints2jfeats: Joints2Jfeats, **kwargs): 13 | self.joints2jfeats = joints2jfeats 14 | 15 | def Datastruct(self, **kwargs): 16 | return XYZDatastruct(_joints2jfeats=self.joints2jfeats, 17 | transforms=self, 18 | **kwargs) 19 | 20 | def __repr__(self): 21 | return "XYZTransform()" 22 | 23 | 24 | @dataclass 25 | class XYZDatastruct(Datastruct): 26 | transforms: XYZTransform 27 | _joints2jfeats: Joints2Jfeats 28 | 29 | features: Optional[Tensor] = None 30 | joints_: Optional[Tensor] = None 31 | jfeats_: Optional[Tensor] = None 32 | 33 | def __post_init__(self): 34 | self.datakeys = ["features", "joints_", "jfeats_"] 35 | # starting point 36 | if self.features is not None and self.jfeats_ is None: 37 | self.jfeats_ = self.features 38 | 39 | @property 40 | def joints(self): 41 | # Cached value 42 | if self.joints_ is not None: 43 | return self.joints_ 44 | 45 | # self.jfeats_ should be defined 46 | assert self.jfeats_ is not None 47 | 48 | self._joints2jfeats.to(self.jfeats.device) 49 | self.joints_ = self._joints2jfeats.inverse(self.jfeats) 50 | return self.joints_ 51 | 52 | @property 53 | def jfeats(self): 54 | # Cached value 55 | if self.jfeats_ is not None: 56 | return self.jfeats_ 57 | 58 | # self.joints_ should be defined 59 | assert self.joints_ is not None 60 | 61 | self._joints2jfeats.to(self.joints.device) 62 | self.jfeats_ = self._joints2jfeats(self.joints) 63 | return self.jfeats_ 64 | 65 | def __len__(self): 66 | return len(self.jfeats) 67 | -------------------------------------------------------------------------------- /mld/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpthu17/GraphMotion/d56feba6f6d39867b933ca752244fcbbb703a4b8/mld/utils/__init__.py -------------------------------------------------------------------------------- /mld/utils/demo_utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | from pathlib import Path 3 | 4 | 5 | # load example data 6 | def load_example_input(txt_path): 7 | file = open(txt_path, "r") 8 | Lines = file.readlines() 9 | count = 0 10 | texts, lens = [], [] 11 | # Strips the newline character 12 | for line in Lines: 13 | count += 1 14 | s = line.strip() 15 | s_l = s.split(" ")[0] 16 | s_t = s[(len(s_l) + 1):] 17 | lens.append(int(s_l)) 18 | texts.append(s_t) 19 | print("Length-{}: {}".format(s_l, s_t)) 20 | return texts, lens 21 | 22 | 23 | # render batch 24 | def render_batch(npy_dir, execute_python="./scripts/visualize_motion.sh", mode="sequence"): 25 | os.system(f"{execute_python} {npy_dir} {mode}") 26 | 27 | 28 | # render 29 | def render(execute_python, npy_path, jointtype, cfg_path): 30 | # execute_python = "/apdcephfs/share_1227775/shingxchen/libs/blender_bpy/blender-2.93.2-linux-x64/blender" 31 | # execute_python = "/apdcephfs/share_1227775/mingzhenzhu/jiangbiao/libs/blender-2.93.2-linux-x64/blender" 32 | export_scripts = "render.py" 33 | 34 | os.system( 35 | f"{execute_python} --background --python {export_scripts} -- --cfg={cfg_path} --npy={npy_path} --joint_type={jointtype}" 36 | ) 37 | 38 | fig_path = Path(str(npy_path).replace(".npy", ".png")) 39 | return fig_path 40 | 41 | 42 | # origin render 43 | # def render(npy_path, jointtype): 44 | # execute_python = '/apdcephfs/share_1227775/shingxchen/libs/blender_bpy/blender-2.93.2-linux-x64/blender' 45 | # export_scripts = 'render.py' 46 | 47 | # os.system(f"{execute_python} --background --python {export_scripts} -- npy={npy_path} jointstype={jointtype}") 48 | 49 | # fig_path = Path(str(npy_path).replace(".npy",".png")) 50 | # return fig_path 51 | 52 | # export fbx with hand params from pkl files 53 | # refer to /apdcephfs/share_1227775/shingxchen/AIMotion/TMOST/scripts/fbx_output_smplx.py 54 | def export_fbx_hand(pkl_path): 55 | input = pkl_path 56 | output = pkl_path.replace(".pkl", ".fbx") 57 | 58 | execute_python = "/apdcephfs/share_1227775/shingxchen/libs/blender_bpy/blender-2.93.2-linux-x64/blender" 59 | export_scripts = "./scripts/fbx_output_smplx.py" 60 | os.system( 61 | f"{execute_python} -noaudio --background --python {export_scripts}\ 62 | --input {input} \ 63 | --output {output}" 64 | ) 65 | 66 | 67 | # export fbx without hand params from pkl files 68 | # refer to /apdcephfs/share_1227775/shingxchen/AIMotion/TMOST/scripts/fbx_output.py 69 | def export_fbx(pkl_path): 70 | input = pkl_path 71 | output = pkl_path.replace(".pkl", ".fbx") 72 | 73 | execute_python = "/apdcephfs/share_1227775/shingxchen/libs/blender_bpy/blender-2.93.2-linux-x64/blender" 74 | export_scripts = "./scripts/fbx_output.py" 75 | os.system( 76 | f"{execute_python} -noaudio --background --python {export_scripts}\ 77 | --input {input} \ 78 | --output {output}" 79 | ) 80 | -------------------------------------------------------------------------------- /mld/utils/easyconvert.py: -------------------------------------------------------------------------------- 1 | import mld.utils.geometry as geometry 2 | 3 | 4 | def nfeats_of(rottype): 5 | if rottype in ["rotvec", "axisangle"]: 6 | return 3 7 | elif rottype in ["rotquat", "quaternion"]: 8 | return 4 9 | elif rottype in ["rot6d", "6drot", "rotation6d"]: 10 | return 6 11 | elif rottype in ["rotmat"]: 12 | return 9 13 | else: 14 | return TypeError("This rotation type doesn't have features.") 15 | 16 | 17 | def axis_angle_to(newtype, rotations): 18 | if newtype in ["matrix"]: 19 | rotations = geometry.axis_angle_to_matrix(rotations) 20 | return rotations 21 | elif newtype in ["rotmat"]: 22 | rotations = geometry.axis_angle_to_matrix(rotations) 23 | rotations = matrix_to("rotmat", rotations) 24 | return rotations 25 | elif newtype in ["rot6d", "6drot", "rotation6d"]: 26 | rotations = geometry.axis_angle_to_matrix(rotations) 27 | rotations = matrix_to("rot6d", rotations) 28 | return rotations 29 | elif newtype in ["rotquat", "quaternion"]: 30 | rotations = geometry.axis_angle_to_quaternion(rotations) 31 | return rotations 32 | elif newtype in ["rotvec", "axisangle"]: 33 | return rotations 34 | else: 35 | raise NotImplementedError 36 | 37 | 38 | def matrix_to(newtype, rotations): 39 | if newtype in ["matrix"]: 40 | return rotations 41 | if newtype in ["rotmat"]: 42 | rotations = rotations.reshape((*rotations.shape[:-2], 9)) 43 | return rotations 44 | elif newtype in ["rot6d", "6drot", "rotation6d"]: 45 | rotations = geometry.matrix_to_rotation_6d(rotations) 46 | return rotations 47 | elif newtype in ["rotquat", "quaternion"]: 48 | rotations = geometry.matrix_to_quaternion(rotations) 49 | return rotations 50 | elif newtype in ["rotvec", "axisangle"]: 51 | rotations = geometry.matrix_to_axis_angle(rotations) 52 | return rotations 53 | else: 54 | raise NotImplementedError 55 | 56 | 57 | def to_matrix(oldtype, rotations): 58 | if oldtype in ["matrix"]: 59 | return rotations 60 | if oldtype in ["rotmat"]: 61 | rotations = rotations.reshape((*rotations.shape[:-2], 3, 3)) 62 | return rotations 63 | elif oldtype in ["rot6d", "6drot", "rotation6d"]: 64 | rotations = geometry.rotation_6d_to_matrix(rotations) 65 | return rotations 66 | elif oldtype in ["rotquat", "quaternion"]: 67 | rotations = geometry.quaternion_to_matrix(rotations) 68 | return rotations 69 | elif oldtype in ["rotvec", "axisangle"]: 70 | rotations = geometry.axis_angle_to_matrix(rotations) 71 | return rotations 72 | else: 73 | raise NotImplementedError 74 | -------------------------------------------------------------------------------- /mld/utils/fixseed.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | import random 4 | 5 | 6 | def fixseed(seed): 7 | random.seed(seed) 8 | np.random.seed(seed) 9 | torch.manual_seed(seed) 10 | 11 | 12 | SEED = 10 13 | EVALSEED = 0 14 | # Provoc warning: not fully functionnal yet 15 | # torch.set_deterministic(True) 16 | torch.backends.cudnn.benchmark = False 17 | 18 | fixseed(SEED) 19 | -------------------------------------------------------------------------------- /mld/utils/logger.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | import os 3 | import time 4 | import logging 5 | from omegaconf import OmegaConf 6 | from pytorch_lightning.utilities.rank_zero import rank_zero_only 7 | 8 | 9 | def create_logger(cfg, phase='train'): 10 | # root dir set by cfg 11 | root_output_dir = Path(cfg.FOLDER) 12 | # set up logger 13 | if not root_output_dir.exists(): 14 | print('=> creating {}'.format(root_output_dir)) 15 | root_output_dir.mkdir() 16 | 17 | cfg_name = cfg.NAME 18 | model = cfg.model.model_type 19 | cfg_name = os.path.basename(cfg_name).split('.')[0] 20 | 21 | final_output_dir = root_output_dir / model / cfg_name 22 | cfg.FOLDER_EXP = str(final_output_dir) 23 | 24 | time_str = time.strftime('%Y-%m-%d-%H-%M-%S') 25 | 26 | new_dir(cfg, phase, time_str, final_output_dir) 27 | 28 | head = '%(asctime)-15s %(message)s' 29 | logger = config_logger(final_output_dir, time_str, phase, head) 30 | if logger is None: 31 | logger = logging.getLogger() 32 | logger.setLevel(logging.CRITICAL) 33 | logging.basicConfig(format=head) 34 | return logger 35 | 36 | 37 | @rank_zero_only 38 | def config_logger(final_output_dir, time_str, phase, head): 39 | log_file = '{}_{}_{}.log'.format('log', time_str, phase) 40 | final_log_file = final_output_dir / log_file 41 | logging.basicConfig(filename=str(final_log_file)) 42 | logger = logging.getLogger() 43 | logger.setLevel(logging.INFO) 44 | console = logging.StreamHandler() 45 | formatter = logging.Formatter(head) 46 | console.setFormatter(formatter) 47 | logging.getLogger('').addHandler(console) 48 | file_handler = logging.FileHandler(final_log_file, 'w') 49 | file_handler.setFormatter(logging.Formatter(head)) 50 | file_handler.setLevel(logging.INFO) 51 | logging.getLogger('').addHandler(file_handler) 52 | return logger 53 | 54 | 55 | @rank_zero_only 56 | def new_dir(cfg, phase, time_str, final_output_dir): 57 | # new experiment folder 58 | cfg.TIME = str(time_str) 59 | if os.path.exists( 60 | final_output_dir) and cfg.TRAIN.RESUME is None and not cfg.DEBUG: 61 | file_list = sorted(os.listdir(final_output_dir), reverse=True) 62 | for item in file_list: 63 | if item.endswith('.log'): 64 | os.rename(str(final_output_dir), 65 | str(final_output_dir) + '_' + cfg.TIME) 66 | break 67 | final_output_dir.mkdir(parents=True, exist_ok=True) 68 | # write config yaml 69 | config_file = '{}_{}_{}.yaml'.format('config', time_str, phase) 70 | final_config_file = final_output_dir / config_file 71 | OmegaConf.save(config=cfg, f=final_config_file) 72 | -------------------------------------------------------------------------------- /mld/utils/misc.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | 4 | def to_numpy(tensor): 5 | if torch.is_tensor(tensor): 6 | return tensor.cpu().numpy() 7 | elif type(tensor).__module__ != 'numpy': 8 | raise ValueError("Cannot convert {} to numpy array".format( 9 | type(tensor))) 10 | return tensor 11 | 12 | 13 | def to_torch(ndarray): 14 | if type(ndarray).__module__ == 'numpy': 15 | return torch.from_numpy(ndarray) 16 | elif not torch.is_tensor(ndarray): 17 | raise ValueError("Cannot convert {} to torch tensor".format( 18 | type(ndarray))) 19 | return ndarray 20 | 21 | 22 | def cleanexit(): 23 | import sys 24 | import os 25 | try: 26 | sys.exit(0) 27 | except SystemExit: 28 | os._exit(0) 29 | 30 | -------------------------------------------------------------------------------- /mld/utils/sample_utils.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from pathlib import Path 3 | logger = logging.getLogger(__name__) 4 | 5 | def cfg_mean_nsamples_resolution(cfg): 6 | if cfg.mean and cfg.number_of_samples > 1: 7 | logger.error("All the samples will be the mean.. cfg.number_of_samples=1 will be forced.") 8 | cfg.number_of_samples = 1 9 | 10 | return cfg.number_of_samples == 1 11 | 12 | 13 | def get_path(sample_path: Path, is_amass: bool, gender: str, split: str, onesample: bool, mean: bool, fact: float): 14 | extra_str = ("_mean" if mean else "") if onesample else "_multi" 15 | fact_str = "" if fact == 1 else f"{fact}_" 16 | gender_str = gender + "_" if is_amass else "" 17 | path = sample_path / f"{fact_str}{gender_str}{split}{extra_str}" 18 | return path 19 | -------------------------------------------------------------------------------- /mld/utils/tensors.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | 4 | def lengths_to_mask(lengths): 5 | max_len = max(lengths) 6 | mask = torch.arange(max_len, device=lengths.device).expand( 7 | len(lengths), max_len) < lengths.unsqueeze(1) 8 | return mask 9 | 10 | 11 | def collate_tensors(batch): 12 | dims = batch[0].dim() 13 | max_size = [max([b.size(i) for b in batch]) for i in range(dims)] 14 | size = (len(batch),) + tuple(max_size) 15 | canvas = batch[0].new_zeros(size=size) 16 | for i, b in enumerate(batch): 17 | sub_tensor = canvas[i] 18 | for d in range(dims): 19 | sub_tensor = sub_tensor.narrow(d, 0, b.size(d)) 20 | sub_tensor.add_(b) 21 | return canvas 22 | 23 | 24 | def collate(batch): 25 | databatch = [b[0] for b in batch] 26 | labelbatch = [b[1] for b in batch] 27 | lenbatch = [len(b[0][0][0]) for b in batch] 28 | 29 | databatchTensor = collate_tensors(databatch) 30 | labelbatchTensor = torch.as_tensor(labelbatch) 31 | lenbatchTensor = torch.as_tensor(lenbatch) 32 | 33 | maskbatchTensor = lengths_to_mask(lenbatchTensor) 34 | # x - [bs, njoints, nfeats, lengths] 35 | # - nfeats, the representation of a joint 36 | # y - [bs] 37 | # mask - [bs, lengths] 38 | # lengths - [bs] 39 | batch = {"x": databatchTensor, "y": labelbatchTensor, 40 | "mask": maskbatchTensor, 'lengths': lenbatchTensor} 41 | return batch 42 | 43 | 44 | # slow version with padding 45 | def collate_data3d_slow(batch): 46 | batchTensor = {} 47 | for key in batch[0].keys(): 48 | databatch = [b[key] for b in batch] 49 | batchTensor[key] = collate_tensors(databatch) 50 | batch = batchTensor 51 | # theta - [bs, lengths, 85], theta shape (85,) 52 | # - (np.array([1., 0., 0.]), pose(72), shape(10)), axis=0) 53 | # kp_2d - [bs, lengths, njoints, nfeats], nfeats (x,y,weight) 54 | # kp_3d - [bs, lengths, njoints, nfeats], nfeats (x,y,z) 55 | # w_smpl - [bs, lengths] zeros 56 | # w_3d - [bs, lengths] zeros 57 | return batch 58 | 59 | def collate_data3d(batch): 60 | batchTensor = {} 61 | for key in batch[0].keys(): 62 | databatch = [b[key] for b in batch] 63 | if key == "paths": 64 | batchTensor[key] = databatch 65 | else: 66 | batchTensor[key] = torch.stack(databatch,axis=0) 67 | batch = batchTensor 68 | # theta - [bs, lengths, 85], theta shape (85,) 69 | # - (np.array([1., 0., 0.]), pose(72), shape(10)), axis=0) 70 | # kp_2d - [bs, lengths, njoints, nfeats], nfeats (x,y,weight) 71 | # kp_3d - [bs, lengths, njoints, nfeats], nfeats (x,y,z) 72 | # w_smpl - [bs, lengths] zeros 73 | # w_3d - [bs, lengths] zeros 74 | return batch 75 | -------------------------------------------------------------------------------- /pictures/fig0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpthu17/GraphMotion/d56feba6f6d39867b933ca752244fcbbb703a4b8/pictures/fig0.png -------------------------------------------------------------------------------- /pictures/fig1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpthu17/GraphMotion/d56feba6f6d39867b933ca752244fcbbb703a4b8/pictures/fig1.png -------------------------------------------------------------------------------- /pictures/fig2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpthu17/GraphMotion/d56feba6f6d39867b933ca752244fcbbb703a4b8/pictures/fig2.png -------------------------------------------------------------------------------- /pictures/fig3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpthu17/GraphMotion/d56feba6f6d39867b933ca752244fcbbb703a4b8/pictures/fig3.png -------------------------------------------------------------------------------- /pictures/fig4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpthu17/GraphMotion/d56feba6f6d39867b933ca752244fcbbb703a4b8/pictures/fig4.png -------------------------------------------------------------------------------- /pictures/fig5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jpthu17/GraphMotion/d56feba6f6d39867b933ca752244fcbbb703a4b8/pictures/fig5.png -------------------------------------------------------------------------------- /prepare/download_pretrained_models.sh: -------------------------------------------------------------------------------- 1 | mkdir -p checkpoints/ 2 | cd checkpoints/ 3 | echo -e "The pretrained models will stored in the 'checkpoints' folder\n" 4 | mkdir -p humanml3d_checkpoint/ 5 | cd humanml3d_checkpoint/ 6 | # humanml3d.ckpt 7 | gdown "https://drive.google.com/file/d/1FNT3JpcjMhHDUh0U5LuF3sINk1cynOZ8" 8 | 9 | echo -e "Downloading done!" 10 | -------------------------------------------------------------------------------- /prepare/download_smpl_model.sh: -------------------------------------------------------------------------------- 1 | mkdir -p deps/ 2 | cd deps/ 3 | 4 | echo "The smpl model will be stored in the './deps' folder" 5 | 6 | # HumanAct12 poses 7 | echo "Downloading" 8 | gdown "https://drive.google.com/uc?id=1qrFkPZyRwRGd0Q3EY76K8oJaIgs_WK9i" 9 | echo "Extracting" 10 | tar xfzv smpl.tar.gz 11 | echo "Cleaning" 12 | rm smpl.tar.gz 13 | 14 | echo "Downloading done!" 15 | -------------------------------------------------------------------------------- /prepare/download_t2m_evaluators.sh: -------------------------------------------------------------------------------- 1 | mkdir -p deps/ 2 | cd deps/ 3 | 4 | echo "The t2m evaluators will be stored in the './deps' folder" 5 | 6 | # HumanAct12 poses 7 | echo "Downloading" 8 | gdown "https://drive.google.com/uc?id=1AYsmEG8I3fAAoraT4vau0GnesWBWyeT8" 9 | echo "Extracting" 10 | tar xfzv t2m.tar.gz 11 | echo "Cleaning" 12 | rm t2m.tar.gz 13 | 14 | echo "Downloading done!" 15 | -------------------------------------------------------------------------------- /prepare/prepare_bert.sh: -------------------------------------------------------------------------------- 1 | cd deps/ 2 | # yum -y install git-lfs 3 | git lfs install 4 | git clone https://huggingface.co/distilbert-base-uncased 5 | cd .. 6 | -------------------------------------------------------------------------------- /prepare/prepare_clip.sh: -------------------------------------------------------------------------------- 1 | cd deps/ 2 | git lfs install 3 | git clone https://huggingface.co/openai/clip-vit-large-patch14 4 | cd .. 5 | -------------------------------------------------------------------------------- /prepare/prepare_kit.sh: -------------------------------------------------------------------------------- 1 | # down kit https://motion-annotation.humanoids.kit.edu/dataset/ 2 | # refer to https://github.com/anindita127/Complextext2animation 3 | 4 | # use kit_dataset_path = /apdcephfs/share_1227775/shingxchen/datasets/KIT/KIT-ML-Default 5 | python src/data.py -- xx 6 | 7 | python dataProcessing/meanVariance.py -mask '[0]' -feats_kind rifke -dataset KITMocap -path2data /apdcephfs/share_1227775/shingxchen/datasets/KIT/KIT-ML-Default -f_new 8 -------------------------------------------------------------------------------- /prepare/requirements_render.txt: -------------------------------------------------------------------------------- 1 | # for rendering in blender python 2 | pytest-shutil 3 | matplotlib 4 | tqdm 5 | hydra-core 6 | six 7 | -------------------------------------------------------------------------------- /prepare/smplh.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo "Extraction of the archives" 3 | echo 4 | 5 | cd deps/smplh 6 | mkdir tmp 7 | cd tmp 8 | 9 | tar xfv ../smplh.tar.xz 10 | unzip ../mano_v1_2.zip 11 | 12 | cd ../../../ 13 | echo 14 | echo "Done!" 15 | echo 16 | echo "Clean and merge models" 17 | echo 18 | 19 | python prepare/merge_smplh_mano.py --smplh-fn deps/smplh/tmp/male/model.npz --mano-left-fn deps/smplh/tmp/mano_v1_2/models/MANO_LEFT.pkl --mano-right-fn deps/smplh/tmp/mano_v1_2/models/MANO_RIGHT.pkl --output-folder deps/smplh/ 20 | 21 | python prepare/merge_smplh_mano.py --smplh-fn deps/smplh/tmp/female/model.npz --mano-left-fn deps/smplh/tmp/mano_v1_2/models/MANO_LEFT.pkl --mano-right-fn deps/smplh/tmp/mano_v1_2/models/MANO_RIGHT.pkl --output-folder deps/smplh/ 22 | 23 | python prepare/merge_smplh_mano.py --smplh-fn deps/smplh/tmp/neutral/model.npz --mano-left-fn deps/smplh/tmp/mano_v1_2/models/MANO_LEFT.pkl --mano-right-fn deps/smplh/tmp/mano_v1_2/models/MANO_RIGHT.pkl --output-folder deps/smplh/ 24 | 25 | echo 26 | echo "Done!" 27 | echo 28 | echo "Deleting tmp files" 29 | rm -rf deps/smplh/tmp/ 30 | echo 31 | echo "Done!" 32 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pytorch_lightning==2.0.0 2 | torchmetrics==0.7 3 | omegaconf 4 | shortuuid 5 | tqdm 6 | pandas 7 | # sklearn 8 | chumpy 9 | transformers 10 | psutil 11 | einops 12 | yacs 13 | wandb 14 | rich 15 | # for fitting 16 | smplx==0.1.28 17 | trimesh==3.9.24 18 | # pyyaml==5.4.1 19 | h5py 20 | joblib==1.2.0 21 | scikit-image 22 | spacy 23 | diffusers==0.14.0 24 | ftfy 25 | tensorboard 26 | -------------------------------------------------------------------------------- /scripts/fit_motion.sh: -------------------------------------------------------------------------------- 1 | python -m fit --dir $1 --save_folder $2 --cuda True -------------------------------------------------------------------------------- /scripts/fit_motion_parallel.sh: -------------------------------------------------------------------------------- 1 | # parallel render 2 | for i in `seq 0 7` 3 | do 4 | for j in `seq 0 2` 5 | do 6 | CUDA_VISIBLE_DEVICES=$i python -m fit --dir $1 --save_folder $2 --cuda True & 7 | echo $j & 8 | done 9 | done 10 | 11 | wait 12 | echo "all weakup" 13 | 14 | # # parallel render 15 | # for i in `seq 0 25` 16 | # do 17 | # CUDA_VISIBLE_DEVICES=$3 python -m fit --dir $1 --save_folder $2 --cuda True & 18 | # echo $i 19 | # done 20 | # wait 21 | # echo "all weakup" 22 | 23 | 24 | # # gpu parallel render 25 | # for i in `seq 0 7` 26 | # do 27 | # CUDA_VISIBLE_DEVICES=$i python -m fit --dir $1 --save_folder $2 --cuda True & 28 | # echo $i 29 | # done 30 | # wait 31 | # echo "all weakup" -------------------------------------------------------------------------------- /scripts/plys2npy.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | from argparse import ArgumentParser 4 | from pathlib import Path 5 | 6 | import natsort 7 | import numpy as np 8 | import torch 9 | import trimesh 10 | from tqdm import tqdm 11 | 12 | 13 | def main(): 14 | parser = ArgumentParser() 15 | 16 | group = parser.add_argument_group("Params") 17 | group.add_argument( 18 | "--ply_dir", 19 | type=str, 20 | required=True, 21 | help="ply set", 22 | ) 23 | group.add_argument( 24 | "--out_dir", 25 | type=str, 26 | required=True, 27 | help="output folder", 28 | ) 29 | params = parser.parse_args() 30 | plys2npy(params.ply_dir, params.out_dir) 31 | 32 | def plys2npy(ply_dir, out_dir): 33 | ply_dir = Path(ply_dir) 34 | paths = [] 35 | file_list = natsort.natsorted(os.listdir(ply_dir)) 36 | for item in file_list: 37 | if item.endswith(".ply") and not item.endswith("_gt.ply"): 38 | paths.append(os.path.join(ply_dir, item)) 39 | 40 | 41 | meshs = np.zeros((len(paths), 6890, 3)) 42 | for i, path in enumerate(paths): 43 | mesh = trimesh.load_mesh(path, process=False) 44 | vs = mesh.vertices 45 | assert vs.shape == (6890, 3) 46 | meshs[i] = vs 47 | 48 | basename = os.path.basename(ply_dir) 49 | if basename.startswith("SMPLFit_"): 50 | basename = basename[len("SMPLFit_"):] 51 | file_name = os.path.join(out_dir, basename+ "_mesh.npy") 52 | np.save(file_name, meshs) 53 | 54 | 55 | if __name__ == "__main__": 56 | main() 57 | -------------------------------------------------------------------------------- /scripts/visualize_motion.sh: -------------------------------------------------------------------------------- 1 | # # for npy folder 2 | for j in `seq 0 2` 3 | do 4 | CUDA_VISIBLE_DEVICES=0 /apdcephfs/share_1227775/shingxchen/libs/blender_bpy/blender-2.93.2-linux-x64/blender --background --python render.py -- --cfg=./configs/render.yaml --dir=$1 --mode=$2 --joint_type=HumanML3D 5 | done 6 | 7 | # for single npy 8 | # /apdcephfs/share_1227775/shingxchen/libs/blender_bpy/blender-2.93.2-linux-x64/blender --background --python render.py -- --cfg=./configs/render_cx.yaml --npy=$1 --joint_type=HumanML3D 9 | -------------------------------------------------------------------------------- /scripts/visualize_motion_parallel.sh: -------------------------------------------------------------------------------- 1 | # # parallel fit 2 | # for i in `seq 0 7` 3 | # do 4 | # for j in `seq 0 2` 5 | # do 6 | # CUDA_VISIBLE_DEVICES=$i python -m fit --dir $1 --save_folder $2 --cuda True & 7 | # echo $j & 8 | # done 9 | # done 10 | # wait 11 | # echo "all weakup" 12 | 13 | 14 | # parallel render 15 | for i in `seq 0 7` 16 | do 17 | for j in `seq 0 2` 18 | do 19 | CUDA_VISIBLE_DEVICES=$i /apdcephfs/share_1227775/shingxchen/libs/blender_bpy/blender-2.93.2-linux-x64/blender --background --python render.py -- --cfg=./configs/render.yaml --dir=$1 --mode=$2 --joint_type=HumanML3D & 20 | echo $i 21 | done 22 | done 23 | wait 24 | echo "all weakup" 25 | --------------------------------------------------------------------------------