├── emokine ├── __init__.py ├── kinematic │ ├── __init__.py │ └── kinematic_features.py ├── technical_validation.py ├── plotting.py ├── mvnx.py └── utils.py ├── assets ├── pld_example.gif ├── blender_mvnx.png ├── faceblur_example.jpg ├── kinematics_table.png ├── silhouette_example.jpg ├── blender_addon │ ├── io_anim_mvnx.zip │ └── README.md ├── techval_examples │ ├── foreground_stats.png │ ├── kinematics_single.png │ └── histograms_selection.png └── requirements.txt ├── 1a_mvnx_to_csv.py ├── .gitignore ├── 4a_techval_compute.py ├── 3a_kinematic_features.py ├── 2a_silhouettes.py ├── 2b_face_blurs.py ├── README.md ├── 4b_techval_plots.py ├── 1b_mvnx_blender.py └── LICENSE /emokine/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /assets/pld_example.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andres-fr/emokine/HEAD/assets/pld_example.gif -------------------------------------------------------------------------------- /assets/blender_mvnx.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andres-fr/emokine/HEAD/assets/blender_mvnx.png -------------------------------------------------------------------------------- /assets/faceblur_example.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andres-fr/emokine/HEAD/assets/faceblur_example.jpg -------------------------------------------------------------------------------- /assets/kinematics_table.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andres-fr/emokine/HEAD/assets/kinematics_table.png -------------------------------------------------------------------------------- /assets/silhouette_example.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andres-fr/emokine/HEAD/assets/silhouette_example.jpg -------------------------------------------------------------------------------- /assets/blender_addon/io_anim_mvnx.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andres-fr/emokine/HEAD/assets/blender_addon/io_anim_mvnx.zip -------------------------------------------------------------------------------- /assets/techval_examples/foreground_stats.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andres-fr/emokine/HEAD/assets/techval_examples/foreground_stats.png -------------------------------------------------------------------------------- /assets/techval_examples/kinematics_single.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andres-fr/emokine/HEAD/assets/techval_examples/kinematics_single.png -------------------------------------------------------------------------------- /assets/techval_examples/histograms_selection.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andres-fr/emokine/HEAD/assets/techval_examples/histograms_selection.png -------------------------------------------------------------------------------- /assets/blender_addon/README.md: -------------------------------------------------------------------------------- 1 | This is a snapshot of our Blender MVNX add-on. 2 | Full repository can be found at https://github.com/andres-fr/blender-mvnx-io 3 | -------------------------------------------------------------------------------- /1a_mvnx_to_csv.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | """ 6 | This module loads an MVNX file and extracts its data in tabular format for the 7 | specified fields. 8 | """ 9 | 10 | 11 | from pathlib import Path 12 | import os 13 | # for OmegaConf 14 | from dataclasses import dataclass 15 | from typing import Optional, List 16 | # 17 | from omegaconf import OmegaConf, MISSING 18 | # 19 | from emokine.mvnx import Mvnx, MvnxToTabular 20 | 21 | 22 | # ############################################################################## 23 | # # GLOBALS 24 | # ############################################################################## 25 | @dataclass 26 | class ConfDef: 27 | """ 28 | :cvar MVNX_PATH: Path to the MVNX file to be converted. 29 | :cvar OUT_DIR: Path to directory where results are saved. 30 | :cvar SCHEMA_PATH: Optional path to an MVNX schema to be tested against. 31 | :cvar FIELDS: List of parameters to be extracted like position, speed... 32 | see complete list in ``MvnxToTabular.ALLOWED_FIELDS``. If none given, 33 | all fields will be extracted 34 | """ 35 | MVNX_PATH: str = MISSING 36 | OUT_DIR: str = MISSING 37 | SCHEMA_PATH: Optional[str] = None 38 | FIELDS: Optional[List[str]] = None 39 | 40 | 41 | # ############################################################################## 42 | # # MAIN ROUTINE 43 | # ############################################################################## 44 | if __name__ == "__main__": 45 | CONF = OmegaConf.structured(ConfDef()) 46 | cli_conf = OmegaConf.from_cli() 47 | CONF = OmegaConf.merge(CONF, cli_conf) 48 | print("\n\nCONFIGURATION:") 49 | print(OmegaConf.to_yaml(CONF), end="\n\n\n") 50 | 51 | # load MVNX + extract tabular data for selected fields (all fields if None) 52 | m = Mvnx(CONF.MVNX_PATH, CONF.SCHEMA_PATH) 53 | processor = MvnxToTabular(m) 54 | dataframes = processor(CONF.FIELDS) 55 | 56 | # create out_dir if doesn't exist and save results 57 | Path(CONF.OUT_DIR).mkdir(parents=True, exist_ok=True) 58 | for df_name, df in dataframes.items(): 59 | outpath = os.path.join(CONF.OUT_DIR, df_name + ".csv") 60 | df.to_csv(outpath, index=False) 61 | print("saved dataframe to", outpath) 62 | -------------------------------------------------------------------------------- /emokine/kinematic/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | 3 | 4 | """ 5 | This module contains kinematic feature functionality that is also used 6 | elsewhere. 7 | """ 8 | 9 | 10 | import numpy as np 11 | from shapely.geometry import MultiPoint 12 | from scipy.spatial import ConvexHull 13 | 14 | 15 | # ############################################################################## 16 | # # MAD COMPUTATION 17 | # ############################################################################## 18 | def median_absdev(arr): 19 | """ 20 | Returns the MAD from a given array (dimensions will be flattened). 21 | MAD(x) is defined as the median(abs(x-median(x))) 22 | """ 23 | flat = arr.flatten() 24 | result = np.median(np.absolute(flat - np.median(flat))) 25 | return result 26 | 27 | 28 | # ############################################################################## 29 | # # CONVEX HULL COMPUTATION 30 | # ############################################################################## 31 | def get_2d_convex_hull(data_row): 32 | """ 33 | :param data_row: A 1D numpy array, expected to be a succession of 34 | x, y, z values. 35 | :returns: ``(area, ch)``, Where area of the 2D convex hull is formed 36 | by the x,y points, as a ratio, where 1 means covering the full area 37 | and 0 no area. The ``ch`` is the full ConvexHull object 38 | """ 39 | xvals = data_row[0::3] 40 | yvals = data_row[1::3] 41 | xyvals = np.stack([xvals, yvals]).T 42 | # ch = ConvexHull(xyvals) 43 | # area = ch.volume # for 2D, volume is actually surface 44 | mp = MultiPoint(xyvals) 45 | area = mp.convex_hull.area 46 | return area, mp 47 | 48 | 49 | def get_3d_convex_hull(data_row): 50 | """ 51 | :param data_row: A 1D numpy array, expected to be a succession of 52 | x, y, z values. 53 | :returns: ``(area, ch)``, Where area of the 2D convex hull is formed 54 | by the x,y points, as a ratio, where 1 means covering the full area 55 | and 0 no area. The ``ch`` is the full ConvexHull object 56 | """ 57 | xvals = data_row[0::3] 58 | yvals = data_row[1::3] 59 | zvals = data_row[2::3] 60 | xyzvals = np.stack([xvals, yvals, zvals]).T 61 | ch = ConvexHull(xyzvals) 62 | volume = ch.volume 63 | mp = MultiPoint(xyzvals) 64 | return volume, mp 65 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Repo-specific 2 | output/* 3 | EmokineDataset 4 | 5 | 6 | # Byte-compiled / optimized / DLL files 7 | __pycache__/ 8 | *.py[cod] 9 | *$py.class 10 | 11 | # C extensions 12 | *.so 13 | 14 | # Distribution / packaging 15 | .Python 16 | build/ 17 | develop-eggs/ 18 | dist/ 19 | downloads/ 20 | eggs/ 21 | .eggs/ 22 | lib/ 23 | lib64/ 24 | parts/ 25 | sdist/ 26 | var/ 27 | wheels/ 28 | pip-wheel-metadata/ 29 | share/python-wheels/ 30 | *.egg-info/ 31 | .installed.cfg 32 | *.egg 33 | MANIFEST 34 | 35 | # PyInstaller 36 | # Usually these files are written by a python script from a template 37 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 38 | *.manifest 39 | *.spec 40 | 41 | # Installer logs 42 | pip-log.txt 43 | pip-delete-this-directory.txt 44 | 45 | # Unit test / coverage reports 46 | htmlcov/ 47 | .tox/ 48 | .nox/ 49 | .coverage 50 | .coverage.* 51 | .cache 52 | nosetests.xml 53 | coverage.xml 54 | *.cover 55 | *.py,cover 56 | .hypothesis/ 57 | .pytest_cache/ 58 | 59 | # Translations 60 | *.mo 61 | *.pot 62 | 63 | # Django stuff: 64 | *.log 65 | local_settings.py 66 | db.sqlite3 67 | db.sqlite3-journal 68 | 69 | # Flask stuff: 70 | instance/ 71 | .webassets-cache 72 | 73 | # Scrapy stuff: 74 | .scrapy 75 | 76 | # Sphinx documentation 77 | docs/_build/ 78 | 79 | # PyBuilder 80 | target/ 81 | 82 | # Jupyter Notebook 83 | .ipynb_checkpoints 84 | 85 | # IPython 86 | profile_default/ 87 | ipython_config.py 88 | 89 | # pyenv 90 | .python-version 91 | 92 | # pipenv 93 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 94 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 95 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 96 | # install all needed dependencies. 97 | #Pipfile.lock 98 | 99 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 100 | __pypackages__/ 101 | 102 | # Celery stuff 103 | celerybeat-schedule 104 | celerybeat.pid 105 | 106 | # SageMath parsed files 107 | *.sage.py 108 | 109 | # Environments 110 | .env 111 | .venv 112 | env/ 113 | venv/ 114 | ENV/ 115 | env.bak/ 116 | venv.bak/ 117 | 118 | # Spyder project settings 119 | .spyderproject 120 | .spyproject 121 | 122 | # Rope project settings 123 | .ropeproject 124 | 125 | # mkdocs documentation 126 | /site 127 | 128 | # mypy 129 | .mypy_cache/ 130 | .dmypy.json 131 | dmypy.json 132 | 133 | # Pyre type checker 134 | .pyre/ 135 | -------------------------------------------------------------------------------- /emokine/technical_validation.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | """ 6 | This module contains reusable functionality to perform technical validation 7 | of EMOKINE-style datasets. 8 | """ 9 | 10 | 11 | from multiprocessing import Pool 12 | from itertools import repeat 13 | # 14 | import numpy as np 15 | from shapely.ops import unary_union # union of polygons 16 | import skimage 17 | # 18 | from .utils import load_bw_vid, load_bg_vid, JsonBlenderPositions 19 | from .kinematic import get_2d_convex_hull 20 | 21 | 22 | # ############################################################################## 23 | # # COMPUTE BRIGHTNESS RATIOS AND DENSITIES FOR BW VIDEOS 24 | # ############################################################################## 25 | def techval_bw_vid(path, ignore_below=0): 26 | """ 27 | Given a path to a video, assumed to be black-and-white, returns a pair 28 | ``(vid_ratio, histogram)``, where the former is the number of non-black 29 | pixels divided by the total pixels (across the whole video), and the 30 | latter is a single integer frame which, for each pixel, contains the 31 | number of times it was non-black across the whole sequence. 32 | 33 | :param ignore_below: Any frames with less than this many non-black pixels 34 | will be ignored. 35 | """ 36 | vid, vid_fps = load_bw_vid(path, ignore_below) 37 | vid_ratio = vid.sum() / vid.size 38 | histogram = vid.sum(axis=0) 39 | return vid_ratio, histogram 40 | 41 | 42 | def techval_bw_vid_multi(paths, num_processes=1, ignore_below=0): 43 | """ 44 | Multiprocessing version of ``techval_bw_vid``, accepts multiple paths and 45 | distributes them across multiple processes. 46 | 47 | See ``techval_bw_vid`` for parameter descriptions. 48 | """ 49 | with Pool(num_processes) as pool: 50 | ratios, hist = zip( 51 | *pool.starmap(techval_bw_vid, zip(paths, repeat(ignore_below)))) 52 | return ratios, hist 53 | 54 | 55 | # ############################################################################## 56 | # # COMPUTE BRIGHTNESS RATIOS AND DENSITIES FOR AVATAR VIDEOS 57 | # ############################################################################## 58 | def techval_avatar_vid(path, bg_rgb=(208, 240, 241), bg_rgb_margin=(1, 1, 1), 59 | ignore_below=100, ignore_above=100_000): 60 | """ 61 | Avatar videos have a flat background, but they are not black-and-white. 62 | This function converts them to binary by identifying the given color range 63 | as background (black/false) and everything else as foreground (white/true). 64 | 65 | :param path: Path to the avatar video. 66 | :param bg_rgb: RGB color of the background (uint8 triple). 67 | :param bg_rgb_margin: Any pixel of ``bg_rgb`` color, plus minus this range, 68 | will be considered background. Set to ``(0, 0, 0)`` for precise 69 | extraction. 70 | :param ignore_below: Any frames with less than this many foreground pixels 71 | will be ignored. 72 | :param ignore_above: Any frames with more than this many foreground pixels 73 | will be ignored. 74 | """ 75 | vid, vid_fps = load_bg_vid( 76 | path, bg_rgb, bg_rgb_margin, ignore_below, ignore_above) 77 | vid_ratio = vid.sum() / vid.size 78 | histogram = vid.sum(axis=0) 79 | return vid_ratio, histogram 80 | 81 | 82 | def techval_avatar_vid_multi(paths, bg_rgb=(208, 240, 241), 83 | bg_rgb_margin=(1, 1, 1), 84 | ignore_below=100, ignore_above=100_000, 85 | num_processes=1): 86 | """ 87 | Multiprocessing version of ``techval_avatar_vid``, accepts multiple paths 88 | and distributes them across multiple processes. 89 | 90 | See ``techval_avatar_vid`` for parameter descriptions. 91 | """ 92 | aa, bb = techval_avatar_vid(paths[0], bg_rgb, bg_rgb_margin) 93 | with Pool(num_processes) as pool: 94 | ratios, hist = zip(*pool.starmap(techval_avatar_vid, zip( 95 | paths, repeat(bg_rgb), repeat(bg_rgb_margin), 96 | repeat(ignore_below), repeat(ignore_above)))) 97 | return ratios, hist 98 | 99 | 100 | # ############################################################################## 101 | # # COMPUTE AREA RATIOS, DENSITIES AND BOUNDS FOR CAMPOS DATA 102 | # ############################################################################## 103 | def techval_campos(path, output_hw): 104 | """ 105 | :param path: Path to a CamPos JSON file as the ones present in ``EMOKINE`` 106 | (and/or the ones produced by the ``emokine`` scripts). 107 | :returns: A tuple ``(poly_ratio, hist, (min_x, min_y, max_x, max_y))``. 108 | 109 | The CamPos data presents the human keypoints in the sequence from the 110 | perspective of a given camera. This allows us to compute bounding 2D 111 | polygons, as well as the leftmost, rightmost... etc positions given that 112 | camera perspective. 113 | 114 | The returned values are respectively: The ratio of polygon surface divided 115 | by total surface, a single integer frame which, for each pixel, contains 116 | the number of times it was inside a polygon across the whole sequence, and 117 | the extremal values found in the whole sequence. 118 | 119 | See the ``TechVal`` plots in the ``EMOKINE`` dataset for an example of 120 | the outcome. 121 | """ 122 | out_h, out_w = output_hw 123 | jbp = JsonBlenderPositions(path, which_pos="sphere_pos") 124 | # 125 | polygons = [get_2d_convex_hull(jbp.data.iloc[i].array)[1].convex_hull 126 | for i in range(len(jbp.data))] 127 | seq_hull = unary_union(polygons) 128 | min_x, min_y, max_x, max_y = seq_hull.bounds 129 | # 130 | hist = np.zeros(output_hw, dtype=np.int64) 131 | for poly in polygons: 132 | xxx, yyy = poly.boundary.xy 133 | yyy, xxx = skimage.draw.polygon((np.array(yyy) * out_h).round(), 134 | (np.array(xxx) * out_w).round()) 135 | hist[yyy, xxx] += 1 136 | # 137 | poly_ratio = sum(p.area for p in polygons) / len(polygons) 138 | hist = hist[::-1] # flip vertical dimension to go from top to bottom 139 | return poly_ratio, hist, (min_x, min_y, max_x, max_y) 140 | 141 | 142 | def techval_campos_multi(paths, output_hw, num_processes=1): 143 | """ 144 | Multiprocessing version of ``techval_campos``, accepts multiple paths 145 | and distributes them across multiple processes. 146 | 147 | See ``techval_campos`` for parameter descriptions. 148 | """ 149 | with Pool(num_processes) as pool: 150 | ratios, hist, xyxy_bounds = zip( 151 | *pool.starmap(techval_campos, zip(paths, repeat(output_hw)))) 152 | return ratios, hist, xyxy_bounds 153 | -------------------------------------------------------------------------------- /assets/requirements.txt: -------------------------------------------------------------------------------- 1 | name: emokine 2 | channels: 3 | - pytorch 4 | - anaconda 5 | - conda-forge 6 | - defaults 7 | dependencies: 8 | - _libgcc_mutex=0.1=main 9 | - _openmp_mutex=5.1=1_gnu 10 | - alsa-lib=1.2.3.2=h166bdaf_0 11 | - antlr-python-runtime=4.9.3=pyhd8ed1ab_1 12 | - av=9.1.0=py39h8d29763_0 13 | - blas=1.0=mkl 14 | - blosc=1.21.1=hd32f23e_0 15 | - bottleneck=1.3.5=py39h7deecbd_0 16 | - brotli=1.0.9=h5eee18b_7 17 | - brotli-bin=1.0.9=h5eee18b_7 18 | - brunsli=0.1=h2531618_0 19 | - bzip2=1.0.8=h7f98852_4 20 | - c-ares=1.18.1=h7f8727e_0 21 | - ca-certificates=2023.01.10=h06a4308_0 22 | - certifi=2022.12.7=pyhd8ed1ab_0 23 | - cfitsio=3.470=h5893167_7 24 | - charls=2.2.0=h2531618_0 25 | - cudatoolkit=11.3.1=h9edb442_10 26 | - cycler=0.11.0=pyhd8ed1ab_0 27 | - cytoolz=0.12.0=py39h5eee18b_0 28 | - dask-core=2022.7.0=py39h06a4308_0 29 | - dbus=1.13.6=h48d8840_2 30 | - expat=2.4.8=h27087fc_0 31 | - ffmpeg=4.3.2=hca11adc_0 32 | - fontconfig=2.14.0=h8e229c2_0 33 | - freetype=2.10.4=h0708190_1 34 | - fsspec=2022.11.0=py39h06a4308_0 35 | - geos=3.10.2=h9c3ff4c_0 36 | - gettext=0.19.8.1=h0b5b191_1005 37 | - giflib=5.2.1=h5eee18b_1 38 | - glib=2.68.4=h9c3ff4c_0 39 | - glib-tools=2.68.4=h9c3ff4c_0 40 | - gmp=6.2.1=h58526e2_0 41 | - gnutls=3.6.13=h85f3911_1 42 | - gst-plugins-base=1.18.5=hf529b03_0 43 | - gstreamer=1.18.5=h76c114f_0 44 | - icu=68.2=h9c3ff4c_0 45 | - imagecodecs=2021.7.30=py39h44211f0_1 46 | - imageio=2.19.3=py39h06a4308_0 47 | - importlib_resources=5.12.0=pyhd8ed1ab_0 48 | - intel-openmp=2022.1.0=h9e868ea_3769 49 | - jbig=2.1=h7f98852_2003 50 | - jpeg=9e=h166bdaf_1 51 | - jxrlib=1.1=h7b6447c_2 52 | - krb5=1.19.4=h568e23c_0 53 | - lame=3.100=h7f98852_1001 54 | - lcms2=2.12=hddcbb42_0 55 | - ld_impl_linux-64=2.38=h1181459_1 56 | - lerc=2.2.1=h9c3ff4c_0 57 | - libaec=1.0.6=h9c3ff4c_0 58 | - libblas=3.9.0=16_linux64_mkl 59 | - libbrotlicommon=1.0.9=h5eee18b_7 60 | - libbrotlidec=1.0.9=h5eee18b_7 61 | - libbrotlienc=1.0.9=h5eee18b_7 62 | - libcblas=3.9.0=16_linux64_mkl 63 | - libclang=11.1.0=default_ha53f305_1 64 | - libcurl=7.87.0=h91b91d3_0 65 | - libdeflate=1.7=h7f98852_5 66 | - libedit=3.1.20221030=h5eee18b_0 67 | - libev=4.33=h7f8727e_1 68 | - libevent=2.1.10=h9b69904_4 69 | - libffi=3.3=he6710b0_2 70 | - libgcc-ng=11.2.0=h1234567_1 71 | - libgfortran-ng=12.2.0=h69a702a_19 72 | - libgfortran5=12.2.0=h337968e_19 73 | - libglib=2.68.4=h3e27bee_0 74 | - libgomp=11.2.0=h1234567_1 75 | - libiconv=1.17=h166bdaf_0 76 | - liblapack=3.9.0=16_linux64_mkl 77 | - libllvm11=11.1.0=hf817b99_2 78 | - libnghttp2=1.46.0=hce63b2e_0 79 | - libogg=1.3.4=h7f98852_1 80 | - libopenblas=0.3.20=pthreads_h78a6416_0 81 | - libopus=1.3.1=h7f98852_1 82 | - libpng=1.6.37=h21135ba_2 83 | - libpq=13.3=hd57d9b9_0 84 | - libssh2=1.10.0=h8f2d780_0 85 | - libstdcxx-ng=12.2.0=h46fd767_19 86 | - libtiff=4.3.0=hf544144_1 87 | - libuuid=2.32.1=h7f98852_1000 88 | - libuv=1.43.0=h7f98852_0 89 | - libvorbis=1.3.7=h9c3ff4c_0 90 | - libwebp=1.2.2=h55f646e_0 91 | - libwebp-base=1.2.2=h7f98852_1 92 | - libxcb=1.13=h7f98852_1004 93 | - libxkbcommon=1.0.3=he3ba5ed_0 94 | - libxml2=2.9.12=h72842e0_0 95 | - libxslt=1.1.33=h15afd5d_2 96 | - libzopfli=1.0.3=he6710b0_0 97 | - locket=1.0.0=py39h06a4308_0 98 | - lxml=4.8.0=py39hb9d737c_2 99 | - lz4-c=1.9.3=h9c3ff4c_1 100 | - matplotlib=3.7.1=py39hf3d152e_0 101 | - matplotlib-base=3.7.1=py39h417a72b_1 102 | - mkl=2022.1.0=hc2b9512_224 103 | - munkres=1.1.4=pyh9f0ad1d_0 104 | - mysql-common=8.0.25=ha770c72_2 105 | - mysql-libs=8.0.25=hfa10184_2 106 | - ncurses=6.4=h6a678d5_0 107 | - nettle=3.6=he412f7d_0 108 | - networkx=2.8.4=py39h06a4308_0 109 | - nomkl=2.0=0 110 | - nspr=4.32=h9c3ff4c_1 111 | - nss=3.69=hb5efdd6_1 112 | - numexpr=2.8.0=py39h194a79d_102 113 | - numpy=1.22.3=py39hc58783e_2 114 | - olefile=0.46=pyh9f0ad1d_1 115 | - omegaconf=2.3.0=pyhd8ed1ab_0 116 | - openh264=2.1.1=h780b84a_0 117 | - openjpeg=2.4.0=hb52868f_1 118 | - openssl=1.1.1s=h7f8727e_0 119 | - pandas=1.5.2=py39h417a72b_0 120 | - partd=1.2.0=pyhd3eb1b0_1 121 | - pcre=8.45=h9c3ff4c_0 122 | - pillow=9.3.0=py39h6a678d5_2 123 | - pip=23.0.1=py39h06a4308_0 124 | - pthread-stubs=0.4=h36c2ea0_1001 125 | - pyparsing=3.0.9=pyhd8ed1ab_0 126 | - pyqt=5.12.3=py39hf3d152e_8 127 | - pyqt-impl=5.12.3=py39hde8b62d_8 128 | - pyqt5-sip=4.19.18=py39he80948d_8 129 | - pyqtchart=5.12=py39h0fcd23e_8 130 | - pyqtwebengine=5.12.1=py39h0fcd23e_8 131 | - python=3.9.0=hdb3f193_2 132 | - python-dateutil=2.8.2=pyhd3eb1b0_0 133 | - python-wget=3.2=py_0 134 | - python_abi=3.9=2_cp39 135 | - pytorch=1.10.1=py3.9_cuda11.3_cudnn8.2.0_0 136 | - pytorch-mutex=1.0=cuda 137 | - pytz=2023.3=pyhd8ed1ab_0 138 | - pywavelets=1.4.1=py39h5eee18b_0 139 | - pyyaml=6.0=py39h5eee18b_1 140 | - qt=5.12.9=hda022c4_4 141 | - readline=8.2=h5eee18b_0 142 | - scikit-image=0.19.3=py39h6a678d5_1 143 | - scipy=1.8.1=py39he49c0e8_0 144 | - seaborn=0.12.2=py39h06a4308_0 145 | - setuptools=65.6.3=py39h06a4308_0 146 | - shapely=1.8.2=py39h73b9895_1 147 | - six=1.16.0=pyhd3eb1b0_1 148 | - snappy=1.1.9=h295c915_0 149 | - sqlite=3.41.1=h5eee18b_0 150 | - tifffile=2021.7.2=pyhd3eb1b0_2 151 | - tk=8.6.12=h1ccaba5_0 152 | - toolz=0.12.0=py39h06a4308_0 153 | - torchaudio=0.10.1=py39_cu113 154 | - torchvision=0.11.2=py39_cu113 155 | - tornado=6.1=py39hb9d737c_3 156 | - typing_extensions=4.5.0=pyha770c72_0 157 | - tzdata=2023c=h04d1e81_0 158 | - unicodedata2=14.0.0=py39hb9d737c_1 159 | - wheel=0.38.4=py39h06a4308_0 160 | - x264=1!161.3030=h7f98852_1 161 | - xorg-libxau=1.0.9=h7f98852_0 162 | - xorg-libxdmcp=1.1.3=h7f98852_0 163 | - xz=5.2.10=h5eee18b_1 164 | - yaml=0.2.5=h7b6447c_0 165 | - zfp=0.5.5=h295c915_6 166 | - zipp=3.15.0=pyhd8ed1ab_0 167 | - zlib=1.2.13=h5eee18b_0 168 | - zstd=1.5.0=ha95c52a_0 169 | - pip: 170 | - absl-py==1.4.0 171 | - appdirs==1.4.4 172 | - black==21.4b2 173 | - cachetools==5.3.0 174 | - charset-normalizer==3.1.0 175 | - click==8.1.3 176 | - cloudpickle==2.2.1 177 | - contourpy==1.0.7 178 | - detectron2==0.6+cu113 179 | - face-segmentation-pytorch==1.1.dev11+ge46087c 180 | - fonttools==4.39.3 181 | - future==0.18.3 182 | - fvcore==0.1.5.post20221221 183 | - google-auth==2.17.1 184 | - google-auth-oauthlib==1.0.0 185 | - grpcio==1.53.0 186 | - hydra-core==1.3.2 187 | - idna==3.4 188 | - importlib-metadata==6.1.0 189 | - iopath==0.1.9 190 | - kiwisolver==1.4.4 191 | - markdown==3.4.3 192 | - markupsafe==2.1.2 193 | - mypy-extensions==1.0.0 194 | - oauthlib==3.2.2 195 | - packaging==23.0 196 | - pathspec==0.11.1 197 | - portalocker==2.7.0 198 | - protobuf==4.22.1 199 | - pyasn1==0.4.8 200 | - pyasn1-modules==0.2.8 201 | - pycocotools==2.0.6 202 | - pydot==1.4.2 203 | - regex==2023.3.23 204 | - requests==2.28.2 205 | - requests-oauthlib==1.3.1 206 | - rsa==4.9 207 | - tabulate==0.9.0 208 | - tensorboard==2.12.1 209 | - tensorboard-data-server==0.7.0 210 | - tensorboard-plugin-wit==1.8.1 211 | - termcolor==2.2.0 212 | - toml==0.10.2 213 | - tqdm==4.65.0 214 | - urllib3==1.26.15 215 | - werkzeug==2.2.3 216 | - yacs==0.1.8 217 | -------------------------------------------------------------------------------- /emokine/plotting.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | """ 6 | Functionality to visually inspect 3D keypoint sequences, mostly for debugging 7 | and data inspection. 8 | """ 9 | 10 | 11 | import matplotlib.pyplot as plt 12 | import matplotlib.colors as mcolors 13 | # import matplotlib.animation as pla 14 | 15 | 16 | # ############################################################################# 17 | # # GLOBALS 18 | # ############################################################################# 19 | # ["aliceblue", "antiquewhite", "aqua", "aquamarine", "azure", 20 | # "beige", "bisque", "black", "blanchedalmond", "blue", "blueviolet", 21 | # "brown", "burlywood", "cadetblue", "chartreuse", "chocolate", 22 | # "coral", "cornflowerblue", "cornsilk", "crimson", "cyan", 23 | # "darkblue", "darkcyan", "darkgoldenrod", "darkgray", "darkgreen", 24 | # "darkgrey", "darkkhaki", "darkmagenta", "darkolivegreen", 25 | # "darkorange", "darkorchid", "darkred", "darksalmon", "darkseagreen", 26 | # "darkslateblue", "darkslategray", "darkslategrey", "darkturquoise", 27 | # "darkviolet", "deeppink", "deepskyblue", "dimgray", "dimgrey", 28 | # "dodgerblue", "firebrick", "floralwhite", "forestgreen", "fuchsia", 29 | # "gainsboro", "ghostwhite", "gold", "goldenrod", "gray", "green", 30 | # "greenyellow", "grey", "honeydew", "hotpink", "indianred", "indigo", 31 | # "ivory", "khaki", "lavender", "lavenderblush", "lawngreen", 32 | # "lemonchiffon", "lightblue", "lightcoral", "lightcyan", 33 | # "lightgoldenrodyellow", "lightgray", "lightgreen", "lightgrey", 34 | # "lightpink", "lightsalmon", "lightseagreen", "lightskyblue", 35 | # "lightslategray", "lightslategrey", "lightsteelblue", "lightyellow", 36 | # "lime", "limegreen", "linen", "magenta", "maroon", 37 | # "mediumaquamarine", "mediumblue", "mediumorchid", "mediumpurple", 38 | # "mediumseagreen", "mediumslateblue", "mediumspringgreen", 39 | # "mediumturquoise", "mediumvioletred", "midnightblue", "mintcream", 40 | # "mistyrose", "moccasin", "navajowhite", "navy", "oldlace", "olive", 41 | # "olivedrab", "orange", "orangered", "orchid", "palegoldenrod", 42 | # "palegreen", "paleturquoise", "palevioletred", "papayawhip", 43 | # "peachpuff", "peru", "pink", "plum", "powderblue", "purple", 44 | # "rebeccapurple", "red", "rosybrown", "royalblue", "saddlebrown", 45 | # "salmon", "sandybrown", "seagreen", "seashell", "sienna", "silver", 46 | # "skyblue", "slateblue", "slategray", "slategrey", "snow", 47 | # "springgreen", "steelblue", "tan", "teal", "thistle", "tomato", 48 | # "turquoise", "violet", "wheat", "white", "whitesmoke", "yellow", 49 | # "yellowgreen"] 50 | ALL_PLT_COLORS = {k.replace("tab:", ""): v # RGB colors 51 | for k, v in mcolors.CSS4_COLORS.items()} 52 | 53 | 54 | # pelvis to head 7 items. Then 4 each: right arm, left arm, right leg, left leg 55 | MVNX_COLORS = tuple(ALL_PLT_COLORS[k] for k in 56 | ["salmon", "indianred", "firebrick", "maroon", 57 | "lightgrey", "darkgrey", "black", 58 | "mediumpurple", "blueviolet", "darkviolet", "indigo", 59 | "lightsteelblue", "cornflowerblue", "royalblue", "navy", 60 | "cyan", "darkturquoise", "lightseagreen", "teal", 61 | # "khaki", "gold", "goldenrod", "darkgoldenrod", 62 | "greenyellow", "limegreen", "forestgreen", "olivedrab"]) 63 | 64 | PI = 3.141592653589793 65 | 66 | 67 | # ############################################################################# 68 | # # xx 69 | # ############################################################################# 70 | def plot_3d_pose(xyz_pos_values, colors=MVNX_COLORS, diameter=1, 71 | title="3D Pose", 72 | x_range_mm=(-2000, 2000), y_range_mm=(-2000, 2000), 73 | z_range_mm=(0, 4000)): 74 | """ 75 | :param xyz_pos_frames: A list in the form ``[frame1, frame2, ...]`` 76 | where each frame is a list in the form ``[(x1, y1, z1), ...]``, each 77 | xyz triple corresponding to the 3D position of a keypoint. All 78 | keypointsmust always be in the same order, frames must be ordered by 79 | time. 80 | 81 | 3D scatterplot of a specific pose. 82 | """ 83 | surface = PI * (diameter / 2) ** 2 84 | fig = plt.figure() 85 | ax = fig.add_subplot(projection="3d") 86 | xxx, yyy, zzz = zip(*xyz_pos_values) 87 | if colors is not None: 88 | assert len(colors) == len(xyz_pos_values), "#colors must match #(xyz)!" 89 | ax_scatter = ax.scatter( 90 | xxx, yyy, zzz, marker="o", depthshade=False, c=colors, s=surface) 91 | # 92 | ax.set_xlabel("X") 93 | ax.set_ylabel("Y") 94 | ax.set_zlabel("Z") 95 | ax_title = ax.set_title(title) 96 | # 97 | ax.set_xlim3d(x_range_mm) 98 | ax.set_ylim3d(y_range_mm) 99 | ax.set_zlim3d(z_range_mm) 100 | # 101 | return fig, ax, ax_scatter, ax_title 102 | 103 | 104 | class PoseAnimation3D: 105 | """ 106 | Functor providing 3D scatterplot frames in a way that can be animated by 107 | the PLT animation engine. See ``animation_3d_mvnx`` for an usage example. 108 | """ 109 | 110 | def __init__(self, xyz_pos_frames, scat, title): 111 | """ 112 | :param xyz_pos_frames: A list in the form ``[frame1, frame2, ...]`` 113 | where each frame is a list in the form ``[(x1, y1, z1), ...]``, each 114 | xyz triple corresponding to the 3D position of a keypoint. All 115 | keypointsmust always be in the same order, frames must be ordered by 116 | time. 117 | """ 118 | self.xyz_pos_frames = xyz_pos_frames 119 | self.scat = scat 120 | self.title = title 121 | self.ori_title = title.get_text() 122 | 123 | def __call__(self, frame_idx): 124 | """ 125 | """ 126 | # https://stackoverflow.com/a/41609238/4511978 127 | xxx, yyy, zzz = zip(*self.xyz_pos_frames[frame_idx]) 128 | self.scat._offsets3d[0][:] = xxx 129 | self.scat._offsets3d[1][:] = yyy 130 | self.scat._offsets3d[2][:] = zzz 131 | # 132 | self.title.set_text(self.ori_title + f" frame={frame_idx}") 133 | # 134 | return self.scat, self.title 135 | 136 | 137 | def animation_3d_mvnx(mvnx_pos_dataframe, colors=MVNX_COLORS, diameter=10, 138 | begin_frame=None, end_frame=None, 139 | skip_every=5, repeat=True): 140 | """ 141 | :param mvnx_pos_dataframe: A Pandas dataframe with columns like 142 | ``Pelvix_x, Pelvis_y, ...``. Use ``df.loc[:, "ori":"dest"]`` to 143 | slice columns 144 | 145 | Usage example: 146 | 147 | m = Mvnx(MVNX_PATH, SCHEMA_PATH) 148 | processor = MvnxToTabular(m) 149 | dataframes = processor(FIELDS) 150 | animation_3d_mvnx(dataframes["position"], MVNX_COLORS) 151 | """ 152 | xyz_pos_values = [ 153 | row.values.reshape(-1, 3) * 1000 for f_idx, row in 154 | mvnx_pos_dataframe.loc[:, "Pelvis_x":"LeftToe_z"].iterrows()] 155 | if begin_frame is None: 156 | begin_frame = 0 157 | if end_frame is None: 158 | end_frame = len(xyz_pos_values) 159 | # frame_range = range(begin_frame, end_frame, skip_every) 160 | fig, ax, ax_scat, ax_title = plot_3d_pose(xyz_pos_values[0], colors=colors, 161 | diameter=diameter, 162 | z_range_mm=(0, 2500)) 163 | # ani = pla.FuncAnimation( 164 | # fig, PoseAnimation3D(xyz_pos_values, ax_scat, ax_title), 165 | # frame_range, interval=1, repeat=repeat, blit=False) 166 | plt.show() 167 | -------------------------------------------------------------------------------- /4a_techval_compute.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding:utf-8 -*- 3 | 4 | 5 | """ 6 | Given the path to the ``EmokineDataset``, this script computes the data 7 | used to produce the technical validation plots, and saves it as a pickle file. 8 | Technical validation data is useful to explore and understand general 9 | properties of the dataset, like e.g. the distribution between foreground 10 | and background pixels. 11 | 12 | See the ``README``, and its companion script, ``4b_techval_plots.py``, for 13 | more details. 14 | """ 15 | 16 | 17 | import glob 18 | import os 19 | import pickle 20 | # for OmegaConf 21 | from dataclasses import dataclass 22 | from typing import Optional, Tuple 23 | # 24 | from omegaconf import OmegaConf, MISSING 25 | # 26 | from emokine.utils import load_bw_vid 27 | from emokine.technical_validation import techval_bw_vid_multi 28 | from emokine.technical_validation import techval_avatar_vid_multi 29 | from emokine.technical_validation import techval_campos_multi 30 | # 31 | # import matplotlib.pyplot as plt 32 | 33 | 34 | # ############################################################################## 35 | # # HELPERS 36 | # ############################################################################## 37 | def accept_condition(path, ignore_explanation=True): 38 | """ 39 | Return true if path is a file. If ``ignore_explanation`` is true, return 40 | false also for paths that contain the 'explanation' substring. 41 | """ 42 | result = os.path.isfile(path) 43 | if ignore_explanation and "explanation" in path: 44 | result = False 45 | return result 46 | 47 | 48 | # ############################################################################## 49 | # # CLI 50 | # ############################################################################## 51 | @dataclass 52 | class ConfDef: 53 | """ 54 | :cvar EMOKINE_PATH: Path to the EmokineDataset. 55 | :cvar OUTPUT_DIR: Where to store the computed techval data. 56 | :cvar OUTPUT_NAME: Name of the produced techval file, in pickle format. 57 | :cvar IGNORE_EXPLANATION: If true (default), ``explanation`` files in 58 | EmokineDataset won't be included in the technical validation. 59 | :cvar AVATAR_BG_RGB: RGB color value of the background in the Avatar 60 | stimuli, used to perform background extraction. 61 | :cvar AVATAR_BG_RGB_MARGIN: Margin around ``AVATAR_BG_RGB`` to consider a 62 | given color part of the background 63 | :cvar TRUNCATE: Useful for debugging, if given, only this many files will 64 | be processed. 65 | :cvar NUM_PROCESSES: Can be used to speed up processing in multi-core CPUs. 66 | """ 67 | EMOKINE_PATH: str = MISSING 68 | OUTPUT_DIR: str = MISSING 69 | OUTPUT_NAME: str = "techval.pickle" 70 | IGNORE_EXPLANATION: bool = True 71 | # 72 | AVATAR_BG_RGB: Tuple[int] = (208, 240, 241) 73 | AVATAR_BG_RGB_MARGIN: Tuple[int] = (0, 0, 0) 74 | # 75 | TRUNCATE: Optional[int] = None 76 | NUM_PROCESSES: int = 1 77 | 78 | 79 | # ############################################################################## 80 | # # MAIN ROUTINE 81 | # ############################################################################## 82 | if __name__ == "__main__": 83 | CONF = OmegaConf.structured(ConfDef()) 84 | cli_conf = OmegaConf.from_cli() 85 | CONF = OmegaConf.merge(CONF, cli_conf) 86 | print("\n\nCONFIGURATION:") 87 | print(OmegaConf.to_yaml(CONF), end="\n\n") 88 | 89 | # ########################################################################## 90 | # # GET PATHS 91 | # ########################################################################## 92 | sil_path = os.path.join(CONF.EMOKINE_PATH, "Stimuli", "Silhouette") 93 | avatar_path = os.path.join(CONF.EMOKINE_PATH, "Stimuli", "Avatar") 94 | pld_path = os.path.join(CONF.EMOKINE_PATH, "Stimuli", "PLD") 95 | cam_path = os.path.join(CONF.EMOKINE_PATH, "Data", "CamPos") 96 | kin_path = os.path.join(CONF.EMOKINE_PATH, "Data", "Kinematic") 97 | # 98 | sil_paths = [p for p in glob.glob(os.path.join(sil_path, "**", "*"), 99 | recursive=True) 100 | if accept_condition(p, CONF.IGNORE_EXPLANATION)] 101 | avatar_paths = [p for p in glob.glob(os.path.join(avatar_path, "**", "*"), 102 | recursive=True) 103 | if accept_condition(p, CONF.IGNORE_EXPLANATION)] 104 | pld_paths = [p for p in glob.glob(os.path.join(pld_path, "**", "*"), 105 | recursive=True) 106 | if accept_condition(p, CONF.IGNORE_EXPLANATION)] 107 | campos_paths = [p for p in glob.glob(os.path.join(cam_path, "**", "*"), 108 | recursive=True) 109 | if accept_condition(p, CONF.IGNORE_EXPLANATION)] 110 | kin_paths = [p for p in glob.glob(os.path.join(kin_path, "**", "*"), 111 | recursive=True) 112 | if accept_condition(p, CONF.IGNORE_EXPLANATION)] 113 | # 114 | sil_paths = sorted(sil_paths)[:CONF.TRUNCATE] 115 | avatar_paths = sorted(avatar_paths)[:CONF.TRUNCATE] 116 | pld_paths = sorted(pld_paths)[:CONF.TRUNCATE] 117 | campos_paths = sorted(campos_paths)[:CONF.TRUNCATE] 118 | kin_paths = sorted(kin_paths)[:CONF.TRUNCATE] 119 | 120 | # ########################################################################## 121 | # # SILHOUETTE/PLD/AVATAR 122 | # ########################################################################## 123 | print("Processing silhouette videos...") 124 | sil_ratios, sil_hists = techval_bw_vid_multi( 125 | sil_paths, ignore_below=0, 126 | num_processes=CONF.NUM_PROCESSES) 127 | print("Processing PLD videos...") 128 | pld_ratios, pld_hists = techval_bw_vid_multi( 129 | pld_paths, ignore_below=0, 130 | num_processes=CONF.NUM_PROCESSES) 131 | print("Processing Avatar videos...") 132 | avatar_ratios, avatar_hists = techval_avatar_vid_multi( 133 | avatar_paths, CONF.AVATAR_BG_RGB, CONF.AVATAR_BG_RGB_MARGIN, 134 | ignore_below=100, ignore_above=100_000, 135 | num_processes=CONF.NUM_PROCESSES) 136 | 137 | ########################################################################## 138 | # CAMPOS 139 | ########################################################################## 140 | campos_hw = load_bw_vid(sil_paths[0], ignore_below=0)[0][0].shape 141 | print("Processing CamPos files...") 142 | campos_ratios, campos_hists, campos_xyxy_bounds = techval_campos_multi( 143 | campos_paths, campos_hw, CONF.NUM_PROCESSES) 144 | min_x, min_y, max_x, max_y = zip(*campos_xyxy_bounds) 145 | 146 | ########################################################################## 147 | # SAVE RESULTS 148 | ########################################################################## 149 | techval_data = { 150 | "sil_paths": sil_paths, 151 | "avatar_paths": avatar_paths, 152 | "pld_paths": pld_paths, 153 | "campos_paths": campos_paths, 154 | "kin_paths": kin_paths, 155 | # 156 | "sil_ratios": sil_ratios, "sil_hists": sil_hists, 157 | "pld_ratios": pld_ratios, "pld_hists": pld_hists, 158 | "avatar_ratios": avatar_ratios, "avatar_hists": avatar_hists, 159 | "campos_ratios": campos_ratios, "campos_hists": campos_hists, 160 | "campos_xyxy_bounds": campos_xyxy_bounds} 161 | # 162 | outpath = os.path.join(CONF.OUTPUT_DIR, CONF.OUTPUT_NAME) 163 | # 164 | with open(outpath, "wb") as f: 165 | pickle.dump(techval_data, f) 166 | print("Saved result to", outpath) 167 | -------------------------------------------------------------------------------- /3a_kinematic_features.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding:utf-8 -*- 3 | 4 | 5 | """ 6 | Given a path to a silhouette video and its corresponding MVNX sequence and 7 | JSON CamPos data (as extracted by ``1b_mvnx_blender.py``, see README), this 8 | script extracts statistics for several kinematic features, informative of the 9 | type and quantity of movement. 10 | The kinematics are exported as a CSV file, with one row per keypoint and colum 11 | per feature. 12 | """ 13 | 14 | 15 | from pathlib import Path 16 | # for OmegaConf 17 | from dataclasses import dataclass 18 | from typing import Optional, Tuple 19 | # 20 | from omegaconf import OmegaConf, MISSING 21 | import numpy as np 22 | import pandas as pd 23 | from shapely.ops import unary_union # union of polygons 24 | # 25 | from emokine.mvnx import Mvnx, MvnxToTabular 26 | from emokine.utils import load_bw_vid, JsonBlenderPositions 27 | from emokine.kinematic import median_absdev 28 | from emokine.kinematic import get_2d_convex_hull, get_3d_convex_hull 29 | from emokine.kinematic.kinematic_features import quantity_of_motion 30 | from emokine.kinematic.kinematic_features import dimensionless_jerk, \ 31 | mvnx_3d_mean_max_mad_magnitudes, limb_contraction, cmass_distances, \ 32 | head_angle 33 | 34 | 35 | # ############################################################################## 36 | # # CLI 37 | # ############################################################################## 38 | @dataclass 39 | class ConfDef: 40 | """ 41 | :cvar SILHOUETTE_PATH: Path to b&w video expected to contain a moving 42 | silhouette. Expected to be consistent with the JSON and MVNX files. 43 | :cvar JSON_PATH: Path to a JSON file compatible with the 44 | ``emokine.utils.JsonBlenderPositions`` class, like the ones produces 45 | by the ``mvnx_blender`` script. The file contains a sequence of 3D 46 | positions represented from the perspective of a given camera. Expected 47 | to be consistent with the SILHOUETTE and MVNX files. 48 | :cvar MVNX_PATH: Path to a MVNX sequence, expected to be consistent with 49 | the SILHOUETTE and JSON files. 50 | :cvar CSV_OUTPATH: Output path to save the computed kinematic features 51 | as CSV. 52 | :cvar KEYPOINTS: For the keypoint-specific kinematic features, which 53 | MVNX keypoints to compute (e.g. left shoulder, left toe...). Default 54 | is the full body configuration by MVNX. 55 | :cvar QOM_SMI_SECONDS: When computing quantity of motions, a range of SMI 56 | frames from the past is used. This tells in seconds how many frames. 57 | :SIL_IGNORE_FRAMES_BELOW: When loading the silhouette video, any frames 58 | with less than this many nonzero pixels will be ignored. 59 | """ 60 | SILHOUETTE_PATH: str = MISSING 61 | JSON_PATH: str = MISSING 62 | MVNX_PATH: str = MISSING 63 | CSV_OUTPATH: str = MISSING 64 | KEYPOINTS: Tuple[str] = ( 65 | "Pelvis", "L5", "L3", "T12", "T8", "Neck", "Head", 66 | "RightShoulder", "RightUpperArm", "RightForeArm", "RightHand", 67 | "LeftShoulder", "LeftUpperArm", "LeftForeArm", "LeftHand", 68 | "RightUpperLeg", "RightLowerLeg", "RightFoot", "RightToe", 69 | "LeftUpperLeg", "LeftLowerLeg", "LeftFoot", "LeftToe") 70 | # 71 | MVNX_SCHEMA: Optional[str] = None 72 | # 73 | QOM_SMI_SECONDS: float = 0.2 # e.g. 0.2 is 5 frames at 25fps 74 | # Hack to avoid current artifacts in silhouette vid, shouldn't be needed in 75 | # the future. Any frames with less than this many nonzeros are ignored 76 | SIL_IGNORE_FRAMES_BELOW: float = 100 77 | 78 | 79 | # ############################################################################## 80 | # # MAIN FUNCTION 81 | # ############################################################################## 82 | def main(silhouette_path, json_path, mvnx_path, keypoints, 83 | qom_smi_seconds=0.2, 84 | mvnx_schema=None, ignore_frames_below=100, 85 | verbose=False): 86 | """ 87 | Convenience wrapper for the main routine, see docstring of the script's 88 | ConfDef for info on the input parameters. 89 | 90 | :returns: A pandas dataframe with one joint per row and one kinematic 91 | feature per column. 92 | """ 93 | # load silhouette video, JSON blender positions and MVNX 94 | sil, sil_fps = load_bw_vid( 95 | silhouette_path, ignore_below=ignore_frames_below) 96 | _, sil_h, sil_w = sil.shape 97 | sil_dur_s = float(len(sil)) / sil_fps 98 | # 99 | jbp = JsonBlenderPositions(json_path, which_pos="bone_head") 100 | jbp_dur_s = len(jbp.data) / jbp.fps 101 | # 102 | m = Mvnx(mvnx_path, mvnx_schema) 103 | mvnx_fps = int(m.mvnx.subject.attrib["frameRate"]) 104 | mvnx_dataframes = MvnxToTabular(m)() 105 | mvnx_len = len(next(iter(mvnx_dataframes.values()))) 106 | mvnx_dur_s = mvnx_len / mvnx_fps 107 | # 108 | print("\nSilhouette duration in seconds:", sil_dur_s) 109 | print("JSON duration in seconds:", jbp_dur_s) 110 | print("MVNX duration in seconds:", mvnx_dur_s) 111 | 112 | # # compute silhouette features ########################################### 113 | # quantity of motion 114 | qom, _, _ = quantity_of_motion( 115 | sil, smi_frames=round(qom_smi_seconds * sil_fps)) 116 | mean_qom = np.mean(qom) 117 | mad_qom = median_absdev(np.array(qom)) 118 | integral_qom = np.sum(qom) 119 | 120 | # # compute JSON features ################################################# 121 | # 2D convex hull: 122 | # per-frame, global hull, and union of per-frame hulls. 123 | # Union can't be greater than global, and it will be similar unless 124 | # many jumps, crouching or other vertical movements happen. 125 | ch2d, multipoints2d = zip(*[get_2d_convex_hull(jbp.data.iloc[i].array) 126 | for i in range(len(jbp.data))]) 127 | mean_ch2d = np.mean(ch2d) 128 | mad_ch2d = median_absdev(np.array(ch2d)) 129 | global_ch2d = get_2d_convex_hull(jbp.data.to_numpy().flatten())[0] 130 | union_ch2d = unary_union([x.convex_hull for x in multipoints2d]).area 131 | 132 | # # compute MVNX features ################################################# 133 | # 3D convex hull: analogous to 2D 134 | ch3d, multipoints3d = zip( 135 | *[get_3d_convex_hull(mvnx_dataframes["position"].iloc[i, 2:].array) 136 | for i in range(len(mvnx_dataframes["position"]))]) 137 | mean_ch3d = np.mean(ch3d) 138 | mad_ch3d = median_absdev(np.array(ch3d)) 139 | global_ch3d = get_3d_convex_hull( 140 | mvnx_dataframes["position"].iloc[:, 2:].to_numpy().flatten())[0] 141 | union_ch3d = unary_union([x.convex_hull for x in multipoints3d]).area 142 | 143 | # # compute MVNX features 144 | # built-in features 145 | mean_pos, max_pos, mad_pos = mvnx_3d_mean_max_mad_magnitudes( 146 | mvnx_dataframes["position"], keypoints) 147 | mean_vels, max_vels, mad_vels = mvnx_3d_mean_max_mad_magnitudes( 148 | mvnx_dataframes["velocity"], keypoints) 149 | mean_accels, max_accels, mad_accels = mvnx_3d_mean_max_mad_magnitudes( 150 | mvnx_dataframes["acceleration"], keypoints) 151 | (mean_ang_vels, max_ang_vels, 152 | mad_ang_vels) = mvnx_3d_mean_max_mad_magnitudes( 153 | mvnx_dataframes["angularVelocity"], keypoints) 154 | (mean_ang_accels, max_ang_accels, 155 | mad_ang_accels) = mvnx_3d_mean_max_mad_magnitudes( 156 | mvnx_dataframes["angularAcceleration"], keypoints) 157 | # dimensionless jerk 158 | dimensionless_jerks_vmean = dimensionless_jerk( 159 | mvnx_dataframes["velocity"], keypoints, srate=mvnx_fps) 160 | # limb contracton from Poyo Solanas (dist. between limbs and head) 161 | lcont = limb_contraction(mvnx_dataframes["position"]) 162 | lcont_t_mean = lcont.mean(axis=1) 163 | lcont_global_mean = lcont.mean() 164 | lcont_t_mad = median_absdev(lcont_t_mean) 165 | # our limb contraction: dist between each keypoint and center of mass 166 | cmass_dists = cmass_distances( 167 | mvnx_dataframes["position"], mvnx_dataframes["centerOfMass"], 168 | keypoints) 169 | cmass_dists_mean_per_kp = {k: v.mean() for k, v in cmass_dists.items()} 170 | cmass_dists_mad_per_kp = {k: median_absdev(v) for k, v 171 | in cmass_dists.items()} 172 | # neck-head angle: vert is global, rel is against the t8-neck line 173 | head_t_angle_vert, head_t_angle_rel = head_angle( 174 | mvnx_dataframes["position"]) 175 | mean_head_angle_vert = head_t_angle_vert.mean() 176 | mad_head_angle_vert = median_absdev(head_t_angle_vert) 177 | mean_head_angle_rel = head_t_angle_rel.mean() 178 | mad_head_angle_rel = median_absdev(head_t_angle_rel) 179 | 180 | # animation_3d_mvnx(mvnx_dataframes["position"]) 181 | # arr=mvnx_dataframes["position"]["LeftHand_z"]; plt.clf(); plt.plot(arr); plt.show() 182 | # plt.clf(); plt.plot(qom); plt.show() 183 | # plt.clf(); plt.plot(ch2d); plt.show() 184 | # plt.clf(); plt.axis("equal"); plt.scatter(xvals.to_numpy(), yvals.to_numpy()); plt.show() 185 | # plt.clf(); plt.plot(ch3d); plt.show() 186 | # arr = lcont_t_mean; plt.clf(); plt.plot(arr); plt.show() 187 | 188 | # Prepare CSV output 189 | columns = ["MVNX frame rate", "MVNX num frames", 190 | "silhouette frame rate", "silhouette num frames", 191 | # row_basic ends here 192 | "keypoint", 193 | "avg. vel.", "max. vel.", "MAD vel.", 194 | "avg. accel.", "max. accel.", "MAD accel.", 195 | "avg. angular vel.", "max. angular vel.", "MAD angular vel.", 196 | "avg. angular accel.", "max. angular accel.", 197 | "MAD angular accel.", 198 | # 199 | "dimensionless jerk", 200 | "avg. limb contraction", 201 | "MAD limb contraction", 202 | "avg. CoM dist.", "MAD CoM dist.", 203 | "avg. head angle (w.r.t. vertical)", 204 | "MAD head angle (w.r.t. vertical)", 205 | "avg. head angle (w.r.t. back)", 206 | "MAD head angle (w.r.t. back)", 207 | "avg. QoM", 208 | "MAD QoM", 209 | "integral QoM", 210 | "avg. convex hull 2D", "MAD convex hull 2D", 211 | "global convex hull 2D", "union convex hull 2D", 212 | "avg. convex hull 3D", "MAD convex hull 3D", 213 | "global convex hull 3D", "union convex hull 3D"] 214 | print("\n", columns) 215 | 216 | row_basic = [mvnx_fps, mvnx_len, sil_fps, len(sil)] 217 | df = pd.DataFrame(columns=columns) 218 | for i, kp in enumerate(keypoints): 219 | r = row_basic + [ 220 | kp, 221 | mean_vels[kp], max_vels[kp], mad_vels[kp], 222 | mean_accels[kp], max_accels[kp], mad_accels[kp], 223 | mean_ang_vels[kp], max_ang_vels[kp], mad_ang_vels[kp], 224 | mean_ang_accels[kp], max_ang_accels[kp], mad_ang_accels[kp], 225 | dimensionless_jerks_vmean[kp], 226 | lcont_global_mean, lcont_t_mad, 227 | cmass_dists_mean_per_kp[kp], cmass_dists_mad_per_kp[kp], 228 | mean_head_angle_vert, mad_head_angle_vert, 229 | mean_head_angle_rel, mad_head_angle_rel, 230 | mean_qom, mad_qom, integral_qom, 231 | mean_ch2d, mad_ch2d, global_ch2d, union_ch2d, 232 | mean_ch3d, mad_ch3d, global_ch3d, union_ch3d] 233 | df.loc[i] = r 234 | if verbose: 235 | print(", ".join(map(str, r))) 236 | # 237 | return df 238 | 239 | 240 | # ############################################################################## 241 | # # MAIN ROUTINE 242 | # ############################################################################## 243 | if __name__ == "__main__": 244 | CONF = OmegaConf.structured(ConfDef()) 245 | cli_conf = OmegaConf.from_cli() 246 | CONF = OmegaConf.merge(CONF, cli_conf) 247 | print("\n\nCONFIGURATION:") 248 | print(OmegaConf.to_yaml(CONF), end="\n\n") 249 | 250 | df = main(CONF.SILHOUETTE_PATH, CONF.JSON_PATH, CONF.MVNX_PATH, 251 | CONF.KEYPOINTS, CONF.QOM_SMI_SECONDS, 252 | CONF.MVNX_SCHEMA, CONF.SIL_IGNORE_FRAMES_BELOW) 253 | 254 | Path(CONF.CSV_OUTPATH).parent.mkdir(parents=True, exist_ok=True) 255 | df.to_csv(CONF.CSV_OUTPATH, index=False) 256 | print("\nSaved CSV to", CONF.CSV_OUTPATH) 257 | -------------------------------------------------------------------------------- /emokine/kinematic/kinematic_features.py: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | 3 | 4 | """ 5 | This module implements several kinematic features of a single-person movement, 6 | that can be extracted from different data modalities like silhouette videos 7 | and MVNX MoCap data. 8 | """ 9 | 10 | 11 | import numpy as np 12 | # 13 | from . import median_absdev 14 | 15 | 16 | # ############################################################################## 17 | # # FROM SILHOUETTES 18 | # ############################################################################## 19 | def silhouette_motion_mask(fhw_segment): 20 | """ 21 | :param fhw_segment: Boolean array with shape ``(frames, h, w)``. 22 | :returns: Boolean array with shape ``(h, w)``, generated by adding 23 | together all frames, and subtracting the last one, to reflect the 24 | variations that happened across the ``f-1`` frames. 25 | """ 26 | last_frame = fhw_segment[-1] 27 | smi = fhw_segment.any(axis=0) # gather all activations 28 | smi[last_frame.nonzero()] = 0 # remove last frame 29 | return smi, last_frame 30 | 31 | 32 | def quantity_of_motion(bool_fhw_vid, smi_frames=5): 33 | """ 34 | :param fhw_segment: Boolean array with shape ``(frames, h, w)``, containing 35 | the silhouettes. Note that ``False`` is interpreted as background. 36 | :param int smi_frames: How many frames will be used to compute the SMIs. 37 | :returns: The triple of lists ``(all_qom, smi_areas, sil_areas)``. Each 38 | list has ``frames-smi_frames+1`` elements, where the first list element 39 | corresponds to ``frame=smi_frames``, and the next elements follow. The 40 | ``smi_areas`` contain the number of SMI active pixels, ``sil_areas`` 41 | contain the number of pixels for the area of the silhouette in the last 42 | SMI frame, and ``all_qom`` contain the rations between SMI areas and 43 | silhouette areas. 44 | 45 | For more details see also section 3.1 in "Recognising Human Emotions from 46 | Body Movement and Gesture Dynamics" (Castellano, Villalba, Camurri 2007). 47 | """ 48 | f, h, w = bool_fhw_vid.shape 49 | smi_areas, sil_areas, all_qom = [], [], [] 50 | for i in range(0, f - smi_frames + 1): 51 | smi, last_frame = silhouette_motion_mask(bool_fhw_vid[i:i+smi_frames]) 52 | # 53 | area_smi = smi.sum() 54 | area_silhouette = last_frame.sum() 55 | qom = float(area_smi) / area_silhouette 56 | # 57 | smi_areas.append(area_smi) 58 | sil_areas.append(area_silhouette) 59 | all_qom.append(qom) 60 | 61 | return all_qom, smi_areas, sil_areas 62 | 63 | 64 | # ############################################################################## 65 | # # FROM MVNX MOCAP 66 | # ############################################################################## 67 | def mvnx_3d_mean_max_mad_magnitudes(df, keypoints, 68 | beg_frame=None, end_frame=None): 69 | """ 70 | Given a pandas ``df`` for a 3-dimensional MVNX feature, this function 71 | gathers the (x,y,z) values for all the given ``keypoints`` at the 72 | given ``[beg, end)`` range, computes the ``norm(xyz)`` magnitudes, 73 | and returns the mean, max and MAD magnitude for each keypoint. 74 | 75 | :param df: data frame as the ones provided by ``MvnxToTabular`` 76 | :param keypoints: Collection containing e.g. ``"Pelvis", "L5", "L3"...`` 77 | :returns: the pair ``(mean_vals, max_vals)``, as dictionaries in form 78 | ``{kp_name: value, ...}`` 79 | """ 80 | mean_vals = {} 81 | max_vals = {} 82 | mad_vals = {} 83 | if beg_frame is None: 84 | beg_frame = 0 85 | if end_frame is None: 86 | end_frame = len(df) 87 | for kp in keypoints: 88 | # xyz shape is (frames, 3) 89 | xyz = df.loc[beg_frame:end_frame-1, kp+"_x":kp+"_z"].values 90 | magnitudes = np.linalg.norm(xyz, axis=1) 91 | mean_vals[kp] = magnitudes.mean() 92 | max_vals[kp] = max(magnitudes) 93 | mad_vals[kp] = median_absdev(magnitudes) 94 | # 95 | return mean_vals, max_vals, mad_vals 96 | 97 | 98 | def dimensionless_jerk(mvnx_vel_dataframe, keypoints, beg_frame=None, 99 | end_frame=None, srate=1): 100 | """ 101 | :param mvnx_vel_dataframe: Data frame as the ones provided by 102 | ``MvnxToTabular`` containing the keypoint velocities. 103 | :param keypoints: Collection containing e.g. ``"Pelvis", "L5", "L3"...`` 104 | :param srate: The jerk is integrated for the time between beg and end. But 105 | here we have a discrete sum, so knowing the sample rate allows us to 106 | adjust "dt" in the integral. Of course the result is dimensionless, but 107 | having high frequencies with a srate of 1 may return extremely high 108 | values, so keeping everything on SI helps, also to compare among 109 | different srates. 110 | 111 | :returns: A dict ``{kp: val, ...}`` where val is the dimensionless jerk 112 | between beg and end for the kp, using the mean velocity as denominator. 113 | For an explanation see: "Sensitivity of Smoothness Measures 114 | to Movement Duration, Amplitude and Arrests (Hogan and Sternad)". 115 | """ 116 | result = {} 117 | # 118 | if beg_frame is None: 119 | beg_frame = 0 120 | if end_frame is None: 121 | end_frame = len(mvnx_vel_dataframe) 122 | # 123 | for kp in keypoints: 124 | # note the end+1 to gather 2 extra samples if possible. 125 | # we don't integrate from accel because it is way more noisy than vel 126 | # also integrating from pos is bad, discretizes a lot 127 | vel_xyz = mvnx_vel_dataframe.loc[ 128 | beg_frame:end_frame+1, kp+"_x":kp+"_z"].values 129 | acc_xyz = (vel_xyz[1:] - vel_xyz[:-1]) * srate 130 | jerk_xyz = (acc_xyz[1:] - acc_xyz[:-1]) * srate 131 | # 132 | vel_norm = np.linalg.norm(vel_xyz, axis=1) 133 | jerk_norm = np.linalg.norm(jerk_xyz, axis=1) 134 | # 135 | # 136 | # jerk_sq_integral = (jerk_norm**2).sum() / srate 137 | jerk_sq_integral = jerk_norm.sum() / srate # WE DON'T NEED NORM**2 138 | time_cubed = ((end_frame - beg_frame) / srate)**3 139 | mean_vel_squared = vel_norm.mean() ** 2 140 | # 141 | dless_jerk = (time_cubed * jerk_sq_integral) / mean_vel_squared 142 | # !!!! 143 | dless_jerk *= mean_vel_squared 144 | 145 | result[kp] = dless_jerk 146 | # vel2_xyz = mvnx_whole_dataframe["velocity"].loc[ 147 | # beg_frame:end_frame+1, kp+"_x":kp+"_z"].values 148 | # acc2_xyz = (vel2_xyz[1:] - vel2_xyz[:-1]) * srate 149 | # plt.clf(); plt.plot(vel_xyz); plt.show() 150 | # plt.clf(); plt.plot(jerk_xyz); plt.show() 151 | # plt.clf(); plt.plot(vel_norm); plt.show() 152 | # plt.clf(); plt.plot(jerk_norm); plt.show() 153 | # 154 | return result 155 | 156 | 157 | def limb_contraction(mvnx_pos_dataframe, beg_frame=None, end_frame=None): 158 | """ 159 | Limb contraction from Poyo Solanas 2020: For each frame, this metric 160 | is the mean euclideam distance between the 4 endpoints and the head. 161 | :param mvnx_pos_dataframe: Data frame as the ones provided by 162 | ``MvnxToTabular`` containing the keypoint positions. 163 | :returns: an array of shape ``(frames, 4)``, where each entry is the 3D 164 | eucl. distance from ``(rHand, lHand, rToe, lToe)`` to ``head``. 165 | """ 166 | if beg_frame is None: 167 | beg_frame = 0 168 | if end_frame is None: 169 | end_frame = len(mvnx_pos_dataframe) 170 | xyz_pos_head = mvnx_pos_dataframe.loc[beg_frame:end_frame-1, 171 | "Head_x":"Head_z"] 172 | xyz_pos_lhand = mvnx_pos_dataframe.loc[beg_frame:end_frame-1, 173 | "LeftHand_x":"LeftHand_z"] 174 | xyz_pos_rhand = mvnx_pos_dataframe.loc[beg_frame:end_frame-1, 175 | "RightHand_x":"RightHand_z"] 176 | xyz_pos_rtoe = mvnx_pos_dataframe.loc[beg_frame:end_frame-1, 177 | "RightToe_x":"RightToe_z"] 178 | xyz_pos_ltoe = mvnx_pos_dataframe.loc[beg_frame:end_frame-1, 179 | "LeftToe_x":"LeftToe_z"] 180 | # 181 | eucl_rhand = np.linalg.norm( 182 | xyz_pos_head.values - xyz_pos_rhand.values, axis=1) 183 | eucl_lhand = np.linalg.norm( 184 | xyz_pos_head.values - xyz_pos_lhand.values, axis=1) 185 | eucl_rtoe = np.linalg.norm( 186 | xyz_pos_head.values - xyz_pos_rtoe.values, axis=1) 187 | eucl_ltoe = np.linalg.norm( 188 | xyz_pos_head.values - xyz_pos_ltoe.values, axis=1) 189 | # 190 | result = np.stack([eucl_rhand, eucl_lhand, eucl_rtoe, eucl_ltoe]).T 191 | assert result.shape == (end_frame - beg_frame, 4), \ 192 | "This shouldn't happen!" 193 | return result 194 | 195 | 196 | def cmass_distances(mvnx_pos_dataframe, mvnx_cmass_dataframe, 197 | keypoints, beg_frame=None, end_frame=None): 198 | """ 199 | :param mvnx_pos_dataframe: Data frame as the ones provided by 200 | ``MvnxToTabular`` containing the keypoint positions. 201 | :param mvnx_cmass_dataframe: Data frame as the ones provided by 202 | ``MvnxToTabular`` containing the full-body center of mass. 203 | :param keypoints: Collection containing e.g. ``"Pelvis", "L5", "L3"...`` 204 | :returns: A dict in the form ``kp: arr``, with one kp key per 205 | ``PROTOTYPE_KEYPOINT``, and arr has one entry per frame between 206 | ``beg_frame`` and ``end_frame`` containing the eucl. distance. 207 | """ 208 | if beg_frame is None: 209 | beg_frame = 0 210 | if end_frame is None: 211 | end_frame = len(mvnx_pos_dataframe) 212 | # 213 | xyz_cmass = mvnx_cmass_dataframe.loc[beg_frame:end_frame-1, 214 | "centerOfMass_x":"centerOfMass_z"] 215 | result = {} 216 | for kp in keypoints: 217 | xyz = mvnx_pos_dataframe.loc[beg_frame:end_frame-1, 218 | kp+"_x":kp+"_z"].values 219 | dist_per_frame = np.linalg.norm(xyz_cmass - xyz, axis=1) 220 | result[kp] = dist_per_frame 221 | return result 222 | 223 | 224 | def head_angle(mvnx_pos_dataframe, beg_frame=None, end_frame=None): 225 | """ 226 | :param mvnx_pos_dataframe: Data frame as the ones provided by 227 | ``MvnxToTabular`` containing the keypoint positions. 228 | :returns: A tuple ``(a, b)``. A is the absolute angle per frame compared 229 | to the vertical line (regardless of the direction). B measures the 230 | head inclination with respect to the body, as follows: MVNX provides a 231 | small 4-keypoint diamond on the chest area, formed by a 'vertical' line 232 | from T8 (5th) to Neck (6th), and a 'horizontal' line from LeftShoulder 233 | (8th) to RightShoulder (12th). The provided head inclination is the angle 234 | between the neck-to-head vector and the T8-to-Neck one. If we want to 235 | ignore lateral inclination, we can use the Shoulder-to-Shoulder line to 236 | find the plane, and then we just need to compute the angle between the 237 | Neck-to-Head vector and its projection on the plane. 238 | 239 | ..Note:: 240 | We assume that all relative angles are in the same direction, towards 241 | the chest, and never towards the back. This may be robustified using 242 | the angle info from MVNX. All angles are given in radians. 243 | """ 244 | if beg_frame is None: 245 | beg_frame = 0 246 | if end_frame is None: 247 | end_frame = len(mvnx_pos_dataframe) 248 | # COMPUTE RELATIVE ANGLES 249 | xyz_pos_t8 = mvnx_pos_dataframe.loc[beg_frame:end_frame-1, 250 | "T8_x":"T8_z"].values 251 | xyz_pos_neck = mvnx_pos_dataframe.loc[beg_frame:end_frame-1, 252 | "Neck_x":"Neck_z"].values 253 | xyz_pos_head = mvnx_pos_dataframe.loc[beg_frame:end_frame-1, 254 | "Head_x":"Head_z"].values 255 | # 256 | t8_to_neck = xyz_pos_neck - xyz_pos_t8 257 | t8_to_neck_unit = t8_to_neck / np.linalg.norm(t8_to_neck, axis=1)[:, None] 258 | # 259 | neck_to_head = xyz_pos_head - xyz_pos_neck 260 | neck_to_head_unit = neck_to_head / np.linalg.norm( 261 | neck_to_head, axis=1)[:, None] 262 | # 263 | relative_cosines = (t8_to_neck_unit * neck_to_head_unit).sum(axis=1) 264 | assert (relative_cosines > 0).all(), "Head angle >90deg. Person broken??" 265 | relative_radians = np.arccos(np.clip(relative_cosines, -1.0, 1.0)) 266 | # NOW COMPUTE ABSOLUTE ANGLES: the z component of the unit head is the cos 267 | # abs angles are always w.r.t [0, 0, 1] 268 | vertical_cosines = neck_to_head_unit[:, 2] 269 | vertical_radians = np.arccos(np.clip(vertical_cosines, -1.0, 1.0)) 270 | # to convert rads to degrees, multiply by 180/np.pi 271 | return vertical_radians, relative_radians 272 | -------------------------------------------------------------------------------- /2a_silhouettes.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | """ 6 | Given a folder with N images assumed to belong to a sequence where a human 7 | moves on a fixed background, this script performs DL human segmentation, 8 | saving a binarized version for each image, where the background is black and 9 | the human silhouette is white. 10 | """ 11 | 12 | 13 | import os 14 | import multiprocessing 15 | from itertools import repeat 16 | # for OmegaConf 17 | from dataclasses import dataclass 18 | from typing import Optional 19 | # 20 | import numpy as np 21 | import torch 22 | from torchvision.transforms import Resize 23 | from PIL import Image 24 | import wget 25 | from omegaconf import OmegaConf, MISSING 26 | # 27 | from detectron2.config import LazyConfig, instantiate 28 | from detectron2.data import MetadataCatalog 29 | from detectron2 import model_zoo 30 | from detectron2.checkpoint import DetectionCheckpointer 31 | from detectron2.structures import ROIMasks 32 | from detectron2.utils.memory import retry_if_cuda_oom 33 | from detectron2.layers.mask_ops import paste_masks_in_image 34 | 35 | # 36 | from emokine.utils import get_lab_distance, otsu_hist_median 37 | from emokine.utils import resize_hw 38 | # import matplotlib.pyplot as plt 39 | 40 | 41 | # ############################################################################## 42 | # # HELPERS 43 | # ############################################################################## 44 | def human_segmentation_setup( 45 | url="new_baselines/mask_rcnn_regnety_4gf_dds_FPN_400ep_LSJ.py", 46 | model_dir=os.path.join("output", "model_snapshots", "segmentation")): 47 | """ 48 | :param url: Model URL from the Detectron2 URL model zoo. 49 | :param model_dir: Where the pre-trained model will be saved 50 | :returns: ``(model, cfg, classes)``, where ``model`` is the pre-trained 51 | pytorch model, ``cfg`` is the detectron2 config, and ``classes`` is 52 | a list of class names as output by the model. 53 | 54 | More info: 55 | https://github.com/facebookresearch/detectron2/blob/main/MODEL_ZOO.md 56 | https://detectron2.readthedocs.io/en/latest/tutorials/lazyconfigs.html 57 | """ 58 | # load config and replace every SyncBN with BN 59 | cfg = LazyConfig.load(model_zoo.get_config_file(url)) 60 | if cfg.model.backbone.norm == "SyncBN": 61 | cfg.model.backbone.norm = "BN" 62 | if cfg.model.backbone.bottom_up.norm == "SyncBN": 63 | cfg.model.backbone.bottom_up.norm = "BN" 64 | # sanity checks 65 | assert cfg.model.input_format in {"BGR", "RGB"}, cfg.model.input_format 66 | 67 | # list of class names as output by the model 68 | classes = MetadataCatalog.get( 69 | cfg.dataloader.test.dataset.names).thing_classes 70 | 71 | # load model from zoo 72 | model = instantiate(cfg.model) 73 | 74 | # load model parameters from zoo, download if not existing 75 | model_ckpt = DetectionCheckpointer(model, save_dir=model_dir) 76 | try: 77 | # try to load pre-downloaded snapshot 78 | ckpt = os.path.join(model_dir, os.listdir(model_dir)[0]) 79 | model_ckpt.load(ckpt) 80 | except (FileNotFoundError, IndexError): 81 | os.makedirs(model_dir, exist_ok=True) 82 | # we assume this is the first time and download the snapshot 83 | ckpt_url = model_zoo.get_checkpoint_url(url) 84 | down_path = os.path.join(model_dir, os.path.basename(ckpt_url)) 85 | print("Downloading", ckpt_url, "\nto", down_path) 86 | wget.download(ckpt_url, down_path) 87 | # and retry to load snapshot 88 | ckpt = os.path.join(model_dir, os.listdir(model_dir)[0]) 89 | model_ckpt.load(ckpt) 90 | # 91 | return model, cfg, classes 92 | 93 | 94 | def silhouette_inference(img_t, model): 95 | """ 96 | :param img_t: Float tensor of shape ``(3, h, w)``. 97 | :param model: Detectron2 model of type ``GeneralizedRCNN``. 98 | 99 | This function works like ``model.inference`` in detectron2 but omits the 100 | final thresholding, returning a soft output as uint8. 101 | 102 | https://detectron2.readthedocs.io/en/latest/tutorials/models.html#partially-execute-a-model 103 | https://github.com/facebookresearch/detectron2/blob/38af375052d3ae7331141bc1a22cfa2713b02987/detectron2/modeling/meta_arch/rcnn.py#L178 104 | https://github.com/facebookresearch/detectron2/blob/38af375052d3ae7331141bc1a22cfa2713b02987/detectron2/layers/mask_ops.py#L74 105 | """ 106 | _, h, w = img_t.shape 107 | result = model.inference( 108 | [{"image": img_t, "height": h, "width": w}], 109 | do_postprocess=False)[0] 110 | # expand predicted masks to HxW heatmaps 111 | roi_masks = ROIMasks(result.pred_masks[:, 0, :, :]).tensor 112 | heatmaps = retry_if_cuda_oom(paste_masks_in_image)( 113 | roi_masks, result.pred_boxes.tensor, (h, w), 114 | threshold=-1) 115 | # replace masks with expanded heatmaps and return 116 | result.pred_masks = heatmaps 117 | return result 118 | 119 | 120 | # ############################################################################## 121 | # # CLI 122 | # ############################################################################## 123 | @dataclass 124 | class ConfDef: 125 | """ 126 | :cvar IMGS_DIR: Path to a directory containing only the images to be 127 | processed. It is assumed that all images have the same shape and format, 128 | and come from a sequence containing people on a static background. 129 | :cvar SEG_URL: Detectron2 URL for the segmentation model, e.g. 130 | ``new_baselines/mask_rcnn_regnety_4gf_dds_FPN_400ep_LSJ.py`` 131 | :cvar SEG_MODEL_DIR: Where the segmentation model from detectron2 is 132 | expected to be downloaded/stored. 133 | :cvar TARGET_CLASS: Name of the class that we want to extract silhouettes 134 | from. It must be a class supported by the detectron2 model. 135 | :cvar MIN_IDX: Once sorted by their name, images will be processed starting 136 | with index 0, unless a different starting index is given here. 137 | :cvar MAX_IDX: Once sorted by their name, images will be processed until 138 | the last one, unless a different max index is given here. 139 | :cvar SKIP_N: Once sorted by their name, images will be processed one 140 | by one. If this parameter is N, one out of N will be processed, and the 141 | rest skipped. 142 | :cvar SIL_INPUT_SIZE: Given size ``h*w`` for all images in pixels, a 143 | rescaling will be applied for the DL inference, so that the smaller 144 | dimension equals this parameter, and aspect ratio is maintained. 145 | :cvar BG_ESTIMATION_RATIO: The lowest ``x`` DL predicted values across the 146 | sequence will be taken into account and averaged to estimate the 147 | background color. This parameter determines how many of the lowest will 148 | be used, as a proportion (ratio) of the whole sequence. 149 | :cvar MEDIAN_FILT_SIZE: After thresholding, spatial median filter is 150 | applied. This determines the size of the filter. 151 | :cvar INCLUDE_DL_ABOVE: If given, DL predictions equal or greater than this 152 | threshold are guaranteed to be part of the solution. 153 | :cvar DEVICE: PyTorch device, e.g. ``cuda`` or ``cpu`` 154 | """ 155 | IMGS_DIR: str = MISSING 156 | SEG_URL: str = "new_baselines/mask_rcnn_regnety_4gf_dds_FPN_400ep_LSJ.py" 157 | SEG_MODEL_DIR: str = os.path.join("output", "model_snapshots", 158 | "segmentation") 159 | TARGET_CLASS: str = "person" 160 | # 161 | MIN_IDX: Optional[int] = None 162 | MAX_IDX: Optional[int] = None 163 | SKIP_N: Optional[int] = None 164 | # 165 | DEVICE: str = "cuda" if torch.cuda.is_available() else "cpu" 166 | SIL_INPUT_SIZE: Optional[int] = 400 167 | BG_ESTIMATION_RATIO: float = 0.02 168 | MEDIAN_FILT_SIZE: int = 5 169 | INCLUDE_DL_ABOVE: Optional[float] = None 170 | 171 | 172 | # ############################################################################## 173 | # # MAIN ROUTINE 174 | # ############################################################################## 175 | if __name__ == "__main__": 176 | 177 | CONF = OmegaConf.structured(ConfDef()) 178 | cli_conf = OmegaConf.from_cli() 179 | CONF = OmegaConf.merge(CONF, cli_conf) 180 | print("\n\nCONFIGURATION:") 181 | print(OmegaConf.to_yaml(CONF), end="\n\n\n") 182 | 183 | img_paths = sorted([os.path.join(CONF.IMGS_DIR, p) 184 | for p in os.listdir(CONF.IMGS_DIR)])[ 185 | CONF.MIN_IDX:CONF.MAX_IDX:CONF.SKIP_N] 186 | assert img_paths, "Empty images dir?" 187 | num_imgs = len(img_paths) 188 | 189 | model, cfg, classes = human_segmentation_setup( 190 | CONF.SEG_URL, CONF.SEG_MODEL_DIR) 191 | target_cls_idx = classes.index(CONF.TARGET_CLASS) 192 | model = model.to(CONF.DEVICE) 193 | model.eval() 194 | 195 | # Human DL segmentation 196 | img_modes = [] 197 | img_arrs = [] 198 | dl_heatmaps = [] 199 | with torch.no_grad(): 200 | for i, ip in enumerate(img_paths): 201 | print(f"DL inference: [{i}/{num_imgs}]: {ip}") 202 | # load (h, w, c) image (typically RGB uint8) and optionally resize: 203 | img = Image.open(ip) 204 | arr = np.array(img).astype(np.float32) 205 | img_arrs.append(arr) 206 | 207 | if CONF.SIL_INPUT_SIZE is not None: 208 | img_resized = Resize(CONF.SIL_INPUT_SIZE)(img) 209 | else: 210 | img_resized = img 211 | 212 | # convert to float32 tensor, permute, swap channels 213 | t = torch.as_tensor(np.array(img_resized).astype( 214 | np.float32)).permute(2, 0, 1).to(CONF.DEVICE) 215 | assert img.mode in {"BGR", "RGB"}, img.mode 216 | if img.mode != cfg.model.input_format: 217 | t = t.flip(0) # flip channels 218 | img_modes.append(img.mode) 219 | 220 | # perform inference 221 | out = silhouette_inference(t, model) 222 | 223 | # extract masks corresponding to target class 224 | target_out = out.pred_masks[ 225 | torch.where(out.pred_classes == target_cls_idx)] 226 | if target_out.numel() >= 1: 227 | target_out = target_out.max(dim=0)[0] 228 | else: # if no masks found, we get all zeros 229 | target_out = torch.zeros_like(out.pred_masks[0]) 230 | # optionally resize back and gather result 231 | if CONF.SIL_INPUT_SIZE is not None: 232 | target_out = resize_hw(target_out, img.size[::-1]) 233 | dl_heatmaps.append(target_out.cpu()) 234 | img_arrs = np.stack(img_arrs) 235 | dl_heatmaps = torch.stack(dl_heatmaps).numpy() 236 | # idx=10; plt.clf(); plt.imshow(dl_heatmaps[idx]); plt.show() 237 | 238 | # get background as an average of colors with lowest human detection 239 | # bg_uncertainty gives away regions where person detection covers the 240 | # bg and we are not cetain about the color. 241 | print(f"Extracting background...") 242 | pick_k = round(CONF.BG_ESTIMATION_RATIO * len(img_arrs)) 243 | k_idxs = np.argpartition(dl_heatmaps, pick_k, axis=0, 244 | kind='introselect', order=None)[:pick_k] 245 | mean_bg = np.take_along_axis( 246 | img_arrs, k_idxs[:, :, :, None], axis=0).mean(axis=0) 247 | bg_uncertainty = np.take_along_axis( 248 | dl_heatmaps, k_idxs, axis=0).mean(axis=0) / 255.0 249 | assert (bg_uncertainty >= 0).all() and (bg_uncertainty <= 1).all(), \ 250 | "bg_uncertainty should be between 0 and 1!" 251 | del k_idxs 252 | # plt.clf(); plt.imshow(mean_bg / 255); plt.show() 253 | # plt.clf(); plt.imshow(bg_uncertainty); plt.show() 254 | 255 | print(f"Computing LAB residuals...") 256 | with multiprocessing.Pool() as pool: 257 | residual_dists = np.stack( 258 | pool.starmap(get_lab_distance, 259 | zip(img_arrs, repeat(mean_bg), repeat(np.float32)))) 260 | residual_dists /= residual_dists.max() 261 | # idx=10; plt.clf(); plt.imshow(residual_dists[idx]); plt.show() 262 | 263 | print("Fusion and thresholding...") 264 | residual_dists *= (dl_heatmaps / 255) 265 | 266 | # INCLUDE_DL_ABOVE: Optional[float] = None 267 | fusion = ((dl_heatmaps / 255) * bg_uncertainty) + (residual_dists * 268 | (1 - bg_uncertainty)) 269 | with multiprocessing.Pool() as pool: 270 | fusion_masks = pool.starmap( 271 | otsu_hist_median, 272 | zip(fusion, repeat(CONF.MEDIAN_FILT_SIZE))) 273 | # idx=10; plt.clf(); plt.imshow(fusion[idx]); plt.show() 274 | # idx=10; plt.clf(); plt.imshow(fusion_masks[idx]); plt.show() 275 | 276 | if CONF.INCLUDE_DL_ABOVE is not None: 277 | thresh = round(CONF.INCLUDE_DL_ABOVE * 255) 278 | dl_masks = (dl_heatmaps >= thresh) 279 | fusion_masks = [fm | dlm for fm, dlm in zip(fusion_masks, dl_masks)] 280 | # idx=10; plt.clf(); plt.imshow(dl_masks[idx]); plt.show() 281 | 282 | for i, (ip, fm) in enumerate(zip(img_paths, fusion_masks)): 283 | outpath = ip + "__silhouette.png" 284 | Image.fromarray(fm).save(outpath) 285 | print(f"[{i}/{num_imgs}]", "Saved to", outpath) 286 | -------------------------------------------------------------------------------- /2b_face_blurs.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | """ 6 | This script performs face masking/blurring on a collection of images expected 7 | to contain one person each. It performs the following steps: 8 | 1. Loads image 9 | 2. Performs human keypoint detection and isolates the most prominent person 10 | 3. Isolates selected keypoints and averages their location to find the head 11 | 4. Extracts a patch around the head and performs face detection (segmentation) 12 | 5. Optionally, transforms the predicted mask into a fitted ellipse 13 | 6. Saves output as binary mask or as original image with the blurred mask. 14 | 15 | The pipeline includes potential resizing before steps 2 and 4, to ensure that 16 | the neural networks being used get images of appropriate scale. 17 | """ 18 | 19 | 20 | import os 21 | # for OmegaConf 22 | from dataclasses import dataclass 23 | from typing import Optional, List 24 | # 25 | import numpy as np 26 | import torch 27 | from torchvision.transforms import Resize 28 | from PIL import Image, ImageFilter 29 | import wget 30 | from omegaconf import OmegaConf, MISSING 31 | # 32 | from detectron2 import model_zoo 33 | from detectron2.checkpoint import DetectionCheckpointer 34 | from detectron2.modeling import build_model 35 | # 36 | import face_segmentation_pytorch as fsp 37 | # 38 | from emokine.utils import make_elliptic_mask, resize_crop_bbox 39 | # import matplotlib.pyplot as plt 40 | 41 | 42 | # ############################################################################## 43 | # # HELPERS 44 | # ############################################################################## 45 | def human_keypoints_setup( 46 | url="COCO-Keypoints/keypoint_rcnn_X_101_32x8d_FPN_3x.yaml", 47 | model_dir=os.path.join("output", "model_snapshots", "keypoints")): 48 | """ 49 | :param url: Model URL from the Detectron2 URL model zoo. 50 | :param model_dir: Where the pre-trained model will be saved 51 | :returns: ``(model, cfg, meta)``, where ``model`` is the pre-trained 52 | pytorch model, ``cfg`` is the detectron2 config, and ``meta`` is 53 | 54 | More info: 55 | https://github.com/facebookresearch/detectron2/blob/main/MODEL_ZOO.md 56 | """ 57 | # load config 58 | cfg = model_zoo.get_config(url) 59 | # sanity checks 60 | assert cfg.INPUT.FORMAT in {"BGR", "RGB"}, cfg.model.input_format 61 | model_dir = os.path.join("output", "model_snapshots", "keypoints") 62 | # 63 | meta = {"size_range": (cfg.INPUT.MIN_SIZE_TEST, cfg.INPUT.MAX_SIZE_TEST)} 64 | # load model from zoo 65 | model = build_model(cfg) 66 | model_ckpt = DetectionCheckpointer(model, save_dir=model_dir) 67 | try: 68 | # try to load pre-downloaded snapshot 69 | ckpt = os.path.join(model_dir, os.listdir(model_dir)[0]) 70 | model_ckpt.load(ckpt) 71 | except (FileNotFoundError, IndexError): 72 | os.makedirs(model_dir, exist_ok=True) 73 | # we assume this is the first time and download the snapshot 74 | ckpt_url = model_zoo.get_checkpoint_url(url) 75 | down_path = os.path.join(model_dir, os.path.basename(ckpt_url)) 76 | print("Downloading", ckpt_url, "\nto", down_path) 77 | wget.download(ckpt_url, down_path) 78 | # and retry to load snapshot 79 | ckpt = os.path.join(model_dir, os.listdir(model_dir)[0]) 80 | model_ckpt.load(ckpt) 81 | # 82 | return model, cfg, meta 83 | 84 | 85 | def main_person_inference(img_t, model, threshold=0.5): 86 | """ 87 | :param img_t: Float tensor of shape ``(3, h, w)``. 88 | :param model: Detectron2 model that provides ``pred_keypoints``. 89 | :returns: None if no person with conficence above threshold is found, and 90 | the ``(K, 3)`` keypoints otherwise (signaling ``x, y, confidence``). 91 | """ 92 | # predict head position 93 | _, h, w = img_t.shape 94 | result = model.inference( 95 | [{"image": img_t, "height": h, "width": w}])[0]["instances"] 96 | person_accepted = (result.scores >= threshold) 97 | if person_accepted.sum() > 0: 98 | main_person_idx = result.scores.argmax().item() 99 | main_person_kps = result.pred_keypoints[main_person_idx] 100 | return main_person_kps 101 | # 102 | return None 103 | 104 | 105 | def thresh_avg_kps(kps, thresh=0.5): 106 | """ 107 | :param kps: Tensor of shape ``(num_kps, 3)``, where each row contains 108 | a ``(x, y, score)`` triple. 109 | :returns: ``(x_avg, y_avg), n``, where ``n`` determines how many points 110 | were used to obtain the average (i.e. how many had a confidence above 111 | the threshold). If ``n=0``, returns ``None, 0``. 112 | """ 113 | kps = kps[kps[:, 2] >= thresh] 114 | num_kps = len(kps) 115 | if num_kps == 0: 116 | return None, 0 117 | else: 118 | x_avg, y_avg = kps.mean(dim=0)[:2] 119 | return (x_avg.item(), y_avg.item()), num_kps 120 | 121 | 122 | # ############################################################################## 123 | # # CLI 124 | # ############################################################################## 125 | @dataclass 126 | class ConfDef: 127 | """ 128 | :cvar IMGS_DIR: Path to a directory containing only the images to be 129 | processed. It is assumed that all images have the same shape and format, 130 | and come from a sequence containing people on a static background. 131 | :cvar KP_URL: Detectron2 URL for the keypoint estimation model, e.g. 132 | ``COCO-Keypoints/keypoint_rcnn_X_101_32x8d_FPN_3x.yaml`` 133 | :cvar KP_MODEL_DIR: Where the keypoint estimation model from detectron2 is 134 | expected to be downloaded/stored. 135 | :cvar FM_MODEL_DIR: Where the face masking model is expected to be 136 | downloaded/stored. 137 | :cvar MIN_IDX: Once sorted by their name, images will be processed starting 138 | with index 0, unless a different starting index is given here. 139 | :cvar MAX_IDX: Once sorted by their name, images will be processed until 140 | the last one, unless a different max index is given here. 141 | :cvar SKIP_N: Once sorted by their name, images will be processed one 142 | by one. If this parameter is N, one out of N will be processed, and the 143 | rest skipped. 144 | :cvar DEVICE: PyTorch device, e.g. ``cuda`` or ``cpu`` 145 | :cvar KP_INPUT_SIZE: Given size ``h*w`` for an input image in pixels, a 146 | rescaling will be applied for the keypoint estimation inference, so that 147 | the smaller dimension equals this param, and aspect ratio is maintained. 148 | :cvar FM_INPUT_SIZE: The input to the face masking model will be rescaled 149 | so that the smaller dimension equals this param. Note that the Nirkin et 150 | al model works best in the ballpark of 400-500 (in pixels). 151 | :cvar PERSON_THRESHOLD: Between 0 and 1, any detected person below this 152 | confidence will not be considered. 153 | :cvar KP_THRESHOLD: Between 0 and 1, any detected keypoints below this 154 | confidence will not be considered 155 | :cvar FM_THRESHOLD: Once the face heatmap is computed as a matrix of values 156 | between 0 and 1, the mask is extracted by applying this threshold. 157 | :cvar BBOX_SIZE: Once the ``KP_SELECTION`` is found, a bounding box of 158 | this size (in pixels) will be drawn around its middlepoint. This will be 159 | the patch send to the face masking model, after resizing via 160 | ``FM_INPUT_SIZE``. 161 | :cvar GAUSS_BLUR_STD: If not given, results will be stored as black/white 162 | binary face masks. If given, results will be the images, where the region 163 | covered by the mask is blurred out using the given standard deviation, 164 | in pixels. 165 | :cvar BORDERS_BLUR_STD: If ``GAUSS_BLUR_STD`` is active, this parameter 166 | regulated the sharpness of the transition between the blurred and 167 | non-blurred regions, in pixels. 168 | :cvar ELLIPTIC_MASK: If not given, the mask is the actual output from the 169 | face masking model. If given, the mask is fitted to an ellipse and this 170 | parameter determines the scale of the resulting ellipse (e.g. 2 means 171 | the ellipse axes will be 2 times longer). It is a ratio. 172 | :cvar KP_CLASSES: The keypoint detection model is expected to output K 173 | triples in the form ``(x_location, y_location, confidence)``, each 174 | corresponding to a specific keypoint. This list of length K determines 175 | the name of the corresponding keypoints. 176 | :cvar KP_SELECTION: To extract the bounding box sent to the face mask 177 | model, we identify a selection of keypoints and take their average. This 178 | list determines which keypoints will be taken into account. 179 | """ 180 | IMGS_DIR: str = MISSING 181 | KP_URL: str = "COCO-Keypoints/keypoint_rcnn_X_101_32x8d_FPN_3x.yaml" 182 | KP_MODEL_DIR: str = os.path.join("output", "model_snapshots", "keypoints") 183 | FM_MODEL_DIR: str = os.path.join("output", "model_snapshots", "face_mask") 184 | # 185 | MIN_IDX: Optional[int] = None 186 | MAX_IDX: Optional[int] = None 187 | SKIP_N: Optional[int] = None 188 | # 189 | DEVICE: str = "cuda" if torch.cuda.is_available() else "cpu" 190 | KP_INPUT_SIZE: Optional[int] = 800 191 | FM_INPUT_SIZE: Optional[int] = 450 192 | # 193 | PERSON_THRESHOLD: float = 0.5 194 | KP_THRESHOLD: float = 0.5 195 | FM_THRESHOLD: float = 0.5 196 | # 197 | BBOX_SIZE: int = 100 198 | GAUSS_BLUR_STD: Optional[float] = None 199 | BORDERS_BLUR_STD: float = 3 200 | ELLIPTIC_MASK: Optional[float] = None 201 | 202 | # 203 | KP_CLASSES: List[str] = ("nose", 204 | "left_eye", "right_eye", 205 | "left_ear", "right_ear", 206 | "left_shoulder", "right_shoulder", 207 | "left_elbow", "right_elbow", 208 | "left_wrist", "right_wrist", 209 | "left_hip", "right_hip", 210 | "left_knee", "right_knee", 211 | "left_ankle", "right_ankle") 212 | KP_SELECTION: List[str] = ("nose", "left_eye", "right_eye", 213 | "left_ear", "right_ear") 214 | 215 | 216 | # ############################################################################## 217 | # # MAIN ROUTINE 218 | # ############################################################################## 219 | if __name__ == "__main__": 220 | 221 | CONF = OmegaConf.structured(ConfDef()) 222 | cli_conf = OmegaConf.from_cli() 223 | CONF = OmegaConf.merge(CONF, cli_conf) 224 | print("\n\nCONFIGURATION:") 225 | print(OmegaConf.to_yaml(CONF), end="\n\n\n") 226 | 227 | img_paths = sorted([os.path.join(CONF.IMGS_DIR, p) 228 | for p in os.listdir(CONF.IMGS_DIR)])[ 229 | CONF.MIN_IDX:CONF.MAX_IDX:CONF.SKIP_N] 230 | assert img_paths, "Empty images dir?" 231 | num_imgs = len(img_paths) 232 | 233 | # load keypoint estimation model 234 | kp_model, cfg, meta = human_keypoints_setup(CONF.KP_URL, CONF.KP_MODEL_DIR) 235 | min_size_range, max_size_range = meta["size_range"] 236 | assert (min_size_range <= CONF.KP_INPUT_SIZE <= max_size_range), \ 237 | "KP_INPUT_SIZE must be in range {(min_size_range, max_size_range)!}" 238 | 239 | kp_sel_idxs = [CONF.KP_CLASSES.index(k) for k in CONF.KP_SELECTION] 240 | kp_model = kp_model.to(CONF.DEVICE) 241 | kp_model.eval() 242 | 243 | # load face mask model 244 | bbox_half = CONF.BBOX_SIZE / 2 245 | fm_model = fsp.model.FaceSegmentationNet() 246 | fsp.utils.load_model_parameters(fm_model, CONF.FM_MODEL_DIR) 247 | fm_model = fm_model.to(CONF.DEVICE) 248 | fm_model.eval() 249 | fm_mean_bgr = torch.tensor(fm_model.MEAN_BGR).type(torch.float32).to( 250 | CONF.DEVICE) 251 | 252 | # Main loop 253 | with torch.no_grad(): 254 | for i, ip in enumerate(img_paths): 255 | print(f"DL inference: [{i}/{num_imgs}]: {ip}") 256 | # load (h, w, c) image (typically RGB uint8) and optionally resize: 257 | img = Image.open(ip).convert("RGB") 258 | arr = np.array(img).astype(np.float32) 259 | 260 | if CONF.KP_INPUT_SIZE is not None: 261 | img_resized = Resize(CONF.KP_INPUT_SIZE)(img) 262 | else: 263 | img_resized = img 264 | 265 | # convert to float32 tensor, permute, swap channels 266 | t = torch.as_tensor(np.array(img_resized).astype( 267 | np.float32)).permute(2, 0, 1).to(CONF.DEVICE) 268 | assert img.mode in {"BGR", "RGB"}, img.mode 269 | if img.mode != cfg.INPUT.FORMAT: 270 | t = t.flip(0) # flip channels 271 | 272 | # mask inference 273 | face_heatmap = torch.zeros_like(t[0]) 274 | kps = main_person_inference(t, kp_model, CONF.PERSON_THRESHOLD) 275 | if kps is not None: 276 | kp_avg, _ = thresh_avg_kps(kps[kp_sel_idxs], CONF.KP_THRESHOLD) 277 | if kp_avg is not None: 278 | # if head found, extract bbox around it and crop tensor 279 | x_avg, y_avg = kp_avg 280 | h, w = t.shape[-2:] 281 | x0, x1, y0, y1 = resize_crop_bbox( 282 | x_avg - bbox_half, x_avg + bbox_half, 283 | y_avg - bbox_half, y_avg + bbox_half, 284 | max_x=w, max_y=h, expansion_ratio=1.0) 285 | t_crop = t[:, y0:y1, x0:x1] 286 | # normalize and resize 287 | t_crop = fsp.utils.normalize_range( 288 | t_crop, torch.float32, out_range=(0, 255)) 289 | t_crop = t_crop.permute(1, 2, 0).sub( 290 | fm_mean_bgr).permute(2, 0, 1) 291 | if CONF.FM_INPUT_SIZE is not None: 292 | t_crop = Resize(CONF.FM_INPUT_SIZE)(t_crop) 293 | # perform face segmentation 294 | hm = fm_model(t_crop.unsqueeze(0), as_pmap=True)[0] 295 | # plt.clf(); plt.imshow(t_crop[0].cpu()); plt.show() 296 | # plt.clf(); plt.imshow(hm.cpu()); plt.show() 297 | 298 | # paste hm on global domain 299 | if CONF.FM_INPUT_SIZE is not None: 300 | hm_sz = Resize(min(x1-x0, y1-y0))(hm.unsqueeze(0))[0] 301 | face_heatmap[y0:y1, x0:x1] = hm_sz 302 | # extract global mask, possibly convert to elliptic 303 | face_mask = (face_heatmap >= CONF.FM_THRESHOLD).cpu().numpy() 304 | if CONF.ELLIPTIC_MASK is not None: 305 | face_mask = make_elliptic_mask( 306 | face_mask, stretch=CONF.ELLIPTIC_MASK) 307 | # plt.clf(); plt.imshow(face_heatmap.cpu()); plt.show() 308 | # plt.clf(); plt.imshow(face_mask); plt.show() 309 | 310 | if CONF.GAUSS_BLUR_STD is None: 311 | # save binary mask 312 | outpath = ip + "__facemask.png" 313 | Image.fromarray(face_mask).save(outpath) 314 | else: 315 | mask_blur_fn = ImageFilter.GaussianBlur( 316 | radius=CONF.BORDERS_BLUR_STD) 317 | img_blur_fn = ImageFilter.GaussianBlur( 318 | radius=CONF.GAUSS_BLUR_STD) 319 | outpath = ip + "__faceblur.jpg" 320 | mask = Image.fromarray(face_mask).convert("L").filter( 321 | mask_blur_fn) 322 | blur_img = img.filter(img_blur_fn) 323 | mix = Image.composite(blur_img, img, mask) 324 | mix.save(outpath) 325 | print(f"[{i}/{num_imgs}]", "Saved to", outpath) 326 | -------------------------------------------------------------------------------- /emokine/mvnx.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | This module contains functionality concerning the adaption of the 6 | XSENS MVNX (the XML version of their proprietary MVN format) into 7 | our Python setup. 8 | The adaption tries to be as MVN-version-agnostc as possible. Still, 9 | it is possible to validate the file against a given schema. 10 | 11 | The official explanation can be found in section 14.4 of the 12 | *XSENS MVN User Manual*: 13 | 14 | https://usermanual.wiki/Document/MVNUserManual.1147412416.pdf 15 | """ 16 | 17 | 18 | import pandas as pd 19 | import numpy as np 20 | from lxml import etree, objectify # https://lxml.de/validation.html 21 | # 22 | from .utils import str_to_vec 23 | 24 | 25 | # ############################################################################# 26 | # ## GLOBALS 27 | # ############################################################################# 28 | KNOWN_STR_FIELDS = {"tc", "type"} 29 | KNOWN_INT_FIELDS = {"segmentCount", "sensorCount", "jointCount", 30 | "time", "index", "ms"} # "audio_sample" 31 | KNOWN_FLOAT_VEC_FIELDS = {"orientation", "position", "velocity", 32 | "acceleration", "angularVelocity", 33 | "angularAcceleration", "sensorFreeAcceleration", 34 | "sensorMagneticField", "sensorOrientation", 35 | "jointAngle", "jointAngleXZY", "jointAngleErgo", 36 | "centerOfMass"} 37 | 38 | 39 | # ############################################################################# 40 | # ## HELPERS 41 | # ############################################################################# 42 | def process_dict(d, str_fields, int_fields, fvec_fields): 43 | """ 44 | :returns: a copy of the given dict ``d`` where the values (expected str) 45 | whose keys are in the specified fields are converted to the specified 46 | type. E.g. If ``int_fields`` contains the ``index`` string and the given 47 | dict contains the ``index`` key, the corresponding value will be 48 | converted via ``int()``. 49 | """ 50 | result = {} 51 | for k, v in d.items(): 52 | if k in str_fields: 53 | result[k] = str(v) 54 | elif k in int_fields: 55 | result[k] = int(v) 56 | elif k in fvec_fields: 57 | result[k] = str_to_vec(v) 58 | else: 59 | result[k] = v 60 | return result 61 | 62 | 63 | # ############################################################################# 64 | # ## MVNX CLASS 65 | # ############################################################################# 66 | class Mvnx: 67 | """ 68 | This class imports and adapts an XML file (expected to be in MVNX format) 69 | to a Python-friendly representation. See this module's docstring for usage 70 | examples and more information. 71 | """ 72 | def __init__(self, mvnx_path, mvnx_schema_path=None, 73 | str_fields=KNOWN_STR_FIELDS, int_fields=KNOWN_INT_FIELDS, 74 | float_vec_fields=KNOWN_FLOAT_VEC_FIELDS): 75 | """ 76 | :param str mvnx_path: a valid path pointing to the XML file to load 77 | :param str mvnx_schema_path: (optional): if given, the given MVNX will 78 | be validated against this XML schema definition. 79 | :param collection fields: List of strings with field names that are 80 | converted to the specified type when calling ``extract_frame_info``. 81 | """ 82 | self.mvnx_path = mvnx_path 83 | # 84 | mvnx = etree.parse(mvnx_path, etree.ETCompatXMLParser()) 85 | # if a schema is given, load it and validate mvn 86 | if mvnx_schema_path is not None: 87 | self.schema = etree.XMLSchema(file=mvnx_schema_path) 88 | self.schema.assertValid(mvnx) 89 | # 90 | self.mvnx = objectify.fromstring(etree.tostring(mvnx)) 91 | # 92 | self.str_fields = str_fields 93 | self.int_fields = int_fields 94 | self.fvec_fields = float_vec_fields 95 | 96 | def export(self, filepath, pretty_print=True, extra_comment=""): 97 | """ 98 | Saves the current ``mvnx`` attribute to the given file path as XML and 99 | adds the ``self.mvnx.attrib["pythonComment"]`` attribute with 100 | a timestamp. 101 | """ 102 | # 103 | with open(filepath, "w") as f: 104 | if extra_comment: 105 | self.mvnx.attrib["pythonComment"] = str(extra_comment) 106 | s = etree.tostring( 107 | self.mvnx, pretty_print=pretty_print).decode("utf-8") 108 | f.write(s) 109 | print("[Mvnx] exported to", filepath) 110 | if extra_comment: 111 | print(" Comment:", extra_comment) 112 | 113 | # EXTRACTORS: LIKE "GETTERS" BUT RETURN A MODIFIED COPY OF THE CONTENTS 114 | def extract_frame_info(self): 115 | """ 116 | :returns: The tuple ``(frames_metadata, config_frames, normal_frames)`` 117 | """ 118 | f_meta, config_f, normal_f = self.extract_frames(self.mvnx, 119 | self.str_fields, 120 | self.int_fields, 121 | self.fvec_fields) 122 | frames_metadata = f_meta 123 | config_frames = config_f 124 | normal_frames = normal_f 125 | # 126 | assert (frames_metadata["segmentCount"] == 127 | len(self.extract_segments())), "Inconsistent segmentCount?" 128 | return frames_metadata, config_frames, normal_frames 129 | 130 | @staticmethod 131 | def extract_frames(mvnx, str_fields, int_fields, fvec_fields): 132 | """ 133 | The bulk of the MVNX file is the ``mvnx->subject->frames`` section. 134 | This function parses it and returns its information in a 135 | python-friendly format, mainly via the ``process_dict`` function. 136 | 137 | :param mvnx: An XML tree, expected to be in MVNX format 138 | :param collection fields: Collection of strings with field names that 139 | are converted to the specified type (fvec is a vector of floats). 140 | 141 | :returns: a tuple ``(frames_metadata, config_frames, normal_frames)`` 142 | where the metadata is a dict in the form ``{'segmentCount': 23, 143 | 'sensorCount': 17, 'jointCount': 22}``, the config frames are the 144 | first 3 frame entries (expected to contain special config info) 145 | and the normal_frames are all frames starting from the 4th. 146 | Fields found in the given int and vec field lists will be converted 147 | and the rest will remain as XML nodes. 148 | """ 149 | frames_metadata = process_dict(mvnx.subject.frames.attrib, 150 | str_fields, int_fields, fvec_fields) 151 | # first 3 frames are config. types: "identity", "tpose", "tpose-isb" 152 | all_frames = mvnx.subject.frames.getchildren() 153 | # rest of frames contain proper data. type: "normal" 154 | config_frames = [process_dict({**f.__dict__, **f.attrib}, 155 | str_fields, int_fields, fvec_fields) 156 | for f in all_frames[:3]] 157 | normal_frames = [process_dict({**f.__dict__, **f.attrib}, 158 | str_fields, int_fields, fvec_fields) 159 | for f in all_frames[3:]] 160 | return frames_metadata, config_frames, normal_frames 161 | 162 | def extract_segments(self): 163 | """ 164 | :returns: A list of the segment names in ``self.mvnx.subject.segments`` 165 | ordered by id (starting at 1 and incrementing +1). 166 | """ 167 | segments = [ch.attrib["label"] if str(i) == ch.attrib["id"] else None 168 | for i, ch in enumerate( 169 | self.mvnx.subject.segments.iterchildren(), 1)] 170 | assert all([s is not None for s in segments]),\ 171 | "Segments aren't ordered by id?" 172 | return segments 173 | 174 | def extract_joints(self): 175 | """ 176 | :returns: A tuple (X, Y). The element X is a list of the joint names 177 | ordered as they appear in the MVNX file. 178 | The element Y is a list in the original MVNX ordering, in the form 179 | [((seg_ori, point_ori), (seg_dest, point_dest)), ...], where each 180 | element contains 4 strings summarizing the origin->destiny of a 181 | connection. 182 | """ 183 | names, connectors = [], [] 184 | for j in self.mvnx.subject.joints.iterchildren(): 185 | names.append(j.attrib["label"]) 186 | # 187 | seg_ori, point_ori = j.connector1.text.split("/") 188 | seg_dest, point_dest = j.connector2.text.split("/") 189 | connectors.append(((seg_ori, point_ori), (seg_dest, point_dest))) 190 | return names, connectors 191 | 192 | 193 | # ############################################################################# 194 | # ## MVNX TO CSV 195 | # ############################################################################# 196 | class MvnxToTabular: 197 | """ 198 | This class is specialized to convert full-body MVNX data, as provided by 199 | the XSENS system and parsed by the ``Mvnx`` companion class, into tabular 200 | form. 201 | 202 | To use it, instantiate it with the desired MVNX object, and call it with 203 | the desired frame fields to be extracted. Usage example: see the 204 | ``1a_mvnx_to_csv.py`` script. 205 | """ 206 | 207 | ALLOWED_FIELDS = {"orientation", "position", "velocity", "acceleration", 208 | "angularVelocity", "angularAcceleration", "footContacts", 209 | "jointAngle", "centerOfMass", "ms"} 210 | # vectors of NUM_SEGMENTS*n where n is 4 or 3 respectively 211 | SEGMENT_4D_FIELDS = {"orientation"} 212 | SEGMENT_3D_FIELDS = {"position", "velocity", 213 | "acceleration", "angularVelocity", 214 | "angularAcceleration"} 215 | # check manual, 22.7.3, "JointAngle"s are in ZXY if not specified 216 | JOINT_ANGLE_3D_LIST = ["L5S1", "L4L3", "L1T12", "C1Head", 217 | "C7LeftShoulder", "LeftShoulder", "LeftShoulderXZY", 218 | "LeftElbow", "LeftWrist", "Lefthip", "LeftKnee", 219 | "LeftAnkle", "LeftBallFoot", 220 | "C7RightShoulder", "RightShoulder", 221 | "RightShoulderXZY", 222 | "RightElbow", "RightWrist", "Righthip", "RightKnee", 223 | "RightAnkle", "RightBallFoot"] 224 | # a 4D boolean vector 225 | FOOT_CONTACTS = ["left_heel_on_ground", "left_toe_on_ground", 226 | "right_heel_on_ground", "right_toe_on_ground"] 227 | 228 | def __init__(self, mvnx): 229 | """ 230 | :param mvnx: An instance of the ``Mvnx`` class to be converted. 231 | """ 232 | assert mvnx.mvnx.subject.attrib["configuration"] == "FullBody", \ 233 | "This processor works only in FullBody MVNX configurations" 234 | # extract skeleton and frame info 235 | joints, seg_detail, seg_names_sorted = self._parse_skeleton(mvnx) 236 | frames_metadata, config_f, normal_f = mvnx.extract_frame_info() 237 | # 238 | self.mvnx = mvnx 239 | self.seg_names_sorted = seg_names_sorted 240 | self.seg_detail = seg_detail 241 | self.joints = joints 242 | self.n_seg = len(self.seg_names_sorted) 243 | self.n_j = len(self.joints) 244 | # 245 | self.frames_metadata = frames_metadata 246 | self.config_frames = config_f 247 | self.normal_frames = normal_f 248 | 249 | @staticmethod 250 | def _parse_skeleton(mvnx): 251 | """ 252 | The MVNX files provide a description of the 'rig', or skeleton being 253 | captured. This method extracts this definition and returns it in 254 | a convenient form for other (non-protected) methods to use. 255 | """ 256 | # Retrieve every segment keypoint with XYZ positions 257 | segproc = lambda seg: np.fromstring( 258 | seg.pos_b.text, dtype=np.float32, sep=" ") 259 | seg_detail = {(s.attrib["label"], ch.attrib["label"]): segproc(ch) 260 | for s in mvnx.mvnx.subject.segments.iterchildren() 261 | for ch in s.points.iterchildren()} 262 | # Retrieve every joint as a relation of segment details 263 | joints = [(j.attrib["label"], 264 | j.connector1, 265 | seg_detail[tuple(j.connector1.text.split("/"))], 266 | j.connector2, 267 | seg_detail[tuple(j.connector2.text.split("/"))]) 268 | for j in mvnx.mvnx.subject.joints.iterchildren()] 269 | # Retrieve segment names sorted by ID 270 | seg_names_sorted = [s.attrib["label"] for s in 271 | sorted(mvnx.mvnx.subject.segments.iterchildren(), 272 | key=lambda elt: int(elt.attrib["id"]))] 273 | return joints, seg_detail, seg_names_sorted 274 | 275 | def __call__(self, frame_fields=None): 276 | """ 277 | :param frame_fields: Collection of MVNX fields to be extracted as 278 | columns. For full list, see ``self.ALLOWED_FIELDS`` (default if 279 | none given). 280 | """ 281 | # sanity check 282 | if frame_fields is None: 283 | frame_fields = self.ALLOWED_FIELDS 284 | assert all([f in self.ALLOWED_FIELDS for f in frame_fields]), \ 285 | f"Error! allowed fields: {self.ALLOWED_FIELDS}" 286 | # 287 | dataframes = {} 288 | # process 3d segment data 289 | for fld in self.SEGMENT_3D_FIELDS: 290 | columns_3d = ["frame_idx", "ms"] + ["_".join([c, dim]) 291 | for c in self.seg_names_sorted 292 | for dim in ["x", "y", "z"]] 293 | if fld in frame_fields: 294 | print("processing", fld) 295 | df = pd.DataFrame(([frm["index"], frm["ms"]] + frm[fld] 296 | for frm in self.normal_frames), 297 | columns=columns_3d) 298 | dataframes[fld] = df 299 | # process 4d segment data 300 | for fld in self.SEGMENT_4D_FIELDS: 301 | columns_4d = ["frame_idx", "ms"] + [ 302 | "_".join([c, dim]) for c in self.seg_names_sorted 303 | for dim in ["q0", "q1", "q2", "q3"]] 304 | if fld in frame_fields: 305 | print("processing", fld) 306 | df = pd.DataFrame(([frm["index"], frm["ms"]] + frm[fld] 307 | for frm in self.normal_frames), 308 | columns=columns_4d) 309 | dataframes[fld] = df 310 | # process foot contacts 311 | fld = "footContacts" 312 | if fld in frame_fields: 313 | print("processing", fld) 314 | columns_foot = ["frame_idx", "ms"] + self.FOOT_CONTACTS 315 | df = pd.DataFrame(([frm["index"], frm["ms"]] + 316 | [bool(x) for x in frm[fld].text.split(" ")] 317 | for frm in self.normal_frames), 318 | columns=columns_foot) 319 | dataframes[fld] = df 320 | # process center of mass 321 | fld = "centerOfMass" 322 | if fld in frame_fields: 323 | print("processing", fld) 324 | columns_com = ["frame_idx", "ms", "centerOfMass_x", 325 | "centerOfMass_y", "centerOfMass_z"] 326 | df = pd.DataFrame(([frm["index"], frm["ms"]] + 327 | frm[fld] 328 | for frm in self.normal_frames), 329 | columns=columns_com) 330 | dataframes[fld] = df 331 | # 332 | return dataframes 333 | -------------------------------------------------------------------------------- /emokine/utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | """ 6 | General utilities. 7 | """ 8 | 9 | 10 | import os 11 | import math 12 | from datetime import datetime 13 | import json 14 | from ast import literal_eval as make_tuple 15 | from math import floor, ceil 16 | # 17 | import pytz 18 | import numpy as np 19 | from PIL import Image, ImageDraw 20 | import pandas as pd 21 | import torch 22 | from torchvision.io import read_video 23 | import skimage 24 | from skimage.filters import threshold_multiotsu, \ 25 | apply_hysteresis_threshold, median 26 | import matplotlib.pyplot as plt 27 | import matplotlib.font_manager as plt_fm 28 | 29 | 30 | # ############################################################################## 31 | # # IMAGE PROCESSING 32 | # ############################################################################## 33 | def get_lab_distance(img1, img2, out_dtype=np.float32): 34 | """ 35 | :img1: Array of shape ``(h, w, 3)`` in RGB format 36 | :img2: Array of shape ``(h, w, 3)`` in RGB format 37 | :returns: Array of shape ``(h, w)`` with the LAB distances based on 38 | the CIEDE2000 specification ``skimage.color.deltaE_ciede2000``. 39 | """ 40 | img1 = skimage.color.rgb2lab(img1) 41 | img2 = skimage.color.rgb2lab(img2) 42 | dist = skimage.color.deltaE_ciede2000(img1, img2).astype(out_dtype) 43 | return dist 44 | 45 | 46 | def otsu_hist_median(arr, median_size=15): 47 | """ 48 | :param arr: Heatmap with floats as an array of shape ``(h, w)`` 49 | :returns: 50 | """ 51 | num_unique_vals = len(np.unique(arr)) 52 | if num_unique_vals <= 1: 53 | result = np.zeros_like(arr, dtype=bool) 54 | else: 55 | otsu_low_t, otsu_hi_t = threshold_multiotsu(arr) 56 | result = apply_hysteresis_threshold(arr, otsu_low_t, otsu_hi_t) 57 | result = median(result, footprint=np.ones((median_size, median_size))) 58 | return result 59 | 60 | 61 | def make_elliptic_mask(msk, stretch=1.0): 62 | """ 63 | Inspired by: 64 | https://matplotlib.org/stable/gallery/statistics/confidence_ellipse.html 65 | """ 66 | yyy, xxx = msk.nonzero() 67 | if len(yyy) == 0: 68 | return msk 69 | 70 | # extract first and second-order information 71 | center_xy = np.array([xxx.mean(), yyy.mean()]) 72 | cov = np.cov(xxx, yyy) 73 | ew, ev = np.linalg.eigh(cov) 74 | stretch_xy = ew ** 0.5 * stretch * 4 # maybe there is a good reason for x4 75 | 76 | # draw ellipse with given resolution and rotate by corr 77 | patch_wh = np.int32(np.ceil(stretch_xy)) + 2 78 | patch = Image.new("L", tuple(patch_wh)) 79 | draw = ImageDraw.Draw(patch) 80 | draw.ellipse([1, 1, *(patch_wh - 2)], fill="white") 81 | # 82 | angle = np.degrees(np.arctan2(*(ev @ [0, 1]))) 83 | patch = patch.rotate(angle, expand=True) 84 | patch = patch.crop(patch.getbbox()) 85 | 86 | # paste 87 | result = Image.new("1", msk.T.shape) 88 | patch_wh_half = np.float32(patch.size) / 2 89 | patch_xy = [int(x) for x in (center_xy - patch_wh_half).round()] 90 | result.paste(patch, patch_xy) 91 | result = np.array(result) 92 | # plt.clf(); plt.imshow(msk.astype(np.uint8) + result); plt.show() 93 | return result 94 | 95 | 96 | def resize_crop_bbox(x0, x1, y0, y1, 97 | min_x=0, max_x=None, min_y=0, max_y=None, 98 | expansion_ratio=1.0): 99 | """ 100 | Given a bounding box as float coordinates, this function (optionally) 101 | resizes it around its center, rounds the result to integers and 102 | (optionally) clips the result within given boundaries. 103 | The ``x0, x1, y0, y1`` parameters represent the input bounding box 104 | coordinates. The optional ``min`` and ``max`` parameters clip the 105 | output bounding box coordinates. Note that the clipping parameters will 106 | themselves be rounded towards 0 before clipping is applied. 107 | """ 108 | center_x, center_y = 0.5 * (x0 + x1), 0.5 * (y0 + y1) 109 | w, h = x1 - x0, y1 - y0 110 | w_exp_half = expansion_ratio * w * 0.5 111 | h_exp_half = expansion_ratio * h * 0.5 112 | # 113 | x0 = round(center_x - w_exp_half) 114 | x1 = round(center_x + w_exp_half) 115 | y0 = round(center_y - h_exp_half) 116 | y1 = round(center_y + h_exp_half) 117 | # 118 | if min_x is not None: 119 | x0 = max(x0, ceil(min_x)) 120 | if max_x is not None: 121 | x1 = min(x1, floor(max_x)) 122 | if min_y is not None: 123 | y0 = max(y0, ceil(min_y)) 124 | if max_y is not None: 125 | y1 = min(y1, floor(max_y)) 126 | # 127 | return (x0, x1, y0, y1) 128 | 129 | 130 | def resize_hw(t, new_hw): 131 | """ 132 | :param t: Tensor of shape ``(h, w)`` 133 | :returns: Tensor of shape ``new_hw``. 134 | """ 135 | input_type = t.dtype 136 | t = t.type(torch.float32).unsqueeze(0).unsqueeze(0) 137 | t = torch.nn.functional.interpolate(t, new_hw, mode="bilinear")[0, 0] 138 | t = t.type(input_type) 139 | return t 140 | 141 | 142 | # ############################################################################## 143 | # # STRING PROCESSING 144 | # ############################################################################## 145 | def make_timestamp(timezone="Europe/London", with_tz_output=True): 146 | """ 147 | Output example: day, month, year, hour, min, sec, milisecs: 148 | 10_Feb_2018_20:10:16.151 149 | """ 150 | ts = datetime.now(tz=pytz.timezone(timezone)).strftime( 151 | "%Y_%m_%d_%H:%M:%S.%f")[:-3] 152 | if with_tz_output: 153 | return "%s(%s)" % (ts, timezone) 154 | else: 155 | return ts 156 | 157 | 158 | def delta_str_to_seconds(delta_ts, ts_format="%H:%M:%S.%f"): 159 | """ 160 | :param delta_ts: A string representing duration in the given ``ts_format``. 161 | :returns: A float corresponding to the duration in ``delta_ts`` as seconds. 162 | """ 163 | dt = datetime.strptime(delta_ts, ts_format) - datetime.strptime("0", "%S") 164 | result = dt.total_seconds() 165 | return result 166 | 167 | 168 | def seconds_to_hhmmssxxx_str(seconds): 169 | """ 170 | :param float seconds: Float number in seconds 171 | :returns: String in the form ``..hh:mm:ss.xxx``. Note that hh can have 172 | more than 2 digits if enough seconds given. The ``xxx`` miliseconds are 173 | always given in 3 digits, rounded. 174 | """ 175 | hours, rest = divmod(seconds, 3600) 176 | minutes, rest = divmod(rest, 60) 177 | rest = round(rest, 3) 178 | seconds, miliseconds = divmod(round(rest * 1000), 1000) 179 | result = "{:02d}:{:02d}:{:02d}.{:03d}".format( 180 | int(hours), int(minutes), int(seconds), round(miliseconds, 3)) 181 | return result 182 | 183 | 184 | # ############################################################################## 185 | # # I/O 186 | # ############################################################################## 187 | def load_imgs(dir_path, out_hw=None, extension=".png", verbose=False): 188 | """ 189 | """ 190 | img_paths = sorted([os.path.join(dir_path, x) 191 | for x in os.listdir(dir_path) 192 | if x.endswith(extension)]) 193 | # 194 | imgs = [] 195 | ori_imgs = [] 196 | for ip in img_paths: 197 | if verbose: 198 | print("loading", ip) 199 | image = Image.open(ip) 200 | arr = np.array(image) 201 | ori_imgs.append(arr) 202 | if out_hw is not None: 203 | out_h, out_w = out_hw 204 | image = image.resize((out_w, out_h)) 205 | arr = np.array(image) 206 | imgs.append(arr) 207 | # 208 | return imgs, ori_imgs 209 | 210 | 211 | class JsonBlenderPositions: 212 | """ 213 | The Blender script renders ``world_to_camera_view`` coordinates for 214 | a given MVNX sequence and saves them as JSON. This class loads the 215 | JSON file into a ``self.data`` pandas DataFrame for convenient processing. 216 | """ 217 | 218 | def __init__(self, json_path, which_pos="sphere_pos"): 219 | """ 220 | :param str which_pos: One of ``sphere_pos, bone_head, bone_tail``. 221 | """ 222 | with open(json_path, "r") as f: 223 | j = json.load(f) 224 | self.fps = int(j[0]["frame_rate"]) 225 | self.pos_explanation = j[0]["pos_explanation"] 226 | # 227 | flattened = [] 228 | for frame in j[1:]: 229 | flat = self.flatten_frame(frame, which_pos) 230 | flattened.append(flat) 231 | self.data = pd.DataFrame(flattened) 232 | # 233 | self.path = json_path 234 | self.which_pos = which_pos 235 | 236 | @staticmethod 237 | def flatten_frame(frame, which_pos="sphere_pos"): 238 | """ 239 | The frame is expected to be a dictionary with a ``frame`` field 240 | (ignored) and other string fields in the form ``"(kp_name, ico_name)"`` 241 | each one of them containing a dictionary with ``pos->[x, y, z]`` 242 | positions. 243 | 244 | The ``which_pos`` parameter tells which one of those positions to take. 245 | """ 246 | flat = {f"{make_tuple(k)[0]}_{coord}": v[which_pos][i] 247 | for k, v in frame.items() if k != "frame" 248 | for i, coord in enumerate(["x", "y", "z"])} 249 | return flat 250 | 251 | 252 | def load_bw_vid(path, ignore_below=1): 253 | """ 254 | :param path: Assumed to be the path to a black and white video, so that 255 | a threshold exactly between the "black" and the "white" color values. 256 | :param ignore_below: If a given frame in the video has less than this 257 | number of entries above threshold, the frame will be skipped 258 | :returns: A boolean numpy array of shape ``(frames, h, w)``. 259 | """ 260 | vid, _, fpsdict = read_video(path, pts_unit="pts") 261 | fps = fpsdict["video_fps"] 262 | # Convert vid(frames, h, w, ch) to boolean(frames, h, w) 263 | result = [] 264 | for i, f in enumerate(vid, 1): 265 | thresh = f.max().item() / 2 266 | f_bool = (f > thresh).any(dim=-1) 267 | num_true = f_bool.sum().item() 268 | if num_true >= ignore_below: 269 | result.append(f_bool.numpy()) 270 | else: 271 | print(i, "WARNING: ignored frame with", num_true, "active!") 272 | result = np.stack(result) 273 | return result, fps 274 | 275 | 276 | def load_bg_vid(path, bg_rgb=[0, 0, 0], margin_rgb=[0, 0, 0], 277 | ignore_below=1, ignore_above=None): 278 | """ 279 | :param path: Assumed to be the path to a color video, with a background 280 | corresponding to the ``bg_rgb`` color, +/- margin. 281 | :param ignore_below: If a given frame in the video has less than this 282 | number of detected non-background pixels, the frame will be skipped 283 | :param ignore_below: If a given frame in the video has more than this 284 | number of detected non-background pixels, (e.g. if the background is not 285 | static) the frame will be skipped. Optional. 286 | :returns: A boolean numpy array of shape ``(frames, h, w)``, where the 287 | pixels close to ``bg_rgb`` are false, and true otherwise. 288 | """ 289 | vid, _, fpsdict = read_video(path, pts_unit="pts") 290 | fps = fpsdict["video_fps"] 291 | t, h, w, c = vid.shape 292 | bg_rgb = torch.tensor(bg_rgb) 293 | assert c == 3 == len(bg_rgb), "Expected color (RGB) video!" 294 | # 295 | r_lo, r_hi = bg_rgb[0] - margin_rgb[0], bg_rgb[0] + margin_rgb[0] 296 | g_lo, g_hi = bg_rgb[1] - margin_rgb[1], bg_rgb[1] + margin_rgb[1] 297 | b_lo, b_hi = bg_rgb[2] - margin_rgb[2], bg_rgb[2] + margin_rgb[2] 298 | # 299 | result = (r_lo <= vid[:, :, :, 0]) 300 | result &= (r_hi >= vid[:, :, :, 0]) 301 | result = (g_lo <= vid[:, :, :, 1]) 302 | result &= (g_hi >= vid[:, :, :, 1]) 303 | result = (b_lo <= vid[:, :, :, 2]) 304 | result &= (b_hi >= vid[:, :, :, 2]) 305 | result = (~result).numpy() 306 | # 307 | num_fg = [(i, x.sum()) for i, x in enumerate(result)] 308 | idxs = [(i, x) for i, x in num_fg if x >= ignore_below] 309 | if ignore_above is not None: 310 | idxs = [(i, x) for i, x in idxs if x <= ignore_above] 311 | idxs, _ = zip(*idxs) 312 | result = result[idxs, :, :] 313 | # import matplotlib.pyplot as plt 314 | # plt.clf(); plt.imshow(result[10]); plt.show() 315 | return result, fps 316 | 317 | 318 | # ############################################################################## 319 | # # DS 320 | # ############################################################################## 321 | def find_nearest_sorted(array, value): 322 | """ 323 | * WARNING:: 324 | ``array`` must be sorted! 325 | 326 | From https://stackoverflow.com/a/26026189 327 | """ 328 | idx = np.searchsorted(array, value, side="left") 329 | if idx > 0 and (idx == len(array) or 330 | math.fabs(value - array[idx-1]) < 331 | math.fabs(value - array[idx])): 332 | return array[idx-1] 333 | else: 334 | return array[idx] 335 | 336 | 337 | def str_to_vec(x): 338 | """ 339 | Converts a node with a text like '1.23, 2.34 ...' into a list 340 | like [1.23, 2.34, ...] 341 | """ 342 | try: 343 | return [float(y) for y in x.text.split(" ")] 344 | except Exception as e: 345 | print("Could not convert to vector (skip conversion):", e) 346 | return x 347 | 348 | 349 | def split_list_in_equal_chunks(l, chunk_size): 350 | """ 351 | """ 352 | return [l[i:i + chunk_size] for i in range(0, len(l), chunk_size)] 353 | 354 | 355 | # ############################################################################## 356 | # # PLOTTING 357 | # ############################################################################## 358 | class PltFontManager: 359 | """ 360 | Sometimes matplotlib finds the system font paths, but setting them can 361 | still be challenging due to using the wrong name or matplotlib complaining 362 | about missing elements. 363 | 364 | This manager static class is intended to aid with that, by facilitating 365 | the found paths and their corresponding font names, and providing a 366 | ``set_font`` convenience method that will provide the allowed names if 367 | the given one didn't work out. 368 | """ 369 | @staticmethod 370 | def get_font_paths(): 371 | """ 372 | :returns: List of paths to the fonts found by matplotlib in the system. 373 | """ 374 | result = plt_fm.findSystemFonts() 375 | return result 376 | 377 | @classmethod 378 | def get_font_names(cls): 379 | """ 380 | :returns: A tuple ``(fontnames, errors)``, where ``fontnames`` is a 381 | list with the valid font names that can be used, and ``errors`` is 382 | a dictionary in the form ``font_name: error`` containing fonts that 383 | couldn't be successfully loaded. 384 | """ 385 | fpaths = cls.get_font_paths() 386 | fnames = set() 387 | errors = {} 388 | for fp in fpaths: 389 | try: 390 | fname = plt_fm.FontProperties(fname=fp).get_name() 391 | fnames.add(fname) 392 | except Exception as e: 393 | errors[fp] = e 394 | # 395 | return fnames, errors 396 | 397 | @classmethod 398 | def set_font(cls, font_name="Liberation Mono"): 399 | """ 400 | :param font_name: A valid font name as the ones retrieved by the 401 | ``get_font_names`` method 402 | 403 | This method attempts to set the given font. If that is not possible, 404 | it will inform the user and provide a list of available fonts. 405 | """ 406 | try: 407 | plt_fm.findfont(font_name, fallback_to_default=False) 408 | plt.rcParams["font.family"] = font_name 409 | except ValueError as ve: 410 | print(ve) 411 | print("Available fonts:") 412 | print(sorted(cls.get_font_names()[0])) 413 | 414 | 415 | def outlined_hist(ax, data, **hist_kwargs): 416 | """ 417 | Histogram with outlined borders 418 | """ 419 | hist_kwargs.pop("histtype", None) 420 | lines1 = ax.hist(data, histtype="stepfilled", **hist_kwargs) 421 | # 422 | hist_kwargs.pop("alpha", None) 423 | hist_kwargs.pop("color", None) 424 | hist_kwargs.pop("label", None) 425 | lines2 = ax.hist(data, alpha=1, histtype="step", color="black", 426 | **hist_kwargs) 427 | # 428 | return lines1, lines2 429 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # EMOKINE 2 | 3 | 4 | [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.7821844.svg)](https://doi.org/10.5281/zenodo.7821844) 5 | 6 | This repository was developed for the curation of the [EmokineDataset](https://zenodo.org/record/7821844) pilot dataset, but it can be used to curate further datasets of similar characteristics. If you use our work, please consider citing us! 7 | 8 | ``` 9 | @misc{emokine, 10 | title={{EMOKINE}: A Software Package and Computational Framework for Scaling Up the Creation of Highly Controlled Emotional Full-Body Movement Datasets, 11 | author="Christensen, Julia F. and Fernandez, Andres and Smith, Rebecca and Michalareas, Georgios and Yazdi, Sina H. N. and Farahi, Fahima and Schmidt, Eva-Madeleine and Bahmanian, Nasimeh and Roig, Gemma", 12 | year={2024}, 13 | } 14 | ``` 15 | 16 | The rest of this document provides instructions to reproduce the following actions: 17 | 18 | 1. Getting the `EmokineDataset` (optional). 19 | 2. Installing the dependencies to run the `emokine` Python library on Ubuntu (which provides the functionality listed here, and was used to generate the `EmokineDataset`). 20 | 3. Converting MoCap data from [MVNX](https://base.xsens.com/s/article/MVNX-Version-4-File-Structure?language=en_US) (as provided by the XSENS system) into CSV. 21 | 4. Extracting black-and-white silhouettes from dancer videos. 22 | 5. Extracting (detailed or elliptical) face-masks from dancer images, and optionally applying a blur. 23 | 6. Computing the kinematic features present in the `EmokineDataset`. 24 | 7. Reproducing the `technical validation` statistics from our paper (optional). 25 | 26 | 27 | Note that we use the `EmokineDataset` as basis for illustration and reproducibility purposes: Researchers interested in applying this software to data from other systems can do so with little to no adaptions (see explanation below in this README). Our code is released under an open source [LICENSE](LICENSE), and can be adapted, extended and applied to further data. Any feedback and contributions are welcome! 28 | 29 | 30 | Here are visual examples of the rendering capabilities of the library, and below are the installation and usage instructions. 31 | 32 | ![silhouette example](assets/silhouette_example.jpg) 33 | ![faceblur example](assets/faceblur_example.jpg) 34 | ![pld example](assets/pld_example.gif) 35 | 36 | 37 | 38 | ------------------------------------------------------------------------------- 39 | # Software Installation: 40 | 41 | Instructions tested on Ubuntu 20.04, Python 3.9 and Blender 3.4, but should work with other configurations too. 42 | 43 | ### Python dependencies 44 | 45 | ``` 46 | # create, activate and configure environment 47 | conda create -n emokine python==3.9 48 | conda activate emokine 49 | conda install -n base conda-libmamba-solver 50 | conda config --set solver libmamba 51 | 52 | # deep learning-related dependencies 53 | conda install -c conda-forge av 54 | python -m pip install detectron2 -f https://dl.fbaipublicfiles.com/detectron2/wheels/cu113/torch1.10/index.html 55 | conda install pytorch==1.10.1 torchvision==0.11.2 torchaudio==0.10.1 cudatoolkit=11.3 -c pytorch -c conda-forge 56 | conda install -c conda-forge python-wget 57 | conda install -c anaconda scikit-image 58 | pip install git+https://github.com/andres-fr/face_segmentation_pytorch.git 59 | 60 | # dependencies for general processing and analysis 61 | conda install -c conda-forge omegaconf 62 | conda install -c anaconda pandas 63 | conda install -c conda-forge matplotlib 64 | conda install -c anaconda seaborn 65 | conda install -c conda-forge pytz 66 | conda install -c anaconda pillow 67 | conda install -c anaconda lxml 68 | conda install -c conda-forge shapely 69 | conda install -c conda-forge libstdcxx-ng 70 | ``` 71 | 72 | A comprehensive description of a working environment can be seen [here](assets/requirements.txt) 73 | 74 | ### Setting up Blender with our MVNX add-on: 75 | 76 | This is needed if we have XSENS MoCap data in the form of `.mvnx` files and we want to project their 3D positions onto a given camera perspectiven and e.g. generate PLD stimuli (the black background with white dots). The following picture gives an intuition of the process: 77 | 78 | ![blender mvnx rendering](assets/blender_mvnx.png) 79 | 80 | 1. Install Blender following instructions at https://www.blender.org/ (e.g. `sudo snap install blender --classic`). 81 | 2. Install our [Blender MVNX add-on](assets/blender_addon/io_anim_mvnx.zip) by running `blender`, going to `Edit->Preferences->Add-Ons->Install` and selecting the `./assets/blender_addon/io_anim_mvnx.zip` file. 82 | 3. Install dependencies as explained [here](https://blender.stackexchange.com/a/140343/70695) (commands for Ubuntu below). 83 | 84 | ``` 85 | # First find BPYTHON, the path to Blender's python executable. On 3.4, can be found as follows 86 | BPYTHON=`blender -b --python-expr "import sys; print(sys.executable)" | grep "/python/bin/python"` 87 | 88 | # Then install and update pip 89 | $BPYTHON -m ensurepip 90 | $BPYTHON -m pip install --upgrade pip 91 | 92 | # Install the dependencies 93 | $BPYTHON -m pip install pytz 94 | $BPYTHON -m pip install lxml 95 | ``` 96 | 97 | 98 | 99 | ------------------------------------------------------------------------------- 100 | # The `EmokineDataset` 101 | 102 | > :point_right: The pilot dataset can be downloaded from [here](https://zenodo.org/record/7821844). 103 | 104 | `EmokineDataset` features a single dancer performing 63 short sequences, which have been recorded and analyzed in different ways. This pilot dataset is organized in 3 folders: 105 | * `Stimuli`: The sequences are presented in 4 visual presentations that can be used as stimulus in observer experiments: 106 | 1. `Silhouette`: Videos with a white silhouette of the dancer on black background. 107 | 2. `FLD` (Full-Light Display): video recordings with the performer's face blurred out. 108 | 3. `PLD` (Point-Light Display): videos featuring a black background with white circles corresponding to the selected body landmarks. 109 | 4. `Avatar`: Videos produced by the `XSENS` motion capture propietary software, featuring a robot-like avatar performing the captured movements on a light blue background. 110 | * `Data`: In order to facilitate computation and analysis of the stimuli, this pilot dataset also includes several data formats: 111 | 1. `MVNX`: Raw motion capture data directly recorded from the XSENS motion capture system. 112 | 2. `CSV`: Translation of a subset of the `MVNX` sequences into `CSV`, included for easier integration with mainstream analysis software tools). The subset includes the following features: `acceleration`, `angularAcceleration`, `angularVelocity`, `centerOfMass`, `footContacts`, `orientation`, `position` and `velocity`. 113 | 3. `CamPos`: While the MVNX provides 3D positions with respect to a global frame of reference, the `CamPos` [JSON](https://www.json.org/json-en.html) files represent the `position` from the perspective of the camera used to render the `PLD` videos. Specifically, their 3D positions are given with respect to the camera as `(x, y, z)`, where `(x, y)` go from `(0, 0)` (left, bottom) to `(1, 1)` (right, top), and `z` is the distance between the camera and the point in meters. It can be useful to get a 2-dimensional projection of the dancer position (simply ignore `z`). 114 | 4. `Kinematic`: Analysis of a selection of relevant kinematic features, using information from `MVNX`, `Silhouette` and `CamPos`, provided in tabular form. 115 | * `Validation`: Data and experiments reported in our paper as part of the pilot dataset validation, to support its meaningfulness and usefulness for downstream tasks. 116 | 1. `TechVal`: A collection of plots presenting relevant statistics of the pilot dataset. 117 | 2. `ObserverExperiment`: Results in tabular form of an online study conducted with human participants, tasked to recognize emotions of the stimuli and rate their beauty. 118 | 119 | More specifically, the 63 unique sequences are divided into 9 unique choreographies, each one being performed once as an explanation, and then 6 times with different intended emotions (angry, content, fearful, joy, neutral and sad). Once downloaded, the pilot dataset should have the following structure: 120 | 121 | ``` 122 | EmokineDataset 123 | ├── Stimuli 124 | │ ├── Avatar 125 | │ ├── FLD 126 | │ ├── PLD 127 | │ └── Silhouette 128 | ├── Data 129 | │ ├── CamPos 130 | │ ├── CSV 131 | │ ├── Kinematic 132 | │ ├── MVNX 133 | │ └── TechVal 134 | └── Validation 135 | ├── TechVal 136 | └── ObserverExperiment 137 | ``` 138 | 139 | Where each `` of the stimuli, `MVNX`, `CamPos` and `Kinematic` have this structure: 140 | 141 | ``` 142 | 143 | ├── explanation 144 | │ ├── _seq1_explanation. 145 | │ ├── ... 146 | │ └── _seq9_explanation. 147 | ├── _seq1_angry. 148 | ├── _seq1_content. 149 | ├── _seq1_fearful. 150 | ├── _seq1_joy. 151 | ├── _seq1_neutral. 152 | ├── _seq1_sad. 153 | ... 154 | └── _seq9_sad. 155 | ``` 156 | 157 | The `CSV` directory is slightly different, because instead of a single file for each `seq` and `emotion`, it features a folder containing a `.csv` file for each one of the 8 features being extracted (acceleration, velocity...). We refer readers to our paper for more details on the data and companion software. 158 | 159 | 160 | ------------------------------------------------------------------------------- 161 | # Using the `emokine` software 162 | 163 | > :point_right: Although the [`EmokineDataset`](https://zenodo.org/record/7821844) pilot dataset was recorded via the XSENS system, most of the software provided here can be directly applied to data obtained from other motion capture systems with little to no modification. Specifically, only the files `1a_mvsn_to_csv.py` and `1b_mvnx_blender.py` are relevant to the MVNX formatted data. This data is then converted to tabular format (see e.g. [positionExample.csv](assets/positionExample.csv)), which is then consumed by the remaining scripts (together with plain video data whenever needed). Researchers intending to use this software with MoCap data from other systems simply need to ensure that their MoCap data follows the same tabular format, and that they have video data available whenever needed (e.g. for silhouette extraction). 164 | 165 | In this section we showcase the relevant `emokine` functionality, by running the software on the `EmokineDataset`. The purpose of each script, as well sa its input parameters that can be consulted as follows: 166 | 167 | ``` 168 | python -c "x = __import__('1a_mvnx_to_csv'); print(x.__doc__); print(x.ConfDef.__doc__)" 169 | blender -b --python-use-system-env --python 1b_mvnx_blender.py -- -h 170 | python -c "x = __import__('2a_silhouettes'); print(x.__doc__); print(x.ConfDef.__doc__)" 171 | python -c "x = __import__('2b_face_blurs'); print(x.__doc__); print(x.ConfDef.__doc__)" 172 | python -c "x = __import__('3a_kinematic_features'); print(x.__doc__); print(x.ConfDef.__doc__)" 173 | python -c "x = __import__('4a_techval_compute'); print(x.__doc__); print(x.ConfDef.__doc__)" 174 | python -c "x = __import__('4b_techval_plots'); print(x.__doc__); print(x.ConfDef.__doc__)" 175 | ``` 176 | 177 | Check also the source code for more documentation. 178 | 179 | 180 | ### Convert MVNX to CSV 181 | 182 | The following command can be used to generate the `CSV` modality from the `MVNX` modality: 183 | 184 | ``` 185 | for mvnx in EmokineDataset/Data/MVNX/*.mvnx; do python 1a_mvnx_to_csv.py MVNX_PATH=$mvnx OUT_DIR=output/$(basename ${mvnx})/csv; done 186 | ``` 187 | 188 | ### Use Blender to extract camera positions and rendering of PLDs 189 | 190 | The following command generates the `CamPos` modality from the `MVNX` modality: 191 | 192 | ``` 193 | for mvnx in EmokineDataset/Data/MVNX/*.mvnx; do blender -b --python-use-system-env --python 1b_mvnx_blender.py -- -x $mvnx -o "output/"$(basename ${mvnx})"/blender_out"; done 194 | ``` 195 | 196 | To additionally render the `PLD` modality as a set of images, add the `-r` flag as follows: 197 | 198 | ``` 199 | for mvnx in EmokineDataset/Data/MVNX/*.mvnx; do blender -b --python-use-system-env --python 1b_mvnx_blender.py -- -x $mvnx -o output/$(basename ${mvnx})/blender_out -r; done 200 | ``` 201 | 202 | And to render it as video, add the `-v` flag: 203 | 204 | ``` 205 | for mvnx in EmokineDataset/Data/MVNX/*.mvnx; do blender -b --python-use-system-env --python 1b_mvnx_blender.py -- -x $mvnx -o output/$(basename ${mvnx})/blender_out -r -v; done 206 | ``` 207 | 208 | ### Automated rendering of silhouettes and face blurs 209 | 210 | First, the video that we want to process must be converted to a sequence of images, e.g. with the following command: 211 | 212 | ``` 213 | ffmpeg -i -vf yadif -filter:v scale=-1:800 -qscale:v 2 /%05d.jpg 214 | ``` 215 | 216 | Note that to make computation faster in this example we are reducing the output to have a height of `800` pixels. THe `-qscale` parameter determines the output quality (the lower the better, best is 1). 217 | 218 | 219 | Then, we can extract the silhouettes as follows (given a directory with images, results will be stored in that same directory): 220 | 221 | 222 | ``` 223 | python 2a_silhouettes.py MIN_IDX=0 SKIP_N=3 MEDIAN_FILT_SIZE=5 INCLUDE_DL_ABOVE=0.99 IMGS_DIR=<...> 224 | python 2a_silhouettes.py MIN_IDX=1 SKIP_N=3 MEDIAN_FILT_SIZE=5 INCLUDE_DL_ABOVE=0.99 IMGS_DIR=<...> 225 | python 2a_silhouettes.py MIN_IDX=2 SKIP_N=3 MEDIAN_FILT_SIZE=5 INCLUDE_DL_ABOVE=0.99 IMGS_DIR=<...> 226 | ``` 227 | 228 | Note that this can be a memory-hungry process, so in this example we skip one every 3 frames to reduce the size of the sequence, and then we start at indexes 0, 1 and 2 to compute the full sequence. 229 | 230 | **Face mask extraction**: 231 | 232 | ``` 233 | python 2b_face_blurs.py IMGS_DIR=<...> PERSON_THRESHOLD=0.1 KP_THRESHOLD=0.3 FM_THRESHOLD=0.7 BBOX_SIZE=100 234 | ``` 235 | 236 | **Elliptic face mask extraction**: 237 | 238 | ``` 239 | python 2b_face_blurs.py IMGS_DIR=<...> PERSON_THRESHOLD=0.1 KP_THRESHOLD=0.3 FM_THRESHOLD=0.7 BBOX_SIZE=100 ELLIPTIC_MASK=1.1 240 | ``` 241 | 242 | **Face blur extraction**: 243 | 244 | ``` 245 | python 2b_face_blurs.py IMGS_DIR=<...> PERSON_THRESHOLD=0.1 KP_THRESHOLD=0.3 FM_THRESHOLD=0.7 BBOX_SIZE=100 GAUSS_BLUR_STD=8 246 | ``` 247 | 248 | **Elliptic face blur extraction**: 249 | 250 | ``` 251 | python 2b_face_blurs.py IMGS_DIR=<...> PERSON_THRESHOLD=0.1 KP_THRESHOLD=0.3 FM_THRESHOLD=0.7 BBOX_SIZE=100 ELLIPTIC_MASK=1.1 GAUSS_BLUR_STD=8 252 | ``` 253 | 254 | 255 | 256 | ### Kinematic Feature Extraction 257 | 258 | The means to extract kinematic features from a given sequence depend on the specific features and nature of the data. For example, if the data is in MVNX format, we already have the keypoint positions, whereas video data needs a keypoint estimation step. The following table summarizes the features being computed, and their corresponding modalities (more details in the paper): 259 | 260 | ![kinematic features](assets/kinematics_table.png) 261 | 262 | 263 | To exemplify this computation, the following command uses the `3a_kinematic_features.py` script to generate the `Kinematic` modality from the `MVNX, Silhouette, CamPos` modalities in `EMOKINE`: 264 | 265 | ``` 266 | for mvnx in EmokineDataset/Data/MVNX/*.mvnx; do sil=${mvnx//Data\/MVNX\/MVNX/Stimuli\/Silhouette\/Silhouette}; sil=${sil/mvnx/mp4}; json=${mvnx//MVNX/CamPos}; json=${json/mvnx/json}; out=${mvnx//MVNX/Kinematic_rerun}; out=${out/mvnx/csv}; python 3a_kinematic_features.py MVNX_PATH=$mvnx SILHOUETTE_PATH=$sil JSON_PATH=$json CSV_OUTPATH=$out; done 267 | 268 | # Optionally compute also the examples 269 | for mvnx in EmokineDataset/Data/MVNX/explanation/*.mvnx; do sil=${mvnx//Data\/MVNX\/explanation\/MVNX/Stimuli\/Silhouette\/explanation\/Silhouette}; sil=${sil/mvnx/mp4}; json=${mvnx//MVNX/CamPos}; json=${json/mvnx/json}; out=${mvnx//MVNX/Kinematic_rerun}; out=${out/mvnx/csv}; python 3a_kinematic_features.py MVNX_PATH=$mvnx SILHOUETTE_PATH=$sil JSON_PATH=$json CSV_OUTPATH=$out; done 270 | ``` 271 | 272 | 273 | ### Technical validation 274 | 275 | 276 | The following 2 commands compute and plot the technical validation data, respectively. The first command can take around 20 minutes and generates the `techval.pickle` file (around 3.6GB), and the second command generates the plots from `techval.pickle`: 277 | 278 | ``` 279 | python 4a_techval_compute.py EMOKINE_PATH=EmokineDataset OUTPUT_DIR=output NUM_PROCESSES=8 280 | python 4b_techval_plots.py TECHVAL_PICKLE_PATH=output/techval.pickle OUTPUT_DIR=output/ 281 | ``` 282 | 283 | The following command was used to post-process the images (trimming borders). Note that `mogrify` is the in-place version of `imagemagick`, which must be installed in the system for this to work: 284 | 285 | ``` 286 | for i in output/*.png; do mogrify -trim $i; done 287 | ``` 288 | 289 | The results are the technical validation images included in our paper, and also as the `TechVal`supplement in the `EmokineDataset`. Here is an example of the output: 290 | 291 | ![](assets/techval_examples/foreground_stats.png) 292 | 293 | ![](assets/techval_examples/histograms_selection.png) 294 | 295 | ![](assets/techval_examples/kinematics_single.png) 296 | -------------------------------------------------------------------------------- /4b_techval_plots.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding:utf-8 -*- 3 | 4 | 5 | """ 6 | Given the path to the technical validation data file as produced by 7 | ``4a_techval_compute.py``, this script renders and saves to disk the 8 | corresponding validation plots. 9 | 10 | When called on the ``EmokineDataset``, it produces the plots included under 11 | ``data/TechVal``, and used in the EMOKINE paper. 12 | See the ``README``, and its companion script, ``4b_techval_compute.py``, for 13 | more details. 14 | """ 15 | 16 | 17 | import os 18 | import pickle 19 | from collections import defaultdict 20 | # for OmegaConf 21 | from dataclasses import dataclass 22 | from typing import Optional, Tuple 23 | # 24 | from omegaconf import OmegaConf, MISSING 25 | import numpy as np 26 | import pandas as pd 27 | import matplotlib.pyplot as plt 28 | from mpl_toolkits.axes_grid1 import make_axes_locatable 29 | import seaborn as sns 30 | # 31 | from emokine.utils import PltFontManager, outlined_hist 32 | 33 | 34 | # ############################################################################## 35 | # # CLI 36 | # ############################################################################## 37 | @dataclass 38 | class ConfDef: 39 | """ 40 | :cvar EMOKINE_PATH: Path to the ``EmokineDataset``. 41 | :cvar OUTPUT_DIR: Where to store the rendered plots. 42 | :cvar HIST_MARGINS: List of ``(bottom, top, left, right, hspace, wspace)`` 43 | margins, given as ratios. 44 | :cvar HIST_SELECTION: Arbitrary subset of sequences to be featured in the 45 | histogram selection, in the form ``(seq1_angry, seq7_neutral, ...)`` 46 | 47 | The ``EMOTIONS`` and ``KEYPOINTS`` parameters are collections of strings 48 | that, by default, include all of ``EmokineDataset``. The ``KIN_FEATURES`` 49 | and ``KIN_FEATURES_SINGLE`` also cover all the computed features, the only 50 | difference is that the latter is computed only once for the full body (i.e. 51 | it is not a function of the keypoint). 52 | """ 53 | TECHVAL_PICKLE_PATH: str = MISSING 54 | OUTPUT_DIR: Optional[str] = None 55 | # filenames 56 | HIST_SEL_NAME: str = "histograms_selection.png" 57 | HIST_NAME: str = "histograms_{}.png" 58 | FG_NAME: str = "foreground_stats.png" 59 | KIN_NAME: str = "kinematics_{}.png" 60 | # kinematic selection 61 | EMOTIONS: Tuple[str] = ("angry", "content", "fearful", "joy", "neutral", 62 | "sad") 63 | KEYPOINTS: Tuple[str] = ("Pelvis", "L5", "L3", "T12", "T8", "Neck", "Head", 64 | "RightShoulder", "RightUpperArm", "RightForeArm", 65 | "RightHand", "LeftShoulder", "LeftUpperArm", 66 | "LeftForeArm", "LeftHand", "RightUpperLeg", 67 | "RightLowerLeg", "RightFoot", "RightToe", 68 | "LeftUpperLeg", "LeftLowerLeg", "LeftFoot", 69 | "LeftToe" 70 | ) 71 | KIN_FEATURES: Tuple[str] = ("avg. vel.", "avg. accel.", 72 | "avg. angular vel.", "avg. angular accel.", 73 | "dimensionless jerk", 74 | "avg. CoM dist.") 75 | KIN_FEATURES_SINGLE: Tuple[str] = ( 76 | "avg. limb contraction", 77 | "avg. head angle (w.r.t. vertical)", 78 | "avg. head angle (w.r.t. back)", 79 | "avg. QoM", 80 | "avg. convex hull 2D", 81 | "avg. convex hull 3D") 82 | # 83 | HIST_SELECTION: Tuple[str] = ("seq1_angry", "seq1_neutral", 84 | "seq7_angry", "seq7_neutral") 85 | # 86 | DPI: int = 300 87 | TRANSPARENT_BG: bool = True 88 | PLT_TEXT_FONT: str = "Carlito" # "Carlito" "Arial" "Open Sans" 89 | FIG_TITLE_SIZE: int = 50 90 | AXIS_TITLE_SIZE: int = 30 91 | AXIS_TICK_SIZE: int = 15 92 | UNITS_SIZE: int = 25 93 | LEGEND_SIZE: int = 14 94 | # 95 | HIST_MARGINS: Tuple[float] = (0.02, 0.85, 0.05, 0.95, 0.03, 0.08) 96 | FG_MARGINS: Tuple[float] = (0.08, 0.9, 0.07, 0.95, 0.03, 0.2) 97 | FG_ALPHA: float = 0.85 98 | # 99 | KIN_MARGINS: Tuple[float] = (0.08, 0.9, 0.18, 0.98, 0.12, 0.05) 100 | KIN_SINGLE_MARGINS: Tuple[float] = (0.08, 0.9, 0.08, 0.98, 0.3, 0.1) 101 | KIN_ALPHA: float = 0.85 102 | KIN_BINS: int = 50 103 | 104 | 105 | # ############################################################################## 106 | # # MAIN ROUTINE 107 | # ############################################################################## 108 | if __name__ == "__main__": 109 | CONF = OmegaConf.structured(ConfDef()) 110 | cli_conf = OmegaConf.from_cli() 111 | CONF = OmegaConf.merge(CONF, cli_conf) 112 | print("\n\nCONFIGURATION:") 113 | print(OmegaConf.to_yaml(CONF), end="\n\n") 114 | 115 | # ########################################################################## 116 | # # GLOBAL CONFIG 117 | # ########################################################################## 118 | PltFontManager.set_font(CONF.PLT_TEXT_FONT) 119 | plt.rcParams["figure.titlesize"] = CONF.FIG_TITLE_SIZE 120 | plt.rcParams["axes.titlesize"] = CONF.AXIS_TITLE_SIZE 121 | plt.rcParams["xtick.labelsize"] = CONF.AXIS_TICK_SIZE 122 | plt.rcParams["ytick.labelsize"] = CONF.AXIS_TICK_SIZE 123 | plt.rcParams["legend.fontsize"] = CONF.LEGEND_SIZE 124 | # 125 | hist_margins = dict(zip(("bottom", "top", "left", "right", "hspace", 126 | "wspace"), CONF.HIST_MARGINS)) 127 | fg_margins = dict(zip(("bottom", "top", "left", "right", "hspace", 128 | "wspace"), CONF.FG_MARGINS)) 129 | kin_margins = dict(zip(("bottom", "top", "left", "right", "hspace", 130 | "wspace"), CONF.KIN_MARGINS)) 131 | kin_single_margins = dict(zip(("bottom", "top", "left", "right", "hspace", 132 | "wspace"), CONF.KIN_SINGLE_MARGINS)) 133 | 134 | feature_titles = { 135 | "avg. vel.": "Mean Velocity", 136 | "avg. accel.": "Mean Acceleration", 137 | "avg. angular vel.": "Mean Angular Velocity", 138 | "avg. angular accel.": "Mean Angular Acceleration", 139 | "dimensionless jerk": "Dimensionless Jerk", 140 | "avg. limb contraction": "Mean Limb Contraction", 141 | "avg. CoM dist.": "Mean CoM Distance", 142 | "avg. head angle (w.r.t. vertical)": 143 | "avg. head angle (w.r.t. vertical)", 144 | "avg. head angle (w.r.t. back)": 145 | "avg. head angle (w.r.t. back)", 146 | "avg. QoM": "Mean Quantity of Motion", 147 | "avg. convex hull 2D": "Mean 2D Convex Hull", 148 | "avg. convex hull 3D": "Mean 3D Convex Hull"} 149 | 150 | feature_units = { 151 | "avg. vel.": r"$\frac{m}{s}$", 152 | "avg. accel.": r"$\frac{m}{s^2}$", 153 | "avg. angular vel.": r"$\frac{rad}{s}$", 154 | "avg. angular accel.": r"$\frac{rad}{s^2}$", 155 | "dimensionless jerk": "dimensionless", 156 | "avg. limb contraction": "m", 157 | "avg. CoM dist.": "m", 158 | "avg. head angle (w.r.t. vertical)": 159 | "Radians (w.r.t. vertical)", 160 | "avg. head angle (w.r.t. back)": 161 | "Radians (w.r.t. Back)", 162 | "avg. QoM": "dimensionless", 163 | "avg. convex hull 2D": "dimensionless", 164 | "avg. convex hull 3D": r"$m^3$" 165 | } 166 | 167 | cb_colors = sns.color_palette("colorblind", 10) 168 | 169 | # ########################################################################## 170 | # # LOAD TECHVAL DATA 171 | # ########################################################################## 172 | with open(CONF.TECHVAL_PICKLE_PATH, "rb") as f: 173 | techval = pickle.load(f) 174 | # 175 | sil_paths = techval["sil_paths"] 176 | avatar_paths = techval["avatar_paths"] 177 | pld_paths = techval["pld_paths"] 178 | campos_paths = techval["campos_paths"] 179 | kin_paths = techval["kin_paths"] 180 | # 181 | pld_ratios = techval["pld_ratios"] 182 | sil_ratios = techval["sil_ratios"] 183 | avatar_ratios = techval["avatar_ratios"] 184 | campos_ratios = techval["campos_ratios"] 185 | # 186 | sil_hists = techval["sil_hists"] 187 | pld_hists = techval["pld_hists"] 188 | avatar_hists = techval["avatar_hists"] 189 | campos_hists = techval["campos_hists"] 190 | # 191 | campos_xyxy_bounds = techval["campos_xyxy_bounds"] 192 | # 193 | kin_dfs = {p: pd.read_csv(p) for p in kin_paths} 194 | 195 | # ########################################################################## 196 | # # SILHOUETTE/PLD/AVATAR HISTOGRAMS 197 | # ########################################################################## 198 | def imshow_with_cbar(fig, ax, data, cmap="YlGnBu", cbar_ticks=None, 199 | cbar_loc="right", cbar_orientation="vertical", 200 | cbar_size="3%", cbar_pad=0.05): 201 | """ 202 | Plots an ``imshow`` with a colorbar attached to the axis and with 203 | controlled relative size and position. 204 | """ 205 | divider = make_axes_locatable(ax) 206 | cax = divider.append_axes(cbar_loc, size=cbar_size, pad=cbar_pad) 207 | plot = ax.imshow(data, cmap=cmap) 208 | if plot is None: 209 | print("Warning: Empty plot!") 210 | else: 211 | fig.colorbar(plot, cax=cax, orientation=cbar_orientation, 212 | ticks=cbar_ticks) 213 | 214 | def plot_histograms(idxs, cmap="YlGnBu", figsize=(20, 15), 215 | title="Pixel Histograms", cbar_ticks=None): 216 | """ 217 | Ad-hoc helper function that uses the global histograms (``sil_hists, 218 | campos_hists, ...``) to plot multiple histograms in their respective 219 | axes. 220 | :returns: The pair ``(fig, axes)`` with the plots. 221 | """ 222 | fig, axes = plt.subplots(nrows=len(idxs), ncols=4, figsize=figsize) 223 | for row_i, idx in enumerate(idxs): 224 | # plot histograms as images 225 | imshow_with_cbar(fig, axes[row_i, 0], np.log(sil_hists[idx]), 226 | cmap, cbar_ticks, "right", "vertical", "3%", 0.05) 227 | imshow_with_cbar(fig, axes[row_i, 1], np.log(campos_hists[idx]), 228 | cmap, cbar_ticks, "right", "vertical", "3%", 0.05) 229 | imshow_with_cbar(fig, axes[row_i, 2], np.log(pld_hists[idx]), 230 | cmap, cbar_ticks, "right", "vertical", "3%", 0.05) 231 | imshow_with_cbar(fig, axes[row_i, 3], np.log(avatar_hists[idx]), 232 | cmap, cbar_ticks, "right", "vertical", "3%", 0.05) 233 | # remove ticks 234 | for ax in axes[row_i]: 235 | ax.get_xaxis().set_ticks([]) 236 | ax.get_yaxis().set_ticks([]) 237 | # set column titles 238 | axes[0, 0].set_title("Silhouette") 239 | axes[0, 1].set_title("CamPos Convex Hull") 240 | axes[0, 2].set_title("PLD") 241 | axes[0, 3].set_title("Avatar") 242 | # set row titles 243 | for i, idx in enumerate(idxs): 244 | lbl = os.path.splitext(os.path.basename(sil_paths[idx]))[0] 245 | lbl = "_".join(lbl.split("_")[1:]) 246 | axes[i, 0].set_ylabel(lbl, rotation=90, 247 | fontsize=CONF.AXIS_TITLE_SIZE) 248 | # 249 | if title is not None: 250 | fig.suptitle(title) 251 | return fig, axes 252 | 253 | hist_selection_idxs = [[idx for idx, sp in enumerate(sil_paths) 254 | if sel in sp][0] for sel in CONF.HIST_SELECTION] 255 | fig, _ = plot_histograms(hist_selection_idxs, figsize=(20, 12), 256 | title=None, cbar_ticks=[1, 2, 3, 4, 5]) 257 | fig.subplots_adjust(**hist_margins) 258 | 259 | if CONF.OUTPUT_DIR is None: 260 | fig.show() 261 | breakpoint() 262 | else: 263 | outpath = os.path.join(CONF.OUTPUT_DIR, CONF.HIST_SEL_NAME) 264 | fig.savefig(outpath, dpi=CONF.DPI, transparent=CONF.TRANSPARENT_BG) 265 | print("Saved figure to", outpath) 266 | 267 | full_idxs = [list(range(54))[i:i+12] for i in range(0, 54, 12)] 268 | for i, idxs in enumerate(full_idxs, 1): 269 | fig_w, fig_h = 20, (3 * len(idxs)) 270 | fig, _ = plot_histograms(idxs, figsize=(fig_w, fig_h), 271 | title=None, cbar_ticks=[1, 2, 3, 4, 5]) 272 | fig.subplots_adjust(**hist_margins) 273 | 274 | if CONF.OUTPUT_DIR is None: 275 | fig.show() 276 | breakpoint() 277 | else: 278 | outpath = os.path.join(CONF.OUTPUT_DIR, CONF.HIST_NAME.format(i)) 279 | fig.savefig(outpath, dpi=CONF.DPI, transparent=CONF.TRANSPARENT_BG) 280 | print("Saved figure to", outpath) 281 | 282 | # ########################################################################## 283 | # # FOREGROUND STATS (PIXEL RATIO, BOUNDARIES) 284 | # ########################################################################## 285 | def plot_fg(bins=1000, brightness_range=(0, 0.05), alpha=0.7, 286 | figsize=(15, 5), top_margin=0.1, y_log=True): 287 | """ 288 | Ad-hoc helper function that uses the global ratios (``sil_ratios, 289 | pld_ratios, ...``) to plot multiple histograms in their respective 290 | axes. 291 | :returns: The pair ``(fig, axes)`` with the plots. 292 | """ 293 | lim_range = [0, 1] 294 | min_x, min_y, max_x, max_y = zip(*campos_xyxy_bounds) 295 | fig, axes = plt.subplots(ncols=3, figsize=figsize) 296 | # 297 | outlined_hist(axes[0], sil_ratios, bins=bins, range=brightness_range, 298 | log=y_log, alpha=alpha, color=cb_colors[1], 299 | label="Silhouette") 300 | outlined_hist(axes[0], pld_ratios, bins=bins, range=brightness_range, 301 | log=y_log, alpha=alpha, color=cb_colors[2], label="PLD") 302 | outlined_hist(axes[0], avatar_ratios, bins=bins, 303 | range=brightness_range, log=y_log, alpha=alpha, 304 | color=cb_colors[4], label="Avatar") 305 | outlined_hist(axes[1], min_x, bins=bins, range=lim_range, 306 | log=y_log, alpha=alpha, color=cb_colors[0], 307 | label="Horiz. minima") 308 | outlined_hist(axes[1], max_x, bins=bins, range=lim_range, 309 | log=y_log, alpha=alpha, color=cb_colors[5], 310 | label="Horiz. maxima") 311 | outlined_hist(axes[2], min_y, bins=bins, range=lim_range, 312 | log=y_log, alpha=alpha, color=cb_colors[0], 313 | label="Vert. minima") 314 | outlined_hist(axes[2], max_y, bins=bins, range=lim_range, 315 | log=y_log, alpha=alpha, color=cb_colors[5], 316 | label="Vert. maxima") 317 | # 318 | axes[0].set_title("Foreground ratios") 319 | axes[1].set_title("CamPos Limits") 320 | axes[2].set_title("CamPos Limits") 321 | # 322 | axes[0].set_xlim(brightness_range) 323 | axes[1].set_xlim(lim_range) 324 | axes[2].set_xlim(lim_range) 325 | axes[1].set_xticks([0, 0.5, 1]) 326 | axes[1].set_xticklabels(["left", "center", "right"]) 327 | axes[2].set_xticks([0, 0.5, 1]) 328 | axes[2].set_xticklabels(["bottom", "center", "top"]) 329 | # 330 | ylabel = "Number of videos" 331 | if y_log: 332 | ylabel += " (scale: log)" 333 | axes[0].set_ylabel(ylabel, fontsize=CONF.AXIS_TITLE_SIZE) 334 | # 335 | for ax in axes: 336 | y0, y1 = ax.get_ylim() 337 | y1 *= (1 + top_margin) 338 | ax.set_ylim((y0, y1)) 339 | # 340 | axes[0].legend(loc="upper right") 341 | axes[1].legend(loc="upper right") 342 | axes[2].legend(loc="upper right") 343 | # 344 | return fig, axes 345 | 346 | fig, axes = plot_fg(bins=50, brightness_range=(0, 0.05), 347 | alpha=CONF.FG_ALPHA, top_margin=1.2) 348 | fig.subplots_adjust(**fg_margins) 349 | 350 | if CONF.OUTPUT_DIR is None: 351 | fig.show() 352 | breakpoint() 353 | else: 354 | outpath = os.path.join(CONF.OUTPUT_DIR, CONF.FG_NAME) 355 | fig.savefig(outpath, dpi=CONF.DPI, transparent=CONF.TRANSPARENT_BG) 356 | print("Saved figure to", outpath) 357 | 358 | # ########################################################################## 359 | # # PER-JOINT KINEMATICS 360 | # ########################################################################## 361 | def kinematic_histograms(kin_dfs, feature, keypoints, emotions, 362 | num_bins=50, color=cb_colors[0], alpha=0.85, 363 | figsize=(23, 36), 364 | num_xticks=3, xtick_digits=2, xtick_rotation=0, 365 | omit_last_tick=True, 366 | xlabel_pos="center", xlabel_offset=75, 367 | xlabel_bbox_edge="black", 368 | xlabel_bbox_fill=(0.9, 0.9, 0.9)): 369 | """ 370 | Ad-hoc helper function that uses the kinematic data to plot multiple 371 | histograms for a given kinematic ``feature``. The histograms are 372 | arranged such that there is one emotion per column, and one keypoint 373 | per row. 374 | :returns: The pair ``(fig, axes)`` with the plots. 375 | """ 376 | fig, axes = plt.subplots(nrows=len(keypoints), ncols=len(emotions), 377 | figsize=figsize, sharex=True) 378 | # retrieve kinematic data by emotion and keypoint for this feature 379 | data = defaultdict(dict) 380 | for i, emotion in enumerate(emotions): 381 | for j, kp in enumerate(keypoints): 382 | data[emotion][kp] = np.array([df[df["keypoint"] == kp][feature] 383 | for k, df in kin_dfs.items() 384 | if emotion in k]).flatten() 385 | # plot histograms and gather hist modes 386 | modes = defaultdict(dict) 387 | x_ranges = [] 388 | for i, emotion in enumerate(emotions): 389 | # get x-domain range for this row 390 | # min_x = min(min(v) for v in data[emotion].values()) 391 | min_x = 0 392 | max_x = max(max(v) for v in data[emotion].values()) 393 | x_range = (min_x, max_x) 394 | x_ranges.append(x_range) 395 | for j, kp in enumerate(keypoints): 396 | (counts, _, _), _ = outlined_hist( 397 | axes[j][i], data[emotion][kp], bins=num_bins, 398 | range=x_range, density=True, 399 | alpha=alpha, color=color) 400 | modes[emotion][kp] = max(counts) 401 | # set ax row and column titles 402 | for i, emotion in enumerate(emotions): 403 | axes[0][i].set_title(emotion.capitalize()) 404 | for i, kp in enumerate(keypoints): 405 | axes[i][0].set_ylabel( 406 | kp, rotation=0, ha="right", va="center", 407 | fontsize=CONF.AXIS_TITLE_SIZE) # global! 408 | # remove unnecessary axis ticks/labels 409 | for i, axrow in enumerate(axes[::-1]): 410 | for j, ax in enumerate(axrow): 411 | if j >= 0: 412 | ax.set_yticks([]) 413 | ax.set_yticks([], minor=True) 414 | if i >= 1: 415 | ax.get_xaxis().set_visible(False) 416 | # set y ranges for histograms based on max mode 417 | max_mode = max([max(d.values()) for d in modes.values()]) 418 | for axrow in axes: 419 | for ax in axrow: 420 | ax.set_ylim((0, max_mode * 1.05)) 421 | 422 | # set axis x-labels and x-ticks 423 | x_mins, x_maxs = zip(*x_ranges) 424 | for ax in axes[-1]: 425 | xticks = np.linspace(min(x_mins), max(x_maxs), 426 | num_xticks).round(xtick_digits) 427 | if omit_last_tick: 428 | xticks = xticks[:-1] 429 | ax.set_xticks(xticks) 430 | for xtl in ax.get_xticklabels(): 431 | xtl.set_rotation(xtick_rotation) 432 | xlbl = ax.set_xlabel(feature_units[feature], loc=xlabel_pos, 433 | labelpad=xlabel_offset, 434 | fontsize=CONF.UNITS_SIZE) # global! 435 | xlbl.set_bbox(dict(facecolor=xlabel_bbox_fill, 436 | edgecolor=xlabel_bbox_edge)) 437 | # 438 | return fig, axes 439 | 440 | # end of 'kinematic_histograms' def 441 | for feat in CONF.KIN_FEATURES: 442 | # get plot 443 | fig, _ = kinematic_histograms( 444 | kin_dfs, feat, CONF.KEYPOINTS, CONF.EMOTIONS, CONF.KIN_BINS, 445 | cb_colors[0], CONF.KIN_ALPHA, 446 | figsize=(23, (1.5 * len(CONF.KEYPOINTS))), 447 | num_xticks=4, xtick_rotation=0, xlabel_offset=10) 448 | fig.suptitle(feature_titles[feat]) 449 | fig.subplots_adjust(**kin_margins) 450 | 451 | # show/save plot 452 | if CONF.OUTPUT_DIR is None: 453 | fig.show() 454 | breakpoint() 455 | else: 456 | outname = CONF.KIN_NAME.format( 457 | feature_titles[feat].replace(" ", "_")) 458 | outpath = os.path.join(CONF.OUTPUT_DIR, outname) 459 | fig.savefig(outpath, dpi=CONF.DPI, transparent=CONF.TRANSPARENT_BG) 460 | print("Saved figure to", outpath) 461 | 462 | # ########################################################################## 463 | # # KINEMATICS (SINGLE) 464 | # ########################################################################## 465 | def kinematic_histograms_single(kin_dfs, features, emotions, 466 | num_bins=50, color=cb_colors[0], 467 | alpha=0.85, figsize=(23, 36), 468 | num_xticks=4, xtick_digits=2, 469 | xtick_rotation=0, omit_last_tick=True, 470 | xlabel_pos="center", xlabel_offset=15, 471 | xlabel_bbox_edge="black", 472 | xlabel_bbox_fill=(0.9, 0.9, 0.9)): 473 | """ 474 | Variation of ``kinematic_histograms`` adapted for 475 | ``KIN_FEATURES_SINGLE``, where all features are in the same plot, with 476 | one emotion per column and one feature per row (since we don't need to 477 | plot one histogram per keypoint). 478 | :returns: The pair ``(fig, axes)`` with the plots. 479 | """ 480 | fig, axes = plt.subplots(nrows=len(emotions), ncols=len(features), 481 | figsize=figsize, sharex=False) 482 | # retrieve kinematic data by emotion and keypoint for this feature 483 | data = defaultdict(dict) 484 | for i, emotion in enumerate(emotions): 485 | for j, feat in enumerate(features): 486 | data[emotion][feat] = np.array([df[feat][0] 487 | for k, df in kin_dfs.items() 488 | if emotion in k]) 489 | 490 | # gather per-feature x ranges 491 | x_ranges = [] 492 | for i, feat in enumerate(features): 493 | min_x = min(min(v[feat]) for v in data.values()) 494 | max_x = max(max(v[feat]) for v in data.values()) 495 | x_ranges.append((min_x, max_x)) 496 | 497 | # plot histograms and gather hist modes 498 | modes = defaultdict(dict) 499 | for i, emotion in enumerate(emotions): 500 | for j, feat in enumerate(features): 501 | (counts, aa, bb), _ = outlined_hist( 502 | axes[i][j], data[emotion][feat], bins=num_bins, 503 | range=x_ranges[j], density=False, 504 | alpha=alpha, color=color) 505 | modes[emotion][feat] = max(counts) 506 | 507 | # set ax row and column titles 508 | for i, feat in enumerate(features): 509 | if "(" in feat: 510 | feat = feat[:feat.index("(")] 511 | axes[0][i].set_title(feat, y=1.08) 512 | for i, emotion in enumerate(emotions): 513 | axes[i][0].set_ylabel( 514 | emotion, rotation=0, ha="right", va="center", 515 | fontsize=CONF.AXIS_TITLE_SIZE) # global! 516 | 517 | # remove unnecessary axis ticks/labels 518 | for axrow in axes: 519 | for ax in axrow: 520 | ax.set_yticks([]) 521 | ax.set_yticks([], minor=True) 522 | 523 | # set y ranges for histograms based on max mode 524 | max_mode = max([max(d.values()) for d in modes.values()]) 525 | for axrow in axes: 526 | for ax in axrow: 527 | ax.set_ylim((0, max_mode * 1.05)) 528 | 529 | # set axis x-labels and x-ticks 530 | for i, (feat, (min_x, max_x)) in enumerate(zip(features, x_ranges)): 531 | xticks = np.linspace(min_x, max_x, num_xticks).round(xtick_digits) 532 | if omit_last_tick: 533 | xticks = xticks[:-1] 534 | for j in range(len(emotions)): 535 | ax = axes[j][i] 536 | ax.set_xlim(min_x, max_x) 537 | ax.set_xticks(xticks) 538 | xlbl = axes[-1][i].set_xlabel( 539 | feature_units[feat], loc=xlabel_pos, 540 | labelpad=xlabel_offset, 541 | fontsize=CONF.UNITS_SIZE) # global! 542 | xlbl.set_bbox(dict(facecolor=xlabel_bbox_fill, 543 | edgecolor=xlabel_bbox_edge)) 544 | 545 | return fig, axes 546 | 547 | # get plot 548 | fig, _ = kinematic_histograms_single( 549 | kin_dfs, CONF.KIN_FEATURES_SINGLE, CONF.EMOTIONS, CONF.KIN_BINS, 550 | cb_colors[0], CONF.KIN_ALPHA, 551 | figsize=(27, (0.55 * len(CONF.KEYPOINTS)))) 552 | fig.subplots_adjust(**kin_single_margins) 553 | 554 | # show/save plot 555 | if CONF.OUTPUT_DIR is None: 556 | fig.show() 557 | breakpoint() 558 | else: 559 | outname = CONF.KIN_NAME.format("single") 560 | outpath = os.path.join(CONF.OUTPUT_DIR, outname) 561 | fig.savefig(outpath, dpi=CONF.DPI, transparent=CONF.TRANSPARENT_BG) 562 | print("Saved figure to", outpath) 563 | -------------------------------------------------------------------------------- /1b_mvnx_blender.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding:utf-8 -*- 3 | 4 | 5 | """ 6 | MVNX MoCap script for Blender. It has the following functionality: 7 | 8 | 1. Load MVNX sequence into Blender as an animated armature 9 | 2. Attach spheres to the armature's bones 10 | 3. Given a camera, calculate pixel-positions (and depth) of the spheres 11 | 4. Given a camera, render sequence in the form of "dots on a flat background" 12 | 13 | Many of the hardcoded parameters in the GLOBALS section are app-specific and 14 | should be revised for different applications, but this script provides the 15 | functionality and structure to do so with ease. 16 | """ 17 | 18 | 19 | from math import radians, cos, sin 20 | import json 21 | import argparse 22 | import sys 23 | import os 24 | # 25 | import lxml 26 | # blender imports 27 | from mathutils import Vector, Euler # mathutils is a blender package 28 | import bpy 29 | from bpy_extras.object_utils import world_to_camera_view 30 | # 31 | from io_anim_mvnx.mvnx_import import load_mvnx_into_blender 32 | 33 | # Blender aliases 34 | C = bpy.context 35 | D = bpy.data 36 | O = bpy.ops 37 | 38 | 39 | # ############################################################################## 40 | # BLENDER-SPECIFIC HELPERS 41 | # ############################################################################## 42 | class ArgumentParserForBlender(argparse.ArgumentParser): 43 | """ 44 | This class is identical to its parent, except for the parse_args method 45 | (see docstring). It resolves the ambiguity generated when calling 46 | Blender from the CLI with a python script, and both Blender and the script 47 | have arguments. E.g., the following call will make Blender crash because 48 | it will try to process the script's -a and -b flags: 49 | >>> blender --python my_script.py -a 1 -b 2 50 | 51 | To bypass this issue this class uses the fact that Blender will ignore all 52 | arguments given after a double-dash ('--'). The approach is that all 53 | arguments before '--' go to Blender, arguments after go to the script. 54 | The following calls work fine: 55 | >>> blender --python my_script.py -- -a 1 -b 2 56 | >>> blender --python my_script.py -- 57 | """ 58 | 59 | def _get_argv_after_doubledash(self): 60 | """ 61 | Given the sys.argv as a list of strings, this method returns the 62 | sublist right after the '--' element (if present, otherwise returns 63 | an empty list). 64 | """ 65 | try: 66 | idx = sys.argv.index("--") 67 | return sys.argv[idx+1:] # the list after '--' 68 | except ValueError: # '--' not in the list: 69 | return [] 70 | 71 | # overrides superclass 72 | def parse_args(self): 73 | """ 74 | This method is expected to behave identically as in the superclass, 75 | except that the sys.argv list will be pre-processed using 76 | _get_argv_after_doubledash before. See the docstring of the class for 77 | usage examples and details. 78 | """ 79 | return super().parse_args(args=self._get_argv_after_doubledash()) 80 | 81 | 82 | def rot_euler_degrees(rot_x, rot_y, rot_z, order="XYZ"): 83 | """ 84 | Returns an Euler rotation object with the given rotations (in degrees) 85 | and rotation order. 86 | """ 87 | return Euler((radians(rot_x), radians(rot_y), radians(rot_z)), order) 88 | 89 | 90 | def update_scene(): 91 | """ 92 | Sometimes changes don't show up due to lazy evaluation. This function 93 | triggers scene update and recalculation of all changes. 94 | """ 95 | C.scene.update() 96 | 97 | 98 | def save_blenderfile(filepath=D.filepath): 99 | """ 100 | Saves blender file 101 | """ 102 | O.wm.save_as_mainfile(filepath=filepath) 103 | 104 | 105 | def open_blenderfile(filepath=D.filepath): 106 | """ 107 | Saves blender file 108 | """ 109 | O.wm.open_mainfile(filepath=filepath) 110 | 111 | 112 | def set_render_resolution_percentage(p=100): 113 | """ 114 | """ 115 | D.scenes[0].render.resolution_percentage = p 116 | 117 | 118 | def get_obj(obj_name): 119 | """ 120 | Actions like undo or entering edit mode invalidate the object references. 121 | This function returns a reference that is always valid, assuming that the 122 | given obj_name is a key of bpy.data.objects. 123 | """ 124 | return D.objects[obj_name] 125 | 126 | 127 | def select_by_name(*names): 128 | """ 129 | Given a variable number of names as strings, tries to select all existing 130 | objects in D.objects by their name. 131 | """ 132 | for name in names: 133 | try: 134 | D.objects[name].select_set(True) 135 | except Exception as e: 136 | print(e) 137 | 138 | 139 | def deselect_by_name(*names): 140 | """ 141 | Given a variable number of names as strings, tries to select all existing 142 | objects in D.objects by their name. 143 | """ 144 | for name in names: 145 | try: 146 | D.objects[name].select_set(False) 147 | except Exception as e: 148 | print(e) 149 | 150 | 151 | def select_all(action="SELECT"): 152 | """ 153 | Action can be SELECT, DESELECT, INVERT, TOGGLE 154 | """ 155 | bpy.ops.object.select_all(action=action) 156 | 157 | 158 | def delete_selected(): 159 | bpy.ops.object.delete() 160 | 161 | 162 | def set_mode(mode="OBJECT"): 163 | """ 164 | """ 165 | bpy.ops.object.mode_set(mode=mode) 166 | 167 | 168 | def purge_unused_data(categories=[D.meshes, D.materials, D.textures, D.images, 169 | D.curves, D.lights, D.cameras, D.screens]): 170 | """ 171 | Blender objects point to data. E.g., a lamp points to a given data lamp 172 | object. Removing the objects doesn't remove the data, which may lead to 173 | data blocks that aren't being used by anyone. Given an ORDERED collection 174 | of categories, this function removes all unused datablocks. 175 | See https://blender.stackexchange.com/a/102046 176 | """ 177 | for cat in categories: 178 | for block in cat: 179 | if block.users == 0: 180 | cat.remove(block) 181 | 182 | 183 | def set_shading_mode(mode="SOLID", screens=[]): 184 | """ 185 | Performs an action analogous to clicking on the display/shade button of 186 | the 3D view. Mode is one of "RENDERED", "MATERIAL", "SOLID", "WIREFRAME". 187 | The change is applied to the given collection of bpy.data.screens. 188 | If none is given, the function is applied to bpy.context.screen (the 189 | active screen) only. E.g. set all screens to rendered mode: 190 | set_shading_mode("RENDERED", D.screens) 191 | """ 192 | screens = screens if screens else [C.screen] 193 | for s in screens: 194 | for spc in s.areas: 195 | if spc.type == "VIEW_3D": 196 | spc.spaces[0].shading.type = mode 197 | break # we expect at most 1 VIEW_3D space 198 | 199 | 200 | def maximize_layout_3d_area(): 201 | """ 202 | TODO: this function assumes Layout is the bpy.context.workspace. 203 | It does the following: 204 | 1. If there is an area with the given name: 205 | 1.1. Minimizes any other maximized window 206 | 1.2. Maximizes the desired area 207 | """ 208 | screen_name = "Layout" 209 | area_name = "VIEW_3D" 210 | screen = D.screens[screen_name] 211 | for a in screen.areas: 212 | if a.type == area_name: 213 | # If screen is already in some fullscreen mode, revert it 214 | if screen.show_fullscreen: 215 | bpy.ops.screen.back_to_previous() 216 | # Set area to fullscreen (dict admits "window","screen","area") 217 | bpy.ops.screen.screen_full_area({"screen": screen, "area": a}) 218 | break 219 | 220 | 221 | if __name__ == "__main__": 222 | # ########################################################################## 223 | # GLOBALS 224 | # ########################################################################## 225 | parser = ArgumentParserForBlender() 226 | parser.add_argument("-x", "--mvnx", type=str, required=True, 227 | help="MVNX motion capture file to be loaded") 228 | parser.add_argument( 229 | "-S", "--mvnx_schema", type=str, default=None, 230 | help="XML validation schema for the given MVNX (optional)") 231 | parser.add_argument("-r", "--render_headless", action="store_true", 232 | help="If given, this script will actually render out") 233 | parser.add_argument("-o", "--output_dir", default=os.path.expanduser("~"), 234 | type=str, help="Output dir for the renderings") 235 | parser.add_argument("-p", "--resolution_percentage", type=int, default=100, 236 | help="Smaller resolution -> faster (but worse) render") 237 | parser.add_argument("-v", "--as_video", action="store_true", 238 | help="if given, MP4 is exported (noisy background)") 239 | args = parser.parse_args() 240 | 241 | RENDER_HEADLESS = args.render_headless 242 | OUT_DIR = os.path.join(args.output_dir, "") # ensure that it is a dir path 243 | try: 244 | os.makedirs(OUT_DIR) 245 | except FileExistsError: 246 | pass 247 | RESOLUTION_PERCENTAGE = args.resolution_percentage 248 | AS_VIDEO = args.as_video 249 | MVNX_PATH = args.mvnx 250 | SCHEMA_PATH = args.mvnx_schema 251 | MVNX_POSITION = (-0.1, -0.07, 0) 252 | MVNX_ROTATION = (0, 0, radians(-6.6)) # euler angle 253 | 254 | BACKGROUND_COLOR = (0, 0, 0, 0) 255 | DOT_COLOR = (100, 100, 100, 0) 256 | DOT_DIAMETER = 0.04 257 | ALL_KEYPOINTS = {"Pelvis", "L5", "L3", "T12", "T8", "Neck", "Head", 258 | "RightShoulder", "RightUpperArm", "RightForeArm", 259 | "RightHand", 260 | "LeftShoulder", "LeftUpperArm", "LeftForeArm", 261 | "LeftHand", 262 | "RightUpperLeg", "RightLowerLeg", "RightFoot", 263 | "RightToe", 264 | "LeftUpperLeg", "LeftLowerLeg", "LeftFoot", 265 | "LeftToe"} 266 | 267 | # Select which PLDs to display 268 | KEYPOINT_SELECTION = { 269 | # "Head", 270 | # "Pelvis", 271 | # "L5", 272 | "T12", 273 | # "Neck", 274 | "RightShoulder", 275 | "RightUpperArm", 276 | # "RightForeArm", 277 | # "RightHand", 278 | "LeftShoulder", 279 | "LeftUpperArm", 280 | # "LeftForeArm", 281 | # "LeftHand", 282 | "RightUpperLeg", 283 | # "RightLowerLeg", 284 | "RightFoot", 285 | # "RightToe", 286 | "LeftUpperLeg", 287 | # "LeftLowerLeg", 288 | # "LeftToe", 289 | "LeftFoot" 290 | } 291 | USED_BONES = KEYPOINT_SELECTION 292 | INIT_SHADING_MODE = "RENDERED" 293 | INIT_3D_MAXIMIZED = False 294 | # renderer 295 | EEVEE_RENDER_SAMPLES = 8 296 | EEVEE_VIEWPORT_SAMPLES = 0 # 1 297 | EEVEE_VIEWPORT_DENOISING = True 298 | RESOLUTION_WH = (1920, 1080) 299 | # sequencer 300 | FRAME_START = 0 # 1000 # 2 # 1 is T-pose if imported with MakeWalk 301 | FRAME_END = None # 1500 # If not None sequence will be at most this 302 | 303 | # In Blender, x points away from the cam, y to the left and z up 304 | # (right-hand rule). Locations are in meters, rotation in degrees. 305 | # Positive rotation on an axis means counter-clockwise when 306 | # the axis points to the cam. 0,0,0 rotation points straight 307 | # to the bottom. 308 | 309 | # SUN_NAME = "SunLight" 310 | # SUN_LOC = Vector((0.0, 0.0, 10.0)) 311 | # SUN_ROT = rot_euler_degrees(0, 0, 0) 312 | # SUN_STRENGTH = 1.0 # in units relative to a reference sun 313 | 314 | FRONTAL_CAM_NAME = "FrontalCam" 315 | FRONTAL_CAM_DIST = 8.16 316 | FRONTAL_CAM_ANGLE = 0 317 | # cam is on the front-right 318 | # FRONTAL_CAM_LOC = (FRONTAL_CAM_DIST * cos(radians(FRONTAL_CAM_ANGLE)), 319 | # FRONTAL_CAM_DIST * sin(radians(FRONTAL_CAM_ANGLE)), 320 | # 1.6) 321 | FRONTAL_CAM_LOC = (11.96, 0.04, 1) 322 | # Vector((8.16, 0, 1.6)) 323 | # human-like view at the origin 324 | FRONTAL_CAM_ROT = rot_euler_degrees(90.0, 0.0, 90.0) 325 | FRONTAL_CAM_LIGHT_NAME = "FrontalCamLight" 326 | FRONTAL_CAM_LIGHT_LOC = Vector((0.0, 1.0, 0.0)) 327 | FRONTAL_CAM_LIGHT_WATTS = 40.0 # intensity of the bulb in watts 328 | FRONTAL_CAM_LIGHT_SHADOW = False 329 | FRONTAL_CAM_FOCAL_LENGTH = 100 # milimeters 330 | # 331 | SIDE_CAM_NAME = "SideCam" 332 | SIDE_CAM_DIST = FRONTAL_CAM_DIST 333 | SIDE_CAM_ANGLE = -60 334 | SIDE_CAM_LOC = (SIDE_CAM_DIST * cos(radians(SIDE_CAM_ANGLE)), 335 | SIDE_CAM_DIST * sin(radians(SIDE_CAM_ANGLE)), 336 | 1.6) 337 | SIDE_CAM_ROT = rot_euler_degrees(86.0, 0.0, 90 + SIDE_CAM_ANGLE) 338 | SIDE_CAM_LIGHT_NAME = "SideCamLight" 339 | SIDE_CAM_LIGHT_LOC = Vector((0.0, 1.0, 0.0)) 340 | SIDE_CAM_LIGHT_WATTS = 40.0 # intensity of the bulb in watts 341 | SIDE_CAM_LIGHT_SHADOW = False 342 | SIDE_CAM_FOCAL_LENGTH = 100 # milimeters 343 | 344 | # ########################################################################## 345 | # MAIN ROUTINE 346 | # ########################################################################## 347 | # general settings 348 | C.scene.world.node_tree.nodes["Background"].inputs[ 349 | "Color"].default_value = BACKGROUND_COLOR 350 | 351 | # rendering 352 | # In older Blender versions, rendering wouldn't go above 60fps, causing 353 | # inconsistencies between data and renderings since MVNX has 240fps. 354 | # So we decided to skip 3 out of 4 frames, still yielding 60fps, which is 355 | # good for our purposes. 356 | C.scene.render.frame_map_old = 100 357 | C.scene.render.frame_map_new = 100 358 | C.scene.frame_step = 4 # jump by 4 frames 359 | 360 | # 361 | C.scene.render.resolution_x = RESOLUTION_WH[0] 362 | C.scene.render.resolution_y = RESOLUTION_WH[1] 363 | C.scene.render.resolution_percentage = RESOLUTION_PERCENTAGE 364 | C.scene.render.engine = "BLENDER_EEVEE" 365 | C.scene.eevee.use_taa_reprojection = EEVEE_VIEWPORT_DENOISING 366 | C.scene.eevee.taa_render_samples = EEVEE_RENDER_SAMPLES 367 | C.scene.eevee.taa_samples = EEVEE_VIEWPORT_SAMPLES 368 | if AS_VIDEO: 369 | C.scene.render.image_settings.file_format = "FFMPEG" 370 | C.scene.render.ffmpeg.format = "MPEG4" 371 | C.scene.render.ffmpeg.codec = "H264" 372 | C.scene.render.ffmpeg.audio_codec = "NONE" 373 | # also HIGH, MEDIUM, LOSSLESS... 374 | C.scene.render.ffmpeg.constant_rate_factor = "PERC_LOSSLESS" 375 | else: 376 | C.scene.render.image_settings.file_format = "PNG" 377 | C.scene.render.image_settings.color_depth = "16" 378 | # 379 | C.scene.render.image_settings.compression = 50 380 | C.scene.render.image_settings.color_mode = "BW" # "RGBA" 381 | C.scene.render.filepath = OUT_DIR 382 | # 383 | # set all 3D screens to RENDERED mode 384 | set_shading_mode(INIT_SHADING_MODE, D.screens) 385 | 386 | # set fullscreen 387 | if INIT_3D_MAXIMIZED: 388 | maximize_layout_3d_area() 389 | 390 | # select and delete all objects 391 | bpy.ops.object.select_all(action="SELECT") 392 | bpy.ops.object.delete() 393 | purge_unused_data() 394 | 395 | # # add a sun 396 | # bpy.ops.object.light_add(type="SUN", location=SUN_LOC, rotation=SUN_ROT) 397 | # C.object.name = SUN_NAME 398 | # C.object.data.name = SUN_NAME 399 | # C.object.data.energy = SUN_STRENGTH 400 | 401 | # add frontal cam 402 | bpy.ops.object.camera_add(location=FRONTAL_CAM_LOC, 403 | rotation=FRONTAL_CAM_ROT) 404 | frontal_cam = C.object 405 | C.object.name = FRONTAL_CAM_NAME 406 | C.object.data.name = FRONTAL_CAM_NAME 407 | C.object.data.lens = FRONTAL_CAM_FOCAL_LENGTH 408 | # add side cam 409 | bpy.ops.object.camera_add(location=SIDE_CAM_LOC, rotation=SIDE_CAM_ROT) 410 | C.object.name = SIDE_CAM_NAME 411 | C.object.data.name = SIDE_CAM_NAME 412 | C.object.data.lens = SIDE_CAM_FOCAL_LENGTH 413 | 414 | # # add light as a child of cam 415 | # bpy.ops.object.light_add(type="POINT", location=CAM_LIGHT_LOC) 416 | # C.object.name = CAM_LIGHT_NAME 417 | # C.object.data.name = CAM_LIGHT_NAME 418 | # C.object.data.energy = CAM_LIGHT_WATTS 419 | # C.object.parent = get_obj(CAM_NAME) 420 | # C.object.data.use_shadow = False 421 | 422 | try: 423 | armature, mvnx = load_mvnx_into_blender( 424 | C, MVNX_PATH, SCHEMA_PATH, 425 | connectivity="CONNECTED", # "INDIVIDUAL", 426 | scale=1.0, 427 | frame_start=FRAME_START, 428 | inherit_rotations=True, 429 | add_identity_pose=False, 430 | add_t_pose=False, 431 | verbose=True) 432 | mvnx_fps = int(mvnx.mvnx.subject.attrib["frameRate"]) 433 | seq_len = len( 434 | armature.animation_data.action.fcurves[0].keyframe_points) 435 | RENDER_FPS = mvnx_fps // C.scene.frame_step # expected: from 240 to 60 436 | except Exception as e: 437 | if isinstance(e, lxml.etree.DocumentInvalid): 438 | print("MNVX didn't pass given validation schema.", 439 | "Remove schema path to bypass validation.") 440 | else: 441 | print("Something went wrong:", e) 442 | if FRAME_END is not None: 443 | assert FRAME_END > FRAME_START, "Frame end must be bigger than start!" 444 | fe = C.scene.frame_end 445 | new_fe = int(FRAME_END) 446 | if new_fe < fe: 447 | fe = new_fe 448 | else: 449 | # subtract 1 because [start, end] instead of [start, end) and Blender 450 | # would render a frame at the end with no animation 451 | FRAME_END = FRAME_START + seq_len - 1 452 | C.scene.frame_end = FRAME_END 453 | 454 | # readjust armature position 455 | armature.location = MVNX_POSITION 456 | armature.rotation_euler = MVNX_ROTATION 457 | 458 | # define glowing material for all spheres 459 | sphere_material = bpy.data.materials.new(name="sphere_material") 460 | sphere_material.use_nodes = True 461 | bsdf_inputs = sphere_material.node_tree.nodes["Principled BSDF"].inputs 462 | bsdf_inputs["Specular"].default_value = 0 463 | bsdf_inputs["Emission"].default_value = DOT_COLOR 464 | 465 | # frames_metadata, config_frames, normal_frames = mvnx.extract_frame_info() 466 | # fcurves = {pb.name: [] for pb in armature.pose.bones} 467 | # spheres = {} 468 | # for b in armature.data.bones: 469 | # bpy.ops.mesh.primitive_ico_sphere_add(subdivisions=3, 470 | # radius=DOT_DIAMETER / 2, 471 | # location=(0, 0, 0)) 472 | # sph = C.object 473 | # spheres[b.name] = sph 474 | # sph.data.materials.append(sphere_material) 475 | 476 | # print(">>>>>>", fcurves, spheres) 477 | 478 | # This snippet creates icospheres at the tail of used_bones 479 | for b in armature.data.bones: 480 | if b.name in USED_BONES: 481 | bpy.ops.mesh.primitive_ico_sphere_add(subdivisions=3, 482 | radius=DOT_DIAMETER / 2, 483 | location=(0, 0, 0)) 484 | sph = C.object 485 | sph.data.materials.append(sphere_material) 486 | sph.parent = armature 487 | sph.parent_type = "BONE" 488 | sph.parent_bone = b.name 489 | # 490 | # if b.name in ["LeftFoot", "RightFoot"]: 491 | # # set heels to the floor if given 492 | # constraint = bone.constraints.new('COPY_ROTATION') 493 | # b.constraints["Child Of"].use_location_z = False 494 | 495 | # ADD CUSTOM SPHERES: 496 | 497 | # add right upper leg 498 | bpy.ops.mesh.primitive_ico_sphere_add(subdivisions=3, 499 | radius=DOT_DIAMETER / 2, 500 | location=(0, 0, 0)) 501 | sph = C.object 502 | sph.data.materials.append(sphere_material) 503 | sph.parent = armature 504 | sph.parent_type = "BONE" 505 | sph.parent_bone = "RightUpperLeg" 506 | sph.location[1] -= armature.pose.bones["RightUpperLeg"].length 507 | # widen hips 508 | sph.location[2] -= armature.pose.bones["RightUpperLeg"].length * 0.15 509 | 510 | # add left upper leg 511 | bpy.ops.mesh.primitive_ico_sphere_add(subdivisions=3, 512 | radius=DOT_DIAMETER / 2, 513 | location=(0, 0, 0)) 514 | sph = C.object 515 | sph.data.materials.append(sphere_material) 516 | sph.parent = armature 517 | sph.parent_type = "BONE" 518 | sph.parent_bone = "LeftUpperLeg" 519 | sph.location[1] -= armature.pose.bones["LeftUpperLeg"].length 520 | # widen hips 521 | sph.location[2] += armature.pose.bones["LeftUpperLeg"].length * 0.15 522 | 523 | # add right lower leg 524 | bpy.ops.mesh.primitive_ico_sphere_add(subdivisions=3, 525 | radius=DOT_DIAMETER / 2, 526 | location=(0, 0, 0)) 527 | sph = C.object 528 | sph.data.materials.append(sphere_material) 529 | sph.parent = armature 530 | sph.parent_type = "BONE" 531 | sph.parent_bone = "RightLowerLeg" 532 | sph.location[1] += armature.pose.bones["RightLowerLeg"].length * 0.15 533 | 534 | # add left lower leg 535 | bpy.ops.mesh.primitive_ico_sphere_add(subdivisions=3, 536 | radius=DOT_DIAMETER / 2, 537 | location=(0, 0, 0)) 538 | sph = C.object 539 | sph.data.materials.append(sphere_material) 540 | sph.parent = armature 541 | sph.parent_type = "BONE" 542 | sph.parent_bone = "LeftLowerLeg" 543 | sph.location[1] += armature.pose.bones["LeftLowerLeg"].length * 0.15 544 | 545 | # add column 546 | bpy.ops.mesh.primitive_ico_sphere_add(subdivisions=3, 547 | radius=DOT_DIAMETER / 2, 548 | location=(0, 0, 0)) 549 | sph = C.object 550 | sph.data.materials.append(sphere_material) 551 | sph.parent = armature 552 | sph.parent_type = "BONE" 553 | sph.parent_bone = "L5" 554 | sph.location[1] -= armature.pose.bones["L5"].length 555 | 556 | # add head 557 | bpy.ops.mesh.primitive_ico_sphere_add(subdivisions=3, 558 | radius=DOT_DIAMETER / 2, 559 | location=(0, 0, 0)) 560 | sph = C.object 561 | sph.data.materials.append(sphere_material) 562 | sph.parent = armature 563 | sph.parent_type = "BONE" 564 | sph.parent_bone = "Head" 565 | sph.location[1] -= armature.pose.bones["Head"].length * 0.618 566 | 567 | # add right hand 568 | bpy.ops.mesh.primitive_ico_sphere_add(subdivisions=3, 569 | radius=DOT_DIAMETER / 2, 570 | location=(0, 0, 0)) 571 | sph = C.object 572 | sph.data.materials.append(sphere_material) 573 | sph.parent = armature 574 | sph.parent_type = "BONE" 575 | sph.parent_bone = "RightHand" 576 | sph.location[1] -= armature.pose.bones["RightHand"].length * 0.618 577 | 578 | # add left hand 579 | bpy.ops.mesh.primitive_ico_sphere_add(subdivisions=3, 580 | radius=DOT_DIAMETER / 2, 581 | location=(0, 0, 0)) 582 | sph = C.object 583 | sph.data.materials.append(sphere_material) 584 | sph.parent = armature 585 | sph.parent_type = "BONE" 586 | sph.parent_bone = "LeftHand" 587 | sph.location[1] -= armature.pose.bones["LeftHand"].length * 0.618 588 | 589 | # Go over the to-be-rendered frames and record the pixel positions of the 590 | # spheres for a given cam 591 | CAM = frontal_cam 592 | CAM_COORDINATES = [ 593 | {"frame_rate": RENDER_FPS, 594 | "frame_rate_explanation": "The given number is how many " + 595 | "frames of this JSON file takes in 1 second, regardless " + 596 | "of their actual frame value.", 597 | "pos_explanation": "The 3D positions are given with " + 598 | "respect to the camera as (x, y, z), where (x, y) go " + 599 | "from (0, 0) (left, bottom) to (1, 1) (right, top), and " + 600 | "z is the distance between the camera and the point in " + 601 | "world units (usually m)."}] 602 | icospheres = [(v.parent_bone, v) for k, v in D.objects.items() 603 | if "Icosphere" in k] 604 | pose_bones = {pb.name: pb for pb in D.objects[armature.name].pose.bones} 605 | for frame_i in range(C.scene.frame_start, C.scene.frame_end + 1, 606 | C.scene.frame_step): 607 | print("Collecting positions for frame >>>", frame_i) 608 | C.scene.frame_set(frame_i) 609 | # C.scene.frame_current = frame_i 610 | # bpy.context.view_layer.update() # does nothing? 611 | data = {"frame": frame_i} 612 | for ico_bone, ico in icospheres: 613 | # get PoseBone global position 614 | pb_tail = pose_bones[ico_bone].tail 615 | pb_head = pose_bones[ico_bone].head 616 | # get PoseBone cam-relative position (x, y) where x goes from 617 | # left (0) to right(1), and y from bottom (0) to top(1) 618 | pb_tail_cam_xyz = world_to_camera_view( 619 | C.scene, frontal_cam, pb_tail) 620 | pb_head_cam_xyz = world_to_camera_view( 621 | C.scene, frontal_cam, pb_head) 622 | # get Icosphere global pos: contained in the last column 623 | # of "matrix_world" 624 | ico_loc_xyz = Vector(ico.matrix_world.transposed()[3][0:3]) 625 | ico_cam_xyz = world_to_camera_view(C.scene, CAM, ico_loc_xyz) 626 | data[str((ico_bone, ico.name))] = {"bone_tail": pb_tail_cam_xyz[:], 627 | "bone_head": pb_head_cam_xyz[:], 628 | "sphere_pos": ico_cam_xyz[:]} 629 | CAM_COORDINATES.append(data) 630 | C.scene.frame_set(0) 631 | # 632 | mvnx_basename = os.path.splitext(os.path.basename(MVNX_PATH))[0] 633 | json_path = os.path.join(OUT_DIR, mvnx_basename + ".json") 634 | with open(json_path, "w", encoding="utf-8") as f: 635 | json.dump(CAM_COORDINATES, f, ensure_ascii=False, indent=4) 636 | print("Saved camera positions to", json_path) 637 | 638 | C.scene.render.filepath += "mvnx_basename" 639 | # Finally render the sequence 640 | C.scene.camera = frontal_cam 641 | if RENDER_HEADLESS: 642 | C.scene.render.fps = RENDER_FPS 643 | bpy.ops.render.render(animation=True) 644 | # bpy.ops.screen.animation_play() 645 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | Preamble 9 | 10 | The GNU General Public License is a free, copyleft license for 11 | software and other kinds of works. 12 | 13 | The licenses for most software and other practical works are designed 14 | to take away your freedom to share and change the works. By contrast, 15 | the GNU General Public License is intended to guarantee your freedom to 16 | share and change all versions of a program--to make sure it remains free 17 | software for all its users. We, the Free Software Foundation, use the 18 | GNU General Public License for most of our software; it applies also to 19 | any other work released this way by its authors. You can apply it to 20 | your programs, too. 21 | 22 | When we speak of free software, we are referring to freedom, not 23 | price. Our General Public Licenses are designed to make sure that you 24 | have the freedom to distribute copies of free software (and charge for 25 | them if you wish), that you receive source code or can get it if you 26 | want it, that you can change the software or use pieces of it in new 27 | free programs, and that you know you can do these things. 28 | 29 | To protect your rights, we need to prevent others from denying you 30 | these rights or asking you to surrender the rights. Therefore, you have 31 | certain responsibilities if you distribute copies of the software, or if 32 | you modify it: responsibilities to respect the freedom of others. 33 | 34 | For example, if you distribute copies of such a program, whether 35 | gratis or for a fee, you must pass on to the recipients the same 36 | freedoms that you received. You must make sure that they, too, receive 37 | or can get the source code. And you must show them these terms so they 38 | know their rights. 39 | 40 | Developers that use the GNU GPL protect your rights with two steps: 41 | (1) assert copyright on the software, and (2) offer you this License 42 | giving you legal permission to copy, distribute and/or modify it. 43 | 44 | For the developers' and authors' protection, the GPL clearly explains 45 | that there is no warranty for this free software. For both users' and 46 | authors' sake, the GPL requires that modified versions be marked as 47 | changed, so that their problems will not be attributed erroneously to 48 | authors of previous versions. 49 | 50 | Some devices are designed to deny users access to install or run 51 | modified versions of the software inside them, although the manufacturer 52 | can do so. This is fundamentally incompatible with the aim of 53 | protecting users' freedom to change the software. The systematic 54 | pattern of such abuse occurs in the area of products for individuals to 55 | use, which is precisely where it is most unacceptable. Therefore, we 56 | have designed this version of the GPL to prohibit the practice for those 57 | products. If such problems arise substantially in other domains, we 58 | stand ready to extend this provision to those domains in future versions 59 | of the GPL, as needed to protect the freedom of users. 60 | 61 | Finally, every program is threatened constantly by software patents. 62 | States should not allow patents to restrict development and use of 63 | software on general-purpose computers, but in those that do, we wish to 64 | avoid the special danger that patents applied to a free program could 65 | make it effectively proprietary. To prevent this, the GPL assures that 66 | patents cannot be used to render the program non-free. 67 | 68 | The precise terms and conditions for copying, distribution and 69 | modification follow. 70 | 71 | TERMS AND CONDITIONS 72 | 73 | 0. Definitions. 74 | 75 | "This License" refers to version 3 of the GNU General Public License. 76 | 77 | "Copyright" also means copyright-like laws that apply to other kinds of 78 | works, such as semiconductor masks. 79 | 80 | "The Program" refers to any copyrightable work licensed under this 81 | License. Each licensee is addressed as "you". "Licensees" and 82 | "recipients" may be individuals or organizations. 83 | 84 | To "modify" a work means to copy from or adapt all or part of the work 85 | in a fashion requiring copyright permission, other than the making of an 86 | exact copy. The resulting work is called a "modified version" of the 87 | earlier work or a work "based on" the earlier work. 88 | 89 | A "covered work" means either the unmodified Program or a work based 90 | on the Program. 91 | 92 | To "propagate" a work means to do anything with it that, without 93 | permission, would make you directly or secondarily liable for 94 | infringement under applicable copyright law, except executing it on a 95 | computer or modifying a private copy. Propagation includes copying, 96 | distribution (with or without modification), making available to the 97 | public, and in some countries other activities as well. 98 | 99 | To "convey" a work means any kind of propagation that enables other 100 | parties to make or receive copies. Mere interaction with a user through 101 | a computer network, with no transfer of a copy, is not conveying. 102 | 103 | An interactive user interface displays "Appropriate Legal Notices" 104 | to the extent that it includes a convenient and prominently visible 105 | feature that (1) displays an appropriate copyright notice, and (2) 106 | tells the user that there is no warranty for the work (except to the 107 | extent that warranties are provided), that licensees may convey the 108 | work under this License, and how to view a copy of this License. If 109 | the interface presents a list of user commands or options, such as a 110 | menu, a prominent item in the list meets this criterion. 111 | 112 | 1. Source Code. 113 | 114 | The "source code" for a work means the preferred form of the work 115 | for making modifications to it. "Object code" means any non-source 116 | form of a work. 117 | 118 | A "Standard Interface" means an interface that either is an official 119 | standard defined by a recognized standards body, or, in the case of 120 | interfaces specified for a particular programming language, one that 121 | is widely used among developers working in that language. 122 | 123 | The "System Libraries" of an executable work include anything, other 124 | than the work as a whole, that (a) is included in the normal form of 125 | packaging a Major Component, but which is not part of that Major 126 | Component, and (b) serves only to enable use of the work with that 127 | Major Component, or to implement a Standard Interface for which an 128 | implementation is available to the public in source code form. A 129 | "Major Component", in this context, means a major essential component 130 | (kernel, window system, and so on) of the specific operating system 131 | (if any) on which the executable work runs, or a compiler used to 132 | produce the work, or an object code interpreter used to run it. 133 | 134 | The "Corresponding Source" for a work in object code form means all 135 | the source code needed to generate, install, and (for an executable 136 | work) run the object code and to modify the work, including scripts to 137 | control those activities. However, it does not include the work's 138 | System Libraries, or general-purpose tools or generally available free 139 | programs which are used unmodified in performing those activities but 140 | which are not part of the work. For example, Corresponding Source 141 | includes interface definition files associated with source files for 142 | the work, and the source code for shared libraries and dynamically 143 | linked subprograms that the work is specifically designed to require, 144 | such as by intimate data communication or control flow between those 145 | subprograms and other parts of the work. 146 | 147 | The Corresponding Source need not include anything that users 148 | can regenerate automatically from other parts of the Corresponding 149 | Source. 150 | 151 | The Corresponding Source for a work in source code form is that 152 | same work. 153 | 154 | 2. Basic Permissions. 155 | 156 | All rights granted under this License are granted for the term of 157 | copyright on the Program, and are irrevocable provided the stated 158 | conditions are met. This License explicitly affirms your unlimited 159 | permission to run the unmodified Program. The output from running a 160 | covered work is covered by this License only if the output, given its 161 | content, constitutes a covered work. This License acknowledges your 162 | rights of fair use or other equivalent, as provided by copyright law. 163 | 164 | You may make, run and propagate covered works that you do not 165 | convey, without conditions so long as your license otherwise remains 166 | in force. You may convey covered works to others for the sole purpose 167 | of having them make modifications exclusively for you, or provide you 168 | with facilities for running those works, provided that you comply with 169 | the terms of this License in conveying all material for which you do 170 | not control copyright. Those thus making or running the covered works 171 | for you must do so exclusively on your behalf, under your direction 172 | and control, on terms that prohibit them from making any copies of 173 | your copyrighted material outside their relationship with you. 174 | 175 | Conveying under any other circumstances is permitted solely under 176 | the conditions stated below. Sublicensing is not allowed; section 10 177 | makes it unnecessary. 178 | 179 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law. 180 | 181 | No covered work shall be deemed part of an effective technological 182 | measure under any applicable law fulfilling obligations under article 183 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or 184 | similar laws prohibiting or restricting circumvention of such 185 | measures. 186 | 187 | When you convey a covered work, you waive any legal power to forbid 188 | circumvention of technological measures to the extent such circumvention 189 | is effected by exercising rights under this License with respect to 190 | the covered work, and you disclaim any intention to limit operation or 191 | modification of the work as a means of enforcing, against the work's 192 | users, your or third parties' legal rights to forbid circumvention of 193 | technological measures. 194 | 195 | 4. Conveying Verbatim Copies. 196 | 197 | You may convey verbatim copies of the Program's source code as you 198 | receive it, in any medium, provided that you conspicuously and 199 | appropriately publish on each copy an appropriate copyright notice; 200 | keep intact all notices stating that this License and any 201 | non-permissive terms added in accord with section 7 apply to the code; 202 | keep intact all notices of the absence of any warranty; and give all 203 | recipients a copy of this License along with the Program. 204 | 205 | You may charge any price or no price for each copy that you convey, 206 | and you may offer support or warranty protection for a fee. 207 | 208 | 5. Conveying Modified Source Versions. 209 | 210 | You may convey a work based on the Program, or the modifications to 211 | produce it from the Program, in the form of source code under the 212 | terms of section 4, provided that you also meet all of these conditions: 213 | 214 | a) The work must carry prominent notices stating that you modified 215 | it, and giving a relevant date. 216 | 217 | b) The work must carry prominent notices stating that it is 218 | released under this License and any conditions added under section 219 | 7. This requirement modifies the requirement in section 4 to 220 | "keep intact all notices". 221 | 222 | c) You must license the entire work, as a whole, under this 223 | License to anyone who comes into possession of a copy. This 224 | License will therefore apply, along with any applicable section 7 225 | additional terms, to the whole of the work, and all its parts, 226 | regardless of how they are packaged. This License gives no 227 | permission to license the work in any other way, but it does not 228 | invalidate such permission if you have separately received it. 229 | 230 | d) If the work has interactive user interfaces, each must display 231 | Appropriate Legal Notices; however, if the Program has interactive 232 | interfaces that do not display Appropriate Legal Notices, your 233 | work need not make them do so. 234 | 235 | A compilation of a covered work with other separate and independent 236 | works, which are not by their nature extensions of the covered work, 237 | and which are not combined with it such as to form a larger program, 238 | in or on a volume of a storage or distribution medium, is called an 239 | "aggregate" if the compilation and its resulting copyright are not 240 | used to limit the access or legal rights of the compilation's users 241 | beyond what the individual works permit. Inclusion of a covered work 242 | in an aggregate does not cause this License to apply to the other 243 | parts of the aggregate. 244 | 245 | 6. Conveying Non-Source Forms. 246 | 247 | You may convey a covered work in object code form under the terms 248 | of sections 4 and 5, provided that you also convey the 249 | machine-readable Corresponding Source under the terms of this License, 250 | in one of these ways: 251 | 252 | a) Convey the object code in, or embodied in, a physical product 253 | (including a physical distribution medium), accompanied by the 254 | Corresponding Source fixed on a durable physical medium 255 | customarily used for software interchange. 256 | 257 | b) Convey the object code in, or embodied in, a physical product 258 | (including a physical distribution medium), accompanied by a 259 | written offer, valid for at least three years and valid for as 260 | long as you offer spare parts or customer support for that product 261 | model, to give anyone who possesses the object code either (1) a 262 | copy of the Corresponding Source for all the software in the 263 | product that is covered by this License, on a durable physical 264 | medium customarily used for software interchange, for a price no 265 | more than your reasonable cost of physically performing this 266 | conveying of source, or (2) access to copy the 267 | Corresponding Source from a network server at no charge. 268 | 269 | c) Convey individual copies of the object code with a copy of the 270 | written offer to provide the Corresponding Source. This 271 | alternative is allowed only occasionally and noncommercially, and 272 | only if you received the object code with such an offer, in accord 273 | with subsection 6b. 274 | 275 | d) Convey the object code by offering access from a designated 276 | place (gratis or for a charge), and offer equivalent access to the 277 | Corresponding Source in the same way through the same place at no 278 | further charge. You need not require recipients to copy the 279 | Corresponding Source along with the object code. If the place to 280 | copy the object code is a network server, the Corresponding Source 281 | may be on a different server (operated by you or a third party) 282 | that supports equivalent copying facilities, provided you maintain 283 | clear directions next to the object code saying where to find the 284 | Corresponding Source. Regardless of what server hosts the 285 | Corresponding Source, you remain obligated to ensure that it is 286 | available for as long as needed to satisfy these requirements. 287 | 288 | e) Convey the object code using peer-to-peer transmission, provided 289 | you inform other peers where the object code and Corresponding 290 | Source of the work are being offered to the general public at no 291 | charge under subsection 6d. 292 | 293 | A separable portion of the object code, whose source code is excluded 294 | from the Corresponding Source as a System Library, need not be 295 | included in conveying the object code work. 296 | 297 | A "User Product" is either (1) a "consumer product", which means any 298 | tangible personal property which is normally used for personal, family, 299 | or household purposes, or (2) anything designed or sold for incorporation 300 | into a dwelling. In determining whether a product is a consumer product, 301 | doubtful cases shall be resolved in favor of coverage. For a particular 302 | product received by a particular user, "normally used" refers to a 303 | typical or common use of that class of product, regardless of the status 304 | of the particular user or of the way in which the particular user 305 | actually uses, or expects or is expected to use, the product. A product 306 | is a consumer product regardless of whether the product has substantial 307 | commercial, industrial or non-consumer uses, unless such uses represent 308 | the only significant mode of use of the product. 309 | 310 | "Installation Information" for a User Product means any methods, 311 | procedures, authorization keys, or other information required to install 312 | and execute modified versions of a covered work in that User Product from 313 | a modified version of its Corresponding Source. The information must 314 | suffice to ensure that the continued functioning of the modified object 315 | code is in no case prevented or interfered with solely because 316 | modification has been made. 317 | 318 | If you convey an object code work under this section in, or with, or 319 | specifically for use in, a User Product, and the conveying occurs as 320 | part of a transaction in which the right of possession and use of the 321 | User Product is transferred to the recipient in perpetuity or for a 322 | fixed term (regardless of how the transaction is characterized), the 323 | Corresponding Source conveyed under this section must be accompanied 324 | by the Installation Information. But this requirement does not apply 325 | if neither you nor any third party retains the ability to install 326 | modified object code on the User Product (for example, the work has 327 | been installed in ROM). 328 | 329 | The requirement to provide Installation Information does not include a 330 | requirement to continue to provide support service, warranty, or updates 331 | for a work that has been modified or installed by the recipient, or for 332 | the User Product in which it has been modified or installed. Access to a 333 | network may be denied when the modification itself materially and 334 | adversely affects the operation of the network or violates the rules and 335 | protocols for communication across the network. 336 | 337 | Corresponding Source conveyed, and Installation Information provided, 338 | in accord with this section must be in a format that is publicly 339 | documented (and with an implementation available to the public in 340 | source code form), and must require no special password or key for 341 | unpacking, reading or copying. 342 | 343 | 7. Additional Terms. 344 | 345 | "Additional permissions" are terms that supplement the terms of this 346 | License by making exceptions from one or more of its conditions. 347 | Additional permissions that are applicable to the entire Program shall 348 | be treated as though they were included in this License, to the extent 349 | that they are valid under applicable law. If additional permissions 350 | apply only to part of the Program, that part may be used separately 351 | under those permissions, but the entire Program remains governed by 352 | this License without regard to the additional permissions. 353 | 354 | When you convey a copy of a covered work, you may at your option 355 | remove any additional permissions from that copy, or from any part of 356 | it. (Additional permissions may be written to require their own 357 | removal in certain cases when you modify the work.) You may place 358 | additional permissions on material, added by you to a covered work, 359 | for which you have or can give appropriate copyright permission. 360 | 361 | Notwithstanding any other provision of this License, for material you 362 | add to a covered work, you may (if authorized by the copyright holders of 363 | that material) supplement the terms of this License with terms: 364 | 365 | a) Disclaiming warranty or limiting liability differently from the 366 | terms of sections 15 and 16 of this License; or 367 | 368 | b) Requiring preservation of specified reasonable legal notices or 369 | author attributions in that material or in the Appropriate Legal 370 | Notices displayed by works containing it; or 371 | 372 | c) Prohibiting misrepresentation of the origin of that material, or 373 | requiring that modified versions of such material be marked in 374 | reasonable ways as different from the original version; or 375 | 376 | d) Limiting the use for publicity purposes of names of licensors or 377 | authors of the material; or 378 | 379 | e) Declining to grant rights under trademark law for use of some 380 | trade names, trademarks, or service marks; or 381 | 382 | f) Requiring indemnification of licensors and authors of that 383 | material by anyone who conveys the material (or modified versions of 384 | it) with contractual assumptions of liability to the recipient, for 385 | any liability that these contractual assumptions directly impose on 386 | those licensors and authors. 387 | 388 | All other non-permissive additional terms are considered "further 389 | restrictions" within the meaning of section 10. If the Program as you 390 | received it, or any part of it, contains a notice stating that it is 391 | governed by this License along with a term that is a further 392 | restriction, you may remove that term. If a license document contains 393 | a further restriction but permits relicensing or conveying under this 394 | License, you may add to a covered work material governed by the terms 395 | of that license document, provided that the further restriction does 396 | not survive such relicensing or conveying. 397 | 398 | If you add terms to a covered work in accord with this section, you 399 | must place, in the relevant source files, a statement of the 400 | additional terms that apply to those files, or a notice indicating 401 | where to find the applicable terms. 402 | 403 | Additional terms, permissive or non-permissive, may be stated in the 404 | form of a separately written license, or stated as exceptions; 405 | the above requirements apply either way. 406 | 407 | 8. Termination. 408 | 409 | You may not propagate or modify a covered work except as expressly 410 | provided under this License. Any attempt otherwise to propagate or 411 | modify it is void, and will automatically terminate your rights under 412 | this License (including any patent licenses granted under the third 413 | paragraph of section 11). 414 | 415 | However, if you cease all violation of this License, then your 416 | license from a particular copyright holder is reinstated (a) 417 | provisionally, unless and until the copyright holder explicitly and 418 | finally terminates your license, and (b) permanently, if the copyright 419 | holder fails to notify you of the violation by some reasonable means 420 | prior to 60 days after the cessation. 421 | 422 | Moreover, your license from a particular copyright holder is 423 | reinstated permanently if the copyright holder notifies you of the 424 | violation by some reasonable means, this is the first time you have 425 | received notice of violation of this License (for any work) from that 426 | copyright holder, and you cure the violation prior to 30 days after 427 | your receipt of the notice. 428 | 429 | Termination of your rights under this section does not terminate the 430 | licenses of parties who have received copies or rights from you under 431 | this License. If your rights have been terminated and not permanently 432 | reinstated, you do not qualify to receive new licenses for the same 433 | material under section 10. 434 | 435 | 9. Acceptance Not Required for Having Copies. 436 | 437 | You are not required to accept this License in order to receive or 438 | run a copy of the Program. Ancillary propagation of a covered work 439 | occurring solely as a consequence of using peer-to-peer transmission 440 | to receive a copy likewise does not require acceptance. However, 441 | nothing other than this License grants you permission to propagate or 442 | modify any covered work. These actions infringe copyright if you do 443 | not accept this License. Therefore, by modifying or propagating a 444 | covered work, you indicate your acceptance of this License to do so. 445 | 446 | 10. Automatic Licensing of Downstream Recipients. 447 | 448 | Each time you convey a covered work, the recipient automatically 449 | receives a license from the original licensors, to run, modify and 450 | propagate that work, subject to this License. You are not responsible 451 | for enforcing compliance by third parties with this License. 452 | 453 | An "entity transaction" is a transaction transferring control of an 454 | organization, or substantially all assets of one, or subdividing an 455 | organization, or merging organizations. If propagation of a covered 456 | work results from an entity transaction, each party to that 457 | transaction who receives a copy of the work also receives whatever 458 | licenses to the work the party's predecessor in interest had or could 459 | give under the previous paragraph, plus a right to possession of the 460 | Corresponding Source of the work from the predecessor in interest, if 461 | the predecessor has it or can get it with reasonable efforts. 462 | 463 | You may not impose any further restrictions on the exercise of the 464 | rights granted or affirmed under this License. For example, you may 465 | not impose a license fee, royalty, or other charge for exercise of 466 | rights granted under this License, and you may not initiate litigation 467 | (including a cross-claim or counterclaim in a lawsuit) alleging that 468 | any patent claim is infringed by making, using, selling, offering for 469 | sale, or importing the Program or any portion of it. 470 | 471 | 11. Patents. 472 | 473 | A "contributor" is a copyright holder who authorizes use under this 474 | License of the Program or a work on which the Program is based. The 475 | work thus licensed is called the contributor's "contributor version". 476 | 477 | A contributor's "essential patent claims" are all patent claims 478 | owned or controlled by the contributor, whether already acquired or 479 | hereafter acquired, that would be infringed by some manner, permitted 480 | by this License, of making, using, or selling its contributor version, 481 | but do not include claims that would be infringed only as a 482 | consequence of further modification of the contributor version. For 483 | purposes of this definition, "control" includes the right to grant 484 | patent sublicenses in a manner consistent with the requirements of 485 | this License. 486 | 487 | Each contributor grants you a non-exclusive, worldwide, royalty-free 488 | patent license under the contributor's essential patent claims, to 489 | make, use, sell, offer for sale, import and otherwise run, modify and 490 | propagate the contents of its contributor version. 491 | 492 | In the following three paragraphs, a "patent license" is any express 493 | agreement or commitment, however denominated, not to enforce a patent 494 | (such as an express permission to practice a patent or covenant not to 495 | sue for patent infringement). To "grant" such a patent license to a 496 | party means to make such an agreement or commitment not to enforce a 497 | patent against the party. 498 | 499 | If you convey a covered work, knowingly relying on a patent license, 500 | and the Corresponding Source of the work is not available for anyone 501 | to copy, free of charge and under the terms of this License, through a 502 | publicly available network server or other readily accessible means, 503 | then you must either (1) cause the Corresponding Source to be so 504 | available, or (2) arrange to deprive yourself of the benefit of the 505 | patent license for this particular work, or (3) arrange, in a manner 506 | consistent with the requirements of this License, to extend the patent 507 | license to downstream recipients. "Knowingly relying" means you have 508 | actual knowledge that, but for the patent license, your conveying the 509 | covered work in a country, or your recipient's use of the covered work 510 | in a country, would infringe one or more identifiable patents in that 511 | country that you have reason to believe are valid. 512 | 513 | If, pursuant to or in connection with a single transaction or 514 | arrangement, you convey, or propagate by procuring conveyance of, a 515 | covered work, and grant a patent license to some of the parties 516 | receiving the covered work authorizing them to use, propagate, modify 517 | or convey a specific copy of the covered work, then the patent license 518 | you grant is automatically extended to all recipients of the covered 519 | work and works based on it. 520 | 521 | A patent license is "discriminatory" if it does not include within 522 | the scope of its coverage, prohibits the exercise of, or is 523 | conditioned on the non-exercise of one or more of the rights that are 524 | specifically granted under this License. You may not convey a covered 525 | work if you are a party to an arrangement with a third party that is 526 | in the business of distributing software, under which you make payment 527 | to the third party based on the extent of your activity of conveying 528 | the work, and under which the third party grants, to any of the 529 | parties who would receive the covered work from you, a discriminatory 530 | patent license (a) in connection with copies of the covered work 531 | conveyed by you (or copies made from those copies), or (b) primarily 532 | for and in connection with specific products or compilations that 533 | contain the covered work, unless you entered into that arrangement, 534 | or that patent license was granted, prior to 28 March 2007. 535 | 536 | Nothing in this License shall be construed as excluding or limiting 537 | any implied license or other defenses to infringement that may 538 | otherwise be available to you under applicable patent law. 539 | 540 | 12. No Surrender of Others' Freedom. 541 | 542 | If conditions are imposed on you (whether by court order, agreement or 543 | otherwise) that contradict the conditions of this License, they do not 544 | excuse you from the conditions of this License. If you cannot convey a 545 | covered work so as to satisfy simultaneously your obligations under this 546 | License and any other pertinent obligations, then as a consequence you may 547 | not convey it at all. For example, if you agree to terms that obligate you 548 | to collect a royalty for further conveying from those to whom you convey 549 | the Program, the only way you could satisfy both those terms and this 550 | License would be to refrain entirely from conveying the Program. 551 | 552 | 13. Use with the GNU Affero General Public License. 553 | 554 | Notwithstanding any other provision of this License, you have 555 | permission to link or combine any covered work with a work licensed 556 | under version 3 of the GNU Affero General Public License into a single 557 | combined work, and to convey the resulting work. The terms of this 558 | License will continue to apply to the part which is the covered work, 559 | but the special requirements of the GNU Affero General Public License, 560 | section 13, concerning interaction through a network will apply to the 561 | combination as such. 562 | 563 | 14. Revised Versions of this License. 564 | 565 | The Free Software Foundation may publish revised and/or new versions of 566 | the GNU General Public License from time to time. Such new versions will 567 | be similar in spirit to the present version, but may differ in detail to 568 | address new problems or concerns. 569 | 570 | Each version is given a distinguishing version number. If the 571 | Program specifies that a certain numbered version of the GNU General 572 | Public License "or any later version" applies to it, you have the 573 | option of following the terms and conditions either of that numbered 574 | version or of any later version published by the Free Software 575 | Foundation. If the Program does not specify a version number of the 576 | GNU General Public License, you may choose any version ever published 577 | by the Free Software Foundation. 578 | 579 | If the Program specifies that a proxy can decide which future 580 | versions of the GNU General Public License can be used, that proxy's 581 | public statement of acceptance of a version permanently authorizes you 582 | to choose that version for the Program. 583 | 584 | Later license versions may give you additional or different 585 | permissions. However, no additional obligations are imposed on any 586 | author or copyright holder as a result of your choosing to follow a 587 | later version. 588 | 589 | 15. Disclaimer of Warranty. 590 | 591 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY 592 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT 593 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY 594 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, 595 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 596 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM 597 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF 598 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 599 | 600 | 16. Limitation of Liability. 601 | 602 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 603 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS 604 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY 605 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE 606 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF 607 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD 608 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), 609 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF 610 | SUCH DAMAGES. 611 | 612 | 17. Interpretation of Sections 15 and 16. 613 | 614 | If the disclaimer of warranty and limitation of liability provided 615 | above cannot be given local legal effect according to their terms, 616 | reviewing courts shall apply local law that most closely approximates 617 | an absolute waiver of all civil liability in connection with the 618 | Program, unless a warranty or assumption of liability accompanies a 619 | copy of the Program in return for a fee. 620 | 621 | END OF TERMS AND CONDITIONS 622 | 623 | How to Apply These Terms to Your New Programs 624 | 625 | If you develop a new program, and you want it to be of the greatest 626 | possible use to the public, the best way to achieve this is to make it 627 | free software which everyone can redistribute and change under these terms. 628 | 629 | To do so, attach the following notices to the program. It is safest 630 | to attach them to the start of each source file to most effectively 631 | state the exclusion of warranty; and each file should have at least 632 | the "copyright" line and a pointer to where the full notice is found. 633 | 634 | 635 | Copyright (C) 636 | 637 | This program is free software: you can redistribute it and/or modify 638 | it under the terms of the GNU General Public License as published by 639 | the Free Software Foundation, either version 3 of the License, or 640 | (at your option) any later version. 641 | 642 | This program is distributed in the hope that it will be useful, 643 | but WITHOUT ANY WARRANTY; without even the implied warranty of 644 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 645 | GNU General Public License for more details. 646 | 647 | You should have received a copy of the GNU General Public License 648 | along with this program. If not, see . 649 | 650 | Also add information on how to contact you by electronic and paper mail. 651 | 652 | If the program does terminal interaction, make it output a short 653 | notice like this when it starts in an interactive mode: 654 | 655 | Copyright (C) 656 | This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 657 | This is free software, and you are welcome to redistribute it 658 | under certain conditions; type `show c' for details. 659 | 660 | The hypothetical commands `show w' and `show c' should show the appropriate 661 | parts of the General Public License. Of course, your program's commands 662 | might be different; for a GUI interface, you would use an "about box". 663 | 664 | You should also get your employer (if you work as a programmer) or school, 665 | if any, to sign a "copyright disclaimer" for the program, if necessary. 666 | For more information on this, and how to apply and follow the GNU GPL, see 667 | . 668 | 669 | The GNU General Public License does not permit incorporating your program 670 | into proprietary programs. If your program is a subroutine library, you 671 | may consider it more useful to permit linking proprietary applications with 672 | the library. If this is what you want to do, use the GNU Lesser General 673 | Public License instead of this License. But first, please read 674 | . 675 | --------------------------------------------------------------------------------