├── README.md ├── __init__.py ├── bface.py ├── byasp.py ├── data ├── yasp_map.json └── yasp_map.py ├── facs_process.py └── yasp ├── sphinxinstall ├── lib │ ├── libpocketsphinx.so │ ├── libpocketsphinx.so.3 │ ├── libpocketsphinx.so.3.0.0 │ ├── libsphinxad.so │ ├── libsphinxad.so.3 │ ├── libsphinxad.so.3.0.0 │ ├── libsphinxbase.so │ ├── libsphinxbase.so.3 │ └── libsphinxbase.so.3.0.0 └── share │ └── pocketsphinx │ └── model │ └── en-us │ ├── cmudict-en-us.dict │ ├── en-us-phone.lm.bin │ ├── en-us.lm.bin │ └── en-us │ ├── README │ ├── feat.params │ ├── mdef │ ├── means │ ├── noisedict │ ├── sendump │ ├── transition_matrices │ └── variances └── yaspbin ├── _yasp.so ├── yasp ├── yasp.py └── yasp_setup.py /README.md: -------------------------------------------------------------------------------- 1 | # YASP 2 | A speech parser blender plug-in 3 | -------------------------------------------------------------------------------- /__init__.py: -------------------------------------------------------------------------------- 1 | # ##### BEGIN GPL LICENSE BLOCK ##### 2 | # 3 | # Copyright (C) 2018 Amir Shehata 4 | # http://www.openmovie.com 5 | # amir.shehata@gmail.com 6 | 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 3 of the License, or 10 | # (at your option) any later version. 11 | 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | 17 | # You should have received a copy of the GNU General Public License 18 | # along with this program. If not, see . 19 | # 20 | # ##### END GPL LICENSE BLOCK ##### 21 | 22 | import bpy 23 | import traceback 24 | from bpy.props import EnumProperty, StringProperty, BoolProperty, IntProperty, FloatProperty 25 | from . import byasp 26 | from . import bface 27 | 28 | bl_info = { 29 | "name": "YASP", 30 | "version": (0, 1), 31 | "blender": (2, 80, 0), 32 | "location": "View3D > UI > YASP", 33 | "author": "Amir Shehata ", 34 | "description": "Yet Another Speech Parser", 35 | "category": "Speech Parser" 36 | } 37 | 38 | classes = ( 39 | byasp.VIEW3D_PT_tools_mb_yasp, 40 | byasp.YASP_OT_mark, 41 | byasp.YASP_OT_unmark, 42 | byasp.YASP_OT_set, 43 | byasp.YASP_OT_unset, 44 | byasp.YASP_OT_next, 45 | byasp.YASP_OT_prev, 46 | byasp.YASP_OT_setallKeyframes, 47 | byasp.YASP_OT_deleteallKeyframes, 48 | byasp.YASP_OT_delete_seq, 49 | bface.VIEW3D_PT_tools_openface, 50 | bface.VIEW3D_PT_pdm2d_openface, 51 | bface.FACE_OT_animate, 52 | bface.FACE_OT_clear_animation, 53 | bface.FACE_OT_pdm_del_animate, 54 | bface.FACE_OT_pdm2d_animate, 55 | bface.FACE_OT_pdm3d_rm_rotation, 56 | ) 57 | 58 | def register(): 59 | for cls in classes: 60 | bpy.utils.register_class(cls) 61 | 62 | bpy.types.Scene.yafr_facs_rig = StringProperty( 63 | name="FACS Rig name", 64 | subtype='FILE_NAME', 65 | default='', 66 | description='name of FACS rig') 67 | 68 | bpy.types.Scene.yafr_videofile = StringProperty( 69 | name="Path to video face reference", 70 | subtype='FILE_PATH', 71 | default='', 72 | description='path to video face reference') 73 | 74 | bpy.types.Scene.yafr_csvfile = StringProperty( 75 | name="Path to facs file", 76 | subtype='FILE_PATH', 77 | default='', 78 | description='path to FACS .csv file') 79 | 80 | bpy.types.Scene.yafr_start_frame = IntProperty( 81 | name="Start Frame", 82 | default = 0, 83 | description='Animation start frame') 84 | 85 | bpy.types.Scene.yafr_openface_ws = IntProperty( 86 | name="Window Size", 87 | default = 1, 88 | description='Smoothing Window Size') 89 | 90 | bpy.types.Scene.yafr_openface_polyorder = IntProperty( 91 | name="Polynomial Order", 92 | description='Polynomial order. Should be less than window size') 93 | 94 | bpy.types.Scene.yafr_openface_au_intensity = FloatProperty( 95 | name="Animation Intensity", 96 | description='Increase intensity of animation by factor') 97 | 98 | bpy.types.Scene.yafr_openface_vgaze_intensity = FloatProperty( 99 | name="Vertical Gaze Intensity", 100 | description='Increase intensity of vertical gaze by factor') 101 | 102 | bpy.types.Scene.yafr_openface_hgaze_intensity = FloatProperty( 103 | name="Horizontal Gaze Intensity", 104 | description='Increase intensity of horizontal gaze by factor') 105 | 106 | bpy.types.Scene.yafr_openface_mouth = BoolProperty( 107 | name="Enable Mouth", 108 | description="Enable mouth animation", 109 | default=True) 110 | 111 | bpy.types.Scene.yafr_openface_head = BoolProperty( 112 | name="Enable Head", 113 | description="Enable head animation", 114 | default=True) 115 | 116 | bpy.types.Scene.yafr_pdm_2d = BoolProperty( 117 | name="2D Plotting", 118 | description="plot 2D data", 119 | default=True) 120 | 121 | bpy.types.Scene.yafr_pdm_plot_all = BoolProperty( 122 | name="plot every point", 123 | description="plot all the data provided", 124 | default=False) 125 | 126 | bpy.types.Scene.yasp_phoneme_rig = StringProperty( 127 | name="Phoneme Rig Name", 128 | subtype='FILE_NAME', 129 | default='', 130 | description='name of phoneme rig') 131 | 132 | bpy.types.Scene.yasp_wave_path = StringProperty( 133 | name="Path to wave file", 134 | subtype='FILE_PATH', 135 | default='', 136 | description='Path to wave file') 137 | 138 | bpy.types.Scene.yasp_transcript_path = StringProperty( 139 | name="Path to transcript file", 140 | subtype='FILE_PATH', 141 | default='', 142 | description='Path to transcript file') 143 | 144 | bpy.types.Scene.yasp_start_frame = IntProperty( 145 | name="Start frame", 146 | description='Start audio on specified frame') 147 | 148 | bpy.types.Scene.yasp_avg_window_size = IntProperty( 149 | name="Avg Window", 150 | description='Average keyframe values within the window') 151 | 152 | bface.set_init_state(True) 153 | 154 | def unregister(): 155 | for cls in reversed(classes): 156 | bpy.utils.unregister_class(cls) 157 | 158 | if __name__ == "__main__": 159 | register() 160 | -------------------------------------------------------------------------------- /bface.py: -------------------------------------------------------------------------------- 1 | import bpy 2 | import logging 3 | import os 4 | import json 5 | from bpy_extras.io_utils import ExportHelper, ImportHelper 6 | from bpy.app.handlers import persistent 7 | import traceback 8 | import time 9 | import ctypes 10 | import sys 11 | import platform 12 | import random 13 | import math 14 | import subprocess 15 | import datetime 16 | from bpy.props import EnumProperty, StringProperty, BoolVectorProperty 17 | from . import facs_process as facs 18 | 19 | logger = logging.getLogger(__name__) 20 | 21 | # Global sliders 22 | global_sliders_set = False 23 | global_sliders = {} 24 | init_state = False 25 | plot_all = False 26 | 27 | def set_rotation_type(rtype): 28 | rotation_types = ('BOUNDING_BOX_CENTER', 'CURSOR', 'INDIVIDUAL_ORIGINS', 'MEDIAN_POINT', 'ACTIVE_ELEMENT') 29 | if not rtype in rotation_types: 30 | raise RuntimeError(rtype, 'not a valid rotation type. Should be: ', rotation_types) 31 | bpy.context.scene.tool_settings.transform_pivot_point = rtype 32 | 33 | def get_rotation_type(): 34 | return bpy.context.scene.tool_settings.transform_pivot_point 35 | 36 | def get_override(area_type, region_type): 37 | for area in bpy.context.screen.areas: 38 | if area.type == area_type: 39 | for region in area.regions: 40 | if region.type == region_type: 41 | override = {'area': area, 'region': region} 42 | return override 43 | #error message if the area or region wasn't found 44 | raise RuntimeError("Wasn't able to find", region_type," in area ", area_type, 45 | "\n Make sure it's open while executing script.") 46 | 47 | def rotate_obj_quaternion(obj, axis='Z', value=0.0): 48 | bpy.context.scene.cursor.location = (0,0,0) 49 | if not axis in ['X', 'Y', 'Z']: 50 | return 51 | for o in bpy.data.objects: 52 | o.select_set(False) 53 | obj.select_set(True) 54 | orig_rt = get_rotation_type() 55 | set_rotation_type('CURSOR') 56 | override = get_override('VIEW_3D', 'WINDOW') 57 | obj.rotation_mode = 'QUATERNION' 58 | bpy.ops.transform.rotate(override, value=value, orient_axis=axis, 59 | orient_type='CURSOR') 60 | set_rotation_type(orig_rt) 61 | 62 | def set_init_state(state): 63 | global init_state 64 | 65 | init_state = state 66 | if init_state: 67 | facs.init_database() 68 | 69 | class FACE_OT_clear_animation(bpy.types.Operator): 70 | bl_idname = "yafr.del_animation" 71 | bl_label = "Delete Animation" 72 | bl_description = "Clear Facial Animation" 73 | 74 | def execute(self, context): 75 | global global_sliders_set 76 | global global_sliders 77 | global init_state 78 | 79 | for obj in bpy.data.objects: 80 | if 'facs_rig_slider_' in obj.name: 81 | obj.animation_data_clear() 82 | obj = get_mb_rig() 83 | if obj: 84 | obj.animation_data_clear() 85 | 86 | global_sliders_set = False 87 | global_sliders = {} 88 | facs.reset_database() 89 | return {'FINISHED'} 90 | 91 | def get_mb_rig(): 92 | rig_names = ['MBLab_skeleton_muscle_ik', 'MBLab_skeleton_base_ik', 'MBLab_skeleton_muscle_fk', 'MBLab_skeleton_base_fk'] 93 | for obj in bpy.data.objects: 94 | if obj.type == 'ARMATURE' and obj.data.name in rig_names: 95 | return obj 96 | return None 97 | 98 | def process_csv_file(csv, ws, po): 99 | try: 100 | js = facs.process_openface_csv(csv, ws, po) 101 | except Exception as e: 102 | msg = 'failed to process results\n'+traceback.format_exc() 103 | logger.critical(msg) 104 | return False, msg 105 | 106 | if not js: 107 | msg = 'Failed to process results' 108 | return False, msg 109 | 110 | return True, 'Success' 111 | 112 | class FACE_OT_animate(bpy.types.Operator): 113 | bl_idname = "yafr.animate_face" 114 | bl_label = "Animate Face" 115 | bl_description = "Create Facial Animation" 116 | 117 | def run_openface(self, openface, video): 118 | outdir = os.path.join(os.path.dirname(openface), 119 | os.path.splitext(os.path.basename(video))[0]+"_processed") 120 | if not os.path.exists(outdir): 121 | os.makedirs(outdir) 122 | rc = subprocess.run([openface, '-verbose', '-f', video, '-out_dir', outdir], 123 | stdout=subprocess.PIPE) 124 | return rc.returncode, rc.stdout.decode(), outdir 125 | 126 | def set_keyframes_hr(self, result, array, attr, head_bone, intensity): 127 | rotation = {'Rx': 1, 'Ry': 2, 'Rz': 3} 128 | 129 | for m in array: 130 | # angle in radians 131 | val = result[m] + (result[m] * intensity) 132 | head_bone.rotation_quaternion[rotation[attr]] = val 133 | head_bone.keyframe_insert('rotation_quaternion', index=rotation[attr], frame=m) 134 | 135 | def get_head_bone(self, mb_rig): 136 | for obj in bpy.data.objects: 137 | obj.select_set(False) 138 | mb_rig.select_set(True) 139 | bpy.context.view_layer.objects.active = mb_rig 140 | bpy.ops.object.mode_set(mode='POSE') 141 | head_bone = None 142 | msg = "Success" 143 | try: 144 | head_bone = bpy.context.object.pose.bones['head'] 145 | except: 146 | msg = "no head bone found" 147 | return None, msg 148 | 149 | return head_bone, msg 150 | 151 | def set_keyframes(self, result, array, slider_bone, intensity, vgi, hgi): 152 | global global_sliders 153 | if bpy.context.scene.yafr_start_frame > 0: 154 | frame_offset = bpy.context.scene.yafr_start_frame - 1 155 | else: 156 | frame_offset = 0 157 | 158 | for m in array: 159 | if not 'GZ' in slider_bone.name: 160 | value = (result[m] / 5) * 0.377 161 | else: 162 | # normalize the gaze values to fit in the -0.189 - 0.189 163 | # range of the gaze slider 164 | # TODO: if we're going to fit that with other rig systems 165 | # we need to be a bit smarter than this. 166 | value = result[m] 167 | if value > 0: 168 | value = min(value, 1) 169 | value = value * 0.189 170 | elif value < 0: 171 | value = max(value, -1) 172 | value = value * 0.189 173 | 174 | gaze_intensity = hgi 175 | # intensify the gaze motion independently 176 | if 'GZ0V' in slider_bone.name: 177 | # reverse sign to get the correct up/down motion 178 | #value = -value 179 | gaze_intensity = vgi 180 | 181 | if value > 0: 182 | value = value + (value * gaze_intensity) 183 | elif value < 0: 184 | value = value - ((value * -1) * gaze_intensity) 185 | if intensity > 0 and not 'GZ' in slider_bone.name: 186 | # don't accept negative values 187 | if value < 0: 188 | value = -value 189 | value = value + (value * intensity) 190 | slider_bone.location[0] = value 191 | slider_bone.keyframe_insert(data_path="location", 192 | frame=m+frame_offset, index=0) 193 | global_sliders[slider_bone.name].append(m) 194 | 195 | def set_every_keyframe(self, result, slider_bone, intensity): 196 | frame = 1 197 | for m in result: 198 | if not 'GZ' in slider_bone.name: 199 | value = (m / 5) * 0.377 200 | else: 201 | value = m 202 | if intensity > 0: 203 | value = value + (value * (intensity/100)) 204 | slider_bone.location[0] = value 205 | if not 'GZ' in slider_bone.name: 206 | slider_bone.keyframe_insert(data_path="location", frame=frame, index=0) 207 | frame = frame + 1 208 | 209 | def set_animation_prereq(self, scn): 210 | if not scn.yafr_facs_rig: 211 | facs_rig = bpy.data.objects.get('MBLab_skeleton_facs_rig') 212 | else: 213 | facs_rig = bpy.data.objects.get(scn.yafr_facs_rig) 214 | if not facs_rig: 215 | return False 216 | 217 | # select the rig and put it in POSE mode 218 | for obj in bpy.data.objects: 219 | obj.select_set(False) 220 | facs_rig.select_set(True) 221 | bpy.context.view_layer.objects.active = facs_rig 222 | bpy.ops.object.mode_set(mode='POSE') 223 | return True 224 | 225 | def animate_face(self, mouth, head, animation_data, intensity, vgi, hgi): 226 | global global_sliders_set 227 | 228 | if not self.set_animation_prereq(bpy.context.scene): 229 | print("Animation prerequisites not set") 230 | return 231 | 232 | # animation already done 233 | if global_sliders_set: 234 | print("Animation already set. Delete animation first") 235 | return 236 | 237 | mouth_aus = ['AU10', 'AU12', 'AU13', 'AU14', 'AU15', 'AU16', 'AU17', 'AU20', 'AU23'] 238 | 239 | for key, value in animation_data.items(): 240 | # don't use specific AUs if mouth is not selected 241 | if key.strip('_r') in mouth_aus and not mouth: 242 | continue 243 | 244 | if 'pose_' in key and not head: 245 | continue 246 | 247 | slider_name = '' 248 | head_animation = False 249 | if 'AU' in key: 250 | slider_name = 'facs_rig_slider_' + key.strip('_r') 251 | elif key == 'gaze_angle_x': 252 | slider_name = 'facs_rig_slider_GZ0H' 253 | elif key == 'gaze_angle_y': 254 | slider_name = 'facs_rig_slider_GZ0V' 255 | elif 'pose_R' in key: 256 | # only look at the head rotation for now 257 | head_animation = True 258 | else: 259 | continue 260 | 261 | result = value[facs.VALUES] 262 | maximas = value[facs.MAXIMAS] 263 | minimas = value[facs.MINIMAS] 264 | 265 | slider_bone = None 266 | mb_rig = None 267 | if head_animation: 268 | mb_rig = get_mb_rig() 269 | if not mb_rig: 270 | msg = "no MB rig found" 271 | logger.critical(msg) 272 | self.report({'ERROR'}, msg) 273 | return 274 | head_bone, msg = self.get_head_bone(mb_rig) 275 | if not head_bone: 276 | logger.critical(msg) 277 | self.report({'ERROR'}, msg) 278 | return 279 | self.set_keyframes_hr(result, maximas, key.strip('pose_'), head_bone, intensity) 280 | self.set_keyframes_hr(result, minimas, key.strip('pose_'), head_bone, intensity) 281 | else: 282 | global_sliders[slider_name] = [] 283 | slider_bone = bpy.context.object.pose.bones.get(slider_name) 284 | if not slider_bone: 285 | logger.critical('slider %s not found', slider_name) 286 | continue 287 | 288 | self.set_keyframes(result, maximas, slider_bone, intensity, vgi, hgi) 289 | self.set_keyframes(result, minimas, slider_bone, intensity, vgi, hgi) 290 | #self.set_every_keyframe(result, slider_bone, intensity, vgi, hgi) 291 | 292 | global_sliders_set = True 293 | 294 | def execute(self, context): 295 | global global_sliders_set 296 | 297 | set_init_state(False) 298 | 299 | scn = context.scene 300 | dirname = os.path.dirname(os.path.realpath(__file__)) 301 | openface = os.path.join(dirname, "openface", "FeatureExtraction") 302 | video = scn.yafr_videofile 303 | csv = scn.yafr_csvfile 304 | ws = scn.yafr_openface_ws 305 | po = scn.yafr_openface_polyorder 306 | intensity = scn.yafr_openface_au_intensity 307 | hgi = scn.yafr_openface_hgaze_intensity 308 | vgi = scn.yafr_openface_vgaze_intensity 309 | mouth = scn.yafr_openface_mouth 310 | head = scn.yafr_openface_head 311 | 312 | if global_sliders_set: 313 | self.report({'ERROR'}, "Delete current animation first") 314 | return {'FINISHED'} 315 | 316 | if po >= ws: 317 | msg = "polyorder must be less than window_length." 318 | logger.critical(msg) 319 | self.report({'ERROR'}, msg) 320 | return {'FINISHED'} 321 | 322 | if ws % 2 == 0: 323 | msg = "window size needs to be an odd number" 324 | logger.critical(msg) 325 | self.report({'ERROR'}, msg) 326 | return {'FINISHED'} 327 | 328 | # csv file provided use that instead of the video file 329 | if csv: 330 | if not os.path.isfile(csv): 331 | if not os.path.isfile(dirname+csv): 332 | msg = "bad csv file provided "+csv 333 | logger.critical(msg) 334 | self.report({'ERROR'}, msg) 335 | return {'FINISHED'} 336 | else: 337 | csv = dirname+csv 338 | 339 | rc, msg = process_csv_file(csv, ws, po) 340 | # animate the data 341 | if rc: 342 | facs_data = facs.get_facs_data() 343 | self.animate_face(mouth, head, facs_data, intensity, vgi, hgi) 344 | return {'FINISHED'} 345 | 346 | self.report({'ERROR'}, msg) 347 | return {'FINISHED'} 348 | 349 | # run openface on the videofile 350 | # TODO: check if openface is an executable and videofile is a video 351 | # file. 352 | if not os.path.isfile(openface): 353 | if not os.path.isfile(dirname+openface): 354 | msg = "Bad path to openFace: " + openface 355 | self.report({'ERROR'}, msg) 356 | return {'FINISHED'} 357 | else: 358 | openface = dirname+openface 359 | if not os.path.isfile(video): 360 | # try another tac 361 | if not os.path.isfile(dirname+video): 362 | msg = "Bad path to video file: " + video 363 | self.report({'ERROR'}, 'Bad path to video file') 364 | return {'FINISHED'} 365 | else: 366 | video = dirname+video 367 | 368 | outdir = '' 369 | try: 370 | rc, output, outdir = self.run_openface(openface, video) 371 | if rc: 372 | self.report({'ERROR'}, ouput) 373 | return {'FINISHED'} 374 | except Exception as e: 375 | logger.critical(e) 376 | msg = 'failed to run openface\n'+traceback.format_exc() 377 | self.report({'ERROR'}, msg) 378 | return {'FINISHED'} 379 | 380 | # process the csv file 381 | csv = os.path.join(outdir, 382 | os.path.splitext(os.path.basename(video))[0]+'.csv') 383 | if not os.path.isfile(csv): 384 | msg = "Failed to process video. No csv file found: "+csv 385 | self.report({'ERROR'}, msg) 386 | return {'FINISHED'} 387 | 388 | # animate the data 389 | rc, msg = process_csv_file(csv, ws, po) 390 | # animate the data 391 | if rc: 392 | facs_data = facs.get_facs_data() 393 | self.animate_face(mouth, head, facs_data, intensity, vgi, hgi) 394 | #frame_end = facs_data['frame'][facs.VALUES][-1] 395 | #bpy.context.scene.frame_end = frame_end 396 | return {'FINISHED'} 397 | 398 | self.report({'ERROR'}, msg) 399 | return {'FINISHED'} 400 | 401 | class FACE_OT_pdm_del_animate(bpy.types.Operator): 402 | bl_idname = "yafr.del_pdm_animation" 403 | bl_label = "Delete" 404 | bl_description = "Experimental feature" 405 | 406 | def execute(self, context): 407 | for obj in bpy.data.objects: 408 | obj.select_set(False) 409 | 410 | for obj in bpy.data.objects: 411 | if 'pdm2d_' in obj.name or 'pdm3d_' in obj.name: 412 | obj.animation_data_clear() 413 | bpy.context.view_layer.objects.active = obj 414 | obj.select_set(True) 415 | bpy.ops.object.delete(use_global=True) 416 | return {'FINISHED'} 417 | 418 | class FACE_OT_pdm3d_rm_rotation(bpy.types.Operator): 419 | bl_idname = "yafr.rm_pdm3d_rotation" 420 | bl_label = "Remove Rotation" 421 | bl_description = "Experimental feature" 422 | 423 | def rotate_obj(self, obj, rx, ry, rz): 424 | if len(rx) != len(ry) or len(rx) != len(rz): 425 | self.report({'ERROR'}, "bad rotation information") 426 | return 427 | for f in range(0, len(rx)): 428 | rotate_obj_quaternion(obj, 'X', rx[f]) 429 | rotate_obj_quaternion(obj, 'Y', ry[f]) 430 | rotate_obj_quaternion(obj, 'Z', rz[f]) 431 | obj.keyframe_insert(data_path="location", frame=f) 432 | #obj.keyframe_insert(data_path="rotation_quaternion", frame=f) 433 | #obj.rotation_mode = 'QUATERNION' 434 | #obj.rotation_quaternion[1] = rx[f] 435 | #obj.rotation_quaternion[2] = ry[f] * -1 436 | #obj.rotation_quaternion[3] = rz[f] * -1 437 | #obj.keyframe_insert(data_path="rotation_quaternion", frame=f, index=1) 438 | #obj.keyframe_insert(data_path="rotation_quaternion", frame=f, index=2) 439 | #obj.keyframe_insert(data_path="rotation_quaternion", frame=f, index=3) 440 | 441 | def execute(self, context): 442 | data = facs.get_facs_data() 443 | 444 | rx = data['pose_Rx'][facs.VALUES] 445 | ry = data['pose_Ry'][facs.VALUES] 446 | rz = data['pose_Rz'][facs.VALUES] 447 | 448 | # Store the current location of the object for this frame. 449 | # find out the 3D location of the object after applying the 450 | # rotation. 451 | # The delta between the current location and the rotated location 452 | # is eliminated by subtracting the X,Y,Z locations. 453 | for obj in bpy.data.objects: 454 | if not 'amir' in obj.name: 455 | #if not 'pdm3d_' in obj.name: 456 | continue 457 | logger.critical("Rotating object %s: %s", obj.name, 458 | str(datetime.datetime.now())) 459 | self.rotate_obj(obj, rx, ry, rz) 460 | 461 | return {'FINISHED'} 462 | 463 | class FACE_OT_pdm2d_animate(bpy.types.Operator): 464 | bl_idname = "yafr.animate_pdm2d_face" 465 | bl_label = "Plot" 466 | bl_description = "Experimental feature" 467 | 468 | def plot_axis(self, obj, axis, result, array, adj=[], div=400): 469 | if plot_all: 470 | f = 0 471 | values = [] 472 | for p in result: 473 | # the adjustment array brings the points to the center 474 | # point. 475 | if len(adj) == len(result): 476 | av = adj[f] 477 | value = (p - av) / div 478 | else: 479 | value = p / div 480 | 481 | if axis == 1 or axis == 2: 482 | value = value * -1 483 | obj.location[axis] = value 484 | obj.keyframe_insert(data_path="location", frame=f, index=axis) 485 | values.append(value) 486 | f = f+1 487 | return values 488 | for m in array: 489 | value = result[m] / div 490 | if axis == 1 or axis == 2: 491 | value = value * -1 492 | obj.location[axis] = value 493 | obj.keyframe_insert(data_path="location", frame=m, index=axis) 494 | 495 | def animate_2d_empty(self, obj, attr, pdm_2d, rigid_data): 496 | y_name = 'y_'+attr.strip('x_') 497 | 498 | x_info = pdm_2d[attr] 499 | y_info = pdm_2d[y_name] 500 | p_tx = rigid_data['p_tx'][facs.VALUES] 501 | p_ty = rigid_data['p_ty'][facs.VALUES] 502 | 503 | x_values = self.plot_axis(obj, 0, x_info[facs.VALUES], 504 | x_info[facs.MAXIMAS], adj=p_tx) 505 | if not plot_all: 506 | self.plot_axis(obj, 0, x_info[facs.VALUES], 507 | x_info[facs.MINIMAS], adj=p_tx) 508 | y_values = self.plot_axis(obj, 1, y_info[facs.VALUES], 509 | y_info[facs.MAXIMAS], adj=p_ty) 510 | if not plot_all: 511 | self.plot_axis(obj, 1, y_info[facs.VALUES], 512 | y_info[facs.MINIMAS], adj=p_ty) 513 | 514 | return x_values, y_values 515 | 516 | def animate_3d_empty(self, obj, attr, pdm_3d, head_pose): 517 | y_name = 'Y_'+attr.strip('X_') 518 | z_name = 'Z_'+attr.strip('X_') 519 | div = 40 520 | 521 | x_info = pdm_3d[attr] 522 | y_info = pdm_3d[y_name] 523 | z_info = pdm_3d[z_name] 524 | tx_adj = head_pose['pose_Tx'][facs.VALUES] 525 | ty_adj = head_pose['pose_Ty'][facs.VALUES] 526 | tz_adj = head_pose['pose_Tz'][facs.VALUES] 527 | 528 | self.plot_axis(obj, 0, x_info[facs.VALUES], 529 | x_info[facs.MAXIMAS], adj=tx_adj, div=div) 530 | if not plot_all: 531 | self.plot_axis(obj, 0, x_info[facs.VALUES], 532 | x_info[facs.MINIMAS], adj=tx_adj, div=div) 533 | self.plot_axis(obj, 1, y_info[facs.VALUES], 534 | x_info[facs.MAXIMAS], adj=ty_adj, div=div) 535 | if not plot_all: 536 | self.plot_axis(obj, 1, y_info[facs.VALUES], 537 | x_info[facs.MINIMAS], adj=ty_adj, div=div) 538 | self.plot_axis(obj, 2, z_info[facs.VALUES], 539 | x_info[facs.MAXIMAS], adj=tz_adj, div=div) 540 | if not plot_all: 541 | self.plot_axis(obj, 2, z_info[facs.VALUES], 542 | x_info[facs.MINIMAS], adj=tz_adj, div=div) 543 | 544 | def delta(self, pp, startpoint, endpoints): 545 | sp = pp[startpoint] 546 | num_frames = len(sp[0]) 547 | resutl = [] 548 | for i in range(0, len(num_frames)): 549 | a = np.array(sp[0][i], sp[1][i]) 550 | eps = [] 551 | for e in endpoints: 552 | eps.append(pp[e][0][i] + pp[e][1][i]) 553 | b = np.array(eps) 554 | dist = scipy.spatial.distance.cdist(a,b) 555 | flat_dist = [item for sublist in dist.tolist() for item in sublist] 556 | avg = sum(flat_dist) / len(flat_dist) 557 | result.append(avg) 558 | return result 559 | 560 | def animate_pdm2d(self, pdm_2d, rigid_data): 561 | # create all the empties 562 | pp = {} 563 | for k, v in pdm_2d.items(): 564 | if 'y_' in k or 'frame' in k or 'timestamp' in k: 565 | continue 566 | bpy.ops.object.empty_add(type='SPHERE', radius=0.01) 567 | empty = bpy.context.view_layer.objects.active 568 | entry = k.strip('x_') 569 | empty.name = 'pdm2d_'+k.strip('x_') 570 | # animate each empty 571 | logger.critical('Start plotting %s: %s', k, 572 | str(datetime.datetime.now())) 573 | x_values, y_values = self.animate_2d_empty(empty, k, pdm_2d, rigid_data) 574 | 575 | # post process the values 576 | if len(x_values) != len(y_values): 577 | print("Unexpected array lengths") 578 | continue 579 | 580 | if not plot_all: 581 | continue 582 | 583 | if int(entry) in [51, 62, 57, 66, 54, 12, 48, 4, 54, 11, 5, 57, 8, 53, 29, 49, 55, 9, 59, 7] + \ 584 | list(range(13, 15)) + list(range(3,1)): 585 | pp[int(entry)] = [x_values, y_values] 586 | 587 | if not plot_all: 588 | return 589 | 590 | # keeping deltas on the following points 591 | # The array is indexed by frames 592 | # upper lip roll: 51-62 593 | # lower lip roll: 57-66 594 | # left lip side: 54-12 595 | # Right lip side: 48-4 596 | # left lip up: 54-[13-15] 597 | # right lip up: 48-[3-1] 598 | # Left lip down: 54-11 599 | # Right lip down: 48-5 600 | # Chin: 57-8 601 | # left upper lip curl: 53-29 602 | # right upper lip curl: 49-29 603 | # left lower lip curl: 55-9 604 | # right lower lip curl: 59-7 605 | deltas = [{'upper-lip-roll': self.delta(pp, 51, [62]), 'lower-lip-roll': self.delta(pp, 57, [66]), 606 | 'left-lip-side': self.delta(pp, 54, [12]), 'right-lip-side': self.delta(pp, 48, [4]), 607 | 'left-lip-up': self.delta(pp, 54, list(range(13-16))), 'right-lip-up': self.delta(pp, 48, list(range(1, 4))), 608 | 'left-lip-down': self.delta(pp, 48, [5]), 'right-lip-down': self.delta(pp, 48, [5]), 609 | 'chin': self.delta(pp, 57, [8]), 'left-upper-lip-curl': self.delta(pp, 53, [29]), 610 | 'right-upper-lip-curl': self.delta(pp, 49, [29]), 611 | 'left-lower-lip-curl': self.delta(pp, 55, [9]), 612 | 'right-lower-lip-curl': self.delta(pp, 59, [7])}] 613 | print(deltas) 614 | # The idea now is that we can use that delta in comparison with 615 | # the basis delta we collected to calculate the percentage of 616 | 617 | def animate_pdm3d(self, pdm_3d, head_pose): 618 | # create all the empties 619 | for k, v in pdm_3d.items(): 620 | if 'Y_' in k or 'Z_' in k or \ 621 | 'frame' in k or 'timestamp' in k: 622 | continue 623 | bpy.ops.object.empty_add(type='SPHERE', radius=0.05) 624 | empty = bpy.context.view_layer.objects.active 625 | empty.name = 'pdm3d_'+k.strip('X_') 626 | # animate each empty 627 | logger.critical('Start plotting %s: %s', k, 628 | str(datetime.datetime.now())) 629 | self.animate_3d_empty(empty, k, pdm_3d, head_pose) 630 | #logger.critical('Finished plotting %s: %s', k, 631 | # str(datetime.datetime.now())) 632 | 633 | def execute(self, context): 634 | global plot_all 635 | 636 | scn = context.scene 637 | dirname = os.path.dirname(os.path.realpath(__file__)) 638 | csv = scn.yafr_csvfile 639 | ws = scn.yafr_openface_ws 640 | po = scn.yafr_openface_polyorder 641 | two_d = scn.yafr_pdm_2d 642 | plot_all = scn.yafr_pdm_plot_all 643 | 644 | set_init_state(False) 645 | 646 | if po >= ws: 647 | msg = "polyorder must be less than window_length." 648 | logger.critical(msg) 649 | self.report({'ERROR'}, msg) 650 | return {'FINISHED'} 651 | 652 | if ws % 2 == 0: 653 | msg = "window size needs to be an odd number" 654 | logger.critical(msg) 655 | self.report({'ERROR'}, msg) 656 | return {'FINISHED'} 657 | 658 | if not csv: 659 | self.report({'ERROR'}, 'No CSV file specified') 660 | return {'FINISHED'} 661 | 662 | if not os.path.isfile(csv): 663 | if not os.path.isfile(dirname+csv): 664 | msg = "bad csv file provided "+csv 665 | logger.critical(msg) 666 | self.report({'ERROR'}, msg) 667 | return {'FINISHED'} 668 | else: 669 | csv = dirname+csv 670 | 671 | # reset and reload the data base 672 | facs.reset_database() 673 | 674 | logger.critical('Start processing CSV: %s', 675 | str(datetime.datetime.now())) 676 | rc, msg = process_csv_file(csv, ws, po) 677 | logger.critical('Finished processing CSV: %s', 678 | str(datetime.datetime.now())) 679 | # animate the data 680 | if not rc: 681 | self.report({'ERROR'}, msg) 682 | return {'FINISHED'} 683 | if two_d: 684 | logger.critical('Start plotting 2D: %s', 685 | str(datetime.datetime.now())) 686 | pdm2d_data = facs.get_pdm2d_data() 687 | rigid_data = facs.get_rigid_data() 688 | self.animate_pdm2d(pdm2d_data, rigid_data) 689 | logger.critical('Finished plotting 2D: %s', 690 | str(datetime.datetime.now())) 691 | else: 692 | logger.critical('Start plotting 3D: %s', 693 | str(datetime.datetime.now())) 694 | pdm3d_data = facs.get_pdm3d_data() 695 | head_pose = facs.get_facs_data() 696 | self.animate_pdm3d(pdm3d_data, head_pose) 697 | logger.critical('Finished plotting 3D: %s', 698 | str(datetime.datetime.now())) 699 | 700 | if two_d: 701 | frame_end = pdm2d_data['frame'][facs.VALUES][-1] 702 | else: 703 | frame_end = pdm3d_data['frame'][facs.VALUES][-1] 704 | bpy.context.scene.frame_end = frame_end 705 | 706 | return {'FINISHED'} 707 | 708 | class VIEW3D_PT_tools_openface(bpy.types.Panel): 709 | bl_label = "Open Face" 710 | bl_space_type = "VIEW_3D" 711 | bl_region_type = "UI" 712 | bl_category = "YAFR" 713 | #bl_options = {'DEFAULT_CLOSED'} 714 | 715 | def draw(self, context): 716 | scn = context.scene 717 | layout = self.layout 718 | wm = context.window_manager 719 | col = layout.column(align=True) 720 | col.label(text="FACS Rig Name") 721 | col.prop(scn, "yafr_facs_rig", text='') 722 | col.label(text="FACS CSV file") 723 | col.prop(scn, "yafr_csvfile", text='') 724 | col.label(text="Video file") 725 | col.prop(scn, "yafr_videofile", text='') 726 | col.label(text="Animation Start Frame") 727 | col.prop(scn, "yafr_start_frame", text='') 728 | col.label(text="Smoothing Window Size") 729 | col.prop(scn, "yafr_openface_ws", text='') 730 | col.label(text="Polynomial Order") 731 | col.prop(scn, "yafr_openface_polyorder", text='') 732 | col.label(text="Animation Intensity") 733 | col.prop(scn, "yafr_openface_au_intensity", text='') 734 | col.label(text="Vertical Gaze Intensity") 735 | col.prop(scn, "yafr_openface_vgaze_intensity", text='') 736 | col.label(text="Horizontal Gaze Intensity") 737 | col.prop(scn, "yafr_openface_hgaze_intensity", text='') 738 | col.prop(scn, "yafr_openface_mouth", text='Mouth Animation') 739 | col.prop(scn, "yafr_openface_head", text='Head Animation') 740 | col = layout.column(align=False) 741 | col.operator('yafr.animate_face', icon='ANIM_DATA') 742 | col = layout.column(align=False) 743 | col.operator('yafr.del_animation', icon='DECORATE_ANIMATE') 744 | 745 | class VIEW3D_PT_pdm2d_openface(bpy.types.Panel): 746 | bl_label = "PDM Experimental" 747 | bl_space_type = "VIEW_3D" 748 | bl_region_type = "UI" 749 | bl_category = "YAFR" 750 | #bl_options = {'DEFAULT_CLOSED'} 751 | 752 | def draw(self, context): 753 | scn = context.scene 754 | layout = self.layout 755 | wm = context.window_manager 756 | col = layout.column(align=True) 757 | col.label(text="Experimental") 758 | col.prop(scn, "yafr_pdm_2d", text='2D Plotting') 759 | col.prop(scn, "yafr_pdm_plot_all", text='Plot All') 760 | col.operator('yafr.animate_pdm2d_face', icon='ANIM_DATA') 761 | col = layout.column(align=False) 762 | col.operator('yafr.rm_pdm3d_rotation', icon='ANIM_DATA') 763 | col = layout.column(align=False) 764 | col.operator('yafr.del_pdm_animation', icon='DECORATE_ANIMATE') 765 | 766 | 767 | -------------------------------------------------------------------------------- /byasp.py: -------------------------------------------------------------------------------- 1 | import bpy 2 | import logging 3 | import os 4 | import json 5 | from bpy_extras.io_utils import ExportHelper, ImportHelper 6 | from bpy.app.handlers import persistent 7 | import time 8 | import ctypes 9 | import sys 10 | import platform 11 | import random 12 | import math 13 | from bpy.props import EnumProperty, StringProperty, BoolVectorProperty 14 | 15 | random.seed(23483) 16 | addon_path = os.path.dirname(os.path.realpath(__file__)) 17 | yasp_model_dir = os.path.join(addon_path, "yasp", "sphinxinstall", "share", "pocketsphinx", "model") 18 | yasp_sphinx_dir = os.path.join(addon_path, "yasp", "sphinxinstall", "lib") 19 | yasp_libs_dir = os.path.join(addon_path, "yasp", "yaspbin") 20 | sys.path.append(yasp_libs_dir) 21 | pocketsphinxlib = None 22 | sphinxadlib = None 23 | sphinxbaselib = None 24 | yasplib = None 25 | libs_loaded = True 26 | 27 | logger = logging.getLogger(__name__) 28 | 29 | # Load the .so files we need 30 | # Then import yasp 31 | # Now we're ready to do some speech parsing 32 | def yasp_load_dep(): 33 | global libs_loaded 34 | pocketsphinx = os.path.join(yasp_sphinx_dir, "libpocketsphinx.so") 35 | sphinxad = os.path.join(yasp_sphinx_dir, "libsphinxad.so") 36 | sphinxbase = os.path.join(yasp_sphinx_dir, "libsphinxbase.so") 37 | yasp = os.path.join(yasp_libs_dir, "_yasp.so") 38 | if not os.path.exists(pocketsphinx) or \ 39 | not os.path.exists(sphinxad) or \ 40 | not os.path.exists(sphinxbase) or \ 41 | not os.path.exists(yasp): 42 | logger.critical("libraries don't exist. Reinstall") 43 | try: 44 | sphinxbaselib = ctypes.cdll.LoadLibrary(os.path.abspath(sphinxbase)) 45 | sphinxadlib = ctypes.cdll.LoadLibrary(os.path.abspath(sphinxad)) 46 | pocketsphinxlib = ctypes.cdll.LoadLibrary(os.path.abspath(pocketsphinx)) 47 | yasplib = ctypes.cdll.LoadLibrary(os.path.abspath(yasp)) 48 | except Exception as e: 49 | logger.critical(e) 50 | logger.critical("Failed to load libraries") 51 | libs_loaded = False 52 | 53 | 54 | if platform.system() == "Linux": 55 | yasp_load_dep() 56 | if libs_loaded: 57 | import yasp 58 | 59 | def get_data_path(): 60 | addon_directory = os.path.dirname(os.path.realpath(__file__)) 61 | data_dir = os.path.join(addon_directory, "data") 62 | if os.path.isdir(data_dir): 63 | return data_dir 64 | else: 65 | logger.critical("can't find data directory") 66 | return None 67 | 68 | # class maps yasp phoneme Mapper 69 | # YASP produces a more nuanced phonemes. We need to reduce that to the set 70 | # of phonemes which we use for the animation 71 | class YASP2MBPhonemeMapper(object): 72 | def __init__(self): 73 | self.yasp_2_mb_phoneme_map = None 74 | data_path = get_data_path() 75 | if not data_path: 76 | logger.critical("CRITICAL", "%s not found. Please check your Blender addons directory. Might need to reinstall YASP", data_path) 77 | raise ValueError("No Data directory") 78 | 79 | map_file = os.path.join(data_path, 'yasp_map.json') 80 | if not map_file: 81 | logger.critical("CRITICAL", "%s not found. Please check your Blender addons directory. Might need to reinstall YASP", map_file) 82 | with open(map_file, 'r') as f: 83 | self.yasp_2_mb_phoneme_map = json.load(f) 84 | 85 | def get_phoneme_animation_data(self, phoneme): 86 | try: 87 | anim_data = self.yasp_2_mb_phoneme_map[phoneme] 88 | except: 89 | return None 90 | return anim_data 91 | 92 | class Bone(object): 93 | def __init__(self, bone): 94 | # keeps the frame number of the keyframe, the value of the 95 | # z rotation and boolean if it's fake or real. 96 | self.animation_data = {} 97 | self.mybone = bone 98 | 99 | def get_name(self): 100 | return self.mybone.name 101 | 102 | # inserts an actual keyframe if actual=True. If actual=False then it 103 | # inserts it in the list, but doesn't create a keyframe. The purpose 104 | # of that is we might want to do a heuristics pass over the keyframes 105 | # to clean it up. 106 | def insert_keyframe(self, frame, value): 107 | self.animation_data[frame] = value 108 | 109 | def del_keyframe(self, frame): 110 | try: 111 | self.mybone.keyframe_delete('rotation_quaternion', index=3, frame=frame) 112 | except: 113 | return 114 | 115 | def del_keyframes(self): 116 | for k, v in self.animation_data.items(): 117 | try: 118 | self.mybone.keyframe_delete('rotation_quaternion', index=3, 119 | frame=k) 120 | except: 121 | return 122 | self.animation_data = {} 123 | 124 | def animate(self): 125 | for k, v in self.animation_data.items(): 126 | self.mybone.rotation_quaternion[3] = v 127 | self.mybone.keyframe_insert('rotation_quaternion', index=3, frame=k) 128 | 129 | # run our list of heuristics over the list of keyframes we have 130 | # 01 Heuristic: If a bone is being reset to 0 and it has been set to 131 | # some other value less than 5 frames before, or is going to be 132 | # set to another value less than 5 frames after, then skip setting 133 | # that bone 134 | # 02 Heuristic: If a bone is being set to some value != 0, but it has 135 | # been set to 0 or some other value < 5 frames before, and it's 136 | # going to be set to another value != 0 < 5 frames after, then 137 | # recalculate the value of this key frame to be the average of the 138 | # two constraining key frames. 139 | # 03 Heuristic: If a bone is being set to some value != 0 but it 140 | # has been set to some value != 0 < 5 frames before, and it's 141 | # going to be set to another value >= 0 < 5 frames after, then set 142 | # the value of that keyframe to be the average of the two 143 | # constraining keyframes. 144 | # 04 Heuristic: if the time between the end of the word and the start 145 | # of the next one is greater than 15 frames, then the mouth should be 146 | # closed. if it's greater than 20 frames then key frames to close the 147 | # mouth are inserted 5 frames after the word and 5 frames before the 148 | # next one 149 | def heuristic_pass(self): 150 | prev_k = 0 151 | key_list = list(self.animation_data.keys()) 152 | value_list = list(self.animation_data.values()) 153 | num_kf = len(key_list) 154 | # the animation data is in chronological order 155 | i = 0 156 | for k in key_list: 157 | if i == 0 or i == (num_kf - 1): 158 | i = i + 1 159 | continue 160 | prev_key = key_list[i-1] 161 | next_key = key_list[i+1] 162 | if k - prev_key <= 2 and next_key - k <= 2: 163 | self.animation_data[k] = (value_list[i-1] + value_list[i+1]) / 2 164 | i = i + 1 165 | return 166 | 167 | def heuristic_pass2(self, window_size): 168 | # Define a window, where the current key frame is in the middle. 169 | # The current key frame is calculated as the average of the values 170 | # of the key frames surrounding it. 171 | if window_size == 0: 172 | return 173 | key_list = list(self.animation_data.keys()) 174 | value_list = list(self.animation_data.values()) 175 | num_kf = len(key_list) 176 | window_left = int(math.floor(float(window_size)/float(2))) 177 | window_right = int(math.ceil(float(window_size)/float(2))) 178 | i = 0 179 | # this is a float for upper/lower calculations just convert it to 180 | # an int 181 | for k in key_list: 182 | fupper = k + window_right 183 | flower = max(0, k - window_left) 184 | upper = 0 185 | lower = 0 186 | # find the largest key frame that would be smaller than upper 187 | for j in range(i, num_kf): 188 | if key_list[j] < fupper: 189 | upper = j 190 | else: 191 | break 192 | for j in range(0, i): 193 | if key_list[j] < flower: 194 | lower = j 195 | else: 196 | break 197 | if key_list[upper] - k > window_right: 198 | upper = i 199 | if k - key_list[lower] > window_left: 200 | lower = i 201 | total_value = 0 202 | for j in range(lower, upper): 203 | total_value = total_value + value_list[j] 204 | # use the float value to calculate the avg 205 | if (upper > lower): 206 | avg = total_value / (upper - lower) 207 | # don't drop the key frame value more than a 1/3 of its 208 | # original value 209 | avg = max(self.animation_data[k] - self.animation_data[k]/3, avg) 210 | else: 211 | avg = self.animation_data[k] 212 | self.animation_data[k] = avg 213 | i = i + 1 214 | 215 | # each sequence can have multiple markers associated with it. 216 | class Sequence(object): 217 | def __init__(self, seq): 218 | self.sequence = seq 219 | self.markers = [] 220 | self.bones = {} 221 | self.bones_set = False 222 | 223 | def set_bones(self, bones): 224 | if self.bones_set == True: 225 | return 226 | 227 | self.bones_set = True 228 | 229 | for bone in bones: 230 | b = Bone(bone) 231 | self.bones[b.get_name()] = b 232 | 233 | # Markers are added in sequential order 234 | def add_marker(self, m): 235 | self.markers.append(m) 236 | 237 | def del_marker(self, m): 238 | if m in self.markers: 239 | self.markers.remove(m) 240 | 241 | def rm_marker_from_scene(self, scn): 242 | for m in self.markers: 243 | scn.timeline_markers.remove(m) 244 | self.markers = [] 245 | 246 | def is_sequence(self, s): 247 | return (self.sequence == s) 248 | 249 | def mark_seq_at_frame(self, mname, frame, scn): 250 | m = scn.timeline_markers.new(mname, frame=frame) 251 | self.add_marker(m) 252 | 253 | def move_to_next_marker(self, scn): 254 | cur_frame = scn.frame_current 255 | found = False 256 | for m in self.markers: 257 | if m.frame > cur_frame: 258 | found = True 259 | break 260 | if found: 261 | scn.frame_current = m.frame 262 | 263 | def move_to_prev_marker(self, scn): 264 | cur_frame = scn.frame_current 265 | found = False 266 | for m in reversed(self.markers): 267 | if m.frame < cur_frame: 268 | found = True 269 | break 270 | if found: 271 | scn.frame_current = m.frame 272 | 273 | def reset_all_bones(self, frame): 274 | for k, bone in self.bones.items(): 275 | bone.insert_keyframe(frame, 0) 276 | 277 | def set_keyframe(self, m, pm, idx): 278 | delta = 0 279 | # Heuristic: If the delta between this marker and the previous 280 | # marker is >= 12 frames then we want to set a rest 281 | # in/out poses 282 | # Heuristic: If the delta is < 12 then we want to have a rest pose 283 | # in the middle 284 | if pm: 285 | delta = m.frame - pm.frame 286 | if delta >= 12: 287 | percent = round(delta * 0.08) 288 | percent2 = round(delta * 0.20) 289 | self.reset_all_bones(pm.frame + percent) 290 | self.set_random_rest_pose(pm.frame + percent2) 291 | self.set_random_rest_pose(m.frame - percent2) 292 | self.reset_all_bones(m.frame - percent) 293 | elif delta < 12 and delta > 7: 294 | self.reset_all_bones(pm.frame + round(delta/2)) 295 | self.set_random_rest_pose(pm.frame + round(delta/2)) 296 | 297 | if idx == 0: 298 | if (m.frame - 1) <= 5: 299 | frame = 1 300 | else: 301 | frame = m.frame - 5 302 | self.reset_all_bones(frame) 303 | self.reset_all_bones(m.frame) 304 | 305 | phonemes = yaspmapper.get_phoneme_animation_data(m.name) 306 | if not phonemes: 307 | logger.critical("Can't find corresponding mapping for:", m.name) 308 | return 309 | for phone in phonemes: 310 | bone_name = 'ph_'+phone[0] 311 | bone = self.bones[bone_name] 312 | bone.insert_keyframe(m.frame, phone[1]) 313 | 314 | 315 | # go through the markers on the selected sequence. 316 | # for each marker look up the marker name in our mapper 317 | # Set the corresponding bones in the list to the values specified. 318 | def animate_all_markers(self): 319 | idx = 0 320 | bpy.ops.pose.select_all(action='DESELECT') 321 | pm = None 322 | # first pass is to create keyframe entries in every bone for each 323 | # marker 324 | for m in self.markers: 325 | self.set_keyframe(m, pm, idx) 326 | pm = m 327 | idx = idx + 1 328 | 329 | self.reset_all_bones(m.frame + 5) 330 | 331 | # second pass is to run a heuristic pass on the animation data and 332 | # animate 333 | for k, bone in self.bones.items(): 334 | bone.heuristic_pass2(float(bpy.context.scene.yasp_avg_window_size)) 335 | bone.animate() 336 | 337 | def animate_marker_at_frame(self, cur_frame): 338 | idx = 0 339 | found = False 340 | pm = None 341 | for m in self.markers: 342 | if m.frame == cur_frame: 343 | found = True 344 | break 345 | pm = m 346 | idx = idx + 1 347 | 348 | if found: 349 | self.set_keyframe(m, pm, idx) 350 | 351 | def del_all_keyframes(self): 352 | scn = bpy.context.scene 353 | for k, bone in self.bones.items(): 354 | bone.del_keyframes() 355 | if not scn.yasp_phoneme_rig: 356 | phoneme_rig = bpy.data.objects.get('MBLab_skeleton_phoneme_rig') 357 | else: 358 | phoneme_rig = bpy.data.objects.get(scn.yasp_phoneme_rig) 359 | phoneme_rig.animation_data_clear() 360 | 361 | def del_keyframe(self, frame): 362 | for k, bone in self.bones.items(): 363 | bone.del_keyframe(frame) 364 | 365 | def set_random_rest_pose(self, frame): 366 | bone = self.bones['ph_REST'] 367 | bone.insert_keyframe(frame, random.uniform(0, 1)) 368 | 369 | class SequenceMgr(object): 370 | def __init__(self): 371 | self.sequences = [] 372 | self.orig_frame_set = False 373 | 374 | def set_orig_frame(self, scn): 375 | if not self.orig_frame_set: 376 | self.orig_frame_start = scn.frame_start 377 | self.orig_frame_end = scn.frame_end 378 | self.orig_frame_set = True 379 | 380 | def add_sequence(self, s): 381 | seq = Sequence(s) 382 | self.sequences.append(seq) 383 | 384 | def del_sequence(self, s): 385 | if s in self.sequences: 386 | self.sequences.remove(s) 387 | 388 | def get_sequence(self, s): 389 | for seq in self.sequences: 390 | if seq.is_sequence(s): 391 | return seq 392 | return None 393 | 394 | def set_bones(self, s, bones): 395 | seq = self.get_sequence(s) 396 | if not seq: 397 | return 398 | seq.set_bones(bones) 399 | 400 | def unmark_sequence(self, s, scn): 401 | seq = self.get_sequence(s) 402 | if not seq: 403 | return 404 | seq.rm_marker_from_scene(scn) 405 | 406 | def rm_seq_from_scene(self, s, scn): 407 | seq = self.get_sequence(s) 408 | if not seq: 409 | return 410 | scn.sequence_editor.sequences.remove(s) 411 | seq.rm_marker_from_scene(scn) 412 | self.del_sequence(seq) 413 | 414 | def mark_seq_at_frame(self, s, mname, frame, scn): 415 | seq = self.get_sequence(s) 416 | if not seq: 417 | return False 418 | seq.mark_seq_at_frame(mname, frame, scn) 419 | 420 | def move_to_next_marker(self, s, scn): 421 | seq = self.get_sequence(s) 422 | if not seq: 423 | return 424 | seq.move_to_next_marker(scn) 425 | 426 | def move_to_prev_marker(self, s, scn): 427 | seq = self.get_sequence(s) 428 | if not seq: 429 | return 430 | seq.move_to_prev_marker(scn) 431 | 432 | def animate_all_markers(self, s): 433 | seq = self.get_sequence(s) 434 | if not seq: 435 | return 436 | seq.animate_all_markers() 437 | 438 | def animate_current(self, s, scn): 439 | seq = self.get_sequence(s) 440 | if not seq: 441 | return 442 | seq.animate_marker_at_frame(scn.frame_current) 443 | 444 | def del_keyframe(self, s, scn): 445 | seq = self.get_sequence(s) 446 | if not seq: 447 | return 448 | seq.del_keyframe(scn.frame_current) 449 | 450 | def del_all_keyframes(self, s, scn): 451 | seq = self.get_sequence(s) 452 | if not seq: 453 | return 454 | seq.del_all_keyframes() 455 | 456 | def restore_start_end_frames(self): 457 | bpy.context.scene.frame_start = self.orig_frame_start 458 | bpy.context.scene.frame_end = self.orig_frame_end 459 | 460 | seqmgr = SequenceMgr() 461 | yaspmapper = YASP2MBPhonemeMapper() 462 | 463 | class YASP_OT_mark(bpy.types.Operator): 464 | bl_idname = "yasp.mark_audio" 465 | bl_label = "Mark" 466 | bl_description = "Run YASP and mark audio" 467 | 468 | def mark_audio(self, json_str, offset, seq, scn): 469 | jdict = json.loads(json_str) 470 | word_list = [] 471 | # iterate over the json dictionary to and create markers 472 | try: 473 | word_list = jdict['words'] 474 | except: 475 | return False 476 | 477 | for word in word_list: 478 | try: 479 | phonemes = word['phonemes'] 480 | except: 481 | return False 482 | for phone in phonemes: 483 | try: 484 | # calculate the frame to insert the marker 485 | frame = round((scn.render.fps/scn.render.fps_base) * 486 | (phone['start'] / 100)) 487 | cur_frame = offset + frame 488 | seqmgr.mark_seq_at_frame(seq, phone['phoneme'], cur_frame, scn) 489 | except Exception as e: 490 | logger.critical(e) 491 | return False 492 | return True 493 | 494 | def free_json_str(self, json_str): 495 | #yasp.yasp_free_json_str(json_str) 496 | return 497 | 498 | def run_yasp(self, wave, transcript, offset): 499 | if not wave or not transcript: 500 | self.report({'ERROR'}, "bad wave or transcript files") 501 | return None 502 | 503 | logs = yasp.yasp_logs() 504 | yasp.yasp_set_modeldir(yasp_model_dir) 505 | yasp.yasp_setup_logging(logs, None, "MB_YASP_Logs") 506 | json_str = yasp.yasp_interpret_get_str(wave, transcript, None) 507 | yasp.yasp_finish_logging(logs) 508 | if not json_str: 509 | self.report({'ERROR'}, "Couldn't parse speech") 510 | return None 511 | if os.path.exists("MB_YASP_Logs"): 512 | os.remove("MB_YASP_Logs") 513 | 514 | return json_str 515 | 516 | def execute(self, context): 517 | scn = context.scene 518 | wave = scn.yasp_wave_path 519 | transcript = scn.yasp_transcript_path 520 | 521 | if not os.path.isfile(wave) or \ 522 | not os.path.isfile(transcript): 523 | self.report({'ERROR'}, 'Bad path to wave or transcript') 524 | return {'FINISHED'} 525 | 526 | if not scn.yasp_start_frame: 527 | start_frame = 1 528 | else: 529 | try: 530 | start_frame = int(scn.yasp_start_frame) 531 | except: 532 | self.report({'ERROR'}, 'Bad start frame') 533 | return {'FINISHED'} 534 | 535 | json_str = self.run_yasp(wave, transcript, start_frame) 536 | if not json_str: 537 | return {'FINISHED'} 538 | 539 | # find a free channel in the sequence editor 540 | channels = [] 541 | for s in scn.sequence_editor.sequences_all: 542 | channels.append(s.channel) 543 | channels.sort() 544 | 545 | channel_select = 1 546 | for c in channels: 547 | if c > channel_select: 548 | break 549 | channel_select = c + 1 550 | #insert the wave file 551 | seq = scn.sequence_editor.sequences.new_sound(os.path.basename(wave), wave, 552 | channel_select, start_frame) 553 | 554 | seqmgr.add_sequence(seq) 555 | 556 | if not self.mark_audio(json_str, start_frame, seq, scn): 557 | seqmgr.rm_seq_from_scene(seq, scn) 558 | self.report({'ERROR'}, 'Failed to mark the audio file') 559 | # some memory management 560 | self.free_json_str(json_str) 561 | return {'FINISHED'} 562 | 563 | # some memory management 564 | self.free_json_str(json_str) 565 | 566 | # set the end frame 567 | end = 0 568 | for s in scn.sequence_editor.sequences_all: 569 | if s.frame_final_end > end: 570 | end = s.frame_final_end 571 | scn.frame_end = end 572 | return {'FINISHED'} 573 | 574 | class YASP_OT_unmark(bpy.types.Operator): 575 | bl_idname = "yasp.unmark_audio" 576 | bl_label = "Unmark" 577 | bl_description = "Unmark the audio file and remove" 578 | 579 | def execute(self, context): 580 | scn = context.scene 581 | 582 | seq = scn.sequence_editor.active_strip 583 | if seq and seq.select: 584 | seqmgr.unmark_sequence(seq, scn) 585 | else: 586 | self.report({'ERROR'}, 'Must select a strip to unmark') 587 | 588 | seqmgr.restore_start_end_frames() 589 | 590 | return {'FINISHED'} 591 | 592 | class YASP_OT_delete_seq(bpy.types.Operator): 593 | bl_idname = "yasp.delete_seq" 594 | bl_label = "Remove Strip" 595 | bl_description = "Delete active sequence" 596 | 597 | def execute(self, context): 598 | scn = context.scene 599 | seq = scn.sequence_editor.active_strip 600 | if seq and seq.select: 601 | seqmgr.rm_seq_from_scene(seq, scn) 602 | if len(scn.sequence_editor.sequences_all) == 0: 603 | seqmgr.restore_start_end_frames() 604 | else: 605 | self.report({'ERROR'}, 'Must select a strip to delete') 606 | 607 | return {'FINISHED'} 608 | 609 | def set_animation_prereq(scn): 610 | seq = scn.sequence_editor.active_strip 611 | if not seq or not seq.select: 612 | return 'STRIP_ERROR', None 613 | 614 | if not scn.yasp_phoneme_rig: 615 | phoneme_rig = bpy.data.objects.get('MBLab_skeleton_phoneme_rig') 616 | else: 617 | phoneme_rig = bpy.data.objects.get(scn.yasp_phoneme_rig) 618 | if not phoneme_rig: 619 | return 'RIG_ERROR', None 620 | 621 | # select the rig and put it in POSE mode 622 | for obj in bpy.data.objects: 623 | obj.select_set(False) 624 | phoneme_rig.select_set(True) 625 | bpy.context.view_layer.objects.active = phoneme_rig 626 | bpy.ops.object.mode_set(mode='POSE') 627 | seqmgr.set_bones(seq, bpy.context.object.pose.bones) 628 | return 'SUCCESS', seq 629 | 630 | 631 | class YASP_OT_setallKeyframes(bpy.types.Operator): 632 | bl_idname = "yasp.set_all_keyframes" 633 | bl_label = "Animate" 634 | bl_description = "Set all marked lip-sync keyframe" 635 | 636 | def execute(self, context): 637 | scn = context.scene 638 | rc, seq = set_animation_prereq(scn) 639 | if (rc == 'STRIP_ERROR'): 640 | self.report({'ERROR'}, "Must select a strip to operate on") 641 | return {'FINISHED'} 642 | elif (rc == 'RIG_ERROR'): 643 | self.report({'ERROR'}, "Phoneme Rig not found") 644 | return {'FINISHED'} 645 | 646 | # insert key frames on all markers. 647 | seqmgr.animate_all_markers(seq) 648 | return {'FINISHED'} 649 | 650 | class YASP_OT_deleteallKeyframes(bpy.types.Operator): 651 | bl_idname = "yasp.delete_all_keyframes" 652 | bl_label = "Remove Animation" 653 | bl_description = "Set all marked lip-sync keyframe" 654 | 655 | def execute(self, context): 656 | scn = context.scene 657 | rc, seq = set_animation_prereq(scn) 658 | if (rc == 'STRIP_ERROR'): 659 | self.report({'ERROR'}, "Must select a strip to operate on") 660 | return {'FINISHED'} 661 | elif (rc == 'RIG_ERROR'): 662 | self.report({'ERROR'}, "Phoneme Rig not found") 663 | return {'FINISHED'} 664 | 665 | seqmgr.del_all_keyframes(seq, scn) 666 | return {'FINISHED'} 667 | 668 | class YASP_OT_set(bpy.types.Operator): 669 | bl_idname = "yasp.set_keyframe" 670 | bl_label = "Set" 671 | bl_description = "Set a lip-sync keyframe" 672 | 673 | def execute(self, context): 674 | scn = context.scene 675 | rc, seq = set_animation_prereq(scn) 676 | if (rc == 'STRIP_ERROR'): 677 | self.report({'ERROR'}, "Must select a strip to operate on") 678 | return {'FINISHED'} 679 | elif (rc == 'RIG_ERROR'): 680 | self.report({'ERROR'}, "Phoneme Rig not found") 681 | return {'FINISHED'} 682 | 683 | seqmgr.animate_current(seq, scn) 684 | return {'FINISHED'} 685 | 686 | class YASP_OT_unset(bpy.types.Operator): 687 | bl_idname = "yasp.del_keyframe" 688 | bl_label = "Unset" 689 | bl_description = "Unset a lip-sync keyframe" 690 | 691 | def execute(self, context): 692 | scn = context.scene 693 | rc, seq = set_animation_prereq(scn) 694 | if (rc == 'STRIP_ERROR'): 695 | self.report({'ERROR'}, "Must select a strip to operate on") 696 | return {'FINISHED'} 697 | elif (rc == 'RIG_ERROR'): 698 | self.report({'ERROR'}, "Phoneme Rig not found") 699 | return {'FINISHED'} 700 | 701 | seqmgr.del_keyframe(seq, scn) 702 | return {'FINISHED'} 703 | 704 | class YASP_OT_next(bpy.types.Operator): 705 | bl_idname = "yasp.next_marker" 706 | bl_label = "next" 707 | bl_description = "Jump to next marker" 708 | 709 | def execute(self, context): 710 | scn = context.scene 711 | 712 | seq = scn.sequence_editor.active_strip 713 | if seq and seq.select: 714 | seqmgr.move_to_next_marker(seq, scn) 715 | else: 716 | self.report({'ERROR'}, "Must select a strip") 717 | 718 | return {'FINISHED'} 719 | 720 | class YASP_OT_prev(bpy.types.Operator): 721 | bl_idname = "yasp.prev_marker" 722 | bl_label = "prev" 723 | bl_description = "Jump to previous marker" 724 | 725 | def execute(self, context): 726 | scn = context.scene 727 | 728 | seq = scn.sequence_editor.active_strip 729 | if seq and seq.select: 730 | seqmgr.move_to_prev_marker(seq, scn) 731 | else: 732 | self.report({'ERROR'}, "Must select a strip") 733 | 734 | return {'FINISHED'} 735 | 736 | class VIEW3D_PT_tools_mb_yasp(bpy.types.Panel): 737 | bl_label = "Speech Parser" 738 | bl_space_type = "VIEW_3D" 739 | bl_region_type = "UI" 740 | bl_category = "YASP" 741 | #bl_options = {'DEFAULT_CLOSED'} 742 | 743 | def draw(self, context): 744 | scn = context.scene 745 | layout = self.layout 746 | wm = context.window_manager 747 | col = layout.column(align=True) 748 | 749 | seqmgr.set_orig_frame(scn) 750 | 751 | if platform.system() != "Linux": 752 | col.label(text="Linux only feature", icon='ERROR') 753 | return 754 | 755 | if not libs_loaded: 756 | col.label(text="Libraries not loaded", icon='ERROR') 757 | return 758 | 759 | col.label(text="Phoneme Rig Name") 760 | col.prop(scn, "yasp_phoneme_rig", text='') 761 | col.label(text="Path to WAV file") 762 | col.prop(scn, "yasp_wave_path", text='') 763 | col.label(text="Path to transcript file") 764 | col.prop(scn, "yasp_transcript_path", text="") 765 | col.label(text="Start on frame") 766 | col.prop(scn, "yasp_start_frame", text="") 767 | col.label(text="Window Size") 768 | col.prop(scn, "yasp_avg_window_size", text="") 769 | col = layout.column(align=True) 770 | row = col.row(align=False) 771 | row.operator('yasp.mark_audio', icon='MARKER_HLT') 772 | row.operator('yasp.unmark_audio', icon='MARKER') 773 | col = layout.column(align=True) 774 | col.operator('yasp.set_all_keyframes', icon='DECORATE_KEYFRAME') 775 | col = layout.column(align=True) 776 | col.operator('yasp.delete_all_keyframes', icon='KEYFRAME') 777 | col = layout.column(align=True) 778 | row = col.row(align=False) 779 | row.operator('yasp.set_keyframe', icon='KEYFRAME_HLT') 780 | row.operator('yasp.del_keyframe', icon='KEYFRAME') 781 | col = layout.column(align=True) 782 | row = col.row(align=False) 783 | row.operator('yasp.prev_marker', icon='PREV_KEYFRAME') 784 | row.operator('yasp.next_marker', icon='NEXT_KEYFRAME') 785 | col = layout.column(align=True) 786 | col.operator('yasp.delete_seq', icon='KEYFRAME') 787 | 788 | 789 | -------------------------------------------------------------------------------- /data/yasp_map.json: -------------------------------------------------------------------------------- 1 | { 2 | "B": [ 3 | [ 4 | "MBP", 5 | 0.262 6 | ] 7 | ], 8 | "D": [ 9 | [ 10 | "CDGKNRSYZ", 11 | 0.221 12 | ], 13 | [ 14 | "WQ", 15 | 0.149 16 | ] 17 | ], 18 | "F": [ 19 | [ 20 | "FV", 21 | 1 22 | ] 23 | ], 24 | "G": [ 25 | [ 26 | "CDGKNRSYZ", 27 | 0.276 28 | ], 29 | [ 30 | "WQ", 31 | 0.076 32 | ] 33 | ], 34 | "K": [ 35 | [ 36 | "CDGKNRSYZ", 37 | 0.276 38 | ], 39 | [ 40 | "WQ", 41 | 0.076 42 | ] 43 | ], 44 | "L": [ 45 | [ 46 | "L", 47 | 0.865 48 | ] 49 | ], 50 | "M": [ 51 | [ 52 | "MBP", 53 | 0.276 54 | ] 55 | ], 56 | "N": [ 57 | [ 58 | "CDGKNRSYZ", 59 | 0.221 60 | ], 61 | [ 62 | "WQ", 63 | 0.149 64 | ] 65 | ], 66 | "P": [ 67 | [ 68 | "MBP", 69 | 0.319 70 | ] 71 | ], 72 | "R": [ 73 | [ 74 | "AI", 75 | 0.183 76 | ], 77 | [ 78 | "CH", 79 | 0.3 80 | ], 81 | [ 82 | "WQ", 83 | 0.15 84 | ] 85 | ], 86 | "S": [ 87 | [ 88 | "CDGKNRSYZ", 89 | 0.21 90 | ], 91 | [ 92 | "CH", 93 | 0.225 94 | ], 95 | [ 96 | "WQ", 97 | 0.233 98 | ] 99 | ], 100 | "T": [ 101 | [ 102 | "CDGKNRSYZ", 103 | 0.221 104 | ], 105 | [ 106 | "WQ", 107 | 0.149 108 | ] 109 | ], 110 | "V": [ 111 | [ 112 | "FV", 113 | 0.6 114 | ], 115 | [ 116 | "CH", 117 | 0.052 118 | ] 119 | ], 120 | "W": [ 121 | [ 122 | "WQ", 123 | 0.368 124 | ], 125 | [ 126 | "U", 127 | 0.1 128 | ] 129 | ], 130 | "Y": [ 131 | [ 132 | "AI", 133 | 0.25 134 | ], 135 | [ 136 | "CH", 137 | 0.12 138 | ], 139 | [ 140 | "WQ", 141 | 0.077 142 | ] 143 | ], 144 | "Z": [ 145 | [ 146 | "CDGKNRSYZ", 147 | 0.21 148 | ], 149 | [ 150 | "CH", 151 | 0.225 152 | ], 153 | [ 154 | "WQ", 155 | 0.233 156 | ] 157 | ], 158 | "AA": [ 159 | [ 160 | "AI", 161 | 0.181 162 | ] 163 | ], 164 | "AE": [ 165 | [ 166 | "AI", 167 | 0.352 168 | ] 169 | ], 170 | "AH": [ 171 | [ 172 | "AI", 173 | 0.208 174 | ], 175 | [ 176 | "WQ", 177 | 0.078 178 | ] 179 | ], 180 | "AO": [ 181 | [ 182 | "O", 183 | 0.208 184 | ], 185 | [ 186 | "U", 187 | 0.176 188 | ] 189 | ], 190 | "AW": [ 191 | [ 192 | "AI", 193 | 0.156 194 | ], 195 | [ 196 | "O", 197 | 0.116 198 | ], 199 | [ 200 | "WQ", 201 | 0.092 202 | ] 203 | ], 204 | "AY": [ 205 | [ 206 | "AI", 207 | 0.3 208 | ] 209 | ], 210 | "CH": [ 211 | [ 212 | "CH", 213 | 0.225 214 | ], 215 | [ 216 | "O", 217 | 0.202 218 | ] 219 | ], 220 | "DH": [ 221 | [ 222 | "TH", 223 | 0.796 224 | ] 225 | ], 226 | "EH": [ 227 | [ 228 | "E", 229 | 0.324 230 | ], 231 | [ 232 | "WQ", 233 | 0.151 234 | ] 235 | ], 236 | "ER": [ 237 | [ 238 | "AI", 239 | 0.1 240 | ], 241 | [ 242 | "WQ", 243 | 0.35 244 | ] 245 | ], 246 | "EY": [ 247 | [ 248 | "AI", 249 | 0.3 250 | ], 251 | [ 252 | "U", 253 | 0.05 254 | ], 255 | [ 256 | "CH", 257 | 0.1 258 | ] 259 | ], 260 | "HH": [ 261 | [ 262 | "AI", 263 | 0.3 264 | ], 265 | [ 266 | "U", 267 | 0.05 268 | ] 269 | ], 270 | "IH": [ 271 | [ 272 | "AI", 273 | 0.09 274 | ], 275 | [ 276 | "E", 277 | 0.165 278 | ] 279 | ], 280 | "IY": [ 281 | [ 282 | "AI", 283 | 0.197 284 | ], 285 | [ 286 | "E", 287 | 0.165 288 | ] 289 | ], 290 | "JH": [ 291 | [ 292 | "CH", 293 | 0.205 294 | ], 295 | [ 296 | "O", 297 | 0.264 298 | ] 299 | ], 300 | "NG": [ 301 | [ 302 | "CDGKNRSYZ", 303 | 0.016 304 | ], 305 | [ 306 | "E", 307 | 0.304 308 | ] 309 | ], 310 | "OW": [ 311 | [ 312 | "AI", 313 | 0.1 314 | ], 315 | [ 316 | "U", 317 | 0.075 318 | ], 319 | [ 320 | "WQ", 321 | 0.5 322 | ] 323 | ], 324 | "OY": [ 325 | [ 326 | "AI", 327 | 0.05 328 | ], 329 | [ 330 | "O", 331 | 0.1 332 | ], 333 | [ 334 | "U", 335 | 0.1 336 | ], 337 | [ 338 | "WQ", 339 | 0.35 340 | ] 341 | ], 342 | "SH": [ 343 | [ 344 | "CH", 345 | 0.342 346 | ], 347 | [ 348 | "O", 349 | 0.082 350 | ], 351 | [ 352 | "WQ", 353 | 0.47 354 | ] 355 | ], 356 | "TH": [ 357 | [ 358 | "TH", 359 | 1 360 | ], 361 | [ 362 | "CH", 363 | 0.2 364 | ] 365 | ], 366 | "UH": [ 367 | [ 368 | "U", 369 | 0.35 370 | ], 371 | [ 372 | "WQ", 373 | 0.5 374 | ] 375 | ], 376 | "UW": [ 377 | [ 378 | "U", 379 | 0.274 380 | ] 381 | ], 382 | "ZH": [ 383 | [ 384 | "CH", 385 | 0.225 386 | ], 387 | [ 388 | "O", 389 | 0.202 390 | ] 391 | ] 392 | } -------------------------------------------------------------------------------- /data/yasp_map.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | phoneme_map = { 4 | 'B': [['MBP', 0.262]], 5 | 'D': [['CDGKNRSYZ', 0.221], ['WQ', 0.149]], 6 | 'F': [['FV', 1]], 7 | 'G': [['CDGKNRSYZ', 0.276], ['WQ', 0.076]], 8 | 'K': [['CDGKNRSYZ', 0.276], ['WQ', 0.076]], 9 | 'L': [['L', 0.865]], 10 | 'M': [['MBP', 0.276]], 11 | 'N': [['CDGKNRSYZ', 0.221], ['WQ', 0.149]], 12 | 'P': [['MBP', 0.319]], 13 | 'R': [['AI', 0.183], ['CH', 0.3], ['WQ', 0.15]], 14 | 'S': [['CDGKNRSYZ', 0.21], ['CH', 0.225], ['WQ', 0.233]], 15 | 'T': [['CDGKNRSYZ', 0.221], ['WQ', 0.149]], 16 | 'V': [['FV', 0.6], ['CH', 0.052]], 17 | 'W': [['WQ', 0.368], ['U', 0.1]], 18 | 'Y': [['AI', 0.25], ['CH', 0.12], ['WQ', 0.077]], 19 | 'Z': [['CDGKNRSYZ', 0.21], ['CH', 0.225], ['WQ', 0.233]], 20 | 'AA': [['AI', 0.181]], 21 | 'AE': [['AI', 0.352]], 22 | 'AH': [['AI', 0.208], ['WQ', 0.078]], 23 | 'AO': [['O', 0.208], ['U', 0.176]], 24 | 'AW': [['AI', 0.156], ['O', 0.116], ['WQ', 0.092]], 25 | 'AY': [['AI', 0.3]], 26 | 'CH': [['CH', 0.225], ['O', 0.202]], 27 | 'DH': [['TH', 0.796]], 28 | 'EH': [['E', 0.324], ['WQ', 0.151]], 29 | 'ER': [['AI', 0.1], ['WQ', 0.35]], 30 | 'EY': [['AI', 0.3], ['U', 0.05], ['CH', 0.1]], 31 | 'HH': [['AI', 0.3], ['U', 0.05]], 32 | 'IH': [['AI', 0.09], ['E', 0.165]], 33 | 'IY': [['AI', 0.197], ['E', 0.165]], 34 | 'JH': [['CH', 0.205], ['O', 0.264]], 35 | 'NG': [['CDGKNRSYZ', 0.016], ['E', 0.304]], 36 | 'OW': [['AI', 0.1], ['U', 0.075], ['WQ', 0.5]], 37 | 'OY': [['AI', 0.05], ['O', 0.1], ['U', 0.1], ['WQ', 0.35]], 38 | 'SH': [['CH', 0.342], ['O', 0.082], ['WQ', 0.470]], 39 | 'TH': [['TH', 1], ['CH', 0.2]], 40 | 'UH': [['U', 0.35], ['WQ', 0.5]], 41 | 'UW': [['U', 0.274]], 42 | 'ZH': [['CH', 0.225], ['O', 0.202]], 43 | } 44 | 45 | f = open('yasp_map.json', 'w') 46 | json.dump(phoneme_map, f, indent=4) 47 | f.close() 48 | -------------------------------------------------------------------------------- /facs_process.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import csv 3 | import json 4 | from scipy.signal import savgol_filter 5 | from scipy.ndimage import gaussian_filter1d 6 | import numpy as np 7 | plt_unsupported = False 8 | try: 9 | import matplotlib.pyplot as plt 10 | except: 11 | plt_unsupported = True 12 | # https://github.com/NumesSanguis/FACSvatar 13 | # https://github.com/TadasBaltrusaitis/OpenFace/wiki/Action-Units 14 | # https://www.cs.cmu.edu/~face/facs.htm 15 | # https://en.wikipedia.org/wiki/Facial_Action_Coding_System 16 | # https://github.com/TadasBaltrusaitis/OpenFace/wiki/Output-Format 17 | # https://facsvatar.readthedocs.io/en/latest/defaultsetup.html 18 | # 19 | 20 | # There are 68 points 0-67 21 | MAX_PDM_ENTRIES = 68 22 | MAX_PDM_NON_RIGID_ENTRIES = 34 23 | MAX_PDM_EYE_LMK = 56 24 | VALUES = 0 25 | MAXIMAS = 1 26 | MINIMAS = 2 27 | 28 | facs_data_items = ['frame', 'timestamp', 'AU01_r', 'AU02_r', 29 | 'AU04_r', 'AU05_r', 'AU06_r', 'AU07_r', 30 | 'AU09_r', 'AU10_r', 'AU12_r', 'AU14_r', 31 | 'AU15_r', 'AU17_r', 'AU20_r', 'AU23_r', 32 | 'AU25_r', 'AU26_r', 'AU45_r', 'gaze_angle_x', 33 | 'gaze_angle_y', 'pose_Rx', 'pose_Ry', 'pose_Rz', 34 | 'pose_Tx', 'pose_Ty', 'pose_Tz'] 35 | 36 | rigid_data_items = ['frame', 'timestamp', 'p_scale', 'p_rx', 37 | 'p_ry', 'p_rz', 'p_tx', 'p_ty'] 38 | 39 | animation_data = {} 40 | pdm_2d = {} 41 | pdm_3d = {} 42 | rigid_data = {} 43 | non_rigid_data = {} 44 | eye_lmk_2d = {} 45 | eye_lmk_3d = {} 46 | 47 | def init_database(): 48 | global animation_data 49 | global pdm_2d 50 | global pdm_3d 51 | global rigid_data 52 | global non_rigid_data 53 | global eye_lmk_2d 54 | global eye_lmk_3d 55 | 56 | for e in facs_data_items: 57 | animation_data[e] = [[], [], []] 58 | 59 | for e in rigid_data_items: 60 | rigid_data[e] = [[], [], []] 61 | 62 | pdm_2d['frame'] = [[], [], []] 63 | pdm_2d['timestamp'] = [[], [], []] 64 | for i in range(0, MAX_PDM_ENTRIES): 65 | name = 'x_'+str(i) 66 | pdm_2d[name] = [[], [], []] 67 | for i in range(0, MAX_PDM_ENTRIES): 68 | name = 'y_'+str(i) 69 | pdm_2d[name] = [[], [], []] 70 | 71 | pdm_3d['frame'] = [[], [], []] 72 | pdm_3d['timestamp'] = [[], [], []] 73 | for i in range(0, MAX_PDM_ENTRIES): 74 | name = 'X_'+str(i) 75 | pdm_3d[name] = [[], [], []] 76 | for i in range(0, MAX_PDM_ENTRIES): 77 | name = 'Y_'+str(i) 78 | pdm_3d[name] = [[], [], []] 79 | for i in range(0, MAX_PDM_ENTRIES): 80 | name = 'Z_'+str(i) 81 | pdm_3d[name] = [[], [], []] 82 | 83 | non_rigid_data['frame'] = [[], [], []] 84 | non_rigid_data['timestamp'] = [[], [], []] 85 | for i in range(0, MAX_PDM_NON_RIGID_ENTRIES): 86 | name = 'p_'+str(i) 87 | non_rigid_data[name] = [[], [], []] 88 | 89 | for i in range(0, MAX_PDM_EYE_LMK): 90 | name = 'eye_lmk_x_'+str(i) 91 | eye_lmk_2d[name] = [[], [], []] 92 | for i in range(0, MAX_PDM_EYE_LMK): 93 | name = 'eye_lmk_y_'+str(i) 94 | eye_lmk_2d[name] = [[], [], []] 95 | 96 | for i in range(0, MAX_PDM_EYE_LMK): 97 | name = 'eye_lmk_X_'+str(i) 98 | eye_lmk_3d[name] = [[], [], []] 99 | for i in range(0, MAX_PDM_EYE_LMK): 100 | name = 'eye_lmk_Y_'+str(i) 101 | eye_lmk_3d[name] = [[], [], []] 102 | for i in range(0, MAX_PDM_EYE_LMK): 103 | name = 'eye_lmk_Z_'+str(i) 104 | eye_lmk_3d[name] = [[], [], []] 105 | 106 | def smooth_array(ar, window_size, polyorder): 107 | # use savgol_filter() to do first path on smooth 108 | # https://scipy.github.io/devdocs/generated/scipy.signal.savgol_filter.html 109 | result = savgol_filter(ar, window_size, polyorder) 110 | # to avoid high frequency/small-amplitude oscillations to help in 111 | # finding peaks and troughs run a gaussian_filter. 112 | # I'm not a math wiz so I got this from here: 113 | # https://stackoverflow.com/questions/47962044/how-to-get-the-correct-peaks-and-troughs-from-an-1d-array 114 | result = gaussian_filter1d(result, window_size).tolist() 115 | 116 | #https://stackoverflow.com/questions/52125211/find-peaks-and-bottoms-of-graph-and-label-them 117 | minimas = (np.diff(np.sign(np.diff(result))) > 0).nonzero()[0] + 1 118 | maximas = (np.diff(np.sign(np.diff(result))) < 0).nonzero()[0] + 1 119 | 120 | return result, maximas.tolist(), minimas.tolist() 121 | 122 | def reset_database(): 123 | global animation_data 124 | global pdm_2d 125 | global rigid_data 126 | global non_rigid_data 127 | global eye_lmk_2d 128 | global eye_lmk_3d 129 | 130 | for k, v in animation_data.items(): 131 | for i in range(0, 3): 132 | v[i].clear() 133 | 134 | for k, v in pdm_2d.items(): 135 | for i in range(0, 3): 136 | v[i].clear() 137 | 138 | for k, v in pdm_3d.items(): 139 | for i in range(0, 3): 140 | v[i].clear() 141 | 142 | for k, v in rigid_data.items(): 143 | for i in range(0, 3): 144 | v[i].clear() 145 | 146 | for k, v in non_rigid_data.items(): 147 | for i in range(0, 3): 148 | v[i].clear() 149 | 150 | for k, v in eye_lmk_2d.items(): 151 | for i in range(0, 3): 152 | v[i].clear() 153 | 154 | for k, v in eye_lmk_3d.items(): 155 | for i in range(0, 3): 156 | v[i].clear() 157 | 158 | def plot_graph(animation_data, name, show=True, pdf_path=''): 159 | if plt_unsupported: 160 | return 161 | # plot the first entry 162 | plt.plot(animation_data['frame'][0], animation_data[name][0], label=name) 163 | 164 | for minima in animation_data[name][2]: 165 | plt.plot(minima, animation_data[name][0][minima], marker="o") 166 | for maxima in animation_data[name][1]: 167 | plt.plot(maxima, animation_data[name][0][maxima], marker="o") 168 | 169 | plt.legend() 170 | if show: 171 | plt.show() 172 | elif pdf_path: 173 | f = plt.figure() 174 | f.savefig(pdf_path, bbox_inches='tight') 175 | 176 | def get_facs_data(): 177 | global animation_data 178 | return animation_data 179 | 180 | def get_pdm2d_data(): 181 | global pdm_2d 182 | return pdm_2d 183 | 184 | def get_pdm3d_data(): 185 | global pdm_3d 186 | return pdm_3d 187 | 188 | def get_rigid_data(): 189 | global rigid_data 190 | return rigid_data 191 | 192 | def get_non_rigid_data(): 193 | global non_rigid_data 194 | return non_rigid_data 195 | 196 | def get_eye_lmk_2d(): 197 | global eye_lmk_2d 198 | return eye_lmk_2d 199 | 200 | def get_eye_lmk_3d(): 201 | global eye_lmk_3d 202 | return eye_lmk_3d 203 | 204 | def smooth_data(d, window_size, polyorder): 205 | # smooth all the data 206 | for k, v in d.items(): 207 | if k == 'frame' or k == 'timestamp': 208 | continue 209 | d[k][VALUES], \ 210 | d[k][MAXIMAS], \ 211 | d[k][MINIMAS] = \ 212 | smooth_array(v[0], window_size, polyorder) 213 | 214 | def process_openface_csv(csv_name, window_size = 5, polyorder = 2): 215 | global animation_data 216 | global pdm_2d 217 | global rigid_data 218 | global non_rigid_data 219 | 220 | with open(csv_name, 'r') as fcsv: 221 | reader = csv.DictReader(fcsv, delimiter=',') 222 | reader = (dict((k.strip(), v.strip()) for k, v in row.items() if v) \ 223 | for row in reader) 224 | for row in reader: 225 | # ignore entries with low confidence 226 | if float(row['confidence']) < 0.7: 227 | continue 228 | # build my local data base 229 | for k, v in animation_data.items(): 230 | v[VALUES].append(float(row[k])) 231 | for k, v in pdm_2d.items(): 232 | v[VALUES].append(float(row[k])) 233 | for k, v in pdm_3d.items(): 234 | v[VALUES].append(float(row[k])) 235 | for k, v in rigid_data.items(): 236 | v[VALUES].append(float(row[k])) 237 | for k, v in non_rigid_data.items(): 238 | v[VALUES].append(float(row[k])) 239 | 240 | # smooth all the data 241 | for k, v in animation_data.items(): 242 | if k == 'frame' or k == 'timestamp': 243 | continue 244 | if 'pose_' in k: 245 | animation_data[k][VALUES], \ 246 | animation_data[k][MAXIMAS], \ 247 | animation_data[k][MINIMAS] = \ 248 | smooth_array(v[VALUES], 11, 5) 249 | else: 250 | animation_data[k][VALUES], \ 251 | animation_data[k][MAXIMAS], \ 252 | animation_data[k][MINIMAS] = \ 253 | smooth_array(v[VALUES], window_size, polyorder) 254 | 255 | # smooth all the data 256 | smooth_data(pdm_2d, window_size, polyorder) 257 | smooth_data(pdm_3d, window_size, polyorder) 258 | smooth_data(rigid_data, window_size, polyorder) 259 | smooth_data(non_rigid_data, window_size, polyorder) 260 | 261 | # export data to JSON 262 | js = json.dumps(animation_data, indent=4) 263 | 264 | return js 265 | 266 | if __name__ == '__main__': 267 | if len(sys.argv) < 2: 268 | print("usage: smooth []") 269 | exit(1) 270 | 271 | csv_name = sys.argv[1] 272 | json_name = '' 273 | 274 | if len(sys.argv) >= 3: 275 | json_name = sys.argv[2] 276 | 277 | js, ad, pdm = process_openface_csv(csv_name) 278 | 279 | if json_name: 280 | jf = open(json_name, 'w') 281 | jf.write(js) 282 | jf.close() 283 | 284 | plot_graph(ad, 'AU04_r') 285 | -------------------------------------------------------------------------------- /yasp/sphinxinstall/lib/libpocketsphinx.so: -------------------------------------------------------------------------------- 1 | libpocketsphinx.so.3.0.0 -------------------------------------------------------------------------------- /yasp/sphinxinstall/lib/libpocketsphinx.so.3: -------------------------------------------------------------------------------- 1 | libpocketsphinx.so.3.0.0 -------------------------------------------------------------------------------- /yasp/sphinxinstall/lib/libpocketsphinx.so.3.0.0: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amirpavlo/BYASP/706ce757f7181a574e8878efd600adcfae898217/yasp/sphinxinstall/lib/libpocketsphinx.so.3.0.0 -------------------------------------------------------------------------------- /yasp/sphinxinstall/lib/libsphinxad.so: -------------------------------------------------------------------------------- 1 | libsphinxad.so.3.0.0 -------------------------------------------------------------------------------- /yasp/sphinxinstall/lib/libsphinxad.so.3: -------------------------------------------------------------------------------- 1 | libsphinxad.so.3.0.0 -------------------------------------------------------------------------------- /yasp/sphinxinstall/lib/libsphinxad.so.3.0.0: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amirpavlo/BYASP/706ce757f7181a574e8878efd600adcfae898217/yasp/sphinxinstall/lib/libsphinxad.so.3.0.0 -------------------------------------------------------------------------------- /yasp/sphinxinstall/lib/libsphinxbase.so: -------------------------------------------------------------------------------- 1 | libsphinxbase.so.3.0.0 -------------------------------------------------------------------------------- /yasp/sphinxinstall/lib/libsphinxbase.so.3: -------------------------------------------------------------------------------- 1 | libsphinxbase.so.3.0.0 -------------------------------------------------------------------------------- /yasp/sphinxinstall/lib/libsphinxbase.so.3.0.0: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amirpavlo/BYASP/706ce757f7181a574e8878efd600adcfae898217/yasp/sphinxinstall/lib/libsphinxbase.so.3.0.0 -------------------------------------------------------------------------------- /yasp/sphinxinstall/share/pocketsphinx/model/en-us/en-us-phone.lm.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amirpavlo/BYASP/706ce757f7181a574e8878efd600adcfae898217/yasp/sphinxinstall/share/pocketsphinx/model/en-us/en-us-phone.lm.bin -------------------------------------------------------------------------------- /yasp/sphinxinstall/share/pocketsphinx/model/en-us/en-us.lm.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amirpavlo/BYASP/706ce757f7181a574e8878efd600adcfae898217/yasp/sphinxinstall/share/pocketsphinx/model/en-us/en-us.lm.bin -------------------------------------------------------------------------------- /yasp/sphinxinstall/share/pocketsphinx/model/en-us/en-us/README: -------------------------------------------------------------------------------- 1 | /* ==================================================================== 2 | * Copyright (c) 2015 Alpha Cephei Inc. All rights 3 | * reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions 7 | * are met: 8 | * 9 | * 1. Redistributions of source code must retain the above copyright 10 | * notice, this list of conditions and the following disclaimer. 11 | * 12 | * 2. Redistributions in binary form must reproduce the above copyright 13 | * notice, this list of conditions and the following disclaimer in 14 | * the documentation and/or other materials provided with the 15 | * distribution. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY ALPHA CEPHEI INC. ``AS IS'' AND. 18 | * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,. 19 | * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 20 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL ALPHA CEPHEI INC. 21 | * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT. 23 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,. 24 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY. 25 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT. 26 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE. 27 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | * 29 | * ==================================================================== 30 | * 31 | */ 32 | 33 | This directory contains generic US english acoustic model trained with 34 | latest sphinxtrain. 35 | -------------------------------------------------------------------------------- /yasp/sphinxinstall/share/pocketsphinx/model/en-us/en-us/feat.params: -------------------------------------------------------------------------------- 1 | -lowerf 130 2 | -upperf 6800 3 | -nfilt 25 4 | -transform dct 5 | -lifter 22 6 | -feat 1s_c_d_dd 7 | -svspec 0-12/13-25/26-38 8 | -agc none 9 | -cmn batch 10 | -varnorm no 11 | -model ptm 12 | -cmninit 41.00,-5.29,-0.12,5.09,2.48,-4.07,-1.37,-1.78,-5.08,-2.05,-6.45,-1.42,1.17 13 | -------------------------------------------------------------------------------- /yasp/sphinxinstall/share/pocketsphinx/model/en-us/en-us/mdef: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amirpavlo/BYASP/706ce757f7181a574e8878efd600adcfae898217/yasp/sphinxinstall/share/pocketsphinx/model/en-us/en-us/mdef -------------------------------------------------------------------------------- /yasp/sphinxinstall/share/pocketsphinx/model/en-us/en-us/means: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amirpavlo/BYASP/706ce757f7181a574e8878efd600adcfae898217/yasp/sphinxinstall/share/pocketsphinx/model/en-us/en-us/means -------------------------------------------------------------------------------- /yasp/sphinxinstall/share/pocketsphinx/model/en-us/en-us/noisedict: -------------------------------------------------------------------------------- 1 | SIL 2 | SIL 3 | SIL 4 | [NOISE] +NSN+ 5 | [SPEECH] +SPN+ 6 | -------------------------------------------------------------------------------- /yasp/sphinxinstall/share/pocketsphinx/model/en-us/en-us/sendump: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amirpavlo/BYASP/706ce757f7181a574e8878efd600adcfae898217/yasp/sphinxinstall/share/pocketsphinx/model/en-us/en-us/sendump -------------------------------------------------------------------------------- /yasp/sphinxinstall/share/pocketsphinx/model/en-us/en-us/transition_matrices: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amirpavlo/BYASP/706ce757f7181a574e8878efd600adcfae898217/yasp/sphinxinstall/share/pocketsphinx/model/en-us/en-us/transition_matrices -------------------------------------------------------------------------------- /yasp/sphinxinstall/share/pocketsphinx/model/en-us/en-us/variances: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amirpavlo/BYASP/706ce757f7181a574e8878efd600adcfae898217/yasp/sphinxinstall/share/pocketsphinx/model/en-us/en-us/variances -------------------------------------------------------------------------------- /yasp/yaspbin/_yasp.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amirpavlo/BYASP/706ce757f7181a574e8878efd600adcfae898217/yasp/yaspbin/_yasp.so -------------------------------------------------------------------------------- /yasp/yaspbin/yasp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amirpavlo/BYASP/706ce757f7181a574e8878efd600adcfae898217/yasp/yaspbin/yasp -------------------------------------------------------------------------------- /yasp/yaspbin/yasp.py: -------------------------------------------------------------------------------- 1 | # This file was automatically generated by SWIG (http://www.swig.org). 2 | # Version 3.0.12 3 | # 4 | # Do not make changes to this file unless you know what you are doing--modify 5 | # the SWIG interface file instead. 6 | 7 | from sys import version_info as _swig_python_version_info 8 | if _swig_python_version_info >= (2, 7, 0): 9 | def swig_import_helper(): 10 | import importlib 11 | pkg = __name__.rpartition('.')[0] 12 | mname = '.'.join((pkg, '_yasp')).lstrip('.') 13 | try: 14 | return importlib.import_module(mname) 15 | except ImportError: 16 | return importlib.import_module('_yasp') 17 | _yasp = swig_import_helper() 18 | del swig_import_helper 19 | elif _swig_python_version_info >= (2, 6, 0): 20 | def swig_import_helper(): 21 | from os.path import dirname 22 | import imp 23 | fp = None 24 | try: 25 | fp, pathname, description = imp.find_module('_yasp', [dirname(__file__)]) 26 | except ImportError: 27 | import _yasp 28 | return _yasp 29 | try: 30 | _mod = imp.load_module('_yasp', fp, pathname, description) 31 | finally: 32 | if fp is not None: 33 | fp.close() 34 | return _mod 35 | _yasp = swig_import_helper() 36 | del swig_import_helper 37 | else: 38 | import _yasp 39 | del _swig_python_version_info 40 | 41 | try: 42 | _swig_property = property 43 | except NameError: 44 | pass # Python < 2.2 doesn't have 'property'. 45 | 46 | try: 47 | import builtins as __builtin__ 48 | except ImportError: 49 | import __builtin__ 50 | 51 | def _swig_setattr_nondynamic(self, class_type, name, value, static=1): 52 | if (name == "thisown"): 53 | return self.this.own(value) 54 | if (name == "this"): 55 | if type(value).__name__ == 'SwigPyObject': 56 | self.__dict__[name] = value 57 | return 58 | method = class_type.__swig_setmethods__.get(name, None) 59 | if method: 60 | return method(self, value) 61 | if (not static): 62 | if _newclass: 63 | object.__setattr__(self, name, value) 64 | else: 65 | self.__dict__[name] = value 66 | else: 67 | raise AttributeError("You cannot add attributes to %s" % self) 68 | 69 | 70 | def _swig_setattr(self, class_type, name, value): 71 | return _swig_setattr_nondynamic(self, class_type, name, value, 0) 72 | 73 | 74 | def _swig_getattr(self, class_type, name): 75 | if (name == "thisown"): 76 | return self.this.own() 77 | method = class_type.__swig_getmethods__.get(name, None) 78 | if method: 79 | return method(self) 80 | raise AttributeError("'%s' object has no attribute '%s'" % (class_type.__name__, name)) 81 | 82 | 83 | def _swig_repr(self): 84 | try: 85 | strthis = "proxy of " + self.this.__repr__() 86 | except __builtin__.Exception: 87 | strthis = "" 88 | return "<%s.%s; %s >" % (self.__class__.__module__, self.__class__.__name__, strthis,) 89 | 90 | try: 91 | _object = object 92 | _newclass = 1 93 | except __builtin__.Exception: 94 | class _object: 95 | pass 96 | _newclass = 0 97 | 98 | class yasp_logs(_object): 99 | __swig_setmethods__ = {} 100 | __setattr__ = lambda self, name, value: _swig_setattr(self, yasp_logs, name, value) 101 | __swig_getmethods__ = {} 102 | __getattr__ = lambda self, name: _swig_getattr(self, yasp_logs, name) 103 | __repr__ = _swig_repr 104 | __swig_setmethods__["lg_error"] = _yasp.yasp_logs_lg_error_set 105 | __swig_getmethods__["lg_error"] = _yasp.yasp_logs_lg_error_get 106 | if _newclass: 107 | lg_error = _swig_property(_yasp.yasp_logs_lg_error_get, _yasp.yasp_logs_lg_error_set) 108 | __swig_setmethods__["lg_info"] = _yasp.yasp_logs_lg_info_set 109 | __swig_getmethods__["lg_info"] = _yasp.yasp_logs_lg_info_get 110 | if _newclass: 111 | lg_info = _swig_property(_yasp.yasp_logs_lg_info_get, _yasp.yasp_logs_lg_info_set) 112 | 113 | def __init__(self): 114 | this = _yasp.new_yasp_logs() 115 | try: 116 | self.this.append(this) 117 | except __builtin__.Exception: 118 | self.this = this 119 | __swig_destroy__ = _yasp.delete_yasp_logs 120 | __del__ = lambda self: None 121 | yasp_logs_swigregister = _yasp.yasp_logs_swigregister 122 | yasp_logs_swigregister(yasp_logs) 123 | 124 | ERR_DEBUG = _yasp.ERR_DEBUG 125 | ERR_INFO = _yasp.ERR_INFO 126 | ERR_INFOCONT = _yasp.ERR_INFOCONT 127 | ERR_WARN = _yasp.ERR_WARN 128 | ERR_ERROR = _yasp.ERR_ERROR 129 | ERR_FATAL = _yasp.ERR_FATAL 130 | ERR_MAX = _yasp.ERR_MAX 131 | 132 | def yasp_interpret(audioFile, transcript, output, genpath): 133 | return _yasp.yasp_interpret(audioFile, transcript, output, genpath) 134 | yasp_interpret = _yasp.yasp_interpret 135 | 136 | def yasp_interpret_get_str(audioFile, transcript, genpath): 137 | return _yasp.yasp_interpret_get_str(audioFile, transcript, genpath) 138 | yasp_interpret_get_str = _yasp.yasp_interpret_get_str 139 | 140 | def yasp_setup_logging(logs, cb, logfile): 141 | return _yasp.yasp_setup_logging(logs, cb, logfile) 142 | yasp_setup_logging = _yasp.yasp_setup_logging 143 | 144 | def yasp_finish_logging(logs): 145 | return _yasp.yasp_finish_logging(logs) 146 | yasp_finish_logging = _yasp.yasp_finish_logging 147 | 148 | def yasp_set_modeldir(modeldir): 149 | return _yasp.yasp_set_modeldir(modeldir) 150 | yasp_set_modeldir = _yasp.yasp_set_modeldir 151 | 152 | def yasp_free_json_str(json): 153 | return _yasp.yasp_free_json_str(json) 154 | yasp_free_json_str = _yasp.yasp_free_json_str 155 | # This file is compatible with both classic and new-style classes. 156 | 157 | 158 | -------------------------------------------------------------------------------- /yasp/yaspbin/yasp_setup.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | 4 | cur_dir = os.getcwd() 5 | src_dir = os.path.join(cur_dir, "yaspbin") 6 | 7 | sys.path.append(src_dir) 8 | --------------------------------------------------------------------------------