├── LICENSE ├── MANIFEST ├── README ├── pyKinectTools ├── __init__.py ├── algs │ ├── BackgroundSubtraction.py │ ├── Dijkstras.py │ ├── DynamicTimeWarping.c │ ├── DynamicTimeWarping.cpp │ ├── DynamicTimeWarping.html │ ├── DynamicTimeWarping.py │ ├── DynamicTimeWarping.pyx │ ├── FeatureExtraction.py │ ├── GeodesicSkeleton.py │ ├── GlobalSignalSystem.py │ ├── GraphAlgs.py │ ├── HistogramOfOpticalFlow.py │ ├── IterativeClosestPoint.py │ ├── LocalOccupancyPattern.cpp │ ├── LocalOccupancyPattern.py │ ├── LocalOccupancyPattern.pyx │ ├── Manifolds.py │ ├── MeanShift.py │ ├── Normals.py │ ├── OrientationEstimate.py │ ├── PeopleTracker.py │ ├── PoseTracking.py │ ├── RandomForestPose.py │ ├── STIP.py │ ├── SkeletonBeliefPropagation.py │ ├── __init__.py │ ├── cPoseTracking.cpp │ ├── cPoseTracking.pyx │ ├── dijkstras.cpp │ ├── dijkstras.pyx │ ├── old:incomplete │ │ ├── GestureRecognizer_PCA.py │ │ ├── NeighborSuperpixels.c │ │ ├── NeighborSuperpixels.pyx │ │ ├── Normals_OpenCL.py │ │ ├── OpticalFlow.py │ │ ├── PictorialStructures.py │ │ ├── Tracking.py │ │ └── move_orientationEstimate.py │ └── smij.py ├── configs │ ├── .DS_Store │ ├── Kinect_Color_CAD_Param.yml │ ├── Kinect_Color_Param.yml │ ├── Kinect_Color_Param_2.yml │ ├── Kinect_Color_Param_old.yml │ ├── Kinect_Depth_Param.yml │ ├── Kinect_Transformation.txt │ ├── OpenNI_Empty.xml │ ├── OpenNI_Infrared.xml │ ├── SamplesConfig.xml │ ├── SamplesConfig_orig.xml │ ├── __init__.py │ └── fist.png ├── dataset_readers │ ├── .idea │ │ ├── .name │ │ ├── dataset_readers.iml │ │ ├── encodings.xml │ │ ├── inspectionProfiles │ │ │ ├── Project_Default.xml │ │ │ └── profiles_settings.xml │ │ ├── misc.xml │ │ ├── modules.xml │ │ ├── other.xml │ │ ├── scopes │ │ │ └── scope_settings.xml │ │ ├── testrunner.xml │ │ ├── vcs.xml │ │ └── workspace.xml │ ├── BasePlayer.py │ ├── CADPlayer.py │ ├── CADViewer.py │ ├── CAD_Repr.py │ ├── CAD_Stats.py │ ├── ChalearnBodyPartReader.py │ ├── EVALPlayer.py │ ├── KinectPlayer.py │ ├── MHADPlayer.py │ ├── MSR_DailyActivities.py │ ├── RealtimePlayer.py │ ├── SMMCPlayer_tmp.py │ ├── __init__.py │ └── chAirGestPlayer ├── scripts │ ├── AMIA_recognitionTests.py │ ├── Button2Time.py │ ├── ButtonRecorderTest.py │ ├── ConvertToPCD.py │ ├── ConvertToPCD_RGBD.py │ ├── DualViewer.py │ ├── ExtractAndVisualizeVideo.py │ ├── HeadOfBed.py │ ├── IterativeClosestPoint_script.py │ ├── MHAD_Viewer.py │ ├── MulticamViewer.py │ ├── Old_Experiments │ │ ├── PoseByGeodesicExtrema.py │ │ ├── PoseByManifolds.py │ │ ├── PoseByPictorialStructures.py │ │ ├── ProfilePose.py │ │ ├── ToFCamera.py │ │ ├── patientMovement.py │ │ └── pcaGesturesMain.py │ ├── PoseInitAndTracking.py │ ├── PoseInitAndTracking_PF.py │ ├── Pose_Dict_Viewer.py │ ├── Pose_tests.py │ ├── RealtimeRecorder.py │ ├── STIPScript.py │ ├── Tracker.py │ ├── TrainRFPose.py │ ├── Viewer.py │ ├── chalearnExperiments.py │ ├── hogTests.py │ ├── labelSkeletons.py │ ├── loadData.py │ ├── mainICU.py │ ├── mainMultiCam.py │ ├── mainRealtime.py │ └── main_MSR-Dataset.py └── utils │ ├── AlignCameras.py │ ├── CameraCalibration.py │ ├── DepthReader.py │ ├── DepthUtils.py │ ├── DualViewer.py │ ├── FeatureUtils.py │ ├── HOGUtils.py │ ├── MultiCameraUtils.py │ ├── RealtimeReader.py │ ├── SkeletonUtils.py │ ├── SocketReader.py │ ├── Utils.py │ ├── VideoViewer.py │ ├── __init__.py │ ├── pointcloud_conversions.py │ └── transformations.py └── setup.py /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2012 Colin Lea. All rights reserved. 2 | 3 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 4 | 5 | Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 6 | Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 7 | 8 | THIS SOFTWARE IS PROVIDED BY THE FREEBSD PROJECT ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FREEBSD PROJECT OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 9 | 10 | The views and conclusions contained in the software and documentation are those of the authors and should not be interpreted as representing official policies, either expressed or implied, of the FreeBSD Project. -------------------------------------------------------------------------------- /MANIFEST: -------------------------------------------------------------------------------- 1 | # file GENERATED by distutils, do NOT edit 2 | README 3 | setup.py 4 | pyKinectTools/__init__.py 5 | pyKinectTools/algs/LocalOccupancyPattern.pyx 6 | pyKinectTools/algs/dijkstras.pyx 7 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | pyKinectTools 2 | Author: Colin Lea 3 | 4 | These instructions are sparse and likely to change. If you would like to use this code the best thing is to email me (colincsl@gmail.com). I am happy to help 5 | 6 | ---Dependencies--- 7 | numpy, scipy, scikit-learn, scikit-image, visvis, opencv 8 | 9 | ---Installation instructions--- 10 | 11 | ** Python data libraries: 12 | If you use a python virtual environment: 13 | pip install numpy scipy matplotlib ipython scikit-learn scikit-image pyside visvis 14 | 15 | ** OpenCV 16 | (Only used for Optical Flow in Histogram of Oriented Optical Flow -- I'm trying to get rid of this dependence!) 17 | If on OSX/Ubuntu(12.04+)/Windows see: https://docs.google.com/document/d/1TsPRI1g_iXQmsCs1VPkLm-9M0T1n724H-wAo7QSNpMY/edit 18 | Otherwise build from source 19 | 20 | ---Algorithms--- 21 | Background subtraction: Adaptive Mixture of Gaussian, Static median/mean model 22 | Feature extraction: Histogram of Oriented Optical Flow, Cuboids, Geodesic Extrema 23 | Simple person tracking 24 | Basic graph Algorithms 25 | Belief Propagation (tree-based) 26 | Iterative Closest Point 27 | Laplacian Eigenmaps manifolds 28 | PCA-based gesture recognition 29 | Superpixels wrapper [deprecated -- use the one in skimage instead] 30 | Others... 31 | 32 | Recording with Kinect: 33 | Data capture program 34 | Video/Skeleton data player (w/ user controller) 35 | 36 | Depth image utilities (e.g. converting from depth->pointcloud) 37 | chalearn annotated dataset reader 38 | -------------------------------------------------------------------------------- /pyKinectTools/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/colincsl/pyKinectTools/a84bb5b7ff9dd613576415932865c2ad435520b3/pyKinectTools/__init__.py -------------------------------------------------------------------------------- /pyKinectTools/algs/Dijkstras.py: -------------------------------------------------------------------------------- 1 | from pyKinectTools_algs_Dijkstras import * -------------------------------------------------------------------------------- /pyKinectTools/algs/DynamicTimeWarping.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from pyKinectTools_algs_dynamic_time_warping import DynamicTimeWarping_c, dtw_path 3 | '''Dynamic Time Warping''' 4 | 5 | 6 | def test(): 7 | from scipy.signal import resample 8 | ref = np.random.random([10,3]) 9 | x = resample(ref, 100) 10 | y = resample(ref, 200) 11 | 12 | if x.ndim == 1: 13 | x = np.atleast_2d(x).T 14 | y = np.atleast_2d(y).T 15 | 16 | x = np.ascontiguousarray(x) 17 | y = np.ascontiguousarray(y) 18 | 19 | DynamicTimeWarping(x,y) 20 | 21 | print "Cost:", cost_matrix[-1,-1] 22 | print "Path:", path.T 23 | 24 | 25 | ''' 26 | def tmp(x, y): 27 | cost_matrix = DynamicTimeWarping_c(x,y) 28 | path = dtw_path(np.ascontiguousarray(cost_matrix)) 29 | 30 | %timeit tmp(x,y) 31 | 32 | Compare to mlpy's implementation (which uses FastDTW?) 33 | import mlpy 34 | %timeit error, dtw_mat, y_ind = mlpy.dtw.dtw_std(x[:,0], y[:,0], dist_only=False, squared=True) 35 | 36 | ''' 37 | 38 | 39 | def DynamicTimeWarping(x, y, return_path=True): 40 | # To allow for multidimensional data, x and y must be at least 2D 41 | if x.ndim == 1: 42 | x = np.atleast_2d(x).T 43 | y = np.atleast_2d(y).T 44 | 45 | x = np.ascontiguousarray(x) 46 | y = np.ascontiguousarray(y) 47 | 48 | cost_matrix = DynamicTimeWarping_c(x,y) 49 | 50 | if not return_path: 51 | return cost_matrix[-1,-1], cost_matrix 52 | 53 | # Compute optimal path from start to end of matrix 54 | path = dtw_path(cost_matrix) 55 | 56 | return cost_matrix[-1,-1], cost_matrix, path 57 | 58 | def euclidian_distance(x, y): 59 | ''' 60 | Note: don't both taking square root 61 | ''' 62 | return np.sum((x - y)**2) 63 | 64 | # def DynamicTimeWarping_py(x, y, return_path=True, distance_fcn='euclidian'): 65 | # ''' 66 | # Implements vanilla DTW 67 | 68 | # http://en.wikipedia.org/wiki/Dynamic_time_warping 69 | # ''' 70 | # if distance_fcn == "euclidian": 71 | # distance_fcn = euclidian_distance 72 | 73 | # x_res = len(x) 74 | # y_res = len(y) 75 | # cost_matrix = np.zeros([x_res, y_res], dtype=np.float) 76 | 77 | # # Caclulate edges first (they only depend on one value) 78 | # for i in xrange(1, x_res): 79 | # cost_matrix[i,0] = cost_matrix[i-1,0] + distance_fcn(x[i], y[0]) 80 | # for j in xrange(1, y_res): 81 | # cost_matrix[0,j] = cost_matrix[0,j-1] + distance_fcn(x[0], y[j]) 82 | 83 | # # Calculate distance at each location 84 | # for i in xrange(1, len(x)): 85 | # for j in xrange(1, len(y)): 86 | # cost = distance_fcn(x[i], y[j]) 87 | # cost_matrix[i,j] = cost + np.min([cost_matrix[i-1,j], cost_matrix[i,j-1], cost_matrix[i-1,j-1]]) 88 | 89 | # # Cost is simply the last element 90 | # max_cost = cost_matrix[-1,-1] 91 | 92 | # if not return_path: 93 | # return max_cost, cost_matrix 94 | 95 | # # Calculate path from start to finish 96 | # path = [[0,0]] 97 | # candidate_pts = np.array([[0,1],[1,0],[1,1]]) 98 | # current_pos = [0,0] 99 | 100 | # while path[-1][0] < x_res-1 or path[-1][1] < y_res-1: 101 | # if path[-1][0] == x_res-1: 102 | # next = 0 103 | # elif path[-1][1] == y_res-1: 104 | # next = 1 105 | # else: 106 | # next = np.argmin(cost_matrix[candidate_pts[:,0], candidate_pts[:,1]]) 107 | # path += [[candidate_pts[next][0], candidate_pts[next][1]]] 108 | 109 | # if next==0: 110 | # candidate_pts += [0,1] 111 | # elif next == 1: 112 | # candidate_pts += [1,0] 113 | # else: 114 | # candidate_pts += [1,1] 115 | 116 | # path = np.array(path) 117 | 118 | # return max_cost, cost_matrix, path 119 | 120 | 121 | def draw_path(cost_matrix, path): 122 | ''' 123 | Draw path from start to finish of DTW cost matrix 124 | (for visualization purposes) 125 | ''' 126 | path = np.array(path) 127 | 128 | max_score = score_matrix[-1,-1] 129 | for p in path: 130 | cost_matrix[p[0], p[1]] = max_score 131 | 132 | return cost_matrix 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | -------------------------------------------------------------------------------- /pyKinectTools/algs/DynamicTimeWarping.pyx: -------------------------------------------------------------------------------- 1 | 2 | ''' 3 | Vanilla implementation of Dynamic Time Warping 4 | Colin Lea (colincsl@gmail.com) 5 | July 2013 6 | ''' 7 | import time 8 | import numpy as np 9 | import cython 10 | cimport cython 11 | cimport numpy as np 12 | from libcpp.vector cimport vector 13 | 14 | np.import_array() 15 | 16 | ctypedef np.float64_t FLOAT 17 | ctypedef np.int16_t INT 18 | 19 | @cython.boundscheck(False) 20 | @cython.wraparound(False) 21 | @cython.infer_types(True) 22 | cdef float distance_fcn(np.ndarray[FLOAT, ndim=1, mode="c"] x, np.ndarray[FLOAT, ndim=1, mode="c"] y): 23 | ''' 24 | Find squared euclidian distance 25 | ''' 26 | cdef np.ndarray[FLOAT, ndim=1, mode="c"] diff = x-y 27 | return sum(diff*diff) 28 | 29 | 30 | @cython.boundscheck(False) 31 | @cython.wraparound(False) 32 | @cython.infer_types(True) 33 | cpdef np.ndarray[FLOAT, ndim=2, mode="c"] DynamicTimeWarping_c(np.ndarray[FLOAT, ndim=2, mode="c"] x, np.ndarray[FLOAT, ndim=2, mode="c"] y): 34 | ''' 35 | Implements vanilla DTW 36 | 37 | http://en.wikipedia.org/wiki/Dynamic_time_warping 38 | ''' 39 | 40 | cdef int x_res, y_res, i, j 41 | x_res = len(x) 42 | y_res = len(y) 43 | 44 | cdef np.ndarray[FLOAT, ndim=2, mode="c"] cost_matrix = np.zeros([x_res, y_res], dtype=np.float) 45 | 46 | # Caclulate edges first (they only depend on one value) 47 | for i in xrange(1, x_res): 48 | cost_matrix[i,0] = cost_matrix[i-1,0] + distance_fcn(x[i], y[0]) 49 | for j in xrange(1, y_res): 50 | cost_matrix[0,j] = cost_matrix[0,j-1] + distance_fcn(x[0], y[j]) 51 | 52 | # Calculate distance at each location 53 | for i in xrange(1, x_res): 54 | for j in xrange(1, y_res): 55 | cost = distance_fcn(x[i], y[j]) 56 | cost_matrix[i,j] = cost + min(min(cost_matrix[i-1,j], cost_matrix[i,j-1]), cost_matrix[i-1,j-1]) 57 | 58 | return cost_matrix 59 | 60 | 61 | @cython.boundscheck(False) 62 | @cython.wraparound(False) 63 | @cython.infer_types(True) 64 | cpdef np.ndarray[INT, ndim=2, mode="c"] dtw_path(np.ndarray[FLOAT, ndim=2, mode="c"] cost_matrix): 65 | # Calculate path from start to finish 66 | cdef int x_res = cost_matrix.shape[0] 67 | cdef int y_res = cost_matrix.shape[1] 68 | cdef vector[int] path_x 69 | cdef vector[int] path_y 70 | path_x.push_back(0) 71 | path_y.push_back(0) 72 | cdef vector[FLOAT] costs 73 | for i in xrange(3): 74 | costs.push_back(0.) 75 | 76 | cdef int idx_x,idx_y, count=0 77 | # Start at top left and move to bottom right of cost matrix 78 | while path_x[count] < x_res-1 or path_y[count] < y_res-1: 79 | if path_x[count] == x_res-1: # If at left edge 80 | path_x.push_back(path_x[count]) 81 | path_y.push_back(path_y[count]+1) 82 | elif path_y[count] == y_res-1: # If at bottom edge 83 | path_x.push_back(path_x[count]+1) 84 | path_y.push_back(path_y[count]) 85 | else: # Normal case 86 | idx_x = path_x[count] 87 | idx_y = path_y[count]+1 88 | costs[0] = cost_matrix[idx_x, idx_y] 89 | idx_x = path_x[count]+1 90 | idx_y = path_y[count] 91 | costs[1] = cost_matrix[idx_x, idx_y] 92 | idx_x = path_x[count]+1 93 | idx_y = path_y[count]+1 94 | costs[2] = cost_matrix[idx_x, idx_y] 95 | 96 | if costs[0] < costs[1] and costs[0] < costs[2]: 97 | path_x.push_back(path_x[count]) 98 | path_y.push_back(path_y[count]+1) 99 | elif costs[1] < costs[0] and costs[0] < costs[2]: 100 | path_x.push_back(path_x[count]+1) 101 | path_y.push_back(path_y[count]) 102 | else: 103 | path_x.push_back(path_x[count]+1) 104 | path_y.push_back(path_y[count]+1) 105 | count += 1 106 | 107 | # Output to list 108 | cdef int path_length = path_x.size() 109 | path = np.empty([path_length, 2], dtype=np.int16) 110 | for i in xrange(path_length): 111 | path[i,0] = path_x[i] 112 | path[i,1] = path_y[i] 113 | 114 | return path 115 | 116 | 117 | -------------------------------------------------------------------------------- /pyKinectTools/algs/GeodesicSkeleton.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import scipy.ndimage as nd 3 | import pyKinectTools.algs.Dijkstras as dgn 4 | 5 | # from pyKinectTools.utils.DepthUtils import * 6 | from pyKinectTools.utils.DepthUtils import depthIm2PosIm 7 | from copy import deepcopy 8 | from skimage.draw import circle 9 | 10 | from IPython import embed 11 | from pylab import * 12 | 13 | def geodesic_extrema_MPI(im_pos, centroid=None, iterations=1, visualize=False, box=None): 14 | ''' 15 | im : im_pos (NxMx3) 16 | ''' 17 | if centroid==None: 18 | try: 19 | centroid = np.array(nd.center_of_mass(im_pos[:,:,2]), dtype=np.int16) 20 | except: 21 | return np.array([]) 22 | 23 | if box is not None: 24 | im_pos = im_pos[box] 25 | im_pos = np.ascontiguousarray(im_pos, dtype=np.int16) 26 | 27 | if visualize: 28 | cost_map = np.zeros([im_pos.shape[0], im_pos.shape[1]], dtype=np.uint16) 29 | extrema = dgn.geodesic_map_MPI(cost_map, im_pos, np.array(centroid, dtype=np.int16), iterations, 1) 30 | cost_map = np.array(extrema[-1]) 31 | extrema = extrema[:-1] 32 | extrema = np.array([x for x in extrema]) 33 | return extrema, cost_map 34 | else: 35 | extrema = np.array(dgn.geodesic_extrema_MPI(im_pos, np.array(centroid, dtype=np.int16), iterations)) 36 | return extrema 37 | 38 | def connect_extrema(im_pos, target, markers, visualize=False): 39 | ''' 40 | im_pos : XYZ positions of each point in image formation (n x m x 3) 41 | ''' 42 | height, width,_ = im_pos.shape 43 | centroid = np.array(target) 44 | 45 | im_pos = np.ascontiguousarray(im_pos.astype(np.int16)) 46 | cost_map = np.ascontiguousarray(np.zeros([height, width], dtype=np.uint16)) 47 | 48 | extrema = dgn.geodesic_map_MPI(cost_map, im_pos, np.array(centroid, dtype=np.int16), 1, 1) 49 | cost_map = extrema[-1] 50 | 51 | trails = [] 52 | for m in markers: 53 | trail = dgn.geodesic_trail(cost_map.copy()+(32000*(im_pos[:,:,2]==0)).astype(np.uint16), np.array(m, dtype=np.int16)) 54 | trails += [trail.copy()] 55 | if visualize: 56 | cost_map = deepcopy(cost_map) 57 | circ = circle(markers[0][0],markers[0][1], 5) 58 | circ = np.array([np.minimum(circ[0], height-1), np.minimum(circ[1], width-1)]) 59 | circ = np.array([np.maximum(circ[0], 0), np.maximum(circ[1], 0)]) 60 | cost_map[circ[0], circ[1]] = 0 61 | for i,t in enumerate(trails[1:]): 62 | # embed() 63 | cost_map[t[:,0], t[:,1]] = 0 64 | circ = circle(markers[i+1][0],markers[i+1][1], 5) 65 | circ = np.array([np.minimum(circ[0], height-1), np.minimum(circ[1], width-1)]) 66 | circ = np.array([np.maximum(circ[0], 0), np.maximum(circ[1], 0)]) 67 | cost_map[circ[0], circ[1]] = 0 68 | return trails, cost_map 69 | else: 70 | return trails 71 | 72 | 73 | 74 | def distance_map(im, centroid, scale=1): 75 | ''' 76 | ---Parameters--- 77 | im_depth : 78 | centroid : 79 | ---Returns--- 80 | distance_map 81 | ''' 82 | 83 | im_depth = np.ascontiguousarray(im.copy()) 84 | objSize = im_depth.shape 85 | max_value = 32000 86 | mask = im_depth > 0 87 | 88 | # Get discrete form of position/depth matrix 89 | # embed() 90 | depth_min = im_depth[mask].min() 91 | depth_max = im_depth[mask].max() 92 | depth_diff = depth_max - depth_min 93 | if depth_diff < 1: 94 | depth_diff = 1 95 | scale_to = scale / float(depth_diff) 96 | 97 | # Ensure the centroid is within the boundaries 98 | # Segfaults if on the very edge(!) so set border as 1 to resolution-2 99 | centroid[0] = centroid[0] if centroid[0] > 0 else 1 100 | centroid[0] = centroid[0] if centroid[0] < im.shape[0]-1 else im.shape[0]-2 101 | centroid[1] = centroid[1] if centroid[1] > 0 else 1 102 | centroid[1] = centroid[1] if centroid[1] < im.shape[1]-1 else im.shape[1]-2 103 | 104 | # Scale depth image 105 | im_depth_scaled = np.ascontiguousarray(np.array( (im_depth-depth_min)*scale_to, dtype=np.uint16)) 106 | # im_depth_scaled = np.ascontiguousarray(np.array( (im_depth-depth_min), dtype=np.uint16)) 107 | im_depth_scaled *= mask 108 | 109 | # Initialize all but starting point as max 110 | distance_map = np.zeros([objSize[0],objSize[1]], dtype=np.uint16)+max_value 111 | distance_map[centroid[0], centroid[1]] = 0 112 | 113 | # Set which pixels are in/out of bounds 114 | visited_map = np.zeros_like(distance_map, dtype=np.uint8) 115 | visited_map[-mask] = 255 116 | 117 | centroid = np.array(centroid, dtype=np.int16) 118 | # embed() 119 | dgn.distance_map(distance_map, visited_map, im_depth_scaled.astype(np.uint16), centroid, int(scale)) 120 | 121 | return distance_map.copy() 122 | 123 | 124 | 125 | def generateKeypoints(im, centroid, iterations=10, scale=6): 126 | ''' 127 | ---Parameters--- 128 | im_depth : 129 | centroid : 130 | ---Returns--- 131 | extrema 132 | distance_map 133 | ''' 134 | 135 | x,y = centroid 136 | maps = [] 137 | extrema = [] 138 | # Get N distance maps. For 2..N centroid is previous farthest distance. 139 | for i in range(iterations): 140 | im_dist = distance_map(np.ascontiguousarray(im.copy()), centroid=[x,y], scale=scale) 141 | im_dist[im_dist>=32000] = 0 142 | maps += [im_dist.copy()] 143 | max_ = np.argmax(np.min(np.dstack(maps),-1)) 144 | max_px = np.unravel_index(max_, im.shape) 145 | x,y = max_px 146 | extrema += [[x,y]] 147 | 148 | im_min = np.min(np.dstack(maps),-1) 149 | im_min = (im_min/(float(im_min.max())/255.)).astype(np.uint8) 150 | # Visualize 151 | im -= im[im>0].min() 152 | for c in extrema: 153 | # im_min[c[0]-3:c[0]+4, c[1]-3:c[1]+4] = im_min.max() 154 | im[c[0]-3:c[0]+4, c[1]-3:c[1]+4] = im.max() 155 | 156 | 157 | import cv2 158 | cv2.imshow("Extrema", im/float(im.max())) 159 | # cv2.imshow("Extrema", im_min/float(im_min.max())) 160 | cv2.waitKey(30) 161 | 162 | return extrema, im_min 163 | 164 | 165 | def drawTrail(im, trail, value=255): 166 | for i,j in trail: 167 | im[i,j] = value 168 | return im 169 | 170 | 171 | ''' --- Other functions --- ''' 172 | 173 | def relative_marker_positions(im_pos, markers): 174 | feature_map = np.zeros([im_pos.shape[0], im_pos.shape[1], len(markers)], dtype=np.float) 175 | dgn.relative_distance_features(np.ascontiguousarray(im_pos), np.ascontiguousarray(markers), feature_map) 176 | return feature_map 177 | 178 | def local_histograms(im, n_bins=2, max_bound=255, patch_size=5): 179 | output = dgn.local_histograms(np.ascontiguousarray(im), n_bins, patch_size, max_bound) 180 | return output 181 | 182 | 183 | -------------------------------------------------------------------------------- /pyKinectTools/algs/GlobalSignalSystem.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import numpy as np 3 | import time 4 | 5 | class GlobalSignalSystem: 6 | markers = {} 7 | signals = {} 8 | times = [] 9 | signalCount = 0 10 | markerCount = 0 11 | radius = 0 12 | chart = [] 13 | 14 | def __init__(self, markers=[], radius=30): 15 | 16 | if markers != []: 17 | self.addMarkers(markers) 18 | 19 | self.radius = radius 20 | 21 | 22 | def addMarkers(self, markers): 23 | # Markers should be a dictionary with name:[2xfloat position] 24 | for m in markers.keys(): 25 | self.markers[m] = markers[m] 26 | self.signals[m] = [] 27 | self.markerCount += 1 28 | 29 | def getMarkerPos(self): 30 | markers = [] 31 | for m in self.markers.keys(): 32 | markers.append(self.markers[m]) 33 | 34 | return markers 35 | 36 | def update(self, coms=[], curTime=[]): 37 | if curTime == []: 38 | curTime = time.time() 39 | self.times.append(curTime) 40 | 41 | # If no people, add empty signal for all markers 42 | if coms == []: 43 | for k in self.signals.keys(): 44 | self.signals[k].append(0) 45 | else: 46 | # If people, check if they're within bounds of all markers and add a signal 47 | for m in self.markers.keys(): 48 | # Default to 0 signal 49 | self.signals[m].append(0) 50 | 51 | for c in coms: 52 | if np.sqrt(np.sum((self.markers[m][0:2]-c[0:2])**2)) < self.radius: 53 | self.signals[m][-1] += 1 54 | self.signalCount += 1 55 | 56 | def getChart(self): 57 | # TODO: base this on time, not frames 58 | # TODO: design chart (inc. resolution, add titles, ...) 59 | 60 | rez = [200,600] 61 | signalRez = [(rez[0]-50)/self.markerCount-5, rez[1]-40] 62 | chartIm = np.zeros(rez) 63 | 64 | self.chart = np.empty([self.markerCount, self.signalCount]) 65 | 66 | for m,i in zip(self.markers.keys(), range(len(self.markers.keys()))): 67 | self.chart[i,:] = self.signals[m] 68 | 69 | # 0->2 70 | signalIm = np.interp(np.arange(0, self.chart.shape[1], self.chart.shape[1]/float(signalRez[1])), range(self.chart.shape[1]), self.chart[i]) >= 0.5 71 | signalIm = np.vstack(signalIm).repeat(signalRez[0]-5, 1) 72 | # pdb.set_trace() 73 | chartIm[i*signalRez[0]+50+5 : (i+1)*signalRez[0]+50, 20:20+signalRez[1]] = signalIm.T*200 + 20 74 | 75 | cv2.putText(chartIm, "Signals:", org=(rez[1]/2-50, 25), fontFace=cv2.FONT_HERSHEY_DUPLEX, fontScale=1, color=[100,100,100]) 76 | 77 | # cv2.line(charIm, ) 78 | 79 | # return self.chart 80 | return chartIm 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | -------------------------------------------------------------------------------- /pyKinectTools/algs/IterativeClosestPoint.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Iterative closest point 3 | 4 | Colin Lea 5 | pyKinectTools 6 | 2012 7 | ''' 8 | 9 | import numpy as np 10 | from numpy import dot 11 | from scipy import spatial 12 | 13 | 14 | def IterativeClosestPoint(pts_new, pts_ref, max_iters=25, min_change=.001, pt_tolerance=5000, return_transform=False): 15 | 16 | # pts_new = y 17 | # pts_ref = x 18 | pts_new = pts_new.copy() 19 | pts_ref = pts_ref.copy() 20 | 21 | inital_mean = pts_ref.mean(0) 22 | # pts_new -= inital_mean 23 | # pts_ref -= inital_mean 24 | 25 | nn = spatial.cKDTree(pts_ref) 26 | it = 0 27 | prevErr = 10**10 28 | change = np.inf 29 | R_total = np.eye(3) 30 | T_total = np.zeros(3) 31 | R = R_total 32 | t = T_total 33 | pts_new_current = np.copy(pts_new) 34 | 35 | err_best = np.inf 36 | T_best = None 37 | # T_best = pts_ref - pts_new 38 | R_best = None 39 | 40 | 41 | while it < max_iters and change > min_change: 42 | # pts_new_current = (dot(R_total, pts_new.T).T + T_total) 43 | pts_new_current = (dot(R, pts_new_current.T).T + t) 44 | # scatter(pts_ref[:,0], pts_ref[:,1]) 45 | # scatter(pts_new_current[:,0], pts_new_current[:,1], color='r') 46 | dists, pts = nn.query(pts_new_current) 47 | 48 | goodPts_new = pts_new_current[dists < pt_tolerance] 49 | goodPts_ref = pts_ref[pts[dists numNeigh] = 0 21 | W = inds <= numNeigh 22 | 23 | if 0: 24 | W = np.zeros([len(data), len(data)]) 25 | Ball = neighbors.BallTree(data) 26 | dists, nn = Ball.query(data, numNeigh) 27 | 28 | k=numNeigh 29 | for di in range(len(dists)): 30 | for ki in range(k): 31 | # W[di, nn[di,ki]] = dists[di,ki] 32 | W[di, nn[di,ki]] = 1.0#dists[di,ki] 33 | 34 | 35 | if not heatKernel: 36 | # Binary representation 37 | W = np.maximum(W, W.T) 38 | else: 39 | # Heat kernel based on distances. Ranges between 0-1 40 | W = W**2 41 | W /= np.max(np.max(W)) 42 | W = np.maximum(W, W.T) 43 | W[W!=0] = np.exp(-W[W!=0] / (2*heatSigma**2)) 44 | 45 | diag_ = np.diag(np.sum(W,1)) 46 | 47 | #Calc Laplacian 48 | L = diag_-W 49 | # vals, vecs = np.linalg.eigh(L) 50 | vals, vecs = sparse.linalg.eigsh(np.asarray(L, dtype=np.float), which='SM') # "LM" 51 | 52 | # Only keep positive eigenvals 53 | posInds = np.nonzero(vals>0.01)[0] 54 | posVecs = vecs[:,posInds] 55 | 56 | # posVecs = vecs 57 | 58 | 59 | return posVecs 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | -------------------------------------------------------------------------------- /pyKinectTools/algs/MeanShift.py: -------------------------------------------------------------------------------- 1 | ''' 2 | 3 | NOTE: This is adapted from 4 | http://sociograph.blogspot.com/2011/11/scalable-mean-shift-clustering-in-few.html 5 | ''' 6 | 7 | import numpy as np 8 | from sklearn.neighbors import BallTree, NearestNeighbors 9 | from sklearn.utils import extmath 10 | from sklearn.metrics.pairwise import euclidean_distances 11 | from collections import defaultdict 12 | 13 | 14 | def mean_shift(X, bandwidth, n_seeds, kernel_function='gaussian', max_iterations=100, proximity_thresh=5): 15 | ''' 16 | ---Parameters--- 17 | X : data in form (samples, dims) 18 | bandwidth : radius of nearest neighbors 19 | n_seeds : 20 | kernel_update_function : can be "gaussian" or "flat" or your own kernel 21 | proximity_thresh : minimum distance (in pixels) a new cluster must be away from previous ones 22 | 23 | ---Returns--- 24 | cluster_centers : 25 | cluster_counts : how many pixels are with the neighborhood of each cluster 26 | ''' 27 | 28 | import numpy as np 29 | from sklearn.neighbors import BallTree, NearestNeighbors 30 | from sklearn.utils import extmath 31 | from sklearn.metrics.pairwise import euclidean_distances 32 | from collections import defaultdict 33 | 34 | if kernel_function == 'gaussian': 35 | kernel_update_function = gaussian_kernel 36 | elif kernel_function == 'flat': 37 | kernel_update_function = flat_kernel 38 | else: 39 | kernel_update_function = kernel_function 40 | 41 | 42 | n_points, n_features = X.shape 43 | stop_thresh = 1e-2 * bandwidth # when mean has converged 44 | cluster_centers = [] 45 | cluster_counts = [] 46 | # ball_tree = BallTree(X)# to efficiently look up nearby points 47 | neighbors = NearestNeighbors(radius=bandwidth).fit(X) 48 | 49 | seeds = X[(np.random.uniform(0,X.shape[0], n_seeds)).astype(np.int)] 50 | 51 | # For each seed, climb gradient until convergence or max_iterations 52 | for weighted_mean in seeds: 53 | completed_iterations = 0 54 | while True: 55 | points_within = X[neighbors.radius_neighbors([weighted_mean], bandwidth, return_distance=False)[0]] 56 | old_mean = weighted_mean # save the old mean 57 | weighted_mean = kernel_update_function(old_mean, points_within, bandwidth) 58 | converged = extmath.norm(weighted_mean - old_mean) < stop_thresh 59 | if converged or completed_iterations == max_iterations: 60 | # Only add cluster if it's different enough from other centers 61 | if len(cluster_centers) > 0: 62 | diff_from_prev = [np.linalg.norm(weighted_mean-cluster_centers[i], 2) for i in range(len(cluster_centers))] 63 | if np.min(diff_from_prev) > proximity_thresh: 64 | cluster_centers.append(weighted_mean) 65 | cluster_counts.append(points_within.shape[0]) 66 | else: 67 | cluster_centers.append(weighted_mean) 68 | cluster_counts.append(points_within.shape[0]) 69 | break 70 | completed_iterations += 1 71 | 72 | return cluster_centers, cluster_counts 73 | 74 | 75 | def gaussian_kernel(x, points, bandwidth): 76 | from sklearn.metrics.pairwise import euclidean_distances 77 | distances = euclidean_distances(points, x) 78 | weights = np.exp(-1 * (distances ** 2 / bandwidth ** 2)) 79 | return np.sum(points * weights, axis=0) / np.sum(weights) 80 | 81 | def flat_kernel(x, points, bandwidth): 82 | return np.mean(points, axis=0) 83 | 84 | 85 | def bin_points(X, bin_size, min_bin_freq): 86 | bin_sizes = defaultdict(int) 87 | for point in X: 88 | binned_point = np.cast[np.int32](point / bin_size) 89 | bin_sizes[tuple(binned_point)] += 1 90 | 91 | bin_seeds = np.array([point for point, freq in bin_sizes.iteritems() if freq >= min_bin_freq], dtype=np.float32) 92 | bin_seeds = bin_seeds * bin_size 93 | return bin_seeds -------------------------------------------------------------------------------- /pyKinectTools/algs/OrientationEstimate.py: -------------------------------------------------------------------------------- 1 | from copy import deepcopy 2 | import sys, os, time 3 | import cv2, numpy as np 4 | # sys.path.append('/Users/colin/libs/visionTools/slic-python/') 5 | # import slic 6 | # import Image 7 | # from pyKinectTools.algs.SkeletonBeliefPropagation import * 8 | from pyKinectTools.algs.GraphAlgs import * 9 | from pyKinectTools.algs.GeodesicSkeleton import * 10 | import pyKinectTools.algs.NeighborSuperpixels.NeighborSuperpixels as nsp 11 | 12 | 13 | def roughOrientationEstimate(xyz): 14 | U,_,vT=np.linalg.svd((xyz-xyz.mean(0)), full_matrices=0) 15 | forwardVector = vT[2,:] 16 | 17 | return forwardVector, vT 18 | 19 | 20 | 21 | 22 | 23 | def OrientationEstimate(im, regions, regionXYZ, regionPos): 24 | 25 | ''' Get upper and middle nodes ''' 26 | regionXYZ = np.array(regionXYZ) 27 | midpoint = np.argmin(np.sum((regionXYZ[1:] - np.mean(regionXYZ[1:,0]))**2, 1))+1 28 | uppermostPart = np.argmax(regionXYZ[1:,0])+1 29 | upVec = regionXYZ[uppermostPart] - regionXYZ[midpoint] 30 | upVec /= np.sum(upVec) 31 | 32 | dists = np.sqrt(np.sum((regionXYZ[1:] - regionXYZ[midpoint])**2, 1)) 33 | tmpNodes = np.nonzero((100 < dists) * (dists < 250))[0]+1 34 | 35 | upperMidpoint = (regionXYZ[midpoint] + regionXYZ[uppermostPart]) / 2 36 | dists = np.sqrt(np.sum((regionXYZ[1:] - upperMidpoint)**2, 1)) 37 | 38 | ## --- Head regions 39 | headSize = 250 40 | dists = np.sqrt(np.sum((regionXYZ[1:] - regionXYZ[uppermostPart])**2, 1)) 41 | headRegions = np.nonzero(dists0)] 51 | 52 | ## --- Torso 53 | torsoXYZ = regionXYZ[midpoint] 54 | torsoPos = regionPos[midpoint] 55 | headXYZ = regionXYZ[uppermostPart] 56 | neckXYZ = (torsoXYZ+headXYZ)/2 57 | neckPos = np.sum([regionPos[midpoint],regionPos[uppermostPart]], 0)/2 58 | 59 | 60 | tmpXYZ = posMat[(regions<=midpoint)*(regions>np.max(headRegions))] 61 | tmpXYZ -= tmpXYZ.mean(0) 62 | U,_,vT=svd((tmpXYZ-tmpXYZ.mean(0)), full_matrices=0) 63 | 64 | tmpXYZ2 = xyz 65 | tmpXYZ2 -= tmpXYZ2.mean(0) 66 | _,_,vT2=svd(tmpXYZ2-tmpXYZ2.mean(0), full_matrices=0) 67 | 68 | shoulderAxis = vT[2,:] 69 | bodyAxis = vT2[2,:] 70 | angleDiff = np.arccos(np.dot(bodyAxis,shoulderAxis))*180/np.pi 71 | 72 | 73 | if viz: 74 | print "Angle diff:", angleDiff 75 | 76 | 77 | if viz: 78 | figure(5) 79 | min_ = 1.0*posMat[:,:,1][posMat[:,:,2]>0].min() 80 | max_ = 1.0*posMat[:,:,1].max() - min_ 81 | scatter(xyz[:,0],xyz[:,2], c=1.0*np.array([(posMat[:,:,1][posMat[:,:,2]>0]-min_)*1.0/max_], dtype=float)) 82 | scatter(tmpXYZ[:,0],tmpXYZ[:,2], c='r') 83 | plot([0, 200*vT[2,0]], [0,200*vT[2,2]], c='g', linewidth=5) 84 | plot([0, 200*vT2[2,0]], [0,200*vT2[2,2]], c='k', linewidth=5) 85 | 86 | tmpXYZ2 = (np.asmatrix(vT)*np.asmatrix(tmpXYZ.T)).T 87 | tmpXYZ3 = (np.asmatrix(vT)*np.asmatrix(xyz.T)).T 88 | 89 | shoulders = [np.argmin(tmpXYZ2[:,0]),np.argmax(tmpXYZ2[:,0])] 90 | shoulderPx = [] 91 | for i in shoulders: 92 | shoulderPx.append([np.nonzero((posMat[:,:,2]*(regions<=midpoint)*(regions>np.max(headRegions)))>0)[0][i], np.nonzero((posMat[:,:,2]*(regions<=midpoint)*(regions>np.max(headRegions)))>0)[1][i]]) 93 | 94 | if viz: 95 | figure(6) 96 | plot(tmpXYZ2[:,0],tmpXYZ2[:,1], 'b.') 97 | plot(tmpXYZ2[:,0],tmpXYZ2[:,2], 'g.') 98 | plot(tmpXYZ2[:,1],tmpXYZ2[:,2], 'r.') 99 | 100 | # if viz: 101 | if 1: 102 | im2 = deepcopy(regions) 103 | cv2.circle(im2, (headPos[1], headPos[0]), 10, 5) #Head 104 | cv2.circle(im2, (neckPos[1], neckPos[0]), 5, 5) #Neck 105 | cv2.circle(im2, (torsoPos[1], torsoPos[0]), 5, 5) #Torso 106 | for px in shoulderPx: 107 | cv2.circle(im2, (px[1], px[0]), 5, 5) 108 | # figure(8) 109 | imshow(im2) 110 | 111 | if viz: 112 | 113 | # Neck: 114 | figure(6) 115 | # text(.5, .95, 'Upper body points', horizontalalignment='center') 116 | subplot(2,2,1); axis('equal'); xlabel('X'); ylabel('Y') 117 | plot(-tmpXYZ2[:,0],-tmpXYZ2[:,1], 'b.') 118 | subplot(2,2,2); axis('equal'); xlabel('Z'); ylabel('Y') 119 | plot(-tmpXYZ2[:,2],-tmpXYZ2[:,1], 'b.') 120 | subplot(2,2,3); axis('equal'); xlabel('X'); ylabel('Z') 121 | plot(-tmpXYZ2[:,0],tmpXYZ2[:,2], 'b.') 122 | # Whole body: 123 | figure(7) 124 | subplot(2,2,1); axis('equal'); xlabel('X'); ylabel('Y') 125 | plot(tmpXYZ3[:,0],-tmpXYZ3[:,1], 'b.') 126 | subplot(2,2,2); axis('equal'); xlabel('Z'); ylabel('Y') 127 | plot(-tmpXYZ3[:,2],-tmpXYZ3[:,1], 'b.') 128 | subplot(2,2,3); axis('equal'); xlabel('X'); ylabel('Z') 129 | plot(-tmpXYZ3[:,0],tmpXYZ3[:,2], 'b.') 130 | 131 | # Whole body: 132 | figure(9) 133 | subplot(2,2,1); axis('equal'); xlabel('X'); ylabel('Y') 134 | plot(-xyz[:,1],xyz[:,0], 'b.') 135 | subplot(2,2,2); axis('equal'); xlabel('Z'); ylabel('Y') 136 | plot(-xyz[:,2],xyz[:,0], 'b.') 137 | subplot(2,2,3); axis('equal'); xlabel('X'); ylabel('Z') 138 | plot(-xyz[:,1],xyz[:,2], 'b.') 139 | 140 | 141 | 142 | print "Time:", time.time()-time1 143 | 144 | 145 | 146 | 147 | 148 | -------------------------------------------------------------------------------- /pyKinectTools/algs/STIP.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import scipy.misc 3 | import os, sys 4 | import scipy.ndimage as nd 5 | from sklearn import neighbors 6 | 7 | 8 | ## Gabor filter 9 | def generateGabors(angles, size=[20,20], rho=1): 10 | # angles in degrees 11 | if type(angles) != list: 12 | angles = [angles] 13 | width = size[0] 14 | height = size[1] 15 | 16 | xsbase = np.array(range(width)).T 17 | xs = np.array(xsbase, dtype=float) 18 | for i in range(height-1): 19 | xs = np.vstack([xs, xsbase]) 20 | xs -= 1.0*width/2 21 | xs /= width 22 | 23 | ysbase = np.array(range(height)).T 24 | ys = np.array(ysbase, dtype=float) 25 | for i in range(width-1): 26 | ys = np.vstack([ys, ysbase]) 27 | ys = ys.T 28 | ys -= 1.0*height/2 29 | ys /= height 30 | 31 | ux = 1.0/(2*rho) 32 | uy = 1.0/(2*rho) 33 | 34 | # Gaussian envelope 35 | gauss = np.exp(-0.5*(xs**2/rho**2 + ys**2/rho**2)) 36 | gauss -= gauss.min() 37 | gauss / gauss.max() 38 | 39 | if len(angles) > 1: 40 | gabors = np.empty([size[0], size[1], len(angles)]) 41 | else: 42 | gabors = np.empty([size[0], size[1]]) 43 | 44 | for a in range(len(angles)): 45 | theta = (angles[a])*np.pi/180 46 | s = np.cos(2*np.pi*(ux*(np.cos(theta)*xs+np.sin(theta)*ys) +uy*(np.sin(theta)*ys+np.cos(theta)*xs))) 47 | if len(angles) > 1: 48 | gabors[:,:,a] = s*gauss 49 | else: 50 | gabors[:,:] = s*gauss 51 | 52 | return gabors 53 | 54 | def CuboidDetector(sigma=1.5, tau=4): 55 | 56 | delX = int(2*np.ceil(3*sigma)+1) 57 | delY = delX 58 | delT = int(2*np.ceil(3*tau)+1) 59 | 60 | xs = np.arange(delX, dtype=np.float)[:,np.newaxis].T.repeat(delY, 0) 61 | xs -= 1.0*delX/2 62 | xs /= delX 63 | 64 | ys = np.arange(delY, dtype=np.float)[:,np.newaxis].repeat(delX).reshape([delX, delY]) 65 | ys -= 1.0*delY/2 66 | ys /= delY 67 | 68 | # Gaussian envelope 69 | gauss = np.exp(-0.5*(xs**2/sigma**2 + ys**2/sigma**2)) 70 | gauss -= gauss.min() 71 | gauss / gauss.max() 72 | 73 | omega = 4./tau 74 | t = np.ones(delT) 75 | h_ev = -np.cos(2*np.pi*omega)*np.exp(-t**2 / tau**2) 76 | h_od = -np.sin(2*np.pi*t*omega)*np.exp(-t**2 / tau**2) 77 | 78 | return gauss, h_ev, h_od 79 | 80 | 81 | 82 | def adaptiveNonMaximalSuppression(pts, vals, radius=1): 83 | 84 | tree = neighbors.NearestNeighbors() 85 | tree.fit(pts) 86 | nn = tree.radius_neighbors(pts, radius, return_distance=False) 87 | 88 | outputPts = [] 89 | for i in range(len(pts)): 90 | 91 | if vals[i] >= vals[nn[i]].max(): 92 | outputPts.append(pts[i]) 93 | # print vals[i],vals[nn[i]].max(), nn[i], i 94 | 95 | outputPts = np.array(outputPts) 96 | return outputPts 97 | 98 | 99 | 100 | 101 | 102 | -------------------------------------------------------------------------------- /pyKinectTools/algs/SkeletonBeliefPropagation.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | ''' Belief Propagation on tree''' 4 | 5 | class Node: 6 | children = [] 7 | parent = -1 8 | pos = -1 9 | index = -1 10 | depth = 0 11 | 12 | msgsDown = {} 13 | msgUp = -1 14 | psi = -1 15 | belief = -1 16 | 17 | def __init__(self, parent_=-1, index_=-1, children_=[], pos_=-1, depth_=0): 18 | self.index = index_ 19 | self.pos = pos_ 20 | self.parent = parent_ 21 | self.children = [] 22 | self.depth = depth_ 23 | 24 | if len(children_) > 1: 25 | for i in children_: 26 | if self.parent == -1 or i != self.parent.index: 27 | # print self.index, i 28 | self.children.append(Node(index_=i, parent_=self, children_=edgeDict[i], pos_=regionLabels[i][1], depth_=self.depth+1)) 29 | self.msgsDown[i] = -1 30 | self.msgUp = -1; self.psi = -1; self.belief = -1 31 | 32 | 33 | def getLeaves(self, leaves=set()): 34 | if self.children == []: 35 | leaves.add(self) 36 | else: 37 | for c in self.children: 38 | l = c.getLeaves(leaves) 39 | 40 | if self.parent == -1: 41 | return list(leaves) 42 | else: 43 | return leaves 44 | 45 | def calcPsi(self, hypothesis): 46 | #psi is the unary potentials/messages 47 | # hypothesis = guessPose 48 | self.psi = np.empty([np.shape(hypothesis)[0]]) 49 | for i in xrange(np.shape(hypothesis)[0]): 50 | self.psi[i] = np.sqrt(np.sum((hypothesis[i] - self.pos)**2)) 51 | # print '1', self.psi 52 | self.psi = (np.sum(self.psi) - self.psi)/np.sum(self.psi) 53 | self.psi /= np.sum(self.psi) 54 | # print '4', self.psi 55 | 56 | def setPsi(self, psi): 57 | pass 58 | 59 | def calcMessageUp(self): 60 | for kid in self.children: 61 | if np.all(kid.msgUp == -1): 62 | return 63 | tmpMsg = 1 64 | # print '-' 65 | tmpMsg *= self.psi 66 | # print self.psi 67 | if self.children != []: 68 | for kid in self.children: 69 | # print np.asarray(kid.msgUp).T[0], tmpMsg 70 | # tmpMsg *= np.asarray(kid.msgUp).T[0] 71 | # tmpMsg = np.sum(np.asmatrix(tmpMsg).T*kid.msgUp.T, 0) 72 | tmpMsg = tmpMsg*kid.msgUp 73 | tmpMsg /= np.sum(tmpMsg) 74 | # print tmpMsg, np.asarray(kid.msgUp).T[0] 75 | # pdb.set_trace() 76 | 77 | # pdb.set_trace() 78 | self.msgUp = np.array(transitionMatrix*np.asmatrix(tmpMsg).T).T 79 | self.msgUp /= np.sum(self.msgUp) 80 | 81 | # print self.msgUp, np.asmatrix(tmpMsg).T 82 | 83 | def calcMessagesDown(self): 84 | for c, kid in zip(range(len(self.children)), self.children): 85 | tmpMsg = 1 86 | tmpMsg *= self.psi 87 | if self.parent != -1: 88 | for m in self.parent.msgsDown.keys(): 89 | if m == self.index: 90 | # tmpMsg = np.sum(np.asmatrix(tmpMsg).T*self.parent.msgsDown[m].T, 0) 91 | tmpMsg = tmpMsg*self.parent.msgsDown[m] 92 | tmpMsg /= np.sum(tmpMsg) 93 | 94 | break 95 | 96 | for c2_i, kid2 in zip(range(len(self.children)), self.children): 97 | if kid != kid2: 98 | # pdb.set_trace() 99 | # tmpMsg *= np.array(self.children[c2_i].msgUp).T[0] 100 | 101 | # tmpMsg = np.sum(np.asmatrix(tmpMsg).T*kid.msgUp.T, 0) 102 | tmpMsg = tmpMsg*kid.msgUp 103 | tmpMsg /= np.sum(tmpMsg) 104 | 105 | self.msgsDown[kid.index] = np.array(transitionMatrix*np.asmatrix(tmpMsg).T).T 106 | self.msgsDown[kid.index] /= np.sum(self.msgsDown[kid.index]) 107 | # pdb.set_trace() 108 | # print np.array(transitionMatrix*np.asmatrix(tmpMsg).T).T 109 | 110 | 111 | def calcBelief(self): 112 | self.belief = 1 113 | self.belief *= self.psi 114 | 115 | if self.parent != -1: 116 | for c in self.parent.msgsDown.keys(): 117 | if c == self.index: 118 | # self.belief *= np.array(self.parent.msgsDown[c]).T[0] 119 | self.belief *= self.parent.msgsDown[c][0] 120 | break 121 | 122 | if self.children != []: 123 | for kid in self.children: 124 | if np.any(kid.msgUp == -1): 125 | self.belief = -1 126 | break 127 | else: 128 | # self.belief *= np.asarray(kid.msgUp).T[0] 129 | self.belief *= kid.msgUp[0] 130 | 131 | if np.all(self.belief >= 0): 132 | self.belief /= np.sum(self.belief) 133 | 134 | 135 | def updateLeaves(self): 136 | self.calcMessagesDown() 137 | self.calcBelief() 138 | 139 | if self.children != []: 140 | for c in self.children: 141 | c.updateLeaves() 142 | 143 | def reset(self): 144 | self.msgUp = -1 145 | self.psi = -1 146 | self.belief = -1 147 | self.msgsDown = {} 148 | 149 | if self.children != []: 150 | for c in self.children: 151 | c.reset() 152 | 153 | def calcTotalBelief(self): 154 | 155 | if self.children == []: 156 | return np.max(self.belief) 157 | else: 158 | belief = 1 159 | for kids in self.children: 160 | belief *= kids.calcTotalBelief() 161 | 162 | return belief 163 | 164 | 165 | 166 | def calcAll(self, hypothesis): 167 | 168 | if np.any(self.belief >= 0): 169 | self.reset() 170 | 171 | leaves = self.getLeaves() 172 | oldLeaves = set() 173 | while np.any(self.belief < 0): 174 | newLeaves = set() 175 | for l in leaves: 176 | l.calcPsi(hypothesis) 177 | l.calcMessageUp() 178 | 179 | if np.any(l.msgUp != -1): 180 | oldLeaves.add(l) 181 | if np.any(l.parent != -1) and l.parent not in oldLeaves: 182 | newLeaves.add(l.parent) 183 | leaves = newLeaves 184 | self.calcBelief() 185 | 186 | self.calcMessagesDown() 187 | # self.calcBelief() 188 | 189 | for c in self.children: 190 | c.updateLeaves() 191 | 192 | def drawClass(self): 193 | pt1 = (regionLabels[self.index][2][1],regionLabels[self.index][2][0]) 194 | color_ = int(np.argmax(self.belief)*30) 195 | cv2.circle(imLines, pt1, radius=10, color=color_, thickness=1) 196 | 197 | def drawAll(self): 198 | self.drawClass() 199 | 200 | for kid in self.children: 201 | kid.drawAll() 202 | 203 | 204 | 205 | -------------------------------------------------------------------------------- /pyKinectTools/algs/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/colincsl/pyKinectTools/a84bb5b7ff9dd613576415932865c2ad435520b3/pyKinectTools/algs/__init__.py -------------------------------------------------------------------------------- /pyKinectTools/algs/cPoseTracking.pyx: -------------------------------------------------------------------------------- 1 | """ 2 | DON'T USE THIS FILE. LESS EFFICIENT THAN NUMPY VERSION 3 | 4 | Efficient pose query 5 | """ 6 | 7 | import numpy as np 8 | import cython 9 | cimport cython 10 | cimport numpy as cnp 11 | cnp.import_array() 12 | 13 | cdef extern from "math.h": 14 | int abs(int x) 15 | double sqrt(double x) 16 | double pow(double base, double exp) 17 | 18 | from scipy.spatial import cKDTree 19 | ctypedef cnp.float64_t FLOAT 20 | ctypedef cnp.int16_t INT16 21 | 22 | @cython.boundscheck(False) 23 | @cython.wraparound(False) 24 | @cython.infer_types(True) 25 | cpdef inline cnp.ndarray[FLOAT, ndim=1] query_error(cnp.ndarray[FLOAT, ndim=2] markers, cnp.ndarray[FLOAT, ndim=3] trees, cnp.ndarray[INT16, ndim=1] search_joints): 26 | 27 | cdef int i,ii,j,m 28 | cdef float error_tmp, v 29 | cdef int marker_count = len(markers) 30 | cdef int pose_count = len(trees) 31 | cdef int joint_count = len(search_joints) 32 | 33 | # cdef cnp.ndarray[FLOAT, ndim=3, mode="c"] trees = trees_ 34 | cdef cnp.ndarray[FLOAT, ndim=1, mode="c"] error = np.zeros(pose_count, np.float) 35 | cdef cnp.ndarray[FLOAT, ndim=1, mode="c"] marker = np.zeros_like(markers[0]) 36 | cdef cnp.ndarray[FLOAT, ndim=1, mode="c"] diff = np.zeros(joint_count, np.float) 37 | # trees = trees[:,search_joints] 38 | 39 | for i in range(pose_count): 40 | # error[i] = trees[i].query(markers)[0].sum() 41 | error_tmp = 0 42 | for m in range(marker_count): 43 | marker = markers[m] 44 | for ii in range(joint_count): 45 | diff[ii] = 0. 46 | for j in range(3): 47 | # diff[ii] = diff[ii] + trees[i][search_joints][j] - marker[j] 48 | # diff[ii] = diff[ii] + trees[i][search_joints][j] - 49 | 50 | diff[ii] = diff[ii] + pow(trees[i][search_joints[ii]][j] - marker[j],2.) 51 | diff[ii] = sqrt(diff[ii]) 52 | # min_diff = min(diff) 53 | error_tmp += cnp.min(diff) 54 | # error_tmp += cnp.min(cnp.sqrt(cnp.sum(cnp.pow((trees[i][search_joints] - marker), 2), -1))) 55 | error[i] = error_tmp 56 | 57 | # for i,tree in enumerate(trees): 58 | # error[i] = tree.query(markers)[0].sum() 59 | 60 | 61 | return np.array(error) 62 | -------------------------------------------------------------------------------- /pyKinectTools/algs/old:incomplete/NeighborSuperpixels.pyx: -------------------------------------------------------------------------------- 1 | ''' Build with: python setup.py build_ext --inplace ''' 2 | 3 | import numpy as np 4 | cimport numpy as np 5 | from libc.math cimport abs 6 | 7 | np.import_array() 8 | 9 | ctypedef np.uint8_t UINT8 10 | 11 | '''Compute all edges between segments''' 12 | # @cython.boundscheck(False) 13 | # @cython.wraparound(False) 14 | # @cython.infer_types(True) 15 | # @cython.profile(True) 16 | cpdef inline list getNeighborEdges(np.ndarray[UINT8, ndim=2] regions): 17 | cdef int x,y, ind 18 | cdef int height = regions.shape[0] 19 | cdef int width = regions.shape[1] 20 | cdef list edges = [] 21 | 22 | cdef UINT8* regions_c = regions.data 23 | 24 | # Look left and above current node. If it is a different label, add to edges. 25 | for y in range(1, height): 26 | for x in range(1, width): 27 | ind = y*width + x 28 | ind2 = (y-1)*width + x # Above 29 | ind3 = y*width + x -1 # Left 30 | # If not the same label and both labels are not the background (=0) 31 | if regions_c[ind] - regions_c[ind2] != 0 and regions_c[ind] > 0 and regions_c[ind2] > 0: 32 | edges.append([regions_c[ind], regions_c[ind2]]) 33 | if regions_c[ind] - regions_c[ind3] != 0 and regions_c[ind] > 0 and regions_c[ind3] > 0: 34 | edges.append([regions_c[ind], regions_c[ind3]]) 35 | 36 | return edges 37 | 38 | 39 | cpdef inline list getNeighborProfiles(np.ndarray[UINT8, ndim=2] im, list edges, list edgePositions): 40 | cdef int x,y, ind, dist 41 | cdef int height = im.shape[0] 42 | cdef int width = im.shape[1] 43 | 44 | cdef UINT8* im_c = im.data 45 | 46 | cdef list edgeProfiles = [] 47 | 48 | cdef int currentX, currentY 49 | for e in edges: 50 | currentY = edgePositions[e[0]][0] 51 | currentX = edgePositions[e[0]][1] 52 | endY = edgePositions[e[1]][0] 53 | endX = edgePositions[e[1]][1] 54 | 55 | ind = currentY*width + currentX 56 | edgeProfiles.append([im_c[ind]]) 57 | 58 | while currentX != endX and currentY != endY: 59 | if abs(currentX - endX) > abs(currentY - endY): 60 | if currentX > endX: 61 | currentX -= 1 62 | else: 63 | currentX += 1 64 | else: 65 | if currentY > endY: 66 | currentY -= 1 67 | else: 68 | currentY += 1 69 | ind = currentY*width + currentX 70 | edgeProfiles[-1].append(im_c[ind]) 71 | 72 | return edgeProfiles 73 | 74 | 75 | 76 | 77 | 78 | 79 | -------------------------------------------------------------------------------- /pyKinectTools/algs/old:incomplete/Normals_OpenCL.py: -------------------------------------------------------------------------------- 1 | 2 | ''' 3 | Based on code from: 4 | http://enja.org/2011/02/22/adventures-in-pyopencl-part-1-getting-started-with-python/index.html 5 | ''' 6 | 7 | import pyopencl as cl 8 | import time 9 | 10 | kernel = "__kernel void fcn(__global float* a, __global float* b, __global float* c)\ 11 | {\ 12 | unsigned int i = get_global_id(0);\ 13 | c[i] = a[i] + b[i];\ 14 | }" 15 | 16 | 17 | ctx = cl.create_some_context() 18 | queue = cl.CommandQueue(ctx) 19 | program = cl.Program(ctx, kernel).build() 20 | mf = cl.mem_flags 21 | 22 | # Init cpu 23 | # a = numpy.array(range(10000000), dtype=numpy.float32) 24 | # b = numpy.array(range(10000000), dtype=numpy.float32) 25 | a = np.ones([50,100], dtype=numpy.float32) 26 | b = np.ones([50,100], dtype=numpy.float32) 27 | c = np.empty_like(a) # For output 28 | 29 | # Create OpenCL buffers 30 | a_buf = cl.Buffer(ctx, mf.READ_ONLY | mf.COPY_HOST_PTR, hostbuf=a) 31 | b_buf = cl.Buffer(ctx, mf.READ_ONLY | mf.COPY_HOST_PTR, hostbuf=b) 32 | dest_buf = cl.Buffer(ctx, mf.WRITE_ONLY, b.nbytes) 33 | 34 | #Execute 35 | c = np.empty_like(a) # For output 36 | t1 = time.time() 37 | program.fcn(queue, a.shape, None, a_buf, b_buf, dest_buf) 38 | cl.enqueue_read_buffer(queue, dest_buf, c).wait() 39 | print time.time() - t1 40 | 41 | c = np.empty_like(a) # For output 42 | t1 = time.time() 43 | c2 = a+b 44 | print time.time() - t1 -------------------------------------------------------------------------------- /pyKinectTools/algs/old:incomplete/Tracking.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | import numpy as np 5 | from pylab import * 6 | 7 | def coms2map(coms, mapIm=None, mapRez=None, color=255, centers=[-500, 0, 3000], span=[5000.,5000.]): 8 | 9 | color = np.array(color) 10 | 11 | if mapRez is None: 12 | if mapIm is None: 13 | mapRez = [200,200, 3] 14 | else: 15 | mapRez = mapIm.shape 16 | 17 | if mapIm is None: 18 | mapIm = 0*np.ones(mapRez)*255 19 | 20 | ''' Convert com to xy indcies ''' 21 | xs = np.minimum(np.maximum(mapRez[0]+((coms[:,2]+centers[0])/span[0]*mapRez[0]).astype(np.int), 0),mapRez[0]-1) 22 | ys = np.minimum(np.maximum(mapRez[0]-((coms[:,0]+centers[0])/span[1]*mapRez[1]).astype(np.int), 0),mapRez[1]-1) 23 | mapIm[xs, ys,:] = color 24 | 25 | return mapIm 26 | 27 | def plotCamera(cameraPos, im, centers=[3000,0,-500], span=[5000.,5000.,5000.]): 28 | rez = im.shape 29 | x = (rez[0]+(cameraPos[2]+centers[2])/span[2]*rez[0]).astype(np.int) 30 | y = (rez[1]+(cameraPos[0]+centers[0])/span[0]*rez[1]).astype(np.int) 31 | cv2.circle(im, (y,x), 50, (255,255,0), thickness=10) 32 | 33 | return im 34 | 35 | 36 | def list2time(time): 37 | ''' Multiple time by 100 to account for sequential frames/second''' 38 | time = [int(x) for x in time] 39 | return time[0]*(60*60*24*10) + time[1]*(60*60*10) + time[2]*(60*10)+ time[3]*10 + time[4] 40 | 41 | 42 | 43 | class KalmanFilter: 44 | ''' A position-only kalman filter ''' 45 | 46 | def __init__(self, data=None, ProcessVariance=.001, MeasurementVariance=.00001): 47 | 48 | 49 | dims = data.shape[0] 50 | 51 | self.state = data # State estimate 52 | self.covar = np.eye(dims)*ProcessVariance # Covariance estimate 53 | self.StateTransition = np.eye(dims) 54 | self.ObsTransition = np.eye(dims) 55 | 56 | self.ProcessVar = np.eye(dims)*ProcessVariance 57 | self.MeasurementVar = np.eye(dims)*MeasurementVariance 58 | 59 | def update(self, data): 60 | ''' Predict and update ''' 61 | 62 | if self.state is None: 63 | self.state = data 64 | return 65 | 66 | state_prev = self.state 67 | cov_prev = self.covar 68 | 69 | ''' Prediction ''' 70 | state_est = np.dot(self.StateTransition, state_prev) 71 | covar_est = np.dot(self.StateTransition, np.dot(self.covar, self.StateTransition.T)) + self.MeasurementVar 72 | 73 | ''' Update ''' 74 | stateError = data - np.dot(self.ObsTransition, state_est) 75 | covarError = np.dot(self.ObsTransition, np.dot(self.covar, self.ObsTransition.T)) + self.ProcessVar 76 | weight = np.dot(np.dot(covar_est, self.ObsTransition.T), np.linalg.inv(covarError)) 77 | print weight 78 | self.state = state_est + np.dot(weight, stateError) 79 | 80 | self.covar = np.dot(np.eye(stateError.shape[0]) - np.dot(self.ObsTransition, self.StateTransition), covar_est) 81 | 82 | return self.state 83 | 84 | 85 | # com = np.load('../../ICU_Dec2012_r40_c1_') 86 | 87 | if __name__ == "__main__": 88 | kalman = KalmanFilter(coms[5,[0,2]], ProcessVariance=50, MeasurementVariance=.000001) 89 | x = [] 90 | xn = [] 91 | for i in range(6, 50): 92 | x.append(coms[i, [0,2]]) 93 | xn.append(kalman.update(coms[i, [0,2]])) 94 | # print kalman.covar 95 | 96 | x = np.array(x) 97 | xn = np.array(xn) 98 | plot(x[:,0], x[:,1], 'g', linewidth=1) 99 | plot(xn[:,0], xn[:,1], 'r', linewidth=1) 100 | show() 101 | 102 | 103 | ''' 104 | # # from IPython import embed 105 | # # embed() 106 | # self.weight = cov_prev / (cov_prev + self.MeasurementVar) # K matrix 107 | # print 'w', self.weight 108 | # print 'diff', data-state_prev, self.weight, (data - state_prev) 109 | # print 'data', data 110 | # print 'cov', cov_prev 111 | # self.state = state_prev + self.weight * (data - state_prev) 112 | # self.covar = (1-self.weight)*cov_prev 113 | ''' 114 | 115 | 116 | 117 | ''' 118 | Based on kalman filter at http://www.scipy.org/Cookbook/KalmanFiltering by Andrew D. Straw 119 | 120 | # intial parameters 121 | n_iter = 50 122 | sz = (n_iter,2) # size of array 123 | x = [-0.37727, -.37727] # truth value (typo in example at top of p. 13 calls this z) 124 | z = numpy.random.normal(x,0.1,size=sz) # observations (normal about x, sigma=0.1) 125 | 126 | Q = np.ones([1,2])*1e-5 # process variance 127 | 128 | # allocate space for arrays 129 | state=numpy.zeros(sz) # a posteri estimate of x 130 | P=numpy.zeros(sz) # a posteri error estimate 131 | state_prev=numpy.zeros(sz) # a priori estimate of x 132 | P_prev=numpy.zeros(sz) # a priori error estimate 133 | K=numpy.zeros(sz) # gain or blending factor 134 | 135 | R = np.ones([1,2])*0.01**2 # estimate of measurement variance, change to see effect 136 | 137 | # intial guesses 138 | state[0,:] = 0.0 139 | P[0,:] = 1.0 140 | 141 | for k in range(1,n_iter): 142 | # time update 143 | state_prev[k] = state[k-1] 144 | P_prev[k] = P[k-1]+Q 145 | 146 | # measurement update 147 | K[k] = P_prev[k]/( P_prev[k]+R ) 148 | state[k] = state_prev[k]+K[k]*(z[k]-state_prev[k]) 149 | P[k] = (1-K[k])*P_prev[k] 150 | 151 | figure() 152 | # plot(z,'k+',label='noisy measurements') 153 | # plot(state,'b-',label='a posteri estimate') 154 | 155 | plot(z[:,0], z[:,1], '-b', linewidth=1, label='noisy measurements') 156 | plot(state[:,0], state[:,1], '-g', linewidth=1, label='a posteri estimate') 157 | plot(x[0], x[1], '+r') 158 | # axhline(x,color='g',label='truth value') 159 | # legend() 160 | # xlabel('Iteration') 161 | # ylabel('Voltage') 162 | 163 | # figure() 164 | # valid_iter = range(1,n_iter) # Pminus not valid at step 0 165 | # plot(valid_iter,P_prev[valid_iter],label='a priori error estimate') 166 | # xlabel('Iteration') 167 | # ylabel('$(Voltage)^2$') 168 | # setp( gca(),'ylim',[0,.01]) 169 | # show() 170 | 171 | ''' -------------------------------------------------------------------------------- /pyKinectTools/algs/old:incomplete/move_orientationEstimate.py: -------------------------------------------------------------------------------- 1 | from copy import deepcopy 2 | import sys, os, time 3 | import cv2, numpy as np 4 | sys.path.append('/Users/colin/libs/visionTools/slic-python/') 5 | import slic 6 | import Image 7 | from pyKinectTools.algs.SkeletonBeliefPropagation import * 8 | from pyKinectTools.algs.GraphAlgs import * 9 | from pyKinectTools.algs.GeodesicSkeleton import * 10 | import pyKinectTools.algs.NeighborSuperpixels.NeighborSuperpixels as nsp 11 | 12 | # saved = np.load('tmpPerson_close.npz')['arr_0'].tolist() 13 | # saved = np.load('tmpPerson1.npz')['arr_0'].tolist() 14 | # objects1 = saved['objects']; labelInds1=saved['labels']; out1=saved['out']; d1=saved['d']; com1=saved['com'];featureExt1=saved['features']; posMat=saved['posMat']; xyz=saved['xyz'] 15 | # posMat=saved['posMat']; xyz=saved['xyz'] 16 | 17 | time1 = time.time() 18 | mask_erode = posMat[:,:,2]>0 19 | 20 | im8bit = deepcopy(posMat) 21 | for i in range(3): 22 | im8bit[:,:,i][im8bit[:,:,i]!=0] -= im8bit[:,:,i][im8bit[:,:,i]!=0].min() 23 | im8bit[:,:,i] /= im8bit[:,:,i].max()/256 24 | im8bit = np.array(im8bit, dtype=uint8) 25 | # im8bit = im8bit[:,:,2] 26 | im4d = np.dstack([mask_erode, im8bit]) 27 | # im4d = np.dstack([mask_erode, im8bit, im8bit, im8bit]) 28 | 29 | # regions = slic.slic_n(np.array(im4d, dtype=uint8), 50,10)#2 30 | regions = slic.slic_n(np.array(im4d, dtype=uint8), 50,10)#2 31 | regions *= mask_erode 32 | 33 | avgColor = np.zeros([regions.shape[0],regions.shape[1],3]) 34 | 35 | regionCount = regions.max() 36 | regionLabels = [[0]] 37 | goodRegions = 0 38 | bodyMean = np.array([posMat[mask_erode,0].mean(),posMat[mask_erode,1].mean(),posMat[mask_erode,2].mean()]) 39 | for i in range(1, regionCount+2): 40 | if np.sum(np.equal(regions,i)) < 50: 41 | regions[regions==i] = 0 42 | else: 43 | if 1: #if using x/y/z 44 | meanPos = posMat[regions==i,:].mean(0) 45 | if 0: # If using distance map 46 | meanPos = np.array([posMat[regions==i,:].mean(0)[0], 47 | posMat[regions==i,:].mean(0)[1], 48 | # posMat[regions==i,:].mean(0)[2], 49 | (dists2Tot[regions==i].mean())]) 50 | if 0: # If using depth only 51 | meanPos = np.array([(np.nonzero(regions==i)[0].mean()), 52 | (np.nonzero(regions==i)[1].mean()), 53 | (im8bit[regions==i].mean())]) 54 | avgColor[regions==i,:] = meanPos - bodyMean 55 | if not np.isnan(meanPos[0]) and meanPos[0] != 0.0: 56 | tmp = np.nonzero(regions==i) 57 | argPos = [int(tmp[0].mean()),int(tmp[1].mean())] 58 | regionLabels.append([i, meanPos-bodyMean, argPos]) 59 | goodRegions += 1 60 | regions[regions==i] = goodRegions 61 | # print np.sum(np.equal(regions,i)) 62 | else: 63 | regions[regions==i] = 0 64 | regionCount = regions.max() 65 | 66 | 67 | # ------------# ------------# ------------# ------------# ------------ 68 | 69 | 70 | 71 | 72 | 73 | regionXYZ = ([x[1] for x in regionLabels if x[0] != 0]) 74 | regionPos = ([x[2] for x in regionLabels if x[0] != 0]) 75 | regionXYZ.insert(0,[0,0,0]) 76 | regionPos.insert(0,[0,0]) 77 | 78 | ''' Get upper and middle nodes ''' 79 | regionXYZ = np.array(regionXYZ) 80 | midpoint = np.argmin(np.sum((regionXYZ[1:] - np.mean(regionXYZ[1:,0]))**2, 1))+1 81 | uppermostPart = np.argmax(regionXYZ[1:,0])+1 82 | upVec = regionXYZ[uppermostPart] - regionXYZ[midpoint] 83 | upVec /= np.sum(upVec) 84 | 85 | dists = np.sqrt(np.sum((regionXYZ[1:] - regionXYZ[midpoint])**2, 1)) 86 | tmpNodes = np.nonzero((100 < dists) * (dists < 250))[0]+1 87 | 88 | upperMidpoint = (regionXYZ[midpoint] + regionXYZ[uppermostPart]) / 2 89 | dists = np.sqrt(np.sum((regionXYZ[1:] - upperMidpoint)**2, 1)) 90 | 91 | ## --- Head regions 92 | headSize = 250 93 | dists = np.sqrt(np.sum((regionXYZ[1:] - regionXYZ[uppermostPart])**2, 1)) 94 | headRegions = np.nonzero(dists0)] 104 | 105 | ## --- Torso 106 | torsoXYZ = regionXYZ[midpoint] 107 | torsoPos = regionPos[midpoint] 108 | headXYZ = regionXYZ[uppermostPart] 109 | neckXYZ = (torsoXYZ+headXYZ)/2 110 | neckPos = np.sum([regionPos[midpoint],regionPos[uppermostPart]], 0)/2 111 | 112 | 113 | tmpXYZ = posMat[(regions<=midpoint)*(regions>np.max(headRegions))] 114 | tmpXYZ -= tmpXYZ.mean(0) 115 | U,_,vT=svd((tmpXYZ-tmpXYZ.mean(0)), full_matrices=0) 116 | 117 | tmpXYZ2 = xyz 118 | tmpXYZ2 -= tmpXYZ2.mean(0) 119 | _,_,vT2=svd(tmpXYZ2-tmpXYZ2.mean(0), full_matrices=0) 120 | 121 | shoulderAxis = vT[2,:] 122 | bodyAxis = vT2[2,:] 123 | angleDiff = np.arccos(np.dot(bodyAxis,shoulderAxis))*180/np.pi 124 | 125 | 126 | if viz: 127 | print "Angle diff:", angleDiff 128 | 129 | 130 | if viz: 131 | figure(5) 132 | min_ = 1.0*posMat[:,:,1][posMat[:,:,2]>0].min() 133 | max_ = 1.0*posMat[:,:,1].max() - min_ 134 | scatter(xyz[:,0],xyz[:,2], c=1.0*np.array([(posMat[:,:,1][posMat[:,:,2]>0]-min_)*1.0/max_], dtype=float)) 135 | scatter(tmpXYZ[:,0],tmpXYZ[:,2], c='r') 136 | plot([0, 200*vT[2,0]], [0,200*vT[2,2]], c='g', linewidth=5) 137 | plot([0, 200*vT2[2,0]], [0,200*vT2[2,2]], c='k', linewidth=5) 138 | 139 | tmpXYZ2 = (np.asmatrix(vT)*np.asmatrix(tmpXYZ.T)).T 140 | tmpXYZ3 = (np.asmatrix(vT)*np.asmatrix(xyz.T)).T 141 | 142 | shoulders = [np.argmin(tmpXYZ2[:,0]),np.argmax(tmpXYZ2[:,0])] 143 | shoulderPx = [] 144 | for i in shoulders: 145 | shoulderPx.append([np.nonzero((posMat[:,:,2]*(regions<=midpoint)*(regions>np.max(headRegions)))>0)[0][i], np.nonzero((posMat[:,:,2]*(regions<=midpoint)*(regions>np.max(headRegions)))>0)[1][i]]) 146 | 147 | if viz: 148 | figure(6) 149 | plot(tmpXYZ2[:,0],tmpXYZ2[:,1], 'b.') 150 | plot(tmpXYZ2[:,0],tmpXYZ2[:,2], 'g.') 151 | plot(tmpXYZ2[:,1],tmpXYZ2[:,2], 'r.') 152 | 153 | # if viz: 154 | if 1: 155 | im2 = deepcopy(regions) 156 | cv2.circle(im2, (headPos[1], headPos[0]), 10, 5) #Head 157 | cv2.circle(im2, (neckPos[1], neckPos[0]), 5, 5) #Neck 158 | cv2.circle(im2, (torsoPos[1], torsoPos[0]), 5, 5) #Torso 159 | for px in shoulderPx: 160 | cv2.circle(im2, (px[1], px[0]), 5, 5) 161 | # figure(8) 162 | imshow(im2) 163 | 164 | if viz: 165 | 166 | # Neck: 167 | figure(6) 168 | # text(.5, .95, 'Upper body points', horizontalalignment='center') 169 | subplot(2,2,1); axis('equal'); xlabel('X'); ylabel('Y') 170 | plot(-tmpXYZ2[:,0],-tmpXYZ2[:,1], 'b.') 171 | subplot(2,2,2); axis('equal'); xlabel('Z'); ylabel('Y') 172 | plot(-tmpXYZ2[:,2],-tmpXYZ2[:,1], 'b.') 173 | subplot(2,2,3); axis('equal'); xlabel('X'); ylabel('Z') 174 | plot(-tmpXYZ2[:,0],tmpXYZ2[:,2], 'b.') 175 | # Whole body: 176 | figure(7) 177 | subplot(2,2,1); axis('equal'); xlabel('X'); ylabel('Y') 178 | plot(tmpXYZ3[:,0],-tmpXYZ3[:,1], 'b.') 179 | subplot(2,2,2); axis('equal'); xlabel('Z'); ylabel('Y') 180 | plot(-tmpXYZ3[:,2],-tmpXYZ3[:,1], 'b.') 181 | subplot(2,2,3); axis('equal'); xlabel('X'); ylabel('Z') 182 | plot(-tmpXYZ3[:,0],tmpXYZ3[:,2], 'b.') 183 | 184 | # Whole body: 185 | figure(9) 186 | subplot(2,2,1); axis('equal'); xlabel('X'); ylabel('Y') 187 | plot(-xyz[:,1],xyz[:,0], 'b.') 188 | subplot(2,2,2); axis('equal'); xlabel('Z'); ylabel('Y') 189 | plot(-xyz[:,2],xyz[:,0], 'b.') 190 | subplot(2,2,3); axis('equal'); xlabel('X'); ylabel('Z') 191 | plot(-xyz[:,1],xyz[:,2], 'b.') 192 | 193 | 194 | 195 | print "Time:", time.time()-time1 196 | 197 | 198 | 199 | 200 | 201 | -------------------------------------------------------------------------------- /pyKinectTools/algs/smij.py: -------------------------------------------------------------------------------- 1 | 2 | import os 3 | import optparse 4 | import time 5 | import numpy as np 6 | import joblib 7 | 8 | from pyKinectTools.utils.SkeletonUtils import msr_to_kinect_skel, get_skel_angles 9 | 10 | def smij(skels, confidence=None): 11 | ''' 12 | Ranks joints based on variance 13 | Output: ranked joints, ranked variances 14 | ''' 15 | 16 | skel_angles = np.array([get_CAD_skel_angles(x) for x in skels]) 17 | n_joint_angles = len(skel_angles[0]) 18 | skel_angles_variance = np.var(skel_angles, 0) 19 | skel_angles_variance = np.nan_to_num(skel_angles_variance) 20 | 21 | if confidence is not None: 22 | idx = np.nonzero(confidence==0) 23 | skel_angles_variance[idx] *= 0 24 | 25 | joints_ranked = np.argsort(skel_angles_variance)[::-1] 26 | # skel_angles_variance = [np.var(skel_angles, 0) for i in range(len(skels))] 27 | # joints_ranked = [np.argsort(x)[::-1] for x in skel_angles_variance] 28 | 29 | # Rank joint angles by max variance 30 | return joints_ranked, skel_angles_variance 31 | -------------------------------------------------------------------------------- /pyKinectTools/configs/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/colincsl/pyKinectTools/a84bb5b7ff9dd613576415932865c2ad435520b3/pyKinectTools/configs/.DS_Store -------------------------------------------------------------------------------- /pyKinectTools/configs/Kinect_Color_CAD_Param.yml: -------------------------------------------------------------------------------- 1 | # %YAML:1.0 2 | Number of cameras: 1 3 | Reference camera: 1 4 | Number of images: 24 5 | MeanError: 0.2026565819978714 6 | SDError: 0.1151662990450859 7 | MaxError: 1.1049631834030151 8 | # ************************ 9 | Camera_1: 10 | path: "." 11 | imgWidth: 640 12 | imgHeight: 480 13 | K: !!opencv-matrix 14 | rows: 3 15 | cols: 3 16 | dt: f 17 | data: [ 574.1, 0., 318.61, 18 | 0., 575.8, 230.26, 19 | 0., 0., 1. ] 20 | Dist: !!opencv-matrix 21 | rows: 1 22 | cols: 4 23 | dt: f 24 | data: [ -2.15777672e-02, 1.43756594e-04, -2.94323241e-03, 25 | 6.62778622e-04, -2.92507039e-07] 26 | PosRel: !!opencv-matrix 27 | rows: 1 28 | cols: 6 29 | dt: f 30 | data: [ 0., 0., 0., 0., 0., 0. ] 31 | # ************************ 32 | 33 | 34 | # data: [ 10.7, 0., 348.61, 35 | # 0., 86.39, 226.26, 36 | # 0., 0., 1. ] 37 | -------------------------------------------------------------------------------- /pyKinectTools/configs/Kinect_Color_Param.yml: -------------------------------------------------------------------------------- 1 | # %YAML:1.0 2 | Number of cameras: 1 3 | Reference camera: 1 4 | Number of images: 24 5 | MeanError: 0.2026565819978714 6 | SDError: 0.1151662990450859 7 | MaxError: 1.1049631834030151 8 | # ************************ 9 | Camera_1: 10 | path: "." 11 | imgWidth: 640 12 | imgHeight: 480 13 | K: !!opencv-matrix 14 | rows: 3 15 | cols: 3 16 | dt: f 17 | data: [ 531.49230957, 0., 318.61, 18 | 0., 532.39190674, 230.26, 19 | 0., 0., 1. ] 20 | Dist: !!opencv-matrix 21 | rows: 1 22 | cols: 4 23 | dt: f 24 | data: [ -2.15777672e-02, 1.43756594e-04, -2.94323241e-03, 25 | 6.62778622e-04, -2.92507039e-07] 26 | PosRel: !!opencv-matrix 27 | rows: 1 28 | cols: 6 29 | dt: f 30 | data: [ 0., 0., 0., 0., 0., 0. ] 31 | # ************************ 32 | 33 | 34 | # data: [ 10.7, 0., 348.61, 35 | # 0., 86.39, 226.26, 36 | # 0., 0., 1. ] 37 | -------------------------------------------------------------------------------- /pyKinectTools/configs/Kinect_Color_Param_2.yml: -------------------------------------------------------------------------------- 1 | # %YAML:1.0 2 | Number of cameras: 1 3 | Reference camera: 1 4 | Number of images: 0 5 | MeanError: 0. 6 | SDError: 0. 7 | MaxError: 0. 8 | # ************************ 9 | Camera_1: 10 | path: "." 11 | imgWidth: 640 12 | imgHeight: 480 13 | K: !!opencv-matrix 14 | rows: 3 15 | cols: 3 16 | dt: f 17 | data: [ 5.2921508098293293e+02, 0., 3.1294272028759258e+02, 0., 5.2556393630057437e+02, 18 | 2.5748068171871557e+02, 0., 0., 1. ] 19 | Dist: !!opencv-matrix 20 | rows: 1 21 | cols: 4 22 | dt: f 23 | data: [ 0.19607373, -0.36734107, -2.47962005e-003, 24 | -1.89774996e-003 ] 25 | PosRel: !!opencv-matrix 26 | rows: 1 27 | cols: 6 28 | dt: f 29 | data: [ 0., 0., 0., 0., 0., 0. ] 30 | # ************************ 31 | -------------------------------------------------------------------------------- /pyKinectTools/configs/Kinect_Color_Param_old.yml: -------------------------------------------------------------------------------- 1 | # %YAML:1.0 2 | Number of cameras: 1 3 | Reference camera: 1 4 | Number of images: 24 5 | MeanError: 0.2026565819978714 6 | SDError: 0.1151662990450859 7 | MaxError: 1.1049631834030151 8 | # ************************ 9 | Camera_1: 10 | path: "." 11 | imgWidth: 640 12 | imgHeight: 480 13 | K: !!opencv-matrix 14 | rows: 3 15 | cols: 3 16 | dt: f 17 | data: [ 531.49230957, 0., 296.63775635, 18 | 0., 532.39190674, 252.53335571, 19 | 0., 0., 1. ] 20 | Dist: !!opencv-matrix 21 | rows: 1 22 | cols: 4 23 | dt: f 24 | data: [ 0.19607373, -0.36734107, -2.47962005e-003, 25 | -1.89774996e-003 ] 26 | PosRel: !!opencv-matrix 27 | rows: 1 28 | cols: 6 29 | dt: f 30 | data: [ 0., 0., 0., 0., 0., 0. ] 31 | # ************************ 32 | -------------------------------------------------------------------------------- /pyKinectTools/configs/Kinect_Depth_Param.yml: -------------------------------------------------------------------------------- 1 | # %YAML:1.0 2 | Number of cameras: 1 3 | Reference camera: 1 4 | Number of images: 0 5 | MeanError: 0. 6 | SDError: 0. 7 | MaxError: 0. 8 | # ************************ 9 | Camera_1: 10 | path: "." 11 | imgWidth: 640 12 | imgHeight: 480 13 | K: !!opencv-matrix 14 | rows: 3 15 | cols: 3 16 | dt: f 17 | data: [ 1574, 0., 320, 0., 574, 18 | 242, 0., 0., 1. ] 19 | Dist: !!opencv-matrix 20 | rows: 1 21 | cols: 4 22 | dt: f 23 | data: [ 0.19607373, -0.36734107, -2.47962005e-003, 24 | -1.89774996e-003 ] 25 | PosRel: !!opencv-matrix 26 | rows: 1 27 | cols: 6 28 | dt: f 29 | data: [ 0., 0., 0., 0., 0., 0. ] 30 | # ************************ 31 | 32 | # data: [ 594.21434211923247, 0., 339.30780975300314, 0., 591.04053696870778, 33 | # 242.73913761751615, 0., 0., 1. ] -------------------------------------------------------------------------------- /pyKinectTools/configs/Kinect_Transformation.txt: -------------------------------------------------------------------------------- 1 | Rw=9.9984628826577793e-01 1.2635359098409581e-03 -1.7487233004436643e-02 -1.4779096108364480e-03 9.9992385683542895e-01 -1.2251380107679535e-02 1.7470421412464927e-02 1.2275341476520762e-02 9.9977202419716948e-01 2 | Tw=1.9985242312092553e+01 -7.4423738761617583e-01 -1.0916736334336222e+01 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /pyKinectTools/configs/OpenNI_Empty.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /pyKinectTools/configs/OpenNI_Infrared.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /pyKinectTools/configs/SamplesConfig.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /pyKinectTools/configs/SamplesConfig_orig.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /pyKinectTools/configs/__init__.py: -------------------------------------------------------------------------------- 1 | # __all__ = ["algs", "utils"] -------------------------------------------------------------------------------- /pyKinectTools/configs/fist.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/colincsl/pyKinectTools/a84bb5b7ff9dd613576415932865c2ad435520b3/pyKinectTools/configs/fist.png -------------------------------------------------------------------------------- /pyKinectTools/dataset_readers/.idea/.name: -------------------------------------------------------------------------------- 1 | dataset_readers -------------------------------------------------------------------------------- /pyKinectTools/dataset_readers/.idea/dataset_readers.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /pyKinectTools/dataset_readers/.idea/encodings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /pyKinectTools/dataset_readers/.idea/inspectionProfiles/Project_Default.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 19 | -------------------------------------------------------------------------------- /pyKinectTools/dataset_readers/.idea/inspectionProfiles/profiles_settings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 7 | -------------------------------------------------------------------------------- /pyKinectTools/dataset_readers/.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /pyKinectTools/dataset_readers/.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /pyKinectTools/dataset_readers/.idea/other.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /pyKinectTools/dataset_readers/.idea/scopes/scope_settings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | -------------------------------------------------------------------------------- /pyKinectTools/dataset_readers/.idea/testrunner.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /pyKinectTools/dataset_readers/.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /pyKinectTools/dataset_readers/BasePlayer.py: -------------------------------------------------------------------------------- 1 | 2 | import numpy as np 3 | import cv2 4 | from skimage.morphology import closing, erosion, opening 5 | from pyKinectTools.algs.BackgroundSubtraction import * 6 | 7 | 8 | class BasePlayer(object): 9 | 10 | depthIm = None 11 | colorIm = None 12 | users = None 13 | backgroundModel = None 14 | foregroundMask = None 15 | mask = None 16 | prevcolorIm = None 17 | 18 | def __init__(self, base_dir='./', get_depth=True, get_color=False, 19 | get_skeleton=False, bg_subtraction=False, fill_images=False, 20 | background_model=None, background_param=None): 21 | 22 | self.base_dir = base_dir 23 | self.deviceID = '[]' 24 | 25 | if bg_subtraction and background_model is None: 26 | raise Exception, "Must specify background_model" 27 | 28 | self.get_depth = get_depth 29 | self.get_color = get_color 30 | self.get_skeleton =get_skeleton 31 | 32 | self.enable_bg_subtraction = bg_subtraction 33 | self.fill_images = fill_images 34 | 35 | if background_model is not None: 36 | print 'Setting up background model:', background_model 37 | self.set_bg_model(background_model, background_param) 38 | 39 | 40 | def update_background(self): 41 | try: 42 | self.background_model.update(self.depthIm) 43 | self.mask = self.background_model.get_foreground() 44 | # self.mask = self.get_person() 45 | except: 46 | self.mask = None 47 | 48 | def set_background(self, im): 49 | self.background_model.backgroundModel = im 50 | 51 | def set_bg_model(self, bg_type='box', param=None): 52 | ''' 53 | Types: 54 | 'box'[param=max_depth] 55 | 'static'[param=background] 56 | 'mean' 57 | 'median' 58 | 'adaptive_mog' 59 | ''' 60 | self.enable_bg_subtraction = True 61 | if bg_type == 'box': 62 | self.bgSubtraction = BoxModel(param) 63 | elif bg_type == 'static': 64 | if param==None: 65 | param = self.depthIm 66 | self.bgSubtraction = StaticModel(depthIm=param) 67 | elif bg_type == 'mean': 68 | self.bgSubtraction = MeanModel(depthIm=self.depthIm) 69 | elif bg_type == 'median': 70 | self.bgSubtraction = MedianModel(depthIm=self.depthIm) 71 | elif bg_type == 'adaptive_mog': 72 | self.bgSubtraction = AdaptiveMixtureOfGaussians(self.depthIm, maxGaussians=5, learningRate=0.01, decayRate=0.001, variance=300**2) 73 | else: 74 | print "No background model added" 75 | 76 | self.backgroundModel = self.bgSubtraction.getModel() 77 | 78 | def next(self, frames=1): 79 | pass 80 | 81 | def get_person(self, edge_thresh=200): 82 | mask, _, _, _ = extract_people(self.foregroundMask, minPersonPixThresh=5000, gradThresh=None) 83 | # mask, _, _, _ = extract_people(self.mask, minPersonPixThresh=5000, gradThresh=None) 84 | # mask, _, _, _ = extract_people(self.mask, minPersonPixThresh=5000, gradThresh=edge_thresh) 85 | mask = erosion(mask, np.ones([3,3], np.uint8)) 86 | return mask 87 | 88 | def visualize(self, color=True, depth=True, show_skel=False): 89 | # ''' Find people ''' 90 | if show_skel: 91 | plotUsers(self.depthIm, self.users) 92 | 93 | if self.get_depth and depth: 94 | cv2.imshow("Depth"+self.deviceID, (self.depthIm-1000)/float(self.depthIm.max())) 95 | # cv2.putText(self.deviceID, (5,220), (255,255,255), size=15) 96 | # vv.imshow("Depth", self.depthIm/6000.) 97 | 98 | if self.get_color and color: 99 | cv2.imshow("Color "+self.deviceID, self.colorIm/255.) 100 | # vv.putText("Color "+self.deviceID, self.colorIm, "Day "+self.day_dir+" Time "+self.hour_dir+":"+self.minute_dir+" Dev#"+str(self.dev), (10,220)) 101 | # vv.imshow("Color", self.colorIm) 102 | 103 | cv2.waitKey(10) 104 | 105 | def run(self): 106 | pass 107 | -------------------------------------------------------------------------------- /pyKinectTools/dataset_readers/CADViewer.py: -------------------------------------------------------------------------------- 1 | """ 2 | Main file for viewing data 3 | """ 4 | 5 | import os 6 | import optparse 7 | import time 8 | import cPickle as pickle 9 | import numpy as np 10 | from skimage import color 11 | import scipy.misc as sm 12 | import skimage.draw as draw 13 | import cv, cv2 14 | from pylab import * 15 | 16 | from pyKinectTools.dataset_readers.KinectPlayer import KinectPlayer, display_help 17 | from pyKinectTools.dataset_readers.BasePlayer import BasePlayer 18 | from pyKinectTools.utils.SkeletonUtils import * 19 | from pyKinectTools.algs.smij import smij 20 | # from pyKinectTools.dataset_readers.CADPlayer import CADPlayer 21 | 22 | import mlpy 23 | # from scipy.interpolate import interp1d 24 | from scipy.interpolate import * 25 | from scipy.signal import resample 26 | 27 | 28 | """ Debugging """ 29 | from IPython import embed 30 | 31 | 32 | # 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14 33 | CAD_DISABLED = [7,9,8,10,13,14] 34 | # -------------------------MAIN------------------------------------------ 35 | 36 | def get_skels_per_sequence(skels, labels): 37 | 38 | skel_segs = {} 39 | for i,lab in labels.items(): 40 | start = lab['start'] 41 | stop = lab['stop'] 42 | name = lab['subaction'] 43 | if name in skel_segs.keys(): 44 | skel_segs[name] += [skels[start:stop]] 45 | else: 46 | skel_segs[name] = [skels[start:stop]] 47 | 48 | return skel_segs 49 | 50 | 51 | def main(): 52 | # Setup kinect data player 53 | 54 | record = False 55 | DIR = '/Users/colin/Data/CAD_120/' 56 | # cam = CADPlayer(base_dir=DIR, get_color=False, get_skeleton=True, subjects=[1], actions=range(10), instances=[1]) 57 | cam = CADPlayer(base_dir=DIR, get_color=False, get_depth=True, get_skeleton=True, 58 | subjects=[1,3,4], actions=[1], instances=[0,1,2]) 59 | 60 | if record: 61 | writer = cv2.VideoWriter(filename="CAD_action_{}.avi".format(2), fps=30, 62 | frameSize=(640,480),fourcc=cv.CV_FOURCC('I','4','2','0'), isColor=0) 63 | 64 | framerate = 1 65 | temporal_window = 1*(14) # 14 FPS 66 | skel_angles_all = [] 67 | subactivities = [] 68 | skels_subactivity = {} 69 | while cam.next(framerate): 70 | # window_start = np.maximum(0, cam.frame-temporal_window) 71 | 72 | # Get joint angles 73 | if cam.frame == 1: 74 | skels = cam.skel_stack['pos'] 75 | skel_angles = np.array([get_CAD_skel_angles(x) for x in skels]) 76 | skel_angles = np.nan_to_num(skel_angles) 77 | 78 | tmp = get_skels_per_sequence(skel_angles, cam.subactivity) 79 | for t in tmp: 80 | if t not in skels_subactivity: 81 | skels_subactivity[t] = [] 82 | skels_subactivity[t] += tmp[t] 83 | 84 | skel_angles_all += [skel_angles] 85 | subactivities += [cam.subactivity] 86 | cam.next_sequence() 87 | # break 88 | 89 | # Compute variance over the past second 90 | # cam.skels_pos_conf[CAD_DISABLED] = 0 91 | # smij_features, smij_variance = smij(cam.skel_stack['pos'][window_start:cam.frame], confidence=cam.skels_pos_conf) 92 | # smij_features = smij_features[:3] 93 | # print smij_features 94 | 95 | # Add annotations 96 | cam.depthIm = np.repeat(cam.depthIm[: ,: ,None], 3, -1) 97 | max_val = cam.depthIm.max() 98 | # Label actions 99 | cv2.putText(cam.depthIm, "Action: "+cam.action, (20,60), cv2.FONT_HERSHEY_DUPLEX, 1, (max_val,0,0), thickness=2) 100 | cv2.putText(cam.depthIm, "Subaction: "+cam.subaction, (20,85), cv2.FONT_HERSHEY_DUPLEX, 1, (max_val,0,0), thickness=2) 101 | # Label object affordances 102 | for o in cam.objects: 103 | pt1 = tuple(o['topleft'].astype(np.int)) 104 | pt2 = tuple(o['bottomright'].astype(np.int)) 105 | if not all(pt1) or not all(pt2): 106 | continue 107 | cv2.rectangle(cam.depthIm, pt1, pt2, (0,0,max_val)) 108 | pt1 = (pt1[0], pt1[1]+15) 109 | cv2.putText(cam.depthIm, cam.object_names[o["ID"]], pt1, cv2.FONT_HERSHEY_DUPLEX, .6, (0,0,max_val), thickness=1) 110 | # Label skeleton 111 | if cam.depth_stack is not None: 112 | # Hilight active joints 113 | cam.depthIm = display_skeletons(cam.depthIm, cam.users_uv[0], skel_type='CAD', color=(0,max_val,0), confidence=cam.skels_pos_conf, alt_color=(0,max_val/5,max_val/4)) 114 | 115 | # Highlight SMIJ joints 116 | # smij_top = np.ones(cam.users_uv[0].shape[0], np.int) 117 | # smij_top[smij_features] = 0 118 | # cam.depthIm = display_skeletons(cam.depthIm, cam.users_uv[0], skel_type='CAD', color=(0,max_val/4,0), confidence=smij_top, alt_color=(0,max_val,0)) 119 | 120 | cam.visualize(color=True, depth=True)#, depth_bounds=[0,5000]) 121 | 122 | if record: 123 | writer.write((cam.depthIm/cam.depthIm.max()*255).astype(np.uint8)) 124 | 125 | for it in range(len(skel_angles_all)): 126 | for i,angs in enumerate(skel_angles_all[it].T): 127 | figure(i) 128 | plot(angs) 129 | title(CAD_JOINTS[i]) 130 | # from scipy.signal import cspline1d, cspline1d_eval 131 | 132 | ''' it appears to be matching to the wrong subactivity? in DTW ''' 133 | for i_key,key in enumerate(skels_subactivity): 134 | figure(key) 135 | # title(key) 136 | print "{} iterations of {}".format(len(skels_subactivity[key]), key) 137 | for i_iter in range(len(skels_subactivity[key])): 138 | print 'iter', i_iter 139 | for i,ang in enumerate(skels_subactivity[key][i_iter].T): 140 | x = skels_subactivity[key][0][:,i] 141 | y = ang 142 | y = resample(y, len(x)) 143 | error, dtw_mat, y_ind = mlpy.dtw.dtw_std(x, y, dist_only=False) 144 | error, dtw_mat, y_ind = mlpy.dtw.dtw_subsequence(y, x) 145 | 146 | 147 | subplot(3,4,i+1) 148 | y_new = y[y_ind[0]] 149 | x_new = np.linspace(0, 1, len(y_new)) 150 | # poly = polyfit(x_new, y_new, 5) 151 | # y_spline_ev = poly1d(poly)(x_new) 152 | 153 | nknots = 4 154 | idx_knots = (np.arange(1,len(x_new)-1,(len(x_new)-2)/np.double(nknots))).astype('int') 155 | knots = x_new[idx_knots] 156 | y_spline = splrep(x_new, y_new, t=knots) 157 | y_spline_ev = splev(np.linspace(0, 1, len(y_new)), y_spline) 158 | # plot(y_new) 159 | plot(y_spline_ev) 160 | # show() 161 | # plot(y[y_ind[0]]) 162 | 163 | # subplot(3,10,i+1 + i_iter*10) 164 | # plot(x[y_ind[1]]) 165 | # plot(y[y_ind[0]]) 166 | print i,":", len(ang), len(x), len(y[y_ind[0]]) 167 | title(CAD_JOINTS[i]) 168 | 169 | if i == 10: 170 | break 171 | show() 172 | 173 | # Use bezier curves? 174 | 175 | # import mlpy 176 | # x = skel_angles_all[0].T[1] 177 | # y = skel_angles_all[1].T[1] 178 | # error, dtw_mat, y_ind = mlpy.dtw.dtw_std(y, x, dist_only=False) 179 | # plot(y[y_ind[0]-1]) 180 | # plot(x[y_ind[1]-1]) 181 | 182 | 183 | 184 | 185 | # if record: 186 | # writer.release() 187 | 188 | 189 | if __name__=="__main__": 190 | 191 | parser = optparse.OptionParser() 192 | parser.add_option('-a', '--anon', dest='anon', action="store_true", default=False, help='Enable anonomization') 193 | (opt, args) = parser.parse_args() 194 | 195 | main(opt.anon) 196 | 197 | -------------------------------------------------------------------------------- /pyKinectTools/dataset_readers/CAD_Stats.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from collections import Counter 3 | from pyKinectTools.dataset_readers.CADPlayer import read_labels 4 | 5 | base_dir = '/Users/colin/Data/CAD_120/' 6 | 7 | labels_all = [] 8 | labels = read_labels(base_dir, [1,3,4,5], np.arange(10), np.arange(3)) 9 | try: 10 | while 1: 11 | labels_all += [labels.next()] 12 | except: 13 | pass 14 | 15 | x = labels_all[0] 16 | 17 | actions = [x[0]['activity'] for x in labels_all] 18 | subactions = [[y[1][x]['subaction'] for x in y[1]] for y in labels_all] 19 | subactions = np.hstack(subactions) 20 | objects = np.hstack([x[0]['objects'].values() for x in labels_all]) 21 | affordances = [np.hstack([y[1][x]['objects'].values() for x in y[1]]) for y in labels_all] 22 | affordances = np.hstack(affordances) 23 | 24 | 25 | print "Actions:", unique(actions) 26 | print "Total actions:", len(actions) 27 | print 28 | print "Subactions:", np.unique(subactions) 29 | print "Total subactions:", len(subactions) 30 | print 31 | print "Objects:", unique(objects) 32 | print "Total objects:", len(objects) 33 | print 34 | print "Affordances:", np.unique(affordances) 35 | print "Total affordances:", len(affordances) 36 | 37 | action_hist = Counter(actions) 38 | subaction_hist = Counter(subactions) 39 | object_hist = Counter(objects.tolist()) 40 | affordance_hist = Counter(affordances.tolist()) 41 | 42 | from pylab import bar, show, xticks 43 | 44 | figure(1) 45 | bar(np.arange(10), action_hist.values()) 46 | title("Actions") 47 | xticks(np.arange(len(unique(actions)))+.5, action_hist.keys()) 48 | 49 | figure(2) 50 | bar(np.arange(len(unique(subactions))), subaction_hist.values()) 51 | title("Subactions") 52 | xticks(np.arange(len(unique(subactions)))+.5, subaction_hist.keys()) 53 | 54 | figure(3) 55 | bar(np.arange(len(unique(objects))), object_hist.values()) 56 | title("Objects") 57 | xticks(np.arange(len(unique(objects)))+.5, object_hist.keys()) 58 | 59 | figure(4) 60 | bar(np.arange(len(unique(affordances))), affordance_hist.values()) 61 | title("Affordances") 62 | xticks(np.arange(len(unique(affordances)))+.5, affordance_hist.keys()) 63 | 64 | 65 | 66 | -------------------------------------------------------------------------------- /pyKinectTools/dataset_readers/ChalearnBodyPartReader.py: -------------------------------------------------------------------------------- 1 | ''' 2 | This file extracts depth images for all annotated joint data in the chalearn datasets. 3 | 4 | Colin Lea 5 | June 2012 6 | ''' 7 | 8 | import csv 9 | import cv, cv2 10 | from copy import deepcopy 11 | import scipy.io as io 12 | import random 13 | 14 | import sys 15 | sys.path.append("/Users/colin/libs/pyvision/build/lib.macosx-10.7-intel-2.7/") 16 | from vision import features 17 | 18 | calcHOG = 1 19 | 20 | S_R_HAND = 1 21 | S_L_HAND = 2 22 | S_HEAD = 3 23 | S_R_SHOULDER = 4 24 | S_L_SHOULDER = 5 25 | S_R_ELBOW = 6 26 | S_L_ELBOW = 7 27 | 28 | SKELETON = [S_R_HAND, S_L_HAND, S_HEAD, S_R_SHOULDER, S_L_SHOULDER, S_R_ELBOW, S_L_ELBOW] 29 | 30 | fid = csv.reader(open('/Users/colin/data/chalearn/body_parts.csv')) 31 | tmpData = [] 32 | 33 | for i in fid: 34 | tmpData.append(i) 35 | 36 | #must translate annotated position by [-40, -40] 37 | 38 | data = {} 39 | for i in range(1, len(tmpData)): 40 | devSet = tmpData[i][0] 41 | vidSet = tmpData[i][1] 42 | ident = devSet 43 | vidIdent = 'K_' + vidSet + '.avi' 44 | # ident = str(devSet) 45 | frame = int(tmpData[i][2]) 46 | joint = int(tmpData[i][3]) 47 | uncertain = int(tmpData[i][4]) 48 | left = int(tmpData[i][5]) 49 | top =int(tmpData[i][6]) 50 | width = int(tmpData[i][7]) 51 | height = int(tmpData[i][8]) 52 | pos = (left-40,top-40) 53 | size = (width, height) 54 | 55 | if not data.has_key(ident): 56 | data[ident] = {} 57 | if not data[ident].has_key(vidIdent): 58 | data[ident][vidIdent] = {} 59 | data[ident][vidIdent]['pos'] = {} 60 | data[ident][vidIdent]['size'] = {} 61 | data[ident][vidIdent]['uncertain'] = {} 62 | 63 | data[ident][vidIdent]['frame'] = frame 64 | data[ident][vidIdent]['pos'][joint] = pos 65 | data[ident][vidIdent]['size'][joint] = size 66 | data[ident][vidIdent]['uncertain'][joint] = uncertain 67 | 68 | 69 | im = np.empty([240,320]) 70 | 71 | folder = '/Users/colin/data/chalearn/devel/' 72 | 73 | 74 | bodyPartDepths = {1:[], 2:[], 3:[], 4:[], 5:[], 6:[], 7:[]} 75 | bodyPartFeatures = {1:[], 2:[], 3:[], 4:[], 5:[], 6:[], 7:[]} 76 | mapping = ["r_hand","l_hand","face","r_shoulder","l_shoulder","r_elbow","l_elbow"] 77 | 78 | otherDepths = [] 79 | otherFeatures = [] 80 | 81 | #Save all depth data 82 | for setNum in data.keys(): 83 | for vidNum in data[setNum].keys(): 84 | 85 | cam = cv2.VideoCapture(folder+setNum+"/"+vidNum) 86 | frame = data[setNum][vidNum]['frame'] 87 | cam.set(cv.CV_CAP_PROP_POS_FRAMES,frame) 88 | f, im = cam.retrieve() 89 | im = im*(im 0: 97 | size[b] = (boxSize,boxSize) #!! 98 | pos[b] = (pos[b][0] + int(size[b][0]/2) - boxSize/2, 99 | pos[b][1] + int(size[b][1]/2) - boxSize/2) 100 | # cv2.rectangle(im, pos[b], (pos[b][0]+size[b][0], pos[b][1]+size[b][1]), [200,100,100]) 101 | box = im[pos[b][1]:pos[b][1]+size[b][1], pos[b][0]:pos[b][0]+size[b][0], 2] 102 | # boxMean = int(box.mean()) 103 | # boxMax = box.max() 104 | # box[box=boxMean] -= box[box>0].min() 106 | bodyPartDepths[b].append(deepcopy(box)) 107 | if calcHOG and box.shape[0] > 0 and box.shape[1] > 0 and box.shape[0] == boxSize and box.shape[1] == boxSize: 108 | # tmpBox = np.dstack([box[:,:,2], box[:,:,2], box[:,:,2]]) 109 | tmpBoxD = np.dstack([box,box,box]) 110 | f = features.hog(tmpBoxD, 4) 111 | bodyPartFeatures[b].append(deepcopy(f)) 112 | 113 | # im2 = HOGpicture(f, 4) 114 | # figure(2); imshow(im, interpolation='nearest') 115 | 116 | # for i in range(5): 117 | # posNew = [pos[b][0]+random.randint(30,50), pos[b][1]+random.randint(30,50)] 118 | # box = im[pos[b][1]:pos[b][1]+size[b][1], pos[b][0]:pos[b][0]+size[b][0], 2] 119 | # if calcHOG and box.shape[0] > 0 and box.shape[1] > 0 and box.shape[0] == 30 and box.shape[1] == 30: 120 | # otherDepths.append(deepcopy(box)) 121 | # tmpBoxD = np.dstack([box,box,box]) 122 | # f = features.hog(tmpBoxD, 4) 123 | # otherFeatures.append(deepcopy(f)) 124 | 125 | 126 | 127 | # figure(1); subplot(2,4,b) 128 | # imshow(box) 129 | # figure(2); imshow(im) 130 | 131 | # f, im = cam.read() 132 | # imshow(im) 133 | 134 | if 1: 135 | # cv2.imshow("1", im*(im<150)) 136 | cv2.imshow("1", tmpBoxD) 137 | ret = cv2.waitKey(100) 138 | 139 | if ret >= 0: 140 | break 141 | 142 | 143 | io.savemat("bodyPart_DepthImgs.mat", { 144 | mapping[0]:bodyPartDepths[1], 145 | mapping[1]:bodyPartDepths[2], 146 | mapping[2]:bodyPartDepths[3], 147 | mapping[3]:bodyPartDepths[4], 148 | mapping[4]:bodyPartDepths[5], 149 | mapping[5]:bodyPartDepths[6], 150 | mapping[6]:bodyPartDepths[7], 151 | "other":otherDepths 152 | }) 153 | io.savemat("bodyPart_HOGFeatures.mat", { 154 | mapping[0]:bodyPartFeatures[1], 155 | mapping[1]:bodyPartFeatures[2], 156 | mapping[2]:bodyPartFeatures[3], 157 | mapping[3]:bodyPartFeatures[4], 158 | mapping[4]:bodyPartFeatures[5], 159 | mapping[5]:bodyPartFeatures[6], 160 | mapping[6]:bodyPartFeatures[7], 161 | "other":otherFeatures 162 | }) 163 | 164 | 165 | -------------------------------------------------------------------------------- /pyKinectTools/dataset_readers/EVALPlayer.py: -------------------------------------------------------------------------------- 1 | import os 2 | import cv2 3 | import numpy as np 4 | import scipy.misc as sm 5 | import scipy.ndimage as nd 6 | import itertools as it 7 | from pyKinectTools.utils.DepthUtils import skel2depth, depthIm_to_colorIm#world2depth, depthIm2XYZ, depth2world 8 | from pyKinectTools.utils.SkeletonUtils import msr_to_kinect_skel 9 | from pyKinectTools.algs.BackgroundSubtraction import extract_people 10 | from pyKinectTools.dataset_readers.BasePlayer import BasePlayer 11 | from IPython import embed 12 | 13 | 14 | def get_EVAL_filenames(base_dir): 15 | ''' 16 | ---Returns--- 17 | list of filenames 18 | ''' 19 | filenames = os.listdir(base_dir) 20 | filenames = [f for f in filenames if f[:4]=='seq_' and f[-4:]=='.bin'] 21 | 22 | return filenames 23 | 24 | def read_EVAL_depth_ims(data_file): 25 | file_ = open(data_file, 'rb') 26 | frame_size = 930704 27 | marker_count = 32 28 | joint_count = 32 29 | marker_dtype = np.dtype([("name", 'S256'), ("pos", np.float32, 3)]) 30 | joint_dtype = np.dtype([("id", np.int32), ("pos", np.float32, 3)]) 31 | frame_dtype = np.dtype([('magic', np.int32), ('frame', np.int32), \ 32 | ('image',np.float32, [240*320,3]), ("marker_count", np.int32), \ 33 | ("markers", marker_dtype, marker_count),("joints", joint_dtype, joint_count)]) 34 | 35 | frames_stack = [] 36 | markers_stack = [] 37 | skeleton_stack = [] 38 | i=0 39 | while 1: 40 | try: 41 | frame = file_.read(frame_size) 42 | magic = np.frombuffer(frame[0:4], dtype=np.int32)[0] 43 | frame_num = np.frombuffer(frame[4:8], dtype=np.int32)[0] 44 | byte_index = 8 45 | 46 | pts = np.frombuffer(frame[byte_index:byte_index+4*320*240*3], dtype=np.float32).reshape([240,320,3]) 47 | frames_stack += [np.array(pts)[:,:,:,None]] 48 | byte_index += 4*320*240*3 49 | 50 | m_count = np.frombuffer(frame[byte_index:byte_index+4], dtype=np.int32)[0] 51 | byte_index = 8+3*4*320*240+4 52 | 53 | markers = np.frombuffer(frame[byte_index:byte_index+marker_count*(256+3*4)], dtype=marker_dtype) 54 | markers_stack = markers[:m_count] 55 | byte_index += marker_count*(256+3*4) 56 | 57 | j_count = np.frombuffer(frame[byte_index:byte_index+4], dtype=np.int32)[0] 58 | byte_index += 4 59 | 60 | joints = np.frombuffer(frame[byte_index:byte_index+joint_count*(4*4)], dtype=joint_dtype) 61 | skeleton_stack = joints[:j_count] 62 | i += 1 63 | except: 64 | break 65 | xyz_stack = np.concatenate(frames_stack, -1) 66 | markers_stack = np.hstack(markers_stack) 67 | skeleton_stack = np.hstack(skeleton_stack) 68 | 69 | # Convert to my coordinate system 70 | xyz_stack *= 1000 71 | xyz_stack[:,:,:] *= -1 72 | xyz_stack = xyz_stack[:,:,[1,0,2]] 73 | 74 | return xyz_stack, skeleton_stack 75 | 76 | 77 | 78 | class EVALPlayer(BasePlayer): 79 | 80 | def __init__(self, base_dir='./', get_depth=True, 81 | get_skeleton=True, bg_subtraction=False, fill_images=False, 82 | actions=[1], subjects=[1], positions=[2]): 83 | 84 | self.enable_bg_subtraction = bg_subtraction 85 | self.fill_images = fill_images 86 | self.base_dir = base_dir 87 | self.deviceID = ""#Action {0:d}, Subject {1:d}, Instance {2:d}".format(0, 0, 0) 88 | 89 | self.get_depth = get_depth 90 | self.get_skeleton =get_skeleton 91 | 92 | self.xyz_stack = None 93 | self.skel_stack = None 94 | self.background_model = sm.imread(base_dir+"background_model.png") 95 | 96 | self.filenames = get_EVAL_filenames(base_dir) 97 | 98 | self.player = self.run() 99 | self.next(1) 100 | 101 | def set_background(self, im): 102 | self.backgroundModel = im 103 | 104 | def update_background(self): 105 | '''Background model''' 106 | # self.backgroundModel = self.depthIm*(-self.mask) 107 | # self.foregroundMask = self.mask 108 | pass 109 | 110 | def next(self, frames=1): 111 | ''' 112 | frames : skip (this-1) frames 113 | ''' 114 | # Update frame 115 | try: 116 | for i in range(frames): 117 | self.player.next() 118 | return True 119 | except: 120 | return False 121 | 122 | def run(self): 123 | 124 | # Read data from new file 125 | while len(self.filenames) > 0: 126 | if len(self.filenames) > 0 and self.xyz_stack is None: 127 | data_file = self.filenames.pop() 128 | print 'New video:', data_file 129 | self.deviceID = data_file 130 | self.xyz_stack, self.skeleton_stack = read_EVAL_depth_ims(self.base_dir+data_file) 131 | framecount = self.xyz_stack.shape[-1] 132 | 133 | for i in xrange(framecount): 134 | self.depthIm = self.xyz_stack[:,:,2,i].copy() 135 | self.posIm = self.xyz_stack[:,:,:,i] 136 | 137 | self.mask = (np.abs(self.depthIm - self.background_model) > 500) * (self.depthIm != 0) 138 | self.mask = extract_people(self.depthIm, self.mask, 1000, 500)[0] 139 | 140 | if self.enable_bg_subtraction: 141 | self.depthIm *= self.mask 142 | self.posIm *= self.mask[:,:,None] 143 | 144 | self.users = []#[msr_to_kinect_skel(self.skel_stack[i])] 145 | 146 | yield 147 | 148 | 149 | def get_person(self): 150 | labelIm, maxLabel = nd.label(self.mask) 151 | connComps = nd.find_objects(labelIm, maxLabel) 152 | px_count = [nd.sum(labelIm[c]==l) for c,l in zip(connComps,range(1, maxLabel+1))] 153 | max_box = np.argmax(px_count) 154 | 155 | return labelIm == max_box+1 156 | 157 | def visualize(self, show_skel=False): 158 | 159 | # ''' Find people ''' 160 | # if show_skel: 161 | # self.ret = plotUsers(self.depthIm, self.users) 162 | cv2.imshow("Depth", (self.depthIm-2000)/2000.) 163 | # cv2.putText(self.deviceID, (5,220), (255,255,255), size=15) 164 | cv2.waitKey(10) 165 | -------------------------------------------------------------------------------- /pyKinectTools/dataset_readers/MHADPlayer.py: -------------------------------------------------------------------------------- 1 | import os 2 | import cv2 3 | import numpy as np 4 | import scipy.misc as sm 5 | import png 6 | import itertools as it 7 | from skimage.morphology import closing, erosion, opening 8 | from skimage.draw import circle 9 | from pyKinectTools.utils.DepthUtils import CameraModel, skel2depth, depthIm_to_colorIm, world2depth, world2rgb, get_kinect_transform #depthIm2XYZ, depth2world 10 | from pyKinectTools.utils.SkeletonUtils import mhad_to_kinect_skel, display_skeletons 11 | from pyKinectTools.dataset_readers.BasePlayer import BasePlayer 12 | from pyKinectTools.algs.BackgroundSubtraction import fill_image, StaticModel, extract_people 13 | # from pyKinectTools.utils.VideoViewer import VideoViewer 14 | # vv = VideoViewer() 15 | 16 | from IPython import embed 17 | 18 | ''' 19 | For more information about this dataset see: http://tele-immersion.citris-uc.org/berkeley_mhad/ 20 | ''' 21 | 22 | # def get_kinect_transform(filename): 23 | # transform = np.eye(4) 24 | # with open(filename, 'r') as f: 25 | # # Rotation 26 | # line = f.readline().split() 27 | # line[0] = line[0][3:] 28 | # transform[:3,:3] = np.array(line, np.float).reshape([3,3]) 29 | # # Translation 30 | # line = f.readline().split() 31 | # line[0] = line[0][3:] 32 | # transform[:3,-1] = np.array(line, np.float) 33 | # return transform 34 | 35 | def create_folder_names(device=1, subjects=[1], actions=[1], reps=[1]): 36 | folders = [] 37 | for s in subjects: 38 | for a in actions: 39 | for r in reps: 40 | folders += ["Kin{:02d}/S{:02d}/A{:02d}/R{:02d}/".format(device,s,a, r)] 41 | return folders 42 | 43 | def create_mocap_filenames(subjects=[1], actions=[1], reps=[1]): 44 | folders = [] 45 | for s in subjects: 46 | for a in actions: 47 | for r in reps: 48 | folders += ["moc_s{:02d}_a{:02d}_r{:02d}.txt".format(s,a,r)] 49 | return folders 50 | 51 | def read_pnm(filename): 52 | fd = open(filename,'rb') 53 | format, width, height, samples, maxval = png.read_pnm_header( fd ) 54 | pixels = np.fromfile( fd, dtype='u1' if maxval < 256 else '>u2' ) 55 | return pixels.reshape(height,width,samples) 56 | 57 | def read_depth_ims(folder): 58 | files = os.listdir(folder) 59 | files = [f for f in files if f.find('depth')>=0] 60 | framecount = len(files) 61 | 62 | ims = np.empty([480,640,framecount], dtype=np.int16) 63 | rows, cols = [480,640] 64 | 65 | for i,filename in enumerate(files): 66 | im = read_pnm(folder+filename).reshape([480,640]) 67 | ims[:,:,i] = im 68 | 69 | return ims 70 | 71 | def read_color_ims(folder): 72 | ''' Extracts color images from the MSR Daily Activites dataset 73 | ---Parameters--- 74 | folder : folder name for video 75 | data_type : 'depth' or 'color' 76 | ''' 77 | files = os.listdir(folder) 78 | files = [f for f in files if f.find('color')>=0] 79 | framecount = len(files) 80 | 81 | ims = np.empty([480,640,3,framecount], dtype=np.uint8) 82 | rows, cols, depth = [480,640,3] 83 | 84 | for i,filename in enumerate(files): 85 | im = sm.imread(folder+filename) 86 | ims[:,:,:,i] = im 87 | 88 | return ims 89 | 90 | def read_depth_timestamps(filename): 91 | 92 | data = np.loadtxt(filename) 93 | frames = data[:,0] 94 | if data.shape[1] == 2: 95 | times = data[:,1] 96 | else: 97 | times = data[:,1] + data[:,2]/1000000 98 | # timestamps = np.vstack([frames, times]) 99 | timestamps = np.array(times) 100 | 101 | return timestamps 102 | 103 | def read_mocap(filename): 104 | # files = os.listdir(folder) 105 | 106 | data = np.loadtxt(filename) 107 | frames = data[:,129] 108 | times = data[:,130] 109 | data = data[:,:129] 110 | markers = data.reshape([-1, 43,3]) 111 | mocap = {'frames':frames, 'times':times, 'markers':markers} 112 | 113 | return mocap 114 | 115 | 116 | class MHADPlayer(BasePlayer): 117 | 118 | def __init__(self, kinect=1, subjects=[1], actions=[1], reps=[1], **kwargs): 119 | super(MHADPlayer, self).__init__(**kwargs) 120 | 121 | # Settings 122 | self.deviceID = "Kinect: {:d}".format(kinect) 123 | self.repetitions = reps 124 | # Get data filenames 125 | self.kinect_folder_names = create_folder_names(kinect, subjects, actions, reps) 126 | self.mocap_filenames = create_mocap_filenames(subjects, actions, reps) 127 | # Get calibration 128 | print self.base_dir 129 | self.camera_model = CameraModel(self.base_dir+"Calibration/camcfg_k{:02d}.yml".format(kinect)) 130 | self.kinect_transform = get_kinect_transform(self.base_dir+"Calibration/RwTw_k{:02d}.txt".format(kinect)) 131 | self.camera_model.set_transform(self.kinect_transform) 132 | # Setup background model 133 | self.background_model = StaticModel() 134 | self.set_background(sm.imread(self.base_dir+"Kinect/Kin{:02d}/background_model.png".format(kinect)).clip(0, 4500)) 135 | self.mask = 1 136 | # Initialize 137 | self.player = self.run() 138 | self.next(1) 139 | 140 | def next(self, frames=1): 141 | ''' 142 | frames : skip (this-1) frames 143 | ''' 144 | # Update frame 145 | try: 146 | # if 1: 147 | for i in range(frames): 148 | self.player.next() 149 | return True 150 | except: 151 | print "Done playing video" 152 | return False 153 | 154 | def run(self): 155 | 156 | # Read data from new file 157 | while len(self.kinect_folder_names) > 0: 158 | print 'New video:', self.kinect_folder_names[-1] 159 | # Load videos 160 | self.depth_stack = read_depth_ims(self.base_dir + 'Kinect/' + self.kinect_folder_names[-1]) 161 | self.color_stack = read_color_ims(self.base_dir + 'Kinect/' + self.kinect_folder_names[-1]) 162 | # embed() 163 | self.mocap_stack = read_mocap(self.base_dir+'Mocap/OpticalData/'+self.mocap_filenames[-1]) 164 | self.skel_stack = self.mocap_stack['markers'] 165 | # Load timestamps 166 | time_tmp = self.kinect_folder_names[-1].split('/') 167 | if time_tmp[0] == '': 168 | time_tmp = time_tmp[1:] 169 | kinect_timestamp_name = "time_stamps_{:s}_{:s}_{:s}_{:s}.txt".format('kin01', time_tmp[1], time_tmp[2], time_tmp[3]) 170 | # print kinect_timestamp_name, time_tmp 171 | # kinect_timestamp_name = "time_stamps_{:s}_{:s}_{:s}_{:s}.txt".format(time_tmp[1], time_tmp[2], time_tmp[3], time_tmp[4]) 172 | self.depth_timestamps = read_depth_timestamps(self.base_dir + 'Kinect/Time_stamps/' + kinect_timestamp_name.lower()) 173 | self.mocap_timestamps = self.mocap_stack['times'] 174 | 175 | self.mocap_filenames.pop() 176 | self.kinect_folder_names.pop() 177 | framecount = self.depth_stack.shape[-1] 178 | 179 | for i in xrange(framecount): 180 | self.depthIm = self.depth_stack[:,:,i].clip(0,4500) 181 | self.colorIm = self.color_stack[:,:,[2,1,0],i] 182 | # self.colorIm = self.color_stack[:,:,:,i] 183 | self.depth_timestamp = self.depth_timestamps[i] 184 | # Get mocap data at correct time 185 | while self.mocap_timestamps[0] < self.depth_timestamp: 186 | self.mocap_timestamps = self.mocap_timestamps[1:] 187 | self.skel_stack = self.skel_stack[1:] 188 | 189 | # Transform skeleton to kinect space 190 | self.users = [self.skel_stack[0]] 191 | 192 | # from pylab import * 193 | # figure(i) 194 | # for ii,i in enumerate(self.users[0]): 195 | # scatter(i[0], i[1]) 196 | # annotate(str(ii), (i[0], i[1])) 197 | # axis('equal') 198 | # show() 199 | 200 | # Convert to Kinect format 201 | self.users_msr_tmp = [mhad_to_kinect_skel(self.users[0])] 202 | # self.users_msr[0] *= (self.users_msr[0][:,2]!=0)[:,None] 203 | # Transform to Kinect world space 204 | tmp_pts = np.hstack([self.users_msr_tmp[0],np.ones_like(self.users_msr_tmp[0])])[:,:4] 205 | self.users_msr = [np.dot(self.kinect_transform, tmp_pts.T).T[:,:3]] 206 | # Transform to Kinect image space 207 | self.users_uv_msr = [ self.camera_model.world2im(self.users_msr[0], [480,640]) ] 208 | self.users_uv_msr *= (self.users_msr_tmp[0][:,2]!=0)[:,None] 209 | self.users = self.users_msr 210 | self.users_uv = self.users_uv_msr 211 | 212 | # self.colorIm = display_skeletons(self.colorIm, self.users_uv_msr[0], skel_type='Kinect') 213 | self.update_background() 214 | 215 | if type(self.mask) is not int: 216 | self.mask = opening(self.mask, np.ones([3,3], np.uint8)) 217 | if self.fill_images: 218 | self.depthIm = fill_image(self.depthIm) 219 | self.foregroundMask = self.mask 220 | 221 | yield 222 | 223 | 224 | 225 | 226 | -------------------------------------------------------------------------------- /pyKinectTools/dataset_readers/SMMCPlayer_tmp.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import numpy as np 3 | import scipy.misc as sm 4 | import itertools as it 5 | from pyKinectTools.utils.DepthUtils import skel2depth, depthIm_to_colorIm#world2depth, depthIm2XYZ, depth2world 6 | from pyKinectTools.utils.SkeletonUtils import msr_to_kinect_skel 7 | from IPython import embed 8 | from pyKinectTools.dataset_readers.BaseReader import BaseReader 9 | 10 | class SMMCPlayer(BasePlayer): 11 | 12 | def __init__(self, base_dir='./', get_depth=True, get_color=True, 13 | get_skeleton=True, bg_subtraction=False, fill_images=False, 14 | actions=[1], subjects=[1], positions=[2]): 15 | 16 | self.enable_bg_subtraction = bg_subtraction 17 | self.fill_images = fill_images 18 | self.base_dir = base_dir 19 | self.deviceID = ""#Action {0:d}, Subject {1:d}, Instance {2:d}".format(0, 0, 0) 20 | 21 | self.get_depth = get_depth 22 | self.get_color = get_color 23 | self.get_skeleton =get_skeleton 24 | 25 | self.depth_stack = None 26 | self.mask_stack = None 27 | self.color_stack = None 28 | self.skel_stack = None 29 | 30 | self.filenames = create_MSR_filenames(actions, subjects, positions) 31 | 32 | self.player = self.run() 33 | self.next(1) 34 | 35 | def set_background(self, im): 36 | self.backgroundModel = im 37 | 38 | def update_background(self): 39 | '''Background model''' 40 | # self.backgroundModel = self.depthIm*(-self.mask) 41 | # self.foregroundMask = self.mask 42 | pass 43 | 44 | def next(self, frames=1): 45 | ''' 46 | frames : skip (this-1) frames 47 | ''' 48 | # Update frame 49 | try: 50 | for i in range(frames): 51 | self.player.next() 52 | return True 53 | except: 54 | return False 55 | 56 | def run(self): 57 | 58 | # Read data from new file 59 | while len(self.filenames) > 0: 60 | if len(self.filenames) > 0 and self.depth_stack is None: 61 | print 'New video' 62 | name = self.filenames.pop() 63 | depth_file = self.base_dir + name + "depth.bin" 64 | color_file = self.base_dir + name + "rgb.avi" 65 | skeleton_file = self.base_dir + name + "skeleton.txt" 66 | self.depth_stack, self.mask_stack = read_MSR_depth_ims(depth_file) 67 | self.color_stack = read_MSR_color_ims(color_file) 68 | self.skel_stack,_ = read_MSR_skeletons(skeleton_file) 69 | # Offset! 70 | self.skel_stack[:,:,1] -= 75 71 | framecount = np.min([self.depth_stack.shape[-1],self.color_stack.shape[-1]]) 72 | 73 | for i in xrange(framecount): 74 | mask = self.mask_stack[:,:,i] 75 | if self.enable_bg_subtraction: 76 | self.depthIm = depthIm_to_colorIm(self.depth_stack[:,:,i]*mask) 77 | else: 78 | self.depthIm = depthIm_to_colorIm(self.depth_stack[:,:,i]) 79 | # self.depthIm = self.depth_stack[:,:,i] 80 | self.colorIm = self.color_stack[:,:,:,i] 81 | self.users = [msr_to_kinect_skel(self.skel_stack[i])] 82 | 83 | # tmp = depthIm_to_colorIm(self.depthIm * mask) 84 | self.mask = self.depthIm > 0 85 | self.update_background() 86 | yield 87 | 88 | 89 | def get_person(self, edge_thresh=200): 90 | return self.mask 91 | 92 | def visualize(self, show_skel=False): 93 | 94 | # ''' Find people ''' 95 | if show_skel: 96 | self.ret = plotUsers(self.depthIm, self.users) 97 | 98 | if self.get_depth: 99 | cv2.imshow("Depth", (self.depthIm-1000)/2000.) 100 | # cv2.putText(self.deviceID, (5,220), (255,255,255), size=15) 101 | cv2.waitKey(10) 102 | -------------------------------------------------------------------------------- /pyKinectTools/dataset_readers/__init__.py: -------------------------------------------------------------------------------- 1 | # __all__ = ["algs", "utils"] -------------------------------------------------------------------------------- /pyKinectTools/scripts/Button2Time.py: -------------------------------------------------------------------------------- 1 | """ 2 | Main file for viewing data 3 | """ 4 | 5 | import os 6 | import time 7 | import optparse 8 | 9 | """ Debugging """ 10 | from IPython import embed 11 | 12 | try: 13 | import serial 14 | ser = serial.Serial(port='/dev/tty.usbmodemfa131', baudrate=9600) 15 | # Turn on light 16 | ser.write('3') 17 | except: 18 | raise Exception, "Please install PySerial (pip install pyserial)" 19 | 20 | # -------------------------MAIN------------------------------------------ 21 | 22 | FILE = os.path.expanduser('~')+'/Data/Kinect_Recorder/time.txt' 23 | 24 | def main(filename): 25 | 26 | recording_enabled = True 27 | button_pressed = False 28 | 29 | while 1: 30 | 31 | while ser.inWaiting() > 0: 32 | button_current = ser.readline() 33 | print button_current, button_pressed 34 | if button_pressed != button_current and button_current: 35 | recording_enabled = False 36 | recording_time = time.time() 37 | # Turn on light 38 | ser.write('4') 39 | print 'On' 40 | 41 | # Write time to file 42 | with open(filename, 'a') as f: 43 | f.write(str(time.gmtime())+'\n') 44 | 45 | button_pressed = button_current 46 | 47 | if not recording_enabled: 48 | if time.time() - recording_time > 2: 49 | recording_enabled = True 50 | ser.write('3') 51 | print 'Off' 52 | else: 53 | continue 54 | 55 | 56 | 57 | # Pause at the end 58 | embed() 59 | 60 | 61 | if __name__=="__main__": 62 | 63 | parser = optparse.OptionParser() 64 | parser.add_option('-f', '--file', dest='file', default=FILE, help='Filename for times') 65 | (opt, args) = parser.parse_args() 66 | 67 | main(opt.file) 68 | 69 | -------------------------------------------------------------------------------- /pyKinectTools/scripts/ButtonRecorderTest.py: -------------------------------------------------------------------------------- 1 | """ 2 | Main file for viewing data 3 | """ 4 | 5 | import os 6 | import time 7 | import optparse 8 | import time 9 | import cPickle as pickle 10 | import numpy as np 11 | from skimage import color 12 | import scipy.misc as sm 13 | 14 | from pyKinectTools.dataset_readers.KinectPlayer import KinectPlayer, display_help 15 | # from pyKinectTools.utils.SkeletonUtils import display_skeletons, transform_skels, kinect_to_msr_skel 16 | 17 | """ Debugging """ 18 | from IPython import embed 19 | 20 | try: 21 | import serial 22 | ser = serial.Serial(port='/dev/tty.usbmodemfa131', baudrate=9600) 23 | # Turn on light 24 | ser.write('3') 25 | except: 26 | raise Exception, "Please install PySerial (pip install pyserial)" 27 | 28 | # -------------------------MAIN------------------------------------------ 29 | 30 | def main(anonomization=False): 31 | # Setup kinect data player 32 | cam = KinectPlayer(base_dir='./', bg_subtraction=False, get_depth=True, get_color=True, get_skeleton=False) 33 | recording_enabled = True 34 | button_pressed = False 35 | 36 | if anonomization: 37 | ''' bg_type can be: 38 | 'box'[param=max_depth] 39 | 'static'[param=background] 40 | 'mean' 41 | 'median' 42 | 'adaptive_mog' 43 | 44 | See BasePlayer for more details 45 | ''' 46 | cam.set_bg_model(bg_type='box', param=2500) 47 | 48 | 49 | framerate = 1 50 | while cam.next(framerate): 51 | 52 | while ser.inWaiting() > 0: 53 | button_current = ser.readline() 54 | print button_current, button_pressed 55 | if button_pressed != button_current and button_current: 56 | recording_enabled = False 57 | recording_time = time.time() 58 | # Turn off light 59 | ser.write('4') 60 | print 'Off' 61 | button_pressed = button_current 62 | 63 | if not recording_enabled: 64 | if time.time() - recording_time > 2: 65 | recording_enabled = True 66 | ser.write('3') 67 | print 'On' 68 | else: 69 | continue 70 | 71 | if anonomization and cam.mask is not None: 72 | mask = cam.mask == 0 73 | if cam.colorIm.shape[:2] != cam.mask.shape: 74 | mask = sm.imresize(mask, [480,640]) 75 | cam.colorIm *= mask[:,:,None] 76 | cam.visualize(color=True, depth=True, text=True, colorize=True, depth_bounds=[0,5000]) 77 | # cam.visualize(color=True, depth=True, text=False, colorize=False, depth_bounds=None) 78 | 79 | print 'Done' 80 | 81 | # Pause at the end 82 | embed() 83 | 84 | 85 | if __name__=="__main__": 86 | 87 | parser = optparse.OptionParser() 88 | parser.add_option('-a', '--anon', dest='anon', action="store_true", default=True, help='Enable anonomization') 89 | (opt, args) = parser.parse_args() 90 | 91 | main(opt.anon) 92 | 93 | -------------------------------------------------------------------------------- /pyKinectTools/scripts/ConvertToPCD.py: -------------------------------------------------------------------------------- 1 | """ 2 | Convert jpg+png image/depth files to .pcd files to work with PCL 3 | """ 4 | 5 | import os 6 | import optparse 7 | import numpy as np 8 | import pcl 9 | from pyKinectTools.dataset_readers.KinectPlayer import KinectPlayer, display_help 10 | from IPython import embed 11 | 12 | 13 | DIR = os.path.expanduser('~')+'/Data/PCL_Data/' 14 | 15 | 16 | # -------------------------MAIN------------------------------------------ 17 | 18 | def main(visualize=False, save_dir='~/', sparse_pointcloud=False): 19 | 20 | # Create save directory if it doesn't exist 21 | if not os.path.isdir(save_dir): 22 | os.mkdir(save_dir) 23 | 24 | cam = KinectPlayer(base_dir='./', device=1, bg_subtraction=True, get_depth=True, 25 | get_color=True, get_skeleton=False, background_model='box', background_param=3200) 26 | 27 | cloud = pcl.PointCloud() 28 | 29 | framerate = 1 30 | while cam.next(framerate): 31 | 32 | pts = cam.camera_model.im2PosIm(cam.depthIm).reshape([-1,3]) 33 | cloud.from_array(pts.astype(np.float32)) 34 | depth_name = cam.depthFile.split("_") 35 | filename = "pcl_" + "_".join(depth_name[1:-1]) + "_" + depth_name[-1].split(".")[0] + ".pcd" 36 | 37 | if sparse_pointcloud: 38 | nonzero_idx = np.nonzero(pts[:,2]>0) 39 | cloud.from_array(pts[nonzero_idx].astype(np.float32)) 40 | 41 | cloud.to_file(save_dir+filename, ascii=True) 42 | 43 | if visualize: 44 | cam.visualize(color=True, depth=True, text=True, colorize=False, depth_bounds=[500,3500]) 45 | 46 | embed() 47 | 48 | print 'Done' 49 | 50 | if __name__=="__main__": 51 | 52 | parser = optparse.OptionParser() 53 | parser.add_option('-v', '--visualize', dest='viz', action="store_true", default=False, help='Enable visualization') 54 | parser.add_option('-s', '--sparse', dest='sparse', action="store_true", default=False, help='Save sparse pointcloud') 55 | parser.add_option('-i', '--dir', dest='dir', default=DIR, help='Save directory') 56 | 57 | (opt, args) = parser.parse_args() 58 | 59 | main(visualize=opt.viz, save_dir=opt.dir, sparse_pointcloud=opt.sparse) -------------------------------------------------------------------------------- /pyKinectTools/scripts/ConvertToPCD_RGBD.py: -------------------------------------------------------------------------------- 1 | """ 2 | Convert jpg+png image/depth files to .pcd files to work with PCL 3 | """ 4 | 5 | import os 6 | import optparse 7 | import numpy as np 8 | from pyKinectTools.dataset_readers.KinectPlayer import KinectPlayer 9 | from IPython import embed 10 | 11 | 12 | DIR = os.path.expanduser('~')+'/Data/PCL_Data' 13 | 14 | header = '''# .PCD v0.7 - Point Cloud Data file format 15 | VERSION 0.7 16 | FIELDS x y z rgb 17 | SIZE 4 4 4 4 18 | TYPE F F F F 19 | COUNT 1 1 1 1 20 | WIDTH 76800 21 | HEIGHT 1 22 | VIEWPOINT 0 0 0 1 0 0 0 23 | POINTS 76800 24 | DATA ascii 25 | ''' 26 | 27 | # -------------------------MAIN------------------------------------------ 28 | 29 | def main(visualize=False, save_dir='~/', sparse_pointcloud=False): 30 | 31 | # Create save directory if it doesn't exist 32 | if not os.path.isdir(save_dir): 33 | os.mkdir(save_dir) 34 | 35 | cam = KinectPlayer(base_dir='./', device=1, get_depth=True, 36 | get_color=True, get_skeleton=False) 37 | 38 | framerate = 1 39 | while cam.next(framerate): 40 | # Get depth points 41 | pts = cam.camera_model.im2PosIm(cam.depthIm).reshape([-1,3]) 42 | 43 | # Get RGB points in floating point notation 44 | rgb = cam.colorIm.reshape([-1,3]).astype(np.int) 45 | rgb_float = (np.left_shift(rgb[:,0],16) + np.left_shift(rgb[:,1],8) + rgb[:,2]).astype(np.float) 46 | 47 | # Merge depth+color data 48 | assert pts.shape[0] == rgb_float.shape[0] and pts.ndim==2 \ 49 | and rgb_float.ndim==1, "Wrong number of data dimensions" 50 | pts_rgbd = np.hstack([pts, rgb_float[:,None]]) 51 | 52 | # Filename 53 | depth_name = cam.depthFile.split("_") 54 | filename = "pcl_" + "_".join(depth_name[1:-1]) + "_" + depth_name[-1].split(".")[0] + ".pcd" 55 | 56 | # Save to file 57 | with open(save_dir+"/"+filename, 'w') as f: 58 | f.write(header) 59 | for p in zip(pts_rgbd): 60 | f.write("{} {} {} {}\n".format(*p[0])) 61 | print "Saved frame: {}/{}".format(save_dir, filename) 62 | 63 | if visualize: 64 | cam.visualize(color=True, depth=True, text=True, colorize=False, depth_bounds=[500,3500]) 65 | 66 | print 'Done' 67 | 68 | if __name__=="__main__": 69 | 70 | parser = optparse.OptionParser() 71 | parser.add_option('-v', '--visualize', dest='viz', action="store_true", default=False, help='Enable visualization') 72 | parser.add_option('-i', '--dir', dest='dir', default=DIR, help='Save directory') 73 | 74 | (opt, args) = parser.parse_args() 75 | 76 | main(visualize=opt.viz, save_dir=opt.dir) 77 | -------------------------------------------------------------------------------- /pyKinectTools/scripts/DualViewer.py: -------------------------------------------------------------------------------- 1 | """ 2 | Main file for training multi-camera pose 3 | """ 4 | 5 | import os 6 | import optparse 7 | import time 8 | import cPickle as pickle 9 | import numpy as np 10 | from skimage import color 11 | 12 | # from rgbdActionDatasets.dataset_readers.KinectPlayer import KinectPlayer, display_help 13 | from pyKinectTools.dataset_readers.KinectPlayer import KinectPlayer, display_help 14 | from pyKinectTools.utils.DepthUtils import world2depth, depthIm2XYZ, skel2depth, depth2world 15 | from pyKinectTools.utils.SkeletonUtils import display_skeletons, transform_skels, kinect_to_msr_skel 16 | 17 | """ Debugging """ 18 | from IPython import embed 19 | np.seterr(all='ignore') 20 | 21 | # -------------------------MAIN------------------------------------------ 22 | 23 | def main(visualize=True): 24 | n_cameras = 1 25 | cam = KinectPlayer(base_dir='./', device=1, bg_subtraction=True, get_depth=True, get_color=True, get_skeleton=False, fill_images=False) 26 | if n_cameras == 2: 27 | cam2 = KinectPlayer(base_dir='./', device=2, bg_subtraction=True, get_depth=True, get_color=get_color, get_skeleton=get_skel, fill_images=fill) 28 | # Transformation matrix from first to second camera 29 | try: 30 | data = pickle.load(open("Registration.dat", 'r')) 31 | transform_c1_to_c2 = data['transform'] 32 | transform = True 33 | except: 34 | transform = False 35 | pass 36 | 37 | current_frame = 0 38 | all_joint_ims_z = [] 39 | all_joint_ims_c = [] 40 | framerate = 1 41 | while cam.next(): 42 | # print "Frame ", current_frame 43 | # Update frames 44 | if n_cameras == 2: 45 | cam2.next() 46 | # cam2.sync_cameras(cam) 47 | current_frame+=1 48 | if current_frame%framerate != 0: 49 | # current_frame += 1 50 | continue 51 | 52 | # Transform skels from cam1 to cam2 53 | # cam_skels = [np.array(cam.users[s]['jointPositions'].values()) for s in cam.users.keys()] 54 | # cam_skels = [np.array(cam.users[s]['jointPositions'].values()) for s in cam.users] 55 | # Get rid of bad skels 56 | # cam_skels = [s for s in cam_skels if np.all(s[0] != -1)] 57 | 58 | # if len(cam_skels) == 0: 59 | # continue 60 | 61 | 62 | # Save images 63 | if 0: 64 | joint_ims_z = [] 65 | joint_ims_c = [] 66 | dx = 10 67 | skel_tmp = skel2depth(cam_skels[0], [240,320]) 68 | for j_pos in skel_tmp: 69 | # embed() 70 | joint_ims_z += [cam.depthIm[j_pos[0]-dx:j_pos[0]+dx, j_pos[1]-dx:j_pos[1]+dx]] 71 | joint_ims_c += [cam.colorIm[j_pos[0]-dx:j_pos[0]+dx, j_pos[1]-dx:j_pos[1]+dx]] 72 | if len(joint_ims_z) > 0: 73 | all_joint_ims_z += [joint_ims_z] 74 | all_joint_ims_c += [joint_ims_c] 75 | 76 | if 1: 77 | # if transform: 78 | # cam2_skels = transform_skels(cam_skels, transform_c1_to_c2, 'image') 79 | 80 | # try: 81 | # depth = cam2.get_person() 82 | # if learn: 83 | # rf.add_frame(depth, cam2_skels[0]) 84 | # else: 85 | # rf.infer_pose(depth) 86 | # except: 87 | # pass 88 | if visualize: 89 | # cam2.depthIm = display_skeletons(cam2.depthIm, cam2_skels[0], (5000,), skel_type='Low') 90 | # skel1 = kinect_to_msr_skel(skel2depth(cam_skels[0], [240,320])) 91 | # cam.depthIm = display_skeletons(cam.depthIm, skel1, (5000,), skel_type='Low') 92 | # embed() 93 | cam.visualize(color=True, depth=True, text=True, colorize=True, depth_bounds=[500,3500]) 94 | if n_cameras == 2: 95 | cam2.visualize(color=True, depth=True) 96 | 97 | 98 | embed() 99 | 100 | 101 | print 'Done' 102 | 103 | if __name__=="__main__": 104 | 105 | parser = optparse.OptionParser() 106 | parser.add_option('-v', '--visualize', dest='viz', action="store_true", default=True, help='Enable visualization') 107 | (opt, args) = parser.parse_args() 108 | 109 | main(opt.viz) 110 | 111 | 112 | # '''Profiling''' 113 | # cProfile.runctx('main()', globals(), locals(), filename="ShowSkeletons.profile") 114 | -------------------------------------------------------------------------------- /pyKinectTools/scripts/HeadOfBed.py: -------------------------------------------------------------------------------- 1 | """ 2 | Main file for training multi-camera pose 3 | """ 4 | 5 | import os 6 | import optparse 7 | import time 8 | import cPickle as pickle 9 | import numpy as np 10 | from skimage import color 11 | import cv2 12 | import pcl 13 | import itertools as it 14 | 15 | # import rospy 16 | # from sensor_msgs.msg import PointCloud2 17 | # from std_msgs.msg import String 18 | 19 | # pub_str = rospy.Publisher('str', String) 20 | # pub_cloud = rospy.Publisher('cloud', PointCloud2) 21 | # rospy.init_node('pyPCL') 22 | 23 | 24 | # from rgbdActionDatasets.dataset_readers.KinectPlayer import KinectPlayer, display_help 25 | from pyKinectTools.dataset_readers.KinectPlayer import KinectPlayer, display_help 26 | from pyKinectTools.utils.DepthUtils import world2depth, depthIm2XYZ, skel2depth, depth2world 27 | from pyKinectTools.utils.SkeletonUtils import display_skeletons, transform_skels, kinect_to_msr_skel 28 | from pyKinectTools.algs.BackgroundSubtraction import extract_people 29 | 30 | """ Debugging """ 31 | from IPython import embed 32 | 33 | num_to_rgb = np.array([ 34 | [1,0,0], 35 | [0,1,0], 36 | [0,0,1], 37 | [1,1,0], 38 | [0,1,1], 39 | [1,0,1], 40 | [1,1,1]]) 41 | 42 | # -------------------------MAIN------------------------------------------ 43 | 44 | def find_plane(cloud, dist_thresh=10, max_iter=500): 45 | seg = cloud.make_segmenter_normals(ksearch=10) 46 | seg.set_optimize_coefficients (True); 47 | seg.set_model_type (pcl.SACMODEL_NORMAL_PLANE) 48 | seg.set_distance_threshold(dist_thresh) 49 | seg.set_method_type (pcl.SAC_RANSAC) 50 | seg.set_max_iterations (max_iter) 51 | indices, model = seg.segment() 52 | # idx = np.unravel_index(indices,cam.depthIm.shape) 53 | 54 | return indices,model 55 | 56 | 57 | def main(visualize=True): 58 | n_cameras = 1 59 | cam = KinectPlayer(base_dir='./', device=1, bg_subtraction=True, get_depth=True, 60 | get_color=True, get_skeleton=False, background_model='box', background_param=3200) 61 | 62 | # embed() 63 | cloud = pcl.PointCloud() 64 | angle_measurements = [] 65 | dist_thresh = 10 66 | max_iter = 500 67 | n_planes = 3 68 | 69 | framerate = 1 70 | cam.play_speed = 300 71 | while cam.next(framerate): 72 | # mask = cam.get_person() > 0 73 | # embed() 74 | id_im, id_slices, ids, id_px = extract_people(cam.depthIm, gradThresh=50) 75 | # id_max = np.argmin([np.abs(cam.depthIm[0]/2 - (x[1].stop+x[1].start)/2.) for x in id_slices]) 76 | id_max = np.argmin([np.abs(320/2 - (x[1].stop+x[1].start)/2.) for x in id_slices]) 77 | mask = id_im==(id_max+1) 78 | 79 | cam.depthIm *= mask 80 | 81 | planes_idx = [] 82 | planes_models = [] 83 | plane_im = np.repeat(cam.depthIm[:,:,None], 3, -1).reshape([-1,3]).copy() 84 | # plane_im_out = np.repeat(cam.depthIm[:,:,None]*0, 3, -1).reshape([-1,3]).copy() 85 | plane_im_out = np.zeros([cam.depthIm.shape[0]*cam.depthIm.shape[1], 3], np.float) 86 | 87 | # Find planes 88 | for i in range(n_planes): 89 | # Get valid points 90 | pts = cam.camera_model.im2PosIm(plane_im[:,0].reshape(cam.depthIm.shape)).reshape([-1,3]) 91 | nonzero_idx = np.nonzero((pts[:,2]>0)*(pts[:,2]!=1000) ) 92 | cloud.from_array(pts[nonzero_idx].astype(np.float32)) 93 | 94 | # Find plane 95 | indices, model = find_plane(cloud, dist_thresh, max_iter) 96 | print "Plane:", model, len(indices) 97 | # if len(indices) < 5000: 98 | # continue 99 | planes_idx += [indices] 100 | planes_models += [model] 101 | 102 | # Remove labeled points and highlight on image 103 | tmp = plane_im[nonzero_idx] 104 | tmp[(np.array(indices),)] = num_to_rgb[i]*1000 105 | plane_im[nonzero_idx] = tmp 106 | # plane_im_out[nonzero_idx[(np.array(indices),)]] = tmp 107 | plane_im_out[nonzero_idx[0][indices]] = num_to_rgb[i]*1000 108 | 109 | 110 | 111 | angle_measurements = [] 112 | for combos in it.combinations(planes_models, 2): 113 | angle_measurements += [np.arccos(np.dot(combos[0][:3], combos[1][:3]))*180./np.pi] 114 | print "Current angle:", angle_measurements 115 | # angle_measurements += [np.arccos(np.dot(planes_models[0][:3], planes_models[1][:3]))*180./np.pi] 116 | # print "Current angle:", angle_measurements[-1] 117 | # print "Average angle:", np.mean(angle_measurements) 118 | print "" 119 | 120 | # tmp = plane_im[nonzero_idx] 121 | # tmp[(np.array(planes_idx[1]),)] = np.array([0, 2000, 0]) 122 | # plane_im[nonzero_idx] = tmp 123 | 124 | plane_im = plane_im.reshape([240,320,3]) 125 | plane_im_out = plane_im_out.reshape([240,320,3]) 126 | # cv2.imshow("dd", plane_im/float(plane_im.max())) 127 | cv2.imshow("dd", (plane_im_out/float(plane_im_out.max()))*100. + cam.colorIm/300.) 128 | # cv2.waitKey(50) 129 | 130 | # cam.visualize(color=True, depth=True, text=True, colorize=True, depth_bounds=[500,3500]) 131 | cam.visualize(color=False, depth=True, text=True, colorize=False, depth_bounds=[500,3500]) 132 | # embed() 133 | 134 | embed() 135 | 136 | print 'Done' 137 | 138 | if __name__=="__main__": 139 | 140 | parser = optparse.OptionParser() 141 | parser.add_option('-v', '--visualize', dest='viz', action="store_true", default=True, help='Enable visualization') 142 | (opt, args) = parser.parse_args() 143 | 144 | main(opt.viz) -------------------------------------------------------------------------------- /pyKinectTools/scripts/IterativeClosestPoint_script.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Iterative closest point example 3 | 4 | Colin Lea 5 | pyKinectTools 6 | 2012 7 | ''' 8 | 9 | import numpy as np 10 | from pyKinectTools.algs.IterativeClosestPoint import IterativeClosestPoint 11 | 12 | 13 | template = np.asmatrix(posMat[np.nonzero((regions <= 14) * (regions > 0))]) 14 | template -= template.mean(0) 15 | # fakeRot = np.matrix([[0,1,0], [1,0,0],[0,0,1]]) 16 | ang = 5 * np.pi/180 17 | # fakeRot = np.matrix([[1,0,0], [0,1,0],[0,0,1]]) 18 | fakeRot = np.matrix([[cos(ang),-sin(ang),0], [sin(ang),cos(ang),0],[0,0,1]]) 19 | fakeTrans = np.matrix([0,100,0]).T 20 | pointcloudInit = np.asmatrix(fakeRot*template.T + fakeTrans) 21 | # pointcloudInit = template.T 22 | pointcloudInit += np.random.rand(3, pointcloudInit.shape[1]) 23 | pointcloud = np.copy(pointcloudInit) 24 | 25 | 26 | R, T = IterativeClosestPoint(pointcloud, template, 100, .001) 27 | 28 | -------------------------------------------------------------------------------- /pyKinectTools/scripts/MulticamViewer.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Main file for displaying depth/color/skeleton information and extracting features 3 | ''' 4 | 5 | 6 | import os 7 | import optparse 8 | import time 9 | import cPickle as pickle 10 | import numpy as np 11 | import scipy.ndimage as nd 12 | from skimage import color 13 | from skimage.segmentation import felzenszwalb 14 | import cv2 15 | from pyKinectTools.dataset_readers.KinectPlayer import KinectPlayer, display_help 16 | from pyKinectTools.utils.DepthUtils import *#world2depth, depthIm2XYZ, skel2depth, depth2world 17 | # from pyKinectTools.algs.BackgroundSubtraction import AdaptiveMixtureOfGaussians, fill_image, extract_people 18 | from pyKinectTools.algs.BackgroundSubtraction import extract_people 19 | from pyKinectTools.utils.SkeletonUtils import display_skeletons 20 | from pyKinectTools.algs.GeodesicSkeleton import * 21 | 22 | 23 | ''' Debugging ''' 24 | from IPython import embed 25 | np.seterr(all='ignore') 26 | 27 | # -------------------------MAIN------------------------------------------ 28 | 29 | def main(get_depth, get_color, get_skeleton, visualize): 30 | # fill = True 31 | fill = False 32 | get_color = True 33 | # VP = KinectPlayer(base_dir='./', device=device, get_depth=get_depth, get_color=get_color, get_skeleton=get_skeleton, get_mask=get_mask) 34 | # VP = KinectPlayer(base_dir='./', device=1, bg_subtraction=True, get_depth=get_depth, get_color=get_color, get_skeleton=get_skeleton, fill_images=fill) 35 | VP = KinectPlayer(base_dir='./', device=2, bg_subtraction=True, get_depth=True, get_color=True, get_skeleton=False, fill_images=fill) 36 | cam_count = 1 37 | if cam_count == 2: 38 | VP2 = KinectPlayer(base_dir='./', device=2, bg_subtraction=True, get_depth=True, get_color=True, get_skeleton=False, fill_images=fill) 39 | # Transformation matrix from first to second camera 40 | # data = pickle.load(open("./Registration.dat", 'r')) 41 | # transform_c1_to_c2 = data['transform'] 42 | 43 | # print 'aaaaaaaaaaaaaaaa' 44 | # embed() 45 | 46 | while VP.next(): 47 | # VP.update_background() 48 | # Transform skels from cam1 to cam2 49 | if get_skeleton: 50 | VP_skels = [np.array(VP.users[s]['jointPositions'].values()) for s in VP.users.keys()] 51 | if cam_count == 2: 52 | VP2.sync_cameras(VP) 53 | if get_skeleton: 54 | VP2_skels = transform_skels(VP_skels, transform_c1_to_c2) 55 | 56 | VP.colorIm = VP.colorIm[:,:,[2,1,0]] 57 | 58 | # im_tmp = np.dstack([VP.depthIm/float(VP.depthIm.max()),\ 59 | # VP.depthIm/float(VP.depthIm.max()),\ 60 | # VP.colorIm.mean(-1)/255]) 61 | # im = felzenszwalb(im_tmp, scale=255) 62 | # im = felzenszwalb(VP.depthIm/float(VP.depthIm.max()), scale=255) 63 | 64 | # im = VP.depthIm/float(VP.depthIm.max()) * VP. 65 | # im = felzenszwalb(im, scale=255) 66 | # cv2.imshow("seg", im/float(im.max())) 67 | 68 | # cv2.waitKey(10) 69 | # embed() 70 | 71 | # Geodesic extrema 72 | if 0: 73 | if VP.foregroundMask is not None: 74 | im = VP.depthIm * (VP.foregroundMask>0) 75 | # cv2.imshow("im--", im/float(im.max())) 76 | if 1:#(im>0).sum() > 1000: 77 | # Extract person 78 | labelMask, boxes, labels, px_counts= extract_people(im, VP.foregroundMask, minPersonPixThresh=3000) 79 | # print labels, px_counts 80 | if len(labels) > 0: 81 | max_ind = np.argmax(px_counts) 82 | mask = labelMask==max_ind+1 83 | im[-mask] = 0 84 | 85 | edge_thresh = 200#10 86 | gradients = np.gradient(im) 87 | mag = np.sqrt(gradients[0]**2+gradients[1]**2) 88 | im[mag>edge_thresh] = 0 89 | 90 | # Segmentation experiment 91 | # im_s = VP.depthIm/float(VP.depthIm.max()) 92 | # im_s = felzenszwalb(im, scale=255) 93 | # cv2.imshow("seg", im_s/float(im_s.max())) 94 | 95 | x,y = nd.center_of_mass(im) 96 | if im[x,y] == 0: 97 | tmp = np.nonzero(im>0) 98 | x,y = [tmp[0][0], tmp[1][0]] 99 | # min_map = geodesic_extrema_MPI(im, centroid=[x,y], iterations=15) 100 | extrema = geodesic_extrema_MPI(im, centroid=[x,y], iterations=15) 101 | # embed() 102 | 103 | # for e, i in zip(extrema, range(20)): 104 | # if 0 < i < 6: 105 | # box_color = [255, 0, 0] 106 | # elif i==0: 107 | # box_color = [0, 0, 255] 108 | # else: 109 | # box_color = [255,255,255] 110 | # VP.colorIm[e[0]-4:e[0]+5, e[1]-4:e[1]+5] = box_color 111 | 112 | # cv2.imshow("ExtremaB", min_map/float(min_map.max())) 113 | # cv2.waitKey(500) 114 | cv2.waitKey(20) 115 | 116 | # cv2.imshow("bg model", VP.backgroundModel/float(VP.backgroundModel.max())) 117 | # cv2.imshow("foregroundMask", ((VP.foregroundMask>0)*255).astype(np.uint8)) 118 | if visualize: 119 | if cam_count == 2: 120 | for s in VP2_skels: 121 | VP2.depthIm = display_skeletons(VP2.depthIm, s, (VP2.depthIm.max(),), skel_type='Kinect') 122 | VP2.visualize() 123 | VP2.playback_control() 124 | VP.visualize() 125 | # VP.playback_control() 126 | 127 | 128 | 129 | if __name__=="__main__": 130 | 131 | parser = optparse.OptionParser() 132 | parser.add_option('-s', '--skel', dest='skel', action="store_true", default=False, help='Enable skeleton') 133 | parser.add_option('-d', '--depth', dest='depth', action="store_true", default=False, help='Enable depth images') 134 | parser.add_option('-c', '--color', dest='color', action="store_true", default=False, help='Enable color images') 135 | parser.add_option('-m', '--mask', dest='mask', action="store_true", default=False, help='Enable enternal mask') 136 | parser.add_option('-a', '--anonomize', dest='save', action="store_true", default=False, help='Save anonomized RGB image') 137 | parser.add_option('-f', '--calcFeatures', dest='bgSubtraction', action="store_true", default=False, help='Enable feature extraction') 138 | parser.add_option('-v', '--visualize', dest='viz', action="store_true", default=False, help='Enable visualization') 139 | parser.add_option('-i', '--dev', dest='dev', type='int', default=0, help='Device number') 140 | (opt, args) = parser.parse_args() 141 | 142 | if opt.viz: 143 | display_help() 144 | 145 | if len(args) > 0: 146 | print "Wrong input argument" 147 | elif opt.depth==False and opt.color==False and opt.skel==False: 148 | print "You must supply the program with some arguments." 149 | else: 150 | main(get_depth=opt.depth, get_skeleton=opt.skel, get_color=opt.color, visualize=opt.viz) 151 | 152 | '''Profiling''' 153 | # cProfile.runctx('main()', globals(), locals(), filename="ShowSkeletons.profile") 154 | -------------------------------------------------------------------------------- /pyKinectTools/scripts/Old_Experiments/PoseByGeodesicExtrema.py: -------------------------------------------------------------------------------- 1 | import os, scipy 2 | import scipy.ndimage as nd 3 | from pyKinectTools.utils.DepthUtils import posImage2XYZ 4 | from pyKinectTools.algs.BackgroundSubtraction import extract_people, removeNoise 5 | from pyKinectTools.algs.GeodesicSkeleton import * 6 | from pyKinectTools.algs.PictorialStructures import * 7 | from pyKinectTools.algs.STIP import * 8 | 9 | # dataDir = '/Users/colin/data/ICU_7May2012_Wide_jpg/diffDraw1/' 10 | dataDir = '/Users/colin/data/ICU_7May2012_Close_jpg/diffDraw1/' 11 | # dataDir = '/Users/colin/data/ICU_7May2012_Close_jpg/d1c/' 12 | 13 | '''#################### Load Images #########################''' 14 | 15 | ''' 16 | imgs = array of many images 17 | im = specific image 18 | posMat = 3-dimensional array of XYZ positions at each pixel 19 | xyz = list of points 20 | ''' 21 | 22 | files = os.listdir(dataDir) 23 | files = [int(x[0:-4]) for x in files if x[0]!='.'] 24 | files = np.sort(files) 25 | # sequenceFrameNames = files[610:690] 26 | # sequenceFrameNames = files[2000:2101] 27 | # sequenceFrameNames = files[1200:1300] 28 | sequenceFrameNames = files[::500] 29 | # sequenceFrameNames = files[7200:7230] #:7300 30 | imgs = [] 31 | for i in sequenceFrameNames: 32 | imgs.append(scipy.misc.imread(dataDir+str(i)+'.jpg')) 33 | imgs = np.array(imgs) 34 | 35 | ''' Get posMat from individual image ''' 36 | # t=7 37 | t=13 38 | im = imgs[t] 39 | objectNum = 0 40 | # posMatFull = posImage2XYZ(im, 500, 1250) 41 | posMatFull = posImage2XYZ(im, 500, 2000) 42 | imLabels, objSlices, objInds = extract_people(posMatFull[:,:,2], 10000, True) 43 | if len(objInds)!=0: 44 | t += 1 45 | assert len(objInds)!=0, "Error: No objects" 46 | 47 | posMat = posMatFull[objSlices[objectNum]] 48 | for i in range(3): 49 | posMat[:,:,i] *= (imLabels[objSlices[objectNum]]==objInds[objectNum]) 50 | posMat = removeNoise(posMat, thresh=500) 51 | xyz = posMat[(posMat[:,:,2]>0)*(posMat[:,:,0]!=0),:] 52 | 53 | ''' Get geodesic extrema ''' 54 | t1 = time.time() 55 | regions, regionXYZ, regionLabels, edgeDict = regionGraph(posMat, pixelSize=1500) 56 | regionPos = [x[2] for x in regionLabels[1:]] 57 | regionPos.insert(0, [0,0]) 58 | regionPos = np.array(regionPos) 59 | regionCenter = regionLabels[int(len(regionLabels)/2)][2] 60 | extrema, trail, geoImg = generateKeypoints(posMat, iters=10, centroid=regionCenter, use_centroid=False) 61 | 62 | ''' Gabors ''' 63 | im = posMat[:,:,2] 64 | # grad = np.diff(im) 65 | angles = range(0, 180, 45) 66 | gabors = generateGabors(angles, [30,30], 1.0) 67 | convs = [] 68 | for i in range(len(angles)): 69 | convs.append(nd.convolve(im, gabors[:,:,i])) 70 | # convs.append(nd.convolve(grad, gabors[:,:,i])) 71 | convs = np.array(convs) 72 | mask = im>0 73 | imOut = convs.mean(0) 74 | # imOut *= mask[:,1:] 75 | imOut *= mask 76 | grad = np.gradient(np.asarray(imOut, dtype=float), 1) 77 | mag = np.sqrt(grad[0]**2+grad[1]**2) 78 | imOut -= imOut.min() 79 | imOut /= imOut.max() 80 | # gaborResponse=(1.0-imOut) 81 | gaborResponse=(imOut) 82 | gaborResponse[gaborResponse==1] = 0 83 | gaborResponse2 = np.zeros_like(gaborResponse) 84 | regionCount = regions.max() 85 | for i in range(1, regionCount+1): 86 | # gaborResponse2[regions==i] = gaborResponse[regions==i].mean() 87 | gaborResponse2[regions==i] = gaborResponse[regionPos[i][0],regionPos[i][1]] 88 | 89 | 90 | ''' Pictorial Structures ''' 91 | extrema = np.array(extrema) 92 | geoExtrema = posMat[extrema[:,0],extrema[:,1]] 93 | geoExtrema -= xyz.mean(0) 94 | skeletons, scores = pictorialScores(regionXYZ,regionPos, xyz, edgeDict, regions=regions, geoExtremaPos=extrema, geoExtrema=geoExtrema, sampleThresh=.9, gaborResponse=gaborResponse2) 95 | skeleton = skeletons[-1] 96 | print time.time() - t1 97 | 98 | ''' Display ''' 99 | figure(2); imshow(geoImg) 100 | imLab = labelSkeleton(skeleton, regionLabels, posMat[:,:,2]) 101 | figure(3); imshow(imLab) 102 | # labelGraphImage(regionLabels) 103 | t+=1 104 | 105 | 106 | 107 | # ''' Gabors ''' 108 | # # im = np.diff(posMat[:,:,2]) 109 | # im = posMat[:,:,2] 110 | # angles = range(0, 180, 45/2) 111 | # # gabors = generateGabors(angles, [20,20], 1) 112 | # gabors = generateGabors(angles, [20,20], 1.0) 113 | # # gabors = generateGabors(angles, [5,5], .75) 114 | # convs = [] 115 | # for i in range(len(angles)): 116 | # convs.append(nd.convolve(im, gabors[:,:,i])) 117 | # convs = np.array(convs) 118 | # imOut = convs.max(0)#*mask 119 | # grad = np.gradient(np.asarray(imOut, dtype=float), 1) 120 | # mag = np.sqrt(grad[0]**2+grad[1]**2) 121 | # imOut -= imOut.min() 122 | # imOut /= imOut.max() 123 | -------------------------------------------------------------------------------- /pyKinectTools/scripts/Old_Experiments/PoseByManifolds.py: -------------------------------------------------------------------------------- 1 | 2 | from pyKinectTools.algs.manifolds import * 3 | 4 | 5 | # xyz = featureExt1.xyz[ind] 6 | xyzInds = np.nonzero(posMat[:,:,2]!=0) 7 | xyz = posMat[xyzInds[0], xyzInds[1]] 8 | xyz -= xyz.mean(0) 9 | 10 | ptInterval=1 11 | data = xyz[::ptInterval,:] 12 | # data = xyz[::ptInterval,:][100::] 13 | posVecs = LaplacianEigenmaps(data) 14 | 15 | x= posVecs[:,0]; y= posVecs[:,1]; z= posVecs[:,2] 16 | x2= posVecs[:,4]; y2= posVecs[:,5]; z2= posVecs[:,5] 17 | 18 | 19 | ''' Color segments ''' 20 | maxDim = 4 21 | colorAxis = np.zeros_like(y)+maxDim 22 | for i in range(1,maxDim): 23 | colorAxis[posVecs[:,i]>0] = np.minimum(colorAxis[posVecs[:,i]>0], i) 24 | colorAxis *= 255.0/colorAxis.max() 25 | 26 | 27 | 28 | ''' Visualize ''' 29 | # Viz components 1-3 30 | if 0: 31 | fig = figure(9) 32 | ax = fig.add_subplot(111,projection='3d') 33 | ax.scatter3D(x, y, zs=z, c=colorAxis) 34 | xlabel('X') 35 | ylabel('Y') 36 | # axis('equal') 37 | 38 | # Viz components 4-6 39 | if 0: 40 | fig = figure(10) 41 | ax = fig.add_subplot(111,projection='3d') 42 | ax.scatter3D(x2, y2, zs=z2, c=colorAxis) 43 | xlabel('X') 44 | ylabel('Y') 45 | 46 | # Viz colors on original structure 47 | if 0: 48 | fig = figure(12) 49 | ax = fig.add_subplot(111,projection='3d') 50 | ax.scatter3D(data[:,0], data[:,1], zs=data[:,2], c=colorAxis) 51 | xlabel('X') 52 | ylabel('Y') 53 | axis('equal') 54 | draw() 55 | 56 | # Viz components on single axis 57 | if 1: 58 | fig = figure(13) 59 | cla() 60 | for i in range(1,4): 61 | plot(posVecs[:,i]) 62 | 63 | # Paint original image 64 | if 1: 65 | figure(14) 66 | colorMat = np.zeros_like(posMat) 67 | inds = xyzInds 68 | colorMat[inds[0][::ptInterval][::],inds[1][::ptInterval][::],0] = colorAxis 69 | imshow(colorMat[:,:,0]) 70 | 71 | -------------------------------------------------------------------------------- /pyKinectTools/scripts/Old_Experiments/patientMovement.py: -------------------------------------------------------------------------------- 1 | 2 | # Patient movement 3 | 4 | import cv, cv2 5 | import scipy.ndimage as nd 6 | from mpl_toolkits.mplot3d import axes3d, Axes3D 7 | from scipy import interpolate 8 | 9 | ''' #------------ Spline-based ---------------# ''' 10 | 11 | for fi in range(0,190, 5): 12 | # fi=114 13 | fi += 5 14 | im = imgs[fi] 15 | mask = (im<90)*(im>10) 16 | mask = nd.binary_opening(mask) 17 | im *= mask 18 | 19 | 20 | posMat = posImage2XYZ(im, 500, 2000) 21 | for i in range(3): 22 | posMat[:,:,i] *= mask 23 | inds = np.nonzero(mask) 24 | xyz = np.vstack([inds[0], inds[1], posMat[inds[0],inds[1],2]]).T 25 | 26 | xyzMean = xyz.mean(0) 27 | xyz -= xyzMean 28 | # Get top and bottom 29 | U,_,vT = np.linalg.svd(xyz, full_matrices=False) 30 | tmp = np.asmatrix(vT)*np.asmatrix(xyz.T) 31 | tmp = np.asarray(tmp.T) 32 | 33 | nodeTail = posMat[280, 55] 34 | nodeHead = posMat[360,460]#[235, 625] 35 | # nodeTail = np.asarray(np.asmatrix(vT)*np.asmatrix(nodeTail).T).T[0] 36 | # nodeHead = np.asarray(np.asmatrix(vT)*np.asmatrix(nodeHead).T).T[0] 37 | 38 | n_est=5 39 | # tckp,u = interpolate.splprep(xyz[::100,:].T, s=3,k=1,nest=n_est, ue=nodeTail,ub=nodeHead) 40 | tckp,u = interpolate.splprep(xyz[::100,:].T, s=3,k=1,nest=n_est) 41 | xn,yn,zn=interpolate.splev(linspace(0,1,100),tckp) 42 | figure(3) 43 | subplot(2,2,1); axis('equal'); cla(); 44 | plot(xn,yn, c=[fi/200.0, 0, 0]) 45 | subplot(2,2,2); axis('equal'); cla(); 46 | plot(xn,zn, c=[fi/200.0, 0, 0]) 47 | subplot(2,2,3); axis('equal'); cla(); 48 | plot(yn,zn, c=[fi/200.0, 0, 0]) 49 | 50 | figure(2) 51 | imshow(im) 52 | 53 | # spl = interpolate.splrep(xyz[:,0], xyz[:,1], xb=nodeHead[:2], xe=nodeTail[:2], s=0) 54 | 55 | fig = figure(1); 56 | ax = Axes3D(fig) 57 | xlabel('X'); ylabel('Y'); axis('equal') 58 | ax.plot(xn,yn,zn, c='g') 59 | # ax.scatter(tmp[::100,0],tmp[::100,1],tmp[::100,2]) 60 | 61 | ax.scatter(xyz[::50,0],xyz[::50,1],xyz[::50,2]) 62 | 63 | figure(2) 64 | imshow(posMat[:,:,2]) 65 | 66 | # calculate 3D flow 67 | 68 | # cv2.calcOpticalFlowPyrLK(prevImg, nextImg, prevPts[, nextPts[, status[, err[, winSize[, maxLevel[, criteria[, flags[, minEigThreshold]]]]]]]]) → nextPts, status, err 69 | 70 | 71 | 72 | ''' #------------ FLOW ---------------# ''' 73 | 74 | cv2.namedWindow("flow") 75 | cv2.namedWindow("im") 76 | 77 | totalMag = np.zeros_like(imgs[0], dtype=float) 78 | indsStart = np.nonzero(totalMag==0) 79 | magSums = [] 80 | for i in range(5,len(imgs), 1): 81 | # for i in range(150,len(imgs)): 82 | i2 = i-1 83 | im1 = imgs[i] 84 | mask = (im1<150)*(im1>10) 85 | mask = nd.binary_opening(mask) 86 | # im1 *= mask 87 | 88 | im2 = imgs[i2] 89 | mask = (im2<150)*(im2>10) 90 | mask = nd.binary_opening(mask) 91 | # im2 *= mask 92 | 93 | # im2 *= (im2 < 90) * (im2 > 0) 94 | # poly_n is the pixel neighborhood. 5 or 7 suggested. for 5, use poly_sigma=1.1, for 7 use ps=1.5 95 | flow = cv2.calcOpticalFlowFarneback(im1, im2, pyr_scale=.5, levels=3, winsize=9, iterations=3, poly_n=5, poly_sigma=1.1, flags=cv2.OPTFLOW_FARNEBACK_GAUSSIAN) 96 | mag = np.sqrt(flow[:,:,0]**2+flow[:,:,1]**2) 97 | mag *= (mag>10) 98 | 99 | # indsEnd_x = flow[:,:,0].flatten() 100 | # indsEnd_y = flow[:,:,1].flatten() 101 | 102 | totalMag += mag 103 | magSums.append(mag.sum()) 104 | if magSums[-1] > 200000: 105 | print magSums[-1] 106 | # im = totalMag 107 | im = mag 108 | cv2.imshow("im", im1) 109 | cv2.imshow("flow", im/im.max()) 110 | ret = cv2.waitKey(30) 111 | if ret >= 0: 112 | break 113 | 114 | print totalMag.sum() 115 | 116 | imshow(totalMag) 117 | 118 | 119 | ''' #------------ gabor ---------------# ''' 120 | from pyKinectTools.algs.STIP import generateGabors 121 | 122 | im = im1 123 | mask = (im<90)*(im>10) 124 | angles = range(0, 180, 45) 125 | # gabors = generateGabors(angles, [20,20], 1) 126 | gabors = generateGabors(angles, [30,30], 1) 127 | convs = [] 128 | for i in range(len(angles)): 129 | convs.append(nd.convolve(im, gabors[:,:,i])) 130 | convs = np.array(convs) 131 | imOut = convs.max(0)#*mask 132 | grad = np.gradient(np.asarray(imOut, dtype=float), 1) 133 | mag = np.sqrt(grad[0]**2+grad[1]**2) 134 | mag *= mask 135 | 136 | mask *= (mag<.05*mag.max()) 137 | mask = nd.binary_opening(mask, iterations=2) 138 | imshow(im*mask) 139 | # imshow(imOut) 140 | 141 | 142 | -------------------------------------------------------------------------------- /pyKinectTools/scripts/Old_Experiments/pcaGesturesMain.py: -------------------------------------------------------------------------------- 1 | from pylab import * 2 | import numpy as np 3 | import pdb 4 | import time, sys,os 5 | # sys.path.append(os.getcwd() + "/pcaGestures") 6 | import pyKinectTools.utils.skeletonUtils as SkelModule 7 | from pyKinectTools.algs.pcaGestures import gestureRecognizer as GestModule 8 | 9 | # Set gestures 10 | wave = range(0,5) 11 | circle_counter = range(5,10) 12 | circle_clock = range(10,15) 13 | push_forward = range(15,20) 14 | push_left = range(20,25) 15 | push_right = range(25,30) 16 | swoosh_right = range(30,35) 17 | reach_up = range(35,40) 18 | duck = range(40,45) 19 | kick = range(45,50) 20 | bow = range(50,55) 21 | 22 | GESTURES = [wave, 23 | circle_counter, 24 | circle_clock, 25 | push_forward, 26 | push_left, 27 | push_right, 28 | swoosh_right, 29 | reach_up, 30 | duck, 31 | kick, 32 | bow] 33 | 34 | #Jed Sequences 35 | SEQUENCE_TRUTH = [ 36 | [3,6,5], 37 | [0,1,2], 38 | [3,4,5], 39 | [6,7,8], 40 | [9,0,3], 41 | [1,4,6], 42 | [2,5,7], 43 | [8,6,2], 44 | [9,3,0], 45 | [4,8,1]] 46 | 47 | 48 | if __name__ == "__main__": 49 | 50 | SKELETON = SkelModule.SKELETON 51 | print "Start" 52 | if len(sys.argv) > 1: 53 | skelFolder = sys.argv[1] 54 | else: 55 | skelFolder = "" 56 | 57 | skelFolder = "/Users/colin/data/Gestures/jed_11Gest" 58 | skelSetsTrain = SkelModule.getAllSkeletons(skelFolder, normalizeOrientation=0) 59 | # skelSetsTrain = SkelModule.getAllSkeletons("/Users/colin/data/Gestures/11Gestures_9Feb2012", normalizeOrientation=0) 60 | # skelSetsTest = SkelModule.getAllSkeletons(skelFolder, normalizeOrientation=0) 61 | skelSetsTest = SkelModule.getAllSkeletons("/Users/colin/data/Gestures/11Gestures_9Feb2012", normalizeOrientation=0) 62 | 63 | # Train on 1, test on 4 64 | if 0: 65 | ml_accuracy = [] 66 | for s in range(5): 67 | trainingSetInds = [] 68 | for i in range(10): 69 | trainingSetInds.append([GESTURES[i][s]]) 70 | 71 | g = GestModule.pcaGestureRecognizer(SKELETON, GESTURES) 72 | g.train(skelSetsTrain, trainingSetInds, 'jointsCentered') 73 | tmp = [] 74 | for d in trainingSetInds: tmp.append(d[0]) 75 | testData = set(range(50)).difference(set(tmp)) 76 | for i in testData: 77 | g.test(skelSetsTest[i], i) 78 | g.calcAccuracy(np.array(list(testData))/5) 79 | ml_accuracy.append(g.avgAccuracy) 80 | 81 | print "----------------" 82 | print "Overall accuracy", np.array(ml_accuracy).mean() 83 | print ml_accuracy 84 | 85 | g.calcClassAccuracy() 86 | g.calcClassConfusion() 87 | g.calcJointAccuracy() 88 | g.calcJointConfusion() 89 | figure(1) 90 | subplot(2,1,2) 91 | imshow(g.accuracyImg, interpolation="nearest") 92 | title("Gesture/Joint Accuracy") 93 | 94 | pdb.set_trace() 95 | 96 | 97 | 98 | # Train on 4, test on 1 99 | if 1: 100 | ml_accuracy = [] 101 | for s in range(5): 102 | trainingSetInds = [] 103 | for i in range(10): 104 | tmpA = [] 105 | for j in range(5): 106 | if j != s: 107 | tmpA.append(GESTURES[i][j]) 108 | trainingSetInds.append(tmpA) 109 | 110 | g = GestModule.pcaGestureRecognizer(SKELETON, GESTURES) 111 | g.train(skelSetsTrain, trainingSetInds, 'jointsCentered') 112 | tmp = [] 113 | for d in trainingSetInds: tmp.append(d[0]) 114 | testData = set(range(50)).difference(set(tmp)) 115 | for i in testData: 116 | g.test(skelSetsTest[i], i) 117 | g.calcAccuracy(np.array(list(testData))/5) 118 | ml_accuracy.append(g.avgAccuracy) 119 | 120 | print "----------------" 121 | print "Overall accuracy", np.array(ml_accuracy).mean() 122 | print ml_accuracy 123 | g.calcClassAccuracy() 124 | g.calcClassConfusion() 125 | g.calcJointAccuracy() 126 | g.calcJointConfusion() 127 | pdb.set_trace() 128 | 129 | # Train on 4, test on 1. Vary time. 130 | # Accuracy appears to stay very close to 'whole' dataset 131 | if 0: 132 | import copy 133 | skelSetShort = copy.deepcopy(skelSetsTrain) 134 | for i in range(10): 135 | half_len = int(skelSetsTrain[i][0]['jointsRel'].shape[0]/10) 136 | skelSetShort[i][0]['jointsRel'] = copy.deepcopy(skelSetsTrain[i][0]['jointsRel'][20:half_len+20]) 137 | # skelSetShort[i][0]['jointsRel'] = copy.deepcopy(skelSetsTrain[i][0]['jointsRel'][0:half_len]) 138 | ml_accuracy = [] 139 | for s in range(5): 140 | trainingSetInds = [] 141 | for i in range(10): 142 | tmpA = [] 143 | for j in range(5): 144 | if j != s: 145 | tmpA.append(GESTURES[i][j]) 146 | trainingSetInds.append(tmpA) 147 | 148 | g = GestModule.pcaGestureRecognizer(SKELETON, GESTURES) 149 | g.train(skelSetsTrain, trainingSetInds) 150 | tmp = [] 151 | for d in trainingSetInds: tmp.append(d[0]) 152 | testData = set(range(50)).difference(set(tmp)) 153 | for i in testData: 154 | g.test(skelSetShort[i], i) 155 | g.calcAccuracy(np.array(list(testData))/5) 156 | ml_accuracy.append(g.avgAccuracy) 157 | 158 | print "----------------" 159 | print "Overall accuracy", np.array(ml_accuracy).mean() 160 | print ml_accuracy 161 | 162 | g.calcClassAccuracy() 163 | g.calcClassConfusion() 164 | g.calcJointAccuracy() 165 | g.calcJointConfusion() 166 | pdb.set_trace() 167 | 168 | 169 | # Train on all 5. Test on sequences 170 | if 0: 171 | import copy 172 | skelSetsSeq = SkelModule.getAllSkeletons("/Users/colin/data/Gestures/jed_sequences", 10) 173 | skelSetShort = copy.deepcopy(skelSetsSeq) 174 | 175 | #Test 176 | trainingSetInds = [] 177 | for i in range(10): 178 | tmpA = [] 179 | for j in range(5): 180 | tmpA.append(GESTURES[i][j]) 181 | trainingSetInds.append(tmpA) 182 | g = GestModule.pcaGestureRecognizer(SKELETON, GESTURES) 183 | g.train(skelSetsTrain, trainingSetInds) 184 | 185 | #Test 186 | gestCount = 6 187 | seqsLabel = [] 188 | for i in range(len(SEQUENCE_TRUTH)): # For each gesture 189 | seqLen = int(skelSetsSeq[i][0]['jointsRel'].shape[0]) 190 | gestLen = int(seqLen/gestCount) 191 | seqLabel = [] 192 | for gestNum in range(gestCount): # for each part of the gesture 193 | skelSetShort[i][0]['jointsRel'] = copy.deepcopy(skelSetsSeq[i][0]['jointsRel'][gestNum*gestLen : (gestNum+1)*gestLen]) 194 | seqLabel.append(g.test(skelSetShort[i])) 195 | seqsLabel.append(seqLabel) 196 | print "Test: ", seqsLabel 197 | print "Truth: ", SEQUENCE_TRUTH 198 | # print np.equal(seqsLabel, SEQUENCE_TRUTH)*1 199 | 200 | # ml_accuracy = [] 201 | # tmp = [] 202 | # for d in trainingSetInds: tmp.append(d[0]) 203 | # testData = set(range(50)).difference(set(tmp)) 204 | # for i in testData: 205 | # g.test(skelSetShort[i], i) 206 | # g.calcAccuracy(np.array(list(testData))/5) 207 | # ml_accuracy.append(g.avgAccuracy) 208 | 209 | # print "----------------" 210 | # print "Overall accuracy", np.array(ml_accuracy).mean() 211 | # print ml_accuracy 212 | 213 | -------------------------------------------------------------------------------- /pyKinectTools/scripts/STIPScript.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import scipy.misc 3 | import os, sys 4 | import scipy.ndimage as nd 5 | from pyKinectTools.algs import BackgroundSubtraction 6 | from pyKinectTools.algs.STIP import * 7 | import time 8 | 9 | 10 | angles = range(0, 180, 45) 11 | # angles = range(0, 108*2, 90/5) 12 | gabors = generateGabors(angles, [20,20], 1) 13 | 14 | 15 | ''' --- Clouds of STIP --- ''' 16 | 17 | i_time = 100 18 | 19 | O_ratio = [] 20 | O_speed = [] 21 | C_interests = [] 22 | 23 | # for i_time in range(100,200,10): 24 | for i_time in range(0,len(imgs),5): 25 | ''' Object features ''' 26 | tSpan = 5 27 | #T_new 28 | labeledImg, slices, sliceIndices = extract_people(imgs[i_time]) 29 | labeledImg2, slices2, sliceIndices2 = extract_people(imgs[i_time-tSpan]) 30 | t1 = time.time() 31 | S_interests = [] 32 | if len(sliceIndices)>0 and len(sliceIndices2)>0: 33 | O_new_height = slices[0][0].stop-slices[0][0].start 34 | O_new_width = slices[0][1].stop-slices[0][1].start 35 | O_new_ratio = 1.0*O_new_height/O_new_width 36 | O_new_pos = np.array([slices[0][0].start+O_new_height/2, slices[0][1].start+O_new_width/2, imgs[i_time][slices2[0]].mean()]) 37 | #T_prev 38 | height = slices2[0][0].stop-slices2[0][0].start 39 | width = slices2[0][1].stop-slices2[0][1].start 40 | O_old_ratio = 1.0*height/width 41 | O_old_pos = np.array([slices2[0][0].start+height/2, slices2[0][1].start+width/2, imgs[i_time-tSpan][slices2[0]].mean()]) 42 | 43 | O_ratio.append((O_old_ratio+O_new_ratio)/2) 44 | O_speed.append(np.sqrt(np.sum((O_new_pos-O_old_pos)**2))/tSpan) 45 | 46 | 47 | ''' Temporal features ''' 48 | for t in range(0,20,8): 49 | tSpan = t 50 | imTmp = imgs[i_time]-imgs[i_time-tSpan] 51 | imTmp = imTmp*(imTmp<200)#*(imTmp>100) 52 | imTmp = nd.binary_erosion(imTmp, iterations=5)*(imTmp) 53 | imTmp = np.asarray(imTmp, dtype=float) 54 | convs = [] 55 | for i in range(len(angles)): 56 | convs.append(nd.convolve(imTmp, gabors[:,:,i])) 57 | convs = np.array(convs) 58 | fused = convs.max(0)*(imTmp>0) 59 | 60 | interests = np.nonzero(fused>0) 61 | # interests = np.nonzero(fused>.5*fused.max()) 62 | # interests = localMaximums(fused) 63 | pts = np.array(interests).T 64 | vals = fused[pts[:,0],pts[:,1]] 65 | interests = adaptiveNonMaximalSuppression(pts, vals) 66 | # fused2 = (deepcopy(fused)>0)*1.0 67 | # fused2[interests[:,0], interests[:,1]] = fused2.max()*10 68 | if len(interests) > 0: 69 | s_height = interests[0].max() - interests[0].min() 70 | s_width = interests[1].max() - interests[1].min() 71 | 72 | s_ratio = float(s_height)/s_width 73 | s_pos = [interests[0].mean(), interests[1].mean()]#, imgs[i_time][interests[0].mean(), interests[1].mean()]] 74 | s_density = float(len(interests[0])) / (s_height*s_width) 75 | 76 | # find speed not pos! 77 | # overlap = O_new_pos[0]+O_new_height/2 - 78 | # overlap = np.sum(fused>0 == imgs[i_time-tSpan]>0) 79 | 80 | S_interests.append([s_height, s_width, s_pos[0],s_pos[1], s_density, s_pos[0]-O_new_pos[0], s_pos[1]-O_new_pos[1], float(s_height)/O_new_height, float(s_width)/O_new_width]) 81 | 82 | print "Next frame:", time.time()-t1 83 | C_interests.append(S_interests) 84 | 85 | imshow(fused>200) 86 | 87 | 88 | if 0: 89 | ### 90 | # angles = range(0, 108, 90/5) 91 | ii=10 92 | 93 | imTmp = imgs[ii][70::,0:300] 94 | # imTmp[imTmp>0].min() 95 | mask = nd.binary_erosion(imTmp>0, iterations=5) 96 | angles = range(0, 180, 45) 97 | gabors = generateGabors(angles, [20,20], 1) 98 | convs = [] 99 | for i in range(len(angles)): 100 | convs.append(nd.convolve(imTmp, gabors[:,:,i])) 101 | convs = np.array(convs) 102 | imshow(convs.max(0)*mask) 103 | ii+=5 104 | 105 | 106 | 107 | 108 | -------------------------------------------------------------------------------- /pyKinectTools/scripts/TrainRFPose.py: -------------------------------------------------------------------------------- 1 | """ 2 | Main file for training multi-camera pose 3 | """ 4 | 5 | import os 6 | import optparse 7 | import time 8 | import cPickle as pickle 9 | import numpy as np 10 | from skimage import color 11 | 12 | from pyKinectTools.dataset_readers.KinectPlayer import KinectPlayer, display_help 13 | from pyKinectTools.utils.DepthUtils import world2depth, depthIm2XYZ, skel2depth, depth2world 14 | from pyKinectTools.utils.SkeletonUtils import display_skeletons, transform_skels, kinect_to_msr_skel 15 | # from pyKinectTools.algs.GeodesicSkeleton import generateKeypoints, distance_map 16 | from pyKinectTools.algs.RandomForestPose import RFPose 17 | 18 | """ Debugging """ 19 | from IPython import embed 20 | np.seterr(all='ignore') 21 | 22 | # -------------------------MAIN------------------------------------------ 23 | 24 | def main(visualize=False, learn=False): 25 | # Init both cameras 26 | # fill = True 27 | fill = False 28 | get_color = True 29 | cam = KinectPlayer(base_dir='./', device=1, bg_subtraction=True, get_depth=True, get_color=get_color, get_skeleton=True, fill_images=fill) 30 | cam2 = KinectPlayer(base_dir='./', device=2, bg_subtraction=True, get_depth=True, get_color=get_color, get_skeleton=True, fill_images=fill) 31 | # Transformation matrix from first to second camera 32 | data = pickle.load(open("Registration.dat", 'r')) 33 | transform_c1_to_c2 = data['transform'] 34 | 35 | # Get random forest parameters 36 | if learn: 37 | rf = RFPose(offset_max=100, feature_count=300) 38 | else: 39 | rf = RFPose() 40 | rf.load_forest() 41 | 42 | ii = 0 43 | # cam.next(200) 44 | all_joint_ims_z = [] 45 | all_joint_ims_c = [] 46 | while cam.next() and ii < 200: 47 | # Update frames 48 | cam2.sync_cameras(cam) 49 | if ii%10 != 0: 50 | ii += 1 51 | continue 52 | 53 | # Transform skels from cam1 to cam2 54 | cam_skels = [np.array(cam.users[s]['jointPositions'].values()) for s in cam.users.keys()] 55 | # Get rid of bad skels 56 | cam_skels = [s for s in cam_skels if np.all(s[0] != -1)] 57 | 58 | if len(cam_skels) == 0: 59 | continue 60 | ii+=1 61 | 62 | # Save images 63 | if 1: 64 | joint_ims_z = [] 65 | joint_ims_c = [] 66 | dx = 10 67 | skel_tmp = skel2depth(cam_skels[0], [240,320]) 68 | for j_pos in skel_tmp: 69 | # embed() 70 | joint_ims_z += [cam.depthIm[j_pos[0]-dx:j_pos[0]+dx, j_pos[1]-dx:j_pos[1]+dx]] 71 | joint_ims_c += [cam.colorIm[j_pos[0]-dx:j_pos[0]+dx, j_pos[1]-dx:j_pos[1]+dx]] 72 | if len(joint_ims_z) > 0: 73 | all_joint_ims_z += [joint_ims_z] 74 | all_joint_ims_c += [joint_ims_c] 75 | 76 | if 0: 77 | cam2_skels = transform_skels(cam_skels, transform_c1_to_c2, 'image') 78 | 79 | try: 80 | depth = cam2.get_person() 81 | if learn: 82 | rf.add_frame(depth, cam2_skels[0]) 83 | else: 84 | rf.infer_pose(depth) 85 | 86 | if visualize: 87 | cam2.depthIm = display_skeletons(cam2.depthIm, cam2_skels[0], (5000,), skel_type='Low') 88 | skel1 = kinect_to_msr_skel(skel2depth(cam_skels[0], [240,320])) 89 | cam.depthIm = display_skeletons(cam.depthIm, skel1, (5000,), skel_type='Low') 90 | cam.visualize() 91 | cam2.visualize() 92 | except: 93 | pass 94 | 95 | 96 | embed() 97 | if learn: 98 | print "Starting forest" 99 | rf.learn_forest() 100 | 101 | print 'Done' 102 | 103 | if __name__=="__main__": 104 | 105 | parser = optparse.OptionParser() 106 | parser.add_option('-v', '--visualize', dest='viz', action="store_true", default=False, help='Enable visualization') 107 | parser.add_option('-l', '--learn', dest='learn', action="store_true", default=False, help='Training phase') 108 | parser.add_option('-i', '--infer', dest='infer', action="store_true", default=False, help='Training phase') 109 | (opt, args) = parser.parse_args() 110 | 111 | if len(args) > 0: 112 | print "Wrong input argument" 113 | elif opt.infer==False and opt.learn==False: 114 | print "You must supply the program with either -l (learn parameters) or -i (infer pose)." 115 | else: 116 | if opt.infer: 117 | main(visualize=opt.viz, learn=False) 118 | else: 119 | main(visualize=opt.viz, learn=True) 120 | 121 | # '''Profiling''' 122 | # cProfile.runctx('main()', globals(), locals(), filename="ShowSkeletons.profile") 123 | -------------------------------------------------------------------------------- /pyKinectTools/scripts/Viewer.py: -------------------------------------------------------------------------------- 1 | """ 2 | Main file for viewing data 3 | """ 4 | 5 | import os 6 | import optparse 7 | import time 8 | import cPickle as pickle 9 | import numpy as np 10 | from skimage import color 11 | import scipy.misc as sm 12 | import skimage.draw as draw 13 | import cv, cv2 14 | from pylab import * 15 | 16 | from pyKinectTools.dataset_readers.KinectPlayer import KinectPlayer, display_help 17 | from pyKinectTools.dataset_readers.MHADPlayer import MHADPlayer 18 | 19 | from pyKinectTools.utils.SkeletonUtils import display_skeletons, transform_skels, kinect_to_msr_skel, msr_to_kinect_skel 20 | 21 | """ Debugging """ 22 | from IPython import embed 23 | 24 | 25 | if 0: 26 | ## ICU anonomization test 27 | im = cam.colorIm.copy() 28 | # im[::2,::2,0] = 255-im[::2,::2,0] 29 | # im[::2,::2,1] = 128-im[::2,::2,1] 30 | # im[::2,::2,2] = im[::2,::2,2]-64 31 | tmp = np.random.randint(0, 255, cam.colorIm.shape).astype(np.uint8) 32 | im_in = cam.colorIm[:,:,[2,1,0]] 33 | im_tmp = im_in+tmp 34 | im_out = im_tmp-tmp 35 | 36 | subplot(1,4,1) 37 | imshow(im_in) 38 | subplot(1,4,2) 39 | subplot(1,4,1) 40 | title('Original') 41 | subplot(1,4,2) 42 | title('Template') 43 | imshow(tmp) 44 | subplot(1,4,3) 45 | imshow(im_tmp) 46 | title('Result (Anonomized)') 47 | subplot(1,4,4) 48 | title('De-anonomized') 49 | imshow(im_out) 50 | show() 51 | 52 | 53 | # -------------------------MAIN------------------------------------------ 54 | 55 | def main(anonomization=False): 56 | # Setup kinect data player 57 | # cam = KinectPlayer(base_dir='./', bg_subtraction=False, get_depth=True, get_color=True, get_skeleton=False) 58 | # actions = [2, 5, 7] 59 | actions = [2] 60 | actions_labels = ['JumpingJacks', 'Waving', 'Clapping'] 61 | action = 'JumpingJacks' 62 | subjects = range(1,10) 63 | cam = MHADPlayer(base_dir='/Users/colin/Data/BerkeleyMHAD/', kinect=1, actions=actions, subjects=subjects, reps=[1], get_depth=True, get_color=True, get_skeleton=True, fill_images=False) 64 | if anonomization: 65 | ''' bg_type can be: 66 | 'box'[param=max_depth] 67 | 'static'[param=background] 68 | 'mean' 69 | 'median' 70 | 'adaptive_mog' 71 | 72 | See BasePlayer for more details 73 | ''' 74 | cam.set_bg_model(bg_type='box', param=2600) 75 | 76 | # Test HOG pedestrian detector 77 | # cv2.HOGDescriptor_getDefaultPeopleDetector() 78 | # imshow(color[people]) 79 | 80 | 81 | body_data = [[]] 82 | framerate = 1 83 | prev_n = len(cam.kinect_folder_names) 84 | action_idx = 0 85 | # embed() 86 | while cam.next(framerate): 87 | if len(cam.kinect_folder_names) != prev_n: 88 | # action_idx += 1 89 | body_data += [[]] 90 | prev_n = len(cam.kinect_folder_names) 91 | body_data[-1] += [cam.users_uv[0]] 92 | 93 | continue 94 | 95 | 96 | 97 | if 0: 98 | people_all = list(cv.HOGDetectMultiScale(cv.fromarray(cam.colorIm.mean(-1).astype(np.uint8)), cv.CreateMemStorage(0), hit_threshold=-1.5)) 99 | print len(people_all), " people detected" 100 | for i,people in enumerate(people_all): 101 | # embed() 102 | try: 103 | print people 104 | # tmp = cam.colorIm[people[0][0]:people[0][0]+people[1][0], people[0][1]:people[0][1]+people[1][1]] 105 | # tmp = cam.colorIm[people[0][1]:people[0][1]+people[1][1], people[0][0]:people[0][0]+people[1][0]] 106 | cam.colorIm[people[0][1]:people[0][1]+people[1][1], people[0][0]:people[0][0]+people[1][0]] += 50 107 | # cv2.imshow("Body2 "+str(i), tmp) 108 | # subplot(4, len(people_all)/4, i + 1) 109 | # imshow(tmp) 110 | 111 | # tmp = cam.colorIm[people[0][0]:people[0][0]+people[0][1], people[1][0]:people[1][0]+people[1][1]] 112 | # cv2.imshow("Body1 "+str(i), tmp) 113 | # tmp = cam.colorIm[people[1][0]:people[1][0]+people[1][1], people[0][0]:people[0][0]+people[0][1]] 114 | # print tmp 115 | # cv2.imshow("Body "+str(i), tmp) 116 | except: 117 | pass 118 | # show() 119 | 120 | if anonomization and cam.mask is not None: 121 | 122 | mask = (sm.imresize(cam.mask, [480,640]) > 0).copy() 123 | mask[40:170, 80:140] = True # person in background 124 | px = draw.circle(145,285, 40) 125 | mask[px] = True 126 | 127 | # Version 1 - Blur + Circle 128 | color = cam.colorIm.copy() 129 | color[mask] = cv2.GaussianBlur(color, (29,29), 50)[mask] 130 | 131 | subplot(1,3,1) 132 | imshow(color) 133 | 134 | # Version 2 - Noise on mask 135 | color = cam.colorIm.copy() 136 | noise_key = np.random.randint(0, 255, cam.colorIm.shape).astype(np.uint8) 137 | color[mask] = color[mask]+noise_key[mask] 138 | 139 | subplot(1,3,2) 140 | imshow(color) 141 | 142 | # Version 3 - All noise 143 | color = cam.colorIm.copy() 144 | color = color+noise_key 145 | 146 | subplot(1,3,3) 147 | imshow(color) 148 | 149 | show() 150 | 151 | 152 | # cam.colorIm = cam.colorIm[:,:,[2,1,0]] 153 | # cam.colorIm *= mask[:,:,None] 154 | # cam.visualize(color=True, depth=True, text=True, colorize=True, depth_bounds=[0,4000]) 155 | # cam.visualize() 156 | cam.colorIm = display_skeletons(cam.colorIm, cam.users_uv[0], skel_type='Kinect', color=(0,255,0)) 157 | cam.visualize() 158 | 159 | 160 | body_data[actions_labels[action_idx]] += [cam.users_uv[0]] 161 | # cam.visualize(color=True, depth=True, text=False, colorize=False, depth_bounds=None) 162 | 163 | print 'Done' 164 | 165 | # Pause at the end 166 | 167 | # embed() 168 | import scipy.io as si 169 | si.matlab.savemat(action+".mat", {'data':body_data}) 170 | 171 | 172 | if __name__=="__main__": 173 | 174 | parser = optparse.OptionParser() 175 | parser.add_option('-a', '--anon', dest='anon', action="store_true", default=False, help='Enable anonomization') 176 | (opt, args) = parser.parse_args() 177 | 178 | main(opt.anon) 179 | 180 | -------------------------------------------------------------------------------- /pyKinectTools/scripts/chalearnExperiments.py: -------------------------------------------------------------------------------- 1 | 2 | import scipy.io as io 3 | import cv2 4 | from copy import deepcopy 5 | 6 | ''' Load Data ''' 7 | import pyKinectTools.data as ktData 8 | dataFolder = ktData.__file__ 9 | dataFolder = dataFolder[:dataFolder.find("__init__")] 10 | dep = io.loadmat(dataFolder+'bodyPart_DepthImgs.mat') 11 | feat = io.loadmat(dataFolder+'bodyPart_HOGFeatures.mat') 12 | 13 | 14 | '''Play with image''' 15 | # im = dep['l_hand'][0][0] 16 | ## Sphere test 17 | figure(1) 18 | interval = 10 19 | tIms = [] 20 | for s in range(10,70, 10): 21 | tSize = [s,s] 22 | template = np.zeros(tSize) 23 | cv2.circle(template, (s/2,s/2), 0, 1, s) 24 | # cv2.ellipse(template, center=(s/2,s/2), axes=(s/2,s/4), angle=0, startAngle=0, endAngle=360, color=1, thickness=s/4) 25 | im2 = np.array(deepcopy(im), dtype=np.float) 26 | im2 = np.maximum(im2[1:,1:] - im2[:-1,1:],im2[1:,1:] - im2[1:,:-1]) 27 | # im2[np.abs(im2) > 20] = 0 28 | im2[im2!=0] -= im2.min() 29 | im2 /= im2.max() 30 | # im2[im2!=0] = np.log(im2[im2!=0]) 31 | 32 | # imOut = np.zeros([240,320]) 33 | imOut = np.zeros([(240-tSize[0])/interval, (320-tSize[1])/interval]) 34 | # imOut = np.zeros([(240-100)/interval, (320-100)/interval]) 35 | for i,ii in zip(range(tSize[0]/2,240-tSize[0]/2, interval), range((240-tSize[0])/interval)): 36 | for j,jj in zip(range(tSize[1]/2,320-tSize[1]/2,interval), range((320-tSize[0])/interval)): 37 | # p = [60,60] 38 | p = [i,j] 39 | imOut[ii,jj] = np.sum(im2[p[0]-tSize[0]/2:p[0]+tSize[0]/2, p[1]-tSize[1]/2:p[1]+tSize[1]/2, 2] * template) 40 | 41 | subplot(3,3,s/10) 42 | title(s) 43 | imshow(imOut) 44 | tIms.append(deepcopy(imOut)) 45 | figure(2); imshow(im2) 46 | figure(3); imshow(im) 47 | figure(4); imshow(tIms[5]/tIms[5].max()-cv2.resize(tIms[2], (tIms[5].shape[1],tIms[5].shape[0]))/tIms[2].max()) 48 | # figure(4) 49 | 50 | 51 | -------------------------------------------------------------------------------- /pyKinectTools/scripts/hogTests.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from sklearn import svm 3 | # from sklearn import svm 4 | import scipy.ndimage as nd 5 | import scipy.io 6 | from scipy.io.matlab import loadmat 7 | from pyKinectTools.utils.HOGUtils import * 8 | 9 | sys.path.append("/Users/colin/libs/pyvision/build/lib.macosx-10.7-intel-2.7/") 10 | from vision import features # pyvision library 11 | 12 | import pyKinectTools.data as ktData 13 | dataFolder = ktData.__file__ 14 | dataFolder = dataFolder[:dataFolder.find("__init__")] 15 | 16 | #------- 17 | 18 | # Get training data 19 | trainingFeatures = scipy.io.loadmat(dataFolder+'bodyPart_HOGFeatures.mat') 20 | trainingDepths = scipy.io.loadmat(dataFolder+'bodyPart_DepthImgs.mat') 21 | 22 | bodyLabels = [x for x in trainingFeatures if x[0]!='_' and x!='other'] 23 | featureCount = np.sum([len(trainingFeatures[x]) for x in bodyLabels if x!='other']) 24 | vectorLen = trainingFeatures['r_hand'][0].flatten().shape[0] 25 | 26 | allFeatures = np.empty([featureCount, vectorLen]) 27 | allFeaturesLab = np.empty([featureCount], dtype=str) 28 | allFeaturesLabNum = np.empty([featureCount],dtype=uint8) 29 | 30 | tmp_i = 0 31 | for lab,i in zip(bodyLabels, range(len(bodyLabels))): 32 | for feat in trainingFeatures[lab]: 33 | f = feat.flatten() 34 | f[f<0] = 0 35 | 36 | allFeatures[tmp_i,:] = f 37 | allFeaturesLab[tmp_i] = lab 38 | allFeaturesLabNum[tmp_i] = i+1 39 | tmp_i += 1 40 | 41 | # display all heads 42 | 43 | faces = trainingDepths['face'][0][0] 44 | for i in range(1, len(trainingDepths['face'])): 45 | imTmp = trainingDepths['face'][i][0] 46 | if imTmp.shape[0] != 0: 47 | faces = np.dstack([faces, imTmp]) 48 | 49 | 50 | ''' Gabor experiment ''' 51 | if 0: 52 | angles = range(0, 108*2, 90/5) 53 | gabors = generateGabors(angles, [10,10], .5) 54 | 55 | convs = [] 56 | for i in range(len(angles)): 57 | convs.append(nd.convolve(imTmp, gabors[:,:,i])) 58 | convs = np.array(convs) 59 | 60 | ''' Display all faces ''' 61 | import cv, cv2 62 | cv2.namedWindow("Face", cv2.CV_WINDOW_AUTOSIZE) 63 | for i in range(len(trainingDepths['face'])-1): 64 | # imshow(trainingDepths['face'][i][0]) 65 | tmpImg = np.asarray(trainingDepths['face'][i][0], dtype=float) 66 | ## Normalize the image. Min min/max within standard deviations to help eliminate the background 67 | if not np.all(tmpImg == 0): 68 | imMean = tmpImg.mean() 69 | imSTD = tmpImg.std() 70 | min_ = tmpImg[tmpImg > imMean-1*imSTD].min() 71 | tmpImg -= min_ 72 | max_ = tmpImg[tmpImg < imMean-min_+1*imSTD].max() 73 | tmpImg[tmpImg>max_]=max_ 74 | tmpImg[tmpImg<0]=max_ 75 | tmpImg /= np.float(max_/255.0) 76 | # tmpImg = np.minimum(tmpImg, 255) 77 | 78 | tmpImg = np.asarray(tmpImg, dtype=np.uint8) 79 | cv2.imshow("Face", tmpImg) 80 | ret = cv2.waitKey(50) 81 | if ret >= 0: 82 | break 83 | cv2.destroyWindow("Face") 84 | 85 | allbodyparts = ['face', 'l_hand', 'r_hand'] 86 | hogs = {} 87 | hogRes = 8 88 | for part in allbodyparts: 89 | hogs[part] = [] 90 | 91 | bodypart = part 92 | ''' Get HOG features ''' 93 | for i in range(len(trainingDepths[bodypart])-1): 94 | tmpImg = np.asarray(trainingDepths[bodypart][i][0], dtype=float) 95 | ## Normalize the image. Min min/max within standard deviations to help eliminate the background 96 | if not np.all(tmpImg == 0): 97 | imMean = tmpImg.mean() 98 | imSTD = tmpImg.std() 99 | min_ = tmpImg[tmpImg > imMean-1*imSTD].min() 100 | tmpImg -= min_ 101 | max_ = tmpImg[tmpImg < imMean-min_+1*imSTD].max() 102 | tmpImg[tmpImg>max_]=max_ 103 | tmpImg[tmpImg<0]=max_ 104 | tmpImg /= np.float(max_/255.0) 105 | tmpImg = np.asarray(tmpImg, dtype=np.uint8) 106 | 107 | tmp = np.dstack([tmpImg, tmpImg, tmpImg]) 108 | hogs[part].append(features.hog(tmp, hogRes)) 109 | 110 | 111 | from pyKinectTools.utils.HOGUtils import * 112 | hogList = list(hogs['face']) 113 | for p in hogs['l_hand']: 114 | hogList.append(p) 115 | # hogList.append(hogs['l_hand']) 116 | # hogList = np.array(hogList) 117 | hogVals = np.empty([len(hogList), len(hogList[0].flatten())]) 118 | for i in range(len(hogList)): 119 | # hogList[i] = hogList[i].flatten() 120 | hogVals[i,:] = hogList[i][0].flatten()[0] 121 | hogLabels = np.zeros(len(hogs['face'])+len(hogs['l_hand'])) 122 | # hogLabels[0:len(hogs['face'])]+=1 123 | hogLabels[len(hogs['face']):]+=1 124 | 125 | # im = HOGpicture(hogList[1], hogRes) 126 | # imshow(im, interpolation='nearest') 127 | 128 | svm_ = svm.SVC(kernel='poly', probability=True, C=1, degree=2, gamma=1) 129 | # svm_ = svm.SVC(kernel='rbf', probability=True, nu=.7)#, C=1)#, ) 130 | svm_.fit(hogVals, hogLabels) 131 | 132 | score = svm_.score(hogVals, hogLabels) 133 | probs = svm_.predict_proba(hogVals) 134 | print score 135 | 136 | from sklearn.ensemble import RandomForestClassifier 137 | from sklearn.ensemble import ExtraTreesClassifier 138 | 139 | forest = ExtraTreesClassifier(n_estimators=50, compute_importances=True, n_jobs=7, bootstrap=True, random_state=0, max_features=1) 140 | forest.fit(hogVals, hogLabels) 141 | score = forest.score(hogVals, hogLabels) 142 | print score 143 | 144 | 145 | '''--------------------------------------------''' 146 | 147 | '''Catagorical SVMs''' 148 | #'''Train''' 149 | # svms = [] 150 | # for feat,i in zip(allFeatures, range(len(allFeaturesLab))): 151 | # for i in bodyLabels: 152 | # labels = np.zeros(featureCount) 153 | labels = allFeaturesLabNum 154 | svm_ = svm.SVC(probability=True, C=100) 155 | svm_.fit(allFeatures, labels) 156 | # svms.append(deepcopy(svm_)) 157 | #'''Test training''' 158 | svmPredict = np.empty([featureCount]) 159 | for feat,i in zip(allFeatures, range(featureCount)): 160 | # for j in xrange(len(bodyLabels)): 161 | # svmPredict[i,j] = svm_.predict_proba(feat)[0][1] 162 | svmPredict[i] = svm_.predict(feat) 163 | 164 | 165 | for k in range(len(extrema)-1): 166 | tmpBoxRot = allBoxesRot[k][5:35, 5:35] 167 | tmpBoxD = np.dstack([tmpBoxRot, tmpBoxRot, tmpBoxRot]) 168 | 169 | if tmpBoxRot.shape[0] == 30 and tmpBoxRot.shape[1] == 30: 170 | f = features.hog(tmpBoxD, 4).flatten() 171 | print svm_.predict(f),svm_.predict_proba(f), extrema[k] 172 | 173 | scores = np.empty(featureCount) 174 | for j in xrange(featureCount): 175 | scores[j] = svms[j].predict_proba(f)[0][1] 176 | scores /= np.sum(scores) 177 | 178 | 179 | 180 | '''Exemplar SVMs''' 181 | #'''Train''' 182 | svms = [] 183 | for feat,i in zip(allFeatures, range(len(allFeaturesLab))): 184 | labels = np.zeros(featureCount) 185 | labels[i] = allFeaturesLabNum[i] 186 | svm_ = svm.SVC(probability=True, C=100) 187 | svm_.fit(allFeatures, labels) 188 | svms.append(deepcopy(svm_)) 189 | #'''Test training''' 190 | svmPredict = np.empty([featureCount,featureCount]) 191 | for feat,i in zip(allFeatures, range(featureCount)): 192 | for j in xrange(featureCount): 193 | svmPredict[i,j] = svms[j].predict_proba(feat)[0][1] 194 | 195 | 196 | # labels = np.zeros(len(allFeaturesLab)) 197 | # labels[i] = 1 198 | # # svm_ = svm.NuSVC(nu=.2, probability=True) 199 | # svm_ = svm.SVC(probability=True) 200 | # svm_.fit(allFeatures, labels) 201 | 202 | 203 | # Problem: isn't there a way to make it invariant on the input size (the training data bounding box size is modified for fitting) 204 | #2:face; #3:r_hand; 5:l_hand 205 | 206 | for k in range(len(extrema)): 207 | tmpBoxRot = allBoxesRot[k][10:30, 10:30] 208 | tmpBoxD = np.dstack([tmpBoxRot, tmpBoxRot, tmpBoxRot]) 209 | 210 | f = features.hog(tmpBoxD, 4)[0].flatten() 211 | scores = np.empty(featureCount) 212 | for j in xrange(featureCount): 213 | scores[j] = svms[j].predict_proba(f)[0][1] 214 | scores /= np.sum(scores) 215 | 216 | t = np.vstack([allFeaturesLabNum, scores]).T 217 | max_ = 0 218 | argmax_ = -1 219 | for c in [2,3,5]: 220 | tmp = np.sum(t[t[:,0]==c,1]) / np.sum(t[:,0]==c) 221 | if tmp > max_: 222 | max_ = tmp 223 | argmax_ = c 224 | print argmax_ 225 | 226 | 227 | print bodyLabels[allFeaturesLabNum[argmax(scores)]-1], argmax(scores), extrema[k] 228 | 229 | 230 | 231 | 232 | 233 | for k in range(9): 234 | print k 235 | tmpBoxRot = allBoxesRot[k] 236 | # tmpBoxRot = np.nan_to_num(tmpBoxRot) 237 | tmpBoxRot[tmpBoxRot>0] -= tmpBoxRot[tmpBoxRot>0].min() 238 | tmpBoxRot[tmpBoxRot>0] /= tmpBoxRot.max() 239 | # tmpBoxRot[tmpBoxRot>0] = np.log(tmpBoxRot[tmpBoxRot>0]) 240 | tmpBoxD = np.dstack([tmpBoxRot, tmpBoxRot, tmpBoxRot]) 241 | f = features.hog(tmpBoxD, 4) 242 | 243 | # figure(1); subplot(3,3,k+1) 244 | figure(1); imshow(tmpBoxRot) 245 | 246 | # figure(2); subplot(3,3,k+1) 247 | # imshow(f[:,:,i-1]) 248 | 249 | im = HOGpicture(f, 4) 250 | figure(2); imshow(im, interpolation='nearest') 251 | 252 | 253 | -------------------------------------------------------------------------------- /pyKinectTools/scripts/labelSkeletons.py: -------------------------------------------------------------------------------- 1 | import os, scipy, time 2 | import scipy.ndimage as nd 3 | from pyKinectTools.utils.DepthUtils import posImage2XYZ 4 | from pyKinectTools.algs.PictorialStructures import * 5 | from pyKinectTools.algs.BackgroundSubtraction import extract_people, removeNoise 6 | dataDir = '/Users/colin/data/ICU_7May2012_Close_jpg/diffDraw1/' 7 | # dataDir = '/Users/colin/data/ICU_7May2012_Close_jpg/d1c/' 8 | 9 | import cv, cv2 10 | 11 | global bodyPos 12 | global partIndex 13 | global posMat 14 | 15 | AllBodyPos = [] 16 | bodyPos = [] 17 | bodyTimes = [] 18 | partIndex = 0 19 | 20 | 21 | def onclick(event): 22 | 23 | # global regions 24 | global posMat 25 | global bodyPos 26 | global partIndex 27 | 28 | # print "h0", posMat[int(event.ydata), int(event.xdata),:] 29 | print partIndex 30 | bodyPos.append(posMat[int(event.ydata), int(event.xdata),:]) 31 | 32 | partIndex += 1 33 | if partIndex == 5:#len(bodyPos.keys()): 34 | partIndex = 0 35 | 36 | 37 | '''#################### Load Images #########################''' 38 | 39 | ''' 40 | imgs = array of many images 41 | im = specific image 42 | posMat = 3-dimensional array of XYZ positions at each pixel 43 | xyz = list of points 44 | ''' 45 | 46 | files = os.listdir(dataDir) 47 | files = [int(x[0:-4]) for x in files if x[0]!='.'] 48 | files = np.sort(files) 49 | # sequenceFrameNames = files[0:4000:50] 50 | # sequenceFrameNames = files[7200:7230] 51 | sequenceFrameNames = files[::500] 52 | imgs = [] 53 | for i in sequenceFrameNames: 54 | imgs.append(scipy.misc.imread(dataDir+str(i)+'.jpg')) 55 | imgs = np.array(imgs) 56 | 57 | ''' ------- MAIN --------- ''' 58 | print "Start" 59 | 60 | t = 0 61 | 62 | if t > 0: 63 | if bodyPos == -1 or len(bodyPos) == 7: 64 | AllBodyPos.append(bodyPos) 65 | bodyTimes.append(sequenceFrameNames[t-1]) 66 | else: 67 | print "Error. Redoing the last frame." 68 | t -= 1 69 | else: 70 | AllBodyPos = [] 71 | 72 | bodyPos = [] 73 | 74 | im = imgs[t] 75 | objectNum = 0 76 | posMatFull = posImage2XYZ(im, 500, 2000) 77 | imLabels, objSlices, objInds = extract_people(posMatFull[:,:,2], 10000, True) 78 | if len(objInds)==0: 79 | print"No humans" 80 | bodyPos = -1 81 | else: 82 | posMat = posMatFull[objSlices[objectNum]] 83 | for i in range(3): 84 | posMat[:,:,i] *= (imLabels[objSlices[objectNum]]==objInds[objectNum]) 85 | 86 | posMat = removeNoise(posMat, thresh=500) 87 | 88 | fig = figure(2) 89 | fig.canvas.mpl_connect('button_press_event', onclick) 90 | ax = fig.add_subplot(111) 91 | ax.imshow(posMat[:,:,2]) 92 | 93 | t += 1 94 | 95 | 96 | ''' ------- \MAIN --------- ''' 97 | 98 | ''' ---- Compute stats ----- ''' 99 | 100 | # relativePos = [x[2]-x[1], x[]]] for x in AllBodyPos] 101 | 102 | labels = ["r_shoulder", "r_arm", "r_hand", 103 | "l_shoulder", "l_arm", "l_hand"] 104 | relDistsIndiv = [ 105 | [x[1]-x[0] for x in AllBodyPos if x != -1 and np.any(x[1] != 0)], 106 | [x[2]-x[1] for x in AllBodyPos if x != -1 and np.any(x[2] != 0)], 107 | [x[3]-x[2] for x in AllBodyPos if x != -1 and np.any(x[3] != 0)], 108 | [x[4]-x[0] for x in AllBodyPos if x != -1 and np.any(x[4] != 0)], 109 | [x[5]-x[4] for x in AllBodyPos if x != -1 and np.any(x[5] != 0)], 110 | [x[6]-x[5] for x in AllBodyPos if x != -1 and np.any(x[6] != 0)] 111 | ] 112 | 113 | relDists = [np.mean(x, 0) for x in relDistsIndiv] 114 | absDists = [np.sqrt(np.sum(x**2)) for x in relDists] 115 | relStds = [np.std(x,0) for x in relDistsIndiv] 116 | absStds = [np.std(x) for x in relDistsIndiv] 117 | 118 | scipy.savez("labeledSkels_Every500.npz", 119 | relDistsIndiv=relDistsIndiv, 120 | relDists=relDists, 121 | absDists=absDists, 122 | relStds=relStds, 123 | absStds=absStds, 124 | times=bodyTimes, 125 | dataDir=dataDir) 126 | 127 | 128 | -------------------------------------------------------------------------------- /pyKinectTools/scripts/loadData.py: -------------------------------------------------------------------------------- 1 | import os, scipy 2 | import scipy.ndimage as nd 3 | from pyKinectTools.utils.DepthUtils import posImage2XYZ 4 | from pyKinectTools.algs.BackgroundSubtraction import extract_people, removeNoise 5 | dataDir = '/Users/colin/data/ICU_7May2012_Wide_jpg/diffDraw1/' 6 | dataDir = '/Users/colin/data/ICU_7May2012_Close_jpg/diffDraw1/' 7 | dataDir = '/Users/colin/data/ICU_7May2012_Close_jpg/d1c/' 8 | # cd /Users/colin/data/ICU_7May2012_Wide_jpg/d/1c 9 | 10 | '''#################### Load Images #########################''' 11 | 12 | ''' 13 | imgs = array of many images 14 | im = specific image 15 | posMat = 3-dimensional array of XYZ positions at each pixel 16 | xyz = list of points 17 | ''' 18 | 19 | files = os.listdir(dataDir) 20 | files = [int(x[0:-4]) for x in files if x[0]!='.'] 21 | files = np.sort(files) 22 | # sequenceFrameNames = files[410:640] # to 1101 23 | sequenceFrameNames = files[597:620] #400-600#597, 1100 24 | # sequenceFrameNames = files[2000:2300] #400-600#597, 1100 25 | # He moved around 2100-2200 26 | imgs = [] 27 | for i in sequenceFrameNames: 28 | imgs.append(scipy.misc.imread(dataDir+str(i)+'.jpg')) 29 | imgs = np.array(imgs) 30 | 31 | ''' Get posMat from individual image ''' 32 | im = imgs[9] 33 | objectNum = 0 34 | posMatFull = posImage2XYZ(im, 500, 2000) 35 | # posMatFull = posImage2XYZ(im, 500, 2000) 36 | imLabels, objSlices, objInds = extract_people(posMatFull[:,:,2], 10000, True) 37 | posMat = posMatFull[objSlices[objectNum]] 38 | for i in range(3): 39 | posMat[:,:,i] *= (imLabels[objSlices[objectNum]]==objInds[objectNum]) 40 | 41 | # posMat = removeNoise(posMat, thresh=500) 42 | xyz = posMat[(posMat[:,:,2]>0)*(posMat[:,:,0]!=0),:] 43 | 44 | 45 | 46 | # Edge test 47 | im1 = imgs[0] 48 | im2 = imgs[1] 49 | imgs[0] = np.array(imgs[0], dtype=int16) 50 | imgs[1] = np.array(imgs[1], dtype=int16) 51 | imD = np.array(imgs[0]-imgs[1], dtype=np.int16) 52 | diff = (np.diff(np.abs(imD)>10)) 53 | diff = nd.binary_closing(diff, iterations=5) 54 | # diff = nd.binary_dilation(diff, iterations=3) 55 | imshow(im1[:,0:-1]*(1-diff)) 56 | 57 | im = im1[:,0:-1]*(1-diff) 58 | 59 | 60 | 61 | '''#################### Test HOG #########################''' 62 | 63 | 64 | ''' Load example (uncompressed) img ''' 65 | # saved = np.load('tmpPerson_close.npz')['arr_0'].tolist() 66 | # saved = np.load('tmpPerson1.npz')['arr_0'].tolist() 67 | # objects1 = saved['objects']; labelInds1=saved['labels']; out1=saved['out']; d1=saved['d']; com1=saved['com'];featureExt1=saved['features']; posMat=saved['posMat']; xyz=saved['xyz'] 68 | 69 | 70 | 71 | '''############### Load Time of Flight ##################''' 72 | 73 | # cd /Users/colin/data/TUM/operation_camera_tool/operation_camera_tool/ 74 | 75 | 76 | 77 | '''#################### Other stuff #########################''' 78 | 79 | '''Other''' 80 | # np.savez('tmpPerson1.npz', {'objects':objects1, 'labels':labelInds1, 'out':out1, 'd':d1, 'com':com1, 'features':featureExt1, 'posMat':posMat, 'xyz':xyz}) 81 | -------------------------------------------------------------------------------- /pyKinectTools/utils/CameraCalibration.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Modified considerably from https://github.com/abidrahmank/OpenCV-Python/blob/master/Other_Examples/camera_calibration.py 3 | 4 | Usage: python CameraCalibration.py num_board_cols num_board_rows number_of_views 5 | ''' 6 | 7 | ################################################################################################ 8 | 9 | import time 10 | import sys 11 | import cv 12 | import cv2 13 | import numpy as np 14 | from pyKinectTools.utils.RealtimeReader import * 15 | 16 | n_boards=0 17 | board_w=int(sys.argv[1]) 18 | board_h=int(sys.argv[2]) 19 | n_boards=int(sys.argv[3]) 20 | # no of total corners 21 | board_n=board_w*board_h 22 | #size of board 23 | board_sz=(board_w,board_h) 24 | 25 | # Init point arrays 26 | image_points = np.empty([n_boards*board_n, 2], np.float32) 27 | object_points = np.empty([n_boards*board_n, 3], np.float32) 28 | point_counts=np.empty([n_boards, 1], np.float32) 29 | 30 | # capture frames of specified properties and modification of matrix values 31 | i=0 32 | z=0 # to print number of frames 33 | successes=0 34 | # capture=cv2.VideoCapture(0) 35 | # capture = RealTimeDevice(0,get_skeleton=False, get_infrared=True, 36 | # get_depth=False, get_color=False, 37 | # config_file=pyKinectTools.configs.__path__[0]+'/InfraredConfig.xml') 38 | capture = RealTimeDevice(0,get_skeleton=False) 39 | capture.start() 40 | 41 | 42 | # capturing required number of views 43 | while(successes0] = 0 35 | # im = np.zeros([bs*shape_[0], bs*shape_[1]]) 36 | im = np.zeros([bs*shape_[1], bs*shape_[0]]) 37 | for i in range(0,shape_[1]): 38 | for j in range(0,shape_[0]): 39 | for k in range(9): 40 | # pass 41 | # im[(i-1)*bs:i*bs, (j-1)*bs:j*bs] += bim[:,:,k]*w[j,i,k] 42 | im[(i)*bs:(i+1)*bs, (j)*bs:(j+1)*bs] += bim[:,:,k]*w[j,i,k] 43 | # im[(j-1)*bs:j*bs,(i-1)*bs:i*bs] += bim[:,:,k]*w[i,j,k] 44 | 45 | 46 | return im 47 | 48 | 49 | def overlayHOG(im_, hogIm): 50 | im = np.copy(im_)*1.0 51 | imShape = im.shape 52 | hShape = hogIm.shape 53 | xMin = np.floor((imShape[0]-hShape[0])) 54 | xMax = imShape[0] 55 | yMin = np.floor((imShape[1]-hShape[1])) 56 | yMax = imShape[1] 57 | # print xMin, xMax 58 | # print yMin, yMax 59 | 60 | im[xMin:xMax, yMin:yMax] = np.maximum(im[xMin:xMax, yMin:yMax], hogIm*10.0) 61 | 62 | return im -------------------------------------------------------------------------------- /pyKinectTools/utils/MultiCameraUtils.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Used to stream multiple asynchonrous cameras simultaneously 3 | ''' 4 | 5 | import numpy as np 6 | 7 | class multiCameraTimeline: 8 | 9 | def __init__(self, files, temporalCameraOffsets=[0, 18, 13]): 10 | self.files = files 11 | self.stamp = np.zeros(len(files), dtype=int) 12 | self.fileLengths = [len(x) for x in files] 13 | self.devCount = len(files) 14 | self.temporalCameraOffsets = temporalCameraOffsets 15 | 16 | def __iter__(self): 17 | 18 | while 1: 19 | # Get latest file names/times 20 | tmpFiles = [self.files[i][self.stamp[i]] if self.stamp[i]= self.fileLengths[dev]: 33 | dev = None 34 | for d, i in zip(np.argsort(times), range(len(times))): 35 | if d < self.fileLengths[i]: 36 | dev = d 37 | break 38 | if dev == None: 39 | raise StopIteration() 40 | 41 | self.stamp[dev] += 1 42 | # If at the end of the roll 43 | if tmpFiles[dev] == np.inf: 44 | raise StopIteration() 45 | 46 | yield dev, tmpFiles[dev] 47 | 48 | 49 | def formatFileString(x): 50 | if len(x) == 1: 51 | return x+'0' 52 | else: 53 | return x -------------------------------------------------------------------------------- /pyKinectTools/utils/SocketReader.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | from pyKinectTools.algs.BackgroundSubtraction import * 4 | from pyKinectTools.utils.DepthUtils import * 5 | import numpy as np 6 | from socket import * 7 | import time 8 | 9 | localIP = [ip for ip in gethostbyname_ex(gethostname())[2] if not ip.startswith("127.")][:1] 10 | 11 | 12 | class DeviceClient: 13 | ''' Store info about each client ''' 14 | client = -1 15 | address = -1 16 | 17 | colorIm = [] 18 | depthIm = [] 19 | depthIm8 = [] 20 | constrain = [] 21 | 22 | def __init__(self, client, address): 23 | self.client = client 24 | self.address = address 25 | 26 | 27 | class SocketDeviceHost: 28 | 29 | server = [] 30 | clients = [] 31 | 32 | host = -1 33 | port = -1 34 | address = -1 35 | bufferSize = -1 36 | 37 | currentCam = 0 38 | depthIm = [] 39 | depthIm8 = [] 40 | colorIm = [] 41 | 42 | constrain = [] 43 | maxDists = [] 44 | timeout=.03 45 | 46 | bgModel = [] 47 | bgModel8 = [] 48 | 49 | 50 | def __init__(self, port=29770, host='', bufferSize=4096, maxDists=[4000]): 51 | self.port = self.port 52 | self.host = self.host 53 | self.address = (host, port) 54 | self.bufferSize = bufferSize 55 | self.maxDists = maxDists 56 | 57 | # Start server 58 | self.server = socket(AF_INET, SOCK_STREAM) 59 | self.server.bind((self.address)) 60 | # Set timeout to .1 second 61 | self.timeout = .01 62 | self.server.settimeout(self.timeout) 63 | 64 | self.connectDevices() 65 | 66 | 67 | def connectDevices(self): 68 | self.server.listen(10) 69 | t1 = time.time() 70 | while time.time() - t1 < self.timeout*3: 71 | try: 72 | conn, connAddr = self.server.accept() 73 | 74 | dev = DeviceClient(conn, connAddr) 75 | dev.client.settimeout(self.timeout) 76 | self.clients.append(dev) 77 | except: 78 | pass 79 | 80 | 81 | def setMaxDist(self, dists): 82 | self.maxDists = dists 83 | 84 | def stop(self): 85 | for c in xrange(len(self.clients)): 86 | self.clients[c].client.close() 87 | self.server.close() 88 | 89 | def close(self): 90 | self.stop() 91 | 92 | def flushBuffer(self): 93 | for i in xrange(len(self.clients)): 94 | try: 95 | data_str = self.clients[i].client.recv(self.bufferSize) 96 | except: 97 | pass 98 | 99 | def update(self): 100 | imTypes = {'color':921600, 'depth':307200} 101 | imSize = 99999 102 | imsRaw = [] 103 | for i in xrange(len(self.clients)): 104 | data = '' 105 | time_ = time.time() 106 | imStarted = 0 107 | while len(data) < imSize and time.time()-time_ < self.timeout*3: 108 | data_str = '' 109 | try: 110 | data_str = self.clients[i].client.recv(self.bufferSize) 111 | except: 112 | # print "Buffer is empty" 113 | pass 114 | 115 | if len(data_str) > 0: 116 | # Check for start of image 117 | if imStarted == 0: 118 | for type_ in imTypes: 119 | if type_ in data_str: 120 | imType = type_ 121 | imSize = imTypes[imType] 122 | data = data_str[data_str.find(type_)+len(type_):] 123 | imStarted = 1 124 | # Add image packet 125 | elif imStarted == 1: 126 | # Make sure there isn't a problem with buffer 127 | # Restrict to only allow buffersize (unless near the end of the image) 128 | # if len(data_str) == self.bufferSize or len(data) > imSize-self.bufferSize: 129 | if 'end' not in data_str: 130 | data += data_str 131 | else: 132 | data += data_str[:data_str.find('end')] 133 | 134 | # If image is complete, transform from string->im 135 | if len(data) == imSize: 136 | if imType == 'depth': 137 | d = np.fromstring(str(data), dtype=np.uint8).reshape([480,640]) 138 | self.clients[i].depthIm = d 139 | self.clients[i].depthIm8 = d 140 | else: 141 | d = np.fromstring(str(data), dtype=np.uint8).reshape([480,640, 3]) 142 | self.clients[i].colorIm = d 143 | # data = '' 144 | imStarted = 0 145 | print 'New image:', imType 146 | elif len(data) > imSize: 147 | print "Data longer than image size" 148 | 149 | if imStarted == 0: 150 | for type_ in imTypes: 151 | if type_ in data_str: 152 | imType = type_ 153 | imSize = imTypes[imType] 154 | data = data_str[data_str.find(type_)+len(type_):] 155 | imStarted = 1 156 | 157 | 158 | time_ = time.time() 159 | self.data = data 160 | 161 | 162 | # if imSize != 99999 and len(data) == imSize: 163 | # if imType == 'depth': 164 | # d = np.fromstring(str(data), dtype=np.uint8).reshape([480,640]) 165 | # self.clients[i].depthIm = d 166 | # self.clients[i].depthIm8 = d 167 | # else: 168 | # d = np.fromstring(str(data), dtype=np.uint8).reshape([480,640, 3]) 169 | # self.clients[i].colorIm = d 170 | # # imsRaw.append(d) 171 | # print 'New image' 172 | 173 | 174 | # Update im w/ 'current' camera 175 | self.depthIm = self.clients[self.currentCam].depthIm 176 | self.depthIm8 = self.clients[self.currentCam].depthIm8 177 | self.colorIm = self.clients[self.currentCam].colorIm 178 | 179 | 180 | 181 | class SocketDeviceClient(): 182 | 183 | client = -1 184 | server = -1 185 | 186 | host = -1 187 | port = -1 188 | address = -1 189 | bufferSize = -1 190 | 191 | depthIm = [] 192 | depthIm8 = [] 193 | colorIm = [] 194 | 195 | constrain = [] 196 | maxDists = [] 197 | 198 | # bgModel = [] 199 | # bgModel8 = [] 200 | 201 | def __init__(self, port=29770, host='localhost', bufferSize=2048): 202 | self.host = host 203 | self.port = port 204 | self.address = (host, port) 205 | self.bufferSize = bufferSize 206 | 207 | self.client = socket(AF_INET, SOCK_STREAM) 208 | self.client.connect((self.address)) 209 | self.client.settimeout(.1) 210 | 211 | def connect(self, port=-1, host=-1): 212 | if port > 0: 213 | self.port = port 214 | if host > 0: 215 | self.host = host 216 | self.address = (self.host, self.port) 217 | 218 | self.client.connect((self.address)) 219 | 220 | def close(self): 221 | self.client.close() 222 | 223 | def getSample(self): 224 | import os 225 | import numpy as np 226 | import scipy.misc as sm 227 | import sys 228 | 229 | os.chdir('/Users/colin/Dropbox/Public') 230 | 231 | self.depthIm8 = sm.imread('depthImage.jpg') 232 | self.colorIm = np.dstack([self.depthIm8, self.depthIm8, self.depthIm8]) 233 | 234 | self.client.settimeout(5) 235 | 236 | def setDepth(self, im): 237 | self.depthIm = im 238 | if constrain != [] and self.depthIm.max() > 256: 239 | self.depthIm8 = constrain(im, self.constrain[0], self.constrain[1]) 240 | else: 241 | self.depthIm8 = self.depthIm 242 | 243 | def setColor(self, im): 244 | self.colorIm = im 245 | 246 | def sendIm(self, label="depth", im=[]): 247 | # Turn data into string to send over network 248 | im_str = label 249 | im_str += im.tostring() 250 | im_str += 'end' 251 | imSize = sys.getsizeof(im_str) 252 | 253 | # Send in bite-size chunks 254 | i = 0 255 | 256 | data = '' 257 | while i < imSize+self.bufferSize: 258 | c = self.client.send(im_str[i:i+self.bufferSize]) 259 | # print c 260 | # if c == self.bufferSize or i >= imSize-self.bufferSize: 261 | data += im_str[i:i+c] 262 | i += self.bufferSize 263 | 264 | def sendAll(self): 265 | if self.depthIm8 != []: 266 | self.sendIm("depth", self.depthIm8) 267 | if self.colorIm != []: 268 | self.sendIm("color", self.colorIm) 269 | 270 | 271 | -------------------------------------------------------------------------------- /pyKinectTools/utils/Utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | ''' Create a folder if it doesn't exist ''' 4 | def createDirectory(new_dir): 5 | if not os.path.isdir(new_dir): 6 | for p in xrange(2,len(new_dir.split("/"))+1): 7 | tmp_string = "/".join(new_dir.split('/')[:p]) 8 | if not os.path.isdir(tmp_string): 9 | try: 10 | os.mkdir(tmp_string) 11 | except: 12 | print "error making dir:", tmp_string 13 | 14 | def formatFileString(x): 15 | if len(x) == 1: 16 | return x+'0' 17 | else: 18 | return x 19 | -------------------------------------------------------------------------------- /pyKinectTools/utils/VideoViewer.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Replacement for OpenCVs imshow and waitkey commands 3 | ''' 4 | 5 | import numpy as np 6 | import visvis as vv 7 | import time 8 | from IPython import embed 9 | 10 | 11 | class VideoViewer: 12 | ''' 13 | Uses the library visvis to write images to an opengl window. 14 | It imitates the OpenCV video player commands imshow(), waitKey() and putText(). 15 | ''' 16 | 17 | open_windows = [] 18 | 19 | def __init__(self): 20 | pass 21 | 22 | def _getWindow(self, name): 23 | names = [x['name'] for x in self.open_windows] 24 | try: 25 | index = names.index(name) 26 | return self.open_windows[index] 27 | except: 28 | return None 29 | 30 | def _getWindowFromFigure(self, figure): 31 | figs = [x['canvas'] for x in self.open_windows] 32 | try: 33 | index = figs.index(figure) 34 | return self.open_windows[index] 35 | except: 36 | return None 37 | 38 | def _keyHandler(self, event): 39 | # embed() 40 | win = self._getWindowFromFigure(event.owner) 41 | win['keyEvent'] = event.key 42 | # print event.text, event.key 43 | 44 | def _createWindow(self, name, im, axis): 45 | 46 | vv.figure() 47 | vv.gca() 48 | vv.clf() 49 | fig = vv.imshow(im) 50 | dims = im.shape 51 | 52 | ''' Change color bounds ''' 53 | if im.dtype == np.uint8: 54 | fig.clim.Set(0, 255) 55 | else: 56 | fig.clim.Set(0., 1.) 57 | 58 | fig.GetFigure().title = name 59 | 60 | ''' Show ticks on axes? ''' 61 | if not axis: 62 | fig.GetAxes().axis.visible = False 63 | bgcolor = (0.,0.,0.) 64 | else: 65 | fig.GetAxes().axis.showBox = False 66 | bgcolor = (1.,1.,1.) 67 | 68 | ''' Set background color ''' 69 | fig.GetFigure().bgcolor = bgcolor 70 | fig.GetAxes().bgcolor = bgcolor 71 | 72 | ''' Setup keyboard event handler ''' 73 | fig.eventKeyDown.Bind(self._keyHandler) 74 | 75 | win = {'name':name, 'canvas':fig, 'shape':dims, 'keyEvent':None, 'text':[]} 76 | self.open_windows.append(win) 77 | 78 | return win 79 | 80 | 81 | def destroyWindow(self, name): 82 | ''' 83 | Kill a specific window 84 | ''' 85 | win = self._getWindow(name) 86 | 87 | if win is not None: 88 | win['canvas'].Destroy() 89 | else: 90 | print "No window found" 91 | 92 | vv.update() 93 | 94 | def destroyAll(self): 95 | ''' 96 | Kill all open windows 97 | ''' 98 | for win in self.open_windows: 99 | if win is not None: 100 | win['canvas'].Destroy() 101 | 102 | vv.update() 103 | 104 | def imshow(self, name, im, axis=False): 105 | ''' 106 | Inputs 107 | name: string with figure name 108 | im: numpy array 109 | axis: display axes? 110 | ''' 111 | 112 | win = self._getWindow(name) 113 | 114 | if win is None: 115 | win = self._createWindow(name, im, axis) 116 | else: 117 | ''' If new image is of different dimensions we must create new image''' 118 | if im.shape != win['shape']: 119 | self.destroyWindow(win['name']) 120 | win = self._createWindow(name, im) 121 | else: 122 | [x.Destroy() for x in win['text']] 123 | win['text'] = [] 124 | 125 | win['canvas'].SetData(im) 126 | 127 | 128 | def update(self): 129 | ''' Does not capture key inputs ''' 130 | vv.processEvents() 131 | 132 | def putText(self, figureName, text, pos=(0,0), size=10, color=(255,255,255), font='sans'): 133 | ''' 134 | Fonts: 'mono', 'sans' or 'serif' 135 | Color: Can be character (e.g. 'r', 'g', 'k') or 3-element tuple 136 | 137 | Notes: 138 | *This draws text onto the canvas and not the image. 139 | *You can use symbols (e.g. \leftarrow) and greek letters (e.g. \alpha, \omega,...) 140 | *Bold, italics, superscript, and subscript: \b{Text}, \i{Text}, \sup^{Text}, \sub_{Text} 141 | 142 | See here for examples and formatting: 143 | *http://code.google.com/p/visvis/wiki/example_text 144 | *http://code.google.com/p/visvis/wiki/Text 145 | ''' 146 | # embed() 147 | win = self._getWindow(figureName) 148 | win['canvas'].GetFigure().MakeCurrent() 149 | win['text'].append(vv.Text(win['canvas'].GetAxes(), text=text, x=pos[0], y=pos[1], fontSize=size, fontName=font, color=color)) 150 | 151 | 152 | def waitKey(self, duration=0): 153 | return self.waitAndUpdate(duration) 154 | 155 | def waitAndUpdate(self, duration=.00001): 156 | ''' 157 | Input: 158 | *Duration (in milliseconds) 159 | If negative it will wait until a key is pressed 160 | 161 | Output: 162 | *The first key pressed 163 | ''' 164 | 165 | keys = [] 166 | 167 | ''' Wait for set duration ''' 168 | if duration >= 0: 169 | time.sleep(duration) 170 | vv.processEvents() 171 | keys = [x['keyEvent'] for x in self.open_windows if x['keyEvent'] is not None] 172 | for i in self.open_windows: 173 | i['keyEvent'] = None 174 | else: 175 | ''' Wait until key pressed ''' 176 | while(time.time()-start_time < np.inf): 177 | vv.processEvents() 178 | keys = [x['keyEvent'] for x in self.open_windows if x['keyEvent'] is not None] 179 | time.sleep(.000001) 180 | if len(keys) > 0: 181 | break 182 | 183 | ''' Check for keys pressed ''' 184 | if len(keys) > 0: 185 | return keys.pop() 186 | else: 187 | return 0 188 | 189 | 190 | if __name__=="__main__": 191 | w = VideoViewer() 192 | im = vv.imread('/Users/colin/Desktop/hog.png') 193 | w.imshow("Hi", np.eye(100, dtype=np.uint8), axis=True) 194 | w.imshow("im", im, axis=False) 195 | 196 | for j in range(100): 197 | key = w.waitAndUpdate(1000) 198 | w.imshow("im", im+j*5, axis=False) 199 | print key 200 | 201 | 202 | 203 | -------------------------------------------------------------------------------- /pyKinectTools/utils/__init__.py: -------------------------------------------------------------------------------- 1 | # import .. 2 | # import os,sys 3 | # sys.path.append(os.path.dirname(__file__)) 4 | 5 | # __all__ = ["DepthReader"] 6 | # import DepthReader -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | 2 | ''' 3 | Configure: 4 | python setup.py build 5 | 6 | pyKinectTools 7 | Colin Lea 8 | 2012 9 | ''' 10 | 11 | from distutils.core import setup 12 | from distutils.extension import Extension 13 | from Cython.Distutils import build_ext 14 | import numpy as np 15 | 16 | import Cython.Compiler.Options 17 | Cython.Compiler.Options.annotate = True 18 | # from Cython.Build import cythonize 19 | 20 | ext_modules = [ 21 | Extension("pyKinectTools_algs_Dijkstras", ["pyKinectTools/algs/dijkstras.pyx"],language='c++'), 22 | Extension("pyKinectTools_algs_local_occupancy_pattern", ["pyKinectTools/algs/LocalOccupancyPattern.pyx"],language='c++'), 23 | Extension("pyKinectTools_algs_dynamic_time_warping", ["pyKinectTools/algs/DynamicTimeWarping.pyx"],language='c++'), 24 | # cythonize("pyKinectTools/algs/DynamicTimeWarping.pyx"), 25 | ] 26 | # _Dijkstras 27 | # Extension("pyKinectTools_algs_Pose_Tracking", ["pyKinectTools/algs/cPoseTracking.pyx"],language='c++'), 28 | # Extension("pyKinectTools.NeighborSuperpixels", ["pyKinectTools/algs/NeighborSuperpixels.pyx"]) 29 | # Extension("pyKinectTools.algs.Dijkstras", ["pyKinectTools/algs/dijkstras_new.pyx"],) 30 | # Extension("pyKinectTools.algs.NeighborSuperpixels", ["pyKinectTools/algs/NeighborSuperpixels.pyx"]) 31 | # Extension("pyKinectTools.algs.dijkstrasGraph", ["pyKinectTools/algs/dijkstras.pyx"]) 32 | 33 | for e in ext_modules: 34 | e.pyrex_directives = { 35 | "boundscheck": False, 36 | "wraparound": False, 37 | "infer_types": True 38 | } 39 | e.extra_compile_args = ["-w"] 40 | 41 | print ext_modules 42 | setup( 43 | author = 'Colin Lea', 44 | author_email = 'colincsl@gmail.com', 45 | description = '', 46 | license = "FreeBSD", 47 | version= "0.1", 48 | name = 'pyKinectTools', 49 | cmdclass = {'build_ext': build_ext}, 50 | include_dirs = [np.get_include()], 51 | packages= [ "pyKinectTools", 52 | "pyKinectTools.algs", 53 | "pyKinectTools.utils", 54 | "pyKinectTools.configs", 55 | "pyKinectTools.dataset_readers" 56 | ], 57 | package_data={'':['*.xml', '*.png', '*.yml', '*.txt']}, 58 | ext_modules = ext_modules 59 | ) 60 | 61 | --------------------------------------------------------------------------------