├── scripts ├── jqm_cvi │ ├── __init__.py │ ├── theory.pdf │ ├── setup.py │ ├── LICENSE │ ├── README.md │ ├── basec.pyx │ └── cvi.py ├── bash │ ├── c3d_suturing_feature_extraction.sh │ ├── c3d_sport1m_feature_extraction_video.sh │ ├── c3d_sport1m_feature_extraction_frm.sh │ ├── run_tscdl_experiments.sh │ ├── rcnn_e2e.sh │ ├── e2e_c3d_fe_suturing.sh │ ├── c3d_experiment_2.sh │ ├── c3d_experiment_1.sh │ └── suturing_varying_featurizations.sh ├── run_experiment.sh ├── lcd.py ├── read_binary_blob.m ├── split_kinematics.py ├── convert_pickle.py ├── dtw.py ├── encoding.py ├── generate_input_output_files.py ├── cluster_evaluation.py ├── background_subtraction.py ├── constants_sk.py ├── cv_optflw.py ├── segmenter.py ├── pruning.py ├── examples │ ├── plot_gmm.py │ ├── plot_kmeans_digits.py │ ├── plot_hog.py │ ├── plot_compare_cross_decomposition.py │ ├── plot_lle_digits.py │ └── preWhiten.py ├── forward_pass.py ├── fine_tuning │ ├── assemble_data.py │ └── train_val.prototxt ├── feature_extractor.py ├── filter_visualization.py ├── jaccard.py ├── preprocess.py ├── sift.py ├── tsne.py ├── constants.py ├── tscdl.py ├── recorder.py ├── broken_barh.py ├── plot_tsne.py └── experiments.py ├── files └── mgk-icra16-tscdl.pdf ├── .gitignore ├── readme.md └── toyExample ├── utils ├── mkdir_if_not_exist.m ├── genTarget.m ├── plotTraj.m ├── dynamics.m ├── saveFig.m ├── genLabelFile_baseline.m ├── bg_Image.m ├── genLabelFile.m ├── plotState_baseline.m └── plotState.m ├── readme.txt ├── toyEx_baseline.m ├── toyEx_baseline2.m └── main.m /scripts/jqm_cvi/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /files/mgk-icra16-tscdl.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerkeleyAutomation/tsc-dl/HEAD/files/mgk-icra16-tscdl.pdf -------------------------------------------------------------------------------- /scripts/jqm_cvi/theory.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerkeleyAutomation/tsc-dl/HEAD/scripts/jqm_cvi/theory.pdf -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.p 3 | *.ogv 4 | *.mp4 5 | 6 | toyExample/output/* 7 | 8 | clustering/* 9 | jigsaws/* 10 | config/* -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | TSC-DL: Unsupervised Trajectory Segmentation of Multi-Modal Surgical Demonstrations with Deep Learning 2 | 3 | Project Page: 4 | http://berkeleyautomation.github.io/tsc-dl/ -------------------------------------------------------------------------------- /toyExample/utils/mkdir_if_not_exist.m: -------------------------------------------------------------------------------- 1 | function mkdir_if_not_exist(dirpath) 2 | if dirpath(end) ~= filesep, dirpath = [dirpath filesep]; end 3 | if (exist(dirpath, 'dir') == 0), mkdir(dirpath); end 4 | end -------------------------------------------------------------------------------- /scripts/bash/c3d_suturing_feature_extraction.sh: -------------------------------------------------------------------------------- 1 | GLOG_logtosterr=1 ../../build/tools/extract_image_features.bin prototxt/c3d_suturing_feature_extractor_frm.prototxt conv3d_deepnetA_sport1m_iter_1900000 -1 100 1 prototxt/output_list_prefix_suturing.txt conv5b -------------------------------------------------------------------------------- /toyExample/utils/genTarget.m: -------------------------------------------------------------------------------- 1 | function [ targetNext ] = genTarget( ) 2 | % Load previous generator settings. 3 | % rng(s); 4 | 5 | pd = makedist('Uniform','lower',-10,'upper',10); 6 | targetNext = random(pd, 2,1); 7 | 8 | end 9 | 10 | -------------------------------------------------------------------------------- /toyExample/utils/plotTraj.m: -------------------------------------------------------------------------------- 1 | function [ trajHandle ] = plotTraj( x_traj, axHandle ) 2 | 3 | % persistent trajHandle 4 | % delete(trajHandle) 5 | trajHandle = plot(axHandle, x_traj(1,end), x_traj(2,end),':g*', 'LineWidth', 1); 6 | uistack(trajHandle, 'bottom') 7 | 8 | end 9 | 10 | -------------------------------------------------------------------------------- /scripts/run_experiment.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | for FEATURIZATION in 2 3 4 5 4 | do 5 | for DIMRED in CCA 6 | do 7 | FILE=$FEATURIZATION\_$DIMRED 8 | FILE_NAME=\_$FILE 9 | 10 | python clustering_kinematics.py $FILE_NAME\_Z\_v2 --visual $FILE\.p 11 | python clustering.py $FILE\.p $FILE_NAME\_ZW\_ 12 | done 13 | done -------------------------------------------------------------------------------- /scripts/bash/c3d_sport1m_feature_extraction_video.sh: -------------------------------------------------------------------------------- 1 | mkdir -p output/c3d/v_ApplyEyeMakeup_g01_c01 2 | mkdir -p output/c3d/v_BaseballPitch_g01_c01 3 | GLOG_logtosterr=1 ../../build/tools/extract_image_features.bin prototxt/c3d_sport1m_feature_extractor_video.prototxt conv3d_deepnetA_sport1m_iter_1900000 0 50 1 prototxt/output_list_prefix.txt fc7-1 fc6-1 prob 4 | -------------------------------------------------------------------------------- /scripts/lcd.py: -------------------------------------------------------------------------------- 1 | import IPython 2 | import numpy as np 3 | import utils 4 | import encoding 5 | 6 | def LCD(data): 7 | assert len(data.shape) == 3 8 | M = data.shape[0] 9 | a = data.shape[1] 10 | assert a == data.shape[2] 11 | lcd = np.swapaxes(data, 0, 2) 12 | lcd = lcd.reshape(a * a, M) 13 | assert lcd.shape[0] == a * a 14 | assert lcd.shape[1] == M 15 | return lcd -------------------------------------------------------------------------------- /scripts/jqm_cvi/setup.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from distutils.core import setup 4 | from distutils.extension import Extension 5 | from Cython.Distutils import build_ext 6 | import numpy 7 | 8 | setup( 9 | cmdclass = {'build_ext': build_ext}, 10 | ext_modules = [ 11 | Extension("basec", ["basec.pyx"], 12 | include_dirs=[numpy.get_include()]), 13 | ] 14 | ) 15 | -------------------------------------------------------------------------------- /scripts/bash/c3d_sport1m_feature_extraction_frm.sh: -------------------------------------------------------------------------------- 1 | mkdir -p output/c3d/v_ApplyEyeMakeup_g01_c01 2 | mkdir -p output/c3d/v_BaseballPitch_g01_c01 3 | cd input/frm 4 | tar xvzf v_ApplyEyeMakeup_g01_c01.tar.gz 5 | tar xvzf v_BaseballPitch_g01_c01.tar.gz 6 | cd ../.. 7 | GLOG_logtosterr=1 ../../build/tools/extract_image_features.bin prototxt/c3d_sport1m_feature_extractor_frm.prototxt conv3d_deepnetA_sport1m_iter_1900000 -1 150 1 prototxt/output_list_prefix.txt fc8-1 -------------------------------------------------------------------------------- /scripts/bash/run_tscdl_experiments.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This script runs all TSC-DL experiments 4 | 5 | cd .. 6 | 7 | for configfile in "suturing_ED.yaml" 8 | do 9 | echo "[RUNNING EXPERIMENT]" $configfile 10 | echo $configfile > ../config/defaultconfig 11 | 12 | python tscdl.py W _W_last_ 13 | 14 | python tscdl.py Z _Z_last_ --visual_feature 5_PCA.p 15 | 16 | python tscdl.py ZW _ZW_last_ --visual_feature 5_PCA.p 17 | 18 | done 19 | -------------------------------------------------------------------------------- /toyExample/utils/dynamics.m: -------------------------------------------------------------------------------- 1 | function [x_next] = dynamics( x_curr, u_curr) 2 | 3 | % x = [x; xdot] 4 | % u = [accel in nDim] 5 | 6 | mass = ; 7 | f = 10; %deceleration due to friction 8 | nDim = length(x_curr)/2; 9 | 10 | xddot = [u_curr-f*ones(nDim)]; 11 | 12 | x_next = []; 13 | % s = s0 + u.t + 1/2 a.t^2 with t = 1; 14 | x_next(1:nDim) = x_curr(1:nDim) + x_curr(nDim+1:2*nDim) + 0.5*xddot; 15 | % v = u + a.t 16 | x_next(nDim+1:2*nDim) = x_curr(nDim+1:2*nDim) + xddot; 17 | 18 | end -------------------------------------------------------------------------------- /toyExample/utils/saveFig.m: -------------------------------------------------------------------------------- 1 | function saveFig( figInput, iterCount, outputDir ) 2 | 3 | if nargin == 2 4 | fileName = ['output' filesep 'time' int2str(iterCount)]; 5 | elseif nargin == 3 6 | mkdir_if_not_exist (outputDir); 7 | if outputDir(end) ~= '/', outputDir = [outputDir filesep]; end 8 | fileName = [outputDir num2str(iterCount,'%06d')]; 9 | end 10 | % figInput.PaperPositionMode = 'manual'; 11 | saveas(figInput, fileName, 'jpeg') 12 | 13 | end 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /toyExample/readme.txt: -------------------------------------------------------------------------------- 1 | 4 bits: varyRobotInit, noisyDynamics, varyTargetInit, targetNoise 2 | 3 | Experiments: 4 | 1. Noisy targets : 0001 5 | 2. Varying robot initialization: 1000 6 | 3. Dynamics Noise: 0100 7 | 4. Noisy Target + Dynamics Noise: 0101 8 | 5. Noisy Target + vary robot init: 1001 9 | 6. Noisy Target + vary robot init + Dynamics Noise: 1101 10 | 11 | Strectch Goal 12 | 7. Vary target Init + vary robot init + Dynamics Noise: 1110 (Hardest) 13 | Varying target init and target noise are not necessary simultaneously. 14 | 15 | 16 | -------------------------------------------------------------------------------- /scripts/read_binary_blob.m: -------------------------------------------------------------------------------- 1 | % 2 | % Licensed under the Creative Commons Attribution-NonCommercial 3.0 3 | % License (the "License"). You may obtain a copy of the License at 4 | % https://creativecommons.org/licenses/by-nc/3.0/. 5 | % Unless required by applicable law or agreed to in writing, software 6 | % distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 7 | % WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 8 | % License for the specific language governing permissions and limitations 9 | % under the License. 10 | % 11 | 12 | 13 | function [s, data] = read_binary_blob(fn) 14 | f = fopen(fn, 'r'); 15 | s = fread(f, [1 5], 'int32'); 16 | % s contains size of the blob e.g. num x chanel x length x height x width 17 | m = s(1)*s(2)*s(3)*s(4)*s(5); 18 | % data is the blob binary data in single precision (e.g float in C++) 19 | data = fread(f, [1 m], 'single'); 20 | fclose(f); 21 | end -------------------------------------------------------------------------------- /toyExample/utils/genLabelFile_baseline.m: -------------------------------------------------------------------------------- 1 | function [ ] = genLabelFile_baseline( frameLabels, outputDir, fileName ) 2 | 3 | if nargin <2 4 | fprintf('saving labels file in ./output/ \n') 5 | fileName = ['output' filesep 'frameLabels']; 6 | elseif nargin == 2 7 | mkdir_if_not_exist (outputDir); 8 | if outputDir(end) ~= '/', outputDir = [outputDir filesep]; end 9 | fileName = [outputDir 'frameLabels']; 10 | elseif nargin == 3 11 | mkdir_if_not_exist (outputDir); 12 | if outputDir(end) ~= '/', outputDir = [outputDir filesep]; end 13 | fileName = [outputDir fileName]; 14 | end 15 | 16 | % make textFile 17 | fid = fopen( [fileName '.txt'], 'wt' ); 18 | 19 | % write to textFile 20 | for i = 1:size(frameLabels,1) 21 | fprintf(fid, '%d %d %s \n', frameLabels(i,1), frameLabels(i,2), ['G' num2str(i,'%02d')]); 22 | end 23 | fclose(fid); 24 | 25 | end -------------------------------------------------------------------------------- /scripts/split_kinematics.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | import numpy as np 3 | import utils 4 | import constants 5 | import IPython 6 | 7 | # list_of_demonstrations = ["plane_5", "plane_6","plane_7","plane_8","plane_9", "plane_10"] 8 | list_of_demonstrations = ["lego_2", "lego_3", "lego_4", "lego_5", "lego_6", "lego_7"] 9 | 10 | POSE_DIM = 19 11 | JOINTS_DIM = 24 12 | 13 | for demonstration in list_of_demonstrations: 14 | X = pickle.load(open(constants.PATH_TO_KINEMATICS + demonstration + ".p", "rb")) 15 | assert X.shape[1] == (POSE_DIM + JOINTS_DIM) 16 | 17 | X_transpose = X.T 18 | 19 | X_POSE = X_transpose[:POSE_DIM].T 20 | X_JOINTS = X_transpose[:JOINTS_DIM].T 21 | 22 | assert X_POSE.shape[1] == POSE_DIM 23 | assert X_JOINTS.shape[1] == JOINTS_DIM 24 | 25 | assert X_POSE.shape[0] == X.shape[0] 26 | assert X_JOINTS.shape[0] == X.shape[0] 27 | 28 | pickle.dump(X_POSE, open(constants.PATH_TO_KINEMATICS + demonstration + "_POSE.p", "wb")) 29 | pickle.dump(X_JOINTS, open(constants.PATH_TO_KINEMATICS + demonstration + "_JOINTS.p", "wb")) -------------------------------------------------------------------------------- /scripts/jqm_cvi/LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE -------------------------------------------------------------------------------- /scripts/bash/rcnn_e2e.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | COUNT=1 4 | 5 | for CONTRAST in 0 0.5 6 | do 7 | for BLUR in 0 0.5 1 8 | do 9 | echo Classifing $COUNT BLUR $BLUR and contrast $CONTRAST 10 | 11 | econvert -i feature_extraction/rcnn-region-proposals/image-354.jpg --contrast $CONTRAST --blur $BLUR -o feature_extraction/rcnn-region-proposals/e2e_trial3/image-354_$COUNT.jpg 12 | 13 | sleep 5 14 | 15 | echo ~/caffe/examples/feature_extraction/rcnn-region-proposals/e2e_trial3/image-354_$COUNT.jpg > ~/caffe/examples/_temp/det_input.txt 16 | 17 | ../python/detect.py --crop_mode=selective_search --pretrained_model=../models/bvlc_reference_rcnn_ilsvrc13/bvlc_reference_rcnn_ilsvrc13.caffemodel --model_def=../models/bvlc_reference_rcnn_ilsvrc13/deploy.prototxt --gpu --raw_scale=255 _temp/det_input.txt _temp/det_output.h5 18 | 19 | sleep 5 20 | 21 | python detection.py image-354_$COUNT.jpg /home/animesh/caffe/examples/feature_extraction/rcnn-region-proposals/e2e_trial3/ 22 | 23 | let COUNT=COUNT+1 24 | sleep 5 25 | done 26 | done 27 | 28 | COUNT=1 29 | for CONTRAST in 0 0.5 30 | do 31 | for BLUR in 0 0.5 1 32 | do 33 | echo $COUNT CONTRAST $CONTRAST BLUR $BLUR 34 | let COUNT=COUNT+1 35 | done 36 | done -------------------------------------------------------------------------------- /scripts/convert_pickle.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pickle 3 | 4 | import constants 5 | import utils 6 | import parser 7 | 8 | list_of_joint_states = ["plane_3_js.p", "plane_4_js.p", "plane_5_js.p", 9 | "plane_6_js.p", "plane_7_js.p", "plane_8_js.p", "plane_9_js.p", "plane_10_js.p"] 10 | 11 | list_of_trajectories = ["plane_3.p", "plane_4.p", "plane_5.p", 12 | "plane_6.p", "plane_7.p", "plane_8.p", "plane_9.p", "plane_10.p"] 13 | 14 | list_of_annotations = ["plane_3_capture2.p", "plane_4_capture2.p", "plane_5_capture2.p", 15 | "plane_6_capture2.p", "plane_7_capture2.p", "plane_8_capture2.p", "plane_9_capture2.p", "plane_10_capture2.p"] 16 | 17 | for i in range(len(list_of_annotations)): 18 | print list_of_annotations[i], list_of_joint_states[i], list_of_trajectories[i] 19 | start, end = utils.get_start_end_annotations(constants.PATH_TO_DATA + "annotations/" + list_of_annotations[i]) 20 | X = None 21 | trajectory = pickle.load(open(constants.PATH_TO_KINEMATICS + list_of_joint_states[i], "rb")) 22 | for frm in range(start, end + 1): 23 | traj_point = trajectory[frm] 24 | print traj_point.velocity[16:-12] 25 | vector = list(traj_point.position[16:-12]) + list(traj_point.velocity[16:-12]) 26 | X = utils.safe_concatenate(X, utils.reshape(np.array(vector))) 27 | # pickle.dump(X, open(constants.PATH_TO_KINEMATICS + list_of_trajectories[i],"wb")) -------------------------------------------------------------------------------- /scripts/bash/e2e_c3d_fe_suturing.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This script converts all suturing .avi video files to .mat files which contain the c3d features 4 | FILES=/home/animesh/jigsaws/Suturing_video/video/* 5 | 6 | COUNT=1 7 | for f in $FILES; do 8 | # Make all necessary directories 9 | echo "-------------[DEBUG] $COUNT--------------" 10 | echo " Processing $f file......" 11 | cd /home/animesh/C3D/examples/c3d_feature_extraction/output/c3d/ 12 | mkdir $COUNT 13 | cd /home/animesh/C3D/examples/c3d_feature_extraction/input/frm/ 14 | mkdir $COUNT 15 | cd $COUNT 16 | cp $f . 17 | 18 | # Convert video to jpg images 19 | ffmpeg -i $f -r 16 %06d.jpg 20 | 21 | cd /home/animesh/C3D/examples/c3d_feature_extraction/script/ 22 | echo "-------------[DEBUG]--------------" 23 | echo "Type in number of 16-frame batches for file $f, followed by [ENTER]:" 24 | read NUM_BATCHES 25 | python generate_input_output_files.py input_list_frm_suturing.txt $COUNT output_list_prefix_suturing.txt $COUNT $NUM_BATCHES 26 | cd .. 27 | sh c3d_suturing_feature_extraction.sh 28 | 29 | # Convert feature file type to .mat files 30 | FEATURE_FILES=/home/animesh/C3D/examples/c3d_feature_extraction/output/c3d/$COUNT/* 31 | cd /home/animesh/C3D/examples/c3d_feature_extraction/script 32 | for file in $FEATURE_FILES; do 33 | mkdir $COUNT 34 | matlab -nosplash -nodesktop -nojvm -r "read_binary_blob('$file', '$COUNT');exit" 35 | done 36 | COUNT=$((COUNT+1)) 37 | done -------------------------------------------------------------------------------- /scripts/bash/c3d_experiment_2.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Experiment 1: Script loops through all layers in the C3D Net to extract features 4 | COUNT=1 5 | 6 | for LAYER in fc8-1 7 | do 8 | #First make all the folders 9 | echo Extrating features - $LAYER 10 | cd /home/animesh/C3D/examples/c3d_feature_extraction/output/c3d/experiment2/ 11 | mkdir $LAYER 12 | cd /home/animesh/C3D/examples/c3d_feature_extraction/script/data/experiment2/ 13 | mkdir $LAYER 14 | 15 | # Generate config files needed for Feature extraction 16 | cd /home/animesh/C3D/examples/c3d_feature_extraction/script/ 17 | python generate_input_output_files.py input_list_frm_suturing.txt Suturing_E003_capture2 output_list_prefix_suturing.txt experiment2/$LAYER 0 16 --segments data/Suturing_E003_capture2/Suturing_E003_capture2.p 18 | 19 | # Extract features from C3D 20 | cd /home/animesh/C3D/examples/c3d_feature_extraction/ 21 | GLOG_logtosterr=1 ../../build/tools/extract_image_features.bin prototxt/c3d_suturing_feature_extractor_frm.prototxt conv3d_deepnetA_sport1m_iter_1900000 -1 100 1 prototxt/output_list_prefix_suturing.txt $LAYER 22 | cp /home/animesh/C3D/examples/c3d_feature_extraction/output/c3d/experiment2/$LAYER/* script/data/experiment2/$LAYER/ 23 | sleep 5 24 | 25 | echo Plotting - $LAYER 26 | 27 | # Plot PCA of features 28 | cd script 29 | python tsne.py sutE5_cap2 $LAYER data/experiment2/$LAYER/ --a data/Suturing_E003_capture2/Suturing_E003_capture2.p 30 | COUNT=COUNT+1 31 | done -------------------------------------------------------------------------------- /scripts/bash/c3d_experiment_1.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Experiment 1: Script loops through all layers in the C3D Net to extract features 4 | COUNT=1 5 | 6 | for LAYER in conv2a conv3a conv3b conv4a conv4b conv5a conv5b fc6-1 fc7-1 fc8-1 7 | do 8 | # First make all the folders 9 | echo Extrating features - $LAYER 10 | cd /home/animesh/C3D/examples/c3d_feature_extraction/output/c3d/experiment1/ 11 | mkdir $LAYER 12 | cd /home/animesh/C3D/examples/c3d_feature_extraction/script/data/experiment1/ 13 | mkdir $LAYER 14 | 15 | # Generate config files needed for Feature extraction 16 | cd /home/animesh/C3D/examples/c3d_feature_extraction/script/ 17 | python generate_input_output_files.py input_list_frm_suturing.txt Suturing_E005_capture2 output_list_prefix_suturing.txt experiment1/$LAYER 0 16 --segments data/Suturing_E005_capture2/Suturing_E005_capture2.p 18 | 19 | # Extract features from C3D 20 | cd /home/animesh/C3D/examples/c3d_feature_extraction/ 21 | GLOG_logtosterr=1 ../../build/tools/extract_image_features.bin prototxt/c3d_suturing_feature_extractor_frm.prototxt conv3d_deepnetA_sport1m_iter_1900000 -1 100 1 prototxt/output_list_prefix_suturing.txt $LAYER 22 | cp /home/animesh/C3D/examples/c3d_feature_extraction/output/c3d/experiment1/$LAYER/* script/data/experiment1/$LAYER/ 23 | sleep 5 24 | 25 | echo Plotting - $LAYER 26 | 27 | #Plot PCA of features 28 | cd script 29 | 30 | python tsne.py sutE5_cap2 $LAYER data/experiment1/$LAYER/ --a data/Suturing_E005_capture2/Suturing_E005_capture2.p 31 | COUNT=COUNT+1 32 | done -------------------------------------------------------------------------------- /toyExample/utils/bg_Image.m: -------------------------------------------------------------------------------- 1 | function bg_Image(Image_file,Fig_position,reduc_fact,figHandle) 2 | % legend_Image(Image_file,Fig_position,reduc_fact) 3 | % Image_file= Image File name (full) 4 | % Fig_position: position on figure [1 2 3 4]= [U/L U/R D/R D/L] 5 | % reduc_fact : Ratio Image_size/Figure/size 6 | 7 | %% Figure example 8 | % hist(rand(1,2000),100); 9 | Dim1=get(figHandle,'position'); 10 | L =Dim1(1); D=Dim1(2); W=Dim1(3); H=Dim1(4); 11 | %% 12 | % Calculate the Image position on figure 13 | % reduction factor of the size 14 | im_W=W/reduc_fact; 15 | im_H=H/reduc_fact; 16 | 17 | switch Fig_position 18 | case 1 % Position 1 : Upper/Left 19 | im_L=L; 20 | im_D=D+H-im_H; 21 | case 2 % Position 2 : Upper/Right 22 | im_L=L+W-im_W; 23 | im_D=D+H-im_H; 24 | case 3 % Position 3 : Down/Right 25 | im_L=L+W-im_W; 26 | im_D=D; 27 | case 4 % Position 4 : Down/Left 28 | im_L=L; 29 | im_D=D; 30 | end 31 | ha = axes('units','normalized', ... 32 | 'position',[im_L im_D im_W im_H]); 33 | %% 34 | % Load in a background image and display it using the correct 35 | I=imread(Image_file); 36 | hi = imagesc(I); 37 | % colormap gray 38 | %% 39 | % Turn the handlevisibility off so that we don't inadvertently plot into the axes again 40 | % Also, make the axes invisible 41 | set(ha,'handlevisibility','off','visible','off'); 42 | 43 | % this creates transparency, you probably dont need it: 44 | % set(hi,'alphadata',.5) 45 | % % move the image to the top: 46 | uistack(ha,'bottom'); 47 | %% 48 | end 49 | -------------------------------------------------------------------------------- /scripts/dtw.py: -------------------------------------------------------------------------------- 1 | from numpy import array, zeros, argmin, inf 2 | from numpy.linalg import norm 3 | 4 | 5 | # File adapted from pierre-rouanet, May 2013 6 | 7 | def dtw(x, y, dist=lambda x, y: norm(x - y, ord=1)): 8 | """ Computes the DTW of two sequences. 9 | :param array x: N1*M array 10 | :param array y: N2*M array 11 | :param func dist: distance used as cost measure (default L1 norm) 12 | Returns the minimum distance, the accumulated cost matrix and the wrap path. 13 | """ 14 | x = array(x) 15 | if len(x.shape) == 1: 16 | x = x.reshape(-1, 1) 17 | y = array(y) 18 | if len(y.shape) == 1: 19 | y = y.reshape(-1, 1) 20 | 21 | r, c = len(x), len(y) 22 | 23 | D = zeros((r + 1, c + 1)) 24 | D[0, 1:] = inf 25 | D[1:, 0] = inf 26 | 27 | for i in range(r): 28 | for j in range(c): 29 | D[i+1, j+1] = dist(x[i], y[j]) 30 | 31 | for i in range(r): 32 | for j in range(c): 33 | D[i+1, j+1] += min(D[i, j], D[i, j+1], D[i+1, j]) 34 | 35 | D = D[1:, 1:] 36 | 37 | dist = D[-1, -1] / sum(D.shape) 38 | 39 | return dist, D, _trackeback(D) 40 | 41 | 42 | def _trackeback(D): 43 | i, j = array(D.shape) - 1 44 | p, q = [i], [j] 45 | while (i > 0 and j > 0): 46 | tb = argmin((D[i-1, j-1], D[i-1, j], D[i, j-1])) 47 | 48 | if (tb == 0): 49 | i = i - 1 50 | j = j - 1 51 | elif (tb == 1): 52 | i = i - 1 53 | elif (tb == 2): 54 | j = j - 1 55 | 56 | p.insert(0, i) 57 | q.insert(0, j) 58 | 59 | p.insert(0, 0) 60 | q.insert(0, 0) 61 | return (array(p), array(q)) -------------------------------------------------------------------------------- /scripts/encoding.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import IPython 4 | import numpy as np 5 | import matplotlib.pyplot as plt 6 | from sklearn import (manifold, datasets, decomposition, ensemble, lda, 7 | random_projection, preprocessing, covariance, cluster, neighbors) 8 | import utils 9 | 10 | def encode_cluster_normalize(X): 11 | return encode_VLAD(X, 5) 12 | 13 | def difference_vectors(X, cluster_predictions, clusters_centers): 14 | PC = X.shape[1] 15 | K = len(clusters_centers) 16 | u_k = {} 17 | num_frms = X.shape[0] 18 | for frm in range(num_frms): 19 | frm_NN_cluster = cluster_predictions[frm] 20 | c = clusters_centers[frm_NN_cluster] 21 | diff = X[frm] - c 22 | if frm_NN_cluster not in u_k: 23 | u_k[frm_NN_cluster] = diff 24 | else: 25 | sum_k = u_k[frm_NN_cluster] 26 | sum_k += diff 27 | u_k[frm_NN_cluster] = sum_k 28 | vlad = u_k[0] 29 | vlad = vlad.reshape(1, vlad.shape[0]) 30 | for k in range(1, K): 31 | K_cluster = u_k[k] 32 | K_cluster = K_cluster.reshape(1, K_cluster.shape[0]) 33 | 34 | # Intra Normalization 35 | K_cluster = preprocessing.normalize(K_cluster, norm = 'l2') 36 | vlad = np.concatenate((vlad, K_cluster), axis = 1) 37 | 38 | # L2 Normalization 39 | vlad = preprocessing.normalize(vlad, norm = 'l2') 40 | return vlad 41 | 42 | def encode_VLAD(X, K = 5): 43 | 44 | # X = utils.pca(X, PC = 256) 45 | 46 | kmeans = cluster.KMeans(init = 'k-means++', n_clusters = K) 47 | kmeans.fit(X) 48 | clusters_centers = kmeans.cluster_centers_ 49 | assert len(clusters_centers) == K 50 | 51 | # neigh = neighbors.KNeighborsClassifier(n_neighbors = 1) 52 | # Y = np.arange(K) 53 | # neigh.fit(clusters_centers, Y) 54 | # cluster_predictions = neigh.predict(X) 55 | # assert len(cluster_predictions) == X.shape[0] 56 | 57 | cluster_predictions = kmeans.predict(X) 58 | 59 | return difference_vectors(X, cluster_predictions, clusters_centers) 60 | 61 | -------------------------------------------------------------------------------- /scripts/generate_input_output_files.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import IPython 3 | import pickle 4 | 5 | # Author: Adithya Murali 6 | # UC Berkeley, 2015 7 | 8 | # Script to autogenerate config files specifying input frames for C3D 9 | 10 | def generate_files(args): 11 | 12 | input_list_path = "input/frm/" 13 | output_list_path = "output/c3d/" 14 | prototxt_path = "../prototxt/" 15 | 16 | input_list = open(prototxt_path + args.input_list_name, 'w') 17 | output_list = open(prototxt_path + args.output_list_name, 'w') 18 | fps = int(args.fps) 19 | 20 | if args.segments: 21 | all_segments = pickle.load(open(args.segments, "rb")) 22 | for index in all_segments: 23 | for elem in all_segments[index]: 24 | j = elem[0] 25 | while j <= (elem[1]): 26 | input_list.write(input_list_path + args.input_folder + "/ " + str(j) + " 0\n") 27 | output_list.write(output_list_path + args.output_folder + "/"+ str(j)+"\n") 28 | j += fps 29 | else: 30 | i = 1 31 | while i < (int(args.num_minutes) - 16): 32 | input_list.write(input_list_path + args.input_folder + "/ " + str(i) + " 0\n") 33 | output_list.write(output_list_path + args.output_folder + "/"+ str(i)+"\n") 34 | i += fps 35 | 36 | 37 | if __name__ == "__main__": 38 | 39 | parser = argparse.ArgumentParser() 40 | 41 | parser.add_argument("input_list_name", help = "input list to be populated") 42 | parser.add_argument("input_folder", help = "Location of input images") 43 | parser.add_argument("output_list_name", help = "ouput list to be populated") 44 | parser.add_argument("output_folder", help = "Location of output features") 45 | parser.add_argument("num_minutes", help = "Number of 16-frame batches") 46 | parser.add_argument("fps", help = "Frame rate - Frames per second", default = 1) 47 | parser.add_argument("--segments", help = "Pickle file with segments") 48 | 49 | args = parser.parse_args() 50 | generate_files(args) -------------------------------------------------------------------------------- /toyExample/utils/genLabelFile.m: -------------------------------------------------------------------------------- 1 | function [ ] = genLabelFile( frameLabels, outputDir, fileName ) 2 | 3 | if nargin <2 4 | fprintf('saving labels file in ./output/ \n') 5 | fileName = ['output' filesep 'frameLabels']; 6 | elseif nargin == 2 7 | mkdir_if_not_exist (outputDir); 8 | if outputDir(end) ~= '/', outputDir = [outputDir filesep]; end 9 | fileName = [outputDir 'frameLabels']; 10 | elseif nargin == 3 11 | mkdir_if_not_exist (outputDir); 12 | if outputDir(end) ~= '/', outputDir = [outputDir filesep]; end 13 | fileName = [outputDir fileName]; 14 | end 15 | 16 | % make textFile 17 | fid = fopen( [fileName '.txt'], 'wt' ); 18 | 19 | % write to textFile 20 | t= 1; 21 | label = {}; 22 | while t < size(frameLabels,2) 23 | if frameLabels(:, t) == frameLabels(:, t+1)%if all values are same assign rotation 24 | if t == 1, label{t} = 'G2'; end 25 | label{t+1} = 'G2';%rotation 26 | elseif frameLabels(3, t) == frameLabels(3, t+1) 27 | if t == 1, label{t} = 'G1'; end 28 | label{t+1} = 'G1';%movement 29 | elseif frameLabels(1:2, t) == frameLabels(1:2, t+1) 30 | if t == 1, label{t} = 'G2'; end 31 | label{t+1} = 'G2';%rotation 32 | else 33 | fprintf('Can only have two labels--check data \n') 34 | disp(frameLabels(:,t)) 35 | end 36 | t = t+1; 37 | end 38 | 39 | startFrame = 1; 40 | for i = 1:length(label)-1 41 | if ~strcmp(label{i}, label{i+1}) 42 | %write to file 43 | fprintf(fid, '%d %d %s \n', startFrame, i, label{startFrame}); 44 | startFrame = i+1; %re-init counter 45 | elseif i == length(label)-1 %for last surgeme 46 | fprintf(fid, '%d %d %s \n', startFrame, i+1, label{startFrame}); 47 | end 48 | end 49 | 50 | fclose(fid); 51 | 52 | end -------------------------------------------------------------------------------- /toyExample/utils/plotState_baseline.m: -------------------------------------------------------------------------------- 1 | function [ figOutput ] = plotState_baseline( x_curr, target_curr, fAx, background) 2 | if nargin < 2 3 | fprintf('plotState needs at least 2 arguments') 4 | 5 | elseif nargin == 2 6 | if ~exist('x_curr', 'var') 7 | x_curr = []; 8 | end 9 | if ~exist('target_curr', 'var') 10 | target_curr = [1,1]; 11 | end 12 | 13 | elseif nargin == 3 14 | if ~exist('fAx', 'var')%Input axis object to plot on 15 | figInput = figure(); 16 | fAx = axes('parent',figInput); 17 | hold (fAx, 'on') 18 | axesLim = [-10 10 -10 10]; 19 | axis(fAx, axesLim); 20 | hold on 21 | end 22 | elseif nargin ==4 23 | if ~exist('background', 'var') 24 | % background = load('images/whiteBG.jpg'); 25 | set(figInput,'color','w'); 26 | elseif background == 'default' 27 | background = 'images/living-room.jpg'; 28 | bg_Image(background, 1, 1, figInput) 29 | else 30 | bg_Image(['images' filesep background], 1, 1, figInput) 31 | end 32 | end 33 | 34 | %% plot target as flash light 35 | hTarget = plot(fAx,target_curr(1),target_curr(2), ... 36 | 'Marker','o', ... 37 | 'MarkerEdgeColor', 'r', ... 38 | 'MarkerFaceColor', 'y', ... 39 | 'MarkerSize',40 ); 40 | % hTarget.MarkerHandle.EdgeColorData = uint8(255*[1;0;0;0.5]); 41 | 42 | %% plot robot as oriented traingle 43 | hRobot_body = plot(fAx,x_curr(1),x_curr(2), ... 44 | 'Marker','o', ... 45 | 'MarkerFaceColor', 'b',... 46 | 'MarkerEdgeColor', 'b', ... 47 | 'MarkerSize',25 ); 48 | % hRobot.MarkerHandle.EdgeColorData = uint8(255*[0;0;1;0.5]); 49 | % hRobot.MarkerHandle.FaceColorData = uint8(255*[0;0;1;0.5]); 50 | 51 | %% 52 | figOutput = [hTarget, hRobot_body ]; %return figure handle 53 | 54 | 55 | end 56 | 57 | 58 | 59 | -------------------------------------------------------------------------------- /scripts/jqm_cvi/README.md: -------------------------------------------------------------------------------- 1 | jqm_cvi 2 | ======= 3 | 4 | Small module with Cluster Validity Indices (CVI) 5 | ------------------------------------------------ 6 | 7 | Dunn and Davius Bouldin indices are implemented. It follows the equations presented in theory.pdf. 8 | 9 | > base.py : Python + NumPy 10 | > 11 | > basec.pyx : Python + NumPy optimized with Cython 12 | > 13 | > basec.pyx tested in Windows 8.1 x64, Python 3.4 and compiled with VS2010 (python setup.py build_ext -i) 14 | 15 | Functions: 16 | ---------- 17 | 18 | **dunn(k_list)**: 19 | > Slow implementation of Dunn index that depends on numpy 20 | > 21 | > -- basec.pyx Cython implementation is much faster but slower than dunn_fast() 22 | 23 | ```python 24 | """ Dunn index [CVI] 25 | 26 | Parameters 27 | ---------- 28 | k_list : list of np.arrays 29 | A list containing a numpy array for each cluster |c| = number of clusters 30 | c[K] is np.array([N, p]) (N : number of samples in cluster K, p : sample dimension) 31 | """ 32 | ``` 33 | 34 | **dunn_fast(points, labels)**: 35 | > Fast implementation of Dunn index that depends on numpy and sklearn.pairwise 36 | > 37 | > -- No Cython implementation 38 | 39 | ```python 40 | """ Dunn index - FAST (using sklearn pairwise euclidean_distance function) 41 | 42 | Parameters 43 | ---------- 44 | points : np.array 45 | np.array([N, p]) of all points 46 | labels: np.array 47 | np.array([N]) labels of all points 48 | """ 49 | ``` 50 | 51 | **davisbouldin(k_list, k_centers)**: 52 | > Implementation of Davis Boulding index that depends on numpy 53 | > 54 | > -- basec.pyx Cython implementation is much faster 55 | 56 | ```python 57 | """ Davis Bouldin Index 58 | 59 | Parameters 60 | ---------- 61 | k_list : list of np.arrays 62 | A list containing a numpy array for each cluster |c| = number of clusters 63 | c[K] is np.array([N, p]) (N : number of samples in cluster K, p : sample dimension) 64 | k_centers : np.array 65 | The array of the cluster centers (prototypes) of type np.array([K, p]) 66 | """ 67 | ``` -------------------------------------------------------------------------------- /toyExample/utils/plotState.m: -------------------------------------------------------------------------------- 1 | function [ figOutput ] = plotState( x_curr, target_curr, fAx, background) 2 | if nargin < 2 3 | fprintf('plotState needs at least 2 arguments') 4 | 5 | elseif nargin == 2 6 | if ~exist('x_curr', 'var') 7 | x_curr = []; 8 | end 9 | if ~exist('target_curr', 'var') 10 | target_curr = [1,1]; 11 | end 12 | 13 | elseif nargin == 3 14 | if ~exist('fAx', 'var')%Input axis object to plot on 15 | figInput = figure(); 16 | fAx = axes('parent',figInput); 17 | hold (fAx, 'on') 18 | axesLim = [-10 10 -10 10]; 19 | axis(fAx, axesLim); 20 | hold on 21 | end 22 | elseif nargin ==4 23 | if ~exist('background', 'var') 24 | % background = load('images/whiteBG.jpg'); 25 | set(figInput,'color','w'); 26 | elseif background == 'default' 27 | background = 'images/living-room.jpg'; 28 | bg_Image(background, 1, 1, figInput) 29 | else 30 | bg_Image(['images' filesep background], 1, 1, figInput) 31 | end 32 | end 33 | 34 | %% plot target as flash light 35 | hTarget = plot(fAx,target_curr(1),target_curr(2), ... 36 | 'Marker','o', ... 37 | 'MarkerEdgeColor', 'r', ... 38 | 'MarkerFaceColor', 'y', ... 39 | 'MarkerSize',40 ); 40 | % hTarget.MarkerHandle.EdgeColorData = uint8(255*[1;0;0;0.5]); 41 | 42 | %% plot robot as oriented traingle 43 | hRobot_body = plot(fAx,x_curr(1),x_curr(2), ... 44 | 'Marker','o', ... 45 | 'MarkerFaceColor', 'b',... 46 | 'MarkerEdgeColor', 'b', ... 47 | 'MarkerSize',25 ); 48 | % hRobot.MarkerHandle.EdgeColorData = uint8(255*[0;0;1;0.5]); 49 | % hRobot.MarkerHandle.FaceColorData = uint8(255*[0;0;1;0.5]); 50 | theta = x_curr(3); 51 | r = 1; 52 | hRobot_heading = plot (fAx,[x_curr(1), x_curr(1)+r*cos(theta)], ... 53 | [x_curr(2), x_curr(2)+r*sin(theta)], ... 54 | 'b-', 'LineWidth', 3); 55 | 56 | %% 57 | figOutput = [hTarget, hRobot_body, hRobot_heading ]; %return figure handle 58 | 59 | 60 | end 61 | 62 | 63 | 64 | -------------------------------------------------------------------------------- /scripts/cluster_evaluation.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from sklearn import mixture 3 | import IPython 4 | import utils 5 | import scipy.spatial.distance as distance 6 | 7 | def split(points, predictions): 8 | points_list = {} 9 | for i in range(len(predictions)): 10 | label = predictions[i] 11 | if label not in points_list: 12 | points_list[label] = utils.reshape(points[i]) 13 | else: 14 | curr = points_list[label] 15 | curr = np.concatenate((curr, utils.reshape(points[i])), axis = 0) 16 | points_list[label] = curr 17 | return points_list 18 | 19 | def dunn_index(points, predictions, means): 20 | if len(points) == 0: 21 | return [None, None, None] 22 | points_in_clusters = split(points, predictions) 23 | delta_list_1 = [] 24 | delta_list_2 = [] 25 | delta_list_3 = [] 26 | 27 | # Wikipedia Definition No. 1 for Delta - maximum distance between all point-pairs in cluster 28 | for cluster in points_in_clusters.keys(): 29 | if len(points_in_clusters[cluster]) > 1: 30 | try: 31 | delta_list_1.append(max(distance.pdist(points_in_clusters[cluster], 'euclidean'))) 32 | except ValueError as e: 33 | print e 34 | IPython.embed() 35 | 36 | # Wikipedia Definition No. 2 for Delta - mean distance between all point-pairs in cluster 37 | for cluster in points_in_clusters.keys(): 38 | if len(points_in_clusters[cluster]) > 1: 39 | delta_list_2.append(np.mean(distance.pdist(points_in_clusters[cluster], 'euclidean'))) 40 | 41 | # Wikipedia Definition No. 3 for Delta - distance of all points from mean 42 | for cluster in points_in_clusters.keys(): 43 | if len(points_in_clusters[cluster]) > 1: 44 | delta_list_3.append(np.mean(distance.cdist(points_in_clusters[cluster], utils.reshape(means[cluster]), 'euclidean'))) 45 | 46 | del_list = distance.pdist(means, 'euclidean') 47 | 48 | try: 49 | dunn_index_1 = min(del_list) / max(delta_list_1) 50 | dunn_index_2 = min(del_list) / max(delta_list_2) 51 | dunn_index_3 = min(del_list) / max(delta_list_3) 52 | except ValueError as e: 53 | print e 54 | return [None, None, None] 55 | 56 | return [dunn_index_1, dunn_index_2, dunn_index_3] 57 | 58 | if __name__ == "__main__": 59 | points = np.arange(4000).reshape(10, 400) 60 | 61 | g = mixture.GMM(n_components=4) 62 | g.fit(points) 63 | 64 | predictions = g.predict(points) 65 | means = g.means_ 66 | 67 | 68 | -------------------------------------------------------------------------------- /scripts/background_subtraction.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import cv2 3 | import argparse 4 | import sys 5 | import matplotlib.pyplot as plt 6 | import IPython 7 | 8 | import parser 9 | import utils 10 | import constants 11 | 12 | def run_video_with_bsub(cap, func, kernel = None, params = None): 13 | if params is not None: 14 | fgbg = func(params[0], params[1], params[2], params[3]) 15 | else: 16 | fgbg = func() 17 | SAVE_PATH = constants.PATH_TO_DATA + constants.NEW_BGSUB_FOLDER + "Suturing_E003_capture2/" 18 | i = 1 19 | while(1): 20 | ret, frame = cap.read() 21 | print i 22 | fgmask = fgbg.apply(frame) 23 | mask_rbg = cv2.cvtColor(fgmask, cv2.COLOR_GRAY2BGR) 24 | draw = frame & mask_rbg 25 | if frame == None and mask_rbg == None: 26 | sys.exit() 27 | if kernel is not None: 28 | fgmask = cv2.morphologyEx(fgmask, cv2.MORPH_OPEN, kernel) 29 | 30 | # cv2.namedWindow("frame", cv2.WND_PROP_FULLSCREEN) 31 | # cv2.setWindowProperty("frame", cv2.WND_PROP_FULLSCREEN, cv2.cv.CV_WINDOW_FULLSCREEN) 32 | cv2.imshow('frame', draw) 33 | cv2.imwrite(SAVE_PATH + utils.get_frame_fig_name(i), draw) 34 | k = cv2.waitKey(30) & 0xff 35 | i += 1 36 | if k == 27: 37 | break 38 | 39 | cap.release() 40 | cv2.destroyAllWindows() 41 | 42 | if __name__ == "__main__": 43 | parser = argparse.ArgumentParser() 44 | parser.add_argument("video", help = "A: Suturing: B: Pizza making") 45 | parser.add_argument("type", help = "1: MOG 2: MOG2 3: Ken's algorithm") 46 | args = parser.parse_args() 47 | cap = None 48 | if args.video == 'A': 49 | cap = cv2.VideoCapture('/home/animesh/DeepMilestones/jigsaws/Suturing_video/frames/Suturing_E003_capture2/cropped_scaled.avi') 50 | elif args.video == 'B': 51 | cap = cv2.VideoCapture('/home/animesh/C3D/examples/c3d_feature_extraction/input/frm/pizza8/videos/cropped_scaled.avi') 52 | else: 53 | print "Invalid video type" 54 | sys.exit() 55 | 56 | if (int(args.type) == 1): 57 | params = (500, 10, 0.9, 1) 58 | run_video_with_bsub(cap, cv2.BackgroundSubtractorMOG, params = None) 59 | elif (int(args.type) == 2): 60 | run_video_with_bsub(cap, cv2.BackgroundSubtractorMOG2) 61 | elif (int(args.type) == 3): 62 | kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(3,3)) 63 | run_video_with_bsub(cap, cv2.createBackgroundSubtractorGMG, kernel = kernel) 64 | else: 65 | print "Error Type" 66 | sys.exit() -------------------------------------------------------------------------------- /scripts/constants_sk.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | CAFFE_ROOT = '/home/animesh/caffe/' 4 | 5 | PATH_TO_DATA = "/home/animesh/DeepMilestones/jigsaws/Suturing_video/" 6 | 7 | PATH_TO_CLUSTERING_RESULTS = "/home/animesh/DeepMilestones/clustering_FCED/" 8 | 9 | PATH_TO_KINEMATICS = "/home/animesh/DeepMilestones/jigsaws/Suturing_kinematics/kinematics/AllGestures/" 10 | 11 | color_map = {1:'b', 2:'g', 3:'r', 4:'c', 5: 'm', 6:'y', 7:'k', 8:'#4B0082', 9: '#9932CC', 10: '#E9967A', 11: '#800000', 12: '#008080'} 12 | 13 | alphabet_map = {1: "A", 2: "B", 3: "C", 4: "D", 5: "E", 6: "F", 7: "G", 8: "H", 9:"I", 10: "J", 14 | 11: "K", 12: "L", 13: "M", 14: "N", 15: "O", 16: "P"} 15 | 16 | # Extra colors added: 17 | # E9967A is beige/dark salmon 18 | # 4B0082 is Indigo 19 | # 800000 is Maroon 20 | # 008080 IS Teal 21 | 22 | alex_net_layers = ['conv1', 'conv2', 'conv3', 'conv4', 'conv5', 'pool5', 'fc6', 'fc7'] 23 | 24 | vgg_layers = ['conv1_1', 'conv1_2', 'conv2_1', 'conv2_2', 'conv3_1', 'conv3_2', 'conv3_3', 'conv4_1', 'conv4_2', 'conv4_3','conv5_1', 'conv5_2', 'conv5_3', 'pool5'] 25 | 26 | PATH_TO_SAVE_FIG = '/home/animesh/DeepMilestones/plots_sk/' 27 | 28 | # constants.VGG_MEAN = np.array([123.68, 116.779, 103.939]) RGB 0-255 scale 29 | VGG_MEAN = np.array([ 0.48501961, 0.45795686, 0.40760392]) 30 | 31 | NET_PARAMS = {"AlexNet": [CAFFE_ROOT + 'models/bvlc_reference_caffenet/deploy.prototxt', CAFFE_ROOT + 'models/bvlc_reference_caffenet/bvlc_reference_caffenet.caffemodel', 32 | alex_net_layers], "VGG_SOS": [CAFFE_ROOT + 'models/vgg_sos/deploy.prototxt', CAFFE_ROOT + 'models/vgg_sos/VGG16_SalObjSub.caffemodel', vgg_layers], 33 | "VGG": [CAFFE_ROOT + 'models/vgg/deploy.prototxt', CAFFE_ROOT + 'models/vgg/VGG_ILSVRC_16_layers.caffemodel', vgg_layers]} 34 | 35 | CONFIG_FILE = "meta_file_Suturing.txt" 36 | 37 | VIDEO_FOLDER = "video/" 38 | 39 | NEW_FRAMES_FOLDER = "frames/" 40 | 41 | NEW_BGSUB_FOLDER = "bgsubframes/" 42 | 43 | TRANSCRIPTIONS_FOLDER = "transcriptions/" 44 | 45 | ANNOTATIONS_FOLDER = "annotations/" 46 | 47 | ALEXNET_FEATURES_FOLDER = "alexnetfeatures_2/" 48 | 49 | VGG_FEATURES_FOLDER = "vggfeatures_2/" 50 | 51 | PROC_FEATURES_FOLDER = "features_FCED/" 52 | 53 | CROP_PARAMS = {"capture2": "\"crop=330:260:150:150\"", "capture1": "\"crop=330:260:200:170\""} 54 | 55 | map_surgeme_label = {'G1': 1, "G2": 2, "G3": 3, "G4": 4, "G5": 5, "G6": 6, "G7": 7, "G8": 8, "G9": 9, 56 | "G10": 10, "G12": 12, "G11": 11, "G13": 13, "G14": 14, "G15": 15, "G16": 16, "G17": 17} 57 | 58 | train_test_ratio = 0.3 59 | -------------------------------------------------------------------------------- /scripts/cv_optflw.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import cv2 3 | import IPython 4 | 5 | cap = cv2.VideoCapture("/home/animesh/DeepMilestones/jigsaws/Suturing_video/frames/Suturing_E003_capture1/cropped_scaled.avi") 6 | while not cap.isOpened(): 7 | cap = cv2.VideoCapture("/home/animesh/DeepMilestones/jigsaws/Suturing_video/frames/Suturing_E003_capture1/cropped_scaled.avi") 8 | cv2.waitKey(1000) 9 | print "Wait for the header" 10 | 11 | num_features_tracked = 10 12 | 13 | # params for ShiTomasi corner detection 14 | feature_params = dict( maxCorners = num_features_tracked, 15 | qualityLevel = 0.3, 16 | minDistance = 7, 17 | blockSize = 7 ) 18 | 19 | # Parameters for lucas kanade optical flow 20 | lk_params = dict( winSize = (15,15), 21 | maxLevel = 2, 22 | criteria = (cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT, 10, 0.03)) 23 | 24 | # Create some random colors 25 | color = np.random.randint(0,255,(num_features_tracked,3)) 26 | 27 | # Take first frame and find corners in it 28 | ret, old_frame = cap.read() 29 | old_gray = cv2.cvtColor(old_frame, cv2.COLOR_BGR2GRAY) 30 | p0 = cv2.goodFeaturesToTrack(old_gray, mask = None, **feature_params) 31 | 32 | #print old_frame 33 | 34 | # Create a mask image for drawing purposes 35 | mask = np.zeros_like(old_frame) 36 | 37 | j = 1 38 | pleaseWait = False 39 | 40 | while(1): 41 | print j 42 | ret,frame = cap.read() 43 | 44 | frame_gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) 45 | 46 | # calculate optical flow 47 | p1, st, err = cv2.calcOpticalFlowPyrLK(old_gray, frame_gray, p0, None, **lk_params) 48 | 49 | # Select good points 50 | good_new = p1[st==1] 51 | good_old = p0[st==1] 52 | 53 | # draw the tracks 54 | for i,(new,old) in enumerate(zip(good_new,good_old)): 55 | a,b = new.ravel() 56 | c,d = old.ravel() 57 | cv2.line(mask, (a,b),(c,d), color[i].tolist(), 2) 58 | cv2.circle(frame,(a,b),5,color[i].tolist(),-1) 59 | img = cv2.add(frame,mask) 60 | 61 | if pleaseWait: 62 | pleaseWait = False 63 | IPython.embed() 64 | 65 | # cv2.namedWindow("frame", cv2.WND_PROP_FULLSCREEN) 66 | # cv2.setWindowProperty("frame", cv2.WND_PROP_FULLSCREEN, cv2.cv.CV_WINDOW_FULLSCREEN) 67 | cv2.imshow("frame",img) 68 | k = cv2.waitKey(30) & 0xff 69 | if k == 27: 70 | break 71 | 72 | # Now update the previous frame and previous points 73 | old_gray = frame_gray.copy() 74 | p0 = good_new.reshape(-1,1,2) 75 | j += 1 76 | 77 | cv2.destroyAllWindows() 78 | cap.release() -------------------------------------------------------------------------------- /scripts/segmenter.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | import IPython 3 | import argparse 4 | 5 | # Author: Adithya Murali 6 | # UC Berkeley, 2015 7 | 8 | # Script to assist in manual segmentation of videos. Saves segment data, index and labels 9 | # in Pickle file and txt document. 10 | 11 | class Segmenter(): 12 | def __init__(self, output_name, output_path): 13 | self.output_name = output_name 14 | self.map_index_data = {} 15 | self.map_index_labels = {} 16 | self.output_path = output_path 17 | pass 18 | def segment(self): 19 | # Specify segments and index 20 | exit = False 21 | i = 1 22 | while not exit: 23 | user_ret = raw_input("Enter new segment name: ") 24 | self.map_index_labels[i] = str(user_ret) 25 | self.map_index_data[i] = [] 26 | i += 1 27 | user_ret = raw_input("Done with specifing segments?[y/n]") 28 | if user_ret == 'y': 29 | exit = True 30 | print "----------------Collecting Segments ----------------------" 31 | exit = False 32 | while not exit: 33 | self.print_all_index_labels() 34 | index = int(raw_input("Which index?")) 35 | start_frm = int(raw_input("Starting frame?")) 36 | end_frm = int(raw_input("End frame?")) 37 | new_segment = (start_frm, end_frm) 38 | segment_list = self.map_index_data[index] 39 | segment_list.append(new_segment) 40 | self.map_index_data[index] = segment_list 41 | user_ret = raw_input("Done with specifing segments?[y/n]") 42 | if user_ret == 'y': 43 | exit = True 44 | print self.map_index_data 45 | IPython.embed() 46 | 47 | def write_to_picke(self): 48 | pickle.dump(self.map_index_data, open(self.output_path + self.output_name + ".p", "wb")) 49 | 50 | def write_to_txt(self): 51 | f = open(self.output_path + self.output_name + ".txt", "wb") 52 | f.write("Index and Labels:\n") 53 | for index in self.map_index_labels: 54 | f.write(str(index) + " " + self.map_index_labels[index] + '\n') 55 | f.write("\n") 56 | f.write("Index and Data\n") 57 | for index in self.map_index_data: 58 | data = self.map_index_data[index] 59 | f.write(str(index) + " " + str(self.map_index_data[index]) + '\n') 60 | f.write("\n") 61 | f.write("~ Fin ~ ") 62 | def print_all_index_labels(self): 63 | for index in self.map_index_labels.keys(): 64 | print str(index) + ": " + self.map_index_labels[index] 65 | 66 | if __name__ == "__main__": 67 | parser = argparse.ArgumentParser() 68 | parser.add_argument("output_name", help = "Please specify output name for pickle file and txt annotation") 69 | parser.add_argument("output_path", help = "Path to output files") 70 | args = parser.parse_args() 71 | seg = Segmenter(args.output_name, args.output_path) 72 | seg.segment() 73 | seg.write_to_txt() 74 | seg.write_to_picke() 75 | -------------------------------------------------------------------------------- /scripts/pruning.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import IPython 4 | import pickle 5 | import numpy as np 6 | 7 | import parser 8 | import constants 9 | import utils 10 | 11 | experts = ["E001", "E002", "E003", "E004", "E005", "D001", "D002", "D003", "D004", "D005"] 12 | 13 | intermediates = ["F001", "F002", "F003", "F004", "F005", "C001", "C002", "C003", "C004", "C005"] 14 | 15 | novices = ["G001", "G002", "G003", "G004", "G005", "B001", "B002", "B003", "B004", "B005", 16 | "I001", "I002", "I003", "I004", "I005", "H001", "H002", "H003", "H004", "H005"] 17 | 18 | def weighted_score(list_of_demonstrations, list_of_frm_demonstrations): 19 | """ 20 | Implements weighted pruning for given demonstrations represented in list_of_frm_demonstrations. 21 | Returns weighted score. 22 | """ 23 | 24 | if constants.TASK_NAME not in ["suturing", "needle_passing"]: 25 | return None 26 | 27 | if not constants.WEIGHTED_PRUNING_MODE: 28 | return None 29 | 30 | N = float(len(list_of_demonstrations)) 31 | uniform_weight = 1/N 32 | 33 | # Weights for skill-weighted pruning, where each demonstration has the same weight for 34 | map_demonstration2weight = {} 35 | 36 | # Weight is inversely proportional to completion time. 37 | map_demonstration2weight_t = {} 38 | 39 | for demonstration in list_of_demonstrations: 40 | 41 | # Base weight 42 | weight = uniform_weight 43 | 44 | PATH_TO_ANNOTATION = constants.PATH_TO_DATA + constants.ANNOTATIONS_FOLDER + demonstration + "_" + constants.CAMERA + ".p" 45 | start, end = utils.get_start_end_annotations(PATH_TO_ANNOTATION) 46 | weight_t = 1.0/(end - start) 47 | 48 | # Stripped demonstration task (Suturing_, Needle_passing_, etc.) from demonstration name 49 | demonstration_name = demonstration.split("_")[-1] 50 | 51 | if demonstration_name in experts: 52 | weight *= constants.WEIGHT_EXPERT 53 | elif demonstration_name in intermediates: 54 | weight *= constants.WEIGHT_INTERMEDIATE 55 | else: 56 | if demonstration_name not in novices: 57 | print "ERROR: Unidentified Demonstration" 58 | IPython.embed() 59 | map_demonstration2weight[demonstration] = weight 60 | map_demonstration2weight_t[demonstration] = weight_t 61 | 62 | normalization_factor = sum(map_demonstration2weight.values()) 63 | normalization_factor_t = sum(map_demonstration2weight_t.values()) 64 | 65 | #Weight normalization 66 | for demonstration in list_of_demonstrations: 67 | weight = map_demonstration2weight[demonstration] 68 | map_demonstration2weight[demonstration] = weight/float(normalization_factor) 69 | 70 | weight_t = map_demonstration2weight_t[demonstration] 71 | map_demonstration2weight_t[demonstration] = weight_t/float(normalization_factor_t) 72 | 73 | score = 0.0 74 | for demonstration in set(list_of_frm_demonstrations): 75 | score += map_demonstration2weight[demonstration] 76 | 77 | score_t = 0.0 78 | for demonstration in set(list_of_frm_demonstrations): 79 | score_t += map_demonstration2weight_t[demonstration] 80 | 81 | return score_t -------------------------------------------------------------------------------- /scripts/examples/plot_gmm.py: -------------------------------------------------------------------------------- 1 | """ 2 | ================================= 3 | Gaussian Mixture Model Ellipsoids 4 | ================================= 5 | 6 | Plot the confidence ellipsoids of a mixture of two Gaussians with EM 7 | and variational Dirichlet process. 8 | 9 | Both models have access to five components with which to fit the 10 | data. Note that the EM model will necessarily use all five components 11 | while the DP model will effectively only use as many as are needed for 12 | a good fit. This is a property of the Dirichlet Process prior. Here we 13 | can see that the EM model splits some components arbitrarily, because it 14 | is trying to fit too many components, while the Dirichlet Process model 15 | adapts it number of state automatically. 16 | 17 | This example doesn't show it, as we're in a low-dimensional space, but 18 | another advantage of the Dirichlet process model is that it can fit 19 | full covariance matrices effectively even when there are less examples 20 | per cluster than there are dimensions in the data, due to 21 | regularization properties of the inference algorithm. 22 | """ 23 | import itertools 24 | 25 | import numpy as np 26 | from scipy import linalg 27 | import matplotlib.pyplot as plt 28 | import matplotlib as mpl 29 | import IPython 30 | 31 | from sklearn import mixture 32 | 33 | # Number of samples per component 34 | n_samples = 500 35 | 36 | # Generate random sample, two components 37 | np.random.seed(0) 38 | C = np.array([[0., -0.1], [1.7, .4]]) 39 | X = np.r_[np.dot(np.random.randn(n_samples, 2), C), 40 | .7 * np.random.randn(n_samples, 2) + np.array([-6, 3])] 41 | 42 | # Fit a mixture of Gaussians with EM using five components 43 | gmm = mixture.GMM(n_components=5, covariance_type='full') 44 | gmm.fit(X) 45 | 46 | # Fit a Dirichlet process mixture of Gaussians using five components 47 | dpgmm = mixture.DPGMM(n_components=5, covariance_type='full') 48 | dpgmm.fit(X) 49 | 50 | IPython.embed() 51 | 52 | color_iter = itertools.cycle(['r', 'g', 'b', 'c', 'm']) 53 | 54 | for i, (clf, title) in enumerate([(gmm, 'GMM'), 55 | (dpgmm, 'Dirichlet Process GMM')]): 56 | splot = plt.subplot(2, 1, 1 + i) 57 | Y_ = clf.predict(X) 58 | IPython.embed() 59 | for i, (mean, covar, color) in enumerate(zip( 60 | clf.means_, clf._get_covars(), color_iter)): 61 | v, w = linalg.eigh(covar) 62 | u = w[0] / linalg.norm(w[0]) 63 | # as the DP will not use every component it has access to 64 | # unless it needs it, we shouldn't plot the redundant 65 | # components. 66 | if not np.any(Y_ == i): 67 | continue 68 | plt.scatter(X[Y_ == i, 0], X[Y_ == i, 1], .8, color=color) 69 | 70 | # Plot an ellipse to show the Gaussian component 71 | angle = np.arctan(u[1] / u[0]) 72 | angle = 180 * angle / np.pi # convert to degrees 73 | ell = mpl.patches.Ellipse(mean, v[0], v[1], 180 + angle, color=color) 74 | ell.set_clip_box(splot.bbox) 75 | ell.set_alpha(0.5) 76 | splot.add_artist(ell) 77 | 78 | plt.xlim(-10, 10) 79 | plt.ylim(-3, 6) 80 | plt.xticks(()) 81 | plt.yticks(()) 82 | plt.title(title) 83 | 84 | plt.show() 85 | -------------------------------------------------------------------------------- /scripts/bash/suturing_varying_featurizations.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cd .. 4 | 5 | echo "suturing.yaml" > ../config/defaultconfig 6 | 7 | echo "[KINEMATICS - W]" 8 | python tscdl.py W W 9 | 10 | echo "[VISUAL - Z] ", SIFT_1.p 11 | python tscdl.py Z Z_SIFT_1 --visual_feature SIFT_1.p 12 | 13 | echo "[VISUAL - Z] ", SIFT_2.p 14 | python tscdl.py Z Z_SIFT_2 --visual_feature SIFT_2.p 15 | 16 | echo "[VISUAL - Z] ", 2_PCA.p 17 | python tscdl.py Z Z_2_PCA --visual_feature 2_PCA.p 18 | 19 | echo "[VISUAL - Z] ", 2_CCA.p 20 | python tscdl.py Z Z_2_CCA --visual_feature 2_CCA.p 21 | 22 | echo "[VISUAL - Z] ", 2_GRP.p 23 | python tscdl.py Z Z_2_GRP --visual_feature 2_GRP.p 24 | 25 | echo "[VISUAL - Z] ", 3_PCA.p 26 | python tscdl.py Z Z_3_PCA --visual_feature 3_PCA.p 27 | 28 | echo "[VISUAL - Z] ", 3_GRP.p 29 | python tscdl.py Z Z_3_GRP --visual_feature 3_GRP.p 30 | 31 | echo "[VISUAL - Z] ", 3_CCA.p 32 | python tscdl.py Z Z_3_CCA --visual_feature 3_CCA.p 33 | 34 | echo "[VISUAL - Z] ", 4_PCA.p 35 | python tscdl.py Z Z_4_PCA --visual_feature 4_PCA.p 36 | 37 | echo "[VISUAL - Z] ", 4_GRP.p 38 | python tscdl.py Z Z_4_GRP --visual_feature 4_GRP.p 39 | 40 | echo "[VISUAL - Z] ", 4_CCA.p 41 | python tscdl.py Z Z_4_CCA --visual_feature 4_CCA.p 42 | 43 | echo "[VISUAL - Z] ", 5_PCA.p 44 | python tscdl.py Z Z_5_PCA --visual_feature 5_PCA.p 45 | 46 | echo "[VISUAL - Z] ", 5_GRP.p 47 | python tscdl.py Z Z_5_GRP --visual_feature 5_GRP.p 48 | 49 | echo "[VISUAL - Z] ", 5_CCA.p 50 | python tscdl.py Z Z_5_CCA --visual_feature 5_CCA.p 51 | 52 | echo "[VISUAL - Z] ", 7_PCA.p 53 | python tscdl.py Z Z_7_PCA --visual_feature 7_PCA.p 54 | 55 | echo "[VISUAL - Z] ", 7_GRP.p 56 | python tscdl.py Z Z_7_GRP --visual_feature 7_GRP.p 57 | 58 | echo "[VISUAL - Z] ", 7_CCA.p 59 | python tscdl.py Z Z_7_CCA --visual_feature 7_CCA.p 60 | 61 | 62 | echo "[KINEMATICS+VISUAL - ZW]" 63 | python tscdl.py ZW ZW_SIFT_1 --visual_feature SIFT_1.p 64 | echo "[KINEMATICS+VISUAL - ZW]" 65 | python tscdl.py ZW ZW_SIFT_2 --visual_feature SIFT_2.p 66 | echo "[KINEMATICS+VISUAL - ZW]" 67 | python tscdl.py ZW ZW_2_PCA --visual_feature 2_PCA.p 68 | echo "[KINEMATICS+VISUAL - ZW]" 69 | python tscdl.py ZW ZW_2_CCA --visual_feature 2_CCA.p 70 | echo "[KINEMATICS+VISUAL - ZW]" 71 | python tscdl.py ZW ZW_2_GRP --visual_feature 2_GRP.p 72 | echo "[KINEMATICS+VISUAL - ZW]" 73 | python tscdl.py ZW ZW_3_PCA --visual_feature 3_PCA.p 74 | echo "[KINEMATICS+VISUAL - ZW]" 75 | python tscdl.py ZW ZW_3_GRP --visual_feature 3_GRP.p 76 | echo "[KINEMATICS+VISUAL - ZW]" 77 | python tscdl.py ZW ZW_3_CCA --visual_feature 3_CCA.p 78 | echo "[KINEMATICS+VISUAL - ZW]" 79 | python tscdl.py ZW ZW_4_PCA --visual_feature 4_PCA.p 80 | echo "[KINEMATICS+VISUAL - ZW]" 81 | python tscdl.py ZW ZW_4_GRP --visual_feature 4_GRP.p 82 | echo "[KINEMATICS+VISUAL - ZW]" 83 | python tscdl.py ZW ZW_4_CCA --visual_feature 4_CCA.p 84 | echo "[KINEMATICS+VISUAL - ZW]" 85 | python tscdl.py ZW ZW_5_PCA --visual_feature 5_PCA.p 86 | echo "[KINEMATICS+VISUAL - ZW]" 87 | python tscdl.py ZW ZW_5_GRP --visual_feature 5_GRP.p 88 | echo "[KINEMATICS+VISUAL - ZW]" 89 | python tscdl.py ZW ZW_5_CCA --visual_feature 5_CCA.p 90 | echo "[KINEMATICS+VISUAL - ZW]" 91 | python tscdl.py ZW ZW_7_PCA --visual_feature 7_PCA.p 92 | echo "[KINEMATICS+VISUAL - ZW]" 93 | python tscdl.py ZW ZW_7_GRP --visual_feature 7_GRP.p 94 | echo "[KINEMATICS+VISUAL - ZW]" 95 | python tscdl.py ZW ZW_7_CCA --visual_feature 7_CCA.p -------------------------------------------------------------------------------- /scripts/forward_pass.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import numpy as np 4 | import matplotlib.pyplot as plt 5 | import IPython 6 | import sys 7 | import pickle 8 | import caffe 9 | import argparse 10 | import cv2 11 | 12 | import utils 13 | import constants 14 | 15 | sys.path.insert(0, constants.CAFFE_ROOT + 'python') 16 | 17 | plt.rcParams['figure.figsize'] = (10, 10) 18 | plt.rcParams['image.interpolation'] = 'nearest' 19 | plt.rcParams['image.cmap'] = 'gray' 20 | 21 | class CNNFeatureExtractor: 22 | def __init__(self, net_name = "AlexNet"): 23 | self.net = None 24 | self.transformer = None 25 | self.init_caffe(net_name) 26 | 27 | def forward_pass(self, PATH_TO_DATA, annotations, list_of_layers = constants.alex_net_layers, 28 | sampling_rate = 1, batch_size = -1, LCD = False, no_plot_mode = False): 29 | return self.process_individual_frames(PATH_TO_DATA, annotations, 30 | list_of_layers, sampling_rate) 31 | 32 | def process_individual_frames(self, PATH_TO_DATA, annotations, list_of_layers, sampling_rate): 33 | X = {} 34 | map_index_data = pickle.load(open(annotations, "rb")) 35 | 36 | segments = utils.get_chronological_sequences(map_index_data) 37 | for seg in segments: 38 | print str(seg) 39 | frm_num = seg[0] 40 | while frm_num <= seg[1]: 41 | print frm_num, annotations 42 | im = caffe.io.load_image(utils.get_full_image_path(PATH_TO_DATA, frm_num)) 43 | self.net.blobs['data'].data[...] = self.transformer.preprocess('data', im) 44 | out = self.net.forward() 45 | for layer in list_of_layers: 46 | if layer == 'input': 47 | data = cv2.imread(full_image_path) 48 | else: 49 | data = self.net.blobs[layer].data[0] 50 | data = utils.flatten(data) 51 | utils.dict_insert(layer, data, X) 52 | frm_num += sampling_rate 53 | return X 54 | 55 | 56 | def init_caffe(self, net_name): 57 | caffe.set_mode_gpu() 58 | net_params = constants.NET_PARAMS[net_name] 59 | self.net = caffe.Net(net_params[0], net_params[1], caffe.TEST) 60 | self.transformer = caffe.io.Transformer({'data': self.net.blobs['data'].data.shape}) 61 | self.transformer.set_transpose('data', (2,0,1)) 62 | # self.transformer.set_mean('data', np.load(constants.CAFFE_ROOT + 'python/caffe/imagenet/ilsvrc_2012_mean.npy').mean(1).mean(1)) # mean pixel 63 | self.transformer.set_raw_scale('data', 255) # the reference model operates on images in [0,255] range instead of [0,1] 64 | self.transformer.set_channel_swap('data', (2,1,0)) # the reference model has channels in BGR order instead of RGB 65 | if net_name == "AlexNet": 66 | self.net.blobs['data'].reshape(1,3,227,227) 67 | else: 68 | self.net.blobs['data'].reshape(1,3,224,224) 69 | 70 | 71 | if __name__ == "__main__": 72 | parser = argparse.ArgumentParser() 73 | parser.add_argument("net", help = "Net, Choose from: AlexNet, VGG, VGG_SOS") 74 | parser.add_argument("figure_name", help = "Please specify MAIN file name") 75 | parser.add_argument("PATH_TO_DATA", help="Please specify the path to the images") 76 | parser.add_argument("annotations", help = "Annotated frames") 77 | parser.add_argument("--PATH_TO_DATA_2", help="Please specify the path to second set of images", default = None) 78 | parser.add_argument("--annotations_2", help = "Annotated frames for second set of images", default = None) 79 | args = parser.parse_args() 80 | fe = CNNFeatureExtractor(args.net) 81 | layers = constants.NET_PARAMS[args.net][2] 82 | 83 | X = fe.forward_pass(args.PATH_TO_DATA, args.annotations, list_of_layers = layers, sampling_rate = 1) 84 | utils.plot_all_layers(X, args.net, args.figure_name, list_of_layers = layers) 85 | 86 | -------------------------------------------------------------------------------- /scripts/fine_tuning/assemble_data.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import numpy as np 4 | import sys 5 | import os 6 | import IPython 7 | import time 8 | import random 9 | from sklearn.cross_validation import train_test_split 10 | 11 | sys.path.insert(0, "/home/animesh/DeepMilestones/scripts/") 12 | import utils 13 | import constants 14 | import parser 15 | 16 | FPS = 30 17 | 18 | def convert_video_to_frames(list_of_videos): 19 | 20 | for video in list_of_videos: 21 | # for camera in ["capture1", "capture2"]: 22 | for camera in ["capture1",]: 23 | # os.chdir(constants.PATH_TO_DATA) 24 | video_file_name = video + "_" + camera 25 | # mkdir_path = constants.NEW_FRAMES_FOLDER + video_file_name 26 | # print "mkdir " + mkdir_path 27 | # os.mkdir(mkdir_path) 28 | 29 | # command = "cp " + constants.VIDEO_FOLDER + video_file_name + ".avi " + mkdir_path 30 | # print command 31 | # os.system(command) 32 | 33 | command = constants.PATH_TO_DATA + constants.NEW_FRAMES_FOLDER + video_file_name 34 | print "cd " + command 35 | os.chdir(constants.PATH_TO_DATA + constants.NEW_FRAMES_FOLDER + video_file_name) 36 | 37 | command = "ffmpeg -i " + video_file_name + ".avi -filter:v " + constant.CROP_PARAMS[camera] + " cropped.avi" 38 | print command 39 | os.system(command) 40 | 41 | command = "ffmpeg -i cropped.avi -vf scale=640:480 cropped_scaled.avi" 42 | print command 43 | os.system(command) 44 | 45 | command = "ffmpeg -i cropped_scaled.avi -r " + str(FPS) + " ./%6d.jpg" 46 | print command 47 | os.system(command) 48 | 49 | time.sleep(1) 50 | pass 51 | 52 | 53 | def write_to_file(file_name, data): 54 | if not os.path.exists(file_name): 55 | file = open(file_name, 'w') 56 | for elem in data: 57 | file.write(elem) 58 | file.close() 59 | 60 | def generate_train_val_test_files(list_of_videos): 61 | list_of_data = [] 62 | 63 | for video in list_of_videos: 64 | for camera in ["capture1", "capture2"]: 65 | os.chdir(constants.PATH_TO_DATA) 66 | transcriptions_file = TRANSCRIPTIONS_FOLDER + video + ".txt" 67 | with open(transcriptions_file, "rb") as f: 68 | for line in f: 69 | line = line.split() 70 | start = int(line[0]) 71 | end = int(line[1]) 72 | surgeme = line[2] 73 | label = constants.map_surgeme_label[surgeme] 74 | i = start 75 | while i <= end: 76 | data = constants.PATH_TO_DATA + constants.NEW_FRAMES_FOLDER + video + "_" + camera + "/" + utils.get_frame_fig_name(i) + " "+ str(label) + " \n" 77 | list_of_data.append(data) 78 | i += 1 79 | random.shuffle(list_of_data) 80 | 81 | N = len(list_of_data) 82 | 83 | train, test = train_test_split(list_of_data, test_size = constants.train_test_ratio) 84 | 85 | val = test[:len(test)/2] 86 | test = test[len(test)/2:] 87 | 88 | train_file_name = constants.PATH_TO_DATA + "train.txt" 89 | val_file_name = constants.PATH_TO_DATA + "val.txt" 90 | test_file_name = constants.PATH_TO_DATA + "test.txt" 91 | 92 | IPython.embed() 93 | 94 | write_to_file(train_file_name, train) 95 | write_to_file(val_file_name, val) 96 | write_to_file(test_file_name, test) 97 | 98 | if __name__ == "__main__": 99 | # list_of_videos = parser.generate_list_of_videos(constants.PATH_TO_DATA + constants.CONFIG_FILE) 100 | list_of_videos = ["Needle_Passing_E001", "Needle_Passing_E003", "Needle_Passing_E004", "Needle_Passing_E005", 101 | "Needle_Passing_D001", "Needle_Passing_D002","Needle_Passing_D003", "Needle_Passing_D004", "Needle_Passing_D005"] 102 | convert_video_to_frames(list_of_videos) 103 | # generate_train_val_test_files(list_of_videos) 104 | -------------------------------------------------------------------------------- /toyExample/toyEx_baseline.m: -------------------------------------------------------------------------------- 1 | clc 2 | clear all 3 | close all 4 | 5 | %% 6 | numExamples = 5; 7 | numPoints = 5; 8 | stepSize = 0.75; 9 | seed_RNG = rng(10,'twister'); 10 | listX = random('unif', -10, 10, 2, numPoints ); 11 | c = ['b', 'r', 'c', 'm' , 'y']; 12 | a = 50; %markersize 13 | noiseLevel = [10, 25, 50, 75, 100];% as percent of noise 14 | 15 | delete(['baseline' filesep '*.*']); %clear the output folder 16 | 17 | %% Without Noise 18 | xcurr = listX(:,1); 19 | xTraj_clean = xcurr; 20 | scatter(xcurr(1), xcurr(2),a, 'filled', 'MarkerEdgeColor', c(1), 'MarkerFaceColor', c(1)) 21 | hold on 22 | axis([-12 12 -12 12]) 23 | axis equal 24 | 25 | xTranscriptions = []; 26 | for i = 1: numPoints-1 27 | x2 = listX(:,i+1); 28 | diff = x2 - xcurr; 29 | xTranscriptions(i, 1) = size(xTraj_clean, 2); %start frame 30 | while (norm(diff)>0) 31 | diff = x2 - xcurr; 32 | if norm(diff)>stepSize 33 | xcurr = xcurr + stepSize*diff./norm(diff); 34 | else%last step 35 | xcurr = xcurr+ diff; 36 | xTranscriptions(i, 2) = size(xTraj_clean, 2); %end frame 37 | if i == numPoints-1 %for last iteration so the traj length is same 38 | xTranscriptions(i, 2) = xTranscriptions(i, 2)+1; 39 | end 40 | end 41 | scatter(xcurr(1), xcurr(2),a, 'filled', 'MarkerEdgeColor', c(i), 'MarkerFaceColor', c(i)) 42 | xTraj_clean = [xTraj_clean, xcurr]; 43 | end 44 | end 45 | 46 | hold off 47 | fileName = ['baseline' filesep 'simpleExpt' '_noiseLevel000']; 48 | save([fileName '.mat'], ... 49 | 'listX', 'xTraj_clean') 50 | save([fileName '-transcriptions' '.mat'], 'xTranscriptions') 51 | saveas(gcf, fileName, 'jpeg'); 52 | 53 | 54 | %% With Noise 55 | for n = 1: length(noiseLevel) 56 | for e = 1:numExamples 57 | xcurr = listX(:,1); 58 | xTraj_noise = xcurr; 59 | figure() 60 | scatter(xcurr(1), xcurr(2),a, 'filled', 'MarkerEdgeColor', c(1), 'MarkerFaceColor', c(1)) 61 | hold on 62 | axis([-12 12 -12 12]) 63 | axis equal 64 | xTranscriptions = []; 65 | for i = 1: numPoints-1 66 | x2 = listX(:,i+1); 67 | diff = x2 - xcurr; 68 | xTranscriptions(i, 1) = size(xTraj_noise, 2); %start frame 69 | while (norm(diff)>0) 70 | diff = x2 - xcurr; 71 | if norm(diff)>stepSize 72 | xcurr = xcurr + stepSize*diff./norm(diff) + ... 73 | 0.01*noiseLevel(n)*random('unif', -stepSize, stepSize, 2, 1); 74 | else%last step 75 | xcurr = xcurr+ diff; 76 | xTranscriptions(i, 2) = size(xTraj_noise, 2); %end frame 77 | if i == numPoints-1 %for last iteration so the traj length is same 78 | xTranscriptions(i, 2) = xTranscriptions(i, 2)+1; 79 | end 80 | end 81 | scatter(xcurr(1), xcurr(2),a, 'filled', 'MarkerEdgeColor', c(i), 'MarkerFaceColor', c(i)) 82 | xTraj_noise = [xTraj_noise, xcurr]; 83 | end 84 | end 85 | 86 | hold off 87 | fileName = ['baseline' filesep 'simpleExpt' '_noiseLevel' ... 88 | num2str(noiseLevel(n),'%03d') '_' num2str(e,'%02d')]; 89 | save([fileName '.mat'], ... 90 | 'listX', 'xTraj_noise') 91 | save([fileName '-transcriptions' '.mat'], 'xTranscriptions') 92 | saveas(gcf, fileName, 'jpeg'); 93 | end 94 | end -------------------------------------------------------------------------------- /scripts/feature_extractor.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import IPython 3 | import pickle 4 | import numpy as np 5 | 6 | import constants 7 | 8 | from forward_pass import CNNFeatureExtractor 9 | 10 | # layers_of_interest = {"VGG": ['conv5_1', 'conv5_3'], "VGG_SOS": ['conv5_1', 'conv5_3'], "AlexNet": ["conv3", "conv4", "pool5"]} 11 | 12 | layers_of_interest = {"VGG": ['conv5_3'], "VGG_SOS": ['conv5_3'], "AlexNet": ["conv4"]} 13 | 14 | features_folder = {"VGG": constants.VGG_FEATURES_FOLDER, "VGG_SOS": constants.VGG_FEATURES_FOLDER, "AlexNet": constants.ALEXNET_FEATURES_FOLDER} 15 | 16 | def forward_pass_entire_dataset(list_of_demonstrations, net_name, camera): 17 | """ 18 | Function performs forward pass of the frames corresponding to the demonstration 19 | through specified 2D CNN. 20 | Input: List of demonstrations, the Net (VGG, AlexNet) and camera (capture1, caputure2) 21 | """ 22 | net = CNNFeatureExtractor(net_name) 23 | 24 | total = len(list_of_demonstrations) 25 | i = 1 26 | for video in list_of_demonstrations: 27 | print "-------------------- " + str(i) +"--out-of-"+ str(total)+"-----------------" 28 | PATH_TO_ANNOTATION = constants.PATH_TO_DATA + constants.ANNOTATIONS_FOLDER + video + "_" + camera + ".p" 29 | PATH_TO_DATA = constants.PATH_TO_DATA + constants.NEW_FRAMES_FOLDER + video + "_" + camera + "/" 30 | get_cnn_features_pickle_dump(net, video + "_" + camera, PATH_TO_DATA, PATH_TO_ANNOTATION, net_name) 31 | i += 1 32 | 33 | def get_cnn_features_pickle_dump(net, fname, PATH_TO_DATA, annotations, net_name): 34 | """ 35 | Extracts CNN features for frames located at PATH_TO_DATA, saves them as pickle file with file name fname. 36 | """ 37 | 38 | list_of_layers = layers_of_interest[net_name] 39 | Z = net.forward_pass(PATH_TO_DATA, annotations, list_of_layers = list_of_layers, sampling_rate = 1, no_plot_mode = True) 40 | for key in Z.keys(): 41 | pickle.dump(Z[key], open(constants.PATH_TO_DATA + features_folder[net_name] + key + "_" + net_name + "_" + fname + ".p", "wb")) 42 | 43 | if __name__ == "__main__": 44 | # list_of_demonstrations = ['Suturing_D005', 'Suturing_C001', 'Suturing_C002', 'Suturing_C003', 'Suturing_C004', 'Suturing_C005', 45 | # 'Suturing_F001', 'Suturing_F002', 'Suturing_F003', 'Suturing_F004', 'Suturing_F005'] 46 | 47 | # list_of_demonstrations = ['Suturing_H001', 'Suturing_G003'] 48 | 49 | # list_of_demonstrations = ['Suturing_G002', 'Suturing_G004', 'Suturing_G005', 50 | # 'Suturing_H003', 'Suturing_H004', 'Suturing_H005', 51 | # 'Suturing_I001', 'Suturing_I002', 'Suturing_I003', 'Suturing_I004', 'Suturing_I005'] 52 | 53 | # list_of_demonstrations = ['Suturing_G003', 'Suturing_G004', 'Suturing_G005', 'Suturing_I004', 'Suturing_I005'] 54 | 55 | # list_of_demonstrations = ['Suturing_I001', 'Suturing_I002', 'Suturing_I003', 'Suturing_I004', 'Suturing_I005'] 56 | 57 | # list_of_demonstrations = ["Needle_Passing_E001", "Needle_Passing_E003", "Needle_Passing_E004", "Needle_Passing_E005", 58 | # "Needle_Passing_D001", "Needle_Passing_D002","Needle_Passing_D003", "Needle_Passing_D004", "Needle_Passing_D005"] 59 | 60 | # list_of_demonstrations = ["1001_01", "1001_02", "1001_03", "1001_04", "1001_05"] 61 | 62 | # list_of_demonstrations = ["plane_5", 63 | # "plane_6", "plane_7", "plane_8", "plane_9", "plane_10"] 64 | 65 | # list_of_demonstrations = ["people_0", "people_1", "people_2", "people_3", "people_4", "people_5", "people_6"] 66 | 67 | # list_of_demonstrations = ["lego_2", "lego_3", "lego_4", "lego_5", "lego_6", "lego_7"] 68 | 69 | # list_of_demonstrations = ["people2_2", "people2_3", "people2_4", "people2_5", "people2_6", "people2_7"] 70 | 71 | list_of_demonstrations = ["Needle_Passing_E001", "Needle_Passing_E003", "Needle_Passing_E004", 72 | "Needle_Passing_E005", "Needle_Passing_C001", "Needle_Passing_C002"] 73 | 74 | forward_pass_entire_dataset(list_of_demonstrations, "VGG", constants.CAMERA) 75 | # forward_pass_entire_dataset(list_of_demonstrations, "AlexNet", constants.CAMERA) 76 | -------------------------------------------------------------------------------- /scripts/examples/plot_kmeans_digits.py: -------------------------------------------------------------------------------- 1 | print(__doc__) 2 | 3 | from time import time 4 | import numpy as np 5 | import matplotlib.pyplot as plt 6 | 7 | from sklearn import metrics 8 | from sklearn.cluster import KMeans 9 | from sklearn.datasets import load_digits 10 | from sklearn.decomposition import PCA 11 | from sklearn.preprocessing import scale 12 | import IPython 13 | 14 | np.random.seed(42) 15 | 16 | digits = load_digits() 17 | data = scale(digits.data) 18 | 19 | n_samples, n_features = data.shape 20 | n_digits = len(np.unique(digits.target)) 21 | labels = digits.target 22 | 23 | sample_size = 300 24 | 25 | print("n_digits: %d, \t n_samples %d, \t n_features %d" 26 | % (n_digits, n_samples, n_features)) 27 | 28 | 29 | print(79 * '_') 30 | print('% 9s' % 'init' 31 | ' time inertia homo compl v-meas ARI AMI silhouette') 32 | 33 | 34 | def bench_k_means(estimator, name, data): 35 | t0 = time() 36 | estimator.fit(data) 37 | print('% 9s %.2fs %i %.3f %.3f %.3f %.3f %.3f %.3f' 38 | % (name, (time() - t0), estimator.inertia_, 39 | metrics.homogeneity_score(labels, estimator.labels_), 40 | metrics.completeness_score(labels, estimator.labels_), 41 | metrics.v_measure_score(labels, estimator.labels_), 42 | metrics.adjusted_rand_score(labels, estimator.labels_), 43 | metrics.adjusted_mutual_info_score(labels, estimator.labels_), 44 | metrics.silhouette_score(data, estimator.labels_, 45 | metric='euclidean', 46 | sample_size=sample_size))) 47 | 48 | bench_k_means(KMeans(init='k-means++', n_clusters=n_digits, n_init=10), 49 | name="k-means++", data=data) 50 | 51 | bench_k_means(KMeans(init='random', n_clusters=n_digits, n_init=10), 52 | name="random", data=data) 53 | 54 | # in this case the seeding of the centers is deterministic, hence we run the 55 | # kmeans algorithm only once with n_init=1 56 | pca = PCA(n_components=n_digits).fit(data) 57 | bench_k_means(KMeans(init=pca.components_, n_clusters=n_digits, n_init=1), 58 | name="PCA-based", 59 | data=data) 60 | print(79 * '_') 61 | 62 | ############################################################################### 63 | # Visualize the results on PCA-reduced data 64 | 65 | IPython.embed() 66 | reduced_data = PCA(n_components=2).fit_transform(data) 67 | kmeans = KMeans(init='k-means++', n_clusters=n_digits, n_init=10) 68 | kmeans.fit(reduced_data) 69 | 70 | # Step size of the mesh. Decrease to increase the quality of the VQ. 71 | h = .02 # point in the mesh [x_min, m_max]x[y_min, y_max]. 72 | 73 | # Plot the decision boundary. For that, we will assign a color to each 74 | x_min, x_max = reduced_data[:, 0].min() + 1, reduced_data[:, 0].max() - 1 75 | y_min, y_max = reduced_data[:, 1].min() + 1, reduced_data[:, 1].max() - 1 76 | xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h)) 77 | 78 | # Obtain labels for each point in mesh. Use last trained model. 79 | Z = kmeans.predict(np.c_[xx.ravel(), yy.ravel()]) 80 | 81 | # Put the result into a color plot 82 | Z = Z.reshape(xx.shape) 83 | plt.figure(1) 84 | plt.clf() 85 | plt.imshow(Z, interpolation='nearest', 86 | extent=(xx.min(), xx.max(), yy.min(), yy.max()), 87 | cmap=plt.cm.Paired, 88 | aspect='auto', origin='lower') 89 | 90 | plt.plot(reduced_data[:, 0], reduced_data[:, 1], 'k.', markersize=2) 91 | # Plot the centroids as a white X 92 | centroids = kmeans.cluster_centers_ 93 | plt.scatter(centroids[:, 0], centroids[:, 1], 94 | marker='x', s=169, linewidths=3, 95 | color='w', zorder=10) 96 | plt.title('K-means clustering on the digits dataset (PCA-reduced data)\n' 97 | 'Centroids are marked with white cross') 98 | plt.xlim(x_min, x_max) 99 | plt.ylim(y_min, y_max) 100 | plt.xticks(()) 101 | plt.yticks(()) 102 | plt.show() -------------------------------------------------------------------------------- /scripts/filter_visualization.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | import IPython 4 | 5 | # Make sure that caffe is on the python path: 6 | caffe_root = '../' # this file is expected to be in {caffe_root}/examples 7 | import sys 8 | sys.path.insert(0, caffe_root + 'python') 9 | 10 | import caffe 11 | 12 | plt.rcParams['figure.figsize'] = (10, 10) 13 | plt.rcParams['image.interpolation'] = 'nearest' 14 | plt.rcParams['image.cmap'] = 'gray' 15 | 16 | import os 17 | # if not os.path.isfile(caffe_root + 'models/bvlc_reference_caffenet/bvlc_reference_caffenet.caffemodel'): 18 | # print("Downloading pre-trained CaffeNet model...") 19 | # !../scripts/download_model_binary.py ../models/bvlc_reference_caffenet 20 | 21 | path_to_image = "/home/animesh/research/" 22 | image_name = "image-354.jpg" 23 | 24 | 25 | # take an array of shape (n, height, width) or (n, height, width, channels) 26 | # and visualize each (height, width) thing in a grid of size approx. sqrt(n) by sqrt(n) 27 | def vis_square(data, padsize=1, padval=0, title="no-title"): 28 | data -= data.min() 29 | data /= data.max() 30 | 31 | # force the number of filters to be square 32 | n = int(np.ceil(np.sqrt(data.shape[0]))) 33 | padding = ((0, n ** 2 - data.shape[0]), (0, padsize), (0, padsize)) + ((0, 0),) * (data.ndim - 3) 34 | data = np.pad(data, padding, mode='constant', constant_values=(padval, padval)) 35 | 36 | # tile the filters into an image 37 | data = data.reshape((n, n) + data.shape[1:]).transpose((0, 2, 1, 3) + tuple(range(4, data.ndim + 1))) 38 | data = data.reshape((n * data.shape[1], n * data.shape[3]) + data.shape[4:]) 39 | 40 | plt.figure() 41 | plt.title(title) 42 | plt.imshow(data) 43 | # plt.savefig("feature_extraction/feature_images/" + title + image_name) 44 | 45 | caffe.set_mode_cpu() 46 | net = caffe.Net(caffe_root + 'models/bvlc_reference_caffenet/deploy.prototxt', 47 | caffe_root + 'models/bvlc_reference_caffenet/bvlc_reference_caffenet.caffemodel', 48 | caffe.TEST) 49 | 50 | # input preprocessing: 'data' is the name of the input blob == net.inputs[0] 51 | transformer = caffe.io.Transformer({'data': net.blobs['data'].data.shape}) 52 | transformer.set_transpose('data', (2,0,1)) 53 | transformer.set_mean('data', np.load(caffe_root + 'python/caffe/imagenet/ilsvrc_2012_mean.npy').mean(1).mean(1)) # mean pixel 54 | transformer.set_raw_scale('data', 255) # the reference model operates on images in [0,255] range instead of [0,1] 55 | transformer.set_channel_swap('data', (2,1,0)) # the reference model has channels in BGR order instead of RGB 56 | 57 | net.blobs['data'].reshape(1,3,227,227) 58 | net.blobs['data'].data[...] = transformer.preprocess('data', caffe.io.load_image(path_to_image + image_name)) 59 | out = net.forward() 60 | # print("Predicted class is #{}.".format(out['prob'].argmax())) 61 | 62 | IPython.embed() 63 | 64 | plt.imshow(transformer.deprocess('data', net.blobs['data'].data[0])) 65 | 66 | conv1filters = net.params['conv1'][0].data 67 | vis_square(conv1filters.transpose(0, 2, 3, 1), title = 'conv1_filters') 68 | 69 | feat_conv1 = net.blobs['conv1'].data[0] 70 | vis_square(feat_conv1, padval=1, title = "conv1_features") 71 | 72 | conv2filters = net.params['conv2'][0].data 73 | vis_square(conv2filters[:48].reshape(48**2, 5, 5), title = "conv2_filters") 74 | 75 | feat_conv2 = net.blobs['conv2'].data[0] 76 | vis_square(feat_conv2, padval=1, title = "conv2_features") 77 | 78 | feat_conv3 = net.blobs['conv3'].data[0] 79 | vis_square(feat_conv3, padval=0.5, title = "conv3_features") 80 | 81 | feat_conv4 = net.blobs['conv4'].data[0] 82 | vis_square(feat_conv4, padval=0.5, title = "conv4_features") 83 | 84 | feat_conv5 = net.blobs['conv5'].data[0] 85 | vis_square(feat_conv5, padval=0.5, title = "conv5_features") 86 | 87 | feat_pool5 = net.blobs['pool5'].data[0] 88 | vis_square(feat_pool5, padval=1, title = "pool5_features") 89 | 90 | # load labels 91 | imagenet_labels_filename = caffe_root + 'data/ilsvrc12/synset_words.txt' 92 | labels = np.loadtxt(imagenet_labels_filename, str, delimiter='\t') 93 | # sort top k predictions from softmax output 94 | #top_k = net.blobs['prob'].data[0].flatten().argsort()[-1:-6:-1] 95 | #print labels[top_k] 96 | 97 | IPython.embed() 98 | 99 | -------------------------------------------------------------------------------- /scripts/jqm_cvi/basec.pyx: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | #cython: language_level=3, boundscheck=False 3 | __author__ = "Joaquim Viegas" 4 | 5 | """ JQM_CV - Cython implementations of Dunn and Davis Bouldin clustering validity indices 6 | 7 | dunn(k_list): 8 | Slow implementation of Dunn index that depends on numpy 9 | -- basec.pyx Cython implementation is much faster but flower than dunn_fast() 10 | dunn_fast(points, labels): 11 | Fast implementation of Dunn index that depends on numpy and sklearn.pairwise 12 | -- No Cython implementation 13 | davisbouldin(k_list, k_centers): 14 | Implementation of Davis Boulding index that depends on numpy 15 | -- basec.pyx Cython implementation is much faster 16 | """ 17 | 18 | import numpy as np 19 | cimport numpy as np 20 | from cython.view cimport array as cvarray 21 | from sklearn.metrics.pairwise import euclidean_distances 22 | 23 | cdef double d_euc(double[:] arr1, double[:] arr2): 24 | """ Euclidean distance 25 | ... 26 | """ 27 | cdef double total = 0 28 | 29 | for i in range(arr1.shape[0]): 30 | total += (arr1[i] - arr2[i])*(arr1[i] - arr2[i]) 31 | 32 | return total**.5 33 | 34 | cdef double dunn_delta(double [:, :] ck, double [:, :] cl): 35 | cdef: 36 | double [:, :] values = np.zeros([ck.shape[0], cl.shape[0]], dtype=np.float64) 37 | Py_ssize_t i, j 38 | 39 | for i in range(ck.shape[0]): 40 | for j in range(cl.shape[0]): 41 | values[i, j] = d_euc(ck[i], cl[j]) 42 | 43 | return np.min(values) 44 | 45 | cdef double dunn_big_delta(double [:, :] ci): 46 | cdef: 47 | double [:,:] values = np.zeros([ci.shape[0], ci.shape[0]], dtype=np.float64) 48 | Py_ssize_t i, j 49 | 50 | for i in range(ci.shape[0]): 51 | for j in range(ci.shape[0]): 52 | values[i, j] = d_euc(ci[i], ci[j]) 53 | 54 | return np.max(values) 55 | 56 | def dunn(k_list): 57 | """ Dunn index [CVI] 58 | 59 | Parameters 60 | ---------- 61 | k_list : list of np.arrays 62 | A list containing a numpy array for each cluster |c| = number of clusters 63 | c[K] is np.array([N, p]) (N : number of samples in cluster K, p : sample dimension) 64 | """ 65 | cdef: 66 | Py_ssize_t len_k_list = len(k_list) 67 | double [:,:] deltas = np.ones([len_k_list, len_k_list], dtype=np.float64)*100000 68 | double [:,:] big_deltas = np.zeros([len_k_list, 1], dtype=np.float64) 69 | Py_ssize_t k, l 70 | 71 | for k in range(0, len_k_list): 72 | for l in range(0, k): 73 | deltas[k, l] = dunn_delta(k_list[k], k_list[l]) 74 | for l in range(k+1, len_k_list): 75 | deltas[k, l] = dunn_delta(k_list[k], k_list[l]) 76 | 77 | big_deltas[k] = dunn_big_delta(k_list[k]) 78 | res = np.min(deltas)/np.max(big_deltas)*1 79 | return res 80 | 81 | cdef double big_s(double [:, :] x, double [:] center): 82 | cdef: 83 | Py_ssize_t len_x = x.shape[0] 84 | double total = 0 85 | 86 | for i in range(len_x): 87 | total += d_euc(x[i], center) 88 | 89 | return total/len_x 90 | 91 | def davisbouldin(k_list, k_centers): 92 | """ Davis Bouldin Index 93 | 94 | Parameters 95 | ---------- 96 | k_list : list of np.arrays 97 | A list containing a numpy array for each cluster |c| = number of clusters 98 | c[K] is np.array([N, p]) (N : number of samples in cluster K, p : sample dimension) 99 | k_centers : np.array 100 | The array of the cluster centers (prototypes) of type np.array([K, p]) 101 | """ 102 | cdef: 103 | Py_ssize_t len_k_list = len(k_list) 104 | Py_ssize_t k, j 105 | double [:] big_ss = np.zeros([len_k_list], dtype=np.float64) 106 | double [:, :] d_eucs = np.zeros([len_k_list, len_k_list], dtype=np.float64) 107 | double db = 0 108 | double [:] values = np.zeros([len_k_list-1], dtype=np.float64) 109 | 110 | for k in range(len_k_list): 111 | big_ss[k] = big_s(k_list[k], k_centers[k]) 112 | 113 | for k in range(len_k_list): 114 | for l in range(0, len_k_list): 115 | d_eucs[k, l] = d_euc(k_centers[k], k_centers[l]) 116 | 117 | for k in range(len_k_list): 118 | for l in range(0, k): 119 | values[l] = (big_ss[k] + big_ss[l])/d_eucs[k, l] 120 | for l in range(k+1, len_k_list): 121 | values[l-1] = (big_ss[k] + big_ss[l])/d_eucs[k, l] 122 | 123 | db += np.max(values) 124 | res = db/len_k_list 125 | return res 126 | -------------------------------------------------------------------------------- /scripts/jaccard.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import pickle 4 | import numpy as np 5 | import IPython 6 | import argparse 7 | import itertools 8 | from scipy import linalg 9 | import matplotlib.pyplot as plt 10 | import matplotlib as mpl 11 | import matplotlib.cm as cmx 12 | import matplotlib.colors as colors 13 | 14 | import constants 15 | import utils 16 | from clustering import TSCDL_multimodal, get_list_of_demo_combinations, post_evaluation_multimodal 17 | from clustering_kinematics import TSCDL_singlemodal 18 | 19 | from sklearn import (mixture, preprocessing, neighbors, metrics, cross_decomposition) 20 | from decimal import Decimal 21 | from sklearn.metrics import (adjusted_rand_score, adjusted_mutual_info_score, normalized_mutual_info_score, 22 | mutual_info_score, homogeneity_score, completeness_score, recall_score, precision_score) 23 | 24 | def get_cmap(N): 25 | '''Returns a function that maps each index in 0, 1, ... N-1 to a distinct 26 | RGB color.''' 27 | color_norm = colors.Normalize(vmin=0, vmax=N-1) 28 | scalar_map = cmx.ScalarMappable(norm=color_norm, cmap='hsv') 29 | def map_index_to_rgb_color(index): 30 | return scalar_map.to_rgba(index) 31 | return map_index_to_rgb_color 32 | 33 | class TSCDL_manual_multimodal(TSCDL_multimodal): 34 | 35 | def __init__(self, DEBUG, list_of_demonstrations, featfile, trialname): 36 | super(TSCDL_manual_multimodal, self).__init__(DEBUG, list_of_demonstrations, featfile, trialname) 37 | 38 | # This ensures no pruning 39 | self.representativeness = -1.0 40 | 41 | def generate_change_points_2(self): 42 | """ 43 | Generates changespoints by clustering across demonstrations. 44 | """ 45 | cp_index = 0 46 | 47 | for demonstration in self.list_of_demonstrations: 48 | W = self.data_W[demonstration] 49 | Z = self.data_Z[demonstration] 50 | 51 | PATH_TO_ANNOTATION = constants.PATH_TO_DATA + constants.ANNOTATIONS_FOLDER + demonstration + "_" + str(constants.CAMERA) + ".p" 52 | annotations = pickle.load(open(PATH_TO_ANNOTATION, "rb")) 53 | manual_labels = utils.get_chronological_sequences(annotations) 54 | start, end = utils.get_start_end_annotations(PATH_TO_ANNOTATION) 55 | 56 | for elem in manual_labels: 57 | frm = elem[1] 58 | change_pt_W = W[(frm - start)/self.sr] 59 | change_pt_Z = Z[(frm - start)/self.sr] 60 | change_pt = utils.safe_concatenate(change_pt_W, change_pt_Z) 61 | 62 | self.append_cp_array(change_pt) 63 | self.map_cp2demonstrations[cp_index] = demonstration 64 | self.map_cp2frm[cp_index] = frm 65 | self.list_of_cp.append(cp_index) 66 | cp_index += 1 67 | 68 | 69 | class TSCDL_manual_singlemodal(TSCDL_singlemodal): 70 | 71 | def __init__(self, DEBUG, list_of_demonstrations, fname, log, vision_mode = False, feat_fname = None): 72 | super(TSCDL_manual_singlemodal, self).__init__(DEBUG, list_of_demonstrations, fname, log, vision_mode, feat_fname) 73 | 74 | # This ensures no pruning 75 | self.representativeness = -1.0 76 | 77 | def generate_change_points_2(self): 78 | """ 79 | Generates changespoints by clustering across demonstrations. 80 | """ 81 | cp_index = 0 82 | 83 | for demonstration in self.list_of_demonstrations: 84 | X = self.data_X[demonstration] 85 | 86 | PATH_TO_ANNOTATION = constants.PATH_TO_DATA + constants.ANNOTATIONS_FOLDER + demonstration + "_" + str(constants.CAMERA) + ".p" 87 | annotations = pickle.load(open(PATH_TO_ANNOTATION, "rb")) 88 | manual_labels = utils.get_chronological_sequences(annotations) 89 | start, end = utils.get_start_end_annotations(PATH_TO_ANNOTATION) 90 | 91 | for elem in manual_labels: 92 | frm = elem[1] 93 | change_pt = X[(frm - start)/self.sr] 94 | 95 | self.append_cp_array(utils.reshape(change_pt)) 96 | self.map_cp2demonstrations[cp_index] = demonstration 97 | self.map_cp2frm[cp_index] = frm 98 | self.list_of_cp.append(cp_index) 99 | cp_index += 1 100 | 101 | if __name__ == "__main__": 102 | argparser = argparse.ArgumentParser() 103 | argparser.add_argument("feat_fname", help = "Pickle file of visual features", default = 4) 104 | argparser.add_argument("fname", help = "Pickle file of visual features", default = 4) 105 | args = argparser.parse_args() 106 | 107 | combinations = get_list_of_demo_combinations(list_of_demonstrations) 108 | 109 | i = 1 110 | all_metrics = [] 111 | 112 | for elem in combinations: 113 | print "---- k-Fold Cross Validation, Run "+ str(i) + " out of " + str(len(combinations)) + " ----" 114 | mc = TSCDL_manual_multimodal(False, list(elem), args.feat_fname, args.fname) 115 | all_metrics.append(mc.do_everything()) 116 | i += 1 117 | 118 | print "----------- CALCULATING THE ODDS ------------" 119 | post_evaluation_multimodal(all_metrics, args.fname, list_of_demonstrations, args.feat_fname) 120 | -------------------------------------------------------------------------------- /scripts/examples/plot_hog.py: -------------------------------------------------------------------------------- 1 | """ 2 | =============================== 3 | Histogram of Oriented Gradients 4 | =============================== 5 | 6 | The `Histogram of Oriented Gradient 7 | `__ (HOG) feature 8 | descriptor [1]_ is popular for object detection. 9 | 10 | In the following example, we compute the HOG descriptor and display 11 | a visualisation. 12 | 13 | Algorithm overview 14 | ------------------ 15 | 16 | Compute a Histogram of Oriented Gradients (HOG) by 17 | 18 | 1. (optional) global image normalisation 19 | 2. computing the gradient image in x and y 20 | 3. computing gradient histograms 21 | 4. normalising across blocks 22 | 5. flattening into a feature vector 23 | 24 | The first stage applies an optional global image normalisation 25 | equalisation that is designed to reduce the influence of illumination 26 | effects. In practice we use gamma (power law) compression, either 27 | computing the square root or the log of each colour channel. 28 | Image texture strength is typically proportional to the local surface 29 | illumination so this compression helps to reduce the effects of local 30 | shadowing and illumination variations. 31 | 32 | The second stage computes first order image gradients. These capture 33 | contour, silhouette and some texture information, while providing 34 | further resistance to illumination variations. The locally dominant 35 | colour channel is used, which provides colour invariance to a large 36 | extent. Variant methods may also include second order image derivatives, 37 | which act as primitive bar detectors - a useful feature for capturing, 38 | e.g. bar like structures in bicycles and limbs in humans. 39 | 40 | The third stage aims to produce an encoding that is sensitive to 41 | local image content while remaining resistant to small changes in 42 | pose or appearance. The adopted method pools gradient orientation 43 | information locally in the same way as the SIFT [2]_ 44 | feature. The image window is divided into small spatial regions, 45 | called "cells". For each cell we accumulate a local 1-D histogram 46 | of gradient or edge orientations over all the pixels in the 47 | cell. This combined cell-level 1-D histogram forms the basic 48 | "orientation histogram" representation. Each orientation histogram 49 | divides the gradient angle range into a fixed number of 50 | predetermined bins. The gradient magnitudes of the pixels in the 51 | cell are used to vote into the orientation histogram. 52 | 53 | The fourth stage computes normalisation, which takes local groups of 54 | cells and contrast normalises their overall responses before passing 55 | to next stage. Normalisation introduces better invariance to illumination, 56 | shadowing, and edge contrast. It is performed by accumulating a measure 57 | of local histogram "energy" over local groups of cells that we call 58 | "blocks". The result is used to normalise each cell in the block. 59 | Typically each individual cell is shared between several blocks, but 60 | its normalisations are block dependent and thus different. The cell 61 | thus appears several times in the final output vector with different 62 | normalisations. This may seem redundant but it improves the performance. 63 | We refer to the normalised block descriptors as Histogram of Oriented 64 | Gradient (HOG) descriptors. 65 | 66 | The final step collects the HOG descriptors from all blocks of a dense 67 | overlapping grid of blocks covering the detection window into a combined 68 | feature vector for use in the window classifier. 69 | 70 | References 71 | ---------- 72 | 73 | .. [1] Dalal, N. and Triggs, B., "Histograms of Oriented Gradients for 74 | Human Detection," IEEE Computer Society Conference on Computer 75 | Vision and Pattern Recognition, 2005, San Diego, CA, USA. 76 | 77 | .. [2] David G. Lowe, "Distinctive image features from scale-invariant 78 | keypoints," International Journal of Computer Vision, 60, 2 (2004), 79 | pp. 91-110. 80 | 81 | """ 82 | import matplotlib.pyplot as plt 83 | 84 | from skimage.feature import hog 85 | from skimage import data, color, exposure 86 | import IPython 87 | import cv2 88 | 89 | IPython.embed() 90 | image = color.rgb2gray(cv2.imread("../images/sample2.jpg")) 91 | 92 | fd, hog_image = hog(image, orientations=8, pixels_per_cell=(16, 16), 93 | cells_per_block=(1, 1), visualise=True) 94 | 95 | IPython.embed() 96 | 97 | fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(8, 4)) 98 | 99 | ax1.axis('off') 100 | ax1.imshow(image, cmap=plt.cm.gray) 101 | ax1.set_title('Input image') 102 | 103 | # Rescale histogram for better display 104 | hog_image_rescaled = exposure.rescale_intensity(hog_image, in_range=(0, 0.02)) 105 | 106 | ax2.axis('off') 107 | ax2.imshow(hog_image_rescaled, cmap=plt.cm.gray) 108 | ax2.set_title('Histogram of Oriented Gradients') 109 | plt.show() 110 | -------------------------------------------------------------------------------- /scripts/jqm_cvi/cvi.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | __author__ = "Joaquim Viegas" 3 | 4 | """ JQM_CV - Python implementations of Dunn and Davis Bouldin clustering validity indices 5 | 6 | dunn(k_list): 7 | Slow implementation of Dunn index that depends on numpy 8 | -- basec.pyx Cython implementation is much faster but flower than dunn_fast() 9 | dunn_fast(points, labels): 10 | Fast implementation of Dunn index that depends on numpy and sklearn.pairwise 11 | -- No Cython implementation 12 | davisbouldin(k_list, k_centers): 13 | Implementation of Davis Boulding index that depends on numpy 14 | -- basec.pyx Cython implementation is much faster 15 | """ 16 | 17 | import numpy as np 18 | import IPython 19 | from sklearn.metrics.pairwise import euclidean_distances 20 | 21 | def delta(ck, cl): 22 | values = np.ones([len(ck), len(cl)])*10000 23 | 24 | for i in range(0, len(ck)): 25 | for j in range(0, len(cl)): 26 | values[i, j] = np.linalg.norm(ck[i]-cl[j]) 27 | 28 | return np.min(values) 29 | 30 | def big_delta(ci): 31 | values = np.zeros([len(ci), len(ci)]) 32 | 33 | for i in range(0, len(ci)): 34 | for j in range(0, len(ci)): 35 | values[i, j] = np.linalg.norm(ci[i]-ci[j]) 36 | 37 | return np.max(values) 38 | 39 | def dunn(k_list): 40 | """ Dunn index [CVI] 41 | 42 | Parameters 43 | ---------- 44 | k_list : list of np.arrays 45 | A list containing a numpy array for each cluster |c| = number of clusters 46 | c[K] is np.array([N, p]) (N : number of samples in cluster K, p : sample dimension) 47 | """ 48 | deltas = np.ones([len(k_list), len(k_list)])*1000000 49 | big_deltas = np.zeros([len(k_list), 1]) 50 | l_range = list(range(0, len(k_list))) 51 | 52 | for k in l_range: 53 | for l in (l_range[0:k]+l_range[k+1:]): 54 | deltas[k, l] = delta(k_list[k], k_list[l]) 55 | 56 | big_deltas[k] = big_delta(k_list[k]) 57 | 58 | di = np.min(deltas)/np.max(big_deltas) 59 | return di 60 | 61 | def delta_fast(ck, cl, distances): 62 | values = distances[np.where(ck)][:, np.where(cl)] 63 | values = values[np.nonzero(values)] 64 | 65 | return np.min(values) 66 | 67 | def big_delta_fast(ci, distances): 68 | values = distances[np.where(ci)][:, np.where(ci)] 69 | values = values[np.nonzero(values)] 70 | 71 | return np.max(values) 72 | 73 | def dunn_fast(points, labels): 74 | """ Dunn index - FAST (using sklearn pairwise euclidean_distance function) 75 | 76 | Parameters 77 | ---------- 78 | points : np.array 79 | np.array([N, p]) of all points 80 | labels: np.array 81 | np.array([N]) labels of all points 82 | """ 83 | distances = euclidean_distances(points) 84 | ks = np.sort(np.unique(labels)) 85 | 86 | deltas = np.ones([len(ks), len(ks)])*1000000 87 | big_deltas = np.zeros([len(ks), 1]) 88 | 89 | l_range = list(range(0, len(ks))) 90 | 91 | for k in l_range: 92 | for l in (l_range[0:k]+l_range[k+1:]): 93 | deltas[k, l] = delta_fast((labels == ks[k]), (labels == ks[l]), distances) 94 | 95 | big_deltas[k] = big_delta_fast((labels == ks[k]), distances) 96 | 97 | di = np.min(deltas)/np.max(big_deltas) 98 | return di 99 | 100 | 101 | def big_s(x, center): 102 | len_x = len(x) 103 | total = 0 104 | 105 | for i in range(len_x): 106 | total += np.linalg.norm(x[i]-center) 107 | 108 | return total/len_x 109 | 110 | def davisbouldin(k_list, k_centers): 111 | """ Davis Bouldin Index 112 | 113 | Parameters 114 | ---------- 115 | k_list : list of np.arrays 116 | A list containing a numpy array for each cluster |c| = number of clusters 117 | c[K] is np.array([N, p]) (N : number of samples in cluster K, p : sample dimension) 118 | k_centers : np.array 119 | The array of the cluster centers (prototypes) of type np.array([K, p]) 120 | """ 121 | len_k_list = len(k_list) 122 | big_ss = np.zeros([len_k_list], dtype=np.float64) 123 | d_eucs = np.zeros([len_k_list, len_k_list], dtype=np.float64) 124 | db = 0 125 | 126 | for k in range(len_k_list): 127 | big_ss[k] = big_s(k_list[k], k_centers[k]) 128 | 129 | for k in range(len_k_list): 130 | for l in range(0, len_k_list): 131 | d_eucs[k, l] = np.linalg.norm(k_centers[k]-k_centers[l]) 132 | 133 | for k in range(len_k_list): 134 | values = np.zeros([len_k_list-1], dtype=np.float64) 135 | for l in range(0, k): 136 | values[l] = (big_ss[k] + big_ss[l])/d_eucs[k, l] 137 | for l in range(k+1, len_k_list): 138 | values[l-1] = (big_ss[k] + big_ss[l])/d_eucs[k, l] 139 | 140 | db += np.max(values) 141 | res = db/len_k_list 142 | return res 143 | -------------------------------------------------------------------------------- /scripts/preprocess.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import numpy as np 4 | import sys 5 | import os 6 | import IPython 7 | 8 | import utils 9 | import constants 10 | import parser 11 | import time 12 | 13 | def convert_video_to_frames(list_of_videos, cropping_params): 14 | 15 | for video_file_name in list_of_videos: 16 | 17 | command = constants.PATH_TO_DATA + constants.NEW_FRAMES_FOLDER + video_file_name 18 | print "cd " + command 19 | os.chdir(constants.PATH_TO_DATA + constants.NEW_FRAMES_FOLDER + video_file_name) 20 | 21 | command = "ffmpeg -i " + video_file_name + ".avi -filter:v " + cropping_params[video_file_name] + " cropped.avi" 22 | print command 23 | os.system(command) 24 | 25 | command = "ffmpeg -i cropped.avi -vf scale=640:480 cropped_scaled.avi" 26 | print command 27 | os.system(command) 28 | 29 | command = "ffmpeg -i cropped_scaled.avi -r " + str(30) + " ./%6d.jpg" 30 | print command 31 | os.system(command) 32 | 33 | time.sleep(1) 34 | pass 35 | 36 | def preprocess(list_of_demonstrations): 37 | camera = constants.CAMERA 38 | 39 | for demonstration in list_of_demonstrations: 40 | PATH_TO_ANNOTATION = constants.PATH_TO_DATA + constants.ANNOTATIONS_FOLDER + demonstration + "_" + camera + ".p" 41 | start, end = utils.get_start_end_annotations(PATH_TO_ANNOTATION) 42 | 43 | OLD_FRM_PATH = constants.PATH_TO_DATA + "frames_unprocessed/" + demonstration + "_" + camera + "/" 44 | NEW_FRM_PATH = constants.PATH_TO_DATA + constants.NEW_FRAMES_FOLDER + demonstration + "_" + camera + "/" 45 | 46 | command = "mkdir " + NEW_FRM_PATH 47 | print command 48 | os.mkdir(NEW_FRM_PATH) 49 | 50 | for frm in range(start, end + 1): 51 | OLD_FRM_NAME = utils.get_full_image_path(OLD_FRM_PATH, frm) 52 | 53 | NEW_FRM_NAME = utils.get_full_image_path(NEW_FRM_PATH, frm) 54 | NEW_FRM_NAME_UNSCALED = utils.get_full_image_path(NEW_FRM_PATH + "unscaled_", frm) 55 | 56 | command = "ffmpeg -i " + OLD_FRM_NAME + " -filter:v " + constants.CROP_PARAMS[camera] + " " + NEW_FRM_NAME_UNSCALED 57 | print command 58 | os.system(command) 59 | 60 | command = "ffmpeg -i " + NEW_FRM_NAME_UNSCALED + " -vf scale=640:480 " + NEW_FRM_NAME 61 | print command 62 | os.system(command) 63 | 64 | command = "rm " + NEW_FRM_NAME_UNSCALED 65 | print command 66 | os.system(command) 67 | 68 | if __name__ == "__main__": 69 | 70 | # list_of_demonstrations = ["plane_5",] 71 | # list_of_demonstrations = ["plane_6", "plane_7", "plane_8", "plane_9", "plane_10"] 72 | 73 | # list_of_demonstrations = ["people_0", "people_1", "people_2", "people_3", "people_4", "people_5", "people_6"] 74 | 75 | # list_of_demonstrations = ["lego_2", "lego_3", "lego_4", "lego_5", "lego_6", "lego_7"] 76 | 77 | # list_of_demonstrations = ["people2_2", "people2_3", "people2_4", "people2_5", "people2_6", "people2_7"] 78 | 79 | list_of_demonstrations = ["Needle_Passing_B001_capture1", "Needle_Passing_B002_capture1", "Needle_Passing_B003_capture1", 80 | "Needle_Passing_B004_capture1", "Needle_Passing_C001_capture1", "Needle_Passing_C002_capture1", "Needle_Passing_C003_capture1", 81 | "Needle_Passing_C004_capture1","Needle_Passing_C005_capture1", "Needle_Passing_F001_capture1", "Needle_Passing_F003_capture1", 82 | "Needle_Passing_F004_capture1", "Needle_Passing_H002_capture1", "Needle_Passing_H004_capture1","Needle_Passing_H005_capture1", 83 | "Needle_Passing_I002_capture1", "Needle_Passing_I003_capture1", "Needle_Passing_I004_capture1", "Needle_Passing_I005_capture1"] 84 | 85 | cropping_params = {"Needle_Passing_B001_capture1": "crop=434:366:125:107", "Needle_Passing_B002_capture1": "crop=437:368:109:98", 86 | "Needle_Passing_B003_capture1": "crop=439:367:122:104", "Needle_Passing_B004_capture1": "crop=441:367:118:97", 87 | "Needle_Passing_C001_capture1": "crop=371:392:152:82", "Needle_Passing_C002_capture1": "crop=364:392:156:82", 88 | "Needle_Passing_C003_capture1": "crop=362:393:156:82", "Needle_Passing_C004_capture1": "crop=364:390:152:82", 89 | "Needle_Passing_C005_capture1": "crop=363:392:156:82", "Needle_Passing_D001_capture1": "crop=415:398:96:83", 90 | "Needle_Passing_D002_capture1": "crop=414:398:96:83", "Needle_Passing_D003_capture1": "crop=413:398:96:83", 91 | "Needle_Passing_D004_capture1": "crop=414:398:96:83", "Needle_Passing_D005_capture1": "crop=389:398:105:83", 92 | "Needle_Passing_E001_capture1": "crop=433:396:142:1", "Needle_Passing_E003_capture1": "crop=422:361:112:84", 93 | "Needle_Passing_E004_capture1": "crop=422:361:112:84", "Needle_Passing_E005_capture1": "crop=422:361:112:84", 94 | "Needle_Passing_F001_capture1": "crop=202:188:58:40", "Needle_Passing_F003_capture1": "crop=202:189:58:40", 95 | "Needle_Passing_F004_capture1": "crop=202:188:58:40", "Needle_Passing_H002_capture1": "crop=193:188:82:41", 96 | "Needle_Passing_H004_capture1": "crop=191:188:84:41", "Needle_Passing_H005_capture1": "crop=191:188:84:41", 97 | "Needle_Passing_I002_capture1": "crop=352:364:108:68", "Needle_Passing_I003_capture1": "crop=178:180:68:42", 98 | "Needle_Passing_I004_capture1": "crop=351:363:153:83", "Needle_Passing_I005_capture1": "crop=347:363:153:83"} 99 | 100 | 101 | 102 | convert_video_to_frames(list_of_demonstrations, cropping_params) 103 | # preprocess(list_of_demonstrations) 104 | pass -------------------------------------------------------------------------------- /scripts/examples/plot_compare_cross_decomposition.py: -------------------------------------------------------------------------------- 1 | """ 2 | =================================== 3 | Compare cross decomposition methods 4 | =================================== 5 | 6 | Simple usage of various cross decomposition algorithms: 7 | - PLSCanonical 8 | - PLSRegression, with multivariate response, a.k.a. PLS2 9 | - PLSRegression, with univariate response, a.k.a. PLS1 10 | - CCA 11 | 12 | Given 2 multivariate covarying two-dimensional datasets, X, and Y, 13 | PLS extracts the 'directions of covariance', i.e. the components of each 14 | datasets that explain the most shared variance between both datasets. 15 | This is apparent on the **scatterplot matrix** display: components 1 in 16 | dataset X and dataset Y are maximally correlated (points lie around the 17 | first diagonal). This is also true for components 2 in both dataset, 18 | however, the correlation across datasets for different components is 19 | weak: the point cloud is very spherical. 20 | """ 21 | print(__doc__) 22 | 23 | import numpy as np 24 | import matplotlib.pyplot as plt 25 | from sklearn.cross_decomposition import PLSCanonical, PLSRegression, CCA 26 | 27 | ############################################################################### 28 | # Dataset based latent variables model 29 | 30 | n = 500 31 | # 2 latents vars: 32 | l1 = np.random.normal(size=n) 33 | l2 = np.random.normal(size=n) 34 | 35 | latents = np.array([l1, l1, l2, l2]).T 36 | X = latents + np.random.normal(size=4 * n).reshape((n, 4)) 37 | Y = latents + np.random.normal(size=4 * n).reshape((n, 4)) 38 | 39 | X_train = X[:n / 2] 40 | Y_train = Y[:n / 2] 41 | X_test = X[n / 2:] 42 | Y_test = Y[n / 2:] 43 | 44 | print("Corr(X)") 45 | print(np.round(np.corrcoef(X.T), 2)) 46 | print("Corr(Y)") 47 | print(np.round(np.corrcoef(Y.T), 2)) 48 | 49 | ############################################################################### 50 | # Canonical (symmetric) PLS 51 | 52 | # Transform data 53 | # ~~~~~~~~~~~~~~ 54 | plsca = PLSCanonical(n_components=2) 55 | plsca.fit(X_train, Y_train) 56 | X_train_r, Y_train_r = plsca.transform(X_train, Y_train) 57 | X_test_r, Y_test_r = plsca.transform(X_test, Y_test) 58 | 59 | # Scatter plot of scores 60 | # ~~~~~~~~~~~~~~~~~~~~~~ 61 | # 1) On diagonal plot X vs Y scores on each components 62 | plt.figure(figsize=(12, 8)) 63 | plt.subplot(221) 64 | plt.plot(X_train_r[:, 0], Y_train_r[:, 0], "ob", label="train") 65 | plt.plot(X_test_r[:, 0], Y_test_r[:, 0], "or", label="test") 66 | plt.xlabel("x scores") 67 | plt.ylabel("y scores") 68 | plt.title('Comp. 1: X vs Y (test corr = %.2f)' % 69 | np.corrcoef(X_test_r[:, 0], Y_test_r[:, 0])[0, 1]) 70 | plt.xticks(()) 71 | plt.yticks(()) 72 | plt.legend(loc="best") 73 | 74 | plt.subplot(224) 75 | plt.plot(X_train_r[:, 1], Y_train_r[:, 1], "ob", label="train") 76 | plt.plot(X_test_r[:, 1], Y_test_r[:, 1], "or", label="test") 77 | plt.xlabel("x scores") 78 | plt.ylabel("y scores") 79 | plt.title('Comp. 2: X vs Y (test corr = %.2f)' % 80 | np.corrcoef(X_test_r[:, 1], Y_test_r[:, 1])[0, 1]) 81 | plt.xticks(()) 82 | plt.yticks(()) 83 | plt.legend(loc="best") 84 | 85 | # 2) Off diagonal plot components 1 vs 2 for X and Y 86 | plt.subplot(222) 87 | plt.plot(X_train_r[:, 0], X_train_r[:, 1], "*b", label="train") 88 | plt.plot(X_test_r[:, 0], X_test_r[:, 1], "*r", label="test") 89 | plt.xlabel("X comp. 1") 90 | plt.ylabel("X comp. 2") 91 | plt.title('X comp. 1 vs X comp. 2 (test corr = %.2f)' 92 | % np.corrcoef(X_test_r[:, 0], X_test_r[:, 1])[0, 1]) 93 | plt.legend(loc="best") 94 | plt.xticks(()) 95 | plt.yticks(()) 96 | 97 | plt.subplot(223) 98 | plt.plot(Y_train_r[:, 0], Y_train_r[:, 1], "*b", label="train") 99 | plt.plot(Y_test_r[:, 0], Y_test_r[:, 1], "*r", label="test") 100 | plt.xlabel("Y comp. 1") 101 | plt.ylabel("Y comp. 2") 102 | plt.title('Y comp. 1 vs Y comp. 2 , (test corr = %.2f)' 103 | % np.corrcoef(Y_test_r[:, 0], Y_test_r[:, 1])[0, 1]) 104 | plt.legend(loc="best") 105 | plt.xticks(()) 106 | plt.yticks(()) 107 | plt.show() 108 | 109 | ############################################################################### 110 | # PLS regression, with multivariate response, a.k.a. PLS2 111 | 112 | n = 1000 113 | q = 3 114 | p = 10 115 | X = np.random.normal(size=n * p).reshape((n, p)) 116 | B = np.array([[1, 2] + [0] * (p - 2)] * q).T 117 | # each Yj = 1*X1 + 2*X2 + noize 118 | Y = np.dot(X, B) + np.random.normal(size=n * q).reshape((n, q)) + 5 119 | 120 | pls2 = PLSRegression(n_components=3) 121 | pls2.fit(X, Y) 122 | print("True B (such that: Y = XB + Err)") 123 | print(B) 124 | # compare pls2.coefs with B 125 | print("Estimated B") 126 | print(np.round(pls2.coefs, 1)) 127 | pls2.predict(X) 128 | 129 | ############################################################################### 130 | # PLS regression, with univariate response, a.k.a. PLS1 131 | 132 | n = 1000 133 | p = 10 134 | X = np.random.normal(size=n * p).reshape((n, p)) 135 | y = X[:, 0] + 2 * X[:, 1] + np.random.normal(size=n * 1) + 5 136 | pls1 = PLSRegression(n_components=3) 137 | pls1.fit(X, y) 138 | # note that the number of compements exceeds 1 (the dimension of y) 139 | print("Estimated betas") 140 | print(np.round(pls1.coefs, 1)) 141 | 142 | ############################################################################### 143 | # CCA (PLS mode B with symmetric deflation) 144 | 145 | cca = CCA(n_components=2) 146 | cca.fit(X_train, Y_train) 147 | X_train_r, Y_train_r = plsca.transform(X_train, Y_train) 148 | X_test_r, Y_test_r = plsca.transform(X_test, Y_test) 149 | -------------------------------------------------------------------------------- /scripts/sift.py: -------------------------------------------------------------------------------- 1 | import os, sys 2 | import IPython 3 | import pickle 4 | import cv2 5 | import numpy as np 6 | import matplotlib.pyplot as plt 7 | 8 | import constants 9 | import utils 10 | 11 | def min_kp_SIFT(PATH_TO_DATA): 12 | print("Calculating #of features for video: " + PATH_TO_DATA) 13 | cap = cv2.VideoCapture(PATH_TO_DATA) 14 | sift = cv2.SIFT(nfeatures = 10) 15 | result = [] 16 | i = 0 17 | while(1): 18 | ret, frame = cap.read() 19 | if not ret: 20 | break; 21 | kp, des = sift.detectAndCompute(frame, None) 22 | print(i) 23 | result.append(len(kp)) 24 | i += 1 25 | 26 | cap.release() 27 | return min(result) 28 | 29 | def run_surf_frame(PATH_TO_IMAGE, n_features = 10): 30 | img = cv2.imread(PATH_TO_IMAGE) 31 | # gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) 32 | sift = cv2.SURF(4000) 33 | # kp = sift.detect(gray, None) 34 | # img = cv2.drawKeypoints(gray, kp) 35 | kp = sift.detect(img, None) 36 | img2 = cv2.drawKeypoints(img, kp) 37 | cv2.imshow("frame", img2) 38 | return len(kp) 39 | 40 | def generate_sift_features(): 41 | list_of_demonstrations = ["plane_9",] 42 | for demonstration in list_of_demonstrations: 43 | print "SIFT for ", demonstration 44 | PATH_TO_ANNOTATION = constants.PATH_TO_DATA + constants.ANNOTATIONS_FOLDER + demonstration + "_" + str(constants.CAMERA) + ".p" 45 | 46 | X1 = None 47 | X2 = None 48 | n_features = 20 49 | sift = cv2.SIFT(nfeatures = n_features) 50 | 51 | start, end = utils.get_start_end_annotations(PATH_TO_ANNOTATION) 52 | for frm in range(start, end + 1): 53 | # if ((frm % 3) == 0): 54 | PATH_TO_IMAGE = utils.get_full_image_path(constants.PATH_TO_DATA + constants.NEW_FRAMES_FOLDER + demonstration + "_" + constants.CAMERA + "/", frm) 55 | 56 | print PATH_TO_IMAGE 57 | img = cv2.imread(PATH_TO_IMAGE) 58 | kp, des = sift.detectAndCompute(img, None) 59 | img = cv2.drawKeypoints(img, kp) 60 | cv2.imshow('sift',img) 61 | cv2.imwrite('../sift_images/' + demonstration + "/" + str(frm) +".jpg",img) 62 | 63 | vector1 = [] 64 | vector2 = [] 65 | kp.sort(key = lambda x: x.response, reverse = True) 66 | for kp_elem in kp: 67 | vector1 += [kp_elem.response, kp_elem.pt[0], kp_elem.pt[1], kp_elem.size, kp_elem.angle] 68 | vector2 += [kp_elem.pt[0], kp_elem.pt[1]] 69 | try: 70 | X1 = utils.safe_concatenate(X1, utils.reshape(np.array(vector1[:n_features * 5]))) 71 | X2 = utils.safe_concatenate(X2, utils.reshape(np.array(vector2[:n_features * 2]))) 72 | except ValueError as e: 73 | IPython.embed() 74 | 75 | pickle.dump(X1, open("sift_features/SIFT_" + demonstration + "_1.p", "wb")) 76 | pickle.dump(X2, open("sift_features/SIFT_" + demonstration + "_2.p", "wb")) 77 | 78 | def run_sift_images(): 79 | list_of_demonstrations = ["Suturing_E001",] 80 | for demonstration in list_of_demonstrations: 81 | print "SIFT for ", demonstration 82 | PATH_TO_ANNOTATION = constants.PATH_TO_DATA + constants.ANNOTATIONS_FOLDER + demonstration + "_" + str(constants.CAMERA) + ".p" 83 | 84 | start, end = utils.get_start_end_annotations(PATH_TO_ANNOTATION) 85 | for frm in range(start, end + 1): 86 | if ((frm % 3) == 0): 87 | PATH_TO_IMAGE = utils.get_full_image_path(constants.PATH_TO_DATA + constants.NEW_FRAMES_FOLDER + demonstration + "_" + constants.CAMERA + "/", frm) 88 | 89 | print PATH_TO_IMAGE 90 | img = cv2.imread(PATH_TO_IMAGE) 91 | sift = cv2.SIFT(nfeatures = 50) 92 | kp, des = sift.detectAndCompute(img, None) 93 | img = cv2.drawKeypoints(img, kp) 94 | cv2.imshow('sift',img) 95 | cv2.imwrite('../sift_images/' + str(frm) +".jpg",img) 96 | 97 | 98 | def run_sift(PATH_TO_DATA, count, n_features = 20): 99 | cap = cv2.VideoCapture(PATH_TO_DATA) 100 | sift = cv2.SIFT(nfeatures = n_features) 101 | i = 0 102 | X1 = None 103 | X2 = None 104 | IPython.embed() 105 | while(1): 106 | print str(count) + " "+ str(i) 107 | ret, frame = cap.read() 108 | if not ret: 109 | break; 110 | kp, des = sift.detectAndCompute(frame, None) 111 | 112 | img = cv2.drawKeypoints(frame, kp) 113 | 114 | cv2.imshow('sift',img) 115 | vector1 = [] 116 | vector2 = [] 117 | kp.sort(key = lambda x: x.response, reverse = True) 118 | for kp_elem in kp: 119 | vector1 += [kp_elem.response, kp_elem.pt[0], kp_elem.pt[1], kp_elem.size, kp_elem.angle] 120 | vector2 += [kp_elem.pt[0], kp_elem.pt[1]] 121 | # vector2 = utils.reshape(des.flatten()) 122 | try: 123 | X1 = utils.safe_concatenate(X1, utils.reshape(np.array(vector1[:n_features * 5]))) 124 | X2 = utils.safe_concatenate(X2, utils.reshape(np.array(vector2[:n_features * 2]))) 125 | except ValueError as e: 126 | IPython.embed() 127 | 128 | cap.release() 129 | cv2.destroyAllWindows() 130 | return X1, X2 131 | 132 | if __name__ == "__main__": 133 | # list_of_demonstrations = ['Suturing_E001','Suturing_E002', 'Suturing_E003', 'Suturing_E004', 'Suturing_E005', 134 | # 'Suturing_D001','Suturing_D002', 'Suturing_D003', 'Suturing_D004', 'Suturing_D005', 135 | # 'Suturing_C001','Suturing_C002', 'Suturing_C003', 'Suturing_C004', 'Suturing_C005', 136 | # 'Suturing_F001','Suturing_F002', 'Suturing_F003', 'Suturing_F004', 'Suturing_F005'] 137 | generate_sift_features() 138 | # j = 0 139 | # for demonstration in list_of_demonstrations: 140 | # X1, X2 = run_sift("/Users/adithyamurali/dev/DeepMilestones/sift_videos/"+ demonstration + ".mp4", j) 141 | # # pickle.dump(X1, open("SIFT_" + demonstration + "_1.p", "wb")) 142 | # # pickle.dump(X2, open("SIFT_" + demonstration + "_2.p", "wb")) 143 | # j += 1 -------------------------------------------------------------------------------- /toyExample/toyEx_baseline2.m: -------------------------------------------------------------------------------- 1 | clear all 2 | close all 3 | % clc 4 | addpath(genpath(pwd)) 5 | 6 | %% Experiment specifc details 7 | %flags for experiment params 8 | noisyDynamics = false; 9 | targetNoise = true; 10 | observationNoise = true; 11 | 12 | flag_plotTraj = false; 13 | % In order of increasing difficulty 14 | % exptList = ['0001', '1000', '0100', '0101', '1001', '1101', '1110']; 15 | 16 | exptSetup = [num2str(noisyDynamics) num2str(targetNoise) num2str(observationNoise)]; 17 | 18 | fprintf('Running Sim for [noisyDynamics, targetNoise]:%s \n',... 19 | exptSetup); 20 | %make correct directory structure 21 | mkdir_if_not_exist ('baseline2'); 22 | delete(['baseline2' filesep '*.*']); %clear the output folder 23 | 24 | outputDirPrefix = ['baseline2' filesep exptSetup]; 25 | mkdir_if_not_exist (outputDirPrefix); 26 | 27 | kinDIR = [outputDirPrefix filesep exptSetup '_kinematics']; 28 | mkdir_if_not_exist (kinDIR); 29 | 30 | vidTrans= [outputDirPrefix filesep exptSetup '_video' filesep 'transcriptions']; 31 | mkdir_if_not_exist (vidTrans); 32 | vidFrames = [outputDirPrefix filesep exptSetup '_video' filesep 'frames']; 33 | mkdir_if_not_exist (vidFrames); 34 | 35 | %% 36 | 37 | numDemos = 5; 38 | maxStep = 1; 39 | xVel = 0.1; 40 | 41 | % dynamics noise is used as a percent of maxStep 42 | if noisyDynamics, dynamicsNoise = 0.25; end 43 | % observation noise is used as a percent of maxStep 44 | if observationNoise, obsNoise = 0.25; end 45 | % target noise level absolute values. 46 | if targetNoise, targetNoiseLevel = 1; end 47 | 48 | % initialize random number generator 49 | seed_RNG = rng(10,'twister'); 50 | numTargets = 5; 51 | % listX = random('unif', -10, 10, 2, numTargets ); 52 | target_list = [1, 1; 53 | 3, 10; 54 | 5, 2; 55 | 7, 9; 56 | 10, 1]'; 57 | 58 | if targetNoise 59 | target_list = target_list + random('unif', -maxStep, maxStep, 2, numTargets); 60 | end 61 | 62 | %% Initialize 63 | for expt = 1:numDemos 64 | fprintf('%s: %02d \n', exptSetup, expt); 65 | 66 | %% init params for loop 67 | outputDir = [vidFrames filesep exptSetup '_' num2str(expt,'%02d') '_capture1']; 68 | mkdir_if_not_exist (outputDir); 69 | if outputDir(end) ~= '/', outputDir = [outputDir filesep]; end 70 | 71 | target_buffer = [];%this buffer list is updated online as targets are reached 72 | 73 | %state representation is [x,y] 74 | x_curr = target_list (:,1); 75 | 76 | x_traj = x_curr; 77 | 78 | figHandle = figure(); 79 | axHandle = axes('parent',figHandle); 80 | hold (axHandle, 'on') 81 | axisLim = [-5 15 -5 15]; 82 | axis(axHandle, axisLim); 83 | axis(axHandle, 'manual') 84 | axis(axHandle, 'off') 85 | 86 | delete([outputDir '*.*']); %clear the output folder 87 | 88 | iterCount = 1; 89 | xTranscriptions = []; 90 | % This models as x(t+1) = x(t) + u(t) 91 | %% Iterate 92 | for targetCount = 2:numTargets 93 | % sample new target 94 | % target_curr = genTarget(); 95 | target_curr = target_list(:, targetCount); 96 | target_buffer = [target_buffer , target_curr]; 97 | 98 | if targetCount>2 99 | delete(objHandles) 100 | end 101 | objHandles = plotState_baseline(x_curr, target_curr, axHandle ); 102 | saveFig( figHandle, iterCount, outputDir ); 103 | 104 | xTranscriptions(targetCount, 1) = size(x_traj, 2); %start frame 105 | %% go to target in equisized steps. 106 | diff = target_curr - x_curr(1:2); 107 | 108 | while (norm(diff)>0) 109 | diff = target_curr - x_curr(1:2); 110 | if norm(diff)>maxStep 111 | if ~noisyDynamics % Perfect update 112 | x_curr(1:2)= x_curr(1:2) + maxStep*diff./norm(diff); 113 | else%noisy update based on an attracter 114 | x_curr(1:2)= x_curr(1:2) + maxStep*diff./norm(diff) +... 115 | dynamicsNoise*random('unif',-maxStep,maxStep,2,1); 116 | end 117 | 118 | else %last step to reach to target 119 | x_curr(1:2) = x_curr(1:2)+ diff; 120 | xTranscriptions(targetCount, 2) = size(x_traj, 2); %end frame 121 | if targetCount == numTargets %for last iteration so the traj length is same 122 | xTranscriptions(targetCount, 2) = xTranscriptions(targetCount, 2)+1; 123 | end 124 | end 125 | 126 | delete(objHandles) 127 | objHandles = plotState_baseline(x_curr, target_curr, axHandle ); 128 | if ~observationNoise 129 | x_traj = [x_traj, x_curr]; %record trajectory 130 | else 131 | %Only record trajectory with noise, vision sees clean traj 132 | x_traj = [x_traj, ... 133 | x_curr + obsNoise*random('unif',-maxStep,maxStep,2,1)]; 134 | end 135 | 136 | if flag_plotTraj 137 | trajHandle = plotTraj( x_traj, axHandle ); 138 | end 139 | 140 | iterCount = iterCount+1; 141 | saveFig( figHandle, iterCount, outputDir ); 142 | 143 | end 144 | 145 | %% increment target counter 146 | %targetCount = targetCount +1; 147 | end 148 | 149 | save([kinDIR filesep exptSetup '_' num2str(expt,'%02d') '.mat'],... 150 | 'target_buffer','x_traj') 151 | genLabelFile_baseline (xTranscriptions, vidTrans, [exptSetup '_' num2str(expt,'%02d')] ); 152 | 153 | close all 154 | end 155 | -------------------------------------------------------------------------------- /scripts/tsne.py: -------------------------------------------------------------------------------- 1 | import matlab.engine as mateng 2 | from time import time 3 | import IPython 4 | import numpy as np 5 | import matplotlib.pyplot as plt 6 | from matplotlib import offsetbox 7 | from sklearn import (manifold, datasets, decomposition, ensemble, lda, random_projection, preprocessing, covariance) 8 | from scipy import linalg 9 | import argparse 10 | import pickle 11 | import cv2 12 | import utils 13 | 14 | def parse(): 15 | eng = mateng.start_matlab() 16 | 17 | [s, X] = eng.read_binary_blob(PATH_TO_DATA + '1.conv5b', nargout = 2) 18 | i = 17; 19 | while i <= 1489: 20 | [s, data] = eng.read_binary_blob(PATH_TO_DATA + str(i) + '.conv5b', nargout = 2) 21 | data = np.array(data) 22 | X = np.concatenate((X, data), axis = 0) 23 | i += 16 24 | return X 25 | 26 | def parse_annotations_pickle(annotations_file, PATH_TO_DATA, layer): 27 | index_map = {} 28 | label_map = {} 29 | eng = mateng.start_matlab() 30 | map_index_data = pickle.load(open(annotations_file, "rb")) 31 | X = None 32 | i = 0 33 | for index in map_index_data: 34 | print "Parsing label " + str(index) 35 | segments = map_index_data[index] 36 | for seg in segments: 37 | j = seg[0] 38 | while j <= seg[1]: 39 | print j 40 | if X is None: 41 | [s, X] = eng.read_binary_blob(PATH_TO_DATA + str(j) + '.' + args.layer, nargout = 2) 42 | else: 43 | [s, data] = eng.read_binary_blob(PATH_TO_DATA + str(j) + '.' + args.layer, nargout = 2) 44 | data = np.array(data) 45 | X = np.concatenate((X, data), axis = 0) 46 | index_map[i] = j 47 | label_map[i] = index 48 | j += 16 49 | i += 1 50 | return X, label_map, index_map 51 | 52 | def plot_all(X): 53 | tsne = manifold.TSNE(n_components=2, init='pca', random_state=0) 54 | #---------------------------------------------------------------------- 55 | # Pre-processing 56 | print "t-SNE Scaling" 57 | X_scaled = preprocessing.scale(X) #zero mean, unit variance 58 | X_tsne_scaled = tsne.fit_transform(X_scaled) 59 | 60 | #normalize the data (scaling individual samples to have unit norm) 61 | print "t-SNE L2 Norm" 62 | X_normalized = preprocessing.normalize(X, norm='l2') 63 | X_tsne_norm = tsne.fit_transform(X_normalized) 64 | 65 | 66 | #whiten the data 67 | print "t-SNE Whitening" 68 | # the mean computed by the scaler is for the feature dimension. 69 | # We want the normalization to be in feature dimention. 70 | # Zero mean for each sample assumes stationarity which is not necessarily true for CNN features. 71 | # X: NxD where N is number of examples and D is number of features. 72 | 73 | # scaler = preprocessing.StandardScaler(with_std=False).fit(X) 74 | scaler = preprocessing.StandardScaler().fit(X) #this scales each feature to have std-dev 1 75 | X_centered = scaler.transform(X) 76 | 77 | # U, s, Vh = linalg.svd(X_centered) 78 | shapeX = X_centered.shape 79 | IPython.embed() 80 | # this is DxD matrix where D is the feature dimension 81 | # still to figure out: It seems computation is not a problem but carrying around a 50kx50k matrix is memory killer! 82 | sig = (1/shapeX[0]) * np.dot(X_centered.T, X_centered) 83 | sig2= covariance.empirical_covariance(X_centered, assume_centered=True) #estimated -- this is better. 84 | sig3, shrinkage= covariance.oas(X_centered, assume_centered=True) #estimated 85 | 86 | U, s, Vh = linalg.svd(sig, full_matrices=False) 87 | eps = 1e-2 # this affects how many low- freq eigevalues are eliminated 88 | invS = np.diag (np.reciprocal(np.sqrt(s+eps))) 89 | 90 | #PCA_whiten 91 | X_pca = np.dot(invS, np.dot(U.T, X_centered)) 92 | X_tsne_pca = tsne.fit_transform(X_pca) 93 | 94 | #whiten the data (ZCA) 95 | X_zca = np.dot(U, X_pca) 96 | X_tsne_zca = tsne.fit_transform(X_zca) 97 | 98 | return X_tsne_scaled, X_tsne_norm, X_tsne_pca, X_tsne_zca 99 | 100 | if __name__ == "__main__": 101 | parser = argparse.ArgumentParser() 102 | parser.add_argument("file_name", help = "Please specify MAIN file name") 103 | parser.add_argument("layer", help = "Please specify layer") 104 | parser.add_argument("PATH_TO_DATA", help="Please specify the path to the feature data") 105 | parser.add_argument("--a", help = "Annotated frames") 106 | parser.add_argument("--PATH_TO_DATA_2", help="Please specify the path to 2nd set of feature data") 107 | parser.add_argument("--a_2", help="Annotated frames for 2nd set of data") 108 | parser.add_argument("--image", help="Parse image mode", default = None) 109 | args = parser.parse_args() 110 | if args.a_2 and args.PATH_TO_DATA_2 and not args.image: 111 | X1, label_map_1, index_map_1 = parse_annotations_pickle(args.a, args.PATH_TO_DATA, args.layer) 112 | X2, label_map_2, index_map_2 = parse_annotations_pickle(args.a_2, args.PATH_TO_DATA_2, args.layer) 113 | X1_pca = utils.pca(X1) 114 | X2_pca = utils.pca(X2) 115 | plot_annotated_joint(X1_pca, X2_pca, label_map_1, index_map_1, label_map_2, index_map_2, figure_name = args.file_name +".png", title = "PCA " + args.layer) 116 | elif args.image and not args.PATH_TO_DATA_2: 117 | X, label_map, index_map = utils.parse_annotations_images(args.a, args.PATH_TO_DATA) 118 | pickle.dump(X, open(args.file_name + "_allimages.p", "wb")) 119 | pickle.dump(label_map, open(args.file_name + "_labelmap.p", "wb")) 120 | pickle.dump(index_map, open(args.file_name + "_indexmap.p", "wb")) 121 | IPython.embed() 122 | X_pca = utils.pca(X) 123 | X_tsne = utils.tsne(X) 124 | X_tsne_pca = utils.tsne_pca(X) 125 | utils.plot_annotated_embedding(X_pca, label_map, index_map, args.file_name + '_' + args.layer + '_pca.png', 'PCA ' + args.layer) 126 | utils.plot_annotated_embedding(X_tsne, label_map, index_map, args.file_name + '_' + args.layer + '_tsne.png', 't-SNE ' + args.layer) 127 | utils.plot_annotated_embedding(X_tsne_pca, label_map, index_map, args.file_name + '_' + args.layer + '_tsne_pca.png', 't-SNE (PCA Input) ' + args.layer) 128 | else: 129 | if args.a: 130 | X, label_map, index_map = parse_annotations_pickle(args.a, args.PATH_TO_DATA, args.layer) 131 | else: 132 | X, label_map, index_map = parse_annotations(args) 133 | 134 | X_pca = utils.pca(X) 135 | X_tsne = utils.tsne(X) 136 | X_tsne_pca = utils.tsne_pca(X) 137 | utils.plot_annotated_embedding(X_pca, label_map, index_map, args.file_name + '_' + args.layer + '_pca.png', 'PCA - C3D ' + args.layer) 138 | utils.plot_annotated_embedding(X_tsne, label_map, index_map, args.file_name + '_' + args.layer + '_tsne.png', 't-SNE - C3D ' + args.layer) 139 | utils.plot_annotated_embedding(X_tsne_pca, label_map, index_map, args.file_name + '_' + args.layer + '_tsne_pca.png', 't-SNE(PCA input) - C3D ' + args.layer) -------------------------------------------------------------------------------- /scripts/constants.py: -------------------------------------------------------------------------------- 1 | import yaml 2 | import numpy as np 3 | import os 4 | import sys 5 | 6 | default_config = {'ALEXNET_FEATURES_FOLDER': 'alexnetfeatures/', 'ALPHA_W_CP': 750, 'ALPHA_ZW_CP': 1000, 'ALPHA_Z_CP': 0.001, 7 | 'CAMERA': 'capture2', 'CONFIG_FILE': 'meta_file_Suturing.txt', 'CROP_PARAMS_CAPTURE_1': 'crop=330:260:200:170', 8 | 'CROP_PARAMS_CAPTURE_2': 'crop=330:260:150:150', 'DPGMM_DIVISOR': 45, 'DPGMM_DIVISOR_L1': 15, 'KINEMATICS_DIM': 38, 9 | 'N_COMPONENTS_CP': 5, 'N_COMPONENTS_CP_W': 5, 'N_COMPONENTS_CP_Z': 15, 'N_COMPONENTS_L1': 20, 10 | 'N_COMPONENTS_L1_W': 15, 'N_COMPONENTS_L1_Z': 15, 'N_COMPONENTS_L2': 2, 'N_COMPONENTS_TIME_W': 40, 11 | 'N_COMPONENTS_TIME_Z': 40, 'N_COMPONENTS_TIME_ZW': 40, 'PATH_TO_CLUSTERING_RESULTS': 'clustering_suturing_test/', 12 | 'PATH_TO_DATA': 'Suturing_video/', 'PATH_TO_KINEMATICS': 'Suturing_kinematics/kinematics/AllGestures/', 13 | 'PROC_FEATURES_FOLDER': 'features_E12345/', 'PRUNING_FACTOR_T': 0.49, 'PRUNING_FACTOR_W': 0.75, 'PRUNING_FACTOR_Z': 0.49, 14 | 'PRUNING_FACTOR_ZW': 0.49, 'REMOTE': 1, 'SIFT_FEATURES_FOLDER': 'siftfeatures/', 'SIMULATION': False, 15 | 'SR': 3, 'TASK_NAME': 'suturing', 'TEMPORAL_WINDOW_W': 2, 'TEMPORAL_WINDOW_Z': 2, 'TEMPORAL_WINDOW_ZW': 2, 16 | 'VGG_FEATURES_FOLDER': 'vggfeatures_2/', 'WEIGHTED_PRUNING_MODE': False, 'WEIGHT_EXPERT': 30, 'WEIGHT_INTERMEDIATE': 2} 17 | 18 | def parse_yaml(yaml_fname): 19 | config = yaml.load(open(yaml_fname, 'r')) 20 | return config 21 | 22 | class Config: 23 | def __init__(self, yaml_fname): 24 | self.__vars = parse_yaml("../config/"+yaml_fname) 25 | self.__yaml_fname = yaml_fname 26 | 27 | def get(self, var_name): 28 | if var_name in self.__vars: 29 | return self.__vars[var_name] 30 | if var_name not in default_config: 31 | print "ERROR: Incorrect variable name, ", str(var_name) 32 | sys.exit() 33 | else: 34 | print "ERROR: Using default parameter for ", str(var_name) 35 | return default_config[var_name] 36 | 37 | f = open('../config/defaultconfig', 'r+') 38 | config = Config(f.readline().strip('\n')) 39 | 40 | # Constants for Path resolution 41 | REPO_ROOT = os.path.abspath(os.path.join(os.getcwd(), os.pardir)) 42 | CAFFE_ROOT = '/home/animesh/caffe/' 43 | TASK_NAME = config.get('TASK_NAME') 44 | PATH_TO_DATA = REPO_ROOT + "/data/" + config.get('PATH_TO_DATA') 45 | PATH_TO_CLUSTERING_RESULTS = REPO_ROOT + "/clustering/" + config.get('PATH_TO_CLUSTERING_RESULTS') 46 | 47 | if not os.path.exists(PATH_TO_CLUSTERING_RESULTS): 48 | print PATH_TO_CLUSTERING_RESULTS, " does not exist; new folder created" 49 | os.mkdir(PATH_TO_CLUSTERING_RESULTS) 50 | 51 | PATH_TO_KINEMATICS = REPO_ROOT + "/data/" + config.get('PATH_TO_KINEMATICS') 52 | PATH_TO_OPENCV_2_4_9 = "~/opencv_2.4.9/opencv-2.4.9/lib/" 53 | PATH_TO_SAVE_FIG = REPO_ROOT + "/plots/" 54 | CONFIG_FILE = config.get('CONFIG_FILE') 55 | VIDEO_FOLDER = "video/" 56 | NEW_FRAMES_FOLDER = "frames/" 57 | NEW_BGSUB_FOLDER = "bgsubframes/" 58 | TRANSCRIPTIONS_FOLDER = "transcriptions/" 59 | ANNOTATIONS_FOLDER = "annotations/" 60 | ALEXNET_FEATURES_FOLDER = config.get("ALEXNET_FEATURES_FOLDER") 61 | VGG_FEATURES_FOLDER = config.get("VGG_FEATURES_FOLDER") 62 | SIFT_FEATURES_FOLDER = config.get("SIFT_FEATURES_FOLDER") 63 | PROC_FEATURES_FOLDER = config.get("PROC_FEATURES_FOLDER") 64 | CROP_PARAMS_CAPTURE_1 = config.get("CROP_PARAMS_CAPTURE_1") 65 | CROP_PARAMS_CAPTURE_2 = config.get("CROP_PARAMS_CAPTURE_2") 66 | 67 | # Nice color maps 68 | if TASK_NAME in ["100", "010", "011", "plane", "lego", "people", "people2"]: 69 | color_map = {1:'#00A598', 2:'#ffc72c', 3:'#e04e39', 4:'#00b5e2', 5: '#B9D3B6', 6:'#00b2a9', 7:'k', 8:'#ffc72c', 9: '#9932CC', 10: '#E9967A', 11: '#584F29', 12: '#008080'} 70 | else: 71 | color_map = {1:'#00A598', 2:'#00b2a9', 3:'#e04e39', 4:'#ffc72c', 5: '#B9D3B6', 6:'#00b5e2', 7:'k', 8:'#ffc72c', 9: '#9932CC', 10: '#E9967A', 11: '#584F29', 12: '#008080'} 72 | 73 | # Surgeme representations 74 | alphabet_map = {} 75 | 76 | for i in range(500): 77 | alphabet_map[i] = "label" + str(i) 78 | 79 | map_surgeme_label = {'G1': 1, "G2": 2, "G3": 3, "G4": 4, "G5": 5, "G6": 6, "G7": 7, "G8": 8, "G9": 9, 80 | "G10": 10, "G12": 12, "G11": 11, "G13": 13, "G14": 14, "G15": 15, "G16": 16, "G17": 17} 81 | 82 | # Caffe CNN variables 83 | caffe_conv_dimensions = {'conv3': (384, 13), 'conv4':(384, 13), 'pool5': (256, 6), 'conv5_3': (512, 14), 'conv5_1': (512, 14)} 84 | alex_net_layers = ['conv1', 'conv2', 'conv3', 'conv4', 'conv5', 'pool5', 'fc6', 'fc7'] 85 | vgg_layers = ['conv1_1', 'conv1_2', 'conv2_1', 'conv2_2', 'conv3_1', 'conv3_2', 'conv3_3', 'conv4_1', 'conv4_2', 'conv4_3','conv5_1', 'conv5_2', 'conv5_3', 'pool5'] 86 | NET_PARAMS = {"AlexNet": [CAFFE_ROOT + 'models/bvlc_reference_caffenet/deploy.prototxt', CAFFE_ROOT + 'models/bvlc_reference_caffenet/bvlc_reference_caffenet.caffemodel', 87 | alex_net_layers], "VGG_SOS": [CAFFE_ROOT + 'models/vgg_sos/deploy.prototxt', CAFFE_ROOT + 'models/vgg_sos/VGG16_SalObjSub.caffemodel', vgg_layers], 88 | "VGG": [CAFFE_ROOT + 'models/vgg/deploy.prototxt', CAFFE_ROOT + 'models/vgg/VGG_ILSVRC_16_layers.caffemodel', vgg_layers]} 89 | 90 | # Misc. Parameters 91 | CROP_PARAMS = {"capture1": CROP_PARAMS_CAPTURE_1, "capture2": CROP_PARAMS_CAPTURE_2} 92 | SR = config.get("SR") 93 | CAMERA = config.get("CAMERA") 94 | 95 | # Clustering Parameters 96 | PRUNING_FACTOR_W = config.get("PRUNING_FACTOR_W") 97 | PRUNING_FACTOR_Z = config.get("PRUNING_FACTOR_Z") 98 | PRUNING_FACTOR_ZW = config.get("PRUNING_FACTOR_ZW") 99 | PRUNING_FACTOR_T = config.get("PRUNING_FACTOR_T") 100 | REMOTE = config.get("REMOTE") 101 | SIMULATION = config.get("SIMULATION") 102 | KINEMATICS_DIM = config.get("KINEMATICS_DIM") 103 | N_COMPONENTS_CP = config.get("N_COMPONENTS_CP") 104 | N_COMPONENTS_L1 = config.get("N_COMPONENTS_L1") 105 | N_COMPONENTS_L2 = config.get("N_COMPONENTS_L2") 106 | N_COMPONENTS_CP_W = config.get("N_COMPONENTS_CP_W") 107 | N_COMPONENTS_L1_W = config.get("N_COMPONENTS_L1_W") 108 | N_COMPONENTS_CP_Z = config.get("N_COMPONENTS_CP_Z") 109 | N_COMPONENTS_L1_Z = config.get("N_COMPONENTS_L1_Z") 110 | N_COMPONENTS_TIME_W = config.get("N_COMPONENTS_TIME_W") 111 | N_COMPONENTS_TIME_Z = config.get("N_COMPONENTS_TIME_Z") 112 | N_COMPONENTS_TIME_ZW = config.get("N_COMPONENTS_TIME_ZW") 113 | TEMPORAL_WINDOW_ZW = config.get("TEMPORAL_WINDOW_ZW") 114 | TEMPORAL_WINDOW_W = config.get("TEMPORAL_WINDOW_W") 115 | TEMPORAL_WINDOW_Z = config.get("TEMPORAL_WINDOW_Z") 116 | ALPHA_W_CP = config.get("ALPHA_W_CP") 117 | ALPHA_Z_CP = config.get("ALPHA_Z_CP") 118 | ALPHA_ZW_CP = config.get("ALPHA_ZW_CP") 119 | DPGMM_DIVISOR = config.get("DPGMM_DIVISOR") 120 | DPGMM_DIVISOR_L1 = config.get("DPGMM_DIVISOR_L1") 121 | WEIGHT_EXPERT = config.get("WEIGHT_EXPERT") 122 | WEIGHT_INTERMEDIATE = config.get("WEIGHT_INTERMEDIATE") 123 | WEIGHTED_PRUNING_MODE = config.get("WEIGHTED_PRUNING_MODE") 124 | 125 | train_test_ratio = 0.3 126 | -------------------------------------------------------------------------------- /toyExample/main.m: -------------------------------------------------------------------------------- 1 | clear all 2 | close all 3 | % clc 4 | addpath(genpath(pwd)) 5 | 6 | %% Experiment specifc details 7 | %flags for experiment params 8 | varyRobotInit = false; 9 | noisyDynamics = true; 10 | varyTargetInit = false; 11 | targetNoise = false; 12 | 13 | % In order of increasing difficulty 14 | % exptList = ['0001', '1000', '0100', '0101', '1001', '1101', '1110']; 15 | 16 | exptSetup = [num2str(varyRobotInit) num2str(noisyDynamics)... 17 | num2str(varyTargetInit) num2str(targetNoise)]; 18 | 19 | fprintf('Running Sim for [varyRobotInit, noisyDynamics, varyTargetInit, targetNoise]:%s \n',... 20 | exptSetup); 21 | %make correct directory structure 22 | mkdir_if_not_exist ('output'); 23 | 24 | outputDirPrefix = ['output' filesep exptSetup]; 25 | mkdir_if_not_exist (outputDirPrefix); 26 | 27 | kinDIR = [outputDirPrefix filesep exptSetup '_kinematics']; 28 | mkdir_if_not_exist (kinDIR); 29 | 30 | vidTrans= [outputDirPrefix filesep exptSetup '_video' filesep 'transcriptions']; 31 | mkdir_if_not_exist (vidTrans); 32 | vidFrames = [outputDirPrefix filesep exptSetup '_video' filesep 'frames']; 33 | mkdir_if_not_exist (vidFrames); 34 | 35 | numDemos = 5; 36 | flag_plotTraj = true; 37 | 38 | % initialize random number generator 39 | seed_RNG = rng(1,'twister'); 40 | % if strcmp(exptSetup, '0101') 41 | % seed_RNG = rng(10,'twister');%for exptsetup only 0101 42 | % end 43 | 44 | %% Initialize 45 | for expt = 1:numDemos 46 | fprintf('%s: %02d \n', exptSetup, expt); 47 | 48 | %% init params for loop 49 | outputDir = [vidFrames filesep exptSetup '_' num2str(expt,'%02d') '_capture1']; 50 | mkdir_if_not_exist (outputDir); 51 | if outputDir(end) ~= '/', outputDir = [outputDir filesep]; end 52 | 53 | % pause('on') %for save fig 54 | 55 | numTargets = 3; 56 | 57 | %to use the same targets as first iteration 58 | if expt==1 59 | target_list = random('unif', -10, 10, 2, numTargets); 60 | end 61 | target_buffer = [];%this buffer list is updated online as targets are reached 62 | 63 | %generate new targets for every expt run 64 | if expt>1 && varyTargetInit 65 | target_list = random('unif', -10, 10, 2, numTargets); 66 | end 67 | 68 | %state representation is [x,y,theta (radian)] 69 | if ~varyRobotInit 70 | x_curr = [0;0;0]; 71 | else 72 | %rng(100,'twister'); 73 | x_curr = [random('unif',-10,10, 2,1); random('unif',-pi, pi) ]; 74 | %reset the seed after generating random robot position so that we 75 | %get same target positions 76 | %rng(seed_RNG); 77 | end 78 | 79 | x_traj = x_curr; 80 | 81 | maxStep = 1;%only works for maxStep greater than one 82 | rotStepSize = pi/18; 83 | % dynamics noise is used as a percent of maxStep 84 | if noisyDynamics, dynamicsNoise = 0.5; end 85 | % target noise level absolute values. 86 | if targetNoise, targetNoiseLevel = 1; end 87 | 88 | figHandle = figure(); 89 | axHandle = axes('parent',figHandle); 90 | hold (axHandle, 'on') 91 | axisLim = [-12 12 -12 12]; 92 | axis(axHandle, axisLim); 93 | axis(axHandle, 'manual') 94 | axis(axHandle, 'off') 95 | 96 | delete([outputDir '*.*']); %clear the output folder 97 | 98 | iterCount = 1; 99 | 100 | % This models as x(t+1) = x(t) + u(t) 101 | 102 | %% Iterate 103 | for targetCount = 1:numTargets 104 | % sample new target 105 | % target_curr = genTarget(); 106 | target_curr = target_list(:, targetCount); 107 | 108 | if targetNoise 109 | target_curr = target_curr+ ... 110 | random('unif',-targetNoiseLevel,targetNoiseLevel, 2,1); 111 | % make sure target is in [-10,10]x[-10,10] 112 | if sum(abs(target_curr)>10)>=1 113 | target_curr(abs(target_curr)>10) = ... 114 | 10*sign(target_curr(abs(target_curr)>10)); 115 | end 116 | end 117 | 118 | target_buffer = [target_buffer , target_curr]; 119 | 120 | if targetCount>1 121 | delete(objHandles) 122 | end 123 | objHandles = plotState(x_curr, target_curr, axHandle ); 124 | saveFig( figHandle, iterCount, outputDir ); 125 | 126 | %% search for target -- No Noise in rotation 127 | diff = target_curr - x_curr(1:2); 128 | headingToTarget = atan2 (diff(2), diff(1)); 129 | 130 | sDir = -pi:rotStepSize:pi; 131 | sDir = sort([sDir(2:end), headingToTarget, x_curr(3)]); 132 | ind_curr = find(sDir == x_curr(3) ); 133 | ind_target = find(sDir == headingToTarget ); 134 | 135 | %to handle cyclic nature of rotations. 136 | %add ind_curr+1 to not include current state as one step. 137 | if min(ind_curr) >= ind_target 138 | searchArray = [sDir(max(ind_curr)+1:end), sDir(1:ind_target)]; 139 | else 140 | searchArray = sDir(max(ind_curr)+1:ind_target); 141 | end 142 | 143 | for s = 1:length(searchArray) 144 | x_curr(3) = searchArray (s); 145 | delete(objHandles) 146 | objHandles = plotState(x_curr, target_curr, axHandle ); 147 | x_traj = [x_traj, x_curr]; %record trajectory 148 | 149 | iterCount = iterCount+1; 150 | saveFig( figHandle, iterCount, outputDir ); 151 | end 152 | 153 | %% go to target in equisized steps. 154 | 155 | while (norm(diff)>0) 156 | diff = target_curr - x_curr(1:2); 157 | if norm(diff)>maxStep 158 | if ~noisyDynamics % Perfect update 159 | x_curr(1:2)= x_curr(1:2) + maxStep*diff./norm(diff); 160 | else%noisy update based on an attracter 161 | x_curr(1:2)= x_curr(1:2) + maxStep*diff./norm(diff) +... 162 | dynamicsNoise*random('unif',-maxStep,maxStep,2,1); 163 | end 164 | 165 | else %last step to reach to target 166 | x_curr(1:2) = x_curr(1:2)+ diff; 167 | end 168 | 169 | delete(objHandles) 170 | objHandles = plotState(x_curr, target_curr, axHandle ); 171 | x_traj = [x_traj, x_curr]; %record trajectory 172 | if flag_plotTraj 173 | trajHandle = plotTraj( x_traj, axHandle ); 174 | end 175 | 176 | iterCount = iterCount+1; 177 | saveFig( figHandle, iterCount, outputDir ); 178 | 179 | end 180 | 181 | %% increment target counter 182 | %targetCount = targetCount +1; 183 | end 184 | 185 | save([kinDIR filesep exptSetup '_' num2str(expt,'%02d') '.mat'],... 186 | 'target_buffer','x_traj') 187 | genLabelFile (x_traj, vidTrans, [exptSetup '_' num2str(expt,'%02d')] ); 188 | 189 | close all 190 | end 191 | -------------------------------------------------------------------------------- /scripts/tscdl.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import pickle 4 | import sys 5 | import os 6 | import argparse 7 | import sys 8 | import IPython 9 | 10 | import constants 11 | import utils 12 | from clustering import TSCDL_multimodal, post_evaluation_multimodal, get_list_of_demo_combinations 13 | from clustering_kinematics import TSCDL_singlemodal, post_evaluation_singlemodal 14 | from broken_barh import plot_broken_barh_from_pickle 15 | from jaccard import TSCDL_manual_multimodal, TSCDL_manual_singlemodal 16 | 17 | # One script to rule all other scripts 18 | # Adithya, 2016 19 | 20 | def check_visual_features(args): 21 | """ 22 | Function to check for visual features 23 | """ 24 | if not args.visual_feature: 25 | print "ERROR: Please specify visual feature: --visual_feature .p" 26 | os.remove(constants.PATH_TO_CLUSTERING_RESULTS + args.output_fname + ".txt") 27 | sys.exit() 28 | 29 | if __name__ == "__main__": 30 | argparser = argparse.ArgumentParser() 31 | argparser.add_argument("mode", help = "python tscdl.py Z _test_ --visual_feature 5_PCA.p Choose from 1) W (Kinematics only),\n 2) Z (Visual features only),\n 3) ZW (Kinematics+Vision),\n 4) plot,\n 5) control-W,\n 6) control-Z,\n 7) control-ZW") 32 | argparser.add_argument("output_fname", help = "Pickle file of visual features") 33 | argparser.add_argument("--visual_feature", help = "Pickle file of visual features") 34 | argparser.add_argument("--W", help = "Cache of data running TSCDL on Kinematics only") 35 | argparser.add_argument("--Z", help = "Cache of data running TSCDL on Vision only") 36 | argparser.add_argument("--ZW", help = "Cache of data running TSCDL on Both kinematics and vision") 37 | args = argparser.parse_args() 38 | 39 | list_of_demonstrations = constants.config.get("list_of_demonstrations") 40 | combinations = get_list_of_demo_combinations(list_of_demonstrations) 41 | all_metrics = [] 42 | 43 | if args.mode == "ZW": 44 | check_visual_features(args) 45 | 46 | file = open(constants.PATH_TO_CLUSTERING_RESULTS + args.output_fname + ".txt", "wb") 47 | for i, elem in enumerate(combinations): 48 | print "---- Kinematics + Vision Combination #"+ str(i + 1) + "----" 49 | mc = TSCDL_multimodal(False, list(elem), args.visual_feature, args.output_fname) 50 | all_metrics.append(mc.do_everything()) 51 | 52 | print "----------- CALCULATING THE ODDS ------------" 53 | post_evaluation_multimodal(all_metrics, file, args.output_fname, list_of_demonstrations, args.visual_feature) 54 | file.close() 55 | 56 | elif args.mode == "W": 57 | 58 | file = open(constants.PATH_TO_CLUSTERING_RESULTS + args.output_fname + ".txt", "wb") 59 | for i, elem in enumerate(combinations): 60 | utils.print_and_write("\n----------- Kinematics-only Combination #" + str(i + 1) + " -------------\n", file) 61 | print "\n----------- Kinematics-only Combination #" + str(i + 1) + " -------------\n" 62 | print elem 63 | mc = TSCDL_singlemodal(False, list(elem), args.output_fname + str(i), file, False, None) 64 | all_metrics.append(mc.do_everything()) 65 | 66 | print "----------- CALCULATING THE ODDS ------------" 67 | post_evaluation_singlemodal(all_metrics, file, args.output_fname, False, list_of_demonstrations) 68 | file.close() 69 | 70 | elif args.mode == "Z": 71 | check_visual_features(args) 72 | 73 | file = open(constants.PATH_TO_CLUSTERING_RESULTS + args.output_fname + ".txt", "wb") 74 | for i, elem in enumerate(combinations): 75 | utils.print_and_write("\n----------- Vision-only Combination #" + str(i + 1) + " -------------\n", file) 76 | print "\n----------- Vision-only Combination #" + str(i + 1) + " -------------\n" 77 | print elem 78 | mc = TSCDL_singlemodal(False, list(elem), args.output_fname + str(i), file, True, args.visual_feature) 79 | all_metrics.append(mc.do_everything()) 80 | 81 | print "----------- CALCULATING THE ODDS ------------" 82 | post_evaluation_singlemodal(all_metrics, file, args.output_fname, True, list_of_demonstrations) 83 | file.close() 84 | 85 | elif args.mode == "plot": 86 | if not (args.Z and args.ZW and args.W): 87 | print "ERROR: Please specify pickle files for all TSCDL experiments" 88 | sys.exit() 89 | 90 | W = pickle.load(open(args.W, "rb")) 91 | Z = pickle.load(open(args.Z, "rb")) 92 | ZW = pickle.load(open(args.ZW, "rb")) 93 | 94 | assert len(W.keys()) == len(Z.keys()) == len(ZW.keys()) 95 | for demonstration in W.keys(): 96 | labels_manual = W[demonstration]['plot_labels_manual'] 97 | colors_manual = W[demonstration]['plot_colors_manual'] 98 | labels_W = W[demonstration]['plot_labels_automatic'] 99 | colors_W = W[demonstration]['plot_colors_automatic'] 100 | labels_Z = Z[demonstration]['plot_labels_automatic'] 101 | colors_Z = Z[demonstration]['plot_colors_automatic'] 102 | labels_ZW = ZW[demonstration]['plot_labels_automatic'] 103 | colors_ZW = ZW[demonstration]['plot_colors_automatic'] 104 | 105 | 106 | plot_broken_barh_from_pickle(demonstration, args.output_fname + "_" + demonstration, labels_manual, colors_manual, 107 | labels_W, colors_W, labels_Z, colors_Z, labels_ZW, colors_ZW) 108 | 109 | if args.mode == "control-ZW": 110 | check_visual_features(args) 111 | 112 | file = open(constants.PATH_TO_CLUSTERING_RESULTS + args.output_fname + ".txt", "wb") 113 | for i, elem in enumerate(combinations): 114 | print "---- CONTROL: Kinematics + Vision Combination #"+ str(i + 1) + "----" 115 | mc = TSCDL_manual_multimodal(False, list(elem), args.visual_feature, args.output_fname) 116 | all_metrics.append(mc.do_everything()) 117 | 118 | print "----------- CALCULATING THE ODDS ------------" 119 | post_evaluation_multimodal(all_metrics, file, args.output_fname, list_of_demonstrations, args.visual_feature) 120 | file.close() 121 | 122 | elif args.mode == "control-W": 123 | 124 | file = open(constants.PATH_TO_CLUSTERING_RESULTS + args.output_fname + ".txt", "wb") 125 | for i, elem in enumerate(combinations): 126 | utils.print_and_write("\n----------- Kinematics-only Combination #" + str(i + 1) + " -------------\n", file) 127 | print "\n----------- CONTROL: Kinematics-only Combination #" + str(i + 1) + " -------------\n" 128 | print elem 129 | mc = TSCDL_manual_singlemodal(False, list(elem), args.output_fname + str(i), file, False, None) 130 | all_metrics.append(mc.do_everything()) 131 | 132 | print "----------- CALCULATING THE ODDS ------------" 133 | post_evaluation_singlemodal(all_metrics, file, args.output_fname, False, list_of_demonstrations) 134 | file.close() 135 | 136 | elif args.mode == "control-Z": 137 | check_visual_features(args) 138 | 139 | file = open(constants.PATH_TO_CLUSTERING_RESULTS + args.output_fname + ".txt", "wb") 140 | for i, elem in enumerate(combinations): 141 | utils.print_and_write("\n----------- Vision-only Combination #" + str(i + 1) + " -------------\n", file) 142 | print "\n----------- CONTROL: Vision-only Combination #" + str(i + 1) + " -------------\n" 143 | print elem 144 | mc = TSCDL_manual_singlemodal(False, list(elem), args.output_fname + str(i), file, True, args.visual_feature) 145 | all_metrics.append(mc.do_everything()) 146 | 147 | print "----------- CALCULATING THE ODDS ------------" 148 | post_evaluation_singlemodal(all_metrics, file, args.output_fname, True, list_of_demonstrations) 149 | file.close() 150 | else: 151 | print "ERROR: Please specify a valid mode" 152 | -------------------------------------------------------------------------------- /scripts/recorder.py: -------------------------------------------------------------------------------- 1 | import sys 2 | # sys.path.append() 3 | 4 | import roslib 5 | import rospy 6 | import cPickle as pickle 7 | import numpy as np 8 | import os 9 | import IPython 10 | import time 11 | import tf 12 | import utils 13 | import time 14 | 15 | from sensor_msgs.msg import JointState, Image 16 | from std_msgs.msg import String, Float32 17 | from tf import transformations 18 | 19 | import cv 20 | import cv2 21 | import cv_bridge 22 | import numpy as np 23 | import signal 24 | 25 | import constants 26 | 27 | def get_frame_fig_name(frm_num): 28 | """ 29 | Useful for parsing frames and loading into memory. 30 | """ 31 | if len(str(frm_num)) == 1: 32 | return "00000" + str(frm_num) + ".jpg" 33 | elif len(str(frm_num)) == 2: 34 | return "0000" + str(frm_num) + ".jpg" 35 | elif len(str(frm_num)) == 3: 36 | return "000" + str(frm_num) + ".jpg" 37 | elif len(str(frm_num)) == 4: 38 | return "00" + str(frm_num) + ".jpg" 39 | else: 40 | pass 41 | 42 | class Recording(object): 43 | 44 | def __init__(self, trial_name): 45 | 46 | # Preparing folders 47 | self.task_name = constants.TASK_NAME 48 | 49 | self.record_kinematics = False 50 | if self.task_name in ["plane", "lego"]: 51 | self.record_kinematics = True 52 | 53 | self.trial_name = trial_name 54 | self.kinematics_folder = constants.PATH_TO_KINEMATICS 55 | self.video_folder = constants.PATH_TO_DATA + constants.NEW_FRAMES_FOLDER 56 | 57 | # Make folders for frames 58 | command = self.video_folder + self.task_name + "_" + self.trial_name + "_capture1/" 59 | print "mkdir " + command 60 | os.makedirs(command) 61 | command = self.video_folder + self.task_name + "_" + self.trial_name + "_capture2/" 62 | print "mkdir " + command 63 | os.makedirs(command) 64 | 65 | self.data = None 66 | self.frequency = 10 # max save framerate is 10 67 | self.period = 1.0/self.frequency 68 | 69 | # Data to record 70 | self.left_image = None 71 | self.right_image = None 72 | self.psm1_gripper = None 73 | self.psm2_gripper = None 74 | self.psm1_pose = None 75 | self.psm2_pose = None 76 | self.joint_state = None 77 | self.listener = tf.TransformListener() 78 | 79 | # Subscribers for images 80 | rospy.Subscriber("/wide_stereo/left/image_rect_color", Image, self.left_image_callback, queue_size=1) 81 | rospy.Subscriber("/wide_stereo/right/image_rect_color", Image, self.right_image_callback, queue_size=1) 82 | 83 | if self.record_kinematics: 84 | # Subscribers for kinematics 85 | rospy.Subscriber("/joint_states", JointState, self.joint_state_callback) 86 | 87 | self.bridge = cv_bridge.CvBridge() 88 | self.r_l = 0 89 | self.r_r = 0 90 | 91 | signal.signal(signal.SIGINT, self.signal_handler) 92 | 93 | def left_image_callback(self, msg): 94 | self.r_l += 1 95 | self.left_image = self.bridge.imgmsg_to_cv2(msg, "bgr8") 96 | 97 | def right_image_callback(self, msg): 98 | self.r_r += 1 99 | self.right_image = self.bridge.imgmsg_to_cv2(msg, "bgr8") 100 | 101 | def joint_state_callback(self, msg): 102 | self.joint_state = msg 103 | 104 | def save_and_quit(self): 105 | if self.record_kinematics: 106 | rospy.loginfo("Saving data to pickle file.") 107 | try: 108 | print "Saving pickle file to ", self.kinematics_folder + self.task_name +"_" + self.trial_name + ".p" 109 | pickle.dump(self.data, open(self.kinematics_folder + self.task_name +"_" + self.trial_name + ".p", "wb")) 110 | except Exception as e: 111 | print "Exception: ", e 112 | rospy.logwarn('Failed to save registration') 113 | IPython.embed() 114 | print "Fin" 115 | sys.exit() 116 | 117 | def signal_handler(self, signum, something): 118 | self.save_and_quit() 119 | 120 | def start_recording(self): 121 | 122 | print "Recorder Loop" 123 | while self.left_image is None or self.right_image is None: 124 | pass 125 | 126 | if self.record_kinematics: 127 | while (1): 128 | try: 129 | (trans,rot) = self.listener.lookupTransform('/r_gripper_tool_frame', '/base_link', rospy.Time(0)) 130 | break 131 | except (tf.ExtrapolationException): 132 | print "ExtrapolationException" 133 | rospy.sleep(0.1) 134 | continue 135 | 136 | frm = 0 137 | wait_thresh = 0 138 | prev_r_l = self.r_l 139 | prev_r_r = self.r_r 140 | 141 | trans_vel = np.array([0.0, 0.0, 0.0]) 142 | rot_vel = np.array([0.0, 0.0, 0.0]) 143 | 144 | prev_trans = None 145 | prev_rot = None 146 | 147 | for i in range(9999999): 148 | print frm 149 | rospy.sleep(self.period) 150 | 151 | start = time.time() 152 | 153 | cv2.imwrite(self.video_folder + self.task_name + "_" + self.trial_name + "_capture1/" + str(get_frame_fig_name(frm)), self.left_image) 154 | cv2.imwrite(self.video_folder + self.task_name + "_" + self.trial_name + "_capture2/" + str(get_frame_fig_name(frm)), self.right_image) 155 | 156 | if self.record_kinematics: 157 | 158 | (trans, quaternion) = self.listener.lookupTransform('/r_gripper_tool_frame', '/base_link', rospy.Time(0)) 159 | r_matrix = utils.quaternion2rotation(quaternion) 160 | rot = transformations.euler_from_matrix(r_matrix) 161 | r_gripper_angle = self.joint_state.position[-17] 162 | 163 | if frm != 0: 164 | trans_vel = (trans - prev_trans) / self.period 165 | rot_vel = (rot - prev_rot) / self.period 166 | 167 | prev_trans = np.array(trans) 168 | prev_rot = np.array(rot) 169 | 170 | js_pos = self.joint_state.position[16:-12] 171 | js_vel = self.joint_state.velocity[16:-12] 172 | 173 | W = list(trans) + list(r_matrix.flatten()) + list(trans_vel) + list(rot_vel) 174 | 175 | # Gripper angle is r_gripper_joint 176 | W.append(r_gripper_angle) 177 | 178 | W = W + list(js_pos) + list(js_vel) 179 | 180 | self.data = utils.safe_concatenate(self.data, utils.reshape(np.array(W))) 181 | 182 | frm += 1 183 | 184 | if ((self.r_l == prev_r_l) and (self.r_r == prev_r_r)): 185 | print "Not recording anymore?" 186 | wait_thresh += 1 187 | if wait_thresh > 5: 188 | self.save_and_quit() 189 | 190 | prev_r_l = self.r_l 191 | prev_r_r = self.r_r 192 | 193 | end = time.time() 194 | 195 | print end - start 196 | 197 | if __name__ == "__main__": 198 | rospy.init_node("recorder_node") 199 | if len(sys.argv) < 2: 200 | print "ERROR: Please provide correct arguments" 201 | sys.exit() 202 | recording = Recording(sys.argv[1]) 203 | recording.start_recording() -------------------------------------------------------------------------------- /scripts/fine_tuning/train_val.prototxt: -------------------------------------------------------------------------------- 1 | name: "SuturingCaffeNet" 2 | layer { 3 | name: "data" 4 | type: "ImageData" 5 | top: "data" 6 | top: "label" 7 | include { 8 | phase: TRAIN 9 | } 10 | transform_param { 11 | mirror: true 12 | crop_size: 227 13 | mean_file: "data/ilsvrc12/imagenet_mean.binaryproto" 14 | } 15 | image_data_param { 16 | source: "/home/animesh/DeepMilestones/scripts/fine_tuning/train.txt" 17 | batch_size: 50 18 | new_height: 256 19 | new_width: 256 20 | } 21 | } 22 | layer { 23 | name: "data" 24 | type: "ImageData" 25 | top: "data" 26 | top: "label" 27 | include { 28 | phase: TEST 29 | } 30 | transform_param { 31 | mirror: false 32 | crop_size: 227 33 | mean_file: "data/ilsvrc12/imagenet_mean.binaryproto" 34 | } 35 | image_data_param { 36 | source: "/home/animesh/DeepMilestones/scripts/fine_tuning/val.txt" 37 | batch_size: 50 38 | new_height: 256 39 | new_width: 256 40 | } 41 | } 42 | layer { 43 | name: "conv1" 44 | type: "Convolution" 45 | bottom: "data" 46 | top: "conv1" 47 | param { 48 | lr_mult: 1 49 | decay_mult: 1 50 | } 51 | param { 52 | lr_mult: 2 53 | decay_mult: 0 54 | } 55 | convolution_param { 56 | num_output: 96 57 | kernel_size: 11 58 | stride: 4 59 | weight_filler { 60 | type: "gaussian" 61 | std: 0.01 62 | } 63 | bias_filler { 64 | type: "constant" 65 | value: 0 66 | } 67 | } 68 | } 69 | layer { 70 | name: "relu1" 71 | type: "ReLU" 72 | bottom: "conv1" 73 | top: "conv1" 74 | } 75 | layer { 76 | name: "pool1" 77 | type: "Pooling" 78 | bottom: "conv1" 79 | top: "pool1" 80 | pooling_param { 81 | pool: MAX 82 | kernel_size: 3 83 | stride: 2 84 | } 85 | } 86 | layer { 87 | name: "norm1" 88 | type: "LRN" 89 | bottom: "pool1" 90 | top: "norm1" 91 | lrn_param { 92 | local_size: 5 93 | alpha: 0.0001 94 | beta: 0.75 95 | } 96 | } 97 | layer { 98 | name: "conv2" 99 | type: "Convolution" 100 | bottom: "norm1" 101 | top: "conv2" 102 | param { 103 | lr_mult: 1 104 | decay_mult: 1 105 | } 106 | param { 107 | lr_mult: 2 108 | decay_mult: 0 109 | } 110 | convolution_param { 111 | num_output: 256 112 | pad: 2 113 | kernel_size: 5 114 | group: 2 115 | weight_filler { 116 | type: "gaussian" 117 | std: 0.01 118 | } 119 | bias_filler { 120 | type: "constant" 121 | value: 1 122 | } 123 | } 124 | } 125 | layer { 126 | name: "relu2" 127 | type: "ReLU" 128 | bottom: "conv2" 129 | top: "conv2" 130 | } 131 | layer { 132 | name: "pool2" 133 | type: "Pooling" 134 | bottom: "conv2" 135 | top: "pool2" 136 | pooling_param { 137 | pool: MAX 138 | kernel_size: 3 139 | stride: 2 140 | } 141 | } 142 | layer { 143 | name: "norm2" 144 | type: "LRN" 145 | bottom: "pool2" 146 | top: "norm2" 147 | lrn_param { 148 | local_size: 5 149 | alpha: 0.0001 150 | beta: 0.75 151 | } 152 | } 153 | layer { 154 | name: "conv3" 155 | type: "Convolution" 156 | bottom: "norm2" 157 | top: "conv3" 158 | param { 159 | lr_mult: 1 160 | decay_mult: 1 161 | } 162 | param { 163 | lr_mult: 2 164 | decay_mult: 0 165 | } 166 | convolution_param { 167 | num_output: 384 168 | pad: 1 169 | kernel_size: 3 170 | weight_filler { 171 | type: "gaussian" 172 | std: 0.01 173 | } 174 | bias_filler { 175 | type: "constant" 176 | value: 0 177 | } 178 | } 179 | } 180 | layer { 181 | name: "relu3" 182 | type: "ReLU" 183 | bottom: "conv3" 184 | top: "conv3" 185 | } 186 | layer { 187 | name: "conv4" 188 | type: "Convolution" 189 | bottom: "conv3" 190 | top: "conv4" 191 | param { 192 | lr_mult: 1 193 | decay_mult: 1 194 | } 195 | param { 196 | lr_mult: 2 197 | decay_mult: 0 198 | } 199 | convolution_param { 200 | num_output: 384 201 | pad: 1 202 | kernel_size: 3 203 | group: 2 204 | weight_filler { 205 | type: "gaussian" 206 | std: 0.01 207 | } 208 | bias_filler { 209 | type: "constant" 210 | value: 1 211 | } 212 | } 213 | } 214 | layer { 215 | name: "relu4" 216 | type: "ReLU" 217 | bottom: "conv4" 218 | top: "conv4" 219 | } 220 | layer { 221 | name: "conv5" 222 | type: "Convolution" 223 | bottom: "conv4" 224 | top: "conv5" 225 | param { 226 | lr_mult: 1 227 | decay_mult: 1 228 | } 229 | param { 230 | lr_mult: 2 231 | decay_mult: 0 232 | } 233 | convolution_param { 234 | num_output: 256 235 | pad: 1 236 | kernel_size: 3 237 | group: 2 238 | weight_filler { 239 | type: "gaussian" 240 | std: 0.01 241 | } 242 | bias_filler { 243 | type: "constant" 244 | value: 1 245 | } 246 | } 247 | } 248 | layer { 249 | name: "relu5" 250 | type: "ReLU" 251 | bottom: "conv5" 252 | top: "conv5" 253 | } 254 | layer { 255 | name: "pool5" 256 | type: "Pooling" 257 | bottom: "conv5" 258 | top: "pool5" 259 | pooling_param { 260 | pool: MAX 261 | kernel_size: 3 262 | stride: 2 263 | } 264 | } 265 | layer { 266 | name: "fc6" 267 | type: "InnerProduct" 268 | bottom: "pool5" 269 | top: "fc6" 270 | param { 271 | lr_mult: 1 272 | decay_mult: 1 273 | } 274 | param { 275 | lr_mult: 2 276 | decay_mult: 0 277 | } 278 | inner_product_param { 279 | num_output: 4096 280 | weight_filler { 281 | type: "gaussian" 282 | std: 0.005 283 | } 284 | bias_filler { 285 | type: "constant" 286 | value: 1 287 | } 288 | } 289 | } 290 | layer { 291 | name: "relu6" 292 | type: "ReLU" 293 | bottom: "fc6" 294 | top: "fc6" 295 | } 296 | layer { 297 | name: "drop6" 298 | type: "Dropout" 299 | bottom: "fc6" 300 | top: "fc6" 301 | dropout_param { 302 | dropout_ratio: 0.5 303 | } 304 | } 305 | layer { 306 | name: "fc7" 307 | type: "InnerProduct" 308 | bottom: "fc6" 309 | top: "fc7" 310 | # Note that lr_mult can be set to 0 to disable any fine-tuning of this, and any other, layer 311 | param { 312 | lr_mult: 1 313 | decay_mult: 1 314 | } 315 | param { 316 | lr_mult: 2 317 | decay_mult: 0 318 | } 319 | inner_product_param { 320 | num_output: 4096 321 | weight_filler { 322 | type: "gaussian" 323 | std: 0.005 324 | } 325 | bias_filler { 326 | type: "constant" 327 | value: 1 328 | } 329 | } 330 | } 331 | layer { 332 | name: "relu7" 333 | type: "ReLU" 334 | bottom: "fc7" 335 | top: "fc7" 336 | } 337 | layer { 338 | name: "drop7" 339 | type: "Dropout" 340 | bottom: "fc7" 341 | top: "fc7" 342 | dropout_param { 343 | dropout_ratio: 0.5 344 | } 345 | } 346 | layer { 347 | name: "fc8_suturing" 348 | type: "InnerProduct" 349 | bottom: "fc7" 350 | top: "fc8_suturing" 351 | # lr_mult is set to higher than for other layers, because this layer is starting from random while the others are already trained 352 | param { 353 | lr_mult: 10 354 | decay_mult: 1 355 | } 356 | param { 357 | lr_mult: 20 358 | decay_mult: 0 359 | } 360 | inner_product_param { 361 | num_output: 11 362 | weight_filler { 363 | type: "gaussian" 364 | std: 0.01 365 | } 366 | bias_filler { 367 | type: "constant" 368 | value: 0 369 | } 370 | } 371 | } 372 | layer { 373 | name: "loss" 374 | type: "SoftmaxWithLoss" 375 | bottom: "fc8_suturing" 376 | bottom: "label" 377 | top: "loss" 378 | } 379 | layer { 380 | name: "accuracy" 381 | type: "Accuracy" 382 | bottom: "fc8_suturing" 383 | bottom: "label" 384 | top: "accuracy" 385 | include { 386 | phase: TEST 387 | } 388 | } 389 | -------------------------------------------------------------------------------- /scripts/examples/plot_lle_digits.py: -------------------------------------------------------------------------------- 1 | """ 2 | ============================================================================= 3 | Manifold learning on handwritten digits: Locally Linear Embedding, Isomap... 4 | ============================================================================= 5 | 6 | An illustration of various embeddings on the digits dataset. 7 | 8 | The RandomTreesEmbedding, from the :mod:`sklearn.ensemble` module, is not 9 | technically a manifold embedding method, as it learn a high-dimensional 10 | representation on which we apply a dimensionality reduction method. 11 | However, it is often useful to cast a dataset into a representation in 12 | which the classes are linearly-separable. 13 | 14 | t-SNE will be initialized with the embedding that is generated by PCA in 15 | this example, which is not the default setting. It ensures global stability 16 | of the embedding, i.e., the embedding does not depend on random 17 | initialization. 18 | """ 19 | 20 | # Authors: Fabian Pedregosa 21 | # Olivier Grisel 22 | # Mathieu Blondel 23 | # Gael Varoquaux 24 | # License: BSD 3 clause (C) INRIA 2011 25 | 26 | print(__doc__) 27 | from time import time 28 | 29 | import IPython 30 | import numpy as np 31 | import matplotlib.pyplot as plt 32 | from matplotlib import offsetbox 33 | from sklearn import (manifold, datasets, decomposition, ensemble, lda, 34 | random_projection) 35 | 36 | digits = datasets.load_digits(n_class=6) 37 | X = digits.data 38 | y = digits.target 39 | n_samples, n_features = X.shape 40 | n_neighbors = 30 41 | IPython.embed() 42 | 43 | #---------------------------------------------------------------------- 44 | # Scale and visualize the embedding vectors 45 | def plot_embedding(X, title=None): 46 | x_min, x_max = np.min(X, 0), np.max(X, 0) 47 | X = (X - x_min) / (x_max - x_min) 48 | 49 | plt.figure() 50 | ax = plt.subplot(111) 51 | for i in range(X.shape[0]): 52 | plt.text(X[i, 0], X[i, 1], str(digits.target[i]), 53 | color=plt.cm.Set1(y[i] / 10.), 54 | fontdict={'weight': 'bold', 'size': 9}) 55 | 56 | if hasattr(offsetbox, 'AnnotationBbox'): 57 | # only print thumbnails with matplotlib > 1.0 58 | shown_images = np.array([[1., 1.]]) # just something big 59 | for i in range(digits.data.shape[0]): 60 | dist = np.sum((X[i] - shown_images) ** 2, 1) 61 | if np.min(dist) < 4e-3: 62 | # don't show points that are too close 63 | continue 64 | shown_images = np.r_[shown_images, [X[i]]] 65 | imagebox = offsetbox.AnnotationBbox( 66 | offsetbox.OffsetImage(digits.images[i], cmap=plt.cm.gray_r), 67 | X[i]) 68 | ax.add_artist(imagebox) 69 | plt.xticks([]), plt.yticks([]) 70 | if title is not None: 71 | plt.title(title) 72 | 73 | 74 | #---------------------------------------------------------------------- 75 | # Plot images of the digits 76 | n_img_per_row = 20 77 | img = np.zeros((10 * n_img_per_row, 10 * n_img_per_row)) 78 | for i in range(n_img_per_row): 79 | ix = 10 * i + 1 80 | for j in range(n_img_per_row): 81 | iy = 10 * j + 1 82 | img[ix:ix + 8, iy:iy + 8] = X[i * n_img_per_row + j].reshape((8, 8)) 83 | 84 | plt.imshow(img, cmap=plt.cm.binary) 85 | plt.xticks([]) 86 | plt.yticks([]) 87 | plt.title('A selection from the 64-dimensional digits dataset') 88 | 89 | 90 | #---------------------------------------------------------------------- 91 | # Random 2D projection using a random unitary matrix 92 | print("Computing random projection") 93 | rp = random_projection.SparseRandomProjection(n_components=2, random_state=42) 94 | X_projected = rp.fit_transform(X) 95 | plot_embedding(X_projected, "Random Projection of the digits") 96 | 97 | 98 | #---------------------------------------------------------------------- 99 | # Projection on to the first 2 principal components 100 | 101 | print("Computing PCA projection") 102 | t0 = time() 103 | X_pca = decomposition.TruncatedSVD(n_components=2).fit_transform(X) 104 | plot_embedding(X_pca, 105 | "Principal Components projection of the digits (time %.2fs)" % 106 | (time() - t0)) 107 | 108 | #---------------------------------------------------------------------- 109 | # Projection on to the first 2 linear discriminant components 110 | 111 | print("Computing LDA projection") 112 | X2 = X.copy() 113 | X2.flat[::X.shape[1] + 1] += 0.01 # Make X invertible 114 | t0 = time() 115 | X_lda = lda.LDA(n_components=2).fit_transform(X2, y) 116 | plot_embedding(X_lda, 117 | "Linear Discriminant projection of the digits (time %.2fs)" % 118 | (time() - t0)) 119 | 120 | 121 | #---------------------------------------------------------------------- 122 | # Isomap projection of the digits dataset 123 | print("Computing Isomap embedding") 124 | t0 = time() 125 | X_iso = manifold.Isomap(n_neighbors, n_components=2).fit_transform(X) 126 | print("Done.") 127 | plot_embedding(X_iso, 128 | "Isomap projection of the digits (time %.2fs)" % 129 | (time() - t0)) 130 | 131 | 132 | #---------------------------------------------------------------------- 133 | # Locally linear embedding of the digits dataset 134 | print("Computing LLE embedding") 135 | clf = manifold.LocallyLinearEmbedding(n_neighbors, n_components=2, 136 | method='standard') 137 | t0 = time() 138 | X_lle = clf.fit_transform(X) 139 | print("Done. Reconstruction error: %g" % clf.reconstruction_error_) 140 | plot_embedding(X_lle, 141 | "Locally Linear Embedding of the digits (time %.2fs)" % 142 | (time() - t0)) 143 | 144 | 145 | #---------------------------------------------------------------------- 146 | # Modified Locally linear embedding of the digits dataset 147 | print("Computing modified LLE embedding") 148 | clf = manifold.LocallyLinearEmbedding(n_neighbors, n_components=2, 149 | method='modified') 150 | t0 = time() 151 | X_mlle = clf.fit_transform(X) 152 | print("Done. Reconstruction error: %g" % clf.reconstruction_error_) 153 | plot_embedding(X_mlle, 154 | "Modified Locally Linear Embedding of the digits (time %.2fs)" % 155 | (time() - t0)) 156 | 157 | 158 | #---------------------------------------------------------------------- 159 | # HLLE embedding of the digits dataset 160 | print("Computing Hessian LLE embedding") 161 | clf = manifold.LocallyLinearEmbedding(n_neighbors, n_components=2, 162 | method='hessian') 163 | t0 = time() 164 | X_hlle = clf.fit_transform(X) 165 | print("Done. Reconstruction error: %g" % clf.reconstruction_error_) 166 | plot_embedding(X_hlle, 167 | "Hessian Locally Linear Embedding of the digits (time %.2fs)" % 168 | (time() - t0)) 169 | 170 | 171 | #---------------------------------------------------------------------- 172 | # LTSA embedding of the digits dataset 173 | print("Computing LTSA embedding") 174 | clf = manifold.LocallyLinearEmbedding(n_neighbors, n_components=2, 175 | method='ltsa') 176 | t0 = time() 177 | X_ltsa = clf.fit_transform(X) 178 | print("Done. Reconstruction error: %g" % clf.reconstruction_error_) 179 | plot_embedding(X_ltsa, 180 | "Local Tangent Space Alignment of the digits (time %.2fs)" % 181 | (time() - t0)) 182 | 183 | #---------------------------------------------------------------------- 184 | # MDS embedding of the digits dataset 185 | print("Computing MDS embedding") 186 | clf = manifold.MDS(n_components=2, n_init=1, max_iter=100) 187 | t0 = time() 188 | X_mds = clf.fit_transform(X) 189 | print("Done. Stress: %f" % clf.stress_) 190 | plot_embedding(X_mds, 191 | "MDS embedding of the digits (time %.2fs)" % 192 | (time() - t0)) 193 | 194 | #---------------------------------------------------------------------- 195 | # Random Trees embedding of the digits dataset 196 | print("Computing Totally Random Trees embedding") 197 | hasher = ensemble.RandomTreesEmbedding(n_estimators=200, random_state=0, 198 | max_depth=5) 199 | t0 = time() 200 | X_transformed = hasher.fit_transform(X) 201 | pca = decomposition.TruncatedSVD(n_components=2) 202 | X_reduced = pca.fit_transform(X_transformed) 203 | 204 | plot_embedding(X_reduced, 205 | "Random forest embedding of the digits (time %.2fs)" % 206 | (time() - t0)) 207 | 208 | #---------------------------------------------------------------------- 209 | # Spectral embedding of the digits dataset 210 | print("Computing Spectral embedding") 211 | embedder = manifold.SpectralEmbedding(n_components=2, random_state=0, 212 | eigen_solver="arpack") 213 | t0 = time() 214 | X_se = embedder.fit_transform(X) 215 | 216 | plot_embedding(X_se, 217 | "Spectral embedding of the digits (time %.2fs)" % 218 | (time() - t0)) 219 | 220 | #---------------------------------------------------------------------- 221 | # t-SNE embedding of the digits dataset 222 | print("Computing t-SNE embedding") 223 | tsne = manifold.TSNE(n_components=2, init='pca', random_state=0) 224 | t0 = time() 225 | X_tsne = tsne.fit_transform(X) 226 | 227 | plot_embedding(X_tsne, 228 | "t-SNE embedding of the digits (time %.2fs)" % 229 | (time() - t0)) 230 | 231 | plt.show() 232 | -------------------------------------------------------------------------------- /scripts/examples/preWhiten.py: -------------------------------------------------------------------------------- 1 | import matlab.engine as mateng 2 | from time import time 3 | import IPython 4 | import numpy as np 5 | import matplotlib.pyplot as plt 6 | from matplotlib import offsetbox 7 | from sklearn import (manifold, datasets, decomposition, ensemble, lda, random_projection, preprocessing, covariance) 8 | from scipy import linalg 9 | import argparse 10 | import pickle 11 | 12 | # PATH_TO_DATA = 'data/8fps_nooverlap_conv5b_cropped_annotations/' 13 | PATH_TO_DATA = 'data/brownie/' 14 | 15 | all_segments = {'1': [(176, 248), (496, 566), (739, 780), (977, 1075), (1176,1236), (1352,1405)], '2': [(585, 640), (796, 830), (1267, 1284)], 16 | '3': [(695, 720), (857, 889), (1126, 1156), (1315, 1333), (1414, 1450)]} 17 | 18 | color_map = {1:'b', 2:'g', 3:'r', 4:'c', 5: 'm', 6:'y', 7:'k', 8:'#000000', 9: '#9932CC'} 19 | index_map = {} 20 | label_map = {} 21 | 22 | def parse(): 23 | eng = mateng.start_matlab() 24 | 25 | [s, X] = eng.read_binary_blob(PATH_TO_DATA + '1.conv5b', nargout = 2) 26 | i = 17; 27 | while i <= 1489: 28 | [s, data] = eng.read_binary_blob(PATH_TO_DATA + str(i) + '.conv5b', nargout = 2) 29 | data = np.array(data) 30 | X = np.concatenate((X, data), axis = 0) 31 | i += 16 32 | return X 33 | 34 | def parse_annotations(): 35 | eng = mateng.start_matlab() 36 | 37 | [s, X] = eng.read_binary_blob(PATH_TO_DATA + '176.conv5b', nargout = 2) 38 | color = 'b' 39 | i = 0 40 | for label in all_segments: 41 | print "Parsing label " + label 42 | segments = all_segments[label] 43 | if label == '2': 44 | color = 'r' 45 | elif label == '3': 46 | color = 'g' 47 | else: 48 | color = 'b' 49 | for seg in segments: 50 | j = seg[0] 51 | while j <= seg[1]: 52 | if j == 176: 53 | color_map[i] = color 54 | index_map[i] = j 55 | j += 16 56 | i += 1 57 | continue 58 | [s, data] = eng.read_binary_blob(PATH_TO_DATA + str(j) + '.conv5b', nargout = 2) 59 | data = np.array(data) 60 | X = np.concatenate((X, data), axis = 0) 61 | color_map[i] = color 62 | index_map[i] = j 63 | j += 16 64 | i += 1 65 | return X 66 | 67 | def parse_annotations_pickle(annotations): 68 | eng = mateng.start_matlab() 69 | map_index_data = pickle.load(open(annotations, "rb")) 70 | X = None 71 | i = 0 72 | for index in map_index_data: 73 | print "Parsing label " + str(index) 74 | segments = map_index_data[index] 75 | color = np.random.rand(3,1) 76 | for seg in segments: 77 | j = seg[0] 78 | while j <= seg[1]: 79 | if X is None: 80 | [s, X] = eng.read_binary_blob(PATH_TO_DATA + str(j) + '.conv5b', nargout = 2) 81 | else: 82 | [s, data] = eng.read_binary_blob(PATH_TO_DATA + str(j) + '.conv5b', nargout = 2) 83 | data = np.array(data) 84 | X = np.concatenate((X, data), axis = 0) 85 | index_map[i] = j 86 | label_map[i] = index 87 | j += 16 88 | i += 1 89 | return X 90 | 91 | #---------------------------------------------------------------------- 92 | # Scale and visualize the embedding vectors 93 | def plot_embedding(X, figure_name, title=None): 94 | x_min, x_max = np.min(X, 0), np.max(X, 0) 95 | X = (X - x_min) / (x_max - x_min) 96 | plt.figure() 97 | for i in range(X.shape[0]): 98 | plt.text(X[i, 0], X[i, 1], str(i*16 + 1), color=plt.cm.Set1((i*16 + 1)/ 100), fontdict={'weight': 'bold', 'size': 9}) 99 | plt.xticks([]), plt.yticks([]) 100 | if title is not None: 101 | plt.title(title) 102 | plt.savefig('plots/'+figure_name) 103 | 104 | # Scale and visualize the embedding vectors 105 | def plot_annotated_embedding(X, figure_name, title=None): 106 | x_min, x_max = np.min(X, 0), np.max(X, 0) 107 | X = (X - x_min) / (x_max - x_min) 108 | plt.figure() 109 | r_x = r_y = [] 110 | g_x = g_y = [] 111 | b_x = b_y = [] 112 | for i in range(X.shape[0]): 113 | frm_num = index_map[i] 114 | color = color_map[i] 115 | if color == 'r': 116 | plt.text(X[i, 0], X[i, 1], 'o', color=color, fontdict={'weight': 'bold', 'size': 9}) 117 | elif color == 'g': 118 | plt.text(X[i, 0], X[i, 1], 'x', color=color, fontdict={'weight': 'bold', 'size': 9}) 119 | else: 120 | plt.text(X[i, 0], X[i, 1], '*', color=color, fontdict={'weight': 'bold', 'size': 13}) 121 | # if color == 'r': 122 | # r_x.append(X[i, 0]) 123 | # r_y.append(X[i, 1]) 124 | # elif color == 'g': 125 | # g_x.append(X[i, 0]) 126 | # g_y.append(X[i, 1]) 127 | # else: 128 | # b_x.append(X[i, 0]) 129 | # b_y.append(X[i, 1]) 130 | # plt.plot(r_x, r_y, 'rs') 131 | # plt.plot(g_x, g_y, 'go') 132 | # plt.plot(b_x, b_y, 'b^') 133 | plt.xticks([]), plt.yticks([]) 134 | if title is not None: 135 | plt.title(title) 136 | plt.savefig('plots/'+figure_name) 137 | 138 | # Scale and visualize the embedding vectors 139 | def plot_annotated_embedding_2(X, figure_name, title=None): 140 | x_min, x_max = np.min(X, 0), np.max(X, 0) 141 | X = (X - x_min) / (x_max - x_min) 142 | plt.figure() 143 | r_x = r_y = [] 144 | g_x = g_y = [] 145 | b_x = b_y = [] 146 | for i in range(X.shape[0]): 147 | frm_num = index_map[i] 148 | plt.text(X[i, 0], X[i, 1], 'x', color=color_map[label_map[i]], fontdict={'weight': 'bold', 'size': 5}) 149 | plt.xticks([]), plt.yticks([]) 150 | if title is not None: 151 | plt.title(title) 152 | plt.savefig('plots/'+figure_name) 153 | 154 | def pca(X): 155 | print("Computing PCA embedding") 156 | scaler = preprocessing.StandardScaler(with_std = False).fit(X) 157 | X_centered = scaler.transform(X) 158 | X_pca = decomposition.TruncatedSVD(n_components=2).fit_transform(X_centered) 159 | return X_pca 160 | 161 | def tsne_pca(X): 162 | print("Computing PCA -> t-SNE embedding") 163 | scaler = preprocessing.StandardScaler().fit(X) 164 | X_centered = scaler.transform(X) 165 | X_pca = decomposition.TruncatedSVD(n_components=40).fit_transform(X_centered) 166 | tsne = manifold.TSNE(init = 'pca') 167 | X_tsne = tsne.fit_transform(X_pca) 168 | return X_tsne 169 | 170 | def tsne(X): 171 | #---------------------------------------------------------------------- 172 | # t-SNE embedding of the digits dataset 173 | print("Computing t-SNE embedding") 174 | tsne = manifold.TSNE(n_components=2, init='pca', random_state=0) 175 | X_tsne = tsne.fit_transform(X) 176 | return X_tsne 177 | 178 | def plot_all(X): 179 | tsne = manifold.TSNE(n_components=2, init='pca', random_state=0) 180 | #---------------------------------------------------------------------- 181 | # Pre-processing 182 | print "t-SNE Scaling" 183 | X_scaled = preprocessing.scale(X) #zero mean, unit variance 184 | X_tsne_scaled = tsne.fit_transform(X_scaled) 185 | 186 | #normalize the data (scaling individual samples to have unit norm) 187 | print "t-SNE L2 Norm" 188 | X_normalized = preprocessing.normalize(X, norm='l2') 189 | X_tsne_norm = tsne.fit_transform(X_normalized) 190 | 191 | 192 | #whiten the data 193 | print "t-SNE Whitening" 194 | # the mean computed by the scaler is for the feature dimension. 195 | # We want the normalization to be in feature dimention. 196 | # Zero mean for each sample assumes stationarity which is not necessarily true for CNN features. 197 | # X: NxD where N is number of examples and D is number of features. 198 | 199 | # scaler = preprocessing.StandardScaler(with_std=False).fit(X) 200 | scaler = preprocessing.StandardScaler().fit(X) #this scales each feature to have std-dev 1 201 | X_centered = scaler.transform(X) 202 | 203 | # U, s, Vh = linalg.svd(X_centered) 204 | shapeX = X_centered.shape 205 | IPython.embed() 206 | # this is DxD matrix where D is the feature dimension 207 | # still to figure out: It seems computation is not a problem but carrying around a 50kx50k matrix is memory killer! 208 | sig = (1/shapeX[0]) * np.dot(X_centered.T, X_centered) 209 | sig2= covariance.empirical_covariance(X_centered, assume_centered=True) #estimated -- this is better. 210 | sig3, shrinkage= covariance.oas(X_centered, assume_centered=True) #estimated 211 | 212 | U, s, Vh = linalg.svd(sig, full_matrices=False) 213 | eps = 1e-2 # this affects how many low- freq eigevalues are eliminated 214 | invS = np.diag (np.reciprocal(np.sqrt(s+eps))) 215 | 216 | #PCA_whiten 217 | X_pca = np.dot(invS, np.dot(U.T, X_centered)) 218 | X_tsne_pca = tsne.fit_transform(X_pca) 219 | 220 | #whiten the data (ZCA) 221 | X_zca = np.dot(U, X_pca) 222 | X_tsne_zca = tsne.fit_transform(X_zca) 223 | 224 | return X_tsne_scaled, X_tsne_norm, X_tsne_pca, X_tsne_zca 225 | 226 | if __name__ == "__main__": 227 | parser = argparse.ArgumentParser() 228 | parser.add_argument("--a", help = "Annotated frames") 229 | args = parser.parse_args() 230 | if args.a: 231 | X = parse_annotations_pickle(args.a) 232 | else: 233 | X = parse_annotations() 234 | 235 | # parser.add_argument("plot", help = "Choose from pca, tsne, etc.") 236 | # parser.add_argument("figure_name", help = "Figure name to be saved") 237 | # plotter_func = locals()[args.plot] 238 | # plotter_func(args.figure_name, X) 239 | 240 | X_pca = pca(X) 241 | X_tsne = tsne(X) 242 | plot_annotated_embedding_2(X = X_pca, figure_name = 'brownie_pca_stdFalse.png', title = 'PCA conv5b') 243 | plot_annotated_embedding_2(X = X_tsne, figure_name = 'brownie_tsne.png', title = 't-SNE conv5b') 244 | 245 | # X_tsne_scaled, X_tsne_norm, X_tsne_pca, X_tsne_zca = plot_all(X) 246 | # plotNames = ["X_tsne_scaled", "X_tsne_norm", "X_tsne_pca", "X_tsne_zca"] 247 | 248 | # for plotName in plotNames: 249 | # plot_annotated_embedding_2 (X=eval(plotName), figure_name='brownie_'+plotName, title=plotName+'_conv5b') 250 | -- INSERT -- 249,1 Bot 251 | 252 | -------------------------------------------------------------------------------- /scripts/broken_barh.py: -------------------------------------------------------------------------------- 1 | """ 2 | Make a "broken" horizontal bar plot, i.e., one with gaps 3 | """ 4 | import matplotlib.pyplot as plt 5 | import numpy as np 6 | import sys 7 | import os 8 | import IPython 9 | import pickle 10 | 11 | import utils 12 | import constants 13 | import parser 14 | 15 | from sklearn import mixture 16 | from dtw import dtw 17 | 18 | def get_ticks(manual_labels): 19 | """ 20 | Returns x-coordinates of tick marks from manual label transitions. 21 | """ 22 | ticks = [] 23 | for elem in manual_labels[:-1]: 24 | ticks.append(elem[0] + elem[1]) 25 | return ticks 26 | 27 | def setup_manual_labels(segments): 28 | """ 29 | Takes in segment start,end in DeepMilestones annotations format 30 | and converts it to format for Mathplotlib broken_barh. 31 | """ 32 | list_of_start_end = [] 33 | list_of_colors = [] 34 | 35 | for key in segments.keys(): 36 | color = constants.color_map[key] 37 | 38 | for elem in segments[key]: 39 | list_of_start_end.append((elem[0], elem[1] - elem[0])) 40 | list_of_colors.append(color) 41 | 42 | return list_of_start_end, tuple(list_of_colors) 43 | 44 | def compute_dtw(time_sequence_1, time_sequence_2): 45 | dist, cost, path = dtw(sorted(time_sequence_1), sorted(time_sequence_2), 46 | dist = lambda x, y: np.linalg.norm(x - y, ord=2)) 47 | return dist 48 | 49 | 50 | def setup_automatic_labels_2(list_of_frms, color): 51 | list_of_start_end = [] 52 | list_of_colors = [] 53 | for elem in list_of_frms: 54 | list_of_start_end.append((elem[0], elem[1] - elem[0])) 55 | list_of_colors.append(color) 56 | return list_of_start_end, tuple(list_of_colors) 57 | 58 | def setup_automatic_labels(list_of_frms, color): 59 | """ 60 | Takes in changepoints and converts it to format 61 | for Mathplotlib broken_barh. 62 | """ 63 | list_of_start_end = [] 64 | list_of_colors = [] 65 | 66 | if constants.SIMULATION: 67 | bar_size = 0.5 68 | elif constants.TASK_NAME == "plane" or constants.TASK_NAME == "lego": 69 | bar_size = 2 70 | else: 71 | bar_size = 10 72 | 73 | 74 | for elem in list_of_frms: 75 | list_of_start_end.append((elem - (bar_size/2), bar_size)) 76 | list_of_colors.append(color) 77 | return list_of_start_end, tuple(list_of_colors) 78 | 79 | 80 | def get_time_clusters(data, T_COMPONENTS): 81 | # k-fold validation (Leave one out) 82 | numDemos = len(data.keys()) + 1 83 | sizeTestSet = numDemos - 1 84 | 85 | list_of_frms = [] 86 | all_frms = [] 87 | for key in data.keys(): 88 | elem = data[key] 89 | list_of_frms.append(elem) 90 | all_frms += elem 91 | 92 | N_COMPONENTS = min(T_COMPONENTS, len(all_frms)) 93 | time_cluster = mixture.GMM(n_components = N_COMPONENTS, covariance_type='full', n_iter = 50000, thresh = 5e-7) 94 | 95 | X = np.array(all_frms) 96 | X = X.reshape(len(all_frms), 1) 97 | time_cluster.fit(X) 98 | Y = time_cluster.predict(X) 99 | 100 | means = time_cluster.means_ 101 | covars = time_cluster.covars_ 102 | 103 | #dpgmm = mixture.DPGMM(n_components = numDemos * 3, covariance_type='diag', n_iter = 10000, alpha = 1000, thresh= 1e-10) 104 | #dpgmm.fit(X) 105 | #Y = dpgmm.predict(X) 106 | #means = dpgmm.means_ 107 | 108 | list_of_elem = [] 109 | 110 | list_of_means = [] 111 | 112 | for i in range(len(Y)): 113 | list_of_elem.append((Y[i], X[i], means[Y[i]][0], np.sqrt(covars[Y[i]][0][0]))) 114 | 115 | list_of_elem = sorted(list_of_elem, key = lambda x:x[1][0] ) 116 | 117 | dict_time_clusters = {} 118 | for elem in list_of_elem: 119 | utils.dict_insert_list(elem[0], elem[1], dict_time_clusters) 120 | 121 | list_time_clusters = [] 122 | list_time_clusters_noPrune = [] 123 | for cluster in dict_time_clusters.keys(): 124 | # get all frames in this cluster 125 | cluster_frames = dict_time_clusters[cluster] 126 | setClusterFrames = set([elem[0] for elem in cluster_frames]) 127 | # test if frames in cluster are representative of the test set 128 | rep = [] 129 | for id in range(sizeTestSet): 130 | elemSet = set(list_of_frms[id]) 131 | commonElem = elemSet.intersection(setClusterFrames) 132 | id_in_cluster = 1. if len(commonElem) > 0 else 0. 133 | rep.append(id_in_cluster) 134 | 135 | pruneCluster = True if sum(rep)/sizeTestSet < constants.PRUNING_FACTOR_T else False 136 | 137 | min_frm = min(cluster_frames) 138 | max_frm = max(cluster_frames) 139 | 140 | mean = means[cluster][0] 141 | list_of_means.append(int(mean)) 142 | std = np.sqrt(covars[cluster][0][0]) 143 | 144 | leftFrame = max(min_frm[0], mean - std) 145 | rightFrame = min(max_frm[0], mean + std) 146 | 147 | list_time_clusters_noPrune.append((leftFrame, rightFrame)) 148 | # keep for plotting is pruneFlag = 0 149 | if not(pruneCluster): 150 | # list_time_clusters.append((min_frm[0], max_frm[0])) 151 | list_time_clusters.append((leftFrame, rightFrame)) 152 | 153 | print "Number of Clusters pruned in Time Clustering: ", len(list_time_clusters_noPrune) - len(list_time_clusters) 154 | 155 | labels_automatic, colors_automatic = setup_automatic_labels_2(list_time_clusters, "k") 156 | labels_automatic_0, colors_automatic_0 = setup_automatic_labels(list_of_frms[0], "k") 157 | return labels_automatic, colors_automatic, labels_automatic_0, colors_automatic_0, list_of_means, list_of_frms 158 | 159 | def plot_broken_barh_all(demonstration, data_W, data_Z, data_ZW, save_fname = None, save_fname2 = None): 160 | """ 161 | Plots time-clusters for W, K, KW. 162 | """ 163 | 164 | PATH_TO_ANNOTATION = constants.PATH_TO_DATA + constants.ANNOTATIONS_FOLDER + demonstration + "_" + constants.CAMERA + ".p" 165 | start, end = utils.get_start_end_annotations(constants.PATH_TO_DATA + constants.ANNOTATIONS_FOLDER + demonstration + "_" + constants.CAMERA + ".p") 166 | length = end - start 167 | segments = pickle.load(open(PATH_TO_ANNOTATION, "rb")) 168 | 169 | TASK = constants.TASK_NAME 170 | if (TASK in ["lego", "plane"]): 171 | end = end + 20 172 | elif (TASK in ["000", "010", "011", "100"]): 173 | end = end + 2 174 | else: 175 | end = end + 50 176 | 177 | labels_manual, colors_manual = setup_manual_labels(segments) 178 | labels_automatic_W, colors_automatic_W, labels_automatic_W_0, colors_automatic_W_0, means_W, list_of_frms_W = get_time_clusters(data_W, constants.N_COMPONENTS_TIME_W) 179 | labels_automatic_Z, colors_automatic_Z, labels_automatic_Z_0, colors_automatic_Z_0, means_Z, list_of_frms_Z = get_time_clusters(data_Z, constants.N_COMPONENTS_TIME_Z) 180 | labels_automatic_ZW, colors_automatic_ZW, labels_automatic_ZW_0, colors_automatic_ZW_0, means_ZW, list_of_frms_ZW = get_time_clusters(data_ZW, constants.N_COMPONENTS_TIME_ZW) 181 | 182 | fig, ax = plt.subplots() 183 | ax.broken_barh(labels_manual, (17, 2), facecolors = colors_manual) 184 | ax.broken_barh(labels_automatic_W, (13, 2), facecolors = colors_automatic_W) 185 | ax.broken_barh(labels_automatic_Z, (9, 2), facecolors = colors_automatic_Z) 186 | ax.broken_barh(labels_automatic_ZW, (5, 2), facecolors = colors_automatic_ZW) 187 | 188 | ax.set_ylim(3,21) 189 | ax.set_xlim(0, end) 190 | ax.set_xlabel('Frame number') 191 | ax.set_yticks([6, 10, 14, 18]) 192 | ax.set_yticklabels(['ZW','Z','W', 'Manual']) 193 | 194 | if save_fname: 195 | plt.savefig(save_fname) 196 | else: 197 | plt.show() 198 | pass 199 | 200 | fig, ax = plt.subplots() 201 | ax.broken_barh(labels_manual, (17, 2), facecolors = colors_manual) 202 | ax.broken_barh(labels_automatic_W_0, (13, 2), facecolors = colors_automatic_W_0) 203 | ax.broken_barh(labels_automatic_Z_0, (9, 2), facecolors = colors_automatic_Z_0) 204 | ax.broken_barh(labels_automatic_ZW_0, (5, 2), facecolors = colors_automatic_ZW_0) 205 | 206 | ax.set_ylim(3,21) 207 | ax.set_xlim(0, end) 208 | ax.set_xlabel('Frame number') 209 | ax.set_yticks([6, 10, 14, 18]) 210 | ax.set_yticklabels(['ZW_0','Z_0','W_0', 'Manual']) 211 | 212 | if save_fname2: 213 | plt.savefig(save_fname2) 214 | else: 215 | plt.show() 216 | pass 217 | 218 | time_sequence_1 = [elem[0] + elem[1] for elem in labels_manual] 219 | 220 | time_sequence_2 = means_W 221 | dtw_score_W = compute_dtw(time_sequence_1, time_sequence_2) 222 | 223 | time_sequence_2 = means_Z 224 | dtw_score_Z = compute_dtw(time_sequence_1, time_sequence_2) 225 | 226 | time_sequence_2 = means_ZW 227 | dtw_score_ZW = compute_dtw(time_sequence_1, time_sequence_2) 228 | 229 | dtw_score_W_normalized = dtw_score_W/float(length) * 100 230 | dtw_score_Z_normalized = dtw_score_Z/float(length) * 100 231 | dtw_score_ZW_normalized = dtw_score_ZW/float(length) * 100 232 | 233 | return dtw_score_W, dtw_score_Z, dtw_score_ZW, dtw_score_W_normalized, dtw_score_Z_normalized, dtw_score_ZW_normalized, length 234 | 235 | def preprocess_labels(list_of_labels): 236 | processed_list_of_labels = [] 237 | for elem in list_of_labels: 238 | if elem[1] < 4.0: 239 | processed_list_of_labels.append((elem[0], elem[1] + 4.0)) 240 | else: 241 | processed_list_of_labels.append(elem) 242 | return processed_list_of_labels 243 | 244 | def plot_broken_barh_from_pickle(demonstration, output_fname, labels_manual, colors_manual, labels_automatic_W, 245 | colors_automatic_W, labels_automatic_Z, colors_automatic_Z, labels_automatic_ZW, colors_automatic_ZW): 246 | 247 | PATH_TO_ANNOTATION = constants.PATH_TO_DATA + constants.ANNOTATIONS_FOLDER + demonstration + "_" + constants.CAMERA + ".p" 248 | start, end = utils.get_start_end_annotations(constants.PATH_TO_DATA + constants.ANNOTATIONS_FOLDER + demonstration + "_" + constants.CAMERA + ".p") 249 | length = end - start 250 | 251 | fig, ax = plt.subplots() 252 | 253 | # Plot 1) Manual 2) Time clusters 254 | ax.broken_barh(labels_manual, (17, 2), facecolors = colors_manual) 255 | ax.broken_barh(preprocess_labels(labels_automatic_W), (13, 2), facecolors = colors_automatic_W) 256 | ax.broken_barh(preprocess_labels(labels_automatic_Z), (9, 2), facecolors = colors_automatic_Z) 257 | ax.broken_barh(preprocess_labels(labels_automatic_ZW), (5, 2), facecolors = colors_automatic_ZW) 258 | 259 | TASK = constants.TASK_NAME 260 | if (TASK in ["lego", "plane"]): 261 | end = end + 20 262 | elif (TASK in ["000", "010", "011", "100"]): 263 | end = end + 10 264 | else: 265 | end = end + 50 266 | 267 | ticks = get_ticks(labels_manual) 268 | ax.set_ylim(3,21) 269 | ax.set_xlim(0, end) 270 | ax.set_xlabel('Frame number') 271 | ax.set_yticks([6, 10, 14, 18]) 272 | ax.set_yticklabels(['Both (k + z)','Visual (z)','Kinematics (k)', 'Manual']) 273 | 274 | if output_fname: 275 | plt.savefig(output_fname) 276 | else: 277 | plt.show() 278 | pass 279 | 280 | def plot_broken_barh(demonstration, data, save_fname = None, T = 10): 281 | """ 282 | Parameters: 283 | ----------- 284 | demonstration: String name of demonstration without camera specification , e.g. "Suturing_E001" 285 | 286 | list_of_frms_[1,2,3,4]: List of changepoint frames for each of 4 different clustering experiments. 287 | Use this to compare manually vs. automatically generated transition points. 288 | 289 | * For now, the list_of_frms are constrained to 4 for visualization sanity sake. 290 | """ 291 | 292 | numDemos = min(5, len(data.keys()) + 1) 293 | sizeTestSet = numDemos - 1 294 | 295 | PATH_TO_ANNOTATION = constants.PATH_TO_DATA + constants.ANNOTATIONS_FOLDER + demonstration + "_" + constants.CAMERA + ".p" 296 | start, end = utils.get_start_end_annotations(constants.PATH_TO_DATA + constants.ANNOTATIONS_FOLDER + demonstration + "_" + constants.CAMERA + ".p") 297 | length = end - start 298 | segments = pickle.load(open(PATH_TO_ANNOTATION, "rb")) 299 | 300 | fig, ax = plt.subplots() 301 | # Generate labels for 1) Manual 2) Time clusters 302 | 303 | 304 | labels_automatic_0, colors_automatic_0, labels_automatic_1, colors_automatic_1, means, list_of_frms = get_time_clusters(data, T) 305 | 306 | labels_manual, colors_manual = setup_manual_labels(segments) 307 | 308 | # Plot 1) Manual 2) Time clusters 309 | ax.broken_barh(labels_manual, (25, 2), facecolors = colors_manual) 310 | ax.broken_barh(labels_automatic_0, (21, 2), facecolors = colors_automatic_0) 311 | 312 | list_of_plot_ranges = [(17, 2), (13, 2), (9, 2), (5, 2)] 313 | 314 | for i in range(min(sizeTestSet, 4)): 315 | labels_automatic, colors_automatic = setup_automatic_labels(list_of_frms[i], "k") 316 | ax.broken_barh(labels_automatic, list_of_plot_ranges[i], facecolors = colors_automatic) 317 | 318 | TASK = constants.TASK_NAME 319 | if (TASK in ["lego", "plane"]): 320 | end = end + 20 321 | elif (TASK in ["000", "010", "011", "100"]): 322 | end = end + 10 323 | else: 324 | end = end + 50 325 | 326 | ticks = get_ticks(labels_manual) 327 | ax.set_ylim(3,29) 328 | ax.set_xlim(0, end) 329 | ax.set_xlabel('Frame number') 330 | # ax.set_xticks(ticks) 331 | ax.set_yticks([6, 10, 14, 18, 22, 26]) 332 | ax.set_yticklabels(['Automatic4','Automatic3','Automatic2', 'Automatic1','Time Clustering', 'Manual']) 333 | 334 | if save_fname: 335 | plt.savefig(save_fname) 336 | else: 337 | plt.show() 338 | pass 339 | 340 | time_sequence_1 = [elem[0] + elem[1] for elem in labels_manual ] 341 | time_sequence_2 = means 342 | 343 | dtw_score = compute_dtw(time_sequence_1, time_sequence_2) 344 | normalized_dtw_score = dtw_score/float(length) * 100 345 | 346 | return dtw_score, normalized_dtw_score, length, labels_manual, colors_manual, labels_automatic_0, colors_automatic_0 347 | -------------------------------------------------------------------------------- /scripts/plot_tsne.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Script to plot t-SNE visualizations 4 | 5 | import numpy as np 6 | import matplotlib.pyplot as plt 7 | import pickle 8 | import IPython 9 | import sys 10 | import cv2 11 | import matplotlib.image as mpimg 12 | from sklearn import (decomposition, preprocessing, manifold) 13 | 14 | import utils 15 | import constants 16 | 17 | PATH_TO_FIGURE = "../tsne_plots/" 18 | colormap_name = "coolwarm" # We found this to be the best! 19 | 20 | start = 0.00 # Suturing_E001 21 | end = 1.0 # Suturing_E001 22 | 23 | def generate_SIFT(): 24 | data = pickle.load(open("sift_features/SIFT_plane_9_1.p", "rb")) 25 | X_pca = utils.pca(data, PC = 2) 26 | X_tsne = utils.tsne(data) 27 | data_dimred = [X_pca, X_tsne] 28 | pickle.dump(data_dimred, open("SIFT_plane_9_dimred.p", "wb")) 29 | 30 | def generate_raw_image_pixels(list_of_demonstrations): 31 | """ 32 | PCA and t-SNE on raw image pixels 33 | """ 34 | 35 | # Design matrix of raw image pixels 36 | X = None 37 | 38 | for demonstration in list_of_demonstrations: 39 | print "Raw image pixels ", demonstration 40 | PATH_TO_ANNOTATION = constants.PATH_TO_DATA + constants.ANNOTATIONS_FOLDER + demonstration + "_" + str(constants.CAMERA) + ".p" 41 | 42 | start, end = utils.get_start_end_annotations(PATH_TO_ANNOTATION) 43 | for frm in range(start, end + 1): 44 | if ((frm % 6) == 0): 45 | PATH_TO_IMAGE = utils.get_full_image_path(constants.PATH_TO_DATA + constants.NEW_FRAMES_FOLDER + demonstration + "_" + constants.CAMERA + "/", frm) 46 | print demonstration, str(frm) 47 | img = utils.reshape(cv2.imread(PATH_TO_IMAGE).flatten()) 48 | X = utils.safe_concatenate(X, img) 49 | 50 | X_pca = utils.pca(X, PC = 2) 51 | X_tsne = utils.tsne(X) 52 | data_dimred = [X_pca, X_tsne] 53 | pickle.dump(X_tsne, open("raw_pixel_" + demonstration + "_dimred.p", "wb")) 54 | 55 | def plot_scatter_continous(X, figure_name, title = None, colormap_name = "Accent", 56 | changepoints = None, labels = None, plotter = None, end_frame = 1000000, interactive_mode = False): 57 | 58 | if plotter == None: 59 | plotter = plt 60 | plt.figure() 61 | 62 | start_frame_chgpts = np.floor(X.shape[0] * start) 63 | end_frame_chgpts = np.floor(X.shape[0] * end) 64 | 65 | X = X[start_frame_chgpts:end_frame_chgpts,:] 66 | 67 | x_min, x_max = np.min(X, 0), np.max(X, 0) 68 | X = (X - x_min) / (x_max - x_min) 69 | coords_x = [] 70 | coords_y = [] 71 | colors = np.linspace(0, 1, X.shape[0]) 72 | mymap = plt.get_cmap(colormap_name) 73 | 74 | for i in range(X.shape[0]): 75 | if i <= end_frame: 76 | x_coord = X[i, 0] 77 | y_coord = X[i, 1] 78 | coords_x.append(x_coord) 79 | coords_y.append(y_coord) 80 | 81 | if interactive_mode: 82 | plotter.scatter(np.array(coords_x), np.array(coords_y), s = 40, color = 'm') 83 | else: 84 | plotter.scatter(np.array(coords_x), np.array(coords_y), s = 40, c = colors, edgecolors = 'None', cmap = mymap) 85 | 86 | # Plotting changepoints on the scatter plot 87 | if changepoints: 88 | changepoints_x = [] 89 | changepoints_y = [] 90 | 91 | for frame_number in changepoints: 92 | if (frame_number < end_frame_chgpts) and (frame_number > start_frame_chgpts) and (frame_number <= end_frame): 93 | x_coord = X[frame_number - int(start_frame_chgpts), 0] 94 | y_coord = X[frame_number - int(start_frame_chgpts), 1] 95 | changepoints_x.append(x_coord) 96 | changepoints_y.append(y_coord) 97 | if interactive_mode: 98 | plotter.scatter(np.array(changepoints_x), np.array(changepoints_y), s = 100, marker = "^", color = None, edgecolors = 'k') 99 | else: 100 | plotter.scatter(np.array(changepoints_x), np.array(changepoints_y), s = 60, marker = "^", color="k", edgecolors = 'None') 101 | 102 | #Plotting labels 103 | if labels: 104 | manual_label_x = [] 105 | manual_label_y = [] 106 | for frame_number in labels: 107 | if (frame_number < end_frame_chgpts) and (frame_number > start_frame_chgpts) and (frame_number <= end_frame): 108 | x_coord = X[frame_number - int(start_frame_chgpts), 0] 109 | y_coord = X[frame_number - int(start_frame_chgpts), 1] 110 | manual_label_x.append(x_coord) 111 | manual_label_y.append(y_coord) 112 | if interactive_mode: 113 | plotter.scatter(np.array(manual_label_x), np.array(manual_label_y), s = 100, color = 'c') 114 | else: 115 | plotter.scatter(np.array(manual_label_x), np.array(manual_label_y), s = 80, facecolors='none', edgecolors = 'g') 116 | plt.savefig(PATH_TO_FIGURE + figure_name + "_" + colormap_name + ".jpg") 117 | 118 | def plot_VGG(data, demonstration, changepoints = None, plotter = None, labels = None, end_frame = 1000000, interactive_mode = False, plot_pca = False, plot_tsne = False): 119 | 120 | # list_of_layers = ['conv5_3', 'conv4_3', 'pool5'] 121 | 122 | list_of_layers = ['conv5_3'] 123 | 124 | PATH_TO_ANNOTATION = constants.PATH_TO_DATA + constants.ANNOTATIONS_FOLDER + demonstration + "_" + str(constants.CAMERA) + ".p" 125 | annotations = pickle.load(open(PATH_TO_ANNOTATION, "rb")) 126 | 127 | for layer in list_of_layers: 128 | [X_pca, X_tsne, X_grp] = data[layer] 129 | 130 | X_pca = utils.sample_matrix(X_pca, 2) 131 | X_tsne = utils.sample_matrix(X_tsne, 2) 132 | 133 | print "VGG", layer, X_tsne.shape 134 | if plot_pca: 135 | plot_scatter_continous(X_pca, "plot_VGG_PCA" + layer+ "_" + str(start) + "_"+str(end), 136 | colormap_name = colormap_name, changepoints = changepoints, labels = labels, plotter = plotter, end_frame = end_frame, interactive_mode = interactive_mode) 137 | if plot_tsne: 138 | if interactive_mode: 139 | plot_scatter_continous(X_tsne, "plot_VGG_tSNE" + layer+ "_" + str(start) + "_"+str(end), 140 | colormap_name = colormap_name, changepoints = changepoints, labels = labels, plotter = plotter, end_frame = end_frame, interactive_mode = interactive_mode) 141 | else: 142 | plot_scatter_continous(X_tsne, "plot_VGG_tSNE" + layer+ "_" + str(end_frame), 143 | colormap_name = colormap_name, changepoints = changepoints, labels = labels, plotter = plotter, end_frame = end_frame, interactive_mode = interactive_mode) 144 | 145 | def plot_AlexNet(data, demonstration, changepoints = None, plotter = None, labels = None, end_frame = 1000000, interactive_mode = False, plot_pca = False, plot_tsne = False): 146 | 147 | # list_of_layers = ["conv3", "conv4", "pool5"] 148 | 149 | list_of_layers = ["pool5"] 150 | 151 | PATH_TO_ANNOTATION = constants.PATH_TO_DATA + constants.ANNOTATIONS_FOLDER + demonstration + "_" + str(constants.CAMERA) + ".p" 152 | annotations = pickle.load(open(PATH_TO_ANNOTATION, "rb")) 153 | 154 | for layer in list_of_layers: 155 | [X_pca, X_tsne, X_grp] = data[layer] 156 | 157 | X_pca = utils.sample_matrix(X_pca, 2) 158 | X_tsne = utils.sample_matrix(X_tsne, 2) 159 | 160 | print "AlexNet", layer, X_tsne.shape 161 | 162 | if plot_pca: 163 | plot_scatter_continous(X_pca, "plot_AlexNet_PCA" + layer+ "_" + str(start)+"_"+str(end), 164 | colormap_name = colormap_name, changepoints = changepoints, labels = labels, plotter = plotter, end_frame = end_frame, interactive_mode = interactive_mode) 165 | if plot_tsne: 166 | if interactive_mode: 167 | plot_scatter_continous(X_tsne, "plot_AlexNet_tSNE" + layer + "_" + str(end_frame), 168 | colormap_name = colormap_name, changepoints = changepoints, labels = labels, plotter = plotter, end_frame = end_frame, interactive_mode = interactive_mode) 169 | else: 170 | plot_scatter_continous(X_tsne, "plot_AlexNet_tSNE" + layer + "_" + str(start) + "_" + str(end), colormap_name = colormap_name, changepoints = changepoints, labels = labels, plotter = plotter, end_frame = end_frame, interactive_mode = interactive_mode) 171 | 172 | def plot_raw_image_pixels(data, demonstration, changepoints = None, plotter = None, labels = None, end_frame = 1000000, interactive_mode = False, plot_pca = False, plot_tsne = False): 173 | 174 | [X_pca, X_tsne] = data 175 | 176 | print "Raw Pixels", X_tsne.shape 177 | 178 | PATH_TO_ANNOTATION = constants.PATH_TO_DATA + constants.ANNOTATIONS_FOLDER + demonstration + "_" + str(constants.CAMERA) + ".p" 179 | annotations = pickle.load(open(PATH_TO_ANNOTATION, "rb")) 180 | 181 | if plot_pca: 182 | plot_scatter_continous(X_pca, "plot_raw_pixels_PCA" + "_" + str(start) + "_" + str(end), 183 | colormap_name = colormap_name, changepoints = changepoints, labels = labels, plotter = plotter, end_frame = end_frame, interactive_mode = interactive_mode) 184 | 185 | if plot_tsne: 186 | plot_scatter_continous(X_tsne, "plot_raw_pixels_tSNE" + "_" + str(start) + "_" + str(end), 187 | colormap_name = colormap_name, changepoints = changepoints, labels = labels, plotter = plotter, end_frame = end_frame, interactive_mode = interactive_mode) 188 | 189 | def plot_SIFT(data, demonstration, changepoints = None, plotter = None, labels = None, end_frame = 1000000, interactive_mode = False, plot_pca = False, plot_tsne = False): 190 | 191 | [X_pca, X_tsne] = data 192 | X_pca = utils.sample_matrix(X_pca, 6) 193 | X_tsne = utils.sample_matrix(X_tsne, 6) 194 | 195 | print "SIFT", X_tsne.shape 196 | 197 | PATH_TO_ANNOTATION = constants.PATH_TO_DATA + constants.ANNOTATIONS_FOLDER + demonstration + "_" + str(constants.CAMERA) + ".p" 198 | annotations = pickle.load(open(PATH_TO_ANNOTATION, "rb")) 199 | # labels = utils.get_chronological_sequences(annotations) 200 | 201 | if plot_pca: 202 | plot_scatter_continous(X_pca, "plot_SIFT_PCA" + "_" + str(start) + "_" + str(end), 203 | colormap_name = colormap_name, changepoints = changepoints, labels = labels, plotter = plotter) 204 | if plot_tsne: 205 | plot_scatter_continous(X_tsne, "plot_SIFT_tSNE" + "_" + str(start) + "_" + str(end), 206 | colormap_name = colormap_name, changepoints = changepoints, labels = labels, plotter = plotter, end_frame = end_frame, interactive_mode = interactive_mode) 207 | 208 | def plot_all_same_figure(): 209 | changepoints = pickle.load(open("pickle_files/Suturing_E001_changepoints_Z1.p", "rb")) 210 | jackknife_index = 1 211 | demonstration = "Suturing_E001" 212 | 213 | f, (ax1, ax2, ax3) = plt.subplots(1, 3, sharey=True) 214 | 215 | # AlexNet 216 | data = pickle.load(open("pickle_files/Suturing_E001_AlexNet_dimred.p", "rb")) 217 | ax3.set_title('AlexNet') 218 | plot_AlexNet(data, changepoints[jackknife_index], demonstration, plotter = ax3) 219 | 220 | # SIFT 221 | data = pickle.load(open("pickle_files/Suturing_E001_SIFT_dimred.p", "rb")) 222 | ax2.set_title('SIFT') 223 | plot_SIFT(data, changepoints[jackknife_index], demonstration, plotter = ax2) 224 | 225 | # Raw Image Pixels 226 | data = pickle.load(open("pickle_files/Suturing_E001_raw_pixel_dimred.p", "rb")) 227 | ax1.set_title('Raw Image Pixels') 228 | plot_raw_image_pixels(data, changepoints[jackknife_index], demonstration, plotter = ax1) 229 | 230 | plt.show() 231 | 232 | def plot_all(): 233 | # Suturing_E001 234 | # changepoints = pickle.load(open("pickle_files/Suturing_E001_changepoints_Z1.p", "rb")) 235 | # jackknife_index = 1 236 | demonstration = "Suturing_E001" 237 | 238 | PATH_TO_ANNOTATION = constants.PATH_TO_DATA + constants.ANNOTATIONS_FOLDER + demonstration + "_" + str(constants.CAMERA) + ".p" 239 | start_d, end_d = utils.get_start_end_annotations(PATH_TO_ANNOTATION) 240 | 241 | # changepoints = changepoints[jackknife_index] 242 | changepoints_processed = [] 243 | 244 | results = pickle.load(open("pickle_files/Suturing_E001_changepoints_Z2.p", "rb")) 245 | jackknife_index = 3 246 | demonstration = "Suturing_E001" 247 | 248 | changepoints = results[demonstration]['changepoints'][jackknife_index] 249 | labels = results[demonstration]['plot_labels_automatic'] 250 | 251 | for elem in changepoints: 252 | changepoints_processed.append((elem[0] - start_d)/6) 253 | 254 | labels_processed = [] 255 | for elem in labels: 256 | labels_processed.append((elem[0] - start_d)/6) 257 | 258 | # VGG 259 | data = pickle.load(open("pickle_files/Suturing_E001_VGG_dimred.p", "rb")) 260 | plot_VGG(data, demonstration, changepoints = None, labels = None, plot_tsne = True) 261 | 262 | # AlexNet 263 | data = pickle.load(open("pickle_files/Suturing_E001_AlexNet_dimred.p", "rb")) 264 | plot_AlexNet(data, demonstration, changepoints = None, labels = None, plot_tsne = True) 265 | 266 | # Raw Pixels 267 | data = pickle.load(open("pickle_files/Suturing_E001_raw_pixel_dimred.p", "rb")) 268 | plot_raw_image_pixels(data, demonstration, changepoints = None, labels = None, plot_tsne = True) 269 | 270 | # SIFT 271 | data = pickle.load(open("pickle_files/Suturing_E001_SIFT_dimred.p", "rb")) 272 | plot_SIFT(data, demonstration, changepoints = None, labels = None, plot_tsne = True) 273 | 274 | def plot_interactive(): 275 | results = pickle.load(open("pickle_files/Suturing_E001_changepoints_Z2.p", "rb")) 276 | jackknife_index = 3 277 | demonstration = "Suturing_E001" 278 | 279 | changepoints = results[demonstration]['changepoints'][jackknife_index] 280 | labels = results[demonstration]['plot_labels_automatic'] 281 | 282 | PATH_TO_ANNOTATION = constants.PATH_TO_DATA + constants.ANNOTATIONS_FOLDER + demonstration + "_" + str(constants.CAMERA) + ".p" 283 | start_d, end_d = utils.get_start_end_annotations(PATH_TO_ANNOTATION) 284 | 285 | changepoints_processed = [] 286 | for elem in changepoints: 287 | changepoints_processed.append((elem[0] - start_d)/12) 288 | 289 | labels_processed = [] 290 | for elem in labels: 291 | labels_processed.append((elem[0] - start_d)/12) 292 | 293 | total_frames = [int(elem[0]) for elem in changepoints] 294 | total_frames += [int(elem[0]) for elem in labels] 295 | total_frames.sort() 296 | 297 | for end_frame in total_frames: 298 | 299 | # Frame 300 | ax1 = plt.subplot(121) 301 | PATH_TO_FIGURE = constants.PATH_TO_DATA + constants.NEW_FRAMES_FOLDER + demonstration + "_" + str(constants.CAMERA) + "/" 302 | im = mpimg.imread(utils.get_full_image_path(PATH_TO_FIGURE, end_frame)) 303 | ax1.set_title(str(end_frame)) 304 | ax1.xaxis.set_visible(False) 305 | ax1.yaxis.set_visible(False) 306 | ax1.imshow(im) 307 | 308 | # AlexNet 309 | ax2 = plt.subplot(122) 310 | data = pickle.load(open("pickle_files/Suturing_E001_AlexNet_dimred.p", "rb")) 311 | ax2.set_title('AlexNet') 312 | ax2.set_ylim([-0.1, 1.1]) 313 | ax2.set_xlim([-0.1, 1.1]) 314 | ax2.xaxis.set_visible(False) 315 | ax2.yaxis.set_visible(False) 316 | plot_AlexNet(data, demonstration, changepoints = changepoints_processed, plotter = ax2, labels = labels_processed, plot_tsne = True, end_frame = (end_frame - start_d)/12, interactive_mode = True) 317 | 318 | if __name__ == "__main__": 319 | 320 | if PATH_TO_FIGURE is None: 321 | print "ERROR: Please specify path to pyplot savefig" 322 | sys.exit() 323 | 324 | # plot_all() 325 | plot_interactive() -------------------------------------------------------------------------------- /scripts/experiments.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import pickle 4 | import numpy as np 5 | import random 6 | import os 7 | import argparse 8 | import sys 9 | import IPython 10 | import itertools 11 | 12 | import constants 13 | import parser 14 | import utils 15 | import cluster_evaluation 16 | import broken_barh 17 | import clustering 18 | import clustering_kinematics 19 | 20 | from clustering import MilestonesClustering 21 | from clustering_kinematics import KinematicsClustering 22 | 23 | PATH_TO_FEATURES = constants.PATH_TO_DATA + constants.PROC_FEATURES_FOLDER 24 | 25 | def evaluate_multi_modal(metrics, file): 26 | 27 | mutual_information_1 = [] 28 | normalized_mutual_information_1 = [] 29 | adjusted_mutual_information_1 = [] 30 | homogeneity_1 = [] 31 | precision_1_micro = [] 32 | recall_1_micro = [] 33 | precision_1_macro = [] 34 | recall_1_macro = [] 35 | precision_1_weighted = [] 36 | recall_1_weighted = [] 37 | 38 | mutual_information_2 = [] 39 | normalized_mutual_information_2 = [] 40 | adjusted_mutual_information_2 = [] 41 | homogeneity_2 = [] 42 | precision_2_micro = [] 43 | recall_2_micro = [] 44 | precision_2_macro = [] 45 | recall_2_macro = [] 46 | precision_2_weighted = [] 47 | recall_2_weighted = [] 48 | 49 | silhoutte_level_1 = [] 50 | dunn1_level_1 = [] 51 | dunn2_level_1 = [] 52 | dunn3_level_1 = [] 53 | 54 | silhoutte_level_2 = [] 55 | dunn1_level_2 = [] 56 | dunn2_level_2 = [] 57 | dunn3_level_2 = [] 58 | 59 | list_of_frms = {} 60 | 61 | for elem in metrics: 62 | precision_1_micro.append(elem[0]["precision_micro"]) 63 | precision_2_micro.append(elem[1]["precision_micro"]) 64 | 65 | precision_1_macro.append(elem[0]["precision_macro"]) 66 | precision_2_macro.append(elem[1]["precision_macro"]) 67 | 68 | precision_1_weighted.append(elem[0]["precision_weighted"]) 69 | precision_2_weighted.append(elem[1]["precision_weighted"]) 70 | 71 | recall_1_micro.append(elem[0]["recall_micro"]) 72 | recall_2_micro.append(elem[1]["recall_micro"]) 73 | 74 | recall_1_macro.append(elem[0]["recall_macro"]) 75 | recall_2_macro.append(elem[1]["recall_macro"]) 76 | 77 | recall_1_weighted.append(elem[0]["recall_weighted"]) 78 | recall_2_weighted.append(elem[1]["recall_weighted"]) 79 | 80 | mutual_information_1.append(elem[0]["mutual_info_score"]) 81 | mutual_information_2.append(elem[1]["mutual_info_score"]) 82 | 83 | normalized_mutual_information_1.append(elem[0]["normalized_mutual_info_score"]) 84 | normalized_mutual_information_2.append(elem[1]["normalized_mutual_info_score"]) 85 | 86 | adjusted_mutual_information_1.append(elem[0]["adjusted_mutual_info_score"]) 87 | adjusted_mutual_information_2.append(elem[1]["adjusted_mutual_info_score"]) 88 | 89 | homogeneity_1.append(elem[0]["homogeneity_score"]) 90 | homogeneity_2.append(elem[1]["homogeneity_score"]) 91 | 92 | silhoutte_level_1.append(elem[2]["level1"]) 93 | dunn1_level_1.append(elem[3]["level1"]) 94 | dunn2_level_1.append(elem[4]["level1"]) 95 | dunn3_level_1.append(elem[5]["level1"]) 96 | 97 | silhoutte_level_2 += elem[6] 98 | dunn1_level_2 += elem[7] 99 | dunn2_level_2 += elem[8] 100 | dunn3_level_2 += elem[9] 101 | 102 | viz = elem[10] 103 | for demonstration in viz.keys(): 104 | utils.dict_insert_list(demonstration, viz[demonstration], list_of_frms) 105 | 106 | 107 | utils.print_and_write_2("precision_1_micro", np.mean(precision_1_micro), np.std(precision_1_micro), file) 108 | utils.print_and_write_2("precision_2_micro", np.mean(precision_2_micro), np.std(precision_2_micro), file) 109 | 110 | utils.print_and_write_2("precision_1_macro", np.mean(precision_1_macro), np.std(precision_1_macro), file) 111 | utils.print_and_write_2("precision_2_macro", np.mean(precision_2_macro), np.std(precision_2_macro), file) 112 | 113 | utils.print_and_write_2("precision_1_weighted", np.mean(precision_1_weighted), np.std(precision_1_weighted), file) 114 | utils.print_and_write_2("precision_2_weighted", np.mean(precision_2_weighted), np.std(precision_2_weighted), file) 115 | 116 | utils.print_and_write_2("recall_1_micro", np.mean(recall_1_micro), np.std(recall_1_micro), file) 117 | utils.print_and_write_2("recall_2_micro", np.mean(recall_2_micro), np.std(recall_2_micro), file) 118 | 119 | utils.print_and_write_2("recall_1_macro", np.mean(recall_1_macro), np.std(recall_1_macro), file) 120 | utils.print_and_write_2("recall_2_macro", np.mean(recall_2_macro), np.std(recall_2_macro), file) 121 | 122 | utils.print_and_write_2("recall_1_weighted", np.mean(recall_1_weighted), np.std(recall_1_weighted), file) 123 | utils.print_and_write_2("recall_2_weighted", np.mean(recall_2_weighted), np.std(recall_2_weighted), file) 124 | 125 | utils.print_and_write_2("mutual_info_1", np.mean(mutual_information_1), np.std(mutual_information_1), file) 126 | utils.print_and_write_2("mutual_info_2", np.mean(mutual_information_2), np.std(mutual_information_2), file) 127 | 128 | utils.print_and_write_2("normalized_mutual_info_1", np.mean(normalized_mutual_information_1), np.std(normalized_mutual_information_1), file) 129 | utils.print_and_write_2("normalized_mutual_info_2", np.mean(normalized_mutual_information_2), np.std(normalized_mutual_information_2), file) 130 | 131 | utils.print_and_write_2("adjusted_mutual_info_1", np.mean(adjusted_mutual_information_1), np.std(adjusted_mutual_information_1), file) 132 | utils.print_and_write_2("adjusted_mutual_info_2", np.mean(adjusted_mutual_information_2), np.std(adjusted_mutual_information_2), file) 133 | 134 | utils.print_and_write_2("homogeneity_1", np.mean(homogeneity_1), np.std(homogeneity_1), file) 135 | utils.print_and_write_2("homogeneity_2", np.mean(homogeneity_2), np.std(homogeneity_2), file) 136 | 137 | utils.print_and_write_2("silhoutte_level_1", np.mean(silhoutte_level_1), np.std(silhoutte_level_1), file) 138 | utils.print_and_write_2("silhoutte_level_2", np.mean(silhoutte_level_2), np.std(silhoutte_level_2), file) 139 | 140 | utils.print_and_write_2("dunn1_level1", np.mean(dunn1_level_1), np.std(dunn1_level_1), file) 141 | utils.print_and_write_2("dunn2_level1", np.mean(dunn2_level_1), np.std(dunn2_level_1), file) 142 | utils.print_and_write_2("dunn3_level1", np.mean(dunn3_level_1), np.std(dunn3_level_1), file) 143 | 144 | utils.print_and_write_2("dunn1_level2", np.mean(dunn1_level_2), np.std(dunn1_level_2), file) 145 | utils.print_and_write_2("dunn2_level2", np.mean(dunn2_level_2), np.std(dunn2_level_2), file) 146 | utils.print_and_write_2("dunn3_level2", np.mean(dunn3_level_2), np.std(dunn3_level_2), file) 147 | 148 | return list_of_frms 149 | 150 | def evaluate_single_modal(metrics, file): 151 | mutual_information_1 = [] 152 | normalized_mutual_information_1 = [] 153 | adjusted_mutual_information_1 = [] 154 | homogeneity_1 = [] 155 | precision_1_micro = [] 156 | recall_1_micro = [] 157 | precision_1_macro = [] 158 | recall_1_macro = [] 159 | precision_1_weighted = [] 160 | recall_1_weighted = [] 161 | silhouette_scores = [] 162 | 163 | 164 | dunn1_level_1 = [] 165 | dunn2_level_1 = [] 166 | dunn3_level_1 = [] 167 | 168 | list_of_frms = {} 169 | 170 | for elem in metrics: 171 | 172 | mutual_information_1.append(elem[0]["mutual_info_score"]) 173 | normalized_mutual_information_1.append(elem[0]["normalized_mutual_info_score"]) 174 | adjusted_mutual_information_1.append(elem[0]["adjusted_mutual_info_score"]) 175 | homogeneity_1.append(elem[0]["homogeneity_score"]) 176 | 177 | silhouette_scores.append(elem[1]) 178 | 179 | dunn1_level_1.append(elem[2]["level1"]) 180 | dunn2_level_1.append(elem[3]["level1"]) 181 | dunn3_level_1.append(elem[4]["level1"]) 182 | 183 | precision_1_micro.append(elem[0]["precision_micro"]) 184 | precision_1_macro.append(elem[0]["precision_macro"]) 185 | precision_1_weighted.append(elem[0]["precision_weighted"]) 186 | 187 | recall_1_micro.append(elem[0]["recall_micro"]) 188 | recall_1_macro.append(elem[0]["recall_macro"]) 189 | recall_1_weighted.append(elem[0]["recall_weighted"]) 190 | 191 | viz = elem[5] 192 | for demonstration in viz.keys(): 193 | utils.dict_insert_list(demonstration, viz[demonstration], list_of_frms) 194 | 195 | utils.print_and_write_2("precision_1_micro", np.mean(precision_1_micro), np.std(precision_1_micro), file) 196 | utils.print_and_write_2("precision_1_macro", np.mean(precision_1_macro), np.std(precision_1_macro), file) 197 | utils.print_and_write_2("precision_1_weighted", np.mean(precision_1_weighted), np.std(precision_1_weighted), file) 198 | 199 | utils.print_and_write_2("recall_1_micro", np.mean(recall_1_micro), np.std(recall_1_micro), file) 200 | utils.print_and_write_2("recall_1_macro", np.mean(recall_1_macro), np.std(recall_1_macro), file) 201 | utils.print_and_write_2("recall_1_weighted", np.mean(recall_1_weighted), np.std(recall_1_weighted), file) 202 | 203 | utils.print_and_write_2("mutual_info", np.mean(mutual_information_1), np.std(mutual_information_1), file) 204 | utils.print_and_write_2("normalized_mutual_info", np.mean(normalized_mutual_information_1), np.std(normalized_mutual_information_1), file) 205 | utils.print_and_write_2("adjusted_mutual_info", np.mean(adjusted_mutual_information_1), np.std(adjusted_mutual_information_1), file) 206 | utils.print_and_write_2("silhouette_scores", np.mean(silhouette_scores), np.std(silhouette_scores), file) 207 | 208 | utils.print_and_write_2("homogeneity", np.mean(homogeneity_1), np.std(homogeneity_1), file) 209 | 210 | utils.print_and_write_2("dunn1", np.mean(dunn1_level_1), np.std(dunn1_level_1), file) 211 | utils.print_and_write_2("dunn2", np.mean(dunn2_level_1), np.std(dunn2_level_1), file) 212 | utils.print_and_write_2("dunn3", np.mean(dunn3_level_1), np.std(dunn3_level_1), file) 213 | 214 | return list_of_frms 215 | 216 | def post_evaluation_all(metrics_W, metrics_Z, metrics_ZW, file, fname, list_of_demonstrations): 217 | 218 | utils.print_and_write("\nXXXXXXXXXXXX Metric for Kinematics W XXXXXXXXXXXX\n", file) 219 | list_of_frms_W = evaluate_single_modal(metrics_W, file) 220 | 221 | utils.print_and_write("\nXXXXXXXXXXXX Metric for Visual Z XXXXXXXXXXXX\n", file) 222 | list_of_frms_Z = evaluate_single_modal(metrics_Z, file) 223 | 224 | utils.print_and_write("\nXXXXXXXXXXXX Metric for Kinematics + Visual ZW XXXXXXXXXXXX\n", file) 225 | list_of_frms_ZW = evaluate_multi_modal(metrics_ZW, file) 226 | 227 | list_of_dtw_values_W = [] 228 | list_of_dtw_values_Z = [] 229 | list_of_dtw_values_ZW = [] 230 | list_of_dtw_values_W_normalized = [] 231 | list_of_dtw_values_Z_normalized = [] 232 | list_of_dtw_values_ZW_normalized = [] 233 | list_of_lengths = [] 234 | 235 | for demonstration in list_of_demonstrations: 236 | list_of_frms_demonstration_W = list_of_frms_W[demonstration] 237 | list_of_frms_demonstration_Z = list_of_frms_Z[demonstration] 238 | list_of_frms_demonstration_ZW = list_of_frms_ZW[demonstration] 239 | 240 | assert len(list_of_frms_demonstration_W) == len(list_of_demonstrations) - 1 241 | assert len(list_of_frms_demonstration_Z) == len(list_of_demonstrations) - 1 242 | assert len(list_of_frms_demonstration_ZW) == len(list_of_demonstrations) - 1 243 | data_W = {} 244 | data_Z = {} 245 | data_ZW = {} 246 | 247 | for i in range(len(list_of_frms_demonstration_W)): 248 | data_W[i] = list_of_frms_demonstration_W[0] 249 | data_Z[i] = list_of_frms_demonstration_Z[0] 250 | data_ZW[i] = list_of_frms_demonstration_ZW[0] 251 | 252 | save_fig = constants.PATH_TO_CLUSTERING_RESULTS + demonstration + "_" + fname + "_A.jpg" 253 | save_fig2 = constants.PATH_TO_CLUSTERING_RESULTS + demonstration + "_" + fname + "_B.jpg" 254 | 255 | dtw_score_W, dtw_score_Z, dtw_score_ZW, dtw_score_W_normalized, dtw_score_Z_normalized, dtw_score_ZW_normalized, length = broken_barh.plot_broken_barh_all(demonstration, 256 | data_W, data_Z, data_ZW, save_fig, save_fig2) 257 | list_of_dtw_values_W.append(dtw_score_W) 258 | list_of_dtw_values_Z.append(dtw_score_Z) 259 | list_of_dtw_values_ZW.append(dtw_score_ZW) 260 | list_of_dtw_values_W_normalized.append(dtw_score_W_normalized) 261 | list_of_dtw_values_Z_normalized.append(dtw_score_Z_normalized) 262 | list_of_dtw_values_ZW_normalized.append(dtw_score_ZW_normalized) 263 | list_of_lengths.append(length) 264 | 265 | utils.print_and_write_2("dtw_score_W", np.mean(list_of_dtw_values_W), np.std(list_of_dtw_values_W), file) 266 | utils.print_and_write_2("dtw_score_Z", np.mean(list_of_dtw_values_Z), np.std(list_of_dtw_values_Z), file) 267 | utils.print_and_write_2("dtw_score_ZW", np.mean(list_of_dtw_values_ZW), np.std(list_of_dtw_values_ZW), file) 268 | utils.print_and_write_2("dtw_score_W_normalized", np.mean(list_of_dtw_values_W_normalized), np.std(list_of_dtw_values_W_normalized), file) 269 | utils.print_and_write_2("dtw_score_Z_normalized", np.mean(list_of_dtw_values_Z_normalized), np.std(list_of_dtw_values_Z_normalized), file) 270 | utils.print_and_write_2("dtw_score_ZW_normalized", np.mean(list_of_dtw_values_ZW_normalized), np.std(list_of_dtw_values_ZW_normalized), file) 271 | utils.print_and_write(str(list_of_lengths), file) 272 | 273 | if __name__ == "__main__": 274 | argparser = argparse.ArgumentParser() 275 | argparser.add_argument("visual", help = "Specify visual features, e.g. 4_PCA.p", default = False) 276 | argparser.add_argument("fname", help = "File name", default = 4) 277 | args = argparser.parse_args() 278 | 279 | # list_of_demonstrations = ["100_01", "100_02", "100_03", "100_04", "100_05"] 280 | # list_of_demonstrations = ["011_01", "011_02", "011_03", "011_04", "011_05"] 281 | list_of_demonstrations = ["plane_6", "plane_7", "plane_8", "plane_9", "plane_10"] 282 | 283 | # list_of_demonstrations = ['Suturing_E001', 'Suturing_E002','Suturing_E003', 'Suturing_E004', 'Suturing_E005'] 284 | 285 | combinations = clustering.get_list_of_demo_combinations(list_of_demonstrations) 286 | 287 | feat_fname = args.visual 288 | 289 | metrics_W = [] 290 | metrics_Z = [] 291 | metrics_ZW = [] 292 | 293 | log = open(constants.PATH_TO_CLUSTERING_RESULTS + args.fname + ".txt", "wb") 294 | 295 | utils.print_and_write("\nXXXXXXXXXXXX Kinematics W XXXXXXXXXXXX\n", log) 296 | print "\nXXXXXXXXXXXX Kinematics W XXXXXXXXXXXX\n" 297 | 298 | for i in range(len(combinations)): 299 | utils.print_and_write("\n----------- Combination #" + str(i) + " -------------\n", log) 300 | print "\n----------- Combination #" + str(i) + " -------------\n" 301 | print combinations[i] 302 | mc = KinematicsClustering(False, list(combinations[i]), args.fname + str(i), log, False, feat_fname) 303 | metrics_W.append(mc.do_everything()) 304 | 305 | utils.print_and_write("\nXXXXXXXXXXXX Visual Z XXXXXXXXXXXX\n", log) 306 | print "\nXXXXXXXXXXXX Visual Z XXXXXXXXXXXX\n" 307 | 308 | for i in range(len(combinations)): 309 | utils.print_and_write("\n----------- Combination #" + str(i) + " -------------\n", log) 310 | print "\n----------- Combination #" + str(i) + " -------------\n" 311 | print combinations[i] 312 | mc = KinematicsClustering(False, list(combinations[i]), args.fname + str(i), log, True, feat_fname) 313 | metrics_Z.append(mc.do_everything()) 314 | 315 | utils.print_and_write("\nXXXXXXXXXXXX Kinematics + Vision ZW XXXXXXXXXXXX\n", log) 316 | print "\nXXXXXXXXXXXX Kinematics + Visual ZW XXXXXXXXXXXX\n" 317 | 318 | for i in range(len(combinations)): 319 | print "---- k-Fold Cross Validation, Run "+ str(i) + " out of " + str(len(combinations)) + " ----" 320 | mc = MilestonesClustering(False, list(combinations[i]), feat_fname, args.fname) 321 | metrics_ZW.append(mc.do_everything()) 322 | 323 | post_evaluation_all(metrics_W, metrics_Z, metrics_ZW, log, args.fname, list_of_demonstrations) 324 | 325 | log.close() --------------------------------------------------------------------------------