├── LICENSE ├── README.MD ├── __init__.py ├── dataset_script ├── CREATE_DATA.py ├── __init__.py ├── __init__.pyc ├── count_events_with_gt.py ├── count_events_with_gt.pyc ├── data_io_cnn.py ├── data_io_cnn.pyc ├── events_to_img.py ├── events_to_img.pyc └── merge_event_groundtruth.py └── main_keras ├── __init__.py ├── __init__.pyc ├── predict.py ├── spatial_lstm.py ├── train.py ├── vgg16.py └── vgg16.pyc /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Anh Nguyen 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.MD: -------------------------------------------------------------------------------- 1 | ### Requirements 2 | - Keras 2.0.3 3 | - Tensorflow >= 1.0 (used 1.1.0) 4 | 5 | ### Download dataset 6 | - Download datafrom from: http://rpg.ifi.uzh.ch/davis_data.html 7 | - Unzip and copy to **/event_data/raw_data** foler (see dataset_location_1.png, dataset_location2.png) 8 | 9 | ### Create train/test data 10 | - `cd dataset_script` 11 | - `python CREATE_DATA.py` 12 | - change **list_scene** inside **CREATE_DATA.py** for other scene 13 | 14 | ### Run train script: 15 | - `cd main_keras` 16 | - `python train.py --gpu GPU_ID --scene SCENCE_ID` 17 | - For example: `python train.py --gpu 0 --scene shapes_rotation` --> will train on GPU 0 and shapes_rotation sequence 18 | 19 | ### Predict & evaluate 20 | - `cd main_keras` 21 | - `python predict.py` 22 | - Change **in_scene_id**, **in_net_id**, **in_model_file_name** in this file for different scene 23 | 24 | 25 | If you find this code useful in your research, please consider citing: 26 | 27 | @article{nguyen6dof_lstmpose, 28 | author = {Anh Nguyen and 29 | Thanh{-}Toan Do and 30 | Darwin G. Caldwell and 31 | Nikos G. Tsagarakis}, 32 | title = {Real-Time Pose Estimation for Event Cameras with Stacked Spatial {LSTM} 33 | Networks}, 34 | journal = {IEEE Conference on Computer Vision and Pattern Recognition Workshops (CVPRW)}, 35 | year = {2019} 36 | } 37 | -------------------------------------------------------------------------------- /__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /dataset_script/CREATE_DATA.py: -------------------------------------------------------------------------------- 1 | # create test image based on the percentage of the input events 2 | 3 | import os 4 | import sys 5 | import numpy as np 6 | import scipy.misc as spm 7 | 8 | import random 9 | random.seed(99999) 10 | 11 | 12 | #root_path = '/home/anguyen/workspace/paper_src/2018.icra.event.source' # not .source/dataset --> wrong folder 13 | cwd = os.getcwd() 14 | print 'current dir: ', cwd 15 | root_path = os.path.abspath(os.path.join(cwd, os.pardir)) # get parent path 16 | print 'root path: ', root_path 17 | sys.path.insert(0, root_path) 18 | 19 | from dataset_script.events_to_img import convert_event_to_img, create_empty_img, padzero 20 | from dataset_script.data_io_cnn import add_img_id_to_gt, main_convert_percentage 21 | from dataset_script.count_events_with_gt import main_count_event_gt 22 | 23 | ##split_id = 'img_pose_all' ## random split 24 | split_id = 'img_pose_all_novel_split' ## novel split 25 | 26 | def create_index_event_img_file(in_count_events_gt_file, index_event_img_file): 27 | fcounter = open(in_count_events_gt_file, 'r') 28 | findex = open(index_event_img_file, 'w') 29 | 30 | all_lines = fcounter.read().split('\n') 31 | #print 'all line: ', all_lines 32 | 33 | ins_counter = 0 34 | 35 | for l in all_lines: 36 | if l.isdigit(): 37 | #print 'current l: ', l 38 | ins_counter += long(l) 39 | #print 'current index: ', ins_counter 40 | findex.write(str(ins_counter) + '\n') 41 | 42 | 43 | def write_txt(list_data, txt_file): 44 | fwriter = open(txt_file, 'w') 45 | for l in list_data: 46 | fwriter.write(l) 47 | 48 | fwriter.close() 49 | 50 | 51 | def create_train_test_list(grth_with_img_id_path, train_path, test_path): ## random split 52 | #grth_with_img_id_path = '/home/anguyen/workspace/dataset/Event/processed/shapes_rotation/groundtruth_with_img_id.txt' 53 | list_grth_id = list(open(grth_with_img_id_path, 'r')) 54 | 55 | # shuffle list 56 | random.shuffle(list_grth_id) 57 | 58 | num_train = int(0.7 * len(list_grth_id)) 59 | num_test = len(list_grth_id) - num_train 60 | print 'total train sample: ', num_train 61 | print 'total test sample: ', num_test 62 | print list_grth_id[0] 63 | 64 | # slice first num_train sample as training 65 | list_train = list_grth_id[0:num_train] 66 | print 'len list train: ', len(list_train) 67 | 68 | list_test = list(set(list_grth_id) - set(list_train)) 69 | print 'len list test: ', len(list_test) 70 | 71 | # save to txt file 72 | write_txt(list_train, train_path) 73 | write_txt(list_test, test_path) 74 | 75 | 76 | 77 | def create_train_test_list_novel_split(grth_with_img_id_path, train_path, test_path): ## 1st 70% for training, last 30% for testing 78 | #grth_with_img_id_path = '/home/anguyen/workspace/dataset/Event/processed/shapes_rotation/groundtruth_with_img_id.txt' 79 | list_grth_id = list(open(grth_with_img_id_path, 'r')) 80 | 81 | total_length = len(list_grth_id) 82 | print 'total grth id len: ', total_length 83 | 84 | split_index = int(total_length * 0.7) 85 | 86 | # slice first 50 as training 87 | list_train = list_grth_id[0:split_index] 88 | print 'len list train: ', len(list_train) 89 | 90 | list_test = list(set(list_grth_id) - set(list_train)) # the rest as testing 91 | print 'len list test: ', len(list_test) 92 | 93 | # shuffle train and test list - no need to keep sequential order 94 | random.shuffle(list_train) 95 | random.shuffle(list_test) 96 | 97 | # save to txt file 98 | write_txt(list_train, train_path) 99 | write_txt(list_test, test_path) 100 | 101 | 102 | def create_percentage_image(kp, index_event_img_file, in_count_events_gt_file, in_raw_events_file, out_percentage_img_foler): 103 | # read index file 104 | findex = open(index_event_img_file, 'r') 105 | all_index = findex.read().split('\n') 106 | #print 'all index: ', all_index 107 | 108 | list_event = list(open(in_raw_events_file, 'r')) 109 | 110 | 111 | # 1st image is empty - no previous events 112 | img = create_empty_img() 113 | fname = padzero(0) + '.png' 114 | spm.imsave(os.path.join(out_percentage_img_foler, fname), img) # save 1 chanel as greyscale 115 | 116 | 117 | start_index = 0 118 | for i in range(len(all_index)): 119 | if all_index[i].isdigit(): 120 | end_index = int(all_index[i]) 121 | 122 | print '-----------------------' 123 | print 'i: ', i 124 | print 'start index: ', start_index 125 | print 'end index : ', end_index 126 | 127 | total_events = end_index - start_index 128 | keep_num_events = int(total_events * (float(kp)/100.0)) 129 | print 'total events for image: ', total_events 130 | print 'keep num events: ', keep_num_events 131 | 132 | new_start_index = start_index + (total_events - keep_num_events) 133 | print 'new start index: ', new_start_index 134 | 135 | img = convert_event_to_img(new_start_index, end_index, list_event) 136 | #cv2.imshow('img', img) 137 | #cv2.waitKey(0) 138 | 139 | # save image 140 | fname = padzero(i+1) + '.png' 141 | spm.imsave(os.path.join(out_percentage_img_foler, fname), img) # save 1 chanel as greyscale 142 | 143 | 144 | # update new start index 145 | start_index = end_index 146 | 147 | 148 | # debug 149 | #if i == 80: break 150 | 151 | 152 | 153 | 154 | def create_folder_structure(scene_id, split_id): 155 | 156 | print 'CREATING DATA FOR: ', scene_id 157 | # create folder structure 158 | raw_folder = os.path.join(root_path, 'event_data', 'raw_data') 159 | processed_folder = os.path.join(root_path, 'event_data', 'processed') 160 | if not os.path.exists(raw_folder): 161 | os.makedirs(raw_folder) 162 | if not os.path.exists(processed_folder): 163 | os.makedirs(processed_folder) 164 | 165 | 166 | 167 | #scene_raw_folder = os.path.join(raw_folder, scene_id) #/home/anguyen/workspace/paper_src/2018.icra.event.source/event_data/raw_data/shapes_rotation 168 | scene_raw_folder = os.path.join(raw_folder, scene_id) 169 | if not os.path.exists(scene_raw_folder): 170 | print 'ERROR: NO RAW DATA FOR: ', scene_id, 'SCENE!' 171 | 172 | 173 | #scene_processed_folder = os.path.join(processed_folder, scene_id) #/home/anguyen/workspace/paper_src/2018.icra.event.source/event_data/processed/shapes_rotation 174 | scene_processed_folder = os.path.join(processed_folder, scene_id, split_id) #/home/anguyen/workspace/paper_src/2018.icra.event.source/event_data/processed/shapes_rotation 175 | #print 'processed folder: ', scene_processed_folder 176 | if not os.path.exists(scene_processed_folder): 177 | os.makedirs(scene_processed_folder) 178 | 179 | return scene_raw_folder, scene_processed_folder 180 | 181 | 182 | 183 | 184 | 185 | def create_files(scene_raw_folder, scene_processed_folder): 186 | 187 | in_gt_file = scene_raw_folder + '/groundtruth.txt' 188 | in_event_file = scene_raw_folder + '/events.txt' 189 | 190 | print '-------------- COUNT NUMBER OF EVENTS --------------' 191 | counter_event_grt_path = scene_processed_folder + '/count_events_gt.txt' 192 | if not os.path.exists(counter_event_grt_path): 193 | main_count_event_gt(in_event_file, in_gt_file, counter_event_grt_path) 194 | else: 195 | print 'FILE: ', counter_event_grt_path ,' already exists. Not create new! Delete old file if you want to re-run.' 196 | 197 | 198 | print '-------------- ADD IMAGE ID TO GROUNDTRUTH --------------' 199 | grth_with_img_id_path = scene_processed_folder + '/groundtruth_with_img_id.txt' 200 | if not os.path.exists(grth_with_img_id_path): 201 | add_img_id_to_gt(in_gt_file, grth_with_img_id_path) 202 | else: 203 | print 'FILE: ', grth_with_img_id_path, ' already exists. Not create new! Delete old file if you want to re-run.' 204 | 205 | 206 | print '-------------- CREATE TRAIN + TEST LIST --------------' 207 | train_path = scene_processed_folder + '/train.txt' 208 | test_path = scene_processed_folder + '/test.txt' 209 | if not os.path.exists(train_path): 210 | #create_train_test_list(grth_with_img_id_path, train_path, test_path) 211 | create_train_test_list_novel_split(grth_with_img_id_path, train_path, test_path) 212 | else: 213 | print 'FILE: ', train_path, ' already exists. Not create new! Delete old file if you want to re-run.' 214 | 215 | print '-------------- CREATE INDEX EVENT FILE --------------' 216 | index_event_img_file = scene_processed_folder + '/index_event_img.txt' 217 | if not os.path.exists(index_event_img_file): 218 | create_index_event_img_file(counter_event_grt_path, index_event_img_file) 219 | else: 220 | print 'FILE: ', index_event_img_file, ' already exists. Not create new! Delete old file if you want to re-run.' 221 | 222 | 223 | def create_images_from_events(list_percentage, scene_raw_folder, scene_processed_folder): 224 | 225 | main_percentage_folder = os.path.join(scene_processed_folder, 'percentage_img') 226 | if not os.path.exists(main_percentage_folder): 227 | os.makedirs(main_percentage_folder) 228 | 229 | 230 | in_count_events_gt_file = scene_processed_folder + '/count_events_gt.txt' 231 | in_raw_events_file = scene_raw_folder + '/events.txt' 232 | 233 | # create index file 234 | #index_event_img_file = '/home/anguyen/workspace/paper_src/2018.icra.event.source/event_data/processed/boxes_translation/index_event_img.txt' 235 | index_event_img_file = scene_processed_folder + '/index_event_img.txt' 236 | 237 | 238 | for kp in list_percentage: 239 | out_percentage_img_foler = os.path.join(main_percentage_folder, str(kp)) 240 | if not os.path.exists(out_percentage_img_foler): 241 | os.makedirs(out_percentage_img_foler) 242 | # only create if not exists 243 | create_percentage_image(kp, index_event_img_file, in_count_events_gt_file, in_raw_events_file, out_percentage_img_foler) 244 | else: 245 | print 'FOLDER: ', out_percentage_img_foler, ' already exists. SKIP!' 246 | 247 | 248 | def convert_images_to_pkl(list_percentage, scene_processed_folder): 249 | 250 | 251 | for keep_id in list_percentage: 252 | #image_event_folder = '/home/anguyen/workspace/paper_src/2018.icra.event.source/event_data/processed/shapes_rotation/percentage_img/10' 253 | image_event_folder = os.path.join(scene_processed_folder, 'percentage_img', str(keep_id)) 254 | 255 | out_folder = os.path.join(scene_processed_folder, 'percentage_pkl', str(keep_id)) 256 | if not os.path.exists(out_folder): 257 | os.makedirs(out_folder) 258 | # only create if not exists 259 | main_convert_percentage(scene_processed_folder, image_event_folder, out_folder, keep_id) 260 | else: 261 | print 'FOLDER: ', out_folder, ' already exists. SKIP!' 262 | 263 | 264 | def main(): 265 | ''' 266 | 0. Create folder structure 267 | 1. Create files: count_events_gt.txt, groundtruth_with_img_id.txt, index_event_img.txt, train.txt, test.txt 268 | 2. Create images from events 269 | 3. Convert list of images to 1 single .pkl file 270 | ''' 271 | 272 | 273 | list_scene = ['shapes_6dof'] 274 | #list_scene = ['poster_translation'] 275 | #list_scene = ['poster_6dof'] 276 | 277 | for ls in list_scene: 278 | 279 | #scene_id = 'boxes_translation' 280 | scene_id = ls 281 | 282 | 283 | #list_percentage = [10, 20, 30, 40, 50, 60, 70, 80, 90, 100] 284 | list_percentage = [100] 285 | 286 | # 0. Create folder structure 287 | scene_raw_folder, scene_processed_folder = create_folder_structure(scene_id, split_id) 288 | 289 | # 1. Create files 290 | create_files(scene_raw_folder, scene_processed_folder) 291 | 292 | # # 2. Create images from events 293 | create_images_from_events(list_percentage, scene_raw_folder, scene_processed_folder) 294 | # 295 | # # 3. Convert list of images to 1 single .pkl file 296 | convert_images_to_pkl(list_percentage, scene_processed_folder) 297 | 298 | 299 | 300 | if __name__ == '__main__': 301 | 302 | main() 303 | 304 | print 'ALL DONE!' -------------------------------------------------------------------------------- /dataset_script/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /dataset_script/__init__.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nqanh/pose_relocalization/0f2a2e125ec015c6fb70c1fa5d730954eaf8f416/dataset_script/__init__.pyc -------------------------------------------------------------------------------- /dataset_script/count_events_with_gt.py: -------------------------------------------------------------------------------- 1 | # count number of events occur until they have GT 2 | 3 | import os 4 | import numpy as np 5 | from distlib.util import in_venv 6 | 7 | 8 | def main_count_event_gt(in_event_path, in_grt_path, out_event_grt_path): 9 | # in_event_path = '/home/anguyen/workspace/dataset/Event/raw_data/shapes_rotation/events.txt' 10 | # in_grt_path = '/home/anguyen/workspace/dataset/Event/raw_data/shapes_rotation/groundtruth.txt' 11 | # out_event_grt_path = '/home/anguyen/workspace/dataset/Event/processed/shapes_rotation/count_events_gt.txt' 12 | 13 | # read file 14 | list_event = list(open(in_event_path, 'r')) 15 | list_grt = list(open(in_grt_path, 'r')) 16 | print 'total #events: ', len(list_event) 17 | print 'total #grt : ', len(list_grt) 18 | 19 | 20 | fwriter = open(out_event_grt_path, 'w') 21 | 22 | # start_index = 0 23 | # for ev in list_event: 24 | # ev = ev.replace('\n', '') 25 | # 26 | # ev_line = ev.split(' ') 27 | # ev_time_stamp = float(ev_line[0]) 28 | # 29 | # 30 | # 31 | # print 'event time: ', ev_time_stamp 32 | # 33 | # #t1 = 0.0 34 | # # find biggest time stamp in grt 35 | # for i in range(start_index, len(list_grt)): 36 | # gt_arr = list_grt[i].split(' ') 37 | # gt_time = float(gt_arr[0]) 38 | # 39 | # if ev_time_stamp < gt_time: 40 | # print 'found: ', list_grt[i] 41 | # out_line = ev + ' ' + gt_arr[1] + ' ' + gt_arr[2] + ' ' + gt_arr[3] + ' ' + gt_arr[4] + ' ' + gt_arr[5] + ' ' + gt_arr[6] + ' ' + gt_arr[7] # + '\n' 42 | # fwriter.write(out_line) 43 | # 44 | # start_index = i # set index for next loop - ignore the pass 45 | # 46 | # break # important!!! 47 | 48 | 49 | 50 | start_index = 0 ## index for looping all events - jump over for faster 51 | #start_index = 112355460 # DEBUG 52 | 53 | list_counter = np.zeros((len(list_grt), ), dtype=np.int) ## count number of events in each gt 54 | 55 | 56 | 57 | total_events = len(list_event) 58 | # get last ev_time index 59 | #last_ev_time = list_event[total_events-1].split(' ')[0] 60 | #print 'last ev_time: ', last_ev_time 61 | 62 | #start_gt = 11947 # DEBUG 63 | 64 | #for ind, gt in enumerate(list_grt): 65 | #for gt_ind in range(start_gt, len(list_grt)): 66 | for gt_ind in range(len(list_grt)): 67 | # gt_arr = gt.split(' ') 68 | # gt_time = float(gt_arr[0]) 69 | gt_arr = list_grt[gt_ind].split(' ') 70 | gt_time = float(gt_arr[0]) 71 | 72 | print '-----------------------------' 73 | # print 'start index: ', start_index 74 | # print 'list_vent[start_index]', list_event[start_index] 75 | 76 | for i in range(start_index, total_events): 77 | # if start_index >= len(list_event): # in "boxes_translation", some last grt don't have any events 78 | # print 'FOUND start_index >= len(list_event ', '-- start_index=', start_index, '-- len(list_event)=', len(list_event) 79 | # list_counter[ind] = 0 80 | # break 81 | 82 | ev_arr = list_event[i].split(' ') 83 | ev_time = float(ev_arr[0]) 84 | #print 'ev_time: ', ev_time 85 | 86 | # if ev_time == last_ev_time: # reach to the last event --> set start index to the final list 87 | # start_index = total_events 88 | # break 89 | if ev_time < gt_time: 90 | #list_counter[ind] += 1 ## found 1 events with time < gt 91 | list_counter[gt_ind] += 1 ## found 1 events with time < gt 92 | if start_index + list_counter[gt_ind] == total_events: 93 | print 'REACH FINAL EVENT LINE' 94 | start_index = total_events 95 | break 96 | 97 | else: 98 | start_index = i ## for next loop 99 | break 100 | 101 | 102 | #print 'ind: ', ind 103 | print 'ind: ', gt_ind 104 | #print 'gt : ', gt 105 | print 'gt: ', list_grt[gt_ind] 106 | #print 'list_coutner[ind]: ', list_counter[ind] 107 | print 'list_counter[ind]: ', list_counter[gt_ind] 108 | print '-----------------------------' 109 | 110 | #break # for testing 111 | 112 | print 'list counter: ', list_counter 113 | print 'max counter : ', max(list_counter) 114 | 115 | for ct in list_counter: 116 | fwriter.write(str(ct) + '\n') 117 | 118 | #fwriter.close() 119 | 120 | if __name__ == "__main__": 121 | 122 | #main_count_event_gt() 123 | in_event_file = '/home/anguyen/workspace/dataset/Event/raw_data/boxes_translation/events.txt' 124 | list_events = list(open(in_event_file, 'r')) 125 | 126 | print 'total events: ', len(list_events) 127 | 128 | 129 | 130 | print 'ALL DONE!' -------------------------------------------------------------------------------- /dataset_script/count_events_with_gt.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nqanh/pose_relocalization/0f2a2e125ec015c6fb70c1fa5d730954eaf8f416/dataset_script/count_events_with_gt.pyc -------------------------------------------------------------------------------- /dataset_script/data_io_cnn.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pickle 3 | import numpy as np 4 | import random 5 | import cv2 6 | 7 | from keras.preprocessing import image 8 | from dataset_script.events_to_img import padzero 9 | 10 | random.seed(99999) 11 | 12 | def load_data(main_path, pkl_file): 13 | try: 14 | print 'Loading pickle file ....', os.path.join(main_path, pkl_file) 15 | X, Y = pickle.load(open(os.path.join(main_path, pkl_file), 'rb')) 16 | print '........................ done!' 17 | except Exception: 18 | print 'ERROR: File ', pkl_file, 'does not exist!' 19 | 20 | return X, Y 21 | 22 | 23 | def write_data(list_data, img_folder, out_folder, pkl_file): 24 | X, Y = [], [] 25 | 26 | for l in list_data: 27 | l = l.rstrip('\n') 28 | gt_arr = l.split(' ') 29 | 30 | # Y 31 | px = float(gt_arr[1]) 32 | py = float(gt_arr[2]) 33 | pz = float(gt_arr[3]) 34 | qx = float(gt_arr[4]) 35 | qy = float(gt_arr[5]) 36 | qz = float(gt_arr[6]) 37 | qw = float(gt_arr[7]) 38 | current_y = [px, py, pz, qx, qy, qz, qw] 39 | Y.append(current_y) 40 | 41 | # print debug 42 | 43 | # X 44 | img_id = gt_arr[8] 45 | img_path = os.path.join(img_folder, img_id) 46 | X.append(img_path) 47 | 48 | print '--------------------------------' 49 | print 'current l: ', l 50 | print 'current y: ', current_y 51 | print 'img id: ', img_id 52 | 53 | 54 | # process Image 55 | for ind, val in enumerate(X): 56 | 57 | img = image.load_img(val, target_size=(224, 224)) ## change all pixel values 58 | X[ind] = image.img_to_array(img) 59 | 60 | # TODO: zero mean? 61 | X[ind] /= 255. # fix NAN loss problem with X[ind] /= 255. 62 | 63 | # print 'X[ind] shape: ', X[ind].shape 64 | # cv2.imshow('img', X[ind]) 65 | # cv2.waitKey(0) 66 | 67 | print 'Process input images: ', ind, '/', len(list_data), ' -- img path: ', val 68 | 69 | #break 70 | 71 | # convert to numpy array ## memory leak??? 72 | #X = np.array(X) 73 | #Y = np.array(Y) 74 | 75 | #save to pickle file for next time 76 | print 'Saving pickle file ... ', os.path.join(out_folder, pkl_file), '... done!' 77 | pickle.dump((X, Y), open(os.path.join(out_folder, pkl_file), 'wb')) 78 | 79 | 80 | 81 | def save_txt(list_data, out_folder, txt_file): 82 | file_path = os.path.join(out_folder, txt_file) 83 | fwriter = open(file_path, 'w') 84 | for l in list_data: 85 | fwriter.write(l) 86 | 87 | fwriter.close() 88 | 89 | def add_img_id_to_gt(in_grth_path, grth_with_img_id_path): 90 | #in_grth_path = '/home/anguyen/workspace/dataset/Event/raw_data/shapes_rotation/groundtruth.txt' 91 | 92 | list_grth = list(open(in_grth_path, 'r')) 93 | print 'len grth: ', len(list_grth) 94 | 95 | #grth_with_img_id_path = '/home/anguyen/workspace/dataset/Event/processed/shapes_rotation/groundtruth_with_img_id.txt' 96 | fwriter = open(grth_with_img_id_path, 'w') 97 | 98 | for i in range(len(list_grth)): 99 | gt_line = list_grth[i].rstrip('\n') 100 | gt_line = gt_line + ' ' + padzero(i) + '.png' + '\n' 101 | print 'gt line: ', gt_line 102 | fwriter.write(gt_line) 103 | #break 104 | #fwriter.close() 105 | 106 | def main_create_train_test(grth_with_img_id_path, img_folder, out_folder, is_small_500_set): 107 | 108 | #grth_with_img_id_path = '/home/anguyen/workspace/dataset/Event/processed/shapes_rotation/groundtruth_with_img_id.txt' 109 | list_grth_id = list(open(grth_with_img_id_path, 'r')) 110 | 111 | ## for testing 112 | if is_small_500_set: 113 | #list_grth_id = list_grth_id[0:1000] 114 | #list_grth_id = list_grth_id[0:100] 115 | print 'GET ONLY 500 FIRST IMAGE!' 116 | list_grth_id = list_grth_id[0:500] 117 | 118 | # shuffle list 119 | random.shuffle(list_grth_id) 120 | 121 | num_train = int(0.7 * len(list_grth_id)) 122 | num_test = len(list_grth_id) - num_train 123 | print 'total train sample: ', num_train 124 | print 'total test sample: ', num_test 125 | print list_grth_id[0] 126 | 127 | # slice first num_train sample as training 128 | list_train = list_grth_id[0:num_train] 129 | print 'len list train: ', len(list_train) 130 | 131 | list_test = list(set(list_grth_id) - set(list_train)) 132 | print 'len list test: ', len(list_test) 133 | 134 | 135 | #img_folder = '/home/anguyen/workspace/dataset/Event/processed/shapes_rotation/event_img' 136 | #out_folder = '/home/anguyen/workspace/dataset/Event/processed/shapes_rotation' 137 | 138 | train_pkl_file = 'train.pkl' 139 | test_pkl_file = 'test.pkl' 140 | 141 | 142 | #train_pkl_file = 'train_cnn_all.pkl' 143 | #test_pkl_file = 'test_cnn_all.pkl' 144 | 145 | #train_pkl_file = 'train_cnn_small.pkl' 146 | #test_pkl_file = 'test_cnn_small.pkl' 147 | 148 | #train_pkl_file = 'train_cnn_small_100.pkl' 149 | #test_pkl_file = 'test_cnn_small_100.pkl' 150 | 151 | #train_pkl_file = 'train_cnn_small_500.pkl' 152 | #test_pkl_file = 'test_cnn_small_500.pkl' 153 | 154 | # save to txt file 155 | train_txt_file = train_pkl_file.replace('.pkl', '.txt') 156 | test_txt_file = test_pkl_file.replace('.pkl', '.txt') 157 | save_txt(list_train, out_folder, train_txt_file) 158 | save_txt(list_test, out_folder, test_txt_file) 159 | 160 | # save to pkl file 161 | write_data(list_train, img_folder, out_folder, train_pkl_file) 162 | write_data(list_test, img_folder, out_folder, test_pkl_file) 163 | 164 | 165 | 166 | 167 | def main_create_percentage_test(grth_with_img_id_path, img_folder, out_folder): 168 | ''' 169 | create test.pkl for sub percentage image folder 170 | ''' 171 | 172 | #grth_with_img_id_path = '/home/anguyen/workspace/dataset/Event/processed/shapes_rotation/groundtruth_with_img_id.txt' 173 | list_grth_id = list(open(grth_with_img_id_path, 'r')) 174 | 175 | # ## for testing 176 | # if is_small_500_set: 177 | # #list_grth_id = list_grth_id[0:1000] 178 | # #list_grth_id = list_grth_id[0:100] 179 | # print 'GET ONLY 500 FIRST IMAGE!' 180 | # list_grth_id = list_grth_id[0:500] 181 | 182 | # shuffle list 183 | random.shuffle(list_grth_id) 184 | 185 | num_train = int(0.7 * len(list_grth_id)) 186 | num_test = len(list_grth_id) - num_train 187 | print 'total train sample: ', num_train 188 | print 'total test sample: ', num_test 189 | print list_grth_id[0] 190 | 191 | # slice first num_train sample as training 192 | list_train = list_grth_id[0:num_train] 193 | print 'len list train: ', len(list_train) 194 | 195 | list_test = list(set(list_grth_id) - set(list_train)) 196 | print 'len list test: ', len(list_test) 197 | 198 | 199 | #img_folder = '/home/anguyen/workspace/dataset/Event/processed/shapes_rotation/event_img' 200 | #out_folder = '/home/anguyen/workspace/dataset/Event/processed/shapes_rotation' 201 | 202 | train_pkl_file = 'train.pkl' 203 | test_pkl_file = 'test.pkl' 204 | 205 | 206 | #train_pkl_file = 'train_cnn_all.pkl' 207 | #test_pkl_file = 'test_cnn_all.pkl' 208 | 209 | #train_pkl_file = 'train_cnn_small.pkl' 210 | #test_pkl_file = 'test_cnn_small.pkl' 211 | 212 | #train_pkl_file = 'train_cnn_small_100.pkl' 213 | #test_pkl_file = 'test_cnn_small_100.pkl' 214 | 215 | #train_pkl_file = 'train_cnn_small_500.pkl' 216 | #test_pkl_file = 'test_cnn_small_500.pkl' 217 | 218 | # save to txt file 219 | train_txt_file = train_pkl_file.replace('.pkl', '.txt') 220 | test_txt_file = test_pkl_file.replace('.pkl', '.txt') 221 | save_txt(list_train, out_folder, train_txt_file) 222 | save_txt(list_test, out_folder, test_txt_file) 223 | 224 | # save to pkl file 225 | #write_data(list_train, img_folder, out_folder, train_pkl_file) ## dont need to create train.pkl 226 | write_data(list_test, img_folder, out_folder, test_pkl_file) 227 | 228 | 229 | 230 | def main_convert_percentage(scene_processed_folder, img_folder, out_folder, keep_id): 231 | ''' 232 | create test.pkl for sub percentage image folder 233 | create train.pkl when keep_id is 100 --> use all events 234 | ''' 235 | 236 | train_txt_path = scene_processed_folder + '/train.txt' 237 | test_txt_path = scene_processed_folder + '/test.txt' 238 | 239 | list_train = list(open(train_txt_path, 'r')) 240 | list_test = list(open(test_txt_path, 'r')) 241 | 242 | # save to pkl file 243 | train_pkl_file = 'train.pkl' 244 | test_pkl_file = 'test.pkl' 245 | if keep_id == 100: 246 | write_data(list_train, img_folder, out_folder, train_pkl_file) 247 | 248 | write_data(list_test, img_folder, out_folder, test_pkl_file) 249 | 250 | 251 | 252 | if __name__ == '__main__': 253 | 254 | grth_with_img_id_path = '/home/anguyen/workspace/dataset/Event/processed/shapes_rotation/groundtruth_with_img_id.txt' 255 | list_grth_id = list(open(grth_with_img_id_path, 'r')) 256 | 257 | ## for testing 258 | #list_grth_id = list_grth_id[0:1000] 259 | #list_grth_id = list_grth_id[0:100] 260 | #list_grth_id = list_grth_id[0:500] 261 | 262 | # shuffle list 263 | random.shuffle(list_grth_id) 264 | 265 | num_train = int(0.7 * len(list_grth_id)) 266 | num_test = len(list_grth_id) - num_train 267 | print 'total train sample: ', num_train 268 | print 'total test sample: ', num_test 269 | print list_grth_id[0] 270 | 271 | # slice first num_train sample as training 272 | list_train = list_grth_id[0:num_train] 273 | print 'len list train: ', len(list_train) 274 | 275 | list_test = list(set(list_grth_id) - set(list_train)) 276 | print 'len list test: ', len(list_test) 277 | 278 | 279 | img_folder = '/home/anguyen/workspace/dataset/Event/processed/shapes_rotation/event_img' 280 | out_folder = '/home/anguyen/workspace/dataset/Event/processed/shapes_rotation' 281 | 282 | train_pkl_file = 'train_cnn_all.pkl' 283 | test_pkl_file = 'test_cnn_all.pkl' 284 | 285 | #train_pkl_file = 'train_cnn_small.pkl' 286 | #test_pkl_file = 'test_cnn_small.pkl' 287 | 288 | #train_pkl_file = 'train_cnn_small_100.pkl' 289 | #test_pkl_file = 'test_cnn_small_100.pkl' 290 | 291 | #train_pkl_file = 'train_cnn_small_500.pkl' 292 | #test_pkl_file = 'test_cnn_small_500.pkl' 293 | 294 | # save to txt file 295 | train_txt_file = train_pkl_file.replace('.pkl', '.txt') 296 | test_txt_file = test_pkl_file.replace('.pkl', '.txt') 297 | save_txt(list_train, out_folder, train_txt_file) 298 | save_txt(list_test, out_folder, test_txt_file) 299 | 300 | # save to pkl file 301 | write_data(list_train, img_folder, out_folder, train_pkl_file) 302 | write_data(list_test, img_folder, out_folder, test_pkl_file) 303 | 304 | 305 | 306 | print 'ALL DONE!' 307 | 308 | -------------------------------------------------------------------------------- /dataset_script/data_io_cnn.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nqanh/pose_relocalization/0f2a2e125ec015c6fb70c1fa5d730954eaf8f416/dataset_script/data_io_cnn.pyc -------------------------------------------------------------------------------- /dataset_script/events_to_img.py: -------------------------------------------------------------------------------- 1 | # transform list of events to images 2 | 3 | import os 4 | import numpy as np 5 | import scipy.misc as spm 6 | 7 | import cv2 8 | 9 | # set print all 10 | #np.set_printoptions(threshold=np.nan) 11 | 12 | 13 | 14 | def padzero(x): 15 | if x>=0 and x<10: 16 | return '0000000' + str(x) 17 | elif x>=10 and x<100: 18 | return '000000' + str(x) 19 | elif x>=100 and x<1000: 20 | return '00000' + str(x) 21 | elif x>=1000 and x<10000: 22 | return '0000' + str(x) 23 | elif x>=10000 and x<100000: 24 | return '000' + str(x) 25 | elif x>=100000 and x<1000000: 26 | return '00' + str(x) 27 | elif x>=1000000 and x<10000000: 28 | return '0' + str(x) 29 | else: 30 | return '___error___' 31 | 32 | def create_empty_img(): 33 | img = np.full((180, 240), 0.5) 34 | img[0][0] = 1.0 # enable 1 pixel just fix scipy imsave problem 35 | img[1][1] = 0.0 36 | 37 | return img 38 | 39 | def convert_event_to_img(start_index, end_index, list_event): 40 | #print '-- current start_index: ', start_index 41 | #print '-- current end_index: ', end_index 42 | 43 | #if (start_index == end_index or start_index == end_index + 1): 44 | if (start_index == end_index or end_index == start_index + 1): 45 | print 'Warning: found start index = end_index (+1). No events? - return empty image' 46 | img = create_empty_img() 47 | return img 48 | 49 | 50 | #img = np.zeros((240, 180)) 51 | img = np.full((180, 240), 0.5) # create one channel - (row = y, column = x) 52 | #print 'img: ', img 53 | #print 'img shape: ', img.shape 54 | #img = np.dstack((img, img, img)) ## stack to three channels here???? 55 | #print 'img shape: ', img.shape 56 | 57 | for i in range(start_index, end_index): 58 | #print 'i=', i 59 | ev_line = list_event[i].rstrip('\n') 60 | #print 'event line: ', ev_line 61 | ev_arr = ev_line.split(' ') 62 | ex = int(ev_arr[1]) 63 | ey = int(ev_arr[2]) 64 | ep = float(ev_arr[3]) 65 | 66 | img[ey][ex] = ep # row=y, col=x 67 | 68 | 69 | #print 'img: ', img 70 | 71 | # save to folder 72 | 73 | 74 | return img 75 | 76 | 77 | def main_event_to_image(in_event_path, in_counter_path, out_event_img_folder): 78 | # in_event_path = '/home/anguyen/workspace/dataset/Event/raw_data/shapes_rotation/events.txt' 79 | # in_counter_path = '/home/anguyen/workspace/dataset/Event/processed/shapes_rotation/count_events_gt.txt' 80 | # out_event_img_folder = '/home/anguyen/workspace/dataset/Event/processed/shapes_rotation/event_img' 81 | 82 | 83 | list_event = list(open(in_event_path, 'r')) 84 | list_counter = list(open(in_counter_path, 'r')) 85 | 86 | print 'num events: ', len(list_event) 87 | print 'num coutner: ', len(list_counter) 88 | 89 | # 1st image is empty - no previous events 90 | img = create_empty_img() 91 | fname = padzero(0) + '.png' 92 | spm.imsave(os.path.join(out_event_img_folder, fname), img) # save 1 chanel as greyscale 93 | 94 | # create an image from previous events 95 | total_event = 0 96 | start_index = 0 97 | for i in range(len(list_counter)-1): 98 | end_index = total_event + int(list_counter[i]) 99 | 100 | print '-----------------------' 101 | print 'i: ', i 102 | print 'start index: ', start_index 103 | print 'end index : ', end_index 104 | 105 | img = convert_event_to_img(start_index, end_index, list_event) 106 | #cv2.imshow('img', img) 107 | #cv2.waitKey(0) 108 | 109 | # save image 110 | fname = padzero(i+1) + '.png' 111 | spm.imsave(os.path.join(out_event_img_folder, fname), img) # save 1 chanel as greyscale 112 | 113 | 114 | # update new start index 115 | start_index = end_index 116 | total_event += int(list_counter[i]) 117 | 118 | # if i == 61: 119 | # break 120 | 121 | 122 | if __name__ == "__main__": 123 | 124 | #main_event_to_image() 125 | 126 | # # 1st image is empty - no previous events 127 | # img = create_empty_img() 128 | # fname = padzero(0) + '.png' 129 | # spm.imsave(os.path.join(out_event_img_folder, fname), img) # save 1 chanel as greyscale 130 | # 131 | # # create an image from previous events 132 | # total_event = 0 133 | # start_index = 0 134 | # for i in range(len(list_counter)-1): 135 | # end_index = total_event + int(list_counter[i]) 136 | # 137 | # print '-----------------------' 138 | # print 'i: ', i 139 | # print 'start index: ', start_index 140 | # print 'end index : ', end_index 141 | # 142 | # img = convert_event_to_img(start_index, end_index) 143 | # #cv2.imshow('img', img) 144 | # #cv2.waitKey(0) 145 | # 146 | # # save image 147 | # fname = padzero(i+1) + '.png' 148 | # spm.imsave(os.path.join(out_event_img_folder, fname), img) # save 1 chanel as greyscale 149 | # 150 | # 151 | # # update new start index 152 | # start_index = end_index 153 | # total_event += int(list_counter[i]) 154 | # 155 | # # if i == 61: 156 | # # break 157 | 158 | 159 | 160 | print 'ALL DONE!' -------------------------------------------------------------------------------- /dataset_script/events_to_img.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nqanh/pose_relocalization/0f2a2e125ec015c6fb70c1fa5d730954eaf8f416/dataset_script/events_to_img.pyc -------------------------------------------------------------------------------- /dataset_script/merge_event_groundtruth.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import sys 4 | 5 | 6 | in_event_path = '/home/anguyen/workspace/dataset/Event/raw_data/shapes_rotation/events.txt' 7 | in_grt_path = '/home/anguyen/workspace/dataset/Event/raw_data/shapes_rotation/groundtruth.txt' 8 | 9 | out_event_grt_path = '/home/anguyen/workspace/dataset/Event/processed/shapes_rotation/event_grt.txt' 10 | 11 | # read file 12 | list_event = list(open(in_event_path, 'r')) 13 | list_grt = list(open(in_grt_path, 'r')) 14 | print 'total #events: ', len(list_event) 15 | print 'total #grt : ', len(list_grt) 16 | 17 | list_egt = list(open(out_event_grt_path, 'r')) 18 | print 'total #egt : ', len(list_egt) 19 | 20 | 21 | # fwriter = open(out_event_grt_path, 'w') 22 | # 23 | # start_index = 0 24 | # for ev in list_event: 25 | # ev = ev.replace('\n', '') 26 | # 27 | # ev_line = ev.split(' ') 28 | # ev_time_stamp = float(ev_line[0]) 29 | # 30 | # 31 | # 32 | # print 'event time: ', ev_time_stamp 33 | # 34 | # #t1 = 0.0 35 | # # find biggest time stamp in grt 36 | # for i in range(start_index, len(list_grt)): 37 | # gt_arr = list_grt[i].split(' ') 38 | # gt_time = float(gt_arr[0]) 39 | # 40 | # if ev_time_stamp < gt_time: 41 | # print 'found: ', list_grt[i] 42 | # out_line = ev + ' ' + gt_arr[1] + ' ' + gt_arr[2] + ' ' + gt_arr[3] + ' ' + gt_arr[4] + ' ' + gt_arr[5] + ' ' + gt_arr[6] + ' ' + gt_arr[7] # + '\n' 43 | # fwriter.write(out_line) 44 | # 45 | # start_index = i # set index for next loop - ignore the pass 46 | # 47 | # break # important!!! 48 | 49 | 50 | 51 | 52 | print 'ALL DONE!' 53 | 54 | 55 | -------------------------------------------------------------------------------- /main_keras/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /main_keras/__init__.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nqanh/pose_relocalization/0f2a2e125ec015c6fb70c1fa5d730954eaf8f416/main_keras/__init__.pyc -------------------------------------------------------------------------------- /main_keras/predict.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import time 4 | import math 5 | from keras.models import load_model 6 | 7 | 8 | 9 | import sys 10 | 11 | cwd = os.getcwd() 12 | #print 'current dir: ', cwd 13 | root_path = os.path.abspath(os.path.join(cwd, os.pardir)) # get parent path 14 | #print 'root path: ', root_path 15 | sys.path.insert(0, root_path) 16 | 17 | from dataset_script.data_io_cnn import load_data 18 | 19 | 20 | #split_id = 'img_pose_500' 21 | #split_id = 'img_pose_all' 22 | split_id = 'img_pose_all_novel_split' 23 | 24 | def predict_pose(in_scene_id, in_net_id, in_model_file_name): 25 | ''' 26 | only predict, not evaluate results (run evaluate.py script to evaluate) 27 | ''' 28 | 29 | main_output = os.path.join(root_path, 'output') 30 | #scene_id = 'shapes_rotation' 31 | #scene_id = 'boxes_translation' 32 | scene_id = in_scene_id 33 | 34 | net_id = in_net_id 35 | 36 | model_file_name = in_model_file_name 37 | 38 | log_model = 'log_model' 39 | predition = 'prediction' 40 | 41 | # save prediction result to file 42 | predition_path = os.path.join(main_output, scene_id, net_id, split_id, predition) 43 | prediction_file = os.path.join(predition_path, 'prediction.pre') 44 | 45 | # load test data 46 | #dataset_folder = '/home/anguyen/workspace/dataset/Event/processed/' 47 | dataset_folder = os.path.join(root_path, 'event_data', 'processed') 48 | data_path = os.path.join(dataset_folder, scene_id, split_id) 49 | testX, testY = load_data(data_path, 'test.pkl') 50 | # convert to numpy array 51 | testX = np.array(testX) 52 | testY = np.array(testY) 53 | 54 | # load trained model 55 | log_model_path = os.path.join(main_output, scene_id, net_id, split_id, log_model) 56 | model_file = os.path.join(log_model_path, model_file_name) 57 | 58 | trained_model = load_model(model_file) 59 | 60 | predicted_result = trained_model.predict(testX) 61 | 62 | print 'writing predicted result to file ...' 63 | fwriter = open(prediction_file, 'w') 64 | for p in predicted_result: 65 | out_line = str(p[0]) + ' ' + str(p[1]) + ' ' + str(p[2]) + ' ' + str(p[3]) + ' ' + str(p[4]) + ' ' + str(p[5]) + ' ' + str(p[6]) + '\n' 66 | print out_line 67 | fwriter.write(out_line) 68 | 69 | 70 | def predict_and_evaluate(in_scene_id, in_net_id, in_model_file_name): 71 | ''' 72 | do prediction and evaluation for main results (use all events) 73 | ''' 74 | 75 | main_output = os.path.join(root_path, 'output') 76 | #scene_id = 'shapes_rotation' 77 | #scene_id = 'boxes_translation' 78 | scene_id = in_scene_id 79 | 80 | 81 | #net_id = 'vgg_lstm2' 82 | #net_id = 'vgg' 83 | #net_id = 'vgg_conv_lstm2' 84 | #net_id = 'vgg_lsltm2_no_dense' 85 | net_id = in_net_id 86 | 87 | #split_id = 'img_pose_500' 88 | #split_id = 'img_pose_all' 89 | 90 | #model_file_name = 'full_model_epoch_e600.hdf5' 91 | #model_file_name = 'full_model_epoch_e1000.hdf5' 92 | #model_file_name = 'full_model_epoch_e1200.hdf5' 93 | model_file_name = in_model_file_name 94 | 95 | log_model = 'log_model' 96 | predition = 'prediction' 97 | 98 | # save prediction result to file 99 | predition_path = os.path.join(main_output, scene_id, net_id, split_id, predition) 100 | prediction_file = os.path.join(predition_path, 'prediction.pre') 101 | 102 | # load test data 103 | #dataset_folder = '/home/anguyen/workspace/dataset/Event/processed/' 104 | dataset_folder = os.path.join(root_path, 'event_data', 'processed') 105 | data_path = os.path.join(dataset_folder, scene_id, split_id) 106 | testX, testY = load_data(data_path, 'test.pkl') 107 | # convert to numpy array 108 | testX = np.array(testX) 109 | testY = np.array(testY) 110 | 111 | # load trained model 112 | log_model_path = os.path.join(main_output, scene_id, net_id, split_id, log_model) 113 | model_file = os.path.join(log_model_path, model_file_name) 114 | 115 | trained_model = load_model(model_file) 116 | 117 | start_time = time.time() 118 | 119 | # run prediction 120 | predicted_result = trained_model.predict(testX) 121 | 122 | elapsed_time = time.time() - start_time 123 | print 'Predict: ', len(testX), ' event images in: ', elapsed_time, ' seconds --> ', (elapsed_time*1000)/len(testX), 'miliseconds per image' 124 | 125 | print 'Writing results to file ...' 126 | fwriter = open(prediction_file, 'w') 127 | for p in predicted_result: 128 | out_line = str(p[0]) + ' ' + str(p[1]) + ' ' + str(p[2]) + ' ' + str(p[3]) + ' ' + str(p[4]) + ' ' + str(p[5]) + ' ' + str(p[6]) + '\n' 129 | #print out_line 130 | fwriter.write(out_line) 131 | 132 | 133 | # evaluate result 134 | # initilize 'results' to store all results 135 | results = np.zeros((len(predicted_result),2)) 136 | i = 0 137 | 138 | # statistic 139 | counter_good_pred = 0 # count #image has < 0.05m AND 2deg accuracy 140 | 141 | #for line_grth, line_pred in zip(grth_content, pred_content): 142 | for line_pred, line_grth in zip(predicted_result, testY): 143 | # parse content to use PoseNet code: https://github.com/alexgkendall/caffe-posenet/blob/master/posenet/scripts/test_posenet.py 144 | pose_x = [line_grth[0], line_grth[1], line_grth[2]] 145 | pose_q = [line_grth[3], line_grth[4], line_grth[5], line_grth[6]] 146 | 147 | predicted_x = [line_pred[0], line_pred[1], line_pred[2]] 148 | predicted_q = [line_pred[3], line_pred[4], line_pred[5], line_pred[6]] 149 | 150 | # print '-------------------------------------------' 151 | # print 'i : ', i 152 | # print 'pose x : ', pose_x 153 | # print 'pose q : ', pose_q 154 | # print 'Predicted x : ', predicted_x 155 | # print 'Predicted q : ', predicted_q 156 | 157 | 158 | 159 | # convert to numpy 160 | pose_q = np.array(pose_q) 161 | pose_x = np.array(pose_x) 162 | predicted_q = np.array(predicted_q) 163 | predicted_x = np.array(predicted_x) 164 | 165 | #Compute Individual Sample Error 166 | q1 = pose_q / np.linalg.norm(pose_q) 167 | q2 = predicted_q / np.linalg.norm(predicted_q) 168 | d = abs(np.sum(np.multiply(q1,q2))) 169 | theta = 2 * np.arccos(d) * 180/math.pi 170 | error_x = np.linalg.norm(pose_x-predicted_x) 171 | 172 | results[i,:] = [error_x,theta] 173 | 174 | if error_x < 0.08 and theta < 4.0: 175 | counter_good_pred+=1 176 | 177 | i = i + 1 178 | #print 'Error XYZ (m): ', error_x, ' Error Q (degrees): ', theta, '\n' 179 | 180 | median_result = np.median(results,axis=0) 181 | median_error = 'Median_error ' + str(median_result[0]) + ' m and ' + str(median_result[1]) + ' degrees' 182 | print median_error 183 | 184 | average_result = np.average(results, axis=0) 185 | average_error = 'Average_error ' + str(average_result[0]) + ' m and ' + str(average_result[1]) + ' degrees' 186 | print average_error 187 | 188 | acc_statistic = (float(counter_good_pred)/float(len(testY))) * 100 189 | print 'Total good prediction: ', counter_good_pred 190 | print 'Accuracy: ', acc_statistic 191 | 192 | # save error results to file 193 | error_all_path = os.path.join(main_output, scene_id, net_id, split_id, predition, 'error_all.txt') 194 | error_summary_path = os.path.join(main_output, scene_id, net_id, split_id, predition, 'error_summary.txt') 195 | 196 | np.savetxt(error_all_path, results, delimiter=' ', fmt='%02.20f') 197 | 198 | fout = open(error_summary_path, 'w') 199 | #fout.write(median_error + '\n') 200 | #fout.write(average_error + '\n') 201 | fout.write('scene_id median_error_met average_error_met median_error_deg average_error_deg accuracy_statistic\n') 202 | fout.write(scene_id + ' ' + str(median_result[0]) + ' ' + str(average_result[0]) + ' ' + str(median_result[1]) + ' ' + str(average_result[1]) + ' ' + str(acc_statistic)) 203 | 204 | 205 | 206 | 207 | def predict_and_evaluate_percentage_img(in_scene_id, in_net_id, trained_model, current_percentage): 208 | ''' 209 | do prediction and evaluation for percentage images 210 | ''' 211 | 212 | main_output = os.path.join(root_path, 'output') 213 | #scene_id = 'shapes_rotation' 214 | #scene_id = 'boxes_translation' 215 | scene_id = in_scene_id 216 | 217 | 218 | #net_id = 'vgg_lstm2' 219 | #net_id = 'vgg' 220 | #net_id = 'vgg_conv_lstm2' 221 | #net_id = 'vgg_lsltm2_no_dense' 222 | net_id = in_net_id 223 | 224 | 225 | 226 | 227 | #predition = 'prediction' 228 | 229 | # out result paths 230 | percentage_path = os.path.join(main_output, scene_id, net_id, split_id, 'results_all', str(current_percentage)) 231 | # save prediction result to file 232 | #predition_path = os.path.join(main_output, scene_id, net_id, split_id, predition) 233 | #predition_path = os.path.join(percentage_path, predition) 234 | predition_path = percentage_path 235 | if not os.path.exists(predition_path): 236 | os.makedirs(predition_path) 237 | prediction_file = os.path.join(predition_path, 'prediction.pre') 238 | 239 | # load test data 240 | #dataset_folder = '/home/anguyen/workspace/dataset/Event/processed/' 241 | dataset_folder = os.path.join(root_path, 'event_data', 'processed') 242 | data_path = os.path.join(dataset_folder, scene_id, split_id, 'percentage_pkl', str(current_percentage)) 243 | #data_path = '/home/anguyen/workspace/paper_src/2018.icra.event.source/event_data/processed/shapes_rotation/img_pose_all/percentage_test/10' 244 | testX, testY = load_data(data_path, 'test.pkl') 245 | # convert to numpy array 246 | testX = np.array(testX) 247 | testY = np.array(testY) 248 | 249 | # # load trained model 250 | # log_model_path = os.path.join(main_output, scene_id, net_id, split_id, log_model) 251 | # model_file = os.path.join(log_model_path, model_file_name) 252 | # trained_model = load_model(model_file) 253 | 254 | start_time = time.time() 255 | 256 | # run prediction 257 | predicted_result = trained_model.predict(testX) 258 | 259 | elapsed_time = time.time() - start_time 260 | print 'Predict: ', len(testX), ' event images in: ', elapsed_time, ' seconds --> ', (elapsed_time*1000)/len(testX), 'miliseconds per image' 261 | 262 | print 'Writing results to file ...' 263 | fwriter = open(prediction_file, 'w') 264 | for p in predicted_result: 265 | out_line = str(p[0]) + ' ' + str(p[1]) + ' ' + str(p[2]) + ' ' + str(p[3]) + ' ' + str(p[4]) + ' ' + str(p[5]) + ' ' + str(p[6]) + '\n' 266 | #print out_line 267 | fwriter.write(out_line) 268 | 269 | 270 | # evaluate result 271 | # initilize 'results' to store all results 272 | results = np.zeros((len(predicted_result),2)) 273 | i = 0 274 | 275 | # statistic 276 | counter_good_pred = 0 # count #image has < 0.05m AND 2deg accuracy 277 | 278 | #for line_grth, line_pred in zip(grth_content, pred_content): 279 | for line_pred, line_grth in zip(predicted_result, testY): 280 | # parse content to use PoseNet code: https://github.com/alexgkendall/caffe-posenet/blob/master/posenet/scripts/test_posenet.py 281 | pose_x = [line_grth[0], line_grth[1], line_grth[2]] 282 | pose_q = [line_grth[3], line_grth[4], line_grth[5], line_grth[6]] 283 | 284 | predicted_x = [line_pred[0], line_pred[1], line_pred[2]] 285 | predicted_q = [line_pred[3], line_pred[4], line_pred[5], line_pred[6]] 286 | 287 | # print '-------------------------------------------' 288 | # print 'i : ', i 289 | # print 'pose x : ', pose_x 290 | # print 'pose q : ', pose_q 291 | # print 'Predicted x : ', predicted_x 292 | # print 'Predicted q : ', predicted_q 293 | 294 | 295 | 296 | # convert to numpy 297 | pose_q = np.array(pose_q) 298 | pose_x = np.array(pose_x) 299 | predicted_q = np.array(predicted_q) 300 | predicted_x = np.array(predicted_x) 301 | 302 | #Compute Individual Sample Error 303 | q1 = pose_q / np.linalg.norm(pose_q) 304 | q2 = predicted_q / np.linalg.norm(predicted_q) 305 | d = abs(np.sum(np.multiply(q1,q2))) 306 | 307 | # # fix NAN isues 308 | # if d>1.0: # d>1.0 --> arccoss(d) = nan 309 | # d=1.0 310 | 311 | theta = 2 * np.arccos(d) * 180/math.pi 312 | error_x = np.linalg.norm(pose_x-predicted_x) 313 | 314 | results[i,:] = [error_x,theta] 315 | 316 | # print 'd=', d 317 | # # CHECK IF theta is NAN 318 | # if math.isnan(theta): 319 | # print 'Found theta is NAN -- d=', d, '-- arccos(d)=', np.arccos(d) 320 | 321 | if error_x < 0.08 and theta < 4.0: 322 | counter_good_pred+=1 323 | 324 | i = i + 1 325 | #print 'Error XYZ (m): ', error_x, ' Error Q (degrees): ', theta, '\n' 326 | 327 | median_result = np.median(results,axis=0) 328 | median_error = 'Median_error ' + str(median_result[0]) + ' m and ' + str(median_result[1]) + ' degrees' 329 | print median_error 330 | 331 | average_result = np.average(results, axis=0) 332 | average_error = 'Average_error ' + str(average_result[0]) + ' m and ' + str(average_result[1]) + ' degrees' 333 | print average_error 334 | 335 | acc_statistic = (float(counter_good_pred)/float(len(testY))) * 100 336 | print 'Total good prediction: ', counter_good_pred 337 | print 'Accuracy: ', acc_statistic 338 | 339 | # save error results to file 340 | error_all_path = os.path.join(predition_path, 'error_all.txt') 341 | error_summary_path = os.path.join(predition_path, 'error_summary.txt') 342 | 343 | np.savetxt(error_all_path, results, delimiter=' ', fmt='%02.20f') 344 | 345 | fout = open(error_summary_path, 'w') 346 | #fout.write(median_error + '\n') 347 | #fout.write(average_error + '\n') 348 | fout.write('scene_id current_percentage median_error_met average_error_met median_error_deg average_error_deg accuracy_statistic\n') 349 | results_string = scene_id + ' lstm' + str(current_percentage) + ' '+ str(median_result[0]) + ' ' + str(average_result[0]) + ' ' + str(median_result[1]) + ' ' + str(average_result[1]) + ' ' + str(acc_statistic) 350 | #fout.write(scene_id + ' ' + str(median_result[0]) + ' ' + str(average_result[0]) + ' ' + str(median_result[1]) + ' ' + str(average_result[1]) + ' ' + str(acc_statistic)) 351 | fout.write(results_string) 352 | 353 | return results_string 354 | 355 | 356 | def main_predict_and_evaluate_percentage_img(in_scene_id, in_net_id, in_model_file_name): 357 | 358 | # load 1 model to test all list percentage 359 | #log_model_path = os.path.join(root_path, 'output', in_scene_id, in_net_id, 'img_pose_all', 'log_model') 360 | log_model_path = os.path.join(root_path, 'output', in_scene_id, in_net_id, split_id, 'log_model') 361 | model_file = os.path.join(log_model_path, in_model_file_name) 362 | trained_model = load_model(model_file) 363 | print 'Loading model: ', model_file, '...done!' 364 | 365 | all_results = [] 366 | 367 | 368 | #list_percentage = [10, 20, 30, 40, 50, 60, 70, 80, 90, 100] 369 | list_percentage = [100] 370 | 371 | for l in list_percentage: 372 | 373 | print '----------------------------------' 374 | current_percentage = l 375 | #current_percentage = 100 376 | results_string = predict_and_evaluate_percentage_img(in_scene_id, in_net_id, trained_model, current_percentage) 377 | print 'CURRENT RESULT STRING: ', results_string 378 | all_results.append(results_string) 379 | 380 | 381 | # write all_results to file 382 | #out_summary_all_folder = '/home/anguyen/workspace/paper_src/2018.icra.event.source/output/shapes_rotation/vgg_lstm2/img_pose_all' 383 | out_summary_all_folder = os.path.join(root_path, 'output', in_scene_id, in_net_id, split_id) 384 | out_summary_file = out_summary_all_folder + '/ALL_RESULTS.TXT' 385 | fsum = open(out_summary_file, 'w') 386 | 387 | for ar in all_results: 388 | fsum.write(ar + '\n') 389 | 390 | 391 | 392 | if __name__ == '__main__': 393 | 394 | in_scene_id = 'shapes_6dof' 395 | in_net_id = 'vgg_lstm2' 396 | 397 | in_model_file_name = 'full_model_epoch_e1000.hdf5' 398 | 399 | main_predict_and_evaluate_percentage_img(in_scene_id, in_net_id, in_model_file_name) 400 | 401 | 402 | print 'ALL DONE!' 403 | -------------------------------------------------------------------------------- /main_keras/spatial_lstm.py: -------------------------------------------------------------------------------- 1 | from keras.applications.vgg16 import VGG16 2 | from keras.preprocessing import image 3 | from keras.applications.vgg16 import preprocess_input, decode_predictions 4 | import numpy as np 5 | from keras.models import Model 6 | from keras.layers import Flatten, Dense, Concatenate 7 | import matplotlib.pyplot as plt 8 | 9 | import sys 10 | root_path = '/home/anguyen/workspace/paper_src/2018.icra.event.source' # not .source/dataset --> wrong folder 11 | sys.path.insert(0, root_path) 12 | 13 | from dataset.data_io_cnn import load_data 14 | from keras.callbacks import ModelCheckpoint, History 15 | 16 | 17 | import keras 18 | 19 | import os 20 | 21 | from keras.layers import Conv2D, MaxPooling2D, Flatten, Dropout 22 | from keras.layers import Input, LSTM, Embedding, Dense, TimeDistributed 23 | from keras.models import Model, Sequential 24 | 25 | from keras.utils import plot_model 26 | 27 | from dataset.data_io_cnn_imu import load_data 28 | from main_keras.vgg16 import VGG16 29 | 30 | 31 | if __name__ == '__main__': 32 | img_model = VGG16(None) 33 | #print 'image model shape: ', img_model.shape 34 | #print 'image model shape: ', img_model.output_shape # img_model is a MODEL not TENSOR 35 | #print 'model summary: ', img_model.summary() 36 | 37 | # Now let's get a tensor with the output of our vision model: 38 | img_input = Input(shape=(224, 224, 3)) 39 | encoded_img = img_model(img_input) # encoded_img is a TENSOR 40 | 41 | print 'encoded img shape: ', encoded_img.shape 42 | 43 | # try to add lstm 44 | for i in range(64): 45 | input = Input(shape=(64,1)) 46 | lstm = LSTM(256, input_shape=(64,1), name='row_' + str(i)) (input) 47 | 48 | 49 | print 'ALL DONE!' 50 | -------------------------------------------------------------------------------- /main_keras/train.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import matplotlib 4 | import matplotlib.pyplot as plt 5 | import argparse 6 | #from dataset_script.CREATE_DATA import split_id 7 | 8 | 9 | #force matplotlib use any Xwindows 10 | matplotlib.use('Agg') 11 | 12 | import sys 13 | cwd = os.getcwd() 14 | print 'current dir: ', cwd 15 | root_path = os.path.abspath(os.path.join(cwd, os.pardir)) # get parent path 16 | print 'root path: ', root_path 17 | sys.path.insert(0, root_path) 18 | 19 | import tensorflow as tf 20 | 21 | from dataset_script.data_io_cnn import load_data 22 | from main_keras.vgg16 import VGG_LSTM2 23 | 24 | 25 | ##split_id = 'img_pose_all' ## random split 26 | split_id = 'img_pose_all_novel_split' ## novel split 27 | 28 | 29 | def train_net(model, train_image, train_pose, log_model_path, log_plot_path): 30 | 31 | # loss function 32 | model.compile(loss="mean_squared_error", optimizer="sgd", lr=0.00001, decay=1e-6, momentum=0.9) # OK WITH VGG-LSTM2 33 | 34 | # REDUCE learning rate - NOT COVERAGE FASTER 35 | #model.compile(loss="mean_squared_error", optimizer="sgd", lr=0.000001, decay=1e-6, momentum=0.9) # OK WITH VGG-LSTM2 36 | 37 | # use for loop to fit 38 | num_epoch = 20000 39 | loss_history = [] 40 | val_history = [] 41 | for e in range(num_epoch): 42 | print 'EPOCH e =', e, '/', num_epoch 43 | train_history = model.fit(train_image, train_pose, batch_size=20, epochs=1, validation_split=0.05, verbose=1) #verbose control output print) 44 | #print '\n' 45 | 46 | if e > 0: # ignore first epochs 47 | loss_history.append(train_history.history['loss']) 48 | val_history.append(train_history.history['val_loss']) 49 | 50 | #if e > 5 and e % 500 == 0: # save model every 500 epochs 51 | if e > 0 and e % 50 == 0: # save model every 100 epochs 52 | model_file_name = 'full_model_epoch_e' + str(e) + '.hdf5' 53 | out_model_path = os.path.join(log_model_path, model_file_name) 54 | model.save(out_model_path) 55 | print 'saved model at epoch e =', e, 'ok!' 56 | 57 | if e > 0 and e % 10 == 0: # save plot loss 58 | # list all data in history 59 | print(train_history.history.keys()) 60 | # summarize history for loss 61 | plt.plot(loss_history, color='r') 62 | plt.plot(val_history, color='g') 63 | plt.title('model loss') 64 | plt.ylabel('loss') 65 | plt.xlabel('epoch') 66 | plt.legend(['train', 'val'], loc='upper right') 67 | 68 | plot_file_name = 'plot_epoch_e' + str(e) + '.png' 69 | out_plot_path = os.path.join(log_plot_path, plot_file_name) 70 | plt.savefig(out_plot_path) 71 | #plt.show() 72 | 73 | 74 | 75 | def parse_args(): 76 | """Parse input arguments.""" 77 | parser = argparse.ArgumentParser(description='Just another deepsh*t!') 78 | parser.add_argument('--gpu', dest='gpu_id', help='GPU device id to use - default is 0', 79 | default='0', type=str) 80 | parser.add_argument('--scene', dest='scene_id', 81 | help='Scene ID to train - default is `shapes_rotation` ', 82 | default='shapes_rotation', type=str) 83 | parser.add_argument('--weight', dest='weight_path', 84 | help='Pretrained weight - default is `None` ', 85 | default=None, type=str) 86 | 87 | args = parser.parse_args() 88 | 89 | return args 90 | 91 | if __name__ == '__main__': 92 | 93 | # parse arg 94 | args = parse_args() 95 | 96 | # config gpu 97 | gpu_id = args.gpu_id 98 | gpu_id_string = '/gpu:' + gpu_id 99 | #with tf.device('/gpu:0'): 100 | #with tf.device('/cpu:0'): 101 | with tf.device(gpu_id_string): 102 | print 'Using gpu: ', gpu_id_string 103 | 104 | pretrained_weight_path = args.weight_path 105 | if pretrained_weight_path != None: 106 | print 'USING PRETRAINED WEIGHT: ', pretrained_weight_path 107 | else: 108 | print 'TRAIN FROM SCRATCH' 109 | 110 | 111 | model = VGG_LSTM2(pretrained_weight_path) 112 | model.summary() 113 | 114 | main_output = os.path.join(root_path, 'output') 115 | 116 | # scene/sequence id 117 | #scene_id = 'shapes_rotation' 118 | scene_id = args.scene_id 119 | 120 | net_id = 'vgg_lstm2' 121 | 122 | 123 | out_net_id_path = os.path.join(main_output, scene_id, net_id) 124 | print 'out net id path: ', out_net_id_path 125 | 126 | # log training model, plot, etc. 127 | log_model = 'log_model' 128 | log_plot = 'log_plot' 129 | #predition = 'prediction' # not use 130 | 131 | log_model_path = os.path.join(main_output, scene_id, net_id, split_id, log_model) 132 | log_plot_path = os.path.join(main_output, scene_id, net_id, split_id, log_plot) 133 | #predition_path = os.path.join(main_output, scene_id, net_id, split_id, predition) 134 | if not os.path.exists(log_model_path): 135 | os.makedirs(log_model_path) 136 | if not os.path.exists(log_plot_path): 137 | os.makedirs(log_plot_path) 138 | #if not os.path.exists(predition_path): 139 | # os.makedirs(predition_path) 140 | 141 | # load data 142 | dataset_folder = os.path.join(root_path, 'event_data', 'processed' ) 143 | #data_path = os.path.join(dataset_folder, scene_id, split_id) 144 | data_path = os.path.join(dataset_folder, scene_id, split_id, 'percentage_pkl', '100') ## -->> always test with all events i.e. 100% 145 | #print 'data path: ', data_path 146 | train_image, train_pose = load_data(data_path, 'train.pkl') 147 | print 'convert to numpy ...' 148 | train_image = np.array(train_image) 149 | train_pose = np.array(train_pose) 150 | print 'train_image shape: ', train_image.shape 151 | print 'train_pose shape: ', train_pose.shape 152 | 153 | # train 154 | train_net(model, train_image, train_pose, log_model_path, log_plot_path) 155 | 156 | print 'ALL DONE!' 157 | -------------------------------------------------------------------------------- /main_keras/vgg16.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | cwd = os.getcwd() 5 | print 'current dir: ', cwd 6 | root_path = os.path.abspath(os.path.join(cwd, os.pardir)) # get parent path 7 | print 'root path: ', root_path 8 | sys.path.insert(0, root_path) 9 | 10 | 11 | from keras.layers import Conv2D, MaxPooling2D, Flatten, Dropout, Reshape, Input, GRU 12 | from keras.layers import Input, LSTM, Embedding, Dense, TimeDistributed 13 | from keras.models import Model, Sequential 14 | 15 | 16 | def VGG_LSTM2(weights_path=None): 17 | model = Sequential() 18 | # block 1 19 | model.add(Conv2D(64, (3, 3), activation='relu', padding='same', input_shape=(224, 224, 3))) 20 | model.add(Conv2D(64, (3, 3), activation='relu')) 21 | model.add(MaxPooling2D((2, 2))) 22 | 23 | # block 2 24 | model.add(Conv2D(128, (3, 3), activation='relu', padding='same')) 25 | model.add(Conv2D(128, (3, 3), activation='relu')) 26 | model.add(MaxPooling2D((2, 2))) 27 | 28 | # block 3 29 | model.add(Conv2D(256, (3, 3), activation='relu', padding='same')) 30 | model.add(Conv2D(256, (3, 3), activation='relu')) 31 | model.add(Conv2D(256, (3, 3), activation='relu')) 32 | model.add(MaxPooling2D((2, 2))) 33 | 34 | # block 4 35 | model.add(Conv2D(512, (3, 3), activation='relu', padding='same')) 36 | model.add(Conv2D(512, (3, 3), activation='relu')) 37 | model.add(Conv2D(512, (3, 3), activation='relu')) 38 | model.add(MaxPooling2D((2, 2))) 39 | 40 | # block 5 41 | model.add(Conv2D(512, (3, 3), activation='relu', padding='same')) 42 | model.add(Conv2D(512, (3, 3), activation='relu')) 43 | model.add(Conv2D(512, (3, 3), activation='relu')) 44 | model.add(MaxPooling2D((2, 2))) 45 | 46 | # final block 47 | model.add(Flatten()) 48 | model.add(Dense(4096, activation='relu')) 49 | model.add(Dense(4096, activation='relu')) 50 | model.add(Dropout(0.5)) 51 | 52 | # reshape 4096 = 64 * 64 53 | model.add(Reshape((64, 64))) 54 | 55 | # stack lstm 56 | model.add(LSTM(256, return_sequences=True, name='lstm1')) 57 | model.add(LSTM(256, return_sequences=False, name='lstm2')) 58 | 59 | model.add(Dense(512, activation='relu')) 60 | model.add(Dense(7)) # regress pose 61 | 62 | if weights_path: 63 | model.load_weights(weights_path) 64 | 65 | return model 66 | 67 | 68 | 69 | -------------------------------------------------------------------------------- /main_keras/vgg16.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nqanh/pose_relocalization/0f2a2e125ec015c6fb70c1fa5d730954eaf8f416/main_keras/vgg16.pyc --------------------------------------------------------------------------------