├── CMakeLists.txt ├── DL_Scripts ├── MNIST.py ├── basic.py ├── checkpoint ├── detect_ros.py ├── export_inference_graph.py ├── fusion.py ├── generate_tfrecord.py ├── image_recognition.py ├── lidar_image_pub.py ├── multi_sensor_fusion.py ├── object_detection.py ├── pointcloud.py ├── radar_publisher _segment.py ├── radar_publisher.py ├── ros_fusion.py ├── ros_multi_sensor_fusion.py ├── save_model.py ├── test.py ├── train.py └── xml_to_csv.py ├── README.md ├── Results ├── Screenshot from 2020-06-04 12-10-31.png ├── Screenshot from 2020-06-22 15-14-01.png ├── VID-20200623-WA0008.mp4 └── output.gif ├── bag └── loop_bag.launch ├── include └── nuscenes2bag │ ├── DatasetTypes.hpp │ ├── EgoPoseConverter.hpp │ ├── FileProgress.hpp │ ├── Filesystem.hpp │ ├── ImageDirectoryConverter.hpp │ ├── LidarDirectoryConverter.hpp │ ├── LidarDirectoryConverterXYZIR.hpp │ ├── MetaDataProvider.hpp │ ├── MetaDataReader.hpp │ ├── MetaDataTypes.hpp │ ├── NuScenes2Bag.hpp │ ├── PclRadarObject.hpp │ ├── RadarDirectoryConverter.hpp │ ├── RunEvery.hpp │ ├── SceneConverter.hpp │ ├── ToDebugString.hpp │ ├── thread_pool.hpp │ └── utils.hpp ├── msg ├── RadarObject.msg └── RadarObjects.msg ├── output_inference_graph_v2 ├── checkpoint ├── frozen_inference_graph.pb ├── model.ckpt.data-00000-of-00001 ├── model.ckpt.index ├── model.ckpt.meta ├── pipeline.config ├── saved_model │ └── saved_model.pb └── v1.config ├── package.xml ├── src ├── EgoPoseConverter.cpp ├── FileProgress.cpp ├── ImageDirectoryConverter.cpp ├── LidarDirectoryConverter.cpp ├── LidarDirectoryConverterXYZIR.cpp ├── MetaData.cpp ├── MetaDataReader.cpp ├── NuScenes2Bag.cpp ├── RadarDirectoryConverter.cpp ├── SceneConverter.cpp ├── main.cpp ├── pcd_to_image.cpp ├── pointcloud_im.cpp └── utils.cpp └── thirdparty └── json └── nlohmann └── json.hpp /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | 2 | cmake_minimum_required(VERSION 3.8) # C++17 support was added in CMake 3.8 3 | 4 | project(nuscenes2bag) 5 | 6 | #set(CMAKE_CXX_STANDARD 11) 7 | set(CMAKE_CXX_STANDARD 14) 8 | 9 | set(CMAKE_BUILD_TYPE Release) # Debug, Release, RelWithDebInfo 10 | 11 | find_package(Boost 1.58.0 COMPONENTS filesystem thread REQUIRED) # Ubuntu 16.04 12 | #find_package(Boost 1.65.0 COMPONENTS filesystem thread REQUIRED) # Ubuntu 18.04 13 | 14 | find_package(catkin 15 | REQUIRED 16 | rosbag 17 | sensor_msgs 18 | cv_bridge 19 | pcl_conversions 20 | pcl_msgs 21 | message_generation 22 | geometry_msgs 23 | std_msgs 24 | nav_msgs 25 | tf) 26 | 27 | find_package(OpenCV REQUIRED core) 28 | 29 | add_message_files(FILES RadarObjects.msg RadarObject.msg) 30 | 31 | generate_messages(DEPENDENCIES 32 | std_msgs 33 | geometry_msgs 34 | tf 35 | nav_msgs) 36 | 37 | catkin_package(INCLUDE_DIRS 38 | include 39 | thirdparty/json/ 40 | DEPENDS 41 | OpenCV) 42 | 43 | include_directories(SYSTEM 44 | thirdparty/json/ 45 | ${catkin_INCLUDE_DIRS} 46 | ${Boost_INCLUDE_DIRS} 47 | ${OpenCV_INCLUDE_DIRS}) 48 | include_directories(include) 49 | 50 | find_package(PCL REQUIRED COMPONENTS common io) 51 | 52 | set(SRCS 53 | src/EgoPoseConverter.cpp 54 | src/ImageDirectoryConverter.cpp 55 | src/LidarDirectoryConverter.cpp 56 | src/LidarDirectoryConverterXYZIR.cpp 57 | src/RadarDirectoryConverter.cpp 58 | src/NuScenes2Bag.cpp 59 | src/FileProgress.cpp 60 | src/MetaDataReader.cpp 61 | src/MetaData.cpp 62 | src/SceneConverter.cpp 63 | src/utils.cpp) 64 | 65 | add_executable(${PROJECT_NAME} ${SRCS} src/main.cpp) 66 | 67 | target_compile_options(${PROJECT_NAME} 68 | PRIVATE 69 | -Wall 70 | -Wextra 71 | -pedantic 72 | ) 73 | 74 | target_compile_definitions(${PROJECT_NAME} PRIVATE "-DCMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD}") 75 | 76 | add_dependencies(${PROJECT_NAME} ${${PROJECT_NAME}_EXPORTED_TARGETS} 77 | ${catkin_EXPORTED_TARGETS}) 78 | 79 | target_link_libraries(${PROJECT_NAME} 80 | ${OpenCV_LIBRARIES} 81 | ${PCL_COMMON_LIBRARY} 82 | ${PCL_IO_LIBRARY} 83 | ${catkin_LIBRARIES}) 84 | 85 | install(DIRECTORY include/${PROJECT_NAME}/ thirdparty/json/ 86 | DESTINATION ${CATKIN_PACKAGE_INCLUDE_DESTINATION} 87 | FILES_MATCHING 88 | PATTERN "*.h" 89 | PATTERN ".svn" EXCLUDE) 90 | -------------------------------------------------------------------------------- /DL_Scripts/MNIST.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | tf.compat.v1.enable_eager_execution( 4 | config=None, device_policy=None, execution_mode=None 5 | ) 6 | mnist = tf.keras.datasets.mnist 7 | 8 | (x_train, y_train), (x_test, y_test) = mnist.load_data() 9 | x_train, x_test = x_train / 255.0, x_test / 255.0 10 | model = tf.keras.models.Sequential([ 11 | tf.keras.layers.Flatten(input_shape=(28, 28)), 12 | tf.keras.layers.Dense(128, activation='relu'), 13 | tf.keras.layers.Dropout(0.2), 14 | tf.keras.layers.Dense(10) 15 | ]) 16 | 17 | predictions = model(x_train[:1]).numpy() 18 | predictions 19 | 20 | tf.nn.softmax(predictions).numpy() 21 | 22 | loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) 23 | loss_fn(y_train[:1], predictions).numpy() 24 | model.compile(optimizer='adam', 25 | loss=loss_fn, 26 | metrics=['accuracy']) 27 | 28 | model.fit(x_train, y_train, epochs=5) 29 | model.evaluate(x_test, y_test, verbose=2) 30 | 31 | probability_model = tf.keras.Sequential([model,tf.keras.layers.Softmax()]) 32 | probability_model(x_test[:5]) 33 | -------------------------------------------------------------------------------- /DL_Scripts/basic.py: -------------------------------------------------------------------------------- 1 | #The most basic DL model ever 2 | import tensorflow as tf 3 | import numpy as np 4 | from tensorflow import keras 5 | 6 | model = tf.keras.Sequential([keras.layers.Dense(units=1, input_shape=[1])]) 7 | 8 | model.compile(optimizer='sgd', loss='mean_squared_error') 9 | 10 | xs = np.array([-1.0, 0.0, 1.0, 2.0, 3.0, 4.0], dtype=float) 11 | ys = np.array([-3.0, -1.0, 1.0, 3.0, 5.0, 7.0], dtype=float) 12 | 13 | model.fit(xs, ys, epochs=500) 14 | print(model.predict([10.0])) 15 | 16 | -------------------------------------------------------------------------------- /DL_Scripts/checkpoint: -------------------------------------------------------------------------------- 1 | model_checkpoint_path: "model.ckpt-10696" 2 | all_model_checkpoint_paths: "model.ckpt-10517" 3 | all_model_checkpoint_paths: "model.ckpt-10562" 4 | all_model_checkpoint_paths: "model.ckpt-10606" 5 | all_model_checkpoint_paths: "model.ckpt-10651" 6 | all_model_checkpoint_paths: "model.ckpt-10696" 7 | -------------------------------------------------------------------------------- /DL_Scripts/detect_ros.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os 4 | import sys 5 | import cv2 6 | import cv_bridge 7 | import numpy as np 8 | import time 9 | try: 10 | import tensorflow as tf 11 | except ImportError: 12 | print("unable to import TensorFlow. Is it installed?") 13 | sys.exit(1) 14 | 15 | # ROS related imports 16 | import rospy 17 | from std_msgs.msg import String , Header 18 | from sensor_msgs.msg import Image 19 | from cv_bridge import CvBridge, CvBridgeError 20 | from vision_msgs.msg import Detection2D, Detection2DArray, ObjectHypothesisWithPose 21 | from PIL import Image as img 22 | 23 | # Object detection module imports 24 | import object_detection 25 | from object_detection.utils import label_map_util 26 | from object_detection.utils import visualization_utils as vis_util 27 | from collections import defaultdict 28 | from io import StringIO 29 | from matplotlib import pyplot as plt 30 | import cv2 31 | from IPython.display import display 32 | # SET FRACTION OF GPU YOU WANT TO USE HERE 33 | GPU_FRACTION = 0.4 34 | 35 | ######### Set model here ############ 36 | MODEL_NAME = 'output_inference_graph_v2' 37 | # Path to frozen detection graph. This is the actual model that is used for the object detection. 38 | PATH_TO_CKPT = MODEL_NAME + '/frozen_inference_graph.pb' 39 | # List of the strings that is used to add correct label for each box. 40 | PATH_TO_LABELS = os.path.join('annotations','label_map.pbtxt') 41 | NUM_CLASSES = 2 42 | 43 | detection_graph = tf.Graph() 44 | with detection_graph.as_default(): 45 | od_graph_def = tf.GraphDef() 46 | with tf.gfile.GFile(PATH_TO_CKPT, 'rb') as fid: 47 | serialized_graph = fid.read() 48 | od_graph_def.ParseFromString(serialized_graph) 49 | tf.import_graph_def(od_graph_def, name='') 50 | 51 | ## Loading label map 52 | # Label maps map indices to category names, so that when our convolution network predicts `5`, 53 | # we know that this corresponds to `airplane`. Here we use internal utility functions, 54 | # but anything that returns a dictionary mapping integers to appropriate string labels would be fine 55 | label_map = label_map_util.load_labelmap(PATH_TO_LABELS) 56 | categories = label_map_util.convert_label_map_to_categories(label_map, max_num_classes=NUM_CLASSES, use_display_name=True) 57 | category_index = label_map_util.create_category_index(categories) 58 | 59 | # Setting the GPU options to use fraction of gpu that has been set 60 | config = tf.ConfigProto() 61 | config.gpu_options.per_process_gpu_memory_fraction = GPU_FRACTION 62 | 63 | # Detection 64 | 65 | class Detector: 66 | 67 | def __init__(self): 68 | self.object_pub = rospy.Publisher("objects", Detection2DArray, queue_size=1) 69 | self.bridge = CvBridge() 70 | self.image_sub = rospy.Subscriber("image_lidar", Image, self.image_cb) 71 | self.sess = tf.Session(graph=detection_graph,config=config) 72 | #self.front_image_sub=rospy.Subscriber("cam_front/raw",Image,self.front_image_cb) 73 | #self.img_array=np.zeros((450,300)) 74 | self.front_image_sub=rospy.Subscriber("debug_image",Image,self.front_image_cb) 75 | self.img_array=np.zeros((640,480)) 76 | 77 | def front_image_cb(self,data): 78 | self.img_array=cv2.resize(self.bridge.imgmsg_to_cv2(data),(640,480)) 79 | #self.img_array=cv2.resize(self.bridge.imgmsg_to_cv2(data),(450,300)) 80 | time.sleep(0.2) 81 | #cv2.imshow("window",self.img_array) 82 | #cv2.waitKey(1) 83 | 84 | def drawBoundingBox(self,imgcv,array): 85 | cv2.rectangle(imgcv,(int(array[1]),250),(int(array[0]),320),(0,255,0),1) 86 | #cv2.rectangle(imgcv,(_x1,_y1),(_x2,_y2),(0,255,0),cv2.FILLED) 87 | #cv2.putText(imgcv,label,(x1,y1),cv2.FONT_HERSHEY_COMPLEX,0.5,(0,0,0),1) 88 | #cv2.imshow("window",imgcv) 89 | #cv2.waitKey(1) 90 | 91 | def image_cb(self, data): 92 | objArray = Detection2DArray() 93 | try: 94 | cv_image = self.bridge.imgmsg_to_cv2(data) 95 | except CvBridgeError as e: 96 | print(e) 97 | # the array based representation of the image will be used later in order to prepare the 98 | # result image with boxes and labels on it. 99 | image=cv_image[:,:,0] 100 | mod_image=np.zeros((600,600,3)) 101 | mod_image[:, :, 0]=image 102 | mod_image[:, :, 1]=image 103 | mod_image[:, :, 2]=image 104 | image_np = mod_image.astype(np.uint8) 105 | # Expand dimensions since the model expects images to have shape: [1, None, None, 3] 106 | image_np_expanded = np.expand_dims(image_np, axis=0) 107 | image_tensor = detection_graph.get_tensor_by_name('image_tensor:0') 108 | # Each box represents a part of the image where a particular object was detected. 109 | boxes = detection_graph.get_tensor_by_name('detection_boxes:0') 110 | # Each score represent how level of confidence for each of the objects. 111 | # Score is shown on the result image, together with the class label. 112 | scores = detection_graph.get_tensor_by_name('detection_scores:0') 113 | classes = detection_graph.get_tensor_by_name('detection_classes:0') 114 | num_detections = detection_graph.get_tensor_by_name('num_detections:0') 115 | (boxes, scores, classes, num_detections) = self.sess.run([boxes, scores, classes, num_detections], 116 | feed_dict={image_tensor: image_np_expanded}) 117 | objects,array=vis_util.visualize_boxes_and_labels_on_image_array( 118 | image_np, 119 | np.squeeze(boxes), 120 | np.squeeze(classes).astype(np.int32), 121 | np.squeeze(scores), 122 | category_index, 123 | use_normalized_coordinates=True, 124 | line_thickness=2) 125 | if(array!=None): 126 | array[0]=array[0]*640#450 127 | array[1]=array[1]*640#450 128 | array[2]=array[2]*480#300 129 | array[3]=array[3]*480#300 130 | print(array) 131 | self.drawBoundingBox(self.img_array,array) 132 | #display(img.fromarray(image_np)) 133 | #cv2.imshow("window",image_np)) 134 | h1, w1 = image_np.shape[:2] 135 | h2, w2 = self.img_array.shape[:2] 136 | #create empty matrix 137 | vis = np.zeros((max(h1, h2), w1+w2,3), np.uint8) 138 | #combine 2 images 139 | vis[:h1,:w1,:3] = image_np 140 | vis[:h2,w1:w1+w2,:3] =self.img_array 141 | cv2.imshow("window",vis) 142 | cv2.waitKey(1) 143 | 144 | if __name__=='__main__': 145 | rospy.init_node('detector_node') 146 | obj=Detector() 147 | try: 148 | rospy.spin() 149 | except KeyboardInterrupt: 150 | print("ShutDown") 151 | cv2.destroyAllWindows() 152 | -------------------------------------------------------------------------------- /DL_Scripts/export_inference_graph.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | r"""Tool to export an object detection model for inference. 17 | Prepares an object detection tensorflow graph for inference using model 18 | configuration and a trained checkpoint. Outputs inference 19 | graph, associated checkpoint files, a frozen inference graph and a 20 | SavedModel (https://tensorflow.github.io/serving/serving_basic.html). 21 | The inference graph contains one of three input nodes depending on the user 22 | specified option. 23 | * `image_tensor`: Accepts a uint8 4-D tensor of shape [None, None, None, 3] 24 | * `encoded_image_string_tensor`: Accepts a 1-D string tensor of shape [None] 25 | containing encoded PNG or JPEG images. Image resolutions are expected to be 26 | the same if more than 1 image is provided. 27 | * `tf_example`: Accepts a 1-D string tensor of shape [None] containing 28 | serialized TFExample protos. Image resolutions are expected to be the same 29 | if more than 1 image is provided. 30 | and the following output nodes returned by the model.postprocess(..): 31 | * `num_detections`: Outputs float32 tensors of the form [batch] 32 | that specifies the number of valid boxes per image in the batch. 33 | * `detection_boxes`: Outputs float32 tensors of the form 34 | [batch, num_boxes, 4] containing detected boxes. 35 | * `detection_scores`: Outputs float32 tensors of the form 36 | [batch, num_boxes] containing class scores for the detections. 37 | * `detection_classes`: Outputs float32 tensors of the form 38 | [batch, num_boxes] containing classes for the detections. 39 | * `raw_detection_boxes`: Outputs float32 tensors of the form 40 | [batch, raw_num_boxes, 4] containing detection boxes without 41 | post-processing. 42 | * `raw_detection_scores`: Outputs float32 tensors of the form 43 | [batch, raw_num_boxes, num_classes_with_background] containing class score 44 | logits for raw detection boxes. 45 | * `detection_masks`: Outputs float32 tensors of the form 46 | [batch, num_boxes, mask_height, mask_width] containing predicted instance 47 | masks for each box if its present in the dictionary of postprocessed 48 | tensors returned by the model. 49 | Notes: 50 | * This tool uses `use_moving_averages` from eval_config to decide which 51 | weights to freeze. 52 | Example Usage: 53 | -------------- 54 | python export_inference_graph \ 55 | --input_type image_tensor \ 56 | --pipeline_config_path path/to/ssd_inception_v2.config \ 57 | --trained_checkpoint_prefix path/to/model.ckpt \ 58 | --output_directory path/to/exported_model_directory 59 | The expected output would be in the directory 60 | path/to/exported_model_directory (which is created if it does not exist) 61 | with contents: 62 | - inference_graph.pbtxt 63 | - model.ckpt.data-00000-of-00001 64 | - model.ckpt.info 65 | - model.ckpt.meta 66 | - frozen_inference_graph.pb 67 | + saved_model (a directory) 68 | Config overrides (see the `config_override` flag) are text protobufs 69 | (also of type pipeline_pb2.TrainEvalPipelineConfig) which are used to override 70 | certain fields in the provided pipeline_config_path. These are useful for 71 | making small changes to the inference graph that differ from the training or 72 | eval config. 73 | Example Usage (in which we change the second stage post-processing score 74 | threshold to be 0.5): 75 | python export_inference_graph \ 76 | --input_type image_tensor \ 77 | --pipeline_config_path path/to/ssd_inception_v2.config \ 78 | --trained_checkpoint_prefix path/to/model.ckpt \ 79 | --output_directory path/to/exported_model_directory \ 80 | --config_override " \ 81 | model{ \ 82 | faster_rcnn { \ 83 | second_stage_post_processing { \ 84 | batch_non_max_suppression { \ 85 | score_threshold: 0.5 \ 86 | } \ 87 | } \ 88 | } \ 89 | }" 90 | """ 91 | import tensorflow as tf 92 | from google.protobuf import text_format 93 | from object_detection import exporter 94 | from object_detection.protos import pipeline_pb2 95 | 96 | slim = tf.contrib.slim 97 | flags = tf.app.flags 98 | 99 | flags.DEFINE_string('input_type', 'image_tensor', 'Type of input node. Can be ' 100 | 'one of [`image_tensor`, `encoded_image_string_tensor`, ' 101 | '`tf_example`]') 102 | flags.DEFINE_string('input_shape', None, 103 | 'If input_type is `image_tensor`, this can explicitly set ' 104 | 'the shape of this input tensor to a fixed size. The ' 105 | 'dimensions are to be provided as a comma-separated list ' 106 | 'of integers. A value of -1 can be used for unknown ' 107 | 'dimensions. If not specified, for an `image_tensor, the ' 108 | 'default shape will be partially specified as ' 109 | '`[None, None, None, 3]`.') 110 | flags.DEFINE_string('pipeline_config_path', None, 111 | 'Path to a pipeline_pb2.TrainEvalPipelineConfig config ' 112 | 'file.') 113 | flags.DEFINE_string('trained_checkpoint_prefix', None, 114 | 'Path to trained checkpoint, typically of the form ' 115 | 'path/to/model.ckpt') 116 | flags.DEFINE_string('output_directory', None, 'Path to write outputs.') 117 | flags.DEFINE_string('config_override', '', 118 | 'pipeline_pb2.TrainEvalPipelineConfig ' 119 | 'text proto to override pipeline_config_path.') 120 | flags.DEFINE_boolean('write_inference_graph', False, 121 | 'If true, writes inference graph to disk.') 122 | tf.app.flags.mark_flag_as_required('pipeline_config_path') 123 | tf.app.flags.mark_flag_as_required('trained_checkpoint_prefix') 124 | tf.app.flags.mark_flag_as_required('output_directory') 125 | FLAGS = flags.FLAGS 126 | 127 | 128 | def main(_): 129 | pipeline_config = pipeline_pb2.TrainEvalPipelineConfig() 130 | with tf.gfile.GFile(FLAGS.pipeline_config_path, 'r') as f: 131 | text_format.Merge(f.read(), pipeline_config) 132 | text_format.Merge(FLAGS.config_override, pipeline_config) 133 | if FLAGS.input_shape: 134 | input_shape = [ 135 | int(dim) if dim != '-1' else None 136 | for dim in FLAGS.input_shape.split(',') 137 | ] 138 | else: 139 | input_shape = None 140 | exporter.export_inference_graph( 141 | FLAGS.input_type, pipeline_config, FLAGS.trained_checkpoint_prefix, 142 | FLAGS.output_directory, input_shape=input_shape, 143 | write_inference_graph=FLAGS.write_inference_graph) 144 | 145 | 146 | if __name__ == '__main__': 147 | tf.app.run() 148 | -------------------------------------------------------------------------------- /DL_Scripts/fusion.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | from tensorflow import keras 4 | from tensorflow.keras import layers 5 | print(tf.__version__) 6 | from tensorflow.keras.models import Sequential,Model 7 | from tensorflow.keras.layers import Dense,Conv2D, Flatten, Dropout, MaxPooling2D,concatenate 8 | from tensorflow.keras.preprocessing.image import ImageDataGenerator 9 | import cv2 10 | import os 11 | import matplotlib.pyplot as plt 12 | import datetime 13 | 14 | tf.keras.backend.clear_session() 15 | 16 | IMG_HEIGHT=150 17 | IMG_WIDTH=150 18 | input_image=keras.Input(shape=(IMG_HEIGHT,IMG_WIDTH,3)) 19 | input_lidar=keras.Input(shape=(IMG_HEIGHT,IMG_WIDTH,3)) 20 | 21 | class JoinedGenerator(keras.utils.Sequence): 22 | def __init__(self, generator1, generator2): 23 | self.generator1 = generator1 24 | self.generator2 = generator2 25 | 26 | def __len__(self): 27 | return len(self.generator1) 28 | 29 | def __getitem__(self, i): 30 | x1, y1 = self.generator1[i] 31 | x2, y2 = self.generator2[i] 32 | return [x1, x2], y1 33 | 34 | def on_epoch_end(self): 35 | self.generator1.on_epoch_end() 36 | self.generator2.on_epoch_end() 37 | 38 | datagen_lidar = ImageDataGenerator() 39 | # load and iterate training dataset 40 | train_lidar = datagen_lidar.flow_from_directory('data_lidar/train/', target_size=(IMG_HEIGHT, IMG_WIDTH),class_mode='binary', batch_size=2) 41 | # load and iterate validation dataset 42 | val_lidar = datagen_lidar.flow_from_directory('data_lidar/validation/', target_size=(IMG_HEIGHT, IMG_WIDTH),class_mode='binary', batch_size=2) 43 | # load and iterate test dataset 44 | test_lidar = datagen_lidar.flow_from_directory('data_lidar/test/', target_size=(IMG_HEIGHT, IMG_WIDTH),class_mode='binary', batch_size=2) 45 | 46 | datagen = ImageDataGenerator() 47 | # load and iterate training dataset 48 | train_it = datagen.flow_from_directory('data/train/', target_size=(IMG_HEIGHT, IMG_WIDTH),class_mode='binary', batch_size=2) 49 | # load and iterate validation dataset 50 | val_it = datagen.flow_from_directory('data/validation/', target_size=(IMG_HEIGHT, IMG_WIDTH),class_mode='binary', batch_size=2) 51 | # load and iterate test dataset 52 | test_it = datagen.flow_from_directory('data/test/', target_size=(IMG_HEIGHT, IMG_WIDTH),class_mode='binary', batch_size=2) 53 | 54 | x=Conv2D(16, 3, padding='same', activation='relu')(input_image) 55 | output_image=MaxPooling2D()(x) 56 | 57 | model_image = keras.Model(inputs=input_image,outputs=output_image, name='image_input') 58 | model_image.summary() 59 | 60 | y=Conv2D(16, 3, padding='same', activation='relu')(input_lidar) 61 | output_lidar=MaxPooling2D()(y) 62 | 63 | model_lidar = keras.Model(inputs=input_lidar,outputs=output_lidar, name='lidar_input') 64 | model_lidar.summary() 65 | 66 | combined = concatenate([model_image.output, model_lidar.output]) 67 | 68 | z=Conv2D(32, 3, padding='same', activation='relu')(combined) 69 | z=MaxPooling2D()(z) 70 | z=Conv2D(64, 3, padding='same', activation='relu')(z) 71 | z=MaxPooling2D()(z) 72 | z=Conv2D(16, 3, padding='same', activation='relu')(z) 73 | z=MaxPooling2D()(z) 74 | z=Flatten()(z) 75 | z=Dense(512, activation='relu')(z) 76 | z=Dense(128, activation='relu')(z) 77 | z = Dense(1, activation="linear")(z) 78 | fusion = Model(inputs=[model_lidar.input, model_image.input], outputs=z) 79 | #keras.utils.plot_model(fusion, 'fusion.png', show_shapes=True) 80 | 81 | fusion.compile(optimizer='adam',loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),metrics=['accuracy']) 82 | 83 | checkpoint_path = "training_1/cp_fusion.ckpt" 84 | checkpoint_dir = os.path.dirname(checkpoint_path) 85 | 86 | # # Create a callback that saves the model's weights 87 | cp_callback = tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_path, 88 | save_weights_only=True, 89 | verbose=1) 90 | 91 | log_dir = "logs/fit/" + datetime.datetime.now().strftime("%Y%m%d-%H%M%S") 92 | tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1) 93 | 94 | training_generator = JoinedGenerator(train_it, train_lidar) 95 | validation_generator = JoinedGenerator(val_it, val_lidar) 96 | test_generator = JoinedGenerator(test_it, test_lidar) 97 | history = fusion.fit_generator( 98 | training_generator, 99 | steps_per_epoch=40, 100 | epochs=10, 101 | validation_data=validation_generator, 102 | validation_steps=10, 103 | callbacks=[cp_callback]#tensorboard_callback] 104 | ) 105 | loss= fusion.evaluate_generator(test_generator, steps=4) 106 | fusion.save_weights('./checkpoints/my_checkpoint') 107 | 108 | image_mod=cv2.imread("1.jpg") 109 | image_mod=cv2.resize(image_mod,(150,150)) 110 | 111 | image_mod_lidar=cv2.imread("3.jpg") 112 | image_mod_lidar=cv2.resize(image_mod_lidar,(150,150)) 113 | val=fusion.predict([image_mod.reshape(-1,150,150,3),image_mod_lidar.reshape(-1,150,150,3)]) 114 | print(val) 115 | if(val[0][0]<0): 116 | print("car") 117 | else: 118 | print("pedestrian") 119 | 120 | image_mod=cv2.imread("2.jpg") 121 | image_mod=cv2.resize(image_mod,(150,150)) 122 | 123 | image_mod_lidar=cv2.imread("5.jpg") 124 | image_mod_lidar=cv2.resize(image_mod_lidar,(150,150)) 125 | val=fusion.predict([image_mod.reshape(-1,150,150,3),image_mod_lidar.reshape(-1,150,150,3)]) 126 | print(val) 127 | if(val[0][0]<0): 128 | print("car") 129 | else: 130 | print("pedestrian") 131 | -------------------------------------------------------------------------------- /DL_Scripts/generate_tfrecord.py: -------------------------------------------------------------------------------- 1 | 2 | """ 3 | Usage: 4 | # Create train data: 5 | python generate_tfrecord.py --label=