├── .gitattributes ├── .gitignore ├── README.md ├── calibration_pickle.p ├── car_detection.py ├── object_detection.ipynb ├── output_vid ├── lane_tracker.mp4 ├── obj_detection.mp4 └── project_video.mp4 ├── test.py ├── test_img ├── bbox-example-image.jpg ├── test1.jpg ├── test2.jpg └── test3.jpg ├── tracker.py └── video_gen.py /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | non-vehicles/ 2 | vehicles/ 3 | __pycache__/ 4 | .ipynb_checkpoints/ 5 | project.ipynb 6 | helper_functions.py 7 | lane.mp4 8 | 9 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Advanced Lane Detection and Object Detection 2 | 3 | Check out video_gen.py for the Lane Detection code and object_detection.ipynb for the actual object detection code. 4 | 5 | Oh, and I even wrote an article on this breaking the topic down. https://srianumakonda.medium.com/pairing-lane-detection-with-object-detection-665b30462952. 6 | 7 | Have fun! 8 | -------------------------------------------------------------------------------- /calibration_pickle.p: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/srianumakonda/Advanced-Lane-Detection-and-Object-Detection/4d24d437bfdcdbbe02bf7a496eeb65b142289ae8/calibration_pickle.p -------------------------------------------------------------------------------- /car_detection.py: -------------------------------------------------------------------------------- 1 | import matplotlib.image as mpimg 2 | import matplotlib.pyplot as plt 3 | import numpy as np 4 | import pickle 5 | import cv2 6 | from scipy.ndimage.measurements import label 7 | from moviepy.editor import VideoFileClip 8 | from IPython.display import HTML 9 | from skimage.feature import hog 10 | 11 | 12 | def convert_color(img, conv='RGB2YCrCb'): 13 | if conv == 'RGB2YCrCb': 14 | return cv2.cvtColor(img, cv2.COLOR_RGB2YCrCb) 15 | if conv == 'BGR2YCrCb': 16 | return cv2.cvtColor(img, cv2.COLOR_BGR2YCrCb) 17 | if conv == 'RGB2LUV': 18 | return cv2.cvtColor(img, cv2.COLOR_RGB2LUV) 19 | 20 | def get_hog_features(img, orient, pix_per_cell, cell_per_block, 21 | vis=False, feature_vec=True): 22 | # Call with two outputs if vis==True 23 | if vis == True: 24 | features, hog_image = hog(img, orientations=orient, 25 | pixels_per_cell=(pix_per_cell, pix_per_cell), 26 | cells_per_block=(cell_per_block, cell_per_block), 27 | block_norm= 'L2-Hys', 28 | transform_sqrt=False, 29 | visualise=vis, feature_vector=feature_vec) 30 | return features, hog_image 31 | # Otherwise call with one output 32 | else: 33 | features = hog(img, orientations=orient, 34 | pixels_per_cell=(pix_per_cell, pix_per_cell), 35 | cells_per_block=(cell_per_block, cell_per_block), 36 | block_norm= 'L2-Hys', 37 | transform_sqrt=False, 38 | visualize=vis, feature_vector=feature_vec) 39 | return features 40 | 41 | def bin_spatial(img, size=(32, 32)): 42 | color1 = cv2.resize(img[:,:,0], size).ravel() 43 | color2 = cv2.resize(img[:,:,1], size).ravel() 44 | color3 = cv2.resize(img[:,:,2], size).ravel() 45 | return np.hstack((color1, color2, color3)) 46 | 47 | def color_hist(img, nbins=32): #bins_range=(0, 256) 48 | # Compute the histogram of the color channels separately 49 | channel1_hist = np.histogram(img[:,:,0], bins=nbins) 50 | channel2_hist = np.histogram(img[:,:,1], bins=nbins) 51 | channel3_hist = np.histogram(img[:,:,2], bins=nbins) 52 | # Concatenate the histograms into a single feature vector 53 | hist_features = np.concatenate((channel1_hist[0], channel2_hist[0], channel3_hist[0])) 54 | # Return the individual histograms, bin_centers and feature vector 55 | return hist_features 56 | 57 | # get attributes of our svc object 58 | svc = pickle.load(open("svc_pickle.p","rb")) 59 | X_scaler = pickle.load(open("X_scaler.p","rb")) 60 | print(X_scaler) 61 | orient = 8 62 | pix_per_cell = 8 63 | cell_per_block = 2 64 | spatial_size = (16,16) 65 | hist_bins = (32,32) 66 | 67 | # Define a single function that can extract features using hog sub-sampling and make predictions 68 | def find_cars(img, ystart, ystop, scale, svc, X_scaler, orient, pix_per_cell, cell_per_block, spatial_size, hist_bins): 69 | 70 | draw_img = np.copy(img) 71 | img = img.astype(np.float32)/255 72 | 73 | img_tosearch = img[ystart:ystop,:,:] 74 | ctrans_tosearch = convert_color(img_tosearch, conv='RGB2YCrCb') 75 | if scale != 1: 76 | imshape = ctrans_tosearch.shape 77 | ctrans_tosearch = cv2.resize(ctrans_tosearch, (np.int(imshape[1]/scale), np.int(imshape[0]/scale))) 78 | 79 | ch1 = ctrans_tosearch[:,:,0] 80 | ch2 = ctrans_tosearch[:,:,1] 81 | ch3 = ctrans_tosearch[:,:,2] 82 | 83 | # Define blocks and steps as above 84 | nxblocks = (ch1.shape[1] // pix_per_cell) - cell_per_block + 1 85 | nyblocks = (ch1.shape[0] // pix_per_cell) - cell_per_block + 1 86 | nfeat_per_block = orient*cell_per_block**2 87 | 88 | # 64 was the orginal sampling rate, with 8 cells and 8 pix per cell 89 | window = 64 90 | nblocks_per_window = (window // pix_per_cell) - cell_per_block + 1 91 | cells_per_step = 2 # Instead of overlap, define how many cells to step 92 | nxsteps = (nxblocks - nblocks_per_window) // cells_per_step + 1 93 | nysteps = (nyblocks - nblocks_per_window) // cells_per_step + 1 94 | 95 | # Compute individual channel HOG features for the entire image 96 | hog1 = get_hog_features(ch1, orient, pix_per_cell, cell_per_block, feature_vec=False) 97 | hog2 = get_hog_features(ch2, orient, pix_per_cell, cell_per_block, feature_vec=False) 98 | hog3 = get_hog_features(ch3, orient, pix_per_cell, cell_per_block, feature_vec=False) 99 | 100 | for xb in range(nxsteps): 101 | for yb in range(nysteps): 102 | ypos = yb*cells_per_step 103 | xpos = xb*cells_per_step 104 | # Extract HOG for this patch 105 | hog_feat1 = hog1[ypos:ypos+nblocks_per_window, xpos:xpos+nblocks_per_window].ravel() 106 | hog_feat2 = hog2[ypos:ypos+nblocks_per_window, xpos:xpos+nblocks_per_window].ravel() 107 | hog_feat3 = hog3[ypos:ypos+nblocks_per_window, xpos:xpos+nblocks_per_window].ravel() 108 | hog_features = np.hstack((hog_feat1, hog_feat2, hog_feat3)) 109 | 110 | xleft = xpos*pix_per_cell 111 | ytop = ypos*pix_per_cell 112 | 113 | # Extract the image patch 114 | subimg = cv2.resize(ctrans_tosearch[ytop:ytop+window, xleft:xleft+window], (64,64)) 115 | 116 | # Get color features 117 | spatial_features = bin_spatial(subimg, size=spatial_size) 118 | hist_features = color_hist(subimg, nbins=hist_bins) 119 | 120 | # Scale features and make a prediction 121 | test_features = X_scaler.transform(np.hstack((spatial_features, hist_features, hog_features)).reshape(1, -1)) 122 | # test_features = X_scaler.transform(np.hstack((shape_feat, hist_feat)).reshape(1, -1)) 123 | test_prediction = svc.predict(test_features) 124 | 125 | if test_prediction == 1: 126 | xbox_left = np.int(xleft*scale) 127 | ytop_draw = np.int(ytop*scale) 128 | win_draw = np.int(window*scale) 129 | cv2.rectangle(draw_img,(xbox_left, ytop_draw+ystart),(xbox_left+win_draw,ytop_draw+win_draw+ystart),(0,0,255),6) 130 | 131 | return draw_img 132 | 133 | def add_heat(heatmap, bbox_list): 134 | # Iterate through list of bboxes 135 | for box in bbox_list: 136 | # Add += 1 for all pixels inside each bbox 137 | # Assuming each "box" takes the form ((x1, y1), (x2, y2)) 138 | heatmap[box[0][1]:box[1][1], box[0][0]:box[1][0]] += 1 139 | 140 | # Return updated heatmap 141 | return heatmap# Iterate through list of bboxes 142 | 143 | def apply_threshold(heatmap, threshold): 144 | # Zero out pixels below the threshold 145 | heatmap[heatmap <= threshold] = 0 146 | # Return thresholded map 147 | return heatmap 148 | 149 | def draw_labeled_bboxes(img, labels): 150 | # Iterate through all detected cars 151 | for car_number in range(1, labels[1]+1): 152 | # Find pixels with each car_number label value 153 | nonzero = (labels[0] == car_number).nonzero() 154 | # Identify x and y values of those pixels 155 | nonzeroy = np.array(nonzero[0]) 156 | nonzerox = np.array(nonzero[1]) 157 | # Define a bounding box based on min/max x and y 158 | bbox = ((np.min(nonzerox), np.min(nonzeroy)), (np.max(nonzerox), np.max(nonzeroy))) 159 | # Draw the box on the image 160 | cv2.rectangle(img, bbox[0], bbox[1], (0,0,255), 6) 161 | # Return the image 162 | return img 163 | 164 | def process_image(img): 165 | ystart = 400 166 | ystop = 656 167 | scale = 1.5 168 | out_img = find_cars(img, ystart, ystop, scale, svc, X_scaler, orient, pix_per_cell, cell_per_block, spatial_size, hist_bins) 169 | heat = np.zeros_like(img[:,:,0]).astype(np.float) 170 | heat = add_heat(heat,out_img) 171 | heat = apply_threshold(heat,1) 172 | heatmap = np.clip(heat, 0, 255) 173 | labels = label(heatmap) 174 | draw_img = draw_labeled_bboxes(np.copy(img), labels) 175 | return draw_img 176 | 177 | output_vid = 'final_tracker.mp4' 178 | input_vid = 'lane_tracker.mp4' 179 | 180 | clip1 = VideoFileClip(input_vid) 181 | video_clip = clip1.fl_image(process_image) 182 | video_clip.write_videofile(output_vid, audio=False) -------------------------------------------------------------------------------- /object_detection.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "returning-thursday", 6 | "metadata": {}, 7 | "source": [ 8 | "# Imports" 9 | ] 10 | }, 11 | { 12 | "cell_type": "code", 13 | "execution_count": null, 14 | "id": "deluxe-hearing", 15 | "metadata": {}, 16 | "outputs": [], 17 | "source": [ 18 | "import matplotlib.image as mpimg\n", 19 | "import matplotlib.pyplot as plt\n", 20 | "import numpy as np\n", 21 | "import cv2\n", 22 | "import os\n", 23 | "import time\n", 24 | "import pickle\n", 25 | "from sklearn.svm import LinearSVC\n", 26 | "from sklearn.preprocessing import StandardScaler\n", 27 | "from skimage.feature import hog\n", 28 | "from sklearn.model_selection import train_test_split\n", 29 | "from scipy.ndimage.measurements import label\n", 30 | "from moviepy.editor import VideoFileClip\n", 31 | "from IPython.display import HTML\n", 32 | "from skimage.feature import hog\n", 33 | "import random" 34 | ] 35 | }, 36 | { 37 | "cell_type": "markdown", 38 | "id": "given-columbus", 39 | "metadata": {}, 40 | "source": [ 41 | "# Loading in Data" 42 | ] 43 | }, 44 | { 45 | "cell_type": "code", 46 | "execution_count": null, 47 | "id": "color-socket", 48 | "metadata": {}, 49 | "outputs": [], 50 | "source": [ 51 | "def grab_files(root):\n", 52 | " dataset = []\n", 53 | " for path, dirs, files in os.walk(root):\n", 54 | " for file in files:\n", 55 | " if '.DS_Store' not in file:\n", 56 | " dataset.append(os.path.join(path, file))\n", 57 | " return dataset" 58 | ] 59 | }, 60 | { 61 | "cell_type": "code", 62 | "execution_count": null, 63 | "id": "smoking-chemistry", 64 | "metadata": {}, 65 | "outputs": [], 66 | "source": [ 67 | "cars = grab_files(\"./vehicles\")\n", 68 | "notcars = grab_files(\"./non-vehicles\")" 69 | ] 70 | }, 71 | { 72 | "cell_type": "code", 73 | "execution_count": null, 74 | "id": "architectural-grenada", 75 | "metadata": {}, 76 | "outputs": [], 77 | "source": [ 78 | "color_space = 'YCrCb' \n", 79 | "orient = 8\n", 80 | "pix_per_cell = 8 \n", 81 | "cell_per_block = 2 \n", 82 | "hog_channel = 'ALL'\n", 83 | "spatial_size = (16, 16) \n", 84 | "hist_bins = 32\n", 85 | "spatial_feat = True\n", 86 | "hist_feat = True\n", 87 | "hog_feat = True \n", 88 | "y_start_stop = [400,700]\n", 89 | "scale = 1.5\n", 90 | "xy_window = (64,64)\n", 91 | "xy_overlap = (0.85,0.85)" 92 | ] 93 | }, 94 | { 95 | "cell_type": "code", 96 | "execution_count": null, 97 | "id": "palestinian-mercury", 98 | "metadata": {}, 99 | "outputs": [], 100 | "source": [ 101 | "def bin_spatial(img, size=(32, 32)):\n", 102 | " color1 = cv2.resize(img[:,:,0], size).ravel()\n", 103 | " color2 = cv2.resize(img[:,:,1], size).ravel()\n", 104 | " color3 = cv2.resize(img[:,:,2], size).ravel()\n", 105 | " return np.hstack((color1, color2, color3))\n", 106 | "\n", 107 | "def color_hist(img, nbins=32, bins_range=(0, 256)):\n", 108 | " channel1_hist = np.histogram(img[:,:,0], bins=nbins, range=bins_range)\n", 109 | " channel2_hist = np.histogram(img[:,:,1], bins=nbins, range=bins_range)\n", 110 | " channel3_hist = np.histogram(img[:,:,2], bins=nbins, range=bins_range)\n", 111 | " hist_features = np.concatenate((channel1_hist[0], channel2_hist[0], channel3_hist[0]))\n", 112 | " return hist_features\n", 113 | "\n", 114 | "def convert_image(image, cspace):\n", 115 | "\n", 116 | " if cspace != 'RGB':\n", 117 | " if cspace == 'HSV':\n", 118 | " feature_image = cv2.cvtColor(image, cv2.COLOR_RGB2HSV)\n", 119 | " elif cspace == 'LUV':\n", 120 | " feature_image = cv2.cvtColor(image, cv2.COLOR_RGB2LUV)\n", 121 | " elif cspace == 'HLS':\n", 122 | " feature_image = cv2.cvtColor(image, cv2.COLOR_RGB2HLS)\n", 123 | " elif cspace == 'YUV':\n", 124 | " feature_image = cv2.cvtColor(image, cv2.COLOR_RGB2YUV)\n", 125 | " elif cspace == 'YCrCb':\n", 126 | " feature_image = cv2.cvtColor(image, cv2.COLOR_RGB2YCrCb)\n", 127 | " else: \n", 128 | " feature_image = np.copy(image) \n", 129 | " \n", 130 | " return feature_image" 131 | ] 132 | }, 133 | { 134 | "cell_type": "code", 135 | "execution_count": null, 136 | "id": "designing-tragedy", 137 | "metadata": {}, 138 | "outputs": [], 139 | "source": [ 140 | "def get_hog_features(img, orient, pix_per_cell, cell_per_block, \n", 141 | " vis=False, feature_vec=True):\n", 142 | " if vis == True:\n", 143 | " features, hog_image = hog(img, orientations=orient, \n", 144 | " pixels_per_cell=(pix_per_cell, pix_per_cell),\n", 145 | " block_norm= 'L2-Hys',\n", 146 | " cells_per_block=(cell_per_block, cell_per_block), \n", 147 | " transform_sqrt=True, \n", 148 | " visualize=vis, feature_vector=feature_vec)\n", 149 | " return features, hog_image\n", 150 | " else: \n", 151 | " features = hog(img, orientations=orient, \n", 152 | " pixels_per_cell=(pix_per_cell, pix_per_cell),\n", 153 | " cells_per_block=(cell_per_block, cell_per_block), \n", 154 | " block_norm= 'L2-Hys',\n", 155 | " transform_sqrt=True, \n", 156 | " visualize=vis, feature_vector=feature_vec)\n", 157 | " return features\n", 158 | "\n", 159 | "def single_img_features(img, color_space='YCrCb', spatial_size=(16, 16),\n", 160 | " hist_bins=32, orient=8, \n", 161 | " pix_per_cell=8, cell_per_block=2, hog_channel=\"ALL\",\n", 162 | " spatial_feat=True, hist_feat=True, hog_feat=True): \n", 163 | " img_features = []\n", 164 | " feature_image = convert_image(img,color_space) \n", 165 | " \n", 166 | " if spatial_feat == True:\n", 167 | " spatial_features = bin_spatial(feature_image, size=spatial_size)\n", 168 | " img_features.append(spatial_features)\n", 169 | " if hist_feat == True:\n", 170 | " hist_features = color_hist(feature_image, nbins=hist_bins)\n", 171 | " img_features.append(hist_features)\n", 172 | " if hog_feat == True:\n", 173 | " if hog_channel == 'ALL':\n", 174 | " hog_features = []\n", 175 | " for channel in range(feature_image.shape[2]):\n", 176 | " hog_features.extend(get_hog_features(feature_image[:,:,channel], \n", 177 | " orient, pix_per_cell, cell_per_block, \n", 178 | " vis=False, feature_vec=True)) \n", 179 | " else:\n", 180 | " hog_features = get_hog_features(feature_image[:,:,hog_channel], orient, \n", 181 | " pix_per_cell, cell_per_block, vis=False, feature_vec=True)\n", 182 | " img_features.append(hog_features)\n", 183 | "\n", 184 | " return np.concatenate(img_features)\n", 185 | "\n", 186 | "def extract_features(imgs, color_space='YCrCb', spatial_size=(16,16),\n", 187 | " hist_bins=32, orient=8, \n", 188 | " pix_per_cell=8, cell_per_block=2, hog_channel=\"ALL\",\n", 189 | " spatial_feat=True, hist_feat=True, hog_feat=True):\n", 190 | " features = []\n", 191 | " for file in imgs:\n", 192 | " file_features = []\n", 193 | " # Read in each one by one\n", 194 | " image = cv2.imread(file)\n", 195 | " feature_image = convert_image(image,color_space) \n", 196 | "\n", 197 | " if spatial_feat == True:\n", 198 | " spatial_features = bin_spatial(feature_image, size=spatial_size)\n", 199 | " file_features.append(spatial_features)\n", 200 | " if hist_feat == True:\n", 201 | " hist_features = color_hist(feature_image, nbins=hist_bins)\n", 202 | " file_features.append(hist_features)\n", 203 | " if hog_feat == True:\n", 204 | " if hog_channel == 'ALL':\n", 205 | " hog_features = []\n", 206 | " for channel in range(feature_image.shape[2]):\n", 207 | " hog_features.append(get_hog_features(feature_image[:,:,channel], \n", 208 | " orient, pix_per_cell, cell_per_block, \n", 209 | " vis=False, feature_vec=True))\n", 210 | " hog_features = np.ravel(hog_features) \n", 211 | " else:\n", 212 | " hog_features = get_hog_features(feature_image[:,:,hog_channel], orient, \n", 213 | " pix_per_cell, cell_per_block, vis=False, feature_vec=True)\n", 214 | " file_features.append(hog_features)\n", 215 | " features.append(np.concatenate(file_features))\n", 216 | " return features" 217 | ] 218 | }, 219 | { 220 | "cell_type": "code", 221 | "execution_count": null, 222 | "id": "solid-russia", 223 | "metadata": {}, 224 | "outputs": [], 225 | "source": [ 226 | "car_features = extract_features(cars, color_space=color_space, \n", 227 | " spatial_size=spatial_size, hist_bins=hist_bins, \n", 228 | " orient=orient, pix_per_cell=pix_per_cell, \n", 229 | " cell_per_block=cell_per_block, \n", 230 | " hog_channel=hog_channel, spatial_feat=spatial_feat, \n", 231 | " hist_feat=hist_feat, hog_feat=hog_feat)\n", 232 | "notcar_features = extract_features(notcars, color_space=color_space, \n", 233 | " spatial_size=spatial_size, hist_bins=hist_bins, \n", 234 | " orient=orient, pix_per_cell=pix_per_cell, \n", 235 | " cell_per_block=cell_per_block, \n", 236 | " hog_channel=hog_channel, spatial_feat=spatial_feat, \n", 237 | " hist_feat=hist_feat, hog_feat=hog_feat)\n", 238 | "\n", 239 | "X = np.vstack((car_features, notcar_features)).astype(np.float64)\n", 240 | "y = np.hstack((np.ones(len(car_features)), np.zeros(len(notcar_features))))\n", 241 | "\n", 242 | "rand_state = np.random.randint(0, 100)\n", 243 | "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=rand_state)" 244 | ] 245 | }, 246 | { 247 | "cell_type": "code", 248 | "execution_count": null, 249 | "id": "dependent-gentleman", 250 | "metadata": {}, 251 | "outputs": [], 252 | "source": [ 253 | "f, (ax1, ax2) = plt.subplots(1, 2, figsize=(24, 9))\n", 254 | "\n", 255 | "i = cv2.cvtColor(cv2.imread(random.choice(cars)),cv2.COLOR_BGR2RGB)\n", 256 | "j = cv2.cvtColor(cv2.imread(random.choice(notcars)),cv2.COLOR_BGR2RGB)\n", 257 | "\n", 258 | "car_feat = single_img_features(img=i, color_space=color_space, spatial_size=spatial_size,\n", 259 | " hist_bins=hist_bins, orient=orient, \n", 260 | " pix_per_cell=pix_per_cell, cell_per_block=cell_per_block, hog_channel=hog_channel,\n", 261 | " spatial_feat=spatial_feat, hist_feat=hist_feat, hog_feat=hog_feat)\n", 262 | "notcar_feat = single_img_features(img=j, color_space=color_space, spatial_size=spatial_size,\n", 263 | " hist_bins=hist_bins, orient=orient, \n", 264 | " pix_per_cell=pix_per_cell, cell_per_block=cell_per_block, hog_channel=hog_channel,\n", 265 | " spatial_feat=spatial_feat, hist_feat=hist_feat, hog_feat=hog_feat)\n", 266 | "ax1.plot(car_feat)\n", 267 | "ax1.set_title(\"Car Spatial features\")\n", 268 | "ax2.plot(notcar_feat)\n", 269 | "ax2.set_title(\"Non-car Spatial features\")" 270 | ] 271 | }, 272 | { 273 | "cell_type": "code", 274 | "execution_count": null, 275 | "id": "detailed-appreciation", 276 | "metadata": {}, 277 | "outputs": [], 278 | "source": [ 279 | "car1 = cv2.cvtColor(cv2.imread(cars[0]),cv2.COLOR_BGR2RGB)\n", 280 | "not_car1 = cv2.cvtColor(cv2.imread(notcars[0]),cv2.COLOR_BGR2RGB)\n", 281 | "\n", 282 | "_, hog1 = get_hog_features(car1, orient, pix_per_cell, cell_per_block, vis=True, feature_vec=True)\n", 283 | "_, hog2 = get_hog_features(not_car1, orient, pix_per_cell, cell_per_block, vis=True, feature_vec=True)\n", 284 | "\n", 285 | "f, (ax1, ax2, ax3, ax4) = plt.subplots(1, 4, figsize=(20,10))\n", 286 | "ax1.set_title('Vehicle')\n", 287 | "ax1.imshow(car1, cmap='gray')\n", 288 | "ax2.set_title('Vehicle hog')\n", 289 | "ax2.imshow(hog1, cmap='gray')\n", 290 | "ax3.set_title('Non-vehicle')\n", 291 | "ax3.imshow(not_car1, cmap='gray')\n", 292 | "ax4.set_title('Non-vehicle hog')\n", 293 | "ax4.imshow(hog2, cmap='gray')" 294 | ] 295 | }, 296 | { 297 | "cell_type": "markdown", 298 | "id": "existing-brick", 299 | "metadata": {}, 300 | "source": [ 301 | "# SVM Model" 302 | ] 303 | }, 304 | { 305 | "cell_type": "code", 306 | "execution_count": null, 307 | "id": "ambient-nicholas", 308 | "metadata": {}, 309 | "outputs": [], 310 | "source": [ 311 | "X_scaler = StandardScaler().fit(X_train)\n", 312 | "X_train = X_scaler.transform(X_train)\n", 313 | "X_test = X_scaler.transform(X_test)\n", 314 | "\n", 315 | "print('Using:',orient,'orientations',pix_per_cell,\n", 316 | " 'pixels per cell and', cell_per_block,'cells per block')\n", 317 | "print('Feature vector length:', len(X_train[0]))\n", 318 | "\n", 319 | "svc = LinearSVC()\n", 320 | "\n", 321 | "t=time.time()\n", 322 | "\n", 323 | "svc.fit(X_train, y_train)\n", 324 | "\n", 325 | "t2 = time.time()\n", 326 | "\n", 327 | "print(round(t2-t, 2), 'Seconds to train SVC...')\n", 328 | "\n", 329 | "print('Test Accuracy of SVC = ', round(svc.score(X_test, y_test), 4))" 330 | ] 331 | }, 332 | { 333 | "cell_type": "code", 334 | "execution_count": null, 335 | "id": "atlantic-entertainment", 336 | "metadata": {}, 337 | "outputs": [], 338 | "source": [ 339 | "def draw_boxes(img, bboxes, color=(0, 0, 255), thick=6):\n", 340 | " imcopy = np.copy(img)\n", 341 | " for bbox in bboxes:\n", 342 | " cv2.rectangle(imcopy, bbox[0], bbox[1], color, thick)\n", 343 | " return imcopy\n", 344 | "\n", 345 | "def slide_window(img, x_start_stop=[None, None], y_start_stop=[400,700], \n", 346 | " xy_window=(64, 64), xy_overlap=(0.85, 0.85)):\n", 347 | " \n", 348 | " \n", 349 | " x_start_stop = [0,img.shape[1]]\n", 350 | " if y_start_stop[0] == None:\n", 351 | " y_start_stop[0] = 0\n", 352 | " if y_start_stop[1] == None:\n", 353 | " y_start_stop[1] = img.shape[0] \n", 354 | " xspan = x_start_stop[1] - x_start_stop[0]\n", 355 | " yspan = y_start_stop[1] - y_start_stop[0]\n", 356 | " nx_pix_per_step = np.int(xy_window[0]*(1 - xy_overlap[0]))\n", 357 | " ny_pix_per_step = np.int(xy_window[1]*(1 - xy_overlap[1]))\n", 358 | " nx_buffer = np.int(xy_window[0]*(xy_overlap[0]))\n", 359 | " ny_buffer = np.int(xy_window[1]*(xy_overlap[1]))\n", 360 | " nx_windows = np.int((xspan-nx_buffer)/nx_pix_per_step) \n", 361 | " ny_windows = np.int((yspan-ny_buffer)/ny_pix_per_step) \n", 362 | "\n", 363 | " window_list = []\n", 364 | " for ys in range(ny_windows):\n", 365 | " for xs in range(nx_windows):\n", 366 | " # Calculate window position\n", 367 | " startx = xs*nx_pix_per_step + x_start_stop[0]\n", 368 | " endx = startx + xy_window[0]\n", 369 | " starty = ys*ny_pix_per_step + y_start_stop[0]\n", 370 | " endy = starty + xy_window[1]\n", 371 | " window_list.append(((startx, starty), (endx, endy)))\n", 372 | " return window_list \n", 373 | "\n", 374 | "def find_cars(img, clf, scaler, color_space='YCrCb', \n", 375 | " spatial_size=(16, 16), hist_bins=32, \n", 376 | " hist_range=(0, 256), orient=8, \n", 377 | " pix_per_cell=8, cell_per_block=2, \n", 378 | " hog_channel=\"ALL\", spatial_feat=True, \n", 379 | " hist_feat=True, hog_feat=True):\n", 380 | "\n", 381 | " windows = slide_window(img, y_start_stop=y_start_stop, xy_window=xy_window, xy_overlap=xy_overlap)\n", 382 | " on_windows = []\n", 383 | " for window in windows:\n", 384 | " test_img = cv2.resize(img[window[0][1]:window[1][1], window[0][0]:window[1][0]], (64,64)) \n", 385 | " features = single_img_features(test_img, color_space=color_space, \n", 386 | " spatial_size=spatial_size, hist_bins=hist_bins, \n", 387 | " orient=orient, pix_per_cell=pix_per_cell, \n", 388 | " cell_per_block=cell_per_block, \n", 389 | " hog_channel=hog_channel, spatial_feat=spatial_feat, \n", 390 | " hist_feat=hist_feat, hog_feat=hog_feat)\n", 391 | " test_features = scaler.transform(np.array(features).reshape(1, -1))\n", 392 | " prediction = clf.predict(test_features)\n", 393 | " if prediction == 1:\n", 394 | " on_windows.append(window)\n", 395 | " return on_windows" 396 | ] 397 | }, 398 | { 399 | "cell_type": "code", 400 | "execution_count": null, 401 | "id": "fewer-relative", 402 | "metadata": {}, 403 | "outputs": [], 404 | "source": [ 405 | "img = cv2.cvtColor(cv2.imread('test_img/test1.jpg'),cv2.COLOR_BGR2RGB)\n", 406 | "car_loc = find_cars(img, svc, X_scaler, color_space=color_space, \n", 407 | " spatial_size=spatial_size, hist_bins=hist_bins, \n", 408 | " orient=orient, pix_per_cell=pix_per_cell, \n", 409 | " cell_per_block=cell_per_block, \n", 410 | " hog_channel=hog_channel, spatial_feat=spatial_feat, \n", 411 | " hist_feat=hist_feat, hog_feat=hog_feat) \n", 412 | "out_img = draw_boxes(image, car_loc)\n", 413 | "plt.imshow(out_img)" 414 | ] 415 | }, 416 | { 417 | "cell_type": "markdown", 418 | "id": "civil-exception", 419 | "metadata": {}, 420 | "source": [ 421 | "# Heat Pipeline + False Positives" 422 | ] 423 | }, 424 | { 425 | "cell_type": "code", 426 | "execution_count": null, 427 | "id": "registered-delivery", 428 | "metadata": {}, 429 | "outputs": [], 430 | "source": [ 431 | "def add_heat(heatmap, bbox_list):\n", 432 | " for box in bbox_list:\n", 433 | " heatmap[box[0][1]:box[1][1], box[0][0]:box[1][0]] += 1\n", 434 | " return heatmap\n", 435 | " \n", 436 | "def apply_threshold(heatmap, threshold):\n", 437 | " heatmap[heatmap <= threshold] = 0\n", 438 | " return heatmap\n", 439 | "\n", 440 | "def draw_labeled_bboxes(img, labels):\n", 441 | " for car_number in range(1, labels[1]+1):\n", 442 | " nonzero = (labels[0] == car_number).nonzero()\n", 443 | " nonzeroy = np.array(nonzero[0])\n", 444 | " nonzerox = np.array(nonzero[1])\n", 445 | " bbox = ((np.min(nonzerox), np.min(nonzeroy)), (np.max(nonzerox), np.max(nonzeroy)))\n", 446 | " cv2.rectangle(img, bbox[0], bbox[1], (0,0,255), 6)\n", 447 | " return img\n", 448 | "\n", 449 | "def heat_pipeline(img,car_loc):\n", 450 | " heat = np.zeros_like(img[:,:,0]).astype(np.float)\n", 451 | " heat = add_heat(heat,car_loc)\n", 452 | " heat = apply_threshold(heat,1)\n", 453 | " heatmap = np.clip(heat, 0, 255)\n", 454 | " labels = label(heatmap)\n", 455 | " \n", 456 | " return heatmap, labels" 457 | ] 458 | }, 459 | { 460 | "cell_type": "code", 461 | "execution_count": null, 462 | "id": "adolescent-mortality", 463 | "metadata": {}, 464 | "outputs": [], 465 | "source": [ 466 | "def vis_img(img):\n", 467 | " \n", 468 | " car_loc = find_cars(img, svc, X_scaler, color_space=color_space, \n", 469 | " spatial_size=spatial_size, hist_bins=hist_bins, \n", 470 | " orient=orient, pix_per_cell=pix_per_cell, \n", 471 | " cell_per_block=cell_per_block, \n", 472 | " hog_channel=hog_channel, spatial_feat=spatial_feat, \n", 473 | " hist_feat=hist_feat, hog_feat=hog_feat) \n", 474 | " heatmap, labels = heat_pipeline(img,car_loc)\n", 475 | " draw_img = draw_labeled_bboxes(np.copy(img), labels)\n", 476 | " \n", 477 | " f, (ax1, ax2) = plt.subplots(1, 2, figsize=(24, 9))\n", 478 | " ax1.imshow(draw_img)\n", 479 | " ax2.imshow(heatmap, cmap='hot')\n", 480 | " plt.show()" 481 | ] 482 | }, 483 | { 484 | "cell_type": "code", 485 | "execution_count": null, 486 | "id": "square-hours", 487 | "metadata": {}, 488 | "outputs": [], 489 | "source": [ 490 | "img = cv2.cvtColor(cv2.imread('test_img/test1.jpg'),cv2.COLOR_BGR2RGB)\n", 491 | "vis_img(img)" 492 | ] 493 | }, 494 | { 495 | "cell_type": "markdown", 496 | "id": "fantastic-montana", 497 | "metadata": {}, 498 | "source": [ 499 | "# HOG Subsampling Window Search" 500 | ] 501 | }, 502 | { 503 | "cell_type": "code", 504 | "execution_count": null, 505 | "id": "loving-edward", 506 | "metadata": {}, 507 | "outputs": [], 508 | "source": [ 509 | "def hog_sub_find_cars(img, color_space, ystart, ystop, scale, svc, X_scaler, orient, pix_per_cell, cell_per_block, spatial_size, hist_bins, window):\n", 510 | " \n", 511 | " draw_img = np.copy(img)\n", 512 | " \n", 513 | " img_tosearch = img[ystart:ystop,:,:]\n", 514 | " \n", 515 | " if color_space != 'RGB':\n", 516 | " ctrans_tosearch = convert_image(img_tosearch,color_space)\n", 517 | " else: \n", 518 | " ctrans_tosearch = np.copy(img_tosearch) \n", 519 | " \n", 520 | " if scale != 1:\n", 521 | " imshape = ctrans_tosearch.shape\n", 522 | " ctrans_tosearch = cv2.resize(ctrans_tosearch, (np.int(imshape[1]/scale), np.int(imshape[0]/scale)))\n", 523 | " \n", 524 | " ch1 = ctrans_tosearch[:,:,0]\n", 525 | " ch2 = ctrans_tosearch[:,:,1]\n", 526 | " ch3 = ctrans_tosearch[:,:,2]\n", 527 | "\n", 528 | " nxblocks = (ch1.shape[1] // pix_per_cell) - cell_per_block + 1\n", 529 | " nyblocks = (ch1.shape[0] // pix_per_cell) - cell_per_block + 1 \n", 530 | " nfeat_per_block = orient*cell_per_block**2\n", 531 | "\n", 532 | " window = 64\n", 533 | " nblocks_per_window = (window // pix_per_cell) - cell_per_block + 1\n", 534 | " cells_per_step = 1 # Instead of overlap, define how many cells to step\n", 535 | " nxsteps = (nxblocks - nblocks_per_window) // cells_per_step \n", 536 | " nysteps = (nyblocks - nblocks_per_window) // cells_per_step \n", 537 | "\n", 538 | " hog1 = get_hog_features(ch1, orient, pix_per_cell, cell_per_block, feature_vec=False)\n", 539 | " hog2 = get_hog_features(ch2, orient, pix_per_cell, cell_per_block, feature_vec=False)\n", 540 | " hog3 = get_hog_features(ch3, orient, pix_per_cell, cell_per_block, feature_vec=False)\n", 541 | " \n", 542 | " car_windows=[]\n", 543 | " for xb in range(nxsteps):\n", 544 | " for yb in range(nysteps):\n", 545 | " ypos = yb*cells_per_step\n", 546 | " xpos = xb*cells_per_step\n", 547 | "\n", 548 | " hog_feat1 = hog1[ypos:ypos+nblocks_per_window, xpos:xpos+nblocks_per_window].ravel() \n", 549 | " hog_feat2 = hog2[ypos:ypos+nblocks_per_window, xpos:xpos+nblocks_per_window].ravel() \n", 550 | " hog_feat3 = hog3[ypos:ypos+nblocks_per_window, xpos:xpos+nblocks_per_window].ravel() \n", 551 | " hog_features = np.hstack((hog_feat1, hog_feat2, hog_feat3))\n", 552 | "\n", 553 | " xleft = xpos*pix_per_cell\n", 554 | " ytop = ypos*pix_per_cell\n", 555 | "\n", 556 | " subimg = cv2.resize(ctrans_tosearch[ytop:ytop+window, xleft:xleft+window], (64,64))\n", 557 | "\n", 558 | " spatial_features = bin_spatial(subimg, size=spatial_size)\n", 559 | " hist_features = color_hist(subimg, nbins=hist_bins)\n", 560 | "\n", 561 | " test_features = X_scaler.transform(np.hstack((spatial_features, hist_features, hog_features)).reshape(1, -1)) \n", 562 | " test_prediction = svc.predict(test_features)\n", 563 | " \n", 564 | " if test_prediction == 1:\n", 565 | " xbox_left = np.int(xleft*scale)\n", 566 | " ytop_draw = np.int(ytop*scale)\n", 567 | " win_draw = np.int(window*scale)\n", 568 | " cv2.rectangle(draw_img,(xbox_left, ytop_draw+ystart),(xbox_left+win_draw,ytop_draw+win_draw+ystart),(0,0,255),6) \n", 569 | " car_windows.append(((xbox_left, ytop_draw+ystart),(xbox_left+win_draw,ytop_draw+win_draw+ystart)))\n", 570 | " \n", 571 | " return car_windows" 572 | ] 573 | }, 574 | { 575 | "cell_type": "code", 576 | "execution_count": null, 577 | "id": "alternative-tribute", 578 | "metadata": {}, 579 | "outputs": [], 580 | "source": [ 581 | "read_img = mpimg.imread('test_img/test1.jpg')\n", 582 | "car_loc = hog_sub_find_cars(img=read_img, color_space=color_space, ystart=400, ystop=550, scale=scale, svc=svc, X_scaler=X_scaler, \n", 583 | " orient=orient, pix_per_cell=pix_per_cell, cell_per_block=cell_per_block, spatial_size=spatial_size, hist_bins=hist_bins, window=64) \n", 584 | "\n", 585 | "heatmap, labels = heat_pipeline(read_img,car_loc)\n", 586 | "draw_img = draw_labeled_bboxes(np.copy(read_img), labels)\n", 587 | "\n", 588 | "f, (ax1, ax2) = plt.subplots(1, 2, figsize=(24, 9))\n", 589 | "ax1.imshow(draw_img)\n", 590 | "ax2.imshow(heatmap, cmap='hot')\n", 591 | "plt.show()" 592 | ] 593 | }, 594 | { 595 | "cell_type": "markdown", 596 | "id": "widespread-plasma", 597 | "metadata": {}, 598 | "source": [ 599 | "# Writing out video " 600 | ] 601 | }, 602 | { 603 | "cell_type": "code", 604 | "execution_count": null, 605 | "id": "standard-subcommittee", 606 | "metadata": {}, 607 | "outputs": [], 608 | "source": [ 609 | "def process_img(image):\n", 610 | " car_loc = find_cars(image, svc, X_scaler, color_space=color_space, \n", 611 | " spatial_size=spatial_size, hist_bins=hist_bins, \n", 612 | " orient=orient, pix_per_cell=pix_per_cell, \n", 613 | " cell_per_block=cell_per_block, \n", 614 | " hog_channel=hog_channel, spatial_feat=spatial_feat, \n", 615 | " hist_feat=hist_feat, hog_feat=hog_feat) \n", 616 | " heatmap, labels = heat_pipeline(image,car_loc)\n", 617 | " draw_img = draw_labeled_bboxes(np.copy(image), labels)\n", 618 | " return draw_img" 619 | ] 620 | }, 621 | { 622 | "cell_type": "code", 623 | "execution_count": null, 624 | "id": "victorian-console", 625 | "metadata": {}, 626 | "outputs": [], 627 | "source": [ 628 | "def process_img(img):\n", 629 | " \n", 630 | " car_loc = hog_sub_find_cars(img=img, color_space=color_space, ystart=y_start_stop[0], ystop=y_start_stop[1], scale=scale, svc=svc, X_scaler=X_scaler, \n", 631 | " orient=orient, pix_per_cell=pix_per_cell, cell_per_block=cell_per_block, spatial_size=spatial_size, hist_bins=hist_bins, window=xy_window[0]) \n", 632 | " heatmap, labels = heat_pipeline(img,car_loc)\n", 633 | " draw_img = draw_labeled_bboxes(np.copy(img), labels) \n", 634 | " return draw_img" 635 | ] 636 | }, 637 | { 638 | "cell_type": "code", 639 | "execution_count": null, 640 | "id": "beautiful-surge", 641 | "metadata": {}, 642 | "outputs": [], 643 | "source": [ 644 | "output_vid = 'final_tracker4.mp4'\n", 645 | "input_vid = 'project_video.mp4'\n", 646 | "# input_vid = 'lane_tracker.mp4'\n", 647 | "\n", 648 | "clip1 = VideoFileClip(input_vid)\n", 649 | "video_clip = clip1.fl_image(process_img)\n", 650 | "video_clip.write_videofile(output_vid, audio=False, fps=30, threads=12)" 651 | ] 652 | }, 653 | { 654 | "cell_type": "code", 655 | "execution_count": null, 656 | "id": "iraqi-schedule", 657 | "metadata": {}, 658 | "outputs": [], 659 | "source": [ 660 | "HTML(\"\"\"\n", 661 | "\n", 664 | "\"\"\".format(output_vid))" 665 | ] 666 | }, 667 | { 668 | "cell_type": "code", 669 | "execution_count": null, 670 | "id": "crude-advance", 671 | "metadata": {}, 672 | "outputs": [], 673 | "source": [] 674 | } 675 | ], 676 | "metadata": { 677 | "kernelspec": { 678 | "display_name": "Python 3", 679 | "language": "python", 680 | "name": "python3" 681 | }, 682 | "language_info": { 683 | "codemirror_mode": { 684 | "name": "ipython", 685 | "version": 3 686 | }, 687 | "file_extension": ".py", 688 | "mimetype": "text/x-python", 689 | "name": "python", 690 | "nbconvert_exporter": "python", 691 | "pygments_lexer": "ipython3", 692 | "version": "3.8.5" 693 | } 694 | }, 695 | "nbformat": 4, 696 | "nbformat_minor": 5 697 | } 698 | -------------------------------------------------------------------------------- /output_vid/lane_tracker.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/srianumakonda/Advanced-Lane-Detection-and-Object-Detection/4d24d437bfdcdbbe02bf7a496eeb65b142289ae8/output_vid/lane_tracker.mp4 -------------------------------------------------------------------------------- /output_vid/obj_detection.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/srianumakonda/Advanced-Lane-Detection-and-Object-Detection/4d24d437bfdcdbbe02bf7a496eeb65b142289ae8/output_vid/obj_detection.mp4 -------------------------------------------------------------------------------- /output_vid/project_video.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/srianumakonda/Advanced-Lane-Detection-and-Object-Detection/4d24d437bfdcdbbe02bf7a496eeb65b142289ae8/output_vid/project_video.mp4 -------------------------------------------------------------------------------- /test.py: -------------------------------------------------------------------------------- 1 | import matplotlib.image as mpimg 2 | import matplotlib.pyplot as plt 3 | import numpy as np 4 | import cv2 5 | import os 6 | import time 7 | import pickle 8 | from sklearn.svm import LinearSVC 9 | from sklearn.preprocessing import StandardScaler 10 | from skimage.feature import hog 11 | from sklearn.model_selection import train_test_split 12 | from scipy.ndimage.measurements import label 13 | from moviepy.editor import VideoFileClip 14 | from IPython.display import HTML 15 | from skimage.feature import hog 16 | 17 | def single_img_features(img, color_space='YCrCb', spatial_size=(16, 16), 18 | hist_bins=32, orient=8, 19 | pix_per_cell=8, cell_per_block=2, hog_channel="ALL", 20 | spatial_feat=True, hist_feat=True, hog_feat=True): 21 | img_features = [] 22 | if color_space != 'RGB': 23 | if color_space == 'HSV': 24 | feature_image = cv2.cvtColor(img, cv2.COLOR_RGB2HSV) 25 | elif color_space == 'LUV': 26 | feature_image = cv2.cvtColor(img, cv2.COLOR_RGB2LUV) 27 | elif color_space == 'HLS': 28 | feature_image = cv2.cvtColor(img, cv2.COLOR_RGB2HLS) 29 | elif color_space == 'YUV': 30 | feature_image = cv2.cvtColor(img, cv2.COLOR_RGB2YUV) 31 | elif color_space == 'YCrCb': 32 | feature_image = cv2.cvtColor(img, cv2.COLOR_RGB2YCrCb) 33 | else: feature_image = np.copy(img) 34 | if spatial_feat == True: 35 | spatial_features = bin_spatial(feature_image, size=spatial_size) 36 | img_features.append(spatial_features) 37 | if hist_feat == True: 38 | hist_features = color_hist(feature_image, nbins=hist_bins) 39 | img_features.append(hist_features) 40 | if hog_feat == True: 41 | if hog_channel == 'ALL': 42 | hog_features = [] 43 | for channel in range(feature_image.shape[2]): 44 | hog_features.extend(get_hog_features(feature_image[:,:,channel], 45 | orient, pix_per_cell, cell_per_block, 46 | vis=False, feature_vec=True)) 47 | else: 48 | hog_features = get_hog_features(feature_image[:,:,hog_channel], orient, 49 | pix_per_cell, cell_per_block, vis=False, feature_vec=True) 50 | img_features.append(hog_features) 51 | 52 | return np.concatenate(img_features) 53 | 54 | def search_windows(img, windows, clf, scaler, color_space='YCrCb', 55 | spatial_size=(16, 16), hist_bins=32, 56 | hist_range=(0, 256), orient=8, 57 | pix_per_cell=8, cell_per_block=2, 58 | hog_channel="ALL", spatial_feat=True, 59 | hist_feat=True, hog_feat=True): 60 | 61 | on_windows = [] 62 | for window in windows: 63 | test_img = cv2.resize(img[window[0][1]:window[1][1], window[0][0]:window[1][0]], (64, 64)) 64 | features = single_img_features(test_img, color_space=color_space, 65 | spatial_size=spatial_size, hist_bins=hist_bins, 66 | orient=orient, pix_per_cell=pix_per_cell, 67 | cell_per_block=cell_per_block, 68 | hog_channel=hog_channel, spatial_feat=spatial_feat, 69 | hist_feat=hist_feat, hog_feat=hog_feat) 70 | test_features = scaler.transform(np.array(features).reshape(1, -1)) 71 | prediction = clf.predict(test_features) 72 | if prediction == 1: 73 | on_windows.append(window) 74 | return on_windows 75 | 76 | def get_hog_features(img, orient, pix_per_cell, cell_per_block, 77 | vis=False, feature_vec=True): 78 | if vis == True: 79 | features, hog_image = hog(img, orientations=orient, 80 | pixels_per_cell=(pix_per_cell, pix_per_cell), 81 | block_norm= 'L2-Hys', 82 | cells_per_block=(cell_per_block, cell_per_block), 83 | transform_sqrt=True, 84 | visualize=vis, feature_vector=feature_vec) 85 | return features, hog_image 86 | else: 87 | features = hog(img, orientations=orient, 88 | pixels_per_cell=(pix_per_cell, pix_per_cell), 89 | cells_per_block=(cell_per_block, cell_per_block), 90 | block_norm= 'L2-Hys', 91 | transform_sqrt=True, 92 | visualize=vis, feature_vector=feature_vec) 93 | return features 94 | 95 | def bin_spatial(img, size=(32, 32)): 96 | color1 = cv2.resize(img[:,:,0], size).ravel() 97 | color2 = cv2.resize(img[:,:,1], size).ravel() 98 | color3 = cv2.resize(img[:,:,2], size).ravel() 99 | return np.hstack((color1, color2, color3)) 100 | 101 | def color_hist(img, nbins=32, bins_range=(0, 256)): 102 | channel1_hist = np.histogram(img[:,:,0], bins=nbins, range=bins_range) 103 | channel2_hist = np.histogram(img[:,:,1], bins=nbins, range=bins_range) 104 | channel3_hist = np.histogram(img[:,:,2], bins=nbins, range=bins_range) 105 | hist_features = np.concatenate((channel1_hist[0], channel2_hist[0], channel3_hist[0])) 106 | return hist_features 107 | 108 | def extract_features(imgs, color_space='RGB', spatial_size=(32, 32), 109 | hist_bins=32, orient=9, 110 | pix_per_cell=8, cell_per_block=2, hog_channel="ALL", 111 | spatial_feat=True, hist_feat=True, hog_feat=True): 112 | features = [] 113 | for file in imgs: 114 | file_features = [] 115 | # Read in each one by one 116 | image = mpimg.imread(file) 117 | if color_space != 'RGB': 118 | if color_space == 'HSV': 119 | feature_image = cv2.cvtColor(image, cv2.COLOR_RGB2HSV) 120 | elif color_space == 'LUV': 121 | feature_image = cv2.cvtColor(image, cv2.COLOR_RGB2LUV) 122 | elif color_space == 'HLS': 123 | feature_image = cv2.cvtColor(image, cv2.COLOR_RGB2HLS) 124 | elif color_space == 'YUV': 125 | feature_image = cv2.cvtColor(image, cv2.COLOR_RGB2YUV) 126 | elif color_space == 'YCrCb': 127 | feature_image = cv2.cvtColor(image, cv2.COLOR_RGB2YCrCb) 128 | else: feature_image = np.copy(image) 129 | 130 | if spatial_feat == True: 131 | spatial_features = bin_spatial(feature_image, size=spatial_size) 132 | file_features.append(spatial_features) 133 | if hist_feat == True: 134 | hist_features = color_hist(feature_image, nbins=hist_bins) 135 | file_features.append(hist_features) 136 | if hog_feat == True: 137 | if hog_channel == 'ALL': 138 | hog_features = [] 139 | for channel in range(feature_image.shape[2]): 140 | hog_features.append(get_hog_features(feature_image[:,:,channel], 141 | orient, pix_per_cell, cell_per_block, 142 | vis=False, feature_vec=True)) 143 | hog_features = np.ravel(hog_features) 144 | else: 145 | hog_features = get_hog_features(feature_image[:,:,hog_channel], orient, 146 | pix_per_cell, cell_per_block, vis=False, feature_vec=True) 147 | file_features.append(hog_features) 148 | features.append(np.concatenate(file_features)) 149 | return features 150 | 151 | def slide_window(img, x_start_stop=[None, None], y_start_stop=[400,700], 152 | xy_window=(64, 64), xy_overlap=(0.85, 0.85)): 153 | 154 | if x_start_stop[0] == None: 155 | x_start_stop[0] = 0 156 | if x_start_stop[1] == None: 157 | x_start_stop[1] = img.shape[1] 158 | if y_start_stop[0] == None: 159 | y_start_stop[0] = 0 160 | if y_start_stop[1] == None: 161 | y_start_stop[1] = img.shape[0] 162 | xspan = x_start_stop[1] - x_start_stop[0] 163 | yspan = y_start_stop[1] - y_start_stop[0] 164 | nx_pix_per_step = np.int(xy_window[0]*(1 - xy_overlap[0])) 165 | ny_pix_per_step = np.int(xy_window[1]*(1 - xy_overlap[1])) 166 | nx_buffer = np.int(xy_window[0]*(xy_overlap[0])) 167 | ny_buffer = np.int(xy_window[1]*(xy_overlap[1])) 168 | nx_windows = np.int((xspan-nx_buffer)/nx_pix_per_step) 169 | ny_windows = np.int((yspan-ny_buffer)/ny_pix_per_step) 170 | 171 | window_list = [] 172 | for ys in range(ny_windows): 173 | for xs in range(nx_windows): 174 | # Calculate window position 175 | startx = xs*nx_pix_per_step + x_start_stop[0] 176 | endx = startx + xy_window[0] 177 | starty = ys*ny_pix_per_step + y_start_stop[0] 178 | endy = starty + xy_window[1] 179 | window_list.append(((startx, starty), (endx, endy))) 180 | return window_list 181 | 182 | def draw_boxes(img, bboxes, color=(0, 0, 255), thick=6): 183 | imcopy = np.copy(img) 184 | for bbox in bboxes: 185 | cv2.rectangle(imcopy, bbox[0], bbox[1], color, thick) 186 | return imcopy 187 | 188 | cars = [] 189 | notcars = [] 190 | 191 | for img in os.listdir("./vehicles_smallset/"): 192 | cars.append("./vehicles_smallset/"+img) 193 | 194 | for img in os.listdir("./non-vehicles_smallset/"): 195 | notcars.append("./non-vehicles_smallset/"+img) 196 | 197 | # sample_size = 500 198 | # cars = cars[0:sample_size] 199 | # notcars = notcars[0:sample_size] 200 | 201 | color_space = 'YCrCb' 202 | orient = 8 203 | pix_per_cell = 8 204 | cell_per_block = 2 205 | hog_channel = 'ALL' 206 | spatial_size = (16, 16) 207 | hist_bins = 32 208 | spatial_feat = True 209 | hist_feat = True 210 | hog_feat = True 211 | y_start_stop = [None, None] 212 | 213 | car_features = extract_features(cars, color_space=color_space, 214 | spatial_size=spatial_size, hist_bins=hist_bins, 215 | orient=orient, pix_per_cell=pix_per_cell, 216 | cell_per_block=cell_per_block, 217 | hog_channel=hog_channel, spatial_feat=spatial_feat, 218 | hist_feat=hist_feat, hog_feat=hog_feat) 219 | notcar_features = extract_features(notcars, color_space=color_space, 220 | spatial_size=spatial_size, hist_bins=hist_bins, 221 | orient=orient, pix_per_cell=pix_per_cell, 222 | cell_per_block=cell_per_block, 223 | hog_channel=hog_channel, spatial_feat=spatial_feat, 224 | hist_feat=hist_feat, hog_feat=hog_feat) 225 | 226 | X = np.vstack((car_features, notcar_features)).astype(np.float64) 227 | y = np.hstack((np.ones(len(car_features)), np.zeros(len(notcar_features)))) 228 | 229 | rand_state = np.random.randint(0, 100) 230 | X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=rand_state) 231 | 232 | X_scaler = StandardScaler().fit(X_train) 233 | X_train = X_scaler.transform(X_train) 234 | X_test = X_scaler.transform(X_test) 235 | 236 | print('Using:',orient,'orientations',pix_per_cell, 237 | 'pixels per cell and', cell_per_block,'cells per block') 238 | print('Feature vector length:', len(X_train[0])) 239 | 240 | svc = LinearSVC() 241 | 242 | t=time.time() 243 | 244 | svc.fit(X_train, y_train) 245 | 246 | t2 = time.time() 247 | 248 | print(round(t2-t, 2), 'Seconds to train SVC...') 249 | 250 | print('Test Accuracy of SVC = ', round(svc.score(X_test, y_test), 4)) 251 | 252 | image = mpimg.imread('bbox-example-image.jpg') 253 | draw_image = np.copy(image) 254 | 255 | windows = slide_window(image, x_start_stop=[None, None], y_start_stop=y_start_stop, 256 | xy_window=(96, 96), xy_overlap=(0.5, 0.5)) 257 | 258 | hot_windows = search_windows(image, windows, svc, X_scaler, color_space=color_space, 259 | spatial_size=spatial_size, hist_bins=hist_bins, 260 | orient=orient, pix_per_cell=pix_per_cell, 261 | cell_per_block=cell_per_block, 262 | hog_channel=hog_channel, spatial_feat=spatial_feat, 263 | hist_feat=hist_feat, hog_feat=hog_feat) 264 | 265 | window_img = draw_boxes(draw_image, hot_windows, color=(0, 0, 255), thick=6) 266 | 267 | # plt.imshow(window_img) 268 | # plt.show() 269 | 270 | def add_heat(heatmap, bbox_list): 271 | for box in bbox_list: 272 | heatmap[box[0][1]:box[1][1], box[0][0]:box[1][0]] += 1 273 | return heatmap 274 | 275 | def apply_threshold(heatmap, threshold): 276 | heatmap[heatmap <= threshold] = 0 277 | return heatmap 278 | 279 | def draw_labeled_bboxes(img, labels): 280 | for car_number in range(1, labels[1]+1): 281 | nonzero = (labels[0] == car_number).nonzero() 282 | nonzeroy = np.array(nonzero[0]) 283 | nonzerox = np.array(nonzero[1]) 284 | bbox = ((np.min(nonzerox), np.min(nonzeroy)), (np.max(nonzerox), np.max(nonzeroy))) 285 | cv2.rectangle(img, bbox[0], bbox[1], (0,0,255), 6) 286 | # Return the image 287 | return img 288 | 289 | def process_image(img): 290 | windows = slide_window(img, xy_window=(84,84)) 291 | hot_windows = search_windows(img, windows, svc, X_scaler) 292 | heat = np.zeros_like(img[:,:,0]).astype(np.float) 293 | heat = add_heat(heat,hot_windows) 294 | heat = apply_threshold(heat,1) 295 | heatmap = np.clip(heat, 0, 255) 296 | labels = label(heatmap) 297 | draw_img = draw_labeled_bboxes(np.copy(img), labels) 298 | return draw_img 299 | 300 | output_vid = 'final_tracker.mp4' 301 | input_vid = 'lane_tracker.mp4' 302 | 303 | clip1 = VideoFileClip(input_vid) 304 | video_clip = clip1.fl_image(process_image) 305 | video_clip.write_videofile(output_vid, audio=False, threads=12) -------------------------------------------------------------------------------- /test_img/bbox-example-image.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/srianumakonda/Advanced-Lane-Detection-and-Object-Detection/4d24d437bfdcdbbe02bf7a496eeb65b142289ae8/test_img/bbox-example-image.jpg -------------------------------------------------------------------------------- /test_img/test1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/srianumakonda/Advanced-Lane-Detection-and-Object-Detection/4d24d437bfdcdbbe02bf7a496eeb65b142289ae8/test_img/test1.jpg -------------------------------------------------------------------------------- /test_img/test2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/srianumakonda/Advanced-Lane-Detection-and-Object-Detection/4d24d437bfdcdbbe02bf7a496eeb65b142289ae8/test_img/test2.jpg -------------------------------------------------------------------------------- /test_img/test3.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/srianumakonda/Advanced-Lane-Detection-and-Object-Detection/4d24d437bfdcdbbe02bf7a496eeb65b142289ae8/test_img/test3.jpg -------------------------------------------------------------------------------- /tracker.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import cv2 3 | class tracker(): 4 | 5 | def __init__(self, Mywindow_width, Mywindow_height, Mymargin, My_ym = 1, My_xm = 1, Mysmooth_factor = 15): 6 | self.recent_centers = [] 7 | self.window_width = Mywindow_width 8 | self.window_height = Mywindow_height 9 | self.margin = Mymargin 10 | self.ym_per_pix = My_ym 11 | self.xm_per_pix = My_xm 12 | self.smooth_factor = Mysmooth_factor 13 | 14 | def find_window_centroids(self, warped): 15 | window_width = self.window_width 16 | window_height = self.window_height 17 | margin = self.margin 18 | 19 | window_centroids = [] 20 | window = np.ones(window_width) 21 | 22 | l_sum = np.sum(warped[int(3*warped.shape[0]/4):,:int(warped.shape[1]/2)],axis=0) 23 | l_center = np.argmax(np.convolve(window,l_sum))-window_width/2 24 | 25 | r_sum = np.sum(warped[int(3*warped.shape[0]/4):,int(warped.shape[1]/2):],axis=0) 26 | r_center = np.argmax(np.convolve(window,r_sum))-window_width/2+int(warped.shape[1]/2) 27 | 28 | window_centroids.append((l_center,r_center)) 29 | 30 | for level in range(1,(int)(warped.shape[0]/window_height)): 31 | image_layer = np.sum(warped[int(warped.shape[0]-(level+1)*window_height):int(warped.shape[0]-level*window_height),:],axis=0) 32 | conv_signal = np.convolve(window,image_layer) 33 | 34 | offset = window_width/2 35 | l_min_index = int(max(l_center+offset-margin,0)) 36 | l_max_index = int(min(l_center+offset+margin,warped.shape[1])) 37 | l_center = np.argmax(conv_signal[l_min_index:l_max_index])+l_min_index-offset 38 | 39 | r_min_index = int(max(r_center+offset-margin,0)) 40 | r_max_index = int(min(r_center+offset+margin,warped.shape[1])) 41 | r_center = np.argmax(conv_signal[r_min_index:r_max_index])+r_min_index-offset 42 | 43 | window_centroids.append((l_center,r_center)) 44 | 45 | self.recent_centers.append(window_centroids) 46 | return np.average(self.recent_centers[-self.smooth_factor:],axis=0) -------------------------------------------------------------------------------- /video_gen.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import cv2 3 | import pickle 4 | import glob 5 | from tracker import tracker 6 | from moviepy.editor import VideoFileClip 7 | from IPython.display import HTML 8 | 9 | dist_pickle = pickle.load(open("calibration_pickle.p", "rb")) 10 | mtx = dist_pickle["mtx"] 11 | dist = dist_pickle["dist"] 12 | 13 | def abs_sobel_thresh(img, orient='x', sobel_kernel=3, thresh=(0,255)): 14 | gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY) 15 | if orient == 'x': 16 | abs_sobel = np.absolute(cv2.Sobel(gray, cv2.CV_64F, 1, 0)) 17 | if orient == 'y': 18 | abs_sobel = np.absolute(cv2.Sobel(gray, cv2.CV_64F, 0, 1)) 19 | 20 | scaled_sobel = np.uint8(255*abs_sobel/np.max(abs_sobel)) 21 | binary_output = np.zeros_like(scaled_sobel) 22 | 23 | binary_output[(scaled_sobel >= thresh[0]) & (scaled_sobel <= thresh[1])] = 1 24 | return binary_output 25 | 26 | def mag_thresh(img, sobel_kernel=3, mag_thresh=(0,255)): 27 | gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY) 28 | sobelx = cv2.Sobel(gray, cv2.CV_64F, 1, 0, ksize=sobel_kernel) 29 | sobely = cv2.Sobel(gray, cv2.CV_64F, 0, 1, ksize=sobel_kernel) 30 | 31 | gradmag = np.sqrt(sobelx**2+sobely**2) 32 | scale_factor = np.max(gradmag)/255 33 | gradmag = (gradmag/scale_factor).astype(np.uint8) 34 | 35 | binary_output = np.zeros_like(gradmag) 36 | binary_output[(gradmag >= mag_thresh[0]) & (gradmag <= mag_thresh[1])] = 1 37 | return binary_output 38 | 39 | def dir_threshold(image, sobel_kernel=3, thresh=(0, np.pi/2)): 40 | gray = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY) 41 | sobelx = cv2.Sobel(gray, cv2.CV_64F, 1, 0, ksize=sobel_kernel) 42 | sobely = cv2.Sobel(gray, cv2.CV_64F, 0, 1, ksize=sobel_kernel) 43 | 44 | with np.errstate(divide='ignore', invalid='ignore'): 45 | absgraddir = np.absolute(np.arctan(sobely/sobelx)) 46 | binary_output = np.zeros_like(absgraddir) 47 | binary_output[(absgraddir >= thresh[0]) & (absgraddir <= thresh[1])] = 1 48 | return binary_output 49 | 50 | def color_threshold(image, sthresh=(0,255), vthresh=(0,255), lthresh=(0,255)): 51 | hls = cv2.cvtColor(image, cv2.COLOR_RGB2HLS) 52 | s_channel = hls[:,:,2] 53 | s_binary = np.zeros_like(s_channel) 54 | s_binary[(s_channel >= sthresh[0]) & (s_channel <= sthresh[1])] = 1 55 | 56 | hsv = cv2.cvtColor(image, cv2.COLOR_RGB2HSV) 57 | v_channel = hsv[:,:,2] 58 | v_binary = np.zeros_like(v_channel) 59 | v_binary[(v_channel >= vthresh[0]) & (v_channel <= vthresh[1])] = 1 60 | 61 | output = np.zeros_like(s_channel) 62 | output[(s_binary == 1) & (v_binary == 1)] = 1 63 | return output 64 | 65 | def window_mask(width, height, img_ref, center, level): 66 | output = np.zeros_like(img_ref) 67 | output[int(img_ref.shape[0] - (level + 1) * height):int(img_ref.shape[0] - level * height), 68 | max(0, int(center - width / 2)):min(int(center + width / 2), img_ref.shape[1])] = 1 69 | return output 70 | 71 | def process_image(image): 72 | 73 | img = cv2.undistort(image, mtx, dist, None, mtx) 74 | 75 | preprocessImage = np.zeros_like(img[:,:,0]) 76 | gradx = abs_sobel_thresh(img, orient='x', thresh=(12,255)) 77 | grady = abs_sobel_thresh(img, orient='x', thresh=(25,255)) 78 | c_binary = color_threshold(img, sthresh=(100,255),vthresh=(50,255)) 79 | preprocessImage[((gradx==1)&(grady==1)|(c_binary==1))] = 255 80 | 81 | img_size = (img.shape[1], img.shape[0]) 82 | bot_width = .75 #changed from .76 83 | mid_width = .1 #changed this value - seemed to work a lot better than 0.08 84 | height_pct = .62 85 | bottom_trim = .935 86 | src = np.float32([[img.shape[1]*(.5-mid_width/2),img.shape[0]*height_pct],[img.shape[1]*(.5+mid_width/2),img.shape[0]*height_pct], 87 | [img.shape[1]*(.5+bot_width/2),img.shape[0]*bottom_trim],[img.shape[1]*(.5-bot_width/2),img.shape[0]*bottom_trim]]) 88 | offset = img_size[0]*.25 89 | dst = np.float32([[offset, 0], [img_size[0]-offset, 0],[img_size[0]-offset, img_size[1]],[offset, img_size[1]]]) 90 | 91 | M = cv2.getPerspectiveTransform(src, dst) 92 | Minv = cv2.getPerspectiveTransform(dst, src) 93 | warped = cv2.warpPerspective(preprocessImage, M, img_size,flags=cv2.INTER_LINEAR) 94 | 95 | window_width = 25 96 | window_height = 80 97 | curve_centers = tracker(Mywindow_width=window_width,Mywindow_height=window_height,Mymargin=25,My_ym=10/720,My_xm=4/384,Mysmooth_factor=15) 98 | window_centroids = curve_centers.find_window_centroids(warped) 99 | 100 | l_points = np.zeros_like(warped) 101 | r_points = np.zeros_like(warped) 102 | 103 | rightx = [] 104 | leftx = [] 105 | 106 | for level in range(0,len(window_centroids)): 107 | l_mask = window_mask(window_width,window_height,warped,window_centroids[level][0],level) 108 | r_mask = window_mask(window_width,window_height,warped,window_centroids[level][1],level) 109 | 110 | leftx.append(window_centroids[level][0]) 111 | rightx.append(window_centroids[level][1]) 112 | 113 | l_points[(l_points==255)|((l_mask==1))] = 255 114 | r_points[(r_points==255)|((r_mask==1))] = 255 115 | 116 | template = np.array(r_points+l_points,np.uint8) 117 | zero_channel = np.zeros_like(template) 118 | template = np.array(cv2.merge((zero_channel,template,zero_channel)),np.uint8) 119 | warpage = np.array(cv2.merge((warped,warped,warped)),np.uint8) 120 | result = cv2.addWeighted(warpage,1,template,0.5,0.0) 121 | 122 | yvals = range(0,warped.shape[0]) 123 | res_yvals = np.arange(warped.shape[0]-(window_height/2),0,-window_height) 124 | 125 | left_fit = np.polyfit(res_yvals,leftx,2) 126 | left_fitx = left_fit[0]*yvals*yvals + left_fit[1]*yvals + left_fit[2] 127 | left_fitx = np.array(left_fitx,np.int32) 128 | 129 | right_fit = np.polyfit(res_yvals,rightx,2) 130 | right_fitx = right_fit[0]*yvals*yvals + right_fit[1]*yvals + right_fit[2] 131 | right_fitx = np.array(right_fitx,np.int32) 132 | 133 | left_lane = np.array(list(zip(np.concatenate((left_fitx-window_width/2,left_fitx[::-1]+window_width/2),axis=0),np.concatenate((yvals,yvals[::-1]),axis=0))),np.int32) 134 | 135 | right_lane = np.array(list(zip(np.concatenate((right_fitx-window_width/2,right_fitx[::-1]+window_width/2),axis=0),np.concatenate((yvals,yvals[::-1]),axis=0))),np.int32) 136 | 137 | inner_lane = np.array(list(zip(np.concatenate((left_fitx+window_width/2,right_fitx[::-1]+window_width/2),axis=0),np.concatenate((yvals,yvals[::-1]),axis=0))),np.int32) 138 | 139 | road = np.zeros_like(img) 140 | road_bkg = np.zeros_like(img) 141 | cv2.fillPoly(road,[left_lane],color=[255,0,0]) 142 | cv2.fillPoly(road,[inner_lane],color=[0,255,0]) 143 | cv2.fillPoly(road,[right_lane],color=[0,0,255]) 144 | cv2.fillPoly(road_bkg,[left_lane],color=[255,255,255]) 145 | cv2.fillPoly(road_bkg,[right_lane],color=[255,255,255]) 146 | 147 | road_warped = cv2.warpPerspective(road,Minv,img_size,flags=cv2.INTER_LINEAR) 148 | road_warped_bkg = cv2.warpPerspective(road_bkg,Minv,img_size,flags=cv2.INTER_LINEAR) 149 | 150 | base = cv2.addWeighted(img, 1.0, road_warped_bkg, -1.0, 0.0) 151 | result = cv2.addWeighted(base,1.0,road_warped,0.7,0.0) 152 | 153 | #measure pixels in y and x directions 154 | ym_per_pix = curve_centers.ym_per_pix 155 | xm_per_pix = curve_centers.xm_per_pix 156 | 157 | curve_fit_cr = np.polyfit(np.array(res_yvals,np.float32)*ym_per_pix,np.array(leftx,np.float32)*xm_per_pix,2) 158 | curverad = ((1+(2*curve_fit_cr[0]*yvals[-1]*ym_per_pix+curve_fit_cr[1])**2)**1.5)/np.absolute(2*curve_fit_cr[0]) #remember that it's the equation from the lesson (derivatives) - radius of curvature 159 | 160 | camera_center = (left_fitx[-1] + right_fitx[-1])/2 161 | center_diff = (camera_center-warped.shape[1]/2)*xm_per_pix 162 | side_pos = 'left' 163 | if center_diff <= 0: 164 | side_pos = 'right' 165 | 166 | cv2.putText(result,'Radius of curvature = '+str(round(curverad,3))+'(m)',(50,50),cv2.FONT_HERSHEY_SIMPLEX,1,(255,255,255),2) 167 | cv2.putText(result,'Vehicle is '+str(abs(round(center_diff,3)))+'m '+side_pos+' of center',(50,100), cv2.FONT_HERSHEY_SIMPLEX,1,(255,255,255),2) 168 | 169 | return result 170 | 171 | output_vid = 'lane_tracker.mp4' 172 | input_vid = 'project_video.mp4' 173 | 174 | clip1 = VideoFileClip(input_vid) 175 | video_clip = clip1.fl_image(process_image) 176 | video_clip.write_videofile(output_vid, audio=False) --------------------------------------------------------------------------------