├── License.txt ├── README.md ├── driving_data.py ├── model.py ├── run.py ├── run_dataset.py ├── save ├── checkpoint ├── model.ckpt └── model.ckpt.meta ├── steering_wheel_image.jpg └── train.py /License.txt: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2016 Sully Chen 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Autopilot-TensorFlow 2 | A TensorFlow implementation of this [Nvidia paper](https://arxiv.org/pdf/1604.07316.pdf) with some changes. For a summary of the design process and FAQs, see [this medium article I wrote](https://medium.com/@sullyfchen/how-a-high-school-junior-made-a-self-driving-car-705fa9b6e860). 3 | 4 | # IMPORTANT 5 | Absolutely, under NO circumstance, should one ever pilot a car using computer vision software trained with this code (or any home made software for that matter). It is extremely dangerous to use your own self-driving software in a car, even if you think you know what you're doing, not to mention it is quite illegal in most places and any accidents will land you in huge lawsuits. 6 | 7 | This code is purely for research and statistics, absolutley NOT for application or testing of any sort. 8 | 9 | # How to Use 10 | Download the [dataset](https://github.com/SullyChen/driving-datasets) and extract into the repository folder 11 | 12 | Use `python train.py` to train the model 13 | 14 | Use `python run.py` to run the model on a live webcam feed 15 | 16 | Use `python run_dataset.py` to run the model on the dataset 17 | 18 | To visualize training using Tensorboard use `tensorboard --logdir=./logs`, then open http://0.0.0.0:6006/ into your web browser. 19 | -------------------------------------------------------------------------------- /driving_data.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import random 3 | import numpy as np 4 | 5 | xs = [] 6 | ys = [] 7 | 8 | #points to the end of the last batch 9 | train_batch_pointer = 0 10 | val_batch_pointer = 0 11 | 12 | #read data.txt 13 | with open("driving_dataset/data.txt") as f: 14 | for line in f: 15 | xs.append("driving_dataset/" + line.split()[0]) 16 | #the paper by Nvidia uses the inverse of the turning radius, 17 | #but steering wheel angle is proportional to the inverse of turning radius 18 | #so the steering wheel angle in radians is used as the output 19 | ys.append(float(line.split()[1]) * 3.14159265 / 180) 20 | 21 | #get number of images 22 | num_images = len(xs) 23 | 24 | #shuffle list of images 25 | c = list(zip(xs, ys)) 26 | random.shuffle(c) 27 | xs, ys = zip(*c) 28 | 29 | train_xs = xs[:int(len(xs) * 0.8)] 30 | train_ys = ys[:int(len(xs) * 0.8)] 31 | 32 | val_xs = xs[-int(len(xs) * 0.2):] 33 | val_ys = ys[-int(len(xs) * 0.2):] 34 | 35 | num_train_images = len(train_xs) 36 | num_val_images = len(val_xs) 37 | 38 | def LoadTrainBatch(batch_size): 39 | global train_batch_pointer 40 | x_out = [] 41 | y_out = [] 42 | for i in range(0, batch_size): 43 | x_out.append(cv2.resize(cv2.imread(train_xs[(train_batch_pointer + i) % num_train_images])[-150:], (200, 66)) / 255.0) 44 | y_out.append([train_ys[(train_batch_pointer + i) % num_train_images]]) 45 | train_batch_pointer += batch_size 46 | return x_out, y_out 47 | 48 | def LoadValBatch(batch_size): 49 | global val_batch_pointer 50 | x_out = [] 51 | y_out = [] 52 | for i in range(0, batch_size): 53 | x_out.append(cv2.resize(cv2.imread(val_xs[(val_batch_pointer + i) % num_val_images])[-150:], (200, 66)) / 255.0) 54 | y_out.append([val_ys[(val_batch_pointer + i) % num_val_images]]) 55 | val_batch_pointer += batch_size 56 | return x_out, y_out 57 | -------------------------------------------------------------------------------- /model.py: -------------------------------------------------------------------------------- 1 | import tensorflow.compat.v1 as tf 2 | tf.disable_v2_behavior() 3 | import scipy 4 | 5 | def weight_variable(shape): 6 | initial = tf.truncated_normal(shape, stddev=0.1) 7 | return tf.Variable(initial) 8 | 9 | def bias_variable(shape): 10 | initial = tf.constant(0.1, shape=shape) 11 | return tf.Variable(initial) 12 | 13 | def conv2d(x, W, stride): 14 | return tf.nn.conv2d(x, W, strides=[1, stride, stride, 1], padding='VALID') 15 | 16 | x = tf.placeholder(tf.float32, shape=[None, 66, 200, 3]) 17 | y_ = tf.placeholder(tf.float32, shape=[None, 1]) 18 | 19 | x_image = x 20 | 21 | #first convolutional layer 22 | W_conv1 = weight_variable([5, 5, 3, 24]) 23 | b_conv1 = bias_variable([24]) 24 | 25 | h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1, 2) + b_conv1) 26 | 27 | #second convolutional layer 28 | W_conv2 = weight_variable([5, 5, 24, 36]) 29 | b_conv2 = bias_variable([36]) 30 | 31 | h_conv2 = tf.nn.relu(conv2d(h_conv1, W_conv2, 2) + b_conv2) 32 | 33 | #third convolutional layer 34 | W_conv3 = weight_variable([5, 5, 36, 48]) 35 | b_conv3 = bias_variable([48]) 36 | 37 | h_conv3 = tf.nn.relu(conv2d(h_conv2, W_conv3, 2) + b_conv3) 38 | 39 | #fourth convolutional layer 40 | W_conv4 = weight_variable([3, 3, 48, 64]) 41 | b_conv4 = bias_variable([64]) 42 | 43 | h_conv4 = tf.nn.relu(conv2d(h_conv3, W_conv4, 1) + b_conv4) 44 | 45 | #fifth convolutional layer 46 | W_conv5 = weight_variable([3, 3, 64, 64]) 47 | b_conv5 = bias_variable([64]) 48 | 49 | h_conv5 = tf.nn.relu(conv2d(h_conv4, W_conv5, 1) + b_conv5) 50 | 51 | #FCL 1 52 | W_fc1 = weight_variable([1152, 1164]) 53 | b_fc1 = bias_variable([1164]) 54 | 55 | h_conv5_flat = tf.reshape(h_conv5, [-1, 1152]) 56 | h_fc1 = tf.nn.relu(tf.matmul(h_conv5_flat, W_fc1) + b_fc1) 57 | 58 | keep_prob = tf.placeholder(tf.float32) 59 | h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob) 60 | 61 | #FCL 2 62 | W_fc2 = weight_variable([1164, 100]) 63 | b_fc2 = bias_variable([100]) 64 | 65 | h_fc2 = tf.nn.relu(tf.matmul(h_fc1_drop, W_fc2) + b_fc2) 66 | 67 | h_fc2_drop = tf.nn.dropout(h_fc2, keep_prob) 68 | 69 | #FCL 3 70 | W_fc3 = weight_variable([100, 50]) 71 | b_fc3 = bias_variable([50]) 72 | 73 | h_fc3 = tf.nn.relu(tf.matmul(h_fc2_drop, W_fc3) + b_fc3) 74 | 75 | h_fc3_drop = tf.nn.dropout(h_fc3, keep_prob) 76 | 77 | #FCL 3 78 | W_fc4 = weight_variable([50, 10]) 79 | b_fc4 = bias_variable([10]) 80 | 81 | h_fc4 = tf.nn.relu(tf.matmul(h_fc3_drop, W_fc4) + b_fc4) 82 | 83 | h_fc4_drop = tf.nn.dropout(h_fc4, keep_prob) 84 | 85 | #Output 86 | W_fc5 = weight_variable([10, 1]) 87 | b_fc5 = bias_variable([1]) 88 | 89 | y = tf.multiply(tf.atan(tf.matmul(h_fc4_drop, W_fc5) + b_fc5), 2) #scale the atan output 90 | -------------------------------------------------------------------------------- /run.py: -------------------------------------------------------------------------------- 1 | import tensorflow.compat.v1 as tf 2 | tf.disable_v2_behavior() 3 | import model 4 | import cv2 5 | from subprocess import call 6 | import os 7 | 8 | #check if on windows OS 9 | windows = False 10 | if os.name == 'nt': 11 | windows = True 12 | 13 | sess = tf.InteractiveSession() 14 | saver = tf.train.Saver() 15 | saver.restore(sess, "save/model.ckpt") 16 | 17 | img = cv2.imread('steering_wheel_image.jpg',0) 18 | rows,cols = img.shape 19 | 20 | smoothed_angle = 0 21 | 22 | cap = cv2.VideoCapture(0) 23 | while(cv2.waitKey(10) != ord('q')): 24 | ret, frame = cap.read() 25 | image = cv2.resize(frame, (200, 66)) / 255.0 26 | degrees = model.y.eval(feed_dict={model.x: [image], model.keep_prob: 1.0})[0][0] * 180 / 3.14159265 27 | if not windows: 28 | call("clear") 29 | print("Predicted steering angle: " + str(degrees) + " degrees") 30 | cv2.imshow('frame', frame) 31 | #make smooth angle transitions by turning the steering wheel based on the difference of the current angle 32 | #and the predicted angle 33 | smoothed_angle += 0.2 * pow(abs((degrees - smoothed_angle)), 2.0 / 3.0) * (degrees - smoothed_angle) / abs(degrees - smoothed_angle) 34 | M = cv2.getRotationMatrix2D((cols/2,rows/2),-smoothed_angle,1) 35 | dst = cv2.warpAffine(img,M,(cols,rows)) 36 | cv2.imshow("steering wheel", dst) 37 | 38 | cap.release() 39 | cv2.destroyAllWindows() 40 | -------------------------------------------------------------------------------- /run_dataset.py: -------------------------------------------------------------------------------- 1 | import tensorflow.compat.v1 as tf 2 | tf.disable_v2_behavior() 3 | import model 4 | import cv2 5 | from subprocess import call 6 | import os 7 | 8 | #check if on windows OS 9 | windows = False 10 | if os.name == 'nt': 11 | windows = True 12 | 13 | sess = tf.InteractiveSession() 14 | saver = tf.train.Saver() 15 | saver.restore(sess, "save/model.ckpt") 16 | 17 | img = cv2.imread('steering_wheel_image.jpg',0) 18 | rows,cols = img.shape 19 | 20 | smoothed_angle = 0 21 | 22 | i = 0 23 | while(cv2.waitKey(10) != ord('q')): 24 | full_image = cv2.imread("driving_dataset/" + str(i) + ".jpg") 25 | image = cv2.resize(full_image[-150:], (200, 66)) / 255.0 26 | degrees = model.y.eval(feed_dict={model.x: [image], model.keep_prob: 1.0})[0][0] * 180.0 / 3.14159265 27 | if not windows: 28 | call("clear") 29 | print("Predicted steering angle: " + str(degrees) + " degrees") 30 | cv2.imshow("frame", full_image) 31 | #make smooth angle transitions by turning the steering wheel based on the difference of the current angle 32 | #and the predicted angle 33 | smoothed_angle += 0.2 * pow(abs((degrees - smoothed_angle)), 2.0 / 3.0) * (degrees - smoothed_angle) / abs(degrees - smoothed_angle) 34 | M = cv2.getRotationMatrix2D((cols/2,rows/2),-smoothed_angle,1) 35 | dst = cv2.warpAffine(img,M,(cols,rows)) 36 | cv2.imshow("steering wheel", dst) 37 | i += 1 38 | 39 | cv2.destroyAllWindows() 40 | -------------------------------------------------------------------------------- /save/checkpoint: -------------------------------------------------------------------------------- 1 | model_checkpoint_path: "model.ckpt" 2 | all_model_checkpoint_paths: "model.ckpt" 3 | -------------------------------------------------------------------------------- /save/model.ckpt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SullyChen/Autopilot-TensorFlow/33b9e5f91a1c25fbe5cf5644b01f4d9d9a48193e/save/model.ckpt -------------------------------------------------------------------------------- /save/model.ckpt.meta: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SullyChen/Autopilot-TensorFlow/33b9e5f91a1c25fbe5cf5644b01f4d9d9a48193e/save/model.ckpt.meta -------------------------------------------------------------------------------- /steering_wheel_image.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SullyChen/Autopilot-TensorFlow/33b9e5f91a1c25fbe5cf5644b01f4d9d9a48193e/steering_wheel_image.jpg -------------------------------------------------------------------------------- /train.py: -------------------------------------------------------------------------------- 1 | import os 2 | import tensorflow.compat.v1 as tf 3 | tf.disable_v2_behavior() 4 | from tensorflow.core.protobuf import saver_pb2 5 | import driving_data 6 | import model 7 | 8 | LOGDIR = './save' 9 | 10 | sess = tf.InteractiveSession() 11 | 12 | L2NormConst = 0.001 13 | 14 | train_vars = tf.trainable_variables() 15 | 16 | loss = tf.reduce_mean(tf.square(tf.subtract(model.y_, model.y))) + tf.add_n([tf.nn.l2_loss(v) for v in train_vars]) * L2NormConst 17 | train_step = tf.train.AdamOptimizer(1e-4).minimize(loss) 18 | sess.run(tf.global_variables_initializer()) 19 | 20 | # create a summary to monitor cost tensor 21 | tf.summary.scalar("loss", loss) 22 | # merge all summaries into a single op 23 | merged_summary_op = tf.summary.merge_all() 24 | 25 | saver = tf.train.Saver(write_version = saver_pb2.SaverDef.V2) 26 | 27 | # op to write logs to Tensorboard 28 | logs_path = './logs' 29 | summary_writer = tf.summary.FileWriter(logs_path, graph=tf.get_default_graph()) 30 | 31 | epochs = 30 32 | batch_size = 100 33 | 34 | # train over the dataset about 30 times 35 | for epoch in range(epochs): 36 | for i in range(int(driving_data.num_images/batch_size)): 37 | xs, ys = driving_data.LoadTrainBatch(batch_size) 38 | train_step.run(feed_dict={model.x: xs, model.y_: ys, model.keep_prob: 0.8}) 39 | if i % 10 == 0: 40 | xs, ys = driving_data.LoadValBatch(batch_size) 41 | loss_value = loss.eval(feed_dict={model.x:xs, model.y_: ys, model.keep_prob: 1.0}) 42 | print("Epoch: %d, Step: %d, Loss: %g" % (epoch, epoch * batch_size + i, loss_value)) 43 | 44 | # write logs at every iteration 45 | summary = merged_summary_op.eval(feed_dict={model.x:xs, model.y_: ys, model.keep_prob: 1.0}) 46 | summary_writer.add_summary(summary, epoch * driving_data.num_images/batch_size + i) 47 | 48 | if i % batch_size == 0: 49 | if not os.path.exists(LOGDIR): 50 | os.makedirs(LOGDIR) 51 | checkpoint_path = os.path.join(LOGDIR, "model.ckpt") 52 | filename = saver.save(sess, checkpoint_path) 53 | print("Model saved in file: %s" % filename) 54 | 55 | print("Run the command line:\n" \ 56 | "--> tensorboard --logdir=./logs " \ 57 | "\nThen open http://0.0.0.0:6006/ into your web browser") 58 | --------------------------------------------------------------------------------