├── .gitignore ├── LICENSE ├── README.md ├── autonomous.py ├── collect.py ├── preview.py ├── requirements.txt ├── settings.json ├── suiron ├── __init__.py ├── core │ ├── SuironCV.py │ ├── SuironIO.py │ ├── SuironML.py │ ├── SuironVZ.py │ └── __init__.py ├── tools │ ├── autonomous_test.py │ └── ml_test.py └── utils │ ├── __init__.py │ ├── datasets.py │ ├── file_finder.py │ ├── functions.py │ ├── img_serializer.py │ └── preview.py ├── train.py ├── visualize_collect.py └── visualize_predict.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | 27 | # PyInstaller 28 | # Usually these files are written by a python script from a template 29 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 30 | *.manifest 31 | *.spec 32 | 33 | # Installer logs 34 | pip-log.txt 35 | pip-delete-this-directory.txt 36 | 37 | # Unit test / coverage reports 38 | htmlcov/ 39 | .tox/ 40 | .coverage 41 | .coverage.* 42 | .cache 43 | nosetests.xml 44 | coverage.xml 45 | *,cover 46 | .hypothesis/ 47 | 48 | # Translations 49 | *.mo 50 | *.pot 51 | 52 | # Django stuff: 53 | *.log 54 | local_settings.py 55 | 56 | # Flask stuff: 57 | instance/ 58 | .webassets-cache 59 | 60 | # Scrapy stuff: 61 | .scrapy 62 | 63 | # Sphinx documentation 64 | docs/_build/ 65 | 66 | # PyBuilder 67 | target/ 68 | 69 | # IPython Notebook 70 | .ipynb_checkpoints 71 | 72 | # pyenv 73 | .python-version 74 | 75 | # celery beat schedule file 76 | celerybeat-schedule 77 | 78 | # dotenv 79 | .env 80 | 81 | # VSCode 82 | .vscode 83 | 84 | # Data 85 | data/ 86 | mnist/ 87 | checkpoints/ 88 | 89 | # virtualenv 90 | venv/ 91 | ENV/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | 96 | # Rope project settings 97 | .ropeproject 98 | 99 | # Models 100 | cnn_* 101 | nn_* 102 | checkpoint 103 | *.ckpt 104 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2016 Kendrick Tan 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Suiron 2 | ### Machine Learning for RC Cars 3 | 4 | ## Prediction visualization (green = actual, blue = prediction) 5 | ![](https://thumbs.gfycat.com/DarlingForkedAcaciarat-size_restricted.gif) 6 | 7 | ## Click the video below to see it in action! 8 | [![IMAGE ALT TEXT](http://img.youtube.com/vi/tFwCyHdAWf0/0.jpg)](https://youtu.be/tFwCyHdAWf0 "Machine Learning Car") 9 | 10 | ## Dependencies 11 | #### __Python 2.7__ was chosen as it was supported by all the libraries used at the time 12 | ``` 13 | sudo apt-get update 14 | sudo apt-get upgrade 15 | sudo apt-get install python-opencv python-dev 16 | 17 | export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.10.0-cp27-none-linux_x86_64.whl 18 | sudo pip install --upgrade $TF_BINARY_URL 19 | 20 | sudo pip install -r requirements.txt 21 | ``` 22 | 23 | ## Collecting data 24 | ``` 25 | python collect.py 26 | ``` 27 | 28 | ## Visualizing collected data 29 | ``` 30 | python visualize_collect.py 31 | ``` 32 | 33 | ## Training data 34 | ``` 35 | python train.py 36 | ``` 37 | 38 | ## Visualizing predicted data 39 | ``` 40 | python visualize_predict.py 41 | ``` 42 | 43 | # References 44 | 45 | Blog Post detailing how the hardware and software communicate - [Communicating between RC Car and the On-board Computer - Jabelone](http://jabelone.com.au/blog/make-autonomous-car-code-included/) 46 | 47 | 48 | Communication between hardware and software repo - [car-controller](https://github.com/jabelone/car-controller) 49 | 50 | Neural Network architecture was based on NVIDIA's Self-driving car paper - [End-To-End Learning for Self-Driving Cars](https://arxiv.org/pdf/1604.07316v1.pdf) 51 | -------------------------------------------------------------------------------- /autonomous.py: -------------------------------------------------------------------------------- 1 | import time 2 | import json 3 | import numpy as np 4 | 5 | from suiron.core.SuironIO import SuironIO 6 | from suiron.core.SuironML import get_cnn_model 7 | from suiron.utils.functions import cnn_to_raw 8 | 9 | # Image settings 10 | with open('settings.json') as d: 11 | SETTINGS = json.load(d) 12 | 13 | # IO Class to serial ports (read and write em) 14 | print('Initiating I/O operations...') 15 | suironio = SuironIO(width=SETTINGS['width'], height=SETTINGS['height'], depth=SETTINGS['depth']) 16 | suironio.init_saving() 17 | suironio.motor_stop() 18 | 19 | # CNN Model 20 | print('Initiating CNN model...') 21 | servo_model = get_cnn_model(SETTINGS['servo_cnn_name'], SETTINGS['width'], SETTINGS['height'], SETTINGS['depth']) 22 | servo_model.load(SETTINGS['servo_cnn_name'] + '.ckpt') 23 | 24 | print('Warming up camera...') 25 | time.sleep(5) 26 | # Auto calibration for camera 27 | for i in range(50): 28 | suironio.get_frame() 29 | 30 | raw_input('Press any key to autonomous mode now...') 31 | suironio.motor_write_fixed() 32 | while True: 33 | try: 34 | # Get current frame 35 | c_frame = suironio.get_frame_prediction() 36 | 37 | # Get predictions 38 | y_ = servo_model.predict([c_frame]) 39 | s_o = cnn_to_raw(y_[0]) 40 | 41 | # Write outputs to servo 42 | suironio.servo_write(y_[0]) 43 | 44 | except KeyboardInterrupt: 45 | suironio.motor_stop() 46 | suironio.servo_straighten() 47 | 48 | print('Exiting autonomous mode...') 49 | -------------------------------------------------------------------------------- /collect.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import os 3 | import time 4 | import json 5 | import numpy as np 6 | 7 | from suiron.core.SuironIO import SuironIO 8 | 9 | # Load image settings 10 | with open('settings.json') as d: 11 | SETTINGS = json.load(d) 12 | 13 | # Instantiatees our IO class 14 | suironio = SuironIO(width=SETTINGS['width'], height=SETTINGS['height'], depth=SETTINGS['depth']) 15 | suironio.init_saving() 16 | 17 | # Allows time for the camerae to warm up 18 | print('Warming up...') 19 | time.sleep(5) 20 | 21 | raw_input('Press any key to conitnue') 22 | print('Recording data...') 23 | while True: 24 | try: 25 | suironio.record_inputs() 26 | except KeyboardInterrupt: 27 | break 28 | 29 | 30 | print('Saving file...') 31 | suironio.save_inputs() -------------------------------------------------------------------------------- /preview.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | 3 | cap = cv2.VideoCapture(0) 4 | 5 | while True: 6 | ret, frame = cap.read() 7 | 8 | cv2.imshow('frame', frame) 9 | if cv2.waitKey(1) & 0xFF == ord('q'): 10 | break 11 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | h5py==2.6.0 2 | numpy==1.11.1 3 | pandas==0.18.1 4 | pyserial==3.1.1 5 | scikit-image==0.12.3 6 | scikit-learn==0.17.1 7 | scipy==0.18.0 8 | tflearn==0.2.1 -------------------------------------------------------------------------------- /settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "width": 72, 3 | "height": 48, 4 | "depth": 3, 5 | "servo_cnn_name": "cnn_servo_model", 6 | "motor_nn_name": "nn_motor_model" 7 | } -------------------------------------------------------------------------------- /suiron/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kendricktan/suiron/8e51e186ae83a412eb09c732a8edd1e3c15ac0bf/suiron/__init__.py -------------------------------------------------------------------------------- /suiron/core/SuironCV.py: -------------------------------------------------------------------------------- 1 | """ 2 | SuironCV contains functions that does some preprocessing on the images 3 | before it is fed into the feed forward network 4 | """" 5 | import math 6 | import cv2 7 | import numpy as np 8 | 9 | # Median blur 10 | def get_median_blur(gray_frame): 11 | return cv2.medianBlur(gray_frame, 5) 12 | 13 | # Canny edge detection 14 | def get_canny(gray_frame): 15 | return cv2.Canny(gray_frame, 50, 200, apertureSize=3) 16 | 17 | # Hough lines 18 | def get_lane_lines(inframe): 19 | frame = inframe.copy() 20 | ret_frame = np.zeros(frame.shape, np.uint8) 21 | 22 | # We converted it into RGB when we normalized it 23 | gray = cv2.cvtColor(frame, cv2.COLOR_RGB2GRAY) 24 | 25 | gray = get_median_blur(gray) 26 | canny = get_canny(gray) 27 | 28 | # Hough lines 29 | # threshold = number of 'votes' before hough algorithm considers it a line 30 | lines = cv2.HoughLinesP(canny, 1, np.pi/180, threshold=25, minLineLength=40, maxLineGap=100) 31 | 32 | try: 33 | r = lines.shape[0] 34 | except AttributeError: 35 | r = 0 36 | 37 | for i in range(0): 38 | for x1, y1, x2, y2 in lines[i]: 39 | # Degrees as its easier for me to conceptualize 40 | angle = math.atan2(y1-y2, x1-x2)*180/np.pi 41 | 42 | # If it looks like a left or right lane 43 | # Draw it onto the new image 44 | if 100 < angle < 170 or -170 < angle < -100: 45 | cv2.line(ret_frame, (x1, y1), (x2, y2), (255, 255, 255), 10) 46 | 47 | return ret_frame 48 | -------------------------------------------------------------------------------- /suiron/core/SuironIO.py: -------------------------------------------------------------------------------- 1 | import time 2 | import random 3 | import numpy as np 4 | import pandas as pd 5 | import cv2, os, serial, csv 6 | import matplotlib.pyplot as plt 7 | 8 | from suiron.utils.functions import cnn_to_raw 9 | from suiron.utils.img_serializer import serialize_image 10 | from suiron.utils.file_finder import get_new_filename 11 | 12 | class SuironIO: 13 | """ 14 | Class which handles input output aspect of the suiron 15 | - Reads inputs from webcam and normalizes them 16 | - Also reads serial input and write them to file 17 | """ 18 | 19 | # Constructor 20 | def __init__(self, width=72, height=48, depth=3, serial_location='/dev/ttyUSB0', baudrate=57600): 21 | # Image settings 22 | self.width = int(width) 23 | self.height = int(height) 24 | self.depth = int(depth) 25 | 26 | # Video IO 27 | self.cap = cv2.VideoCapture(0) # Use first capture device 28 | 29 | # Serial IO 30 | self.ser = None 31 | if os.path.exists(serial_location): 32 | print('Found %s, starting to read from it...' % serial_location) 33 | self.ser = serial.Serial(serial_location, baudrate) 34 | self.outfile = None 35 | 36 | # In-memory variable to record data 37 | # to prevent too much I/O 38 | self.frame_results = [] 39 | self.servo_results = [] 40 | self.motorspeed_results = [] 41 | 42 | """ Functions below are used for inputs (recording data) """ 43 | # Initialize settings before saving 44 | def init_saving(self, folder='data', filename='output_', extension='.csv'): 45 | fileoutname = get_new_filename(folder=folder, filename=filename, extension=extension) 46 | 47 | # Filename to save serial data and image data 48 | # Output file 49 | outfile = open(fileoutname, 'w') # Truncate file first 50 | self.outfile = open(fileoutname, 'a') 51 | 52 | # Saves both inputs 53 | def record_inputs(self): 54 | # Frame is just a numpy array 55 | frame = self.get_frame() 56 | 57 | # Serial inputs is a dict with key 'servo', and 'motor' 58 | serial_inputs = self.get_serial() 59 | 60 | # If its not in manual mode then proceed 61 | if serial_inputs: 62 | servo = serial_inputs['servo'] 63 | motor = serial_inputs['motor'] 64 | 65 | # Append to memory 66 | # tolist so it actually appends the entire thing 67 | self.frame_results.append(serialize_image(frame)) 68 | self.servo_results.append(servo) 69 | self.motorspeed_results.append(motor) 70 | 71 | # Get motor inputs, steering inputs etc 72 | def get_serial(self): 73 | # For debugging 74 | serial_raw = '-1,-1\n' 75 | if self.ser: 76 | # Polling for consistent data 77 | self.ser.write('d') 78 | serial_raw = self.ser.readline() 79 | serial_processed = self.normalize_serial(serial_raw) 80 | return serial_processed 81 | 82 | # Gets frame 83 | def get_frame(self): 84 | ret, frame = self.cap.read() 85 | 86 | # If we get a frame, save it 87 | if not ret: 88 | raise IOError('No image found!') 89 | 90 | frame = self.normalize_frame(frame) 91 | return frame 92 | 93 | # Gets frame for prediction 94 | def get_frame_prediction(self): 95 | ret, frame = self.cap.read() 96 | 97 | # if we get a frame 98 | if not ret: 99 | raise IOError('No image found!') 100 | 101 | frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) 102 | frame = cv2.resize(frame, (self.width, self.height), interpolation=cv2.INTER_CUBIC) 103 | frame = frame.astype('uint8') 104 | 105 | return frame 106 | 107 | 108 | # Normalizes inputs so we don't have to worry about weird 109 | # characters e.g. \r\n 110 | def normalize_serial(self, line): 111 | # Assuming that it receives 112 | # servo, motor 113 | 114 | # 'error' basically means that 115 | # its in manual mode 116 | try: 117 | line = line.replace('\n', '').split(',') 118 | line_dict = {'servo': int(line[0]), 'motor': int(line[1])} 119 | return line_dict 120 | except: 121 | return None 122 | 123 | # Normalizes frame so we don't have BGR as opposed to RGB 124 | def normalize_frame(self, frame): 125 | frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) 126 | frame = cv2.resize(frame, (self.width, self.height), interpolation=cv2.INTER_CUBIC) 127 | frame = frame.flatten() 128 | frame = frame.astype('uint8') 129 | return frame 130 | 131 | # Saves files 132 | def save_inputs(self): 133 | raw_data = { 134 | 'image': self.frame_results, 135 | 'servo': self.servo_results, 136 | 'motor': self.motorspeed_results 137 | } 138 | df = pd.DataFrame(raw_data, columns=['image', 'servo', 'motor']) 139 | df.to_csv(self.outfile) 140 | 141 | """ Functions below are used for ouputs (controlling servo/motor) """ 142 | # Controls the servo given the numpy array outputted by 143 | # the neural network 144 | def servo_write(self, np_y): 145 | servo_out = cnn_to_raw(np_y) 146 | 147 | if (servo_out < 90): 148 | servo_out *= 0.85 149 | 150 | elif (servo_out > 90): 151 | servo_out *= 1.15 152 | 153 | self.ser.write('steer,' + str(servo_out) + '\n') 154 | time.sleep(0.02) 155 | 156 | # Sets the motor at a fixed speed 157 | def motor_write_fixed(self): 158 | self.ser.write('motor,80\n') 159 | time.sleep(0.02) 160 | 161 | # Stops motors 162 | def motor_stop(self): 163 | self.ser.write('motor,90\n') 164 | time.sleep(0.02) 165 | 166 | # Staightens servos 167 | def servo_straighten(self): 168 | self.ser.write('steer,90') 169 | time.sleep(0.02) 170 | 171 | def __del__(self): 172 | if self.outfile: 173 | self.outfile.close() -------------------------------------------------------------------------------- /suiron/core/SuironML.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | import tflearn 4 | 5 | from tflearn.data_utils import shuffle, to_categorical 6 | from tflearn.layers.core import input_data, dropout, fully_connected 7 | from tflearn.layers.conv import conv_2d, max_pool_2d 8 | from tflearn.layers.normalization import local_response_normalization 9 | from tflearn.layers.estimator import regression 10 | 11 | # NVIDIA's CNN architecture 12 | # (used for unprocessed data) 13 | def get_cnn_model(checkpoint_path='cnn_servo_model', width=72, height=48, depth=3, session=None): 14 | 15 | # Inputs 16 | network = input_data(shape=[None, height, width, depth], name='input') 17 | 18 | # Convolution no.1 19 | # Relu introduces non linearity into training 20 | network = conv_2d(network, 8, [5, 3], activation='relu') 21 | 22 | # Convolution no.2 23 | network = conv_2d(network, 12, [5, 8], activation='relu') 24 | 25 | # Convolution no.3 26 | network = conv_2d(network, 16, [5, 16], activation='relu') 27 | 28 | # Convolution no.4 29 | network = conv_2d(network, 24, [3, 20], activation='relu') 30 | 31 | # Convolution no.5 32 | network = conv_2d(network, 24, [3, 24], activation='relu') 33 | 34 | # Fully connected no.1 35 | network = fully_connected(network, 256, activation='relu') 36 | network = dropout(network, 0.8) 37 | 38 | # Fully connected no.2 39 | network = fully_connected(network, 100, activation='relu') 40 | network = dropout(network, 0.8) 41 | 42 | # Fully connected no.3 43 | network = fully_connected(network, 50, activation='relu') 44 | network = dropout(network, 0.8) 45 | 46 | # Fully connected no.4 47 | network = fully_connected(network, 10, activation='relu') 48 | network = dropout(network, 0.8) 49 | 50 | # Fully connected no.5 51 | network = fully_connected(network, 1, activation='tanh') 52 | 53 | # Regression 54 | network = regression(network, loss='mean_square', metric='accuracy', learning_rate=1e-4,name='target') 55 | 56 | # Verbosity yay nay 57 | # 0 = nothing 58 | model = tflearn.DNN(network, tensorboard_verbose=2, checkpoint_path=checkpoint_path, session=session) 59 | return model 60 | 61 | # Simple One hidden layer NN to determine the (linear?) relationship between 62 | # the steering angle and motor value 63 | def get_nn_model(checkpoint_path='nn_motor_model', session=None): 64 | # Input is a single value (raw motor value) 65 | network = input_data(shape=[None, 1], name='input') 66 | 67 | # Hidden layer no.1, 68 | network = fully_connected(network, 12, activation='linear') 69 | 70 | # Output layer 71 | network = fully_connected(network, 1, activation='tanh') 72 | 73 | # regression 74 | network = regression(network, loss='mean_square', metric='accuracy', name='target') 75 | 76 | # Verbosity yay nay 77 | model = tflearn.DNN(network, tensorboard_verbose=3, checkpoint_path=checkpoint_path, session=session) 78 | return model -------------------------------------------------------------------------------- /suiron/core/SuironVZ.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import cv2 3 | import pandas as pd 4 | 5 | from suiron.utils.functions import raw_to_cnn, cnn_to_raw, raw_motor_to_rgb 6 | from suiron.utils.img_serializer import deserialize_image 7 | 8 | # Visualize images 9 | # With and without any predictions 10 | def visualize_data(filename, width=72, height=48, depth=3, cnn_model=None): 11 | """ 12 | When cnn_model is specified it'll show what the cnn_model predicts (red) 13 | as opposed to what inputs it actually received (green) 14 | """ 15 | data = pd.DataFrame.from_csv(filename) 16 | 17 | for i in data.index: 18 | cur_img = data['image'][i] 19 | cur_throttle = int(data['servo'][i]) 20 | cur_motor = int(data['motor'][i]) 21 | 22 | # [1:-1] is used to remove '[' and ']' from string 23 | cur_img_array = deserialize_image(cur_img) 24 | y_input = cur_img_array.copy() # NN input 25 | 26 | # And then rescale it so we can more easily preview it 27 | cur_img_array = cv2.resize(cur_img_array, (480, 320), interpolation=cv2.INTER_CUBIC) 28 | 29 | # Extra debugging info (e.g. steering etc) 30 | cv2.putText(cur_img_array, "frame: %s" % str(i), (5,35), cv2.FONT_HERSHEY_SIMPLEX, 1, 255) 31 | cv2.line(cur_img_array, (240, 300), (240-(90-cur_throttle), 200), (0, 255, 0), 3) 32 | 33 | # Motor values 34 | # RGB 35 | cv2.line(cur_img_array, (50, 160), (50, 160-(90-cur_motor)), raw_motor_to_rgb(cur_motor), 3) 36 | 37 | # If we wanna visualize our cnn_model 38 | if cnn_model: 39 | y = cnn_model.predict([y_input]) 40 | servo_out = cnn_to_raw(y[0]) 41 | cv2.line(cur_img_array, (240, 300), (240-(90-int(servo_out)), 200), (0, 0, 255), 3) 42 | 43 | # Can determine the motor our with a simple exponential equation 44 | # x = abs(servo_out-90) 45 | # motor_out = (7.64*e^(-0.096*x)) - 1 46 | # motor_out = 90 - motor_out 47 | x_ = abs(servo_out - 90) 48 | motor_out = (7.64*np.e**(-0.096*x_)) - 1 49 | motor_out = int(80 - motor_out) # Only wanna go forwards 50 | cv2.line(cur_img_array, (100, 160), (100, 160-(90-motor_out)), raw_motor_to_rgb(motor_out), 3) 51 | print(motor_out, cur_motor) 52 | 53 | # Show frame 54 | # Convert to BGR cause thats how OpenCV likes it 55 | cv2.imshow('frame', cv2.cvtColor(cur_img_array, cv2.COLOR_RGB2BGR)) 56 | if cv2.waitKey(0) & 0xFF == ord('q'): 57 | break -------------------------------------------------------------------------------- /suiron/core/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kendricktan/suiron/8e51e186ae83a412eb09c732a8edd1e3c15ac0bf/suiron/core/__init__.py -------------------------------------------------------------------------------- /suiron/tools/autonomous_test.py: -------------------------------------------------------------------------------- 1 | import serial, time 2 | 3 | ser = serial.Serial('/dev/ttyUSB0', 57600) 4 | 5 | time.sleep(2) 6 | 7 | print('Locking left...') 8 | ser.write('steer,40\n') 9 | time.sleep(1) 10 | 11 | print('Locking right...') 12 | ser.write('steer,140\n') 13 | time.sleep(1) 14 | 15 | print('motor forward') 16 | ser.write('steer,90\n') 17 | time.sleep(0.02) 18 | ser.write('motor,78\n') 19 | time.sleep(5) 20 | 21 | print('motor stop') 22 | ser.write('motor,90\n') 23 | 24 | -------------------------------------------------------------------------------- /suiron/tools/ml_test.py: -------------------------------------------------------------------------------- 1 | from suiron.core.SuironML import get_cnn_model, get_nn_model 2 | 3 | # Test CNN with MNIST dataset 4 | def test_with_mnist(): 5 | import tflearn.datasets.mnist as mnist 6 | 7 | X, Y, testX, testY = mnist.load_data(one_hot=True) 8 | X = X.reshape([-1, 28, 28, 1]) 9 | testX = testX.reshape([-1, 28, 28, 1]) 10 | 11 | model = get_cnn_model(width=28, height=28, depth=1) 12 | model.fit({'input': X}, {'target': Y}, n_epoch=20, 13 | validation_set=({'input': testX}, {'target': testY}), 14 | snapshot_step=100, show_metric=True, run_id='cnn_mnist') 15 | 16 | test_with_mnist() 17 | -------------------------------------------------------------------------------- /suiron/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kendricktan/suiron/8e51e186ae83a412eb09c732a8edd1e3c15ac0bf/suiron/utils/__init__.py -------------------------------------------------------------------------------- /suiron/utils/datasets.py: -------------------------------------------------------------------------------- 1 | """ 2 | datasets.py provides functions to help condense data 'collect.py' into 3 | numpy arrays which can be fed into the CNN/NN 4 | """ 5 | 6 | import numpy as np 7 | import pandas as pd 8 | 9 | from suiron.utils.img_serializer import deserialize_image 10 | from suiron.utils.functions import raw_to_cnn 11 | 12 | # Gets servo dataset 13 | def get_servo_dataset(filename, start_index=0, end_index=None): 14 | data = pd.DataFrame.from_csv(filename) 15 | 16 | # Outputs 17 | x = [] 18 | 19 | # Servo ranges from 40-150 20 | servo = [] 21 | 22 | for i in data.index[start_index:end_index]: 23 | # Don't want noisy data 24 | if data['servo'][i] < 40 or data['servo'][i] > 150: 25 | continue 26 | 27 | # Append 28 | x.append(deserialize_image(data['image'][i])) 29 | servo.append(raw_to_cnn(data['servo'][i])) 30 | 31 | return x, servo 32 | 33 | # Gets motor output dataset 34 | # Assumption is that motor and servo has 35 | # some sort of relationship 36 | def get_motor_dataset(filename, start_index=0, end_index=None): 37 | data = pd.DataFrame.from_csv(filename) 38 | 39 | # Servo 40 | servo = [] 41 | 42 | # Motor ranges from 40-150 43 | motor = [] 44 | 45 | for i in data.index[start_index:end_index]: 46 | # Don't want noisy data 47 | if data['motor'][i] < 40 or data['motor'][i] > 150: 48 | continue 49 | 50 | if data['servo'][i] < 40 or data['servo'][i] > 150: 51 | continue 52 | 53 | # Append 54 | servo.append(raw_to_cnn(data['servo'][i])) 55 | motor.append(raw_to_cnn(data['motor'][i], min_arduino=60.0, max_arduino=90.0)) 56 | 57 | return servo, motor -------------------------------------------------------------------------------- /suiron/utils/file_finder.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | # Check if folder exists 4 | def check_folder_exists(folder='data'): 5 | if not os.path.exists(folder): 6 | os.mkdir(folder) 7 | 8 | # Get numeric number for new file (e.g. output_{x}) 9 | def get_iter_no(folder='data', filename='output_', extension='.csv'): 10 | iter_name = 0 11 | while os.path.exists(os.path.join(folder, filename+str(iter_name)+extension)): 12 | iter_name += 1 13 | return iter_name 14 | 15 | # Gets relative filename for new file (e.g. folders etc joined) 16 | def get_relative_filename(iter_name, folder='data', filename='output_', extension='.csv'): 17 | fileoutname = filename + str(iter_name) + extension 18 | fileoutname = os.path.join(folder, fileoutname) 19 | return fileoutname 20 | 21 | # Get latest filename for outfile 22 | def get_new_filename(folder='data', filename='output_', extension='.csv'): 23 | check_folder_exists(folder) 24 | iter_name = get_iter_no(folder=folder, filename=filename, extension=extension) 25 | return get_relative_filename(iter_name, folder=folder, filename=filename, extension=extension) 26 | 27 | # Get latest output filename 28 | def get_latest_filename(folder='data', filename='output_', extension='.csv'): 29 | check_folder_exists(folder) 30 | iter_name = get_iter_no(folder=folder, filename=filename, extension=extension) - 1 31 | if iter_name == -1: 32 | raise IOError('No file found!') 33 | return get_relative_filename(iter_name, folder=folder, filename=filename, extension=extension) -------------------------------------------------------------------------------- /suiron/utils/functions.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | #Map function from arduino 4 | def arduino_map(x, in_min, in_max, out_min, out_max): 5 | return (x - in_min) * (out_max - out_min) / (in_max - in_min) + out_min 6 | 7 | # Converts raw values to target (Y) values 8 | # for the convolutional neural network 9 | def raw_to_cnn(y, min_arduino=40.0, max_arduino=150.0): 10 | # Servo values 11 | # Map from 40-140 to 1-10 and 12 | # Convert to values between 0-1 because neurons can only contain 13 | # between 0 and 1 14 | y_ = arduino_map(y, min_arduino, max_arduino, 0.0, 1.0) 15 | return [y_] 16 | 17 | # Converts convolutional neural network outputs 18 | # to raw outputs 19 | def cnn_to_raw(y, min_arduino=40.0, max_arduino=150.0): 20 | # Get highest index value and map 21 | # it back 22 | y_ = y[np.argmax(y)] 23 | 24 | # degrees to output 25 | y_ = arduino_map(y_, 0.0, 1.0, min_arduino, max_arduino) 26 | 27 | return y_ 28 | 29 | # Motor to RGB color based on speed 30 | def raw_motor_to_rgb(x): 31 | if x <= 90: 32 | if x < 70: 33 | return (255, 0, 0) 34 | elif x < 80: 35 | return (255, 165, 0) 36 | else: 37 | return (0, 255, 0) 38 | elif x > 90: 39 | if x > 120: 40 | return (255, 0, 0) 41 | elif x > 110: 42 | return (255, 165, 0) 43 | else: 44 | return (0, 255, 0) -------------------------------------------------------------------------------- /suiron/utils/img_serializer.py: -------------------------------------------------------------------------------- 1 | """ 2 | Serializes and deserializes images 3 | after converted to the DataFrame format 4 | """ 5 | import cv2 6 | import numpy as np 7 | 8 | # Deserializes image from data frame dump 9 | def deserialize_image(df_dump, width=72, height=48, depth=3): 10 | # [1:-1] is used to remove '[' and ']' from dataframe string 11 | df_dump = np.fromstring(df_dump[1:-1], sep=', ', dtype='uint8') 12 | df_dump = np.resize(df_dump, (height, width, depth)) 13 | 14 | return df_dump 15 | 16 | # Serializes image to data frame dump 17 | def serialize_image(frame): 18 | return frame.tolist() -------------------------------------------------------------------------------- /suiron/utils/preview.py: -------------------------------------------------------------------------------- 1 | """ 2 | Simple file to view what the webcam is viewing 3 | """ 4 | 5 | import cv2 6 | 7 | cap = cv2.VideoCapture(0) 8 | while True: 9 | ret, frame = cap.read() 10 | 11 | cv2.imshow('frame', frame) 12 | if cv2.waitKey(1) & 0xFF == ord('q'): 13 | break 14 | 15 | cv2.destroyAllWindows() -------------------------------------------------------------------------------- /train.py: -------------------------------------------------------------------------------- 1 | import sys, os 2 | import json 3 | import numpy as np 4 | 5 | from suiron.utils.datasets import get_servo_dataset 6 | from suiron.core.SuironML import get_cnn_model 7 | 8 | # Load image settings 9 | with open('settings.json') as d: 10 | SETTINGS = json.load(d) 11 | 12 | # Our datasets 13 | print('[!] Loading dataset...') 14 | X = [] 15 | SERVO = [] 16 | DATA_FILES = ['data/output_0.csv', 'data/output_1.csv', 'data/output_2.csv', 'data/output_3.csv', 'data/output_4.csv'] 17 | for d in DATA_FILES: 18 | c_x, c_servo = get_servo_dataset(d) 19 | X = X + c_x 20 | SERVO = SERVO + c_servo 21 | 22 | X = np.array(X) 23 | SERVO = np.array(SERVO) 24 | print('[!] Finished loading dataset...') 25 | 26 | # One NN for servo, one for motor 27 | # for now, outputs = 10 28 | servo_model = get_cnn_model(SETTINGS['servo_cnn_name'], SETTINGS['width'], SETTINGS['height'], SETTINGS['depth']) 29 | 30 | # Loads previous model if specified 31 | if len(sys.argv) > 1: 32 | servo_model.load(sys.argv[1]) 33 | 34 | servo_model.fit({'input': X}, {'target': SERVO}, n_epoch=10000, 35 | validation_set=0.1, show_metric=True, snapshot_epoch=False, 36 | snapshot_step=10000, run_id=SETTINGS['servo_cnn_name']) 37 | servo_model.save(SETTINGS['servo_cnn_name'] + '.ckpt') -------------------------------------------------------------------------------- /visualize_collect.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import json 3 | 4 | from suiron.core.SuironVZ import visualize_data 5 | from suiron.utils.file_finder import get_latest_filename 6 | 7 | # Load image settings 8 | with open('settings.json') as d: 9 | SETTINGS = json.load(d) 10 | 11 | # Visualize latest filename 12 | filename = get_latest_filename() 13 | 14 | # If we specified which file 15 | if len(sys.argv) > 1: 16 | filename = sys.argv[1] 17 | 18 | visualize_data(filename, width=SETTINGS['width'], height=SETTINGS['height'], depth=SETTINGS['depth']) -------------------------------------------------------------------------------- /visualize_predict.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import json 3 | import numpy as np 4 | 5 | import tensorflow as tf 6 | 7 | from suiron.utils.file_finder import get_latest_filename 8 | from suiron.core.SuironML import get_cnn_model, get_nn_model 9 | from suiron.core.SuironVZ import visualize_data 10 | 11 | # Image settings 12 | with open('settings.json') as d: 13 | SETTINGS = json.load(d) 14 | 15 | 16 | # Load up our CNN 17 | servo_model = None 18 | servo_model = get_cnn_model(SETTINGS['servo_cnn_name'], SETTINGS['width'], SETTINGS['height'], SETTINGS['depth']) 19 | try: 20 | servo_model.load(SETTINGS['servo_cnn_name'] + '.ckpt') 21 | except Exception as e: 22 | print(e) 23 | 24 | # Visualize latest filename 25 | filename = get_latest_filename() 26 | 27 | # If we specified which file 28 | if len(sys.argv) > 1: 29 | filename = sys.argv[1] 30 | 31 | # Visualize it 32 | visualize_data(filename, SETTINGS['width'], SETTINGS['height'], SETTINGS['depth'], cnn_model=servo_model) --------------------------------------------------------------------------------