├── .gitignore
├── LICENSE
├── README.md
├── adas_cas.py
├── efficientdet-d0-tf
├── efficientdet-d0-tf.bin
├── efficientdet-d0-tf.mapping
└── efficientdet-d0-tf.xml
├── images
├── ADAS_Architecture.jpeg
├── CodeCogsEqn.gif
├── Sensor_Fusion_Output.gif
├── adas.gif
├── bom.jpg
├── dummy.jpg
├── equation.jpeg
├── homo.jpeg
├── lidar_cam_geometry.jpeg
├── lidar_cam_sf.jpeg
├── lidar_converstion_formula.jpeg
├── thumbnail_play.png
└── trafficSign.gif
├── lidar_getdist.py
├── models.lst
├── pulse_mqtt.py
└── ssdlite_mobilenet_v2
├── ssdlite_mobilenet_v2.bin
├── ssdlite_mobilenet_v2.mapping
└── ssdlite_mobilenet_v2.xml
/.gitignore:
--------------------------------------------------------------------------------
1 | # Byte-compiled / optimized / DLL files
2 | __pycache__/
3 | *.py[cod]
4 | *$py.class
5 |
6 | # C extensions
7 | *.so
8 |
9 | # Distribution / packaging
10 | .Python
11 | build/
12 | develop-eggs/
13 | dist/
14 | downloads/
15 | eggs/
16 | .eggs/
17 | lib/
18 | lib64/
19 | parts/
20 | sdist/
21 | var/
22 | wheels/
23 | pip-wheel-metadata/
24 | share/python-wheels/
25 | *.egg-info/
26 | .installed.cfg
27 | *.egg
28 | MANIFEST
29 |
30 | # PyInstaller
31 | # Usually these files are written by a python script from a template
32 | # before PyInstaller builds the exe, so as to inject date/other infos into it.
33 | *.manifest
34 | *.spec
35 |
36 | # Installer logs
37 | pip-log.txt
38 | pip-delete-this-directory.txt
39 |
40 | # Unit test / coverage reports
41 | htmlcov/
42 | .tox/
43 | .nox/
44 | .coverage
45 | .coverage.*
46 | .cache
47 | nosetests.xml
48 | coverage.xml
49 | *.cover
50 | *.py,cover
51 | .hypothesis/
52 | .pytest_cache/
53 |
54 | # Translations
55 | *.mo
56 | *.pot
57 |
58 | # Django stuff:
59 | *.log
60 | local_settings.py
61 | db.sqlite3
62 | db.sqlite3-journal
63 |
64 | # Flask stuff:
65 | instance/
66 | .webassets-cache
67 |
68 | # Scrapy stuff:
69 | .scrapy
70 |
71 | # Sphinx documentation
72 | docs/_build/
73 |
74 | # PyBuilder
75 | target/
76 |
77 | # Jupyter Notebook
78 | .ipynb_checkpoints
79 |
80 | # IPython
81 | profile_default/
82 | ipython_config.py
83 |
84 | # pyenv
85 | .python-version
86 |
87 | # pipenv
88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies
90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not
91 | # install all needed dependencies.
92 | #Pipfile.lock
93 |
94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow
95 | __pypackages__/
96 |
97 | # Celery stuff
98 | celerybeat-schedule
99 | celerybeat.pid
100 |
101 | # SageMath parsed files
102 | *.sage.py
103 |
104 | # Environments
105 | .env
106 | .venv
107 | env/
108 | venv/
109 | ENV/
110 | env.bak/
111 | venv.bak/
112 |
113 | # Spyder project settings
114 | .spyderproject
115 | .spyproject
116 |
117 | # Rope project settings
118 | .ropeproject
119 |
120 | # mkdocs documentation
121 | /site
122 |
123 | # mypy
124 | .mypy_cache/
125 | .dmypy.json
126 | dmypy.json
127 |
128 | # Pyre type checker
129 | .pyre/
130 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | Creative Commons Legal Code
2 |
3 | CC0 1.0 Universal
4 |
5 | CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE
6 | LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN
7 | ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS
8 | INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES
9 | REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS
10 | PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM
11 | THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED
12 | HEREUNDER.
13 |
14 | Statement of Purpose
15 |
16 | The laws of most jurisdictions throughout the world automatically confer
17 | exclusive Copyright and Related Rights (defined below) upon the creator
18 | and subsequent owner(s) (each and all, an "owner") of an original work of
19 | authorship and/or a database (each, a "Work").
20 |
21 | Certain owners wish to permanently relinquish those rights to a Work for
22 | the purpose of contributing to a commons of creative, cultural and
23 | scientific works ("Commons") that the public can reliably and without fear
24 | of later claims of infringement build upon, modify, incorporate in other
25 | works, reuse and redistribute as freely as possible in any form whatsoever
26 | and for any purposes, including without limitation commercial purposes.
27 | These owners may contribute to the Commons to promote the ideal of a free
28 | culture and the further production of creative, cultural and scientific
29 | works, or to gain reputation or greater distribution for their Work in
30 | part through the use and efforts of others.
31 |
32 | For these and/or other purposes and motivations, and without any
33 | expectation of additional consideration or compensation, the person
34 | associating CC0 with a Work (the "Affirmer"), to the extent that he or she
35 | is an owner of Copyright and Related Rights in the Work, voluntarily
36 | elects to apply CC0 to the Work and publicly distribute the Work under its
37 | terms, with knowledge of his or her Copyright and Related Rights in the
38 | Work and the meaning and intended legal effect of CC0 on those rights.
39 |
40 | 1. Copyright and Related Rights. A Work made available under CC0 may be
41 | protected by copyright and related or neighboring rights ("Copyright and
42 | Related Rights"). Copyright and Related Rights include, but are not
43 | limited to, the following:
44 |
45 | i. the right to reproduce, adapt, distribute, perform, display,
46 | communicate, and translate a Work;
47 | ii. moral rights retained by the original author(s) and/or performer(s);
48 | iii. publicity and privacy rights pertaining to a person's image or
49 | likeness depicted in a Work;
50 | iv. rights protecting against unfair competition in regards to a Work,
51 | subject to the limitations in paragraph 4(a), below;
52 | v. rights protecting the extraction, dissemination, use and reuse of data
53 | in a Work;
54 | vi. database rights (such as those arising under Directive 96/9/EC of the
55 | European Parliament and of the Council of 11 March 1996 on the legal
56 | protection of databases, and under any national implementation
57 | thereof, including any amended or successor version of such
58 | directive); and
59 | vii. other similar, equivalent or corresponding rights throughout the
60 | world based on applicable law or treaty, and any national
61 | implementations thereof.
62 |
63 | 2. Waiver. To the greatest extent permitted by, but not in contravention
64 | of, applicable law, Affirmer hereby overtly, fully, permanently,
65 | irrevocably and unconditionally waives, abandons, and surrenders all of
66 | Affirmer's Copyright and Related Rights and associated claims and causes
67 | of action, whether now known or unknown (including existing as well as
68 | future claims and causes of action), in the Work (i) in all territories
69 | worldwide, (ii) for the maximum duration provided by applicable law or
70 | treaty (including future time extensions), (iii) in any current or future
71 | medium and for any number of copies, and (iv) for any purpose whatsoever,
72 | including without limitation commercial, advertising or promotional
73 | purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each
74 | member of the public at large and to the detriment of Affirmer's heirs and
75 | successors, fully intending that such Waiver shall not be subject to
76 | revocation, rescission, cancellation, termination, or any other legal or
77 | equitable action to disrupt the quiet enjoyment of the Work by the public
78 | as contemplated by Affirmer's express Statement of Purpose.
79 |
80 | 3. Public License Fallback. Should any part of the Waiver for any reason
81 | be judged legally invalid or ineffective under applicable law, then the
82 | Waiver shall be preserved to the maximum extent permitted taking into
83 | account Affirmer's express Statement of Purpose. In addition, to the
84 | extent the Waiver is so judged Affirmer hereby grants to each affected
85 | person a royalty-free, non transferable, non sublicensable, non exclusive,
86 | irrevocable and unconditional license to exercise Affirmer's Copyright and
87 | Related Rights in the Work (i) in all territories worldwide, (ii) for the
88 | maximum duration provided by applicable law or treaty (including future
89 | time extensions), (iii) in any current or future medium and for any number
90 | of copies, and (iv) for any purpose whatsoever, including without
91 | limitation commercial, advertising or promotional purposes (the
92 | "License"). The License shall be deemed effective as of the date CC0 was
93 | applied by Affirmer to the Work. Should any part of the License for any
94 | reason be judged legally invalid or ineffective under applicable law, such
95 | partial invalidity or ineffectiveness shall not invalidate the remainder
96 | of the License, and in such case Affirmer hereby affirms that he or she
97 | will not (i) exercise any of his or her remaining Copyright and Related
98 | Rights in the Work or (ii) assert any associated claims and causes of
99 | action with respect to the Work, in either case contrary to Affirmer's
100 | express Statement of Purpose.
101 |
102 | 4. Limitations and Disclaimers.
103 |
104 | a. No trademark or patent rights held by Affirmer are waived, abandoned,
105 | surrendered, licensed or otherwise affected by this document.
106 | b. Affirmer offers the Work as-is and makes no representations or
107 | warranties of any kind concerning the Work, express, implied,
108 | statutory or otherwise, including without limitation warranties of
109 | title, merchantability, fitness for a particular purpose, non
110 | infringement, or the absence of latent or other defects, accuracy, or
111 | the present or absence of errors, whether or not discoverable, all to
112 | the greatest extent permissible under applicable law.
113 | c. Affirmer disclaims responsibility for clearing rights of other persons
114 | that may apply to the Work or any use thereof, including without
115 | limitation any person's Copyright and Related Rights in the Work.
116 | Further, Affirmer disclaims responsibility for obtaining any necessary
117 | consents, permissions or other rights required for any use of the
118 | Work.
119 | d. Affirmer understands and acknowledges that Creative Commons is not a
120 | party to this document and has no duty or obligation with respect to
121 | this CC0 or use of the Work.
122 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # ADAS: Collision Avoidance System on Indian Roads
2 | _Real-world implementation of ADAS L0 - CAS on Indian Roads - using LIDAR-Camera Low-Level Sensor Fusion_
3 |
4 | This solution aims **to augment even the least expensive cars in India with an ultra-cheap ADAS Level 0, i.e. collision avoidance and smart surround-view.** Modern cars with a forward-collision warning (FCW) system or autonomous emergency braking (AEB) are expensive, but we can augment such functions on old cars, at a low cost.
5 |
6 | # Project Demo
7 |
8 | **Watch the gadget plying Indian Roads**, giving driver-assist alerts after perceiving the surrounding objects, their depth and direction.
9 |
10 | [](https://youtu.be/6XW5p3OjkIA)
11 |
12 |
13 | **Project Blog:**
14 | https://towardsdatascience.com/adas-collision-avoidance-system-on-indian-cars-bac64cc8a863
15 |
16 | **Source Code:**
17 | https://github.com/AdroitAnandAI/ADAS-Collision-Avoidance-System-on-Indian-Roads
18 |
19 | **Linkedin:**
20 | http://linkedin.com/in/ananduthaman
21 |
22 |
23 | # Bill of Materials
24 |
25 |
26 | 
27 |
28 |
29 |
30 | # How to Use?
31 |
32 | **Please execute the commands in this order**, so that by the time the object detection module runs the LIDAR and Flash modules will be up and running. Object detection module is the publish node while the other two are subscriber nodes.
33 |
34 | To run the LIDAR distance estimation Subscribe module:
35 | ```python
36 | python3 lidar_getdist.py
37 | ```
38 |
39 | To run the Flash Warning Subscriber Module:
40 | ```python
41 | python3 pulse_mqtt.py
42 | ```
43 |
44 | To run the main object detection Publish Module:
45 | ```python
46 | python3 adas_cas.py -i 0 -m ssdlite_mobilenet_v2/ssdlite_mobilenet_v2.xml -at ssd --labels ~/open_model_zoo/data/dataset_classes/coco_91cl.txt -d MYRIAD --output_resolution 640x480
47 | ```
48 |
49 | # Project Description
50 |
51 | The idea is to **use a battery-powered Pi connected with a LIDAR, Pi Cam, LED SHIM, and NCS 2, mounted on the car bonnet to perceive frontal objects with their depth & direction.** This not only enables a forward-collision warning system but also smart driver assistance that gives alerts about traffic signs or pedestrians, walking along the roadside, or crossing the road.
52 |
53 | Cameras generally have higher resolution than LiDAR but cameras have a limited FOV and can't estimate distance. **While rotating LIDAR has a 360° field of view, Pi Cam has only 62x48 degrees Horizontal x Vertical FoV.** As we deal with multiple sensors here, we need to employ **visual fusion techniques** to integrate the sensor output, that is to get the distance and angle of an obstacle in front of the vehicle. Let's first discuss the theoretical foundation of sensor fusion before hands-on implementation.
54 |
55 | ## The Idea of Sensor Fusion
56 |
57 | Each sensor has its own advantages and disadvantages. Take, for instance, RADARs are low in resolution, but are good at measurement without a line of sight. **In an autonomous car, often a combination of LiDARs, RADARs, and Cameras are used to perceive the environment.** This way we can **compensate for the disadvantages, by combining the advantages of all sensors.**
58 |
59 | - **Camera:** excellent to understand a scene or to classify objects
60 | - **LIDAR:** excellent to estimate distances using pulsed laser waves
61 | - **RADAR:** can measure the speed of obstacles using Doppler Effect
62 |
63 | The camera is a 2D Sensor from which features like bounding boxes, traffic lights, lane divisions can be identified. LIDAR is a 3D Sensor that outputs a set of point clouds. **The fusion technique finds a correspondence between points detected by LIDAR and points detected by the camera.** To use LiDARs and Cameras in unison to build ADAS, the 3D sensor output needs to be fused with 2D sensor output by doing the following steps.
64 |
65 | 1. **Project the LiDAR point clouds (3D) onto the 2D image.**
66 | 2. **Do object detection** using an algorithm like YOLOv4.
67 | 3. **Match the ROI** to find the interested LiDAR projected points.
68 |
69 | By doing the 3 steps above, the surrounding objects would be measured and classified using LIDAR-Camera fusion.
70 |
71 | ## LIDAR-Camera Sensor Fusion Considerations
72 | When a raw image from a cam is merged with raw data from RADAR or LIDAR then it's called **Low-Level Fusion or Early Fusion.** In Late Fusion, detection is done before the fusion. Note that there are many challenges to project the 3D LIDAR point cloud on a 2D image. **The relative orientation and translation between the two sensors must be considered in performing fusion.**
73 |
74 | - **Rotation:** The coordinate system of LIDAR and Camera can be different. Distance on the LIDAR may be on the z-axis, while it is x-axis on the camera. **We need to apply rotation on the LIDAR point cloud to make the coordinate system the same**, i.e. multiply each LIDAR point with the Rotation matrix.
75 | - **Translation:** In an autonomous car, the LIDAR can be at the center top and the camera on the sides. The position of LIDAR and camera in each installation can be different. Based on the relative sensor position, **translate the LIDAR Points by multiplying with a Translation matrix.
76 | - **Stereo Rectification:** For stereo camera setup, we need to do Stereo Rectification to make the left and right images co-planar. Thus, we need to multiply with matrix R0 to align everything along the horizontal Epipolar line.
77 | - **Intrinsic calibration:** Calibration is the step where you tell your camera how **to convert a point in the 3D world into a pixel.** To account for this, we need to multiply with an intrinsic calibration matrix containing factory calibrated values.
78 |
79 | To sum it up, we **need to multiply LIDAR points with all the 4 matrices** to project on the camera image.
80 |
81 | **To project a point X in 3D onto a point Y in 2D,
82 |
83 |
84 | 
85 |
86 |
87 | - **P = Camera Intrinsic Calibration matrix
88 | - **R0 = Stereo Rectification matrix
89 | - **R|t = Rotation & Translation to go from LIDAR to Camera
90 | - **X = Point in 3D space
91 | - **Y = Point in 2D Image
92 |
93 | Note that we have combined both the rigid body transformations, rotation, and translation, in one matrix, R|t. Putting it together, the 3 matrices, P, R0, and R|t account for extrinsic and intrinsic calibration to project LIDAR points onto the camera image. However, the matrix values highly **depend on our custom sensor installation.**
94 |
95 | ## Real-World Implementation with RPi & RPLIDAR A1
96 |
97 | First, we need to assemble the Pi with RPLIDAR A1, Pi Cam, LED SHIM, and NCS 2. 2D LIDAR is used instead of 3D LIDAR as we aim to make the gadget, cheapest possible. The unit is powered by a 5V 3A 10K mAH battery. For ease of assembly, a LIDAR mount is 3D printed and attached to the RPi. Part of the mount design is taken from the STL files obtained from 
98 |
99 | Connect RPLIDAR A1 with the USB adapter that is connected to the Pi USB using a micro-USB cable. LIDAR's adapter provides power and converts LIDAR's internal UART serial interface to a USB interface. Use an Aux-to-Aux cable to connect RPi to speakers. Due to physical constraints, an **LED SHIM is used instead of Blinkt** to signal warning messages. While the total cost of the ADAS gadget is just around US$ 150–200, one may have to shell out at least $10–20K more, to get a car model with such advanced features.
100 |
101 | **Let's imagine, a 3D LIDAR is connected** the same way as above. First, **we will try to solve 3D LIDAR-Camera Sensor Fusion** on the above gadget. **Then we will see the variation for 2D LIDAR-Camera Fusion, so as to make it work on RPLIDAR A1.**
102 |
103 |
104 | 
105 |
106 |
107 | ## 3D LIDAR-Camera Sensor Fusion
108 |
109 | It is clear from the above discussion that **we need to do rotation, translation, stereo rectification, and intrinsic calibration to project LIDAR points on the image.** We will try to apply the above formula based on the custom gadget that we built.
110 |
111 | From the above image, you can estimate the **Pi Cam is 10 mm below the LIDAR scan plane. i.e. a translation of [0, -10, 0] along the 3D-axis. Consider Velodyne HDL-64E as our 3D LIDAR, which requires 180° rotation to align the coordinate system with Pi Cam. We can compute the R|t matrix now.**
112 |
113 | As we use a monocular camera here, the **stereo rectification matrix will be an identity matrix.** We can make the intrinsic calibration matrix based on the hardware spec of Pi Cam V2.
114 |
115 |
116 | 
117 |
118 |
119 | For the **RaspberryPi V2 camera,**
120 | - **Focal Length (FL)** = 3.04 mm
121 | - **FL Pixels** = focal length * sx, where sx = real world to pixels ratio
122 | - **Focal Length * sx** = 3.04mm * (1/ 0.00112 mm per px) = 2714.3 px
123 |
124 | Due to a mismatch in shape, the matrices cannot be multiplied. To make it work, we need to **transition from Euclidean to Homogeneous coordinates by adding 0's and 1's as the last row or column.** After doing the multiplication we need to **convert back to Homogeneous coordinates.**
125 |
126 |
127 |
128 | 
129 |
130 |
131 | You can see the **3DLIDAR-CAM sensor fusion projection output after applying the projection formula on the 3D point cloud.** The input sensor data from 360° Velodyne HDL-64E and camera is downloaded [9] and fed in.
132 |
133 |
134 | 
135 |
136 |
137 | However, the 3D LiDAR cost is a barrier to build a cheap solution. We can **instead use cheap 2D LiDAR with necessary tweaks**, as it only scans a single horizontal line.
138 |
139 | ## 2D LIDAR-Camera Sensor Fusion
140 |
141 | Our gadget is equipped with 2D RP LIDAR A1 to minimize the cost. **This LIDAR scans the environment in a 2D plane, orthogonal to the camera plane. The rotating scan will estimate the distance to the obstacle, for each angle from 0° to 360°. Due to the placement of LIDAR w.r.t. Pi Cam in the gadget, the camera is at +90° on the LIDAR geometry. However, note that the Field of View of Pi cam V2 is 62°x48° in horizontal x vertical direction respectively.**
142 |
143 | The **integrated front view of the device** is as shown below.
144 |
145 |
146 |
147 | 
148 |
149 |
150 | As **both the LIDAR and Camera sensor data is available in the frontal 62° arc, we need to fuse the data.** In the LIDAR scan plane, the camera data starts from +59° to +59° + 62° = 121°. **We can run object detection on the image** to get bounding boxes for the objects of interest. Eg: human, car, bike, traffic light, etc. **Since 2D LIDAR has only width information, consider only the x_min and x_max of each bounding box.**
151 |
152 | We need to compute the LIDAR angle corresponding to an image pixel, in order to estimate the distance to the pixel. **To find the distance to the object inside the bounding box, compute θ_min and θ_max corresponding to x_min & x_max using the below formula**, based on the above diagram,
153 |
154 |
155 | 
156 |
157 |
158 | Now you can **find the distance to each angle between θ_min and θ_max based on the latest LIDAR scan data. Then compute the median distance of all LIDAR points that subtends the object bounding box to estimate the object depth.** If the distance is below a threshold, then trigger a warning based on the angle. Repeat warning, only if the box center shift by a significant distance in subsequent frames.
159 |
160 | **To build a non-blocking system flow, a modular architecture is employed wherein, each independent node is dependent on different hardware components.** i.e., the "object detection" node uses Movidius for inference, whereas the "distance estimation" node takes LIDAR data as input, while the "Alert" module signals to Pimoroni Blinkt and the speaker. The modules are communicated via MQTT messages on respective topics.
161 |
162 |
163 | # Architecture Diagram
164 |
165 |
166 |
167 | 
168 |
169 |
170 | The **time synchronization module takes care of the "data relevance factor" for Sensor Fusion.** For ADAS, the location of detected objects by 'Node 1' may change, as the objects move. Thus, the distance estimate to the bounding box could go wrong after 2–3 seconds (while the message can remain in the MQTT queue). In order to synchronize, current time = 60*minutes + seconds is appended to the message (to ignore lagged messages).
171 |
172 | The model output is sent from Node 1 to Node 2, where the LiDAR-Cam sensor fusion happens, further pushing a message to Node 3. **For the system to function, the 3 MQTT nodes should work in tandem, orchestrated by MQTT messages, published, and subscribed on respective topics.**
173 |
174 | **This ADAS device can be connected to the CAN bus** to let it accelerate, steer, or apply the brake. RPi doesn't have a built-in CAN Bus, but its GPIO includes SPI Bus, which is supported by a number of CAN controllers like MCP2515. So, autonomous emergency braking (AEB) and collision avoidance system (CAS) can be done by connecting this device to the CAN bus.
175 |
176 | # Tweaking for Indian Conditions
177 |
178 | The Indian traffic conundrum is so unique that it demands custom solutions. To start with, **we need to train object detection models with Indian vehicles such as trucks, tempos, vans, autos, cycle rickshaws, etc.**
179 |
180 | Further, to enhance smart surround view, **we need to train the model with Indian traffic signs and signboards** to give more meaningful driver-assist warnings on Indian roads. It's a common sight in India, the animals like cows, pigs, buffaloes, goats, dogs, etc., cross the roads and highways. Hence, it's beneficial to detect them as well. To find out the ROI from images, we have used SSD MobileNet trained on COCO filtered by potential objects. To detect only people and vehicles, you can use this model also to get better speed and accuracy.
181 |
182 | For the PoC, **see the output of SSD-Mobilenet model trained to classify Indian traffic signs against Indian signboards.** You can further classify the traffic sign to decipher the exact meaning of the sign.
183 |
184 |
185 |
186 | 
187 |
188 |
189 | _The annotated Indian Traffic Sign dataset is provided by Datacluster Labs, India. They are yet to finish the annotation of "Indian Vehicles" database. It's just a matter of training time to make this gadget, tailor-made for India._
190 |
191 |
192 | **Project Blog:**
193 | https://towardsdatascience.com/adas-collision-avoidance-system-on-indian-cars-bac64cc8a863
194 |
195 | **Source Code:**
196 | https://github.com/AdroitAnandAI/ADAS-Collision-Avoidance-System-on-Indian-Roads
197 |
198 | **Linkedin:**
199 | http://linkedin.com/in/ananduthaman
200 |
201 |
202 | # References
203 | 1. **LIDAR-Camera Sensor Fusion High Level:** https://www.thinkautonomous.ai/blog/?p=lidar-and-camera-sensor-fusion-in-self-driving-cars
204 | 2. **LIDAR data scan code stub by Adafruit:** https://learn.adafruit.com/remote-iot-environmental-sensor/code
205 | 3. **Camera Calibration and Intrinsic Matrix Estimation:** https://www.cc.gatech.edu/classes/AY2016/cs4476_fall/results/proj3/html/agartia3/index.html
206 | 4. **Visual Fusion** For Autonomous Cars Course at PyImageSearch University: https://www.pyimagesearch.com/pyimagesearch-university/
207 | 5. LIDAR Distance Estimation:https://en.wikipedia.org/wiki/Lidar
208 | 6. RPLIDAR A1 M8 Hardware Specification:https://www.generationrobots.com/media/rplidar-a1m8-360-degree-laser-scanner-development-kit-datasheet-1.pdf
209 | 7. Model Training, Data Cleansing & Augmentation:www.roboflow.com
210 | 8. _The Indian traffic sign model has been trained using the Traffic dataset offered by Datacluster Labs, India._
211 |
--------------------------------------------------------------------------------
/adas_cas.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 | """
3 |
4 | The python object detection from OpenVINO demos is taken and extensively
5 | modified to add the ADAS functionality. This module will get the imagea
6 | and identity the bounding boxes. Then it will publish an MQTT message with
7 | the theta min and max, label, time info on "object/getdistance" topic. The
8 | data is decoded by other lidar_getdist.py node to get the distance of the
9 | object and it will send a message to pulse_mqtt node to flash the warning.
10 |
11 | The stub code is taken from the OpenVINO demos and extensively modified to
12 | add the above functions.
13 |
14 | Copyright (C) 2018-2021 Intel Corporation
15 |
16 | Licensed under the Apache License, Version 2.0 (the "License");
17 | you may not use this file except in compliance with the License.
18 | You may obtain a copy of the License at
19 |
20 | http://www.apache.org/licenses/LICENSE-2.0
21 |
22 | Unless required by applicable law or agreed to in writing, software
23 | distributed under the License is distributed on an "AS IS" BASIS,
24 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
25 | See the License for the specific language governing permissions and
26 | limitations under the License.
27 | """
28 |
29 |
30 | import colorsys
31 | import logging
32 | import random
33 | import os
34 | import sys
35 | from collections import deque
36 | from argparse import ArgumentParser, SUPPRESS
37 | from pathlib import Path
38 | from time import perf_counter
39 | import time
40 |
41 | import paho.mqtt.client as mqtt
42 | import cv2
43 | import numpy as np
44 | from openvino.inference_engine import IECore
45 |
46 | # sys.path.append(str(Path(__file__).resolve().parents[2] / 'common/python'))
47 |
48 | sys.path.append('/home/pi/open_model_zoo/demos/common/python')
49 | sys.path.append('/opt/intel/openvino_2021/deployment_tools/')
50 |
51 | import models
52 | import monitors
53 | from pipelines import get_user_config, AsyncPipeline
54 | from images_capture import open_images_capture
55 | from performance_metrics import PerformanceMetrics
56 | from helpers import resolution
57 |
58 | logging.basicConfig(format='[ %(levelname)s ] %(message)s', level=logging.INFO, stream=sys.stdout)
59 | log = logging.getLogger()
60 |
61 | objLastFrames = deque(maxlen=30)
62 | objMidsLastFrames = deque(maxlen=30)
63 | noObjFrames = 0
64 |
65 | # This is the Publisher
66 | client = mqtt.Client()
67 | client.connect("localhost", 1883, 600)
68 |
69 |
70 | def build_argparser():
71 | parser = ArgumentParser(add_help=False)
72 | args = parser.add_argument_group('Options')
73 | args.add_argument('-h', '--help', action='help', default=SUPPRESS, help='Show this help message and exit.')
74 | args.add_argument('-m', '--model', help='Required. Path to an .xml file with a trained model.',
75 | required=True, type=Path)
76 | args.add_argument('-at', '--architecture_type', help='Required. Specify model\' architecture type.',
77 | type=str, required=True, choices=('ssd', 'yolo', 'yolov4', 'faceboxes', 'centernet', 'ctpn',
78 | 'retinaface', 'ultra_lightweight_face_detection',
79 | 'retinaface-pytorch'))
80 | args.add_argument('-i', '--input', required=True,
81 | help='Required. An input to process. The input must be a single image, '
82 | 'a folder of images, video file or camera id.')
83 | args.add_argument('-d', '--device', default='CPU', type=str,
84 | help='Optional. Specify the target device to infer on; CPU, GPU, HDDL or MYRIAD is '
85 | 'acceptable. The demo will look for a suitable plugin for device specified. '
86 | 'Default value is CPU.')
87 |
88 | common_model_args = parser.add_argument_group('Common model options')
89 | common_model_args.add_argument('--labels', help='Optional. Labels mapping file.', default=None, type=str)
90 | common_model_args.add_argument('-t', '--prob_threshold', default=0.6, type=float,
91 | help='Optional. Probability threshold for detections filtering.')
92 | common_model_args.add_argument('--keep_aspect_ratio', action='store_true', default=False,
93 | help='Optional. Keeps aspect ratio on resize.')
94 | common_model_args.add_argument('--input_size', default=(600, 600), type=int, nargs=2,
95 | help='Optional. The first image size used for CTPN model reshaping. '
96 | 'Default: 600 600. Note that submitted images should have the same resolution, '
97 | 'otherwise predictions might be incorrect.')
98 |
99 | infer_args = parser.add_argument_group('Inference options')
100 | infer_args.add_argument('-nireq', '--num_infer_requests', help='Optional. Number of infer requests',
101 | default=0, type=int)
102 | infer_args.add_argument('-nstreams', '--num_streams',
103 | help='Optional. Number of streams to use for inference on the CPU or/and GPU in throughput '
104 | 'mode (for HETERO and MULTI device cases use format '
105 | ':,: or just ).',
106 | default='', type=str)
107 | infer_args.add_argument('-nthreads', '--num_threads', default=None, type=int,
108 | help='Optional. Number of threads to use for inference on CPU (including HETERO cases).')
109 |
110 | io_args = parser.add_argument_group('Input/output options')
111 | io_args.add_argument('--loop', default=False, action='store_true',
112 | help='Optional. Enable reading the input in a loop.')
113 | io_args.add_argument('-o', '--output', required=False,
114 | help='Optional. Name of the output file(s) to save.')
115 | io_args.add_argument('-limit', '--output_limit', required=False, default=1000, type=int,
116 | help='Optional. Number of frames to store in output. '
117 | 'If 0 is set, all frames are stored.')
118 | io_args.add_argument('--no_show', help="Optional. Don't show output.", action='store_true')
119 | io_args.add_argument('--output_resolution', default=None, type=resolution,
120 | help='Optional. Specify the maximum output window resolution '
121 | 'in (width x height) format. Example: 1280x720. '
122 | 'Input frame size used by default.')
123 | io_args.add_argument('-u', '--utilization_monitors', default='', type=str,
124 | help='Optional. List of monitors to show initially.')
125 |
126 | input_transform_args = parser.add_argument_group('Input transform options')
127 | input_transform_args.add_argument('--reverse_input_channels', default=False, action='store_true',
128 | help='Optional. Switch the input channels order from '
129 | 'BGR to RGB.')
130 | input_transform_args.add_argument('--mean_values', default=None, type=float, nargs=3,
131 | help='Optional. Normalize input by subtracting the mean '
132 | 'values per channel. Example: 255 255 255')
133 | input_transform_args.add_argument('--scale_values', default=None, type=float, nargs=3,
134 | help='Optional. Divide input by scale values per channel. '
135 | 'Division is applied after mean values subtraction. '
136 | 'Example: 255 255 255')
137 |
138 | debug_args = parser.add_argument_group('Debug options')
139 | debug_args.add_argument('-r', '--raw_output_message', help='Optional. Output inference results raw values showing.',
140 | default=False, action='store_true')
141 | return parser
142 |
143 |
144 | class ColorPalette:
145 | def __init__(self, n, rng=None):
146 | assert n > 0
147 |
148 | if rng is None:
149 | rng = random.Random(0xACE)
150 |
151 | candidates_num = 100
152 | hsv_colors = [(1.0, 1.0, 1.0)]
153 | for _ in range(1, n):
154 | colors_candidates = [(rng.random(), rng.uniform(0.8, 1.0), rng.uniform(0.5, 1.0))
155 | for _ in range(candidates_num)]
156 | min_distances = [self.min_distance(hsv_colors, c) for c in colors_candidates]
157 | arg_max = np.argmax(min_distances)
158 | hsv_colors.append(colors_candidates[arg_max])
159 |
160 | self.palette = [self.hsv2rgb(*hsv) for hsv in hsv_colors]
161 |
162 | @staticmethod
163 | def dist(c1, c2):
164 | dh = min(abs(c1[0] - c2[0]), 1 - abs(c1[0] - c2[0])) * 2
165 | ds = abs(c1[1] - c2[1])
166 | dv = abs(c1[2] - c2[2])
167 | return dh * dh + ds * ds + dv * dv
168 |
169 | @classmethod
170 | def min_distance(cls, colors_set, color_candidate):
171 | distances = [cls.dist(o, color_candidate) for o in colors_set]
172 | return np.min(distances)
173 |
174 | @staticmethod
175 | def hsv2rgb(h, s, v):
176 | return tuple(round(c * 255) for c in colorsys.hsv_to_rgb(h, s, v))
177 |
178 | def __getitem__(self, n):
179 | return self.palette[n % len(self.palette)]
180 |
181 | def __len__(self):
182 | return len(self.palette)
183 |
184 |
185 | def get_model(ie, args):
186 | input_transform = models.InputTransform(args.reverse_input_channels, args.mean_values, args.scale_values)
187 | common_args = (ie, args.model, input_transform)
188 | if args.architecture_type in ('ctpn', 'yolo', 'yolov4', 'retinaface',
189 | 'retinaface-pytorch') and not input_transform.is_trivial:
190 | raise ValueError("{} model doesn't support input transforms.".format(args.architecture_type))
191 |
192 | if args.architecture_type == 'ssd':
193 | return models.SSD(*common_args, labels=args.labels, keep_aspect_ratio_resize=args.keep_aspect_ratio)
194 | elif args.architecture_type == 'ctpn':
195 | return models.CTPN(ie, args.model, input_size=args.input_size, threshold=args.prob_threshold)
196 | elif args.architecture_type == 'yolo':
197 | return models.YOLO(ie, args.model, labels=args.labels,
198 | threshold=args.prob_threshold, keep_aspect_ratio=args.keep_aspect_ratio)
199 | elif args.architecture_type == 'yolov4':
200 | return models.YoloV4(ie, args.model, labels=args.labels,
201 | threshold=args.prob_threshold, keep_aspect_ratio=args.keep_aspect_ratio)
202 | elif args.architecture_type == 'faceboxes':
203 | return models.FaceBoxes(*common_args, threshold=args.prob_threshold)
204 | elif args.architecture_type == 'centernet':
205 | return models.CenterNet(*common_args, labels=args.labels, threshold=args.prob_threshold)
206 | elif args.architecture_type == 'retinaface':
207 | return models.RetinaFace(ie, args.model, threshold=args.prob_threshold)
208 | elif args.architecture_type == 'ultra_lightweight_face_detection':
209 | return models.UltraLightweightFaceDetection(*common_args, threshold=args.prob_threshold)
210 | elif args.architecture_type == 'retinaface-pytorch':
211 | return models.RetinaFacePyTorch(ie, args.model, threshold=args.prob_threshold)
212 | else:
213 | raise RuntimeError('No model type or invalid model type (-at) provided: {}'.format(args.architecture_type))
214 |
215 |
216 | def isAnnounced(label, x_mid, y_mid):
217 |
218 | for idx, objs in enumerate(objLastFrames):
219 | for idy, lbl in enumerate(objs):
220 |
221 | if lbl == label:
222 | lastMidPt = objMidsLastFrames[idx][idy]
223 |
224 | # Increase this value to reduce repeated announcements and vice versa
225 | if np.abs(lastMidPt[0] - x_mid) + np.abs(lastMidPt[1] - y_mid) < 100:
226 |
227 | return True
228 |
229 | return False
230 |
231 |
232 |
233 | def announceDetection(frame, frameCount, detections, labels, threshold, output_transform):
234 |
235 | global noObjFrames
236 | size = frame.shape[:2]
237 | x_width = 1280
238 | y_height = 720
239 |
240 | objectsInFrame = []
241 | objectMidPts = []
242 |
243 | for detection in detections:
244 |
245 | if detection.score > threshold:
246 |
247 | class_id = int(detection.id)-1
248 |
249 | # Potential Objects for self driving car
250 | # person
251 | # bicycle
252 | # car
253 | # motorcycle
254 | # bus
255 | # truck
256 | # traffic light
257 | # street sign
258 | # stop sign
259 | if class_id not in [0, 1, 2, 3, 5, 7, 9, 11, 12]:
260 | continue
261 |
262 | det_label = labels[class_id] if labels and len(labels) >= class_id else '#{}'.format(class_id)
263 | xmin = max(int(detection.xmin), 0)
264 | ymin = max(int(detection.ymin), 0)
265 | xmax = min(int(detection.xmax), x_width)
266 | ymax = min(int(detection.ymax), y_height)
267 |
268 | if xmin > xmax:
269 | swap = xmin
270 | xmin = xmax
271 | xmax = swap
272 |
273 | x_mid = np.mean([xmin, xmax])
274 | y_mid = np.mean([ymin, ymax])
275 |
276 | if not isAnnounced(det_label, x_mid, y_mid):
277 |
278 | # theta min and max corresponds to Pi cam FoV angle
279 | # Picam has 62 degrees horizontal FoV. Need to
280 | # convert to LIDAR angles at LIDAR node.
281 | theta_min = xmin / (x_width / 62)
282 | theta_max = xmax / (x_width / 62)
283 | # print('X pixels =' + str(xmin) + ' image width = ' + str(size[1]) +
284 | # ' theta_min = ' + str(theta_min) + ' theta_max = ' + str(theta_max))
285 |
286 | now = time.localtime()
287 |
288 | client.publish("object/getdistance", str(det_label) + '|' +
289 | str(theta_min) + '|' + str(theta_max) + '|' + str(now.tm_min * 60 + now.tm_sec))
290 |
291 | objectsInFrame.append(det_label)
292 | objectMidPts.append((x_mid, y_mid))
293 |
294 | # List of objects and its mid points in last 30 frames will eb stored in dqueue
295 | if len(objectsInFrame) > 0:
296 | objLastFrames.extend([objectsInFrame])
297 | objMidsLastFrames.extend([objectMidPts])
298 | noObjFrames = 0
299 | else:
300 | noObjFrames += 1
301 |
302 | if noObjFrames >= 30:
303 | objMidsLastFrames.clear()
304 | objLastFrames.clear()
305 | noObjFrames = 0
306 |
307 |
308 |
309 | def draw_detections(frame, detections, palette, labels, threshold, output_transform):
310 | size = frame.shape[:2]
311 | frame = output_transform.resize(frame)
312 | boxcount = 0
313 | for detection in detections:
314 | if detection.score > threshold:
315 | boxcount = boxcount + 1
316 | class_id = int(detection.id)-1
317 | color = palette[class_id]
318 | det_label = labels[class_id] if labels and len(labels) >= class_id else '#{}'.format(class_id)
319 | xmin = max(int(detection.xmin), 0)
320 | ymin = max(int(detection.ymin), 0)
321 | xmax = min(int(detection.xmax), size[1])
322 | ymax = min(int(detection.ymax), size[0])
323 | xmin, ymin, xmax, ymax = output_transform.scale([xmin, ymin, xmax, ymax])
324 |
325 | cv2.rectangle(frame, (xmin, ymin), (xmax, ymax), color, 2)
326 | cv2.putText(frame, '{} {:.1%} {}'.format(det_label, detection.score, boxcount),
327 | (xmin, ymin - 7), cv2.FONT_HERSHEY_COMPLEX, 0.6, color, 1)
328 | if isinstance(detection, models.DetectionWithLandmarks):
329 | for landmark in detection.landmarks:
330 | landmark = output_transform.scale(landmark)
331 | cv2.circle(frame, (int(landmark[0]), int(landmark[1])), 2, (0, 255, 255), 2)
332 | return frame
333 |
334 |
335 | def print_raw_results(size, detections, labels, threshold):
336 | log.info(' Class ID | Confidence | XMIN | YMIN | XMAX | YMAX ')
337 | for detection in detections:
338 | if detection.score > threshold:
339 | xmin = max(int(detection.xmin), 0)
340 | ymin = max(int(detection.ymin), 0)
341 | xmax = min(int(detection.xmax), size[1])
342 | ymax = min(int(detection.ymax), size[0])
343 | class_id = int(detection.id)-1
344 | det_label = labels[class_id] if labels and len(labels) >= class_id else '#{}'.format(class_id)
345 | log.info('{:^9} | {:10f} | {:4} | {:4} | {:4} | {:4} '
346 | .format(det_label, detection.score, xmin, ymin, xmax, ymax))
347 |
348 |
349 | def main():
350 | args = build_argparser().parse_args()
351 |
352 | log.info('Initializing Inference Engine...')
353 | ie = IECore()
354 |
355 | plugin_config = get_user_config(args.device, args.num_streams, args.num_threads)
356 |
357 | log.info('Loading network...')
358 |
359 | model = get_model(ie, args)
360 |
361 | detector_pipeline = AsyncPipeline(ie, model, plugin_config,
362 | device=args.device, max_num_requests=args.num_infer_requests)
363 |
364 | cap = open_images_capture(args.input, args.loop)
365 | # print(cap)
366 |
367 | next_frame_id = 0
368 | next_frame_id_to_show = 0
369 |
370 | log.info('Starting inference...')
371 | print("To close the application, press 'CTRL+C' here or switch to the output window and press ESC key")
372 |
373 | palette = ColorPalette(len(model.labels) if model.labels else 100)
374 | metrics = PerformanceMetrics()
375 | presenter = None
376 | output_transform = None
377 | video_writer = cv2.VideoWriter()
378 |
379 | while True:
380 | if detector_pipeline.callback_exceptions:
381 | raise detector_pipeline.callback_exceptions[0]
382 | # Process all completed requests
383 | results = detector_pipeline.get_result(next_frame_id_to_show)
384 | # print('test')
385 | # print(results)
386 | if results:
387 | objects, frame_meta = results
388 | frame = frame_meta['frame']
389 | start_time = frame_meta['start_time']
390 |
391 | if len(objects) and args.raw_output_message:
392 | print_raw_results(frame.shape[:2], objects, model.labels, args.prob_threshold)
393 |
394 | presenter.drawGraphs(frame)
395 | frame = draw_detections(frame, objects, palette, model.labels, args.prob_threshold, output_transform)
396 | announceDetection(frame, next_frame_id, objects, model.labels, args.prob_threshold, output_transform)
397 |
398 | metrics.update(start_time, frame)
399 |
400 | if video_writer.isOpened() and (args.output_limit <= 0 or next_frame_id_to_show <= args.output_limit-1):
401 | video_writer.write(frame)
402 | next_frame_id_to_show += 1
403 | # print(args.no_show)
404 | if not args.no_show:
405 | cv2.imshow('Detection Results', frame)
406 | key = cv2.waitKey(1)
407 |
408 | ESC_KEY = 27
409 | # Quit.
410 | if key in {ord('q'), ord('Q'), ESC_KEY}:
411 | break
412 | presenter.handleKey(key)
413 | continue
414 |
415 | if detector_pipeline.is_ready():
416 | # Get new image/frame
417 | start_time = perf_counter()
418 | frame = cap.read()
419 | if frame is None:
420 | if next_frame_id == 0:
421 | raise ValueError("Can't read an image from the input")
422 | break
423 | if next_frame_id == 0:
424 | output_transform = models.OutputTransform(frame.shape[:2], args.output_resolution)
425 | if args.output_resolution:
426 | output_resolution = output_transform.new_resolution
427 | else:
428 | output_resolution = (frame.shape[1], frame.shape[0])
429 | presenter = monitors.Presenter(args.utilization_monitors, 55,
430 | (round(output_resolution[0] / 4), round(output_resolution[1] / 8)))
431 | if args.output and not video_writer.open(args.output, cv2.VideoWriter_fourcc(*'MJPG'),
432 | cap.fps(), output_resolution):
433 | raise RuntimeError("Can't open video writer")
434 | # Submit for inference
435 | detector_pipeline.submit_data(frame, next_frame_id, {'frame': frame, 'start_time': start_time})
436 | next_frame_id += 1
437 |
438 | else:
439 | # Wait for empty request
440 | detector_pipeline.await_any()
441 |
442 | detector_pipeline.await_all()
443 | # Process completed requests
444 | for next_frame_id_to_show in range(next_frame_id_to_show, next_frame_id):
445 | results = detector_pipeline.get_result(next_frame_id_to_show)
446 | while results is None:
447 | results = detector_pipeline.get_result(next_frame_id_to_show)
448 | objects, frame_meta = results
449 | frame = frame_meta['frame']
450 | start_time = frame_meta['start_time']
451 |
452 | if len(objects) and args.raw_output_message:
453 | print_raw_results(frame.shape[:2], objects, model.labels, args.prob_threshold)
454 |
455 | presenter.drawGraphs(frame)
456 | frame = draw_detections(frame, objects, palette, model.labels, args.prob_threshold, output_transform)
457 | metrics.update(start_time, frame)
458 |
459 | if video_writer.isOpened() and (args.output_limit <= 0 or next_frame_id_to_show <= args.output_limit-1):
460 | video_writer.write(frame)
461 |
462 | if not args.no_show:
463 | cv2.imshow('Detection Results', frame)
464 | key = cv2.waitKey(1)
465 |
466 | ESC_KEY = 27
467 | # Quit.
468 | if key in {ord('q'), ord('Q'), ESC_KEY}:
469 | break
470 | presenter.handleKey(key)
471 |
472 | metrics.print_total()
473 | print(presenter.reportMeans())
474 |
475 |
476 | if __name__ == '__main__':
477 |
478 | sys.exit(main() or 0)
479 |
--------------------------------------------------------------------------------
/efficientdet-d0-tf/efficientdet-d0-tf.bin:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AdroitAnandAI/ADAS-Car-using-Raspberry-Pi/3775f9bc1b03d564cc6e16f1b159c721d1ecf21b/efficientdet-d0-tf/efficientdet-d0-tf.bin
--------------------------------------------------------------------------------
/images/ADAS_Architecture.jpeg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AdroitAnandAI/ADAS-Car-using-Raspberry-Pi/3775f9bc1b03d564cc6e16f1b159c721d1ecf21b/images/ADAS_Architecture.jpeg
--------------------------------------------------------------------------------
/images/CodeCogsEqn.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AdroitAnandAI/ADAS-Car-using-Raspberry-Pi/3775f9bc1b03d564cc6e16f1b159c721d1ecf21b/images/CodeCogsEqn.gif
--------------------------------------------------------------------------------
/images/Sensor_Fusion_Output.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AdroitAnandAI/ADAS-Car-using-Raspberry-Pi/3775f9bc1b03d564cc6e16f1b159c721d1ecf21b/images/Sensor_Fusion_Output.gif
--------------------------------------------------------------------------------
/images/adas.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AdroitAnandAI/ADAS-Car-using-Raspberry-Pi/3775f9bc1b03d564cc6e16f1b159c721d1ecf21b/images/adas.gif
--------------------------------------------------------------------------------
/images/bom.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AdroitAnandAI/ADAS-Car-using-Raspberry-Pi/3775f9bc1b03d564cc6e16f1b159c721d1ecf21b/images/bom.jpg
--------------------------------------------------------------------------------
/images/dummy.jpg:
--------------------------------------------------------------------------------
1 |
2 |
--------------------------------------------------------------------------------
/images/equation.jpeg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AdroitAnandAI/ADAS-Car-using-Raspberry-Pi/3775f9bc1b03d564cc6e16f1b159c721d1ecf21b/images/equation.jpeg
--------------------------------------------------------------------------------
/images/homo.jpeg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AdroitAnandAI/ADAS-Car-using-Raspberry-Pi/3775f9bc1b03d564cc6e16f1b159c721d1ecf21b/images/homo.jpeg
--------------------------------------------------------------------------------
/images/lidar_cam_geometry.jpeg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AdroitAnandAI/ADAS-Car-using-Raspberry-Pi/3775f9bc1b03d564cc6e16f1b159c721d1ecf21b/images/lidar_cam_geometry.jpeg
--------------------------------------------------------------------------------
/images/lidar_cam_sf.jpeg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AdroitAnandAI/ADAS-Car-using-Raspberry-Pi/3775f9bc1b03d564cc6e16f1b159c721d1ecf21b/images/lidar_cam_sf.jpeg
--------------------------------------------------------------------------------
/images/lidar_converstion_formula.jpeg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AdroitAnandAI/ADAS-Car-using-Raspberry-Pi/3775f9bc1b03d564cc6e16f1b159c721d1ecf21b/images/lidar_converstion_formula.jpeg
--------------------------------------------------------------------------------
/images/thumbnail_play.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AdroitAnandAI/ADAS-Car-using-Raspberry-Pi/3775f9bc1b03d564cc6e16f1b159c721d1ecf21b/images/thumbnail_play.png
--------------------------------------------------------------------------------
/images/trafficSign.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AdroitAnandAI/ADAS-Car-using-Raspberry-Pi/3775f9bc1b03d564cc6e16f1b159c721d1ecf21b/images/trafficSign.gif
--------------------------------------------------------------------------------
/lidar_getdist.py:
--------------------------------------------------------------------------------
1 | """
2 |
3 | The getObjectDistance (angle_min, angle_max) function will get the
4 | distance of LIDAR points between angle_min and angle_max. Then it takes
5 | the median of distances and return it. This function is used as a
6 | supporting method for the ADAS system after bounding box detection of
7 | objects. The x_min and x_max of bounding boxes are used to compute the
8 | angle_min and angle_max which is passed in to the getObjectDistance() in
9 | this module to estimate the object distance.
10 |
11 | The code is extensively modified from the base code provided by ADAFruit.
12 |
13 | Adafruit invests time and resources providing this open source code.
14 | Please support Adafruit and open source hardware by purchasing
15 | products from Adafruit!
16 |
17 | Written by Dave Astels for Adafruit Industries
18 | Copyright (c) 2019 Adafruit Industries
19 | Licensed under the MIT license.
20 |
21 | All text above must be included in any redistribution.
22 | """
23 |
24 | import os
25 | from math import cos, sin, pi, floor
26 | import math
27 | import paho.mqtt.client as mqtt
28 | import pygame
29 | from adafruit_rplidar import RPLidar
30 | import numpy as np
31 |
32 | # Screen width & height
33 | W = 640
34 | H = 480
35 |
36 | SCAN_BYTE = b'\x20'
37 | SCAN_TYPE = 129
38 |
39 |
40 | import ledshim
41 | import colorsys
42 | import time
43 | from sys import exit
44 |
45 |
46 | def make_gaussian(fwhm):
47 | x = np.arange(0, ledshim.NUM_PIXELS, 1, float)
48 | y = x[:, np.newaxis]
49 | x0, y0 = 3.5, (ledshim.NUM_PIXELS / 2) - 0.5
50 | fwhm = fwhm
51 | gauss = np.exp(-4 * np.log(2) * ((x - x0) ** 2 + (y - y0) ** 2) / fwhm ** 2)
52 | return gauss
53 |
54 |
55 | def flashLight():
56 | print('inside flashlight')
57 | for _ in range(10):
58 |
59 | ledshim.set_clear_on_exit()
60 |
61 | for z in list(range(1, 10)[::-1]) + list(range(1, 10)):
62 | fwhm = 15.0 / z
63 | gauss = make_gaussian(fwhm)
64 | print('after make gaussian')
65 | start = time.time()
66 | y = 4
67 | for x in range(ledshim.NUM_PIXELS):
68 | h = 0.5
69 | s = 1.0
70 | v = gauss[x, y]
71 | rgb = colorsys.hsv_to_rgb(h, s, v)
72 | r, g, b = [int(255.0 * i) for i in rgb]
73 | ledshim.set_pixel(x, r, g, b)
74 | ledshim.show()
75 | end = time.time()
76 | t = end - start
77 | if t < 0.04:
78 | time.sleep(0.04 - t)
79 |
80 |
81 | # Setup the RPLidar
82 | PORT_NAME = '/dev/ttyUSB0'
83 | lidar = RPLidar(None, PORT_NAME)
84 |
85 | # used to scale data to fit on the screen
86 | max_distance = 0
87 |
88 | #pylint: disable=redefined-outer-name,global-statement
89 | def process_data(data):
90 | global max_distance
91 | lcd.fill((0,0,0))
92 | point = ( int(W / 2) , int(H / 2) )
93 |
94 | pygame.draw.circle(lcd,pygame.Color(255, 255, 255),point,10 )
95 | pygame.draw.circle(lcd,pygame.Color(100, 100, 100),point,100 , 1 )
96 | pygame.draw.line( lcd,pygame.Color(100, 100, 100) , ( 0, int(H/2)),( W , int(H/2) ) )
97 | pygame.draw.line( lcd,pygame.Color(100, 100, 100) , ( int(W/2),0),( int(W/2) , H ) )
98 |
99 | for angle in range(360):
100 | distance = data[angle]
101 | if distance > 0: # ignore initially ungathered data points
102 | max_distance = max([min([5000, distance]), max_distance])
103 | radians = angle * pi / 180.0
104 | x = distance * cos(radians)
105 | y = distance * sin(radians)
106 | point = ( int(W / 2) + int(x / max_distance * (W/2)), int(H/2) + int(y / max_distance * (H/2) ))
107 | pygame.draw.circle(lcd,pygame.Color(255, 0, 0),point,2 )
108 | pygame.display.update()
109 |
110 |
111 | scan_data = [0]*360
112 |
113 | def _process_scan(raw):
114 | '''Processes input raw data and returns measurment data'''
115 | new_scan = bool(raw[0] & 0b1)
116 | inversed_new_scan = bool((raw[0] >> 1) & 0b1)
117 | quality = raw[0] >> 2
118 | if new_scan == inversed_new_scan:
119 | raise RPLidarException('New scan flags mismatch')
120 | check_bit = raw[1] & 0b1
121 | if check_bit != 1:
122 | raise RPLidarException('Check bit not equal to 1')
123 | angle = ((raw[1] >> 1) + (raw[2] << 7)) / 64.
124 | distance = (raw[3] + (raw[4] << 8)) / 4.
125 | return new_scan, quality, angle, distance
126 |
127 | def lidar_measurments(self, max_buf_meas=500):
128 |
129 | lidar.set_pwm(800)
130 | status, error_code = self.health
131 |
132 | cmd = SCAN_BYTE
133 | self._send_cmd(cmd)
134 | dsize, is_single, dtype = self._read_descriptor()
135 | if dsize != 5:
136 | raise RPLidarException('Wrong info reply length')
137 | if is_single:
138 | raise RPLidarException('Not a multiple response mode')
139 | if dtype != SCAN_TYPE:
140 | raise RPLidarException('Wrong response data type')
141 | while True:
142 | raw = self._read_response(dsize)
143 | self.log_bytes('debug', 'Received scan response: ', raw)
144 | if max_buf_meas:
145 | data_in_buf = self._serial_port.in_waiting
146 | if data_in_buf > max_buf_meas*dsize:
147 | self.log('warning',
148 | 'Too many measurments in the input buffer: %d/%d. '
149 | 'Clearing buffer...' %
150 | (data_in_buf//dsize, max_buf_meas))
151 | self._serial_port.read(data_in_buf//dsize*dsize)
152 | yield _process_scan(raw)
153 |
154 | def lidar_scans(self, max_buf_meas=500, min_len=5):
155 |
156 | scan = []
157 | iterator = lidar_measurments(lidar,max_buf_meas)
158 | for new_scan, quality, angle, distance in iterator:
159 | if new_scan:
160 | if len(scan) > min_len:
161 | yield scan
162 | scan = []
163 | if quality > 0 and distance > 0:
164 | scan.append((quality, angle, distance))
165 |
166 | def getObjectDistance (angle_min, angle_max):
167 |
168 | minDist = 0
169 | lidar = RPLidar(None, PORT_NAME)
170 |
171 | try:
172 | for scan in lidar_scans(lidar):
173 |
174 | for (_, angle, distance) in scan:
175 | # print("Angle = " + str(angle) + "distance == " + str(distance))
176 | scan_data[min([359, floor(angle)])] = distance
177 |
178 |
179 | # fetching all non zero distance values between subtended angles
180 | allDists = [scan_data[i] for i in range(360)
181 | if i >= angle_min and i <= angle_max and scan_data[i] > 0]
182 |
183 | # if half the distance values are filled in then break
184 | if (2 * len(allDists) > angle_max - angle_min):
185 |
186 | minDist = np.median(allDists)
187 | lidar.stop()
188 | lidar.disconnect()
189 |
190 | return minDist
191 |
192 | except KeyboardInterrupt:
193 | print('Stopping LIDAR Scan')
194 |
195 | def roundtoTen(x):
196 | return int(math.ceil(x/10.0)) * 10
197 |
198 | def on_connect(client, userdata, flags, rc):
199 | print("Connected with result code " + str(rc))
200 | client.subscribe("object/getdistance")
201 |
202 |
203 | def on_message(client, userdata, msg):
204 |
205 | # print(msg.payload.decode())
206 |
207 | word = msg.payload.decode()
208 |
209 | # objAttributes contains label,
210 | # theta min and max separated by |
211 | objAttributes = word.split('|')
212 |
213 | now = time.localtime()
214 | if (now.tm_min * 60 + now.tm_sec - int(objAttributes[3]) >= 1):
215 | return
216 |
217 | theta1 = float(objAttributes[1])
218 | theta2 = float(objAttributes[2])
219 |
220 | dist = getObjectDistance(int(theta1) + 90 + 59, int(theta2) + 90 + 59)
221 |
222 | # convert distance from mm to cms
223 | dist = round(float (dist / 1000), 1)
224 |
225 | # print('The distance to the object = ' + str(dist))
226 |
227 | theta_mid = int((theta1 + theta2) / 2)
228 | print(' Object is at an angle of ' + str(theta_mid))
229 |
230 | # if near then announce an alert!
231 | # Passing the hue value on MQTT. 0 = Red. 0.3 = Green
232 | if (dist < 2.0):
233 | # print('setting alert msg')
234 | announceText = "ALERT ALERT "
235 | client.publish("object/flashlight", "0.0")
236 | else:
237 | announceText = ""
238 | client.publish("object/flashlight", "0.3")
239 |
240 | announceText = announceText + str(objAttributes[0]) + ' at ' + str(dist) + ' meters. '
241 |
242 | # theta_mid can vary from 0 to 62 degrees
243 | if theta_mid > 40:
244 | # print('Right Side')
245 | os.system(
246 | 'espeak \"' + announceText + str (roundtoTen(abs(theta_mid - 31))) + ' degrees right\"')
247 | elif theta_mid < 21:
248 | # print('Left Side')
249 | os.system(
250 | 'espeak \"' + announceText + str (roundtoTen(abs(31 - theta_mid))) + ' degrees left\"')
251 | else:
252 | # theta_mid will be > 20 and < 40, if here
253 | if theta_mid < 30:
254 | direction = ' Right '
255 | else:
256 | direction = ' Left '
257 | # Alert to slide to opposite direction at theta + 30 degrees
258 | os.system(
259 | 'espeak \"' + announceText + 'Slide ' + direction + str(abs(roundtoTen(abs(31 - theta_mid)) + 30)) + 'degrees \"')
260 |
261 |
262 | client = mqtt.Client()
263 | client.connect("localhost", 1883, 600)
264 |
265 | client.on_connect = on_connect
266 | client.on_message = on_message
267 |
268 | try:
269 | client.loop_forever()
270 | # To catch SigINT
271 | except KeyboardInterrupt:
272 | client.disconnect()
273 |
--------------------------------------------------------------------------------
/models.lst:
--------------------------------------------------------------------------------
1 | # This file can be used with the --list option of the model downloader.
2 | # For --architecture_type=centernet
3 | ctdet_coco_dlav0_384
4 | ctdet_coco_dlav0_512
5 | # For --architecture_type=ctpn
6 | ctpn
7 | # For --architecture_type=faceboxes
8 | faceboxes-pytorch
9 | # For --architecture_type=retinaface-pytorch
10 | retinaface-resnet50-pytorch
11 | # For --architecture_type=ssd
12 | efficientdet-d0-tf
13 | efficientdet-d1-tf
14 | face-detection-????
15 | face-detection-adas-????
16 | face-detection-retail-????
17 | faster-rcnn-resnet101-coco-sparse-60-0001
18 | pedestrian-and-vehicle-detector-adas-????
19 | pedestrian-detection-adas-????
20 | pelee-coco
21 | person-detection-????
22 | person-detection-retail-0013
23 | person-vehicle-bike-detection-????
24 | product-detection-0001
25 | retinanet-tf
26 | rfcn-resnet101-coco-tf
27 | ssd300
28 | ssd512
29 | ssd_mobilenet_v1_coco
30 | ssd_mobilenet_v1_fpn_coco
31 | ssd_mobilenet_v2_coco
32 | ssd_resnet50_v1_fpn_coco
33 | ssd-resnet34-1200-onnx
34 | ssdlite_mobilenet_v2
35 | vehicle-detection-????
36 | vehicle-detection-adas-????
37 | vehicle-license-plate-detection-barrier-????
38 | # For --architecture_type=ultra_lightweight_face_detection
39 | ultra-lightweight-face-detection-rfb-320
40 | ultra-lightweight-face-detection-slim-320
41 | # For --architecture_type=yolo
42 | mobilefacedet-v1-mxnet
43 | person-vehicle-bike-detection-crossroad-yolov3-1020
44 | yolo-v1-tiny-tf
45 | yolo-v2-ava-0001
46 | yolo-v2-ava-sparse-??-0001
47 | yolo-v2-tf
48 | yolo-v2-tiny-ava-0001
49 | yolo-v2-tiny-ava-sparse-??-0001
50 | yolo-v2-tiny-tf
51 | yolo-v2-tiny-vehicle-detection-0001
52 | yolo-v3-tf
53 | yolo-v3-tiny-tf
54 | # For --architecture_type=yolov4
55 | yolo-v4-tf
56 | yolo-v4-tiny-tf
57 |
--------------------------------------------------------------------------------
/pulse_mqtt.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 |
3 | import os
4 | from math import cos, sin, pi, floor
5 | import paho.mqtt.client as mqtt
6 | import colorsys
7 | import time
8 | from sys import exit
9 |
10 | try:
11 | import numpy as np
12 | except ImportError:
13 | exit('This script requires the numpy module\nInstall with: sudo pip install numpy')
14 |
15 | import ledshim
16 |
17 | ledshim.set_clear_on_exit()
18 |
19 |
20 | def make_gaussian(fwhm):
21 | x = np.arange(0, ledshim.NUM_PIXELS, 1, float)
22 | y = x[:, np.newaxis]
23 | x0, y0 = 3.5, (ledshim.NUM_PIXELS / 2) - 0.5
24 | fwhm = fwhm
25 | gauss = np.exp(-4 * np.log(2) * ((x - x0) ** 2 + (y - y0) ** 2) / fwhm ** 2)
26 | return gauss
27 |
28 |
29 | def flashLight(colorCode):
30 | # print('inside flashlight')
31 | for _ in range(5):
32 |
33 | ledshim.set_clear_on_exit()
34 |
35 | for z in list(range(1, 10)[::-1]) + list(range(1, 10)):
36 | fwhm = 15.0 / z
37 | gauss = make_gaussian(fwhm)
38 |
39 | start = time.time()
40 | y = 4
41 | for x in range(ledshim.NUM_PIXELS):
42 | h = colorCode
43 | s = 1.0
44 | v = gauss[x, y]
45 | rgb = colorsys.hsv_to_rgb(h, s, v)
46 | r, g, b = [int(255.0 * i) for i in rgb]
47 | ledshim.set_pixel(x, r, g, b)
48 | ledshim.show()
49 | end = time.time()
50 | t = end - start
51 | if t < 0.04:
52 | time.sleep(0.04 - t)
53 |
54 |
55 |
56 | def on_connect(client, userdata, flags, rc):
57 | print("Connected with result code " + str(rc))
58 | client.subscribe("object/flashlight")
59 |
60 |
61 | def on_message(client, userdata, msg):
62 |
63 | colorCode = msg.payload.decode()
64 | flashLight(float(colorCode))
65 |
66 |
67 | client = mqtt.Client()
68 | client.connect("localhost", 1883, 600)
69 |
70 | client.on_connect = on_connect
71 | client.on_message = on_message
72 |
73 | try:
74 | client.loop_forever()
75 | # To catch SigINT
76 | except KeyboardInterrupt:
77 | client.disconnect()
--------------------------------------------------------------------------------
/ssdlite_mobilenet_v2/ssdlite_mobilenet_v2.bin:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AdroitAnandAI/ADAS-Car-using-Raspberry-Pi/3775f9bc1b03d564cc6e16f1b159c721d1ecf21b/ssdlite_mobilenet_v2/ssdlite_mobilenet_v2.bin
--------------------------------------------------------------------------------