├── .gitattributes ├── Models ├── ReadmePhotos │ ├── Callback.PNG │ ├── CameraView.png │ ├── MATLABScript.PNG │ ├── ModelComponents.png │ └── ModelPhoto.PNG ├── obstacleAvoidanceUsingOpticalFlow.slx ├── startScript.m └── virtualworld │ ├── slv3.x3d │ ├── uav_arducopter.x3d │ ├── uav_environment.x3d │ ├── uav_heliport.png │ └── uav_protos.x3d ├── README.md ├── SECURITY.md └── license.txt /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /Models/ReadmePhotos/Callback.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mathworks-robotics/obstacle-avoidance-using-camera/d2735e13c1962e880ec1da6b56a4c813ce96e576/Models/ReadmePhotos/Callback.PNG -------------------------------------------------------------------------------- /Models/ReadmePhotos/CameraView.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mathworks-robotics/obstacle-avoidance-using-camera/d2735e13c1962e880ec1da6b56a4c813ce96e576/Models/ReadmePhotos/CameraView.png -------------------------------------------------------------------------------- /Models/ReadmePhotos/MATLABScript.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mathworks-robotics/obstacle-avoidance-using-camera/d2735e13c1962e880ec1da6b56a4c813ce96e576/Models/ReadmePhotos/MATLABScript.PNG -------------------------------------------------------------------------------- /Models/ReadmePhotos/ModelComponents.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mathworks-robotics/obstacle-avoidance-using-camera/d2735e13c1962e880ec1da6b56a4c813ce96e576/Models/ReadmePhotos/ModelComponents.png -------------------------------------------------------------------------------- /Models/ReadmePhotos/ModelPhoto.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mathworks-robotics/obstacle-avoidance-using-camera/d2735e13c1962e880ec1da6b56a4c813ce96e576/Models/ReadmePhotos/ModelPhoto.PNG -------------------------------------------------------------------------------- /Models/obstacleAvoidanceUsingOpticalFlow.slx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mathworks-robotics/obstacle-avoidance-using-camera/d2735e13c1962e880ec1da6b56a4c813ce96e576/Models/obstacleAvoidanceUsingOpticalFlow.slx -------------------------------------------------------------------------------- /Models/startScript.m: -------------------------------------------------------------------------------- 1 | % Obstacle Avoidance using Optical Flow 2 | % Copyright 2021 The MathWorks, Inc. 3 | 4 | % Model Initialization 5 | % Start Location [X Z Y] 6 | dronestartlocation=[-1.5 0 4]; 7 | 8 | % Final Location [X Z Y] 9 | helipadLocation = [14 0.29 4]; 10 | 11 | % Linear Velocity [Vx Vz Vy] 12 | linearVelocity = [0.6 0.05 .2]; 13 | 14 | % Angular Velocity during Optical Flow navigation 15 | % represented as [pitch yaw roll] 16 | angularVelocity = [0 0 -0.01]; 17 | 18 | % X-coordinate to switch from Optical flow controller to PID Controller 19 | switchingXLimit = 12.0; 20 | 21 | % Angular Velocity during landing 22 | angularVelocityforLanding = [0 0 0.01]; 23 | 24 | % Hover Height before landing 25 | finalHoverHeight = 0.8; 26 | 27 | % Threshold distance to execute the PID controller. 28 | delta = 0.1; 29 | 30 | %Sample Time 31 | Ts = 0.1; 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /Models/virtualworld/slv3.x3d: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | 164 | 165 | 166 | 167 | 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | 184 | 185 | 186 | 187 | 188 | 189 | 190 | 191 | 192 | 193 | 194 | 195 | 196 | 197 | 198 | 199 | 200 | 201 | 202 | 203 | 204 | 205 | 206 | 207 | 208 | 209 | 210 | 211 | 212 | 213 | 214 | 215 | 216 | 217 | 218 | 219 | 220 | 221 | 222 | 223 | 224 | 225 | 226 | 227 | 228 | 229 | 230 | 231 | 232 | 233 | 234 | 235 | 236 | 237 | 238 | 239 | 240 | 241 | 242 | 243 | 244 | 245 | 246 | 247 | 248 | 249 | 250 | 251 | 252 | 253 | 254 | 255 | 256 | 257 | 258 | 259 | 260 | 261 | 262 | 263 | 264 | 265 | 266 | 267 | 268 | 269 | 270 | 271 | 272 | 273 | 274 | 275 | 276 | 277 | 278 | 279 | 280 | 281 | 282 | 283 | 284 | 285 | 286 | 287 | 288 | 289 | 290 | 291 | 292 | 293 | 294 | 295 | 296 | 297 | 298 | 299 | 300 | 301 | 302 | 303 | 304 | 305 | 306 | 307 | 308 | 309 | 310 | 311 | 312 | 313 | 314 | 315 | 316 | 317 | 318 | 319 | 320 | 321 | 322 | 323 | 324 | 325 | 326 | 327 | 328 | 329 | 330 | -------------------------------------------------------------------------------- /Models/virtualworld/uav_arducopter.x3d: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | 164 | 165 | 166 | 167 | 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | 184 | 185 | 186 | 187 | 188 | 189 | 190 | 191 | 192 | 193 | 194 | 195 | 196 | 197 | 198 | 199 | 200 | 201 | 202 | 203 | 204 | 205 | 206 | 207 | 208 | 209 | 210 | 211 | 212 | 213 | 214 | 215 | 216 | 217 | 218 | 219 | 220 | 221 | 222 | 223 | 224 | 225 | 226 | 227 | 228 | 229 | 230 | 231 | 232 | 233 | 234 | 235 | 236 | 237 | 238 | 239 | 240 | 241 | 242 | 243 | 244 | 245 | 246 | 247 | 248 | 249 | 250 | 251 | 252 | 253 | 254 | 255 | 256 | 257 | 258 | 259 | 260 | 261 | 262 | 263 | 264 | 265 | 266 | 267 | 268 | 269 | 270 | 271 | 272 | 273 | 274 | 275 | 276 | 277 | 278 | 279 | 280 | 281 | 282 | 283 | 284 | 285 | 286 | 287 | 288 | 289 | 290 | 291 | 292 | 293 | 294 | 295 | 296 | 297 | 298 | 299 | 300 | 301 | 302 | 303 | 304 | 305 | 306 | 307 | 308 | 309 | 310 | 311 | 312 | 313 | 314 | 315 | 316 | 317 | 318 | 319 | 320 | 321 | 322 | 323 | 324 | 325 | 326 | 327 | 328 | 329 | 330 | 331 | 332 | 333 | 334 | 335 | 336 | 337 | 338 | -------------------------------------------------------------------------------- /Models/virtualworld/uav_environment.x3d: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | -------------------------------------------------------------------------------- /Models/virtualworld/uav_heliport.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mathworks-robotics/obstacle-avoidance-using-camera/d2735e13c1962e880ec1da6b56a4c813ce96e576/Models/virtualworld/uav_heliport.png -------------------------------------------------------------------------------- /Models/virtualworld/uav_protos.x3d: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Obstacle Avoidance with Camera Sensor using Simulink 2 | 3 | This submission contains the implementation of [optical flow algorithm](https://www.mathworks.com/help/vision/ref/opticalflow.html) for obstacle avoidance. Here, the drone to traverses through a [Simulink® 3D Animation](https://www.mathworks.com/products/3d-animation.html) arena using optical flow algorithm calculated using the drone's front view camera. 4 | This model aims at helping you to get started to use computer vision along with control systems in your ground/aerial robots. 5 | Using this model, you can get your UAV/UGV to avoid obstacles using the front-facing (bird's eye view) vision data of the vehicle. 6 | 7 | Check out the video explaining this example: 8 | [https://www.youtube.com/watch?v=YTmq13xGnLg](https://www.youtube.com/watch?v=YTmq13xGnLg) 9 | 10 | ## About the model ## 11 | 12 | The model shows the workflow to implement an optical flow algorithm for navigating a drone through the arena which has poles as the obstacles. 13 | 14 | 15 | The following steps describe the workflow: 16 | 17 | 1. Getting the input image from the on-board camera. 18 | 2. Calculating the optical flow lines from the image using *Horn-Schunck* method 19 | 3. Implementing an algorithm for steering the vehicle from the calculated flow lines 20 | 4. Implementing a switching based logic using [Stateflow® ](https://www.mathworks.com/products/stateflow.html) to switch from optical flow algorithm to a PID Control algorithm. 21 | 5. Visualizing the drone navigating the obstacles through the VR Sink block (used to set the UAV position/rotation and visual properties of virtual world objects) associated with the virtual world. 22 | 23 | 24 | When you open the model and execute it, the drone will take off from the helipad, navigate by evading the obstacles, switch to PID Controller once it detects that it is close to the landing area and eventually, land on the landing helipad. 25 | 26 | 27 | 28 | 29 | 30 | ## Model ## 31 | 32 | **obstacleAvoidanceUsingOpticalFlow.slx** contains a model which implements Optical Flow algorithm for navigating through obstacles and PID Control algorithm for Drone Landing. 33 | 34 | **startScript.mlx** contains the constants used in the model. 35 | 36 | **virtualworld** folder contains the files required for showing the quadcopter in the Simulink 3D Animation World 37 | 38 | **uav_environment.wrl** shows the three-dimensional environment used for the simulation. 39 | 40 | *Note:* Before running the model, please make sure all these files are in the current folder 41 | 42 | ## How to run the model? ## 43 | - Open the Simulink model *obstacleAvoidanceUsingOpticalFlow.slx*. 44 | 45 | - The model automatically adds the *virtualworld* folder to the path. Check *PostLoadFcn* [Model Callback](https://www.mathworks.com/help/simulink/ug/model-callbacks.html). 46 | 47 | - The model automatically loads *startScript.m* that initializes the parameters required to run the model.Check *InitFcn* [Model Callback](https://www.mathworks.com/help/simulink/ug/model-callbacks.html). 48 | 49 | 50 | 51 | - Run the model. 52 | 53 | ## Assumptions in the model ## 54 | 55 | - While designing the system, the drone is considered as a point mass and the drone dynamics are not considered. 56 | - The feedback from the drone is taken from the Simulink 3D Animation sensors. 57 | - The model assumes that the user knows at what point the controller switches from using optical flow to using PID controller. 58 | - The model also assumes that the x-coordinate landing location is greater than the x-coordinate of the point of transition from optical flow controller to PID Controlelr. 59 | 60 | *Note:* Do not change the *Bird's Eye* View in the *On-board Camera* and *Virtual Reality Sensors* block. This will affect the optical flow algorithm since we need the images from the front camera. 61 | 62 | 63 | 64 | 65 | 66 | ## About the Subsystems ## 67 | 68 | 69 | 70 | 71 | - *Sensor Inputs:* Simulink 3D Animation Sensor inputs (camera view and estimated position) 72 | - *Optical Flow Lines Calculation:* Calculates the optical flow lines for the image using 'Horn-Schunk' method. 73 | - *Optical Flow Algorithm for Obstacle Avoidance:* Controller for steering the vehicle: 74 | - *FlowDifferences* MATLAB® Function: The Optical Flow on the right and left is calculated by the system. 75 | - *Optical flow algorithm and PID Controller for landing:* 76 | - *DronePositionEstimation:* 77 | 1. *OpticalFlowAlgorithm:* The sign of the linear velocity (*V_Y*)is assigned depending on Optical Flow on Left and Right 78 | 2. *ControlAlgorithm:* Switch to the PID controller to hover above the destination coordinate. 79 | 3. *DroneLanding:* Descend the drone to the floor. 80 | - *DroneOrientationEstimation:* 81 | 1. *OpticalFlowAlgorithm:* The sign of the angular velocity (*W_Y*)is assigned depending on Optical Flow on Left and Right 82 | 2. *ControlAlgorithm:* Find the orientation from angular velocities 83 | - *3D Visualization:* Once the Position and Orientation outputs are estimated in the model, they are sent to the VR Sink block which are then used to set the UAV position / rotation. 84 | - *Distance between landing helipad and present location:* This subsystem calculates the distance between the current point and the Helipad Lcoation. If the drone gets closer to the helipad, it stops the simulation. The propeller will be turned off since the sensor at the landing helipad will get activated. 85 | 86 | ## How to change the start location, land location and other parameters? ## 87 | All the variables used in the model are lcoated in the *startScript.m* file. To change any of the paramters, change the variable value in the file. You can *Run* the file to ensure that the variables are updated. 88 | 89 | 90 | 91 | 92 | 93 | 94 | - *droneStartLocation*: The location from where the drone will take off. 95 | - *helipadLocation*: The location where the drone will land. Presently this is assigned as the center ocation of the helipad. This can be modified to a value where the drone needs to land. 96 | - *linearVelocity*: The linear velocity [V_X V_Z V_Y] during optical flow navigation. 97 | - *angularVelocity*: The angular velocity [W_X W_Z W_Y] during optical flow navigation. 98 | - *angularVelocityforLanding*: The angular velocity of the drone during landing. 99 | - *switchingXLimit*: The X-coordinate to switch from Optical flow controller to PID Controller 100 | - *finalHoverHeight*: The height to reach over the helipad before landing. 101 | - *delta*: The threshold distance to execute the PID controller. The drone will start landing if the present coordinate of the drone is within this *delta* value. 102 | 103 | 104 | 105 | ## How to tune linear and angular velocities? ## 106 | 107 | The linear and angular velocities defined in *startScript.mlx* need to tuned if the Simulink 3D Animation environment is modified. The following are the steps to tune the same: 108 | 1. Ensure that with a given linear velocity, the drone doesn't move out of the arena. 109 | 2. *V_Y* is the velocity which will be determined depending on the output from the optical flow algorithm. Here, the magnitude of V_Y remains the same but just the sign is changed. If: 110 | - *vleftflow* > *vrightflow*, then *V_Y* is positive. Hence, the drone will steer to its right. 111 | - *vrightflow* > *vleftflow*, then *V_Y* is negative. Hence, the drone will steer to its left. 112 | 3. Similarly, the angular velocity *W_Y* will be determined depending on the output from the optical flow algorithm. Here, the magnitude of W_Y remains the same but just the sign is changed.If: 113 | - *vleftflow* > *vrightflow* then *W_Y* is negative (clockwise) 114 | - *vrightflow* > *vleftflow* then *W_Y* is positive (anti-clockwise) 115 | 4. If the drone was not be able to bypass all the obstacles with the given *V_Y*, change *V_Y* along with some incremental change in roll velocity. 116 | 5. Always try with a smaller angular velocity values to ensure less oscillations since the decisions are made depending on the flow differences.For example, if there are lots of obstacles on both the sides of the drone, the sign of *flowDifference* will flip rapidly. In case the angular velocities are high here, the drone will have too many oscillations which are not desirable. 117 | 118 | *Note*: The first component in linear velocity is the *V_X*, the second component in linear velocity is the *V_Z*, the third component in linear velocity is the *V_Y*. 119 | 120 | ## Product Requirements ## 121 | 122 | The model use the following MathWorks products: 123 | 124 | 1. MATLAB 125 | 2. Simulink® 126 | 3. Simulink 3D Animation 127 | 4. Stateflow 128 | 5. Aerospace Blockset™ 129 | 7. Aerospace Toolbox 130 | 8. Computer Vision Toolbox™ 131 | 132 | In case of any questions, please reach out to us at roboticsarena@mathworks.com. 133 | 134 | Copyright 2021 The MathWorks, Inc. 135 | 136 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Reporting Security Vulnerabilities 2 | 3 | If you believe you have discovered a security vulnerability, please report it to 4 | [security@mathworks.com](mailto:security@mathworks.com). Please see 5 | [MathWorks Vulnerability Disclosure Policy for Security Researchers](https://www.mathworks.com/company/aboutus/policies_statements/vulnerability-disclosure-policy.html) 6 | for additional information. -------------------------------------------------------------------------------- /license.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2020, The MathWorks, Inc. 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 5 | 6 | 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 7 | 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 8 | 3. In all cases, the software is, and all modifications and derivatives of the software shall be, licensed to you solely for use in conjunction with MathWorks products and service offerings. 9 | 10 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --------------------------------------------------------------------------------