├── .idea ├── .gitignore ├── inspectionProfiles │ └── profiles_settings.xml ├── misc.xml ├── modules.xml ├── uavSim-3ac1c.iml └── vcs.xml ├── README.md ├── __pycache__ ├── utils.cpython-311.pyc └── utils.cpython-38.pyc ├── config ├── cpp.json └── manhattan32_cpp.json ├── example └── scenarios │ ├── manhattan_cpp.json │ └── manhattan_cpp_target.png ├── logs └── training │ ├── manhattan32_cpp │ ├── test │ │ └── events.out.tfevents.1695304762.DESKTOP-41HH9JU.6676.1.v2 │ ├── train │ │ └── events.out.tfevents.1695304762.DESKTOP-41HH9JU.6676.2.v2 │ └── training │ │ └── events.out.tfevents.1695304762.DESKTOP-41HH9JU.6676.0.v2 │ ├── manhattan32_cpp_mc │ ├── test │ │ └── events.out.tfevents.1695105448.DESKTOP-41HH9JU.9600.1.v2 │ ├── train │ │ └── events.out.tfevents.1695105449.DESKTOP-41HH9JU.9600.2.v2 │ └── training │ │ └── events.out.tfevents.1695105448.DESKTOP-41HH9JU.9600.0.v2 │ └── scenario_random │ ├── test │ └── events.out.tfevents.1695311938.DESKTOP-41HH9JU.12184.1.v2 │ ├── train │ └── events.out.tfevents.1695311939.DESKTOP-41HH9JU.12184.2.v2 │ └── training │ └── events.out.tfevents.1695311938.DESKTOP-41HH9JU.12184.0.v2 ├── main.py ├── main_mc.py ├── main_scenario.py ├── models ├── checkpoint ├── manhattan32_cpp_best.data-00000-of-00001 ├── manhattan32_cpp_best.index ├── manhattan32_cpp_unfinished.data-00000-of-00001 └── manhattan32_cpp_unfinished.index ├── res ├── manhattan32.png └── manhattan32_shadowing.npy ├── src ├── CPP │ ├── Display.py │ ├── Environment.py │ ├── Grid.py │ ├── Physics.py │ ├── RandomTargetGenerator.py │ ├── Rewards.py │ ├── SimpleSquareCamera.py │ ├── State.py │ ├── __init__.py │ └── __pycache__ │ │ ├── Display.cpython-311.pyc │ │ ├── Display.cpython-38.pyc │ │ ├── Environment.cpython-311.pyc │ │ ├── Environment.cpython-38.pyc │ │ ├── Grid.cpython-311.pyc │ │ ├── Grid.cpython-38.pyc │ │ ├── Physics.cpython-311.pyc │ │ ├── Physics.cpython-38.pyc │ │ ├── RandomTargetGenerator.cpython-311.pyc │ │ ├── RandomTargetGenerator.cpython-38.pyc │ │ ├── Rewards.cpython-311.pyc │ │ ├── Rewards.cpython-38.pyc │ │ ├── SimpleSquareCamera.cpython-311.pyc │ │ ├── SimpleSquareCamera.cpython-38.pyc │ │ ├── State.cpython-311.pyc │ │ ├── State.cpython-38.pyc │ │ ├── __init__.cpython-311.pyc │ │ └── __init__.cpython-38.pyc ├── DDQN │ ├── Agent.py │ ├── ReplayMemory.py │ ├── Trainer.py │ ├── __init__.py │ └── __pycache__ │ │ ├── Agent.cpython-311.pyc │ │ ├── Agent.cpython-38.pyc │ │ ├── ReplayMemory.cpython-311.pyc │ │ ├── ReplayMemory.cpython-38.pyc │ │ ├── Trainer.cpython-311.pyc │ │ ├── Trainer.cpython-38.pyc │ │ ├── __init__.cpython-311.pyc │ │ └── __init__.cpython-38.pyc ├── Map │ ├── Map.py │ ├── Shadowing.py │ ├── __init__.py │ └── __pycache__ │ │ ├── Map.cpython-311.pyc │ │ ├── Map.cpython-38.pyc │ │ ├── Shadowing.cpython-311.pyc │ │ ├── Shadowing.cpython-38.pyc │ │ ├── __init__.cpython-311.pyc │ │ └── __init__.cpython-38.pyc ├── ModelStats.py ├── StateUtils.py ├── __init__.py ├── __pycache__ │ ├── ModelStats.cpython-311.pyc │ ├── ModelStats.cpython-38.pyc │ ├── StateUtils.cpython-311.pyc │ ├── StateUtils.cpython-38.pyc │ ├── __init__.cpython-311.pyc │ └── __init__.cpython-38.pyc └── base │ ├── BaseDisplay.py │ ├── BaseGrid.py │ ├── BaseState.py │ ├── Environment.py │ ├── GridActions.py │ ├── GridPhysics.py │ ├── GridRewards.py │ ├── __init__.py │ └── __pycache__ │ ├── BaseDisplay.cpython-311.pyc │ ├── BaseDisplay.cpython-38.pyc │ ├── BaseGrid.cpython-311.pyc │ ├── BaseGrid.cpython-38.pyc │ ├── BaseState.cpython-311.pyc │ ├── BaseState.cpython-38.pyc │ ├── Environment.cpython-311.pyc │ ├── Environment.cpython-38.pyc │ ├── GridActions.cpython-311.pyc │ ├── GridActions.cpython-38.pyc │ ├── GridPhysics.cpython-311.pyc │ ├── GridPhysics.cpython-38.pyc │ ├── GridRewards.cpython-311.pyc │ ├── GridRewards.cpython-38.pyc │ ├── __init__.cpython-311.pyc │ └── __init__.cpython-38.pyc └── utils.py /.idea/.gitignore: -------------------------------------------------------------------------------- 1 | # Default ignored files 2 | /shelf/ 3 | /workspace.xml 4 | -------------------------------------------------------------------------------- /.idea/inspectionProfiles/profiles_settings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 6 | -------------------------------------------------------------------------------- /.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/uavSim-3ac1c.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 14 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Table of contents 2 | 3 | * [Introduction](#introduction) 4 | * [Requirements](#requirements) 5 | * [How to use](#how-to-use) 6 | * [Resources](#resources) 7 | * [Reference](#reference) 8 | * [License](# 9 | * license) 10 | 11 | ## Introduction 12 | 13 | This repository contains an implementation of the double deep Q-learning (DDQN) approach to control a UAV on a coverage path planning mission, including global-local map processing. 14 | 15 | 16 | ## Requirements 17 | 18 | ``` 19 | python==3.8.10 20 | 21 | or newer 22 | numpy==1.24.4 or newer 23 | keras==2.10.0 or newer 24 | tensorflow==2.10.0 or newer 25 | matplotlib==3.7.1 or newer 26 | scikit-image==0.16.2 or newer 27 | tqdm==4.45.0 or newer 28 | opencv-python=4.7.0 or newer 29 | ``` 30 | 31 | 32 | ## How to use 33 | 34 | Train a new DDQN model with the parameters of your choice in the specified config file for CPP or DH: 35 | 36 | ``` 37 | python main.py --cpp --gpu --config config/manhattan32_cpp.json --id manhattan32_cpp 38 | 39 | --cpp Activates CPP 40 | --gpu Activates GPU acceleration for DDQN training 41 | --config Path to config file in json format 42 | --id Overrides standard name for logfiles and model 43 | --generate_config Enable only to write default config from default values in the code 44 | ``` 45 | 46 | Evaluate a model through Monte Carlo analysis over the random parameter space for the performance indicators 'Successful Landing', 'Collection Ratio', 'Collection Ratio and Landed' as defined in the paper (plus 'Boundary Counter' counting safety controller activations), e.g. for 1000 Monte Carlo iterations: 47 | 48 | ``` 49 | python main_mc.py --cpp --weights example/models/manhattan32_cpp --config config/manhattan32_cpp.json --id manhattan32_cpp_mc --samples 1000 50 | 51 | 52 | --cpp Activates CPP 53 | --weights Path to weights of trained model 54 | --config Path to config file in json format 55 | --id Name for exported files 56 | --samples Number of Monte Carlo over random scenario parameters 57 | --seed Seed for repeatability 58 | --show Pass '--show True' for individual plots of scenarios and allow plot saving 59 | ``` 60 | 61 | For an example run of pretrained agents the following commands can be used: 62 | ``` 63 | python main_scenario.py --cpp --config config/manhattan32_cpp.json --weights example/models/manhattan32_cpp --scenario example/scenarios/manhattan_cpp.json --video 64 | 65 | ``` 66 | 67 | ## Resources 68 | 69 | The city environments from the paper 'manhattan32' are included in the 'res' directory. Map information is formatted as PNG files with one pixel representing on grid world cell. The pixel color determines the type of cell according to 70 | 71 | * red #ff0000 no-fly zone (NFZ) 72 | * blue #0000ff start and landing zone 73 | * yellow #ffff00 buildings blocking wireless links (also obstacles for flying) 74 | 75 | If you would like to create a new map, you can use any tool to design a PNG with the same pixel dimensions as the desired map and the above color codes. 76 | 77 | The shadowing maps, defining for each position and each IoT device whether there is a line-of-sight (LoS) or non-line-of-sight (NLoS) connection, are computed automatically the first time a new map is used for training and then saved to the 'res' directory as an NPY file. The shadowing maps are further used to determine which cells the field of view of the camera in a CPP scenario. 78 | 79 | 80 | ## Reference 81 | 82 | If using this code for research purposes, please cite: 83 | 84 | [1] M. Theile, H. Bayerlein, R. Nai, D. Gesbert, M. Caccamo, “UAV Path Planning using Global and Local Map Information with Deep Reinforcement Learning" arXiv:2010.06917 [cs.RO], 2020. 85 | 86 | ``` 87 | @article{Theile2020, 88 | author = {Mirco Theile and Harald Bayerlein and Richard Nai and David Gesbert and Marco Caccamo}, 89 | title = {{UAV} Path Planning using Global and Local Map Information with Deep Reinforcement Learning}, 90 | journal = {arXiv:2010.06917 [cs.RO]}, 91 | year = {2020}, 92 | url = {https://arxiv.org/abs/2010.06917} 93 | } 94 | ``` 95 | 96 | 97 | ## License 98 | 99 | This code is under a BSD license. -------------------------------------------------------------------------------- /__pycache__/utils.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/__pycache__/utils.cpython-311.pyc -------------------------------------------------------------------------------- /__pycache__/utils.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/__pycache__/utils.cpython-38.pyc -------------------------------------------------------------------------------- /config/cpp.json: -------------------------------------------------------------------------------- 1 | { 2 | "model_stats_params": { 3 | "save_model": "models/save_model", 4 | "moving_average_length": 200, 5 | "log_file_name": "20201022-194431", 6 | "training_images": false 7 | }, 8 | "grid_params": { 9 | "movement_range": [ 10 | 100, 11 | 200 12 | ], 13 | "map_path": "res/manhattan32.png", 14 | "generator_params": { 15 | "shape": [ 16 | 32, 17 | 32 18 | ], 19 | "coverage_range": [ 20 | 0.2, 21 | 0.8 22 | ], 23 | "shape_range": [ 24 | 1, 25 | 5 26 | ] 27 | } 28 | }, 29 | "reward_params": { 30 | "boundary_penalty": 1.0, 31 | "empty_battery_penalty": 150.0, 32 | "movement_penalty": 0.2, 33 | "cell_multiplier": 0.4 34 | }, 35 | "trainer_params": { 36 | "batch_size": 128, 37 | "num_steps": 10000, 38 | "rm_pre_fill_ratio": 0.0, 39 | "rm_pre_fill_random": true, 40 | "eval_period": 5, 41 | "rm_size": 50000, 42 | "load_model": "" 43 | }, 44 | "agent_params": { 45 | "conv_layers": 2, 46 | "conv_kernel_size": 5, 47 | "conv_kernels": 16, 48 | "hidden_layer_size": 256, 49 | "hidden_layer_num": 3, 50 | "learning_rate": 3e-05, 51 | "alpha": 0.005, 52 | "gamma": 0.95, 53 | "soft_max_scaling": 0.1, 54 | "use_global_local": true, 55 | "global_map_scaling": 3, 56 | "local_map_size": 17, 57 | "print_summary": false 58 | }, 59 | "physics_params": { 60 | "camera_params": { 61 | "half_length": 2, 62 | "map_path": "res/manhattan32.png" 63 | } 64 | } 65 | } -------------------------------------------------------------------------------- /config/manhattan32_cpp.json: -------------------------------------------------------------------------------- 1 | { 2 | "model_stats_params": { 3 | "save_model": "models/manhattan32_cpp", 4 | "moving_average_length": 200, 5 | "log_file_name": "manhattan32_cpp", 6 | "training_images": false 7 | }, 8 | "grid_params": { 9 | "movement_range": [ 10 | 50, 11 | 200 12 | ], 13 | "map_path": "res/manhattan32.png", 14 | "generator_params": { 15 | "coverage_range": [ 16 | 0.2, 17 | 0.5 18 | ], 19 | "shape_range": [ 20 | 3, 21 | 8 22 | ] 23 | } 24 | }, 25 | "reward_params": { 26 | "boundary_penalty": 1.0, 27 | "empty_battery_penalty": 150.0, 28 | "movement_penalty": 0.2, 29 | "cell_multiplier": 0.4 30 | }, 31 | "trainer_params": { 32 | "batch_size": 128, 33 | "num_steps": 1e5, 34 | "rm_pre_fill_ratio": 0.5, 35 | "rm_pre_fill_random": true, 36 | "eval_period": 5, 37 | "rm_size": 50000, 38 | "load_model": "" 39 | }, 40 | "agent_params": { 41 | "conv_layers": 2, 42 | "conv_kernel_size": 5, 43 | "conv_kernels": 16, 44 | "hidden_layer_size": 256, 45 | "hidden_layer_num": 3, 46 | "learning_rate": 3e-05, 47 | "alpha": 0.005, 48 | "gamma": 0.95, 49 | "soft_max_scaling": 0.1, 50 | "use_global_local": true, 51 | "global_map_scaling": 3, 52 | "local_map_size": 17, 53 | "print_summary": false 54 | }, 55 | "physics_params": { 56 | "camera_params": { 57 | "half_length": 2, 58 | "map_path": "res/manhattan32.png" 59 | } 60 | } 61 | } -------------------------------------------------------------------------------- /example/scenarios/manhattan_cpp.json: -------------------------------------------------------------------------------- 1 | { 2 | "movement_budget": 100, 3 | "position_idx": 4, 4 | "target_path": "example/scenarios/manhattan_cpp_target.png" 5 | } 6 | 7 | -------------------------------------------------------------------------------- /example/scenarios/manhattan_cpp_target.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/example/scenarios/manhattan_cpp_target.png -------------------------------------------------------------------------------- /logs/training/manhattan32_cpp/test/events.out.tfevents.1695304762.DESKTOP-41HH9JU.6676.1.v2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/logs/training/manhattan32_cpp/test/events.out.tfevents.1695304762.DESKTOP-41HH9JU.6676.1.v2 -------------------------------------------------------------------------------- /logs/training/manhattan32_cpp/train/events.out.tfevents.1695304762.DESKTOP-41HH9JU.6676.2.v2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/logs/training/manhattan32_cpp/train/events.out.tfevents.1695304762.DESKTOP-41HH9JU.6676.2.v2 -------------------------------------------------------------------------------- /logs/training/manhattan32_cpp/training/events.out.tfevents.1695304762.DESKTOP-41HH9JU.6676.0.v2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/logs/training/manhattan32_cpp/training/events.out.tfevents.1695304762.DESKTOP-41HH9JU.6676.0.v2 -------------------------------------------------------------------------------- /logs/training/manhattan32_cpp_mc/test/events.out.tfevents.1695105448.DESKTOP-41HH9JU.9600.1.v2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/logs/training/manhattan32_cpp_mc/test/events.out.tfevents.1695105448.DESKTOP-41HH9JU.9600.1.v2 -------------------------------------------------------------------------------- /logs/training/manhattan32_cpp_mc/train/events.out.tfevents.1695105449.DESKTOP-41HH9JU.9600.2.v2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/logs/training/manhattan32_cpp_mc/train/events.out.tfevents.1695105449.DESKTOP-41HH9JU.9600.2.v2 -------------------------------------------------------------------------------- /logs/training/manhattan32_cpp_mc/training/events.out.tfevents.1695105448.DESKTOP-41HH9JU.9600.0.v2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/logs/training/manhattan32_cpp_mc/training/events.out.tfevents.1695105448.DESKTOP-41HH9JU.9600.0.v2 -------------------------------------------------------------------------------- /logs/training/scenario_random/test/events.out.tfevents.1695311938.DESKTOP-41HH9JU.12184.1.v2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/logs/training/scenario_random/test/events.out.tfevents.1695311938.DESKTOP-41HH9JU.12184.1.v2 -------------------------------------------------------------------------------- /logs/training/scenario_random/train/events.out.tfevents.1695311939.DESKTOP-41HH9JU.12184.2.v2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/logs/training/scenario_random/train/events.out.tfevents.1695311939.DESKTOP-41HH9JU.12184.2.v2 -------------------------------------------------------------------------------- /logs/training/scenario_random/training/events.out.tfevents.1695311938.DESKTOP-41HH9JU.12184.0.v2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/logs/training/scenario_random/training/events.out.tfevents.1695311938.DESKTOP-41HH9JU.12184.0.v2 -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | 4 | from src.CPP.Environment import CPPEnvironmentParams, CPPEnvironment 5 | 6 | 7 | from utils import * 8 | 9 | 10 | def main_cpp(p): 11 | env = CPPEnvironment(p) 12 | 13 | env.run() 14 | 15 | 16 | 17 | 18 | if __name__ == "__main__": 19 | 20 | parser = argparse.ArgumentParser() 21 | parser.add_argument('--gpu', action='store_true', help='Activates usage of GPU') 22 | parser.add_argument('--generate_config', action='store_true', help='Enable to write default config only') 23 | parser.add_argument('--config', default=None, help='Path to config file') 24 | parser.add_argument('--id', default=None, help='If set overrides the logfile name and the save name') 25 | 26 | parser.add_argument('--params', nargs='*', default=None) 27 | 28 | parser.add_argument('--cpp', action='store_true', help='Run Coverage Path Planning') 29 | 30 | 31 | args = parser.parse_args() 32 | 33 | if args.generate_config: 34 | if args.cpp: 35 | generate_config(CPPEnvironmentParams(), "config/cpp.json") 36 | else: 37 | print("Specify which config to generate") 38 | exit(0) 39 | 40 | if args.config is None: 41 | print("Config file needed!") 42 | exit(1) 43 | 44 | if not args.gpu: 45 | os.environ["CUDA_VISIBLE_DEVICES"] = "-1" 46 | 47 | params = read_config(args.config) 48 | 49 | if args.params is not None: 50 | params = override_params(params, args.params) 51 | 52 | if args.id is not None: 53 | params.model_stats_params.save_model = "models/" + args.id 54 | params.model_stats_params.log_file_name = args.id 55 | 56 | if args.cpp: 57 | main_cpp(params) 58 | 59 | -------------------------------------------------------------------------------- /main_mc.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | 4 | import numpy as np 5 | 6 | from src.CPP.Environment import CPPEnvironmentParams, CPPEnvironment 7 | 8 | from utils import override_params, read_config 9 | 10 | from tensorboard.backend.event_processing.event_accumulator import EventAccumulator 11 | import tensorflow as tf 12 | 13 | 14 | def eval_logs(event_path): 15 | event_acc = EventAccumulator(event_path, size_guidance={'tensors': 100000}) 16 | event_acc.Reload() 17 | 18 | _, _, vals = zip(*event_acc.Tensors('successful_landing')) 19 | has_landed = [tf.make_ndarray(val) for val in vals] 20 | 21 | _, _, vals = zip(*event_acc.Tensors('cr')) 22 | cr = [tf.make_ndarray(val) for val in vals] 23 | 24 | _, _, vals = zip(*event_acc.Tensors('cral')) 25 | cral = [tf.make_ndarray(val) for val in vals] 26 | 27 | _, _, vals = zip(*event_acc.Tensors('boundary_counter')) 28 | boundary_counter = [tf.make_ndarray(val) for val in vals] 29 | 30 | print("Successful Landing:", sum(has_landed) / len(has_landed)) 31 | print("Collection ratio:", sum(cr) / len(cr)) 32 | print("Collection ratio and landed:", sum(cral) / len(cral)) 33 | 34 | 35 | 36 | 37 | def cpp_mc(args, params: CPPEnvironmentParams): 38 | try: 39 | env = CPPEnvironment(params) 40 | env.agent.load_weights(args.weights) 41 | 42 | env.eval(int(args.samples), show=args.show) 43 | except AttributeError: 44 | print("Not overriding log dir, eval existing:") 45 | 46 | eval_logs("logs/training/" + args.id + "/test") 47 | 48 | 49 | if __name__ == "__main__": 50 | parser = argparse.ArgumentParser() 51 | parser.add_argument('--weights', required=True, help='Path to weights') 52 | parser.add_argument('--config', required=True, help='Config file for agent shaping') 53 | parser.add_argument('--id', required=False, help='Id for exported files') 54 | parser.add_argument('--samples', required=True, help='Id for exported files') 55 | parser.add_argument('--seed', default=None, help="Seed for repeatability") 56 | parser.add_argument('--show', default=False, help="Show individual plots, allows saving") 57 | parser.add_argument('--params', nargs='*', default=None) 58 | 59 | 60 | # CPP Params 61 | parser.add_argument('--cpp', action='store_true', help='Run Coverage Path Planning') 62 | 63 | args = parser.parse_args() 64 | 65 | if args.seed: 66 | np.random.seed(int(args.seed)) 67 | 68 | params = read_config(args.config) 69 | 70 | if args.params is not None: 71 | params = override_params(params, args.params) 72 | 73 | if args.id is not None: 74 | params.model_stats_params.save_model = "models/" + args.id 75 | params.model_stats_params.log_file_name = args.id 76 | 77 | os.environ["CUDA_VISIBLE_DEVICES"] = "-1" 78 | 79 | if args.cpp: 80 | cpp_mc(args, params) 81 | 82 | -------------------------------------------------------------------------------- /main_scenario.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | 4 | import numpy as np 5 | 6 | from src.CPP.Environment import CPPEnvironment 7 | 8 | from utils import override_params, read_config, get_bool_user 9 | 10 | 11 | def save_video(env): 12 | if get_bool_user('Save run?', False): 13 | save_as_default = "video.mp4" 14 | save_as = input('Save as: [{}]\n'.format(save_as_default)) 15 | if save_as == '': 16 | save_as = save_as_default 17 | frame_rate_default = 4 18 | frame_rate = input('Frame rate: [{}]\n'.format(frame_rate_default)) 19 | if frame_rate == '': 20 | frame_rate = frame_rate_default 21 | frame_rate = int(frame_rate) 22 | 23 | draw_path = get_bool_user('Show path?', False) 24 | env.display.create_video(env.grid.map_image, env.stats.trajectory, save_as, frame_rate, draw_path=draw_path) 25 | 26 | 27 | def scenario(args, params): 28 | env = None 29 | if args.cpp: 30 | env = CPPEnvironment(params) 31 | 32 | else: 33 | print("Need --cpp or --dh") 34 | exit(1) 35 | 36 | env.agent.load_weights(args.weights) 37 | 38 | init_state = None 39 | if args.scenario: 40 | scenario = read_config(args.scenario) 41 | init_state = env.grid.create_scenario(scenario) 42 | 43 | env.test_episode(init_state) 44 | env.display.display_episode(env.grid.map_image, env.stats.trajectory, plot=True) 45 | if args.video: 46 | save_video(env) 47 | 48 | 49 | if __name__ == "__main__": 50 | parser = argparse.ArgumentParser() 51 | parser.add_argument('--weights', required=True, help='Path to weights') 52 | parser.add_argument('--config', required=True, help='Config file for agent shaping') 53 | parser.add_argument('--scenario', default=None, help='Config file for scenario') 54 | parser.add_argument('--seed', default=None, help="Seed for repeatability") 55 | parser.add_argument('--video', action='store_true', help="Will ask to create video after plotting") 56 | parser.add_argument('--params', nargs='*', default=None) 57 | 58 | 59 | # CPP Params 60 | parser.add_argument('--cpp', action='store_true', help='Run Coverage Path Planning') 61 | 62 | args = parser.parse_args() 63 | 64 | if args.seed: 65 | np.random.seed(int(args.seed)) 66 | 67 | params = read_config(args.config) 68 | 69 | if args.params is not None: 70 | params = override_params(params, args.params) 71 | 72 | params.model_stats_params.save_model = "models/save" 73 | if args.seed: 74 | params.model_stats_params.log_file_name = "scenario_" + str(args.seed) 75 | else: 76 | params.model_stats_params.log_file_name = "scenario_random" 77 | 78 | os.environ["CUDA_VISIBLE_DEVICES"] = "-1" 79 | 80 | 81 | scenario(args, params) 82 | -------------------------------------------------------------------------------- /models/checkpoint: -------------------------------------------------------------------------------- 1 | model_checkpoint_path: "manhattan32_cpp_unfinished" 2 | all_model_checkpoint_paths: "manhattan32_cpp_unfinished" 3 | -------------------------------------------------------------------------------- /models/manhattan32_cpp_best.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/models/manhattan32_cpp_best.data-00000-of-00001 -------------------------------------------------------------------------------- /models/manhattan32_cpp_best.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/models/manhattan32_cpp_best.index -------------------------------------------------------------------------------- /models/manhattan32_cpp_unfinished.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/models/manhattan32_cpp_unfinished.data-00000-of-00001 -------------------------------------------------------------------------------- /models/manhattan32_cpp_unfinished.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/models/manhattan32_cpp_unfinished.index -------------------------------------------------------------------------------- /res/manhattan32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/res/manhattan32.png -------------------------------------------------------------------------------- /res/manhattan32_shadowing.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/res/manhattan32_shadowing.npy -------------------------------------------------------------------------------- /src/CPP/Display.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | from src.Map.Map import Map 4 | from src.base.BaseDisplay import BaseDisplay 5 | 6 | 7 | class CPPDisplay(BaseDisplay): 8 | 9 | def __init__(self): 10 | super().__init__() 11 | 12 | def display_episode(self, env_map: Map, trajectory, plot=False, save_path=None): 13 | 14 | first_state = trajectory[0][0] 15 | final_state = trajectory[-1][3] 16 | 17 | fig_size = 5.5 18 | fig, ax = plt.subplots(1, 1, figsize=[fig_size, fig_size]) 19 | value_map = final_state.coverage * 1.0 + (~final_state.coverage) * 0.75 20 | 21 | self.create_grid_image(ax=ax, env_map=env_map, value_map=value_map, green=first_state.target) 22 | 23 | self.draw_start_and_end(trajectory) 24 | 25 | # plot trajectory arrows 26 | for exp in trajectory: 27 | self.draw_movement(exp[0].position, exp[3].position, color="black") 28 | 29 | # save image and return 30 | if save_path is not None: 31 | fig.savefig(save_path, bbox_inches='tight', 32 | format='png', dpi=300) 33 | if plot: 34 | plt.show() 35 | 36 | return self.create_tf_image() 37 | 38 | def display_state(self, env_map, initial_state, state, plot=False): 39 | fig_size = 5.5 40 | fig, ax = plt.subplots(1, 1, figsize=[fig_size, fig_size]) 41 | value_map = state.coverage * 1.0 + (~state.coverage) * 0.75 42 | 43 | self.create_grid_image(ax=ax, env_map=env_map, value_map=value_map, green=initial_state.target) 44 | 45 | color = "green" if state.landed else "r" 46 | plt.scatter(state.position[0] + 0.5, state.position[1] + 0.5, 47 | s=self.marker_size, marker="D", color=color) 48 | 49 | if plot: 50 | plt.show() 51 | 52 | return self.create_tf_image() 53 | 54 | def draw_map(self, map_in): 55 | rgb = map_in[0, :, :, :3] 56 | rgb = np.stack([rgb[:, :, 0], rgb[:, :, 2], rgb[:, :, 1]], axis=2) 57 | plt.imshow(rgb.astype(float)) 58 | plt.show() 59 | 60 | def draw_maps(self, total_map, global_map, local_map): 61 | fig, ax = plt.subplots(1, 3) 62 | rgb = total_map[0, :, :, :3] 63 | rgb = np.stack([rgb[:, :, 0], rgb[:, :, 2], rgb[:, :, 1]], axis=2) 64 | ax[0].imshow(rgb.astype(float)) 65 | 66 | rgb = global_map[0, :, :, :3] 67 | rgb = np.stack([rgb[:, :, 0], rgb[:, :, 2], rgb[:, :, 1]], axis=2) 68 | ax[1].imshow(rgb.astype(float)) 69 | 70 | rgb = local_map[0, :, :, :3] 71 | rgb = np.stack([rgb[:, :, 0], rgb[:, :, 2], rgb[:, :, 1]], axis=2) 72 | ax[2].imshow(rgb.astype(float)) 73 | plt.show() 74 | -------------------------------------------------------------------------------- /src/CPP/Environment.py: -------------------------------------------------------------------------------- 1 | import copy 2 | 3 | from src.DDQN.Agent import DDQNAgentParams, DDQNAgent 4 | from src.CPP.Display import CPPDisplay 5 | from src.CPP.Grid import CPPGrid, CPPGridParams 6 | from src.CPP.Physics import CPPPhysics, CPPPhysicsParams 7 | from src.CPP.State import CPPState 8 | from src.CPP.Rewards import CPPRewardParams, CPPRewards 9 | 10 | from src.DDQN.Trainer import DDQNTrainerParams, DDQNTrainer 11 | from src.base.Environment import BaseEnvironment, BaseEnvironmentParams 12 | from src.base.GridActions import GridActions 13 | 14 | 15 | class CPPEnvironmentParams(BaseEnvironmentParams): 16 | def __init__(self): 17 | super().__init__() 18 | self.grid_params = CPPGridParams() 19 | self.reward_params = CPPRewardParams() 20 | self.trainer_params = DDQNTrainerParams() 21 | self.agent_params = DDQNAgentParams() 22 | self.physics_params = CPPPhysicsParams() 23 | 24 | 25 | class CPPEnvironment(BaseEnvironment): 26 | 27 | def __init__(self, params: CPPEnvironmentParams): 28 | self.display = CPPDisplay() 29 | super().__init__(params, self.display) 30 | 31 | self.grid = CPPGrid(params.grid_params, self.stats) 32 | self.rewards = CPPRewards(params.reward_params, stats=self.stats) 33 | self.physics = CPPPhysics(params=params.physics_params, stats=self.stats) 34 | self.agent = DDQNAgent(params.agent_params, self.grid.get_example_state(), self.physics.get_example_action(), 35 | stats=self.stats) 36 | self.trainer = DDQNTrainer(params=params.trainer_params, agent=self.agent) 37 | 38 | -------------------------------------------------------------------------------- /src/CPP/Grid.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | import src.Map.Map as ImageLoader 4 | from src.CPP.State import CPPState, CPPScenario 5 | from src.CPP.RandomTargetGenerator import RandomTargetGenerator, RandomTargetGeneratorParams 6 | from src.base.BaseGrid import BaseGrid, BaseGridParams 7 | 8 | 9 | class CPPGridParams(BaseGridParams): 10 | def __init__(self): 11 | super().__init__() 12 | self.generator_params = RandomTargetGeneratorParams() 13 | 14 | 15 | class CPPGrid(BaseGrid): 16 | 17 | def __init__(self, params: CPPGridParams, stats): 18 | super().__init__(params, stats) 19 | self.params = params 20 | 21 | self.generator = RandomTargetGenerator(params.generator_params, self.map_image.get_size()) 22 | self.target_zone = self.generator.generate_target(self.map_image.obstacles) 23 | 24 | def init_episode(self): 25 | self.target_zone = self.generator.generate_target(self.map_image.obstacles) 26 | 27 | state = CPPState(self.map_image) 28 | state.reset_target(self.target_zone) 29 | 30 | idx = np.random.randint(0, len(self.starting_vector)) 31 | state.position = self.starting_vector[idx] 32 | 33 | state.movement_budget = np.random.randint(low=self.params.movement_range[0], 34 | high=self.params.movement_range[1] + 1) 35 | 36 | state.initial_movement_budget = state.movement_budget 37 | state.landed = False 38 | state.terminal = False 39 | 40 | return state 41 | 42 | def create_scenario(self, scenario: CPPScenario): 43 | state = CPPState(self.map_image) 44 | target = ImageLoader.load_target(scenario.target_path, self.map_image.obstacles) 45 | state.reset_target(target) 46 | state.position = self.starting_vector[scenario.position_idx] 47 | state.movement_budget = scenario.movement_budget 48 | state.initial_movement_budget = scenario.movement_budget 49 | return state 50 | 51 | def init_scenario(self, state: CPPState): 52 | self.target_zone = state.target 53 | 54 | return state 55 | 56 | def get_example_state(self): 57 | state = CPPState(self.map_image) 58 | state.reset_target(self.target_zone) 59 | state.position = [0, 0] 60 | state.movement_budget = 0 61 | state.initial_movement_budget = 0 62 | return state 63 | 64 | def get_target_zone(self): 65 | return self.target_zone 66 | -------------------------------------------------------------------------------- /src/CPP/Physics.py: -------------------------------------------------------------------------------- 1 | from src.CPP.State import CPPState 2 | from src.CPP.SimpleSquareCamera import SimpleSquareCameraParams, SimpleSquareCamera 3 | from src.ModelStats import ModelStats 4 | from src.base.GridActions import GridActions, GridActionsNoHover 5 | from src.base.GridPhysics import GridPhysics 6 | 7 | 8 | class CPPPhysicsParams: 9 | def __init__(self): 10 | self.camera_params = SimpleSquareCameraParams() 11 | 12 | 13 | class CPPPhysics(GridPhysics): 14 | def __init__(self, params: CPPPhysicsParams, stats: ModelStats): 15 | super().__init__() 16 | self.landed = False 17 | 18 | self.camera = None 19 | 20 | self.params = params 21 | 22 | self.register_functions(stats) 23 | 24 | def register_functions(self, stats: ModelStats): 25 | stats.set_evaluation_value_callback(self.get_cral) 26 | 27 | stats.add_log_data_callback('cral', self.get_cral) 28 | stats.add_log_data_callback('cr', self.get_coverage_ratio) 29 | stats.add_log_data_callback('successful_landing', self.has_landed) 30 | stats.add_log_data_callback('boundary_counter', self.get_boundary_counter) 31 | stats.add_log_data_callback('landing_attempts', self.get_landing_attempts) 32 | stats.add_log_data_callback('movement_ratio', self.get_movement_ratio) 33 | 34 | def reset(self, state: CPPState): 35 | GridPhysics.reset(self, state) 36 | self.landed = False 37 | 38 | self.camera = SimpleSquareCamera(self.params.camera_params) 39 | 40 | def step(self, action: GridActions): 41 | self.movement_step(action) 42 | if not self.state.terminal: 43 | self.vision_step() 44 | 45 | if self.state.landed: 46 | self.landed = True 47 | 48 | return self.state 49 | 50 | def vision_step(self): 51 | view = self.camera.computeView(self.state.position, 0) 52 | self.state.add_explored(view) 53 | 54 | def get_example_action(self): 55 | return GridActionsNoHover.LAND 56 | 57 | def is_in_landing_zone(self): 58 | return self.state.is_in_landing_zone() 59 | 60 | def get_coverage_ratio(self): 61 | return self.state.get_coverage_ratio() 62 | 63 | def get_movement_budget_used(self): 64 | return self.state.initial_movement_budget - self.state.movement_budget 65 | 66 | def get_cral(self): 67 | return self.get_coverage_ratio() * self.landed 68 | 69 | def get_boundary_counter(self): 70 | return self.boundary_counter 71 | 72 | def get_landing_attempts(self): 73 | return self.landing_attempts 74 | 75 | def get_movement_ratio(self): 76 | return float(self.get_movement_budget_used()) / float(self.state.initial_movement_budget) 77 | 78 | def has_landed(self): 79 | return self.landed 80 | -------------------------------------------------------------------------------- /src/CPP/RandomTargetGenerator.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from skimage.draw import random_shapes 3 | import logging 4 | 5 | 6 | class RandomTargetGeneratorParams: 7 | def __init__(self): 8 | self.coveragfe_range = (0.2, 0.8) 9 | self.shape_range = (1, 5) 10 | 11 | 12 | class RandomTargetGenerator: 13 | 14 | def __init__(self, params: RandomTargetGeneratorParams, shape): 15 | self.params = params 16 | self.shape = shape 17 | 18 | def generate_target(self, obstacles): 19 | 20 | area = np.product(self.shape) 21 | 22 | target = self.__generate_random_shapes_area( 23 | self.params.shape_range[0], 24 | self.params.shape_range[1], 25 | area * self.params.coverage_range[0], 26 | area * self.params.coverage_range[1] 27 | ) 28 | 29 | return target & ~obstacles 30 | 31 | def __generate_random_shapes(self, min_shapes, max_shapes): 32 | img, _ = random_shapes(self.shape, max_shapes, min_shapes=min_shapes, multichannel=False, 33 | allow_overlap=True, random_seed=np.random.randint(2**16 - 1)) 34 | # Numpy random usage for random seed unifies random seed which can be set for repeatability 35 | attempt = np.array(img != 255, dtype=bool) 36 | return attempt, np.sum(attempt) 37 | 38 | def __generate_random_shapes_area(self, min_shapes, max_shapes, min_area, max_area, retry=100): 39 | for attemptno in range(retry): 40 | attempt, area = self.__generate_random_shapes(min_shapes, max_shapes) 41 | if min_area is not None and min_area > area: 42 | continue 43 | if max_area is not None and max_area < area: 44 | continue 45 | return attempt 46 | print("Here") 47 | logging.warning("Was not able to generate shapes with given area constraint in allowed number of tries." 48 | " Randomly returning next attempt.") 49 | attempt, area = self.__generate_random_shapes(min_shapes, max_shapes) 50 | logging.warning("Size is: ", area) 51 | return attempt 52 | 53 | def __generate_exclusive_shapes(self, exclusion, min_shapes, max_shapes): 54 | attempt, area = self.__generate_random_shapes(min_shapes, max_shapes) 55 | attempt = attempt & (~exclusion) 56 | area = np.sum(attempt) 57 | return attempt, area 58 | 59 | # Create target image and then subtract exclusion area 60 | def __generate_exclusive_shapes_area(self, exclusion, min_shapes, max_shapes, min_area, max_area, retry=100): 61 | for attemptno in range(retry): 62 | attempt, area = self.__generate_exclusive_shapes(exclusion, min_shapes, max_shapes) 63 | if min_area is not None and min_area > area: 64 | continue 65 | if max_area is not None and max_area < area: 66 | continue 67 | return attempt 68 | 69 | logging.warning("Was not able to generate shapes with given area constraint in allowed number of tries." 70 | " Randomly returning next attempt.") 71 | attempt, area = self.__generate_exclusive_shapes(exclusion, min_shapes, max_shapes) 72 | logging.warning("Size is: ", area) 73 | return attempt 74 | -------------------------------------------------------------------------------- /src/CPP/Rewards.py: -------------------------------------------------------------------------------- 1 | from src.CPP.State import CPPState 2 | from src.base.GridActions import GridActions 3 | from src.base.GridRewards import GridRewardParams, GridRewards 4 | 5 | 6 | class CPPRewardParams(GridRewardParams): 7 | def __init__(self): 8 | super().__init__() 9 | self.cell_multiplier = 0.4 10 | 11 | 12 | # Class used to track rewards 13 | class CPPRewards(GridRewards): 14 | 15 | def __init__(self, reward_params: CPPRewardParams, stats): 16 | super().__init__(stats) 17 | self.params = reward_params 18 | self.reset() 19 | 20 | def calculate_reward(self, state: CPPState, action: GridActions, next_state: CPPState): 21 | reward = self.calculate_motion_rewards(state, action, next_state) 22 | 23 | # Reward the collected data 24 | reward += self.params.cell_multiplier * (state.get_remaining_cells() - next_state.get_remaining_cells()) 25 | 26 | # Cumulative reward 27 | self.cumulative_reward += reward 28 | 29 | return reward 30 | -------------------------------------------------------------------------------- /src/CPP/SimpleSquareCamera.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | 4 | from src.Map.Shadowing import load_or_create_shadowing 5 | from src.Map.Map import load_map 6 | 7 | 8 | class SimpleSquareCameraParams: 9 | def __init__(self): 10 | self.half_length = 2 11 | self.map_path = "res/downtown.png" 12 | 13 | 14 | class SimpleSquareCamera: 15 | 16 | def __init__(self, params: SimpleSquareCameraParams): 17 | self.params = params 18 | total_map = load_map(self.params.map_path) 19 | self.obstacles = total_map.obstacles 20 | self.size = self.obstacles.shape[:2] 21 | self.obstruction_map = load_or_create_shadowing(self.params.map_path) 22 | 23 | def computeView(self, position, attitude): 24 | view = np.zeros(self.size, dtype=bool) 25 | camera_width = self.params.half_length 26 | x_pos, y_pos = position[0], position[1] 27 | x_size, y_size = self.size[0], self.size[1] 28 | view[max(0, y_pos - camera_width):min(y_size, y_pos + camera_width + 1), 29 | max(0, x_pos - camera_width):min(x_size, x_pos + camera_width + 1)] = True 30 | 31 | view &= ~self.obstacles 32 | view &= ~self.obstruction_map[y_pos, x_pos] 33 | return view 34 | -------------------------------------------------------------------------------- /src/CPP/State.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from src.Map.Map import Map 4 | from src.StateUtils import pad_centered 5 | from src.base.BaseState import BaseState 6 | 7 | 8 | class CPPScenario: 9 | def __init__(self): 10 | self.target_path = "" 11 | self.position_idx = 0 12 | self.movement_budget = 100 13 | 14 | 15 | class CPPState(BaseState): 16 | def __init__(self, map_init: Map): 17 | super().__init__(map_init) 18 | self.target = None 19 | self.position = None 20 | self.movement_budget = None 21 | self.landed = False 22 | self.terminal = False 23 | 24 | self.initial_movement_budget = None 25 | self.initial_target_cell_count = 0 26 | self.coverage = None 27 | 28 | def reset_target(self, target): 29 | self.target = target 30 | self.initial_target_cell_count = np.sum(target) 31 | self.coverage = np.zeros(self.target.shape, dtype=bool) 32 | 33 | def get_remaining_cells(self): 34 | return np.sum(self.target) 35 | 36 | def get_total_cells(self): 37 | return self.initial_target_cell_count 38 | 39 | def get_coverage_ratio(self): 40 | return 1.0 - float(np.sum(self.get_remaining_cells())) / float(self.initial_target_cell_count) 41 | 42 | def get_scalars(self): 43 | return np.array([self.movement_budget]) 44 | 45 | def get_num_scalars(self): 46 | return 1 47 | 48 | def get_boolean_map(self): 49 | padded_red = pad_centered(self, np.concatenate([np.expand_dims(self.no_fly_zone, -1), 50 | np.expand_dims(self.obstacles, -1)], axis=-1), 1) 51 | padded_rest = pad_centered(self, np.concatenate([np.expand_dims(self.landing_zone, -1), 52 | np.expand_dims(self.target, -1)], axis=-1), 0) 53 | return np.concatenate([padded_red, padded_rest], axis=-1) 54 | 55 | def get_boolean_map_shape(self): 56 | return self.get_boolean_map().shape 57 | 58 | def get_float_map(self): 59 | shape = list(self.get_boolean_map().shape) 60 | shape[2] = 0 61 | float_map = np.zeros(tuple(shape), dtype=float) 62 | return float_map 63 | 64 | def get_float_map_shape(self): 65 | return self.get_float_map().shape 66 | 67 | def is_in_landing_zone(self): 68 | return self.landing_zone[self.position[1]][self.position[0]] 69 | 70 | def is_in_no_fly_zone(self): 71 | # Out of bounds is implicitly nfz 72 | if 0 <= self.position[1] < self.no_fly_zone.shape[0] and 0 <= self.position[0] < self.no_fly_zone.shape[1]: 73 | return self.no_fly_zone[self.position[1], self.position[0]] 74 | return True 75 | 76 | def add_explored(self, view): 77 | self.target &= ~view 78 | self.coverage |= view 79 | 80 | def set_terminal(self, terminal): 81 | self.terminal = terminal 82 | 83 | def set_landed(self, landed): 84 | self.landed = landed 85 | 86 | def set_position(self, position): 87 | self.position = position 88 | 89 | def decrement_movement_budget(self): 90 | self.movement_budget -= 1 91 | 92 | def is_terminal(self): 93 | return self.terminal 94 | -------------------------------------------------------------------------------- /src/CPP/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/CPP/__init__.py -------------------------------------------------------------------------------- /src/CPP/__pycache__/Display.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/CPP/__pycache__/Display.cpython-311.pyc -------------------------------------------------------------------------------- /src/CPP/__pycache__/Display.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/CPP/__pycache__/Display.cpython-38.pyc -------------------------------------------------------------------------------- /src/CPP/__pycache__/Environment.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/CPP/__pycache__/Environment.cpython-311.pyc -------------------------------------------------------------------------------- /src/CPP/__pycache__/Environment.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/CPP/__pycache__/Environment.cpython-38.pyc -------------------------------------------------------------------------------- /src/CPP/__pycache__/Grid.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/CPP/__pycache__/Grid.cpython-311.pyc -------------------------------------------------------------------------------- /src/CPP/__pycache__/Grid.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/CPP/__pycache__/Grid.cpython-38.pyc -------------------------------------------------------------------------------- /src/CPP/__pycache__/Physics.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/CPP/__pycache__/Physics.cpython-311.pyc -------------------------------------------------------------------------------- /src/CPP/__pycache__/Physics.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/CPP/__pycache__/Physics.cpython-38.pyc -------------------------------------------------------------------------------- /src/CPP/__pycache__/RandomTargetGenerator.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/CPP/__pycache__/RandomTargetGenerator.cpython-311.pyc -------------------------------------------------------------------------------- /src/CPP/__pycache__/RandomTargetGenerator.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/CPP/__pycache__/RandomTargetGenerator.cpython-38.pyc -------------------------------------------------------------------------------- /src/CPP/__pycache__/Rewards.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/CPP/__pycache__/Rewards.cpython-311.pyc -------------------------------------------------------------------------------- /src/CPP/__pycache__/Rewards.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/CPP/__pycache__/Rewards.cpython-38.pyc -------------------------------------------------------------------------------- /src/CPP/__pycache__/SimpleSquareCamera.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/CPP/__pycache__/SimpleSquareCamera.cpython-311.pyc -------------------------------------------------------------------------------- /src/CPP/__pycache__/SimpleSquareCamera.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/CPP/__pycache__/SimpleSquareCamera.cpython-38.pyc -------------------------------------------------------------------------------- /src/CPP/__pycache__/State.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/CPP/__pycache__/State.cpython-311.pyc -------------------------------------------------------------------------------- /src/CPP/__pycache__/State.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/CPP/__pycache__/State.cpython-38.pyc -------------------------------------------------------------------------------- /src/CPP/__pycache__/__init__.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/CPP/__pycache__/__init__.cpython-311.pyc -------------------------------------------------------------------------------- /src/CPP/__pycache__/__init__.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/CPP/__pycache__/__init__.cpython-38.pyc -------------------------------------------------------------------------------- /src/DDQN/Agent.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | from tensorflow.keras import Model 4 | from tensorflow.keras.layers import Conv2D, Dense, Flatten, Concatenate, Input, AvgPool2D 5 | 6 | import numpy as np 7 | 8 | 9 | def print_node(x): 10 | print(x) 11 | return x 12 | 13 | 14 | class DDQNAgentParams: 15 | def __init__(self): 16 | # Convolutional part config 17 | self.conv_layers = 2 18 | self.conv_kernel_size = 5 19 | self.conv_kernels = 16 20 | 21 | # Fully Connected config 22 | self.hidden_layer_size = 256 23 | self.hidden_layer_num = 3 24 | 25 | # Training Params 26 | self.learning_rate = 3e-5 27 | self.alpha = 0.005 28 | self.gamma = 0.95 29 | 30 | # Exploration strategy 31 | self.soft_max_scaling = 0.1 32 | 33 | # Global-Local Map 34 | self.use_global_local = True 35 | self.global_map_scaling = 3 36 | self.local_map_size = 17 37 | 38 | # Printing 39 | self.print_summary = False 40 | 41 | 42 | class DDQNAgent(object): 43 | 44 | def __init__(self, params: DDQNAgentParams, example_state, example_action, stats=None): 45 | 46 | self.params = params 47 | gamma = tf.constant(self.params.gamma, dtype=float) 48 | self.align_counter = 0 49 | 50 | self.boolean_map_shape = example_state.get_boolean_map_shape() 51 | self.float_map_shape = example_state.get_float_map_shape() 52 | self.scalars = example_state.get_num_scalars() 53 | self.num_actions = len(type(example_action)) 54 | self.num_map_channels = self.boolean_map_shape[2] + self.float_map_shape[2] 55 | 56 | # Create shared inputs 57 | boolean_map_input = Input(shape=self.boolean_map_shape, name='boolean_map_input', dtype=tf.bool) 58 | float_map_input = Input(shape=self.float_map_shape, name='float_map_input', dtype=tf.float32) 59 | scalars_input = Input(shape=(self.scalars,), name='scalars_input', dtype=tf.float32) 60 | action_input = Input(shape=(), name='action_input', dtype=tf.int64) 61 | reward_input = Input(shape=(), name='reward_input', dtype=tf.float32) 62 | termination_input = Input(shape=(), name='termination_input', dtype=tf.bool) 63 | q_star_input = Input(shape=(), name='q_star_input', dtype=tf.float32) 64 | states = [boolean_map_input, 65 | float_map_input, 66 | scalars_input] 67 | 68 | map_cast = tf.cast(boolean_map_input, dtype=tf.float32) 69 | padded_map = tf.concat([map_cast, float_map_input], axis=3) 70 | 71 | self.q_network = self.build_model(padded_map, scalars_input, states) 72 | self.target_network = self.build_model(padded_map, scalars_input, states, 'target_') 73 | self.hard_update() 74 | 75 | if self.params.use_global_local: 76 | self.global_map_model = Model(inputs=[boolean_map_input, float_map_input], 77 | outputs=self.global_map) 78 | self.local_map_model = Model(inputs=[boolean_map_input, float_map_input], 79 | outputs=self.local_map) 80 | self.total_map_model = Model(inputs=[boolean_map_input, float_map_input], 81 | outputs=self.total_map) 82 | 83 | q_values = self.q_network.output 84 | q_target_values = self.target_network.output 85 | 86 | # Define Q* in min(Q - (r + gamma_terminated * Q*))^2 87 | max_action = tf.argmax(q_values, axis=1, name='max_action', output_type=tf.int64) 88 | max_action_target = tf.argmax(q_target_values, axis=1, name='max_action', output_type=tf.int64) 89 | one_hot_max_action = tf.one_hot(max_action, depth=self.num_actions, dtype=float) 90 | q_star = tf.reduce_sum(tf.multiply(one_hot_max_action, q_target_values, name='mul_hot_target'), axis=1, 91 | name='q_star') 92 | self.q_star_model = Model(inputs=states, outputs=q_star) 93 | 94 | # Define Bellman loss 95 | one_hot_rm_action = tf.one_hot(action_input, depth=self.num_actions, on_value=1.0, off_value=0.0, dtype=float) 96 | one_cold_rm_action = tf.one_hot(action_input, depth=self.num_actions, on_value=0.0, off_value=1.0, dtype=float) 97 | q_old = tf.stop_gradient(tf.multiply(q_values, one_cold_rm_action)) 98 | gamma_terminated = tf.multiply(tf.cast(tf.math.logical_not(termination_input), tf.float32), gamma) 99 | q_update = tf.expand_dims(tf.add(reward_input, tf.multiply(q_star_input, gamma_terminated)), 1) 100 | q_update_hot = tf.multiply(q_update, one_hot_rm_action) 101 | q_new = tf.add(q_update_hot, q_old) 102 | q_loss = tf.losses.MeanSquaredError()(q_new, q_values) 103 | self.q_loss_model = Model( 104 | inputs=[boolean_map_input, float_map_input, scalars_input, action_input, reward_input, 105 | termination_input, q_star_input], 106 | outputs=q_loss) 107 | 108 | # Exploit act model 109 | self.exploit_model = Model(inputs=states, outputs=max_action) 110 | self.exploit_model_target = Model(inputs=states, outputs=max_action_target) 111 | 112 | # Softmax explore model 113 | softmax_scaling = tf.divide(q_values, tf.constant(self.params.soft_max_scaling, dtype=float)) 114 | softmax_action = tf.math.softmax(softmax_scaling, name='softmax_action') 115 | self.soft_explore_model = Model(inputs=states, outputs=softmax_action) 116 | 117 | self.q_optimizer = tf.optimizers.Adam(learning_rate=params.learning_rate, amsgrad=True) 118 | 119 | if self.params.print_summary: 120 | self.q_loss_model.summary() 121 | 122 | if stats: 123 | stats.set_model(self.target_network) 124 | 125 | def build_model(self, map_proc, states_proc, inputs, name=''): 126 | 127 | flatten_map = self.create_map_proc(map_proc, name) 128 | 129 | layer = Concatenate(name=name + 'concat')([flatten_map, states_proc]) 130 | for k in range(self.params.hidden_layer_num): 131 | layer = Dense(self.params.hidden_layer_size, activation='relu', name=name + 'hidden_layer_all_' + str(k))( 132 | layer) 133 | output = Dense(self.num_actions, activation='linear', name=name + 'output_layer')(layer) 134 | 135 | model = Model(inputs=inputs, outputs=output) 136 | 137 | return model 138 | 139 | def create_map_proc(self, conv_in, name): 140 | 141 | if self.params.use_global_local: 142 | # Forking for global and local map 143 | # Global Map 144 | global_map = tf.stop_gradient( 145 | AvgPool2D((self.params.global_map_scaling, self.params.global_map_scaling))(conv_in)) 146 | 147 | self.global_map = global_map 148 | self.total_map = conv_in 149 | 150 | for k in range(self.params.conv_layers): 151 | global_map = Conv2D(self.params.conv_kernels, self.params.conv_kernel_size, activation='relu', 152 | strides=(1, 1), 153 | name=name + 'global_conv_' + str(k + 1))(global_map) 154 | 155 | flatten_global = Flatten(name=name + 'global_flatten')(global_map) 156 | 157 | # Local Map 158 | crop_frac = float(self.params.local_map_size) / float(self.boolean_map_shape[0]) 159 | local_map = tf.stop_gradient(tf.image.central_crop(conv_in, crop_frac)) 160 | self.local_map = local_map 161 | 162 | for k in range(self.params.conv_layers): 163 | local_map = Conv2D(self.params.conv_kernels, self.params.conv_kernel_size, activation='relu', 164 | strides=(1, 1), 165 | name=name + 'local_conv_' + str(k + 1))(local_map) 166 | 167 | flatten_local = Flatten(name=name + 'local_flatten')(local_map) 168 | 169 | return Concatenate(name=name + 'concat_flatten')([flatten_global, flatten_local]) 170 | else: 171 | conv_map = Conv2D(self.params.conv_kernels, self.params.conv_kernel_size, activation='relu', strides=(1, 1), 172 | name=name + 'map_conv_0')(conv_in) 173 | for k in range(self.params.conv_layers - 1): 174 | conv_map = Conv2D(self.params.conv_kernels, self.params.conv_kernel_size, activation='relu', 175 | strides=(1, 1), 176 | name=name + 'map_conv_' + str(k + 1))(conv_map) 177 | 178 | flatten_map = Flatten(name=name + 'flatten')(conv_map) 179 | return flatten_map 180 | 181 | def act(self, state): 182 | return self.get_soft_max_exploration(state) 183 | 184 | def get_random_action(self): 185 | return np.random.randint(0, self.num_actions) 186 | 187 | def get_exploitation_action(self, state): 188 | 189 | boolean_map_in = state.get_boolean_map()[tf.newaxis, ...] 190 | float_map_in = state.get_float_map()[tf.newaxis, ...] 191 | scalars = np.array(state.get_scalars(), dtype=np.single)[tf.newaxis, ...] 192 | 193 | return self.exploit_model([boolean_map_in, float_map_in, scalars]).numpy()[0] 194 | 195 | def get_soft_max_exploration(self, state): 196 | 197 | boolean_map_in = state.get_boolean_map()[tf.newaxis, ...] 198 | float_map_in = state.get_float_map()[tf.newaxis, ...] 199 | scalars = np.array(state.get_scalars(), dtype=np.single)[tf.newaxis, ...] 200 | p = self.soft_explore_model([boolean_map_in, float_map_in, scalars]).numpy()[0] 201 | return np.random.choice(range(self.num_actions), size=1, p=p) 202 | 203 | def get_exploitation_action_target(self, state): 204 | 205 | boolean_map_in = state.get_boolean_map()[tf.newaxis, ...] 206 | float_map_in = state.get_float_map()[tf.newaxis, ...] 207 | scalars = np.array(state.get_scalars(), dtype=np.single)[tf.newaxis, ...] 208 | 209 | return self.exploit_model_target([boolean_map_in, float_map_in, scalars]).numpy()[0] 210 | 211 | def hard_update(self): 212 | self.target_network.set_weights(self.q_network.get_weights()) 213 | 214 | def soft_update(self, alpha): 215 | weights = self.q_network.get_weights() 216 | target_weights = self.target_network.get_weights() 217 | self.target_network.set_weights( 218 | [w_new * alpha + w_old * (1. - alpha) for w_new, w_old in zip(weights, target_weights)]) 219 | 220 | def train(self, experiences): 221 | boolean_map = experiences[0] 222 | float_map = experiences[1] 223 | scalars = tf.convert_to_tensor(experiences[2], dtype=tf.float32) 224 | action = tf.convert_to_tensor(experiences[3], dtype=tf.int64) 225 | reward = experiences[4] 226 | next_boolean_map = experiences[5] 227 | next_float_map = experiences[6] 228 | next_scalars = tf.convert_to_tensor(experiences[7], dtype=tf.float32) 229 | terminated = experiences[8] 230 | 231 | q_star = self.q_star_model( 232 | [next_boolean_map, next_float_map, next_scalars]) 233 | 234 | # Train Value network 235 | with tf.GradientTape() as tape: 236 | q_loss = self.q_loss_model( 237 | [boolean_map, float_map, scalars, action, reward, 238 | terminated, q_star]) 239 | q_grads = tape.gradient(q_loss, self.q_network.trainable_variables) 240 | self.q_optimizer.apply_gradients(zip(q_grads, self.q_network.trainable_variables)) 241 | 242 | self.soft_update(self.params.alpha) 243 | 244 | def save_weights(self, path_to_weights): 245 | self.target_network.save_weights(path_to_weights) 246 | 247 | def save_model(self, path_to_model): 248 | self.target_network.save(path_to_model) 249 | 250 | def load_weights(self, path_to_weights): 251 | self.q_network.load_weights(path_to_weights) 252 | self.hard_update() 253 | 254 | def get_global_map(self, state): 255 | boolean_map_in = state.get_boolean_map()[tf.newaxis, ...] 256 | float_map_in = state.get_float_map()[tf.newaxis, ...] 257 | return self.global_map_model([boolean_map_in, float_map_in]).numpy() 258 | 259 | def get_local_map(self, state): 260 | boolean_map_in = state.get_boolean_map()[tf.newaxis, ...] 261 | float_map_in = state.get_float_map()[tf.newaxis, ...] 262 | return self.local_map_model([boolean_map_in, float_map_in]).numpy() 263 | 264 | def get_total_map(self, state): 265 | boolean_map_in = state.get_boolean_map()[tf.newaxis, ...] 266 | float_map_in = state.get_float_map()[tf.newaxis, ...] 267 | return self.total_map_model([boolean_map_in, float_map_in]).numpy() 268 | -------------------------------------------------------------------------------- /src/DDQN/ReplayMemory.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def shape(exp): 5 | if type(exp) is np.ndarray: 6 | return list(exp.shape) 7 | else: 8 | return [] 9 | 10 | 11 | def type_of(exp): 12 | if type(exp) is np.ndarray: 13 | return exp.dtype 14 | else: 15 | return type(exp) 16 | 17 | 18 | class ReplayMemory(object): 19 | """ 20 | Replay memory class for RL 21 | """ 22 | 23 | def __init__(self, size): 24 | self.k = 0 25 | self.head = -1 26 | self.full = False 27 | self.size = size 28 | self.memory = None 29 | 30 | def initialize(self, experience): 31 | self.memory = [np.zeros(shape=[self.size] + shape(exp), dtype=type_of(exp)) for exp in experience] 32 | 33 | def store(self, experience): 34 | if self.memory is None: 35 | self.initialize(experience) 36 | if len(experience) != len(self.memory): 37 | raise Exception('Experience not the same size as memory', len(experience), '!=', len(self.memory)) 38 | 39 | for e, mem in zip(experience, self.memory): 40 | mem[self.k] = e 41 | 42 | self.head = self.k 43 | self.k += 1 44 | if self.k >= self.size: 45 | self.k = 0 46 | self.full = True 47 | 48 | def sample(self, batch_size): 49 | r = self.size 50 | if not self.full: 51 | r = self.k 52 | random_idx = np.random.choice(r, size=batch_size, replace=False) 53 | random_idx[0] = self.head # Always add the latest one 54 | 55 | return [mem[random_idx] for mem in self.memory] 56 | 57 | def get(self, start, length): 58 | return [mem[start:start + length] for mem in self.memory] 59 | 60 | def get_size(self): 61 | if self.full: 62 | return self.size 63 | return self.k 64 | 65 | def get_max_size(self): 66 | return self.size 67 | 68 | def reset(self): 69 | self.k = 0 70 | self.head = -1 71 | self.full = False 72 | 73 | def shuffle(self): 74 | self.memory = self.sample(self.get_size()) 75 | -------------------------------------------------------------------------------- /src/DDQN/Trainer.py: -------------------------------------------------------------------------------- 1 | from src.DDQN.Agent import DDQNAgent 2 | from src.DDQN.ReplayMemory import ReplayMemory 3 | import tqdm 4 | 5 | 6 | class DDQNTrainerParams: 7 | def __init__(self): 8 | self.batch_size = 128 9 | self.num_steps = 1e6 10 | self.rm_pre_fill_ratio = 0.5 11 | self.rm_pre_fill_random = True 12 | self.eval_period = 5 13 | self.rm_size = 50000 14 | self.load_model = "" 15 | 16 | 17 | class DDQNTrainer: 18 | def __init__(self, params: DDQNTrainerParams, agent: DDQNAgent): 19 | self.params = params 20 | self.replay_memory = ReplayMemory(size=params.rm_size) 21 | self.agent = agent 22 | 23 | if self.params.load_model != "": 24 | print("Loading model", self.params.load_model, "for agent") 25 | self.agent.load_weights(self.params.load_model) 26 | 27 | self.prefill_bar = None 28 | 29 | def add_experience(self, state, action, reward, next_state): 30 | self.replay_memory.store((state.get_boolean_map(), 31 | state.get_float_map(), 32 | state.get_scalars(), 33 | action, 34 | reward, 35 | next_state.get_boolean_map(), 36 | next_state.get_float_map(), 37 | next_state.get_scalars(), 38 | next_state.terminal)) 39 | 40 | def train_agent(self): 41 | if self.params.batch_size > self.replay_memory.get_size(): 42 | return 43 | mini_batch = self.replay_memory.sample(self.params.batch_size) 44 | 45 | self.agent.train(mini_batch) 46 | 47 | def should_fill_replay_memory(self): 48 | target_size = self.replay_memory.get_max_size() * self.params.rm_pre_fill_ratio 49 | if self.replay_memory.get_size() >= target_size or self.replay_memory.full: 50 | if self.prefill_bar: 51 | self.prefill_bar.update(target_size - self.prefill_bar.n) 52 | self.prefill_bar.close() 53 | return False 54 | 55 | if self.prefill_bar is None: 56 | print("Filling replay memory") 57 | self.prefill_bar = tqdm.tqdm(total=target_size) 58 | 59 | self.prefill_bar.update(self.replay_memory.get_size() - self.prefill_bar.n) 60 | 61 | return True 62 | -------------------------------------------------------------------------------- /src/DDQN/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/DDQN/__init__.py -------------------------------------------------------------------------------- /src/DDQN/__pycache__/Agent.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/DDQN/__pycache__/Agent.cpython-311.pyc -------------------------------------------------------------------------------- /src/DDQN/__pycache__/Agent.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/DDQN/__pycache__/Agent.cpython-38.pyc -------------------------------------------------------------------------------- /src/DDQN/__pycache__/ReplayMemory.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/DDQN/__pycache__/ReplayMemory.cpython-311.pyc -------------------------------------------------------------------------------- /src/DDQN/__pycache__/ReplayMemory.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/DDQN/__pycache__/ReplayMemory.cpython-38.pyc -------------------------------------------------------------------------------- /src/DDQN/__pycache__/Trainer.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/DDQN/__pycache__/Trainer.cpython-311.pyc -------------------------------------------------------------------------------- /src/DDQN/__pycache__/Trainer.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/DDQN/__pycache__/Trainer.cpython-38.pyc -------------------------------------------------------------------------------- /src/DDQN/__pycache__/__init__.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/DDQN/__pycache__/__init__.cpython-311.pyc -------------------------------------------------------------------------------- /src/DDQN/__pycache__/__init__.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/DDQN/__pycache__/__init__.cpython-38.pyc -------------------------------------------------------------------------------- /src/Map/Map.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from skimage import io 3 | 4 | 5 | class Map: 6 | def __init__(self, map_data): 7 | self.start_land_zone = map_data[:, :, 2].astype(bool) 8 | self.nfz = map_data[:, :, 0].astype(bool) 9 | self.obstacles = map_data[:, :, 1].astype(bool) 10 | 11 | def get_starting_vector(self): 12 | similar = np.where(self.start_land_zone) 13 | return list(zip(similar[1], similar[0])) 14 | 15 | def get_free_space_vector(self): 16 | free_space = np.logical_not( 17 | np.logical_or(self.obstacles, self.start_land_zone)) 18 | free_idcs = np.where(free_space) 19 | return list(zip(free_idcs[1], free_idcs[0])) 20 | 21 | def get_size(self): 22 | return self.start_land_zone.shape[:2] 23 | 24 | 25 | def load_image(path): 26 | if type(path) is not str: 27 | raise TypeError('path needs to be a string') 28 | data = io.imread(path, as_gray=True) 29 | return np.array(data, dtype=bool) 30 | 31 | 32 | def save_image(path, image): 33 | if type(path) is not str: 34 | raise TypeError('path needs to be a string') 35 | if image.dtype == bool: 36 | io.imsave(path, image * np.uint8(255)) 37 | else: 38 | io.imsave(path, image) 39 | 40 | 41 | def load_map(path): 42 | if type(path) is not str: 43 | raise TypeError('path needs to be a string') 44 | data = io.imread(path, as_gray=False) 45 | return Map(data) 46 | 47 | 48 | def load_target(path, obstacles=None): 49 | if type(path) is not str: 50 | raise TypeError('path needs to be a string') 51 | 52 | data = np.array(io.imread(path, as_gray=True), dtype=bool) 53 | if obstacles is not None: 54 | data = data & ~obstacles 55 | return data 56 | -------------------------------------------------------------------------------- /src/Map/Shadowing.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | import tqdm 4 | from src.Map.Map import load_map 5 | 6 | 7 | def bresenham(x0, y0, x1, y1, obstacles, shadow_map): 8 | x_dist = abs(x0 - x1) 9 | y_dist = -abs(y0 - y1) 10 | x_step = 1 if x1 > x0 else -1 11 | y_step = 1 if y1 > y0 else -1 12 | 13 | error = x_dist + y_dist 14 | 15 | # shadowed = False 16 | shadow_map[y0, x0] = False 17 | 18 | while x0 != x1 or y0 != y1: 19 | if 2 * error - y_dist > x_dist - 2 * error: 20 | # horizontal step 21 | error += y_dist 22 | x0 += x_step 23 | else: 24 | # vertical step 25 | error += x_dist 26 | y0 += y_step 27 | 28 | if obstacles[y0, x0]: 29 | # shadowed = True 30 | return 31 | 32 | # if shadowed: 33 | shadow_map[y0, x0] = False 34 | 35 | 36 | def calculate_shadowing(map_path, save_as): 37 | print("Calculating shadowing maps") 38 | total_map = load_map(map_path) 39 | obstacles = total_map.obstacles 40 | size = total_map.obstacles.shape[0] 41 | total = size * size 42 | 43 | total_shadow_map = np.ones((size, size, size, size), dtype=bool) 44 | with tqdm.tqdm(total=total) as pbar: 45 | for i, j in np.ndindex(total_map.obstacles.shape): 46 | shadow_map = np.ones((size, size), dtype=bool) 47 | 48 | for x in range(size): 49 | bresenham(i, j, x, 0, obstacles, shadow_map) 50 | bresenham(i, j, x, size - 1, obstacles, shadow_map) 51 | bresenham(i, j, 0, x, obstacles, shadow_map) 52 | bresenham(i, j, size - 1, x, obstacles, shadow_map) 53 | 54 | total_shadow_map[j, i] = shadow_map 55 | pbar.update(1) 56 | 57 | np.save(save_as, total_shadow_map) 58 | return total_shadow_map 59 | 60 | 61 | def load_or_create_shadowing(map_path): 62 | shadow_file_name = os.path.splitext(map_path)[0] + "_shadowing.npy" 63 | if os.path.exists(shadow_file_name): 64 | return np.load(shadow_file_name) 65 | else: 66 | return calculate_shadowing(map_path, shadow_file_name) 67 | -------------------------------------------------------------------------------- /src/Map/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/Map/__init__.py -------------------------------------------------------------------------------- /src/Map/__pycache__/Map.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/Map/__pycache__/Map.cpython-311.pyc -------------------------------------------------------------------------------- /src/Map/__pycache__/Map.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/Map/__pycache__/Map.cpython-38.pyc -------------------------------------------------------------------------------- /src/Map/__pycache__/Shadowing.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/Map/__pycache__/Shadowing.cpython-311.pyc -------------------------------------------------------------------------------- /src/Map/__pycache__/Shadowing.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/Map/__pycache__/Shadowing.cpython-38.pyc -------------------------------------------------------------------------------- /src/Map/__pycache__/__init__.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/Map/__pycache__/__init__.cpython-311.pyc -------------------------------------------------------------------------------- /src/Map/__pycache__/__init__.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/Map/__pycache__/__init__.cpython-38.pyc -------------------------------------------------------------------------------- /src/ModelStats.py: -------------------------------------------------------------------------------- 1 | import collections 2 | import datetime 3 | import os 4 | import shutil 5 | 6 | import tensorflow as tf 7 | import numpy as np 8 | import progressbar 9 | import distutils.util 10 | 11 | 12 | class ModelStatsParams: 13 | def __init__(self, 14 | save_model='models/save_model', 15 | moving_average_length=50): 16 | self.save_model = save_model 17 | self.moving_average_length = moving_average_length 18 | self.log_file_name = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") 19 | self.training_images = False 20 | 21 | 22 | class ModelStats: 23 | 24 | def __init__(self, params: ModelStatsParams, display, force_override=False): 25 | self.params = params 26 | self.display = display 27 | 28 | self.evaluation_value_callback = None 29 | self.env_map_callback = None 30 | self.log_value_callbacks = [] 31 | self.trajectory = [] 32 | 33 | self.log_dir = "logs/training/" + params.log_file_name 34 | self.tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=self.log_dir, 35 | histogram_freq=100) 36 | self.model = None 37 | 38 | if os.path.isdir(self.log_dir): 39 | if force_override: 40 | shutil.rmtree(self.log_dir) 41 | else: 42 | print(self.log_dir, 'already exists.') 43 | resp = input('Override log file? [Y/n]\n') 44 | if resp == '' or distutils.util.strtobool(resp): 45 | print('Deleting old log dir') 46 | shutil.rmtree(self.log_dir) 47 | else: 48 | raise AttributeError('Okay bye') 49 | 50 | self.training_log_writer = tf.summary.create_file_writer(self.log_dir + '/training') 51 | self.testing_log_writer = tf.summary.create_file_writer(self.log_dir + '/test') 52 | 53 | self.evaluation_deque = collections.deque(maxlen=params.moving_average_length) 54 | self.eval_best = -float('inf') 55 | self.bar = None 56 | 57 | def set_evaluation_value_callback(self, callback: callable): 58 | self.evaluation_value_callback = callback 59 | 60 | def add_experience(self, experience): 61 | self.trajectory.append(experience) 62 | 63 | def set_model(self, model): 64 | self.tensorboard_callback.set_model(model) 65 | self.model = model 66 | 67 | def set_env_map_callback(self, callback: callable): 68 | self.env_map_callback = callback 69 | 70 | def add_log_data_callback(self, name: str, callback: callable): 71 | self.log_value_callbacks.append((name, callback)) 72 | 73 | def log_training_data(self, step): 74 | 75 | with self.training_log_writer.as_default(): 76 | self.log_data(step, self.params.training_images) 77 | 78 | def log_testing_data(self, step): 79 | with self.testing_log_writer.as_default(): 80 | self.log_data(step) 81 | 82 | if self.evaluation_value_callback: 83 | self.evaluation_deque.append(self.evaluation_value_callback()) 84 | 85 | def log_data(self, step, images=True): 86 | 87 | for callback in self.log_value_callbacks: 88 | tf.summary.scalar(callback[0], callback[1](), step=step) 89 | if images: 90 | trajectory = self.display.display_episode(self.env_map_callback(), trajectory=self.trajectory) 91 | tf.summary.image('trajectory', trajectory, 92 | step=step) 93 | 94 | def save_if_best(self): 95 | if len(self.evaluation_deque) < self.params.moving_average_length: 96 | return 97 | 98 | eval_mean = np.mean(self.evaluation_deque) 99 | if eval_mean > self.eval_best: 100 | self.eval_best = eval_mean 101 | if self.params.save_model != '': 102 | print('Saving best with:', eval_mean) 103 | self.model.save_weights(self.params.save_model + '_best') 104 | 105 | def get_log_dir(self): 106 | return self.log_dir 107 | 108 | def training_ended(self): 109 | 110 | if self.params.save_model != '': 111 | self.model.save_weights(self.params.save_model + '_unfinished') 112 | print('Model saved as', self.params.save_model + '_unfinished') 113 | 114 | def save_episode(self, save_path): 115 | f = open(save_path + ".txt", "w") 116 | 117 | for callback in self.log_value_callbacks: 118 | f.write(callback[0] + ' ' + str(callback[1]()) + '\n') 119 | f.close() 120 | 121 | def on_episode_begin(self, episode_count): 122 | self.tensorboard_callback.on_epoch_begin(episode_count) 123 | self.trajectory = [] 124 | 125 | def on_episode_end(self, episode_count): 126 | self.tensorboard_callback.on_epoch_end(episode_count) 127 | 128 | -------------------------------------------------------------------------------- /src/StateUtils.py: -------------------------------------------------------------------------------- 1 | import math 2 | import numpy as np 3 | 4 | 5 | def pad_centered(state, map_in, pad_value): 6 | padding_rows = math.ceil(state.no_fly_zone.shape[0] / 2.0) 7 | padding_cols = math.ceil(state.no_fly_zone.shape[1] / 2.0) 8 | position_x, position_y = state.position 9 | position_row_offset = padding_rows - position_y 10 | position_col_offset = padding_cols - position_x 11 | return np.pad(map_in, 12 | pad_width=[[padding_rows + position_row_offset - 1, padding_rows - position_row_offset], 13 | [padding_cols + position_col_offset - 1, padding_cols - position_col_offset], 14 | [0, 0]], 15 | mode='constant', 16 | constant_values=pad_value) 17 | -------------------------------------------------------------------------------- /src/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/__init__.py -------------------------------------------------------------------------------- /src/__pycache__/ModelStats.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/__pycache__/ModelStats.cpython-311.pyc -------------------------------------------------------------------------------- /src/__pycache__/ModelStats.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/__pycache__/ModelStats.cpython-38.pyc -------------------------------------------------------------------------------- /src/__pycache__/StateUtils.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/__pycache__/StateUtils.cpython-311.pyc -------------------------------------------------------------------------------- /src/__pycache__/StateUtils.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/__pycache__/StateUtils.cpython-38.pyc -------------------------------------------------------------------------------- /src/__pycache__/__init__.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/__pycache__/__init__.cpython-311.pyc -------------------------------------------------------------------------------- /src/__pycache__/__init__.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/__pycache__/__init__.cpython-38.pyc -------------------------------------------------------------------------------- /src/base/BaseDisplay.py: -------------------------------------------------------------------------------- 1 | import io 2 | import tensorflow as tf 3 | from tqdm import tqdm 4 | 5 | from src.Map.Map import Map 6 | import numpy as np 7 | import matplotlib.pyplot as plt 8 | from skimage.color import rgb2hsv, hsv2rgb 9 | from matplotlib import patches 10 | import cv2 11 | 12 | from PIL import Image 13 | 14 | 15 | class BaseDisplay: 16 | def __init__(self): 17 | self.arrow_scale = 14 18 | self.marker_size = 15 19 | 20 | def create_grid_image(self, ax, env_map: Map, value_map, green=None): 21 | area_y_max, area_x_max = env_map.get_size() 22 | 23 | if green is None: 24 | green = np.zeros((area_y_max, area_x_max)) 25 | 26 | nfz = np.expand_dims(env_map.nfz, -1) 27 | lz = np.expand_dims(env_map.start_land_zone, -1) 28 | green = np.expand_dims(green, -1) 29 | 30 | neither = np.logical_not(np.logical_or(np.logical_or(nfz, lz), green)) 31 | 32 | base = np.zeros((area_y_max, area_x_max, 3)) 33 | 34 | nfz_color = base.copy() 35 | nfz_color[..., 0] = 0.8 36 | 37 | lz_color = base.copy() 38 | lz_color[..., 2] = 0.8 39 | 40 | green_color = base.copy() 41 | green_color[..., 1] = 0.8 42 | 43 | neither_color = np.ones((area_y_max, area_x_max, 3), dtype=float) 44 | grid_image = green_color * green + nfz_color * nfz + lz_color * lz + neither_color * neither 45 | 46 | hsv_image = rgb2hsv(grid_image) 47 | hsv_image[..., 2] *= value_map.astype('float32') 48 | 49 | grid_image = hsv2rgb(hsv_image) 50 | 51 | if (area_x_max, area_y_max) == (64, 64): 52 | tick_labels_x = np.arange(0, area_x_max, 4) 53 | tick_labels_y = np.arange(0, area_y_max, 4) 54 | self.arrow_scale = 14 55 | self.marker_size = 6 56 | elif (area_x_max, area_y_max) == (32, 32): 57 | tick_labels_x = np.arange(0, area_x_max, 2) 58 | tick_labels_y = np.arange(0, area_y_max, 2) 59 | self.arrow_scale = 8 60 | self.marker_size = 15 61 | elif (area_x_max, area_y_max) == (50, 50): 62 | tick_labels_x = np.arange(0, area_x_max, 4) 63 | tick_labels_y = np.arange(0, area_y_max, 4) 64 | self.arrow_scale = 12 65 | self.marker_size = 8 66 | else: 67 | tick_labels_x = np.arange(0, area_x_max, 1) 68 | tick_labels_y = np.arange(0, area_y_max, 1) 69 | self.arrow_scale = 5 70 | self.marker_size = 15 71 | 72 | plt.sca(ax) 73 | plt.gca().set_aspect('equal', adjustable='box') 74 | plt.xticks(tick_labels_x) 75 | plt.yticks(tick_labels_y) 76 | plt.axis([0, area_x_max, area_y_max, 0]) 77 | ax.imshow(grid_image.astype(float), extent=[0, area_x_max, area_y_max, 0]) 78 | # plt.axis('off') 79 | 80 | obst = env_map.obstacles 81 | for i in range(area_x_max): 82 | for j in range(area_y_max): 83 | if obst[j, i]: 84 | rect = patches.Rectangle((i, j), 1, 1, fill=None, hatch='////', edgecolor="Black") 85 | ax.add_patch(rect) 86 | 87 | # offset to shift tick labels 88 | locs, labels = plt.xticks() 89 | locs_new = [x + 0.5 for x in locs] 90 | plt.xticks(locs_new, tick_labels_x) 91 | 92 | locs, labels = plt.yticks() 93 | locs_new = [x + 0.5 for x in locs] 94 | plt.yticks(locs_new, tick_labels_y) 95 | 96 | def draw_start_and_end(self, trajectory): 97 | first_state = trajectory[0][0] 98 | final_state = trajectory[-1][3] 99 | 100 | plt.scatter(first_state.position[0] + 0.5, first_state.position[1] + 0.5, s=self.marker_size, marker="D", 101 | color="w") 102 | 103 | if final_state.landed: 104 | plt.scatter(final_state.position[0] + 0.5, final_state.position[1] + 0.5, 105 | s=self.marker_size, marker="D", color="green") 106 | else: 107 | plt.scatter(final_state.position[0] + 0.5, final_state.position[1] + 0.5, 108 | s=self.marker_size, marker="D", color="r") 109 | 110 | def draw_movement(self, from_position, to_position, color): 111 | y = from_position[1] 112 | x = from_position[0] 113 | dir_y = to_position[1] - y 114 | dir_x = to_position[0] - x 115 | if dir_x == 0 and dir_y == 0: 116 | plt.scatter(x + 0.5, y + 0.5, marker="X", color=color) 117 | else: 118 | if abs(dir_x) >= 1 or abs(dir_y) >= 1: 119 | plt.quiver(x + 0.5, y + 0.5, dir_x, -dir_y, color=color, 120 | scale=self.arrow_scale, scale_units='inches') 121 | else: 122 | plt.quiver(x + 0.5, y + 0.5, dir_x, -dir_y, color=color, 123 | scale=self.arrow_scale, scale_units='inches') 124 | 125 | def create_tf_image(self): 126 | buf = io.BytesIO() 127 | plt.savefig(buf, format='png', dpi=180, bbox_inches='tight') 128 | buf.seek(0) 129 | plt.close('all') 130 | combined_image = tf.image.decode_png(buf.getvalue(), channels=3) 131 | return tf.expand_dims(combined_image, 0) 132 | 133 | def create_video(self, map_image, trajectory, save_path, frame_rate, draw_path=True): 134 | fourcc = cv2.VideoWriter_fourcc(*'mp4v') 135 | video = None 136 | print("Creating video, if it freezes tap ctrl...") 137 | for k in tqdm(range(len(trajectory))): 138 | if draw_path: 139 | frame = np.squeeze(self.display_episode(map_image, trajectory[:k + 1]).numpy()) 140 | else: 141 | frame = np.squeeze(self.display_state(map_image, trajectory[0][0], trajectory[k][0]).numpy()) 142 | r = frame[..., 0] 143 | g = frame[..., 1] 144 | b = frame[..., 2] 145 | img = np.stack([b, g, r], axis=2) 146 | if not video: 147 | height, width, channels = img.shape 148 | print("Creating video writer for {}, with the shape {}".format(save_path, (height, width))) 149 | video = cv2.VideoWriter(save_path, fourcc, frame_rate, (width, height)) 150 | video.write(img) 151 | 152 | if not draw_path: 153 | frame = np.squeeze(self.display_episode(map_image, trajectory).numpy()) 154 | r = frame[..., 0] 155 | g = frame[..., 1] 156 | b = frame[..., 2] 157 | img = np.stack([b, g, r], axis=2) 158 | video.write(img) 159 | 160 | video.release() 161 | 162 | def display_episode(self, map_image, trajectory, plot=False, save_path=None) -> tf.Tensor: 163 | pass 164 | 165 | def display_state(self, env_map, initial_state, state, plot=False) -> tf.Tensor: 166 | pass 167 | -------------------------------------------------------------------------------- /src/base/BaseGrid.py: -------------------------------------------------------------------------------- 1 | from src.ModelStats import ModelStats 2 | import src.Map.Map as Map 3 | 4 | 5 | class BaseGridParams: 6 | def __init__(self): 7 | self.movement_range = (100, 200) 8 | self.map_path = 'res/downtown.png' 9 | 10 | 11 | class BaseGrid: 12 | def __init__(self, params: BaseGridParams, stats: ModelStats): 13 | self.map_image = Map.load_map(params.map_path) 14 | self.shape = self.map_image.start_land_zone.shape 15 | self.starting_vector = self.map_image.get_starting_vector() 16 | stats.set_env_map_callback(self.get_map_image) 17 | 18 | def get_map_image(self): 19 | return self.map_image 20 | 21 | def get_grid_size(self): 22 | return self.shape 23 | 24 | def get_no_fly(self): 25 | return self.map_image.nfz 26 | 27 | def get_landing_zone(self): 28 | return self.map_image.start_land_zone 29 | -------------------------------------------------------------------------------- /src/base/BaseState.py: -------------------------------------------------------------------------------- 1 | from src.Map.Map import Map 2 | 3 | 4 | class BaseState: 5 | def __init__(self, map_init: Map): 6 | self.no_fly_zone = map_init.nfz 7 | self.obstacles = map_init.obstacles 8 | self.landing_zone = map_init.start_land_zone 9 | 10 | @property 11 | def shape(self): 12 | return self.landing_zone.shape[:2] 13 | -------------------------------------------------------------------------------- /src/base/Environment.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import tqdm 3 | import distutils.util 4 | 5 | from src.ModelStats import ModelStatsParams, ModelStats 6 | from src.base.BaseDisplay import BaseDisplay 7 | from src.base.GridActions import GridActions 8 | 9 | 10 | class BaseEnvironmentParams: 11 | def __init__(self): 12 | self.model_stats_params = ModelStatsParams() 13 | 14 | 15 | class BaseEnvironment: 16 | def __init__(self, params: BaseEnvironmentParams, display: BaseDisplay): 17 | self.stats = ModelStats(params.model_stats_params, display=display) 18 | self.trainer = None 19 | self.agent = None 20 | self.grid = None 21 | self.rewards = None 22 | self.physics = None 23 | self.display = display 24 | self.episode_count = 0 25 | self.step_count = 0 26 | 27 | def fill_replay_memory(self): 28 | 29 | while self.trainer.should_fill_replay_memory(): 30 | 31 | state = copy.deepcopy(self.init_episode()) 32 | while not state.terminal: 33 | next_state = self.step(state, random=self.trainer.params.rm_pre_fill_random) 34 | state = copy.deepcopy(next_state) 35 | 36 | def train_episode(self): 37 | state = copy.deepcopy(self.init_episode()) 38 | self.stats.on_episode_begin(self.episode_count) 39 | while not state.is_terminal(): 40 | state = self.step(state) 41 | self.trainer.train_agent() 42 | 43 | self.stats.on_episode_end(self.episode_count) 44 | self.stats.log_training_data(step=self.step_count) 45 | 46 | self.episode_count += 1 47 | 48 | def run(self): 49 | 50 | self.fill_replay_memory() 51 | 52 | print('Running ', self.stats.params.log_file_name) 53 | 54 | bar = tqdm.tqdm(total=int(self.trainer.params.num_steps)) 55 | last_step = 0 56 | while self.step_count < self.trainer.params.num_steps: 57 | bar.update(self.step_count - last_step) 58 | last_step = self.step_count 59 | self.train_episode() 60 | 61 | if self.episode_count % self.trainer.params.eval_period == 0: 62 | self.test_episode() 63 | 64 | self.stats.save_if_best() 65 | 66 | self.stats.training_ended() 67 | 68 | def step(self, state, random=False): 69 | if random: 70 | action = self.agent.get_random_action() 71 | else: 72 | action = self.agent.act(state) 73 | next_state = self.physics.step(GridActions(action)) 74 | reward = self.rewards.calculate_reward(state, GridActions(action), next_state) 75 | self.trainer.add_experience(state, action, reward, next_state) 76 | self.stats.add_experience((state, action, reward, copy.deepcopy(next_state))) 77 | self.step_count += 1 78 | return copy.deepcopy(next_state) 79 | 80 | def test_episode(self, scenario=None): 81 | state = copy.deepcopy(self.init_episode(scenario)) 82 | self.stats.on_episode_begin(self.episode_count) 83 | while not state.terminal: 84 | action = self.agent.get_exploitation_action_target(state) 85 | next_state = self.physics.step(GridActions(action)) 86 | reward = self.rewards.calculate_reward(state, GridActions(action), next_state) 87 | self.stats.add_experience((copy.deepcopy(state), action, reward, copy.deepcopy(next_state))) 88 | state = copy.deepcopy(next_state) 89 | 90 | self.stats.on_episode_end(self.episode_count) 91 | self.stats.log_testing_data(step=self.step_count) 92 | 93 | def test_scenario(self, scenario): 94 | state = copy.deepcopy(self.init_episode(scenario)) 95 | while not state.terminal: 96 | action = self.agent.get_exploitation_action_target(state) 97 | state = self.physics.step(GridActions(action)) 98 | 99 | def init_episode(self, init_state=None): 100 | if init_state: 101 | state = copy.deepcopy(self.grid.init_scenario(init_state)) 102 | else: 103 | state = copy.deepcopy(self.grid.init_episode()) 104 | 105 | self.rewards.reset() 106 | self.physics.reset(state) 107 | return state 108 | 109 | def eval(self, episodes, show=False): 110 | for _ in tqdm.tqdm(range(episodes)): 111 | self.test_episode() 112 | self.step_count += 1 # Increase step count so that logging works properly 113 | 114 | if show: 115 | self.display.display_episode(self.grid.map_image, self.stats.trajectory, plot=True) 116 | 117 | resp = input('Save run? [y/N]\n') 118 | try: 119 | if distutils.util.strtobool(resp): 120 | save_as = input('Save as: [run_' + str(self.step_count) + ']\n') 121 | if save_as == '': 122 | save_as = 'run_' + str(self.step_count) 123 | self.display.display_episode(self.grid.map_image, self.stats.trajectory, plot=False, 124 | save_path=save_as + '.png') 125 | self.stats.save_episode(save_as) 126 | print("Saved as run_" + str(self.step_count)) 127 | except ValueError: 128 | pass 129 | print("next then") 130 | 131 | def eval_scenario(self, init_state): 132 | self.test_scenario(init_state) 133 | 134 | self.display.display_episode(self.grid.map_image, self.stats.trajectory, plot=True) 135 | 136 | resp = input('Save run? [y/N]\n') 137 | try: 138 | if distutils.util.strtobool(resp): 139 | save_as = input('Save as: [scenario]\n') 140 | if save_as == '': 141 | save_as = 'scenario' 142 | self.display.display_episode(self.grid.map_image, self.stats.trajectory, plot=False, 143 | save_path=save_as + '.png') 144 | self.stats.save_episode(save_as) 145 | print("Saved as", save_as) 146 | except ValueError: 147 | pass 148 | -------------------------------------------------------------------------------- /src/base/GridActions.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | 3 | 4 | class GridActions(Enum): 5 | NORTH = 0 6 | EAST = 1 7 | SOUTH = 2 8 | WEST = 3 9 | LAND = 4 10 | HOVER = 5 11 | 12 | 13 | class GridActionsNoHover(Enum): 14 | NORTH = 0 15 | EAST = 1 16 | SOUTH = 2 17 | WEST = 3 18 | LAND = 4 19 | -------------------------------------------------------------------------------- /src/base/GridPhysics.py: -------------------------------------------------------------------------------- 1 | from src.base.GridActions import GridActions 2 | 3 | 4 | class GridPhysics: 5 | def __init__(self): 6 | self.landing_attempts = 0 7 | self.boundary_counter = 0 8 | self.state = None 9 | 10 | def movement_step(self, action: GridActions): 11 | old_position = self.state.position 12 | x, y = old_position 13 | 14 | if action == GridActions.NORTH: 15 | y += 1 16 | elif action == GridActions.SOUTH: 17 | y -= 1 18 | elif action == GridActions.WEST: 19 | x -= 1 20 | elif action == GridActions.EAST: 21 | x += 1 22 | elif action == GridActions.LAND: 23 | self.landing_attempts += 1 24 | if self.state.is_in_landing_zone(): 25 | self.state.set_landed(True) 26 | 27 | self.state.set_position([x, y]) 28 | if self.state.is_in_no_fly_zone(): 29 | # Reset state 30 | self.boundary_counter += 1 31 | x, y = old_position 32 | self.state.set_position([x, y]) 33 | 34 | self.state.decrement_movement_budget() 35 | self.state.set_terminal(self.state.landed or (self.state.movement_budget == 0)) 36 | 37 | return x, y 38 | 39 | def reset(self, state): 40 | self.landing_attempts = 0 41 | self.boundary_counter = 0 42 | self.state = state 43 | -------------------------------------------------------------------------------- /src/base/GridRewards.py: -------------------------------------------------------------------------------- 1 | from src.base.GridActions import GridActions 2 | 3 | 4 | class GridRewardParams: 5 | def __init__(self): 6 | self.boundary_penalty = 1.0 7 | self.empty_battery_penalty = 150.0 8 | self.movement_penalty = 0.2 9 | 10 | 11 | class GridRewards: 12 | def __init__(self, stats): 13 | self.params = GridRewardParams() 14 | self.cumulative_reward: float = 0.0 15 | 16 | stats.add_log_data_callback('cumulative_reward', self.get_cumulative_reward) 17 | 18 | def get_cumulative_reward(self): 19 | return self.cumulative_reward 20 | 21 | def calculate_motion_rewards(self, state, action: GridActions, next_state): 22 | reward = 0.0 23 | if not next_state.landed: 24 | # Penalize battery Consumption 25 | reward -= self.params.movement_penalty 26 | 27 | # Penalize not moving (This happens when it either tries to land or fly into a boundary or hovers or fly into 28 | # a cell occupied by another agent) 29 | if state.position == next_state.position and not next_state.landed and not action == GridActions.HOVER: 30 | reward -= self.params.boundary_penalty 31 | 32 | # Penalize battery dead 33 | if next_state.movement_budget == 0 and not next_state.landed: 34 | reward -= self.params.empty_battery_penalty 35 | 36 | return reward 37 | 38 | def reset(self): 39 | self.cumulative_reward = 0 40 | -------------------------------------------------------------------------------- /src/base/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/base/__init__.py -------------------------------------------------------------------------------- /src/base/__pycache__/BaseDisplay.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/base/__pycache__/BaseDisplay.cpython-311.pyc -------------------------------------------------------------------------------- /src/base/__pycache__/BaseDisplay.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/base/__pycache__/BaseDisplay.cpython-38.pyc -------------------------------------------------------------------------------- /src/base/__pycache__/BaseGrid.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/base/__pycache__/BaseGrid.cpython-311.pyc -------------------------------------------------------------------------------- /src/base/__pycache__/BaseGrid.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/base/__pycache__/BaseGrid.cpython-38.pyc -------------------------------------------------------------------------------- /src/base/__pycache__/BaseState.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/base/__pycache__/BaseState.cpython-311.pyc -------------------------------------------------------------------------------- /src/base/__pycache__/BaseState.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/base/__pycache__/BaseState.cpython-38.pyc -------------------------------------------------------------------------------- /src/base/__pycache__/Environment.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/base/__pycache__/Environment.cpython-311.pyc -------------------------------------------------------------------------------- /src/base/__pycache__/Environment.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/base/__pycache__/Environment.cpython-38.pyc -------------------------------------------------------------------------------- /src/base/__pycache__/GridActions.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/base/__pycache__/GridActions.cpython-311.pyc -------------------------------------------------------------------------------- /src/base/__pycache__/GridActions.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/base/__pycache__/GridActions.cpython-38.pyc -------------------------------------------------------------------------------- /src/base/__pycache__/GridPhysics.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/base/__pycache__/GridPhysics.cpython-311.pyc -------------------------------------------------------------------------------- /src/base/__pycache__/GridPhysics.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/base/__pycache__/GridPhysics.cpython-38.pyc -------------------------------------------------------------------------------- /src/base/__pycache__/GridRewards.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/base/__pycache__/GridRewards.cpython-311.pyc -------------------------------------------------------------------------------- /src/base/__pycache__/GridRewards.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/base/__pycache__/GridRewards.cpython-38.pyc -------------------------------------------------------------------------------- /src/base/__pycache__/__init__.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/base/__pycache__/__init__.cpython-311.pyc -------------------------------------------------------------------------------- /src/base/__pycache__/__init__.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kashifmug/UAV_PathPlanning_using_DDQN/003ee444b5733d389bdfd4ed74a49bf56c4f26fd/src/base/__pycache__/__init__.cpython-38.pyc -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- 1 | import distutils 2 | import json 3 | 4 | from types import SimpleNamespace as Namespace 5 | 6 | 7 | def getattr_recursive(obj, s): 8 | if isinstance(s, list): 9 | split = s 10 | else: 11 | split = s.split('/') 12 | return getattr_recursive(getattr(obj, split[0]), split[1:]) if len(split) > 1 else getattr(obj, split[0]) 13 | 14 | 15 | def setattr_recursive(obj, s, val): 16 | if isinstance(s, list): 17 | split = s 18 | else: 19 | split = s.split('/') 20 | return setattr_recursive(getattr(obj, split[0]), split[1:], val) if len(split) > 1 else setattr(obj, split[0], val) 21 | 22 | 23 | def generate_config(params, file_path): 24 | print("Saving Configs") 25 | f = open(file_path, "w") 26 | json_data = json.dumps(params.__dict__, default=lambda o: o.__dict__, indent=4) 27 | f.write(json_data) 28 | f.close() 29 | 30 | 31 | def read_config(config_path): 32 | print('Parse Params file here from ', config_path, ' and pass into main') 33 | json_data = open(config_path, "r").read() 34 | return json.loads(json_data, object_hook=lambda d: Namespace(**d)) 35 | 36 | 37 | def override_params(params, overrides): 38 | assert (len(overrides) % 2 == 0) 39 | for k in range(0, len(overrides), 2): 40 | oldval = getattr_recursive(params, overrides[k]) 41 | if type(oldval) == bool: 42 | to_val = bool(distutils.util.strtobool(overrides[k + 1])) 43 | else: 44 | to_val = type(oldval)(overrides[k + 1]) 45 | setattr_recursive(params, overrides[k], 46 | to_val) 47 | print("Overriding param", overrides[k], "from", oldval, "to", to_val) 48 | 49 | return params 50 | 51 | 52 | def get_bool_user(message, default: bool): 53 | if default: 54 | default_string = '[Y/n]' 55 | else: 56 | default_string = '[y/N]' 57 | resp = input('{} {}\n'.format(message, default_string)) 58 | try: 59 | if distutils.util.strtobool(resp): 60 | return True 61 | else: 62 | return False 63 | except ValueError: 64 | return default 65 | --------------------------------------------------------------------------------