├── src ├── examples │ ├── more_advanced_example.py │ └── follow_center_line.py └── deep_racer_framework.py ├── LICENSE ├── notes └── sample_params.txt └── README.md /src/examples/more_advanced_example.py: -------------------------------------------------------------------------------- 1 | from src.deep_racer_framework import Framework 2 | 3 | 4 | def get_reward(framework: Framework): 5 | speed_factor = framework.progress_speed 6 | if abs(framework.skew) > 40: 7 | speed_factor = speed_factor / 2 8 | elif abs(framework.skew) < 20 and abs(framework.slide) < 15: 9 | speed_factor += 1 10 | speed_factor = min(5.0, speed_factor) 11 | 12 | return min(1000.0, pow(framework.distance_from_extreme_edge * 10, speed_factor)) -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 dmh23 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /src/examples/follow_center_line.py: -------------------------------------------------------------------------------- 1 | # Example of rewarding the agent to follow center line 2 | # Taken from AWS DeepRacer documentation 3 | 4 | from src.deep_racer_framework import Framework 5 | 6 | 7 | # ----------------------------------------------------------------------------- 8 | # Firstly, here is the *ORIGINAL* version of the reward function written by AWS 9 | # ----------------------------------------------------------------------------- 10 | 11 | def reward_function(params): 12 | # Read input parameters 13 | track_width = params['track_width'] 14 | distance_from_center = params['distance_from_center'] 15 | 16 | # Calculate 3 markers that are at varying distances away from the center line 17 | marker_1 = 0.1 * track_width 18 | marker_2 = 0.25 * track_width 19 | marker_3 = 0.5 * track_width 20 | 21 | # Give higher reward if the car is closer to center line and vice versa 22 | if distance_from_center <= marker_1: 23 | reward = 1.0 24 | elif distance_from_center <= marker_2: 25 | reward = 0.5 26 | elif distance_from_center <= marker_3: 27 | reward = 0.1 28 | else: 29 | reward = 1e-3 # likely crashed/ close to off track 30 | 31 | return float(reward) 32 | 33 | 34 | # ----------------------------------------------------------------------------- 35 | # And now, here it is using the DRF 36 | # 37 | # Changes compared to the AWS original are marked with an asterisk (*) 38 | # 39 | # This is a trivial example but of course the framework also contains many attributes not available directly from AWS. 40 | # 41 | # And even this simple example is easier to type from scratch because most IDEs and editors can check the references 42 | # to framework attributes, whereas references to AWS "params" are only checked at run-time 43 | # ----------------------------------------------------------------------------- 44 | 45 | def get_reward(framework: Framework): # (*) different function name and signature 46 | 47 | # Read input parameters 48 | track_width = framework.track_width # (*) get value from framework 49 | distance_from_center = framework.distance_from_center # (*) get value from framework 50 | 51 | # Calculate 3 markers that are at varying distances away from the center line 52 | marker_1 = 0.1 * track_width 53 | marker_2 = 0.25 * track_width 54 | marker_3 = 0.5 * track_width 55 | 56 | # Give higher reward if the car is closer to center line and vice versa 57 | if distance_from_center <= marker_1: 58 | reward = 1.0 59 | elif distance_from_center <= marker_2: 60 | reward = 0.5 61 | elif distance_from_center <= marker_3: 62 | reward = 0.1 63 | else: 64 | reward = 1e-3 # likely crashed/ close to off track 65 | 66 | return reward # (*) No need to convert to a float 67 | -------------------------------------------------------------------------------- /notes/sample_params.txt: -------------------------------------------------------------------------------- 1 | 2 | all_wheels_on_track 3 | closest_waypoints 4 | distance_from_center 5 | is_crashed 6 | is_left_of_center 7 | is_offtrack 8 | is_reversed 9 | heading 10 | progress 11 | projection_distance 12 | speed 13 | steering_angle 14 | steps 15 | track_length 16 | track_width 17 | waypoints 18 | x 19 | y 20 | 21 | 22 | 23 | 24 | EXTRACTED ON 9th FEB 2021 (re-ordered to match AWS documentation below) 25 | ========================= 26 | 27 | { 28 | 'all_wheels_on_track': False, 29 | 'x': -6.175130999630964, 30 | 'y': -3.89226815099667, 31 | 'closest_objects': [0, 0], 32 | 'closest_waypoints': [16, 17] 33 | 'distance_from_center': 0.6613717351801577, 34 | 'is_crashed': False, 35 | 'is_left_of_center': True, 36 | 'is_offtrack': False, 37 | 'is_reversed': False, 38 | 'heading': 31.08439540895211, 39 | 'objects_distance': [], 40 | 'objects_heading': [], 41 | 'objects_left_of_center': [], 42 | 'objects_location': [], 43 | 'objects_speed': [], 44 | 'progress': 2.891042542136657, 45 | 'speed': 1.5, 46 | 'steering_angle': -10.0, 47 | 'steps': 17.0, 48 | 'track_length': 59.35085588912497, 49 | 'track_width': 1.3945737857649045, 50 | 'waypoints': [(-8.721938610076904, -1.364508032798767), (-8.586678504943848, -1.6246190071105957), (-8.451693534851074, -1.8848654627799988), (-8.318925142288208, -2.1461880207061768), (-8.1847825050354, -2.4068459272384644), (-8.048007249832153, -2.6661975383758545), (-7.9164206981658936, -2.9280951023101807), (-7.791316747665405, -3.1929264068603516), (-7.668456792831421, -3.458854556083679), (-7.5386834144592285, -3.721458077430725), (-7.391506671905518, -3.9753745794296265), (-7.207223415374756, -4.204581141471863), (-6.981844186782837, -4.39617645740509), (-6.716232538223267, -4.52928352355957), (-6.422172546386719, -4.586187958717346), (-6.183715581893921, -4.58315634727478), (-5.982163906097412, -4.536198973655701), (-5.7925519943237305, -4.439626455307007), (-5.554929971694946, -4.239877462387085), (-5.37417459487915, -3.98416006565094), (-5.247910022735596, -3.6927220821380615), (-5.159286022186279, -3.3820430040359497), (-5.076839447021484, -3.059462070465088), (-4.986419439315796, -2.806283950805664), (-4.87703800201416, -2.58838152885437), (-4.769734621047974, -2.443451464176178), (-4.641028165817261, -2.3174564838409424), (-4.485800504684448, -2.218119502067566), (-4.321366548538208, -2.139563500881195), (-4.154363393783569, -2.0763309597969055), (-3.8522030115127563, -1.9732429385185242), (-3.5456424951553345, -1.8543224930763245), (-3.254565954208374, -1.6941035389900208), (-2.9652040004730225, -1.469386637210846), (-2.736297607421875, -1.2384593784809113), (-2.4944114685058594, -1.0013168901205063), (-2.3684579730033875, -0.896468922495842), (-2.2316759824752808, -0.807245284318924), (-2.076523005962372, -0.7339699976146221), (-1.9103770852088928, -0.687925923615694), (-1.7363024950027466, -0.6639129660907201), (-1.5622275471687317, -0.6399000100791454), (-1.227791965007782, -0.5939073115587234), (-0.9169197082519531, -0.5201173275709152), (-0.618903785943985, -0.40787631273269653), (-0.325309157371521, -0.2720489352941513), (-0.026595767587423325, -0.145352303981781), (0.12446743249893188, -0.1055918037891388), (0.2797711044549942, -0.08667925000190735), (0.430124893784523, -0.08720162510871887), (0.5766128450632095, -0.11898225545883179), (0.7237322926521301, -0.17410439252853394), (0.8603569269180298, -0.2510853409767151), (1.1265714466571808, -0.46551504731178284), (1.382514476776123, -0.7256386950612068), (1.6460735201835632, -0.9860905706882477), (1.9098804593086243, -1.18031245470047), (2.187691569328308, -1.2930217683315277), (2.4780325889587402, -1.3329910933971405), (2.773878574371338, -1.332958608865738), (3.068732976913452, -1.3365693092346191), (3.348672032356262, -1.3795991241931915), (3.618680000305176, -1.4772025346755981), (3.8765169382095337, -1.6283919215202332), (4.124377965927124, -1.811739981174469), (4.374072074890137, -1.9945585131645203), (4.635909080505371, -2.1500189900398254), (4.911490440368652, -2.2667890191078186), (5.198182106018066, -2.341328978538513), (5.491522312164307, -2.37667053937912), (5.786585569381714, -2.374807059764862), (6.0858659744262695, -2.3319250345230103), (6.374570608139038, -2.253949522972107), (6.652871131896973, -2.1459155678749084), (6.923996925354004, -2.021554470062256), (7.192794561386108, -1.8925054669380188), (7.462010622024536, -1.7642699480056763), (7.734374523162845, -1.6425002217292772), (8.009332418441769, -1.5263626575469984), (8.284817457199097, -1.4113675355911255), (8.560301303863525, -1.2964639961719513), (8.835959434509277, -1.1817838251590729), (9.112760543823242, -1.0700470209121704), (9.378759860992432, -0.9362864643335342), (9.608596801757812, -0.750214695930481), (9.783488750457764, -0.5144959986209869), (9.891698360443115, -0.2435300052165985), (9.923367977142334, 0.045416850596666336), (9.87643814086914, 0.33251510187983513), (9.757167339324951, 0.5990577563643456), (9.578029155731201, 0.8321249037981033), (9.352334976196289, 1.0231253653764725), (9.096649169921875, 1.1750081777572632), (8.826563358306885, 1.3012796640396118), (8.55599594116211, 1.4269342124462128), (8.276055812835693, 1.5306991338729858), (7.990830183029175, 1.6205081343650818), (7.705917596817017, 1.71117103099823), (7.420769214630127, 1.8011590242385864), (7.135752439498901, 1.8915260434150696), (6.850665092468262, 1.9816880226135254), (6.5656163692474365, 2.0719590187072754), (6.280541658401489, 2.162172496318817), (5.99548602104187, 2.252415955066681), (5.710418462753296, 2.3426429629325867), (5.4253575801849365, 2.43287992477417), (5.14029598236084, 2.523111045360565), (4.8552281856536865, 2.613345444202423), (4.570167541503906, 2.7035789489746094), (4.285102605819702, 2.7938109636306763), (4.000042915344238, 2.884047508239746), (3.7149770259857178, 2.974273920059204), (3.429919481277466, 3.0645190477371216), (3.144846558570862, 3.1547319889068604), (2.8598004579544067, 3.245004415512085), (2.5747119188308716, 3.3351694345474243), (2.2896900177001953, 3.425526976585388), (2.004556953907013, 3.515535593032837), (1.7160229682922363, 3.6160316467285156), (1.4499804973602295, 3.7311683893203735), (1.186974048614502, 3.8420413732528687), (1.0048611760139465, 3.8890329599380493), (0.8436891138553619, 3.91313099861145), (0.687198668718338, 3.910848021507263), (0.6028326451778412, 3.885154962539673), (0.5239479541778564, 3.843032121658325), (0.4347173422574997, 3.786695122718811), (0.31835207086987793, 3.6744788885116577), (0.21723096072673798, 3.55594539642334), (0.12277120351791382, 3.408610463142395), (0.04058440029621124, 3.258126974105835), (-0.14629779756069183, 2.8814334869384766), (-0.33939504995942116, 2.4862420558929443), (-0.5591442435979843, 2.1619624495506287), (-0.6870620101690292, 2.0482369661331177), (-0.8333538770675659, 1.9644349217414856), (-1.027192622423172, 1.9029214978218079), (-1.2333354949951172, 1.8794960379600525), (-1.4284414649009705, 1.9028074741363525), (-1.6448314785957336, 1.9815595149993896), (-1.8656255006790161, 2.1473699808120728), (-2.0365350246429443, 2.3898534774780273), (-2.167572498321533, 2.732468008995056), (-2.22360897064209, 2.988304018974304), (-2.2928425073623657, 3.2837610244750977), (-2.400964081287384, 3.573250889778137), (-2.5971245765686035, 3.884732961654663), (-2.694763422012329, 3.9881839752197266), (-2.8116849660873413, 4.069127082824707), (-3.052322030067444, 4.161665439605713), (-3.3104424476623535, 4.193146467208862), (-3.563464045524597, 4.164375066757202), (-3.7344651222229004, 4.101981997489929), (-3.8817275762557983, 3.9929449558258057), (-3.994374632835388, 3.848942518234253), (-4.070441961288452, 3.681480884552002), (-4.142942547798157, 3.49771249294281), (-4.198321461677551, 3.1183059215545654), (-4.206498503684998, 2.760501980781555), (-4.202826499938965, 2.4237594604492188), (-4.2204166650772095, 2.0746145248413086), (-4.272469520568848, 1.737552523612976), (-4.340022563934326, 1.467886596918106), (-4.400842070579529, 1.347464919090271), (-4.4879302978515625, 1.2229550778865814), (-4.57945990562439, 1.1363904178142548), (-4.68793249130249, 1.0631908774375916), (-4.814671516418457, 1.000778004527092), (-4.950603008270264, 0.9535165876150131), (-5.2082929611206055, 0.9288836121559143), (-5.478792428970337, 0.918044775724411), (-5.7284770011901855, 0.9260439723730087), (-5.974202394485474, 0.9705317169427872), (-6.144105911254883, 1.0276133120059967), (-6.29846453666687, 1.1150914132595062), (-6.395999193191528, 1.2059332132339478), (-6.482736110687256, 1.3090826272964478), (-6.5752599239349365, 1.4237359762191772), (-6.715615510940552, 1.6999114751815796), (-6.843124628067017, 2.061498463153839), (-6.933829307556152, 2.409706950187683), (-6.995930433273315, 2.677325963973999), (-7.064676523208618, 2.9729509353637695), (-7.143395185470581, 3.2737594842910767), (-7.236890554428101, 3.574544906616211), (-7.3488609790802, 3.8572219610214233), (-7.500182867050171, 4.119042873382568), (-7.644118785858154, 4.275362849235535), (-7.802390813827515, 4.381262540817261), (-7.979835510253906, 4.454203009605408), (-8.139649629592896, 4.493025422096252), (-8.139649629592896, 4.493025422096252), (-8.302613735198975, 4.519863486289978), (-8.537531852722168, 4.551554560661316), (-8.797042846679688, 4.582455515861511), (-9.075617790222168, 4.616629600524902), (-9.374495506286621, 4.6415855884552), (-9.669283390045166, 4.667464017868042), (-9.968292713165283, 4.684208035469055), (-10.25966501235962, 4.669860482215881), (-10.557689666748047, 4.586061954498291), (-10.806339740753174, 4.425511479377747), (-10.990524768829346, 4.196977376937866), (-11.096714973449707, 3.9255080223083496), (-11.124760150909424, 3.637893557548523), (-11.083415031433105, 3.3479504585266113), (-10.987834930419922, 3.0720374584198), (-10.855430126190186, 2.810674548149109), (-10.707069873809814, 2.55742347240448), (-10.560279369354248, 2.303284525871277), (-10.4181809425354, 2.046660542488098), (-10.283358097076416, 1.7863335013389587), (-10.152853965759277, 1.523941993713379), (-10.023746967315674, 1.260889321565628), (-9.895512104034424, 0.9974376261234283), (-9.768439292907715, 0.7334512770175934), (-9.640498161315918, 0.46986236423254013), (-9.511675357818604, 0.20667827129363933), (-9.382283210754395, -0.056237801909447514), (-9.252198219299316, -0.31883347081020474), (-9.12059736251831, -0.5807124674320221), (-8.9877290725708, -0.8419884741306305), (-8.855119228363037, -1.1033860743045807), (-8.721938610076904, -1.364508032798767)], 51 | 52 | # These are not documented! 53 | 54 | 'object_in_camera': False, 55 | 'projection_distance': 4.683401287333071, 56 | 'objects_distance_from_center': [], 57 | } 58 | 59 | Documentation is here: 60 | https://docs.aws.amazon.com/deepracer/latest/developerguide/deepracer-reward-function-input.html 61 | 62 | { 63 | "all_wheels_on_track": Boolean, # flag to indicate if the agent is on the track 64 | "x": float, # agent's x-coordinate in meters 65 | "y": float, # agent's y-coordinate in meters 66 | "closest_objects": [int, int], # zero-based indices of the two closest objects to the agent's current position of (x, y). 67 | "closest_waypoints": [int, int], # indices of the two nearest waypoints. 68 | "distance_from_center": float, # distance in meters from the track center 69 | "is_crashed": Boolean, # Boolean flag to indicate whether the agent has crashed. 70 | "is_left_of_center": Boolean, # Flag to indicate if the agent is on the left side to the track center or not. 71 | "is_offtrack": Boolean, # Boolean flag to indicate whether the agent has gone off track. 72 | "is_reversed": Boolean, # flag to indicate if the agent is driving clockwise (True) or counter clockwise (False). 73 | "heading": float, # agent's yaw in degrees 74 | "objects_distance": [float, ], # list of the objects' distances in meters between 0 and track_length in relation to the starting line. 75 | "objects_heading": [float, ], # list of the objects' headings in degrees between -180 and 180. 76 | "objects_left_of_center": [Boolean, ], # list of Boolean flags indicating whether elements' objects are left of the center (True) or not (False). 77 | "objects_location": [(float, float),], # list of object locations [(x,y), ...]. 78 | "objects_speed": [float, ], # list of the objects' speeds in meters per second. 79 | "progress": float, # percentage of track completed 80 | "speed": float, # agent's speed in meters per second (m/s) 81 | "steering_angle": float, # agent's steering angle in degrees 82 | "steps": int, # number steps completed 83 | "track_length": float, # track length in meters. 84 | "track_width": float, # width of the track 85 | "waypoints": [(float, float), ] # list of (x,y) as milestones along the track center 86 | } 87 | 88 | 89 | 90 | 91 | 92 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # deep_racer_framework v1.2.0 2 | 3 | ## Introduction 4 | 5 | To use this framework, simply: 6 | - Copy the **entire file** src/deep_racer_framework.py 7 | - Scroll down to the end and write your own version of the get_reward() function 8 | - Use one or more of the attributes from the "framework" input parameter to calculate a reward greater than zero 9 | 10 | Notice how most IDEs, and even the editor provided in the AWS DeepRacer console itself, suggest names as you start to type attributes. This helps you write your reward function quickly and accurately. 11 | 12 | For a few simple ideas of what's possible in a reward function, see the "src/examples" directory. 13 | 14 | ## Parameters - Summary 15 | 16 | | Name | Datatype | Range | Accuracy | Units | AWS Param | 17 | | :---- | :-------- | :----- | :-------- | :----- | :--------- | 18 | | x | float | Any | Exact | Meters | x | 19 | | y | float | Any | Exact | Meters | y | 20 | | all_wheels_on_track | bool | True or False | Exact | | all_wheels_on_track | 21 | | is_left_of_center | bool | True or False | Exact | | is_left_of_center | 22 | | is_right_of_center | bool | True or False | Exact | 23 | | distance_from_center | float | \>= 0.0 | Exact | Meters | distance_from_center | 24 | | distance_from_edge | float | \>= 0.0 | Exact | Meters | 25 | | distance_from_extreme_edge | float | \>= 0.0 | Approximate | Meters | 26 | | waypoints | List | | Exact | | waypoints | 27 | | closest_waypoint_id | int | \>= 0 | Exact | List index | 28 | | closest_waypoint_x | float | Any | Exact | Meters | | 29 | | closest_waypoint_y | float | Any | Exact | Meters | | 30 | | previous_waypoint_id | int | \>=0 | Exact | List index | closest_waypoints[0] | 31 | | previous_waypoint_x | float | Any | Exact | Meters | | 32 | | previous_waypoint_y | float | Any | Exact | Meters | | 33 | | next_waypoint_id | int | \>= 0 | Exact | List index | closest_waypoints[1] | 34 | | next_waypoint_x | float | Any | Exact | Meters | | 35 | | next_waypoint_y | float | Any | Exact | Meters | | 36 | | distance_from_closest_waypoint | float | \>= 0.0 | Exact | Meters | 37 | | just_passed_waypoint_ids | List | | Exact | List index | | 38 | | time_at_waypoint | List | | Approximate | Seconds | | 39 | | steps | int | \>= 1 | Exact | Steps | steps | 40 | | progress | float | 0.0 to 100.0 | Exact | Percent | progress | 41 | | time | float | \>= 0.0 | Approximate | Seconds | 42 | | predicted_lap_time | float | \>= 0.0 | Approximate | Seconds | 43 | | total_distance | float | \>= 0.0 | Approximate | Meters | 44 | | is_final_step | bool | True or False | Exact | 45 | | is_crashed | bool | True or False | Exact | | is_crashed | 46 | | is_off_track | bool | True or False | Exact | | is_offtrack | 47 | | is_reversed | bool | True or False | Exact | | is_reversed | 48 | | is_complete_lap | bool | True or False | Exact | 49 | | track_speed | float | \>= 0.0 | Approximate | Meters per Second | 50 | | max_possible_track_speed | float | \> 0.0 | Approximate | Meters per Second | 51 | | progress_speed | float | \>= 0.0 | Approximate | Meters per Second | 52 | | action_speed | float | \> 0.0 | Exact | Meters per Second | speed | 53 | | action_steering_angle | float | -30.0 to 30.0 | Exact | Degrees | steering_angle | 54 | | is_steering_left | bool | True or False | Exact | 55 | | is_steering_right | bool | True or False | Exact | 56 | | is_steering_straight | bool | True or False | Exact | 57 | | action_sequence_length | int | \>= 1 | Exact | Steps | 58 | | heading | float | -180.0 to 180.0 | Exact | Degrees | heading | 59 | | track_bearing | float | -180.0 to 180.0 | Exact | Degrees | 60 | | true_bearing | float | -180.0 to 180.0 | Approximate | Degrees | 61 | | corner_cutting | float | \>= 0.0 | Exact | 62 | | projected_distance | float | \>= 0.0 | Approximate | Meters | 63 | | projected_progress_distance | float | \>= 0.0 | Approximate | Meters | 64 | | projected_finish_left | bool | True or False | Approximate | 65 | | slide | float | -180.0 to 180.0 | Approximate | Degrees | 66 | | skew | float | -180.0 to 180.0 | Approximate | Degrees | 67 | | max_slide | float | -180.0 to 180.0 | Approximate | Degrees | 68 | | max_skew | float | -180.0 to 180.0 | Approximate | Degrees | 69 | | track_length | float | \>= 0.0 | Exact | Meters | track_length | 70 | | track_width | float | \>= 0.0 | Exact | Meters | track_width | 71 | | has_objects | bool | True or False | Exact | 72 | | objects_location | TODO | | | | objects_location | 73 | | front_object_id | TODO | 74 | | distance_to_front_object | TODO | 75 | | front_object_is_left_of_centre | TODO | 76 | | rear_object_id | TODO | 77 | | distance_to_rear_object | TODO | 78 | | rear_object_is_left_of_centre | TODO | 79 | | step_when_passed_object | TODO | 80 | | projected_hit_object | TODO | 81 | 82 | ## Parameters - Explained 83 | 84 | #### Car Location 85 | - **x** - The x co-ordinate of the car's location 86 | - **y** - The y co-ordinate of the car's location 87 | - **all_wheels_on_track** - Value of _True_ means all four wheels are on the track 88 | - **is_left_of_center** - Value of _True_ means the centre of the car is left of the center line 89 | - **is_right_of_center** - Value of _True_ means the centre of the car is right of the center line 90 | - **distance_from_center** - The distance of the center of the car from the center line of the track 91 | - **distance_from_edge** - The distance of the center of the car from the edge of the track (zero means the center of the car is on or beyond the edge) 92 | - **distance_from_extreme_edge** - The _approximate_ distance of the car from actually going off track (zero means the car is either off track, or dangerously close) 93 | 94 | Note: Regarding the edge of the track, remember that the car is still on the track so long as at least one wheel is on the track, and furthermore the car itself is quite wide relative to the track ... so when **distance_from_edge** is zero, in reality the car is still fairly safely on the track ... this is why **distance_from_extreme_edge** is also provided as a better estimate of how far the car really is from being judged off track. 95 | 96 | 97 | #### Waypoints 98 | - **waypoints** - List of all track waypoints, same as AWS DeepRacer parameter 99 | - **closest_waypoint_x** - The x co-ordinate of the closest waypoint to the car 100 | - **closest_waypoint_y** - The y co-ordinate of the closest waypoint to the car 101 | - **closest_waypoint_id** - The index number of the closest waypoint to the car (index indicates position in the **waypoints** list) 102 | - **previous_waypoint_x** / **y** / **id** - Similarly, the x, y and index of the waypoint immediately behind the car 103 | - **next_waypoint_x** / **y** / **id** - Similarly, the x, y and index of the waypoint immediately in front of the car 104 | - **distance_from_closest_waypoint** - The distance of the car from the closest waypoint 105 | - **just_passed_waypoint_ids** - A list of waypoint index(es) just passed (each waypoint is only passed at most once per episode) 106 | - **time_at_waypoint** - A list of the **time** when each waypoint was passed in this episode - or _None_ if the waypoint is not yet passed - the list index is the waypoint id 107 | 108 | #### Progress Indications 109 | - **steps** - Number of steps completed so far in this episode, including this step (so it's basically the step number) 110 | - **progress** - Progress towards a complete lap as a percentage in the range 0 to 100 111 | - **time** - The _approximate_ number of seconds that the car has taken so far 112 | - **predicted_lap_time** - Estimate for complete lap time based on progress so far (e.g. if car has covered half the track in 6 secs, then the predicted lap time is 12 seconds, and so on) 113 | - **total_distance** - Total distance travelled so far in this episode 114 | - **is_final_step** - Value of _true_ means the car has ended this episode, this is the last call to the reward function, so this is your chance to give a special reward for a fast lap, or to penalise not finishing successfully (see next section, "Episode Final Status", to discover why the episode is ending, good or bad!) 115 | 116 | #### Episode Final Status 117 | - **is_crashed** - Value of _true_ means the car has crashed into an object 118 | - **is_off_track** - Value of _true_ means the car has gone off track, i.e. none of its wheels are on the track 119 | - **is_reversed** - Value of _true_ means the car is going the wrong way round the track i.e. it probably spun and ended up pointing the wrong way! 120 | - **is_complete_lap** - Value of _true_ means the car has finished a lap 121 | 122 | Note: All values of these are _false_ until the very last step, when these are set to indicate the reason for reaching the end of the episode 123 | 124 | #### Actual Speed 125 | - **track_speed** - The speed the car is currently actually travelling at 126 | - **max_possible_track_speed** - An estimate of the maximum possible **track_speed** for the number of steps completed so far (it takes approximately 25 steps to reach a potential top speed of 4 m/s) 127 | - **progress_speed** - The speed of the car relative to the center line; if the car is travelling along the centre line, then it will be the same as the **track_speed**; if it is cutting a corner, the **progress_speed** will be higher; or if it is going sideways or taking the outside of a corner, then the **progress_speed** will be lower 128 | 129 | Note: These are real measures of the car's speed, unlike the **action_speed**, see below 130 | Note: More precisely, the **progress_speed** is the speed the car is effectively completing the track as measured along the waypoints / center line (it is called "progress" speed since this is the method for measuring **progress**, see above) 131 | 132 | #### Action Chosen 133 | - **action_speed** - The speed of the action chosen from the action space 134 | - **action_steering_angle** - The steering angle of the action chosen from the action space 135 | - **is_steering_left** - Value of _true_ means the chosen action is steering left 136 | - **is_steering_right** - Value of _true_ means the chosen action is steering right 137 | - **is_steering_straight** - Value of _true_ means the chosen action is steering straight ahead 138 | - **action_sequence_length** - Counts number of consecutive times the same action has been chosen, including this step (hence always >= 1) 139 | 140 | Note: A sequence length of 1 means the chosen action is different from the last step; a value >= 2 indicates the same action has been chosen again 141 | 142 | #### Direction of Travel etc. 143 | - **heading** - The heading of the car in degrees, which means this is where the car is "pointing" (also think of this as being the direction the camera is "looking") 144 | - **track_bearing** - The bearing of the track in degrees, based on the waypoints / center line 145 | - **true_bearing** - The actual bearing the car is travelling along, which might differ from the **heading** especially on bends, or if the car is out of control (spinning etc.) 146 | - **corner_cutting** - TODO 147 | 148 | #### Projections 149 | - **projected_distance** - The remaining distance the car will travel before coming off the track or hitting an object if it continues at the current **true_bearing** 150 | - **projected_progress_distance** - The remaining distance the car will travel relative to the centre line before coming off the track or hitting an object if it continues at the current **true_bearing** 151 | - **projected_finish_left** - Value of _true_ means the car will come off track or hit an object on the left-hand side if it continues at the current **true_bearing** 152 | - **projected_hit_object** - See "Object Avoidance", below 153 | 154 | Note: The **projected_progress_distance** is a similar concept to the **progress_speed** as described above, i.e. it measures the likely "progress" of the car along the waypoints 155 | 156 | #### Indications of Sliding/Skidding etc. 157 | - **slide** - The difference in degrees between **heading** and **true_bearing**, you decide what is reasonable but typically somewhere between 10 and 20 degrees difference marks the change from controlled behaviour to sliding/skidding/spinning (i.e. uncontrolled, unless you want to encourage rally turns round a tight corner!) 158 | - **skew** - The difference in degrees between **track_bearing** and **true_bearing**, so a value close to zero indicates the car is following the track center line (which might be good for straight sections), whereas higher values indicate driving across the track (which might be good for cutting corners) 159 | - **max_slide** - The greatest value of **slide** during this episode, provided so you can reward or penalise behaviour such as skidding for the remainder of the episode (e.g. you might reward the car for recovering and continuing, or you might continue to penalise the rest of the episode to prevent loss of control in future) 160 | - **max_skew** - Similarly, the greatest value of **skew** during this episode (probably less useful than **max_slide**?!) 161 | 162 | #### Track Characteristics 163 | - **track_length** - Total length of the track (measured along the waypoints / center line) 164 | - **track_width** - Width of the track 165 | 166 | #### Object Avoidance 167 | - **has_objects** - Value of _true_ means there are objects to be avoided 168 | - **objects_location** - An array of the location of every object, same as the AWS DeepRacer parameter 169 | - **front_object_id** - The index number of the closest object in front of the car (use this index to access the **objects_location**) 170 | - **distance_to_front_object** - The distance to the closest object in front of the car 171 | - **front_object_is_left_of_centre** - Value of _true_ means the closest object in front of the car is on the left side 172 | - **rear_object_id** - The index number of the closest object behind the car (use this index to access the **objects_location** or **step_when_passed_object**) 173 | - **distance_to_rear_object** - The distance to the closest object behind the car 174 | - **rear_object_is_left_of_centre** - Value of _true_ means the closest object behind the car is on the left side 175 | - **step_when_passed_object** - An array indicating which at which step each object was passed (a value of _None_ means the object is not yet passed) 176 | - **projected_hit_object** - Value of _true_ means the car is on a direct course to crash into an object (based on the **true_bearing**) 177 | 178 | 179 | -------------------------------------------------------------------------------- /src/deep_racer_framework.py: -------------------------------------------------------------------------------- 1 | # 2 | # DeepRacer Framework 3 | # 4 | # Version 1.2.0 5 | # 6 | # Copyright (c) 2021 dmh23 7 | # 8 | 9 | import math 10 | 11 | 12 | # ------------------------------------------------------------------------------- 13 | # 14 | # CONSTANTS 15 | # 16 | # ------------------------------------------------------------------------------- 17 | 18 | class ParamNames: 19 | ALL_WHEELS_ON_TRACK = "all_wheels_on_track" 20 | CLOSEST_WAYPOINTS = "closest_waypoints" 21 | DISTANCE_FROM_CENTER = "distance_from_center" 22 | IS_CRASHED = "is_crashed" 23 | IS_LEFT_OF_CENTER = "is_left_of_center" 24 | IS_OFFTRACK = "is_offtrack" 25 | IS_REVERSED = "is_reversed" 26 | HEADING = "heading" 27 | PROGRESS = "progress" 28 | PROJECTION_DISTANCE = "projection_distance" 29 | SPEED = "speed" 30 | STEERING_ANGLE = "steering_angle" 31 | STEPS = "steps" 32 | TRACK_LENGTH = "track_length" 33 | TRACK_WIDTH = "track_width" 34 | WAYPOINTS = "waypoints" 35 | X = "x" 36 | Y = "y" 37 | CLOSEST_OBJECTS = "closest_objects" 38 | OBJECTS_DISTANCE = "objects_distance" 39 | OBJECTS_DISTANCE_FROM_CENTER = "objects_distance_from_center" 40 | OBJECTS_HEADING = "objects_heading" 41 | OBJECTS_LEFT_OF_CENTER = "objects_left_of_center" 42 | OBJECTS_LOCATION = "objects_location" 43 | OBJECTS_SPEED = "objects_speed" 44 | OBJECT_IN_CAMERA = "object_in_camera" 45 | 46 | 47 | class RealWorld: 48 | STEPS_PER_SECOND = 15 49 | 50 | VEHICLE_LENGTH = 0.365 51 | VEHICLE_WIDTH = 0.225 52 | 53 | BOX_OBSTACLE_WIDTH = 0.38 54 | BOX_OBSTACLE_LENGTH = 0.24 55 | 56 | MAX_SPEEDS = [None, 0.01, 0.02, 0.04, 0.1, 0.15, 0.25, 0.4, 0.6, 0.9, 1.1, 1.3, 1.5, 1.7, 2.0, 2.2, 57 | 2.3, 2.6, 2.7, 3.1, 3.3, 3.4, 3.6, 3.8, 4.0] 58 | 59 | SAFE_CAR_OVERHANG = min(VEHICLE_LENGTH, VEHICLE_WIDTH) / 2 60 | 61 | 62 | # ------------------------------------------------------------------------------- 63 | # 64 | # GEOMETRY 65 | # 66 | # ------------------------------------------------------------------------------- 67 | 68 | def get_distance_between_points(first, second): 69 | (x1, y1) = first 70 | (x2, y2) = second 71 | 72 | x_diff = x2 - x1 73 | y_diff = y2 - y1 74 | 75 | return math.sqrt(x_diff * x_diff + y_diff * y_diff) 76 | 77 | 78 | def get_bearing_between_points(start, finish): 79 | (start_x, start_y) = start 80 | (finish_x, finish_y) = finish 81 | 82 | direction_in_radians = math.atan2(finish_y - start_y, finish_x - start_x) 83 | return math.degrees(direction_in_radians) 84 | 85 | 86 | def get_angle_in_proper_range(angle): 87 | if angle >= 180: 88 | return angle - 360 89 | elif angle <= -180: 90 | return 360 + angle 91 | else: 92 | return angle 93 | 94 | 95 | def get_turn_between_directions(current, required): 96 | difference = required - current 97 | return get_angle_in_proper_range(difference) 98 | 99 | 100 | def is_point_between(point, start, finish): 101 | bearing_from_start = get_bearing_between_points(start, point) 102 | bearing_to_finish = get_bearing_between_points(point, finish) 103 | return abs(get_turn_between_directions(bearing_from_start, bearing_to_finish)) < 1 104 | 105 | 106 | def get_point_at_bearing(start_point, bearing: float, distance: float): 107 | (x, y) = start_point 108 | 109 | radians_to_target = math.radians(bearing) 110 | 111 | x2 = x + math.cos(radians_to_target) * distance 112 | y2 = y + math.sin(radians_to_target) * distance 113 | 114 | return x2, y2 115 | 116 | 117 | # Intersection of two lines comes from Wikipedia 118 | # https://en.wikipedia.org/wiki/Line%E2%80%93line_intersection#Given_two_points_on_each_line 119 | 120 | def get_intersection_of_two_lines(line_a_point_1, line_a_point_2, 121 | line_b_point_1, line_b_point_2): 122 | (x1, y1) = line_a_point_1 123 | (x2, y2) = line_a_point_2 124 | (x3, y3) = line_b_point_1 125 | (x4, y4) = line_b_point_2 126 | 127 | denominator = ((x1 - x2) * (y3 - y4)) - ((y1 - y2) * (x3 - x4)) 128 | 129 | if denominator == 0.0: 130 | return None 131 | 132 | z1 = (x1 * y2) - (y1 * x2) 133 | z2 = (x3 * y4) - (y3 * x4) 134 | 135 | x = ((z1 * (x3 - x4)) - ((x1 - x2) * z2)) / denominator 136 | y = ((z1 * (y3 - y4)) - ((y1 - y2) * z2)) / denominator 137 | 138 | return x, y 139 | 140 | 141 | # ------------------------------------------------------------------------------- 142 | # 143 | # WAYPOINT INFO CACHE 144 | # 145 | # ------------------------------------------------------------------------------- 146 | 147 | def get_edge_point(previous, mid, future, direction_offset: int, distance: float): 148 | assert direction_offset in [90, -90] 149 | assert previous != mid 150 | 151 | (previous_x, previous_y) = previous 152 | (mid_x, mid_y) = mid 153 | (next_x, next_y) = future 154 | 155 | degrees_to_mid_point = math.degrees(math.atan2(mid_y - previous_y, mid_x - previous_x)) 156 | if mid == future: 157 | track_heading_degrees = degrees_to_mid_point 158 | else: 159 | degrees_from_mid_point = math.degrees(math.atan2(next_y - mid_y, next_x - mid_x)) 160 | degrees_difference = get_turn_between_directions(degrees_to_mid_point, degrees_from_mid_point) 161 | track_heading_degrees = degrees_to_mid_point + degrees_difference / 2 162 | 163 | radians_to_edge_point = math.radians(track_heading_degrees + direction_offset) 164 | 165 | x = mid_x + math.cos(radians_to_edge_point) * distance 166 | y = mid_y + math.sin(radians_to_edge_point) * distance 167 | 168 | return x, y 169 | 170 | 171 | class ProcessedWaypoint: 172 | def __init__(self, point, left_safe, right_safe): 173 | (self.x, self.y) = point 174 | self.left_safe = left_safe 175 | self.right_safe = right_safe 176 | 177 | 178 | def get_processed_waypoints(waypoints, track_width): 179 | if waypoints[0] == waypoints[-1]: 180 | previous = waypoints[-2] 181 | else: 182 | previous = waypoints[-1] 183 | 184 | left_safe = previous 185 | right_safe = previous 186 | 187 | edge_error_tolerance = 0.01 188 | 189 | processed_waypoints = [] 190 | 191 | for i, w in enumerate(waypoints): 192 | # Tracks often contain a repeated waypoint, suspect this is deliberate to mess up waypoint algorithms! 193 | if previous != w: 194 | if i < len(waypoints) - 1: 195 | future = waypoints[i + 1] 196 | else: 197 | future = waypoints[0] 198 | 199 | previous_left = left_safe 200 | previous_right = right_safe 201 | 202 | left_safe = get_edge_point(previous, w, future, 90, track_width / 2 + RealWorld.SAFE_CAR_OVERHANG) 203 | if get_distance_between_points(previous_left, left_safe) < edge_error_tolerance: 204 | left_safe = previous_left 205 | 206 | right_safe = get_edge_point(previous, w, future, -90, track_width / 2 + RealWorld.SAFE_CAR_OVERHANG) 207 | if get_distance_between_points(previous_right, right_safe) < edge_error_tolerance: 208 | right_safe = previous_right 209 | 210 | previous = w 211 | 212 | processed_waypoints.append(ProcessedWaypoint(w, left_safe, right_safe)) 213 | 214 | return processed_waypoints 215 | 216 | 217 | # ------------------------------------------------------------------------------- 218 | # 219 | # REMEMBER A PREVIOUS STEP IN THIS EPISODE 220 | # 221 | # ------------------------------------------------------------------------------- 222 | 223 | class HistoricStep: 224 | def __init__(self, framework, previous_step): 225 | self.x = framework.x 226 | self.y = framework.y 227 | self.progress = framework.progress 228 | self.action_speed = framework.action_speed 229 | self.action_steering_angle = framework.action_steering_angle 230 | self.closest_waypoint_id = framework.closest_waypoint_id 231 | self.next_waypoint_id = framework.next_waypoint_id 232 | self.slide = framework.slide 233 | 234 | if previous_step: 235 | self.distance = get_distance_between_points((previous_step.x, previous_step.y), (self.x, self.y)) 236 | else: 237 | self.distance = 0.0 # Causes issues if we use: framework.progress / 100 * framework.track_length 238 | 239 | 240 | # ------------------------------------------------------------------------------- 241 | # 242 | # FRAMEWORK 243 | # 244 | # ------------------------------------------------------------------------------- 245 | 246 | class Framework: 247 | def __init__(self, params): 248 | # Real PRIVATE variables set here 249 | self._processed_waypoints = get_processed_waypoints(params[ParamNames.WAYPOINTS], 250 | params[ParamNames.TRACK_WIDTH]) 251 | self._history = [] 252 | self._previous_front_object = -1 253 | 254 | # Definitions only of variables to use in your reward method, real values are set during process_params() 255 | self.x = 0.0 256 | self.y = 0.0 257 | self.start_waypoint_id = 0 258 | self.all_wheels_on_track = True 259 | self.previous_waypoint_id = 0 260 | self.previous_waypoint_x = 0.0 261 | self.previous_waypoint_y = 0.0 262 | self.next_waypoint_id = 0 263 | self.next_waypoint_x = 0.0 264 | self.next_waypoint_y = 0.0 265 | self.closest_waypoint_id = 0 266 | self.closest_waypoint_x = 0.0 267 | self.closest_waypoint_y = 0.0 268 | self.distance_from_closest_waypoint = 0.0 269 | self.distance_from_center = 0.0 270 | self.distance_from_edge = 0.0 271 | self.distance_from_extreme_edge = 0.0 272 | self.is_left_of_center = False 273 | self.is_right_of_center = False 274 | self.is_crashed = False 275 | self.is_off_track = False 276 | self.is_reversed = False 277 | self.is_complete_lap = False 278 | self.steps = 0 279 | self.time = 0.0 280 | self.is_final_step = False 281 | self.progress = 0.0 282 | self.predicted_lap_time = 0.0 283 | self.waypoints = [] 284 | self.track_length = 0.0 285 | self.track_width = 0.0 286 | self.track_speed = 0.0 287 | self.progress_speed = 0.0 288 | # self.progress_speeds = [] 289 | self.action_speed = 0.0 290 | self.action_steering_angle = 0.0 291 | self.action_sequence_length = 0 292 | self.is_steering_left = False 293 | self.is_steering_right = False 294 | self.is_steering_straight = False 295 | self.heading = 0.0 296 | self.track_bearing = 0.0 297 | self.true_bearing = 0.0 298 | self.slide = 0.0 299 | self.skew = 0.0 300 | self.max_slide = 0.0 301 | self.recent_max_slide = 0.0 302 | self.max_skew = 0.0 303 | self.total_distance = 0.0 304 | self.objects_location = [] 305 | self.just_passed_waypoint_ids = [] 306 | self.time_at_waypoint = [] 307 | self.projected_distance = 0.0 308 | self.projected_progress_distance = 0.0 309 | self.projected_finish_left = False 310 | self.max_possible_track_speed = 0.0 311 | self.corner_cutting = 0.0 312 | 313 | # New stuff for OA ################################ 314 | self.has_objects = False 315 | self.step_when_passed_object = [-1] * 20 316 | self.front_object_id = None 317 | self.rear_object_id = None 318 | self.distance_to_front_object = None 319 | self.distance_to_rear_object = None 320 | self.front_object_is_left_of_centre = False 321 | self.rear_object_is_left_of_centre = False 322 | self.projected_hit_object = False 323 | 324 | def process_params(self, params): 325 | self.x = float(params[ParamNames.X]) 326 | self.y = float(params[ParamNames.Y]) 327 | 328 | self.all_wheels_on_track = bool(params[ParamNames.ALL_WHEELS_ON_TRACK]) 329 | 330 | self.previous_waypoint_id = int(params[ParamNames.CLOSEST_WAYPOINTS][0]) 331 | self.previous_waypoint_x, self.previous_waypoint_y = params[ParamNames.WAYPOINTS][self.previous_waypoint_id] 332 | self.next_waypoint_id = int(params[ParamNames.CLOSEST_WAYPOINTS][1]) 333 | self.next_waypoint_x, self.next_waypoint_y = params[ParamNames.WAYPOINTS][self.next_waypoint_id] 334 | 335 | distance_to_previous_waypoint = get_distance_between_points((self.x, self.y), params[ParamNames.WAYPOINTS][ 336 | self.previous_waypoint_id]) 337 | distance_to_next_waypoint = get_distance_between_points( 338 | (self.x, self.y), params[ParamNames.WAYPOINTS][self.next_waypoint_id]) 339 | if distance_to_previous_waypoint < distance_to_next_waypoint: 340 | self.closest_waypoint_id = self.previous_waypoint_id 341 | self.closest_waypoint_x = self.previous_waypoint_x 342 | self.closest_waypoint_y = self.previous_waypoint_y 343 | self.distance_from_closest_waypoint = distance_to_previous_waypoint 344 | else: 345 | self.closest_waypoint_id = self.next_waypoint_id 346 | self.closest_waypoint_x = self.next_waypoint_x 347 | self.closest_waypoint_y = self.next_waypoint_y 348 | self.distance_from_closest_waypoint = distance_to_next_waypoint 349 | 350 | self.distance_from_center = float(params[ParamNames.DISTANCE_FROM_CENTER]) 351 | self.distance_from_edge = float(max(0.0, params[ParamNames.TRACK_WIDTH] / 2 - self.distance_from_center)) 352 | self.distance_from_extreme_edge = \ 353 | float(max(0.0, (params[ParamNames.TRACK_WIDTH] + RealWorld.VEHICLE_WIDTH) / 2 - self.distance_from_center)) 354 | 355 | self.is_left_of_center = bool(params[ParamNames.IS_LEFT_OF_CENTER]) 356 | self.is_right_of_center = not self.is_left_of_center 357 | 358 | self.is_crashed = bool(params[ParamNames.IS_CRASHED]) 359 | self.is_off_track = bool(params[ParamNames.IS_OFFTRACK]) 360 | self.is_reversed = bool(params[ParamNames.IS_REVERSED]) 361 | 362 | self.steps = int(round(params[ParamNames.STEPS])) 363 | self.time = self.steps / RealWorld.STEPS_PER_SECOND 364 | self.progress = float(params[ParamNames.PROGRESS]) 365 | self.is_complete_lap = self.progress == 100.0 366 | self.is_final_step = self.is_complete_lap or self.is_crashed or self.is_off_track or self.is_reversed 367 | if self.progress > 0: 368 | self.predicted_lap_time = round(100 / self.progress * self.steps / RealWorld.STEPS_PER_SECOND, 2) 369 | else: 370 | self.predicted_lap_time = 0.0 371 | 372 | self.waypoints = params[ParamNames.WAYPOINTS] 373 | self.track_length = params[ParamNames.TRACK_LENGTH] 374 | self.track_width = params[ParamNames.TRACK_WIDTH] 375 | 376 | self.action_speed = params[ParamNames.SPEED] 377 | self.action_steering_angle = params[ParamNames.STEERING_ANGLE] 378 | 379 | self.is_steering_straight = abs(self.action_steering_angle) < 0.01 380 | self.is_steering_left = self.action_steering_angle > 0 and not self.is_steering_straight 381 | self.is_steering_right = self.action_steering_angle < 0 and not self.is_steering_straight 382 | 383 | self.heading = params[ParamNames.HEADING] 384 | self.track_bearing = get_bearing_between_points( 385 | (self.previous_waypoint_x, self.previous_waypoint_y), 386 | (self.next_waypoint_x, self.next_waypoint_y)) 387 | 388 | self.max_possible_track_speed = RealWorld.MAX_SPEEDS[min(self.steps, len(RealWorld.MAX_SPEEDS) - 1)] 389 | 390 | self.objects_location = params[ParamNames.OBJECTS_LOCATION] 391 | 392 | # 393 | # Print object info for use by DRG 394 | # 395 | 396 | # Not step 1 because there's still a bug (?!) that means the reward function is not called until step 2!!! 397 | if self.steps == 2 and len(self.objects_location) > 0: 398 | print("DRG-OBJECTS:", self.objects_location) 399 | 400 | # 401 | # Record history 402 | # 403 | 404 | if self.steps <= 2: 405 | self._history = [] 406 | self.time_at_waypoint = [None] * len(self.waypoints) 407 | self.step_when_passed_object = [-1] * 20 408 | self._previous_front_object = -1 409 | 410 | if self._history: 411 | previous_step = self._history[-1] 412 | else: 413 | previous_step = None 414 | 415 | this_step = HistoricStep(self, previous_step) 416 | self._history.append(this_step) 417 | 418 | # 419 | # Calculations that use the history 420 | # 421 | 422 | if previous_step: 423 | if previous_step.x != self.x or previous_step.y != self.y: # Otherwise keep existing true_bearing 424 | if self.progress - previous_step.progress >= 0.05: 425 | self.true_bearing = get_bearing_between_points((previous_step.x, previous_step.y), (self.x, self.y)) 426 | if (previous_step.action_speed == self.action_speed and 427 | previous_step.action_steering_angle == self.action_steering_angle): 428 | self.action_sequence_length += 1 429 | else: 430 | self.action_sequence_length = 1 431 | 432 | speed_calculate_steps = self._history[-6:] 433 | speed_calculate_distance = sum(s.distance for s in speed_calculate_steps) 434 | speed_calculate_time = len(speed_calculate_steps) / RealWorld.STEPS_PER_SECOND 435 | self.track_speed = speed_calculate_distance / speed_calculate_time 436 | 437 | progress_speed_distance = (self.progress - speed_calculate_steps[0].progress) / 100 * self.track_length 438 | progress_speed_calculate_time = (len(speed_calculate_steps) - 1) / RealWorld.STEPS_PER_SECOND 439 | self.progress_speed = max(0.0, progress_speed_distance / progress_speed_calculate_time) 440 | 441 | self.just_passed_waypoint_ids = self._get_just_passed_waypoint_ids( 442 | previous_step.next_waypoint_id, self.next_waypoint_id) 443 | 444 | progress_gain = self.progress - previous_step.progress 445 | if progress_gain < 0: 446 | self.corner_cutting = 0 447 | elif this_step.distance == 0: 448 | self.corner_cutting = 1 449 | else: 450 | progress_distance = progress_gain / 100 * self.track_length 451 | self.corner_cutting = progress_distance / this_step.distance 452 | 453 | self.recent_max_slide = 0.0 454 | for h in self._history[-4:]: 455 | if abs(h.slide) > abs(self.recent_max_slide): 456 | self.recent_max_slide = h.slide 457 | 458 | else: 459 | self.action_sequence_length = 1 460 | self.true_bearing = self.heading 461 | self.progress_speed = 0.0 462 | self.track_speed = 0.0 463 | self.total_distance = 0.0 464 | self.max_skew = 0.0 465 | self.max_slide = 0.0 466 | self.recent_max_slide = 0.0 467 | self.just_passed_waypoint_ids = [] 468 | self.start_waypoint_id = self.closest_waypoint_id 469 | if len(self.time_at_waypoint) > self.start_waypoint_id: # FUDGE TO PASS VALIDATION IN CONSOLE 470 | self.time_at_waypoint[self.start_waypoint_id] = self.time 471 | self.corner_cutting = 1 472 | 473 | self.slide = get_turn_between_directions(self.heading, self.true_bearing) 474 | self.skew = get_turn_between_directions(self.track_bearing, self.true_bearing) 475 | self.total_distance += this_step.distance 476 | 477 | if abs(self.slide) > abs(self.max_slide): 478 | self.max_slide = self.slide 479 | if abs(self.skew) > abs(self.max_skew): 480 | self.max_skew = self.skew 481 | 482 | # 483 | # Object Avoidance Calculations 484 | # 485 | 486 | object_locations = params[ParamNames.OBJECTS_LOCATION] 487 | objects_left_of_center = params[ParamNames.OBJECTS_LEFT_OF_CENTER] 488 | closest_objects = params[ParamNames.CLOSEST_OBJECTS] 489 | 490 | self.has_objects = len(object_locations) > 0 491 | if self.has_objects: 492 | self.front_object_id = int(closest_objects[1]) 493 | self.rear_object_id = int(closest_objects[0]) 494 | 495 | if self.rear_object_id == self._previous_front_object: 496 | self.step_when_passed_object[self.rear_object_id] = self.steps 497 | 498 | self._previous_front_object = self.front_object_id 499 | 500 | self.distance_to_front_object = get_distance_between_points((self.x, self.y), 501 | object_locations[self.front_object_id]) 502 | self.distance_to_rear_object = get_distance_between_points((self.x, self.y), 503 | object_locations[self.rear_object_id]) 504 | 505 | self.front_object_is_left_of_centre = objects_left_of_center[self.front_object_id] 506 | self.rear_object_is_left_of_centre = objects_left_of_center[self.rear_object_id] 507 | 508 | else: 509 | self.front_object_id = None 510 | self.rear_object_id = None 511 | self.distance_to_front_object = None 512 | self.distance_to_rear_object = None 513 | self.front_object_is_left_of_centre = False 514 | self.rear_object_is_left_of_centre = None 515 | 516 | # 517 | # Projected distance calculation 518 | # 519 | 520 | self.projected_hit_object = False 521 | self.projected_distance, self.projected_progress_distance, self.projected_finish_left = self._calculate_projected_distance_on_track() 522 | if self.has_objects: 523 | object_hit_distance = self._calculate_object_hit_distance(object_locations[self.front_object_id]) 524 | if object_hit_distance is not None and object_hit_distance < self.projected_distance: 525 | self.projected_distance = object_hit_distance 526 | self.projected_hit_object = True 527 | elif len(object_locations) > 1: 528 | second_object_id = self.front_object_id + 1 529 | if second_object_id == len(object_locations): 530 | second_object_id = 0 531 | second_object_hit_distance = self._calculate_object_hit_distance(object_locations[second_object_id]) 532 | if second_object_hit_distance is not None and second_object_hit_distance < self.projected_distance: 533 | self.projected_distance = second_object_hit_distance 534 | 535 | def _calculate_projected_distance_on_track(self): 536 | heading = get_angle_in_proper_range(self.true_bearing) 537 | point = (self.x, self.y) 538 | 539 | previous_left = self._processed_waypoints[self.previous_waypoint_id].left_safe 540 | previous_right = self._processed_waypoints[self.previous_waypoint_id].right_safe 541 | 542 | (previous_progress_distance, next_progress_distance) = self._calculate_progress_distances( 543 | point, self.waypoints[self.previous_waypoint_id], self.waypoints[self.next_waypoint_id], 544 | self.is_left_of_center, self.distance_from_center) 545 | progress_distance = 0.0 546 | is_first_step = True 547 | 548 | previous_waypoint = self.waypoints[self.previous_waypoint_id] 549 | for w in self._processed_waypoints[self.next_waypoint_id:] + self._processed_waypoints[:self.next_waypoint_id]: 550 | off_track_distance, off_track_point, off_left = self._get_off_track_distance_and_point(point, heading, 551 | previous_left, 552 | previous_right, w) 553 | 554 | if off_track_distance is None: 555 | previous_left = w.left_safe 556 | previous_right = w.right_safe 557 | if is_first_step: 558 | is_first_step = False 559 | progress_distance = next_progress_distance 560 | else: 561 | progress_distance += get_distance_between_points((w.x, w.y), previous_waypoint) 562 | previous_waypoint = (w.x, w.y) 563 | elif off_track_distance == 0.0: 564 | return 0.0, 0.0, False 565 | else: 566 | (final_previous_progress_distance, final_next_progress_distance) = self._calculate_progress_distances( 567 | off_track_point, previous_waypoint, (w.x, w.y), off_left, 568 | self.track_width / 2 + RealWorld.SAFE_CAR_OVERHANG) 569 | 570 | if is_first_step: 571 | progress_distance = next_progress_distance - final_next_progress_distance 572 | else: 573 | progress_distance += final_previous_progress_distance 574 | return off_track_distance, progress_distance, off_left 575 | 576 | @staticmethod 577 | def _get_off_track_distance_and_point(point, heading: float, previous_left, previous_right, processed_waypoint): 578 | left_safe = processed_waypoint.left_safe 579 | right_safe = processed_waypoint.right_safe 580 | 581 | direction_to_left_target = get_bearing_between_points(point, left_safe) 582 | direction_to_right_target = get_bearing_between_points(point, right_safe) 583 | 584 | relative_direction_to_left_target = get_turn_between_directions(heading, direction_to_left_target) 585 | relative_direction_to_right_target = get_turn_between_directions(heading, direction_to_right_target) 586 | 587 | if relative_direction_to_left_target >= 0 and relative_direction_to_right_target <= 0: 588 | return None, None, None 589 | else: 590 | point2 = get_point_at_bearing(point, heading, 1) # Just some random distance (1m) 591 | if left_safe == previous_left: 592 | off_track_left = previous_left 593 | else: 594 | off_track_left = get_intersection_of_two_lines(point, point2, left_safe, previous_left) 595 | if right_safe == previous_right: 596 | off_track_right = previous_right 597 | else: 598 | off_track_right = get_intersection_of_two_lines(point, point2, right_safe, previous_right) 599 | 600 | left_bearing = get_bearing_between_points(point, off_track_left) 601 | right_bearing = get_bearing_between_points(point, off_track_right) 602 | 603 | distances = [] 604 | end_points = [] 605 | off_left = [] 606 | if abs(get_turn_between_directions(left_bearing, heading)) < 1: 607 | if is_point_between(off_track_left, left_safe, previous_left): 608 | distances += [get_distance_between_points(point, off_track_left)] 609 | end_points += [off_track_left] 610 | off_left += [True] 611 | if abs(get_turn_between_directions(right_bearing, heading)) < 1: 612 | if is_point_between(off_track_right, right_safe, previous_right): 613 | distances += [get_distance_between_points(point, off_track_right)] 614 | end_points += [off_track_right] 615 | off_left += [False] 616 | 617 | if len(distances) == 2 and distances[1] > distances[0]: 618 | return distances[1], end_points[1], off_left[1] 619 | elif len(distances) > 0: 620 | return distances[0], end_points[0], off_left[0] 621 | else: 622 | return 0.0, None, None 623 | 624 | @staticmethod 625 | def _calculate_progress_distances(point, previous_waypoint, next_waypoint, 626 | is_left, distance_from_centre): 627 | track_bearing = get_bearing_between_points(previous_waypoint, next_waypoint) 628 | 629 | if is_left: 630 | offset = -90 631 | else: 632 | offset = 90 633 | 634 | radians_to_centre_point = math.radians(track_bearing + offset) 635 | 636 | (x, y) = point 637 | centre_point = (x + math.cos(radians_to_centre_point) * distance_from_centre, 638 | y + math.sin(radians_to_centre_point) * distance_from_centre) 639 | 640 | return get_distance_between_points(centre_point, previous_waypoint), get_distance_between_points(centre_point, 641 | next_waypoint) 642 | 643 | def _calculate_object_hit_distance(self, obj_middle): 644 | heading = get_angle_in_proper_range(self.true_bearing) 645 | point = (self.x, self.y) 646 | 647 | point2 = get_point_at_bearing(point, heading, 1) # Just some random distance (1m) to define line 648 | track_bearing = self._get_track_bearing_at_point(obj_middle) 649 | safe_border = min(RealWorld.VEHICLE_WIDTH, RealWorld.VEHICLE_LENGTH) / 3 # Effectively enlarge the box 650 | 651 | front_middle = get_point_at_bearing(obj_middle, track_bearing, RealWorld.BOX_OBSTACLE_LENGTH / 2 + safe_border) 652 | front_left = get_point_at_bearing(front_middle, track_bearing + 90, 653 | RealWorld.BOX_OBSTACLE_WIDTH / 2 + safe_border) 654 | front_right = get_point_at_bearing(front_middle, track_bearing - 90, 655 | RealWorld.BOX_OBSTACLE_WIDTH / 2 + safe_border) 656 | 657 | rear_middle = get_point_at_bearing(obj_middle, track_bearing, -RealWorld.BOX_OBSTACLE_LENGTH / 2 - safe_border) 658 | rear_left = get_point_at_bearing(rear_middle, track_bearing + 90, 659 | RealWorld.BOX_OBSTACLE_WIDTH / 2 + safe_border) 660 | rear_right = get_point_at_bearing(rear_middle, track_bearing - 90, 661 | RealWorld.BOX_OBSTACLE_WIDTH / 2 + safe_border) 662 | 663 | distances = [] 664 | for box_side in [(front_left, front_right), (rear_left, rear_right), 665 | (front_left, rear_left), (front_right, rear_right)]: 666 | (box_point1, box_point2) = box_side 667 | hit_point = get_intersection_of_two_lines(point, point2, box_point1, box_point2) 668 | if hit_point is not None and is_point_between(hit_point, box_point1, box_point2): 669 | # Make sure it's in front of us! 670 | bearing_to_hit_point = get_bearing_between_points(point, hit_point) 671 | if abs(get_turn_between_directions(bearing_to_hit_point, heading)) < 1: 672 | distances.append(get_distance_between_points(point, hit_point)) 673 | 674 | if not distances: 675 | return None 676 | else: 677 | return min(distances) 678 | 679 | def _get_track_bearing_at_point(self, point): 680 | closest_waypoint = self._get_closest_waypoint_id(point) 681 | (before_waypoint, after_waypoint) = self.get_waypoint_ids_before_and_after(point, closest_waypoint) 682 | return get_bearing_between_points(self.waypoints[before_waypoint], 683 | self.waypoints[after_waypoint]) 684 | 685 | def _get_closest_waypoint_id(self, point): 686 | distance = get_distance_between_points(self.waypoints[0], point) 687 | closest_id = 0 688 | for i, w in enumerate(self.waypoints[1:]): 689 | new_distance = get_distance_between_points(w, point) 690 | if new_distance < distance: 691 | distance = new_distance 692 | closest_id = i + 1 693 | return closest_id 694 | 695 | def get_waypoint_ids_before_and_after(self, point, closest_waypoint_id: int, prefer_forwards=False): 696 | assert 0 <= closest_waypoint_id < len(self.waypoints) 697 | 698 | previous_id = self._get_previous_waypoint_id(closest_waypoint_id) 699 | next_id = self._get_next_waypoint_id(closest_waypoint_id) 700 | 701 | previous_waypoint = self.waypoints[previous_id] 702 | next_waypoint = self.waypoints[next_id] 703 | closest_waypoint = self.waypoints[closest_waypoint_id] 704 | 705 | target_dist = get_distance_between_points(closest_waypoint, previous_waypoint) 706 | if target_dist == 0.0: 707 | previous_ratio = 99999.0 708 | else: 709 | previous_ratio = get_distance_between_points(point, previous_waypoint) / target_dist 710 | 711 | target_dist = get_distance_between_points(closest_waypoint, next_waypoint) 712 | if target_dist == 0.0: 713 | next_ratio = 99999.0 714 | else: 715 | next_ratio = get_distance_between_points(point, next_waypoint) / target_dist 716 | 717 | if prefer_forwards: # Make the behind waypoint appear 5% further away 718 | previous_ratio *= 1.05 719 | 720 | if previous_ratio > next_ratio: 721 | return closest_waypoint_id, next_id 722 | else: 723 | return previous_id, closest_waypoint_id 724 | 725 | def _get_next_waypoint_id(self, waypoint_id): 726 | if waypoint_id >= len(self.waypoints) - 1: 727 | return 0 728 | else: 729 | return waypoint_id + 1 730 | 731 | def _get_previous_waypoint_id(self, waypoint_id): 732 | if waypoint_id < 1: 733 | return len(self.waypoints) - 1 734 | else: 735 | return waypoint_id - 1 736 | 737 | def _get_just_passed_waypoint_ids(self, previous_next_waypoint_id, current_next_waypoint_id): 738 | if previous_next_waypoint_id == current_next_waypoint_id: 739 | return [] 740 | 741 | difference = current_next_waypoint_id - previous_next_waypoint_id 742 | 743 | if difference < -10 or 1 <= difference <= 10: 744 | result = [] 745 | w = previous_next_waypoint_id 746 | while w != current_next_waypoint_id: 747 | if self.time_at_waypoint[w] is None: 748 | result.append(w) 749 | self.time_at_waypoint[w] = self.time 750 | w += 1 751 | if w >= len(self.waypoints): 752 | w = 0 753 | 754 | return result 755 | else: 756 | return [] 757 | 758 | def get_track_distance_between_waypoints(self, start: int, finish: int): 759 | distance = 0 760 | assert 0 <= start < len(self.waypoints) 761 | assert 0 <= finish < len(self.waypoints) 762 | 763 | while start != finish: 764 | next_wp = self._get_next_waypoint_id(start) 765 | distance += get_distance_between_points(self.waypoints[start], self.waypoints[next_wp]) 766 | start = next_wp 767 | 768 | return distance 769 | 770 | def get_progress_speed(self, steps: int): 771 | assert steps >= 1 772 | if steps >= len(self._history): 773 | return None 774 | 775 | progress_speed_distance = (self.progress - self._history[-steps - 1].progress) / 100 * self.track_length 776 | progress_speed_calculate_time = steps / RealWorld.STEPS_PER_SECOND 777 | return max(0.0, progress_speed_distance / progress_speed_calculate_time) 778 | 779 | def print_debug(self): 780 | print("x, y ", round(self.x, 3), round(self.y, 3)) 781 | print("all_wheels_on_track ", self.all_wheels_on_track) 782 | print("previous_waypoint_id ", self.previous_waypoint_id) 783 | print("previous_waypoint_x, y ", round(self.previous_waypoint_x, 3), round(self.previous_waypoint_y, 3)) 784 | print("next_waypoint_id ", self.next_waypoint_id) 785 | print("next_waypoint_x, y ", round(self.next_waypoint_x, 3), round(self.next_waypoint_y, 3)) 786 | print("closest_waypoint_id ", self.closest_waypoint_id) 787 | print("closest_waypoint_x, y ", round(self.closest_waypoint_x, 3), round(self.closest_waypoint_y, 3)) 788 | print("distance_from_closest_waypoint ", round(self.distance_from_closest_waypoint, 2)) 789 | print("distance_from_center ", round(self.distance_from_center, 2)) 790 | print("distance_from_edge ", round(self.distance_from_edge, 2)) 791 | print("distance_from_extreme_edge ", round(self.distance_from_extreme_edge, 2)) 792 | print("is_left/right_of_center ", self.is_left_of_center, self.is_right_of_center) 793 | print("is_crashed / reversed ", self.is_crashed, self.is_reversed) 794 | print("is_off_track ", self.is_off_track) 795 | print("is_complete_lap ", self.is_complete_lap) 796 | print("steps, is_final_step ", self.steps, self.is_final_step) 797 | print("time ", round(self.time, 2)) 798 | print("predicted_lap_time ", round(self.predicted_lap_time, 2)) 799 | print("progress ", round(self.progress, 2)) 800 | print("waypoints (SIZE) ", len(self.waypoints)) 801 | print("track_length, width ", round(self.track_length, 2), round(self.track_width, 2)) 802 | print("action_speed ", round(self.action_speed, 2)) 803 | print("action_steering_angle ", round(self.action_steering_angle, 1)) 804 | print("action_sequence_length ", self.action_sequence_length) 805 | print("is_steering_left/right ", self.is_steering_left, self.is_steering_right) 806 | print("is_steering_straight ", self.is_steering_straight) 807 | print("heading ", round(self.heading, 2)) 808 | print("track_bearing ", round(self.track_bearing, 2)) 809 | print("true_bearing ", round(self.true_bearing, 2)) 810 | print("slide / max / recent ", 811 | round(self.slide, 2), round(self.max_slide, 2), round(self.recent_max_slide, 2)) 812 | print("skew / max_skew ", round(self.skew, 2), round(self.max_skew, 2)) 813 | print("total_distance ", round(self.total_distance, 2)) 814 | print("track_speed ", round(self.track_speed, 2)) 815 | print("progress_speed ", round(self.progress_speed, 2)) 816 | print("just_passed_waypoint_ids ", self.just_passed_waypoint_ids) 817 | print("time_at_waypoint ", self.time_at_waypoint) 818 | print("projected_distance ", self.projected_distance) 819 | 820 | 821 | # ------------------------------------------------------------------------------- 822 | # 823 | # REWARD FUNCTION MASTER WRAPPER 824 | # 825 | # ------------------------------------------------------------------------------- 826 | 827 | def reward_function(params): 828 | global framework_global 829 | if not framework_global: 830 | framework_global = Framework(params) 831 | framework_global.process_params(params) 832 | raw_reward = float(get_reward(framework_global)) 833 | if raw_reward > 0: 834 | return raw_reward 835 | else: 836 | tiny_reward = 0.0001 837 | print("WARNING - Invalid reward " + str(raw_reward) + " replaced with " + str(tiny_reward)) 838 | return tiny_reward 839 | 840 | 841 | framework_global = None 842 | 843 | 844 | # ------------------------------------------------------------------------------- 845 | # 846 | # YOUR REWARD FUNCTION GOES HERE ... ... ... ... 847 | # 848 | # ------------------------------------------------------------------------------- 849 | 850 | def get_reward(f: Framework): 851 | print(f.max_possible_track_speed, round(f.projected_distance, 1)) 852 | return f.progress_speed / f.max_possible_track_speed * f.projected_distance + f.steps / 10 853 | --------------------------------------------------------------------------------