├── assets ├── BP_AudiA2.uasset ├── Mat_patch.uasset ├── BP_Ambulance.uasset └── BP_Decal_Billboard_Auto.uasset ├── configs ├── collections │ ├── ss_template.yml │ ├── 2dod_template.yml │ ├── 3dod_template.yml │ └── depth_template.yml ├── sensors │ ├── coco.json │ ├── cityscapes.json │ └── kitti.json ├── simulation_configs.py └── billboards_config_Town10HD.yml ├── utils ├── prepare_carla_input_images.py ├── carla_utils.py ├── sensor_interface.py ├── ego_vehicle.py ├── sstags_carla_to_cityscapes.py ├── npc.py ├── bbox_utils.py └── datasets_utils.py ├── LICENSE ├── requirements.txt ├── README.md └── collect_dataset.py /assets/BP_AudiA2.uasset: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/retis-ai/CARLA-GeAR/HEAD/assets/BP_AudiA2.uasset -------------------------------------------------------------------------------- /assets/Mat_patch.uasset: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/retis-ai/CARLA-GeAR/HEAD/assets/Mat_patch.uasset -------------------------------------------------------------------------------- /assets/BP_Ambulance.uasset: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/retis-ai/CARLA-GeAR/HEAD/assets/BP_Ambulance.uasset -------------------------------------------------------------------------------- /assets/BP_Decal_Billboard_Auto.uasset: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/retis-ai/CARLA-GeAR/HEAD/assets/BP_Decal_Billboard_Auto.uasset -------------------------------------------------------------------------------- /configs/collections/ss_template.yml: -------------------------------------------------------------------------------- 1 | basepath: /path/to/root 2 | root: ss 3 | 4 | town_name: Town10HD 5 | weather: # fixed 6 | 7 | task: ss # ss, 2dod, 3dod, depth 8 | split: train # train, val, test, debug. Might add _othertext to obtain additional splits. 9 | billboard: billboard01 # Depends on town! 10 | patch_folder: # folder where to find the patches 11 | 12 | patch_type: # Not used 13 | net: # Not used 14 | 15 | debug: False 16 | 17 | 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /configs/collections/2dod_template.yml: -------------------------------------------------------------------------------- 1 | basepath: /path/to/root 2 | root: 2dod 3 | 4 | town_name: Town10HD 5 | weather: # fixed 6 | 7 | task: 2dod # ss, 2dod, 3dod, depth 8 | split: train # train, val, test, debug. Might add _othertext to obtain additional splits. 9 | billboard: billboard01 # Depends on town! 10 | patch_folder: # folder where to find the patches 11 | 12 | patch_type: # Not used 13 | net: # Not used 14 | 15 | debug: False 16 | 17 | 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /configs/collections/3dod_template.yml: -------------------------------------------------------------------------------- 1 | basepath: /path/to/root 2 | root: 3dod 3 | 4 | town_name: Town10HD 5 | weather: # fixed 6 | 7 | task: 3dod # ss, 2dod, 3dod, depth 8 | split: train # train, val, test, debug. Might add _othertext to obtain additional splits. 9 | billboard: billboard01 # Depends on town! 10 | patch_folder: # folder where to find the patches 11 | 12 | patch_type: # Not used 13 | net: # Not used 14 | 15 | debug: False 16 | 17 | 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /configs/collections/depth_template.yml: -------------------------------------------------------------------------------- 1 | basepath: /path/to/root 2 | root: depth 3 | 4 | town_name: Town10HD 5 | weather: # fixed 6 | 7 | task: depth # ss, 2dod, 3dod, depth 8 | split: train # train, val, test, debug. Might add _othertext to obtain additional splits. 9 | billboard: billboard01 # Depends on town! 10 | patch_folder: # folder where to find the patches 11 | 12 | patch_type: # Not used 13 | net: # Not used 14 | 15 | debug: False 16 | 17 | 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /utils/prepare_carla_input_images.py: -------------------------------------------------------------------------------- 1 | import glob 2 | import imageio 3 | import numpy as np 4 | 5 | #input_basepath = '/data/dataset_rgb' 6 | #output_basepath = '/data/dataset_carla/leftImg8bit/val/Town03DUP' 7 | 8 | input_basepath = '/data/rgb' 9 | output_basepath = '/data/dataset_carla/leftImg8bit/billboard/Town03DUP' 10 | 11 | img_files = glob.glob('{}/*.png'.format(input_basepath)) 12 | 13 | for imgpth in img_files: 14 | filename = imgpth.split('_')[-1][:-4] 15 | #filename = imgpth.split('/')[-1][:-4] 16 | print('Processing:', filename) 17 | img = imageio.imread(imgpth) 18 | img = img[:, :, :3] 19 | imageio.imwrite('{}/{}_leftImg8bit.png'.format(output_basepath, filename), img) 20 | 21 | -------------------------------------------------------------------------------- /configs/sensors/coco.json: -------------------------------------------------------------------------------- 1 | {"sensors_config": [ 2 | { 3 | "type": "sensor.camera.rgb", 4 | "x": 1.5, 5 | "y": 0.0, 6 | "z": 1.65, 7 | "roll": 0.0, 8 | "pitch": 0.0, 9 | "yaw": 0.0, 10 | "width": 1333, 11 | "height": 800, 12 | "fov": 90, 13 | "id": "RGBCam" 14 | }, 15 | { 16 | "type": "sensor.camera.depth", 17 | "x": 1.5, 18 | "y": 0.0, 19 | "z": 1.65, 20 | "roll": 0.0, 21 | "pitch": 0.0, 22 | "yaw": 0.0, 23 | "width": 1333, 24 | "height": 800, 25 | "fov": 90, 26 | "id": "DepthCam" 27 | } 28 | ] 29 | } -------------------------------------------------------------------------------- /configs/sensors/cityscapes.json: -------------------------------------------------------------------------------- 1 | {"sensors_config": [ 2 | { 3 | "type": "sensor.camera.rgb", 4 | "x": 1.5, 5 | "y": 0.0, 6 | "z": 1.65, 7 | "roll": 0.0, 8 | "pitch": 0.0, 9 | "yaw": 0.0, 10 | "width": 2048, 11 | "height": 1024, 12 | "fov": 90, 13 | "id": "RGBCam" 14 | }, 15 | { 16 | "type": "sensor.camera.semantic_segmentation", 17 | "x": 1.5, 18 | "y": 0.0, 19 | "z": 1.65, 20 | "roll": 0.0, 21 | "pitch": 0.0, 22 | "yaw": 0.0, 23 | "width": 2048, 24 | "height": 1024, 25 | "fov": 90, 26 | "id": "SSCam" 27 | } 28 | ] 29 | } -------------------------------------------------------------------------------- /configs/sensors/kitti.json: -------------------------------------------------------------------------------- 1 | {"sensors_config": [ 2 | { 3 | "type": "sensor.camera.rgb", 4 | "x": 1.5, 5 | "y": 0.0, 6 | "z": 1.65, 7 | "roll": 0.0, 8 | "pitch": 0.0, 9 | "yaw": 0.0, 10 | "width": 1224, 11 | "height": 370, 12 | "fov": 90, 13 | "id": "RGBCam" 14 | }, 15 | { 16 | "type": "sensor.camera.rgb", 17 | "x": 1.5, 18 | "y": 0.54, 19 | "z": 1.65, 20 | "roll": 0.0, 21 | "pitch": 0.0, 22 | "yaw": 0.0, 23 | "width": 1224, 24 | "height": 370, 25 | "fov": 90, 26 | "id": "RGBCamDx" 27 | }, 28 | { 29 | "type": "sensor.camera.depth", 30 | "x": 1.5, 31 | "y": 0.0, 32 | "z": 1.65, 33 | "roll": 0.0, 34 | "pitch": 0.0, 35 | "yaw": 0.0, 36 | "width": 1224, 37 | "height": 370, 38 | "fov": 90, 39 | "id": "DepthCam" 40 | } 41 | ] 42 | } -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 RetisLab - AI 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /configs/simulation_configs.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os 3 | 4 | # Seeds for repeatability 5 | SEED_VALUES = { 6 | 'finetuning': 0, 7 | 'train': 123, 8 | 'val': 131, 9 | 'test': 363, # 363 10 | 'debug': 10, 11 | #'finetuningval': 100, 12 | #'finetuningtest': 1000, 13 | 'video': 60, #Good values: 313 14 | } 15 | 16 | ITERATION_VALUES = { 17 | 'finetuning': 300, 18 | 'train': 100, 19 | 'val': 50, 20 | 'test': 50, 21 | 'debug': 2, 22 | #'finetuningval': 50, ##25, 23 | #'finetuningtest': 50, ##25, 24 | 'video': 1, 25 | } 26 | 27 | # Vehicle brands included in the simulation. Do not include ford or audi. 28 | BRANDS = ['dodge', 'citroen', 'chevrolet', 'bmw', 'toyota', 'mercedes'] 29 | MAX_VEHICLE_NPC = 7 30 | MAX_WALKER_NPC = 200 31 | 32 | def get_spawn_bp(billboard): 33 | if 'no_billboard' in billboard: 34 | return None 35 | elif 'billboard' in billboard: 36 | return 'vehicle.audi.a2' 37 | elif 'truck' in billboard: 38 | return 'vehicle.ford.ambulance' 39 | else: 40 | raise NotImplemented('No billboard name %s is implemented.' % billboard) 41 | 42 | # CarlaUE4 import path for patch. 43 | patch_load_path = '/home/federico/carla/carla/Unreal/CarlaUE4/Content/patches/patch_auto/patch.png' 44 | 45 | 46 | 47 | 48 | 49 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | actionlib==1.12.1 2 | angles==1.9.12 3 | auditwheel==5.1.2 4 | carla @ file:///home/federico/carla/carla/PythonAPI/carla/dist/carla-0.9.13-cp36-cp36m-linux_x86_64.whl 5 | catkin==0.7.29 6 | cycler==0.11.0 7 | decorator==4.4.2 8 | diagnostic_updater==1.9.7 9 | distro==1.7.0 10 | dynamic_reconfigure==1.6.5 11 | gencpp==0.6.5 12 | geneus==2.2.6 13 | genlisp==0.4.16 14 | genmsg==0.5.16 15 | gennodejs==2.0.1 16 | genpy==0.6.16 17 | imageio==2.15.0 18 | importlib-metadata==4.8.3 19 | interactive-markers==1.11.5 20 | kiwisolver==1.3.1 21 | laser_geometry==1.6.7 22 | matplotlib==3.3.4 23 | message_filters==1.14.13 24 | networkx==2.5.1 25 | numpy==1.18.4 26 | Pillow==8.4.0 27 | pkg_resources==0.0.0 28 | pyelftools==0.28 29 | pygame==2.1.2 30 | pyparsing==3.0.7 31 | python-dateutil==2.8.2 32 | python_qt_binding==0.4.4 33 | PyYAML==5.4.1 34 | resource_retriever==1.12.7 35 | rosbag==1.14.13 36 | rosclean==1.14.9 37 | rosgraph==1.14.13 38 | roslaunch==1.14.13 39 | roslib==1.14.9 40 | roslint==0.11.2 41 | roslz4==1.14.13 42 | rosmaster==1.14.13 43 | rosmsg==1.14.13 44 | rosnode==1.14.13 45 | rosparam==1.14.13 46 | rospy==1.14.13 47 | rosservice==1.14.13 48 | rostest==1.14.13 49 | rostopic==1.14.13 50 | rosunit==1.14.9 51 | roswtf==1.14.13 52 | rviz==1.13.29 53 | sensor-msgs==1.12.8 54 | Shapely==1.6.4.post2 55 | six==1.16.0 56 | smclib==1.8.5 57 | tf==1.12.1 58 | tf2_py==0.6.5 59 | tf2_ros==0.6.5 60 | topic_tools==1.14.13 61 | typing_extensions==4.1.1 62 | wiimote==1.14.0 63 | xacro==1.13.17 64 | zipp==3.6.0 65 | -------------------------------------------------------------------------------- /utils/carla_utils.py: -------------------------------------------------------------------------------- 1 | import carla 2 | import time 3 | import queue 4 | 5 | def carla_setup(town_name, SEED, debug=False): 6 | # Connect to Carla Simulator 7 | count_errors = 0 8 | ok_connect = False 9 | while not ok_connect and count_errors < 3: 10 | try: 11 | client = carla.Client(host='localhost', port=2000) 12 | client.set_timeout(5.0) 13 | client.load_world(town_name) 14 | except: 15 | count_errors += 1 16 | time.sleep(1) 17 | finally: 18 | ok_connect = True 19 | 20 | synchronous_mode = True 21 | 22 | world = client.get_world() 23 | settings = world.get_settings() 24 | world.apply_settings(carla.WorldSettings( 25 | no_rendering_mode=not debug, 26 | synchronous_mode=synchronous_mode, 27 | fixed_delta_seconds=1/30, 28 | )) 29 | client.reload_world(False) 30 | 31 | weather_setting = carla.WeatherParameters.ClearNoon # #CloudyNoon, ClearNoon, CloudySunset, ClearSunset, SoftRainNoon 32 | weather_setting.scattering_intensity = 0 33 | world.set_weather(weather_setting) 34 | 35 | traffic_manager = client.get_trafficmanager(8000) 36 | traffic_manager.set_global_distance_to_leading_vehicle(1.0) 37 | traffic_manager.set_synchronous_mode(synchronous_mode) 38 | traffic_manager.set_random_device_seed(SEED) 39 | return client, world, settings 40 | 41 | 42 | def cleanup_env(ego, npc, client): 43 | time.sleep(1) 44 | ego.cleanup() 45 | # print('ego cleaned up.') 46 | npc.cleanup(client) 47 | # print('npc cleaned up.') 48 | 49 | 50 | class CarlaSyncMode(object): 51 | """ 52 | Context manager to synchronize output from different sensors. Synchronous 53 | mode is enabled as long as we are inside this context 54 | 55 | with CarlaSyncMode(world, sensors) as sync_mode: 56 | while True: 57 | data = sync_mode.tick(timeout=1.0) 58 | 59 | """ 60 | 61 | def __init__(self, world, *sensors, **kwargs): 62 | self.world = world 63 | self.sensors = sensors 64 | self.frame = None 65 | self.delta_seconds = 1.0 / kwargs.get('fps', 20) 66 | self._queues = [] 67 | self._settings = None 68 | 69 | def __enter__(self): 70 | self._settings = self.world.get_settings() 71 | self.frame = self.world.apply_settings(carla.WorldSettings( 72 | no_rendering_mode=False, 73 | synchronous_mode=True, 74 | fixed_delta_seconds=self.delta_seconds)) 75 | 76 | 77 | def make_queue(register_event): 78 | q = queue.Queue() 79 | register_event(q.put) 80 | self._queues.append(q) 81 | 82 | make_queue(self.world.on_tick) 83 | for sensor in self.sensors: 84 | make_queue(sensor.listen) 85 | return self 86 | 87 | def tick(self, timeout): 88 | self.frame = self.world.tick() 89 | data = [self._retrieve_data(q, timeout) for q in self._queues] 90 | assert all(x.frame == self.frame for x in data) 91 | return data 92 | 93 | def __exit__(self, *args, **kwargs): 94 | self.world.apply_settings(self._settings) 95 | 96 | def _retrieve_data(self, sensor_queue, timeout): 97 | while True: 98 | data = sensor_queue.get(timeout=timeout) 99 | if data.frame == self.frame: 100 | return data 101 | 102 | 103 | -------------------------------------------------------------------------------- /utils/sensor_interface.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # This work is licensed under the terms of the MIT license. 4 | # For a copy, see . 5 | 6 | """ 7 | This file containts CallBack class and SensorInterface, responsible of 8 | handling the use of sensors for the agents 9 | """ 10 | 11 | import copy 12 | import logging 13 | 14 | try: 15 | from queue import Queue 16 | from queue import Empty 17 | except ImportError: 18 | from Queue import Queue 19 | from Queue import Empty 20 | 21 | import numpy as np 22 | 23 | import carla 24 | 25 | 26 | class SensorReceivedNoData(Exception): 27 | 28 | """ 29 | Exceptions thrown when the sensors used by the agent take too long to receive data 30 | """ 31 | 32 | 33 | class CallBack(object): 34 | 35 | """ 36 | Class the sensors listen to in order to receive their data each frame 37 | """ 38 | 39 | def __init__(self, tag, sensor, data_provider): 40 | """ 41 | Initializes the call back 42 | """ 43 | self._tag = tag 44 | self._data_provider = data_provider 45 | 46 | self._data_provider.register_sensor(tag, sensor) 47 | 48 | def __call__(self, data): 49 | """ 50 | call function 51 | """ 52 | if isinstance(data, carla.Image): 53 | self._parse_image_cb(data, self._tag) 54 | elif isinstance(data, carla.LidarMeasurement): 55 | self._parse_lidar_cb(data, self._tag) 56 | elif isinstance(data, carla.RadarMeasurement): 57 | self._parse_radar_cb(data, self._tag) 58 | elif isinstance(data, carla.GnssMeasurement): 59 | self._parse_gnss_cb(data, self._tag) 60 | elif isinstance(data, carla.IMUMeasurement): 61 | self._parse_imu_cb(data, self._tag) 62 | else: 63 | logging.error('No callback method for this sensor.') 64 | 65 | # Parsing CARLA physical Sensors 66 | def _parse_image_cb(self, image, tag): 67 | """ 68 | parses cameras 69 | """ 70 | self._data_provider.update_sensor(tag, image, image.frame) 71 | 72 | 73 | class SensorInterface(object): 74 | 75 | """ 76 | Class that contains all sensor data 77 | """ 78 | 79 | def __init__(self): 80 | """ 81 | Initializes the class 82 | """ 83 | self._sensors_objects = {} 84 | self._new_data_buffers = Queue() 85 | self._queue_timeout = 10 86 | 87 | def register_sensor(self, tag, sensor): 88 | """ 89 | Registers the sensors 90 | """ 91 | if tag in self._sensors_objects: 92 | raise ValueError("Duplicated sensor tag [{}]".format(tag)) 93 | 94 | self._sensors_objects[tag] = sensor 95 | 96 | def update_sensor(self, tag, data, timestamp): 97 | """ 98 | Updates the sensor 99 | """ 100 | if tag not in self._sensors_objects: 101 | raise ValueError("The sensor with tag [{}] has not been created!".format(tag)) 102 | 103 | self._new_data_buffers.put((tag, timestamp, data)) 104 | 105 | def all_sensors_ready(self): 106 | """ 107 | Checks if all the sensors have sent data at least once 108 | """ 109 | data_dict = self.get_data() 110 | frame_ids = [val[0] for val in data_dict.values()] 111 | if len(set(frame_ids)) == 1: 112 | return True 113 | return False 114 | 115 | 116 | def get_data(self): 117 | """ 118 | Returns the data of a sensor 119 | """ 120 | try: 121 | data_dict = {} 122 | while len(data_dict.keys()) < len(self._sensors_objects.keys()): 123 | 124 | sensor_data = self._new_data_buffers.get(True, self._queue_timeout) 125 | data_dict[sensor_data[0]] = ((sensor_data[1], sensor_data[2])) 126 | 127 | except Empty: 128 | raise SensorReceivedNoData("A sensor took too long to send its data") 129 | 130 | return data_dict 131 | 132 | def reset(self): 133 | self._sensors_objects = {} 134 | self._new_data_buffers = Queue() -------------------------------------------------------------------------------- /utils/ego_vehicle.py: -------------------------------------------------------------------------------- 1 | import random 2 | import time 3 | 4 | import carla 5 | from .sensor_interface import CallBack, SensorInterface 6 | 7 | class EgoVehicle: 8 | def __init__(self, world, sensors_config, verbose=False): 9 | self.world = world 10 | self.start_transform = None 11 | 12 | self.sensor_interface = SensorInterface() 13 | self.sensors = sensors_config 14 | self._sensors_list = [] 15 | self.vehicle = None 16 | self.verbose = verbose 17 | 18 | def setup_sensors(self): 19 | """ 20 | Create the sensors defined by the user and attach them to the ego-vehicle 21 | :param vehicle: ego vehicle 22 | :return: 23 | """ 24 | bp_library = self.world.get_blueprint_library() 25 | for sensor_spec in self.sensors: 26 | # These are the sensors spawned on the carla world 27 | bp = bp_library.find(str(sensor_spec['type'])) 28 | if sensor_spec['type'].startswith('sensor.camera'): 29 | bp.set_attribute('image_size_x', str(sensor_spec['width'])) 30 | bp.set_attribute('image_size_y', str(sensor_spec['height'])) 31 | bp.set_attribute('fov', str(sensor_spec['fov'])) 32 | sensor_location = carla.Location(x=sensor_spec['x'], y=sensor_spec['y'], z=sensor_spec['z']) 33 | sensor_rotation = carla.Rotation(pitch=sensor_spec['pitch'], roll=sensor_spec['roll'], yaw=sensor_spec['yaw']) 34 | 35 | # create sensor 36 | sensor_transform = carla.Transform(sensor_location, sensor_rotation) 37 | sensor = self.world.spawn_actor(bp, sensor_transform, attach_to=self.vehicle) 38 | # setup callback 39 | sensor.listen(CallBack(sensor_spec['id'], sensor, self.sensor_interface)) 40 | self._sensors_list.append(sensor) 41 | 42 | # Tick once to spawn the sensors 43 | self.world.tick() 44 | 45 | def setup_vehicle(self, vehicle_blueprints, start_transform=None): 46 | self.start_transform = start_transform 47 | if start_transform is None: 48 | self.start_transform = random.choice(self.world.get_map().get_spawn_points()) 49 | 50 | self.start_transform.location.z = 0.1 # need to raise vehicle to avoid collision at spawn time 51 | 52 | ## needed for billboard 02 53 | #self.start_transform.rotation.pitch=0.0 54 | #self.start_transform.rotation.yaw=0.0 55 | 56 | 57 | if self.vehicle is None: 58 | 59 | bp = random.choice(vehicle_blueprints) 60 | # sprinter dimenstions raise problems with camera config. 61 | while 'sprinter' in bp.id: 62 | bp = random.choice(vehicle_blueprints) 63 | 64 | self.vehicle = None 65 | count_try = 0 66 | while self.vehicle is None and count_try < 10: 67 | self.vehicle = self.world.try_spawn_actor(bp, self.start_transform) 68 | self.start_transform.location.z += 0.1 69 | count_try += 1 70 | 71 | if count_try >= 10: 72 | raise Exception("Impossible to spawn ego vehicle. It's possible that the spawn config is wrong?") 73 | 74 | if self.vehicle is not None: 75 | if self.verbose: 76 | print('Spawned Ego Vehicle at (%.2f, %.2f, %.2f): ' % (start_transform.location.x, start_transform.location.y, start_transform.location.z)) 77 | 78 | self.vehicle.set_autopilot(enabled=True) 79 | self.world.tick() 80 | 81 | 82 | def get_sensor_data(self): 83 | return self.sensor_interface.get_data() 84 | 85 | def wait_for_sensors(self, num_ticks=25): 86 | for _ in range(num_ticks): 87 | self.sensor_interface.get_data() 88 | self.world.tick() 89 | 90 | if self.verbose: 91 | print('all sensors ready to go...') 92 | 93 | def cleanup(self): 94 | """ 95 | Remove and destroy all sensors 96 | """ 97 | for i, _ in enumerate(self._sensors_list): 98 | if self._sensors_list[i] is not None: 99 | if self.verbose: 100 | print("Destroying: ", self._sensors_list[i]) 101 | self._sensors_list[i].stop() 102 | time.sleep(1) 103 | self._sensors_list[i].destroy() 104 | self._sensors_list[i] = None 105 | self._sensors_list = [] 106 | 107 | self.sensor_interface.reset() 108 | if self.verbose: 109 | print('destroyed sensors.') 110 | 111 | if self.vehicle is not None: 112 | self.vehicle.set_autopilot(False) 113 | self.vehicle.destroy() 114 | self.vehicle = None 115 | 116 | if self.verbose: 117 | print('destroyed ego.') 118 | 119 | 120 | 121 | -------------------------------------------------------------------------------- /configs/billboards_config_Town10HD.yml: -------------------------------------------------------------------------------- 1 | # Name each billboard with billboardXX, or no_billboard, or truck 2 | billboard01: 3 | b1: 4 | location: 5 | x: -25 6 | y: 4 7 | z: 2 8 | rotation: 9 | pitch: 0 10 | roll: -90 11 | yaw: 200 12 | spawn_config: 13 | npc: 14 | min_x: -10 15 | max_x: 20 16 | min_y: 0 17 | max_y: 30 18 | ego: 19 | min_x: 15 20 | max_x: 30 21 | min_y: 0 22 | max_y: 10 23 | 24 | 25 | # GOOD 26 | billboard02: 27 | b1: 28 | location: 29 | x: 88 30 | y: 144 31 | z: 2 32 | rotation: 33 | pitch: 0 34 | roll: -90 35 | yaw: -160 36 | spawn_config: 37 | npc: 38 | min_x: -60 39 | max_x: 20 40 | min_y: -10 41 | max_y: 30 42 | ego: 43 | min_x: -30 # -40 -35 -30 -25 -20 44 | max_x: -15 45 | min_y: 0 46 | max_y: 10 47 | 48 | billboard03: 49 | b1: 50 | location: 51 | x: -28 52 | y: -76 53 | z: 2 54 | rotation: 55 | pitch: 0 56 | roll: -90 57 | yaw: 30 58 | spawn_config: 59 | npc: 60 | min_x: -50 61 | max_x: 50 62 | min_y: 0 63 | max_y: 30 64 | ego: 65 | min_x: 15 66 | max_x: 30 67 | min_y: 0 68 | max_y: 12 69 | 70 | 71 | billboard04: 72 | b1: 73 | location: 74 | x: -122 75 | y: 51 76 | z: 2 77 | rotation: 78 | pitch: 0 79 | roll: -90 80 | yaw: -75 81 | spawn_config: 82 | npc: 83 | min_x: 0 84 | max_x: 30 85 | min_y: -20 86 | max_y: 50 87 | ego: 88 | min_x: 0 89 | max_x: 12 90 | min_y: -30 91 | max_y: -15 92 | 93 | 94 | # DOUBLE 95 | billboard05: 96 | b1: 97 | location: 98 | x: -122 99 | y: 16.5 100 | z: 2 101 | rotation: 102 | pitch: 0 103 | roll: -90 104 | yaw: -180 105 | spawn_config: 106 | npc: 107 | min_x: 0 108 | max_x: 40 109 | min_y: -30 110 | max_y: 30 111 | ego: 112 | min_x: 26 113 | max_x: 40 114 | min_y: -12 115 | max_y: 0 116 | b2: 117 | location: 118 | x: -122 119 | y: 5.6 120 | z: 2 121 | rotation: 122 | pitch: 0 123 | roll: -90 124 | yaw: -180 125 | 126 | 127 | # DOUBLE 128 | billboard06: 129 | b1: 130 | location: 131 | x: 117 132 | y: 24.5 133 | z: 2 134 | rotation: 135 | pitch: 0 136 | roll: -90 137 | yaw: 0 138 | spawn_config: 139 | npc: 140 | min_x: 0 141 | max_x: 40 142 | min_y: -30 143 | max_y: 30 144 | ego: 145 | min_x: -35 146 | max_x: -25 147 | min_y: 0 148 | max_y: 12 149 | b2: 150 | location: 151 | x: 117 152 | y: 33.5 153 | z: 2 154 | rotation: 155 | pitch: 0 156 | roll: -90 157 | yaw: 0 158 | 159 | 160 | # DOUBLE 161 | billboard07: 162 | b1: 163 | location: 164 | x: -51 165 | y: 145 166 | z: 2 167 | rotation: 168 | pitch: 0 169 | roll: -90 170 | yaw: -100 171 | spawn_config: 172 | npc: 173 | min_x: 0 174 | max_x: 40 175 | min_y: -30 176 | max_y: 30 177 | ego: 178 | min_x: -12 179 | max_x: 2 180 | min_y: -24 181 | max_y: -35 182 | b2: 183 | location: 184 | x: -64 185 | y: 145 186 | z: 2 187 | rotation: 188 | pitch: 0 189 | roll: -90 190 | yaw: -80 191 | 192 | 193 | billboard08: 194 | b1: 195 | location: 196 | x: 37.5 197 | y: 72.50 198 | z: 2 199 | rotation: 200 | pitch: 0 201 | roll: -90 202 | yaw: -270 203 | spawn_config: 204 | npc: 205 | min_x: -10 206 | max_x: 30 207 | min_y: -50 208 | max_y: 0 209 | ego: 210 | min_x: 0 211 | max_x: 3 212 | min_y: -30 213 | max_y: -14 214 | 215 | 216 | billboard09: 217 | b1: 218 | location: 219 | x: -19.50 220 | y: 122 221 | z: 2 222 | rotation: 223 | pitch: 0 224 | roll: -90 225 | yaw: 20 226 | spawn_config: 227 | npc: 228 | min_x: -20 229 | max_x: 40 230 | min_y: -10 231 | max_y: 30 232 | ego: 233 | min_x: 15 234 | max_x: 30 235 | min_y: 0 236 | max_y: 12 237 | 238 | truck: 239 | b1: 240 | location: 241 | x: 242 | y: 243 | z: 244 | rotation: 245 | pitch: 246 | roll: 247 | yaw: 248 | spawn_config: 249 | npc: 250 | min_x: -1000 #-10 251 | max_x: 1000 #20 252 | min_y: -1000 #0 253 | max_y: 1000 #20 254 | ego: 255 | min_x: -1000 #-60 256 | max_x: 1000 #-10 257 | min_y: -1000 #-10 258 | max_y: 1000 #10 259 | 260 | 261 | 262 | no_billboard: 263 | b1: 264 | location: 265 | x: 266 | y: 267 | z: 268 | rotation: 269 | pitch: 270 | roll: 271 | yaw: 272 | spawn_config: 273 | npc: 274 | min_x: -1000 #-10 275 | max_x: 1000 #20 276 | min_y: -1000 #0 277 | max_y: 1000 #20 278 | ego: 279 | min_x: -1000 #-60 280 | max_x: 1000 #-10 281 | min_y: -1000 #-10 282 | max_y: 1000 #10 283 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # CARLA-GeAR 2 | This is the code repository for the data generation of our paper "[CARLA-GeAR: A Dataset Generator for Systematic Adversarial Robustness Evaluation of Vision Models](https://arxiv.org/abs/2206.04365)". More info can be found at the main project page: https://carlagear.retis.santannapisa.it/. 3 | 4 | This code requires a working installation of CARLA to run. If you cannot/don't want to install the CARLA simulator, please consider using the datasets we provided in the main page. 5 | 6 | ![image](https://user-images.githubusercontent.com/92364988/172151551-640ca8ec-6159-4f4d-9c96-62ba58f4248e.png) 7 | 8 | ### Citation 9 | If you found our work useful, please consider citing the paper! 10 | ``` 11 | @ARTICLE{10576049, 12 | author={Nesti, Federico and Rossolini, Giulio and D’Amico, Gianluca and Biondi, Alessandro and Buttazzo, Giorgio}, 13 | journal={IEEE Transactions on Intelligent Transportation Systems}, 14 | title={CARLA-GeAR: A Dataset Generator for a Systematic Evaluation of Adversarial Robustness of Deep Learning Vision Models}, 15 | year={2024}, 16 | volume={25}, 17 | number={8}, 18 | pages={9840-9851}, 19 | keywords={Robustness;Autonomous vehicles;Systematics;Object detection;Benchmark testing;Three-dimensional displays;Adversarial machine learning;Autonomous driving;Adversarial robustness;autonomous driving;CARLA simulator;adversarial defenses}, 20 | doi={10.1109/TITS.2024.3412432}} 21 | 22 | ``` 23 | 24 | ### Features 25 | This code allows dataset generation for a systematic evaluation of the adversarial robustness of custom models for 4 different tasks: **Semantic Segmentation** (SS), **2D object detection** (2DOD), **3D Stereo-camera Object Detection** (3DOD), and **Monocular Depth Estimation** (depth). The use of CARLA simulator allows photo-realistic rendering of the meshes, and full control of autonomous driving environment. Hence, it is possible to build several datasets that include surfaces on which an adversary might attach physically-realizable adversarial patches to change the network prediction. 26 | 27 | The adversarial surfaces come in two different "shapes": billboards and a truck. The billboards are placed in fixed (customizable) positions in the selected city (suggested Town10HD), while the truck is spawned in random positions, always in front of the ego vehicle. It is possible to import the patches in CARLA, so that they are rendered realistically. 28 | 29 | 30 | 31 | ### Installation 32 | The code requires a running installation of CARLA. Since different versions might cause unexpected errors, we suggest to use the same setup we selected: 33 | - Ubuntu 18.04 34 | - CARLA 0.9.13 35 | - Python3.6 36 | Please refer to the step-by-step CARLA [installation guide](https://carla.readthedocs.io/en/latest/build_linux/) to install it smoothly. 37 | After a successful installation of CARLA, create a virtualenv and install the Python API following the instructions. 38 | You can then clone the repo in the parent folder where carla is installed. 39 | Then, install the requirements: `pip install -r requirements.txt` using the same venv. 40 | 41 | To allow easier billboard spawning and management we had to change a few of the original blueprints of CARLA. In particular, you will have to replace the `vehicle.audi.a2` (`carla/Unreal/CarlaUE4/Content/Carla/Blueprints/Vehicles/AudiA2/BP_AudiA2.uasset`) and `vehicle.ford.ambulance` (`carla/Unreal/CarlaUE4/Content/Carla/Blueprints/Vehicles/Ambulance/BP_Ambulance.uasset`) with the corresponding ones we provide in the `assets` folder. 42 | Also, copy the files `assets/BP_Decal_Billboard_Auto.uasset` and `assets/Mat_patch.uasset` in `carla/Unreal/CarlaUE4/Content` and create a new folder `carla/Unreal/CarlaUE4/Content/patches/patch_auto/`. 43 | 44 | ### Running the code 45 | To run the generation code, you will need to open the simulator (CarlaUE4.uproject), and run the simulation. 46 | After this you can run the script that will collect a split of a dataset. 47 | `python collect_dataset.py --config ` 48 | The config file is a `.yml` file (we provide examples for each task in the `config` folder), where it's possible to specify the task, the dataset save path, the billboard that must be placed in the city, and the patch that must be loaded (if any). 49 | 50 | ### Dataset format 51 | Each task follows its own data format. For a smoother integration with already-working code, we decided to use the same data-format and folder structure of well-estabilished benchmarks. SS follows the CityScapes dataset format, 2DOD follows the COCO format, 3DOD and depth follow the corresponding Kitti format. 52 | 53 | The saved dataset includes additional information on the cameras used, on the position of the camera and the billboards at each frame. This information might be used to perform digital patch placement during the optimization of patch-based attacks. Please check Patch Generation for further details on this. 54 | 55 | ### Billboard positioning 56 | The dataset generation tool can be used with different objectives: 57 | - Generic, fine-tuning datasets. This can be done by choosing the "no_billboard" option in the "billboard" field of the config file. The car will be spawned randomly in the city. 58 | - Attack datasets. 59 | 60 | In the latter case, there are a few options. If you want to generate a dataset for a specific billboard, select "billboardXX". This will use the spawn info written in the json attack scenario library file. Otherwise, the "truck" option is available. In this scenario, the car is placed behind an "adversarial truck" with a patch on its rear end. 61 | 62 | Please note that in the yml you can select any city you want. However, since Town10HD is the most realistic one, we only provided pre-computed billboards positions only for that city. Different cities have not-so-realistic meshes that will make the networks perform poorly. 63 | 64 | ### Dataset Evaluation and Patch Generation 65 | This repo includes only the data generation tool. If you want to perform patch-based attacks, or evaluate the performance of a network or a defense on any of these datasets, please check our tool at this repo: https://github.com/retis-ai/PatchAttackTool. 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | -------------------------------------------------------------------------------- /utils/sstags_carla_to_cityscapes.py: -------------------------------------------------------------------------------- 1 | """ 2 | Script to map the ground truth semantic segmentation tags returned by Carla to tags from the CityScapes dataset. 3 | Usage: python sstags_carla_to_cityscapes.py --in path/to/input/folder --out path/to/output/folder 4 | 5 | Author: Saasha Nair 6 | """ 7 | 8 | import os 9 | import sys 10 | 11 | import argparse 12 | import imageio 13 | import glob 14 | import numpy as np 15 | np.set_printoptions(threshold=sys.maxsize) 16 | 17 | 18 | CITYSCAPES_LABEL_TO_COLOR_AND_ID = { 19 | 'unlabeled' : [[ 0, 0, 0], 0 ], 20 | 'ego vehicle' : [[ 0, 0, 0], 0 ], # original = 1 21 | 'rectification border' : [[ 0, 0, 0], 0 ], # original = 2 22 | 'out of roi' : [[ 0, 0, 0], 0 ], # original = 3 23 | 'static' : [[ 0, 0, 0], 0 ], # original = 4 24 | 'dynamic' : [[111, 74, 0], 5 ], 25 | 'ground' : [[ 81, 0, 81], 6 ], 26 | 'road' : [[128, 64,128], 7 ], 27 | 'sidewalk' : [[244, 35,232], 8 ], 28 | 'parking' : [[250,170,160], 9 ], 29 | 'rail track' : [[230,150,140], 10], 30 | 'building' : [[ 70, 70, 70], 11], 31 | 'wall' : [[102,102,156], 12], 32 | 'fence' : [[190,153,153], 13], 33 | 'guard rail' : [[180,165,180], 14], 34 | 'bridge' : [[150,100,100], 15], 35 | 'tunnel' : [[150,120, 90], 16], 36 | 'pole' : [[153,153,153], 17], 37 | 'polegroup' : [[153,153,153], 17], # original = 18 38 | 'traffic light' : [[250,170, 30], 19], 39 | 'traffic sign' : [[220,220, 0], 20], 40 | 'vegetation' : [[107,142, 35], 21], 41 | 'terrain' : [[152,251,152], 22], 42 | 'sky' : [[ 70,130,180], 23], 43 | 'person' : [[220, 20, 60], 24], 44 | 'rider' : [[255, 0, 0], 25], 45 | 'car' : [[ 0, 0,142], 26], 46 | 'truck' : [[ 0, 0, 70], 27], 47 | 'bus' : [[ 0, 60,100], 28], 48 | 'caravan' : [[ 0, 0, 90], 29], 49 | 'trailer' : [[ 0, 0,110], 30], 50 | 'train' : [[ 0, 80,100], 31], 51 | 'motorcycle' : [[ 0, 0,230], 32], 52 | 'bicycle' : [[119, 11, 32], 33], 53 | 'license plate' : [[ 0, 0,142], 26 ], # original = -1 54 | } 55 | 56 | 57 | CARLA_COLOR_CITYSCAPES_LABEL = { 58 | (0, 0, 0): 'unlabeled', # unlabeled 59 | (70, 70, 70): 'building', # building 60 | (100, 40,40): 'fence', # fence 61 | (55, 90, 80): 'unlabeled', # other 62 | (220, 20, 60): 'person', # pedestrians 63 | (153, 153, 153): 'pole', # pole 64 | (157, 234, 50): 'road', # roadline 65 | (128, 64, 128): 'road', # road 66 | (244, 35, 232): 'sidewalk', # sidewalk 67 | (107, 142, 35): 'vegetation', # vegetation 68 | (0, 0, 142): 'car', # vehicles 69 | (102, 102, 156): 'wall', # wall 70 | (220, 220, 0): 'traffic sign', # trafficsign 71 | (70, 130, 180): 'sky', # sky 72 | (81, 0, 81): 'ground', # ground 73 | (150, 100, 100): 'bridge', # bridge 74 | (230, 150, 140): 'rail track', #railtrack 75 | (180, 165, 180): 'guard rail', # guardrail 76 | (250, 170, 30): 'traffic light', # trafficlight 77 | (110, 190, 160): 'static', # static 78 | (170, 120, 50): 'dynamic', # dynamic 79 | (45, 60, 150): 'unlabeled', # water 80 | (145, 170, 100): 'terrain', # terrain 81 | } 82 | 83 | def convert_single_image_to_cityscapes(image_to_map): 84 | """ 85 | Function that creates a color image of Cityscapes tags and grayscale image of cityscapes labelIds for a given color image of Carla tags 86 | 87 | Parameters 88 | --- 89 | image_to_map: imageio.core.util.Array 90 | RGB image obtained from Carla's Semantic Segmentation Camera 91 | 92 | Returns 93 | --- 94 | Two numpy arrays, namely cityscapes_color_image and cityscapes_label_image. 95 | cityscapes_label_image has the same shape as image_to_map and contains the RGB equivalents from CityScapes color scheme. 96 | cityscapes_label_image has the same width and height as image_to_map but only 1-channel (i.e. is grayscale) and contains the labelIds from CityScapes color scheme for each of the pixels. 97 | """ 98 | cityscapes_color_image = np.zeros(shape=image_to_map.shape).astype(np.uint8) # RGB with cityscapes color scheme 99 | cityscapes_label_image = np.zeros(shape=image_to_map.shape[:2]).astype(np.uint8) # Grayscale image (1-channel) to store cityscapes labelIds 100 | 101 | for color, label in CARLA_COLOR_CITYSCAPES_LABEL.items(): 102 | cityscapes_color, cityscapes_id = CITYSCAPES_LABEL_TO_COLOR_AND_ID[label] 103 | indices_to_replace = np.where((image_to_map==list(color)).all(axis=2)) 104 | cityscapes_color_image[indices_to_replace] = cityscapes_color 105 | cityscapes_label_image[indices_to_replace] = cityscapes_id 106 | 107 | return cityscapes_color_image, cityscapes_label_image 108 | 109 | def convert_all_images_to_cityscapes(in_basepath, out_basepath): 110 | """ 111 | Function that reads all the images the given input path and applies the Carla --> CityScapes color scheme remapping 112 | 113 | Parameters 114 | --- 115 | in_basepath: str 116 | path to the folder where the input images (i.e., images collected via Carla's Semantic Segmentation Camera) are stored 117 | 118 | out_basepath: str 119 | path to the folder where the output color and grayscale (i.e. labelId) images obtained from the remapping are to be stored 120 | 121 | Returns 122 | --- 123 | None 124 | """ 125 | path_to_images_to_map = glob.glob('{}/*.png'.format(in_basepath)) 126 | 127 | for image_path in path_to_images_to_map: 128 | image_name = os.path.basename(image_path)[:-4] 129 | # image_name = image_path.split('_')[-1][:-4] # discard the info about the path and the trailing '.png' 130 | #image_name = image_path.split('/')[-1][:-4] 131 | print('Mapping: {}.png'.format(image_name)) 132 | 133 | image = imageio.imread(image_path) 134 | image = image[:, :, :3] # convert from Carla's RGBA format to RGB format 135 | 136 | cityscapes_color_image, cityscapes_label_image = convert_single_image_to_cityscapes(image) 137 | 138 | imageio.imwrite('{}/{}gtFine_color.png'.format(out_basepath, image_name), cityscapes_color_image) # output filename format for RGB = aachen_000053_000019_gtFine_color.png 139 | imageio.imwrite('{}/{}gtFine_labelIds.png'.format(out_basepath, image_name), cityscapes_label_image) # output filename format for LabelID = aachen_000053_000019_gtFine_labelIds.png 140 | 141 | if __name__ == '__main__': 142 | parser = argparse.ArgumentParser() 143 | parser.add_argument('--in-folder', type=str, help='path to the folder containing images with Carla\'s SS tags') 144 | parser.add_argument('--out-folder', type=str, help='path to the folder where the remapped images are to be saved') 145 | args = parser.parse_args() 146 | 147 | os.makedirs(args.out_folder, exist_ok=True) # if output folder not found, create it 148 | 149 | convert_all_images_to_cityscapes(args.in_folder, args.out_folder) 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | 164 | 165 | 166 | 167 | 168 | -------------------------------------------------------------------------------- /utils/npc.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import random 3 | import carla 4 | 5 | class NPC: 6 | def __init__(self, world, verbose=False): 7 | self.world = world 8 | self.min_x = 0 9 | self.max_x = 0 10 | self.min_y = 0 11 | self.max_y = 0 12 | 13 | self.vehicle_npcs = [] 14 | self.pedestrian_npcs = [] 15 | self.walker_controllers = [] 16 | self.verbose = verbose 17 | 18 | def set_spawn_area(self, min_x, max_x, min_y, max_y): 19 | self.min_x = min_x 20 | self.max_x = max_x 21 | self.min_y = min_y 22 | self.max_y = max_y 23 | 24 | def setup_vehicle_npcs(self, num_actors, vehicle_blueprints, client): 25 | 26 | waypoints = [] 27 | for idx in range(num_actors): 28 | #if isinstance(self.min_x, tuple) and isinstance(self.max_x, tuple): 29 | # x_range = random.choice([self.min_x, self.max_x]) 30 | # x = np.random.uniform(x_range[0], x_range[1], size=(1, ))[0] 31 | #else: 32 | x = np.random.uniform(self.min_x, self.max_x, size=(1,))[0] 33 | 34 | #if isinstance(self.min_y, tuple) and isinstance(self.max_y, tuple): 35 | # y_range = random.choice([self.min_y, self.max_y]) 36 | # y = np.random.uniform(y_range[0], y_range[1], size=(1, ))[0] 37 | #else: 38 | y = np.random.uniform(self.min_y, self.max_y, size=(1,))[0] 39 | 40 | 41 | #location = random.choice(self.world.get_map().get_spawn_points()) 42 | location = carla.Location(x=x, y=y, z=2.0) 43 | # print(location) 44 | wp = self.world.get_map().get_waypoint(location=location, project_to_road=True, lane_type=carla.LaneType.Driving) # | carla.LaneType.Shoulder) 45 | 46 | if wp is not None: 47 | # print("Changing z") 48 | # print(wp.transform.location.z) 49 | wp.transform.location.z = 2.0 50 | # print(wp.transform.location.z) 51 | waypoints.append(wp) 52 | 53 | num_actors = min(num_actors, len(waypoints)) 54 | bps = np.random.choice(vehicle_blueprints, size=num_actors, replace=True) 55 | 56 | for bp, wp in zip(bps, waypoints): 57 | # wp.transform = 58 | #npc = self.world.try_spawn_actor(bp, wp.transform) 59 | transf = carla.Transform(location=carla.Location(x=wp.transform.location.x, y=wp.transform.location.y, z=0.05), rotation=carla.Rotation(roll=wp.transform.rotation.roll, pitch=wp.transform.rotation.pitch, yaw=wp.transform.rotation.yaw)) 60 | npc = self.world.try_spawn_actor(bp, transf) 61 | # print("Trying to spawn car") 62 | # print(wp.transform) 63 | if npc is not None: 64 | self.vehicle_npcs.append(npc) 65 | if self.verbose: 66 | print("Spawned NPC car in (%.2f, %.2f, %.2f)" % (transf.location.x, transf.location.y, transf.location.z)) 67 | 68 | # Moved in a different function 69 | # [vehicle.set_autopilot(enabled=True) for vehicle in self.vehicle_npcs] 70 | # self.world.tick() 71 | if self.verbose: 72 | print('created {} npc vehicles.'.format(len(self.vehicle_npcs))) 73 | 74 | 75 | def setup_pedestrian_npcs(self, num_actors): 76 | #walker_bps = np.random.choice(self.world.get_blueprint_library().filter('walker.pedestrian.*'), size=num_actors, replace=True) 77 | controller_bp = self.world.get_blueprint_library().find('controller.ai.walker') 78 | 79 | self.world.set_pedestrians_cross_factor(0.3) 80 | 81 | for idx in range(num_actors): 82 | bp = random.choice(self.world.get_blueprint_library().filter('walker.pedestrian.*')) 83 | if bp.has_attribute('is_invincible'): 84 | bp.set_attribute('is_invincible', 'false') 85 | #location = carla.Location(x=x, y=y, z=2.0) 86 | # if isinstance(self.min_x, tuple) and isinstance(self.min_y, tuple): 87 | location = self.world.get_random_location_from_navigation() ## only to be used for finetuning_random!! 88 | # else: 89 | # x = np.random.uniform(self.min_x, self.max_x, size=(1,))[0] 90 | # y = np.random.uniform(self.min_y, self.max_y, size=(1,))[0] 91 | # location = carla.Location(x=x, y=y, z=2.0) 92 | # wp = self.world.get_map().get_waypoint(location=location, project_to_road=True, lane_type=carla.LaneType.Sidewalk | carla.LaneType.Shoulder) #carla.LaneType.Any) 93 | #transform = carla.Transform(location=wp.transform.location, rotation=carla.Rotation()) 94 | #transform = carla.Transform(location=location, rotation=carla.Rotation()) 95 | 96 | npc = None 97 | transf = carla.Transform(location, carla.Rotation(yaw=np.random.uniform(-90, 90))) 98 | # if wp is not None: 99 | # transf = wp.transform 100 | npc = self.world.try_spawn_actor(bp, transf) # waypoint.transform) 101 | 102 | #npc = self.world.try_spawn_actor(bp, transform) # waypoint.transform) 103 | 104 | if npc is not None: 105 | controller = self.world.spawn_actor(controller_bp, carla.Transform(), attach_to=npc) 106 | self.pedestrian_npcs.append(npc) 107 | self.walker_controllers.append(controller) 108 | if self.verbose: 109 | print("Spawned NPC walker in (%.2f, %.2f, %.2f)" % (transf.location.x, transf.location.y, transf.location.z)) 110 | 111 | self.world.tick() 112 | 113 | # Moved in another function 114 | ''' 115 | for controller in self.walker_controllers: 116 | controller.start() 117 | controller.go_to_location(self.world.get_random_location_from_navigation()) 118 | controller.set_max_speed(1 + random.random()) # Between 1 and 2 m/s (default is 1.4 m/s). 119 | ''' 120 | # self.world.tick() 121 | if self.verbose: 122 | print('created {} walkers.'.format(len(self.pedestrian_npcs))) 123 | 124 | def start_controller(self): 125 | for controller in self.walker_controllers: 126 | controller.start() 127 | controller.go_to_location(self.world.get_random_location_from_navigation()) 128 | controller.set_max_speed(1 + random.random()) # Between 1 and 2 m/s (default is 1.4 m/s). 129 | 130 | [vehicle.set_autopilot(enabled=True) for vehicle in self.vehicle_npcs] 131 | for i in range(5): 132 | self.world.tick() 133 | 134 | def cleanup(self, client): 135 | client.apply_batch([carla.command.DestroyActor(vehicle) for vehicle in self.vehicle_npcs]) 136 | if self.verbose: 137 | print('destroyed all vehicle npcs.') 138 | 139 | [controller.stop() for controller in self.walker_controllers] 140 | client.apply_batch([carla.command.DestroyActor(controller) for controller in self.walker_controllers]) 141 | if self.verbose: 142 | print('destroyed all walker controllers.') 143 | 144 | client.apply_batch([carla.command.DestroyActor(pedestrian) for pedestrian in self.pedestrian_npcs]) 145 | if self.verbose: 146 | print('destroyed all pedestrians.') 147 | 148 | self.vehicle_npcs = [] 149 | self.pedestrian_npcs = [] 150 | self.walker_controllers = [] 151 | 152 | 153 | -------------------------------------------------------------------------------- /utils/bbox_utils.py: -------------------------------------------------------------------------------- 1 | import carla 2 | import numpy as np 3 | import matplotlib.pyplot as plt 4 | import datetime 5 | 6 | # Utils for debug drawing of bboxes 7 | CONNECTIONS = [ 8 | [1, 2], 9 | [2, 4], 10 | [4, 3], 11 | [1, 3], 12 | [5, 6], 13 | [6, 8], 14 | [8, 7], 15 | [7, 5], 16 | [1, 5], 17 | [2, 6], 18 | [3, 7], 19 | [4, 8] 20 | ] 21 | 22 | COLOR_LABELS = { 23 | 'car': [0, 0, 142/255], 24 | 'motorcycle': [0, 0., 230/255], 25 | 'truck': [0, 0., 70/255], 26 | 'bicycle': [119/255, 11/255, 32/255], 27 | 'pedestrian': [220/255, 20/255, 60/255], 28 | 'traffic_light': [250/255, 170/255, 30/255], 29 | 'stop_sign': [220/255, 220/255, 0] 30 | } 31 | 32 | # CAR_NAMES = ['vehicle'] 33 | BIKE_NAMES = ['Bike'] 34 | TRUCK_NAMES = ['Truck'] 35 | MOTORCYCLE_NAMES = ['Harley', 'Yamaha', 'Vespa', 'Kawasaki'] 36 | SIGN_NAMES = ['Stop'] 37 | 38 | 39 | 40 | 41 | 42 | class CameraParams(): 43 | def __init__(self, image_size, fov): 44 | self.transform = carla.Transform() 45 | self.image_size = image_size 46 | self.fov = np.pi * fov /180 47 | self.camera_focal = 1/2 * self.image_size[1] / np.tan(self.fov/2) 48 | cpi = np.cos(np.pi/2) 49 | spi = np.sin(np.pi/2) 50 | self.T_cam_offset = np.transpose(np.array([[cpi, 0, spi, 0], [0, 1, 0, 0], [-spi, 0, cpi, 0], [0, 0, 0, 1]]) @ 51 | np.array([[cpi, spi, 0, 0], [-spi, cpi, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]])) 52 | 53 | self.intrinsics = np.array([[self.camera_focal, 0, self.image_size[1]/2], 54 | [0, self.camera_focal, self.image_size[0]/2], 55 | [0, 0, 1]]) 56 | 57 | self.get_extrinsics(self.transform) 58 | 59 | def get_extrinsics(self, camera_transform): 60 | # print(camera_transform) 61 | self.transform = camera_transform 62 | self.camera_transform = camera_transform 63 | loc = np.array([[camera_transform.location.x], [-camera_transform.location.y], [camera_transform.location.z]]) 64 | # ori = [np.pi / 180 * camera_transform.rotation.pitch, -np.pi / 180 * camera_transform.rotation.yaw, np.pi / 180 * camera_transform.rotation.roll] 65 | yaw = -np.pi / 180 * camera_transform.rotation.yaw 66 | 67 | # Compute camera transform 68 | camera_rot_mat = np.array([[np.cos(yaw), -np.sin(yaw), 0], [np.sin(yaw), np.cos(yaw), 0], [0, 0, 1]]) 69 | T_WC_inv = np.block([[camera_rot_mat.T, -camera_rot_mat.T @ loc], [0, 0, 0, 1]]) 70 | self.extrinsics = self.T_cam_offset @ T_WC_inv 71 | # print(self.extrinsics) 72 | 73 | 74 | 75 | def get_camera_params(sensor_config): 76 | return CameraParams((sensor_config['height'], sensor_config['width']), sensor_config['fov']) 77 | 78 | 79 | # Loop on actors and environment objects and collect all bounding boxes. 80 | def get_world_bboxes(world): 81 | bboxes, transforms, labels, actors, orientations = [], [], [], [], [] 82 | 83 | car_actors = {'actors': [actor for actor in world.get_actors() if 'vehicle' in actor.type_id], 'label': 'car'} 84 | walker_actors = {'actors': [actor for actor in world.get_actors() if 'walker' in actor.type_id], 'label': 'pedestrian'} 85 | actor_list = [car_actors, walker_actors] 86 | 87 | vehicle_objects = world.get_environment_objects(carla.CityObjectLabel.Vehicles) 88 | traffic_lights = {'objects': world.get_environment_objects(carla.CityObjectLabel.TrafficLight), 'label': 'traffic_light'} 89 | traffic_signs = {'objects': [obj for obj in world.get_environment_objects(carla.CityObjectLabel.TrafficSigns) if any([name in obj.name for name in SIGN_NAMES])], 'label': 'stop_sign'} 90 | 91 | # Filter vehicles (cars, trucks, bikes, motorbikes) 92 | bike_objects = {'objects': [obj for obj in vehicle_objects if any([name in obj.name for name in BIKE_NAMES])], 'label': 'bicycle'} 93 | moto_objects = {'objects': [obj for obj in vehicle_objects if any([name in obj.name for name in MOTORCYCLE_NAMES])], 'label': 'motorcycle'} 94 | truck_objects = {'objects': [obj for obj in vehicle_objects if any([name in obj.name for name in TRUCK_NAMES])], 'label': 'truck'} 95 | car_objects = {'objects': [obj for obj in vehicle_objects if obj not in bike_objects['objects'] + moto_objects['objects'] + truck_objects['objects']], 'label': 'car'} 96 | 97 | object_list = [bike_objects, moto_objects, truck_objects, car_objects, traffic_lights, traffic_signs] 98 | for obj_dict in object_list: 99 | for obj in obj_dict['objects']: 100 | # print(obj.name) 101 | bboxes.append(obj.bounding_box) 102 | orientations.append(obj.bounding_box.rotation.yaw) 103 | transforms.append(carla.Transform()) # if bbox is extracted from env obj, they are already in world coords 104 | labels.append(obj_dict['label']) 105 | actors.append(obj) 106 | 107 | for actor_dict in actor_list: 108 | for actor in actor_dict['actors']: 109 | # print(obj.name) 110 | bboxes.append(actor.bounding_box) 111 | orientations.append(actor.get_transform().rotation.yaw) 112 | transforms.append(actor.get_transform()) # if bbox is extracted from actor, they are in actor ref frame. 113 | labels.append(actor_dict['label']) 114 | actors.append(actor) 115 | 116 | return bboxes, transforms, labels, actors, orientations 117 | 118 | 119 | # Complete bboxes computation pipeline. 120 | def compute_bboxes(data, camera_params, world): 121 | if 'DepthCam' not in data.keys(): 122 | return None, None, None, None, None, None 123 | h, w = camera_params.image_size 124 | 125 | # Get depth map and normalize 126 | depth_np = np.float32(np.frombuffer(data['DepthCam'][1].raw_data, dtype=np.uint8).reshape(h, w, 4)[:, :, :-1]) 127 | depth_norm = 1000 * (depth_np[:, :, 2] + depth_np[:, :, 1] * 256 + depth_np[:, :, 0] * 256 * 256) / (256 * 256 * 256 - 1) 128 | 129 | # Obtain 3D vertices in world 130 | bboxes, transforms, labels, actors, orientations = get_world_bboxes(world) 131 | vertices = [get_vertices_from_bbox(bbox, transform) for (bbox, transform) in zip(bboxes, transforms)] 132 | # orientations = [act.boundingtransform.rotation.yaw for act in actors] 133 | 134 | # Filter bboxes. 135 | idx_in_image = [] 136 | vert_pix = [] # vertices on the image (in pixels) 137 | vert_cam = [] # vertices in the cam ref frame. 138 | bboxes_in_image = [] 139 | orient = [] 140 | for v_idx, (vert, label, actor, bbox, orientation) in enumerate(zip(vertices, labels, actors, bboxes, orientations)): 141 | vert_candidates, v_cam = project_vertices(vert, camera_params) 142 | if filter_bbox(vert_candidates, depth_norm): 143 | # print("OK") 144 | if 'stop' in label and not is_stop_visible(actor, camera_params): 145 | continue 146 | vert_pix.append(clip_vertices(vert_candidates, camera_params)) 147 | idx_in_image.append(v_idx) 148 | vert_cam.append(v_cam) 149 | bboxes_in_image.append(bbox) 150 | orient.append(orientation) 151 | 152 | # Find 2d bounding boxes 153 | vert_2d = [convert_3d_vertices_to_2d(v) for v in vert_pix] 154 | labels = [labels[idx] for idx in idx_in_image] 155 | 156 | return bboxes_in_image, vert_pix, vert_2d, vert_cam, labels, orient 157 | 158 | 159 | 160 | def get_vertices_from_bbox(bbox, transform): 161 | vert = bbox.get_world_vertices(transform) 162 | vert = np.block([[v.x, -v.y, v.z, 1] for v in vert]) 163 | return vert 164 | 165 | 166 | # Return the positions of the 3D bbox in the image and distance from camera. 167 | def project_vertices(vert, camera_params): 168 | # Vertices in camera coords and projection on image. 169 | vert_cam = camera_params.extrinsics @ vert.T 170 | dist = vert_cam[2:3, :] 171 | vert_cam_ = vert_cam[:3] / dist 172 | vert_pix = camera_params.intrinsics @ vert_cam_ 173 | vert_pix = vert_pix[:2, :].astype(np.int) 174 | return np.concatenate((vert_pix, dist), axis=0).T, vert_cam 175 | 176 | 177 | def convert_3d_vertices_to_2d(vertices): 178 | # print(vertices) 179 | min_x, max_x, min_y, max_y = np.amin(vertices[:, 0]), np.amax(vertices[:, 0]), np.amin(vertices[:, 1]), np.amax(vertices[:, 1]) 180 | # print(min_x, max_x, min_y, max_y) 181 | return np.array([[min_x, min_y], [max_x, min_y], [min_x, max_y], [max_x, max_y]]) 182 | 183 | 184 | # To clip bboxes that are outside the image. 185 | def clip_vertices(vertices, camera_params): 186 | img_size = np.array([[camera_params.image_size[1], camera_params.image_size[0], 1000]]) 187 | return np.clip(vertices, np.zeros((1, 3)), img_size) 188 | 189 | 190 | # Consider only bboxes that are in the image, are not occluded, and are not too small. 191 | def filter_bbox(vertices, depth_map): 192 | # Compute the midpoints of the "connections" between vertices. 193 | midpoints = [(vertices[conn[0]-1, :] + vertices[conn[1]-1, :])/2 for conn in CONNECTIONS] 194 | # is_bbox_in_front = sum([is_point_in_front(mid) for mid in midpoints]) > 3 195 | is_bbox_in_image = sum([is_point_in_front(mid) and is_point_in_image(mid, depth_map.shape[:2]) and not is_point_occluded(mid, depth_map) for mid in midpoints]) > 3 196 | # is_bbox_occluded = sum([is_point_occluded(mid, depth_map) for mid in midpoints]) > 8 197 | 198 | # point_in_image = all(is_point_in_front(vertices[i, :]) and is_point_in_image(vertices[i, :], depth_map.shape[:2]) for i in range(vertices.shape[0])) and not is_bbox_small(vertices) 199 | # point_occluded = point_in_image and all(is_point_occluded(vertices[i, :], depth_map) for i in range(vertices.shape[0])) 200 | return is_bbox_in_image and not is_bbox_small(vertices) 201 | 202 | 203 | def is_stop_visible(actor, camera_params): 204 | stop_yaw = -actor.transform.rotation.yaw - 90 205 | camera_yaw = -camera_params.transform.rotation.yaw 206 | visual_angle = stop_yaw - camera_yaw 207 | visual_angle = (visual_angle + 360) % (360) 208 | # print("STOP:", stop_yaw, camera_yaw, visual_angle) 209 | return abs(visual_angle - 180) < 45 210 | 211 | 212 | # False if vertex is behind camera 213 | def is_point_in_front(point, max_dist=200): 214 | return point[2] > 0.5 and point[2] < max_dist 215 | 216 | 217 | # False if vertex is outside camera sensor 218 | def is_point_in_image(point, image_size): 219 | image_h, image_w = image_size 220 | return point[0] >= 0 and point[0] < image_w and point[1] >= 0 and point[1] < image_h 221 | 222 | 223 | # Checks if point's distance is greater than depth map in that point (and its surroundings). 224 | # Bounding box is occluded if all vertices are occluded. 225 | # TODO improve occlusion filtering: check if midpoints of connections are occluded, instead of vertices. 226 | def is_point_occluded(point, depth_map): 227 | neighbors = [[1, 1], [1, 0], [1, -1], [0, -1], [-1, -1], [-1, 0], [-1, 1], [0, 1], [0, 0]] 228 | occluded = [] 229 | 230 | point_dist = point[-1] 231 | for n in neighbors: 232 | point_n = (point[:2] + np.array(n)).astype(np.int) 233 | occluded.append(False) 234 | if is_point_in_image(point_n, depth_map.shape[:2]) and point_dist > depth_map[point_n[1], point_n[0]]: 235 | # print(point_dist, depth_map[point_n[1], point_n[0]]) 236 | occluded[-1] = True 237 | 238 | # print(occluded) 239 | return sum(occluded) > 4 240 | 241 | 242 | # Checks if bbox has small area. 243 | def is_bbox_small(vertices, min_area=900): 244 | w, h = np.amax(vertices[:, 0]) - np.amin(vertices[:, 0]), np.amax(vertices[:, 1]) - np.amin(vertices[:, 1]) 245 | return w * h < min_area 246 | 247 | # Saves image with bounding boxes (debug) 248 | def save_image_bboxes(filename, image, vertices, labels): 249 | plt.figure() 250 | plt.imshow(image) 251 | if len(vertices) > 0: 252 | is_2d = (vertices[0].shape[0] < 8) 253 | vert_connections = CONNECTIONS 254 | if is_2d: 255 | vert_connections = CONNECTIONS[:4] 256 | 257 | for v_idx, v in enumerate(vertices): 258 | for conn in vert_connections: 259 | plt.plot([v[conn[0]-1, 0], v[conn[1]-1, 0]], [v[conn[0]-1, 1], v[conn[1]-1, 1]], c=COLOR_LABELS[labels[v_idx]], linewidth=0.5) 260 | plt.text(np.amin(v[:, 0]), np.amin(v[:, 1]), labels[v_idx], fontsize='xx-small', c=COLOR_LABELS[labels[v_idx]]) 261 | plt.xticks([]) 262 | plt.yticks([]) 263 | plt.tight_layout() 264 | plt.savefig(filename, dpi=300) 265 | print("File %s saved." % filename) 266 | plt.close() -------------------------------------------------------------------------------- /collect_dataset.py: -------------------------------------------------------------------------------- 1 | import yaml 2 | import argparse 3 | import os 4 | 5 | import random 6 | import numpy as np 7 | import time 8 | 9 | import carla 10 | import json 11 | import subprocess 12 | import warnings 13 | 14 | from utils.datasets_utils import set_dataset_paths, DataSaver, json_billboard_position 15 | from utils.bbox_utils import get_camera_params, compute_bboxes 16 | from utils.carla_utils import carla_setup, cleanup_env 17 | from utils.ego_vehicle import EgoVehicle 18 | from utils.npc import NPC 19 | 20 | 21 | def collect_dataset(task, 22 | dataset_split, 23 | billboard_name, 24 | basepath, 25 | root_path, 26 | patch_type, 27 | net, 28 | patch_folder, 29 | town_name, 30 | debug): 31 | from configs.simulation_configs import SEED_VALUES, ITERATION_VALUES, get_spawn_bp, BRANDS, MAX_VEHICLE_NPC, MAX_WALKER_NPC, patch_load_path 32 | # Set seeds and num_images. 33 | split_v = dataset_split.split('_')[0] 34 | SEED = SEED_VALUES[split_v] 35 | random.seed(SEED) 36 | np.random.seed(SEED) 37 | 38 | ITERATIONS = ITERATION_VALUES[split_v] 39 | STEP_ITERATIONS = 105 if split_v == 'video' else 1 40 | 41 | # TODO include additional info. 42 | debug_str = '=======================\nDATASET GENERATION INFO\nMain Folder: %s\nPatch Folder: %s\nTown Name: %s\n==========================' 43 | print(debug_str % (os.path.join(basepath, root_path), patch_folder, town_name)) 44 | 45 | # Set image paths. dataset_paths is a dict with specific paths for each task. 46 | dataset_paths, sensors_config_file = set_dataset_paths(task, root_path, billboard_name, dataset_split) 47 | # leftImg8bit_path = os.path.join(root_path, billboard_name, 'leftImg8bit', dataset_split, dataset_split) 48 | # gtFine_path = os.path.join(root_path, billboard_name, 'gtFine', dataset_split, dataset_split) 49 | ds = DataSaver(task, basepath, dataset_paths) 50 | 51 | # The billboard config is dependent on town 52 | with open('configs/billboards_config_%s.yml' % town_name, 'r') as f: 53 | billboard_data = yaml.load(f)[billboard_name] 54 | 55 | with open(os.path.join('configs/sensors/', sensors_config_file), "r") as f: 56 | sensors_config = json.load(f)['sensors_config'] 57 | 58 | # Carla setup. 59 | client, world, settings = carla_setup(town_name, SEED, debug=debug) 60 | 61 | # Check number of patches and number of billboards 62 | num_billboards = len(billboard_data) 63 | if 'no_billboard' in billboard_name: 64 | num_billboards = 0 65 | 66 | num_patches = 0 67 | 68 | if patch_folder is not None: 69 | patch_files = [os.path.join(patch_folder, p_file) for p_file in os.listdir(patch_folder) if 'png' in p_file] 70 | num_patches = len(patch_files) 71 | if num_patches > 2: 72 | raise Exception('A maximum of 2 patches are supported for multi-patch') 73 | elif num_patches > 1: 74 | if '_0.png' not in patch_files[0]: 75 | patch_files = patch_files[::-1] 76 | if '_0.png' not in patch_files[0]: 77 | raise Exception('NO PATCH HAS 0 NUMBERING. Patches must be named ..._0.png and _1.png') 78 | if num_patches > num_billboards: 79 | warnings.warn('Number of billboards (%d) and number of patches (%d) does not match. Loading only allowed patches.' % (num_billboards, num_patches)) 80 | num_patches = num_billboards 81 | 82 | 83 | # Find blueprint for patch spawning 84 | billboard_bp = get_spawn_bp(billboard_name) 85 | if billboard_bp is not None: 86 | spawn_billboard_bp = world.get_blueprint_library().filter(billboard_bp)[0] 87 | # print(spawn_billboard_bp) 88 | 89 | billboards_transform = [] 90 | 91 | # Spawn billboards and corresponding patches 92 | truck_actor = None 93 | if billboard_bp is not None: 94 | for i in range(num_billboards): 95 | os.system('rm %s' % patch_load_path) 96 | if i < num_patches: 97 | os.system('cp %s %s' % (patch_files[i], patch_load_path)) 98 | if 'truck' not in billboard_name: 99 | 100 | bb_loc = billboard_data['b%d' % (i+1)]['location'] 101 | bb_rot = billboard_data['b%d' % (i+1)]['rotation'] 102 | 103 | loc = carla.Location(bb_loc['x'], bb_loc['y'], bb_loc['z']) 104 | rot = carla.Rotation(bb_rot['pitch'], bb_rot['yaw'], bb_rot['roll']) 105 | tr = carla.Transform(loc, rot) 106 | 107 | spawned_car = world.spawn_actor(spawn_billboard_bp, tr) 108 | 109 | world.tick() 110 | succ = spawned_car.destroy() 111 | 112 | billboards_transform.append(tr) 113 | 114 | else: 115 | truck_spawn_point = random.choice(world.get_map().get_spawn_points()) 116 | truck_actor = world.spawn_actor(spawn_billboard_bp, truck_spawn_point) 117 | 118 | spawn_config = billboard_data[list(billboard_data.keys())[0]]['spawn_config'] 119 | # Absolute positioning 120 | ego_min_x = spawn_config['ego']['min_x'] 121 | ego_max_x = spawn_config['ego']['max_x'] 122 | ego_min_y = spawn_config['ego']['min_y'] 123 | ego_max_y = spawn_config['ego']['max_y'] 124 | min_x = spawn_config['npc']['min_x'] 125 | max_x = spawn_config['npc']['max_x'] 126 | min_y = spawn_config['npc']['min_y'] 127 | max_y = spawn_config['npc']['max_y'] 128 | if billboard_data['b1']['location']['x'] is not None: 129 | x, y, z = billboards_transform[0].location.x, billboards_transform[0].location.y, billboards_transform[0].location.z 130 | # Relative position wrt billboard01 131 | ego_min_x = x + spawn_config['ego']['min_x'] 132 | ego_max_x = x + spawn_config['ego']['max_x'] 133 | ego_min_y = y + spawn_config['ego']['min_y'] 134 | ego_max_y = y + spawn_config['ego']['max_y'] 135 | min_x = x + spawn_config['npc']['min_x'] 136 | max_x = x + spawn_config['npc']['max_x'] 137 | min_y = y + spawn_config['npc']['min_y'] 138 | max_y = y + spawn_config['npc']['max_y'] 139 | 140 | if 'no_billboard' in billboard_name or 'truck' in billboard_name: 141 | MAX_VEHICLE_NPC *= 100 142 | # MAX_WALKER_NPC *= 100 143 | 144 | vehicle_blueprints = [] 145 | for brand in BRANDS: 146 | vehicle_blueprints.extend([bp for bp in world.get_blueprint_library().filter('vehicle.%s.*' % brand)]) 147 | 148 | print('{} blueprints found for 4-wheeled vehicles.'.format(len(vehicle_blueprints))) #, vehicle_blueprints) 149 | 150 | camera_params = get_camera_params(sensors_config[0]) 151 | camera_params_dx = get_camera_params(sensors_config[0]) 152 | if sensors_config[1]['id'] == 'RGBCamDx': 153 | camera_params_dx = get_camera_params(sensors_config[1]) 154 | 155 | # Meta information 156 | info = { 157 | 'task': task, 158 | 'num_billboards': num_billboards, 159 | 'is_patched': patch_folder is not None, 160 | 'patch_folder': patch_folder, 161 | 'patch_type': patch_type, 162 | 'net': net, 163 | 'creator': 'Federico Nesti', # put your name here! 164 | } 165 | 166 | ego = EgoVehicle(world, sensors_config, verbose=debug) 167 | npc = NPC(world, verbose=debug) 168 | 169 | camera_billboard_positions = [] 170 | if debug: 171 | spectator = world.get_spectator() 172 | times = [] 173 | 174 | try: 175 | for cnt in range(ITERATIONS): 176 | print('Iteration no. %d / %d: ' % (cnt + 1, ITERATIONS)) 177 | 178 | if debug: 179 | start_time = time.time() 180 | ego_spawn_point = random.choice(world.get_map().get_spawn_points()) 181 | 182 | ego_random_point = carla.Location(x=np.random.uniform(ego_min_x, ego_max_x), y=np.random.uniform(ego_min_y, ego_max_y), z=1) 183 | if 'truck' in billboard_name: 184 | spawn_err_count = 0 185 | spawned = False 186 | # CHECK CAR SPAWN 187 | while not spawned and spawn_err_count < 5: 188 | try: 189 | ego_spawn_point = world.get_map().get_waypoint(location=ego_random_point, project_to_road=True, lane_type=carla.LaneType.Driving).transform 190 | truck_distance = np.random.uniform(3, 30) 191 | truck_transf = random.choice(world.get_map().get_waypoint(location=ego_spawn_point.location, project_to_road=True, lane_type=carla.LaneType.Driving).next(truck_distance)).transform 192 | truck_actor.set_transform(truck_transf) 193 | billboards_transform = [truck_transf] 194 | ego.setup_vehicle(vehicle_blueprints, ego_spawn_point) 195 | except: 196 | ego_random_point = carla.Location(x=np.random.uniform(ego_min_x, ego_max_x), y=np.random.uniform(ego_min_y, ego_max_y), z=1) 197 | spawn_err_count += 1 198 | else: 199 | spawned = True 200 | 201 | elif 'no_billboard' not in billboard_name: 202 | 203 | ego_spawn_point = world.get_map().get_waypoint(location=ego_random_point, project_to_road=True, lane_type=carla.LaneType.Driving).transform 204 | 205 | # Spawn ego, then NPCs. 206 | if 'truck' not in billboard_name: 207 | ego.setup_vehicle(vehicle_blueprints, ego_spawn_point) 208 | 209 | if debug: 210 | spectator.set_transform(carla.Transform(carla.Location(z=2)+ego_spawn_point.location, ego_spawn_point.rotation)) 211 | 212 | npc.set_spawn_area(min_x=min_x, max_x=max_x, min_y=min_y, max_y=max_y) 213 | npc.setup_vehicle_npcs(num_actors=random.randint(0, MAX_VEHICLE_NPC), vehicle_blueprints=vehicle_blueprints, client=client) ## standard range (non-fintuning_random): 0 to 50 214 | npc.setup_pedestrian_npcs(num_actors=random.randint(0, MAX_WALKER_NPC)) ## standard range (non-fintuning_random): 0 to 20 215 | 216 | # Start autopilot now that everyone has been spawned. 217 | npc.start_controller() 218 | 219 | # Ego vehicle cameras setup 220 | ego.setup_sensors() 221 | ego.wait_for_sensors() 222 | 223 | 224 | for idx in range(STEP_ITERATIONS): 225 | world.tick() 226 | data = ego.get_sensor_data() 227 | 228 | # For stereo applications, sensors.json must have rgb at 1st place, rgb_dx at 2nd. 229 | camera_transform = ego._sensors_list[0].get_transform() 230 | camera_params.get_extrinsics(camera_transform) 231 | camera_transform_dx = ego._sensors_list[1].get_transform() 232 | camera_params_dx.get_extrinsics(camera_transform_dx) 233 | camera_params_list = [camera_params, camera_params_dx] 234 | 235 | bboxes_in_image, vert_pix, vert_2d, vert_cam, labels, orientations = compute_bboxes(data, camera_params, world) 236 | 237 | # Save Images 238 | ds.save_image(data) 239 | 240 | # Accumulate billboard positions. 241 | camera_billboard_positions.append(json_billboard_position(billboards_transform, [camera_transform, camera_transform_dx], ds.img_count)) 242 | 243 | ds.save_annotations(vert_2d, vert_cam, bboxes_in_image, labels, camera_params_list, split, orientations) 244 | 245 | 246 | 247 | cleanup_env(ego=ego, npc=npc, client=client) 248 | 249 | if debug: 250 | times.append(time.time() - start_time) 251 | 252 | finally: 253 | # Save camera and billboard positions. 254 | ds.save_camera_positions(info, camera_billboard_positions, camera_params_list) 255 | # Clean up Carla world! 256 | world.apply_settings(settings) 257 | cleanup_env(ego=ego, npc=npc, client=client) 258 | if truck_actor is not None: 259 | truck_actor.destroy() 260 | 261 | if debug: 262 | print("Timing analysis: mean %.2f sec/img, std %.2f" % (np.array(times).mean(), np.array(times).std())) 263 | 264 | 265 | if __name__ == '__main__': 266 | parser = argparse.ArgumentParser() 267 | parser.add_argument( 268 | "--config", 269 | nargs="?", 270 | type=str, 271 | default='configs/collection_config.yml', 272 | help="config file (yml format)", 273 | ) 274 | 275 | 276 | args = parser.parse_args() 277 | cfg_file = args.config 278 | 279 | with open(cfg_file, "r") as f: 280 | cfg = yaml.load(f) 281 | 282 | print("CFG", cfg) 283 | root = cfg['root'] 284 | basepath = cfg['basepath'] 285 | 286 | town_name = cfg['town_name'] 287 | # weather: # TODO 288 | 289 | task = cfg['task'] 290 | split = cfg['split'] 291 | billboard = cfg['billboard'] 292 | patch_folder = cfg['patch_folder'] 293 | 294 | patch_type = cfg['patch_type'] 295 | net = cfg['net'] 296 | 297 | debug = cfg['debug'] 298 | 299 | if task not in ('ss', '2dod', '3dod', 'depth'): 300 | raise Exception('Task specified is not supported. Choose one between ss, 2dod, 3dod, depth') 301 | 302 | 303 | collect_dataset(task=task, dataset_split=split, 304 | billboard_name=billboard, 305 | basepath=basepath, 306 | root_path=root, 307 | patch_type=patch_type, 308 | net=net, 309 | patch_folder=patch_folder, 310 | town_name=town_name, 311 | debug=debug) 312 | # To avoid early resets in automatic data collection. 313 | time.sleep(10) 314 | -------------------------------------------------------------------------------- /utils/datasets_utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import functools 3 | import os 4 | import imageio 5 | from .sstags_carla_to_cityscapes import convert_single_image_to_cityscapes 6 | # from .bbox_utils import get_world_bboxes, get_vertices_from_bbox, project_vertices, filter_bbox, is_stop_visible, clip_vertices 7 | import carla 8 | import datetime 9 | import json 10 | 11 | 12 | 13 | class DataSaver(): 14 | ''' 15 | This class takes the data and saves it according to the data structure corresponding to the task. 16 | ''' 17 | 18 | def __init__(self, task, basepath, dataset_paths): 19 | self.task = task 20 | self.dataset_paths = dataset_paths 21 | key2savedata = { 22 | 'ss': self._save_ss_data, 23 | '3dod': self._save_3dod_data, 24 | '2dod': self._save_2dod_data, 25 | 'depth': self._save_depth_data 26 | } 27 | 28 | key2saveann = { 29 | 'ss': self._save_ss_ann, 30 | '3dod': self._save_3dod_ann, 31 | '2dod': self._save_2dod_ann, 32 | 'depth': self._save_depth_ann 33 | } 34 | 35 | 36 | self.save_image = key2savedata[self.task] 37 | self.save_annotations = key2saveann[self.task] 38 | self.basepath = basepath 39 | 40 | self.img_count = 0 41 | self.obj_count = 0 42 | 43 | # COCO stuff 44 | self.annotations = [] 45 | self.images = [] 46 | self.info = { 47 | "year": 2022, 48 | "version": '0.0', 49 | "description": 'COCO-like dataset for CARLA images', 50 | "contributor": "Federico Nesti", 51 | "url": '', 52 | "date_created": datetime.datetime.now() 53 | } 54 | self.licenses = [] 55 | self.filelist = [] 56 | 57 | # Create dirs for image and annotations. 58 | for path in dataset_paths.values(): 59 | if not os.path.isdir(os.path.join(basepath, path)): 60 | os.makedirs(os.path.join(basepath, path)) 61 | 62 | # Task-specific images save functions 63 | def _save_ss_data(self, ss_data): 64 | rgb_filename = os.path.join(self.basepath, self.dataset_paths['images'], 'rgb_%04d.png' % self.img_count) 65 | ss_data['RGBCam'][1].save_to_disk(rgb_filename, carla.ColorConverter.Raw) 66 | ss_filename = os.path.join(self.basepath, self.dataset_paths['labels'], 'rgb_%04d.png' % self.img_count) 67 | ss_data['SSCam'][1].save_to_disk(ss_filename, carla.ColorConverter.CityScapesPalette) 68 | 69 | # convert to cityscapes Tags. 70 | image = imageio.imread(ss_filename) 71 | image = image[:, :, :3] # convert from Carla's RGBA format to RGB format 72 | 73 | cityscapes_color_image, cityscapes_label_image = convert_single_image_to_cityscapes(image) 74 | 75 | ss_color_filename = ss_filename[:-4] + '_gtFine_color.png' 76 | ss_id_filename = ss_filename[:-4] + '_gtFine_labelIds.png' 77 | imageio.imwrite(ss_color_filename, cityscapes_color_image) # output filename format for RGB = aachen_000053_000019_gtFine_color.png 78 | imageio.imwrite(ss_id_filename, cityscapes_label_image) # output filename format for LabelID = aachen_000053_000019_gtFine_labelIds.png 79 | 80 | def _save_3dod_data(self, data_3dod): 81 | rgb_filename = os.path.join(self.basepath, self.dataset_paths['images'], '%06d.png' % self.img_count) 82 | data_3dod['RGBCam'][1].save_to_disk(rgb_filename, carla.ColorConverter.Raw) 83 | rgb_dx_filename = os.path.join(self.basepath, self.dataset_paths['images_dx'], '%06d.png' % self.img_count) 84 | data_3dod['RGBCamDx'][1].save_to_disk(rgb_dx_filename, carla.ColorConverter.Raw) 85 | self.filelist.append('%06d\n' % self.img_count) 86 | 87 | def _save_2dod_data(self, data_2dod): 88 | rgb_filename = os.path.join(self.basepath, self.dataset_paths['images'], 'rgb_%04d.png' % self.img_count) 89 | data_2dod['RGBCam'][1].save_to_disk(rgb_filename, carla.ColorConverter.Raw) 90 | image = imageio.imread(rgb_filename) 91 | self.images.append(coco_format_image(self.img_count, image.shape[:2], os.path.basename(rgb_filename))) 92 | 93 | def _save_depth_data(self, depth_data): 94 | rgb_filename = os.path.join(self.basepath, self.dataset_paths['images'], 'rgb_%04d.png' % self.img_count) 95 | depth_data['RGBCam'][1].save_to_disk(rgb_filename, carla.ColorConverter.Raw) 96 | depth_filename = os.path.join(self.basepath, self.dataset_paths['labels'], 'rgb_%04d.png' % self.img_count) 97 | depth_data['DepthCam'][1].save_to_disk(depth_filename, carla.ColorConverter.Depth) 98 | filename_path = os.path.join(*self.dataset_paths['images'].split('/')[-2:]) 99 | labels_path = os.path.join(*self.dataset_paths['labels'].split('/')[-2:]) 100 | self.filelist.append(os.path.join('/', filename_path, 'rgb_%04d.png' % self.img_count) + \ 101 | ' /' + os.path.join(labels_path, 'rgb_%04d.png' % self.img_count) + '\n') 102 | 103 | 104 | # Task-specific annotation save functions 105 | def _save_ss_ann(self, vert_2d, vert_cam, bboxes_in_image, labels, camera_params_list, dataset_split, orientations): 106 | # Data already saved as image. Increment counter. 107 | self.img_count += 1 108 | 109 | def _save_3dod_ann(self, vert_2d, vert_cam, bboxes_in_image, labels, camera_params_list, dataset_split, orientations): 110 | camera_params, camera_params_dx = camera_params_list 111 | for (vert, v_cam, bbox, label, orient) in zip(vert_2d, vert_cam, bboxes_in_image, labels, orientations): 112 | self.annotations.append(kitti_format_annotation(bbox, vert, v_cam, label, camera_params.transform, orient)) 113 | # Add dummy label (to write a file) 114 | if not self.annotations: 115 | self.annotations.append('%s %.2f %d %.2f %d %d %d %d %.2f %.2f %.2f %.2f %.2f %.2f %.2f\n' % ('DontCare', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)) 116 | annotation_filename = '%06d.txt' % self.img_count 117 | with open(os.path.join(self.basepath, self.dataset_paths['labels'], annotation_filename), "w") as f: 118 | for ann in self.annotations: 119 | f.write(ann) 120 | 121 | with open(os.path.join(self.basepath, self.dataset_paths['calib'], annotation_filename), "w") as f: 122 | f.write(kitti_format_calib(camera_params, camera_params_dx)) 123 | 124 | self.obj_count += 1 125 | self.img_count += 1 126 | self.annotations = [] 127 | filelist = ''.join(self.filelist)[:-1] 128 | with open(os.path.join(self.basepath, self.dataset_paths['splits'], '%s.txt' % dataset_split), "w") as f: 129 | f.write(filelist) 130 | 131 | def _save_2dod_ann(self, vert_2d, vert_cam, bboxes_in_image, labels, camera_params_list, dataset_split, orientations): 132 | for v_idx, vert in enumerate(vert_2d): 133 | self.annotations.append(coco_format_annotation(self.obj_count, self.img_count, labels[v_idx], vert)) 134 | self.obj_count += 1 135 | 136 | with open(os.path.join(self.basepath, self.dataset_paths['labels'], 'coco_annotations.json'), 'w') as f: 137 | json_str = json.dumps({ 138 | "info": self.info, 139 | "images": self.images, 140 | "annotations": self.annotations, 141 | "licenses": self.licenses, 142 | "categories": COCO_CATEGORIES() 143 | }, default=str) 144 | f.write(json_str) 145 | 146 | self.img_count += 1 147 | 148 | def _save_depth_ann(self, vert_2d, vert_cam, bboxes_in_image, labels, camera_params_list, dataset_split, orientations): 149 | filelist = ''.join(self.filelist)[:-1] 150 | with open(os.path.join(self.basepath, self.dataset_paths['filenames'], '%s_list.txt' % dataset_split), "w") as f: 151 | f.write(filelist) 152 | self.img_count += 1 153 | 154 | 155 | 156 | def save_camera_positions(self, info, camera_billboard_positions, camera_params_list): 157 | # if info['task'] != '3dod': 158 | # for key in camera_params_list[1].keys(): 159 | # camera2_info[key] = None 160 | 161 | def camera_params_dict(camera_param): 162 | return_dict = { 163 | 'height': camera_param.image_size[0], 164 | 'width': camera_param.image_size[1], 165 | 'fov': camera_param.fov, 166 | 'focal': camera_param.camera_focal 167 | } 168 | return return_dict 169 | 170 | with open(os.path.join(self.basepath, self.dataset_paths['labels'], 'camera_billboard_info.json'), 'w') as f: 171 | json_str = json.dumps({ 172 | "info": info, 173 | "camera_billboard_positions": camera_billboard_positions, 174 | "patch_mask_corners": {}, 175 | "camera_params": camera_params_dict(camera_params_list[0]), 176 | "camera_dx_params": camera_params_dict(camera_params_list[1]) 177 | }, default=str) 178 | f.write(json_str) 179 | 180 | 181 | 182 | # Returns specific data folder structure for each task 183 | def set_dataset_paths(task, root_path, billboard_name, dataset_split): 184 | # Semantic Segmentation -> cityscapes 185 | if task == 'ss': 186 | dataset_paths = { 187 | 'images': os.path.join(root_path, billboard_name, 'leftImg8bit', dataset_split, dataset_split), 188 | 'labels': os.path.join(root_path, billboard_name, 'gtFine', dataset_split, dataset_split) 189 | } 190 | config_file = 'cityscapes.json' 191 | # 2d Object Detection -> COCO 192 | elif task == '2dod': 193 | dataset_paths = { 194 | 'images': os.path.join(root_path, billboard_name, 'images', dataset_split), 195 | 'labels': os.path.join(root_path, billboard_name, 'annotations', dataset_split) 196 | } 197 | config_file = 'coco.json' 198 | # 3D Object Detection -> Kitti 199 | elif task == '3dod': 200 | dataset_paths = { 201 | 'images': os.path.join(root_path, billboard_name, dataset_split, 'image_2'), 202 | 'images_dx': os.path.join(root_path, billboard_name, dataset_split, 'image_3'), 203 | 'calib': os.path.join(root_path, billboard_name, dataset_split, 'calib'), 204 | 'labels': os.path.join(root_path, billboard_name, dataset_split, 'label_2'), 205 | 'splits': os.path.join(root_path, billboard_name, 'splits') 206 | } 207 | config_file = 'kitti.json' 208 | # Depth Estimation -> Kitti 209 | elif task == 'depth': 210 | dataset_paths = { 211 | 'images': os.path.join(root_path, billboard_name, 'leftImg8bit', dataset_split), 212 | 'labels': os.path.join(root_path, billboard_name, 'gtFine', dataset_split), 213 | 'filenames': os.path.join(root_path, billboard_name, 'filenames'), 214 | } 215 | config_file = 'kitti.json' 216 | return dataset_paths, config_file 217 | 218 | 219 | # write annotation dict in coco format 220 | def coco_format_annotation(ann_id, image_id, category_id, vertices_2d, segmentation=[], is_crowd=0): 221 | x, y = int(vertices_2d[0, 0]), int(vertices_2d[0, 1]) 222 | w, h = int(vertices_2d[-1, 0]) - x, int(vertices_2d[-1, 1]) - y 223 | 224 | annotation = { 225 | "id": ann_id, 226 | "image_id": image_id, 227 | "category_id": COCO_CLASSES[category_id], 228 | "segmentation": segmentation, 229 | "area": w * h, 230 | "bbox": [x, y, w, h], 231 | "iscrowd": is_crowd, 232 | } 233 | # categories[{ 234 | # "id": int, "name": str, "supercategory": str, 235 | # }] 236 | return annotation 237 | 238 | # write image dict in coco format 239 | def coco_format_image(image_id, image_shape, file_name): 240 | 241 | image = { 242 | "id": image_id, 243 | "width": image_shape[1], 244 | "height": image_shape[0], 245 | "file_name": file_name, 246 | "license": 0, 247 | "flickr_url": '', 248 | "coco_url": '', 249 | "date_captured": datetime.datetime.now() 250 | } 251 | # categories[{ 252 | # "id": int, "name": str, "supercategory": str, 253 | # }] 254 | return image 255 | 256 | 257 | def kitti_format_annotation(bbox, vertices_2d, v_cam, label, camera_transform, orientation): 258 | truncate = 0 259 | occluded = 0 260 | 261 | # Fix labels 262 | if not ('car' in label or 'pedestrian' in label): 263 | label = 'DontCare' 264 | else: 265 | label = label[0].upper() + label[1:] 266 | 267 | # 2d bbox 268 | x1, y1 = int(vertices_2d[0, 0]), int(vertices_2d[0, 1]) 269 | x2, y2 = int(vertices_2d[-1, 0]), int(vertices_2d[-1, 1]) 270 | 271 | # 3d bbox 272 | H, W, L = 2 * bbox.extent.z, 2 * bbox.extent.y, 2 * bbox.extent.x 273 | # print(v_cam) 274 | x_c, y_c, z_c = np.mean(v_cam[0, :]), np.mean(v_cam[1, :]), np.mean(v_cam[2, :]) 275 | 276 | # orientation of the bbox wrt camera 277 | r_y = - np.pi/180 * orientation # bbox.rotation.yaw # (bbox.rotation.yaw - camera_transform.rotation.yaw) 278 | r_y = (r_y + np.pi) % (2 * np.pi) - np.pi # Mod(fAng + _PI, _TWO_PI) - _PI 279 | 280 | # alpha is the object viewing angle. 281 | dx = x_c - camera_transform.location.x 282 | dz = z_c - camera_transform.location.z 283 | alpha = np.arctan2(dz, dx) 284 | alpha = (alpha + np.pi) % (2 * np.pi) - np.pi 285 | 286 | return '%s %.2f %d %.2f %d %d %d %d %.2f %.2f %.2f %.2f %.2f %.2f %.2f\n' % (label, truncate, occluded, alpha, x1, y1, x2, y2, H, W, L, x_c, y_c, z_c, r_y) 287 | 288 | 289 | def kitti_format_calib(camera_params2, camera_params3): 290 | P2 = np.block([camera_params2.intrinsics, np.zeros((3, 1))]) @ np.eye(4) #camera_params2.extrinsics 291 | P3 = np.block([camera_params3.intrinsics, np.zeros((3, 1))]) @ (camera_params3.extrinsics @ np.linalg.inv(camera_params2.extrinsics) ) 292 | R0_rect = np.eye(3) 293 | Tr_imu_to_cam = np.block([np.eye(3), np.zeros((3, 1))]) 294 | Tr_velo_to_cam = np.block([np.eye(3), np.zeros((3, 1))]) 295 | 296 | # P0, P1, P2, P3 297 | calib_str = 'P0: ' + ' '.join(['%e' % P2[i, j] for i in range(3) for j in range(4)]) + '\n' 298 | calib_str += 'P1: ' + ' '.join(['%e' % P3[i, j] for i in range(3) for j in range(4)]) + '\n' 299 | calib_str += 'P2: ' + ' '.join(['%e' % P2[i, j] for i in range(3) for j in range(4)]) + '\n' 300 | calib_str += 'P3: ' + ' '.join(['%e' % P3[i, j] for i in range(3) for j in range(4)]) + '\n' 301 | calib_str += 'R0_rect: ' + ' '.join(['%e' % R0_rect[i, j] for i in range(3) for j in range(3)]) + '\n' 302 | calib_str += 'Tr_velo_to_cam: ' + ' '.join(['%e' % Tr_velo_to_cam[i, j] for i in range(3) for j in range(4)]) + '\n' 303 | calib_str += 'Tr_imu_to_velo: ' + ' '.join(['%e' % Tr_imu_to_cam[i, j] for i in range(3) for j in range(4)]) 304 | 305 | return calib_str 306 | 307 | 308 | def json_billboard_position(billboards_transform, cameras_transform, count): 309 | camera_transform, camera_transform_dx = cameras_transform 310 | billboard_data = [None] * 6 311 | billboard2_data = [None] * 6 312 | deg2rad = np.pi / 180 313 | 314 | num_billboards = len(billboards_transform) 315 | if num_billboards > 0: 316 | billboard_data = [billboards_transform[0].location.x, -billboards_transform[0].location.y, billboards_transform[0].location.z, 317 | deg2rad * billboards_transform[0].rotation.roll, deg2rad * billboards_transform[0].rotation.pitch, -deg2rad * billboards_transform[0].rotation.yaw] 318 | if num_billboards > 1: 319 | billboard2_data = [billboards_transform[1].location.x, -billboards_transform[1].location.y, billboards_transform[1].location.z, 320 | deg2rad * billboards_transform[1].rotation.roll, deg2rad * billboards_transform[1].rotation.pitch, -deg2rad * billboards_transform[1].rotation.yaw] 321 | 322 | # TODO include patch mask, or compute it in carla-loader. 323 | return_dict = { 324 | 'id': count, 325 | 'billboard_xyzrpy': billboard_data, 326 | 'billboard2_xyzrpy': billboard2_data, 327 | 'camera_xyzrpy': [camera_transform.location.x, -camera_transform.location.y, camera_transform.location.z, 328 | deg2rad * camera_transform.rotation.roll, deg2rad * camera_transform.rotation.pitch, -deg2rad * camera_transform.rotation.yaw], 329 | 'camera_dx_xyzrpy': [camera_transform_dx.location.x, -camera_transform_dx.location.y, camera_transform_dx.location.z, 330 | deg2rad * camera_transform_dx.rotation.roll, deg2rad * camera_transform_dx.rotation.pitch, -deg2rad * camera_transform_dx.rotation.yaw] 331 | } 332 | return return_dict 333 | 334 | def COCO_CATEGORIES(): 335 | return [{"supercategory": "person","id": 1,"name": "person"}, 336 | {"supercategory": "vehicle","id": 2,"name": "bicycle"}, 337 | {"supercategory": "vehicle","id": 3,"name": "car"}, 338 | {"supercategory": "vehicle","id": 4,"name": "motorcycle"}, 339 | {"supercategory": "vehicle","id": 5,"name": "airplane"}, 340 | {"supercategory": "vehicle","id": 6,"name": "bus"}, 341 | {"supercategory": "vehicle","id": 7,"name": "train"}, 342 | {"supercategory": "vehicle","id": 8,"name": "truck"}, 343 | {"supercategory": "vehicle","id": 9,"name": "boat"}, 344 | {"supercategory": "outdoor","id": 10,"name": "traffic light"}, 345 | {"supercategory": "outdoor","id": 11,"name": "fire hydrant"}, 346 | {"supercategory": "outdoor","id": 13,"name": "stop sign"}, 347 | {"supercategory": "outdoor","id": 14,"name": "parking meter"}, 348 | {"supercategory": "outdoor","id": 15,"name": "bench"}, 349 | {"supercategory": "animal","id": 16,"name": "bird"}, 350 | {"supercategory": "animal","id": 17,"name": "cat"}, 351 | {"supercategory": "animal","id": 18,"name": "dog"}, 352 | {"supercategory": "animal","id": 19,"name": "horse"}, 353 | {"supercategory": "animal","id": 20,"name": "sheep"}, 354 | {"supercategory": "animal","id": 21,"name": "cow"}, 355 | {"supercategory": "animal","id": 22,"name": "elephant"}, 356 | {"supercategory": "animal","id": 23,"name": "bear"}, 357 | {"supercategory": "animal","id": 24,"name": "zebra"}, 358 | {"supercategory": "animal","id": 25,"name": "giraffe"}, 359 | {"supercategory": "accessory","id": 27,"name": "backpack"}, 360 | {"supercategory": "accessory","id": 28,"name": "umbrella"}, 361 | {"supercategory": "accessory","id": 31,"name": "handbag"}, 362 | {"supercategory": "accessory","id": 32,"name": "tie"}, 363 | {"supercategory": "accessory","id": 33,"name": "suitcase"}, 364 | {"supercategory": "sports","id": 34,"name": "frisbee"}, 365 | {"supercategory": "sports","id": 35,"name": "skis"}, 366 | {"supercategory": "sports","id": 36,"name": "snowboard"}, 367 | {"supercategory": "sports","id": 37,"name": "sports ball"}, 368 | {"supercategory": "sports","id": 38,"name": "kite"}, 369 | {"supercategory": "sports","id": 39,"name": "baseball bat"}, 370 | {"supercategory": "sports","id": 40,"name": "baseball glove"}, 371 | {"supercategory": "sports","id": 41,"name": "skateboard"}, 372 | {"supercategory": "sports","id": 42,"name": "surfboard"}, 373 | {"supercategory": "sports","id": 43,"name": "tennis racket"}, 374 | {"supercategory": "kitchen","id": 44,"name": "bottle"}, 375 | {"supercategory": "kitchen","id": 46,"name": "wine glass"}, 376 | {"supercategory": "kitchen","id": 47,"name": "cup"}, 377 | {"supercategory": "kitchen","id": 48,"name": "fork"}, 378 | {"supercategory": "kitchen","id": 49,"name": "knife"}, 379 | {"supercategory": "kitchen","id": 50,"name": "spoon"}, 380 | {"supercategory": "kitchen","id": 51,"name": "bowl"}, 381 | {"supercategory": "food","id": 52,"name": "banana"}, 382 | {"supercategory": "food","id": 53,"name": "apple"}, 383 | {"supercategory": "food","id": 54,"name": "sandwich"}, 384 | {"supercategory": "food","id": 55,"name": "orange"}, 385 | {"supercategory": "food","id": 56,"name": "broccoli"}, 386 | {"supercategory": "food","id": 57,"name": "carrot"}, 387 | {"supercategory": "food","id": 58,"name": "hot dog"}, 388 | {"supercategory": "food","id": 59,"name": "pizza"}, 389 | {"supercategory": "food","id": 60,"name": "donut"}, 390 | {"supercategory": "food","id": 61,"name": "cake"}, 391 | {"supercategory": "furniture","id": 62,"name": "chair"}, 392 | {"supercategory": "furniture","id": 63,"name": "couch"}, 393 | {"supercategory": "furniture","id": 64,"name": "potted plant"}, 394 | {"supercategory": "furniture","id": 65,"name": "bed"}, 395 | {"supercategory": "furniture","id": 67,"name": "dining table"}, 396 | {"supercategory": "furniture","id": 70,"name": "toilet"}, 397 | {"supercategory": "electronic","id": 72,"name": "tv"}, 398 | {"supercategory": "electronic","id": 73,"name": "laptop"}, 399 | {"supercategory": "electronic","id": 74,"name": "mouse"}, 400 | {"supercategory": "electronic","id": 75,"name": "remote"}, 401 | {"supercategory": "electronic","id": 76,"name": "keyboard"}, 402 | {"supercategory": "electronic","id": 77,"name": "cell phone"}, 403 | {"supercategory": "appliance","id": 78,"name": "microwave"}, 404 | {"supercategory": "appliance","id": 79,"name": "oven"}, 405 | {"supercategory": "appliance","id": 80,"name": "toaster"}, 406 | {"supercategory": "appliance","id": 81,"name": "sink"}, 407 | {"supercategory": "appliance","id": 82,"name": "refrigerator"}, 408 | {"supercategory": "indoor","id": 84,"name": "book"}, 409 | {"supercategory": "indoor","id": 85,"name": "clock"}, 410 | {"supercategory": "indoor","id": 86,"name": "vase"}, 411 | {"supercategory": "indoor","id": 87,"name": "scissors"}, 412 | {"supercategory": "indoor","id": 88,"name": "teddy bear"}, 413 | {"supercategory": "indoor","id": 89,"name": "hair drier"}, 414 | {"supercategory": "indoor","id": 90,"name": "toothbrush"}] 415 | 416 | 417 | COCO_CLASSES = { 418 | 'pedestrian': 1, 419 | 'car': 3, 420 | 'bicycle': 2, 421 | 'motorcycle': 4, 422 | 'truck': 8, 423 | 'stop_sign': 13, 424 | 'traffic_light': 10 425 | } 426 | 427 | --------------------------------------------------------------------------------