Module robotic_manipulator_rloa.environment
23 | Sub-modules
28 |-
29 |
robotic_manipulator_rloa.environment.environment
30 | - 31 | 32 | 33 |
robotic_manipulator_rloa.environmentrobotic_manipulator_rloa.environment.environmentrobotic_manipulator_rloafrom .rl_framework import ManipulatorFrameworkrobotic_manipulator_rloa.environmentrobotic_manipulator_rloa.naf_componentsrobotic_manipulator_rloa.rl_frameworkrobotic_manipulator_rloa.utilsrobotic_manipulator_rloa.naf_componentsrobotic_manipulator_rloa.naf_components.naf_algorithmrobotic_manipulator_rloa.naf_components.naf_neural_networkrobotic_manipulator_rloa.utils.collision_detectorfrom dataclasses import dataclass
30 | from typing import List
31 |
32 | import numpy as np
33 | import pybullet as p
34 | from numpy.typing import NDArray
35 |
36 |
37 | @dataclass
38 | class CollisionObject:
39 | """
40 | Dataclass which contains the UID of the manipulator/body and the link number of the joint from which to
41 | calculate distances to other bodies.
42 | """
43 | body: str
44 | link: int
45 |
46 |
47 | class CollisionDetector:
48 |
49 | def __init__(self, collision_object: CollisionObject, obstacle_ids: List[str]):
50 | """
51 | Calculates distances between bodies' joints.
52 | Args:
53 | collision_object: CollisionObject instance, which indicates the body/joint from
54 | which to calculate distances/collisions.
55 | obstacle_ids: Obstacle body UID. Distances are calculated from the joint/body given in the
56 | "collision_object" parameter to the "obstacle_ids" bodies.
57 | """
58 | self.obstacles = obstacle_ids
59 | self.collision_object = collision_object
60 |
61 | def compute_distances(self, max_distance: float = 10.0) -> NDArray:
62 | """
63 | Compute the closest distances from the joint given by the CollisionObject instance in self.collision_object
64 | to the bodies defined in self.obstacles.
65 | Args:
66 | max_distance: Bodies farther apart than this distance are not queried by PyBullet, the return value
67 | for the distance between such bodies will be max_distance.
68 | Returns:
69 | A numpy array of distances, one per pair of collision objects.
70 | """
71 | distances = list()
72 | for obstacle in self.obstacles:
73 |
74 | # Compute the shortest distances between the collision-object and the given obstacle
75 | closest_points = p.getClosestPoints(
76 | self.collision_object.body,
77 | obstacle,
78 | distance=max_distance,
79 | linkIndexA=self.collision_object.link
80 | )
81 |
82 | # If bodies are above max_distance apart, nothing is returned, so
83 | # we just saturate at max_distance. Otherwise, take the minimum
84 | if len(closest_points) == 0:
85 | distances.append(max_distance)
86 | else:
87 | distances.append(np.min([point[8] for point in closest_points]))
88 |
89 | return np.array(distances)
90 |
91 | def compute_collisions_in_manipulator(self, affected_joints: List[int], max_distance: float = 10.) -> NDArray:
92 | """
93 | Compute collisions between manipulator's parts.
94 | Args:
95 | affected_joints: Joints to consider when calculating distances.
96 | max_distance: Maximum distance to be considered. Distances further than this will be ignored, and
97 | the "max_distance" value will be returned.
98 | Returns:
99 | Array where each element corresponds to the distances from a given joint to the other joints.
100 | """
101 | distances = list()
102 | for joint_ind in affected_joints:
103 |
104 | # Collisions with the previous and next joints are omitted, as they will be always in contact
105 | if (self.collision_object.link == joint_ind) or \
106 | (joint_ind == self.collision_object.link - 1) or \
107 | (joint_ind == self.collision_object.link + 1):
108 | continue # pragma: no cover
109 |
110 | # Compute the shortest distances between all object pairs
111 | closest_points = p.getClosestPoints(
112 | self.collision_object.body,
113 | self.collision_object.body,
114 | distance=max_distance,
115 | linkIndexA=self.collision_object.link,
116 | linkIndexB=joint_ind
117 | )
118 |
119 | # If bodies are above max_distance apart, nothing is returned, so
120 | # we just saturate at max_distance. Otherwise, take the minimum
121 | if len(closest_points) == 0:
122 | distances.append(max_distance)
123 | else:
124 | distances.append(np.min([point[8] for point in closest_points]))
125 |
126 | return np.array(distances)
139 | class CollisionDetector
140 | (collision_object: CollisionObject, obstacle_ids: List[str])
141 | Calculates distances between bodies' joints.
144 |collision_objectobstacle_idsclass CollisionDetector:
158 |
159 | def __init__(self, collision_object: CollisionObject, obstacle_ids: List[str]):
160 | """
161 | Calculates distances between bodies' joints.
162 | Args:
163 | collision_object: CollisionObject instance, which indicates the body/joint from
164 | which to calculate distances/collisions.
165 | obstacle_ids: Obstacle body UID. Distances are calculated from the joint/body given in the
166 | "collision_object" parameter to the "obstacle_ids" bodies.
167 | """
168 | self.obstacles = obstacle_ids
169 | self.collision_object = collision_object
170 |
171 | def compute_distances(self, max_distance: float = 10.0) -> NDArray:
172 | """
173 | Compute the closest distances from the joint given by the CollisionObject instance in self.collision_object
174 | to the bodies defined in self.obstacles.
175 | Args:
176 | max_distance: Bodies farther apart than this distance are not queried by PyBullet, the return value
177 | for the distance between such bodies will be max_distance.
178 | Returns:
179 | A numpy array of distances, one per pair of collision objects.
180 | """
181 | distances = list()
182 | for obstacle in self.obstacles:
183 |
184 | # Compute the shortest distances between the collision-object and the given obstacle
185 | closest_points = p.getClosestPoints(
186 | self.collision_object.body,
187 | obstacle,
188 | distance=max_distance,
189 | linkIndexA=self.collision_object.link
190 | )
191 |
192 | # If bodies are above max_distance apart, nothing is returned, so
193 | # we just saturate at max_distance. Otherwise, take the minimum
194 | if len(closest_points) == 0:
195 | distances.append(max_distance)
196 | else:
197 | distances.append(np.min([point[8] for point in closest_points]))
198 |
199 | return np.array(distances)
200 |
201 | def compute_collisions_in_manipulator(self, affected_joints: List[int], max_distance: float = 10.) -> NDArray:
202 | """
203 | Compute collisions between manipulator's parts.
204 | Args:
205 | affected_joints: Joints to consider when calculating distances.
206 | max_distance: Maximum distance to be considered. Distances further than this will be ignored, and
207 | the "max_distance" value will be returned.
208 | Returns:
209 | Array where each element corresponds to the distances from a given joint to the other joints.
210 | """
211 | distances = list()
212 | for joint_ind in affected_joints:
213 |
214 | # Collisions with the previous and next joints are omitted, as they will be always in contact
215 | if (self.collision_object.link == joint_ind) or \
216 | (joint_ind == self.collision_object.link - 1) or \
217 | (joint_ind == self.collision_object.link + 1):
218 | continue # pragma: no cover
219 |
220 | # Compute the shortest distances between all object pairs
221 | closest_points = p.getClosestPoints(
222 | self.collision_object.body,
223 | self.collision_object.body,
224 | distance=max_distance,
225 | linkIndexA=self.collision_object.link,
226 | linkIndexB=joint_ind
227 | )
228 |
229 | # If bodies are above max_distance apart, nothing is returned, so
230 | # we just saturate at max_distance. Otherwise, take the minimum
231 | if len(closest_points) == 0:
232 | distances.append(max_distance)
233 | else:
234 | distances.append(np.min([point[8] for point in closest_points]))
235 |
236 | return np.array(distances)
241 | def compute_collisions_in_manipulator(self, affected_joints: List[int], max_distance: float = 10.0) ‑> numpy.ndarray[typing.Any, numpy.dtype[+ScalarType]]
242 | Compute collisions between manipulator's parts.
245 |affected_jointsmax_distanceArray where each element corresponds to the distances from a given joint to the other joints.
def compute_collisions_in_manipulator(self, affected_joints: List[int], max_distance: float = 10.) -> NDArray:
260 | """
261 | Compute collisions between manipulator's parts.
262 | Args:
263 | affected_joints: Joints to consider when calculating distances.
264 | max_distance: Maximum distance to be considered. Distances further than this will be ignored, and
265 | the "max_distance" value will be returned.
266 | Returns:
267 | Array where each element corresponds to the distances from a given joint to the other joints.
268 | """
269 | distances = list()
270 | for joint_ind in affected_joints:
271 |
272 | # Collisions with the previous and next joints are omitted, as they will be always in contact
273 | if (self.collision_object.link == joint_ind) or \
274 | (joint_ind == self.collision_object.link - 1) or \
275 | (joint_ind == self.collision_object.link + 1):
276 | continue # pragma: no cover
277 |
278 | # Compute the shortest distances between all object pairs
279 | closest_points = p.getClosestPoints(
280 | self.collision_object.body,
281 | self.collision_object.body,
282 | distance=max_distance,
283 | linkIndexA=self.collision_object.link,
284 | linkIndexB=joint_ind
285 | )
286 |
287 | # If bodies are above max_distance apart, nothing is returned, so
288 | # we just saturate at max_distance. Otherwise, take the minimum
289 | if len(closest_points) == 0:
290 | distances.append(max_distance)
291 | else:
292 | distances.append(np.min([point[8] for point in closest_points]))
293 |
294 | return np.array(distances)
298 | def compute_distances(self, max_distance: float = 10.0) ‑> numpy.ndarray[typing.Any, numpy.dtype[+ScalarType]]
299 | Compute the closest distances from the joint given by the CollisionObject instance in self.collision_object 302 | to the bodies defined in self.obstacles.
303 |max_distanceA numpy array of distances, one per pair of collision objects.
def compute_distances(self, max_distance: float = 10.0) -> NDArray:
316 | """
317 | Compute the closest distances from the joint given by the CollisionObject instance in self.collision_object
318 | to the bodies defined in self.obstacles.
319 | Args:
320 | max_distance: Bodies farther apart than this distance are not queried by PyBullet, the return value
321 | for the distance between such bodies will be max_distance.
322 | Returns:
323 | A numpy array of distances, one per pair of collision objects.
324 | """
325 | distances = list()
326 | for obstacle in self.obstacles:
327 |
328 | # Compute the shortest distances between the collision-object and the given obstacle
329 | closest_points = p.getClosestPoints(
330 | self.collision_object.body,
331 | obstacle,
332 | distance=max_distance,
333 | linkIndexA=self.collision_object.link
334 | )
335 |
336 | # If bodies are above max_distance apart, nothing is returned, so
337 | # we just saturate at max_distance. Otherwise, take the minimum
338 | if len(closest_points) == 0:
339 | distances.append(max_distance)
340 | else:
341 | distances.append(np.min([point[8] for point in closest_points]))
342 |
343 | return np.array(distances)
349 | class CollisionObject
350 | (body: str, link: int)
351 | Dataclass which contains the UID of the manipulator/body and the link number of the joint from which to 354 | calculate distances to other bodies.
class CollisionObject:
360 | """
361 | Dataclass which contains the UID of the manipulator/body and the link number of the joint from which to
362 | calculate distances to other bodies.
363 | """
364 | body: str
365 | link: intvar body : strvar link : introbotic_manipulator_rloa.utilsrobotic_manipulator_rloa.utils.collision_detectorrobotic_manipulator_rloa.utils.exceptionsrobotic_manipulator_rloa.utils.loggerrobotic_manipulator_rloa.utils.replay_bufferrobotic_manipulator_rloa.utils.loggerimport logging
30 | from datetime import datetime
31 | from logging import LogRecord
32 | from logging.config import dictConfig
33 | from logging.handlers import RotatingFileHandler
34 |
35 |
36 | class Logger: # pylint: disable=too-many-instance-attributes
37 | """ Stores and processes the logs """
38 |
39 | @staticmethod
40 | def set_logger_setup() -> None:
41 | """
42 | Sets the logger setup with a predefined configuration.
43 | """
44 |
45 | log_config_dict = Logger.generate_logging_config_dict()
46 | dictConfig(log_config_dict)
47 | rotating_file_handler = RotatingFileHandler(filename='training_logs.log', mode='a', maxBytes=50000000,
48 | backupCount=10, encoding='utf-8')
49 | rotating_file_handler.setFormatter(logging.Formatter(
50 | '"%(levelname)s"|"{datetime}"|%(message)s'.format(datetime=datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S.%fZ'))
51 | ))
52 | rotating_file_handler.setLevel(logging.INFO)
53 | logger = get_global_logger()
54 | logger.addHandler(rotating_file_handler)
55 | logger.setLevel(20)
56 |
57 | @staticmethod
58 | def generate_logging_config_dict() -> dict:
59 | """
60 | Generates the configuration dictionary that is used to configure the logger.
61 | Returns:
62 | Configuration dictionary.
63 | """
64 | return {
65 | 'version': 1,
66 | 'disable_existing_loggers': False,
67 | 'formatters': {
68 | 'custom_formatter': {
69 | '()': CustomFormatter,
70 | 'dateformat': '%Y-%m-%dT%H:%M:%S.%06d%z'
71 | },
72 | },
73 | 'handlers': {
74 | 'debug_console_handler': {
75 | 'level': 'NOTSET',
76 | 'formatter': 'custom_formatter',
77 | 'class': 'logging.StreamHandler',
78 | 'stream': 'ext://sys.stdout',
79 | }
80 | },
81 | 'loggers': {
82 | '': {
83 | 'handlers': ['debug_console_handler'],
84 | 'level': 'NOTSET',
85 | },
86 | }
87 | }
88 |
89 |
90 | def get_global_logger() -> logging.Logger:
91 | """
92 | Getter for the logger.
93 | Returns:
94 | Logger instance to be used on the framework.
95 | """
96 | return logging.getLogger(__name__)
97 |
98 |
99 | class CustomFormatter(logging.Formatter):
100 |
101 | def __init__(self, dateformat: str = None):
102 | """
103 | CustomFormatter for the logger.
104 | """
105 | super().__init__()
106 | self.dateformat = dateformat
107 |
108 | def format(self, record: LogRecord) -> str:
109 | """
110 | Formats the provided LogRecord instance.
111 | Returns:
112 | Formatted LogRecord as string.
113 | """
114 | # Set format and colors
115 | grey = "\033[38;20m"
116 | green = "\033[32;20m"
117 | yellow = "\033[33;20m"
118 | red = "\033[31;20m"
119 | bold_red = "\033[31;1m"
120 | reset = "\033[0m"
121 | format_ = '[%(levelname)-8s] - {datetime} - %(message)s'.format(
122 | datetime=datetime.now().astimezone().strftime('%Y-%m-%dT%H:%M:%S.%f%z')
123 | )
124 |
125 | self.FORMATS = {
126 | logging.DEBUG: green + format_ + reset,
127 | logging.INFO: grey + format_ + reset,
128 | logging.WARNING: yellow + format_ + reset,
129 | logging.ERROR: red + format_ + reset,
130 | logging.CRITICAL: bold_red + format_ + reset
131 | }
132 |
133 | log_format = self.FORMATS.get(record.levelno)
134 |
135 | formatter = logging.Formatter(log_format, datefmt=self.dateformat)
136 | return formatter.format(record)
147 | def get_global_logger() ‑> logging.Logger
148 | Getter for the logger.
151 |Logger instance to be used on the framework.
def get_global_logger() -> logging.Logger:
158 | """
159 | Getter for the logger.
160 | Returns:
161 | Logger instance to be used on the framework.
162 | """
163 | return logging.getLogger(__name__)
172 | class CustomFormatter
173 | (dateformat: str = None)
174 | Formatter instances are used to convert a LogRecord to text.
177 |Formatters need to know how a LogRecord is constructed. They are 178 | responsible for converting a LogRecord to (usually) a string which can 179 | be interpreted by either a human or an external system. The base Formatter 180 | allows a formatting string to be specified. If none is supplied, the 181 | style-dependent default value, "%(message)s", "{message}", or 182 | "${message}", is used.
183 |The Formatter can be initialized with a format string which makes use of 184 | knowledge of the LogRecord attributes - e.g. the default value mentioned 185 | above makes use of the fact that the user's message and arguments are pre- 186 | formatted into a LogRecord's message attribute. Currently, the useful 187 | attributes in a LogRecord are described by:
188 |%(name)s 189 | Name of the logger (logging channel) 190 | %(levelno)s 191 | Numeric logging level for the message (DEBUG, INFO, 192 | WARNING, ERROR, CRITICAL) 193 | %(levelname)s 194 | Text logging level for the message ("DEBUG", "INFO", 195 | "WARNING", "ERROR", "CRITICAL") 196 | %(pathname)s 197 | Full pathname of the source file where the logging 198 | call was issued (if available) 199 | %(filename)s 200 | Filename portion of pathname 201 | %(module)s 202 | Module (name portion of filename) 203 | %(lineno)d 204 | Source line number where the logging call was issued 205 | (if available) 206 | %(funcName)s 207 | Function name 208 | %(created)f 209 | Time when the LogRecord was created (time.time() 210 | return value) 211 | %(asctime)s 212 | Textual time when the LogRecord was created 213 | %(msecs)d 214 | Millisecond portion of the creation time 215 | %(relativeCreated)d Time in milliseconds when the LogRecord was created, 216 | relative to the time the logging module was loaded 217 | (typically at application startup time) 218 | %(thread)d 219 | Thread ID (if available) 220 | %(threadName)s 221 | Thread name (if available) 222 | %(process)d 223 | Process ID (if available) 224 | %(message)s 225 | The result of record.getMessage(), computed just as 226 | the record is emitted
227 |CustomFormatter for the logger.
class CustomFormatter(logging.Formatter):
233 |
234 | def __init__(self, dateformat: str = None):
235 | """
236 | CustomFormatter for the logger.
237 | """
238 | super().__init__()
239 | self.dateformat = dateformat
240 |
241 | def format(self, record: LogRecord) -> str:
242 | """
243 | Formats the provided LogRecord instance.
244 | Returns:
245 | Formatted LogRecord as string.
246 | """
247 | # Set format and colors
248 | grey = "\033[38;20m"
249 | green = "\033[32;20m"
250 | yellow = "\033[33;20m"
251 | red = "\033[31;20m"
252 | bold_red = "\033[31;1m"
253 | reset = "\033[0m"
254 | format_ = '[%(levelname)-8s] - {datetime} - %(message)s'.format(
255 | datetime=datetime.now().astimezone().strftime('%Y-%m-%dT%H:%M:%S.%f%z')
256 | )
257 |
258 | self.FORMATS = {
259 | logging.DEBUG: green + format_ + reset,
260 | logging.INFO: grey + format_ + reset,
261 | logging.WARNING: yellow + format_ + reset,
262 | logging.ERROR: red + format_ + reset,
263 | logging.CRITICAL: bold_red + format_ + reset
264 | }
265 |
266 | log_format = self.FORMATS.get(record.levelno)
267 |
268 | formatter = logging.Formatter(log_format, datefmt=self.dateformat)
269 | return formatter.format(record)
278 | def format(self, record: logging.LogRecord) ‑> str
279 | Formats the provided LogRecord instance.
282 |Formatted LogRecord as string.
def format(self, record: LogRecord) -> str:
289 | """
290 | Formats the provided LogRecord instance.
291 | Returns:
292 | Formatted LogRecord as string.
293 | """
294 | # Set format and colors
295 | grey = "\033[38;20m"
296 | green = "\033[32;20m"
297 | yellow = "\033[33;20m"
298 | red = "\033[31;20m"
299 | bold_red = "\033[31;1m"
300 | reset = "\033[0m"
301 | format_ = '[%(levelname)-8s] - {datetime} - %(message)s'.format(
302 | datetime=datetime.now().astimezone().strftime('%Y-%m-%dT%H:%M:%S.%f%z')
303 | )
304 |
305 | self.FORMATS = {
306 | logging.DEBUG: green + format_ + reset,
307 | logging.INFO: grey + format_ + reset,
308 | logging.WARNING: yellow + format_ + reset,
309 | logging.ERROR: red + format_ + reset,
310 | logging.CRITICAL: bold_red + format_ + reset
311 | }
312 |
313 | log_format = self.FORMATS.get(record.levelno)
314 |
315 | formatter = logging.Formatter(log_format, datefmt=self.dateformat)
316 | return formatter.format(record)
322 | class Logger
323 | Stores and processes the logs
class Logger: # pylint: disable=too-many-instance-attributes
331 | """ Stores and processes the logs """
332 |
333 | @staticmethod
334 | def set_logger_setup() -> None:
335 | """
336 | Sets the logger setup with a predefined configuration.
337 | """
338 |
339 | log_config_dict = Logger.generate_logging_config_dict()
340 | dictConfig(log_config_dict)
341 | rotating_file_handler = RotatingFileHandler(filename='training_logs.log', mode='a', maxBytes=50000000,
342 | backupCount=10, encoding='utf-8')
343 | rotating_file_handler.setFormatter(logging.Formatter(
344 | '"%(levelname)s"|"{datetime}"|%(message)s'.format(datetime=datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S.%fZ'))
345 | ))
346 | rotating_file_handler.setLevel(logging.INFO)
347 | logger = get_global_logger()
348 | logger.addHandler(rotating_file_handler)
349 | logger.setLevel(20)
350 |
351 | @staticmethod
352 | def generate_logging_config_dict() -> dict:
353 | """
354 | Generates the configuration dictionary that is used to configure the logger.
355 | Returns:
356 | Configuration dictionary.
357 | """
358 | return {
359 | 'version': 1,
360 | 'disable_existing_loggers': False,
361 | 'formatters': {
362 | 'custom_formatter': {
363 | '()': CustomFormatter,
364 | 'dateformat': '%Y-%m-%dT%H:%M:%S.%06d%z'
365 | },
366 | },
367 | 'handlers': {
368 | 'debug_console_handler': {
369 | 'level': 'NOTSET',
370 | 'formatter': 'custom_formatter',
371 | 'class': 'logging.StreamHandler',
372 | 'stream': 'ext://sys.stdout',
373 | }
374 | },
375 | 'loggers': {
376 | '': {
377 | 'handlers': ['debug_console_handler'],
378 | 'level': 'NOTSET',
379 | },
380 | }
381 | }
386 | def generate_logging_config_dict() ‑> dict
387 | Generates the configuration dictionary that is used to configure the logger.
390 |Configuration dictionary.
@staticmethod
397 | def generate_logging_config_dict() -> dict:
398 | """
399 | Generates the configuration dictionary that is used to configure the logger.
400 | Returns:
401 | Configuration dictionary.
402 | """
403 | return {
404 | 'version': 1,
405 | 'disable_existing_loggers': False,
406 | 'formatters': {
407 | 'custom_formatter': {
408 | '()': CustomFormatter,
409 | 'dateformat': '%Y-%m-%dT%H:%M:%S.%06d%z'
410 | },
411 | },
412 | 'handlers': {
413 | 'debug_console_handler': {
414 | 'level': 'NOTSET',
415 | 'formatter': 'custom_formatter',
416 | 'class': 'logging.StreamHandler',
417 | 'stream': 'ext://sys.stdout',
418 | }
419 | },
420 | 'loggers': {
421 | '': {
422 | 'handlers': ['debug_console_handler'],
423 | 'level': 'NOTSET',
424 | },
425 | }
426 | }
430 | def set_logger_setup() ‑> None
431 | Sets the logger setup with a predefined configuration.
@staticmethod
439 | def set_logger_setup() -> None:
440 | """
441 | Sets the logger setup with a predefined configuration.
442 | """
443 |
444 | log_config_dict = Logger.generate_logging_config_dict()
445 | dictConfig(log_config_dict)
446 | rotating_file_handler = RotatingFileHandler(filename='training_logs.log', mode='a', maxBytes=50000000,
447 | backupCount=10, encoding='utf-8')
448 | rotating_file_handler.setFormatter(logging.Formatter(
449 | '"%(levelname)s"|"{datetime}"|%(message)s'.format(datetime=datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S.%fZ'))
450 | ))
451 | rotating_file_handler.setLevel(logging.INFO)
452 | logger = get_global_logger()
453 | logger.addHandler(rotating_file_handler)
454 | logger.setLevel(20)robotic_manipulator_rloa.utils.replay_bufferimport random
30 | from collections import deque, namedtuple
31 | from typing import Tuple
32 |
33 | import numpy as np
34 | import torch
35 | from numpy.typing import NDArray
36 |
37 |
38 | # deque is a Doubly Ended Queue, provides O(1) complexity for pop and append actions
39 | # namedtuple is a tuple that can be accessed by both its index and attributes
40 |
41 |
42 | class ReplayBuffer:
43 |
44 | def __init__(self, buffer_size: int, batch_size: int, device: torch.device, seed: int):
45 | """
46 | Buffer to store experience tuples. Each experience has the following structure:
47 | (state, action, reward, next_state, done)
48 | Args:
49 | buffer_size: Maximum size for the buffer. Higher buffer size imply higher RAM consumption.
50 | batch_size: Number of experiences to be retrieved from the ReplayBuffer per batch.
51 | device: CUDA device.
52 | seed: Random seed.
53 | """
54 | self.device = device
55 | self.memory = deque(maxlen=buffer_size)
56 | self.batch_size = batch_size
57 | self.experience = namedtuple("Experience", field_names=["state", "action", "reward", "next_state", "done"])
58 | random.seed(seed)
59 |
60 | def add(self, state: NDArray, action: NDArray, reward: float, next_state: NDArray, done: int) -> None:
61 | """
62 | Add a new experience to the Replay Buffer.
63 | Args:
64 | state: NDArray of the current state.
65 | action: NDArray of the action taken from state {state}.
66 | reward: Reward obtained after performing action {action} from state {state}.
67 | next_state: NDArray of the state reached after performing action {action} from state {state}.
68 | done: Integer (0 or 1) indicating whether the next_state is a terminal state.
69 | """
70 | # Create namedtuple object from the experience
71 | exp = self.experience(state, action, reward, next_state, done)
72 | # Add the experience object to memory
73 | self.memory.append(exp)
74 |
75 | def sample(self) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
76 | """
77 | Randomly sample a batch of experiences from memory.
78 | Returns:
79 | Tuple of 5 elements, which are (states, actions, rewards, next_states, dones). Each element
80 | in the tuple is a torch Tensor composed of {batch_size} items.
81 | """
82 | # Randomly sample a batch of experiences
83 | experiences = random.sample(self.memory, k=self.batch_size)
84 |
85 | states = torch.from_numpy(
86 | np.stack([e.state if not isinstance(e.state, tuple) else e.state[0] for e in experiences])).float().to(
87 | self.device)
88 | actions = torch.from_numpy(np.vstack([e.action for e in experiences if e is not None])).long().to(self.device)
89 | rewards = torch.from_numpy(np.vstack([e.reward for e in experiences if e is not None])).float().to(self.device)
90 | next_states = torch.from_numpy(np.stack([e.next_state for e in experiences if e is not None])).float().to(
91 | self.device)
92 | dones = torch.from_numpy(np.vstack([e.done for e in experiences if e is not None]).astype(np.uint8)).float().to(
93 | self.device)
94 |
95 | return states, actions, rewards, next_states, dones
96 |
97 | def __len__(self) -> int:
98 | """
99 | Return the current size of the Replay Buffer
100 | Returns:
101 | Size of Replay Buffer
102 | """
103 | return len(self.memory)
116 | class ReplayBuffer
117 | (buffer_size: int, batch_size: int, device: torch.device, seed: int)
118 | Buffer to store experience tuples. Each experience has the following structure: 121 | (state, action, reward, next_state, done)
122 |buffer_sizebatch_sizedeviceseedclass ReplayBuffer:
138 |
139 | def __init__(self, buffer_size: int, batch_size: int, device: torch.device, seed: int):
140 | """
141 | Buffer to store experience tuples. Each experience has the following structure:
142 | (state, action, reward, next_state, done)
143 | Args:
144 | buffer_size: Maximum size for the buffer. Higher buffer size imply higher RAM consumption.
145 | batch_size: Number of experiences to be retrieved from the ReplayBuffer per batch.
146 | device: CUDA device.
147 | seed: Random seed.
148 | """
149 | self.device = device
150 | self.memory = deque(maxlen=buffer_size)
151 | self.batch_size = batch_size
152 | self.experience = namedtuple("Experience", field_names=["state", "action", "reward", "next_state", "done"])
153 | random.seed(seed)
154 |
155 | def add(self, state: NDArray, action: NDArray, reward: float, next_state: NDArray, done: int) -> None:
156 | """
157 | Add a new experience to the Replay Buffer.
158 | Args:
159 | state: NDArray of the current state.
160 | action: NDArray of the action taken from state {state}.
161 | reward: Reward obtained after performing action {action} from state {state}.
162 | next_state: NDArray of the state reached after performing action {action} from state {state}.
163 | done: Integer (0 or 1) indicating whether the next_state is a terminal state.
164 | """
165 | # Create namedtuple object from the experience
166 | exp = self.experience(state, action, reward, next_state, done)
167 | # Add the experience object to memory
168 | self.memory.append(exp)
169 |
170 | def sample(self) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
171 | """
172 | Randomly sample a batch of experiences from memory.
173 | Returns:
174 | Tuple of 5 elements, which are (states, actions, rewards, next_states, dones). Each element
175 | in the tuple is a torch Tensor composed of {batch_size} items.
176 | """
177 | # Randomly sample a batch of experiences
178 | experiences = random.sample(self.memory, k=self.batch_size)
179 |
180 | states = torch.from_numpy(
181 | np.stack([e.state if not isinstance(e.state, tuple) else e.state[0] for e in experiences])).float().to(
182 | self.device)
183 | actions = torch.from_numpy(np.vstack([e.action for e in experiences if e is not None])).long().to(self.device)
184 | rewards = torch.from_numpy(np.vstack([e.reward for e in experiences if e is not None])).float().to(self.device)
185 | next_states = torch.from_numpy(np.stack([e.next_state for e in experiences if e is not None])).float().to(
186 | self.device)
187 | dones = torch.from_numpy(np.vstack([e.done for e in experiences if e is not None]).astype(np.uint8)).float().to(
188 | self.device)
189 |
190 | return states, actions, rewards, next_states, dones
191 |
192 | def __len__(self) -> int:
193 | """
194 | Return the current size of the Replay Buffer
195 | Returns:
196 | Size of Replay Buffer
197 | """
198 | return len(self.memory)
203 | def add(self, state: numpy.ndarray[typing.Any, numpy.dtype[+ScalarType]], action: numpy.ndarray[typing.Any, numpy.dtype[+ScalarType]], reward: float, next_state: numpy.ndarray[typing.Any, numpy.dtype[+ScalarType]], done: int) ‑> None
204 | Add a new experience to the Replay Buffer.
207 |stateactionrewardnext_statedonedef add(self, state: NDArray, action: NDArray, reward: float, next_state: NDArray, done: int) -> None:
225 | """
226 | Add a new experience to the Replay Buffer.
227 | Args:
228 | state: NDArray of the current state.
229 | action: NDArray of the action taken from state {state}.
230 | reward: Reward obtained after performing action {action} from state {state}.
231 | next_state: NDArray of the state reached after performing action {action} from state {state}.
232 | done: Integer (0 or 1) indicating whether the next_state is a terminal state.
233 | """
234 | # Create namedtuple object from the experience
235 | exp = self.experience(state, action, reward, next_state, done)
236 | # Add the experience object to memory
237 | self.memory.append(exp)
241 | def sample(self) ‑> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]
242 | Randomly sample a batch of experiences from memory.
245 |Tuple of 5 elements, which are (states, actions, rewards, next_states, dones). Each element 247 | in the tuple is a torch Tensor composed of {batch_size} items.
def sample(self) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
253 | """
254 | Randomly sample a batch of experiences from memory.
255 | Returns:
256 | Tuple of 5 elements, which are (states, actions, rewards, next_states, dones). Each element
257 | in the tuple is a torch Tensor composed of {batch_size} items.
258 | """
259 | # Randomly sample a batch of experiences
260 | experiences = random.sample(self.memory, k=self.batch_size)
261 |
262 | states = torch.from_numpy(
263 | np.stack([e.state if not isinstance(e.state, tuple) else e.state[0] for e in experiences])).float().to(
264 | self.device)
265 | actions = torch.from_numpy(np.vstack([e.action for e in experiences if e is not None])).long().to(self.device)
266 | rewards = torch.from_numpy(np.vstack([e.reward for e in experiences if e is not None])).float().to(self.device)
267 | next_states = torch.from_numpy(np.stack([e.next_state for e in experiences if e is not None])).float().to(
268 | self.device)
269 | dones = torch.from_numpy(np.vstack([e.done for e in experiences if e is not None]).astype(np.uint8)).float().to(
270 | self.device)
271 |
272 | return states, actions, rewards, next_states, dones