├── gym_missile_command ├── game │ ├── __init__.py │ ├── batteries.py │ ├── target.py │ ├── cities.py │ ├── enemy_missiles.py │ └── friendly_missiles.py ├── version.py ├── environment │ ├── __init__.py │ └── missile_command.py ├── configuration │ ├── __init__.py │ ├── parser.py │ └── config.py ├── __init__.py ├── utils.py └── examples │ ├── random_agent.py │ └── human_agent.py ├── materials └── human_demo.gif ├── setup.py ├── LICENSE ├── README.md ├── .gitignore └── CONFIG.md /gym_missile_command/game/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /gym_missile_command/version.py: -------------------------------------------------------------------------------- 1 | 1.5 2 | -------------------------------------------------------------------------------- /materials/human_demo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ElieKadoche/gym_missile_command/HEAD/materials/human_demo.gif -------------------------------------------------------------------------------- /gym_missile_command/environment/__init__.py: -------------------------------------------------------------------------------- 1 | from gym_missile_command.environment.missile_command import MissileCommandEnv 2 | -------------------------------------------------------------------------------- /gym_missile_command/configuration/__init__.py: -------------------------------------------------------------------------------- 1 | from gym_missile_command.configuration.config import CONFIG 2 | from gym_missile_command.configuration.parser import update_config 3 | -------------------------------------------------------------------------------- /gym_missile_command/__init__.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | from gymnasium.envs.registration import register 4 | 5 | with open(Path(__file__).parent / "version.py") as _version_file: 6 | __version__ = _version_file.read().strip() 7 | 8 | register( 9 | id="missile-command-v0", 10 | entry_point="gym_missile_command.environment:MissileCommandEnv", 11 | ) 12 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | """Setup file.""" 2 | 3 | from pathlib import Path 4 | 5 | from setuptools import setup 6 | 7 | ROOT = Path(__file__).parent 8 | with open(ROOT / "gym_missile_command" / "version.py") as version_file: 9 | VERSION = version_file.read().strip() 10 | 11 | setup( 12 | name="gym_missile_command", 13 | version=VERSION, 14 | author="Elie KADOCHE", 15 | author_email="eliekadoche78@gmail.com", 16 | install_requires=["gymnasium >= 0.28.1", 17 | "numpy >= 1.24.3", 18 | "opencv-python >= 4.7.0.72", 19 | "pygame >= 2.4.0"], 20 | description="Gym environment of the Atari game, Missile Command.", 21 | packages=["gym_missile_command"], 22 | url="https://github.com/ElieKadoche/gym_missile_command.git", 23 | ) 24 | -------------------------------------------------------------------------------- /gym_missile_command/utils.py: -------------------------------------------------------------------------------- 1 | """Useful functions.""" 2 | 3 | 4 | def get_cv2_xy(height, width, x, y): 5 | """Transform x environment position into x opencv position. 6 | 7 | The origin of the environment is the anti-missiles battery, placed in the 8 | bottom center. But in the render method, the origin is in the top left 9 | corner. It is also good to note that in python-opencv, coordinates are 10 | written (y, x) and not (x, y) like for the environment. 11 | 12 | Args: 13 | height (float): environment height. 14 | width (float): environment width. 15 | x (float): x environment coordinate. 16 | y (float): y environment coordinate. 17 | 18 | Returns: 19 | y (int): x opencv coordinate. 20 | 21 | x (int): x opencv coordinate. 22 | """ 23 | return int(height - y), int(x + (width / 2)) 24 | -------------------------------------------------------------------------------- /gym_missile_command/examples/random_agent.py: -------------------------------------------------------------------------------- 1 | """Random agent.""" 2 | 3 | import gymnasium as gym 4 | 5 | import gym_missile_command 6 | 7 | if __name__ == "__main__": 8 | # Environment configuration (see CONFIG.md) 9 | env_context = {"ENEMY_MISSILES.NUMBER": 42, 10 | "FRIENDLY_MISSILES.EXPLOSION_RADIUS": 17} 11 | 12 | # Create the environment 13 | env = gym.make("missile-command-v0", env_context=env_context) 14 | 15 | # Reset it 16 | observation, _ = env.reset(seed=None) 17 | 18 | # While the episode is not finished 19 | terminated = False 20 | while not terminated: 21 | 22 | # Select an action (here, a random one) 23 | action = env.action_space.sample() 24 | 25 | # One step forward 26 | observation, reward, terminated, _, _ = env.step(action) 27 | 28 | # Render (or not) the environment 29 | env.render() 30 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Elie KADOCHE 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /gym_missile_command/configuration/parser.py: -------------------------------------------------------------------------------- 1 | """Functions to parse the configuration.""" 2 | 3 | import functools 4 | import sys 5 | 6 | from gym_missile_command.configuration import CONFIG 7 | 8 | 9 | def _rgetattr(obj, attr, *args): 10 | """Recursive getattr function.""" 11 | def _getattr(obj, attr): 12 | return getattr(obj, attr, *args) 13 | return functools.reduce(_getattr, [obj] + attr.split(".")) 14 | 15 | 16 | def _rsetattr(obj, attr, val): 17 | """Recursive setattr function.""" 18 | pre, _, post = attr.rpartition('.') 19 | return setattr(_rgetattr(obj, pre) if pre else obj, post, val) 20 | 21 | 22 | def update_config(env_context): 23 | """Update global configuration. 24 | 25 | Args: 26 | env_context (dict): environment configuration. 27 | """ 28 | # For each custom parameter 29 | for key, value in env_context.items(): 30 | 31 | # Be robust to case 32 | key = key.upper() 33 | 34 | # Check if parameter is valid 35 | try: 36 | _ = _rgetattr(CONFIG, key) 37 | except AttributeError as e: 38 | print("Invalid custom configuration: {}.".format(e)) 39 | sys.exit(1) 40 | 41 | # Modify it 42 | _rsetattr(CONFIG, key, value) 43 | -------------------------------------------------------------------------------- /gym_missile_command/configuration/config.py: -------------------------------------------------------------------------------- 1 | """Environment configuration.""" 2 | 3 | from dataclasses import dataclass 4 | 5 | 6 | @dataclass 7 | class CONFIG(): 8 | """Configuration class.""" 9 | 10 | @dataclass 11 | class EPISODE(): 12 | """Episode global configuration.""" 13 | 14 | FPS: int = 144 15 | HEIGHT: int = 466 16 | WIDTH: int = 966 17 | 18 | @dataclass 19 | class BATTERY(): 20 | """Anti-missiles battery configuration.""" 21 | 22 | RADIUS: float = 37.0 23 | 24 | @dataclass 25 | class CITIES(): 26 | """Cities configuration.""" 27 | 28 | NUMBER: int = 6 29 | RADIUS: float = 24.0 30 | 31 | @dataclass 32 | class COLORS(): 33 | """Colors configuration.""" 34 | 35 | BACKGROUND: tuple = (0, 0, 0) 36 | BATTERY: tuple = (255, 255, 255) 37 | CITY: tuple = (0, 0, 255) 38 | ENEMY_MISSILE: tuple = (255, 0, 0) 39 | EXPLOSION: tuple = (255, 255, 0) 40 | FRIENDLY_MISSILE: tuple = (0, 255, 0) 41 | TARGET: tuple = (255, 255, 255) 42 | 43 | @dataclass 44 | class ENEMY_MISSILES(): 45 | """Enemy missiles configuration.""" 46 | 47 | NUMBER: int = 19 48 | PROBA_IN: float = 0.005 49 | RADIUS: float = 4.0 50 | SPEED: float = 1.0 51 | 52 | @dataclass 53 | class FRIENDLY_MISSILES(): 54 | """Friendly missiles configuration.""" 55 | 56 | NUMBER: int = 142 57 | EXPLOSION_RADIUS: float = 37.0 58 | EXPLOSION_SPEED: float = 0.5 59 | RADIUS: float = 7.0 60 | SPEED: float = 7.0 61 | 62 | @dataclass 63 | class OBSERVATION(): 64 | """Observation configuration.""" 65 | 66 | HEIGHT: float = 84 67 | RENDER_PROCESSED_HEIGHT: int = 250 68 | RENDER_PROCESSED_WIDTH: int = 250 69 | WIDTH: float = 84 70 | 71 | @dataclass 72 | class REWARD(): 73 | """Reward configuration.""" 74 | 75 | DESTROYED_CITY: float = -10.0 76 | DESTROYED_ENEMEY_MISSILES: float = 15.0 77 | FRIENDLY_MISSILE_LAUNCHED: float = -4.0 78 | 79 | @dataclass 80 | class TARGET(): 81 | """Target configuration.""" 82 | 83 | SIZE: int = 12 84 | VX: int = 4 85 | VY: int = 4 86 | -------------------------------------------------------------------------------- /gym_missile_command/examples/human_agent.py: -------------------------------------------------------------------------------- 1 | """Human agent.""" 2 | 3 | import sys 4 | 5 | import gymnasium as gym 6 | import pygame 7 | 8 | import gym_missile_command 9 | 10 | # Number of time step to wait before the user can send a new missile 11 | STOP_FIRE_WAIT = 57 12 | 13 | if __name__ == "__main__": 14 | # Create the environment 15 | env = gym.make("missile-command-v0") 16 | 17 | # Reset it 18 | observation, _ = env.reset(seed=None) 19 | 20 | # Initialize PyGame 21 | env.render() 22 | 23 | # The user can naturally pause the environment 24 | pause = False 25 | 26 | # Prevent multiple fire when the user chooses action 5 27 | stop_fire = 0 28 | 29 | # While the episode is not finished 30 | terminated = False 31 | while not terminated: 32 | 33 | # Check if user exits 34 | for event in pygame.event.get(): 35 | 36 | # Exit the environment 37 | if event.type == pygame.QUIT: 38 | pygame.quit() 39 | sys.exit() 40 | 41 | # Pause the environment 42 | if event.type == pygame.KEYDOWN: 43 | if event.key == pygame.K_ESCAPE: 44 | pause = not pause 45 | 46 | # Get keys pressed by user 47 | keystate = pygame.key.get_pressed() 48 | 49 | # Default action is 0 50 | action = 0 51 | 52 | # Target up 53 | if keystate[pygame.K_UP]: 54 | action = 1 55 | 56 | # Target down 57 | if keystate[pygame.K_DOWN]: 58 | action = 2 59 | 60 | # Target left 61 | if keystate[pygame.K_LEFT]: 62 | action = 3 63 | 64 | # Target right 65 | if keystate[pygame.K_RIGHT]: 66 | action = 4 67 | 68 | # Fire missile 69 | if keystate[pygame.K_SPACE]: 70 | if stop_fire == 0: 71 | stop_fire = STOP_FIRE_WAIT 72 | action = 5 73 | 74 | # Decrease stop_fire 75 | if stop_fire > 0: 76 | stop_fire -= 1 77 | 78 | if not pause: 79 | # One step forward 80 | observation, reward, terminated, _, _ = env.step(action) 81 | 82 | # Render (or not) the environment 83 | env.render() 84 | 85 | # Close the environment 86 | env.close() 87 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # gym_missile_command 2 | 3 | Open AI Gym environment of the [Missile Command Atari game](https://en.wikipedia.org/wiki/Missile_Command). 4 | 5 | ![Demonstration (gif)](./materials/human_demo.gif) 6 | 7 | Game 8 | ------------------------------------------ 9 | 10 | The player musts defend 6 cities from incoming enemy ballistic missiles. 11 | To do so, he can fire missiles from an anti-missiles battery. 12 | An episode ends when all enemy missiles or cities are destroyed. 13 | This environment does not reproduce an exact version of the Missile Command Atari game but a simplified one. 14 | 15 | - The anti-missiles battery can not be destroyed. 16 | - There are no levels, all episodes have the same difficulty. 17 | - Enemy missiles do not have an explosion radius and do not split. 18 | 19 | The reward depends on several variables, each one contributing to a specific wanted skill of the agent. 20 | 21 | - Number of cities destroyed, to protect the cities. 22 | - Number of enemy missiles destroyed, to improve accuracy. 23 | - Number of missiles launched, to minimize the use of missiles. 24 | 25 | Installation 26 | ------------------------------------------ 27 | 28 | [Python](https://www.python.org/) 3.8+ is required. 29 | The installation is done with the following commands. 30 | 31 | ```shell 32 | git clone https://github.com/ElieKadoche/gym_missile_command.git 33 | pip install -e ./gym_missile_command 34 | ``` 35 | Examples 36 | ------------------------------------------ 37 | 38 | For a human to play, commands are: arrow keys to move the target and space to fire a missile. 39 | 40 | ```shell 41 | python -m gym_missile_command.examples.random_agent # For a random agent to play 42 | python -m gym_missile_command.examples.human_agent # For a human to play 43 | ``` 44 | 45 | Usage 46 | ------------------------------------------ 47 | 48 | ```python 49 | import gymnasium as gym 50 | 51 | import gym_missile_command 52 | 53 | # Environment configuration (see CONFIG.md) 54 | env_context = {"ENEMY_MISSILES.NUMBER": 42, 55 | "FRIENDLY_MISSILES.EXPLOSION_RADIUS": 17} 56 | 57 | # Create the environment 58 | env = gym.make("missile-command-v0", env_context=env_context) 59 | 60 | # Reset it 61 | observation, info = env.reset(seed=None) 62 | 63 | # While the episode is not finished 64 | terminated = False 65 | while not terminated: 66 | 67 | # Select an action (here, a random one) 68 | action = env.action_space.sample() 69 | 70 | # One step forward 71 | observation, reward, terminated, truncated, info = env.step(action) 72 | 73 | # Render (or not) the environment 74 | env.render() 75 | 76 | # Close the environment 77 | env.close() 78 | ``` 79 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Python 2 | # ------------------------------------------ 3 | 4 | # Byte-compiled / optimized / DLL files 5 | __pycache__/ 6 | *.py[cod] 7 | *$py.class 8 | 9 | # C extensions 10 | *.so 11 | 12 | # Distribution / packaging 13 | .Python 14 | build/ 15 | develop-eggs/ 16 | dist/ 17 | downloads/ 18 | eggs/ 19 | .eggs/ 20 | lib/ 21 | lib64/ 22 | parts/ 23 | sdist/ 24 | var/ 25 | wheels/ 26 | share/python-wheels/ 27 | *.egg-info/ 28 | .installed.cfg 29 | *.egg 30 | MANIFEST 31 | 32 | # PyInstaller 33 | # Usually these files are written by a python script from a template 34 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 35 | *.manifest 36 | *.spec 37 | 38 | # Installer logs 39 | pip-log.txt 40 | pip-delete-this-directory.txt 41 | 42 | # Unit test / coverage reports 43 | htmlcov/ 44 | .tox/ 45 | .nox/ 46 | .coverage 47 | .coverage.* 48 | .cache 49 | nosetests.xml 50 | coverage.xml 51 | *.cover 52 | *.py,cover 53 | .hypothesis/ 54 | .pytest_cache/ 55 | cover/ 56 | 57 | # Translations 58 | *.mo 59 | *.pot 60 | 61 | # Django stuff: 62 | *.log 63 | local_settings.py 64 | db.sqlite3 65 | db.sqlite3-journal 66 | 67 | # Flask stuff: 68 | instance/ 69 | .webassets-cache 70 | 71 | # Scrapy stuff: 72 | .scrapy 73 | 74 | # Sphinx documentation 75 | docs/_build/ 76 | 77 | # PyBuilder 78 | .pybuilder/ 79 | target/ 80 | 81 | # Jupyter Notebook 82 | .ipynb_checkpoints 83 | 84 | # IPython 85 | profile_default/ 86 | ipython_config.py 87 | 88 | # pyenv 89 | # For a library or package, you might want to ignore these files since the code is 90 | # intended to run in multiple environments; otherwise, check them in: 91 | # .python-version 92 | 93 | # pipenv 94 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 95 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 96 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 97 | # install all needed dependencies. 98 | #Pipfile.lock 99 | 100 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 101 | __pypackages__/ 102 | 103 | # Celery stuff 104 | celerybeat-schedule 105 | celerybeat.pid 106 | 107 | # SageMath parsed files 108 | *.sage.py 109 | 110 | # Environments 111 | .env 112 | .venv 113 | env/ 114 | venv/ 115 | ENV/ 116 | env.bak/ 117 | venv.bak/ 118 | 119 | # Spyder project settings 120 | .spyderproject 121 | .spyproject 122 | 123 | # Rope project settings 124 | .ropeproject 125 | 126 | # mkdocs documentation 127 | /site 128 | 129 | # mypy 130 | .mypy_cache/ 131 | .dmypy.json 132 | dmypy.json 133 | 134 | # Pyre type checker 135 | .pyre/ 136 | 137 | # pytype static type analyzer 138 | .pytype/ 139 | 140 | # Cython debug symbols 141 | cython_debug/ 142 | -------------------------------------------------------------------------------- /gym_missile_command/game/batteries.py: -------------------------------------------------------------------------------- 1 | """Anti-missiles batteries.""" 2 | 3 | import cv2 4 | import numpy as np 5 | 6 | from gym_missile_command.configuration import CONFIG 7 | from gym_missile_command.utils import get_cv2_xy 8 | 9 | 10 | class Batteries(): 11 | """Anti-missiles batteries class. 12 | 13 | Attributes: 14 | NB_BATTERY (int): the number of batteries. It only supports 1 battery. 15 | 16 | batteries (numpy array): of size (N, 1) with N the number of batteries, 17 | i.d., 1. The feature is: (0) number of available missiles. 18 | nb_missiles_launched (int): the number of missiles launched. 19 | """ 20 | 21 | NB_BATTERIES = 1 22 | 23 | def __init__(self): 24 | """Initialize 1 battery.""" 25 | self.batteries = np.zeros((self.NB_BATTERIES, 1), dtype=np.float32) 26 | 27 | def reset(self, seed=None): 28 | """Reset batteries. 29 | 30 | Total number of missiles is reset to default. 31 | 32 | Warning: 33 | To fully initialize a Batteries object, init function and reset 34 | function musts be called. 35 | 36 | Args: 37 | seed (int): seed for reproducibility. 38 | """ 39 | self.batteries[:, 0] = CONFIG.FRIENDLY_MISSILES.NUMBER 40 | self.nb_missiles_launched = 0 41 | 42 | def step(self, action): 43 | """Go from current step to next one. 44 | 45 | The missile launched is done in the main environment class. 46 | 47 | Args: 48 | action (int): (0) do nothing, (1) target up, (2) target down, (3) 49 | target left, (4) target right, (5) fire missile. 50 | 51 | Returns: 52 | observation: None. 53 | 54 | reward: None. 55 | 56 | done: None. 57 | 58 | info (dict): additional information of the current time step. It 59 | contains key "can_fire" with associated value "True" if the 60 | anti-missile battery can fire a missile and "False" otherwise. 61 | """ 62 | can_fire = self.batteries[0, 0] > 0 63 | 64 | if action == 5 and can_fire: 65 | self.batteries[0, 0] -= 1 66 | 67 | return None, None, None, {"can_fire": can_fire} 68 | 69 | def render(self, observation): 70 | """Render anti-missiles batteries. 71 | 72 | Todo: 73 | Include the number of available missiles. 74 | 75 | Args: 76 | observation (numpy.array): the current environment observation 77 | representing the pixels. See the object description in the main 78 | environment class for information. 79 | """ 80 | cv2.circle( 81 | img=observation, 82 | center=(get_cv2_xy(CONFIG.EPISODE.HEIGHT, 83 | CONFIG.EPISODE.WIDTH, 84 | 0.0, 85 | 0.0)), 86 | radius=int(CONFIG.BATTERY.RADIUS), 87 | color=CONFIG.COLORS.BATTERY, 88 | thickness=-1, 89 | ) 90 | -------------------------------------------------------------------------------- /gym_missile_command/game/target.py: -------------------------------------------------------------------------------- 1 | """Target.""" 2 | 3 | import cv2 4 | 5 | from gym_missile_command.configuration import CONFIG 6 | from gym_missile_command.utils import get_cv2_xy 7 | 8 | 9 | class Target(): 10 | """Target class. 11 | 12 | Attributes: 13 | x (float): x position. 14 | y (float): y position. 15 | """ 16 | 17 | def __init__(self): 18 | """Initialize target.""" 19 | pass 20 | 21 | def reset(self, seed=None): 22 | """Reset target. 23 | 24 | Warning: 25 | To fully initialize a Batteries object, init function and reset 26 | function musts be called. 27 | 28 | Args: 29 | seed (int): seed for reproducibility. 30 | """ 31 | self.x = 0.0 32 | self.y = CONFIG.EPISODE.HEIGHT / 2 33 | 34 | def step(self, action): 35 | """Go from current step to next one. 36 | 37 | Args: 38 | action (int): (0) do nothing, (1) target up, (2) target down, (3) 39 | target left, (4) target right, (5) fire missile. 40 | 41 | Returns: 42 | observation: None. 43 | 44 | reward: None. 45 | 46 | done: None. 47 | 48 | info: None. 49 | """ 50 | if action == 1: 51 | self.y = min(CONFIG.EPISODE.HEIGHT, self.y + CONFIG.TARGET.VY) 52 | 53 | elif action == 2: 54 | self.y = max(0, self.y - CONFIG.TARGET.VY) 55 | 56 | elif action == 3: 57 | self.x = max(-CONFIG.EPISODE.WIDTH / 2, self.x - CONFIG.TARGET.VX) 58 | 59 | elif action == 4: 60 | self.x = min(CONFIG.EPISODE.WIDTH / 2, self.x + CONFIG.TARGET.VX) 61 | 62 | return None, None, None, None 63 | 64 | def render(self, observation): 65 | """Render target. 66 | 67 | The target is a cross, represented by 4 coordinates, 2 for the 68 | horizontal line and 2 for the vertical line. 69 | 70 | Args: 71 | observation (numpy.array): the current environment observation 72 | representing the pixels. See the object description in the main 73 | environment class for information. 74 | """ 75 | # Horizontal 76 | cv2.line( 77 | img=observation, 78 | pt1=(get_cv2_xy(CONFIG.EPISODE.HEIGHT, 79 | CONFIG.EPISODE.WIDTH, 80 | self.x - CONFIG.TARGET.SIZE, 81 | self.y)), 82 | pt2=(get_cv2_xy(CONFIG.EPISODE.HEIGHT, 83 | CONFIG.EPISODE.WIDTH, 84 | self.x + CONFIG.TARGET.SIZE, 85 | self.y)), 86 | color=CONFIG.COLORS.TARGET, 87 | thickness=1, 88 | ) 89 | 90 | # Vertical 91 | cv2.line( 92 | img=observation, 93 | pt1=(get_cv2_xy(CONFIG.EPISODE.HEIGHT, 94 | CONFIG.EPISODE.WIDTH, 95 | self.x, 96 | self.y + CONFIG.TARGET.SIZE)), 97 | pt2=(get_cv2_xy(CONFIG.EPISODE.HEIGHT, 98 | CONFIG.EPISODE.WIDTH, 99 | self.x, 100 | self.y - CONFIG.TARGET.SIZE)), 101 | color=CONFIG.COLORS.TARGET, 102 | thickness=1, 103 | ) 104 | -------------------------------------------------------------------------------- /CONFIG.md: -------------------------------------------------------------------------------- 1 | # CONFIG 2 | 3 | Below are all the editable variables. 4 | It is highly recommended to read it carefully, as understanding each variable leads to a better understanding of the overall environment. 5 | Default parameters are in [./gym_missile_command/configuration/config.py](./gym_missile_command/configuration/config.py). 6 | 7 | ```python 8 | env_context = { 9 | "EPISODE.FPS": 144, 10 | "EPISODE.HEIGHT": 466, 11 | "EPISODE.WIDTH": 966, 12 | "BATTERY.RADIUS": 37.0, 13 | "CITIES.NUMBER": 6, 14 | "CITIES.RADIUS": 24.0, 15 | "COLORS.BACKGROUND": (0, 0, 0), 16 | "COLORS.BATTERY": (255, 255, 255), 17 | "COLORS.CITY": (0, 0, 255), 18 | "COLORS.ENEMY_MISSILE": (255, 0, 0), 19 | "COLORS.EXPLOSION": (255, 255, 0), 20 | "COLORS.FRIENDLY_MISSILE": (0, 255, 0), 21 | "COLORS.TARGET": (255, 255, 255), 22 | "ENEMY_MISSILES.NUMBER": 19, 23 | "ENEMY_MISSILES.PROBA_IN": 0.005, 24 | "ENEMY_MISSILES.RADIUS": 4.0, 25 | "ENEMY_MISSILES.SPEED": 1.0, 26 | "FRIENDLY_MISSILES.NUMBER": 142, 27 | "FRIENDLY_MISSILES.EXPLOSION_RADIUS": 37.0, 28 | "FRIENDLY_MISSILES.EXPLOSION_SPEED": 0.5, 29 | "FRIENDLY_MISSILES.RADIUS": 7.0, 30 | "FRIENDLY_MISSILES.SPEED": 7.0, 31 | "OBSERVATION.HEIGHT": 84, 32 | "OBSERVATION.RENDER_PROCESSED_HEIGHT": 250, 33 | "OBSERVATION.RENDER_PROCESSED_WIDTH": 250, 34 | "OBSERVATION.WIDTH": 84, 35 | "REWARD.DESTROYED_CITY": -10.0, 36 | "REWARD.DESTROYED_ENEMEY_MISSILES": 15.0, 37 | "REWARD.FRIENDLY_MISSILE_LAUNCHED": -4.0, 38 | "TARGET.SIZE": 12, 39 | "TARGET.VX": 4, 40 | "TARGET.VY": 4, 41 | } 42 | ``` 43 | 44 | ### General 45 | 46 | `"EPISODE.FPS"` 47 | Frame per second for the rendering function. 48 | 49 | `"EPISODE.HEIGHT"` 50 | Height of the rendering window. 51 | 52 | `"EPISODE.WIDTH"` 53 | Width of the rendering window. 54 | 55 | ### Battery 56 | 57 | `"BATTERY.RADIUS"` 58 | Radius of the anti-missile battery object. 59 | 60 | ### Cities 61 | 62 | `"CITIES.NUMBER"` 63 | Number of cities to defend (even integer >= 2). 64 | 65 | `"CITIES.RADIUS"` 66 | Radius of a city object. 67 | 68 | ### Colors 69 | 70 | Colors of each object for the rendering function. 71 | 72 | `"COLORS.BACKGROUND"` 73 | 74 | `"COLORS.BATTERY"` 75 | 76 | `"COLORS.CITY"` 77 | 78 | `"COLORS.ENEMY_MISSILE"` 79 | 80 | `"COLORS.EXPLOSION"` 81 | 82 | `"COLORS.FRIENDLY_MISSILE"` 83 | 84 | `"COLORS.TARGET"` 85 | 86 | ### Enemy missiles 87 | 88 | `"ENEMY_MISSILES.NUMBER"` 89 | Total number of enemy missiles for 1 episode. 90 | 91 | `"ENEMY_MISSILES.PROBA_IN"` 92 | Probability for an enemy missile to appear at a time step. 93 | 94 | `"ENEMY_MISSILES.RADIUS"` 95 | Radius of an enemy missile object. 96 | 97 | `"ENEMY_MISSILES.SPEED"` 98 | Enemy missile speed. 99 | 100 | ### Friendly missiles 101 | 102 | `"FRIENDLY_MISSILES.NUMBER"` 103 | Total number of available friendly missiles. 104 | 105 | `"FRIENDLY_MISSILES.EXPLOSION_RADIUS"` 106 | Maximum explosion radius. 107 | 108 | `"FRIENDLY_MISSILES.EXPLOSION_SPEED"` 109 | Speed of the explosion. 110 | 111 | `"FRIENDLY_MISSILES.RADIUS"` 112 | Radius of a friendly missile object. 113 | 114 | `"FRIENDLY_MISSILES.SPEED"` 115 | Friendly missile speed. 116 | 117 | ### Observation 118 | 119 | An agent takes as input the screen pixels. 120 | The resolution of the environment can be quite big: the computational cost to train an agent can then be high. 121 | For an agent to well perform on the Missile Command Atari game, a smaller resized version of the observation can be enough. 122 | So the environment returns a resized version of the environment observation. 123 | If you wish to not resize the observation, fix these variables to the same values as CONFIG.EPISODE.HEIGHT and CONFIG.EPISODE.WIDTH. 124 | 125 | `"OBSERVATION.HEIGHT"` 126 | Observation height. 127 | 128 | `"OBSERVATION.RENDER_PROCESSED_HEIGHT"` 129 | Render window height of the processed observation. 130 | 131 | `"OBSERVATION.RENDER_PROCESSED_WIDTH"` 132 | Render window width of the processed observation. 133 | 134 | `"OBSERVATION.WIDTH"` 135 | Observation width. 136 | 137 | ### Reward 138 | 139 | `"REWARD.DESTROYED_CITY"` 140 | Reward for each destroyed city. 141 | 142 | `"REWARD.DESTROYED_ENEMEY_MISSILES"` 143 | Reward for each destroyed missile. 144 | 145 | `"REWARD.FRIENDLY_MISSILE_LAUNCHED"` 146 | Reward for each friendly missile launched. 147 | 148 | ### Target 149 | 150 | `"TARGET.SIZE"` 151 | Target size (only for the rendering function). 152 | 153 | `"TARGET.VX"` 154 | Horizontal shifting of the target. 155 | 156 | `"TARGET.VY"` 157 | Vertical shifting of the target. 158 | -------------------------------------------------------------------------------- /gym_missile_command/game/cities.py: -------------------------------------------------------------------------------- 1 | """Cities.""" 2 | 3 | import sys 4 | 5 | import cv2 6 | import numpy as np 7 | 8 | from gym_missile_command.configuration import CONFIG 9 | from gym_missile_command.utils import get_cv2_xy 10 | 11 | 12 | class Cities(): 13 | """Cities class. 14 | 15 | Attributes: 16 | MAX_HEALTH (float): value corresponding to the max health of a city. 17 | Each time an enemy missile destroys a city, it loses 1 point of 18 | heath. At 0, the city is completely destroyed. 19 | 20 | cities (numpy array): of size (N, 3) with N the number of cities. The 21 | features are: (0) x position, (1) y position and (2) integrity 22 | level (0 if destroyed else 1). 23 | """ 24 | 25 | MAX_HEALTH = 1.0 26 | 27 | def __init__(self): 28 | """Initialize cities.""" 29 | # First initializations 30 | # ------------------------------------------ 31 | 32 | # The length of free space for the cities (only for one side) 33 | free_space = 0.5 * CONFIG.EPISODE.WIDTH - CONFIG.BATTERY.RADIUS 34 | 35 | # Creation of the main numpy array 36 | self.cities = np.zeros((CONFIG.CITIES.NUMBER, 3), dtype=np.float32) 37 | 38 | # Check for errors 39 | # ------------------------------------------ 40 | 41 | # Even number of cities for symmetry, bigger or equal to 2 42 | if CONFIG.CITIES.NUMBER % 2 != 0 or CONFIG.CITIES.NUMBER < 2: 43 | sys.exit("Please choose an even number of cities, bigger or equal " 44 | "to 2.") 45 | 46 | # Enough space is needed for objects not to not overlap 47 | if free_space < CONFIG.CITIES.NUMBER * CONFIG.CITIES.RADIUS: 48 | sys.exit("Not enough space for the cities. Increase width, " 49 | "decrease the number of cities or decrease objects " 50 | "radiuses.") 51 | 52 | # Compute x positions 53 | # ------------------------------------------ 54 | 55 | # Gap between cities 56 | gap = (free_space - CONFIG.CITIES.NUMBER * CONFIG.CITIES.RADIUS) \ 57 | / (0.5 * CONFIG.CITIES.NUMBER + 1) 58 | 59 | # First position, last position and step between cities centers 60 | start = CONFIG.BATTERY.RADIUS + gap + CONFIG.CITIES.RADIUS 61 | step = gap + 2 * CONFIG.CITIES.RADIUS 62 | stop = 0.5 * CONFIG.EPISODE.WIDTH - gap 63 | half_cities_nb = int(CONFIG.CITIES.NUMBER / 2) 64 | 65 | # Cities on the left side 66 | self.cities[:half_cities_nb, 0] = -np.arange(start=start, 67 | stop=stop, 68 | step=step, 69 | dtype=np.float32) 70 | 71 | # Cities on the right side 72 | self.cities[half_cities_nb:, 0] = np.arange(start=start, 73 | stop=stop, 74 | step=step, 75 | dtype=np.float32) 76 | 77 | def get_remaining_cities(self): 78 | """Compute healthy cities number. 79 | 80 | Returns:opencv draw multiple circles 81 | nb_remaining_cities (int): the number of remaining cities. 82 | """ 83 | return np.sum(self.cities[:, 2] == self.MAX_HEALTH) 84 | 85 | def reset(self, seed=None): 86 | """Reset the environment. 87 | 88 | Integrity is reset to 1 for all cities. 89 | 90 | Warning: 91 | To fully initialize a Cities object, init function and reset 92 | function musts be called. 93 | 94 | Args: 95 | seed (int): seed for reproducibility. 96 | """ 97 | self.cities[:, 2] = self.MAX_HEALTH 98 | 99 | def step(self, action): 100 | """Go from current step to next one. 101 | 102 | Destructions by enemy missiles are checked in the main environment 103 | class. 104 | 105 | Args: 106 | action (int): (0) do nothing, (1) target up, (2) target down, (3) 107 | target left, (4) target right, (5) fire missile. 108 | 109 | returns: 110 | observation: None. 111 | 112 | reward: None. 113 | 114 | done (bool): True if the episode is finished, i.d. all cities are 115 | destroyed. False otherwise. 116 | 117 | info: None. 118 | """ 119 | done = np.all(self.cities[:, 2] == 0.0) 120 | return None, None, done, None 121 | 122 | def render(self, observation): 123 | """Render cities. 124 | 125 | Todo: 126 | Include the integrity level. 127 | 128 | Args: 129 | observation (numpy.array): the current environment observation 130 | representing the pixels. See the object description in the main 131 | environment class for information. 132 | """ 133 | for x, y, integrity in zip(self.cities[:, 0], 134 | self.cities[:, 1], 135 | self.cities[:, 2]): 136 | if integrity > 0: 137 | cv2.circle( 138 | img=observation, 139 | center=(get_cv2_xy(CONFIG.EPISODE.HEIGHT, 140 | CONFIG.EPISODE.WIDTH, 141 | x, 142 | y)), 143 | radius=int(CONFIG.CITIES.RADIUS), 144 | color=CONFIG.COLORS.CITY, 145 | thickness=-1, 146 | ) 147 | -------------------------------------------------------------------------------- /gym_missile_command/game/enemy_missiles.py: -------------------------------------------------------------------------------- 1 | """Enemy missiles.""" 2 | 3 | from random import Random 4 | 5 | import cv2 6 | import numpy as np 7 | 8 | from gym_missile_command.configuration import CONFIG 9 | from gym_missile_command.utils import get_cv2_xy 10 | 11 | 12 | class EnemyMissiles(): 13 | """Enemy missiles class. 14 | 15 | Enemy missiles are created by the environment. 16 | 17 | Attributes: 18 | enemy_missiles (numpy array): of size (N, 8) with N the number of 19 | enemy missiles present in the environment. The features are: (0) 20 | initial x position, (1) initial y position, (2) current x position, 21 | (3) current y position, (4) final x position, (5) final y position, 22 | (6) horizontal speed vx and (7) vertical speed vy. 23 | nb_missiles_launched (int): the number of enemy missiles launched in 24 | the environment. 25 | """ 26 | 27 | def __init__(self): 28 | """Initialize missiles.""" 29 | pass 30 | 31 | def _launch_missile(self): 32 | """Launch a new missile. 33 | 34 | - 0) Generate initial and final positions. 35 | - 1) Compute speed vectors. 36 | - 2) Add the new missile. 37 | """ 38 | # Generate initial and final positions 39 | # ------------------------------------------ 40 | 41 | # Initial position 42 | x0 = self._rng_python.uniform( 43 | -0.5 * CONFIG.EPISODE.WIDTH, 0.5 * CONFIG.EPISODE.WIDTH) 44 | y0 = CONFIG.EPISODE.HEIGHT 45 | 46 | # Final position 47 | x1 = self._rng_python.uniform( 48 | -0.5 * CONFIG.EPISODE.WIDTH, 0.5 * CONFIG.EPISODE.WIDTH) 49 | y1 = 0.0 50 | 51 | # Compute speed vectors 52 | # ------------------------------------------ 53 | 54 | # Compute norm 55 | norm = np.sqrt(np.square(x1 - x0) + np.square(y1 - y0)) 56 | 57 | # Compute unit vectors 58 | ux = (x1 - x0) / norm 59 | uy = (y1 - y0) / norm 60 | 61 | # Compute speed vectors 62 | vx = CONFIG.ENEMY_MISSILES.SPEED * ux 63 | vy = CONFIG.ENEMY_MISSILES.SPEED * uy 64 | 65 | # Add the new missile 66 | # ------------------------------------------ 67 | 68 | # Create the missile 69 | new_missile = np.array( 70 | [[x0, y0, x0, y0, x1, y1, vx, vy]], 71 | dtype=np.float32, 72 | ) 73 | 74 | # Add it to the others 75 | self.enemy_missiles = np.vstack( 76 | (self.enemy_missiles, new_missile)) 77 | 78 | # Increase number of launched missiles 79 | self.nb_missiles_launched += 1 80 | 81 | def reset(self, seed=None): 82 | """Reset enemy missiles. 83 | 84 | Warning: 85 | To fully initialize a EnemyMissiles object, init function and reset 86 | function must be called. 87 | 88 | Args: 89 | seed (int): seed for reproducibility. 90 | """ 91 | self.enemy_missiles = np.zeros((0, 8), dtype=np.float32) 92 | self.nb_missiles_launched = 0 93 | 94 | # Create random numbers generator 95 | self._rng_python = Random(seed) 96 | 97 | def step(self, action): 98 | """Go from current step to next one. 99 | 100 | - 0) Moving missiles. 101 | - 1) Potentially launch a new missile. 102 | - 2) Remove missiles that hit the ground. 103 | 104 | Collisions with friendly missiles and / or cities are checked in the 105 | main environment class. 106 | 107 | Notes: 108 | From one step to another, a missile could exceed its final 109 | position, so we need to do some verification. This issue is due to 110 | the discrete nature of environment, decomposed in time steps. 111 | 112 | Args: 113 | action (int): (0) do nothing, (1) target up, (2) target down, (3) 114 | target left, (4) target right, (5) fire missile. 115 | 116 | returns: 117 | observation: None. 118 | 119 | reward: None. 120 | 121 | done (bool): True if the episode is finished, i.d. there are no 122 | more enemy missiles in the environment and no more enemy 123 | missiles to be launch. False otherwise. 124 | 125 | info: None. 126 | """ 127 | # Moving missiles 128 | # ------------------------------------------ 129 | 130 | # Compute horizontal and vertical distances to targets 131 | dx = np.abs(self.enemy_missiles[:, 4] - self.enemy_missiles[:, 2]) 132 | dy = np.abs(self.enemy_missiles[:, 5] - self.enemy_missiles[:, 3]) 133 | 134 | # Take the minimum between the actual speed and the distance to target 135 | movement_x = np.minimum(np.abs(self.enemy_missiles[:, 6]), dx) 136 | movement_y = np.minimum(np.abs(self.enemy_missiles[:, 7]), dy) 137 | 138 | # Keep the right sign 139 | movement_x *= np.sign(self.enemy_missiles[:, 6]) 140 | movement_y *= np.sign(self.enemy_missiles[:, 7]) 141 | 142 | # Step t to step t+1 143 | self.enemy_missiles[:, 2] += movement_x 144 | self.enemy_missiles[:, 3] += movement_y 145 | 146 | # Potentially launch a new missile 147 | # ------------------------------------------ 148 | 149 | if self.nb_missiles_launched < CONFIG.ENEMY_MISSILES.NUMBER: 150 | if self._rng_python.random() <= CONFIG.ENEMY_MISSILES.PROBA_IN: 151 | self._launch_missile() 152 | 153 | # Remove missiles that hit the ground 154 | # ------------------------------------------ 155 | 156 | missiles_out_indices = np.squeeze(np.argwhere( 157 | (self.enemy_missiles[:, 2] == self.enemy_missiles[:, 4]) & 158 | (self.enemy_missiles[:, 3] == self.enemy_missiles[:, 5]) 159 | )) 160 | 161 | self.enemy_missiles = np.delete( 162 | self.enemy_missiles, missiles_out_indices, axis=0) 163 | 164 | done = self.enemy_missiles.shape[0] == 0 and \ 165 | self.nb_missiles_launched == CONFIG.ENEMY_MISSILES.NUMBER 166 | return None, None, done, None 167 | 168 | def render(self, observation): 169 | """Render enemy missiles. 170 | 171 | For each enemy missiles, draw a line of its trajectory and the actual 172 | missile. 173 | 174 | Args: 175 | observation (numpy.array): the current environment observation 176 | representing the pixels. See the object description in the main 177 | environment class for information. 178 | """ 179 | for x0, y0, x, y in zip(self.enemy_missiles[:, 0], 180 | self.enemy_missiles[:, 1], 181 | self.enemy_missiles[:, 2], 182 | self.enemy_missiles[:, 3]): 183 | cv2.line( 184 | img=observation, 185 | pt1=(get_cv2_xy(CONFIG.EPISODE.HEIGHT, 186 | CONFIG.EPISODE.WIDTH, 187 | x0, 188 | y0)), 189 | pt2=(get_cv2_xy(CONFIG.EPISODE.HEIGHT, 190 | CONFIG.EPISODE.WIDTH, 191 | x, 192 | y)), 193 | color=CONFIG.COLORS.ENEMY_MISSILE, 194 | thickness=1, 195 | ) 196 | 197 | cv2.circle( 198 | img=observation, 199 | center=(get_cv2_xy(CONFIG.EPISODE.HEIGHT, 200 | CONFIG.EPISODE.WIDTH, 201 | x, 202 | y)), 203 | radius=int(CONFIG.ENEMY_MISSILES.RADIUS), 204 | color=CONFIG.COLORS.ENEMY_MISSILE, 205 | thickness=-1, 206 | ) 207 | -------------------------------------------------------------------------------- /gym_missile_command/game/friendly_missiles.py: -------------------------------------------------------------------------------- 1 | """Friendly missiles.""" 2 | 3 | import cv2 4 | import numpy as np 5 | 6 | from gym_missile_command.configuration import CONFIG 7 | from gym_missile_command.utils import get_cv2_xy 8 | 9 | 10 | class FriendlyMissiles(): 11 | """Friendly missiles class. 12 | 13 | Friendly missiles are send by the user from the anti-missiles battery. 14 | 15 | Attributes: 16 | ORIGIN_X (float): x origin position of the anti-missiles battery. 17 | ORIGIN_Y (float): y origin position of the anti-missiles battery. 18 | 19 | missiles_movement (numpy array): of size (N, 6) with N the number of 20 | missiles in movement in the environment. The features are: (0) 21 | current x position, (1) current y position, (2) final x position, 22 | (3) final y position, (4) horizontal speed vx and (5) vertical 23 | speed (vy). 24 | missiles_explosion (numpy array): of size (M, 3) with M the number of 25 | missiles in explosion. The features are: (0) x position, (1) y 26 | position and (2) explosion level. 27 | """ 28 | 29 | ORIGIN_X = 0.0 30 | ORIGIN_Y = 0.0 31 | 32 | def __init__(self): 33 | """Initialize friendly missiles. 34 | 35 | Notes: 36 | No need to keep trace of the origin fire position because all 37 | missiles are launched from the origin (0, 0). 38 | """ 39 | pass 40 | 41 | def launch_missile(self, target): 42 | """Launch a new missile. 43 | 44 | - 0) Compute speed vectors. 45 | - 1) Add the new missile. 46 | 47 | Notes: 48 | When executing this function, not forget to decrease the number of 49 | available missiles of the anti-missiles battery by 1. 50 | 51 | Args: 52 | target (Target): Target object. 53 | """ 54 | # Compute speed vectors 55 | # ------------------------------------------ 56 | 57 | # Careful not to divide by 0 if target is in (0, 0) 58 | if target.x == self.ORIGIN_X and target.y == self.ORIGIN_Y: 59 | vx, vy = 0.0, 0.0 60 | 61 | # We can safely compute speed vectors 62 | else: 63 | 64 | # Compute norm 65 | norm = np.sqrt(np.square(target.x) + np.square(target.y)) 66 | 67 | # Compute unit vectors 68 | ux = target.x / norm 69 | uy = target.y / norm 70 | 71 | # Compute speed vectors 72 | vx = CONFIG.FRIENDLY_MISSILES.SPEED * ux 73 | vy = CONFIG.FRIENDLY_MISSILES.SPEED * uy 74 | 75 | # Add the new missile 76 | # ------------------------------------------ 77 | 78 | # Create the missile 79 | new_missile = np.array( 80 | [[self.ORIGIN_X, self.ORIGIN_Y, target.x, target.y, vx, vy]], 81 | dtype=np.float32, 82 | ) 83 | 84 | # Add it to the others 85 | self.missiles_movement = np.vstack( 86 | (self.missiles_movement, new_missile)) 87 | 88 | def reset(self, seed=None): 89 | """Reset friendly missiles. 90 | 91 | Warning: 92 | To fully initialize a FriendlyMissiles object, init function and 93 | reset function must be called. 94 | 95 | Args: 96 | seed (int): seed for reproducibility. 97 | """ 98 | self.missiles_movement = np.zeros((0, 6), dtype=np.float32) 99 | self.missiles_explosion = np.zeros((0, 3), dtype=np.float32) 100 | 101 | def step(self, action): 102 | """Go from current step to next one. 103 | 104 | - 0) Moving missiles. 105 | - 1) Exploding missiles. 106 | - 2) New exploding missiles. 107 | - 3) Remove missiles with full explosion. 108 | 109 | Friendly missiles destroying enemy missiles are checked in the main 110 | environment class. 111 | 112 | Notes: 113 | From one step to another, a missile could exceed its final 114 | position, so we need to do some verification. This issue is due to 115 | the discrete nature of environment, decomposed in time steps. 116 | 117 | Args: 118 | action (int): (0) do nothing, (1) target up, (2) target down, (3) 119 | target left, (4) target right, (5) fire missile. 120 | 121 | returns: 122 | observation: None. 123 | 124 | reward: None. 125 | 126 | done: None. 127 | 128 | info: None. 129 | """ 130 | # Moving missiles 131 | # ------------------------------------------ 132 | 133 | # Compute horizontal and vertical distances to targets 134 | dx = np.abs( 135 | self.missiles_movement[:, 2] - self.missiles_movement[:, 0]) 136 | dy = np.abs( 137 | self.missiles_movement[:, 3] - self.missiles_movement[:, 1]) 138 | 139 | # Take the minimum between the actual speed and the distance to target 140 | movement_x = np.minimum(np.abs(self.missiles_movement[:, 4]), dx) 141 | movement_y = np.minimum(np.abs(self.missiles_movement[:, 5]), dy) 142 | 143 | # Keep the good sign 144 | movement_x *= np.sign(self.missiles_movement[:, 4]) 145 | movement_y *= np.sign(self.missiles_movement[:, 5]) 146 | 147 | # Step t to step t+1 148 | self.missiles_movement[:, 0] += movement_x 149 | self.missiles_movement[:, 1] += movement_y 150 | 151 | # Exploding missiles 152 | # ------------------------------------------ 153 | 154 | # Increase by 1 the explosion 155 | self.missiles_explosion[:, 2] += \ 156 | CONFIG.FRIENDLY_MISSILES.EXPLOSION_SPEED 157 | 158 | # New exploding missiles 159 | # ------------------------------------------ 160 | 161 | # Indices of new exploding missiles 162 | new_exploding_missiles_indices = np.argwhere( 163 | (self.missiles_movement[:, 0] == self.missiles_movement[:, 2]) & 164 | (self.missiles_movement[:, 1] == self.missiles_movement[:, 3]) 165 | ) 166 | nb_new_exploding_missiles = new_exploding_missiles_indices.shape[0] 167 | # print(new_exploding_missiles_indices) 168 | 169 | if nb_new_exploding_missiles > 0: 170 | new_exploding_missiles_indices = np.squeeze( 171 | new_exploding_missiles_indices) 172 | 173 | # Get positions 174 | x = self.missiles_movement[new_exploding_missiles_indices, 0] 175 | y = self.missiles_movement[new_exploding_missiles_indices, 1] 176 | 177 | # Remove missiles 178 | self.missiles_movement = np.delete( 179 | self.missiles_movement, new_exploding_missiles_indices, axis=0) 180 | 181 | # Create new ones 182 | new_exploding_missiles = np.zeros( 183 | (nb_new_exploding_missiles, 3), 184 | dtype=np.float32, 185 | ) 186 | 187 | # Affect positions 188 | new_exploding_missiles[:, 0] = x 189 | new_exploding_missiles[:, 1] = y 190 | 191 | # Add them 192 | self.missiles_explosion = np.vstack( 193 | (self.missiles_explosion, new_exploding_missiles)) 194 | 195 | # Remove missiles with full explosion 196 | # ------------------------------------------ 197 | 198 | full_explosion_indices = np.squeeze(np.argwhere( 199 | (self.missiles_explosion[:, 2] > 200 | CONFIG.FRIENDLY_MISSILES.EXPLOSION_RADIUS) 201 | )) 202 | 203 | self.missiles_explosion = np.delete( 204 | self.missiles_explosion, full_explosion_indices, axis=0) 205 | 206 | return None, None, None, None 207 | 208 | def render(self, observation): 209 | """Render friendly missiles. 210 | 211 | Args: 212 | observation (numpy.array): the current environment observation 213 | representing the pixels. See the object description in the main 214 | environment class for information. 215 | """ 216 | # Moving missiles 217 | # ------------------------------------------ 218 | 219 | for x, y in zip(self.missiles_movement[:, 0], 220 | self.missiles_movement[:, 1]): 221 | cv2.line( 222 | img=observation, 223 | pt1=(get_cv2_xy(CONFIG.EPISODE.HEIGHT, 224 | CONFIG.EPISODE.WIDTH, 225 | 0.0, 226 | 0.0)), 227 | pt2=(get_cv2_xy(CONFIG.EPISODE.HEIGHT, 228 | CONFIG.EPISODE.WIDTH, 229 | x, 230 | y)), 231 | color=CONFIG.COLORS.FRIENDLY_MISSILE, 232 | thickness=1, 233 | ) 234 | 235 | cv2.circle( 236 | img=observation, 237 | center=(get_cv2_xy(CONFIG.EPISODE.HEIGHT, 238 | CONFIG.EPISODE.WIDTH, 239 | x, 240 | y)), 241 | radius=int(CONFIG.FRIENDLY_MISSILES.RADIUS), 242 | color=CONFIG.COLORS.FRIENDLY_MISSILE, 243 | thickness=-1, 244 | ) 245 | 246 | # Exploding missiles 247 | # ------------------------------------------ 248 | 249 | for x, y, explosion in zip(self.missiles_explosion[:, 0], 250 | self.missiles_explosion[:, 1], 251 | self.missiles_explosion[:, 2]): 252 | cv2.circle( 253 | img=observation, 254 | center=(get_cv2_xy(CONFIG.EPISODE.HEIGHT, 255 | CONFIG.EPISODE.WIDTH, 256 | x, 257 | y)), 258 | radius=int(explosion), 259 | color=CONFIG.COLORS.EXPLOSION, 260 | thickness=-1, 261 | ) 262 | -------------------------------------------------------------------------------- /gym_missile_command/environment/missile_command.py: -------------------------------------------------------------------------------- 1 | """Main environment class.""" 2 | 3 | import cv2 4 | import gymnasium as gym 5 | import numpy as np 6 | import pygame 7 | from gymnasium import spaces 8 | 9 | from gym_missile_command.configuration import CONFIG, update_config 10 | from gym_missile_command.game.batteries import Batteries 11 | from gym_missile_command.game.cities import Cities 12 | from gym_missile_command.game.enemy_missiles import EnemyMissiles 13 | from gym_missile_command.game.friendly_missiles import FriendlyMissiles 14 | from gym_missile_command.game.target import Target 15 | 16 | 17 | class MissileCommandEnv(gym.Env): 18 | """Missile Command Gym environment. 19 | 20 | Attributes: 21 | NB_ACTIONS (int): the 6 possible actions. (0) do nothing, (1) target 22 | up, (2) target down, (3) target left, (4) target right, (5) fire 23 | missile. 24 | 25 | action_space (gymnasim.spaces.discrete.Discrete): action space. 26 | observation_space (gymnasium.spaces.Box): observation space. 27 | reward_range (tuple): reward range. 28 | 29 | batteries (Batteries): Batteries game object. 30 | cities (Cities): Cities game object. 31 | enemy_missiles (EnemyMissiles): EnemyMissiles game object. 32 | friendly_missiles (FriendlyMissiles): FriendlyMissiles game object. 33 | 34 | observation (numpy.array): of size (CONFIG.EPISODE.WIDTH, 35 | CONFIG.EPISODE.HEIGHT, 3). The observation of the current time] 36 | step, representing the RGB values of each pixel. 37 | reward (float): reward of the current time step. 38 | reward_total (float): reward sum from first time step to current one. 39 | time_step (int): current time step, starts from 0. 40 | """ 41 | 42 | NB_ACTIONS = 6 43 | 44 | def __init__(self, env_context=None): 45 | """Initialize environment. 46 | 47 | Args: 48 | env_context (dict): environment configuration. 49 | """ 50 | # Update configuration 51 | if env_context is not None: 52 | update_config(env_context) 53 | 54 | # Action and observation spaces 55 | self.action_space = spaces.Discrete(self.NB_ACTIONS) 56 | self.observation_space = spaces.Box( 57 | low=0, 58 | high=255, 59 | shape=(CONFIG.OBSERVATION.HEIGHT, CONFIG.OBSERVATION.WIDTH, 3), 60 | dtype=np.uint8, 61 | ) 62 | 63 | # TODO: compute reward bounds 64 | self.reward_range = (-float("inf"), float("inf")) 65 | 66 | # Objects 67 | self.batteries = Batteries() 68 | self.cities = Cities() 69 | self.enemy_missiles = EnemyMissiles() 70 | self.friendly_missiles = FriendlyMissiles() 71 | self.target = Target() 72 | 73 | # No display while no render 74 | self._clock = None 75 | self._display = None 76 | 77 | def _collisions_cities(self): 78 | """Check for cities collisions. 79 | 80 | Check cities destroyed by enemy missiles. 81 | """ 82 | # Cities 83 | cities = self.cities.cities 84 | 85 | # Enemy missiles current positions 86 | enemy_m = self.enemy_missiles.enemy_missiles[:, [2, 3]] 87 | 88 | # Align cities and enemy missiles 89 | cities_dup = np.repeat(cities, enemy_m.shape[0], axis=0) 90 | enemy_m_dup = np.tile(enemy_m, reps=[cities.shape[0], 1]) 91 | 92 | # Compute distances 93 | dx = enemy_m_dup[:, 0] - cities_dup[:, 0] 94 | dy = enemy_m_dup[:, 1] - cities_dup[:, 1] 95 | distances = np.sqrt(np.square(dx) + np.square(dy)) 96 | 97 | # Get cities destroyed by enemy missiles 98 | exploded = distances <= ( 99 | CONFIG.ENEMY_MISSILES.RADIUS + CONFIG.CITIES.RADIUS) 100 | exploded = exploded.astype(int) 101 | exploded = np.reshape(exploded, (cities.shape[0], enemy_m.shape[0])) 102 | 103 | # Get destroyed cities 104 | cities_out = np.argwhere( 105 | (np.sum(exploded, axis=1) >= 1) & 106 | (cities[:, 2] > 0.0) 107 | ) 108 | 109 | # Update time step reward 110 | self.reward += CONFIG.REWARD.DESTROYED_CITY * \ 111 | cities_out.shape[0] 112 | 113 | # Destroy the cities 114 | self.cities.cities[cities_out, 2] -= 1 115 | 116 | def _collisions_missiles(self): 117 | """Check for missiles collisions. 118 | 119 | Check enemy missiles destroyed by friendly exploding missiles. 120 | """ 121 | # Friendly exploding missiles 122 | friendly_exploding = self.friendly_missiles.missiles_explosion 123 | 124 | # Enemy missiles current positions 125 | enemy_missiles = self.enemy_missiles.enemy_missiles[:, [2, 3]] 126 | 127 | # Align enemy missiles and friendly exploding ones 128 | enemy_m_dup = np.repeat(enemy_missiles, 129 | friendly_exploding.shape[0], 130 | axis=0) 131 | friendly_e_dup = np.tile(friendly_exploding, 132 | reps=[enemy_missiles.shape[0], 1]) 133 | 134 | # Compute distances 135 | dx = friendly_e_dup[:, 0] - enemy_m_dup[:, 0] 136 | dy = friendly_e_dup[:, 1] - enemy_m_dup[:, 1] 137 | distances = np.sqrt(np.square(dx) + np.square(dy)) 138 | 139 | # Get enemy missiles inside an explosion radius 140 | inside_radius = distances <= ( 141 | friendly_e_dup[:, 2] + CONFIG.ENEMY_MISSILES.RADIUS) 142 | inside_radius = inside_radius.astype(int) 143 | inside_radius = np.reshape( 144 | inside_radius, 145 | (enemy_missiles.shape[0], friendly_exploding.shape[0]), 146 | ) 147 | 148 | # Remove these missiles 149 | missiles_out = np.argwhere(np.sum(inside_radius, axis=1) >= 1) 150 | self.enemy_missiles.enemy_missiles = np.delete( 151 | self.enemy_missiles.enemy_missiles, 152 | np.squeeze(missiles_out), 153 | axis=0, 154 | ) 155 | 156 | # Compute current reward 157 | nb_missiles_destroyed = missiles_out.shape[0] 158 | self.reward += CONFIG.REWARD.DESTROYED_ENEMEY_MISSILES * \ 159 | nb_missiles_destroyed 160 | 161 | def _compute_observation(self): 162 | """Compute observation.""" 163 | # Reset observation 164 | self.observation = np.zeros( 165 | (CONFIG.EPISODE.WIDTH, CONFIG.EPISODE.HEIGHT, 3), dtype=np.uint8) 166 | self.observation[:, :, 0] = CONFIG.COLORS.BACKGROUND[0] 167 | self.observation[:, :, 1] = CONFIG.COLORS.BACKGROUND[1] 168 | self.observation[:, :, 2] = CONFIG.COLORS.BACKGROUND[2] 169 | 170 | # Draw objects 171 | self.batteries.render(self.observation) 172 | self.cities.render(self.observation) 173 | self.enemy_missiles.render(self.observation) 174 | self.friendly_missiles.render(self.observation) 175 | self.target.render(self.observation) 176 | 177 | def _process_observation(self): 178 | """Process observation. 179 | 180 | This function could be implemented into the agent model, but for 181 | commodity this environment can do it directly. 182 | 183 | The interpolation mode INTER_AREA seems to give the best results. With 184 | other methods, every objects could not be seen at all time steps. 185 | 186 | Returns: 187 | processed_observation (numpy.array): of size 188 | (CONFIG.OBSERVATION.HEIGHT, CONFIG.OBSERVATION.WIDTH, 3), the 189 | resized (or not) observation. 190 | """ 191 | # Process observation 192 | processed_observation = cv2.resize( 193 | self.observation, 194 | (CONFIG.OBSERVATION.HEIGHT, CONFIG.OBSERVATION.WIDTH), 195 | interpolation=cv2.INTER_AREA, 196 | ) 197 | 198 | return processed_observation 199 | 200 | def reset(self, seed=None, options=None): 201 | """Reset the environment. 202 | 203 | Args: 204 | seed (int): seed for reproducibility. 205 | options (dict): additional information. 206 | 207 | Returns: 208 | observation (numpy.array): the processed observation. 209 | 210 | info (dict): auxiliary diagnostic information. 211 | """ 212 | # Reset time step and rewards 213 | self.time_step = 0 214 | self.reward_total = 0.0 215 | self.reward = 0.0 216 | 217 | # Reset objects 218 | self.batteries.reset(seed=seed) 219 | self.cities.reset(seed=seed) 220 | self.enemy_missiles.reset(seed=seed) 221 | self.friendly_missiles.reset(seed=seed) 222 | self.target.reset(seed=seed) 223 | 224 | # Compute observation 225 | self._compute_observation() 226 | 227 | return self._process_observation(), {} 228 | 229 | def step(self, action): 230 | """Go from current step to next one. 231 | 232 | Args: 233 | action (int): 0, 1, 2, 3, 4 or 5, the different actions. 234 | 235 | Returns: 236 | observation (numpy.array): the processed observation. 237 | 238 | reward (float): reward of the current time step. 239 | 240 | terminated (bool): whether a terminal state is reached. 241 | 242 | truncated (bool): whether a truncation condition is reached. 243 | 244 | info (dict): additional information on the current time step. 245 | """ 246 | # Reset current reward 247 | self.reward = 0.0 248 | 249 | # Step functions 250 | _, battery_reward, _, can_fire_dict = self.batteries.step(action) 251 | _, _, done_cities, _ = self.cities.step(action) 252 | _, _, done_enemy_missiles, _ = self.enemy_missiles.step(action) 253 | _, _, _, _ = self.friendly_missiles.step(action) 254 | _, _, _, _ = self.target.step(action) 255 | 256 | # Launch a new missile 257 | if action == 5 and can_fire_dict["can_fire"]: 258 | self.friendly_missiles.launch_missile(self.target) 259 | self.reward += CONFIG.REWARD.FRIENDLY_MISSILE_LAUNCHED 260 | 261 | # Check for collisions 262 | self._collisions_missiles() 263 | self._collisions_cities() 264 | 265 | # Check if episode is finished 266 | terminated = done_cities or done_enemy_missiles 267 | 268 | # Compute observation 269 | self._compute_observation() 270 | 271 | # Update values 272 | self.time_step += 1 273 | self.reward_total += self.reward 274 | 275 | return self._process_observation(), self.reward, terminated, False, {} 276 | 277 | def render(self, mode="raw_observation"): 278 | """Render the environment. 279 | 280 | This function renders the environment observation. To check what the 281 | processed observation looks like, it can also renders it. 282 | 283 | Args: 284 | mode (str): the render mode. Possible values are "raw_observation" 285 | and "processed_observation". 286 | """ 287 | # Get width and height 288 | w, h = CONFIG.EPISODE.WIDTH, CONFIG.EPISODE.HEIGHT 289 | 290 | # Initialize PyGame 291 | if self._display is None: 292 | pygame.init() 293 | pygame.mouse.set_visible(False) 294 | self._clock = pygame.time.Clock() 295 | pygame.display.set_caption("MissileCommand") 296 | self._display = pygame.display.set_mode((w, h)) 297 | 298 | # For debug only, display processed observation 299 | if mode == "processed_observation": 300 | observation = self._process_observation() 301 | surface = pygame.surfarray.make_surface(observation) 302 | surface = pygame.transform.scale(surface, (w, h)) 303 | 304 | # Normal mode 305 | else: 306 | observation = self.observation 307 | surface = pygame.surfarray.make_surface(observation) 308 | 309 | # Display all 310 | self._display.blit(surface, (0, 0)) 311 | pygame.display.update() 312 | 313 | # Limit max FPS 314 | self._clock.tick(CONFIG.EPISODE.FPS) 315 | 316 | def close(self): 317 | """Close the environment.""" 318 | if self._display is not None: 319 | pygame.quit() 320 | --------------------------------------------------------------------------------