├── agents ├── __init__.py └── baseline_sub_agents │ ├── __init__.py │ ├── checkpoint_pointer.txt │ ├── red_states.pkl │ ├── sub_agents.py │ ├── neural_nets.py │ ├── CybORGAgent.py │ ├── bline_CybORGAgent.py │ ├── CybORGActionAgent.py │ ├── configs.py │ ├── train_subagent.py │ ├── evaluation.py │ ├── loadBanditController.py │ ├── train_simple_bandit.py │ ├── BlueTableActionWrapper.py │ ├── StateRepWrapper.py │ ├── newBlueTableWrapper.py │ ├── curiosity.py │ └── newFlatWrapper.py ├── logs ├── training │ ├── placeholder │ └── PPO_RedMeanderAgent_2022-09-14_16-25-58 │ │ ├── PPO_CybORGAgent_88835_00000_0_2022-09-14_16-25-58 │ │ ├── progress.csv │ │ ├── result.json │ │ ├── params.pkl │ │ ├── events.out.tfevents.1663169159.gpu03.doc.ic.ac.uk │ │ └── params.json │ │ ├── basic-variant-state-2022-09-14_16-25-58.json │ │ └── experiment_state-2022-09-14_16-25-58.json ├── various │ ├── PPO_RedMeanderAgent_2022-07-06_16-32-36 │ │ ├── PPO_CybORGAgent_dcaaa_00000_0_2022-07-06_16-32-36 │ │ │ └── checkpoint_001829 │ │ │ │ ├── .is_checkpoint │ │ │ │ ├── checkpoint-1829 │ │ │ │ └── checkpoint-1829.tune_metadata │ │ └── basic-variant-state-2022-07-06_16-32-36.json │ └── SR_B_lineAgent_new52obs-27floats_2022-07-16_16-40-09 │ │ └── PPO_CybORGAgent_931b8_00000_0_2022-07-16_16-40-10 │ │ └── checkpoint_001916 │ │ ├── .is_checkpoint │ │ ├── checkpoint-1916 │ │ └── checkpoint-1916.tune_metadata └── bandits │ └── controller_bandit_2022-07-15_11-08-56 │ ├── bandit_controller_15000.pkl │ ├── bandit_controller_rewards_15000.pkl │ └── log_rewards.eps ├── requirements.txt ├── checkpoint_pointer.txt ├── .gitignore ├── LICENSE ├── terminal_output.txt └── readme.MD /agents/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /logs/training/placeholder: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /agents/baseline_sub_agents/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /agents/baseline_sub_agents/checkpoint_pointer.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /logs/training/PPO_RedMeanderAgent_2022-09-14_16-25-58/PPO_CybORGAgent_88835_00000_0_2022-09-14_16-25-58/progress.csv: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /logs/training/PPO_RedMeanderAgent_2022-09-14_16-25-58/PPO_CybORGAgent_88835_00000_0_2022-09-14_16-25-58/result.json: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /logs/various/PPO_RedMeanderAgent_2022-07-06_16-32-36/PPO_CybORGAgent_dcaaa_00000_0_2022-07-06_16-32-36/checkpoint_001829/.is_checkpoint: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /logs/various/SR_B_lineAgent_new52obs-27floats_2022-07-16_16-40-09/PPO_CybORGAgent_931b8_00000_0_2022-07-16_16-40-10/checkpoint_001916/.is_checkpoint: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /agents/baseline_sub_agents/red_states.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alan-turing-institute/cage-challenge-2-public/HEAD/agents/baseline_sub_agents/red_states.pkl -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | CybORG==2.1 2 | gym==0.19.0 3 | matplotlib==3.5.2 4 | numpy==1.21.4 5 | pandas==1.3.5 6 | prettytable==0.7.2 7 | torch==1.11.0 8 | ray==1.9.0 -------------------------------------------------------------------------------- /checkpoint_pointer.txt: -------------------------------------------------------------------------------- 1 | /Users/mylesfoley/Desktop/Imperial/git/turing/cage-challenge-2/logs/PPO__20220410_184257/PPO_HierEnv_a83db_00000_0_2022-04-10_18-42-57/checkpoint_002500/checkpoint-2500 -------------------------------------------------------------------------------- /logs/bandits/controller_bandit_2022-07-15_11-08-56/bandit_controller_15000.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alan-turing-institute/cage-challenge-2-public/HEAD/logs/bandits/controller_bandit_2022-07-15_11-08-56/bandit_controller_15000.pkl -------------------------------------------------------------------------------- /logs/bandits/controller_bandit_2022-07-15_11-08-56/bandit_controller_rewards_15000.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alan-turing-institute/cage-challenge-2-public/HEAD/logs/bandits/controller_bandit_2022-07-15_11-08-56/bandit_controller_rewards_15000.pkl -------------------------------------------------------------------------------- /logs/training/PPO_RedMeanderAgent_2022-09-14_16-25-58/basic-variant-state-2022-09-14_16-25-58.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alan-turing-institute/cage-challenge-2-public/HEAD/logs/training/PPO_RedMeanderAgent_2022-09-14_16-25-58/basic-variant-state-2022-09-14_16-25-58.json -------------------------------------------------------------------------------- /logs/various/PPO_RedMeanderAgent_2022-07-06_16-32-36/basic-variant-state-2022-07-06_16-32-36.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alan-turing-institute/cage-challenge-2-public/HEAD/logs/various/PPO_RedMeanderAgent_2022-07-06_16-32-36/basic-variant-state-2022-07-06_16-32-36.json -------------------------------------------------------------------------------- /logs/training/PPO_RedMeanderAgent_2022-09-14_16-25-58/PPO_CybORGAgent_88835_00000_0_2022-09-14_16-25-58/params.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alan-turing-institute/cage-challenge-2-public/HEAD/logs/training/PPO_RedMeanderAgent_2022-09-14_16-25-58/PPO_CybORGAgent_88835_00000_0_2022-09-14_16-25-58/params.pkl -------------------------------------------------------------------------------- /logs/various/PPO_RedMeanderAgent_2022-07-06_16-32-36/PPO_CybORGAgent_dcaaa_00000_0_2022-07-06_16-32-36/checkpoint_001829/checkpoint-1829: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alan-turing-institute/cage-challenge-2-public/HEAD/logs/various/PPO_RedMeanderAgent_2022-07-06_16-32-36/PPO_CybORGAgent_dcaaa_00000_0_2022-07-06_16-32-36/checkpoint_001829/checkpoint-1829 -------------------------------------------------------------------------------- /logs/various/PPO_RedMeanderAgent_2022-07-06_16-32-36/PPO_CybORGAgent_dcaaa_00000_0_2022-07-06_16-32-36/checkpoint_001829/checkpoint-1829.tune_metadata: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alan-turing-institute/cage-challenge-2-public/HEAD/logs/various/PPO_RedMeanderAgent_2022-07-06_16-32-36/PPO_CybORGAgent_dcaaa_00000_0_2022-07-06_16-32-36/checkpoint_001829/checkpoint-1829.tune_metadata -------------------------------------------------------------------------------- /logs/various/SR_B_lineAgent_new52obs-27floats_2022-07-16_16-40-09/PPO_CybORGAgent_931b8_00000_0_2022-07-16_16-40-10/checkpoint_001916/checkpoint-1916: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alan-turing-institute/cage-challenge-2-public/HEAD/logs/various/SR_B_lineAgent_new52obs-27floats_2022-07-16_16-40-09/PPO_CybORGAgent_931b8_00000_0_2022-07-16_16-40-10/checkpoint_001916/checkpoint-1916 -------------------------------------------------------------------------------- /logs/training/PPO_RedMeanderAgent_2022-09-14_16-25-58/PPO_CybORGAgent_88835_00000_0_2022-09-14_16-25-58/events.out.tfevents.1663169159.gpu03.doc.ic.ac.uk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alan-turing-institute/cage-challenge-2-public/HEAD/logs/training/PPO_RedMeanderAgent_2022-09-14_16-25-58/PPO_CybORGAgent_88835_00000_0_2022-09-14_16-25-58/events.out.tfevents.1663169159.gpu03.doc.ic.ac.uk -------------------------------------------------------------------------------- /logs/various/SR_B_lineAgent_new52obs-27floats_2022-07-16_16-40-09/PPO_CybORGAgent_931b8_00000_0_2022-07-16_16-40-10/checkpoint_001916/checkpoint-1916.tune_metadata: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alan-turing-institute/cage-challenge-2-public/HEAD/logs/various/SR_B_lineAgent_new52obs-27floats_2022-07-16_16-40-09/PPO_CybORGAgent_931b8_00000_0_2022-07-16_16-40-10/checkpoint_001916/checkpoint-1916.tune_metadata -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | 3 | # Byte-compiled / optimized / DLL files 4 | __pycache__/ 5 | *.py[cod] 6 | *$py.class 7 | 8 | # C extensions 9 | *.so 10 | 11 | # Distribution / packaging 12 | .Python 13 | build/ 14 | develop-eggs/ 15 | dist/ 16 | downloads/ 17 | eggs/ 18 | .eggs/ 19 | lib/ 20 | lib64/ 21 | parts/ 22 | sdist/ 23 | var/ 24 | wheels/ 25 | share/python-wheels/ 26 | *.egg-info/ 27 | .installed.cfg 28 | *.egg 29 | MANIFEST 30 | 31 | #Saved_Models 32 | agents/a2c/saved_models/ 33 | 34 | #IDE 35 | .idea/ -------------------------------------------------------------------------------- /agents/baseline_sub_agents/sub_agents.py: -------------------------------------------------------------------------------- 1 | 2 | sub_agents = {'B_line_trained':'/logs/various/SR_B_lineAgent_new52obs-27floats_2022-07-16_16-40-09/PPO_CybORGAgent_931b8_00000_0_2022-07-16_16-40-10/checkpoint_001916/checkpoint-1916', 3 | 'RedMeander_trained':'/logs/various/PPO_RedMeanderAgent_2022-07-06_16-32-36/PPO_CybORGAgent_dcaaa_00000_0_2022-07-06_16-32-36/checkpoint_001829/checkpoint-1829', 4 | 'bandit_trained': '/logs/bandits/controller_bandit_2022-07-15_11-08-56/bandit_controller_15000.pkl' 5 | } -------------------------------------------------------------------------------- /agents/baseline_sub_agents/neural_nets.py: -------------------------------------------------------------------------------- 1 | from ray.rllib.models.torch.torch_modelv2 import TorchModelV2 2 | from ray.rllib.models.torch.fcnet import FullyConnectedNetwork as TorchFC 3 | # from ray.rllib.models.tf.fcnet import FullyConnectedNetwork 4 | # from ray.rllib.models.tf.tf_modelv2 import TFModelV2 5 | import torch 6 | 7 | class TorchModel(TorchModelV2, torch.nn.Module): 8 | def __init__(self, obs_space, action_space, num_outputs, model_config, 9 | name): 10 | TorchModelV2.__init__(self, obs_space, action_space, num_outputs, model_config, 11 | name) 12 | torch.nn.Module.__init__(self) 13 | 14 | self.model = TorchFC(obs_space, action_space, 15 | num_outputs, model_config, name) 16 | 17 | def forward(self, input_dict, state, seq_lens): 18 | return self.model.forward(input_dict, state, seq_lens) 19 | 20 | def value_function(self): 21 | return self.model.value_function() 22 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 The Alan Turing Institute 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /agents/baseline_sub_agents/CybORGAgent.py: -------------------------------------------------------------------------------- 1 | import gym, inspect, random 2 | from ray.rllib.env.env_context import EnvContext 3 | 4 | from CybORG import CybORG 5 | from CybORG.Agents import B_lineAgent, GreenAgent, BaseAgent, RedMeanderAgent, BlueMonitorAgent 6 | from CybORG.Agents.Wrappers import ChallengeWrapper, OpenAIGymWrapper, EnumActionWrapper, FixedFlatWrapper, ReduceActionSpaceWrapper 7 | 8 | 9 | class CybORGAgent(gym.Env): 10 | max_steps = 100 11 | path = str(inspect.getfile(CybORG)) 12 | path = path[:-10] + '/Shared/Scenarios/Scenario2.yaml' 13 | 14 | agents = { 15 | 'Red': RedMeanderAgent 16 | } 17 | 18 | """The CybORGAgent env""" 19 | 20 | def __init__(self, config: EnvContext): 21 | self.cyborg = CybORG(self.path, 'sim', agents=self.agents) 22 | self.env = ChallengeWrapper(env=self.cyborg, agent_name='Blue') 23 | self.env._max_episode_steps=100 24 | self.steps = 1 25 | self.agent_name = self.env.agent_name 26 | self.action_space = self.env.action_space 27 | self.observation_space = self.env.observation_space 28 | self.action = None 29 | 30 | def reset(self): 31 | self.steps = 1 32 | return self.env.reset() 33 | 34 | def step(self, action=None): 35 | result = self.env.step(action=action) 36 | self.steps += 1 37 | if self.steps == self.max_steps: 38 | return result[0], result[1], True, result[3] 39 | assert (self.steps <= self.max_steps) 40 | return result 41 | 42 | def seed(self, seed=None): 43 | random.seed(seed) 44 | -------------------------------------------------------------------------------- /agents/baseline_sub_agents/bline_CybORGAgent.py: -------------------------------------------------------------------------------- 1 | from cv2 import reduce 2 | import gym, inspect, random 3 | from ray.rllib.env.env_context import EnvContext 4 | import os 5 | import sys 6 | sys.path.insert(0, os.path.abspath('..')) 7 | 8 | from CybORG import CybORG 9 | from CybORG.Agents import B_lineAgent, GreenAgent, BaseAgent, RedMeanderAgent, BlueMonitorAgent 10 | # from CybORG.Agents.Wrappers import ChallengeWrapper 11 | from StateRepWrapper import StateRepWrapper 12 | from pprint import pprint 13 | # from csv import writer 14 | 15 | class CybORGAgent(gym.Env): 16 | max_steps = 100 17 | path = str(inspect.getfile(CybORG)) 18 | path = path[:-10] + '/Shared/Scenarios/Scenario2.yaml' 19 | 20 | agents = { 21 | 'Red': B_lineAgent 22 | } 23 | 24 | """The CybORGAgent env""" 25 | 26 | def __init__(self, config: EnvContext): 27 | self.cyborg = CybORG(self.path, 'sim', agents=self.agents) 28 | 29 | self.env = StateRepWrapper(env=self.cyborg, agent_name='Blue', actionSpace=0, obsSpace=3) 30 | self.steps = 0 31 | self.agent_name = self.env.agent_name 32 | self.action_space = self.env.action_space 33 | print('\033[92m' + '///// initiating CybORG... action space:' + '\033[0m', flush=True) 34 | pprint(self.action_space) 35 | self.observation_space = self.env.observation_space 36 | self.action = None 37 | 38 | def reset(self): 39 | self.steps = 1 40 | return self.env.reset() 41 | 42 | def step(self, action=None): 43 | 44 | result = self.env.step(action=action) 45 | 46 | self.steps += 1 47 | if self.steps == self.max_steps: 48 | return result[0], result[1], True, result[3] 49 | assert (self.steps <= self.max_steps) 50 | return result 51 | 52 | def seed(self, seed=None): 53 | random.seed(seed) 54 | 55 | -------------------------------------------------------------------------------- /agents/baseline_sub_agents/CybORGActionAgent.py: -------------------------------------------------------------------------------- 1 | import gym, inspect, random 2 | from ray.rllib.env.env_context import EnvContext 3 | 4 | from CybORG import CybORG 5 | from CybORG.Agents import B_lineAgent, GreenAgent, BaseAgent, RedMeanderAgent, BlueMonitorAgent 6 | from CybORG.Agents.Wrappers import BaseWrapper, OpenAIGymWrapper, EnumActionWrapper 7 | from BlueTableActionWrapper import BlueTableWrapper 8 | import numpy as np 9 | from gym import Env, spaces 10 | 11 | class CybORGActionAgent(Env, BaseWrapper): 12 | max_steps = 100 13 | #path = str(inspect.getfile(CybORG)) 14 | #path = path[:-10] + '/Shared/Scenarios/Scenario2.yaml' 15 | #agents = { 16 | # 'Red': B_lineAgent # , #RedMeanderAgent, 'Green': GreenAgent 17 | #} 18 | def __init__(self, config): 19 | super().__init__() 20 | self.agent_name = config['agent_name'] 21 | if config['env'] is not None: 22 | self.cyborg = config['env'] 23 | self.agents = { 24 | 'Red': config['attacker'] # , #RedMeanderAgent, 'Green': GreenAgent 25 | } 26 | else: 27 | path = str(inspect.getfile(CybORG)) 28 | path = path[:-10] + '/Shared/Scenarios/Scenario2.yaml' 29 | self.agents = { 30 | 'Red': config['attacker']#B_lineAgent # , #RedMeanderAgent, 'Green': GreenAgent 31 | } 32 | self.cyborg = CybORG(path, 'sim', agents=self.agents) 33 | 34 | #self.agent_name = self.agent_name 35 | self.env = BlueTableWrapper(self.cyborg, output_mode='vector') 36 | self.env = EnumActionWrapper(self.env) 37 | self.env = OpenAIGymWrapper(agent_name=self.agent_name, env=self.env) 38 | 39 | #self.env = self.cyborg 40 | self.action_space = self.env.action_space 41 | self.observation_space = spaces.Box(-1.0, 1.0, shape=(54,), dtype=np.float32) 42 | #self.reward_threshold = reward_threshold 43 | self.max_steps = config['max_steps'] 44 | self.step_counter = 0 45 | #self.agent_name = self.env.agent_name 46 | self.action = None 47 | self.success = None 48 | 49 | def step(self, action=None): 50 | obs, reward, done, info = self.env.step(action=action) 51 | assert (obs.shape[0] == 54) 52 | if np.array_equal(obs[:2], [0,1]): 53 | self.success = False 54 | elif np.array_equal(obs[:2], [1,0]): 55 | self.success = True 56 | else: 57 | self.success = False 58 | #obs = obs[2:] 59 | self.step_counter += 1 60 | if self.max_steps is not None and self.step_counter >= self.max_steps: 61 | done = True 62 | assert (self.step_counter <= self.max_steps) 63 | 64 | return obs, reward, done, info 65 | 66 | def reset(self): 67 | self.step_counter = 0 68 | return self.env.reset() 69 | 70 | def get_attr(self, attribute: str): 71 | return self.env.get_attr(attribute) 72 | 73 | def get_observation(self, agent: str): 74 | return self.env.get_observation(agent) 75 | 76 | def get_agent_state(self, agent: str): 77 | return self.env.get_agent_state(agent) 78 | 79 | def get_action_space(self, agent=None) -> dict: 80 | return self.env.get_action_space(self.agent_name) 81 | 82 | def get_last_action(self, agent): 83 | return self.get_attr('get_last_action')(agent) 84 | 85 | def get_ip_map(self): 86 | return self.get_attr('get_ip_map')() 87 | 88 | def get_rewards(self): 89 | return self.get_attr('get_rewards')() 90 | 91 | def get_reward_breakdown(self, agent: str): 92 | return self.get_attr('get_reward_breakdown')(agent) -------------------------------------------------------------------------------- /agents/baseline_sub_agents/configs.py: -------------------------------------------------------------------------------- 1 | from ray.rllib.agents.ppo.ppo import DEFAULT_CONFIG as PPO_CONFIG 2 | from ray.rllib.models import ModelCatalog 3 | from ray.rllib.agents import Trainer 4 | from CybORGAgent import CybORGAgent 5 | from bline_CybORGAgent import CybORGAgent as bline_CybORGAgent 6 | import os 7 | from neural_nets import * 8 | from ray import tune 9 | ModelCatalog.register_custom_model("CybORG_Torch", TorchModel) 10 | from curiosity import Curiosity 11 | 12 | 13 | meander_config = { 14 | "env": CybORGAgent, 15 | "env_config": { 16 | "null": 0, 17 | }, 18 | "gamma": 0.99, 19 | # Use GPUs iff `RLLIB_NUM_GPUS` env var set to > 0. 20 | "num_gpus": int(os.environ.get("RLLIB_NUM_GPUS", 0)), 21 | "num_envs_per_worker": 4, 22 | "entropy_coeff": 0.001, 23 | "num_sgd_iter": 10, 24 | "horizon": 100, 25 | "rollout_fragment_length": 100, 26 | #"vf_loss_coeff": 1e-5, 27 | #"vf_share_layers": False, 28 | "model": { 29 | # Attention net wrapping (for tf) can already use the native keras 30 | # model versions. For torch, this will have no effect. 31 | "_use_default_native_models": True, 32 | "custom_model": "CybORG_Torch", 33 | 'fcnet_hiddens': [256, 256, 52], 34 | "use_attention": not True, 35 | "use_lstm": not True, 36 | "max_seq_len": 10, 37 | "lstm_use_prev_action": True, 38 | "lstm_use_prev_reward": True, 39 | 40 | }, 41 | "framework": 'torch', 42 | } 43 | 44 | 45 | bline_config = Trainer.merge_trainer_configs( 46 | PPO_CONFIG,{ 47 | "env": bline_CybORGAgent, 48 | "env_config": { 49 | "null": 0, 50 | }, 51 | # Use GPUs iff `RLLIB_NUM_GPUS` env various set to > 0. 52 | "num_gpus": int(os.environ.get("RLLIB_NUM_GPUS", "0")), 53 | "model": { 54 | "custom_model": "CybORG_Torch", 55 | 56 | "vf_share_layers": False, 57 | "fcnet_hiddens": [256, 256], 58 | }, 59 | "lr": 0.0005, 60 | #"momentum": tune.uniform(0, 1), 61 | "num_workers": 0, # parallelism 62 | "framework": "torch", # May also use "tf2", "tfe" or "torch" if supported 63 | "eager_tracing": True, # In order to reach similar execution speed as with static-graph mode (tf default) 64 | "vf_loss_coeff": 1, # Scales down the value function loss for better comvergence with PPO 65 | "clip_param": 0.5, 66 | "vf_clip_param": 5.0, 67 | "exploration_config": { 68 | "type": Curiosity, # <- Use the Curiosity module for exploring. 69 | "framework": "torch", 70 | "eta": 1.0, # Weight for intrinsic rewards before being added to extrinsic ones. 71 | "lr": 0.001, # Learning rate of the curiosity (ICM) module. 72 | "feature_dim": 53, # Dimensionality of the generated feature vectors. 73 | # Setup of the feature net (used to encode observations into feature (latent) vectors). 74 | "feature_net_config": { 75 | "fcnet_hiddens": [], 76 | "fcnet_activation": "relu", 77 | 'framework': 'torch', 78 | #'device': 'cuda:0' 79 | }, 80 | "inverse_net_hiddens": [256], # Hidden layers of the "inverse" model. 81 | "inverse_net_activation": "relu", # Activation of the "inverse" model. 82 | "forward_net_hiddens": [256], # Hidden layers of the "forward" model. 83 | "forward_net_activation": "relu", # Activation of the "forward" model. 84 | "beta": 0.2, # Weight for the "forward" loss (beta) over the "inverse" loss (1.0 - beta). 85 | # Specify, which exploration sub-type to use (usually, the algo's "default" 86 | # exploration, e.g. EpsilonGreedy for DQN, StochasticSampling for PG/SAC). 87 | "sub_exploration": { 88 | "type": "StochasticSampling", 89 | } 90 | } 91 | }) -------------------------------------------------------------------------------- /terminal_output.txt: -------------------------------------------------------------------------------- 1 | /Users/mylesfoley/opt/anaconda3/lib/python3.8/site-packages/paramiko/transport.py:219: CryptographyDeprecationWarning: Blowfish has been deprecated 2 | "class": algorithms.Blowfish, 3 | Using checkpoint file (Controller): /Users/mylesfoley/Desktop/Imperial/git/turing/mia_dev/mindrake-float_obs/logs/bandits/controller_bandit_2022-07-15_11-08-56/bandit_controller_15000.pkl 4 | Using checkpoint file (B-line): /Users/mylesfoley/Desktop/Imperial/git/turing/mia_dev/mindrake-float_obs/logs/various/SR_B_lineAgent_new52obs-27floats_2022-07-16_16-40-09/PPO_CybORGAgent_931b8_00000_0_2022-07-16_16-40-10/checkpoint_001916/checkpoint-1916 5 | Using checkpoint file (Red Meander): /Users/mylesfoley/Desktop/Imperial/git/turing/mia_dev/mindrake-float_obs/logs/various/PPO_LSTM_RedMeanderAgent_2022-07-06_16-32-36/PPO_CybORGAgent_dcaaa_00000_0_2022-07-06_16-32-36/checkpoint_001829/checkpoint-1829 6 | 2022-07-19 10:37:43,930 WARNING ppo.py:143 -- `train_batch_size` (4000) cannot be achieved with your other settings (num_workers=2 num_envs_per_worker=20 rollout_fragment_length=200)! Auto-adjusting `rollout_fragment_length` to 100. 7 | (RolloutWorker pid=69451) /Users/mylesfoley/opt/anaconda3/lib/python3.8/site-packages/paramiko/transport.py:219: CryptographyDeprecationWarning: Blowfish has been deprecated 8 | (RolloutWorker pid=69451) "class": algorithms.Blowfish, 9 | (RolloutWorker pid=69445) /Users/mylesfoley/opt/anaconda3/lib/python3.8/site-packages/paramiko/transport.py:219: CryptographyDeprecationWarning: Blowfish has been deprecated 10 | (RolloutWorker pid=69445) "class": algorithms.Blowfish, 11 | 2022-07-19 10:37:59,509 INFO trainable.py:124 -- Trainable.setup took 15.579 seconds. If your trainable is slow to initialize, consider setting reuse_actors=True to reduce actor creation overheads. 12 | 2022-07-19 10:37:59,595 INFO trainable.py:467 -- Restored on 127.0.0.1 from checkpoint: /Users/mylesfoley/Desktop/Imperial/git/turing/mia_dev/mindrake-float_obs/logs/various/PPO_LSTM_RedMeanderAgent_2022-07-06_16-32-36/PPO_CybORGAgent_dcaaa_00000_0_2022-07-06_16-32-36/checkpoint_001829/checkpoint-1829 13 | 2022-07-19 10:37:59,595 INFO trainable.py:475 -- Current state after restoring: {'_iteration': 1829, '_timesteps_total': 0, '_time_total': 29899.77183365822, '_episodes_total': 73880} 14 | ///// initiating CybORG... action space: 15 | Discrete(145) 16 | 2022-07-19 10:37:59,853 INFO trainable.py:467 -- Restored on 127.0.0.1 from checkpoint: /Users/mylesfoley/Desktop/Imperial/git/turing/mia_dev/mindrake-float_obs/logs/various/SR_B_lineAgent_new52obs-27floats_2022-07-16_16-40-09/PPO_CybORGAgent_931b8_00000_0_2022-07-16_16-40-10/checkpoint_001916/checkpoint-1916 17 | 2022-07-19 10:37:59,853 INFO trainable.py:475 -- Current state after restoring: {'_iteration': 1916, '_timesteps_total': 0, '_time_total': 97031.91204357147, '_episodes_total': 77414} 18 | Using agent LoadBanditBlueAgent, if this is incorrect please update the code to load in your agent 19 | Saving evaluation results to /Users/mylesfoley/Desktop/Imperial/git/cage-challenge-2/CybORG/CybORG/Evaluation/20220719_103759_LoadBanditBlueAgent.txt 20 | using CybORG v2.1, Scenario2 21 | 22 | Average reward for red agent B_lineAgent and steps 30 is: -3.4240000000000004 with a standard deviation of 1.7097533639845728 23 | Average reward for red agent RedMeanderAgent and steps 30 is: -6.769 with a standard deviation of 1.6371751354825856 24 | Average reward for red agent SleepAgent and steps 30 is: 0.0 with a standard deviation of 0.0 25 | Average reward for red agent B_lineAgent and steps 50 is: -6.117999999999996 with a standard deviation of 3.125146622822903 26 | Average reward for red agent RedMeanderAgent and steps 50 is: -10.549999999999999 with a standard deviation of 2.1547926380817506 27 | Average reward for red agent SleepAgent and steps 50 is: 0.0 with a standard deviation of 0.0 28 | Average reward for red agent B_lineAgent and steps 100 is: -12.66499999999999 with a standard deviation of 5.828116715067096 29 | Average reward for red agent RedMeanderAgent and steps 100 is: -17.340000000000003 with a standard deviation of 4.3315614092869925 30 | Average reward for red agent SleepAgent and steps 100 is: 0.0 with a standard deviation of 0.0 31 | -------------------------------------------------------------------------------- /agents/baseline_sub_agents/train_subagent.py: -------------------------------------------------------------------------------- 1 | """Alternative RLLib model based on local training 2 | You can visualize experiment results in ~/ray_results using TensorBoard. 3 | """ 4 | ############################## 5 | import os 6 | import sys 7 | from neural_nets import * 8 | from sub_agents import * 9 | from configs import * 10 | 11 | # Ray imports 12 | import ray 13 | from ray import tune 14 | from ray.tune import grid_search 15 | from ray.tune.schedulers import ASHAScheduler # https://openreview.net/forum?id=S1Y7OOlRZ algo for early stopping 16 | from ray.rllib.agents.trainer import Trainer 17 | import ray.rllib.agents.ppo as ppo 18 | from ray.rllib.agents.ppo import DEFAULT_CONFIG 19 | from ray.rllib.utils.framework import try_import_torch 20 | from ray.rllib.models import ModelCatalog 21 | import time 22 | import os 23 | import sys 24 | sys.path.insert(0, os.path.abspath('..')) 25 | 26 | 27 | from typing import Any 28 | from pprint import pprint 29 | #import torch 30 | from ray.rllib.models.torch.torch_modelv2 import TorchModelV2 31 | from ray.rllib.models.modelv2 import ModelV2 32 | 33 | from CybORGAgent import CybORGAgent 34 | 35 | 36 | 37 | 38 | #os.environ["CUDA_VISIBLE_DEVICES"]="1" 39 | #tf1, tf, tfv = try_import_tf() 40 | torch, nn = try_import_torch() 41 | # torch.device(str("cuda:0")) 42 | 43 | 44 | if __name__ == "__main__": 45 | # set subagent config 46 | config = dict() 47 | if len(sys.argv) > 1: 48 | if sys.argv[1].lower() == 'bline': 49 | config = bline_config 50 | elif sys.argv[1].lower() == 'meander': 51 | config = meander_config 52 | else: 53 | raise ValueError("Please specify which subagent you'd like to train: \"bline\" or \"meander\".") 54 | else: 55 | raise ValueError("Please specify which subagent you'd like to train: \"bline\" or \"meander\".") 56 | 57 | adversary_name = config['env'].agents['Red'].__name__ 58 | # ModelCatalog.register_custom_model("CybORG_hier_Model", TorchModel) 59 | print('\033[92m' + "/"*50 + '\033[0m', flush=True) 60 | print('\033[92m' + "Training defender for " + adversary_name + "..." + '\033[0m', flush=True) 61 | ray.init() 62 | 63 | # gpu availability 64 | print("torch.cuda.is_available()", torch.cuda.is_available()) 65 | torch.device(str("cuda:0")) 66 | if torch.cuda.is_available(): 67 | torch.device(str("cuda:0")) 68 | gpus = 1 69 | else: 70 | gpus = 0 71 | config['num_gpus'] = gpus 72 | 73 | 74 | stop = { 75 | "training_iteration": 10000000, # The number of times tune.report() has been called 76 | "timesteps_total": 10000000, # Total number of timesteps 77 | "episode_reward_mean": -0.1, # When to stop.. it would be great if we could define this in terms 78 | # of a more complex expression which incorporates the episode reward min too 79 | # There is a lot of variance in the episode reward min 80 | } 81 | 82 | log_dir = '../../logs/training/' 83 | algo = ppo.PPOTrainer 84 | analysis = tune.run(algo, 85 | config=bline_config, 86 | name= algo.__name__ + '_' + adversary_name + '_' + time.strftime("%Y-%m-%d_%H-%M-%S"), 87 | local_dir=log_dir, 88 | stop=stop, 89 | checkpoint_at_end=True, 90 | checkpoint_freq=1, 91 | keep_checkpoints_num=3, 92 | checkpoint_score_attr="episode_reward_mean") 93 | 94 | checkpoint_pointer = open("checkpoint_pointer.txt", "w") 95 | last_checkpoint = analysis.get_last_checkpoint( 96 | metric="episode_reward_mean", mode="max" 97 | ) 98 | 99 | checkpoint_pointer.write(last_checkpoint) 100 | print("Best model checkpoint written to: {}".format(last_checkpoint)) 101 | 102 | print(algo.get_weights()) 103 | checkpoint_pointer.close() 104 | ray.shutdown() 105 | 106 | # You can run tensorboard --logdir=logs to visualise the learning processs during and after training 107 | # tensorboard --logdir=thesis_logs -------------------------------------------------------------------------------- /agents/baseline_sub_agents/evaluation.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | import inspect 3 | import time 4 | from statistics import mean, stdev 5 | 6 | from CybORG import CybORG, CYBORG_VERSION 7 | from CybORG.Agents import B_lineAgent, SleepAgent 8 | from CybORG.Agents.SimpleAgents.Meander import RedMeanderAgent 9 | 10 | from CybORG.Agents.Wrappers import ChallengeWrapper 11 | 12 | from loadBanditController import LoadBanditBlueAgent as LoadBlueAgent 13 | from CybORGActionAgent import CybORGActionAgent 14 | MAX_EPS = 100 15 | agent_name = 'Blue' 16 | 17 | def wrap(env): 18 | return ChallengeWrapper(env=env, agent_name='Blue') 19 | 20 | def custom_wrap(config): 21 | return CybORGActionAgent(config) 22 | 23 | def get_git_revision_hash() -> str: 24 | return subprocess.check_output(['git', 'rev-parse', 'HEAD']).decode('ascii').strip() 25 | 26 | 27 | if __name__ == "__main__": 28 | cyborg_version = CYBORG_VERSION 29 | scenario = 'Scenario2' 30 | commit_hash = get_git_revision_hash() 31 | # ask for a name 32 | name = 'Myles Foley' #input('Name: ') 33 | # ask for a team 34 | team = 'Mindrake' #input("Team: ") 35 | # ask for a name for the agent 36 | name_of_agent = 'Test evaluation'#input("Name of technique: ") 37 | 38 | lines = inspect.getsource(wrap) 39 | wrap_line = lines.split('\n')[1].split('return ')[1] 40 | 41 | # Change this line to load your agent 42 | agent = LoadBlueAgent() 43 | 44 | print(f'Using agent {agent.__class__.__name__}, if this is incorrect please update the code to load in your agent') 45 | 46 | file_name = str(inspect.getfile(CybORG))[:-10] + '/Evaluation/' + time.strftime("%Y%m%d_%H%M%S") + f'_{agent.__class__.__name__}.txt' 47 | print(f'Saving evaluation results to {file_name}') 48 | with open(file_name, 'a+') as data: 49 | data.write(f'CybORG v{cyborg_version}, {scenario}, Commit Hash: {commit_hash}\n') 50 | data.write(f'author: {name}, team: {team}, technique: {name_of_agent}\n') 51 | data.write(f"wrappers: {wrap_line}\n") 52 | 53 | path = str(inspect.getfile(CybORG)) 54 | path = path[:-10] + '/Shared/Scenarios/Scenario2.yaml' 55 | 56 | print(f'using CybORG v{cyborg_version}, {scenario}\n') 57 | for num_steps in [30, 50, 100]: 58 | for red_agent in [B_lineAgent, RedMeanderAgent, SleepAgent]: 59 | r_step = {i:[] for i in range(1, num_steps + 1)} 60 | #print(r_step) 61 | #print(type(list(r_step)[0])) 62 | 63 | cyborg = CybORG(path, 'sim', agents={'Red': red_agent}) 64 | #wrapped_cyborg = ChallengeWrapper(env=cyborg, agent_name='Blue') #wrap(cyborg) 65 | 66 | wrapped_cyborg = custom_wrap({'agent_name': 'Blue', 'env':cyborg, 'max_steps': 100, 'attacker': red_agent}) 67 | 68 | observation = wrapped_cyborg.reset() 69 | # observation = cyborg.reset().observation 70 | 71 | action_space = wrapped_cyborg.get_action_space(agent_name) 72 | # action_space = cyborg.get_action_space(agent_name) 73 | total_reward = [] 74 | actions = [] 75 | for i in range(MAX_EPS): 76 | r = [] 77 | a = [] 78 | # cyborg.env.env.tracker.render() 79 | for j in range(num_steps): 80 | #action, agent_selected = agent.get_action(observation, action_space) 81 | action, agent_to_select = agent.get_action(observation, action_space) 82 | observation, rew, done, info = wrapped_cyborg.step(action) 83 | # result = cyborg.step(agent_name, action) 84 | 85 | # Print true table on each step 86 | #true_state = cyborg.get_agent_state('True') 87 | #true_table = true_obs_to_table(true_state,cyborg) 88 | #print(true_table) 89 | if agent_to_select == 0: 90 | agent_to_select = 'Meander' 91 | else: 92 | agent_to_select = 'BLine' 93 | r.append(rew) 94 | r_step[j+1].append(rew) 95 | # r.append(result.reward) 96 | #agent_selected = 'BLine' if agent_selected == 0 else 'RedMeander' 97 | a.append((str(cyborg.get_last_action('Blue')), str(cyborg.get_last_action('Red')), str(agent_to_select))) 98 | agent.end_episode() # Don't forget to dangermouse 99 | total_reward.append(sum(r)) 100 | actions.append(a) 101 | # observation = cyborg.reset().observation 102 | observation = wrapped_cyborg.reset() 103 | r_step = {step: mean(r_step[step]) for step in range(1, num_steps+1)} 104 | 105 | print(f'Average reward for red agent {red_agent.__name__} and steps {num_steps} is: {mean(total_reward)} with a standard deviation of {stdev(total_reward)}') 106 | #print(sum(r_step.values())) 107 | #include average rewards per step 108 | #print(f'Average reward per step for red agent {red_agent.__name__} and steps {num_steps} is: ' +str(r_step)) 109 | with open(file_name, 'a+') as data: 110 | data.write(f'steps: {num_steps}, adversary: {red_agent.__name__}, mean: {mean(total_reward)}, standard deviation {stdev(total_reward)}\n') 111 | for act, sum_rew in zip(actions, total_reward): 112 | data.write(f'actions: {act}, total reward: {sum_rew}\n') 113 | -------------------------------------------------------------------------------- /readme.MD: -------------------------------------------------------------------------------- 1 | ![mindrake](https://user-images.githubusercontent.com/10000317/150498045-b712992c-b569-4654-a35e-65660df3f795.png) 2 | 3 | # Team Mindrake 4 | *"For death or glory"* 5 | 6 | This repository contains the model that won the 3rd place in [CAGE-Challenge-2](https://github.com/cage-challenge/cage-challenge-2). This model has shown clear performance improvement under CAGE-Challenge-2 CybORG environment, compared with our original [winning model](https://github.com/alan-turing-institute/cage-challenge-1/blob/submission_final) in [CAGE-Challenge-1](https://github.com/cage-challenge/cage-challenge-1) under the same environment. 7 | 8 | If you use this repository in your research, please cite it as follows: 9 | 10 | ```bibtex 11 | @inproceedings{foley2022autonomous, 12 | title={Autonomous network defence using reinforcement learning}, 13 | author={Foley, Myles and Hicks, Chris and Highnam, Kate and Mavroudis, Vasilios}, 14 | booktitle={Proceedings of the 2022 ACM on Asia Conference on Computer and Communications Security}, 15 | pages={1252--1254}, 16 | year={2022} 17 | } 18 | ``` 19 | 20 |

21 | 22 | # Model Architecture 23 | Our blue agent here keeps the hierarchical structure, meaning it has a controller sitting on top of two subagents, and each subagent is specialilsed at defending against one type of attacker. The controller receives the obervations of the network at beginning of each episode and pick a specialised subagent to defend the network. Subagents are pretrained Proximal Policy Optimisation (PPO) reinforcement learning agents, which have formed converged policies to defend against their corresponding attackers. 24 | 25 | 26 | 27 | The controller can achive 100% accuracy when choosing the subagent. It uses simple bandit learning algorithm which has been pretrained for 15000 steps. 28 | 29 | The attackers are **MeanderAgent** (has no information about the network so it attacks the hosts at random) and **BLineAgent** (has information about the network so it has clear strategy to exploit operational server). Subagent for **MeaderAgent** uses PPO algorithms and 52-bit observation space, while subagent for **BLineAgent** uses PPO with curiosity and 27-float observation space. 30 | 31 |

32 | 33 | # What is in the repo 34 | ### There are two folders in the main directory:
35 | 36 | `agents/baseline_sub_agents/` -- contains the scripts to load both types of controllers and subagents;
37 | 38 | * `evaluation.py` can evaluate the hierarchical model 39 | * `loadBanditController.py` can retrieve the pretrained controller and subagents, which is used by `evaluation.py` 40 | * BlineAgent defender uses `bline_CybORGAgent.py` to setup the environment; `StateRepWrapper.py` and `newBlueTableWrapper.py` are used to create the 27-float observation space. `curiosity.py` is used to add curiosity in the RL algorithm 41 | * MeanderAgent defender uses `CybORGAgent.py` as the environment, where `ChallengeWrapper` creates 52-bit observation space. 42 | * `configs.py` contrains RL configurations when training both subagents 43 | * `neural_nets.py` includes the customised neural network used in subagents 44 | * `train_simple_bandit.py` is used to train the bandit controller 45 | * `train_subagent.py` is used to train the subagents 46 | 47 | 48 | `logs/` -- contains the pretrained controller and subagent models. 49 | 50 | * `bandits/` contains pretrained bandit controller (i.e. `bandit_controller_15000.pkl`) 51 | * `various/` contains pretrained **MeanderAgent** defender (`PPO_RedMeanderAgent_2022-07-06_16-32-36`) and **BLineAgent** defender (`SR_B_lineAgent_new52obs-27floats_2022-07-16_16-40-09`) 52 | 53 |
54 | 55 | ### CAGE Challenge 2 submission from Team Mindrake. 56 | 57 | Evaluation output file: 58 | - `20220719_103759_LoadBanditBlueAgent.txt` 59 | 60 | Terminal Ouptut file: 61 | - `terminal_output.txt` 62 | 63 | Evaluation Script: 64 | - `/agents/baseline_sub_agents/evaluation.py` 65 | 66 |

67 | 68 | # How to run the code 69 | ## Setup and installation 70 | ### Install [CAGE Challenge](https://github.com/cage-challenge/cage-challenge-2) 71 | 72 | ``` 73 | # Grab the repo 74 | git clone https://github.com/cage-challenge/cage-challenge-2.git 75 | 76 | # from the cage-challenge-2/CybORG directory 77 | pip install -e . 78 | ``` 79 | 80 | ### Install our requirements 81 | 82 | ``` 83 | pip install -r requirements.txt 84 | ``` 85 |
86 | 87 | ## Model training 88 | To train subagnts 89 | ``` 90 | # assume you are in the main directory 91 | cd agents/baseline_sub_agents/ 92 | 93 | # to train BLineAgent defender 94 | python train_subagent.py bline 95 | 96 | # to train MeanderAgent defender 97 | python train_subagent.py meander 98 | 99 | # to train bandit controller 100 | python train_simple_bandit.py 101 | ``` 102 | 103 | ## Model evaluation 104 | ``` 105 | # assume you are in the main directory 106 | cd agents/baseline_sub_agents/ 107 | ``` 108 | 109 | 110 | ### if you are using the pretrained models we provided: 111 | ``` 112 | python evaluation.py 113 | ``` 114 | 115 | ### if you want to use the models you trained yourself: 116 | - Change the model directory in `sub_agents.py` 117 | - `python evaluation.py` 118 | 119 |
120 | 121 | 122 | 123 | -------------------------------------------------------------------------------- /logs/training/PPO_RedMeanderAgent_2022-09-14_16-25-58/PPO_CybORGAgent_88835_00000_0_2022-09-14_16-25-58/params.json: -------------------------------------------------------------------------------- 1 | { 2 | "_disable_preprocessor_api": false, 3 | "_fake_gpus": false, 4 | "_tf_policy_handles_more_than_one_loss": false, 5 | "action_space": null, 6 | "actions_in_input_normalized": false, 7 | "batch_mode": "truncate_episodes", 8 | "callbacks": "", 9 | "clip_actions": false, 10 | "clip_param": 0.5, 11 | "clip_rewards": null, 12 | "collect_metrics_timeout": 180, 13 | "compress_observations": false, 14 | "create_env_on_driver": false, 15 | "custom_eval_function": null, 16 | "custom_resources_per_worker": {}, 17 | "eager_max_retraces": 20, 18 | "eager_tracing": true, 19 | "entropy_coeff": 0.0, 20 | "entropy_coeff_schedule": null, 21 | "env": "", 22 | "env_config": { 23 | "null": 0 24 | }, 25 | "env_task_fn": null, 26 | "evaluation_config": {}, 27 | "evaluation_interval": null, 28 | "evaluation_num_episodes": 10, 29 | "evaluation_num_workers": 0, 30 | "evaluation_parallel_to_training": false, 31 | "exploration_config": { 32 | "beta": 0.2, 33 | "eta": 1.0, 34 | "feature_dim": 53, 35 | "feature_net_config": { 36 | "fcnet_activation": "relu", 37 | "fcnet_hiddens": [], 38 | "framework": "torch" 39 | }, 40 | "forward_net_activation": "relu", 41 | "forward_net_hiddens": [ 42 | 256 43 | ], 44 | "framework": "torch", 45 | "inverse_net_activation": "relu", 46 | "inverse_net_hiddens": [ 47 | 256 48 | ], 49 | "lr": 0.001, 50 | "sub_exploration": { 51 | "type": "StochasticSampling" 52 | }, 53 | "type": "" 54 | }, 55 | "explore": true, 56 | "extra_python_environs_for_driver": {}, 57 | "extra_python_environs_for_worker": {}, 58 | "fake_sampler": false, 59 | "framework": "torch", 60 | "gamma": 0.99, 61 | "grad_clip": null, 62 | "horizon": null, 63 | "ignore_worker_failures": false, 64 | "in_evaluation": false, 65 | "input": "sampler", 66 | "input_config": {}, 67 | "input_evaluation": [ 68 | "is", 69 | "wis" 70 | ], 71 | "kl_coeff": 0.2, 72 | "kl_target": 0.01, 73 | "lambda": 1.0, 74 | "local_tf_session_args": { 75 | "inter_op_parallelism_threads": 8, 76 | "intra_op_parallelism_threads": 8 77 | }, 78 | "log_level": "WARN", 79 | "log_sys_usage": true, 80 | "logger_config": null, 81 | "lr": 0.0005, 82 | "lr_schedule": null, 83 | "metrics_smoothing_episodes": 100, 84 | "min_iter_time_s": 0, 85 | "model": { 86 | "_disable_preprocessor_api": false, 87 | "_time_major": false, 88 | "_use_default_native_models": false, 89 | "attention_dim": 64, 90 | "attention_head_dim": 32, 91 | "attention_init_gru_gate_bias": 2.0, 92 | "attention_memory_inference": 50, 93 | "attention_memory_training": 50, 94 | "attention_num_heads": 1, 95 | "attention_num_transformer_units": 1, 96 | "attention_position_wise_mlp_dim": 32, 97 | "attention_use_n_prev_actions": 0, 98 | "attention_use_n_prev_rewards": 0, 99 | "conv_activation": "relu", 100 | "conv_filters": null, 101 | "custom_action_dist": null, 102 | "custom_model": "CybORG_Torch", 103 | "custom_model_config": {}, 104 | "custom_preprocessor": null, 105 | "dim": 84, 106 | "fcnet_activation": "tanh", 107 | "fcnet_hiddens": [ 108 | 256, 109 | 256 110 | ], 111 | "framestack": true, 112 | "free_log_std": false, 113 | "grayscale": false, 114 | "lstm_cell_size": 256, 115 | "lstm_use_prev_action": false, 116 | "lstm_use_prev_action_reward": -1, 117 | "lstm_use_prev_reward": false, 118 | "max_seq_len": 20, 119 | "no_final_linear": false, 120 | "post_fcnet_activation": "relu", 121 | "post_fcnet_hiddens": [], 122 | "use_attention": false, 123 | "use_lstm": false, 124 | "vf_share_layers": false, 125 | "zero_mean": true 126 | }, 127 | "monitor": -1, 128 | "multiagent": { 129 | "count_steps_by": "env_steps", 130 | "observation_fn": null, 131 | "policies": {}, 132 | "policies_to_train": null, 133 | "policy_map_cache": null, 134 | "policy_map_capacity": 100, 135 | "policy_mapping_fn": null, 136 | "replay_mode": "independent" 137 | }, 138 | "no_done_at_end": false, 139 | "normalize_actions": true, 140 | "num_cpus_for_driver": 1, 141 | "num_cpus_per_worker": 1, 142 | "num_envs_per_worker": 1, 143 | "num_gpus": 0, 144 | "num_gpus_per_worker": 0, 145 | "num_sgd_iter": 30, 146 | "num_workers": 0, 147 | "observation_filter": "NoFilter", 148 | "observation_space": null, 149 | "optimizer": {}, 150 | "output": null, 151 | "output_compress_columns": [ 152 | "obs", 153 | "new_obs" 154 | ], 155 | "output_max_file_size": 67108864, 156 | "placement_strategy": "PACK", 157 | "postprocess_inputs": false, 158 | "preprocessor_pref": "deepmind", 159 | "record_env": false, 160 | "remote_env_batch_wait_ms": 0, 161 | "remote_worker_envs": false, 162 | "render_env": false, 163 | "rollout_fragment_length": 200, 164 | "sample_async": false, 165 | "sample_collector": "", 166 | "seed": null, 167 | "sgd_minibatch_size": 128, 168 | "shuffle_buffer_size": 0, 169 | "shuffle_sequences": true, 170 | "simple_optimizer": -1, 171 | "soft_horizon": false, 172 | "synchronize_filters": true, 173 | "tf_session_args": { 174 | "allow_soft_placement": true, 175 | "device_count": { 176 | "CPU": 1 177 | }, 178 | "gpu_options": { 179 | "allow_growth": true 180 | }, 181 | "inter_op_parallelism_threads": 2, 182 | "intra_op_parallelism_threads": 2, 183 | "log_device_placement": false 184 | }, 185 | "timesteps_per_iteration": 0, 186 | "train_batch_size": 4000, 187 | "use_critic": true, 188 | "use_gae": true, 189 | "vf_clip_param": 5.0, 190 | "vf_loss_coeff": 1, 191 | "vf_share_layers": -1 192 | } -------------------------------------------------------------------------------- /agents/baseline_sub_agents/loadBanditController.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pickle as pkl 3 | from neural_nets import * 4 | import os 5 | from sub_agents import * 6 | import os.path as path 7 | from CybORG.Agents import B_lineAgent, SleepAgent, RedMeanderAgent 8 | from configs import * 9 | # from CybORGActionAgent import CybORGActionAgent 10 | import ray.rllib.agents.ppo as ppo 11 | from ray.rllib.agents.ppo import PPOTrainer 12 | from ray.rllib.models import ModelCatalog 13 | from bline_CybORGAgent import CybORGAgent as bline_CybORGAgent 14 | 15 | class LoadBanditBlueAgent: 16 | 17 | """ 18 | Load the agent model using the latest checkpoint and return it for evaluation 19 | """ 20 | def __init__(self) -> None: 21 | ModelCatalog.register_custom_model("CybORG_hier_Model", TorchModel) 22 | 23 | # Load checkpoint locations of each agent 24 | two_up = path.abspath(path.join(__file__, "../../../")) 25 | self.CTRL_checkpoint_pointer = two_up + sub_agents['bandit_trained'] 26 | self.BL_checkpoint_pointer = two_up + sub_agents['B_line_trained'] 27 | self.RM_checkpoint_pointer = two_up + sub_agents['RedMeander_trained'] 28 | 29 | print("Using checkpoint file (Controller): {}".format(self.CTRL_checkpoint_pointer)) 30 | print("Using checkpoint file (B-line): {}".format(self.BL_checkpoint_pointer)) 31 | print("Using checkpoint file (Red Meander): {}".format(self.RM_checkpoint_pointer)) 32 | 33 | # Restore the controller model 34 | with open(self.CTRL_checkpoint_pointer, "rb") as controller_chkpt: # Must open file in binary mode for pickle 35 | self.controller = pkl.load(controller_chkpt) 36 | 37 | self.bandit_observation = np.array([], dtype=int) 38 | RM_config = meander_config 39 | RM_config["in_evaluation"] = True 40 | RM_config["explore"] = False 41 | 42 | BL_config = bline_config 43 | BL_config["in_evaluation"] = True 44 | BL_config["explore"] = False 45 | 46 | #load agent trained against RedMeanderAgent 47 | self.RM_def = PPOTrainer(config=RM_config, env=CybORGAgent) 48 | self.RM_def.restore(self.RM_checkpoint_pointer) 49 | 50 | #load agent trained against B_lineAgent 51 | BL_config['env'] = bline_CybORGAgent 52 | BL_config["env_config"] = {'agent_name': 'Blue', 'env': None, 'max_steps': 100, 'attacker': B_lineAgent} 53 | self.BL_def = ppo.PPOTrainer(config=BL_config, env=bline_CybORGAgent) 54 | self.BL_def.restore(self.BL_checkpoint_pointer) 55 | 56 | #self.red_agent=-1 57 | self.state = [np.zeros(256, np.float32), 58 | np.zeros(256, np.float32)] 59 | self.step = 0 60 | # heuristics 61 | self.set = False 62 | self.observations = [] 63 | self.adversary = 0 64 | 65 | 66 | def set_red_agent(self, red_agent): 67 | self.red_agent = red_agent 68 | 69 | """Compensate for the different method name""" 70 | def get_action(self, obs, action_space): 71 | #update sliding window 72 | # discover network services sequence 73 | self.step += 1 74 | if self.step < 5: 75 | self.bandit_observation = np.append(self.bandit_observation, obs[2:]) 76 | #return 0, -1 77 | elif self.step == 5: 78 | bandit_obs_hashable = ''.join(str(bit) for bit in self.bandit_observation) 79 | self.adversary = np.argmax(self.controller[bandit_obs_hashable]) 80 | 81 | 82 | #select agent to compute action 83 | """ if self.red_agent == B_lineAgent or self.red_agent == SleepAgent: 84 | agent_to_select = 0 85 | else: #RedMeanderAgent 86 | agent_to_select = 1""" 87 | #agent_to_select = self.controller_agent.compute_single_action(obs) 88 | if self.adversary == 0: 89 | #print('b_line defence') 90 | # get action from agent trained against the B_lineAgent 91 | 92 | # keep track of the lstm state for later use 93 | _, self.state, _ = self.RM_def.compute_single_action(obs[2:], self.state) 94 | bl_obs = self.bits_to_float(obs) 95 | agent_action = self.BL_def.compute_single_action(bl_obs) 96 | elif self.adversary == 1: 97 | # get action from agent trained against the RedMeanderAgent 98 | agent_action, state, _ = self.RM_def.compute_single_action(obs[2:], self.state) 99 | #print('meander defence') 100 | self.state = state 101 | # agent_action = self.RM_def.compute_single_action(self.observation[-1:]) 102 | elif self.adversary == 2: 103 | agent_action = 0 104 | 105 | else: 106 | print('something went terribly wrong, old sport') 107 | return agent_action, self.adversary 108 | 109 | def end_episode(self): 110 | self.set = False 111 | self.state = [np.zeros(256, np.float32), 112 | np.zeros(256, np.float32)] 113 | self.bandit_observation = np.array([], dtype=int) 114 | #self.observations = [] 115 | self.adversary = 0 116 | self.step = 0 117 | 118 | def bits_to_float(self, obs): 119 | float_list = [] 120 | 121 | rest_obs = np.reshape(obs[2:], (13, 4)) 122 | for host in rest_obs: 123 | activity = np.array(host[:2]) 124 | compromised = np.array(host[2:]) 125 | if all(activity == [0,0]): 126 | value = [0.] 127 | elif all(activity == [1,0]): 128 | value = [1.] 129 | elif all(activity == [1,1]): 130 | value = [2.] 131 | else: 132 | raise ValueError('not activity type') 133 | float_list += value 134 | 135 | # Compromised 136 | if all(compromised == [0, 0]): 137 | value = [0.] 138 | elif all(compromised == [1, 0]): 139 | value = [1.] 140 | elif all(compromised == [0,1]): 141 | value = [2.] 142 | elif all(compromised == [1,1]): 143 | value = [3.] 144 | else: 145 | raise ValueError('not compromised type') 146 | float_list += value 147 | 148 | success = obs[:2] 149 | if all(success == [1, 0]): 150 | float_list += [0.] 151 | elif all(success == [0, 0]): 152 | float_list += [1.] 153 | elif all(success == [0, 1]): 154 | float_list += [2.] 155 | 156 | return np.array(float_list) 157 | 158 | -------------------------------------------------------------------------------- /agents/baseline_sub_agents/train_simple_bandit.py: -------------------------------------------------------------------------------- 1 | from CybORG import CybORG 2 | from CybORG.Agents import * 3 | from CybORG.Agents.Wrappers import ChallengeWrapper 4 | import inspect 5 | import numpy as np 6 | from collections import defaultdict 7 | import matplotlib.pyplot as plt 8 | import os 9 | import random 10 | import pickle as pkl 11 | import time 12 | 13 | path = str(inspect.getfile(CybORG)) 14 | path = path[:-10] + '/Shared/Scenarios/Scenario2.yaml' 15 | agents = { 16 | 'Red': B_lineAgent # RedMeanderAgent 17 | } 18 | 19 | env = CybORG(path, 'sim', agents={'Red': B_lineAgent}) 20 | BL_env = ChallengeWrapper(env=env, agent_name='Blue') # wrap(cyborg) 21 | 22 | env = CybORG(path, 'sim', agents={'Red': RedMeanderAgent}) 23 | RM_env = ChallengeWrapper(env=env, agent_name='Blue') # wrap(cyborg) 24 | 25 | env = CybORG(path, 'sim', agents={'Red': SleepAgent}) 26 | SL_env = ChallengeWrapper(env=env, agent_name='Blue') # wrap(cyborg) 27 | 28 | action_map = { 29 | 0: B_lineAgent, 30 | 1: RedMeanderAgent, 31 | 2: SleepAgent, 32 | } 33 | action_selector = list(range(len(action_map))) 34 | 35 | red_states = [] 36 | with open('red_states.pkl', "rb") as bline_states_file: # Must open file in binary mode for pickle 37 | print('Red Agent states loaded from {}'.format(bline_states_file)) 38 | red_states = pkl.load(bline_states_file) 39 | 40 | N_EPISODES = 15000 41 | N_STEPS = 4 42 | EPSILON = 0.01 43 | test_episodes = 1000 44 | def init_bandit(epsilon=0.0001): 45 | # Generate an arbitrary value function and (approximately..) soft policy 46 | Q = {} 47 | N = {} 48 | rewards = {} 49 | #policy = {} 50 | #returns = list() 51 | # Policy is a map from obs to action 52 | for s in red_states: 53 | s_hashable = ''.join(str(bit) for bit in s) 54 | Q[s_hashable] = np.zeros(len(action_map)) 55 | N[s_hashable] = {action:0 for action in action_map.keys()} 56 | rewards[s_hashable] = [] 57 | # #s_hashable = s # ''.join(str(bit) for bit in s) 58 | # #policy[s_hashable] = np.random.dirichlet(np.ones(n_blue_actions) * 100., size=1)[0] 59 | 60 | #Q = np.zeros(len(action_selector))#{action:0 for action in action_map.keys()} 61 | return Q, N, rewards#, policy, returns 62 | 63 | 64 | 65 | Q, N, rewards = init_bandit(EPSILON) 66 | 67 | import pandas as pd 68 | 69 | dataset = pd.DataFrame(['observation', 'label']) 70 | 71 | for episode in range(N_EPISODES): 72 | bandit_obs = np.array([], dtype=int) 73 | attacker = np.random.choice([0, 1, 2], p=[0.475, 0.475, 0.05]) 74 | if attacker == 0: 75 | current_env = BL_env 76 | adversary = B_lineAgent 77 | elif attacker == 1: 78 | current_env = RM_env 79 | adversary = RedMeanderAgent 80 | elif attacker == 2: 81 | current_env = SL_env 82 | adversary = SleepAgent 83 | 84 | 85 | blue_obs = current_env.reset() 86 | 87 | for step in range(N_STEPS): 88 | bandit_obs = np.append(bandit_obs, blue_obs) 89 | blue_obs, rew, done, info = current_env.step(0) 90 | bandit_obs = ''.join(str(bit) for bit in bandit_obs) 91 | 92 | dataset.append({'observation':bandit_obs, 'label':attacker}, ignore_index=True) 93 | 94 | if random.uniform(0, 1) < EPSILON: 95 | action = random.choice(action_selector) 96 | else: 97 | try: 98 | action = np.argmax(Q[bandit_obs]) 99 | except KeyError as e: 100 | # State not seen before 101 | print('State not seen before: {}'.format(bandit_obs)) 102 | # Initialise with a random soft policy 103 | Q[bandit_obs] = np.zeros(len(action_map)) 104 | N[bandit_obs] = {action:0 for action in action_map.keys()} 105 | rewards[bandit_obs] = [] 106 | action = np.argmax(Q[bandit_obs]) 107 | 108 | if action == 0: 109 | #print('guess') 110 | if adversary == action_map[0]: 111 | bandit_rew = 1 112 | else: 113 | bandit_rew = -1 114 | elif action == 1: 115 | #print('guess') 116 | if adversary == action_map[1]: 117 | bandit_rew = 1 118 | else: 119 | bandit_rew = -1 120 | elif action == 2: 121 | bandit_rew = 0 122 | #increment actions seen 123 | N[bandit_obs][action] += 1 124 | # compute Q 125 | Q[bandit_obs][action] = Q[bandit_obs][action] + (1/(N[bandit_obs][action]))*(bandit_rew - Q[bandit_obs][action]) 126 | 127 | rewards[bandit_obs].append(bandit_rew) 128 | 129 | cumulative_average = rewards 130 | for obs in rewards.keys(): 131 | cumulative_average[obs] = np.cumsum(rewards[obs]) / len(rewards) 132 | #print(rewards[obs]) 133 | 134 | bandit_path = '../../logs/training/controller_bandit_{}'.format(time.strftime("%Y-%m-%d_%H-%M-%S")) 135 | os.mkdir(bandit_path) 136 | # plot moving average ctr 137 | obs_seen = [] 138 | for obs in cumulative_average.keys(): 139 | if sum(cumulative_average[obs]) != 0: 140 | plt.plot(cumulative_average[obs]) 141 | obs_seen.append(obs) 142 | plt.xscale('log') 143 | #plt.show() 144 | plt.savefig(bandit_path+'/log_rewards.eps', format='eps') 145 | plt.legend(['state '+ str(i) for i in range(len(obs_seen))]) 146 | plt.savefig(bandit_path+'/log_rewards_legend.eps', format='eps') 147 | 148 | # plot moving average ctr linear 149 | for obs in cumulative_average.keys(): 150 | if sum(cumulative_average[obs]) != 0: 151 | plt.plot(cumulative_average[obs]) 152 | plt.savefig(bandit_path+'/rewards.eps', format='eps') 153 | plt.legend(['state '+ str(i) for i in range(len(obs_seen))]) 154 | #plt.show() 155 | plt.savefig(bandit_path+'/rewards_legend.eps', format='eps') 156 | 157 | errors = 0 158 | 159 | for episode in range(test_episodes): 160 | # agent = B_lineAgent() 161 | if random.choice([0, 1]) == 0: 162 | current_env = BL_env 163 | adversary = B_lineAgent 164 | else: 165 | current_env = RM_env 166 | adversary = RedMeanderAgent 167 | bandit_obs = np.array([], dtype=int) 168 | blue_obs = current_env.reset() 169 | 170 | for step in range(N_STEPS): 171 | bandit_obs = np.append(bandit_obs, blue_obs) 172 | blue_obs, rew, done, info = current_env.step(0) 173 | 174 | blue_obs_hashable = ''.join(str(bit) for bit in bandit_obs) 175 | action = np.argmax(Q[blue_obs_hashable]) 176 | #prediction = statistics.mode(predicitions) 177 | prediction = str(action_map[action].__name__) 178 | if action_map[action] != adversary: 179 | errors += 1 180 | 181 | #print('Adversary: '+ str(adversary.__name__) + ', Predicted: '+ prediction) 182 | #prediction = statistics.mode(alt_predicitions) 183 | 184 | #print('Adversary: '+ str(adversary.__name__) + ', ALT Predicted: '+ prediction) 185 | print(Q) 186 | print('Total errors: {:d}, over episodes: {:d}, success rate of {:.4f}%'.format(errors, test_episodes, (1-errors/test_episodes)*100)) 187 | 188 | 189 | bandit_save = '/'.join([bandit_path, "bandit_controller_{}.pkl".format(N_EPISODES)]) 190 | reward_save = '/'.join([bandit_path, "/bandit_controller_rewards_{}.pkl".format(N_EPISODES)]) 191 | with open(bandit_save, "wb") as bandit_save_file: # Must open file in binary mode for pickle 192 | pkl.dump(Q, bandit_save_file) 193 | with open(reward_save, "wb") as reward_save_file: # Must open file in binary mode for pickle 194 | pkl.dump(rewards, reward_save_file) 195 | 196 | 197 | -------------------------------------------------------------------------------- /agents/baseline_sub_agents/BlueTableActionWrapper.py: -------------------------------------------------------------------------------- 1 | from copy import deepcopy 2 | from prettytable import PrettyTable 3 | import numpy as np 4 | 5 | from CybORG.Shared.Results import Results 6 | from CybORG.Agents.Wrappers.BaseWrapper import BaseWrapper 7 | from CybORG.Agents.Wrappers.TrueTableWrapper import TrueTableWrapper 8 | 9 | 10 | class BlueTableWrapper(BaseWrapper): 11 | def __init__(self, env=None, agent=None, output_mode='table'): 12 | super().__init__(env, agent) 13 | self.env = TrueTableWrapper(env=env, agent=agent) 14 | self.agent = agent 15 | 16 | self.baseline = None 17 | self.output_mode = output_mode 18 | self.blue_info = {} 19 | 20 | def reset(self, agent='Blue'): 21 | result = self.env.reset(agent) 22 | obs = result.observation 23 | if agent == 'Blue': 24 | self._process_initial_obs(obs) 25 | obs = self.observation_change(obs, baseline=True) 26 | obs = np.append([0, 0], obs) 27 | result.observation = obs 28 | return result 29 | 30 | def step(self, agent=None, action=None) -> Results: 31 | result = self.env.step(agent, action) 32 | obs = result.observation 33 | if agent == 'Blue': 34 | changed_obs = self.observation_change(obs) 35 | if 'success' in obs.keys(): 36 | if obs['success'].name == 'TRUE': 37 | obs = np.append([1, 0], changed_obs) 38 | elif obs['success'].name == 'FALSE': 39 | obs = np.append([0, 1], changed_obs) 40 | elif obs['success'].name == 'UNKNOWN': 41 | obs = np.append([0, 0], changed_obs) 42 | else: 43 | print(obs['success'].name) 44 | result.observation = obs 45 | result.action_space = self.action_space_change(result.action_space) 46 | return result 47 | 48 | def get_table(self, output_mode='blue_table'): 49 | if output_mode == 'blue_table': 50 | return self._create_blue_table(success=None) 51 | elif output_mode == 'true_table': 52 | return self.env.get_table() 53 | 54 | def observation_change(self, observation, baseline=False): 55 | obs = observation if type(observation) == dict else observation.data 56 | obs = deepcopy(observation) 57 | success = obs['success'] 58 | 59 | self._process_last_action() 60 | anomaly_obs = self._detect_anomalies(obs) if not baseline else obs 61 | del obs['success'] 62 | # TODO check what info is for baseline 63 | info = self._process_anomalies(anomaly_obs) 64 | if baseline: 65 | for host in info: 66 | info[host][-2] = 'None' 67 | info[host][-1] = 'No' 68 | self.blue_info[host][-1] = 'No' 69 | 70 | self.info = info 71 | 72 | if self.output_mode == 'table': 73 | return self._create_blue_table(success) 74 | elif self.output_mode == 'anomaly': 75 | anomaly_obs['success'] = success 76 | return anomaly_obs 77 | elif self.output_mode == 'raw': 78 | return observation 79 | elif self.output_mode == 'vector': 80 | return self._create_vector(success) 81 | else: 82 | raise NotImplementedError('Invalid output_mode for BlueTableWrapper') 83 | 84 | def _process_initial_obs(self, obs): 85 | obs = obs.copy() 86 | self.baseline = obs 87 | del self.baseline['success'] 88 | for hostid in obs: 89 | if hostid == 'success': 90 | continue 91 | host = obs[hostid] 92 | interface = host['Interface'][0] 93 | subnet = interface['Subnet'] 94 | ip = str(interface['IP Address']) 95 | hostname = host['System info']['Hostname'] 96 | self.blue_info[hostname] = [str(subnet), str(ip), hostname, 'None', 'No'] 97 | return self.blue_info 98 | 99 | def _process_last_action(self): 100 | action = self.get_last_action(agent='Blue') 101 | if action is not None: 102 | name = action.__class__.__name__ 103 | hostname = action.get_params()['hostname'] if name in ('Restore', 'Remove') else None 104 | 105 | if name == 'Restore': 106 | self.blue_info[hostname][-1] = 'No' 107 | elif name == 'Remove': 108 | compromised = self.blue_info[hostname][-1] 109 | if compromised != 'No': 110 | self.blue_info[hostname][-1] = 'Unknown' 111 | 112 | def _detect_anomalies(self, obs): 113 | if self.baseline is None: 114 | raise TypeError( 115 | 'BlueTableWrapper was unable to establish baseline. This usually means the environment was not reset before calling the step method.') 116 | 117 | anomaly_dict = {} 118 | 119 | for hostid, host in obs.items(): 120 | if hostid == 'success': 121 | continue 122 | 123 | host_baseline = self.baseline[hostid] 124 | if host == host_baseline: 125 | continue 126 | 127 | host_anomalies = {} 128 | if 'Files' in host: 129 | baseline_files = host_baseline.get('Files', []) 130 | anomalous_files = [] 131 | for f in host['Files']: 132 | if f not in baseline_files: 133 | anomalous_files.append(f) 134 | if anomalous_files: 135 | host_anomalies['Files'] = anomalous_files 136 | 137 | if 'Processes' in host: 138 | baseline_processes = host_baseline.get('Processes', []) 139 | anomalous_processes = [] 140 | for p in host['Processes']: 141 | if p not in baseline_processes: 142 | anomalous_processes.append(p) 143 | if anomalous_processes: 144 | host_anomalies['Processes'] = anomalous_processes 145 | 146 | if host_anomalies: 147 | anomaly_dict[hostid] = host_anomalies 148 | 149 | return anomaly_dict 150 | 151 | def _process_anomalies(self, anomaly_dict): 152 | info = deepcopy(self.blue_info) 153 | for hostid, host_anomalies in anomaly_dict.items(): 154 | assert len(host_anomalies) > 0 155 | if 'Processes' in host_anomalies: 156 | connection_type = self._interpret_connections(host_anomalies['Processes']) 157 | info[hostid][-2] = connection_type 158 | if connection_type == 'Exploit': 159 | info[hostid][-1] = 'User' 160 | self.blue_info[hostid][-1] = 'User' 161 | if 'Files' in host_anomalies: 162 | malware = [f['Density'] >= 0.9 for f in host_anomalies['Files']] 163 | if any(malware): 164 | info[hostid][-1] = 'Privileged' 165 | self.blue_info[hostid][-1] = 'Privileged' 166 | 167 | return info 168 | 169 | def _interpret_connections(self, activity: list): 170 | num_connections = len(activity) 171 | 172 | ports = set([item['Connections'][0]['local_port'] \ 173 | for item in activity if 'Connections' in item]) 174 | port_focus = len(ports) 175 | 176 | remote_ports = set([item['Connections'][0].get('remote_port') \ 177 | for item in activity if 'Connections' in item]) 178 | if None in remote_ports: 179 | remote_ports.remove(None) 180 | 181 | if num_connections >= 3 and port_focus >= 3: 182 | anomaly = 'Scan' 183 | elif 4444 in remote_ports: 184 | anomaly = 'Exploit' 185 | elif num_connections >= 3 and port_focus == 1: 186 | anomaly = 'Exploit' 187 | elif 'Service Name' in activity[0]: 188 | anomaly = 'None' 189 | else: 190 | anomaly = 'Scan' 191 | 192 | return anomaly 193 | 194 | 195 | def _create_blue_table(self, success): 196 | table = PrettyTable([ 197 | 'Subnet', 198 | 'IP Address', 199 | 'Hostname', 200 | 'Activity', 201 | 'Compromised' 202 | ]) 203 | for hostid in self.info: 204 | table.add_row(self.info[hostid]) 205 | 206 | table.sortby = 'Hostname' 207 | table.success = success 208 | return table 209 | 210 | def _create_vector(self, success): 211 | table = self._create_blue_table(success)._rows 212 | 213 | proto_vector = [] 214 | for row in table: 215 | # Activity 216 | activity = row[3] 217 | if activity == 'None': 218 | value = [0, 0] 219 | elif activity == 'Scan': 220 | value = [1, 0] 221 | elif activity == 'Exploit': 222 | value = [1, 1] 223 | else: 224 | raise ValueError('Table had invalid Access Level') 225 | proto_vector.extend(value) 226 | 227 | # Compromised 228 | compromised = row[4] 229 | if compromised == 'No': 230 | value = [0, 0] 231 | elif compromised == 'Unknown': 232 | value = [1, 0] 233 | elif compromised == 'User': 234 | value = [0, 1] 235 | elif compromised == 'Privileged': 236 | value = [1, 1] 237 | else: 238 | raise ValueError('Table had invalid Access Level') 239 | proto_vector.extend(value) 240 | 241 | return np.array(proto_vector) 242 | 243 | def get_attr(self, attribute: str): 244 | return self.env.get_attr(attribute) 245 | 246 | def get_observation(self, agent: str): 247 | if agent == 'Blue' and self.output_mode == 'table': 248 | output = self.get_table() 249 | else: 250 | output = self.get_attr('get_observation')(agent) 251 | 252 | return output 253 | 254 | def get_agent_state(self, agent: str): 255 | return self.get_attr('get_agent_state')(agent) 256 | 257 | def get_action_space(self, agent): 258 | return self.env.get_action_space(agent) 259 | 260 | def get_last_action(self, agent): 261 | return self.get_attr('get_last_action')(agent) 262 | 263 | def get_ip_map(self): 264 | return self.get_attr('get_ip_map')() 265 | 266 | def get_rewards(self): 267 | return self.get_attr('get_rewards')() 268 | -------------------------------------------------------------------------------- /logs/training/PPO_RedMeanderAgent_2022-09-14_16-25-58/experiment_state-2022-09-14_16-25-58.json: -------------------------------------------------------------------------------- 1 | { 2 | "checkpoints": [ 3 | "{\n \"stub\": false,\n \"trainable_name\": \"PPO\",\n \"trial_id\": \"88835_00000\",\n \"config\": {\n \"num_workers\": 0,\n \"num_envs_per_worker\": 1,\n \"create_env_on_driver\": false,\n \"rollout_fragment_length\": 200,\n \"batch_mode\": \"truncate_episodes\",\n \"gamma\": 0.99,\n \"lr\": 0.0005,\n \"train_batch_size\": 4000,\n \"model\": {\n \"_use_default_native_models\": false,\n \"_disable_preprocessor_api\": false,\n \"fcnet_hiddens\": [\n 256,\n 256\n ],\n \"fcnet_activation\": \"tanh\",\n \"conv_filters\": null,\n \"conv_activation\": \"relu\",\n \"post_fcnet_hiddens\": [],\n \"post_fcnet_activation\": \"relu\",\n \"free_log_std\": false,\n \"no_final_linear\": false,\n \"vf_share_layers\": false,\n \"use_lstm\": false,\n \"max_seq_len\": 20,\n \"lstm_cell_size\": 256,\n \"lstm_use_prev_action\": false,\n \"lstm_use_prev_reward\": false,\n \"_time_major\": false,\n \"use_attention\": false,\n \"attention_num_transformer_units\": 1,\n \"attention_dim\": 64,\n \"attention_num_heads\": 1,\n \"attention_head_dim\": 32,\n \"attention_memory_inference\": 50,\n \"attention_memory_training\": 50,\n \"attention_position_wise_mlp_dim\": 32,\n \"attention_init_gru_gate_bias\": 2.0,\n \"attention_use_n_prev_actions\": 0,\n \"attention_use_n_prev_rewards\": 0,\n \"framestack\": true,\n \"dim\": 84,\n \"grayscale\": false,\n \"zero_mean\": true,\n \"custom_model\": \"CybORG_Torch\",\n \"custom_model_config\": {},\n \"custom_action_dist\": null,\n \"custom_preprocessor\": null,\n \"lstm_use_prev_action_reward\": -1\n },\n \"optimizer\": {},\n \"horizon\": null,\n \"soft_horizon\": false,\n \"no_done_at_end\": false,\n \"env\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059525000000000000008c11626c696e655f4379624f52474167656e74948c0b4379624f52474167656e749493942e\"\n },\n \"observation_space\": null,\n \"action_space\": null,\n \"env_config\": {\n \"null\": 0\n },\n \"remote_worker_envs\": false,\n \"remote_env_batch_wait_ms\": 0,\n \"env_task_fn\": null,\n \"render_env\": false,\n \"record_env\": false,\n \"clip_rewards\": null,\n \"normalize_actions\": true,\n \"clip_actions\": false,\n \"preprocessor_pref\": \"deepmind\",\n \"log_level\": \"WARN\",\n \"callbacks\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059533000000000000008c1a7261792e726c6c69622e6167656e74732e63616c6c6261636b73948c1044656661756c7443616c6c6261636b739493942e\"\n },\n \"ignore_worker_failures\": false,\n \"log_sys_usage\": true,\n \"fake_sampler\": false,\n \"framework\": \"torch\",\n \"eager_tracing\": true,\n \"eager_max_retraces\": 20,\n \"explore\": true,\n \"exploration_config\": {\n \"type\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005951b000000000000008c09637572696f73697479948c09437572696f736974799493942e\"\n },\n \"framework\": \"torch\",\n \"eta\": 1.0,\n \"lr\": 0.001,\n \"feature_dim\": 53,\n \"feature_net_config\": {\n \"fcnet_hiddens\": [],\n \"fcnet_activation\": \"relu\",\n \"framework\": \"torch\"\n },\n \"inverse_net_hiddens\": [\n 256\n ],\n \"inverse_net_activation\": \"relu\",\n \"forward_net_hiddens\": [\n 256\n ],\n \"forward_net_activation\": \"relu\",\n \"beta\": 0.2,\n \"sub_exploration\": {\n \"type\": \"StochasticSampling\"\n }\n },\n \"evaluation_interval\": null,\n \"evaluation_num_episodes\": 10,\n \"evaluation_parallel_to_training\": false,\n \"in_evaluation\": false,\n \"evaluation_config\": {},\n \"evaluation_num_workers\": 0,\n \"custom_eval_function\": null,\n \"sample_async\": false,\n \"sample_collector\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059551000000000000008c357261792e726c6c69622e6576616c756174696f6e2e636f6c6c6563746f72732e73696d706c655f6c6973745f636f6c6c6563746f72948c1353696d706c654c697374436f6c6c6563746f729493942e\"\n },\n \"observation_filter\": \"NoFilter\",\n \"synchronize_filters\": true,\n \"tf_session_args\": {\n \"intra_op_parallelism_threads\": 2,\n \"inter_op_parallelism_threads\": 2,\n \"gpu_options\": {\n \"allow_growth\": true\n },\n \"log_device_placement\": false,\n \"device_count\": {\n \"CPU\": 1\n },\n \"allow_soft_placement\": true\n },\n \"local_tf_session_args\": {\n \"intra_op_parallelism_threads\": 8,\n \"inter_op_parallelism_threads\": 8\n },\n \"compress_observations\": false,\n \"collect_metrics_timeout\": 180,\n \"metrics_smoothing_episodes\": 100,\n \"min_iter_time_s\": 0,\n \"timesteps_per_iteration\": 0,\n \"seed\": null,\n \"extra_python_environs_for_driver\": {},\n \"extra_python_environs_for_worker\": {},\n \"num_gpus\": 0,\n \"_fake_gpus\": false,\n \"num_cpus_per_worker\": 1,\n \"num_gpus_per_worker\": 0,\n \"custom_resources_per_worker\": {},\n \"num_cpus_for_driver\": 1,\n \"placement_strategy\": \"PACK\",\n \"input\": \"sampler\",\n \"input_config\": {},\n \"actions_in_input_normalized\": false,\n \"input_evaluation\": [\n \"is\",\n \"wis\"\n ],\n \"postprocess_inputs\": false,\n \"shuffle_buffer_size\": 0,\n \"output\": null,\n \"output_compress_columns\": [\n \"obs\",\n \"new_obs\"\n ],\n \"output_max_file_size\": 67108864,\n \"multiagent\": {\n \"policies\": {},\n \"policy_map_capacity\": 100,\n \"policy_map_cache\": null,\n \"policy_mapping_fn\": null,\n \"policies_to_train\": null,\n \"observation_fn\": null,\n \"replay_mode\": \"independent\",\n \"count_steps_by\": \"env_steps\"\n },\n \"logger_config\": null,\n \"_tf_policy_handles_more_than_one_loss\": false,\n \"_disable_preprocessor_api\": false,\n \"simple_optimizer\": -1,\n \"monitor\": -1,\n \"use_critic\": true,\n \"use_gae\": true,\n \"lambda\": 1.0,\n \"kl_coeff\": 0.2,\n \"sgd_minibatch_size\": 128,\n \"shuffle_sequences\": true,\n \"num_sgd_iter\": 30,\n \"lr_schedule\": null,\n \"vf_loss_coeff\": 1,\n \"entropy_coeff\": 0.0,\n \"entropy_coeff_schedule\": null,\n \"clip_param\": 0.5,\n \"vf_clip_param\": 5.0,\n \"grad_clip\": null,\n \"kl_target\": 0.01,\n \"vf_share_layers\": -1\n },\n \"local_dir\": \"/vol/bitbucket/yw21218/thesis/final_sub/cage-challenge-2/logs/training/PPO_RedMeanderAgent_2022-09-14_16-25-58\",\n \"evaluated_params\": {},\n \"experiment_tag\": \"0\",\n \"location\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"8005953a000000000000008c0e7261792e74756e652e747269616c948c084c6f636174696f6e9493942981947d94288c08686f73746e616d65944e8c03706964944e75622e\"\n },\n \"placement_group_factory\": \"800595a6000000000000008c1f7261792e74756e652e7574696c732e706c6163656d656e745f67726f757073948c15506c6163656d656e7447726f7570466163746f72799493942981947d94288c155f686561645f62756e646c655f69735f656d70747994898c085f62756e646c6573945d947d948c0343505594473ff000000000000073618c095f7374726174656779948c045041434b948c055f6172677394298c075f6b7761726773947d9475622e\",\n \"stopping_criterion\": {\n \"training_iteration\": 10000000,\n \"timesteps_total\": 10000000,\n \"episode_reward_mean\": -0.1\n },\n \"log_to_file\": [\n null,\n null\n ],\n \"max_failures\": 0,\n \"_last_result\": {\n \"trial_id\": \"88835_00000\"\n },\n \"_default_result_or_future\": null,\n \"last_update_time\": -Infinity,\n \"metric_analysis\": {},\n \"n_steps\": [\n 5,\n 10\n ],\n \"metric_n_steps\": {},\n \"export_formats\": [],\n \"status\": \"RUNNING\",\n \"start_time\": 1663169159.09581,\n \"logdir\": \"/vol/bitbucket/yw21218/thesis/final_sub/cage-challenge-2/logs/training/PPO_RedMeanderAgent_2022-09-14_16-25-58/PPO_CybORGAgent_88835_00000_0_2022-09-14_16-25-58\",\n \"runner\": null,\n \"last_debug\": 0,\n \"error_file\": null,\n \"error_msg\": null,\n \"trial_name_creator\": null,\n \"custom_trial_name\": null,\n \"custom_dirname\": null,\n \"saving_to\": null,\n \"remote_checkpoint_dir_prefix\": null,\n \"sync_function_tpl\": null,\n \"checkpoint_freq\": 1,\n \"checkpoint_at_end\": true,\n \"keep_checkpoints_num\": 3,\n \"checkpoint_score_attr\": \"episode_reward_mean\",\n \"sync_on_checkpoint\": true,\n \"checkpoint_manager\": {\n \"_type\": \"CLOUDPICKLE_FALLBACK\",\n \"value\": \"80059560010000000000008c1b7261792e74756e652e636865636b706f696e745f6d616e61676572948c11436865636b706f696e744d616e616765729493942981947d94288c146b6565705f636865636b706f696e74735f6e756d944b038c165f636865636b706f696e745f73636f72655f6465736394898c165f636865636b706f696e745f73636f72655f61747472948c13657069736f64655f7265776172645f6d65616e948c1c6e65776573745f70657273697374656e745f636865636b706f696e749468008c0a436865636b706f696e749493942981947d94288c0773746f72616765948c0a70657273697374656e74948c0576616c7565944e8c06726573756c74947d9475628c195f6e65776573745f6d656d6f72795f636865636b706f696e7494680b2981947d9428680e8c066d656d6f72799468104e68117d9475628c115f626573745f636865636b706f696e7473945d948c0b5f6d656d62657273686970948f9475622e\"\n },\n \"restore_path\": null,\n \"restoring_from\": null,\n \"num_failures\": 0,\n \"has_new_resources\": false,\n \"results\": \"80054e2e\",\n \"best_result\": \"80054e2e\",\n \"param_config\": \"80054e2e\",\n \"extra_arg\": \"80054e2e\",\n \"_state_json\": null,\n \"_state_valid\": false\n}" 4 | ], 5 | "runner_data": { 6 | "_max_pending_trials": 16, 7 | "_metric": null, 8 | "_total_time": 0, 9 | "_iteration": 27, 10 | "_has_errored": false, 11 | "_fail_fast": false, 12 | "_server_port": null, 13 | "_cached_trial_decisions": {}, 14 | "_queued_trial_decisions": {}, 15 | "_updated_queue": true, 16 | "_result_wait_time": 1, 17 | "_should_stop_experiment": false, 18 | "_local_checkpoint_dir": "/vol/bitbucket/yw21218/thesis/final_sub/cage-challenge-2/logs/training/PPO_RedMeanderAgent_2022-09-14_16-25-58", 19 | "_remote_checkpoint_dir": null, 20 | "_stopper": { 21 | "_type": "CLOUDPICKLE_FALLBACK", 22 | "value": "80059527000000000000008c107261792e74756e652e73746f70706572948c0b4e6f6f7053746f707065729493942981942e" 23 | }, 24 | "_resumed": false, 25 | "_start_time": 1663169158.7733886, 26 | "_last_checkpoint_time": -Infinity, 27 | "_session_str": "2022-09-14_16-25-58", 28 | "checkpoint_file": "/vol/bitbucket/yw21218/thesis/final_sub/cage-challenge-2/logs/training/PPO_RedMeanderAgent_2022-09-14_16-25-58/experiment_state-2022-09-14_16-25-58.json", 29 | "_checkpoint_period": "auto", 30 | "launch_web_server": false 31 | }, 32 | "stats": { 33 | "start_time": 1663169158.7733886, 34 | "timestamp": 1663169171.4356413 35 | } 36 | } -------------------------------------------------------------------------------- /agents/baseline_sub_agents/StateRepWrapper.py: -------------------------------------------------------------------------------- 1 | from gym import Env, spaces 2 | from gym.spaces import Discrete, Tuple 3 | from CybORG.Agents.Wrappers import BaseWrapper, OpenAIGymWrapper, BlueTableWrapper,RedTableWrapper,EnumActionWrapper, FixedFlatWrapper 4 | from CybORG.Agents.SimpleAgents import BaseAgent 5 | from CybORG.Shared import Results 6 | from typing import Union, List 7 | import inspect 8 | from prettytable import PrettyTable 9 | import numpy as np 10 | from pprint import pprint 11 | import random 12 | 13 | # import os 14 | # import sys 15 | # sys.path.insert(0, os.path.abspath('.')) 16 | from newBlueTableWrapper import NewBlueTableWrapper 17 | from newFlatWrapper import NewFixedFlatWrapper 18 | 19 | # import decoy actions 20 | from CybORG.Shared.Actions.ConcreteActions.DecoyApache import DecoyApache 21 | from CybORG.Shared.Actions.ConcreteActions.DecoyFemitter import DecoyFemitter 22 | from CybORG.Shared.Actions.ConcreteActions.DecoyHarakaSMPT import DecoyHarakaSMPT 23 | from CybORG.Shared.Actions.ConcreteActions.DecoySmss import DecoySmss 24 | from CybORG.Shared.Actions.ConcreteActions.DecoySSHD import DecoySSHD 25 | from CybORG.Shared.Actions.ConcreteActions.DecoySvchost import DecoySvchost 26 | from CybORG.Shared.Actions.ConcreteActions.DecoyTomcat import DecoyTomcat 27 | from CybORG.Shared.Actions.ConcreteActions.DecoyVsftpd import DecoyVsftpd 28 | 29 | class NewEnumActionWrapper(BaseWrapper): 30 | def __init__(self, env: Union[type, BaseWrapper] = None, agent: BaseAgent = None, reduce_misinform=False): 31 | super().__init__(env, agent) 32 | self.possible_actions = None 33 | self.reduce_misinform = reduce_misinform 34 | self.action_signature = {} 35 | self.decoy_top = DecoyFemitter 36 | self.decoy_other = [DecoyApache, DecoyHarakaSMPT, DecoySmss, DecoySSHD, DecoySvchost, DecoyTomcat, DecoyVsftpd] 37 | self.decoy_top_idx = None 38 | self.decoy_other_idx = [] 39 | self.get_action_space('Red') 40 | 41 | def step(self, agent=None, action: list = None) -> Results: 42 | if action is not None: 43 | if self.reduce_misinform: 44 | if action[0] == self.decoy_other_idx[0]: 45 | a1 = random.choice(self.decoy_other_idx) 46 | elif action[0] == 6: 47 | a1 = 12 # map to the last type of action in self.possible_actions 48 | else: 49 | a1 = action[0] 50 | else: 51 | a1 = action[0] 52 | if len(self.possible_actions[a1])==1: 53 | action = self.possible_actions[a1] 54 | else: 55 | action = self.possible_actions[a1][action[1]] ## changed here 56 | return super().step(agent, action) 57 | 58 | def action_space_change(self, action_space: dict) -> int: 59 | assert type(action_space) is dict, \ 60 | f"Wrapper required a dictionary action space. " \ 61 | f"Please check that the wrappers below the ReduceActionSpaceWrapper return the action space as a dict " 62 | possible_actions = [] 63 | self.decoy_other_idx = [] 64 | temp = {} 65 | params = ['action'] 66 | for i, action in enumerate(action_space['action']): 67 | # check the decoy index: 68 | if action is self.decoy_top: 69 | self.decoy_top_idx = i 70 | elif action in self.decoy_other: 71 | self.decoy_other_idx += [i] 72 | 73 | # find all possible actions for each type of action 74 | if action not in self.action_signature: 75 | self.action_signature[action] = inspect.signature(action).parameters 76 | param_dict = {} 77 | param_list = [{}] 78 | for p in self.action_signature[action]: 79 | if p == 'priority': 80 | continue 81 | temp[p] = [] 82 | if p not in params: 83 | params.append(p) 84 | 85 | if len(action_space[p]) == 1: 86 | for p_dict in param_list: 87 | p_dict[p] = list(action_space[p].keys())[0] 88 | else: 89 | new_param_list = [] 90 | for p_dict in param_list: 91 | for key, val in action_space[p].items(): 92 | p_dict[p] = key 93 | new_param_list.append({key: value for key, value in p_dict.items()}) 94 | param_list = new_param_list 95 | 96 | # create nested list for each type of action 97 | possible_actions_temp = [] 98 | for p_dict in param_list: 99 | possible_actions_temp.append(action(**p_dict)) 100 | possible_actions += [possible_actions_temp] 101 | 102 | self.possible_actions = possible_actions 103 | # print('\033[92m' + 'possible_actions:' + '\033[0m', flush=True) 104 | # print(np.shape(possible_actions)) 105 | 106 | # find the action space 107 | if self.reduce_misinform: 108 | action1_len = len(action_space['action']) - len(self.decoy_other_idx) + 1 109 | else: 110 | action1_len = len(action_space['action']) 111 | action2_len = len(max(possible_actions, key=lambda action: len(action))) 112 | if action_space['agent'] != {'Red': True}: 113 | assert action2_len == len(action_space['hostname']), f"second action has length of {action2_len}, "\ 114 | f"while there are {len(action_space['hostname'])} number of hosts" 115 | 116 | return [action1_len, action2_len] 117 | 118 | 119 | 120 | class NewOpenAIGymWrapper(Env, BaseWrapper): 121 | def __init__(self, agent_name: str, env: BaseWrapper = None, agent: BaseAgent = None): 122 | super().__init__(env, agent_name) 123 | self.agent_name = agent_name 124 | if isinstance(self.get_action_space(self.agent_name), list): 125 | # self.action_space = spaces.MultiDiscrete(self.get_action_space(self.agent_name)) 126 | action_space_shape = self.get_action_space(self.agent_name) 127 | self.action_space = Tuple([Discrete(action_space_shape[0]), Discrete(action_space_shape[1])]) 128 | else: 129 | assert isinstance(self.get_action_space(self.agent_name), int) 130 | self.action_space = spaces.Discrete(self.get_action_space(self.agent_name)) 131 | box_len = len(self.observation_change(self.env.reset(self.agent_name).observation)) 132 | self.observation_space = spaces.Box(-np.inf, np.inf, shape=(box_len,), dtype=np.float32) 133 | self.reward_range = (float('-inf'), float('inf')) 134 | self.metadata = {} 135 | self.action = None 136 | 137 | def step(self, action: Union[int, List[int]] = None): 138 | self.action = action 139 | # print('OpenAI action', action, flush=True) 140 | 141 | result = self.env.step(self.agent_name, action) 142 | result.observation = self.observation_change(result.observation) 143 | result.action_space = self.action_space_change(result.action_space) 144 | info = vars(result) 145 | return np.array(result.observation), result.reward, result.done, info 146 | 147 | def reset(self, agent=None): 148 | result = self.env.reset(self.agent_name) 149 | result.action_space = self.action_space_change(result.action_space) 150 | result.observation = self.observation_change(result.observation) 151 | return np.array(result.observation) 152 | 153 | def render(self): 154 | if self.agent_name == 'Red': 155 | table = PrettyTable({ 156 | 'Subnet', 157 | 'IP Address', 158 | 'Hostname', 159 | 'Scanned', 160 | 'Access', 161 | }) 162 | for ip in self.get_attr('red_info'): 163 | table.add_row(self.get_attr('red_info')[ip]) 164 | table.sortby = 'IP Address' 165 | if self.action is not None: 166 | _action = self.get_attr('possible_actions')[self.action[0]][self.action[1]] # change action selection 167 | return print(f'\nRed Action: {_action}\n{table}') 168 | elif self.agent_name == 'Blue': 169 | table = PrettyTable({ 170 | 'Subnet', 171 | 'IP Address', 172 | 'Hostname', 173 | 'Activity', 174 | 'Compromised', 175 | }) 176 | for hostid in self.get_attr('info'): 177 | table.add_row(self.get_attr('info')[hostid]) 178 | table.sortby = 'Hostname' 179 | if self.action is not None: 180 | _action = self.get_attr('possible_actions')[self.action[0]][self.action[1]] # change action selection 181 | red_action = self.get_last_action(agent=self.agent_name) 182 | return print(f'\nBlue Action: {_action}\nRed Action: {red_action}\n{table}') 183 | return print(table) 184 | 185 | def get_attr(self,attribute:str): 186 | return self.env.get_attr(attribute) 187 | 188 | def get_observation(self, agent: str): 189 | return self.env.get_observation(agent) 190 | 191 | def get_agent_state(self,agent:str): 192 | return self.get_attr('get_agent_state')(agent) 193 | 194 | def get_action_space(self,agent): 195 | return self.env.get_action_space(agent) 196 | 197 | def get_last_action(self,agent): 198 | return self.get_attr('get_last_action')(agent) 199 | 200 | def get_ip_map(self): 201 | return self.get_attr('get_ip_map')() 202 | 203 | def get_rewards(self): 204 | return self.get_attr('get_rewards')() 205 | 206 | 207 | 208 | class StateRepWrapper(Env,BaseWrapper): 209 | def __init__(self, agent_name: str, actionSpace: int, obsSpace: int, env, agent=None, 210 | reward_threshold=None, max_steps = None): 211 | """ 212 | actionSpace: 213 | 0 -> list of 145 possible actions; 214 | 1 -> tuple(13, 13) autoregressive; 215 | 2 -> tuple(7, 13) reduce misinform action; 216 | obsSpace: 217 | 0 -> table wrapper observation (challenge wrapper); 218 | 1 -> fixed flat wrapper observation; 219 | 2 -> new FlatWrapper 220 | 3 -> new BlueTableWrapper 221 | """ 222 | super().__init__(env, agent) 223 | self.agent_name = agent_name 224 | if agent_name.lower() == 'red': 225 | table_wrapper = RedTableWrapper 226 | elif agent_name.lower() == 'blue': 227 | table_wrapper = BlueTableWrapper 228 | else: 229 | raise ValueError('Invalid Agent Name') 230 | 231 | # observation space 232 | if obsSpace ==0: 233 | env = table_wrapper(env, output_mode='vector') 234 | elif obsSpace == 1: 235 | env = FixedFlatWrapper(env) 236 | elif obsSpace == 2: 237 | env = NewFixedFlatWrapper(env=env) 238 | elif obsSpace == 3: 239 | env = NewBlueTableWrapper(env=env, output_mode='float_vector', port_length=0) 240 | else: 241 | raise ValueError('Please input valid integer for obsSpace argument in StateRepWrapper()') 242 | 243 | # action space 244 | if actionSpace == 0: 245 | env = EnumActionWrapper(env=env) 246 | elif actionSpace == 1: 247 | env = NewEnumActionWrapper(env, reduce_misinform=False) 248 | elif actionSpace == 2: 249 | env = NewEnumActionWrapper(env, reduce_misinform=True) 250 | 251 | env = NewOpenAIGymWrapper(agent_name=agent_name, env=env) 252 | 253 | self.env = env 254 | self.action_space = self.env.action_space 255 | self.observation_space = self.env.observation_space 256 | self.reward_threshold = reward_threshold 257 | self.max_steps = max_steps 258 | self.step_counter = None 259 | 260 | def step(self,action=None): 261 | obs, reward, done, info = self.env.step(action=action) 262 | 263 | self.step_counter += 1 264 | if self.max_steps is not None and self.step_counter >= self.max_steps: 265 | done = True 266 | 267 | return obs, reward, done, info 268 | 269 | def reset(self): 270 | self.step_counter = 0 271 | return self.env.reset() 272 | 273 | def get_attr(self,attribute:str): 274 | return self.env.get_attr(attribute) 275 | 276 | def get_observation(self, agent: str): 277 | return self.env.get_observation(agent) 278 | 279 | def get_agent_state(self,agent:str): 280 | return self.env.get_agent_state(agent) 281 | 282 | def get_action_space(self, agent=None) -> dict: 283 | return self.env.get_action_space(self.agent_name) 284 | 285 | def get_last_action(self,agent): 286 | return self.get_attr('get_last_action')(agent) 287 | 288 | def get_ip_map(self): 289 | return self.get_attr('get_ip_map')() 290 | 291 | def get_rewards(self): 292 | return self.get_attr('get_rewards')() 293 | 294 | def get_reward_breakdown(self, agent: str): 295 | return self.get_attr('get_reward_breakdown')(agent) 296 | 297 | -------------------------------------------------------------------------------- /agents/baseline_sub_agents/newBlueTableWrapper.py: -------------------------------------------------------------------------------- 1 | from copy import deepcopy 2 | from prettytable import PrettyTable 3 | import numpy as np 4 | 5 | from CybORG.Shared.Results import Results 6 | from CybORG.Agents.Wrappers.BaseWrapper import BaseWrapper 7 | from CybORG.Agents.Wrappers.TrueTableWrapper import TrueTableWrapper 8 | from pprint import pprint 9 | 10 | class NewBlueTableWrapper(BaseWrapper): 11 | def __init__(self,env=None,agent=None,output_mode='table', port_length=12): 12 | super().__init__(env,agent) 13 | self.env = TrueTableWrapper(env=env, agent=agent) 14 | self.agent = agent 15 | 16 | self.baseline = None 17 | self.output_mode = output_mode 18 | self.blue_info = {} 19 | self.port_length = port_length 20 | 21 | def reset(self, agent='Blue'): 22 | result = self.env.reset(agent) 23 | obs = result.observation 24 | if agent == 'Blue': 25 | self._process_initial_obs(obs) 26 | obs = self.observation_change(obs,baseline=True) 27 | result.observation = obs 28 | return result 29 | 30 | def step(self, agent=None, action=None) -> Results: 31 | result = self.env.step(agent, action) 32 | obs = result.observation 33 | if agent == 'Blue': 34 | obs = self.observation_change(obs) 35 | result.observation = obs 36 | result.action_space = self.action_space_change(result.action_space) 37 | return result 38 | 39 | def get_table(self,output_mode='blue_table'): 40 | if output_mode == 'blue_table': 41 | return self._create_blue_table(success=None) 42 | elif output_mode == 'true_table': 43 | return self.env.get_table() 44 | 45 | def observation_change(self,observation,baseline=False): 46 | obs = observation if type(observation) == dict else observation.data 47 | # print('==========observation') 48 | # pprint(observation) 49 | obs = deepcopy(observation) 50 | success = obs['success'] 51 | 52 | self._process_last_action() 53 | anomaly_obs = self._detect_anomalies(obs) if not baseline else obs 54 | del obs['success'] 55 | # TODO check what info is for baseline 56 | info = self._process_anomalies(anomaly_obs) 57 | if baseline: 58 | for host in info: 59 | info[host][-3] = 'None' 60 | info[host][-2] = 'None' 61 | info[host][-1] = 'No' 62 | self.blue_info[host][-1] = 'No' 63 | 64 | self.info = info 65 | if self.output_mode == 'table': 66 | return self._create_blue_table(success) 67 | elif self.output_mode == 'anomaly': 68 | anomaly_obs['success'] = success 69 | return anomaly_obs 70 | elif self.output_mode == 'raw': 71 | return observation 72 | elif self.output_mode == 'vector': 73 | return self._create_vector(success) 74 | elif self.output_mode == 'float_vector': 75 | return self._create_float_vector(success, port_length=self.port_length) 76 | else: 77 | raise NotImplementedError('Invalid output_mode for BlueTableWrapper') 78 | 79 | def _process_initial_obs(self, obs): 80 | obs = obs.copy() 81 | self.baseline = obs 82 | del self.baseline['success'] 83 | for hostid in obs: 84 | if hostid == 'success': 85 | continue 86 | host = obs[hostid] 87 | interface = host['Interface'][0] 88 | subnet = interface['Subnet'] 89 | ip = str(interface['IP Address']) 90 | hostname = host['System info']['Hostname'] 91 | 92 | self.blue_info[hostname] = [str(subnet),str(ip),hostname, 'None', 'None','No'] # [-3] is local_ports 93 | return self.blue_info 94 | 95 | def _process_last_action(self): 96 | action = self.get_last_action(agent='Blue') 97 | if action is not None: 98 | name = action.__class__.__name__ 99 | hostname = action.get_params()['hostname'] if name in ('Restore','Remove') else None 100 | 101 | if name == 'Restore': 102 | self.blue_info[hostname][-1] = 'No' 103 | elif name == 'Remove': 104 | compromised = self.blue_info[hostname][-1] 105 | if compromised != 'No': 106 | self.blue_info[hostname][-1] = 'Unknown' 107 | 108 | def _detect_anomalies(self,obs): 109 | if self.baseline is None: 110 | raise TypeError('BlueTableWrapper was unable to establish baseline. This usually means the environment was not reset before calling the step method.') 111 | 112 | anomaly_dict = {} 113 | 114 | for hostid,host in obs.items(): 115 | if hostid == 'success': 116 | continue 117 | 118 | host_baseline = self.baseline[hostid] 119 | if host == host_baseline: 120 | continue 121 | 122 | host_anomalies = {} 123 | if 'Files' in host: 124 | baseline_files = host_baseline.get('Files',[]) 125 | anomalous_files = [] 126 | for f in host['Files']: 127 | if f not in baseline_files: 128 | anomalous_files.append(f) 129 | if anomalous_files: 130 | host_anomalies['Files'] = anomalous_files 131 | 132 | if 'Processes' in host: 133 | baseline_processes = host_baseline.get('Processes',[]) 134 | anomalous_processes = [] 135 | for p in host['Processes']: 136 | if p not in baseline_processes: 137 | anomalous_processes.append(p) 138 | if anomalous_processes: 139 | host_anomalies['Processes'] = anomalous_processes 140 | 141 | if host_anomalies: 142 | anomaly_dict[hostid] = host_anomalies 143 | 144 | return anomaly_dict 145 | 146 | def _process_anomalies(self,anomaly_dict): 147 | info = deepcopy(self.blue_info) 148 | for hostid, host_anomalies in anomaly_dict.items(): 149 | # print('info[hostid]', info[hostid]) 150 | assert len(host_anomalies) > 0 151 | if 'Processes' in host_anomalies: 152 | connection_type, all_ports = self._interpret_connections(host_anomalies['Processes']) 153 | info[hostid][-2] = connection_type 154 | info[hostid][-3] = all_ports 155 | if connection_type == 'Exploit': 156 | info[hostid][-1] = 'User' 157 | self.blue_info[hostid][-1] = 'User' 158 | if 'Files' in host_anomalies: 159 | malware = [f['Density'] >= 0.9 for f in host_anomalies['Files']] 160 | if any(malware): 161 | info[hostid][-1] = 'Privileged' 162 | self.blue_info[hostid][-1] = 'Privileged' 163 | 164 | return info 165 | 166 | def _interpret_connections(self,activity:list): 167 | num_connections = len(activity) 168 | 169 | ports = set([item['Connections'][0]['local_port'] \ 170 | for item in activity if 'Connections' in item]) 171 | all_ports = [item['Connections'][0]['local_port'] \ 172 | for item in activity if 'Connections' in item] 173 | port_focus = len(ports) 174 | 175 | remote_ports = set([item['Connections'][0].get('remote_port') \ 176 | for item in activity if 'Connections' in item]) 177 | if None in remote_ports: 178 | remote_ports.remove(None) 179 | 180 | if num_connections >= 3 and port_focus >=3: 181 | anomaly = 'Scan' 182 | elif 4444 in remote_ports: 183 | anomaly = 'Exploit' 184 | elif num_connections >= 3 and port_focus == 1: 185 | anomaly = 'Exploit' 186 | elif 'Service Name' in activity[0]: 187 | anomaly = 'None' 188 | else: 189 | anomaly = 'Scan' 190 | str_all_ports = ','.join([str(port) for port in ports]) 191 | return anomaly, str_all_ports 192 | 193 | # def _malware_analysis(self,obs,hostname): 194 | # anomaly_dict = {hostname: {'Files': []}} 195 | # if hostname in obs: 196 | # if 'Files' in obs[hostname]: 197 | # files = obs[hostname]['Files'] 198 | # else: 199 | # return anomaly_dict 200 | # else: 201 | # return anomaly_dict 202 | 203 | # for f in files: 204 | # if f['Density'] >= 0.9: 205 | # anomaly_dict[hostname]['Files'].append(f) 206 | 207 | # return anomaly_dict 208 | 209 | 210 | def _create_blue_table(self, success): 211 | table = PrettyTable([ 212 | 'Subnet', 213 | 'IP Address', 214 | 'Hostname', 215 | 'Local Ports', 216 | 'Activity', 217 | 'Compromised' 218 | ]) 219 | for hostid in self.info: 220 | table.add_row(self.info[hostid]) 221 | 222 | table.sortby = 'Hostname' 223 | table.success = success 224 | return table 225 | 226 | def _create_float_vector(self, success, port_length): 227 | table = self._create_blue_table(success)._rows 228 | 229 | proto_vector = [] 230 | for row in table: 231 | # ports 232 | ports = row[3] 233 | # print('ports', ports) 234 | if ports != 'None': 235 | try: 236 | fill_ports = [float(port) for port in ports.split(',')] 237 | except: 238 | fill_ports = [] 239 | 240 | if len(fill_ports) <= port_length: 241 | all_ports = fill_ports + [0.]*(port_length-len(fill_ports)) 242 | else: 243 | all_ports = fill_ports[:port_length] 244 | elif ports == 'None': 245 | all_ports = [0.]*port_length 246 | else: 247 | raise ValueError('Table had invalid Access Level-- ports') 248 | proto_vector.extend(all_ports) 249 | 250 | # Activity 251 | activity = row[4] 252 | if activity == 'None': 253 | value = [0.] 254 | elif activity == 'Scan': 255 | value = [1.] 256 | elif activity == 'Exploit': 257 | value = [2.] 258 | else: 259 | raise ValueError('Table had invalid Access Level') 260 | proto_vector.extend(value) 261 | 262 | # Compromised 263 | compromised = row[5] 264 | if compromised == 'No': 265 | value = [0.] 266 | elif compromised == 'Unknown': 267 | value = [1.] 268 | elif compromised == 'User': 269 | value = [2.] 270 | elif compromised == 'Privileged': 271 | value = [3.] 272 | else: 273 | raise ValueError('Table had invalid Access Level') 274 | proto_vector.extend(value) 275 | 276 | success_type = success.value 277 | if success_type == 1: # true 278 | value = [0.] 279 | elif success_type == 2: 280 | value = [1.] 281 | elif success_type == 3: 282 | value = [2.] 283 | else: 284 | raise ValueError('Table had invalid Access Level-- success value unseen') 285 | proto_vector.extend(value) 286 | 287 | return np.array(proto_vector) 288 | 289 | 290 | def _create_vector(self, success): 291 | table = self._create_blue_table(success)._rows 292 | 293 | proto_vector = [] 294 | for row in table: 295 | # Activity 296 | activity = row[4] 297 | if activity == 'None': 298 | value = [0,0] 299 | elif activity == 'Scan': 300 | value = [1,0] 301 | elif activity == 'Exploit': 302 | value = [1,1] 303 | else: 304 | raise ValueError('Table had invalid Access Level') 305 | proto_vector.extend(value) 306 | 307 | # Compromised 308 | compromised = row[5] 309 | if compromised == 'No': 310 | value = [0, 0] 311 | elif compromised == 'Unknown': 312 | value = [1, 0] 313 | elif compromised == 'User': 314 | value = [0,1] 315 | elif compromised == 'Privileged': 316 | value = [1,1] 317 | else: 318 | raise ValueError('Table had invalid Access Level') 319 | proto_vector.extend(value) 320 | 321 | success_type = success.value 322 | if success_type == 1: 323 | value = [0, 0] 324 | elif success_type == 2: 325 | value = [0, 1] 326 | elif success_type == 3: 327 | value = [1, 1] 328 | else: 329 | raise ValueError('Table had invalid Access Level-- success value unseen') 330 | proto_vector.extend(value) 331 | 332 | return np.array(proto_vector) 333 | 334 | def get_attr(self,attribute:str): 335 | return self.env.get_attr(attribute) 336 | 337 | def get_observation(self, agent: str): 338 | if agent == 'Blue' and self.output_mode == 'table': 339 | output = self.get_table() 340 | else: 341 | output = self.get_attr('get_observation')(agent) 342 | 343 | return output 344 | 345 | def get_agent_state(self,agent:str): 346 | return self.get_attr('get_agent_state')(agent) 347 | 348 | def get_action_space(self,agent): 349 | return self.env.get_action_space(agent) 350 | 351 | def get_last_action(self,agent): 352 | return self.get_attr('get_last_action')(agent) 353 | 354 | def get_ip_map(self): 355 | return self.get_attr('get_ip_map')() 356 | 357 | def get_rewards(self): 358 | return self.get_attr('get_rewards')() 359 | -------------------------------------------------------------------------------- /agents/baseline_sub_agents/curiosity.py: -------------------------------------------------------------------------------- 1 | from gym.spaces import Discrete, MultiDiscrete, Space 2 | import numpy as np 3 | from typing import Optional, Tuple, Union 4 | 5 | from ray.rllib.models.action_dist import ActionDistribution 6 | from ray.rllib.models.catalog import ModelCatalog 7 | from ray.rllib.models.modelv2 import ModelV2 8 | from ray.rllib.models.tf.tf_action_dist import Categorical, MultiCategorical 9 | from ray.rllib.models.torch.misc import SlimFC 10 | from ray.rllib.models.torch.torch_action_dist import ( 11 | TorchCategorical, 12 | TorchMultiCategorical, 13 | ) 14 | from ray.rllib.models.utils import get_activation_fn 15 | from ray.rllib.policy.sample_batch import SampleBatch 16 | from ray.rllib.utils import NullContextManager 17 | from ray.rllib.utils.annotations import override 18 | from ray.rllib.utils.exploration.exploration import Exploration 19 | from ray.rllib.utils.framework import try_import_tf, try_import_torch 20 | from ray.rllib.utils.from_config import from_config 21 | from ray.rllib.utils.tf_utils import get_placeholder, one_hot as tf_one_hot 22 | from ray.rllib.utils.torch_utils import one_hot 23 | from ray.rllib.utils.typing import FromConfigSpec, ModelConfigDict, TensorType 24 | 25 | tf1, tf, tfv = try_import_tf() 26 | torch, nn = try_import_torch() 27 | F = None 28 | if nn is not None: 29 | F = nn.functional 30 | 31 | 32 | class Curiosity(Exploration): 33 | """Implementation of: 34 | [1] Curiosity-driven Exploration by Self-supervised Prediction 35 | Pathak, Agrawal, Efros, and Darrell - UC Berkeley - ICML 2017. 36 | https://arxiv.org/pdf/1705.05363.pdf 37 | 38 | Learns a simplified model of the environment based on three networks: 39 | 1) Embedding observations into latent space ("feature" network). 40 | 2) Predicting the action, given two consecutive embedded observations 41 | ("inverse" network). 42 | 3) Predicting the next embedded obs, given an obs and action 43 | ("forward" network). 44 | 45 | The less the agent is able to predict the actually observed next feature 46 | vector, given obs and action (through the forwards network), the larger the 47 | "intrinsic reward", which will be added to the extrinsic reward. 48 | Therefore, if a state transition was unexpected, the agent becomes 49 | "curious" and will further explore this transition leading to better 50 | exploration in sparse rewards environments. 51 | """ 52 | 53 | def __init__( 54 | self, 55 | action_space: Space, 56 | *, 57 | framework: str, 58 | model: ModelV2, 59 | feature_dim: int = 288, 60 | feature_net_config: Optional[ModelConfigDict] = None, 61 | inverse_net_hiddens: Tuple[int] = (256,), 62 | inverse_net_activation: str = "relu", 63 | forward_net_hiddens: Tuple[int] = (256,), 64 | forward_net_activation: str = "relu", 65 | beta: float = 0.2, 66 | eta: float = 1.0, 67 | lr: float = 1e-3, 68 | sub_exploration: Optional[FromConfigSpec] = None, 69 | **kwargs 70 | ): 71 | """Initializes a Curiosity object. 72 | 73 | Uses as defaults the hyperparameters described in [1]. 74 | 75 | Args: 76 | feature_dim: The dimensionality of the feature (phi) 77 | vectors. 78 | feature_net_config: Optional model 79 | configuration for the feature network, producing feature 80 | vectors (phi) from observations. This can be used to configure 81 | fcnet- or conv_net setups to properly process any observation 82 | space. 83 | inverse_net_hiddens: Tuple of the layer sizes of the 84 | inverse (action predicting) NN head (on top of the feature 85 | outputs for phi and phi'). 86 | inverse_net_activation: Activation specifier for the inverse 87 | net. 88 | forward_net_hiddens: Tuple of the layer sizes of the 89 | forward (phi' predicting) NN head. 90 | forward_net_activation: Activation specifier for the forward 91 | net. 92 | beta: Weight for the forward loss (over the inverse loss, 93 | which gets weight=1.0-beta) in the common loss term. 94 | eta: Weight for intrinsic rewards before being added to 95 | extrinsic ones. 96 | lr: The learning rate for the curiosity-specific 97 | optimizer, optimizing feature-, inverse-, and forward nets. 98 | sub_exploration: The config dict for 99 | the underlying Exploration to use (e.g. epsilon-greedy for 100 | DQN). If None, uses the FromSpecDict provided in the Policy's 101 | default config. 102 | """ 103 | 104 | # change action_space from Tuple to MultiDiscret 105 | if not isinstance(action_space, (Discrete, MultiDiscrete)): 106 | action_space = MultiDiscrete([action_space[0].n, action_space[1].n]) 107 | self.action_space = action_space 108 | 109 | if not isinstance(action_space, (Discrete, MultiDiscrete)): 110 | raise ValueError( 111 | "Only (Multi)Discrete action spaces supported for Curiosity so far!" 112 | ) 113 | 114 | super().__init__(action_space, model=model, framework=framework, **kwargs) 115 | 116 | if self.policy_config["num_workers"] != 0: 117 | raise ValueError( 118 | "Curiosity exploration currently does not support parallelism." 119 | " `num_workers` must be 0!" 120 | ) 121 | 122 | self.feature_dim = feature_dim 123 | if feature_net_config is None: 124 | feature_net_config = self.policy_config["model"].copy() 125 | self.feature_net_config = feature_net_config 126 | self.inverse_net_hiddens = inverse_net_hiddens 127 | self.inverse_net_activation = inverse_net_activation 128 | self.forward_net_hiddens = forward_net_hiddens 129 | self.forward_net_activation = forward_net_activation 130 | 131 | 132 | self.action_dim = ( 133 | self.action_space.n 134 | if isinstance(self.action_space, Discrete) 135 | else np.sum(self.action_space.nvec) 136 | ) 137 | 138 | self.beta = beta 139 | self.eta = eta 140 | self.lr = lr 141 | # TODO: (sven) if sub_exploration is None, use Trainer's default 142 | # Exploration config. 143 | if sub_exploration is None: 144 | raise NotImplementedError 145 | self.sub_exploration = sub_exploration 146 | 147 | # Creates modules/layers inside the actual ModelV2. 148 | self._curiosity_feature_net = ModelCatalog.get_model_v2( 149 | self.model.obs_space, 150 | self.action_space, 151 | self.feature_dim, 152 | model_config=self.feature_net_config, 153 | framework=self.framework, 154 | name="feature_net", 155 | ).to(self.device) 156 | 157 | self._curiosity_inverse_fcnet = self._create_fc_net( 158 | [2 * self.feature_dim] + list(self.inverse_net_hiddens) + [self.action_dim], 159 | self.inverse_net_activation, 160 | name="inverse_net", 161 | ).to(self.device) 162 | 163 | self._curiosity_forward_fcnet = self._create_fc_net( 164 | [self.feature_dim + self.action_dim] 165 | + list(self.forward_net_hiddens) 166 | + [self.feature_dim], 167 | self.forward_net_activation, 168 | name="forward_net", 169 | ).to(self.device) 170 | 171 | # This is only used to select the correct action 172 | self.exploration_submodule = from_config( 173 | cls=Exploration, 174 | config=self.sub_exploration, 175 | action_space=self.action_space, 176 | framework=self.framework, 177 | policy_config=self.policy_config, 178 | model=self.model, 179 | num_workers=self.num_workers, 180 | worker_index=self.worker_index, 181 | ) 182 | 183 | @override(Exploration) 184 | def get_exploration_action( 185 | self, 186 | *, 187 | action_distribution: ActionDistribution, 188 | timestep: Union[int, TensorType], 189 | explore: bool = True 190 | ): 191 | # Simply delegate to sub-Exploration module. 192 | return self.exploration_submodule.get_exploration_action( 193 | action_distribution=action_distribution, timestep=timestep, explore=explore 194 | ) 195 | 196 | @override(Exploration) 197 | def get_exploration_optimizer(self, optimizers): 198 | # Create, but don't add Adam for curiosity NN updating to the policy. 199 | # If we added and returned it here, it would be used in the policy's 200 | # update loop, which we don't want (curiosity updating happens inside 201 | # `postprocess_trajectory`). 202 | if self.framework == "torch": 203 | feature_params = list(self._curiosity_feature_net.parameters()) 204 | inverse_params = list(self._curiosity_inverse_fcnet.parameters()) 205 | forward_params = list(self._curiosity_forward_fcnet.parameters()) 206 | 207 | # Now that the Policy's own optimizer(s) have been created (from 208 | # the Model parameters (IMPORTANT: w/o(!) the curiosity params), 209 | # we can add our curiosity sub-modules to the Policy's Model. 210 | self.model._curiosity_feature_net = self._curiosity_feature_net.to( 211 | self.device 212 | ) 213 | self.model._curiosity_inverse_fcnet = self._curiosity_inverse_fcnet.to( 214 | self.device 215 | ) 216 | self.model._curiosity_forward_fcnet = self._curiosity_forward_fcnet.to( 217 | self.device 218 | ) 219 | self._optimizer = torch.optim.Adam( 220 | forward_params + inverse_params + feature_params, lr=self.lr 221 | ) 222 | else: 223 | self.model._curiosity_feature_net = self._curiosity_feature_net 224 | self.model._curiosity_inverse_fcnet = self._curiosity_inverse_fcnet 225 | self.model._curiosity_forward_fcnet = self._curiosity_forward_fcnet 226 | # Feature net is a RLlib ModelV2, the other 2 are keras Models. 227 | self._optimizer_var_list = ( 228 | self._curiosity_feature_net.base_model.variables 229 | + self._curiosity_inverse_fcnet.variables 230 | + self._curiosity_forward_fcnet.variables 231 | ) 232 | self._optimizer = tf1.train.AdamOptimizer(learning_rate=self.lr) 233 | # Create placeholders and initialize the loss. 234 | if self.framework == "tf": 235 | self._obs_ph = get_placeholder( 236 | space=self.model.obs_space, name="_curiosity_obs" 237 | ) 238 | self._next_obs_ph = get_placeholder( 239 | space=self.model.obs_space, name="_curiosity_next_obs" 240 | ) 241 | self._action_ph = get_placeholder( 242 | space=self.model.action_space, name="_curiosity_action" 243 | ) 244 | ( 245 | self._forward_l2_norm_sqared, 246 | self._update_op, 247 | ) = self._postprocess_helper_tf( 248 | self._obs_ph, self._next_obs_ph, self._action_ph 249 | ) 250 | 251 | return optimizers 252 | 253 | @override(Exploration) 254 | def postprocess_trajectory(self, policy, sample_batch, tf_sess=None): 255 | """Calculates phi values (obs, obs', and predicted obs') and ri. 256 | 257 | Also calculates forward and inverse losses and updates the curiosity 258 | module on the provided batch using our optimizer. 259 | """ 260 | if self.framework != "torch": 261 | self._postprocess_tf(policy, sample_batch, tf_sess) 262 | else: 263 | self._postprocess_torch(policy, sample_batch) 264 | 265 | def _postprocess_tf(self, policy, sample_batch, tf_sess): 266 | # tf1 static-graph: Perform session call on our loss and update ops. 267 | if self.framework == "tf": 268 | forward_l2_norm_sqared, _ = tf_sess.run( 269 | [self._forward_l2_norm_sqared, self._update_op], 270 | feed_dict={ 271 | self._obs_ph: sample_batch[SampleBatch.OBS], 272 | self._next_obs_ph: sample_batch[SampleBatch.NEXT_OBS], 273 | self._action_ph: sample_batch[SampleBatch.ACTIONS], 274 | }, 275 | ) 276 | # tf-eager: Perform model calls, loss calculations, and optimizer 277 | # stepping on the fly. 278 | else: 279 | forward_l2_norm_sqared, _ = self._postprocess_helper_tf( 280 | sample_batch[SampleBatch.OBS], 281 | sample_batch[SampleBatch.NEXT_OBS], 282 | sample_batch[SampleBatch.ACTIONS], 283 | ) 284 | # Scale intrinsic reward by eta hyper-parameter. 285 | sample_batch[SampleBatch.REWARDS] = ( 286 | sample_batch[SampleBatch.REWARDS] + self.eta * forward_l2_norm_sqared 287 | ) 288 | 289 | return sample_batch 290 | 291 | def _postprocess_helper_tf(self, obs, next_obs, actions): 292 | with ( 293 | tf.GradientTape() if self.framework != "tf" else NullContextManager() 294 | ) as tape: 295 | # Push both observations through feature net to get both phis. 296 | phis, _ = self.model._curiosity_feature_net( 297 | {SampleBatch.OBS: tf.concat([obs, next_obs], axis=0)} 298 | ) 299 | phi, next_phi = tf.split(phis, 2) 300 | 301 | # Predict next phi with forward model. 302 | predicted_next_phi = self.model._curiosity_forward_fcnet( 303 | tf.concat([phi, tf_one_hot(actions, self.action_space)], axis=-1) 304 | ) 305 | 306 | # Forward loss term (predicted phi', given phi and action vs 307 | # actually observed phi'). 308 | forward_l2_norm_sqared = 0.5 * tf.reduce_sum( 309 | tf.square(predicted_next_phi - next_phi), axis=-1 310 | ) 311 | forward_loss = tf.reduce_mean(forward_l2_norm_sqared) 312 | 313 | # Inverse loss term (prediced action that led from phi to phi' vs 314 | # actual action taken). 315 | phi_cat_next_phi = tf.concat([phi, next_phi], axis=-1) 316 | dist_inputs = self.model._curiosity_inverse_fcnet(phi_cat_next_phi) 317 | action_dist = ( 318 | Categorical(dist_inputs, self.model) 319 | if isinstance(self.action_space, Discrete) 320 | else MultiCategorical(dist_inputs, self.model, self.action_space.nvec) 321 | ) 322 | # Neg log(p); p=probability of observed action given the inverse-NN 323 | # predicted action distribution. 324 | inverse_loss = -action_dist.logp(tf.convert_to_tensor(actions)) 325 | inverse_loss = tf.reduce_mean(inverse_loss) 326 | 327 | # Calculate the ICM loss. 328 | loss = (1.0 - self.beta) * inverse_loss + self.beta * forward_loss 329 | 330 | # Step the optimizer. 331 | if self.framework != "tf": 332 | grads = tape.gradient(loss, self._optimizer_var_list) 333 | grads_and_vars = [ 334 | (g, v) for g, v in zip(grads, self._optimizer_var_list) if g is not None 335 | ] 336 | update_op = self._optimizer.apply_gradients(grads_and_vars) 337 | else: 338 | update_op = self._optimizer.minimize( 339 | loss, var_list=self._optimizer_var_list 340 | ) 341 | 342 | # Return the squared l2 norm and the optimizer update op. 343 | return forward_l2_norm_sqared, update_op 344 | 345 | def _postprocess_torch(self, policy, sample_batch): 346 | # Push both observations through feature net to get both phis. 347 | 348 | phis, _ = self.model._curiosity_feature_net( 349 | { 350 | SampleBatch.OBS: torch.cat( 351 | [ 352 | torch.from_numpy(sample_batch[SampleBatch.OBS]).to(policy.device), 353 | torch.from_numpy(sample_batch[SampleBatch.NEXT_OBS]).to(policy.device), 354 | ] 355 | ) 356 | } 357 | ) 358 | phi, next_phi = torch.chunk(phis, 2) 359 | actions_tensor = ( 360 | torch.from_numpy(sample_batch[SampleBatch.ACTIONS]).long().to(policy.device) 361 | ) 362 | 363 | # Predict next phi with forward model. 364 | predicted_next_phi = self.model._curiosity_forward_fcnet( 365 | torch.cat([phi, one_hot(actions_tensor, self.action_space).float()], dim=-1) 366 | ).to(self.device) 367 | 368 | # Forward loss term (predicted phi', given phi and action vs actually 369 | # observed phi'). 370 | forward_l2_norm_sqared = 0.5 * torch.sum( 371 | torch.pow(predicted_next_phi - next_phi, 2.0), dim=-1 372 | ) 373 | forward_loss = torch.mean(forward_l2_norm_sqared) 374 | 375 | # Scale intrinsic reward by eta hyper-parameter. 376 | sample_batch[SampleBatch.REWARDS] = ( 377 | sample_batch[SampleBatch.REWARDS] 378 | + self.eta * forward_l2_norm_sqared.detach().cpu().numpy() 379 | ) 380 | 381 | # Inverse loss term (prediced action that led from phi to phi' vs 382 | # actual action taken). 383 | phi_cat_next_phi = torch.cat([phi, next_phi], dim=-1) 384 | dist_inputs = self.model._curiosity_inverse_fcnet(phi_cat_next_phi).to(self.device) 385 | action_dist = ( 386 | TorchCategorical(dist_inputs, self.model) 387 | if isinstance(self.action_space, Discrete) 388 | else TorchMultiCategorical(dist_inputs, self.model, self.action_space.nvec) 389 | ) 390 | # Neg log(p); p=probability of observed action given the inverse-NN 391 | # predicted action distribution. 392 | inverse_loss = -action_dist.logp(actions_tensor) 393 | inverse_loss = torch.mean(inverse_loss) 394 | 395 | # Calculate the ICM loss. 396 | loss = (1.0 - self.beta) * inverse_loss + self.beta * forward_loss 397 | # Perform an optimizer step. 398 | self._optimizer.zero_grad() 399 | loss.backward() 400 | self._optimizer.step() 401 | 402 | # Return the postprocessed sample batch (with the corrected rewards). 403 | return sample_batch 404 | 405 | def _create_fc_net(self, layer_dims, activation, name=None): 406 | """Given a list of layer dimensions (incl. input-dim), creates FC-net. 407 | 408 | Args: 409 | layer_dims (Tuple[int]): Tuple of layer dims, including the input 410 | dimension. 411 | activation (str): An activation specifier string (e.g. "relu"). 412 | 413 | Examples: 414 | If layer_dims is [4,8,6] we'll have a two layer net: 4->8 (8 nodes) 415 | and 8->6 (6 nodes), where the second layer (6 nodes) does not have 416 | an activation anymore. 4 is the input dimension. 417 | """ 418 | layers = ( 419 | [tf.keras.layers.Input(shape=(layer_dims[0],), name="{}_in".format(name))] 420 | if self.framework != "torch" 421 | else [] 422 | ) 423 | 424 | for i in range(len(layer_dims) - 1): 425 | act = activation if i < len(layer_dims) - 2 else None 426 | if self.framework == "torch": 427 | layers.append( 428 | SlimFC( 429 | in_size=layer_dims[i], 430 | out_size=layer_dims[i + 1], 431 | initializer=torch.nn.init.xavier_uniform_, 432 | activation_fn=act, 433 | ) 434 | ) 435 | else: 436 | layers.append( 437 | tf.keras.layers.Dense( 438 | units=layer_dims[i + 1], 439 | activation=get_activation_fn(act), 440 | name="{}_{}".format(name, i), 441 | ) 442 | ) 443 | 444 | if self.framework == "torch": 445 | return nn.Sequential(*layers) 446 | else: 447 | return tf.keras.Sequential(layers) -------------------------------------------------------------------------------- /agents/baseline_sub_agents/newFlatWrapper.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | 3 | from CybORG.Agents.Wrappers.BaseWrapper import BaseWrapper 4 | from CybORG.Shared import Observation 5 | from CybORG.Shared.Actions import ShellSleep 6 | from CybORG.Shared.Enums import OperatingSystemType, SessionType, ProcessName, Path, ProcessType, ProcessVersion, \ 7 | AppProtocol, FileType, ProcessState, Vulnerability, Vendor, PasswordHashType, BuiltInGroups, \ 8 | OperatingSystemDistribution, OperatingSystemVersion, OperatingSystemKernelVersion, Architecture, \ 9 | OperatingSystemPatch, FileVersion 10 | 11 | import inspect, random 12 | from pprint import pprint 13 | import pickle 14 | import os 15 | import numpy as np 16 | import copy 17 | import sys 18 | from csv import writer 19 | 20 | class NewFixedFlatWrapper(BaseWrapper): 21 | def __init__(self, env: BaseWrapper=None, agent=None, log_name='flat.txt'): 22 | super().__init__(env, agent) 23 | self.MAX_HOSTS = 5 # changed 24 | self.MAX_PROCESSES = 11 #100 25 | self.MAX_CONNECTIONS = 1 26 | self.MAX_VULNERABILITIES = 1 27 | self.MAX_INTERFACES = 1 # 4 28 | self.MAX_FILES = 2 # 10 29 | self.MAX_SESSIONS = 0 # 2 30 | self.MAX_USERS = 0 # 5 31 | self.MAX_GROUPS = 0 #1 32 | self.MAX_PATCHES = 0 # 10 33 | self.hostname = {'Defender': 1, 'Enterprise0': 2, 'Enterprise1': 3, 'Enterprise2': 4, \ 34 | 'Op_Host0': 5, 'Op_Host1': 6, 'Op_Host2': 7, 'Op_Server0': 8, 'User0': 9, 'User1': 10, \ 35 | 'User2': 11, 'User3': 12, 'User4': 13} 36 | self.username = {} 37 | self.group_name = {} 38 | self.process_name = {} 39 | self.interface_name = {} 40 | self.path = {} 41 | self.password = {} 42 | self.password_hash = {} 43 | self.file = {} 44 | # added 45 | self.session_pid = {} 46 | self.process_pid = {} 47 | self.process_ppid = {} 48 | # self.subnet_netadd = {} 49 | # self.subnet_prefix = {} 50 | self.ipadd = {} 51 | self.localadd = {} 52 | self.remoteadd = {} 53 | self.remote_port = {} 54 | self.local_port = {} 55 | 56 | self.count = 0 57 | self.log_name = log_name 58 | 59 | def get_action(self, observation, action_space): 60 | 61 | action = self.agent.get_action(self.observation_change(observation), self.action_space_change(action_space)) 62 | 63 | action_class = action['action'] 64 | params = {} 65 | for p in inspect.signature(action_class).parameters: 66 | if p in action: 67 | params[p] = action[p] 68 | else: 69 | action_class = ShellSleep 70 | params = {} 71 | break 72 | action = action_class(**params) 73 | return action 74 | 75 | # def action_space_change(self, action_space: dict) -> dict: 76 | # action_space.pop('process') 77 | # action_space['session'] = {0: True} 78 | # action_space['username'] = {'pi': action_space['username']['pi'], 79 | # 'vagrant': action_space['username']['vagrant']} 80 | # action_space['password'] = {'raspberry': action_space['password']['raspberry'], 81 | # 'vagrant': action_space['password']['vagrant']} 82 | # action_space['port'] = {22: action_space['port'][22]} 83 | # return action_space 84 | 85 | def observation_change(self, obs: dict) -> list: 86 | numeric_obs = copy.deepcopy(obs) 87 | flat_obs = [] 88 | hostid = str(random.randint(0, self.MAX_HOSTS+1)) 89 | 90 | orig_stdout = sys.stdout 91 | f = open('obs_dict.txt', 'a') 92 | sys.stdout = f 93 | self.count +=1 94 | if len(obs.keys()) != 14 and self.count%2==0: 95 | print(f'{self.count/2}-----------') 96 | pprint(obs) 97 | sys.stdout = orig_stdout 98 | f.close() 99 | 100 | 101 | while len(numeric_obs) < self.MAX_HOSTS: 102 | hostid = str(random.randint(0, self.MAX_HOSTS+1)) 103 | if hostid not in numeric_obs.keys(): 104 | numeric_obs[hostid] = {} 105 | 106 | if len(numeric_obs) > self.MAX_HOSTS: 107 | with open(self.log_name, 'a') as file: 108 | file.write(f"MAX_HOSTS: {len(numeric_obs)}\n") 109 | while len(numeric_obs) > self.MAX_HOSTS: 110 | numeric_obs.popitem() 111 | # print('numeric_obs', numeric_obs.keys()) 112 | # raise ValueError("Too many hosts in observation for fixed size") 113 | 114 | # print('numeric_obs.items()', numeric_obs.keys()) 115 | for key_name, host in numeric_obs.items(): 116 | # print('====== ', key_name) 117 | if key_name == 'success': 118 | # flat_obs.append(float(host.value)/3) # @@@ ??? not sure why divided by 3 == maybe there are only three types of success status 119 | flat_obs += list(np.eye(4)[int(host.value)]) # make it as one hot 120 | # flat_obs += list(np.eye(4)[int(host)]) 121 | elif not isinstance(host, dict): 122 | raise ValueError('Host data must be a dict') 123 | else: 124 | if key_name in self.hostname: 125 | # self.hostname[key_name] = len(self.hostname) # @@@ "user 3": 0 (value is the order ofh) 126 | # print(self.hostname.keys()) 127 | flat_obs += list(np.eye(13)[self.hostname[key_name]-1]) # one-hot representation 128 | else: 129 | # with open(self.log_name, 'a') as file: 130 | # file.write("@@@@ no hostname info??") 131 | # flat_obs.append(-1.0) 132 | flat_obs += 13*[-1.0] 133 | print('hostname', len(flat_obs)-1) 134 | # if 'System info' in host: 135 | # if "Hostname" in host["System info"]: 136 | # element = host["System info"]["Hostname"] 137 | # if element not in self.hostname: 138 | # self.hostname[element] = len(self.hostname) # @@@ "user 3": 0 (value is the order ofh) 139 | # # element = self.hostname[element]/self.MAX_HOSTS # normalise it 140 | # # flat_obs.append(float(element)) 141 | # flat_obs += list(np.eye(14)[self.hostname[element]]) # one-hot representation 142 | # else: 143 | # with open(self.log_name, 'a') as file: 144 | # file.write("@@@@ no hostname info??") 145 | # # flat_obs.append(-1.0) 146 | # flat_obs += 14*[-1.0] 147 | 148 | # if "OSType" in host["System info"]: 149 | # if host["System info"]["OSType"] != -1: 150 | # element = host["System info"]["OSType"].value/len(OperatingSystemType.__members__) 151 | # else: 152 | # element = -1 153 | 154 | # flat_obs.append(float(element)) 155 | # else: 156 | # flat_obs.append(-1.0) 157 | 158 | # if "OSDistribution" in host["System info"]: 159 | # if host["System info"]["OSDistribution"] != -1: 160 | # element = host["System info"]["OSDistribution"].value / len(OperatingSystemDistribution.__members__) 161 | # else: 162 | # element = -1 163 | 164 | # flat_obs.append(float(element)) 165 | # else: 166 | # flat_obs.append(-1.0) 167 | 168 | # if "OSVersion" in host["System info"]: 169 | # if host["System info"]["OSVersion"] != -1: 170 | # element = host["System info"]["OSVersion"].value / len(OperatingSystemVersion.__members__) 171 | # else: 172 | # element = -1 173 | 174 | # flat_obs.append(float(element)) 175 | # else: 176 | # flat_obs.append(-1.0) 177 | 178 | # # if "OSKernelVersion" in host["System info"]: 179 | # # if host["System info"]["OSKernelVersion"] != -1: 180 | # # element = host["System info"]["OSKernelVersion"].value / len(OperatingSystemKernelVersion.__members__) 181 | # # else: 182 | # # element = -1 183 | 184 | # # flat_obs.append(float(element)) 185 | # # else: 186 | # # flat_obs.append(-1.0) 187 | 188 | # if "Architecture" in host["System info"]: 189 | # if host["System info"]["Architecture"] != -1: 190 | # element = host["System info"]["Architecture"].value / len(Architecture.__members__) 191 | # else: 192 | # element = -1 193 | 194 | # flat_obs.append(float(element)) 195 | # else: 196 | # flat_obs.append(-1.0) 197 | 198 | # # if 'Local Time' in host["System info"]: 199 | # # element = (host["System info"]['Local Time'] - datetime(2020, 1, 1)).total_seconds() 200 | 201 | # # flat_obs.append(float(element)) 202 | # # else: 203 | # # flat_obs.append(-1.0) 204 | 205 | # if "os_patches" not in host["System info"]: 206 | # host["System info"]["os_patches"] = [] 207 | 208 | # while len(host["System info"]["os_patches"]) < self.MAX_PATCHES: 209 | # host["System info"]["os_patches"].append(-1.0) 210 | # if len(host["System info"]["os_patches"]) > self.MAX_PATCHES: 211 | # with open(self.log_name, 'a') as file: 212 | # file.write(f'MAX_PATCHES, {len(host["System info"]["os_patches"])}\n') 213 | # # raise ValueError("Too many processes in observation for fixed size") 214 | # for patch_idx, patch in enumerate(host["System info"]["os_patches"]): 215 | # if patch != -1: 216 | # element = patch.value / len(OperatingSystemPatch.__members__) 217 | # else: 218 | # element = patch 219 | 220 | # flat_obs.append(float(element)) 221 | # else: 222 | # flat_obs += [-1.0]*4 223 | # for num_patches in range(self.MAX_PATCHES): 224 | # flat_obs.append(-1.0) 225 | 226 | if 'Processes' not in host: 227 | host["Processes"] = [] 228 | while len(host["Processes"]) < self.MAX_PROCESSES: 229 | host["Processes"].append({}) 230 | 231 | if len(host["Processes"]) > self.MAX_PROCESSES: 232 | with open(self.log_name, 'a') as file: 233 | file.write(f'MAX_PROCESSES, {len(host["Processes"])}\n') 234 | while len(host["Processes"]) > self.MAX_PROCESSES: 235 | host["Processes"].pop() 236 | # raise ValueError("Too many processes in observation for fixed size") 237 | 238 | for proc_idx, process in enumerate(host['Processes']): 239 | if "PID" in process: 240 | # element = process["PID"] 241 | # if element not in self.process_pid: 242 | # self.process_pid[element] = len(self.process_pid) 243 | # with open(self.log_name, 'a') as file: 244 | # file.write(f"process_pid, {len(self.process_pid)}\n") 245 | # element = self.process_pid[element] 246 | # flat_obs.append(float(element)) 247 | # flat_obs.append(float(process["PID"])/32768) 248 | flat_obs.append(float(process["PID"])/1000) 249 | else: 250 | flat_obs.append(-1.0) 251 | print('pid', len(flat_obs)-1) 252 | 253 | if "PPID" in process: 254 | element = process["PPID"] 255 | if element not in self.process_ppid: 256 | self.process_ppid[element] = len(self.process_ppid) 257 | # with open(self.log_name, 'a') as file: 258 | # file.write(f"process_ppid, {len(self.process_ppid)}\n") 259 | element = self.process_ppid[element] 260 | flat_obs.append(float(element)) 261 | # flat_obs.append(float(process["PPID"])/32768) 262 | else: 263 | flat_obs.append(-1.0) 264 | print('ppid', len(flat_obs)-1) 265 | # if "Process Name" in process: 266 | # element = process["Process Name"] 267 | # with open(self.log_name, 'a') as file: 268 | # file.write(f"Process Name, {element}\n") 269 | # if element not in self.process_name: 270 | # self.process_name[element] = len(self.process_name) 271 | # element = self.process_name[element] 272 | # flat_obs.append(float(element)) 273 | # else: 274 | # flat_obs.append(-1.0) 275 | 276 | if "Username" in process: 277 | element = process["Username"] 278 | with open(self.log_name, 'a') as file: 279 | file.write(f"Username, {element}\n") 280 | if element not in self.username: 281 | self.username[element] = len(self.username) 282 | element = self.username[element] 283 | flat_obs.append(float(element)) 284 | else: 285 | flat_obs.append(-1.0) 286 | print('Username', len(flat_obs)-1) 287 | # if "Path" in process: 288 | # element = process["Path"] 289 | # with open(self.log_name, 'a') as file: 290 | # file.write(f"Path, {element}\n") 291 | # if element not in self.path: 292 | # self.path[element] = len(self.path) 293 | # element = self.path[element] 294 | # flat_obs.append(float(element)) 295 | # else: 296 | # flat_obs.append(-1.0) 297 | 298 | # if "Known Process" in process: 299 | # if process["Known Process"] != -1: 300 | # element = process["Known Process"].value / len(ProcessName.__members__) 301 | # else: 302 | # element = -1.0 303 | # with open(self.log_name, 'a') as file: 304 | # file.write(f"Known Process, {element}\n") 305 | # flat_obs.append(float(element)) 306 | # else: 307 | # flat_obs.append(-1.0) 308 | 309 | # if "Known Path" in process: 310 | # if process["Known Path"] != -1: 311 | # element = process["Known Path"].value / len(Path.__members__) 312 | # else: 313 | # element = -1.0 314 | # with open(self.log_name, 'a') as file: 315 | # file.write(f"Known Path, {element}\n") 316 | # flat_obs.append(float(element)) 317 | # else: 318 | # flat_obs.append(-1.0) 319 | 320 | # if "Process Type" in process: 321 | # if process["Process Type"] != -1: 322 | # element = process["Process Type"].value / len(ProcessType.__members__) 323 | # else: 324 | # element = -1.0 325 | # with open(self.log_name, 'a') as file: 326 | # file.write(f"Process Type, {element}\n") 327 | # flat_obs.append(float(element)) 328 | # else: 329 | # flat_obs.append(-1.0) 330 | 331 | # if "Process Version" in process: 332 | # if process["Process Version"] != -1: 333 | # element = process["Process Version"].value / len(ProcessVersion.__members__) 334 | # else: 335 | # element = -1.0 336 | # with open(self.log_name, 'a') as file: 337 | # file.write(f"Process Version, {element}\n") 338 | # flat_obs.append(float(element)) 339 | # else: 340 | # flat_obs.append(-1.0) 341 | 342 | if "Connections" not in process: 343 | process["Connections"] = [] 344 | while len(process["Connections"]) < self.MAX_CONNECTIONS: 345 | process["Connections"].append({}) 346 | if len(process["Connections"]) > self.MAX_CONNECTIONS: 347 | with open(self.log_name, 'a') as file: 348 | file.write(f'MAX_CONNECTIONS: {len(process["Connections"])}\n') 349 | 350 | for conn_idx, connection in enumerate(process["Connections"]): 351 | if "local_port" in connection: 352 | element = int(connection["local_port"]) 353 | if element not in self.local_port: 354 | self.local_port[element] = len(self.local_port) 355 | # with open(self.log_name, 'a') as file: 356 | # file.write(f"local_port, {len(self.local_port)}\n") 357 | element = self.local_port[element] 358 | flat_obs.append(float(element)) 359 | # flat_obs.append(float(connection["local_port"])/65535) 360 | else: 361 | flat_obs.append(-1.0) 362 | print('local_port', len(flat_obs)-1) 363 | 364 | if "remote_port" in connection: 365 | element = int(connection["remote_port"]) 366 | if element not in self.remote_port: 367 | self.remote_port[element] = len(self.remote_port) 368 | # with open(self.log_name, 'a') as file: 369 | # file.write(f"remote_port, {len(self.remote_port)}\n") 370 | element = self.remote_port[element] 371 | flat_obs.append(float(element)) 372 | # flat_obs.append(float(connection["remote_port"])/65535) 373 | else: 374 | flat_obs.append(-1.0) 375 | print('remote_port', len(flat_obs)-1) 376 | 377 | if "local_address" in connection: 378 | element = int(connection["local_address"]) 379 | if element not in self.localadd: 380 | self.localadd[element] = len(self.localadd) 381 | # with open(self.log_name, 'a') as file: 382 | # file.write(f"localadd, {len(self.localadd)}\n") 383 | element = self.localadd[element] 384 | flat_obs.append(float(element)) 385 | # flat_obs.append(float(element)/4294967296) 386 | else: 387 | flat_obs.append(-1.0) 388 | print('local_address', len(flat_obs)-1) 389 | 390 | if "remote_address" in connection: 391 | element = int(connection["remote_address"]) 392 | if element not in self.remoteadd: 393 | self.remoteadd[element] = len(self.remoteadd) 394 | # with open(self.log_name, 'a') as file: 395 | # file.write(f"remoteadd, {len(self.remoteadd)}\n") 396 | element = self.remoteadd[element] 397 | flat_obs.append(float(element)) 398 | # flat_obs.append(float(element)/4294967296) 399 | else: 400 | flat_obs.append(-1.0) 401 | print('remote_address', len(flat_obs)-1) 402 | 403 | # if "Application Protocol" in connection: 404 | # if connection["Application Protocol"] != -1: 405 | # element = connection["Application Protocol"].value / len(AppProtocol.__members__) 406 | # else: 407 | # element = -1.0 408 | # with open(self.log_name, 'a') as file: 409 | # file.write(f"Application Protocol, {element}\n") 410 | # flat_obs.append(float(element)) 411 | # else: 412 | # flat_obs.append(-1.0) 413 | 414 | # if "Status" in connection: 415 | # if connection["Status"] != -1: 416 | # element = connection["Status"].value / len(ProcessState.__members__) 417 | # else: 418 | # element = -1.0 419 | # with open(self.log_name, 'a') as file: 420 | # file.write(f"Status, {element}\n") 421 | # flat_obs.append(float(element)) 422 | # else: 423 | # flat_obs.append(-1.0) 424 | 425 | if "Vulnerability" in process: 426 | if len(process["Vulnerability"]) > self.MAX_VULNERABILITIES: 427 | with open(self.log_name, 'a') as file: 428 | file.write(f'MAX_VULNERABILITIES: {len(process["Vulnerability"])}\n') 429 | for idx, element in enumerate(process["Vulnerability"]): 430 | if element != -1: 431 | element = element.value / len(Vulnerability.__members__) 432 | with open(self.log_name, 'a') as file: 433 | file.write(f"Vulnerability, {element}\n") 434 | flat_obs.append(float(element)) 435 | else: 436 | for idx in range(self.MAX_VULNERABILITIES): 437 | flat_obs.append(-1.0) 438 | print('vulnerability', len(flat_obs)-1) 439 | 440 | if "Files" not in host: 441 | host["Files"] = [] 442 | while len(host["Files"]) < self.MAX_FILES: 443 | host["Files"].append({}) 444 | 445 | if len(host["Files"]) > self.MAX_FILES: 446 | with open(self.log_name, 'a') as file: 447 | file.write(f'MAX_FILES: {len(host["Files"])}\n') 448 | while len(host["Files"]) > self.MAX_FILES: 449 | host["Files"].pop() 450 | # raise ValueError("Too many files in observation for fixed size") 451 | 452 | 453 | for file_idx, file in enumerate(host['Files']): 454 | if "Path" in file: 455 | element = file["Path"] 456 | with open(self.log_name, 'a') as afile: 457 | afile.write(f"file Path, {element}\n") 458 | if element not in self.path: 459 | self.path[element] = len(self.path) 460 | element = self.path[element] 461 | flat_obs.append(float(element)) 462 | else: 463 | flat_obs.append(-1.0) 464 | print('file Path', len(flat_obs)-1) 465 | 466 | if "Known Path" in file: 467 | if file["Known Path"] != -1: 468 | element = file["Known Path"].value / len(Path.__members__) 469 | else: 470 | element = -1.0 471 | with open(self.log_name, 'a') as afile: 472 | afile.write(f"file Known Path, {element}\n") 473 | flat_obs.append(float(element)) 474 | else: 475 | flat_obs.append(-1.0) 476 | print('file Known Path', len(flat_obs)-1) 477 | 478 | if "File Name" in file: 479 | element = file["File Name"] 480 | with open(self.log_name, 'a') as afile: 481 | afile.write(f"File Name, {element}\n") 482 | if element not in self.file: 483 | self.file[element] = len(self.file) 484 | element = self.file[element] 485 | flat_obs.append(float(element)) 486 | else: 487 | flat_obs.append(-1.0) 488 | print('file Name', len(flat_obs)-1) 489 | 490 | 491 | if "Known File" in file: 492 | if file["Known File"] != -1: 493 | element = file["Known File"].value / len(FileType.__members__) 494 | else: 495 | element = -1.0 496 | with open(self.log_name, 'a') as afile: 497 | afile.write(f"Known File, {element}\n") 498 | flat_obs.append(float(element)) 499 | else: 500 | flat_obs.append(-1.0) 501 | print('Known file', len(flat_obs)-1) 502 | 503 | # if "Type" in file: 504 | # if file["Type"] != -1: 505 | # element = file["Type"].value / len(FileType.__members__) 506 | # else: 507 | # element = -1.0 508 | # with open(self.log_name, 'a') as afile: 509 | # afile.write(f"File Type, {element}\n") 510 | # flat_obs.append(float(element)) 511 | # else: 512 | # flat_obs.append(-1.0) 513 | 514 | # if "Vendor" in file: 515 | # if file["Vendor"] != -1: 516 | # element = file["Vendor"].value / len(Vendor.__members__) 517 | # else: 518 | # element = -1.0 519 | # with open(self.log_name, 'a') as afile: 520 | # afile.write(f"File Vendor, {element}\n") 521 | # flat_obs.append(float(element)) 522 | # else: 523 | # flat_obs.append(-1.0) 524 | 525 | # if "Version" in file: 526 | # if file["Version"] != -1: 527 | # element = file["Version"].value / len(FileVersion.__members__) 528 | # else: 529 | # element = -1.0 530 | # with open(self.log_name, 'a') as afile: 531 | # afile.write(f"File Version, {element}\n") 532 | # flat_obs.append(float(element)) 533 | # else: 534 | # flat_obs.append(-1.0) 535 | 536 | # if "Username" in file: 537 | # element = file["Username"] 538 | # with open(self.log_name, 'a') as afile: 539 | # afile.write(f"File Username, {element}\n") 540 | # if element not in self.username: 541 | # self.username[element] = len(self.username) 542 | # element = self.username[element] 543 | # flat_obs.append(float(element)) 544 | # else: 545 | # flat_obs.append(-1.0) 546 | 547 | # if "Group Name" in file: 548 | # element = file["Group Name"] 549 | # with open(self.log_name, 'a') as afile: 550 | # afile.write(f"File Group Name, {element}\n") 551 | # if element not in self.group_name: 552 | # self.group_name[element] = len(self.group_name) 553 | # element = self.group_name[element] 554 | # flat_obs.append(float(element)) 555 | # else: 556 | # flat_obs.append(-1.0) 557 | 558 | # if "Last Modified Time" in file: 559 | # #TODO work out how to normalise this value 560 | # element = -1 #(file["Last Modified Time"] - datetime(2020, 1, 1)).total_seconds() 561 | 562 | # flat_obs.append(float(element)) 563 | # else: 564 | # flat_obs.append(-1.0) 565 | 566 | # if "User Permissions" in file: 567 | # element = file["User Permissions"] 568 | # with open(self.log_name, 'a') as afile: 569 | # afile.write(f"File User Permissions, {element}\n") 570 | # flat_obs.append(float(element)/7) 571 | # else: 572 | # flat_obs.append(-1.0) 573 | 574 | # if "Group Permissions" in file: 575 | # element = file["Group Permissions"] 576 | # with open(self.log_name, 'a') as afile: 577 | # afile.write(f"File Group Permissions, {element}\n") 578 | # flat_obs.append(float(element)/7) 579 | # else: 580 | # flat_obs.append(-1.0) 581 | 582 | # if "Default Permissions" in file: 583 | # element = file["Default Permissions"] 584 | # with open(self.log_name, 'a') as afile: 585 | # afile.write(f"File Default Permissions, {element}\n") 586 | # flat_obs.append(float(element)/7) 587 | # else: 588 | # flat_obs.append(-1.0) 589 | 590 | if "User Info" not in host: 591 | host["User Info"] = [] 592 | while len(host["User Info"]) < self.MAX_USERS: 593 | host["User Info"].append({}) 594 | 595 | if len(host["User Info"]) > self.MAX_USERS: 596 | with open(self.log_name, 'a') as file: 597 | file.write(f'MAX_USERS: {len(host["User Info"])}\n') 598 | while len(host["User Info"]) > self.MAX_USERS: 599 | host["User Info"].pop() 600 | # raise ValueError("Too many users in observation for fixed size") 601 | 602 | for user_idx, user in enumerate(host['User Info']): 603 | if "Username" in user: 604 | element = user["Username"] 605 | with open(self.log_name, 'a') as file: 606 | file.write(f"User Info Username, {element}\n") 607 | if element not in self.username: 608 | self.username[element] = len(self.username) 609 | element = self.username[element] 610 | flat_obs.append(float(element)) 611 | else: 612 | flat_obs.append(-1.0) 613 | print('user info username', len(flat_obs)-1) 614 | 615 | if "Password" in user: 616 | element = user["Password"] 617 | with open(self.log_name, 'a') as file: 618 | file.write(f"User Info Password, {element}\n") 619 | if element not in self.password: 620 | self.password[element] = len(self.password) 621 | element = self.password[element] 622 | flat_obs.append(float(element)) 623 | else: 624 | flat_obs.append(-1.0) 625 | print('user info Password', len(flat_obs)-1) 626 | # if "Password Hash" in user: 627 | # element = user["Password Hash"] 628 | # with open(self.log_name, 'a') as file: 629 | # file.write(f"User Info Password Hash, {element}\n") 630 | # if element not in self.password_hash: 631 | # self.password_hash[element] = len(self.password_hash) 632 | # element = self.password_hash[element] 633 | # flat_obs.append(float(element)) 634 | # else: 635 | # flat_obs.append(-1.0) 636 | 637 | # if "Password Hash Type" in user: 638 | # if user["Password Hash Type"] != -1: 639 | # element = user["Password Hash Type"].value / len(PasswordHashType.__members__) 640 | # else: 641 | # element = -1.0 642 | # with open(self.log_name, 'a') as file: 643 | # file.write(f"User Info Password Hash Type, {element}\n") 644 | # flat_obs.append(float(element)) 645 | # else: 646 | # flat_obs.append(-1.0) 647 | 648 | # if "UID" in user: 649 | # with open(self.log_name, 'a') as file: 650 | # file.write(f"User Info UID, {user['UID']}\n") 651 | # flat_obs.append(float(user["UID"])) 652 | # else: 653 | # flat_obs.append(-1.0) 654 | 655 | # if "Logged in" in user: 656 | # with open(self.log_name, 'a') as file: 657 | # file.write(f"User Info Logged in, {user['Logged in']}\n") 658 | # flat_obs.append(float(user["Logged in"])) 659 | # else: 660 | # flat_obs.append(-1.0) 661 | 662 | if "Groups" not in user: 663 | user["Groups"] = [] 664 | while len(user["Groups"]) < self.MAX_GROUPS: 665 | user["Groups"].append({}) 666 | 667 | if len(user['Groups']) > self.MAX_GROUPS: 668 | with open(self.log_name, 'a') as file: 669 | file.write(f'MAX_GROUPS: {len(user["Groups"])}\n') 670 | while len(user['Groups']) > self.MAX_GROUPS: 671 | user['Groups'].pop() 672 | # raise ValueError("Too many groups in observation for fixed size") 673 | for group_idx, group in enumerate(user["Groups"]): 674 | # if 'Builtin Group' in group: 675 | # if group["Builtin Group"] != -1: # TODO test if this is ever not true 676 | # element = group["Builtin Group"].value / len(BuiltInGroups.__members__) 677 | # else: 678 | # element = -1.0 679 | # with open(self.log_name, 'a') as file: 680 | # file.write(f"Groups Builtin Group, {element}\n") 681 | # flat_obs.append(float(element)) 682 | # else: 683 | # flat_obs.append(-1.0) 684 | 685 | # if "Group Name" in group: 686 | # element = user["Group Name"] 687 | # with open(self.log_name, 'a') as file: 688 | # file.write(f"Groups Group Name, {element}\n") 689 | # if element not in self.group_name: 690 | # self.group_name[element] = len(self.group_name) 691 | # element = self.group_name[element] 692 | # flat_obs.append(float(element)) 693 | # else: 694 | # flat_obs.append(-1.0) 695 | 696 | if "GID" in group: 697 | with open(self.log_name, 'a') as file: 698 | file.write(f"Groups GID, {group['GID']}\n") 699 | flat_obs.append(float(group["GID"])) 700 | else: 701 | flat_obs.append(-1.0) 702 | 703 | if "Sessions" not in host: 704 | host["Sessions"] = [] 705 | while len(host["Sessions"]) < self.MAX_SESSIONS: 706 | host["Sessions"].append({}) 707 | 708 | if len(host["Sessions"]) > self.MAX_SESSIONS and len(obs.keys())!=14: 709 | with open(self.log_name, 'a') as file: 710 | file.write(f'MAX_SESSIONS: {len(host["Sessions"])}\n') 711 | while len(host["Sessions"]) > self.MAX_SESSIONS: 712 | host["Sessions"].pop() 713 | # raise ValueError("Too many sessions in observation for fixed size") 714 | 715 | for session_idx, session in enumerate(host['Sessions']): 716 | if "Username" in session: 717 | element = session["Username"] 718 | with open(self.log_name, 'a') as file: 719 | file.write(f"Sessions Username, {element}\n") 720 | if element not in self.username: 721 | self.username[element] = len(self.username) 722 | element = self.username[element] 723 | flat_obs.append(float(element)) 724 | else: 725 | flat_obs.append(-1.0) 726 | 727 | if "Type" in session: 728 | if session["Type"] != -1: 729 | element = session["Type"].value/len(SessionType.__members__) 730 | else: 731 | element = -1.0 732 | with open(self.log_name, 'a') as file: 733 | file.write(f"Sessions Type, {element}\n") 734 | flat_obs.append(float(element)) 735 | else: 736 | flat_obs.append(-1.0) 737 | 738 | if "ID" in session: 739 | with open(self.log_name, 'a') as file: 740 | file.write(f"Sessions ID, {session['ID']}\n") 741 | flat_obs.append(float(session["ID"])/20) 742 | else: 743 | flat_obs.append(-1.0) 744 | 745 | if "Timeout" in session: 746 | with open(self.log_name, 'a') as file: 747 | file.write(f"Sessions Timeout, {session['Timeout']}\n") 748 | flat_obs.append(float(session["Timeout"])) 749 | else: 750 | flat_obs.append(-1.0) 751 | 752 | if "PID" in session: 753 | element = session["PID"] 754 | with open(self.log_name, 'a') as file: 755 | file.write(f"Sessions PID, {element}\n") 756 | if element not in self.session_pid: 757 | self.session_pid[element] = len(self.session_pid) 758 | # with open(self.log_name, 'a') as file: 759 | # file.write(f"session_pid, {len(self.session_pid)}\n") 760 | element = self.session_pid[element] 761 | flat_obs.append(float(element)) 762 | # flat_obs.append(float(session["PID"])/32768) 763 | else: 764 | flat_obs.append(-1.0) 765 | 766 | if 'Interface' not in host: 767 | host["Interface"] = [] 768 | while len(host["Interface"]) < self.MAX_INTERFACES: 769 | host["Interface"].append({}) 770 | 771 | if len(host["Interface"]) > self.MAX_INTERFACES: 772 | with open(self.log_name, 'a') as file: 773 | file.write(f'MAX_INTERFACES: {len(host["Interface"])}\n') 774 | while len(host["Interface"]) > self.MAX_INTERFACES: 775 | host["Interface"].pop() 776 | # raise ValueError("Too many interfaces in observation for fixed size") 777 | 778 | if 'Interface' in host: 779 | for interface_idx, interface in enumerate(host['Interface']): 780 | # if "Interface Name" in interface: 781 | # element = interface["Interface Name"] 782 | # if element not in self.interface_name: 783 | # self.interface_name[element] = len(self.interface_name) 784 | # element = self.interface_name[element] 785 | # flat_obs.append(float( 786 | # element)) 787 | # else: 788 | # flat_obs.append(-1.0) 789 | 790 | # if "Subnet" in interface: 791 | # element = interface["Subnet"].network_address 792 | # if element not in self.subnet_netadd: 793 | # self.subnet_netadd[element] = len(self.subnet_netadd) 794 | # # with open(self.log_name, 'a') as file: 795 | # # file.write(f"subnet_netadd, {len(self.subnet_netadd)}\n") 796 | # element = self.subnet_netadd[element] 797 | # flat_obs.append(float(element)) 798 | # # flat_obs.append(float(int(element.network_address))/4294967296) 799 | # element = interface["Subnet"].prefixlen 800 | # if element not in self.subnet_prefix: 801 | # self.subnet_prefix[element] = len(self.subnet_prefix) 802 | # # with open(self.log_name, 'a') as file: 803 | # # file.write(f"subnet_prefix, {len(self.subnet_prefix)}\n") 804 | # element = self.subnet_prefix[element] 805 | # flat_obs.append(float(element)) 806 | # # flat_obs.append(float(int(element.prefixlen))/4294967296) 807 | # else: 808 | # flat_obs.append(-1.0) 809 | # flat_obs.append(-1.0) 810 | 811 | if "IP Address" in interface: 812 | element = int(interface["IP Address"]) 813 | if element not in self.ipadd: 814 | self.ipadd[element] = len(self.ipadd) 815 | # with open(self.log_name, 'a') as file: 816 | # file.write(f"ipadd, {len(self.ipadd)}\n") 817 | element = self.ipadd[element] 818 | flat_obs.append(float(element)) 819 | # flat_obs.append(float(element)/4294967296) 820 | else: 821 | flat_obs.append(-1.0) 822 | print('ip add', len(flat_obs)-1) 823 | 824 | # print('flat_obs', np.shape(flat_obs), flush=True) 825 | 826 | ## change here for PCA 827 | # dir_path = os.path.dirname(os.path.realpath(__file__)) 828 | # # print('dir_path', dir_path, flush=True) 829 | # pca_obs = pickle.load(open(dir_path+'/pca_10000.pkl', 'rb')) 830 | # obs = np.reshape(flat_obs, (-1, np.shape(flat_obs)[0])) 831 | # obs_transformed = pca_obs.transform(obs) 832 | # # print('obs_transformed.squeeze()', np.shape(obs_transformed.squeeze()), flush=True) 833 | # return obs_transformed.squeeze() 834 | if self.count%2==0: 835 | with open('flat_obs.csv', 'a') as rb_csv: 836 | writer_rb_csv = writer(rb_csv) 837 | writer_rb_csv.writerow(flat_obs) 838 | rb_csv.close() 839 | return flat_obs 840 | 841 | def get_attr(self,attribute:str): 842 | return self.env.get_attr(attribute) 843 | 844 | def get_observation(self, agent: str): 845 | # print('in get_observation()', flush=True) 846 | obs = self.get_attr('get_observation')(agent) 847 | # new_obs = self.observation_change(obs) 848 | # print('new_obs', new_obs, flush=True) 849 | return self.observation_change(obs) 850 | -------------------------------------------------------------------------------- /logs/bandits/controller_bandit_2022-07-15_11-08-56/log_rewards.eps: -------------------------------------------------------------------------------- 1 | %!PS-Adobe-3.0 EPSF-3.0 2 | %%Title: log_rewards.eps 3 | %%Creator: Matplotlib v3.5.1, https://matplotlib.org/ 4 | %%CreationDate: Fri Jul 15 11:08:57 2022 5 | %%Orientation: portrait 6 | %%BoundingBox: 75 223 537 569 7 | %%HiResBoundingBox: 75.600000 223.200000 536.400000 568.800000 8 | %%EndComments 9 | %%BeginProlog 10 | /mpldict 11 dict def 11 | mpldict begin 12 | /_d { bind def } bind def 13 | /m { moveto } _d 14 | /l { lineto } _d 15 | /r { rlineto } _d 16 | /c { curveto } _d 17 | /cl { closepath } _d 18 | /ce { closepath eofill } _d 19 | /box { 20 | m 21 | 1 index 0 r 22 | 0 exch r 23 | neg 0 r 24 | cl 25 | } _d 26 | /clipbox { 27 | box 28 | clip 29 | newpath 30 | } _d 31 | /sc { setcachedevice } _d 32 | %!PS-Adobe-3.0 Resource-Font 33 | %%Creator: Converted from TrueType to Type 3 by Matplotlib. 34 | 10 dict begin 35 | /FontName /DejaVuSans def 36 | /PaintType 0 def 37 | /FontMatrix [0.00048828125 0 0 0.00048828125 0 0] def 38 | /FontBBox [-2090 -948 3673 2524] def 39 | /FontType 3 def 40 | /Encoding [/zero /one /two /three /five] def 41 | /CharStrings 6 dict dup begin 42 | /.notdef 0 def 43 | /zero{1303 0 135 -29 1167 1520 sc 44 | 651 1360 m 45 | 547 1360 469 1309 416 1206 c 46 | 364 1104 338 950 338 745 c 47 | 338 540 364 387 416 284 c 48 | 469 182 547 131 651 131 c 49 | 756 131 834 182 886 284 c 50 | 939 387 965 540 965 745 c 51 | 965 950 939 1104 886 1206 c 52 | 834 1309 756 1360 651 1360 c 53 | 54 | 651 1520 m 55 | 818 1520 946 1454 1034 1321 c 56 | 1123 1189 1167 997 1167 745 c 57 | 1167 494 1123 302 1034 169 c 58 | 946 37 818 -29 651 -29 c 59 | 484 -29 356 37 267 169 c 60 | 179 302 135 494 135 745 c 61 | 135 997 179 1189 267 1321 c 62 | 356 1454 484 1520 651 1520 c 63 | 64 | ce} _d 65 | /one{1303 0 225 0 1114 1493 sc 66 | 254 170 m 67 | 584 170 l 68 | 584 1309 l 69 | 225 1237 l 70 | 225 1421 l 71 | 582 1493 l 72 | 784 1493 l 73 | 784 170 l 74 | 1114 170 l 75 | 1114 0 l 76 | 254 0 l 77 | 254 170 l 78 | 79 | ce} _d 80 | /two{1303 0 150 0 1098 1520 sc 81 | 393 170 m 82 | 1098 170 l 83 | 1098 0 l 84 | 150 0 l 85 | 150 170 l 86 | 227 249 331 356 463 489 c 87 | 596 623 679 709 713 748 c 88 | 778 821 823 882 848 932 c 89 | 874 983 887 1032 887 1081 c 90 | 887 1160 859 1225 803 1275 c 91 | 748 1325 675 1350 586 1350 c 92 | 523 1350 456 1339 385 1317 c 93 | 315 1295 240 1262 160 1217 c 94 | 160 1421 l 95 | 241 1454 317 1478 388 1495 c 96 | 459 1512 523 1520 582 1520 c 97 | 737 1520 860 1481 952 1404 c 98 | 1044 1327 1090 1223 1090 1094 c 99 | 1090 1033 1078 974 1055 919 c 100 | 1032 864 991 800 930 725 c 101 | 913 706 860 650 771 557 c 102 | 682 465 556 336 393 170 c 103 | 104 | ce} _d 105 | /three{1303 0 156 -29 1139 1520 sc 106 | 831 805 m 107 | 928 784 1003 741 1057 676 c 108 | 1112 611 1139 530 1139 434 c 109 | 1139 287 1088 173 987 92 c 110 | 886 11 742 -29 555 -29 c 111 | 492 -29 428 -23 361 -10 c 112 | 295 2 227 20 156 45 c 113 | 156 240 l 114 | 212 207 273 183 340 166 c 115 | 407 149 476 141 549 141 c 116 | 676 141 772 166 838 216 c 117 | 905 266 938 339 938 434 c 118 | 938 522 907 591 845 640 c 119 | 784 690 698 715 588 715 c 120 | 414 715 l 121 | 414 881 l 122 | 596 881 l 123 | 695 881 771 901 824 940 c 124 | 877 980 903 1037 903 1112 c 125 | 903 1189 876 1247 821 1288 c 126 | 767 1329 689 1350 588 1350 c 127 | 533 1350 473 1344 410 1332 c 128 | 347 1320 277 1301 201 1276 c 129 | 201 1456 l 130 | 278 1477 349 1493 416 1504 c 131 | 483 1515 547 1520 606 1520 c 132 | 759 1520 881 1485 970 1415 c 133 | 1059 1346 1104 1252 1104 1133 c 134 | 1104 1050 1080 980 1033 923 c 135 | 986 866 918 827 831 805 c 136 | 137 | ce} _d 138 | /five{1303 0 158 -29 1124 1493 sc 139 | 221 1493 m 140 | 1014 1493 l 141 | 1014 1323 l 142 | 406 1323 l 143 | 406 957 l 144 | 435 967 465 974 494 979 c 145 | 523 984 553 987 582 987 c 146 | 749 987 881 941 978 850 c 147 | 1075 759 1124 635 1124 479 c 148 | 1124 318 1074 193 974 104 c 149 | 874 15 733 -29 551 -29 c 150 | 488 -29 424 -24 359 -13 c 151 | 294 -2 227 14 158 35 c 152 | 158 238 l 153 | 218 205 280 181 344 165 c 154 | 408 149 476 141 547 141 c 155 | 662 141 754 171 821 232 c 156 | 888 293 922 375 922 479 c 157 | 922 583 888 665 821 726 c 158 | 754 787 662 817 547 817 c 159 | 493 817 439 811 385 799 c 160 | 332 787 277 768 221 743 c 161 | 221 1493 l 162 | 163 | ce} _d 164 | end readonly def 165 | 166 | /BuildGlyph { 167 | exch begin 168 | CharStrings exch 169 | 2 copy known not {pop /.notdef} if 170 | true 3 1 roll get exec 171 | end 172 | } _d 173 | 174 | /BuildChar { 175 | 1 index /Encoding get exch get 176 | 1 index /BuildGlyph get exec 177 | } _d 178 | 179 | FontName currentdict end definefont pop 180 | end 181 | %%EndProlog 182 | mpldict begin 183 | 75.6 223.2 translate 184 | 460.8 345.6 0 0 clipbox 185 | gsave 186 | 0 0 m 187 | 460.8 0 l 188 | 460.8 345.6 l 189 | 0 345.6 l 190 | cl 191 | 1.000 setgray 192 | fill 193 | grestore 194 | gsave 195 | 57.6 38.016 m 196 | 414.72 38.016 l 197 | 414.72 304.128 l 198 | 57.6 304.128 l 199 | cl 200 | 1.000 setgray 201 | fill 202 | grestore 203 | 0.800 setlinewidth 204 | 1 setlinejoin 205 | 0 setlinecap 206 | [] 0 setdash 207 | 0.000 setgray 208 | gsave 209 | /o { 210 | gsave 211 | newpath 212 | translate 213 | 0.8 setlinewidth 214 | 1 setlinejoin 215 | 216 | 0 setlinecap 217 | 218 | 0 0 m 219 | 0 -3.5 l 220 | 221 | gsave 222 | 0.000 setgray 223 | fill 224 | grestore 225 | stroke 226 | grestore 227 | } bind def 228 | 73.8327 38.016 o 229 | grestore 230 | gsave 231 | 64.832727 22.094125 translate 232 | 0.000000 rotate 233 | /DejaVuSans 10.0 selectfont 234 | 0.000000 0.976562 moveto 235 | /one glyphshow 236 | 6.362305 0.976562 moveto 237 | /zero glyphshow 238 | /DejaVuSans 7.0 selectfont 239 | 12.820312 4.804688 moveto 240 | /zero glyphshow 241 | grestore 242 | gsave 243 | /o { 244 | gsave 245 | newpath 246 | translate 247 | 0.8 setlinewidth 248 | 1 setlinejoin 249 | 250 | 0 setlinecap 251 | 252 | 0 0 m 253 | 0 -3.5 l 254 | 255 | gsave 256 | 0.000 setgray 257 | fill 258 | grestore 259 | stroke 260 | grestore 261 | } bind def 262 | 174.197 38.016 o 263 | grestore 264 | gsave 265 | 165.196746 23.094125 translate 266 | 0.000000 rotate 267 | /DejaVuSans 10.0 selectfont 268 | 0.000000 0.064063 moveto 269 | /one glyphshow 270 | 6.362305 0.064063 moveto 271 | /zero glyphshow 272 | /DejaVuSans 7.0 selectfont 273 | 12.820312 3.892188 moveto 274 | /one glyphshow 275 | grestore 276 | gsave 277 | /o { 278 | gsave 279 | newpath 280 | translate 281 | 0.8 setlinewidth 282 | 1 setlinejoin 283 | 284 | 0 setlinecap 285 | 286 | 0 0 m 287 | 0 -3.5 l 288 | 289 | gsave 290 | 0.000 setgray 291 | fill 292 | grestore 293 | stroke 294 | grestore 295 | } bind def 296 | 274.561 38.016 o 297 | grestore 298 | gsave 299 | 265.560764 22.094125 translate 300 | 0.000000 rotate 301 | /DejaVuSans 10.0 selectfont 302 | 0.000000 0.976562 moveto 303 | /one glyphshow 304 | 6.362305 0.976562 moveto 305 | /zero glyphshow 306 | /DejaVuSans 7.0 selectfont 307 | 12.820312 4.804688 moveto 308 | /two glyphshow 309 | grestore 310 | gsave 311 | /o { 312 | gsave 313 | newpath 314 | translate 315 | 0.8 setlinewidth 316 | 1 setlinejoin 317 | 318 | 0 setlinecap 319 | 320 | 0 0 m 321 | 0 -3.5 l 322 | 323 | gsave 324 | 0.000 setgray 325 | fill 326 | grestore 327 | stroke 328 | grestore 329 | } bind def 330 | 374.925 38.016 o 331 | grestore 332 | gsave 333 | 365.924783 22.094125 translate 334 | 0.000000 rotate 335 | /DejaVuSans 10.0 selectfont 336 | 0.000000 0.976562 moveto 337 | /one glyphshow 338 | 6.362305 0.976562 moveto 339 | /zero glyphshow 340 | /DejaVuSans 7.0 selectfont 341 | 12.820312 4.804688 moveto 342 | /three glyphshow 343 | grestore 344 | 0.600 setlinewidth 345 | gsave 346 | /o { 347 | gsave 348 | newpath 349 | translate 350 | 0.6 setlinewidth 351 | 1 setlinejoin 352 | 353 | 0 setlinecap 354 | 355 | 0 0 m 356 | 0 -2 l 357 | 358 | gsave 359 | 0.000 setgray 360 | fill 361 | grestore 362 | stroke 363 | grestore 364 | } bind def 365 | 58.2861 38.016 o 366 | grestore 367 | gsave 368 | /o { 369 | gsave 370 | newpath 371 | translate 372 | 0.6 setlinewidth 373 | 1 setlinejoin 374 | 375 | 0 setlinecap 376 | 377 | 0 0 m 378 | 0 -2 l 379 | 380 | gsave 381 | 0.000 setgray 382 | fill 383 | grestore 384 | stroke 385 | grestore 386 | } bind def 387 | 64.1064 38.016 o 388 | grestore 389 | gsave 390 | /o { 391 | gsave 392 | newpath 393 | translate 394 | 0.6 setlinewidth 395 | 1 setlinejoin 396 | 397 | 0 setlinecap 398 | 399 | 0 0 m 400 | 0 -2 l 401 | 402 | gsave 403 | 0.000 setgray 404 | fill 405 | grestore 406 | stroke 407 | grestore 408 | } bind def 409 | 69.2403 38.016 o 410 | grestore 411 | gsave 412 | /o { 413 | gsave 414 | newpath 415 | translate 416 | 0.6 setlinewidth 417 | 1 setlinejoin 418 | 419 | 0 setlinecap 420 | 421 | 0 0 m 422 | 0 -2 l 423 | 424 | gsave 425 | 0.000 setgray 426 | fill 427 | grestore 428 | stroke 429 | grestore 430 | } bind def 431 | 104.045 38.016 o 432 | grestore 433 | gsave 434 | /o { 435 | gsave 436 | newpath 437 | translate 438 | 0.6 setlinewidth 439 | 1 setlinejoin 440 | 441 | 0 setlinecap 442 | 443 | 0 0 m 444 | 0 -2 l 445 | 446 | gsave 447 | 0.000 setgray 448 | fill 449 | grestore 450 | stroke 451 | grestore 452 | } bind def 453 | 121.719 38.016 o 454 | grestore 455 | gsave 456 | /o { 457 | gsave 458 | newpath 459 | translate 460 | 0.6 setlinewidth 461 | 1 setlinejoin 462 | 463 | 0 setlinecap 464 | 465 | 0 0 m 466 | 0 -2 l 467 | 468 | gsave 469 | 0.000 setgray 470 | fill 471 | grestore 472 | stroke 473 | grestore 474 | } bind def 475 | 134.258 38.016 o 476 | grestore 477 | gsave 478 | /o { 479 | gsave 480 | newpath 481 | translate 482 | 0.6 setlinewidth 483 | 1 setlinejoin 484 | 485 | 0 setlinecap 486 | 487 | 0 0 m 488 | 0 -2 l 489 | 490 | gsave 491 | 0.000 setgray 492 | fill 493 | grestore 494 | stroke 495 | grestore 496 | } bind def 497 | 143.984 38.016 o 498 | grestore 499 | gsave 500 | /o { 501 | gsave 502 | newpath 503 | translate 504 | 0.6 setlinewidth 505 | 1 setlinejoin 506 | 507 | 0 setlinecap 508 | 509 | 0 0 m 510 | 0 -2 l 511 | 512 | gsave 513 | 0.000 setgray 514 | fill 515 | grestore 516 | stroke 517 | grestore 518 | } bind def 519 | 151.931 38.016 o 520 | grestore 521 | gsave 522 | /o { 523 | gsave 524 | newpath 525 | translate 526 | 0.6 setlinewidth 527 | 1 setlinejoin 528 | 529 | 0 setlinecap 530 | 531 | 0 0 m 532 | 0 -2 l 533 | 534 | gsave 535 | 0.000 setgray 536 | fill 537 | grestore 538 | stroke 539 | grestore 540 | } bind def 541 | 158.65 38.016 o 542 | grestore 543 | gsave 544 | /o { 545 | gsave 546 | newpath 547 | translate 548 | 0.6 setlinewidth 549 | 1 setlinejoin 550 | 551 | 0 setlinecap 552 | 553 | 0 0 m 554 | 0 -2 l 555 | 556 | gsave 557 | 0.000 setgray 558 | fill 559 | grestore 560 | stroke 561 | grestore 562 | } bind def 563 | 164.47 38.016 o 564 | grestore 565 | gsave 566 | /o { 567 | gsave 568 | newpath 569 | translate 570 | 0.6 setlinewidth 571 | 1 setlinejoin 572 | 573 | 0 setlinecap 574 | 575 | 0 0 m 576 | 0 -2 l 577 | 578 | gsave 579 | 0.000 setgray 580 | fill 581 | grestore 582 | stroke 583 | grestore 584 | } bind def 585 | 169.604 38.016 o 586 | grestore 587 | gsave 588 | /o { 589 | gsave 590 | newpath 591 | translate 592 | 0.6 setlinewidth 593 | 1 setlinejoin 594 | 595 | 0 setlinecap 596 | 597 | 0 0 m 598 | 0 -2 l 599 | 600 | gsave 601 | 0.000 setgray 602 | fill 603 | grestore 604 | stroke 605 | grestore 606 | } bind def 607 | 204.409 38.016 o 608 | grestore 609 | gsave 610 | /o { 611 | gsave 612 | newpath 613 | translate 614 | 0.6 setlinewidth 615 | 1 setlinejoin 616 | 617 | 0 setlinecap 618 | 619 | 0 0 m 620 | 0 -2 l 621 | 622 | gsave 623 | 0.000 setgray 624 | fill 625 | grestore 626 | stroke 627 | grestore 628 | } bind def 629 | 222.083 38.016 o 630 | grestore 631 | gsave 632 | /o { 633 | gsave 634 | newpath 635 | translate 636 | 0.6 setlinewidth 637 | 1 setlinejoin 638 | 639 | 0 setlinecap 640 | 641 | 0 0 m 642 | 0 -2 l 643 | 644 | gsave 645 | 0.000 setgray 646 | fill 647 | grestore 648 | stroke 649 | grestore 650 | } bind def 651 | 234.622 38.016 o 652 | grestore 653 | gsave 654 | /o { 655 | gsave 656 | newpath 657 | translate 658 | 0.6 setlinewidth 659 | 1 setlinejoin 660 | 661 | 0 setlinecap 662 | 663 | 0 0 m 664 | 0 -2 l 665 | 666 | gsave 667 | 0.000 setgray 668 | fill 669 | grestore 670 | stroke 671 | grestore 672 | } bind def 673 | 244.348 38.016 o 674 | grestore 675 | gsave 676 | /o { 677 | gsave 678 | newpath 679 | translate 680 | 0.6 setlinewidth 681 | 1 setlinejoin 682 | 683 | 0 setlinecap 684 | 685 | 0 0 m 686 | 0 -2 l 687 | 688 | gsave 689 | 0.000 setgray 690 | fill 691 | grestore 692 | stroke 693 | grestore 694 | } bind def 695 | 252.295 38.016 o 696 | grestore 697 | gsave 698 | /o { 699 | gsave 700 | newpath 701 | translate 702 | 0.6 setlinewidth 703 | 1 setlinejoin 704 | 705 | 0 setlinecap 706 | 707 | 0 0 m 708 | 0 -2 l 709 | 710 | gsave 711 | 0.000 setgray 712 | fill 713 | grestore 714 | stroke 715 | grestore 716 | } bind def 717 | 259.014 38.016 o 718 | grestore 719 | gsave 720 | /o { 721 | gsave 722 | newpath 723 | translate 724 | 0.6 setlinewidth 725 | 1 setlinejoin 726 | 727 | 0 setlinecap 728 | 729 | 0 0 m 730 | 0 -2 l 731 | 732 | gsave 733 | 0.000 setgray 734 | fill 735 | grestore 736 | stroke 737 | grestore 738 | } bind def 739 | 264.834 38.016 o 740 | grestore 741 | gsave 742 | /o { 743 | gsave 744 | newpath 745 | translate 746 | 0.6 setlinewidth 747 | 1 setlinejoin 748 | 749 | 0 setlinecap 750 | 751 | 0 0 m 752 | 0 -2 l 753 | 754 | gsave 755 | 0.000 setgray 756 | fill 757 | grestore 758 | stroke 759 | grestore 760 | } bind def 761 | 269.968 38.016 o 762 | grestore 763 | gsave 764 | /o { 765 | gsave 766 | newpath 767 | translate 768 | 0.6 setlinewidth 769 | 1 setlinejoin 770 | 771 | 0 setlinecap 772 | 773 | 0 0 m 774 | 0 -2 l 775 | 776 | gsave 777 | 0.000 setgray 778 | fill 779 | grestore 780 | stroke 781 | grestore 782 | } bind def 783 | 304.773 38.016 o 784 | grestore 785 | gsave 786 | /o { 787 | gsave 788 | newpath 789 | translate 790 | 0.6 setlinewidth 791 | 1 setlinejoin 792 | 793 | 0 setlinecap 794 | 795 | 0 0 m 796 | 0 -2 l 797 | 798 | gsave 799 | 0.000 setgray 800 | fill 801 | grestore 802 | stroke 803 | grestore 804 | } bind def 805 | 322.447 38.016 o 806 | grestore 807 | gsave 808 | /o { 809 | gsave 810 | newpath 811 | translate 812 | 0.6 setlinewidth 813 | 1 setlinejoin 814 | 815 | 0 setlinecap 816 | 817 | 0 0 m 818 | 0 -2 l 819 | 820 | gsave 821 | 0.000 setgray 822 | fill 823 | grestore 824 | stroke 825 | grestore 826 | } bind def 827 | 334.986 38.016 o 828 | grestore 829 | gsave 830 | /o { 831 | gsave 832 | newpath 833 | translate 834 | 0.6 setlinewidth 835 | 1 setlinejoin 836 | 837 | 0 setlinecap 838 | 839 | 0 0 m 840 | 0 -2 l 841 | 842 | gsave 843 | 0.000 setgray 844 | fill 845 | grestore 846 | stroke 847 | grestore 848 | } bind def 849 | 344.712 38.016 o 850 | grestore 851 | gsave 852 | /o { 853 | gsave 854 | newpath 855 | translate 856 | 0.6 setlinewidth 857 | 1 setlinejoin 858 | 859 | 0 setlinecap 860 | 861 | 0 0 m 862 | 0 -2 l 863 | 864 | gsave 865 | 0.000 setgray 866 | fill 867 | grestore 868 | stroke 869 | grestore 870 | } bind def 871 | 352.659 38.016 o 872 | grestore 873 | gsave 874 | /o { 875 | gsave 876 | newpath 877 | translate 878 | 0.6 setlinewidth 879 | 1 setlinejoin 880 | 881 | 0 setlinecap 882 | 883 | 0 0 m 884 | 0 -2 l 885 | 886 | gsave 887 | 0.000 setgray 888 | fill 889 | grestore 890 | stroke 891 | grestore 892 | } bind def 893 | 359.378 38.016 o 894 | grestore 895 | gsave 896 | /o { 897 | gsave 898 | newpath 899 | translate 900 | 0.6 setlinewidth 901 | 1 setlinejoin 902 | 903 | 0 setlinecap 904 | 905 | 0 0 m 906 | 0 -2 l 907 | 908 | gsave 909 | 0.000 setgray 910 | fill 911 | grestore 912 | stroke 913 | grestore 914 | } bind def 915 | 365.199 38.016 o 916 | grestore 917 | gsave 918 | /o { 919 | gsave 920 | newpath 921 | translate 922 | 0.6 setlinewidth 923 | 1 setlinejoin 924 | 925 | 0 setlinecap 926 | 927 | 0 0 m 928 | 0 -2 l 929 | 930 | gsave 931 | 0.000 setgray 932 | fill 933 | grestore 934 | stroke 935 | grestore 936 | } bind def 937 | 370.332 38.016 o 938 | grestore 939 | gsave 940 | /o { 941 | gsave 942 | newpath 943 | translate 944 | 0.6 setlinewidth 945 | 1 setlinejoin 946 | 947 | 0 setlinecap 948 | 949 | 0 0 m 950 | 0 -2 l 951 | 952 | gsave 953 | 0.000 setgray 954 | fill 955 | grestore 956 | stroke 957 | grestore 958 | } bind def 959 | 405.137 38.016 o 960 | grestore 961 | 0.800 setlinewidth 962 | gsave 963 | /o { 964 | gsave 965 | newpath 966 | translate 967 | 0.8 setlinewidth 968 | 1 setlinejoin 969 | 970 | 0 setlinecap 971 | 972 | -0 0 m 973 | -3.5 0 l 974 | 975 | gsave 976 | 0.000 setgray 977 | fill 978 | grestore 979 | stroke 980 | grestore 981 | } bind def 982 | 57.6 50.9618 o 983 | grestore 984 | /DejaVuSans 10.000 selectfont 985 | gsave 986 | 987 | 44.240625 47.164961 translate 988 | 0.000000 rotate 989 | 0.000000 0 m /zero glyphshow 990 | grestore 991 | gsave 992 | /o { 993 | gsave 994 | newpath 995 | translate 996 | 0.8 setlinewidth 997 | 1 setlinejoin 998 | 999 | 0 setlinecap 1000 | 1001 | -0 0 m 1002 | -3.5 0 l 1003 | 1004 | gsave 1005 | 0.000 setgray 1006 | fill 1007 | grestore 1008 | stroke 1009 | grestore 1010 | } bind def 1011 | 57.6 96.9946 o 1012 | grestore 1013 | gsave 1014 | 1015 | 44.240625 93.197748 translate 1016 | 0.000000 rotate 1017 | 0.000000 0 m /five glyphshow 1018 | grestore 1019 | gsave 1020 | /o { 1021 | gsave 1022 | newpath 1023 | translate 1024 | 0.8 setlinewidth 1025 | 1 setlinejoin 1026 | 1027 | 0 setlinecap 1028 | 1029 | -0 0 m 1030 | -3.5 0 l 1031 | 1032 | gsave 1033 | 0.000 setgray 1034 | fill 1035 | grestore 1036 | stroke 1037 | grestore 1038 | } bind def 1039 | 57.6 143.027 o 1040 | grestore 1041 | gsave 1042 | 1043 | 37.881250 139.230535 translate 1044 | 0.000000 rotate 1045 | 0.000000 0 m /one glyphshow 1046 | 6.362305 0 m /zero glyphshow 1047 | grestore 1048 | gsave 1049 | /o { 1050 | gsave 1051 | newpath 1052 | translate 1053 | 0.8 setlinewidth 1054 | 1 setlinejoin 1055 | 1056 | 0 setlinecap 1057 | 1058 | -0 0 m 1059 | -3.5 0 l 1060 | 1061 | gsave 1062 | 0.000 setgray 1063 | fill 1064 | grestore 1065 | stroke 1066 | grestore 1067 | } bind def 1068 | 57.6 189.06 o 1069 | grestore 1070 | gsave 1071 | 1072 | 37.881250 185.263322 translate 1073 | 0.000000 rotate 1074 | 0.000000 0 m /one glyphshow 1075 | 6.362305 0 m /five glyphshow 1076 | grestore 1077 | gsave 1078 | /o { 1079 | gsave 1080 | newpath 1081 | translate 1082 | 0.8 setlinewidth 1083 | 1 setlinejoin 1084 | 1085 | 0 setlinecap 1086 | 1087 | -0 0 m 1088 | -3.5 0 l 1089 | 1090 | gsave 1091 | 0.000 setgray 1092 | fill 1093 | grestore 1094 | stroke 1095 | grestore 1096 | } bind def 1097 | 57.6 235.093 o 1098 | grestore 1099 | gsave 1100 | 1101 | 37.881250 231.296109 translate 1102 | 0.000000 rotate 1103 | 0.000000 0 m /two glyphshow 1104 | 6.362305 0 m /zero glyphshow 1105 | grestore 1106 | gsave 1107 | /o { 1108 | gsave 1109 | newpath 1110 | translate 1111 | 0.8 setlinewidth 1112 | 1 setlinejoin 1113 | 1114 | 0 setlinecap 1115 | 1116 | -0 0 m 1117 | -3.5 0 l 1118 | 1119 | gsave 1120 | 0.000 setgray 1121 | fill 1122 | grestore 1123 | stroke 1124 | grestore 1125 | } bind def 1126 | 57.6 281.126 o 1127 | grestore 1128 | gsave 1129 | 1130 | 37.881250 277.328895 translate 1131 | 0.000000 rotate 1132 | 0.000000 0 m /two glyphshow 1133 | 6.362305 0 m /five glyphshow 1134 | grestore 1135 | 1.500 setlinewidth 1136 | 2 setlinecap 1137 | 0.122 0.467 0.706 setrgbcolor 1138 | gsave 1139 | 357.12 266.112 57.6 38.016 clipbox 1140 | -1 50.96173 m 1141 | 73.832727 50.961836 l 1142 | 121.718534 51.245115 l 1143 | 151.931114 51.670033 l 1144 | 174.196746 52.23659 l 1145 | 191.869972 52.944787 l 1146 | 204.409326 53.652984 l 1147 | 215.845138 54.50282 l 1148 | 226.236888 55.494295 l 1149 | 235.698196 56.62741 l 1150 | 244.348184 57.902164 l 1151 | 252.295132 59.318557 l 1152 | 259.014181 60.734951 l 1153 | 268.490673 63.00118 l 1154 | 274.560764 64.842492 l 1155 | 277.915307 65.975607 l 1156 | 278.317038 65.833967 l 1157 | 283.584002 67.816918 l 1158 | 288.599591 69.941508 l 1159 | 293.381097 72.207738 l 1160 | 297.94519 74.615607 l 1161 | 302.307582 77.165115 l 1162 | 306.482878 79.856262 l 1163 | 312.355541 83.963803 l 1164 | 316.209157 87.079869 l 1165 | 316.37648 86.93823 l 1166 | 320.210822 90.337574 l 1167 | 323.875798 93.878557 l 1168 | 329.906502 100.110689 l 1169 | 330.15069 100.393967 l 1170 | 330.15069 100.393967 l 1171 | stroke 1172 | grestore 1173 | 1.000 0.498 0.055 setrgbcolor 1174 | gsave 1175 | 357.12 266.112 57.6 38.016 clipbox 1176 | -1 50.96173 m 1177 | 73.832727 50.961836 l 1178 | 121.718534 51.245115 l 1179 | 151.931114 51.670033 l 1180 | 174.196746 52.23659 l 1181 | 191.869972 52.944787 l 1182 | 204.409326 53.652984 l 1183 | 215.845138 54.50282 l 1184 | 226.236888 55.494295 l 1185 | 235.698196 56.62741 l 1186 | 244.348184 57.902164 l 1187 | 252.295132 59.318557 l 1188 | 259.014181 60.734951 l 1189 | 265.375953 62.292984 l 1190 | 271.397586 63.992656 l 1191 | 277.100562 65.833967 l 1192 | 282.507712 67.816918 l 1193 | 287.641585 69.941508 l 1194 | 292.52361 72.207738 l 1195 | 297.173709 74.615607 l 1196 | 301.610166 77.165115 l 1197 | 305.849634 79.856262 l 1198 | 311.802626 83.963803 l 1199 | 315.7033 87.079869 l 1200 | 319.594733 90.479213 l 1201 | 323.309718 94.020197 l 1202 | 326.995367 97.844459 l 1203 | 328.411926 99.402492 l 1204 | 328.411926 99.402492 l 1205 | stroke 1206 | grestore 1207 | 0.173 0.627 0.173 setrgbcolor 1208 | gsave 1209 | 357.12 266.112 57.6 38.016 clipbox 1210 | -1 51.245009 m 1211 | 73.832727 51.245115 l 1212 | 121.718534 51.528393 l 1213 | 151.931114 51.953311 l 1214 | 174.196746 52.519869 l 1215 | 191.869972 53.228066 l 1216 | 204.409326 53.936262 l 1217 | 215.845138 54.786098 l 1218 | 226.236888 55.777574 l 1219 | 235.698196 56.910689 l 1220 | 244.348184 58.185443 l 1221 | 252.295132 59.601836 l 1222 | 255.108207 60.168393 l 1223 | 255.783997 60.026754 l 1224 | 262.598737 61.584787 l 1225 | 268.988822 63.284459 l 1226 | 274.994475 65.12577 l 1227 | 280.652643 67.108721 l 1228 | 285.659992 69.091672 l 1229 | 290.45466 71.216262 l 1230 | 295.047066 73.482492 l 1231 | 299.448358 75.890361 l 1232 | 303.892758 78.581508 l 1233 | 308.127887 81.414295 l 1234 | 312.172014 84.388721 l 1235 | 316.041189 87.504787 l 1236 | 319.903866 90.904131 l 1237 | 323.593677 94.445115 l 1238 | 327.256371 98.269377 l 1239 | 329.906502 101.243803 l 1240 | 330.028767 101.102164 l 1241 | 333.545798 105.351344 l 1242 | 337.008664 109.883803 l 1243 | 340.409416 114.699541 l 1244 | 343.831616 119.940197 l 1245 | 347.169682 125.464131 l 1246 | 350.499804 131.412984 l 1247 | 353.806257 137.786754 l 1248 | 357.076856 144.585443 l 1249 | 360.302351 151.809049 l 1250 | 365.847462 164.981508 l 1251 | 369.004735 173.621508 l 1252 | 370.186842 176.737574 l 1253 | 370.235408 176.595934 l 1254 | 370.283919 176.737574 l 1255 | 373.462125 186.369049 l 1256 | 377.628753 199.824787 l 1257 | 380.750535 211.014295 l 1258 | 383.877089 223.053639 l 1259 | 386.993033 235.94282 l 1260 | 390.116869 249.823475 l 1261 | 393.970958 268.094951 l 1262 | 397.068198 284.241836 l 1263 | 398.487273 292.032 l 1264 | 398.487273 292.032 l 1265 | stroke 1266 | grestore 1267 | 0.839 0.153 0.157 setrgbcolor 1268 | gsave 1269 | 357.12 266.112 57.6 38.016 clipbox 1270 | -1 50.96173 m 1271 | 73.832727 50.961836 l 1272 | 121.718534 51.245115 l 1273 | 151.931114 51.670033 l 1274 | 174.196746 52.23659 l 1275 | 191.869972 52.944787 l 1276 | 204.409326 53.652984 l 1277 | 215.845138 54.50282 l 1278 | 226.236888 55.494295 l 1279 | 235.698196 56.62741 l 1280 | 244.348184 57.902164 l 1281 | 252.295132 59.318557 l 1282 | 259.014181 60.734951 l 1283 | 265.375953 62.292984 l 1284 | 271.397586 63.992656 l 1285 | 277.100562 65.833967 l 1286 | 282.507712 67.816918 l 1287 | 287.641585 69.941508 l 1288 | 292.52361 72.207738 l 1289 | 297.173709 74.615607 l 1290 | 301.610166 77.165115 l 1291 | 305.849634 79.856262 l 1292 | 309.907217 82.689049 l 1293 | 313.973408 85.805115 l 1294 | 317.854165 89.06282 l 1295 | 321.565984 92.462164 l 1296 | 325.259646 96.144787 l 1297 | 328.915833 100.110689 l 1298 | 332.520162 104.359869 l 1299 | 332.750176 104.643148 l 1300 | 332.750176 104.643148 l 1301 | stroke 1302 | grestore 1303 | 0.580 0.404 0.741 setrgbcolor 1304 | gsave 1305 | 357.12 266.112 57.6 38.016 clipbox 1306 | -1 51.245009 m 1307 | 73.832727 51.245115 l 1308 | 121.718534 51.528393 l 1309 | 151.931114 51.953311 l 1310 | 182.143694 52.661508 l 1311 | 197.325525 53.369705 l 1312 | 210.501205 54.219541 l 1313 | 222.082552 55.211016 l 1314 | 231.223754 56.202492 l 1315 | 236.748549 56.910689 l 1316 | 237.774185 56.769049 l 1317 | 239.755778 57.052328 l 1318 | 240.713785 56.910689 l 1319 | 248.50252 58.185443 l 1320 | 255.783997 59.601836 l 1321 | 262.598737 61.159869 l 1322 | 268.988822 62.859541 l 1323 | 274.994475 64.700852 l 1324 | 280.652643 66.683803 l 1325 | 285.659992 68.666754 l 1326 | 290.45466 70.791344 l 1327 | 295.047066 73.057574 l 1328 | 299.448358 75.465443 l 1329 | 303.892758 78.15659 l 1330 | 308.127887 80.989377 l 1331 | 312.172014 83.963803 l 1332 | 314.673625 85.946754 l 1333 | 314.846935 85.805115 l 1334 | 318.653959 89.06282 l 1335 | 322.44657 92.603803 l 1336 | 329.413982 99.544131 l 1337 | 332.978982 103.793311 l 1338 | 336.485398 108.32577 l 1339 | 339.925643 113.141508 l 1340 | 340.982945 114.699541 l 1341 | 341.077803 114.557902 l 1342 | 344.44989 119.798557 l 1343 | 347.823917 125.464131 l 1344 | 351.10625 131.412984 l 1345 | 354.368685 137.786754 l 1346 | 358.434028 146.285115 l 1347 | 361.682389 153.792 l 1348 | 364.870366 161.723803 l 1349 | 368.045619 170.222164 l 1350 | 371.195525 179.287082 l 1351 | 374.354429 189.060197 l 1352 | 377.505682 199.541508 l 1353 | 381.991144 215.405115 l 1354 | 385.990208 231.127082 l 1355 | 389.901012 247.982164 l 1356 | 393.002939 262.712656 l 1357 | 393.886404 267.103475 l 1358 | 393.886404 267.103475 l 1359 | stroke 1360 | grestore 1361 | 0.549 0.337 0.294 setrgbcolor 1362 | gsave 1363 | 357.12 266.112 57.6 38.016 clipbox 1364 | -1 51.245009 m 1365 | 73.832727 51.245115 l 1366 | 121.718534 51.528393 l 1367 | 151.931114 51.953311 l 1368 | 174.196746 52.519869 l 1369 | 191.869972 53.228066 l 1370 | 204.409326 53.936262 l 1371 | 215.845138 54.786098 l 1372 | 226.236888 55.777574 l 1373 | 235.698196 56.910689 l 1374 | 244.348184 58.185443 l 1375 | 252.295132 59.601836 l 1376 | 259.014181 61.01823 l 1377 | 265.375953 62.576262 l 1378 | 271.397586 64.275934 l 1379 | 277.100562 66.117246 l 1380 | 282.507712 68.100197 l 1381 | 287.641585 70.224787 l 1382 | 292.52361 72.491016 l 1383 | 297.173709 74.898885 l 1384 | 301.610166 77.448393 l 1385 | 302.766402 78.15659 l 1386 | 302.994014 78.15659 l 1387 | 307.313142 80.989377 l 1388 | 311.430081 83.963803 l 1389 | 315.36277 87.079869 l 1390 | 320.66724 91.612328 l 1391 | 324.43462 95.294951 l 1392 | 328.030134 99.119213 l 1393 | 331.587772 103.226754 l 1394 | 336.27432 109.033967 l 1395 | 339.73062 113.849705 l 1396 | 343.11411 118.948721 l 1397 | 346.505479 124.472656 l 1398 | 349.807313 130.279869 l 1399 | 350.652209 131.837902 l 1400 | 350.728211 131.696262 l 1401 | 355.540278 140.90282 l 1402 | 359.564604 149.54282 l 1403 | 362.790359 157.191344 l 1404 | 366.008203 165.406426 l 1405 | 370.380781 177.304131 l 1406 | 373.552182 186.935607 l 1407 | 377.710607 200.391344 l 1408 | 380.864789 211.722492 l 1409 | 384.89447 227.16118 l 1410 | 389.184147 245.149377 l 1411 | 392.306452 259.73823 l 1412 | 395.411084 275.318557 l 1413 | 398.283711 290.757246 l 1414 | 398.283711 290.757246 l 1415 | stroke 1416 | grestore 1417 | 0.890 0.467 0.761 setrgbcolor 1418 | gsave 1419 | 357.12 266.112 57.6 38.016 clipbox 1420 | -1 50.96173 m 1421 | 73.832727 50.961836 l 1422 | 121.718534 51.245115 l 1423 | 151.931114 51.670033 l 1424 | 174.196746 52.23659 l 1425 | 191.869972 52.944787 l 1426 | 204.409326 53.652984 l 1427 | 215.845138 54.50282 l 1428 | 226.236888 55.494295 l 1429 | 235.698196 56.62741 l 1430 | 244.348184 57.902164 l 1431 | 252.295132 59.318557 l 1432 | 259.014181 60.734951 l 1433 | 265.375953 62.292984 l 1434 | 271.397586 63.992656 l 1435 | 277.100562 65.833967 l 1436 | 282.507712 67.816918 l 1437 | 287.641585 69.941508 l 1438 | 292.52361 72.207738 l 1439 | 297.173709 74.615607 l 1440 | 301.375192 77.023475 l 1441 | 301.610166 76.881836 l 1442 | 305.849634 79.572984 l 1443 | 309.907217 82.40577 l 1444 | 313.973408 85.521836 l 1445 | 317.854165 88.779541 l 1446 | 319.594733 90.337574 l 1447 | 319.749574 90.195934 l 1448 | 323.451929 93.736918 l 1449 | 327.126064 97.56118 l 1450 | 330.755243 101.668721 l 1451 | 333.545798 105.068066 l 1452 | 333.545798 105.068066 l 1453 | stroke 1454 | grestore 1455 | 0.498 setgray 1456 | gsave 1457 | 357.12 266.112 57.6 38.016 clipbox 1458 | -1 50.96173 m 1459 | 73.832727 50.961836 l 1460 | 121.718534 51.245115 l 1461 | 151.931114 51.670033 l 1462 | 174.196746 52.23659 l 1463 | 191.869972 52.944787 l 1464 | 204.409326 53.652984 l 1465 | 215.845138 54.50282 l 1466 | 226.236888 55.494295 l 1467 | 227.538105 55.352656 l 1468 | 236.748549 56.48577 l 1469 | 245.211332 57.760525 l 1470 | 253.015604 59.176918 l 1471 | 259.632454 60.593311 l 1472 | 265.910776 62.151344 l 1473 | 271.863767 63.851016 l 1474 | 277.509838 65.692328 l 1475 | 282.869437 67.675279 l 1476 | 287.963266 69.799869 l 1477 | 292.811317 72.066098 l 1478 | 297.43239 74.473967 l 1479 | 301.84388 77.023475 l 1480 | 306.06174 79.714623 l 1481 | 310.10051 82.54741 l 1482 | 314.14952 85.663475 l 1483 | 318.015302 88.92118 l 1484 | 321.861494 92.462164 l 1485 | 325.53122 96.144787 l 1486 | 328.790401 99.68577 l 1487 | 328.790401 99.68577 l 1488 | stroke 1489 | grestore 1490 | 0.737 0.741 0.133 setrgbcolor 1491 | gsave 1492 | 357.12 266.112 57.6 38.016 clipbox 1493 | -1 50.96173 m 1494 | 73.832727 50.961836 l 1495 | 121.718534 51.245115 l 1496 | 151.931114 51.670033 l 1497 | 174.196746 52.23659 l 1498 | 191.869972 52.944787 l 1499 | 204.409326 53.652984 l 1500 | 215.845138 54.50282 l 1501 | 226.236888 55.494295 l 1502 | 235.698196 56.62741 l 1503 | 244.348184 57.902164 l 1504 | 252.295132 59.318557 l 1505 | 259.014181 60.734951 l 1506 | 265.375953 62.292984 l 1507 | 271.397586 63.992656 l 1508 | 279.88793 66.683803 l 1509 | 284.978923 68.666754 l 1510 | 289.845034 70.791344 l 1511 | 294.498788 73.057574 l 1512 | 298.953039 75.465443 l 1513 | 303.220443 78.014951 l 1514 | 307.518261 80.847738 l 1515 | 308.529618 81.555934 l 1516 | 308.729104 81.414295 l 1517 | 312.720292 84.388721 l 1518 | 318.175846 88.779541 l 1519 | 322.008501 92.320525 l 1520 | 325.666375 96.003148 l 1521 | 329.289978 99.969049 l 1522 | 331.587772 102.660197 l 1523 | 331.587772 102.660197 l 1524 | stroke 1525 | grestore 1526 | 0.090 0.745 0.812 setrgbcolor 1527 | gsave 1528 | 357.12 266.112 57.6 38.016 clipbox 1529 | -1 50.96173 m 1530 | 73.832727 50.961836 l 1531 | 121.718534 51.245115 l 1532 | 151.931114 51.670033 l 1533 | 174.196746 52.23659 l 1534 | 191.869972 52.944787 l 1535 | 204.409326 53.652984 l 1536 | 215.845138 54.50282 l 1537 | 226.236888 55.494295 l 1538 | 235.698196 56.62741 l 1539 | 244.348184 57.902164 l 1540 | 252.295132 59.318557 l 1541 | 259.014181 60.734951 l 1542 | 263.730945 61.868066 l 1543 | 264.286208 61.726426 l 1544 | 270.449994 63.426098 l 1545 | 276.270298 65.26741 l 1546 | 281.775131 67.250361 l 1547 | 286.991012 69.374951 l 1548 | 291.942434 71.64118 l 1549 | 296.651696 74.049049 l 1550 | 301.138945 76.598557 l 1551 | 305.422302 79.289705 l 1552 | 309.51804 82.122492 l 1553 | 313.440764 85.096918 l 1554 | 317.367148 88.354623 l 1555 | 321.118929 91.753967 l 1556 | 324.849085 95.43659 l 1557 | 328.53845 99.402492 l 1558 | 328.664608 99.544131 l 1559 | 328.664608 99.544131 l 1560 | stroke 1561 | grestore 1562 | 0.122 0.467 0.706 setrgbcolor 1563 | gsave 1564 | 357.12 266.112 57.6 38.016 clipbox 1565 | -1 51.245009 m 1566 | 73.832727 51.245115 l 1567 | 121.718534 51.528393 l 1568 | 151.931114 51.953311 l 1569 | 174.196746 52.519869 l 1570 | 191.869972 53.228066 l 1571 | 204.409326 53.936262 l 1572 | 215.845138 54.786098 l 1573 | 226.236888 55.777574 l 1574 | 235.698196 56.910689 l 1575 | 244.348184 58.185443 l 1576 | 252.295132 59.601836 l 1577 | 253.015604 59.460197 l 1578 | 259.632454 60.87659 l 1579 | 265.910776 62.434623 l 1580 | 271.863767 64.134295 l 1581 | 277.509838 65.975607 l 1582 | 282.869437 67.958557 l 1583 | 287.963266 70.083148 l 1584 | 292.811317 72.349377 l 1585 | 297.43239 74.757246 l 1586 | 301.84388 77.306754 l 1587 | 306.06174 79.997902 l 1588 | 310.10051 82.830689 l 1589 | 314.14952 85.946754 l 1590 | 319.594733 90.479213 l 1591 | 319.749574 90.620852 l 1592 | 319.903866 90.479213 l 1593 | 323.593677 94.020197 l 1594 | 327.256371 97.844459 l 1595 | 330.875154 101.952 l 1596 | 334.437646 106.34282 l 1597 | 337.934998 111.016918 l 1598 | 343.294597 118.665443 l 1599 | 346.672481 124.189377 l 1600 | 349.962154 129.99659 l 1601 | 353.236477 136.228721 l 1602 | 356.482021 142.88577 l 1603 | 359.688432 149.967738 l 1604 | 363.645603 159.315934 l 1605 | 367.532214 169.372328 l 1606 | 368.854692 172.913311 l 1607 | 368.904764 172.771672 l 1608 | 368.954778 172.913311 l 1609 | 372.734892 183.96118 l 1610 | 375.873312 194.017574 l 1611 | 378.999797 204.782164 l 1612 | 383.052968 219.654295 l 1613 | 387.026067 235.659541 l 1614 | 390.147619 249.540197 l 1615 | 393.261279 264.412328 l 1616 | 397.068198 283.816918 l 1617 | 398.258199 290.332328 l 1618 | 398.258199 290.332328 l 1619 | stroke 1620 | grestore 1621 | 1.000 0.498 0.055 setrgbcolor 1622 | gsave 1623 | 357.12 266.112 57.6 38.016 clipbox 1624 | -1 50.678663 m 1625 | 214.135604 50.678557 l 1626 | 215.845138 50.536918 l 1627 | 324.016177 50.536918 l 1628 | 324.295585 50.395279 l 1629 | 324.711368 50.395279 l 1630 | 324.986369 50.253639 l 1631 | 330.994735 50.253639 l 1632 | 331.232919 50.112 l 1633 | 360.302351 50.112 l 1634 | 360.302351 50.112 l 1635 | stroke 1636 | grestore 1637 | 0.173 0.627 0.173 setrgbcolor 1638 | gsave 1639 | 357.12 266.112 57.6 38.016 clipbox 1640 | -1 51.245009 m 1641 | 73.832727 51.245115 l 1642 | 104.045307 51.386754 l 1643 | 121.718534 51.528393 l 1644 | 134.257887 51.670033 l 1645 | 143.984166 51.811672 l 1646 | 151.931114 51.953311 l 1647 | 158.650163 52.094951 l 1648 | 164.470467 52.23659 l 1649 | 169.60434 52.37823 l 1650 | 174.196746 52.519869 l 1651 | 178.351082 52.661508 l 1652 | 182.143694 52.803148 l 1653 | 185.632558 52.944787 l 1654 | 188.862743 53.086426 l 1655 | 191.869972 53.228066 l 1656 | 194.683047 53.369705 l 1657 | 197.325525 53.511344 l 1658 | 199.81692 53.652984 l 1659 | 202.173577 53.794623 l 1660 | 204.409326 53.936262 l 1661 | 206.535969 54.077902 l 1662 | 208.563662 54.219541 l 1663 | 210.501205 54.36118 l 1664 | 212.356274 54.50282 l 1665 | 214.135604 54.644459 l 1666 | 215.845138 54.786098 l 1667 | 217.490146 54.927738 l 1668 | 219.075323 55.069377 l 1669 | 220.604867 55.211016 l 1670 | 222.082552 55.352656 l 1671 | 223.51178 55.494295 l 1672 | 224.895627 55.635934 l 1673 | 226.236888 55.777574 l 1674 | 227.538105 55.919213 l 1675 | 228.801601 56.060852 l 1676 | 230.0295 56.202492 l 1677 | 231.223754 56.344131 l 1678 | 232.386157 56.48577 l 1679 | 233.518365 56.62741 l 1680 | 234.621906 56.769049 l 1681 | 235.698196 56.910689 l 1682 | 236.748549 57.052328 l 1683 | 237.774185 57.193967 l 1684 | 238.776242 57.335607 l 1685 | 239.755778 57.477246 l 1686 | 240.713785 57.618885 l 1687 | 241.651187 57.760525 l 1688 | 242.568854 57.902164 l 1689 | 243.467598 58.043803 l 1690 | 244.348184 58.185443 l 1691 | 245.211332 58.327082 l 1692 | 246.057718 58.468721 l 1693 | 246.887982 58.610361 l 1694 | 247.702726 58.752 l 1695 | 248.50252 58.893639 l 1696 | 249.287903 59.035279 l 1697 | 250.059384 59.176918 l 1698 | 250.817447 59.318557 l 1699 | 251.562551 59.460197 l 1700 | 252.295132 59.601836 l 1701 | 253.015604 59.743475 l 1702 | 253.72436 59.885115 l 1703 | 254.421775 60.026754 l 1704 | 255.108207 60.168393 l 1705 | 255.783997 60.310033 l 1706 | 256.449468 60.451672 l 1707 | 257.104932 60.593311 l 1708 | 257.750686 60.734951 l 1709 | 258.387011 60.87659 l 1710 | 259.014181 61.01823 l 1711 | 259.632454 61.159869 l 1712 | 260.24208 61.301508 l 1713 | 260.843297 61.443148 l 1714 | 261.436334 61.584787 l 1715 | 262.02141 61.726426 l 1716 | 262.598737 61.868066 l 1717 | stroke 1718 | grestore 1719 | 0.839 0.153 0.157 setrgbcolor 1720 | gsave 1721 | 357.12 266.112 57.6 38.016 clipbox 1722 | -1 50.96173 m 1723 | 73.832727 50.961836 l 1724 | 121.718534 51.245115 l 1725 | 151.931114 51.670033 l 1726 | 174.196746 52.23659 l 1727 | 191.869972 52.944787 l 1728 | 204.409326 53.652984 l 1729 | 215.845138 54.50282 l 1730 | 226.236888 55.494295 l 1731 | 235.698196 56.62741 l 1732 | 244.348184 57.902164 l 1733 | 252.295132 59.318557 l 1734 | 259.014181 60.734951 l 1735 | 265.375953 62.292984 l 1736 | 271.397586 63.992656 l 1737 | 277.100562 65.833967 l 1738 | 282.507712 67.816918 l 1739 | 287.641585 69.941508 l 1740 | 292.52361 72.207738 l 1741 | 293.663218 72.774295 l 1742 | 293.943525 72.632656 l 1743 | 298.452028 75.040525 l 1744 | 302.766402 77.590033 l 1745 | 306.899987 80.28118 l 1746 | 311.054324 83.255607 l 1747 | 315.01956 86.371672 l 1748 | 318.812171 89.629377 l 1749 | 322.591621 93.170361 l 1750 | 326.202844 96.852984 l 1751 | 328.15777 98.977574 l 1752 | 328.15777 98.977574 l 1753 | stroke 1754 | grestore 1755 | 0.580 0.404 0.741 setrgbcolor 1756 | gsave 1757 | 357.12 266.112 57.6 38.016 clipbox 1758 | -1 50.96173 m 1759 | 73.832727 50.961836 l 1760 | 121.718534 51.245115 l 1761 | 151.931114 51.670033 l 1762 | 174.196746 52.23659 l 1763 | 191.869972 52.944787 l 1764 | 204.409326 53.652984 l 1765 | 215.845138 54.50282 l 1766 | 226.236888 55.494295 l 1767 | 235.698196 56.62741 l 1768 | 244.348184 57.902164 l 1769 | 252.295132 59.318557 l 1770 | 259.014181 60.734951 l 1771 | 265.375953 62.292984 l 1772 | 271.397586 63.992656 l 1773 | 277.100562 65.833967 l 1774 | 282.507712 67.816918 l 1775 | 287.641585 69.941508 l 1776 | 292.52361 72.207738 l 1777 | 297.173709 74.615607 l 1778 | 303.669803 78.29823 l 1779 | 307.925624 81.131016 l 1780 | 311.987711 84.105443 l 1781 | 315.872572 87.221508 l 1782 | 319.749574 90.620852 l 1783 | 323.451929 94.161836 l 1784 | 327.126064 97.986098 l 1785 | 331.11399 102.376918 l 1786 | 331.11399 102.376918 l 1787 | stroke 1788 | grestore 1789 | 0.549 0.337 0.294 setrgbcolor 1790 | gsave 1791 | 357.12 266.112 57.6 38.016 clipbox 1792 | -1 51.245009 m 1793 | 73.832727 51.245115 l 1794 | 104.045307 51.386754 l 1795 | 121.718534 51.528393 l 1796 | 134.257887 51.670033 l 1797 | 143.984166 51.811672 l 1798 | 151.931114 51.953311 l 1799 | 158.650163 52.094951 l 1800 | 164.470467 52.23659 l 1801 | 169.60434 52.37823 l 1802 | 174.196746 52.519869 l 1803 | 178.351082 52.661508 l 1804 | 182.143694 52.803148 l 1805 | 185.632558 52.944787 l 1806 | 188.862743 53.086426 l 1807 | 191.869972 53.228066 l 1808 | 194.683047 53.369705 l 1809 | 197.325525 53.511344 l 1810 | 199.81692 53.652984 l 1811 | 202.173577 53.794623 l 1812 | 204.409326 53.936262 l 1813 | 206.535969 54.077902 l 1814 | 208.563662 54.219541 l 1815 | 210.501205 54.36118 l 1816 | 212.356274 54.50282 l 1817 | 214.135604 54.644459 l 1818 | 215.845138 54.786098 l 1819 | 217.490146 54.927738 l 1820 | 219.075323 55.069377 l 1821 | 220.604867 55.211016 l 1822 | 222.082552 55.352656 l 1823 | 223.51178 55.494295 l 1824 | 224.895627 55.635934 l 1825 | 226.236888 55.777574 l 1826 | 227.538105 55.919213 l 1827 | 228.801601 56.060852 l 1828 | 230.0295 56.202492 l 1829 | 231.223754 56.344131 l 1830 | 232.386157 56.48577 l 1831 | 233.518365 56.62741 l 1832 | 234.621906 56.769049 l 1833 | 235.698196 56.910689 l 1834 | 236.748549 57.052328 l 1835 | 237.774185 57.193967 l 1836 | 238.776242 57.335607 l 1837 | 239.755778 57.477246 l 1838 | 240.713785 57.618885 l 1839 | 241.651187 57.760525 l 1840 | 242.568854 57.902164 l 1841 | 243.467598 58.043803 l 1842 | 244.348184 58.185443 l 1843 | 245.211332 58.327082 l 1844 | 246.057718 58.468721 l 1845 | 246.887982 58.610361 l 1846 | 247.702726 58.752 l 1847 | 248.50252 58.893639 l 1848 | 249.287903 59.035279 l 1849 | 250.059384 59.176918 l 1850 | 250.817447 59.318557 l 1851 | 251.562551 59.460197 l 1852 | stroke 1853 | grestore 1854 | 0.890 0.467 0.761 setrgbcolor 1855 | gsave 1856 | 357.12 266.112 57.6 38.016 clipbox 1857 | -1 50.96173 m 1858 | 73.832727 50.961836 l 1859 | 121.718534 51.245115 l 1860 | 151.931114 51.670033 l 1861 | 174.196746 52.23659 l 1862 | 191.869972 52.944787 l 1863 | 204.409326 53.652984 l 1864 | 215.845138 54.50282 l 1865 | 226.236888 55.494295 l 1866 | 235.698196 56.62741 l 1867 | 244.348184 57.902164 l 1868 | 252.295132 59.318557 l 1869 | 259.014181 60.734951 l 1870 | 265.375953 62.292984 l 1871 | 271.397586 63.992656 l 1872 | 277.100562 65.833967 l 1873 | 282.507712 67.816918 l 1874 | 287.641585 69.941508 l 1875 | 292.52361 72.207738 l 1876 | 297.173709 74.615607 l 1877 | 301.610166 77.165115 l 1878 | 305.849634 79.856262 l 1879 | 309.907217 82.689049 l 1880 | 313.973408 85.805115 l 1881 | 317.854165 89.06282 l 1882 | 321.565984 92.462164 l 1883 | 325.259646 96.144787 l 1884 | 328.915833 100.110689 l 1885 | 329.783893 101.102164 l 1886 | 329.783893 101.102164 l 1887 | stroke 1888 | grestore 1889 | 0.498 setgray 1890 | gsave 1891 | 357.12 266.112 57.6 38.016 clipbox 1892 | -1 50.96173 m 1893 | 73.832727 50.961836 l 1894 | 121.718534 51.245115 l 1895 | 151.931114 51.670033 l 1896 | 174.196746 52.23659 l 1897 | 191.869972 52.944787 l 1898 | 204.409326 53.652984 l 1899 | 215.845138 54.50282 l 1900 | 226.236888 55.494295 l 1901 | 235.698196 56.62741 l 1902 | 244.348184 57.902164 l 1903 | 252.295132 59.318557 l 1904 | 253.015604 59.460197 l 1905 | 253.72436 59.318557 l 1906 | 260.843297 60.87659 l 1907 | 266.961129 62.434623 l 1908 | 272.781434 64.134295 l 1909 | 275.423912 64.984131 l 1910 | 275.84916 64.842492 l 1911 | 281.404171 66.825443 l 1912 | 286.662048 68.950033 l 1913 | 291.648914 71.216262 l 1914 | 296.388327 73.624131 l 1915 | 300.90141 76.173639 l 1916 | 305.207055 78.864787 l 1917 | 306.482878 79.714623 l 1918 | 306.691932 79.572984 l 1919 | 310.865223 82.54741 l 1920 | 314.499622 85.380197 l 1921 | 314.673625 85.238557 l 1922 | 318.49517 88.496262 l 1923 | 323.734966 93.312 l 1924 | 327.386289 97.136262 l 1925 | 328.53845 98.411016 l 1926 | 328.664608 98.269377 l 1927 | 330.635001 100.535607 l 1928 | 330.635001 100.535607 l 1929 | stroke 1930 | grestore 1931 | 0.737 0.741 0.133 setrgbcolor 1932 | gsave 1933 | 357.12 266.112 57.6 38.016 clipbox 1934 | -1 50.96173 m 1935 | 73.832727 50.961836 l 1936 | 121.718534 51.245115 l 1937 | 151.931114 51.670033 l 1938 | 174.196746 52.23659 l 1939 | 191.869972 52.944787 l 1940 | 204.409326 53.652984 l 1941 | 215.845138 54.50282 l 1942 | 226.236888 55.494295 l 1943 | 235.698196 56.62741 l 1944 | 244.348184 57.902164 l 1945 | 252.295132 59.318557 l 1946 | 259.014181 60.734951 l 1947 | 262.02141 61.443148 l 1948 | 262.598737 61.301508 l 1949 | 268.988822 63.00118 l 1950 | 274.994475 64.842492 l 1951 | 280.652643 66.825443 l 1952 | 285.659992 68.808393 l 1953 | 290.45466 70.932984 l 1954 | 295.047066 73.199213 l 1955 | 299.448358 75.607082 l 1956 | 305.636492 79.289705 l 1957 | 309.713063 82.122492 l 1958 | 313.796582 85.238557 l 1959 | 317.69243 88.496262 l 1960 | 321.417475 91.895607 l 1961 | 325.123221 95.57823 l 1962 | 328.790401 99.544131 l 1963 | 329.537635 100.393967 l 1964 | 329.537635 100.393967 l 1965 | stroke 1966 | grestore 1967 | 0.090 0.745 0.812 setrgbcolor 1968 | gsave 1969 | 357.12 266.112 57.6 38.016 clipbox 1970 | -1 50.96173 m 1971 | 73.832727 50.961836 l 1972 | 121.718534 51.245115 l 1973 | 151.931114 51.670033 l 1974 | 174.196746 52.23659 l 1975 | 191.869972 52.944787 l 1976 | 204.409326 53.652984 l 1977 | 215.845138 54.50282 l 1978 | 226.236888 55.494295 l 1979 | 235.698196 56.62741 l 1980 | 244.348184 57.902164 l 1981 | 252.295132 59.318557 l 1982 | 259.014181 60.734951 l 1983 | 265.375953 62.292984 l 1984 | 271.397586 63.992656 l 1985 | 277.100562 65.833967 l 1986 | 282.507712 67.816918 l 1987 | 287.641585 69.941508 l 1988 | 292.52361 72.207738 l 1989 | 297.173709 74.615607 l 1990 | 301.610166 77.165115 l 1991 | 305.849634 79.856262 l 1992 | 309.907217 82.689049 l 1993 | 313.973408 85.805115 l 1994 | 317.854165 89.06282 l 1995 | 321.565984 92.462164 l 1996 | 325.259646 96.144787 l 1997 | 328.915833 100.110689 l 1998 | 330.272274 101.668721 l 1999 | 330.272274 101.668721 l 2000 | stroke 2001 | grestore 2002 | 0.122 0.467 0.706 setrgbcolor 2003 | gsave 2004 | 357.12 266.112 57.6 38.016 clipbox 2005 | -1 50.96173 m 2006 | 73.832727 50.961836 l 2007 | 121.718534 51.245115 l 2008 | 151.931114 51.670033 l 2009 | 174.196746 52.23659 l 2010 | 191.869972 52.944787 l 2011 | 204.409326 53.652984 l 2012 | 215.845138 54.50282 l 2013 | 226.236888 55.494295 l 2014 | 235.698196 56.62741 l 2015 | 244.348184 57.902164 l 2016 | 252.295132 59.318557 l 2017 | 259.014181 60.734951 l 2018 | 265.375953 62.292984 l 2019 | 271.397586 63.992656 l 2020 | 277.100562 65.833967 l 2021 | 282.507712 67.816918 l 2022 | 287.641585 69.941508 l 2023 | 292.52361 72.207738 l 2024 | 297.173709 74.615607 l 2025 | 301.610166 77.165115 l 2026 | 305.849634 79.856262 l 2027 | 309.907217 82.689049 l 2028 | 313.973408 85.805115 l 2029 | 315.191503 86.79659 l 2030 | 315.36277 86.654951 l 2031 | 317.367148 88.354623 l 2032 | 317.530092 88.212984 l 2033 | 321.268457 91.612328 l 2034 | 324.986369 95.294951 l 2035 | 328.664608 99.260852 l 2036 | 332.172849 103.368393 l 2037 | 332.864729 104.21823 l 2038 | 332.864729 104.21823 l 2039 | stroke 2040 | grestore 2041 | 1.000 0.498 0.055 setrgbcolor 2042 | gsave 2043 | 357.12 266.112 57.6 38.016 clipbox 2044 | -1 51.245009 m 2045 | 73.832727 51.245115 l 2046 | 104.045307 51.386754 l 2047 | 121.718534 51.528393 l 2048 | 134.257887 51.670033 l 2049 | 143.984166 51.811672 l 2050 | 151.931114 51.953311 l 2051 | 158.650163 52.094951 l 2052 | 164.470467 52.23659 l 2053 | 169.60434 52.37823 l 2054 | 174.196746 52.519869 l 2055 | 178.351082 52.661508 l 2056 | 182.143694 52.803148 l 2057 | 185.632558 52.944787 l 2058 | 188.862743 53.086426 l 2059 | 191.869972 53.228066 l 2060 | 194.683047 53.369705 l 2061 | 197.325525 53.511344 l 2062 | 199.81692 53.652984 l 2063 | 202.173577 53.794623 l 2064 | 204.409326 53.936262 l 2065 | 206.535969 54.077902 l 2066 | 208.563662 54.219541 l 2067 | 210.501205 54.36118 l 2068 | 212.356274 54.50282 l 2069 | 214.135604 54.644459 l 2070 | 215.845138 54.786098 l 2071 | 217.490146 54.927738 l 2072 | 219.075323 55.069377 l 2073 | 220.604867 55.211016 l 2074 | 222.082552 55.352656 l 2075 | 223.51178 55.494295 l 2076 | 224.895627 55.635934 l 2077 | 226.236888 55.777574 l 2078 | 227.538105 55.919213 l 2079 | 228.801601 56.060852 l 2080 | 230.0295 56.202492 l 2081 | 231.223754 56.344131 l 2082 | 232.386157 56.48577 l 2083 | 233.518365 56.62741 l 2084 | 234.621906 56.769049 l 2085 | 235.698196 56.910689 l 2086 | 236.748549 57.052328 l 2087 | 237.774185 57.193967 l 2088 | 238.776242 57.335607 l 2089 | 239.755778 57.477246 l 2090 | 240.713785 57.618885 l 2091 | 241.651187 57.760525 l 2092 | 242.568854 57.902164 l 2093 | 243.467598 58.043803 l 2094 | 244.348184 58.185443 l 2095 | 245.211332 58.327082 l 2096 | 246.057718 58.468721 l 2097 | 246.887982 58.610361 l 2098 | 247.702726 58.752 l 2099 | 248.50252 58.893639 l 2100 | 249.287903 59.035279 l 2101 | 250.059384 59.176918 l 2102 | 250.817447 59.318557 l 2103 | 251.562551 59.460197 l 2104 | 252.295132 59.601836 l 2105 | 253.015604 59.743475 l 2106 | 253.72436 59.885115 l 2107 | 254.421775 60.026754 l 2108 | 255.108207 60.168393 l 2109 | 255.783997 60.310033 l 2110 | 256.449468 60.451672 l 2111 | 257.104932 60.593311 l 2112 | 257.750686 60.734951 l 2113 | 258.387011 60.87659 l 2114 | 259.014181 61.01823 l 2115 | 259.632454 61.159869 l 2116 | 260.24208 61.301508 l 2117 | 260.843297 61.443148 l 2118 | 261.436334 61.584787 l 2119 | 262.02141 61.726426 l 2120 | 262.598737 61.868066 l 2121 | stroke 2122 | grestore 2123 | 0.173 0.627 0.173 setrgbcolor 2124 | gsave 2125 | 357.12 266.112 57.6 38.016 clipbox 2126 | -1 50.96173 m 2127 | 73.832727 50.961836 l 2128 | 121.718534 51.245115 l 2129 | 151.931114 51.670033 l 2130 | 174.196746 52.23659 l 2131 | 191.869972 52.944787 l 2132 | 204.409326 53.652984 l 2133 | 215.845138 54.50282 l 2134 | 226.236888 55.494295 l 2135 | 235.698196 56.62741 l 2136 | 244.348184 57.902164 l 2137 | 252.295132 59.318557 l 2138 | 259.014181 60.734951 l 2139 | 264.834486 62.151344 l 2140 | 265.375953 62.009705 l 2141 | 271.397586 63.709377 l 2142 | 277.100562 65.550689 l 2143 | 281.030027 66.825443 l 2144 | 281.404171 66.683803 l 2145 | 286.662048 68.808393 l 2146 | 291.648914 71.074623 l 2147 | 292.52361 71.499541 l 2148 | 292.811317 71.357902 l 2149 | 297.43239 73.76577 l 2150 | 301.84388 76.315279 l 2151 | 306.06174 79.006426 l 2152 | 310.10051 81.839213 l 2153 | 314.14952 84.955279 l 2154 | 318.015302 88.212984 l 2155 | 321.861494 91.753967 l 2156 | 324.986369 94.870033 l 2157 | 324.986369 94.870033 l 2158 | stroke 2159 | grestore 2160 | 0.839 0.153 0.157 setrgbcolor 2161 | gsave 2162 | 357.12 266.112 57.6 38.016 clipbox 2163 | -1 51.245009 m 2164 | 73.832727 51.245115 l 2165 | 121.718534 51.528393 l 2166 | 151.931114 51.953311 l 2167 | 174.196746 52.519869 l 2168 | 191.869972 53.228066 l 2169 | 204.409326 53.936262 l 2170 | 215.845138 54.786098 l 2171 | 226.236888 55.777574 l 2172 | 235.698196 56.910689 l 2173 | 244.348184 58.185443 l 2174 | 252.295132 59.601836 l 2175 | 259.014181 61.01823 l 2176 | 265.375953 62.576262 l 2177 | 271.397586 64.275934 l 2178 | 277.100562 66.117246 l 2179 | 282.507712 68.100197 l 2180 | 287.641585 70.224787 l 2181 | 290.45466 71.499541 l 2182 | 290.45466 71.499541 l 2183 | stroke 2184 | grestore 2185 | 0.580 0.404 0.741 setrgbcolor 2186 | gsave 2187 | 357.12 266.112 57.6 38.016 clipbox 2188 | -1 50.96173 m 2189 | 73.832727 50.961836 l 2190 | 121.718534 51.245115 l 2191 | 151.931114 51.670033 l 2192 | 174.196746 52.23659 l 2193 | 191.869972 52.944787 l 2194 | 204.409326 53.652984 l 2195 | 215.845138 54.50282 l 2196 | 226.236888 55.494295 l 2197 | 235.698196 56.62741 l 2198 | 244.348184 57.902164 l 2199 | 252.295132 59.318557 l 2200 | 259.014181 60.734951 l 2201 | 261.436334 61.301508 l 2202 | 262.02141 61.301508 l 2203 | 268.490673 63.00118 l 2204 | 274.560764 64.842492 l 2205 | 279.88793 66.683803 l 2206 | 284.978923 68.666754 l 2207 | 289.845034 70.791344 l 2208 | 290.45466 71.074623 l 2209 | 290.756305 71.074623 l 2210 | 295.31864 73.340852 l 2211 | 299.93811 75.890361 l 2212 | 304.335275 78.581508 l 2213 | 308.529618 81.414295 l 2214 | 312.538298 84.388721 l 2215 | 316.37648 87.504787 l 2216 | 321.71399 92.178885 l 2217 | 325.395644 95.861508 l 2218 | 327.256371 97.844459 l 2219 | 327.256371 97.844459 l 2220 | stroke 2221 | grestore 2222 | 0.549 0.337 0.294 setrgbcolor 2223 | gsave 2224 | 357.12 266.112 57.6 38.016 clipbox 2225 | -1 50.96173 m 2226 | 73.832727 50.961836 l 2227 | 121.718534 51.245115 l 2228 | 151.931114 51.670033 l 2229 | 174.196746 52.23659 l 2230 | 191.869972 52.944787 l 2231 | 204.409326 53.652984 l 2232 | 222.082552 54.927738 l 2233 | 231.223754 55.919213 l 2234 | 233.518365 56.202492 l 2235 | 234.621906 56.202492 l 2236 | 243.467598 57.477246 l 2237 | 250.817447 58.752 l 2238 | 257.750686 60.168393 l 2239 | 264.286208 61.726426 l 2240 | 270.449994 63.426098 l 2241 | 276.270298 65.26741 l 2242 | 281.775131 67.250361 l 2243 | 286.991012 69.374951 l 2244 | 291.942434 71.64118 l 2245 | 296.651696 74.049049 l 2246 | 297.43239 74.473967 l 2247 | 297.689544 74.473967 l 2248 | 302.076348 77.023475 l 2249 | 306.272818 79.714623 l 2250 | 310.29295 82.54741 l 2251 | 314.324923 85.663475 l 2252 | 318.175846 88.92118 l 2253 | 322.008501 92.462164 l 2254 | 325.666375 96.144787 l 2255 | 327.64497 98.269377 l 2256 | 327.64497 98.269377 l 2257 | stroke 2258 | grestore 2259 | 0.890 0.467 0.761 setrgbcolor 2260 | gsave 2261 | 357.12 266.112 57.6 38.016 clipbox 2262 | -1 50.96173 m 2263 | 73.832727 50.961836 l 2264 | 121.718534 51.245115 l 2265 | 151.931114 51.670033 l 2266 | 174.196746 52.23659 l 2267 | 191.869972 52.944787 l 2268 | 204.409326 53.652984 l 2269 | 215.845138 54.50282 l 2270 | 226.236888 55.494295 l 2271 | 235.698196 56.62741 l 2272 | 244.348184 57.902164 l 2273 | 252.295132 59.318557 l 2274 | 259.014181 60.734951 l 2275 | 265.375953 62.292984 l 2276 | 271.397586 63.992656 l 2277 | 277.100562 65.833967 l 2278 | 282.507712 67.816918 l 2279 | 287.641585 69.941508 l 2280 | 292.52361 72.207738 l 2281 | 297.173709 74.615607 l 2282 | 301.610166 77.165115 l 2283 | 305.849634 79.856262 l 2284 | 309.907217 82.689049 l 2285 | 313.973408 85.805115 l 2286 | 317.854165 89.06282 l 2287 | 321.565984 92.462164 l 2288 | 325.259646 96.144787 l 2289 | 328.915833 100.110689 l 2290 | 328.915833 100.110689 l 2291 | stroke 2292 | grestore 2293 | 0.498 setgray 2294 | gsave 2295 | 357.12 266.112 57.6 38.016 clipbox 2296 | -1 50.96173 m 2297 | 73.832727 50.961836 l 2298 | 121.718534 51.245115 l 2299 | 134.257887 51.103475 l 2300 | 158.650163 51.528393 l 2301 | 178.351082 52.094951 l 2302 | 194.683047 52.803148 l 2303 | 208.563662 53.652984 l 2304 | 220.604867 54.644459 l 2305 | 230.0295 55.635934 l 2306 | 238.776242 56.769049 l 2307 | 246.887982 58.043803 l 2308 | 254.421775 59.460197 l 2309 | 255.783997 59.743475 l 2310 | 256.449468 59.601836 l 2311 | 263.168517 61.159869 l 2312 | 269.481342 62.859541 l 2313 | 275.423912 64.700852 l 2314 | 281.030027 66.683803 l 2315 | 286.330583 68.808393 l 2316 | 291.353404 71.074623 l 2317 | 295.588533 73.199213 l 2318 | 295.856765 73.057574 l 2319 | 300.422421 75.607082 l 2320 | 304.773344 78.29823 l 2321 | 308.92768 81.131016 l 2322 | 312.90153 84.105443 l 2323 | 316.874628 87.363148 l 2324 | 320.66724 90.762492 l 2325 | 324.43462 94.445115 l 2326 | 328.030134 98.269377 l 2327 | 330.028767 100.535607 l 2328 | 330.028767 100.535607 l 2329 | stroke 2330 | grestore 2331 | 0.737 0.741 0.133 setrgbcolor 2332 | gsave 2333 | 357.12 266.112 57.6 38.016 clipbox 2334 | -1 50.96173 m 2335 | 73.832727 50.961836 l 2336 | 121.718534 51.245115 l 2337 | 151.931114 51.670033 l 2338 | 174.196746 52.23659 l 2339 | 191.869972 52.944787 l 2340 | 204.409326 53.652984 l 2341 | 215.845138 54.50282 l 2342 | 226.236888 55.494295 l 2343 | 235.698196 56.62741 l 2344 | 244.348184 57.902164 l 2345 | 252.295132 59.318557 l 2346 | 259.014181 60.734951 l 2347 | 265.375953 62.292984 l 2348 | 271.397586 63.992656 l 2349 | 277.100562 65.833967 l 2350 | 282.507712 67.816918 l 2351 | 287.641585 69.941508 l 2352 | 292.52361 72.207738 l 2353 | 297.173709 74.615607 l 2354 | 301.610166 77.165115 l 2355 | 305.849634 79.856262 l 2356 | 309.907217 82.689049 l 2357 | 313.973408 85.805115 l 2358 | 317.854165 89.06282 l 2359 | 320.515631 91.470689 l 2360 | 320.66724 91.329049 l 2361 | 324.43462 95.011672 l 2362 | 328.030134 98.835934 l 2363 | 331.587772 102.943475 l 2364 | 334.216389 106.20118 l 2365 | 334.216389 106.20118 l 2366 | stroke 2367 | grestore 2368 | 0.090 0.745 0.812 setrgbcolor 2369 | gsave 2370 | 357.12 266.112 57.6 38.016 clipbox 2371 | -1 51.245009 m 2372 | 73.832727 51.245115 l 2373 | 104.045307 51.386754 l 2374 | 121.718534 51.528393 l 2375 | 134.257887 51.670033 l 2376 | 143.984166 51.811672 l 2377 | 151.931114 51.953311 l 2378 | 158.650163 52.094951 l 2379 | 164.470467 52.23659 l 2380 | 169.60434 52.37823 l 2381 | 174.196746 52.519869 l 2382 | 178.351082 52.661508 l 2383 | 182.143694 52.803148 l 2384 | 185.632558 52.944787 l 2385 | 188.862743 53.086426 l 2386 | 191.869972 53.228066 l 2387 | 194.683047 53.369705 l 2388 | 197.325525 53.511344 l 2389 | 199.81692 53.652984 l 2390 | 202.173577 53.794623 l 2391 | 204.409326 53.936262 l 2392 | 206.535969 54.077902 l 2393 | 208.563662 54.219541 l 2394 | 210.501205 54.36118 l 2395 | 212.356274 54.50282 l 2396 | 214.135604 54.644459 l 2397 | 215.845138 54.786098 l 2398 | 217.490146 54.927738 l 2399 | 219.075323 55.069377 l 2400 | 220.604867 55.211016 l 2401 | 222.082552 55.352656 l 2402 | 223.51178 55.494295 l 2403 | 224.895627 55.635934 l 2404 | 226.236888 55.777574 l 2405 | 227.538105 55.919213 l 2406 | 228.801601 56.060852 l 2407 | 230.0295 56.202492 l 2408 | 231.223754 56.344131 l 2409 | 232.386157 56.48577 l 2410 | 233.518365 56.62741 l 2411 | 234.621906 56.769049 l 2412 | 235.698196 56.910689 l 2413 | 236.748549 57.052328 l 2414 | 237.774185 57.193967 l 2415 | 238.776242 57.335607 l 2416 | 239.755778 57.477246 l 2417 | 240.713785 57.618885 l 2418 | 241.651187 57.760525 l 2419 | 242.568854 57.902164 l 2420 | 243.467598 58.043803 l 2421 | 244.348184 58.185443 l 2422 | 245.211332 58.327082 l 2423 | 246.057718 58.468721 l 2424 | 246.887982 58.610361 l 2425 | 247.702726 58.752 l 2426 | 248.50252 58.893639 l 2427 | 249.287903 59.035279 l 2428 | 250.059384 59.176918 l 2429 | 250.817447 59.318557 l 2430 | 251.562551 59.460197 l 2431 | 252.295132 59.601836 l 2432 | 253.015604 59.743475 l 2433 | 253.72436 59.885115 l 2434 | 254.421775 60.026754 l 2435 | 255.108207 60.168393 l 2436 | 255.783997 60.310033 l 2437 | 256.449468 60.451672 l 2438 | 257.104932 60.593311 l 2439 | 257.750686 60.734951 l 2440 | 258.387011 60.87659 l 2441 | 259.014181 61.01823 l 2442 | 259.632454 61.159869 l 2443 | 260.24208 61.301508 l 2444 | 260.843297 61.443148 l 2445 | 261.436334 61.584787 l 2446 | 262.02141 61.726426 l 2447 | 262.598737 61.868066 l 2448 | 263.168517 62.009705 l 2449 | 263.730945 62.151344 l 2450 | 264.286208 62.292984 l 2451 | 264.834486 62.434623 l 2452 | 265.375953 62.576262 l 2453 | 265.910776 62.717902 l 2454 | stroke 2455 | grestore 2456 | 0.800 setlinewidth 2457 | 0 setlinejoin 2458 | 0.000 setgray 2459 | gsave 2460 | 57.6 38.016 m 2461 | 57.6 304.128 l 2462 | stroke 2463 | grestore 2464 | gsave 2465 | 414.72 38.016 m 2466 | 414.72 304.128 l 2467 | stroke 2468 | grestore 2469 | gsave 2470 | 57.6 38.016 m 2471 | 414.72 38.016 l 2472 | stroke 2473 | grestore 2474 | gsave 2475 | 57.6 304.128 m 2476 | 414.72 304.128 l 2477 | stroke 2478 | grestore 2479 | 2480 | end 2481 | showpage 2482 | --------------------------------------------------------------------------------