From 9d40e95982fa02935aa00a26d68abc3414803b60 Mon Sep 17 00:00:00 2001 From: Marek Wolan Date: Fri, 2 Aug 2024 13:48:12 +0100 Subject: [PATCH 01/95] precommit json end of file fixes --- benchmark/results/v3/v3.2.0/session_metadata/1.json | 2 +- benchmark/results/v3/v3.2.0/session_metadata/2.json | 2 +- benchmark/results/v3/v3.2.0/session_metadata/3.json | 2 +- benchmark/results/v3/v3.2.0/session_metadata/4.json | 2 +- benchmark/results/v3/v3.2.0/session_metadata/5.json | 2 +- benchmark/results/v3/v3.2.0/v3.2.0_benchmark_metadata.json | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/benchmark/results/v3/v3.2.0/session_metadata/1.json b/benchmark/results/v3/v3.2.0/session_metadata/1.json index 794f03e3..bfccfcdc 100644 --- a/benchmark/results/v3/v3.2.0/session_metadata/1.json +++ b/benchmark/results/v3/v3.2.0/session_metadata/1.json @@ -1006,4 +1006,4 @@ "999": 78.49999999999996, "1000": 84.69999999999993 } -} \ No newline at end of file +} diff --git a/benchmark/results/v3/v3.2.0/session_metadata/2.json b/benchmark/results/v3/v3.2.0/session_metadata/2.json index e48c34b9..c35b5ae6 100644 --- a/benchmark/results/v3/v3.2.0/session_metadata/2.json +++ b/benchmark/results/v3/v3.2.0/session_metadata/2.json @@ -1006,4 +1006,4 @@ "999": 97.59999999999975, "1000": 103.34999999999978 } -} \ No newline at end of file +} diff --git a/benchmark/results/v3/v3.2.0/session_metadata/3.json b/benchmark/results/v3/v3.2.0/session_metadata/3.json index 4e2d845c..342e0f7d 100644 --- a/benchmark/results/v3/v3.2.0/session_metadata/3.json +++ b/benchmark/results/v3/v3.2.0/session_metadata/3.json @@ -1006,4 +1006,4 @@ "999": 101.14999999999978, "1000": 80.94999999999976 } -} \ No newline at end of file +} diff --git a/benchmark/results/v3/v3.2.0/session_metadata/4.json b/benchmark/results/v3/v3.2.0/session_metadata/4.json index 6e03a18f..6aaf9ab8 100644 --- a/benchmark/results/v3/v3.2.0/session_metadata/4.json +++ b/benchmark/results/v3/v3.2.0/session_metadata/4.json @@ -1006,4 +1006,4 @@ "999": 118.0500000000001, "1000": 77.95000000000005 } -} \ No newline at end of file +} diff --git a/benchmark/results/v3/v3.2.0/session_metadata/5.json b/benchmark/results/v3/v3.2.0/session_metadata/5.json index ca7ad1e9..05cf76ed 100644 --- a/benchmark/results/v3/v3.2.0/session_metadata/5.json +++ b/benchmark/results/v3/v3.2.0/session_metadata/5.json @@ -1006,4 +1006,4 @@ "999": 55.849999999999916, "1000": 96.95000000000007 } -} \ No newline at end of file +} diff --git a/benchmark/results/v3/v3.2.0/v3.2.0_benchmark_metadata.json b/benchmark/results/v3/v3.2.0/v3.2.0_benchmark_metadata.json index 830e980e..111ae25f 100644 --- a/benchmark/results/v3/v3.2.0/v3.2.0_benchmark_metadata.json +++ b/benchmark/results/v3/v3.2.0/v3.2.0_benchmark_metadata.json @@ -7442,4 +7442,4 @@ } } } -} \ No newline at end of file +} From 0ff88e36726ff7e652047ec6e3a78dc46576c6d3 Mon Sep 17 00:00:00 2001 From: Archer Bowen Date: Mon, 2 Sep 2024 11:50:49 +0100 Subject: [PATCH 02/95] #2840 Initial Implementation completed and tested. --- src/primaite/game/agent/actions.py | 23 +++++++++ .../system/services/terminal/terminal.py | 22 +++++++- tests/conftest.py | 1 + .../actions/test_terminal_actions.py | 51 +++++++++++++++++++ 4 files changed, 96 insertions(+), 1 deletion(-) diff --git a/src/primaite/game/agent/actions.py b/src/primaite/game/agent/actions.py index 2e6189c0..3dc1f514 100644 --- a/src/primaite/game/agent/actions.py +++ b/src/primaite/game/agent/actions.py @@ -1266,6 +1266,28 @@ class NodeSendRemoteCommandAction(AbstractAction): ] +class NodeSendLocalCommandAction(AbstractAction): + """Action which sends a terminal command using a local terminal session.""" + + def __init__(self, manager: "ActionManager", **kwargs) -> None: + super().__init__(manager=manager) + + def form_request(self, node_id: int, username: str, password: str, command: RequestFormat) -> RequestFormat: + """Return the action formatted as a request which can be ingested by the PrimAITE simulation.""" + node_name = self.manager.get_node_name_by_idx(node_id) + return [ + "network", + "node", + node_name, + "service", + "Terminal", + "send_local_command", + username, + password, + {"command": command}, + ] + + class TerminalC2ServerAction(AbstractAction): """Action which causes the C2 Server to send a command to the C2 Beacon to execute the terminal command passed.""" @@ -1372,6 +1394,7 @@ class ActionManager: "SSH_TO_REMOTE": NodeSessionsRemoteLoginAction, "SESSIONS_REMOTE_LOGOFF": NodeSessionsRemoteLogoutAction, "NODE_SEND_REMOTE_COMMAND": NodeSendRemoteCommandAction, + "NODE_SEND_LOCAL_COMMAND": NodeSendLocalCommandAction, } """Dictionary which maps action type strings to the corresponding action class.""" diff --git a/src/primaite/simulator/system/services/terminal/terminal.py b/src/primaite/simulator/system/services/terminal/terminal.py index e98e8555..9b88bbe8 100644 --- a/src/primaite/simulator/system/services/terminal/terminal.py +++ b/src/primaite/simulator/system/services/terminal/terminal.py @@ -208,7 +208,6 @@ class Terminal(Service): status="success", data={}, ) - else: return RequestResponse( status="failure", data={}, @@ -219,6 +218,27 @@ class Terminal(Service): request_type=RequestType(func=remote_execute_request), ) + def local_execute_request(request: RequestFormat, context: Dict) -> RequestResponse: + """Executes a command using a local terminal session.""" + command: str = request[2]["command"] + local_connection = self._process_local_login(username=request[0], password=request[1]) + if local_connection: + outcome = local_connection.execute(command) + if outcome: + return RequestResponse( + status="success", + data={"reason": outcome}, + ) + return RequestResponse( + status="success", + data={"reason": "Local Terminal failed to resolve command. Potentially invalid credentials?"}, + ) + + rm.add_request( + "send_local_command", + request_type=RequestType(func=local_execute_request), + ) + return rm def execute(self, command: List[Any]) -> Optional[RequestResponse]: diff --git a/tests/conftest.py b/tests/conftest.py index 1bbff8f2..8717abfa 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -467,6 +467,7 @@ def game_and_agent(): {"type": "SSH_TO_REMOTE"}, {"type": "SESSIONS_REMOTE_LOGOFF"}, {"type": "NODE_SEND_REMOTE_COMMAND"}, + {"type": "NODE_SEND_LOCAL_COMMAND"}, ] action_space = ActionManager( diff --git a/tests/integration_tests/game_layer/actions/test_terminal_actions.py b/tests/integration_tests/game_layer/actions/test_terminal_actions.py index d011c1e8..d2ea7202 100644 --- a/tests/integration_tests/game_layer/actions/test_terminal_actions.py +++ b/tests/integration_tests/game_layer/actions/test_terminal_actions.py @@ -164,3 +164,54 @@ def test_change_password_logs_out_user(game_and_agent_fixture: Tuple[PrimaiteGam assert server_1.file_system.get_folder("folder123") is None assert server_1.file_system.get_file("folder123", "doggo.pdf") is None + + +def test_local_terminal(game_and_agent_fixture: Tuple[PrimaiteGame, ProxyAgent]): + game, agent = game_and_agent_fixture + + client_1 = game.simulation.network.get_node_by_hostname("client_1") + # create a new user account on server_1 that will be logged into remotely + client_1_usm: UserManager = client_1.software_manager.software["UserManager"] + client_1_usm.add_user("user123", "password", is_admin=True) + + action = ( + "NODE_SEND_LOCAL_COMMAND", + { + "node_id": 0, + "username": "user123", + "password": "password", + "command": ["file_system", "create", "file", "folder123", "doggo.pdf", False], + }, + ) + agent.store_action(action) + game.step() + + assert client_1.file_system.get_folder("folder123") + assert client_1.file_system.get_file("folder123", "doggo.pdf") + + # Change password + action = ( + "NODE_ACCOUNTS_CHANGE_PASSWORD", + { + "node_id": 0, # server_1 + "username": "user123", + "current_password": "password", + "new_password": "different_password", + }, + ) + agent.store_action(action) + game.step() + + action = ( + "NODE_SEND_LOCAL_COMMAND", + { + "node_id": 0, + "username": "user123", + "password": "password", + "command": ["file_system", "create", "file", "folder123", "cat.pdf", False], + }, + ) + agent.store_action(action) + game.step() + + assert client_1.file_system.get_file("folder123", "cat.pdf") is None From a7f00c668dc75932f6cb72de9f8709ce672b58f2 Mon Sep 17 00:00:00 2001 From: Czar Echavez Date: Mon, 2 Sep 2024 15:15:45 +0100 Subject: [PATCH 03/95] #2782: initial impl of files in nodes --- src/primaite/game/game.py | 6 + .../configs/nodes_with_initial_files.yaml | 256 ++++++++++++++++++ .../test_node_file_system_config.py | 47 ++++ 3 files changed, 309 insertions(+) create mode 100644 tests/assets/configs/nodes_with_initial_files.yaml create mode 100644 tests/integration_tests/configuration_file_parsing/test_node_file_system_config.py diff --git a/src/primaite/game/game.py b/src/primaite/game/game.py index 045b2467..befa4032 100644 --- a/src/primaite/game/game.py +++ b/src/primaite/game/game.py @@ -329,6 +329,12 @@ class PrimaiteGame: _LOGGER.error(msg) raise ValueError(msg) + # handle node file system + if node_cfg.get("file_system") is not None and len(node_cfg.get("file_system")) > 0: + for folder in node_cfg.get("file_system"): + for file in node_cfg["file_system"][folder]: + new_node.file_system.create_file(folder_name=folder, file_name=file) + if "users" in node_cfg and new_node.software_manager.software.get("UserManager"): user_manager: UserManager = new_node.software_manager.software["UserManager"] # noqa for user_cfg in node_cfg["users"]: diff --git a/tests/assets/configs/nodes_with_initial_files.yaml b/tests/assets/configs/nodes_with_initial_files.yaml new file mode 100644 index 00000000..3213098b --- /dev/null +++ b/tests/assets/configs/nodes_with_initial_files.yaml @@ -0,0 +1,256 @@ +# Basic Switched network +# +# -------------- -------------- -------------- +# | client_1 |------| switch_1 |------| client_2 | +# -------------- -------------- -------------- +# +io_settings: + save_step_metadata: false + save_pcap_logs: true + save_sys_logs: true + sys_log_level: WARNING + agent_log_level: INFO + save_agent_logs: true + write_agent_log_to_terminal: True + + +game: + max_episode_length: 256 + ports: + - ARP + - DNS + - HTTP + - POSTGRES_SERVER + protocols: + - ICMP + - TCP + - UDP + +agents: + - ref: client_2_green_user + team: GREEN + type: ProbabilisticAgent + observation_space: null + action_space: + action_list: + - type: DONOTHING + - type: NODE_APPLICATION_EXECUTE + action_map: + 0: + action: DONOTHING + options: {} + 1: + action: NODE_APPLICATION_EXECUTE + options: + node_id: 0 + application_id: 0 + options: + nodes: + - node_name: client_2 + applications: + - application_name: WebBrowser + max_folders_per_node: 1 + max_files_per_folder: 1 + max_services_per_node: 1 + max_applications_per_node: 1 + + reward_function: + reward_components: + - type: DUMMY + + agent_settings: + start_settings: + start_step: 5 + frequency: 4 + variance: 3 + + + + - ref: defender + team: BLUE + type: ProxyAgent + + observation_space: + type: CUSTOM + options: + components: + - type: NODES + label: NODES + options: + hosts: + - hostname: client_1 + - hostname: client_2 + - hostname: client_3 + num_services: 1 + num_applications: 0 + num_folders: 1 + num_files: 1 + num_nics: 2 + include_num_access: false + monitored_traffic: + icmp: + - NONE + tcp: + - DNS + include_nmne: false + routers: + - hostname: router_1 + num_ports: 0 + ip_list: + - 192.168.10.21 + - 192.168.10.22 + - 192.168.10.23 + wildcard_list: + - 0.0.0.1 + port_list: + - 80 + - 5432 + protocol_list: + - ICMP + - TCP + - UDP + num_rules: 10 + + - type: LINKS + label: LINKS + options: + link_references: + - switch_1:eth-1<->client_1:eth-1 + - switch_1:eth-2<->client_2:eth-1 + - type: "NONE" + label: ICS + options: {} + + action_space: + action_list: + - type: DONOTHING + + action_map: + 0: + action: DONOTHING + options: {} + options: + nodes: + - node_name: switch + - node_name: client_1 + - node_name: client_2 + - node_name: client_3 + max_folders_per_node: 2 + max_files_per_folder: 2 + max_services_per_node: 2 + max_nics_per_node: 8 + max_acl_rules: 10 + ip_list: + - 192.168.10.21 + - 192.168.10.22 + - 192.168.10.23 + + reward_function: + reward_components: + - type: DATABASE_FILE_INTEGRITY + weight: 0.5 + options: + node_hostname: database_server + folder_name: database + file_name: database.db + + + - type: WEB_SERVER_404_PENALTY + weight: 0.5 + options: + node_hostname: web_server + service_name: web_server_web_service + + + agent_settings: + flatten_obs: true + +simulation: + network: + nodes: + + - type: switch + hostname: switch_1 + num_ports: 8 + + - hostname: client_1 + type: computer + ip_address: 192.168.10.21 + subnet_mask: 255.255.255.0 + default_gateway: 192.168.10.1 + dns_server: 192.168.1.10 + applications: + - type: RansomwareScript + - type: WebBrowser + options: + target_url: http://arcd.com/users/ + - type: DatabaseClient + options: + db_server_ip: 192.168.1.10 + server_password: arcd + - type: DataManipulationBot + options: + port_scan_p_of_success: 0.8 + data_manipulation_p_of_success: 0.8 + payload: "DELETE" + server_ip: 192.168.1.21 + server_password: arcd + - type: DoSBot + options: + target_ip_address: 192.168.10.21 + payload: SPOOF DATA + port_scan_p_of_success: 0.8 + services: + - type: DNSClient + options: + dns_server: 192.168.1.10 + - type: DNSServer + options: + domain_mapping: + arcd.com: 192.168.1.10 + - type: DatabaseService + options: + backup_server_ip: 192.168.1.10 + - type: WebServer + - type: FTPServer + options: + server_password: arcd + - type: NTPClient + options: + ntp_server_ip: 192.168.1.10 + - type: NTPServer + - hostname: client_2 + type: computer + ip_address: 192.168.10.22 + subnet_mask: 255.255.255.0 + default_gateway: 192.168.10.1 + dns_server: 192.168.1.10 + file_system: + downloads: + - "test.txt" + - "suh_con.dn" + root: + - "passwords.txt" + # pre installed services and applications + - hostname: client_3 + type: computer + ip_address: 192.168.10.23 + subnet_mask: 255.255.255.0 + default_gateway: 192.168.10.1 + dns_server: 192.168.1.10 + start_up_duration: 0 + shut_down_duration: 0 + operating_state: "OFF" + # pre installed services and applications + + links: + - endpoint_a_hostname: switch_1 + endpoint_a_port: 1 + endpoint_b_hostname: client_1 + endpoint_b_port: 1 + bandwidth: 200 + - endpoint_a_hostname: switch_1 + endpoint_a_port: 2 + endpoint_b_hostname: client_2 + endpoint_b_port: 1 + bandwidth: 200 diff --git a/tests/integration_tests/configuration_file_parsing/test_node_file_system_config.py b/tests/integration_tests/configuration_file_parsing/test_node_file_system_config.py new file mode 100644 index 00000000..05ef7275 --- /dev/null +++ b/tests/integration_tests/configuration_file_parsing/test_node_file_system_config.py @@ -0,0 +1,47 @@ +# © Crown-owned copyright 2024, Defence Science and Technology Laboratory UK +from pathlib import Path +from typing import Union + +import yaml + +from primaite.game.game import PrimaiteGame +from tests import TEST_ASSETS_ROOT + +BASIC_CONFIG = TEST_ASSETS_ROOT / "configs/nodes_with_initial_files.yaml" + + +def load_config(config_path: Union[str, Path]) -> PrimaiteGame: + """Returns a PrimaiteGame object which loads the contents of a given yaml path.""" + with open(config_path, "r") as f: + cfg = yaml.safe_load(f) + + return PrimaiteGame.from_config(cfg) + + +def test_node_file_system_from_config(): + """Test that the appropriate files are instantiated in nodes when loaded from config.""" + game = load_config(BASIC_CONFIG) + + client_1 = game.simulation.network.get_node_by_hostname("client_1") + + assert client_1.software_manager.software.get("DatabaseService") # database service should be installed + assert client_1.file_system.get_file(folder_name="database", file_name="database.db") # database files should exist + + assert client_1.software_manager.software.get("WebServer") # web server should be installed + assert client_1.file_system.get_file(folder_name="primaite", file_name="index.html") # web files should exist + + client_2 = game.simulation.network.get_node_by_hostname("client_2") + + # database service should not be installed + assert client_2.software_manager.software.get("DatabaseService") is None + # database files should not exist + assert client_2.file_system.get_file(folder_name="database", file_name="database.db") is None + + # web server should not be installed + assert client_2.software_manager.software.get("WebServer") is None + # web files should not exist + assert client_2.file_system.get_file(folder_name="primaite", file_name="index.html") is None + + # TODO file sizes and file types + # TODO assert that files and folders created: + # TODO create empty folders From 5cacbf03373bccd634c8086783222bb45648871a Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Mon, 2 Sep 2024 16:54:13 +0100 Subject: [PATCH 04/95] #2845: Changes to write observation space data to log file. --- src/primaite/session/environment.py | 25 +++++++++++++++++++++++++ src/primaite/session/io.py | 2 ++ 2 files changed, 27 insertions(+) diff --git a/src/primaite/session/environment.py b/src/primaite/session/environment.py index c66663e3..23b86546 100644 --- a/src/primaite/session/environment.py +++ b/src/primaite/session/environment.py @@ -112,6 +112,9 @@ class PrimaiteGymEnv(gymnasium.Env): self.game.update_agents(state) next_obs = self._get_obs() # this doesn't update observation, just gets the current observation + if self.io.settings.obs_space_data: + # Write unflattened observation space to log file. + self._write_obs_space_data(self.agent.observation_manager.current_observation) reward = self.agent.reward_function.current_reward _LOGGER.debug(f"step: {self.game.step_counter}, Blue reward: {reward}") terminated = False @@ -139,6 +142,25 @@ class PrimaiteGymEnv(gymnasium.Env): with open(path, "w") as file: json.dump(data, file) + def _write_obs_space_data(self, obs_space: ObsType) -> None: + """Write the unflattened observation space data to a JSON file. + + :param obs: Observation of the environment (dict) + :type obs: ObsType + """ + output_dir = SIM_OUTPUT.path / f"episode_{self.episode_counter}" / "obs_space_data" + + output_dir.mkdir(parents=True, exist_ok=True) + path = output_dir / f"step_{self.game.step_counter}.json" + + data = { + "episode": self.episode_counter, + "step": self.game.step_counter, + "obs_space_data": obs_space, + } + with open(path, "w") as file: + json.dump(data, file) + def reset(self, seed: Optional[int] = None, options: Optional[Dict] = None) -> Tuple[ObsType, Dict[str, Any]]: """Reset the environment.""" _LOGGER.info( @@ -159,6 +181,9 @@ class PrimaiteGymEnv(gymnasium.Env): state = self.game.get_sim_state() self.game.update_agents(state=state) next_obs = self._get_obs() + if self.io.settings.obs_space_data: + # Write unflattened observation space to log file. + self._write_obs_space_data(self.agent.observation_manager.current_observation) info = {} return next_obs, info diff --git a/src/primaite/session/io.py b/src/primaite/session/io.py index 78d7cb3c..3627e9e9 100644 --- a/src/primaite/session/io.py +++ b/src/primaite/session/io.py @@ -45,6 +45,8 @@ class PrimaiteIO: """The level of sys logs that should be included in the logfiles/logged into terminal.""" agent_log_level: LogLevel = LogLevel.INFO """The level of agent logs that should be included in the logfiles/logged into terminal.""" + obs_space_data: bool = False + """Whether to save observation space data to a log file.""" def __init__(self, settings: Optional[Settings] = None) -> None: """ From fd3d3812f6d8d03b1d44261b06ff39fecd0e2209 Mon Sep 17 00:00:00 2001 From: Archer Bowen Date: Mon, 2 Sep 2024 16:55:43 +0100 Subject: [PATCH 05/95] #2840 Documentation and minor bug fixes found in terminal and session manager. --- CHANGELOG.md | 2 + .../system/services/terminal.rst | 106 ++++++- .../notebooks/Terminal-Processing.ipynb | 274 +++++++++++++++++- .../simulator/system/core/session_manager.py | 2 +- .../system/services/terminal/terminal.py | 8 +- .../actions/test_terminal_actions.py | 1 + 6 files changed, 385 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d08974c..2a855512 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +- New ``NODE_SEND_LOCAL_COMMAND`` action implemented which grants agents the ability to execute commands locally. (Previously limited to remote only) + ### Added - Random Number Generator Seeding by specifying a random number seed in the config file. - Implemented Terminal service class, providing a generic terminal simulation. diff --git a/docs/source/simulation_components/system/services/terminal.rst b/docs/source/simulation_components/system/services/terminal.rst index f982145d..6a1b0204 100644 --- a/docs/source/simulation_components/system/services/terminal.rst +++ b/docs/source/simulation_components/system/services/terminal.rst @@ -30,6 +30,7 @@ Usage - Terminal Clients connect, execute commands and disconnect from remote nodes. - Ensures that users are logged in to the component before executing any commands. - Service runs on SSH port 22 by default. + - Enables Agents to send commands both remotely and locally. Implementation """""""""""""" @@ -39,9 +40,110 @@ Implementation - Extends Service class. - A detailed guide on the implementation and functionality of the Terminal class can be found in the "Terminal-Processing" jupyter notebook. +Command Format +^^^^^^^^^^^^^^ + +``Terminals`` implement their commands through leveraging the pre-existing :doc:`../../request_system`. + +Due to this ``Terminals`` will only accept commands passed within the ``RequestFormat``. + +:py:class:`primaite.game.interface.RequestFormat` + +For example, ``terminal`` command actions when used in ``yaml`` format are formatted as follows: + +.. code-block:: yaml + command: + - "file_system" + - "create" + - "file" + - "downloads" + - "cat.png" + - "False" + +**This command creates file called ``cat.png`` within the ``downloads`` folder.** + +This is then loaded from ``yaml`` into a dictionary containing the terminal command: + +.. code-block:: python + + {"command":["file_system", "create", "file", "downloads", "cat.png", "False"]} + +Which is then parsed to the ``Terminals`` Request Manager to be executed. + +Game Layer Usage (Agents) +======================== + +The below code examples demonstrate how to use terminal related actions in yaml files. + +yaml +"""" + +``NODE_SEND_LOCAL_COMMAND`` +""""""""""""""""""""""""""" + +Agents can execute local commands without needing to perform a separate remote login action (``SSH_TO_REMOTE``). + +.. code-block:: yaml + + ... + ... + action: NODE_SEND_LOCAL_COMMAND + options: + node_id: 0 + username: admin + password: admin + command: # Example command - Creates a file called 'cat.png' in the downloads folder. + - "file_system" + - "create" + - "file" + - "downloads" + - "cat.png" + - "False" + + +``SSH_TO_REMOTE`` +""""""""""""""""" + +Agents are able to use the terminal to login into remote nodes via ``SSH`` which allows for agents to execute commands on remote hosts. + +.. code-block:: yaml + + ... + ... + action: SSH_TO_REMOTE + options: + node_id: 0 + username: admin + password: admin + remote_ip: 192.168.0.10 # Example Ip Address. (The remote host's IP that will be used by ssh) + + +``NODE_SEND_REMOTE_COMMAND`` +"""""""""""""""""""""""""""" + +After remotely login into another host, a agent can use the ``NODE_SEND_REMOTE_COMMAND`` to execute commands across the network remotely. + +.. code-block:: yaml + + ... + ... + action: NODE_SEND_REMOTE_COMMAND + options: + node_id: 0 + remote_ip: 192.168.0.10 + command: + - "file_system" + - "create" + - "file" + - "downloads" + - "cat.png" + - "False" + + + +Simulation Layer Usage +====================== -Usage -===== The below code examples demonstrate how to create a terminal, a remote terminal, and how to send a basic application install command to a remote node. diff --git a/src/primaite/notebooks/Terminal-Processing.ipynb b/src/primaite/notebooks/Terminal-Processing.ipynb index fdf405a7..19ce567e 100644 --- a/src/primaite/notebooks/Terminal-Processing.ipynb +++ b/src/primaite/notebooks/Terminal-Processing.ipynb @@ -9,6 +9,13 @@ "© Crown-owned copyright 2024, Defence Science and Technology Laboratory UK" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Simulation Layer Implementation." + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -198,6 +205,271 @@ "source": [ "computer_b.user_session_manager.show(include_historic=True, include_session_id=True)" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Game Layer Implementation\n", + "\n", + "This notebook section will detail the implementation of how the game layer utilises the terminal to support different agent actions.\n", + "\n", + "The ``Terminal`` is used in a variety of different ways in the game layer. Specifically, the terminal is leveraged to implement the following actions:\n", + "\n", + "\n", + "| Game Layer Action | Simulation Layer |\n", + "|-----------------------------------|--------------------------|\n", + "| ``NODE_SEND_LOCAL_COMMAND`` | Uses the given user credentials, creates a ``LocalTerminalSession`` and executes the given command and returns the ``RequestResponse``.\n", + "| ``SSH_TO_REMOTE`` | Uses the given user credentials and remote IP to create a ``RemoteTerminalSession``.\n", + "| ``NODE_SEND_REMOTE_COMMAND`` | Uses the given remote IP to locate the correct ``RemoteTerminalSession``, executes the given command and returns the ``RequestsResponse``." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Game Layer Setup\n", + "\n", + "Similar to other notebooks, the next code cells create a custom proxy agent to demonstrate how these commands can be leveraged by agents in the ``UC2`` network environment.\n", + "\n", + "If you're unfamiliar with ``UC2`` then please refer to the [UC2-E2E-Demo notebook for further reference](./Data-Manipulation-E2E-Demonstration.ipynb)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import yaml\n", + "from primaite.config.load import data_manipulation_config_path\n", + "from primaite.session.environment import PrimaiteGymEnv" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "custom_terminal_agent = \"\"\"\n", + " - ref: CustomC2Agent\n", + " team: RED\n", + " type: ProxyAgent\n", + " observation_space: null\n", + " action_space:\n", + " action_list:\n", + " - type: DONOTHING\n", + " - type: NODE_SEND_LOCAL_COMMAND\n", + " - type: SSH_TO_REMOTE\n", + " - type: NODE_SEND_REMOTE_COMMAND\n", + " options:\n", + " nodes:\n", + " - node_name: client_1\n", + " max_folders_per_node: 1\n", + " max_files_per_folder: 1\n", + " max_services_per_node: 2\n", + " max_nics_per_node: 8\n", + " max_acl_rules: 10\n", + " ip_list:\n", + " - 192.168.1.21\n", + " - 192.168.1.14\n", + " wildcard_list:\n", + " - 0.0.0.1\n", + " action_map:\n", + " 0:\n", + " action: DONOTHING\n", + " options: {}\n", + " 1:\n", + " action: NODE_SEND_LOCAL_COMMAND\n", + " options:\n", + " node_id: 0\n", + " username: admin\n", + " password: admin\n", + " command:\n", + " - file_system\n", + " - create\n", + " - file\n", + " - downloads\n", + " - dog.png\n", + " - False\n", + " 2:\n", + " action: SSH_TO_REMOTE\n", + " options:\n", + " node_id: 0\n", + " username: admin\n", + " password: admin\n", + " remote_ip: 192.168.10.22\n", + " 3:\n", + " action: NODE_SEND_REMOTE_COMMAND\n", + " options:\n", + " node_id: 0\n", + " remote_ip: 192.168.10.22\n", + " command:\n", + " - file_system\n", + " - create\n", + " - file\n", + " - downloads\n", + " - cat.png\n", + " - False\n", + " reward_function:\n", + " reward_components:\n", + " - type: DUMMY\n", + "\"\"\"\n", + "custom_terminal_agent_yaml = yaml.safe_load(custom_terminal_agent)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "with open(data_manipulation_config_path()) as f:\n", + " cfg = yaml.safe_load(f)\n", + " # removing all agents & adding the custom agent.\n", + " cfg['agents'] = {}\n", + " cfg['agents'] = custom_terminal_agent_yaml\n", + " \n", + "env = PrimaiteGymEnv(env_config=cfg)\n", + "\n", + "client_1: Computer = env.game.simulation.network.get_node_by_hostname(\"client_1\")\n", + "client_2: Computer = env.game.simulation.network.get_node_by_hostname(\"client_2\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Terminal Action | ``NODE_SEND_LOCAL_COMMAND`` \n", + "\n", + "The yaml snippet below shows all the relevant agent options for this action:\n", + "\n", + "```yaml\n", + "\n", + " action_space:\n", + " action_list:\n", + " ...\n", + " - type: NODE_SEND_LOCAL_COMMAND\n", + " ...\n", + " options:\n", + " nodes: # Node List\n", + " - node_name: client_1\n", + " ...\n", + " ...\n", + " action_map:\n", + " 1:\n", + " action: NODE_SEND_LOCAL_COMMAND\n", + " options:\n", + " node_id: 0 # Index 0 at the node list.\n", + " username: admin\n", + " password: admin\n", + " command:\n", + " - file_system\n", + " - create\n", + " - file\n", + " - downloads\n", + " - dog.png\n", + " - False\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "env.step(1)\n", + "client_1.file_system.show(full=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Terminal Action | ``SSH_TO_REMOTE`` \n", + "\n", + "The yaml snippet below shows all the relevant agent options for this action:\n", + "\n", + "```yaml\n", + "\n", + " action_space:\n", + " action_list:\n", + " ...\n", + " - type: SSH_TO_REMOTE\n", + " ...\n", + " options:\n", + " nodes: # Node List\n", + " - node_name: client_1\n", + " ...\n", + " ...\n", + " action_map:\n", + " 2:\n", + " action: SSH_TO_REMOTE\n", + " options:\n", + " node_id: 0 # Index 0 at the node list.\n", + " username: admin\n", + " password: admin\n", + " remote_ip: 192.168.10.22 # client_2's ip address.\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "env.step(2)\n", + "client_2.session_manager.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Terminal Action | ``NODE_SEND_REMOTE_COMMAND``\n", + "\n", + "The yaml snippet below shows all the relevant agent options for this action:\n", + "\n", + "```yaml\n", + "\n", + " action_space:\n", + " action_list:\n", + " ...\n", + " - type: NODE_SEND_REMOTE_COMMAND\n", + " ...\n", + " options:\n", + " nodes: # Node List\n", + " - node_name: client_1\n", + " ...\n", + " ...\n", + " action_map:\n", + " 1:\n", + " action: NODE_SEND_REMOTE_COMMAND\n", + " options:\n", + " node_id: 0 # Index 0 at the node list.\n", + " remote_ip: 192.168.10.22\n", + " commands:\n", + " - file_system\n", + " - create\n", + " - file\n", + " - downloads\n", + " - cat.png\n", + " - False\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "env.step(3)\n", + "client_2.file_system.show(full=True)" + ] } ], "metadata": { @@ -216,7 +488,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.11" + "version": "3.10.12" } }, "nbformat": 4, diff --git a/src/primaite/simulator/system/core/session_manager.py b/src/primaite/simulator/system/core/session_manager.py index b7e2c021..677ff477 100644 --- a/src/primaite/simulator/system/core/session_manager.py +++ b/src/primaite/simulator/system/core/session_manager.py @@ -413,5 +413,5 @@ class SessionManager: table.align = "l" table.title = f"{self.sys_log.hostname} Session Manager" for session in self.sessions_by_key.values(): - table.add_row([session.dst_ip_address, session.dst_port.value, session.protocol.name]) + table.add_row([session.with_ip_address, session.dst_port.value, session.protocol.name]) print(table) diff --git a/src/primaite/simulator/system/services/terminal/terminal.py b/src/primaite/simulator/system/services/terminal/terminal.py index 9b88bbe8..dc7da205 100644 --- a/src/primaite/simulator/system/services/terminal/terminal.py +++ b/src/primaite/simulator/system/services/terminal/terminal.py @@ -208,10 +208,10 @@ class Terminal(Service): status="success", data={}, ) - return RequestResponse( - status="failure", - data={}, - ) + return RequestResponse( + status="failure", + data={}, + ) rm.add_request( "send_remote_command", diff --git a/tests/integration_tests/game_layer/actions/test_terminal_actions.py b/tests/integration_tests/game_layer/actions/test_terminal_actions.py index d2ea7202..c4247d6e 100644 --- a/tests/integration_tests/game_layer/actions/test_terminal_actions.py +++ b/tests/integration_tests/game_layer/actions/test_terminal_actions.py @@ -215,3 +215,4 @@ def test_local_terminal(game_and_agent_fixture: Tuple[PrimaiteGame, ProxyAgent]) game.step() assert client_1.file_system.get_file("folder123", "cat.pdf") is None + client_1.session_manager.show() From 8e6b9f39707e1236f3d5f9e8f85d962b33f0e1d5 Mon Sep 17 00:00:00 2001 From: Czar Echavez Date: Tue, 3 Sep 2024 11:53:23 +0100 Subject: [PATCH 06/95] #2782: added ability to create empty folders + create files with size and types + tests --- src/primaite/game/game.py | 21 ++++++++++++++--- .../configs/nodes_with_initial_files.yaml | 9 +++++--- .../test_node_file_system_config.py | 23 ++++++++++++++++--- 3 files changed, 44 insertions(+), 9 deletions(-) diff --git a/src/primaite/game/game.py b/src/primaite/game/game.py index befa4032..d11f6a19 100644 --- a/src/primaite/game/game.py +++ b/src/primaite/game/game.py @@ -17,6 +17,7 @@ from primaite.game.agent.scripted_agents.random_agent import PeriodicAgent from primaite.game.agent.scripted_agents.tap001 import TAP001 from primaite.game.science import graph_has_cycle, topological_sort from primaite.simulator import SIM_OUTPUT +from primaite.simulator.file_system.file_type import FileType from primaite.simulator.network.airspace import AirSpaceFrequency from primaite.simulator.network.hardware.base import NetworkInterface, NodeOperatingState, UserManager from primaite.simulator.network.hardware.nodes.host.computer import Computer @@ -331,9 +332,23 @@ class PrimaiteGame: # handle node file system if node_cfg.get("file_system") is not None and len(node_cfg.get("file_system")) > 0: - for folder in node_cfg.get("file_system"): - for file in node_cfg["file_system"][folder]: - new_node.file_system.create_file(folder_name=folder, file_name=file) + for folder_idx, folder_obj in enumerate(node_cfg.get("file_system")): + # if the folder is not a Dict, create an empty folder + if not isinstance(folder_obj, Dict): + new_node.file_system.create_folder(folder_name=folder_obj) + else: + folder_name = next(iter(folder_obj)) + for file_idx, file_obj in enumerate(node_cfg["file_system"][folder_idx][folder_name]): + if not isinstance(file_obj, Dict): + new_node.file_system.create_file(folder_name=folder_name, file_name=file_obj) + else: + file_name = next(iter(file_obj)) + new_node.file_system.create_file( + folder_name=folder_name, + file_name=file_name, + size=file_obj[file_name].get("size", 0), + file_type=FileType[file_obj[file_name].get("type", "UNKNOWN").upper()], + ) if "users" in node_cfg and new_node.software_manager.software.get("UserManager"): user_manager: UserManager = new_node.software_manager.software["UserManager"] # noqa diff --git a/tests/assets/configs/nodes_with_initial_files.yaml b/tests/assets/configs/nodes_with_initial_files.yaml index 3213098b..fad6cffd 100644 --- a/tests/assets/configs/nodes_with_initial_files.yaml +++ b/tests/assets/configs/nodes_with_initial_files.yaml @@ -226,11 +226,14 @@ simulation: default_gateway: 192.168.10.1 dns_server: 192.168.1.10 file_system: - downloads: + - empty_folder + - downloads: - "test.txt" - "suh_con.dn" - root: - - "passwords.txt" + - root: + - passwords: + size: 69 + type: TXT # pre installed services and applications - hostname: client_3 type: computer diff --git a/tests/integration_tests/configuration_file_parsing/test_node_file_system_config.py b/tests/integration_tests/configuration_file_parsing/test_node_file_system_config.py index 05ef7275..49e90b54 100644 --- a/tests/integration_tests/configuration_file_parsing/test_node_file_system_config.py +++ b/tests/integration_tests/configuration_file_parsing/test_node_file_system_config.py @@ -5,6 +5,7 @@ from typing import Union import yaml from primaite.game.game import PrimaiteGame +from primaite.simulator.file_system.file_type import FileType from tests import TEST_ASSETS_ROOT BASIC_CONFIG = TEST_ASSETS_ROOT / "configs/nodes_with_initial_files.yaml" @@ -42,6 +43,22 @@ def test_node_file_system_from_config(): # web files should not exist assert client_2.file_system.get_file(folder_name="primaite", file_name="index.html") is None - # TODO file sizes and file types - # TODO assert that files and folders created: - # TODO create empty folders + empty_folder = client_2.file_system.get_folder(folder_name="empty_folder") + assert empty_folder + assert len(empty_folder.files) == 0 # should have no files + + password_file = client_2.file_system.get_file(folder_name="root", file_name="passwords.txt") + assert password_file # should exist + assert password_file.file_type is FileType.TXT + assert password_file.size is 69 + + downloads_folder = client_2.file_system.get_folder(folder_name="downloads") + assert downloads_folder # downloads folder should exist + + test_txt = downloads_folder.get_file(file_name="test.txt") + assert test_txt # test.txt should exist + assert test_txt.file_type is FileType.TXT + + unknown_file_type = downloads_folder.get_file(file_name="suh_con.dn") + assert unknown_file_type # unknown_file_type should exist + assert unknown_file_type.file_type is FileType.UNKNOWN From 26a56bf3608d2f0c7930d8e0b6e5faa0830e092f Mon Sep 17 00:00:00 2001 From: Czar Echavez Date: Tue, 3 Sep 2024 12:37:39 +0100 Subject: [PATCH 07/95] #2782: documentation + adding example to data_manipulation.yaml --- .../nodes/common/common_node_attributes.rst | 33 +++++++++++++++++++ .../_package_data/data_manipulation.yaml | 9 ++++- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/docs/source/configuration/simulation/nodes/common/common_node_attributes.rst b/docs/source/configuration/simulation/nodes/common/common_node_attributes.rst index 7cf11eb4..056422ca 100644 --- a/docs/source/configuration/simulation/nodes/common/common_node_attributes.rst +++ b/docs/source/configuration/simulation/nodes/common/common_node_attributes.rst @@ -54,6 +54,39 @@ Optional. Default value is ``3``. The number of time steps required to occur in order for the node to cycle from ``ON`` to ``SHUTTING_DOWN`` and then finally ``OFF``. +``file_system`` +--------------- + +Optional. + +The file system of the node. This configuration allows nodes to be initialised with files and/or folders. + +The file system takes a list of folders and files. + +Example: + +.. code-block:: yaml + + simulation: + network: + nodes: + - hostname: client_1 + type: computer + ip_address: 192.168.10.11 + subnet_mask: 255.255.255.0 + default_gateway: 192.168.10.1 + file_system: + - empty_folder # example of an empty folder + - downloads: + - "test_1.txt" # files in the downloads folder + - "test_2.txt" + - root: + - passwords: # example of file with size and type + size: 69 # size in bytes + type: TXT # See FileType for list of available file types + +List of file types: :py:mod:`primaite.simulator.file_system.file_type.FileType` + ``users`` --------- diff --git a/src/primaite/config/_package_data/data_manipulation.yaml b/src/primaite/config/_package_data/data_manipulation.yaml index 97442903..2d03609a 100644 --- a/src/primaite/config/_package_data/data_manipulation.yaml +++ b/src/primaite/config/_package_data/data_manipulation.yaml @@ -843,7 +843,14 @@ simulation: dns_server: 192.168.1.10 services: - type: FTPServer - + file_system: + - root: + - backup_script.sh: # example file in backup server + size: 400 + type: SH + - downloads: + - "ChromeSetup.exe" # another example file + - "New Folder" # example of an empty folder - hostname: security_suite type: server ip_address: 192.168.1.110 From 8e57e707b3e1d5eec3b53d6deeb90d7b9289338b Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Tue, 3 Sep 2024 14:38:19 +0100 Subject: [PATCH 08/95] #2845: Changed to store obs data within AgentHistoryItem --- src/primaite/game/agent/interface.py | 18 ++++++++++++++++-- src/primaite/game/game.py | 1 + src/primaite/session/environment.py | 25 ------------------------- src/primaite/session/io.py | 2 -- 4 files changed, 17 insertions(+), 29 deletions(-) diff --git a/src/primaite/game/agent/interface.py b/src/primaite/game/agent/interface.py index 14b97821..aac6c05a 100644 --- a/src/primaite/game/agent/interface.py +++ b/src/primaite/game/agent/interface.py @@ -38,6 +38,9 @@ class AgentHistoryItem(BaseModel): reward_info: Dict[str, Any] = {} + obs_space_data: Optional[ObsType] = None + """The observation space data for this step.""" + class AgentStartSettings(BaseModel): """Configuration values for when an agent starts performing actions.""" @@ -169,12 +172,23 @@ class AbstractAgent(ABC): return request def process_action_response( - self, timestep: int, action: str, parameters: Dict[str, Any], request: RequestFormat, response: RequestResponse + self, + timestep: int, + action: str, + parameters: Dict[str, Any], + request: RequestFormat, + response: RequestResponse, + obs_space_data: ObsType, ) -> None: """Process the response from the most recent action.""" self.history.append( AgentHistoryItem( - timestep=timestep, action=action, parameters=parameters, request=request, response=response + timestep=timestep, + action=action, + parameters=parameters, + request=request, + response=response, + obs_space_data=obs_space_data, ) ) diff --git a/src/primaite/game/game.py b/src/primaite/game/game.py index 045b2467..ed3c84d3 100644 --- a/src/primaite/game/game.py +++ b/src/primaite/game/game.py @@ -186,6 +186,7 @@ class PrimaiteGame: parameters=parameters, request=request, response=response, + obs_space_data=obs, ) def pre_timestep(self) -> None: diff --git a/src/primaite/session/environment.py b/src/primaite/session/environment.py index 23b86546..c66663e3 100644 --- a/src/primaite/session/environment.py +++ b/src/primaite/session/environment.py @@ -112,9 +112,6 @@ class PrimaiteGymEnv(gymnasium.Env): self.game.update_agents(state) next_obs = self._get_obs() # this doesn't update observation, just gets the current observation - if self.io.settings.obs_space_data: - # Write unflattened observation space to log file. - self._write_obs_space_data(self.agent.observation_manager.current_observation) reward = self.agent.reward_function.current_reward _LOGGER.debug(f"step: {self.game.step_counter}, Blue reward: {reward}") terminated = False @@ -142,25 +139,6 @@ class PrimaiteGymEnv(gymnasium.Env): with open(path, "w") as file: json.dump(data, file) - def _write_obs_space_data(self, obs_space: ObsType) -> None: - """Write the unflattened observation space data to a JSON file. - - :param obs: Observation of the environment (dict) - :type obs: ObsType - """ - output_dir = SIM_OUTPUT.path / f"episode_{self.episode_counter}" / "obs_space_data" - - output_dir.mkdir(parents=True, exist_ok=True) - path = output_dir / f"step_{self.game.step_counter}.json" - - data = { - "episode": self.episode_counter, - "step": self.game.step_counter, - "obs_space_data": obs_space, - } - with open(path, "w") as file: - json.dump(data, file) - def reset(self, seed: Optional[int] = None, options: Optional[Dict] = None) -> Tuple[ObsType, Dict[str, Any]]: """Reset the environment.""" _LOGGER.info( @@ -181,9 +159,6 @@ class PrimaiteGymEnv(gymnasium.Env): state = self.game.get_sim_state() self.game.update_agents(state=state) next_obs = self._get_obs() - if self.io.settings.obs_space_data: - # Write unflattened observation space to log file. - self._write_obs_space_data(self.agent.observation_manager.current_observation) info = {} return next_obs, info diff --git a/src/primaite/session/io.py b/src/primaite/session/io.py index 3627e9e9..78d7cb3c 100644 --- a/src/primaite/session/io.py +++ b/src/primaite/session/io.py @@ -45,8 +45,6 @@ class PrimaiteIO: """The level of sys logs that should be included in the logfiles/logged into terminal.""" agent_log_level: LogLevel = LogLevel.INFO """The level of agent logs that should be included in the logfiles/logged into terminal.""" - obs_space_data: bool = False - """Whether to save observation space data to a log file.""" def __init__(self, settings: Optional[Settings] = None) -> None: """ From 61add769c46b6d8a4f255e301e9d19f5d6a7ddfb Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Tue, 3 Sep 2024 17:16:48 +0100 Subject: [PATCH 09/95] #2845: Add test for obs_data_space capture. --- .../observations/test_obs_data_capture.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 tests/integration_tests/game_layer/observations/test_obs_data_capture.py diff --git a/tests/integration_tests/game_layer/observations/test_obs_data_capture.py b/tests/integration_tests/game_layer/observations/test_obs_data_capture.py new file mode 100644 index 00000000..205341d9 --- /dev/null +++ b/tests/integration_tests/game_layer/observations/test_obs_data_capture.py @@ -0,0 +1,25 @@ +from primaite.session.environment import PrimaiteGymEnv +from primaite.session.io import PrimaiteIO +import json +from tests import TEST_ASSETS_ROOT + +DATA_MANIPULATION_CONFIG = TEST_ASSETS_ROOT / "configs" / "data_manipulation.yaml" + +def test_obs_data_in_log_file(): + """Create a log file of AgentHistoryItems and check observation data is + included. Assumes that data_manipulation.yaml has an agent labelled + 'defender' with a non-null observation space. + The log file will be in: + primaite/VERSION/sessions/YYYY-MM-DD/HH-MM-SS/agent_actions + """ + env = PrimaiteGymEnv(DATA_MANIPULATION_CONFIG) + env.reset() + for _ in range(10): + env.step(0) + env.reset() + io = PrimaiteIO() + path = io.generate_agent_actions_save_path(episode=1) + with open(path, 'r') as f: + j = json.load(f) + + assert type(j['0']['defender']['obs_space_data']) == dict From 1822e85eec61710c69db4deaeaeaba2d49053a83 Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Tue, 3 Sep 2024 17:24:21 +0100 Subject: [PATCH 10/95] #2845: Pre-commit fixes --- .../game_layer/observations/test_obs_data_capture.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/integration_tests/game_layer/observations/test_obs_data_capture.py b/tests/integration_tests/game_layer/observations/test_obs_data_capture.py index 205341d9..810b2ad7 100644 --- a/tests/integration_tests/game_layer/observations/test_obs_data_capture.py +++ b/tests/integration_tests/game_layer/observations/test_obs_data_capture.py @@ -1,12 +1,15 @@ +# © Crown-owned copyright 2024, Defence Science and Technology Laboratory UK +import json + from primaite.session.environment import PrimaiteGymEnv from primaite.session.io import PrimaiteIO -import json from tests import TEST_ASSETS_ROOT DATA_MANIPULATION_CONFIG = TEST_ASSETS_ROOT / "configs" / "data_manipulation.yaml" + def test_obs_data_in_log_file(): - """Create a log file of AgentHistoryItems and check observation data is + """Create a log file of AgentHistoryItems and check observation data is included. Assumes that data_manipulation.yaml has an agent labelled 'defender' with a non-null observation space. The log file will be in: @@ -19,7 +22,7 @@ def test_obs_data_in_log_file(): env.reset() io = PrimaiteIO() path = io.generate_agent_actions_save_path(episode=1) - with open(path, 'r') as f: + with open(path, "r") as f: j = json.load(f) - assert type(j['0']['defender']['obs_space_data']) == dict + assert type(j["0"]["defender"]["obs_space_data"]) == dict From f4b1d9a91c5566ca6ba49056479d0e8c21f38abe Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Tue, 3 Sep 2024 17:26:01 +0100 Subject: [PATCH 11/95] #2845: Update CHANGELOG. --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d08974c..e2989247 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- Log observation space data by episode and step. + +## [3.3.0] - 2024-09-04 +### Added - Random Number Generator Seeding by specifying a random number seed in the config file. - Implemented Terminal service class, providing a generic terminal simulation. - Added `User`, `UserManager` and `UserSessionManager` to enable the creation of user accounts and login on Nodes. From 1374a23e14fb9fea35c346747eb8d9edd303c2ca Mon Sep 17 00:00:00 2001 From: Czar Echavez Date: Wed, 4 Sep 2024 10:17:33 +0100 Subject: [PATCH 12/95] #2782: fix spacing in data_manipulation yaml + documentation --- .../simulation/nodes/common/common_node_attributes.rst | 4 ++-- src/primaite/config/_package_data/data_manipulation.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/source/configuration/simulation/nodes/common/common_node_attributes.rst b/docs/source/configuration/simulation/nodes/common/common_node_attributes.rst index 056422ca..c94344fd 100644 --- a/docs/source/configuration/simulation/nodes/common/common_node_attributes.rst +++ b/docs/source/configuration/simulation/nodes/common/common_node_attributes.rst @@ -82,8 +82,8 @@ Example: - "test_2.txt" - root: - passwords: # example of file with size and type - size: 69 # size in bytes - type: TXT # See FileType for list of available file types + size: 69 # size in bytes + type: TXT # See FileType for list of available file types List of file types: :py:mod:`primaite.simulator.file_system.file_type.FileType` diff --git a/src/primaite/config/_package_data/data_manipulation.yaml b/src/primaite/config/_package_data/data_manipulation.yaml index 2d03609a..b36ec707 100644 --- a/src/primaite/config/_package_data/data_manipulation.yaml +++ b/src/primaite/config/_package_data/data_manipulation.yaml @@ -846,8 +846,8 @@ simulation: file_system: - root: - backup_script.sh: # example file in backup server - size: 400 - type: SH + size: 400 + type: SH - downloads: - "ChromeSetup.exe" # another example file - "New Folder" # example of an empty folder From 5608ad5ed5799d0dfb02d5767a5fde0f343ff0e7 Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Wed, 4 Sep 2024 14:25:08 +0100 Subject: [PATCH 13/95] #2845: Change 'obs_space_data' to 'observation'. --- src/primaite/game/agent/interface.py | 6 +++--- src/primaite/game/game.py | 2 +- .../game_layer/observations/test_obs_data_capture.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/primaite/game/agent/interface.py b/src/primaite/game/agent/interface.py index aac6c05a..d5165a71 100644 --- a/src/primaite/game/agent/interface.py +++ b/src/primaite/game/agent/interface.py @@ -38,7 +38,7 @@ class AgentHistoryItem(BaseModel): reward_info: Dict[str, Any] = {} - obs_space_data: Optional[ObsType] = None + observation: Optional[ObsType] = None """The observation space data for this step.""" @@ -178,7 +178,7 @@ class AbstractAgent(ABC): parameters: Dict[str, Any], request: RequestFormat, response: RequestResponse, - obs_space_data: ObsType, + observation: ObsType, ) -> None: """Process the response from the most recent action.""" self.history.append( @@ -188,7 +188,7 @@ class AbstractAgent(ABC): parameters=parameters, request=request, response=response, - obs_space_data=obs_space_data, + observation=observation, ) ) diff --git a/src/primaite/game/game.py b/src/primaite/game/game.py index ed3c84d3..4f21120d 100644 --- a/src/primaite/game/game.py +++ b/src/primaite/game/game.py @@ -186,7 +186,7 @@ class PrimaiteGame: parameters=parameters, request=request, response=response, - obs_space_data=obs, + observation=obs, ) def pre_timestep(self) -> None: diff --git a/tests/integration_tests/game_layer/observations/test_obs_data_capture.py b/tests/integration_tests/game_layer/observations/test_obs_data_capture.py index 810b2ad7..e8bdea22 100644 --- a/tests/integration_tests/game_layer/observations/test_obs_data_capture.py +++ b/tests/integration_tests/game_layer/observations/test_obs_data_capture.py @@ -25,4 +25,4 @@ def test_obs_data_in_log_file(): with open(path, "r") as f: j = json.load(f) - assert type(j["0"]["defender"]["obs_space_data"]) == dict + assert type(j["0"]["defender"]["observation"]) == dict From 2391c485698a645a035333208252c39209c1a9da Mon Sep 17 00:00:00 2001 From: Czar Echavez Date: Thu, 5 Sep 2024 10:18:35 +0100 Subject: [PATCH 14/95] #2782: apply suggestions --- src/primaite/config/_package_data/data_manipulation.yaml | 8 -------- src/primaite/game/game.py | 2 +- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/src/primaite/config/_package_data/data_manipulation.yaml b/src/primaite/config/_package_data/data_manipulation.yaml index b36ec707..2a069971 100644 --- a/src/primaite/config/_package_data/data_manipulation.yaml +++ b/src/primaite/config/_package_data/data_manipulation.yaml @@ -843,14 +843,6 @@ simulation: dns_server: 192.168.1.10 services: - type: FTPServer - file_system: - - root: - - backup_script.sh: # example file in backup server - size: 400 - type: SH - - downloads: - - "ChromeSetup.exe" # another example file - - "New Folder" # example of an empty folder - hostname: security_suite type: server ip_address: 192.168.1.110 diff --git a/src/primaite/game/game.py b/src/primaite/game/game.py index d11f6a19..8e4d4513 100644 --- a/src/primaite/game/game.py +++ b/src/primaite/game/game.py @@ -331,7 +331,7 @@ class PrimaiteGame: raise ValueError(msg) # handle node file system - if node_cfg.get("file_system") is not None and len(node_cfg.get("file_system")) > 0: + if node_cfg.get("file_system"): for folder_idx, folder_obj in enumerate(node_cfg.get("file_system")): # if the folder is not a Dict, create an empty folder if not isinstance(folder_obj, Dict): From e809d89c30d3ba438d4edabfe88ea9c1ba9f226d Mon Sep 17 00:00:00 2001 From: Czar Echavez Date: Thu, 5 Sep 2024 13:47:59 +0100 Subject: [PATCH 15/95] #2842 and #2843: implement add user and disable user actions + tests --- src/primaite/game/agent/actions.py | 34 +++++++ .../simulator/network/hardware/base.py | 16 +++- tests/conftest.py | 2 + .../actions/test_user_account_actions.py | 93 +++++++++++++++++++ 4 files changed, 144 insertions(+), 1 deletion(-) create mode 100644 tests/integration_tests/game_layer/actions/test_user_account_actions.py diff --git a/src/primaite/game/agent/actions.py b/src/primaite/game/agent/actions.py index 2e6189c0..a299788e 100644 --- a/src/primaite/game/agent/actions.py +++ b/src/primaite/game/agent/actions.py @@ -1116,6 +1116,38 @@ class ConfigureC2BeaconAction(AbstractAction): return ["network", "node", node_name, "application", "C2Beacon", "configure", config.__dict__] +class NodeAccountsAddUserAction(AbstractAction): + """Action which changes adds a User.""" + + def __init__(self, manager: "ActionManager", **kwargs) -> None: + super().__init__(manager=manager) + + def form_request(self, node_id: str, username: str, password: str, is_admin: bool) -> RequestFormat: + """Return the action formatted as a request which can be ingested by the PrimAITE simulation.""" + node_name = self.manager.get_node_name_by_idx(node_id) + return ["network", "node", node_name, "service", "UserManager", "add_user", username, password, is_admin] + + +class NodeAccountsDisableUserAction(AbstractAction): + """Action which disables a user.""" + + def __init__(self, manager: "ActionManager", **kwargs) -> None: + super().__init__(manager=manager) + + def form_request(self, node_id: str, username: str) -> RequestFormat: + """Return the action formatted as a request which can be ingested by the PrimAITE simulation.""" + node_name = self.manager.get_node_name_by_idx(node_id) + return [ + "network", + "node", + node_name, + "service", + "UserManager", + "disable_user", + username, + ] + + class NodeAccountsChangePasswordAction(AbstractAction): """Action which changes the password for a user.""" @@ -1368,6 +1400,8 @@ class ActionManager: "C2_SERVER_RANSOMWARE_CONFIGURE": RansomwareConfigureC2ServerAction, "C2_SERVER_TERMINAL_COMMAND": TerminalC2ServerAction, "C2_SERVER_DATA_EXFILTRATE": ExfiltrationC2ServerAction, + "NODE_ACCOUNTS_ADD_USER": NodeAccountsAddUserAction, + "NODE_ACCOUNTS_DISABLE_USER": NodeAccountsDisableUserAction, "NODE_ACCOUNTS_CHANGE_PASSWORD": NodeAccountsChangePasswordAction, "SSH_TO_REMOTE": NodeSessionsRemoteLoginAction, "SESSIONS_REMOTE_LOGOFF": NodeSessionsRemoteLogoutAction, diff --git a/src/primaite/simulator/network/hardware/base.py b/src/primaite/simulator/network/hardware/base.py index ef2d47c3..f49d0a17 100644 --- a/src/primaite/simulator/network/hardware/base.py +++ b/src/primaite/simulator/network/hardware/base.py @@ -857,7 +857,21 @@ class UserManager(Service): """ rm = super()._init_request_manager() - # todo add doc about requeest schemas + # todo add doc about request schemas + rm.add_request( + "add_user", + RequestType( + func=lambda request, context: RequestResponse.from_bool( + self.add_user(username=request[0], password=request[1], is_admin=request[2]) + ) + ), + ) + rm.add_request( + "disable_user", + RequestType( + func=lambda request, context: RequestResponse.from_bool(self.disable_user(username=request[0])) + ), + ) rm.add_request( "change_password", RequestType( diff --git a/tests/conftest.py b/tests/conftest.py index 1bbff8f2..50877378 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -463,6 +463,8 @@ def game_and_agent(): {"type": "C2_SERVER_RANSOMWARE_CONFIGURE"}, {"type": "C2_SERVER_TERMINAL_COMMAND"}, {"type": "C2_SERVER_DATA_EXFILTRATE"}, + {"type": "NODE_ACCOUNTS_ADD_USER"}, + {"type": "NODE_ACCOUNTS_DISABLE_USER"}, {"type": "NODE_ACCOUNTS_CHANGE_PASSWORD"}, {"type": "SSH_TO_REMOTE"}, {"type": "SESSIONS_REMOTE_LOGOFF"}, diff --git a/tests/integration_tests/game_layer/actions/test_user_account_actions.py b/tests/integration_tests/game_layer/actions/test_user_account_actions.py new file mode 100644 index 00000000..fd720315 --- /dev/null +++ b/tests/integration_tests/game_layer/actions/test_user_account_actions.py @@ -0,0 +1,93 @@ +# © Crown-owned copyright 2024, Defence Science and Technology Laboratory UK +import pytest + +from primaite.simulator.network.hardware.nodes.host.computer import Computer + + +@pytest.fixture +def game_and_agent_fixture(game_and_agent): + """Create a game with a simple agent that can be controlled by the tests.""" + game, agent = game_and_agent + + client_1: Computer = game.simulation.network.get_node_by_hostname("client_1") + client_1.start_up_duration = 3 + + return (game, agent) + + +def test_user_account_add_user_action(game_and_agent_fixture): + """Tests the add user account action.""" + game, agent = game_and_agent_fixture + client_1 = game.simulation.network.get_node_by_hostname("client_1") + + assert len(client_1.user_manager.users) == 1 # admin is created by default + assert len(client_1.user_manager.admins) == 1 + + # add admin account + action = ( + "NODE_ACCOUNTS_ADD_USER", + {"node_id": 0, "username": "soccon_diiz", "password": "nuts", "is_admin": True}, + ) + agent.store_action(action) + game.step() + + assert len(client_1.user_manager.users) == 2 # new user added + assert len(client_1.user_manager.admins) == 2 + + # add non admin account + action = ( + "NODE_ACCOUNTS_ADD_USER", + {"node_id": 0, "username": "mike_rotch", "password": "password", "is_admin": False}, + ) + agent.store_action(action) + game.step() + + assert len(client_1.user_manager.users) == 3 # new user added + assert len(client_1.user_manager.admins) == 2 + + +def test_user_account_disable_user_action(game_and_agent_fixture): + """Tests the disable user account action.""" + game, agent = game_and_agent_fixture + client_1 = game.simulation.network.get_node_by_hostname("client_1") + + client_1.user_manager.add_user(username="test", password="icles", is_admin=True) + assert len(client_1.user_manager.users) == 2 # new user added + assert len(client_1.user_manager.admins) == 2 + + test_user = client_1.user_manager.users.get("test") + assert test_user + assert test_user.disabled is not True + + # disable test account + action = ( + "NODE_ACCOUNTS_DISABLE_USER", + { + "node_id": 0, + "username": "test", + }, + ) + agent.store_action(action) + game.step() + assert test_user.disabled + + +def test_user_account_change_password_action(game_and_agent_fixture): + """Tests the change password user account action.""" + game, agent = game_and_agent_fixture + client_1 = game.simulation.network.get_node_by_hostname("client_1") + + client_1.user_manager.add_user(username="test", password="icles", is_admin=True) + + test_user = client_1.user_manager.users.get("test") + assert test_user.password == "icles" + + # change account password + action = ( + "NODE_ACCOUNTS_CHANGE_PASSWORD", + {"node_id": 0, "username": "test", "current_password": "icles", "new_password": "2Hard_2_Hack"}, + ) + agent.store_action(action) + game.step() + + assert test_user.password == "2Hard_2_Hack" From a998b8e22b2fd605583fcc6f455894563c5c4ad5 Mon Sep 17 00:00:00 2001 From: Czar Echavez Date: Thu, 5 Sep 2024 16:47:17 +0100 Subject: [PATCH 16/95] #2345: remove try catch + todo - figure out why db connection cannot be made --- .../system/services/web_server/web_server.py | 54 ++++++++++--------- 1 file changed, 30 insertions(+), 24 deletions(-) diff --git a/src/primaite/simulator/system/services/web_server/web_server.py b/src/primaite/simulator/system/services/web_server/web_server.py index 4fc64e1f..f9f561df 100644 --- a/src/primaite/simulator/system/services/web_server/web_server.py +++ b/src/primaite/simulator/system/services/web_server/web_server.py @@ -108,37 +108,43 @@ class WebServer(Service): :type: payload: HttpRequestPacket """ response = HttpResponsePacket(status_code=HttpStatusCode.NOT_FOUND, payload=payload) - try: - parsed_url = urlparse(payload.request_url) - path = parsed_url.path.strip("/") - if len(path) < 1: + parsed_url = urlparse(payload.request_url) + path = parsed_url.path.strip("/") if parsed_url and parsed_url.path else "" + + if len(path) < 1: + # query succeeded + response.status_code = HttpStatusCode.OK + + if path.startswith("users"): + # get data from DatabaseServer + # get all users + if self._establish_db_connection(): + # unable to create a db connection + response.status_code = HttpStatusCode.INTERNAL_SERVER_ERROR + + if self.db_connection.query("SELECT"): # query succeeded + self.set_health_state(SoftwareHealthState.GOOD) response.status_code = HttpStatusCode.OK + else: + self.set_health_state(SoftwareHealthState.COMPROMISED) + return response - if path.startswith("users"): - # get data from DatabaseServer - # get all users - if not self.db_connection: - self._establish_db_connection() - - if self.db_connection.query("SELECT"): - # query succeeded - self.set_health_state(SoftwareHealthState.GOOD) - response.status_code = HttpStatusCode.OK - else: - self.set_health_state(SoftwareHealthState.COMPROMISED) - - return response - except Exception: # TODO: refactor this. Likely to cause silent bugs. (ADO ticket #2345 ) - # something went wrong on the server - response.status_code = HttpStatusCode.INTERNAL_SERVER_ERROR - return response - - def _establish_db_connection(self) -> None: + def _establish_db_connection(self) -> bool: """Establish a connection to db.""" + # if active db connection, return true + if self.db_connection: + return True + + # otherwise, try to create db connection db_client = self.software_manager.software.get("DatabaseClient") + + if db_client is None: + return False # database client not installed + self.db_connection: DatabaseClientConnection = db_client.get_new_connection() + return self.db_connection is not None def send( self, From 974aee90b37afd3be0cfddb159cddd63892d2bb4 Mon Sep 17 00:00:00 2001 From: "Archer.Bowen" Date: Fri, 6 Sep 2024 14:09:30 +0100 Subject: [PATCH 17/95] #2842 Added additional tests to confirm terminal functionality --- .../actions/test_user_account_actions.py | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/tests/integration_tests/game_layer/actions/test_user_account_actions.py b/tests/integration_tests/game_layer/actions/test_user_account_actions.py index fd720315..bb36ce73 100644 --- a/tests/integration_tests/game_layer/actions/test_user_account_actions.py +++ b/tests/integration_tests/game_layer/actions/test_user_account_actions.py @@ -2,6 +2,8 @@ import pytest from primaite.simulator.network.hardware.nodes.host.computer import Computer +from primaite.simulator.network.hardware.nodes.network.router import ACLAction +from primaite.simulator.network.transmission.transport_layer import Port @pytest.fixture @@ -91,3 +93,84 @@ def test_user_account_change_password_action(game_and_agent_fixture): game.step() assert test_user.password == "2Hard_2_Hack" + + +def test_user_account_create_terminal_action(game_and_agent_fixture): + """Tests that agents can use the terminal to create new users.""" + game, agent = game_and_agent_fixture + + router = game.simulation.network.get_node_by_hostname("router") + router.acl.add_rule(action=ACLAction.PERMIT, src_port=Port.SSH, dst_port=Port.SSH, position=4) + + server_1 = game.simulation.network.get_node_by_hostname("server_1") + server_1_usm = server_1.software_manager.software["UserManager"] + server_1_usm.add_user("user123", "password", is_admin=True) + + action = ( + "SSH_TO_REMOTE", + { + "node_id": 0, + "username": "user123", + "password": "password", + "remote_ip": str(server_1.network_interface[1].ip_address), + }, + ) + agent.store_action(action) + game.step() + assert agent.history[-1].response.status == "success" + + # Create a new user account via terminal. + action = ( + "NODE_SEND_REMOTE_COMMAND", + { + "node_id": 0, + "remote_ip": str(server_1.network_interface[1].ip_address), + "command": ["service", "UserManager", "add_user", "new_user", "new_pass", True], + }, + ) + agent.store_action(action) + game.step() + new_user = server_1.user_manager.users.get("new_user") + assert new_user + assert new_user.password == "new_pass" + assert new_user.disabled is not True + + +def test_user_account_disable_terminal_action(game_and_agent_fixture): + """Tests that agents can use the terminal to disable users.""" + game, agent = game_and_agent_fixture + router = game.simulation.network.get_node_by_hostname("router") + router.acl.add_rule(action=ACLAction.PERMIT, src_port=Port.SSH, dst_port=Port.SSH, position=4) + + server_1 = game.simulation.network.get_node_by_hostname("server_1") + server_1_usm = server_1.software_manager.software["UserManager"] + server_1_usm.add_user("user123", "password", is_admin=True) + + action = ( + "SSH_TO_REMOTE", + { + "node_id": 0, + "username": "user123", + "password": "password", + "remote_ip": str(server_1.network_interface[1].ip_address), + }, + ) + agent.store_action(action) + game.step() + assert agent.history[-1].response.status == "success" + + # Disable a user via terminal + action = ( + "NODE_SEND_REMOTE_COMMAND", + { + "node_id": 0, + "remote_ip": str(server_1.network_interface[1].ip_address), + "command": ["service", "UserManager", "disable_user", "user123"], + }, + ) + agent.store_action(action) + game.step() + + new_user = server_1.user_manager.users.get("user123") + assert new_user + assert new_user.disabled is True From 5ab42ead273934a3132cf47c92cb784a0ccd27bb Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Mon, 9 Sep 2024 09:12:20 +0100 Subject: [PATCH 18/95] #2829: Add check for capture_nmne --- src/primaite/game/agent/observations/nic_observations.py | 7 +++++-- src/primaite/game/game.py | 2 ++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/primaite/game/agent/observations/nic_observations.py b/src/primaite/game/agent/observations/nic_observations.py index 002ee4da..c5da8767 100644 --- a/src/primaite/game/agent/observations/nic_observations.py +++ b/src/primaite/game/agent/observations/nic_observations.py @@ -1,18 +1,21 @@ # © Crown-owned copyright 2024, Defence Science and Technology Laboratory UK from __future__ import annotations -from typing import Dict, Optional +from typing import ClassVar, Dict, Optional from gymnasium import spaces from gymnasium.core import ObsType from primaite.game.agent.observations.observations import AbstractObservation, WhereType from primaite.game.agent.utils import access_from_nested_dict, NOT_PRESENT_IN_STATE +from primaite.simulator.network.nmne import NMNEConfig from primaite.simulator.network.transmission.transport_layer import Port class NICObservation(AbstractObservation, identifier="NETWORK_INTERFACE"): """Status information about a network interface within the simulation environment.""" + capture_nmne: ClassVar[bool] = NMNEConfig().capture_nmne + "A dataclass defining malicious network events to be captured." class ConfigSchema(AbstractObservation.ConfigSchema): """Configuration schema for NICObservation.""" @@ -164,7 +167,7 @@ class NICObservation(AbstractObservation, identifier="NETWORK_INTERFACE"): for port in self.monitored_traffic[protocol]: obs["TRAFFIC"][protocol][Port[port].value] = {"inbound": 0, "outbound": 0} - if self.include_nmne: + if self.capture_nmne and self.include_nmne: obs.update({"NMNE": {}}) direction_dict = nic_state["nmne"].get("direction", {}) inbound_keywords = direction_dict.get("inbound", {}).get("keywords", {}) diff --git a/src/primaite/game/game.py b/src/primaite/game/game.py index 045b2467..9c0f49af 100644 --- a/src/primaite/game/game.py +++ b/src/primaite/game/game.py @@ -10,6 +10,7 @@ from primaite import DEFAULT_BANDWIDTH, getLogger from primaite.game.agent.actions import ActionManager from primaite.game.agent.interface import AbstractAgent, AgentSettings, ProxyAgent from primaite.game.agent.observations.observation_manager import ObservationManager +from primaite.game.agent.observations import NICObservation from primaite.game.agent.rewards import RewardFunction, SharedReward from primaite.game.agent.scripted_agents.data_manipulation_bot import DataManipulationAgent from primaite.game.agent.scripted_agents.probabilistic_agent import ProbabilisticAgent @@ -275,6 +276,7 @@ class PrimaiteGame: links_cfg = network_config.get("links", []) # Set the NMNE capture config NetworkInterface.nmne_config = NMNEConfig(**network_config.get("nmne_config", {})) + NICObservation.capture_nmne = NMNEConfig(**network_config.get("nmne_config", {})).capture_nmne for node_cfg in nodes_cfg: n_type = node_cfg["type"] From 4a48a8d0547f00f186617cd4226d3853fd0e2be3 Mon Sep 17 00:00:00 2001 From: Czar Echavez Date: Mon, 9 Sep 2024 10:54:42 +0100 Subject: [PATCH 19/95] #2345: return error if db connection cannot be made --- .../simulator/system/services/web_server/web_server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/primaite/simulator/system/services/web_server/web_server.py b/src/primaite/simulator/system/services/web_server/web_server.py index f9f561df..0df47999 100644 --- a/src/primaite/simulator/system/services/web_server/web_server.py +++ b/src/primaite/simulator/system/services/web_server/web_server.py @@ -119,9 +119,10 @@ class WebServer(Service): if path.startswith("users"): # get data from DatabaseServer # get all users - if self._establish_db_connection(): + if not self._establish_db_connection(): # unable to create a db connection response.status_code = HttpStatusCode.INTERNAL_SERVER_ERROR + return response if self.db_connection.query("SELECT"): # query succeeded From 3cecf169bafab4b059ee27e3156f365e1bb9f3c9 Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Mon, 9 Sep 2024 16:30:36 +0100 Subject: [PATCH 20/95] #2829: Update and add nmne tests --- .../agent/observations/nic_observations.py | 3 ++- src/primaite/game/game.py | 2 +- .../observations/test_nic_observations.py | 8 +++++++ .../network/test_capture_nmne.py | 22 +++++++++++++++++++ 4 files changed, 33 insertions(+), 2 deletions(-) diff --git a/src/primaite/game/agent/observations/nic_observations.py b/src/primaite/game/agent/observations/nic_observations.py index c5da8767..ed2bb7f9 100644 --- a/src/primaite/game/agent/observations/nic_observations.py +++ b/src/primaite/game/agent/observations/nic_observations.py @@ -14,8 +14,9 @@ from primaite.simulator.network.transmission.transport_layer import Port class NICObservation(AbstractObservation, identifier="NETWORK_INTERFACE"): """Status information about a network interface within the simulation environment.""" + capture_nmne: ClassVar[bool] = NMNEConfig().capture_nmne - "A dataclass defining malicious network events to be captured." + "A Boolean specifying whether malicious network events should be captured." class ConfigSchema(AbstractObservation.ConfigSchema): """Configuration schema for NICObservation.""" diff --git a/src/primaite/game/game.py b/src/primaite/game/game.py index 9afdbea6..64cdf63b 100644 --- a/src/primaite/game/game.py +++ b/src/primaite/game/game.py @@ -9,8 +9,8 @@ from pydantic import BaseModel, ConfigDict from primaite import DEFAULT_BANDWIDTH, getLogger from primaite.game.agent.actions import ActionManager from primaite.game.agent.interface import AbstractAgent, AgentSettings, ProxyAgent -from primaite.game.agent.observations.observation_manager import ObservationManager from primaite.game.agent.observations import NICObservation +from primaite.game.agent.observations.observation_manager import ObservationManager from primaite.game.agent.rewards import RewardFunction, SharedReward from primaite.game.agent.scripted_agents.data_manipulation_bot import DataManipulationAgent from primaite.game.agent.scripted_agents.probabilistic_agent import ProbabilisticAgent diff --git a/tests/integration_tests/game_layer/observations/test_nic_observations.py b/tests/integration_tests/game_layer/observations/test_nic_observations.py index ef789ba7..ced598f0 100644 --- a/tests/integration_tests/game_layer/observations/test_nic_observations.py +++ b/tests/integration_tests/game_layer/observations/test_nic_observations.py @@ -77,6 +77,14 @@ def test_nic(simulation): nic_obs = NICObservation(where=["network", "nodes", pc.hostname, "NICs", 1], include_nmne=True) + # The Simulation object created by the fixture also creates the + # NICObservation class with the NICObservation.capture_nmnme class variable + # set to False. Under normal (non-test) circumstances this class variable + # is set from a config file such as data_manipulation.yaml. So although + # capture_nmne is set to True in the NetworkInterface class it's still False + # in the NICObservation class so we set it now. + nic_obs.capture_nmne = True + # Set the NMNE configuration to capture DELETE/ENCRYPT queries as MNEs nmne_config = { "capture_nmne": True, # Enable the capture of MNEs diff --git a/tests/integration_tests/network/test_capture_nmne.py b/tests/integration_tests/network/test_capture_nmne.py index debf5b1c..1499df9a 100644 --- a/tests/integration_tests/network/test_capture_nmne.py +++ b/tests/integration_tests/network/test_capture_nmne.py @@ -1,5 +1,11 @@ # © Crown-owned copyright 2024, Defence Science and Technology Laboratory UK +from itertools import product + +import yaml + +from primaite.config.load import data_manipulation_config_path from primaite.game.agent.observations.nic_observations import NICObservation +from primaite.session.environment import PrimaiteGymEnv from primaite.simulator.network.container import Network from primaite.simulator.network.hardware.nodes.host.host_node import NIC from primaite.simulator.network.hardware.nodes.host.server import Server @@ -277,3 +283,19 @@ def test_capture_nmne_observations(uc2_network: Network): assert web_nic_obs["outbound"] == expected_nmne assert db_nic_obs["inbound"] == expected_nmne uc2_network.apply_timestep(timestep=0) + + +def test_nmne_parameter_settings(): + """ + Check that the four permutations of the values of capture_nmne and + include_nmne work as expected. + """ + + with open(data_manipulation_config_path(), "r") as f: + cfg = yaml.safe_load(f) + + DEFENDER = 3 + for capture, include in product([True, False], [True, False]): + cfg["simulation"]["network"]["nmne_config"]["capture_nmne"] = capture + cfg["agents"][DEFENDER]["observation_space"]["options"]["components"][0]["options"]["include_nmne"] = include + PrimaiteGymEnv(env_config=cfg) From 82887bdb177258c1d9633b4860833b02c7b640f9 Mon Sep 17 00:00:00 2001 From: Czar Echavez Date: Tue, 10 Sep 2024 10:52:00 +0100 Subject: [PATCH 21/95] #2842: apply PR suggestions --- .../game_layer/actions/test_user_account_actions.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/integration_tests/game_layer/actions/test_user_account_actions.py b/tests/integration_tests/game_layer/actions/test_user_account_actions.py index bb36ce73..2fbf5a8c 100644 --- a/tests/integration_tests/game_layer/actions/test_user_account_actions.py +++ b/tests/integration_tests/game_layer/actions/test_user_account_actions.py @@ -28,7 +28,7 @@ def test_user_account_add_user_action(game_and_agent_fixture): # add admin account action = ( "NODE_ACCOUNTS_ADD_USER", - {"node_id": 0, "username": "soccon_diiz", "password": "nuts", "is_admin": True}, + {"node_id": 0, "username": "admin_2", "password": "e-tronic-boogaloo", "is_admin": True}, ) agent.store_action(action) game.step() @@ -39,7 +39,7 @@ def test_user_account_add_user_action(game_and_agent_fixture): # add non admin account action = ( "NODE_ACCOUNTS_ADD_USER", - {"node_id": 0, "username": "mike_rotch", "password": "password", "is_admin": False}, + {"node_id": 0, "username": "leeroy.jenkins", "password": "no_plan_needed", "is_admin": False}, ) agent.store_action(action) game.step() @@ -53,7 +53,7 @@ def test_user_account_disable_user_action(game_and_agent_fixture): game, agent = game_and_agent_fixture client_1 = game.simulation.network.get_node_by_hostname("client_1") - client_1.user_manager.add_user(username="test", password="icles", is_admin=True) + client_1.user_manager.add_user(username="test", password="password", is_admin=True) assert len(client_1.user_manager.users) == 2 # new user added assert len(client_1.user_manager.admins) == 2 @@ -79,7 +79,7 @@ def test_user_account_change_password_action(game_and_agent_fixture): game, agent = game_and_agent_fixture client_1 = game.simulation.network.get_node_by_hostname("client_1") - client_1.user_manager.add_user(username="test", password="icles", is_admin=True) + client_1.user_manager.add_user(username="test", password="password", is_admin=True) test_user = client_1.user_manager.users.get("test") assert test_user.password == "icles" @@ -87,7 +87,7 @@ def test_user_account_change_password_action(game_and_agent_fixture): # change account password action = ( "NODE_ACCOUNTS_CHANGE_PASSWORD", - {"node_id": 0, "username": "test", "current_password": "icles", "new_password": "2Hard_2_Hack"}, + {"node_id": 0, "username": "test", "current_password": "password", "new_password": "2Hard_2_Hack"}, ) agent.store_action(action) game.step() From 1c6e8b2a95227606ba99a66fb32bf40fe0e1225b Mon Sep 17 00:00:00 2001 From: Charlie Crane Date: Tue, 10 Sep 2024 11:39:04 +0100 Subject: [PATCH 22/95] #2775 - Removed default ARP rule for routers and added logic when handling ARP traffic --- .../simulator/network/hardware/nodes/network/router.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/primaite/simulator/network/hardware/nodes/network/router.py b/src/primaite/simulator/network/hardware/nodes/network/router.py index ceb91695..bfc90984 100644 --- a/src/primaite/simulator/network/hardware/nodes/network/router.py +++ b/src/primaite/simulator/network/hardware/nodes/network/router.py @@ -467,6 +467,14 @@ class AccessControlList(SimComponent): """Check if a packet with the given properties is permitted through the ACL.""" permitted = False rule: ACLRule = None + + # check if the frame is ARP and if ACL rules apply. + if frame.udp: + if frame.is_arp: + permitted = True + rule: ACLRule = None + return permitted, rule + for _rule in self._acl: if not _rule: continue @@ -1257,7 +1265,6 @@ class Router(NetworkNode): Initializes the router's ACL (Access Control List) with default rules, permitting essential protocols like ARP and ICMP, which are necessary for basic network operations and diagnostics. """ - self.acl.add_rule(action=ACLAction.PERMIT, src_port=Port.ARP, dst_port=Port.ARP, position=22) self.acl.add_rule(action=ACLAction.PERMIT, protocol=IPProtocol.ICMP, position=23) def setup_for_episode(self, episode: int): From 19d6fa2174b0d304ff4abc9a159087383d842c6f Mon Sep 17 00:00:00 2001 From: Charlie Crane Date: Wed, 11 Sep 2024 10:12:07 +0100 Subject: [PATCH 23/95] #2775 - Updated to look neater --- .../network/hardware/nodes/network/router.py | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/primaite/simulator/network/hardware/nodes/network/router.py b/src/primaite/simulator/network/hardware/nodes/network/router.py index bfc90984..3b267200 100644 --- a/src/primaite/simulator/network/hardware/nodes/network/router.py +++ b/src/primaite/simulator/network/hardware/nodes/network/router.py @@ -468,13 +468,6 @@ class AccessControlList(SimComponent): permitted = False rule: ACLRule = None - # check if the frame is ARP and if ACL rules apply. - if frame.udp: - if frame.is_arp: - permitted = True - rule: ACLRule = None - return permitted, rule - for _rule in self._acl: if not _rule: continue @@ -1376,6 +1369,12 @@ class Router(NetworkNode): return False + def subject_to_acl(self, frame: Frame) -> bool: + """Check that frame is subject to ACL rules.""" + if frame.ip.protocol == IPProtocol.UDP and frame.udp.dst_port == Port.ARP: + return False + return True + def receive_frame(self, frame: Frame, from_network_interface: RouterInterface): """ Processes an incoming frame received on one of the router's interfaces. @@ -1389,8 +1388,12 @@ class Router(NetworkNode): if self.operating_state != NodeOperatingState.ON: return - # Check if it's permitted - permitted, rule = self.acl.is_permitted(frame) + if self.subject_to_acl(frame=frame): + # Check if it's permitted + permitted, rule = self.acl.is_permitted(frame) + else: + permitted = True + rule = None if not permitted: at_port = self._get_port_of_nic(from_network_interface) From a2005df9f0d0d1d3f32051058d5bc3e4b7cfcedf Mon Sep 17 00:00:00 2001 From: Charlie Crane Date: Wed, 11 Sep 2024 10:56:44 +0100 Subject: [PATCH 24/95] #2775 - Documentation updates --- docs/source/configuration/simulation/nodes/router.rst | 6 +----- docs/source/simulation_components/network/network.rst | 9 +-------- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/docs/source/configuration/simulation/nodes/router.rst b/docs/source/configuration/simulation/nodes/router.rst index ac9d6411..b8741521 100644 --- a/docs/source/configuration/simulation/nodes/router.rst +++ b/docs/source/configuration/simulation/nodes/router.rst @@ -74,7 +74,7 @@ The subnet mask setting for the port. ``acl`` ------- -Sets up the ACL rules for the router. +Sets up the ACL rules for the router to apply to layer-3 traffic. These are not applied to layer-2 traffic such as ARP. e.g. @@ -85,10 +85,6 @@ e.g. ... acl: 1: - action: PERMIT - src_port: ARP - dst_port: ARP - 2: action: PERMIT protocol: ICMP diff --git a/docs/source/simulation_components/network/network.rst b/docs/source/simulation_components/network/network.rst index 636ffbcc..00781307 100644 --- a/docs/source/simulation_components/network/network.rst +++ b/docs/source/simulation_components/network/network.rst @@ -97,17 +97,10 @@ we'll use the following Network that has a client, server, two switches, and a r network.connect(endpoint_a=switch_2.network_interface[1], endpoint_b=client_1.network_interface[1]) network.connect(endpoint_a=switch_1.network_interface[1], endpoint_b=server_1.network_interface[1]) -8. Add ACL rules on the Router to allow ARP and ICMP traffic. +8. Add an ACL rules on the Router to allow ICMP traffic. .. code-block:: python - router_1.acl.add_rule( - action=ACLAction.PERMIT, - src_port=Port.ARP, - dst_port=Port.ARP, - position=22 - ) - router_1.acl.add_rule( action=ACLAction.PERMIT, protocol=IPProtocol.ICMP, From d5f1d0fda184aa4cfdf9f7ae1fb032f7073ed145 Mon Sep 17 00:00:00 2001 From: Charlie Crane Date: Wed, 11 Sep 2024 11:28:27 +0100 Subject: [PATCH 25/95] #2775 - Updated Changelog and bring up to date with dev --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e2989247..77b7bb7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added - Log observation space data by episode and step. +- ACL's are no longer applied to layer-2 traffic. ## [3.3.0] - 2024-09-04 ### Added From f95501f2a872099f616fd5cabfac75d0ddaf4b18 Mon Sep 17 00:00:00 2001 From: Charlie Crane Date: Wed, 11 Sep 2024 15:12:36 +0100 Subject: [PATCH 26/95] #2775 - Purging of more instances where the ARP acl rule is set and no longer necessary. Added a new test to show ARP is unaffected by ACL rules and actioned review comments --- .../simulation_components/network/network.rst | 2 +- .../network/nodes/wireless_router.rst | 1 - src/primaite/simulator/network/creation.py | 2 -- .../network/hardware/nodes/network/router.py | 3 +++ src/primaite/simulator/network/networks.py | 4 ---- .../network/transmission/data_link_layer.py | 2 +- tests/conftest.py | 3 --- .../game_layer/test_actions.py | 14 ++++++------- .../integration_tests/network/test_routing.py | 1 - .../network/test_wireless_router.py | 1 - tests/integration_tests/system/test_arp.py | 20 ++++++++++++++++++- .../_system/_services/test_terminal.py | 1 - 12 files changed, 31 insertions(+), 23 deletions(-) diff --git a/docs/source/simulation_components/network/network.rst b/docs/source/simulation_components/network/network.rst index 00781307..b04d6ecf 100644 --- a/docs/source/simulation_components/network/network.rst +++ b/docs/source/simulation_components/network/network.rst @@ -97,7 +97,7 @@ we'll use the following Network that has a client, server, two switches, and a r network.connect(endpoint_a=switch_2.network_interface[1], endpoint_b=client_1.network_interface[1]) network.connect(endpoint_a=switch_1.network_interface[1], endpoint_b=server_1.network_interface[1]) -8. Add an ACL rules on the Router to allow ICMP traffic. +8. Add an ACL rule on the Router to allow ICMP traffic. .. code-block:: python diff --git a/docs/source/simulation_components/network/nodes/wireless_router.rst b/docs/source/simulation_components/network/nodes/wireless_router.rst index c78c8419..02fe73db 100644 --- a/docs/source/simulation_components/network/nodes/wireless_router.rst +++ b/docs/source/simulation_components/network/nodes/wireless_router.rst @@ -102,7 +102,6 @@ ICMP traffic, ensuring basic network connectivity and ping functionality. network.connect(pc_a.network_interface[1], router_1.router_interface) # Configure Router 1 ACLs - router_1.acl.add_rule(action=ACLAction.PERMIT, src_port=Port.ARP, dst_port=Port.ARP, position=22) router_1.acl.add_rule(action=ACLAction.PERMIT, protocol=IPProtocol.ICMP, position=23) # Configure PC B diff --git a/src/primaite/simulator/network/creation.py b/src/primaite/simulator/network/creation.py index 61a37a90..b801a38e 100644 --- a/src/primaite/simulator/network/creation.py +++ b/src/primaite/simulator/network/creation.py @@ -7,7 +7,6 @@ from primaite.simulator.network.hardware.nodes.host.computer import Computer from primaite.simulator.network.hardware.nodes.network.router import ACLAction, Router from primaite.simulator.network.hardware.nodes.network.switch import Switch from primaite.simulator.network.transmission.network_layer import IPProtocol -from primaite.simulator.network.transmission.transport_layer import Port def num_of_switches_required(num_nodes: int, max_network_interface: int = 24) -> int: @@ -98,7 +97,6 @@ def create_office_lan( default_gateway = IPv4Address(f"192.168.{subnet_base}.1") router = Router(hostname=f"router_{lan_name}", start_up_duration=0) router.power_on() - router.acl.add_rule(action=ACLAction.PERMIT, src_port=Port.ARP, dst_port=Port.ARP, position=22) router.acl.add_rule(action=ACLAction.PERMIT, protocol=IPProtocol.ICMP, position=23) network.add_node(router) router.configure_port(port=1, ip_address=default_gateway, subnet_mask="255.255.255.0") diff --git a/src/primaite/simulator/network/hardware/nodes/network/router.py b/src/primaite/simulator/network/hardware/nodes/network/router.py index 3b267200..e86b1843 100644 --- a/src/primaite/simulator/network/hardware/nodes/network/router.py +++ b/src/primaite/simulator/network/hardware/nodes/network/router.py @@ -1388,6 +1388,9 @@ class Router(NetworkNode): if self.operating_state != NodeOperatingState.ON: return + print("£££££££££££££££££££££££££££££") + print(f"Frame received is: {frame}") + if self.subject_to_acl(frame=frame): # Check if it's permitted permitted, rule = self.acl.is_permitted(frame) diff --git a/src/primaite/simulator/network/networks.py b/src/primaite/simulator/network/networks.py index cb0965eb..ae6476c1 100644 --- a/src/primaite/simulator/network/networks.py +++ b/src/primaite/simulator/network/networks.py @@ -79,8 +79,6 @@ def client_server_routed() -> Network: server_1.power_on() network.connect(endpoint_b=server_1.network_interface[1], endpoint_a=switch_1.network_interface[1]) - router_1.acl.add_rule(action=ACLAction.PERMIT, src_port=Port.ARP, dst_port=Port.ARP, position=22) - router_1.acl.add_rule(action=ACLAction.PERMIT, protocol=IPProtocol.ICMP, position=23) return network @@ -271,8 +269,6 @@ def arcd_uc2_network() -> Network: security_suite.connect_nic(NIC(ip_address="192.168.10.110", subnet_mask="255.255.255.0")) network.connect(endpoint_b=security_suite.network_interface[2], endpoint_a=switch_2.network_interface[7]) - router_1.acl.add_rule(action=ACLAction.PERMIT, src_port=Port.ARP, dst_port=Port.ARP, position=22) - router_1.acl.add_rule(action=ACLAction.PERMIT, protocol=IPProtocol.ICMP, position=23) # Allow PostgreSQL requests diff --git a/src/primaite/simulator/network/transmission/data_link_layer.py b/src/primaite/simulator/network/transmission/data_link_layer.py index 159eca7f..9d8a0a1c 100644 --- a/src/primaite/simulator/network/transmission/data_link_layer.py +++ b/src/primaite/simulator/network/transmission/data_link_layer.py @@ -161,7 +161,7 @@ class Frame(BaseModel): """ Checks if the Frame is an ARP (Address Resolution Protocol) packet. - This is determined by checking if the destination port of the TCP header is equal to the ARP port. + This is determined by checking if the destination port of the UDP header is equal to the ARP port. :return: True if the Frame is an ARP packet, otherwise False. """ diff --git a/tests/conftest.py b/tests/conftest.py index 1bbff8f2..e9aeada8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -350,7 +350,6 @@ def install_stuff_to_sim(sim: Simulation): network.connect(endpoint_a=server_2.network_interface[1], endpoint_b=switch_2.network_interface[2]) # 2: Configure base ACL - router.acl.add_rule(action=ACLAction.PERMIT, src_port=Port.ARP, dst_port=Port.ARP, position=22) router.acl.add_rule(action=ACLAction.PERMIT, protocol=IPProtocol.ICMP, position=23) router.acl.add_rule(action=ACLAction.PERMIT, src_port=Port.DNS, dst_port=Port.DNS, position=1) router.acl.add_rule(action=ACLAction.PERMIT, src_port=Port.HTTP, dst_port=Port.HTTP, position=3) @@ -382,8 +381,6 @@ def install_stuff_to_sim(sim: Simulation): assert acl_rule.src_port == acl_rule.dst_port == Port.DNS elif i == 3: assert acl_rule.src_port == acl_rule.dst_port == Port.HTTP - elif i == 22: - assert acl_rule.src_port == acl_rule.dst_port == Port.ARP elif i == 23: assert acl_rule.protocol == IPProtocol.ICMP elif i == 24: diff --git a/tests/integration_tests/game_layer/test_actions.py b/tests/integration_tests/game_layer/test_actions.py index a1005f34..ecd21a03 100644 --- a/tests/integration_tests/game_layer/test_actions.py +++ b/tests/integration_tests/game_layer/test_actions.py @@ -115,7 +115,7 @@ def test_router_acl_addrule_integration(game_and_agent: Tuple[PrimaiteGame, Prox server_1 = game.simulation.network.get_node_by_hostname("server_1") server_2 = game.simulation.network.get_node_by_hostname("server_2") router = game.simulation.network.get_node_by_hostname("router") - assert router.acl.num_rules == 4 + assert router.acl.num_rules == 3 assert client_1.ping("10.0.2.3") # client_1 can ping server_2 assert server_2.ping("10.0.1.2") # server_2 can ping client_1 @@ -138,8 +138,8 @@ def test_router_acl_addrule_integration(game_and_agent: Tuple[PrimaiteGame, Prox agent.store_action(action) game.step() - # 3: Check that the ACL now has 5 rules, and that client 1 cannot ping server 2 - assert router.acl.num_rules == 5 + # 3: Check that the ACL now has 4 rules, and that client 1 cannot ping server 2 + assert router.acl.num_rules == 4 assert not client_1.ping("10.0.2.3") # Cannot ping server_2 assert client_1.ping("10.0.2.2") # Can ping server_1 assert not server_2.ping( @@ -165,8 +165,8 @@ def test_router_acl_addrule_integration(game_and_agent: Tuple[PrimaiteGame, Prox agent.store_action(action) game.step() - # 5: Check that the ACL now has 6 rules, but that server_1 can still ping server_2 - assert router.acl.num_rules == 6 + # 5: Check that the ACL now has 5 rules, but that server_1 can still ping server_2 + assert router.acl.num_rules == 5 assert server_1.ping("10.0.2.3") # Can ping server_2 @@ -195,8 +195,8 @@ def test_router_acl_removerule_integration(game_and_agent: Tuple[PrimaiteGame, P agent.store_action(action) game.step() - # 3: Check that the ACL now has 3 rules, and that client 1 cannot access example.com - assert router.acl.num_rules == 3 + # 3: Check that the ACL now has 2 rules, and that client 1 cannot access example.com + assert router.acl.num_rules == 2 assert not browser.get_webpage() client_1.software_manager.software.get("DNSClient").dns_cache.clear() assert client_1.ping("10.0.2.2") # pinging still works because ICMP is allowed diff --git a/tests/integration_tests/network/test_routing.py b/tests/integration_tests/network/test_routing.py index 62b58cbd..e234b4e5 100644 --- a/tests/integration_tests/network/test_routing.py +++ b/tests/integration_tests/network/test_routing.py @@ -73,7 +73,6 @@ def multi_hop_network() -> Network: router_1.enable_port(2) # Configure Router 1 ACLs - router_1.acl.add_rule(action=ACLAction.PERMIT, src_port=Port.ARP, dst_port=Port.ARP, position=22) router_1.acl.add_rule(action=ACLAction.PERMIT, protocol=IPProtocol.ICMP, position=23) # Configure PC B diff --git a/tests/integration_tests/network/test_wireless_router.py b/tests/integration_tests/network/test_wireless_router.py index 733de6f6..9a22208b 100644 --- a/tests/integration_tests/network/test_wireless_router.py +++ b/tests/integration_tests/network/test_wireless_router.py @@ -37,7 +37,6 @@ def wireless_wan_network(): network.connect(pc_a.network_interface[1], router_1.network_interface[2]) # Configure Router 1 ACLs - router_1.acl.add_rule(action=ACLAction.PERMIT, src_port=Port.ARP, dst_port=Port.ARP, position=22) router_1.acl.add_rule(action=ACLAction.PERMIT, protocol=IPProtocol.ICMP, position=23) # Configure PC B diff --git a/tests/integration_tests/system/test_arp.py b/tests/integration_tests/system/test_arp.py index be8656aa..6c7e853a 100644 --- a/tests/integration_tests/system/test_arp.py +++ b/tests/integration_tests/system/test_arp.py @@ -1,5 +1,7 @@ # © Crown-owned copyright 2024, Defence Science and Technology Laboratory UK -from primaite.simulator.network.hardware.nodes.network.router import RouterARP +from primaite.simulator.network.hardware.nodes.network.router import ACLAction, Router, RouterARP +from primaite.simulator.network.transmission.network_layer import IPProtocol +from primaite.simulator.network.transmission.transport_layer import Port from primaite.simulator.system.services.arp.arp import ARP from tests.integration_tests.network.test_routing import multi_hop_network @@ -48,3 +50,19 @@ def test_arp_fails_for_network_address_between_routers(multi_hop_network): actual_result = router_1_arp.get_arp_cache_mac_address(router_1.network_interface[1].ip_network.network_address) assert actual_result == expected_result + + +def test_arp_not_affected_by_acl(multi_hop_network): + pc_a = multi_hop_network.get_node_by_hostname("pc_a") + router_1: Router = multi_hop_network.get_node_by_hostname("router_1") + + # Add explicit rule to block ARP traffic. This shouldn't actually stop ARP traffic + # as it operates a different layer within the network. + router_1.acl.add_rule(action=ACLAction.DENY, src_port=Port.ARP, dst_port=Port.ARP, position=23) + + pc_a_arp: ARP = pc_a.software_manager.arp + + expected_result = router_1.network_interface[2].mac_address + actual_result = pc_a_arp.get_arp_cache_mac_address(router_1.network_interface[2].ip_address) + + assert actual_result == expected_result diff --git a/tests/unit_tests/_primaite/_simulator/_system/_services/test_terminal.py b/tests/unit_tests/_primaite/_simulator/_system/_services/test_terminal.py index 41858b90..3c3daa61 100644 --- a/tests/unit_tests/_primaite/_simulator/_system/_services/test_terminal.py +++ b/tests/unit_tests/_primaite/_simulator/_system/_services/test_terminal.py @@ -77,7 +77,6 @@ def wireless_wan_network(): network.connect(pc_a.network_interface[1], router_1.network_interface[2]) # Configure Router 1 ACLs - router_1.acl.add_rule(action=ACLAction.PERMIT, src_port=Port.ARP, dst_port=Port.ARP, position=22) router_1.acl.add_rule(action=ACLAction.PERMIT, protocol=IPProtocol.ICMP, position=23) # add ACL rule to allow SSH traffic From 85863b1972516a9c40b441e23ff2865bd82fe437 Mon Sep 17 00:00:00 2001 From: Charlie Crane Date: Wed, 11 Sep 2024 15:36:51 +0100 Subject: [PATCH 27/95] #2775 - Removed a print statement committed in error and updated the checks done in subject_to_acl following review --- .../simulator/network/hardware/nodes/network/router.py | 5 +---- .../simulator/network/transmission/data_link_layer.py | 4 ++-- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/primaite/simulator/network/hardware/nodes/network/router.py b/src/primaite/simulator/network/hardware/nodes/network/router.py index e86b1843..8cdf3f86 100644 --- a/src/primaite/simulator/network/hardware/nodes/network/router.py +++ b/src/primaite/simulator/network/hardware/nodes/network/router.py @@ -1371,7 +1371,7 @@ class Router(NetworkNode): def subject_to_acl(self, frame: Frame) -> bool: """Check that frame is subject to ACL rules.""" - if frame.ip.protocol == IPProtocol.UDP and frame.udp.dst_port == Port.ARP: + if frame.ip.protocol == IPProtocol.UDP and frame.is_arp: return False return True @@ -1388,9 +1388,6 @@ class Router(NetworkNode): if self.operating_state != NodeOperatingState.ON: return - print("£££££££££££££££££££££££££££££") - print(f"Frame received is: {frame}") - if self.subject_to_acl(frame=frame): # Check if it's permitted permitted, rule = self.acl.is_permitted(frame) diff --git a/src/primaite/simulator/network/transmission/data_link_layer.py b/src/primaite/simulator/network/transmission/data_link_layer.py index 9d8a0a1c..86a6038b 100644 --- a/src/primaite/simulator/network/transmission/data_link_layer.py +++ b/src/primaite/simulator/network/transmission/data_link_layer.py @@ -161,11 +161,11 @@ class Frame(BaseModel): """ Checks if the Frame is an ARP (Address Resolution Protocol) packet. - This is determined by checking if the destination port of the UDP header is equal to the ARP port. + This is determined by checking if the destination and source port of the UDP header is equal to the ARP port. :return: True if the Frame is an ARP packet, otherwise False. """ - return self.udp.dst_port == Port.ARP + return self.udp.dst_port == Port.ARP and self.udp.src_port == Port.ARP @property def is_icmp(self) -> bool: From f908f9b23e43f910ee8b1e732ca546e2d3b954ca Mon Sep 17 00:00:00 2001 From: Charlie Crane Date: Wed, 11 Sep 2024 15:50:14 +0100 Subject: [PATCH 28/95] #2775 - Actioning review comments --- tests/integration_tests/game_layer/test_actions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration_tests/game_layer/test_actions.py b/tests/integration_tests/game_layer/test_actions.py index ecd21a03..a9231632 100644 --- a/tests/integration_tests/game_layer/test_actions.py +++ b/tests/integration_tests/game_layer/test_actions.py @@ -106,7 +106,7 @@ def test_router_acl_addrule_integration(game_and_agent: Tuple[PrimaiteGame, Prox """ Test that the RouterACLAddRuleAction can form a request and that it is accepted by the simulation. - The ACL starts off with 4 rules, and we add a rule, and check that the ACL now has 5 rules. + The ACL starts off with 3 rules, and we add a rule, and check that the ACL now has 4 rules. """ game, agent = game_and_agent From 8bd20275d085fc79b016e690ebabff0b4d52008f Mon Sep 17 00:00:00 2001 From: Czar Echavez Date: Thu, 12 Sep 2024 10:01:12 +0100 Subject: [PATCH 29/95] #2842: fix test --- .../game_layer/actions/test_user_account_actions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration_tests/game_layer/actions/test_user_account_actions.py b/tests/integration_tests/game_layer/actions/test_user_account_actions.py index 2fbf5a8c..f97716c6 100644 --- a/tests/integration_tests/game_layer/actions/test_user_account_actions.py +++ b/tests/integration_tests/game_layer/actions/test_user_account_actions.py @@ -82,7 +82,7 @@ def test_user_account_change_password_action(game_and_agent_fixture): client_1.user_manager.add_user(username="test", password="password", is_admin=True) test_user = client_1.user_manager.users.get("test") - assert test_user.password == "icles" + assert test_user.password == "password" # change account password action = ( From 7c26ca9d79d14bc529368f455909a097c8a4614c Mon Sep 17 00:00:00 2001 From: Czar Echavez Date: Thu, 12 Sep 2024 16:07:14 +0100 Subject: [PATCH 30/95] #2864: add configuration for services_requires_scan and applications_requires_scan --- .../agent/observations/host_observations.py | 24 +++- .../agent/observations/node_observations.py | 12 +- .../observations/software_observation.py | 33 ++++-- .../_game/_agent/test_observations.py | 111 +++++++++++++++++- 4 files changed, 169 insertions(+), 11 deletions(-) diff --git a/src/primaite/game/agent/observations/host_observations.py b/src/primaite/game/agent/observations/host_observations.py index 4419ccc7..3371a99c 100644 --- a/src/primaite/game/agent/observations/host_observations.py +++ b/src/primaite/game/agent/observations/host_observations.py @@ -52,6 +52,14 @@ class HostObservation(AbstractObservation, identifier="HOST"): """ If True, files and folders must be scanned to update the health state. If False, true state is always shown. """ + services_requires_scan: Optional[bool] = None + """ + If True, services must be scanned to update the health state. If False, true state is always shown. + """ + applications_requires_scan: Optional[bool] = None + """ + If True, applications must be scanned to update the health state. If False, true state is always shown. + """ include_users: Optional[bool] = True """If True, report user session information.""" @@ -71,6 +79,8 @@ class HostObservation(AbstractObservation, identifier="HOST"): monitored_traffic: Optional[Dict], include_num_access: bool, file_system_requires_scan: bool, + services_requires_scan: bool, + applications_requires_scan: bool, include_users: bool, ) -> None: """ @@ -106,6 +116,12 @@ class HostObservation(AbstractObservation, identifier="HOST"): :param file_system_requires_scan: If True, the files and folders must be scanned to update the health state. If False, the true state is always shown. :type file_system_requires_scan: bool + :param services_requires_scan: If True, services must be scanned to update the health state. + If False, the true state is always shown. + :type services_requires_scan: bool + :param applications_requires_scan: If True, applications must be scanned to update the health state. + If False, the true state is always shown. + :type applications_requires_scan: bool :param include_users: If True, report user session information. :type include_users: bool """ @@ -119,7 +135,7 @@ class HostObservation(AbstractObservation, identifier="HOST"): # Ensure lists have lengths equal to specified counts by truncating or padding self.services: List[ServiceObservation] = services while len(self.services) < num_services: - self.services.append(ServiceObservation(where=None)) + self.services.append(ServiceObservation(where=None, services_requires_scan=services_requires_scan)) while len(self.services) > num_services: truncated_service = self.services.pop() msg = f"Too many services in Node observation space for node. Truncating service {truncated_service.where}" @@ -127,7 +143,9 @@ class HostObservation(AbstractObservation, identifier="HOST"): self.applications: List[ApplicationObservation] = applications while len(self.applications) < num_applications: - self.applications.append(ApplicationObservation(where=None)) + self.applications.append( + ApplicationObservation(where=None, applications_requires_scan=applications_requires_scan) + ) while len(self.applications) > num_applications: truncated_application = self.applications.pop() msg = f"Too many applications in Node observation space for node. Truncating {truncated_application.where}" @@ -293,5 +311,7 @@ class HostObservation(AbstractObservation, identifier="HOST"): monitored_traffic=config.monitored_traffic, include_num_access=config.include_num_access, file_system_requires_scan=config.file_system_requires_scan, + services_requires_scan=config.services_requires_scan, + applications_requires_scan=config.applications_requires_scan, include_users=config.include_users, ) diff --git a/src/primaite/game/agent/observations/node_observations.py b/src/primaite/game/agent/observations/node_observations.py index e263cadb..85de5396 100644 --- a/src/primaite/game/agent/observations/node_observations.py +++ b/src/primaite/game/agent/observations/node_observations.py @@ -45,7 +45,13 @@ class NodesObservation(AbstractObservation, identifier="NODES"): include_num_access: Optional[bool] = None """Flag to include the number of accesses.""" file_system_requires_scan: bool = True - """If True, the folder must be scanned to update the health state. Tf False, the true state is always shown.""" + """If True, the folder must be scanned to update the health state. If False, the true state is always shown.""" + services_requires_scan: bool = True + """If True, the services must be scanned to update the health state. + If False, the true state is always shown.""" + applications_requires_scan: bool = True + """If True, the applications must be scanned to update the health state. + If False, the true state is always shown.""" include_users: Optional[bool] = True """If True, report user session information.""" num_ports: Optional[int] = None @@ -193,6 +199,10 @@ class NodesObservation(AbstractObservation, identifier="NODES"): host_config.include_num_access = config.include_num_access if host_config.file_system_requires_scan is None: host_config.file_system_requires_scan = config.file_system_requires_scan + if host_config.services_requires_scan is None: + host_config.services_requires_scan = config.services_requires_scan + if host_config.applications_requires_scan is None: + host_config.applications_requires_scan = config.applications_requires_scan if host_config.include_users is None: host_config.include_users = config.include_users diff --git a/src/primaite/game/agent/observations/software_observation.py b/src/primaite/game/agent/observations/software_observation.py index 15cd2447..2075ce43 100644 --- a/src/primaite/game/agent/observations/software_observation.py +++ b/src/primaite/game/agent/observations/software_observation.py @@ -1,7 +1,7 @@ # © Crown-owned copyright 2024, Defence Science and Technology Laboratory UK from __future__ import annotations -from typing import Dict +from typing import Dict, Optional from gymnasium import spaces from gymnasium.core import ObsType @@ -19,7 +19,10 @@ class ServiceObservation(AbstractObservation, identifier="SERVICE"): service_name: str """Name of the service, used for querying simulation state dictionary""" - def __init__(self, where: WhereType) -> None: + services_requires_scan: Optional[bool] = None + """If True, services must be scanned to update the health state. If False, true state is always shown.""" + + def __init__(self, where: WhereType, services_requires_scan: bool) -> None: """ Initialise a service observation instance. @@ -28,6 +31,7 @@ class ServiceObservation(AbstractObservation, identifier="SERVICE"): :type where: WhereType """ self.where = where + self.services_requires_scan = services_requires_scan self.default_observation = {"operating_status": 0, "health_status": 0} def observe(self, state: Dict) -> ObsType: @@ -44,7 +48,9 @@ class ServiceObservation(AbstractObservation, identifier="SERVICE"): return self.default_observation return { "operating_status": service_state["operating_state"], - "health_status": service_state["health_state_visible"], + "health_status": service_state["health_state_visible"] + if self.services_requires_scan + else service_state["health_state_actual"], } @property @@ -70,7 +76,9 @@ class ServiceObservation(AbstractObservation, identifier="SERVICE"): :return: Constructed service observation instance. :rtype: ServiceObservation """ - return cls(where=parent_where + ["services", config.service_name]) + return cls( + where=parent_where + ["services", config.service_name], services_requires_scan=config.services_requires_scan + ) class ApplicationObservation(AbstractObservation, identifier="APPLICATION"): @@ -82,7 +90,12 @@ class ApplicationObservation(AbstractObservation, identifier="APPLICATION"): application_name: str """Name of the application, used for querying simulation state dictionary""" - def __init__(self, where: WhereType) -> None: + applications_requires_scan: Optional[bool] = None + """ + If True, applications must be scanned to update the health state. If False, true state is always shown. + """ + + def __init__(self, where: WhereType, applications_requires_scan: bool) -> None: """ Initialise an application observation instance. @@ -92,6 +105,7 @@ class ApplicationObservation(AbstractObservation, identifier="APPLICATION"): :type where: WhereType """ self.where = where + self.applications_requires_scan = applications_requires_scan self.default_observation = {"operating_status": 0, "health_status": 0, "num_executions": 0} # TODO: allow these to be configured in yaml @@ -128,7 +142,9 @@ class ApplicationObservation(AbstractObservation, identifier="APPLICATION"): return self.default_observation return { "operating_status": application_state["operating_state"], - "health_status": application_state["health_state_visible"], + "health_status": application_state["health_state_visible"] + if self.applications_requires_scan + else application_state["health_state_actual"], "num_executions": self._categorise_num_executions(application_state["num_executions"]), } @@ -161,4 +177,7 @@ class ApplicationObservation(AbstractObservation, identifier="APPLICATION"): :return: Constructed application observation instance. :rtype: ApplicationObservation """ - return cls(where=parent_where + ["applications", config.application_name]) + return cls( + where=parent_where + ["applications", config.application_name], + applications_requires_scan=config.applications_requires_scan, + ) diff --git a/tests/unit_tests/_primaite/_game/_agent/test_observations.py b/tests/unit_tests/_primaite/_game/_agent/test_observations.py index 7f590685..583b9cbd 100644 --- a/tests/unit_tests/_primaite/_game/_agent/test_observations.py +++ b/tests/unit_tests/_primaite/_game/_agent/test_observations.py @@ -4,7 +4,7 @@ from typing import List import pytest import yaml -from primaite.game.agent.observations import ObservationManager +from primaite.game.agent.observations import ObservationManager, ServiceObservation from primaite.game.agent.observations.file_system_observations import FileObservation, FolderObservation from primaite.game.agent.observations.host_observations import HostObservation @@ -130,3 +130,112 @@ class TestFileSystemRequiresScan: [], files=[], num_files=0, include_num_access=False, file_system_requires_scan=False ) assert obs_not_requiring_scan.observe(folder_state)["health_status"] == 3 + + +class TestServiceRequiresScan: + @pytest.mark.parametrize( + ("yaml_option_string", "expected_val"), + ( + ("services_requires_scan: true", True), + ("services_requires_scan: false", False), + (" ", True), + ), + ) + def test_obs_config(self, yaml_option_string, expected_val): + """Check that the default behaviour is to set FileSystemRequiresScan to True.""" + obs_cfg_yaml = f""" + type: CUSTOM + options: + components: + - type: NODES + label: NODES + options: + hosts: + - hostname: domain_controller + - hostname: web_server + services: + - service_name: WebServer + - hostname: database_server + folders: + - folder_name: database + files: + - file_name: database.db + - hostname: backup_server + - hostname: security_suite + - hostname: client_1 + applications: + - application_name: WebBrowser + - hostname: client_2 + num_services: 1 + num_applications: 1 + num_folders: 1 + num_files: 1 + num_nics: 2 + include_num_access: false + {yaml_option_string} + include_nmne: true + monitored_traffic: + icmp: + - NONE + tcp: + - DNS + routers: + - hostname: router_1 + num_ports: 0 + ip_list: + - 192.168.1.10 + - 192.168.1.12 + - 192.168.1.14 + - 192.168.1.16 + - 192.168.1.110 + - 192.168.10.21 + - 192.168.10.22 + - 192.168.10.110 + wildcard_list: + - 0.0.0.1 + port_list: + - 80 + - 5432 + protocol_list: + - ICMP + - TCP + - UDP + num_rules: 10 + + - type: LINKS + label: LINKS + options: + link_references: + - router_1:eth-1<->switch_1:eth-8 + - router_1:eth-2<->switch_2:eth-8 + - switch_1:eth-1<->domain_controller:eth-1 + - switch_1:eth-2<->web_server:eth-1 + - switch_1:eth-3<->database_server:eth-1 + - switch_1:eth-4<->backup_server:eth-1 + - switch_1:eth-7<->security_suite:eth-1 + - switch_2:eth-1<->client_1:eth-1 + - switch_2:eth-2<->client_2:eth-1 + - switch_2:eth-7<->security_suite:eth-2 + - type: "NONE" + label: ICS + options: {{}} + + """ + + cfg = yaml.safe_load(obs_cfg_yaml) + manager = ObservationManager.from_config(cfg) + + hosts: List[HostObservation] = manager.obs.components["NODES"].hosts + for host in hosts: + services: List[ServiceObservation] = host.services + for service in services: + assert service.services_requires_scan == expected_val # Make sure services require scan by default + + def test_services_requires_scan(self): + state = {"health_state_actual": 3, "health_state_visible": 1, "operating_state": 1} + + obs_requiring_scan = ServiceObservation([], services_requires_scan=True) + assert obs_requiring_scan.observe(state)["health_status"] == 1 # should be visible value + + obs_not_requiring_scan = ServiceObservation([], services_requires_scan=False) + assert obs_not_requiring_scan.observe(state)["health_status"] == 3 # should be actual value From 1f937a4c961ae77fa11c57136d3366bd8b073439 Mon Sep 17 00:00:00 2001 From: Czar Echavez Date: Thu, 12 Sep 2024 18:54:18 +0100 Subject: [PATCH 31/95] #2864: config not being passed correctly --- .../agent/observations/host_observations.py | 4 ++++ .../observations/test_node_observations.py | 2 ++ .../test_software_observations.py | 8 ++++++-- .../_game/_agent/test_observations.py | 20 +++++++++++-------- 4 files changed, 24 insertions(+), 10 deletions(-) diff --git a/src/primaite/game/agent/observations/host_observations.py b/src/primaite/game/agent/observations/host_observations.py index 3371a99c..c05b493a 100644 --- a/src/primaite/game/agent/observations/host_observations.py +++ b/src/primaite/game/agent/observations/host_observations.py @@ -281,6 +281,10 @@ class HostObservation(AbstractObservation, identifier="HOST"): folder_config.file_system_requires_scan = config.file_system_requires_scan for nic_config in config.network_interfaces: nic_config.include_nmne = config.include_nmne + for service_config in config.services: + service_config.services_requires_scan = config.services_requires_scan + for application_config in config.applications: + application_config.application_config_requires_scan = config.application_config_requires_scan services = [ServiceObservation.from_config(config=c, parent_where=where) for c in config.services] applications = [ApplicationObservation.from_config(config=c, parent_where=where) for c in config.applications] diff --git a/tests/integration_tests/game_layer/observations/test_node_observations.py b/tests/integration_tests/game_layer/observations/test_node_observations.py index 69d9f106..9d60823b 100644 --- a/tests/integration_tests/game_layer/observations/test_node_observations.py +++ b/tests/integration_tests/game_layer/observations/test_node_observations.py @@ -39,6 +39,8 @@ def test_host_observation(simulation): folders=[], network_interfaces=[], file_system_requires_scan=True, + services_requires_scan=True, + applications_requires_scan=True, include_users=False, ) diff --git a/tests/integration_tests/game_layer/observations/test_software_observations.py b/tests/integration_tests/game_layer/observations/test_software_observations.py index 998aa755..ab9f6e9c 100644 --- a/tests/integration_tests/game_layer/observations/test_software_observations.py +++ b/tests/integration_tests/game_layer/observations/test_software_observations.py @@ -29,7 +29,9 @@ def test_service_observation(simulation): ntp_server = pc.software_manager.software.get("NTPServer") assert ntp_server - service_obs = ServiceObservation(where=["network", "nodes", pc.hostname, "services", "NTPServer"]) + service_obs = ServiceObservation( + where=["network", "nodes", pc.hostname, "services", "NTPServer"], services_requires_scan=True + ) assert service_obs.space["operating_status"] == spaces.Discrete(7) assert service_obs.space["health_status"] == spaces.Discrete(5) @@ -54,7 +56,9 @@ def test_application_observation(simulation): web_browser: WebBrowser = pc.software_manager.software.get("WebBrowser") assert web_browser - app_obs = ApplicationObservation(where=["network", "nodes", pc.hostname, "applications", "WebBrowser"]) + app_obs = ApplicationObservation( + where=["network", "nodes", pc.hostname, "applications", "WebBrowser"], applications_requires_scan=True + ) web_browser.close() observation_state = app_obs.observe(simulation.describe_state()) diff --git a/tests/unit_tests/_primaite/_game/_agent/test_observations.py b/tests/unit_tests/_primaite/_game/_agent/test_observations.py index 583b9cbd..912b672e 100644 --- a/tests/unit_tests/_primaite/_game/_agent/test_observations.py +++ b/tests/unit_tests/_primaite/_game/_agent/test_observations.py @@ -1,4 +1,5 @@ # © Crown-owned copyright 2024, Defence Science and Technology Laboratory UK +import json from typing import List import pytest @@ -142,7 +143,7 @@ class TestServiceRequiresScan: ), ) def test_obs_config(self, yaml_option_string, expected_val): - """Check that the default behaviour is to set FileSystemRequiresScan to True.""" + """Check that the default behaviour is to set service_requires_scan to True.""" obs_cfg_yaml = f""" type: CUSTOM options: @@ -155,19 +156,20 @@ class TestServiceRequiresScan: - hostname: web_server services: - service_name: WebServer + - service_name: DNSClient - hostname: database_server folders: - folder_name: database files: - file_name: database.db - hostname: backup_server + services: + - service_name: FTPServer - hostname: security_suite - hostname: client_1 - applications: - - application_name: WebBrowser - hostname: client_2 - num_services: 1 - num_applications: 1 + num_services: 3 + num_applications: 0 num_folders: 1 num_files: 1 num_nics: 2 @@ -226,10 +228,12 @@ class TestServiceRequiresScan: manager = ObservationManager.from_config(cfg) hosts: List[HostObservation] = manager.obs.components["NODES"].hosts - for host in hosts: + for i, host in enumerate(hosts): services: List[ServiceObservation] = host.services - for service in services: - assert service.services_requires_scan == expected_val # Make sure services require scan by default + for j, service in enumerate(services): + val = service.services_requires_scan + print(f"host {i} service {j} {val}") + assert val == expected_val # Make sure services require scan by default def test_services_requires_scan(self): state = {"health_state_actual": 3, "health_state_visible": 1, "operating_state": 1} From f1ff1f13cf4fc9e4fc8760a51b4d029f1307c8af Mon Sep 17 00:00:00 2001 From: Czar Echavez Date: Fri, 13 Sep 2024 09:08:44 +0100 Subject: [PATCH 32/95] #2864: added applications_requires_scan test --- .../agent/observations/host_observations.py | 2 +- .../_game/_agent/test_observations.py | 116 +++++++++++++++++- 2 files changed, 115 insertions(+), 3 deletions(-) diff --git a/src/primaite/game/agent/observations/host_observations.py b/src/primaite/game/agent/observations/host_observations.py index c05b493a..da054eda 100644 --- a/src/primaite/game/agent/observations/host_observations.py +++ b/src/primaite/game/agent/observations/host_observations.py @@ -284,7 +284,7 @@ class HostObservation(AbstractObservation, identifier="HOST"): for service_config in config.services: service_config.services_requires_scan = config.services_requires_scan for application_config in config.applications: - application_config.application_config_requires_scan = config.application_config_requires_scan + application_config.applications_requires_scan = config.applications_requires_scan services = [ServiceObservation.from_config(config=c, parent_where=where) for c in config.services] applications = [ApplicationObservation.from_config(config=c, parent_where=where) for c in config.applications] diff --git a/tests/unit_tests/_primaite/_game/_agent/test_observations.py b/tests/unit_tests/_primaite/_game/_agent/test_observations.py index 912b672e..935bbdcf 100644 --- a/tests/unit_tests/_primaite/_game/_agent/test_observations.py +++ b/tests/unit_tests/_primaite/_game/_agent/test_observations.py @@ -5,7 +5,7 @@ from typing import List import pytest import yaml -from primaite.game.agent.observations import ObservationManager, ServiceObservation +from primaite.game.agent.observations import ApplicationObservation, ObservationManager, ServiceObservation from primaite.game.agent.observations.file_system_observations import FileObservation, FolderObservation from primaite.game.agent.observations.host_observations import HostObservation @@ -133,7 +133,7 @@ class TestFileSystemRequiresScan: assert obs_not_requiring_scan.observe(folder_state)["health_status"] == 3 -class TestServiceRequiresScan: +class TestServicesRequiresScan: @pytest.mark.parametrize( ("yaml_option_string", "expected_val"), ( @@ -243,3 +243,115 @@ class TestServiceRequiresScan: obs_not_requiring_scan = ServiceObservation([], services_requires_scan=False) assert obs_not_requiring_scan.observe(state)["health_status"] == 3 # should be actual value + + +class TestApplicationsRequiresScan: + @pytest.mark.parametrize( + ("yaml_option_string", "expected_val"), + ( + ("applications_requires_scan: true", True), + ("applications_requires_scan: false", False), + (" ", True), + ), + ) + def test_obs_config(self, yaml_option_string, expected_val): + """Check that the default behaviour is to set applications_requires_scan to True.""" + obs_cfg_yaml = f""" + type: CUSTOM + options: + components: + - type: NODES + label: NODES + options: + hosts: + - hostname: domain_controller + - hostname: web_server + - hostname: database_server + folders: + - folder_name: database + files: + - file_name: database.db + - hostname: backup_server + - hostname: security_suite + - hostname: client_1 + applications: + - application_name: WebBrowser + - hostname: client_2 + applications: + - application_name: WebBrowser + - application_name: DatabaseClient + num_services: 0 + num_applications: 3 + num_folders: 1 + num_files: 1 + num_nics: 2 + include_num_access: false + {yaml_option_string} + include_nmne: true + monitored_traffic: + icmp: + - NONE + tcp: + - DNS + routers: + - hostname: router_1 + num_ports: 0 + ip_list: + - 192.168.1.10 + - 192.168.1.12 + - 192.168.1.14 + - 192.168.1.16 + - 192.168.1.110 + - 192.168.10.21 + - 192.168.10.22 + - 192.168.10.110 + wildcard_list: + - 0.0.0.1 + port_list: + - 80 + - 5432 + protocol_list: + - ICMP + - TCP + - UDP + num_rules: 10 + + - type: LINKS + label: LINKS + options: + link_references: + - router_1:eth-1<->switch_1:eth-8 + - router_1:eth-2<->switch_2:eth-8 + - switch_1:eth-1<->domain_controller:eth-1 + - switch_1:eth-2<->web_server:eth-1 + - switch_1:eth-3<->database_server:eth-1 + - switch_1:eth-4<->backup_server:eth-1 + - switch_1:eth-7<->security_suite:eth-1 + - switch_2:eth-1<->client_1:eth-1 + - switch_2:eth-2<->client_2:eth-1 + - switch_2:eth-7<->security_suite:eth-2 + - type: "NONE" + label: ICS + options: {{}} + + """ + + cfg = yaml.safe_load(obs_cfg_yaml) + manager = ObservationManager.from_config(cfg) + + hosts: List[HostObservation] = manager.obs.components["NODES"].hosts + for i, host in enumerate(hosts): + services: List[ServiceObservation] = host.services + for j, service in enumerate(services): + val = service.services_requires_scan + print(f"host {i} service {j} {val}") + assert val == expected_val # Make sure applications require scan by default + + def test_applications_requires_scan(self): + state = {"health_state_actual": 3, "health_state_visible": 1, "operating_state": 1, "num_executions": 1} + + obs_requiring_scan = ApplicationObservation([], applications_requires_scan=True) + assert obs_requiring_scan.observe(state)["health_status"] == 1 # should be visible value + + obs_not_requiring_scan = ApplicationObservation([], applications_requires_scan=False) + assert obs_not_requiring_scan.observe(state)["health_status"] == 3 # should be actual value From 454789f49461ff285b22d7a5bfbe4890a4c5f335 Mon Sep 17 00:00:00 2001 From: Czar Echavez Date: Fri, 13 Sep 2024 09:34:09 +0100 Subject: [PATCH 33/95] #2864: add to changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 77b7bb7d..56f0c038 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Log observation space data by episode and step. - ACL's are no longer applied to layer-2 traffic. +- Added `services_requires_scan` and `applications_requires_scan` to agent observation space config to allow the agents to be able to see actual health states of services and applications without requiring scans (Default `True`, set to `False` to allow agents to see actual health state without scanning). ## [3.3.0] - 2024-09-04 ### Added From d8c85058edc2b80e9f9afcede098c9d133804d2f Mon Sep 17 00:00:00 2001 From: Charlie Crane Date: Fri, 13 Sep 2024 10:32:09 +0100 Subject: [PATCH 34/95] #2456 - Minor change to arp.show() to include port number --- src/primaite/simulator/system/services/arp/arp.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/primaite/simulator/system/services/arp/arp.py b/src/primaite/simulator/system/services/arp/arp.py index efadf189..9314bea7 100644 --- a/src/primaite/simulator/system/services/arp/arp.py +++ b/src/primaite/simulator/system/services/arp/arp.py @@ -47,7 +47,7 @@ class ARP(Service): :param markdown: If True, format the output as Markdown. Otherwise, use plain text. """ - table = PrettyTable(["IP Address", "MAC Address", "Via"]) + table = PrettyTable(["IP Address", "MAC Address", "Via", "Port"]) if markdown: table.set_style(MARKDOWN) table.align = "l" @@ -58,6 +58,7 @@ class ARP(Service): str(ip), arp.mac_address, self.software_manager.node.network_interfaces[arp.network_interface_uuid].mac_address, + self.software_manager.node.network_interfaces[arp.network_interface_uuid].port_num, ] ) print(table) From 94b30909ee62624181bbaa7187948f41f17b8164 Mon Sep 17 00:00:00 2001 From: Charlie Crane Date: Fri, 13 Sep 2024 10:56:12 +0100 Subject: [PATCH 35/95] #2456 - Updated Changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 77b7bb7d..71341a17 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added - Log observation space data by episode and step. + +### Changed - ACL's are no longer applied to layer-2 traffic. +- ARP .show() method will no include the port number associated with each entry. + ## [3.3.0] - 2024-09-04 ### Added From 9a2fb2a0846cdc8958d7cad03fdbfaa6277e31bc Mon Sep 17 00:00:00 2001 From: Czar Echavez Date: Fri, 13 Sep 2024 11:11:58 +0100 Subject: [PATCH 36/95] #2880: fix action shape for num_ports + test --- src/primaite/game/agent/actions.py | 4 ++-- .../game_layer/test_action_shapes.py | 21 +++++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 tests/integration_tests/game_layer/test_action_shapes.py diff --git a/src/primaite/game/agent/actions.py b/src/primaite/game/agent/actions.py index a299788e..c864f75f 100644 --- a/src/primaite/game/agent/actions.py +++ b/src/primaite/game/agent/actions.py @@ -877,7 +877,7 @@ class FirewallACLRemoveRuleAction(AbstractAction): """Action which removes a rule from a firewall port's ACL.""" def __init__(self, manager: "ActionManager", max_acl_rules: int, **kwargs) -> None: - """Init method for RouterACLRemoveRuleAction. + """Init method for FirewallACLRemoveRuleAction. :param manager: Reference to the ActionManager which created this action. :type manager: ActionManager @@ -1524,7 +1524,7 @@ class ActionManager: "num_nics": max_nics_per_node, "num_acl_rules": max_acl_rules, "num_protocols": len(self.protocols), - "num_ports": len(self.protocols), + "num_ports": len(self.ports), "num_ips": len(self.ip_address_list), "max_acl_rules": max_acl_rules, "max_nics_per_node": max_nics_per_node, diff --git a/tests/integration_tests/game_layer/test_action_shapes.py b/tests/integration_tests/game_layer/test_action_shapes.py new file mode 100644 index 00000000..48500d8f --- /dev/null +++ b/tests/integration_tests/game_layer/test_action_shapes.py @@ -0,0 +1,21 @@ +# © Crown-owned copyright 2024, Defence Science and Technology Laboratory UK +from typing import Tuple + +from primaite.game.agent.interface import ProxyAgent +from primaite.game.game import PrimaiteGame +from tests import TEST_ASSETS_ROOT + +FIREWALL_ACTIONS_NETWORK = TEST_ASSETS_ROOT / "configs/firewall_actions_network.yaml" + + +def test_router_acl_add_rule_action_shape(game_and_agent: Tuple[PrimaiteGame, ProxyAgent]): + """Test to check ROUTER_ADD_ACL_RULE has the expected action shape.""" + game, agent = game_and_agent + + # assert that the shape of the actions is correct + router_acl_add_rule_action = agent.action_manager.actions.get("ROUTER_ACL_ADDRULE") + assert router_acl_add_rule_action.shape.get("source_ip_id") == len(agent.action_manager.ip_address_list) + assert router_acl_add_rule_action.shape.get("dest_ip_id") == len(agent.action_manager.ip_address_list) + assert router_acl_add_rule_action.shape.get("source_port_id") == len(agent.action_manager.ports) + assert router_acl_add_rule_action.shape.get("dest_port_id") == len(agent.action_manager.ports) + assert router_acl_add_rule_action.shape.get("protocol_id") == len(agent.action_manager.protocols) From 17035be0284f1789e026d4cf8328cc97b2035c8b Mon Sep 17 00:00:00 2001 From: Charlie Crane Date: Fri, 13 Sep 2024 11:13:55 +0100 Subject: [PATCH 37/95] #2456 - Actioning review comment --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 71341a17..53b29e85 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - ACL's are no longer applied to layer-2 traffic. -- ARP .show() method will no include the port number associated with each entry. +- ARP .show() method will now include the port number associated with each entry. ## [3.3.0] - 2024-09-04 From c924b9ea46dcd2a701fdf6ba93a3733b72c09a99 Mon Sep 17 00:00:00 2001 From: Charlie Crane Date: Fri, 13 Sep 2024 11:54:17 +0100 Subject: [PATCH 38/95] #2871 - Initial commit of a show_history() function in AbstractAgent --- src/primaite/game/agent/interface.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/primaite/game/agent/interface.py b/src/primaite/game/agent/interface.py index d5165a71..404c2bfe 100644 --- a/src/primaite/game/agent/interface.py +++ b/src/primaite/game/agent/interface.py @@ -4,6 +4,7 @@ from abc import ABC, abstractmethod from typing import Any, Dict, List, Optional, Tuple, TYPE_CHECKING from gymnasium.core import ActType, ObsType +from prettytable import PrettyTable from pydantic import BaseModel, model_validator from primaite.game.agent.actions import ActionManager @@ -126,6 +127,27 @@ class AbstractAgent(ABC): self.history: List[AgentHistoryItem] = [] self.logger = AgentLog(agent_name) + def show_history(self): + """ + Print an agent action provided it's not the DONOTHING action. + + :param agent_name: Name of agent (str). + """ + table = PrettyTable() + table.field_names = ["Step", "Action", "Node", "Application", "Response"] + print(f"Actions for '{self.agent_name}':") + for item in self.history: + if item.action != "DONOTHING": + node, application = "unknown", "unknown" + if (node_id := item.parameters.get("node_id")) is not None: + node = self.action_manager.node_names[node_id] + if (application_id := item.parameters.get("application_id")) is not None: + application = self.action_manager.application_names[node_id][application_id] + if (application_name := item.parameters.get("application_name")) is not None: + application = application_name + table.add_row([item.timestep, item.action, node, application, item.response.status]) + print(table) + def update_observation(self, state: Dict) -> ObsType: """ Convert a state from the simulator into an observation for the agent using the observation space. From cd8fc6d42d153b28b5cd731a4fe19239bfcc327d Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Fri, 13 Sep 2024 12:10:49 +0100 Subject: [PATCH 39/95] #2879: Handle generate_seed_value option --- src/primaite/session/environment.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/primaite/session/environment.py b/src/primaite/session/environment.py index c66663e3..ac9415ac 100644 --- a/src/primaite/session/environment.py +++ b/src/primaite/session/environment.py @@ -26,14 +26,25 @@ except ModuleNotFoundError: _LOGGER.debug("Torch not available for importing") -def set_random_seed(seed: int) -> Union[None, int]: +def set_random_seed(seed: int, generate_seed_value: bool) -> Union[None, int]: """ Set random number generators. + If seed is None or -1 and generate_seed_value is True randomly generate a + seed value. + If seed is > -1 and generate_seed_value is True ignore the latter and use + the provide seed value. + :param seed: int + :param generate_seed_value: bool + :return: None or the int representing the seed used. """ if seed is None or seed == -1: - return None + if generate_seed_value: + rng = np.random.default_rng() + seed = int(rng.integers(low=0, high=2**63)) + else: + return None elif seed < -1: raise ValueError("Invalid random number seed") # Seed python RNG @@ -65,7 +76,8 @@ class PrimaiteGymEnv(gymnasium.Env): """Object that returns a config corresponding to the current episode.""" self.seed = self.episode_scheduler(0).get("game", {}).get("seed") """Get RNG seed from config file. NB: Must be before game instantiation.""" - self.seed = set_random_seed(self.seed) + self.generate_seed_value = self.episode_scheduler(0).get("game", {}).get("generate_seed_value") + self.seed = set_random_seed(self.seed, self.generate_seed_value) self.io = PrimaiteIO.from_config(self.episode_scheduler(0).get("io_settings", {})) """Handles IO for the environment. This produces sys logs, agent logs, etc.""" self.game: PrimaiteGame = PrimaiteGame.from_config(self.episode_scheduler(0)) From 6ebe50c331725c5059f269a59d87bd1dcd4077b3 Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Fri, 13 Sep 2024 12:58:37 +0100 Subject: [PATCH 40/95] #2879: Reduce max seed value to comply with python random seed limit --- src/primaite/session/environment.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/primaite/session/environment.py b/src/primaite/session/environment.py index ac9415ac..0fd21b9f 100644 --- a/src/primaite/session/environment.py +++ b/src/primaite/session/environment.py @@ -42,7 +42,8 @@ def set_random_seed(seed: int, generate_seed_value: bool) -> Union[None, int]: if seed is None or seed == -1: if generate_seed_value: rng = np.random.default_rng() - seed = int(rng.integers(low=0, high=2**63)) + # 2**32-1 is highest value for python RNG seed. + seed = int(rng.integers(low=0, high=2**32-1)) else: return None elif seed < -1: From 08fcf1df19fc811bf9a24aee04d8c5c3239f9678 Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Fri, 13 Sep 2024 12:59:41 +0100 Subject: [PATCH 41/95] #2879: Add generate_seed_value to global options. --- src/primaite/game/game.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/primaite/game/game.py b/src/primaite/game/game.py index 123b6ddd..0e7b8c23 100644 --- a/src/primaite/game/game.py +++ b/src/primaite/game/game.py @@ -80,6 +80,8 @@ class PrimaiteGameOptions(BaseModel): seed: int = None """Random number seed for RNGs.""" + generate_seed_value: bool = False + """Internally generated seed value.""" max_episode_length: int = 256 """Maximum number of episodes for the PrimAITE game.""" ports: List[str] From f2a0eeaca23159da9caa0cd9e55e81f5aaac6875 Mon Sep 17 00:00:00 2001 From: Charlie Crane Date: Fri, 13 Sep 2024 14:11:13 +0100 Subject: [PATCH 42/95] #2871 - Updated show_history() method to use boolean 'include_nothing' for whether to include DONOTHING actions --- src/primaite/game/agent/interface.py | 31 ++++++++++++++++++---------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/src/primaite/game/agent/interface.py b/src/primaite/game/agent/interface.py index 404c2bfe..0ec44d22 100644 --- a/src/primaite/game/agent/interface.py +++ b/src/primaite/game/agent/interface.py @@ -127,25 +127,34 @@ class AbstractAgent(ABC): self.history: List[AgentHistoryItem] = [] self.logger = AgentLog(agent_name) - def show_history(self): + def add_agent_action(self, item: AgentHistoryItem, table: PrettyTable) -> PrettyTable: + """Update the given table with information from given AgentHistoryItem.""" + node, application = "unknown", "unknown" + if (node_id := item.parameters.get("node_id")) is not None: + node = self.action_manager.node_names[node_id] + if (application_id := item.parameters.get("application_id")) is not None: + application = self.action_manager.application_names[node_id][application_id] + if (application_name := item.parameters.get("application_name")) is not None: + application = application_name + table.add_row([item.timestep, item.action, node, application, item.response.status]) + return table + + def show_history(self, include_nothing: bool = False): """ Print an agent action provided it's not the DONOTHING action. - :param agent_name: Name of agent (str). + :param include_nothing: boolean for including DONOTHING actions. Default False. """ table = PrettyTable() table.field_names = ["Step", "Action", "Node", "Application", "Response"] print(f"Actions for '{self.agent_name}':") for item in self.history: - if item.action != "DONOTHING": - node, application = "unknown", "unknown" - if (node_id := item.parameters.get("node_id")) is not None: - node = self.action_manager.node_names[node_id] - if (application_id := item.parameters.get("application_id")) is not None: - application = self.action_manager.application_names[node_id][application_id] - if (application_name := item.parameters.get("application_name")) is not None: - application = application_name - table.add_row([item.timestep, item.action, node, application, item.response.status]) + if item.action == "DONOTHING": + if include_nothing: + table = self.add_agent_action(item=item, table=table) + else: + pass + self.add_agent_action(item=item, table=table) print(table) def update_observation(self, state: Dict) -> ObsType: From 01a2c834ce3c8ff23c90ff098ef2cce04bdd5bab Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Fri, 13 Sep 2024 14:53:15 +0100 Subject: [PATCH 43/95] #2879: Write seed value to log file. --- src/primaite/session/environment.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/primaite/session/environment.py b/src/primaite/session/environment.py index 0fd21b9f..9054106e 100644 --- a/src/primaite/session/environment.py +++ b/src/primaite/session/environment.py @@ -62,6 +62,13 @@ def set_random_seed(seed: int, generate_seed_value: bool) -> Union[None, int]: return seed +def log_seed_value(seed: int): + """Log the selected seed value to file.""" + path = SIM_OUTPUT.path / "seed.log" + with open(path, "w") as file: + file.write(f"Seed value = {seed}") + + class PrimaiteGymEnv(gymnasium.Env): """ Thin wrapper env to provide agents with a gymnasium API. @@ -92,6 +99,8 @@ class PrimaiteGymEnv(gymnasium.Env): _LOGGER.info(f"PrimaiteGymEnv RNG seed = {self.seed}") + log_seed_value(self.seed) + def action_masks(self) -> np.ndarray: """ Return the action mask for the agent. From 5006e41546d37cabb0a505fe0c9e3346dcaebf89 Mon Sep 17 00:00:00 2001 From: Charlie Crane Date: Fri, 13 Sep 2024 15:47:59 +0100 Subject: [PATCH 44/95] #2871 - Updated the show_history() function to receive a list of actions to ignore when printing the history. Defaults to ignoring DONOTHING actions --- src/primaite/game/agent/interface.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/primaite/game/agent/interface.py b/src/primaite/game/agent/interface.py index 0ec44d22..6609dd03 100644 --- a/src/primaite/game/agent/interface.py +++ b/src/primaite/game/agent/interface.py @@ -139,22 +139,23 @@ class AbstractAgent(ABC): table.add_row([item.timestep, item.action, node, application, item.response.status]) return table - def show_history(self, include_nothing: bool = False): + def show_history(self, ignored_actions: Optional[list] = None): """ Print an agent action provided it's not the DONOTHING action. - :param include_nothing: boolean for including DONOTHING actions. Default False. + :param ignored_actions: OPTIONAL: List of actions to be ignored when displaying the history. + If not provided, defaults to ignore DONOTHING actions. """ + if not ignored_actions: + ignored_actions = ["DONOTHING"] table = PrettyTable() table.field_names = ["Step", "Action", "Node", "Application", "Response"] print(f"Actions for '{self.agent_name}':") for item in self.history: - if item.action == "DONOTHING": - if include_nothing: - table = self.add_agent_action(item=item, table=table) - else: - pass - self.add_agent_action(item=item, table=table) + if item.action in ignored_actions: + pass + else: + table = self.add_agent_action(item=item, table=table) print(table) def update_observation(self, state: Dict) -> ObsType: From e0a10928343c650b986da8aa8cd6207786448e0f Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Mon, 16 Sep 2024 09:04:17 +0100 Subject: [PATCH 45/95] #2879: Pre-commit fix. --- src/primaite/session/environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/primaite/session/environment.py b/src/primaite/session/environment.py index 9054106e..07635b70 100644 --- a/src/primaite/session/environment.py +++ b/src/primaite/session/environment.py @@ -43,7 +43,7 @@ def set_random_seed(seed: int, generate_seed_value: bool) -> Union[None, int]: if generate_seed_value: rng = np.random.default_rng() # 2**32-1 is highest value for python RNG seed. - seed = int(rng.integers(low=0, high=2**32-1)) + seed = int(rng.integers(low=0, high=2**32 - 1)) else: return None elif seed < -1: From 215ceaa6e8b5977b231d226715b70d8e88df7f14 Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Mon, 16 Sep 2024 10:08:45 +0100 Subject: [PATCH 46/95] #2879: Fix call to set_random_seed() in reset(). --- src/primaite/session/environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/primaite/session/environment.py b/src/primaite/session/environment.py index 07635b70..db5425e3 100644 --- a/src/primaite/session/environment.py +++ b/src/primaite/session/environment.py @@ -168,7 +168,7 @@ class PrimaiteGymEnv(gymnasium.Env): f"avg. reward: {self.agent.reward_function.total_reward}" ) if seed is not None: - set_random_seed(seed) + set_random_seed(seed, self.generate_seed_value) self.total_reward_per_episode[self.episode_counter] = self.agent.reward_function.total_reward if self.io.settings.save_agent_actions: From f3ca9c55c90fe05b2e43c2c167767037578a0fb7 Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Mon, 16 Sep 2024 16:38:19 +0100 Subject: [PATCH 47/95] #2879: Update tests --- .../game_layer/test_RNG_seed.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/integration_tests/game_layer/test_RNG_seed.py b/tests/integration_tests/game_layer/test_RNG_seed.py index 0c6d567d..508f35e6 100644 --- a/tests/integration_tests/game_layer/test_RNG_seed.py +++ b/tests/integration_tests/game_layer/test_RNG_seed.py @@ -7,6 +7,7 @@ import yaml from primaite.config.load import data_manipulation_config_path from primaite.game.agent.interface import AgentHistoryItem from primaite.session.environment import PrimaiteGymEnv +from primaite.simulator import SIM_OUTPUT @pytest.fixture() @@ -33,6 +34,11 @@ def test_rng_seed_set(create_env): assert a == b + # Check that seed log file was created. + path = SIM_OUTPUT.path / "seed.log" + with open(path, "r") as file: + assert file + def test_rng_seed_unset(create_env): """Test with no RNG seed.""" @@ -48,3 +54,19 @@ def test_rng_seed_unset(create_env): b = [item.timestep for item in env.game.agents["client_2_green_user"].history if item.action != "DONOTHING"] assert a != b + + +def test_for_generated_seed(): + """ + Show that setting generate_seed_value to true producess a valid seed. + """ + with open(data_manipulation_config_path(), "r") as f: + cfg = yaml.safe_load(f) + + cfg["game"]["generate_seed_value"] = True + PrimaiteGymEnv(env_config=cfg) + path = SIM_OUTPUT.path / "seed.log" + with open(path, "r") as file: + data = file.read() + + assert data.split(" ")[3] != None From 078b89856535b0071c76921612b6758f6d48782c Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Tue, 17 Sep 2024 09:30:14 +0100 Subject: [PATCH 48/95] #2879: Update changelog. --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 77b7bb7d..a9f6c891 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Log observation space data by episode and step. - ACL's are no longer applied to layer-2 traffic. +- Random number seed values are recorded in simulation/seed.log if the seed is set in the config file + or `generate_seed_value` is set to `true`. ## [3.3.0] - 2024-09-04 ### Added From 5d7935cde083d662389198b8345fc9194e8351be Mon Sep 17 00:00:00 2001 From: Charlie Crane Date: Tue, 17 Sep 2024 09:39:32 +0100 Subject: [PATCH 49/95] #2871 - Changes to notebooks following updates to action history --- .../Command-&-Control-E2E-Demonstration.ipynb | 12 +++++- .../Data-Manipulation-E2E-Demonstration.ipynb | 11 ++++- .../Getting-Information-Out-Of-PrimAITE.ipynb | 40 ++++++++++++++++++- 3 files changed, 60 insertions(+), 3 deletions(-) diff --git a/src/primaite/notebooks/Command-&-Control-E2E-Demonstration.ipynb b/src/primaite/notebooks/Command-&-Control-E2E-Demonstration.ipynb index b6b13f28..a0599ee4 100644 --- a/src/primaite/notebooks/Command-&-Control-E2E-Demonstration.ipynb +++ b/src/primaite/notebooks/Command-&-Control-E2E-Demonstration.ipynb @@ -1800,6 +1800,16 @@ "\n", "display_obs_diffs(tcp_c2_obs, udp_c2_obs, blue_config_env.game.step_counter)" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "env.game.agents[\"CustomC2Agent\"].show_history()" + ] } ], "metadata": { @@ -1818,7 +1828,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.12" + "version": "3.10.11" } }, "nbformat": 4, diff --git a/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb b/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb index 0460f771..c1b959f5 100644 --- a/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb +++ b/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb @@ -675,6 +675,15 @@ " print(f\"step: {env.game.step_counter}, Red action: {info['agent_actions']['data_manipulation_attacker'].action}, Blue reward:{reward:.2f}\" )" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "env.game.agents[\"data_manipulation_attacker\"].show_history(ignored_actions=[\"\"])" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -708,7 +717,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.12" + "version": "3.10.11" } }, "nbformat": 4, diff --git a/src/primaite/notebooks/Getting-Information-Out-Of-PrimAITE.ipynb b/src/primaite/notebooks/Getting-Information-Out-Of-PrimAITE.ipynb index a832f3cc..e4009822 100644 --- a/src/primaite/notebooks/Getting-Information-Out-Of-PrimAITE.ipynb +++ b/src/primaite/notebooks/Getting-Information-Out-Of-PrimAITE.ipynb @@ -144,6 +144,44 @@ "PRIMAITE_CONFIG[\"developer_mode\"][\"enabled\"] = was_enabled\n", "PRIMAITE_CONFIG[\"developer_mode\"][\"output_sys_logs\"] = was_syslogs_enabled" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Viewing Agent history\n", + "\n", + "It's possible to view the actions carried out by an agent for a given training session using the `show_history()` method. By default, this will be all actions apart from DONOTHING actions." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Run the training session to generate some resultant data.\n", + "for i in range(100):\n", + " env.step(0)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Calling `.show_history()` should show us when the Data Manipulation used the `NODE_APPLICATION_EXECUTE` action." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "attacker = env.game.agents[\"data_manipulation_attacker\"]\n", + "\n", + "attacker.show_history()" + ] } ], "metadata": { @@ -162,7 +200,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.8" + "version": "3.10.11" } }, "nbformat": 4, From c8f6459af6022f2536580f968c04e9d32b15e596 Mon Sep 17 00:00:00 2001 From: Charlie Crane Date: Tue, 17 Sep 2024 10:09:10 +0100 Subject: [PATCH 50/95] #2871 - Changelog and documentation updates, corrected changes in Data manipulation demo notebook --- CHANGELOG.md | 1 + docs/source/configuration/agents.rst | 1 + .../notebooks/Data-Manipulation-E2E-Demonstration.ipynb | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 44f1ec29..b7f8a26e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Log observation space data by episode and step. +- Added `show_history` method to Agents, allowing you to view actions taken by an agent per step. By default, `DONOTHING` actions are omitted. ### Changed - ACL's are no longer applied to layer-2 traffic. diff --git a/docs/source/configuration/agents.rst b/docs/source/configuration/agents.rst index dece94c5..0bc586e8 100644 --- a/docs/source/configuration/agents.rst +++ b/docs/source/configuration/agents.rst @@ -177,3 +177,4 @@ If ``True``, gymnasium flattening will be performed on the observation space bef ----------------- Agents will record their action log for each step. This is a summary of what the agent did, along with response information from requests within the simulation. +A log of the actions taken by the agent can be viewed using the `show_history()` function. By default, this will display all actions taken apart from ``DONOTHING``. diff --git a/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb b/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb index c1b959f5..13533097 100644 --- a/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb +++ b/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb @@ -681,7 +681,7 @@ "metadata": {}, "outputs": [], "source": [ - "env.game.agents[\"data_manipulation_attacker\"].show_history(ignored_actions=[\"\"])" + "env.game.agents[\"data_manipulation_attacker\"].show_history()" ] }, { From ccb91869c4e7b62e5772c09de496c5cc96b7d35a Mon Sep 17 00:00:00 2001 From: Charlie Crane Date: Tue, 17 Sep 2024 10:17:18 +0100 Subject: [PATCH 51/95] #2871 - Minor wording change to description in agents.rst --- docs/source/configuration/agents.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/configuration/agents.rst b/docs/source/configuration/agents.rst index 0bc586e8..74571cf2 100644 --- a/docs/source/configuration/agents.rst +++ b/docs/source/configuration/agents.rst @@ -177,4 +177,4 @@ If ``True``, gymnasium flattening will be performed on the observation space bef ----------------- Agents will record their action log for each step. This is a summary of what the agent did, along with response information from requests within the simulation. -A log of the actions taken by the agent can be viewed using the `show_history()` function. By default, this will display all actions taken apart from ``DONOTHING``. +A summary of the actions taken by the agent can be viewed using the `show_history()` function. By default, this will display all actions taken apart from ``DONOTHING``. From 3a5b75239d64c6febe35ac4bae227e8c804a8f01 Mon Sep 17 00:00:00 2001 From: Charlie Crane Date: Tue, 17 Sep 2024 12:05:40 +0100 Subject: [PATCH 52/95] #2871 - Typo in Changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b7f8a26e..b81e256b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [3.3.0] - 2024-09-04 +## [3.4.0] ### Added - Log observation space data by episode and step. From 4391d7cdd559619a1bbb9c4c8b25936bad2d3290 Mon Sep 17 00:00:00 2001 From: Czar Echavez Date: Tue, 17 Sep 2024 12:19:35 +0100 Subject: [PATCH 53/95] #2445: added the ability to pass the game options thresholds to observations so that relevant observation items can retrieve the thresholds from config --- .../observations/file_system_observations.py | 50 +++++++++++++++++-- .../agent/observations/host_observations.py | 17 ++++++- .../agent/observations/nic_observations.py | 35 ++++++++++--- .../agent/observations/node_observations.py | 6 +++ .../agent/observations/observation_manager.py | 12 +++-- .../game/agent/observations/observations.py | 36 ++++++++++++- .../observations/software_observation.py | 30 +++++++---- src/primaite/game/game.py | 2 +- .../configs/basic_switched_network.yaml | 25 +++++++++- .../test_game_options_config.py | 41 ++++++++++++++- .../test_file_system_observations.py | 32 ++++++++++++ .../observations/test_nic_observations.py | 25 ++++------ .../test_software_observations.py | 27 ++++++++++ 13 files changed, 290 insertions(+), 48 deletions(-) diff --git a/src/primaite/game/agent/observations/file_system_observations.py b/src/primaite/game/agent/observations/file_system_observations.py index 1c73d026..fe959c9f 100644 --- a/src/primaite/game/agent/observations/file_system_observations.py +++ b/src/primaite/game/agent/observations/file_system_observations.py @@ -26,7 +26,13 @@ class FileObservation(AbstractObservation, identifier="FILE"): file_system_requires_scan: Optional[bool] = None """If True, the file must be scanned to update the health state. Tf False, the true state is always shown.""" - def __init__(self, where: WhereType, include_num_access: bool, file_system_requires_scan: bool) -> None: + def __init__( + self, + where: WhereType, + include_num_access: bool, + file_system_requires_scan: bool, + thresholds: Optional[Dict] = {}, + ) -> None: """ Initialise a file observation instance. @@ -48,10 +54,22 @@ class FileObservation(AbstractObservation, identifier="FILE"): if self.include_num_access: self.default_observation["num_access"] = 0 - # TODO: allow these to be configured in yaml - self.high_threshold = 10 - self.med_threshold = 5 - self.low_threshold = 0 + if thresholds.get("file_access") is None: + self.low_threshold = 0 + self.med_threshold = 5 + self.high_threshold = 10 + else: + if self._validate_thresholds( + thresholds=[ + thresholds.get("file_access")["low"], + thresholds.get("file_access")["medium"], + thresholds.get("file_access")["high"], + ], + threshold_identifier="file_access", + ): + self.low_threshold = thresholds.get("file_access")["low"] + self.med_threshold = thresholds.get("file_access")["medium"] + self.high_threshold = thresholds.get("file_access")["high"] def _categorise_num_access(self, num_access: int) -> int: """ @@ -122,6 +140,7 @@ class FileObservation(AbstractObservation, identifier="FILE"): where=parent_where + ["files", config.file_name], include_num_access=config.include_num_access, file_system_requires_scan=config.file_system_requires_scan, + thresholds=config.thresholds, ) @@ -149,6 +168,7 @@ class FolderObservation(AbstractObservation, identifier="FOLDER"): num_files: int, include_num_access: bool, file_system_requires_scan: bool, + thresholds: Optional[Dict] = {}, ) -> None: """ Initialise a folder observation instance. @@ -170,6 +190,23 @@ class FolderObservation(AbstractObservation, identifier="FOLDER"): self.file_system_requires_scan: bool = file_system_requires_scan + if thresholds.get("file_access") is None: + self.low_threshold = 0 + self.med_threshold = 5 + self.high_threshold = 10 + else: + if self._validate_thresholds( + thresholds=[ + thresholds.get("file_access")["low"], + thresholds.get("file_access")["medium"], + thresholds.get("file_access")["high"], + ], + threshold_identifier="file_access", + ): + self.low_threshold = thresholds.get("file_access")["low"] + self.med_threshold = thresholds.get("file_access")["medium"] + self.high_threshold = thresholds.get("file_access")["high"] + self.files: List[FileObservation] = files while len(self.files) < num_files: self.files.append( @@ -177,6 +214,7 @@ class FolderObservation(AbstractObservation, identifier="FOLDER"): where=None, include_num_access=include_num_access, file_system_requires_scan=self.file_system_requires_scan, + thresholds=thresholds, ) ) while len(self.files) > num_files: @@ -248,6 +286,7 @@ class FolderObservation(AbstractObservation, identifier="FOLDER"): for file_config in config.files: file_config.include_num_access = config.include_num_access file_config.file_system_requires_scan = config.file_system_requires_scan + file_config.thresholds = config.thresholds files = [FileObservation.from_config(config=f, parent_where=where) for f in config.files] return cls( @@ -256,4 +295,5 @@ class FolderObservation(AbstractObservation, identifier="FOLDER"): num_files=config.num_files, include_num_access=config.include_num_access, file_system_requires_scan=config.file_system_requires_scan, + thresholds=config.thresholds, ) diff --git a/src/primaite/game/agent/observations/host_observations.py b/src/primaite/game/agent/observations/host_observations.py index 4419ccc7..fa7ceae5 100644 --- a/src/primaite/game/agent/observations/host_observations.py +++ b/src/primaite/game/agent/observations/host_observations.py @@ -151,7 +151,13 @@ class HostObservation(AbstractObservation, identifier="HOST"): self.nics: List[NICObservation] = network_interfaces while len(self.nics) < num_nics: - self.nics.append(NICObservation(where=None, include_nmne=include_nmne, monitored_traffic=monitored_traffic)) + self.nics.append( + NICObservation( + where=None, + include_nmne=include_nmne, + monitored_traffic=monitored_traffic, + ) + ) while len(self.nics) > num_nics: truncated_nic = self.nics.pop() msg = f"Too many network_interfaces in Node observation space for node. Truncating {truncated_nic.where}" @@ -257,12 +263,16 @@ class HostObservation(AbstractObservation, identifier="HOST"): where = parent_where + [config.hostname] # Pass down shared/common config items + for app_config in config.applications: + app_config.thresholds = config.thresholds for folder_config in config.folders: folder_config.include_num_access = config.include_num_access folder_config.num_files = config.num_files folder_config.file_system_requires_scan = config.file_system_requires_scan + folder_config.thresholds = config.thresholds for nic_config in config.network_interfaces: nic_config.include_nmne = config.include_nmne + nic_config.thresholds = config.thresholds services = [ServiceObservation.from_config(config=c, parent_where=where) for c in config.services] applications = [ApplicationObservation.from_config(config=c, parent_where=where) for c in config.applications] @@ -273,7 +283,10 @@ class HostObservation(AbstractObservation, identifier="HOST"): count = 1 while len(nics) < config.num_nics: nic_config = NICObservation.ConfigSchema( - nic_num=count, include_nmne=config.include_nmne, monitored_traffic=config.monitored_traffic + nic_num=count, + include_nmne=config.include_nmne, + monitored_traffic=config.monitored_traffic, + thresholds=config.thresholds, ) nics.append(NICObservation.from_config(config=nic_config, parent_where=where)) count += 1 diff --git a/src/primaite/game/agent/observations/nic_observations.py b/src/primaite/game/agent/observations/nic_observations.py index 002ee4da..48fa11dc 100644 --- a/src/primaite/game/agent/observations/nic_observations.py +++ b/src/primaite/game/agent/observations/nic_observations.py @@ -24,7 +24,13 @@ class NICObservation(AbstractObservation, identifier="NETWORK_INTERFACE"): monitored_traffic: Optional[Dict] = None """A dict containing which traffic types are to be included in the observation.""" - def __init__(self, where: WhereType, include_nmne: bool, monitored_traffic: Optional[Dict] = None) -> None: + def __init__( + self, + where: WhereType, + include_nmne: bool, + monitored_traffic: Optional[Dict] = None, + thresholds: Optional[Dict] = {}, + ) -> None: """ Initialise a network interface observation instance. @@ -44,10 +50,22 @@ class NICObservation(AbstractObservation, identifier="NETWORK_INTERFACE"): self.nmne_inbound_last_step: int = 0 self.nmne_outbound_last_step: int = 0 - # TODO: allow these to be configured in yaml - self.high_nmne_threshold = 10 - self.med_nmne_threshold = 5 - self.low_nmne_threshold = 0 + if thresholds.get("nmne") is None: + self.low_threshold = 0 + self.med_threshold = 5 + self.high_threshold = 10 + else: + if self._validate_thresholds( + thresholds=[ + thresholds.get("nmne")["low"], + thresholds.get("nmne")["medium"], + thresholds.get("nmne")["high"], + ], + threshold_identifier="nmne", + ): + self.low_threshold = thresholds.get("nmne")["low"] + self.med_threshold = thresholds.get("nmne")["medium"] + self.high_threshold = thresholds.get("nmne")["high"] self.monitored_traffic = monitored_traffic if self.monitored_traffic: @@ -86,11 +104,11 @@ class NICObservation(AbstractObservation, identifier="NETWORK_INTERFACE"): :param nmne_count: Number of MNEs detected. :return: Bin number corresponding to the number of MNEs. Returns 0, 1, 2, or 3 based on the detected MNE count. """ - if nmne_count > self.high_nmne_threshold: + if nmne_count > self.high_threshold: return 3 - elif nmne_count > self.med_nmne_threshold: + elif nmne_count > self.med_threshold: return 2 - elif nmne_count > self.low_nmne_threshold: + elif nmne_count > self.low_threshold: return 1 return 0 @@ -224,6 +242,7 @@ class NICObservation(AbstractObservation, identifier="NETWORK_INTERFACE"): where=parent_where + ["NICs", config.nic_num], include_nmne=config.include_nmne, monitored_traffic=config.monitored_traffic, + thresholds=config.thresholds, ) diff --git a/src/primaite/game/agent/observations/node_observations.py b/src/primaite/game/agent/observations/node_observations.py index e263cadb..91bf402e 100644 --- a/src/primaite/game/agent/observations/node_observations.py +++ b/src/primaite/game/agent/observations/node_observations.py @@ -195,6 +195,8 @@ class NodesObservation(AbstractObservation, identifier="NODES"): host_config.file_system_requires_scan = config.file_system_requires_scan if host_config.include_users is None: host_config.include_users = config.include_users + if host_config.thresholds is None: + host_config.thresholds = config.thresholds for router_config in config.routers: if router_config.num_ports is None: @@ -211,6 +213,8 @@ class NodesObservation(AbstractObservation, identifier="NODES"): router_config.num_rules = config.num_rules if router_config.include_users is None: router_config.include_users = config.include_users + if router_config.thresholds is None: + router_config.thresholds = config.thresholds for firewall_config in config.firewalls: if firewall_config.ip_list is None: @@ -225,6 +229,8 @@ class NodesObservation(AbstractObservation, identifier="NODES"): firewall_config.num_rules = config.num_rules if firewall_config.include_users is None: firewall_config.include_users = config.include_users + if firewall_config.thresholds is None: + firewall_config.thresholds = config.thresholds hosts = [HostObservation.from_config(config=c, parent_where=where) for c in config.hosts] routers = [RouterObservation.from_config(config=c, parent_where=where) for c in config.routers] diff --git a/src/primaite/game/agent/observations/observation_manager.py b/src/primaite/game/agent/observations/observation_manager.py index 9b20fdcb..cc32918c 100644 --- a/src/primaite/game/agent/observations/observation_manager.py +++ b/src/primaite/game/agent/observations/observation_manager.py @@ -113,7 +113,9 @@ class NestedObservation(AbstractObservation, identifier="CUSTOM"): instances = dict() for component in config.components: obs_class = AbstractObservation._registry[component.type] - obs_instance = obs_class.from_config(config=obs_class.ConfigSchema(**component.options)) + obs_instance = obs_class.from_config( + config=obs_class.ConfigSchema(**component.options, thresholds=config.thresholds) + ) instances[component.label] = obs_instance return cls(components=instances) @@ -176,7 +178,7 @@ class ObservationManager: return self.obs.space @classmethod - def from_config(cls, config: Optional[Dict]) -> "ObservationManager": + def from_config(cls, config: Optional[Dict], thresholds: Optional[Dict] = {}) -> "ObservationManager": """ Create observation space from a config. @@ -187,11 +189,15 @@ class ObservationManager: AbstractObservation options: this must adhere to the chosen observation type's ConfigSchema nested class. :type config: Dict + :param thresholds: Dictionary containing the observation thresholds. + :type thresholds: Optional[Dict] """ if config is None: return cls(NullObservation()) obs_type = config["type"] obs_class = AbstractObservation._registry[obs_type] - observation = obs_class.from_config(config=obs_class.ConfigSchema(**config["options"])) + observation = obs_class.from_config( + config=obs_class.ConfigSchema(**config["options"], thresholds=thresholds), + ) obs_manager = cls(observation) return obs_manager diff --git a/src/primaite/game/agent/observations/observations.py b/src/primaite/game/agent/observations/observations.py index a9663c56..0b209f52 100644 --- a/src/primaite/game/agent/observations/observations.py +++ b/src/primaite/game/agent/observations/observations.py @@ -1,7 +1,7 @@ # © Crown-owned copyright 2024, Defence Science and Technology Laboratory UK """Manages the observation space for the agent.""" from abc import ABC, abstractmethod -from typing import Any, Dict, Iterable, Optional, Type, Union +from typing import Any, Dict, Iterable, List, Optional, Type, Union from gymnasium import spaces from gymnasium.core import ObsType @@ -19,6 +19,9 @@ class AbstractObservation(ABC): class ConfigSchema(ABC, BaseModel): """Config schema for observations.""" + thresholds: Optional[Dict] = None + """A dict containing the observation thresholds.""" + model_config = ConfigDict(extra="forbid") _registry: Dict[str, Type["AbstractObservation"]] = {} @@ -67,3 +70,34 @@ class AbstractObservation(ABC): def from_config(cls, config: ConfigSchema, parent_where: WhereType = []) -> "AbstractObservation": """Create this observation space component form a serialised format.""" return cls() + + def _validate_thresholds(self, thresholds: List[int] = None, threshold_identifier: Optional[str] = "") -> bool: + """ + Method that checks if the thresholds are non overlapping and in the correct (ascending) order. + + Pass in the thresholds from low to high e.g. + thresholds=[low_threshold, med_threshold, ..._threshold, high_threshold] + + Throws an error if the threshold is not valid + + :param: thresholds: List of thresholds in ascending order. + :type: List[int] + :param: threshold_identifier: The name of the threshold option. + :type: Optional[str] + + :returns: bool + """ + if thresholds is None or len(thresholds) < 2: + raise Exception(f"{threshold_identifier} thresholds are invalid {thresholds}") + for idx in range(1, len(thresholds)): + if not isinstance(thresholds[idx], int): + raise Exception(f"{threshold_identifier} threshold ({thresholds[idx]}) is not a valid int.") + if not isinstance(thresholds[idx - 1], int): + raise Exception(f"{threshold_identifier} threshold ({thresholds[idx]}) is not a valid int.") + + if thresholds[idx] <= thresholds[idx - 1]: + raise Exception( + f"{threshold_identifier} threshold ({thresholds[idx]}) " + f"is greater than or equal to ({thresholds[idx - 1]}.)" + ) + return True diff --git a/src/primaite/game/agent/observations/software_observation.py b/src/primaite/game/agent/observations/software_observation.py index 15cd2447..10adb5c5 100644 --- a/src/primaite/game/agent/observations/software_observation.py +++ b/src/primaite/game/agent/observations/software_observation.py @@ -1,7 +1,7 @@ # © Crown-owned copyright 2024, Defence Science and Technology Laboratory UK from __future__ import annotations -from typing import Dict +from typing import Dict, Optional from gymnasium import spaces from gymnasium.core import ObsType @@ -82,7 +82,7 @@ class ApplicationObservation(AbstractObservation, identifier="APPLICATION"): application_name: str """Name of the application, used for querying simulation state dictionary""" - def __init__(self, where: WhereType) -> None: + def __init__(self, where: WhereType, thresholds: Optional[Dict] = {}) -> None: """ Initialise an application observation instance. @@ -94,16 +94,28 @@ class ApplicationObservation(AbstractObservation, identifier="APPLICATION"): self.where = where self.default_observation = {"operating_status": 0, "health_status": 0, "num_executions": 0} - # TODO: allow these to be configured in yaml - self.high_threshold = 10 - self.med_threshold = 5 - self.low_threshold = 0 + if thresholds.get("app_executions") is None: + self.low_threshold = 0 + self.med_threshold = 5 + self.high_threshold = 10 + else: + if self._validate_thresholds( + thresholds=[ + thresholds.get("app_executions")["low"], + thresholds.get("app_executions")["medium"], + thresholds.get("app_executions")["high"], + ], + threshold_identifier="app_executions", + ): + self.low_threshold = thresholds.get("app_executions")["low"] + self.med_threshold = thresholds.get("app_executions")["medium"] + self.high_threshold = thresholds.get("app_executions")["high"] def _categorise_num_executions(self, num_executions: int) -> int: """ - Represent number of file accesses as a categorical variable. + Represent number of application executions as a categorical variable. - :param num_access: Number of file accesses. + :param num_access: Number of application executions. :return: Bin number corresponding to the number of accesses. """ if num_executions > self.high_threshold: @@ -161,4 +173,4 @@ class ApplicationObservation(AbstractObservation, identifier="APPLICATION"): :return: Constructed application observation instance. :rtype: ApplicationObservation """ - return cls(where=parent_where + ["applications", config.application_name]) + return cls(where=parent_where + ["applications", config.application_name], thresholds=config.thresholds) diff --git a/src/primaite/game/game.py b/src/primaite/game/game.py index 123b6ddd..441ea632 100644 --- a/src/primaite/game/game.py +++ b/src/primaite/game/game.py @@ -531,7 +531,7 @@ class PrimaiteGame: reward_function_cfg = agent_cfg["reward_function"] # CREATE OBSERVATION SPACE - obs_space = ObservationManager.from_config(observation_space_cfg) + obs_space = ObservationManager.from_config(config=observation_space_cfg, thresholds=game.options.thresholds) # CREATE ACTION SPACE action_space = ActionManager.from_config(game, action_space_cfg) diff --git a/tests/assets/configs/basic_switched_network.yaml b/tests/assets/configs/basic_switched_network.yaml index fed0f52d..03cf2207 100644 --- a/tests/assets/configs/basic_switched_network.yaml +++ b/tests/assets/configs/basic_switched_network.yaml @@ -25,7 +25,19 @@ game: - ICMP - TCP - UDP - + thresholds: + nmne: + high: 100 + medium: 25 + low: 5 + file_access: + high: 10 + medium: 5 + low: 2 + app_executions: + high: 5 + medium: 3 + low: 2 agents: - ref: client_2_green_user team: GREEN @@ -79,10 +91,16 @@ agents: options: hosts: - hostname: client_1 + applications: + - application_name: WebBrowser + folders: + - folder_name: root + files: + - file_name: "test.txt" - hostname: client_2 - hostname: client_3 num_services: 1 - num_applications: 0 + num_applications: 1 num_folders: 1 num_files: 1 num_nics: 2 @@ -219,6 +237,9 @@ simulation: options: ntp_server_ip: 192.168.1.10 - type: NTPServer + file_system: + - root: + - "test.txt" - hostname: client_2 type: computer ip_address: 192.168.10.22 diff --git a/tests/integration_tests/configuration_file_parsing/test_game_options_config.py b/tests/integration_tests/configuration_file_parsing/test_game_options_config.py index 32d88c92..2cb5520e 100644 --- a/tests/integration_tests/configuration_file_parsing/test_game_options_config.py +++ b/tests/integration_tests/configuration_file_parsing/test_game_options_config.py @@ -8,7 +8,7 @@ from primaite.config.load import data_manipulation_config_path from primaite.game.game import PrimaiteGame from tests import TEST_ASSETS_ROOT -BASIC_CONFIG = TEST_ASSETS_ROOT / "configs/basic_switched_network.yaml" +BASIC_SWITCHED_NETWORK_CONFIG = TEST_ASSETS_ROOT / "configs/basic_switched_network.yaml" def load_config(config_path: Union[str, Path]) -> PrimaiteGame: @@ -24,3 +24,42 @@ def test_thresholds(): game = load_config(data_manipulation_config_path()) assert game.options.thresholds is not None + + +def test_nmne_threshold(): + """Test that the NMNE thresholds are properly loaded in by observation.""" + game = load_config(BASIC_SWITCHED_NETWORK_CONFIG) + + assert game.options.thresholds["nmne"] is not None + + # get NIC observation + nic_obs = game.agents["defender"].observation_manager.obs.components["NODES"].hosts[0].nics[0] + assert nic_obs.low_threshold == 5 + assert nic_obs.med_threshold == 25 + assert nic_obs.high_threshold == 100 + + +def test_file_access_threshold(): + """Test that the NMNE thresholds are properly loaded in by observation.""" + game = load_config(BASIC_SWITCHED_NETWORK_CONFIG) + + assert game.options.thresholds["file_access"] is not None + + # get file observation + file_obs = game.agents["defender"].observation_manager.obs.components["NODES"].hosts[0].folders[0].files[0] + assert file_obs.low_threshold == 2 + assert file_obs.med_threshold == 5 + assert file_obs.high_threshold == 10 + + +def test_app_executions_threshold(): + """Test that the NMNE thresholds are properly loaded in by observation.""" + game = load_config(BASIC_SWITCHED_NETWORK_CONFIG) + + assert game.options.thresholds["app_executions"] is not None + + # get application observation + app_obs = game.agents["defender"].observation_manager.obs.components["NODES"].hosts[0].applications[0] + assert app_obs.low_threshold == 2 + assert app_obs.med_threshold == 3 + assert app_obs.high_threshold == 5 diff --git a/tests/integration_tests/game_layer/observations/test_file_system_observations.py b/tests/integration_tests/game_layer/observations/test_file_system_observations.py index e2ab2990..cbd9f8c0 100644 --- a/tests/integration_tests/game_layer/observations/test_file_system_observations.py +++ b/tests/integration_tests/game_layer/observations/test_file_system_observations.py @@ -44,6 +44,38 @@ def test_file_observation(simulation): assert observation_state.get("health_status") == 3 # corrupted +def test_config_file_access_categories(simulation): + pc: Computer = simulation.network.get_node_by_hostname("client_1") + file_obs = FileObservation( + where=["network", "nodes", pc.hostname, "file_system", "folders", "root", "files", "dog.png"], + include_num_access=False, + file_system_requires_scan=True, + thresholds={"file_access": {"low": 3, "medium": 6, "high": 9}}, + ) + + assert file_obs.high_threshold == 9 + assert file_obs.med_threshold == 6 + assert file_obs.low_threshold == 3 + + with pytest.raises(Exception): + # should throw an error + FileObservation( + where=["network", "nodes", pc.hostname, "file_system", "folders", "root", "files", "dog.png"], + include_num_access=False, + file_system_requires_scan=True, + thresholds={"file_access": {"low": 9, "medium": 6, "high": 9}}, + ) + + with pytest.raises(Exception): + # should throw an error + FileObservation( + where=["network", "nodes", pc.hostname, "file_system", "folders", "root", "files", "dog.png"], + include_num_access=False, + file_system_requires_scan=True, + thresholds={"file_access": {"low": 3, "medium": 9, "high": 9}}, + ) + + def test_folder_observation(simulation): """Test the folder observation.""" pc: Computer = simulation.network.get_node_by_hostname("client_1") diff --git a/tests/integration_tests/game_layer/observations/test_nic_observations.py b/tests/integration_tests/game_layer/observations/test_nic_observations.py index ef789ba7..cafdec45 100644 --- a/tests/integration_tests/game_layer/observations/test_nic_observations.py +++ b/tests/integration_tests/game_layer/observations/test_nic_observations.py @@ -110,33 +110,28 @@ def test_nic_categories(simulation): nic_obs = NICObservation(where=["network", "nodes", pc.hostname, "NICs", 1], include_nmne=True) - assert nic_obs.high_nmne_threshold == 10 # default - assert nic_obs.med_nmne_threshold == 5 # default - assert nic_obs.low_nmne_threshold == 0 # default + assert nic_obs.high_threshold == 10 # default + assert nic_obs.med_threshold == 5 # default + assert nic_obs.low_threshold == 0 # default -@pytest.mark.skip(reason="Feature not implemented yet") def test_config_nic_categories(simulation): pc: Computer = simulation.network.get_node_by_hostname("client_1") nic_obs = NICObservation( where=["network", "nodes", pc.hostname, "NICs", 1], - low_nmne_threshold=3, - med_nmne_threshold=6, - high_nmne_threshold=9, + thresholds={"nmne": {"low": 3, "medium": 6, "high": 9}}, include_nmne=True, ) - assert nic_obs.high_nmne_threshold == 9 - assert nic_obs.med_nmne_threshold == 6 - assert nic_obs.low_nmne_threshold == 3 + assert nic_obs.high_threshold == 9 + assert nic_obs.med_threshold == 6 + assert nic_obs.low_threshold == 3 with pytest.raises(Exception): # should throw an error NICObservation( where=["network", "nodes", pc.hostname, "NICs", 1], - low_nmne_threshold=9, - med_nmne_threshold=6, - high_nmne_threshold=9, + thresholds={"nmne": {"low": 9, "medium": 6, "high": 9}}, include_nmne=True, ) @@ -144,9 +139,7 @@ def test_config_nic_categories(simulation): # should throw an error NICObservation( where=["network", "nodes", pc.hostname, "NICs", 1], - low_nmne_threshold=3, - med_nmne_threshold=9, - high_nmne_threshold=9, + thresholds={"nmne": {"low": 3, "medium": 9, "high": 9}}, include_nmne=True, ) diff --git a/tests/integration_tests/game_layer/observations/test_software_observations.py b/tests/integration_tests/game_layer/observations/test_software_observations.py index 998aa755..25081585 100644 --- a/tests/integration_tests/game_layer/observations/test_software_observations.py +++ b/tests/integration_tests/game_layer/observations/test_software_observations.py @@ -69,3 +69,30 @@ def test_application_observation(simulation): assert observation_state.get("health_status") == 1 assert observation_state.get("operating_status") == 1 # running assert observation_state.get("num_executions") == 1 + + +def test_application_executions_categories(simulation): + pc: Computer = simulation.network.get_node_by_hostname("client_1") + + app_obs = ApplicationObservation( + where=["network", "nodes", pc.hostname, "applications", "WebBrowser"], + thresholds={"app_executions": {"low": 3, "medium": 6, "high": 9}}, + ) + + assert app_obs.high_threshold == 9 + assert app_obs.med_threshold == 6 + assert app_obs.low_threshold == 3 + + with pytest.raises(Exception): + # should throw an error + ApplicationObservation( + where=["network", "nodes", pc.hostname, "applications", "WebBrowser"], + thresholds={"app_executions": {"low": 9, "medium": 6, "high": 9}}, + ) + + with pytest.raises(Exception): + # should throw an error + ApplicationObservation( + where=["network", "nodes", pc.hostname, "applications", "WebBrowser"], + thresholds={"app_executions": {"low": 3, "medium": 9, "high": 9}}, + ) From 8d3760b5a7e8bf53f8a7e20cabc3a5597ecd897f Mon Sep 17 00:00:00 2001 From: Charlie Crane Date: Tue, 17 Sep 2024 16:19:43 +0100 Subject: [PATCH 54/95] #2871 - Fix notebook failure --- .../notebooks/Getting-Information-Out-Of-PrimAITE.ipynb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/primaite/notebooks/Getting-Information-Out-Of-PrimAITE.ipynb b/src/primaite/notebooks/Getting-Information-Out-Of-PrimAITE.ipynb index e4009822..6a60c1bc 100644 --- a/src/primaite/notebooks/Getting-Information-Out-Of-PrimAITE.ipynb +++ b/src/primaite/notebooks/Getting-Information-Out-Of-PrimAITE.ipynb @@ -160,6 +160,11 @@ "metadata": {}, "outputs": [], "source": [ + "with open(data_manipulation_config_path(), 'r') as f:\n", + " cfg = yaml.safe_load(f)\n", + "\n", + "env = PrimaiteGymEnv(env_config=cfg)\n", + "\n", "# Run the training session to generate some resultant data.\n", "for i in range(100):\n", " env.step(0)" From 0c576746aa1165aac7aa6fc6eebda68a25945249 Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Thu, 19 Sep 2024 11:07:00 +0100 Subject: [PATCH 55/95] #2896: Bump version. --- src/primaite/VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/primaite/VERSION b/src/primaite/VERSION index 15a27998..688932aa 100644 --- a/src/primaite/VERSION +++ b/src/primaite/VERSION @@ -1 +1 @@ -3.3.0 +3.4.0-dev From 88cbb783bc6cd11aed890671be0eb4fa02e371e1 Mon Sep 17 00:00:00 2001 From: Archer Bowen Date: Fri, 20 Sep 2024 13:54:13 +0100 Subject: [PATCH 56/95] #2840 Fixed sphinx user guide formatting issues. --- docs/source/request_system.rst | 2 ++ .../system/services/terminal.rst | 14 +++++++------- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/source/request_system.rst b/docs/source/request_system.rst index f2d2e68d..6b71bf25 100644 --- a/docs/source/request_system.rst +++ b/docs/source/request_system.rst @@ -2,6 +2,8 @@ © Crown-owned copyright 2024, Defence Science and Technology Laboratory UK +.. _request_system: + Request System ************** diff --git a/docs/source/simulation_components/system/services/terminal.rst b/docs/source/simulation_components/system/services/terminal.rst index de6eaf0a..de0bb026 100644 --- a/docs/source/simulation_components/system/services/terminal.rst +++ b/docs/source/simulation_components/system/services/terminal.rst @@ -38,31 +38,31 @@ Implementation - Manages remote connections in a dictionary by session ID. - Processes commands, forwarding to the ``RequestManager`` or ``SessionManager`` where appropriate. - Extends Service class. - - A detailed guide on the implementation and functionality of the Terminal class can be found in the "Terminal-Processing" jupyter notebook. + +A detailed guide on the implementation and functionality of the Terminal class can be found in the "Terminal-Processing" jupyter notebook. Command Format ^^^^^^^^^^^^^^ -``Terminals`` implement their commands through leveraging the pre-existing :doc:`../../request_system`. +Terminals implement their commands through leveraging the pre-existing :ref:`request_system`. -Due to this ``Terminals`` will only accept commands passed within the ``RequestFormat``. +Due to this Terminals will only accept commands passed within the ``RequestFormat``. :py:class:`primaite.game.interface.RequestFormat` For example, ``terminal`` command actions when used in ``yaml`` format are formatted as follows: .. code-block:: yaml + command: - "file_system" - "create" - "file" - "downloads" - "cat.png" - - "False" + - "False -**This command creates file called ``cat.png`` within the ``downloads`` folder.** - -This is then loaded from ``yaml`` into a dictionary containing the terminal command: +This is then loaded from yaml into a dictionary containing the terminal command: .. code-block:: python From e29815305dd4eaf0294b73a6b0d4a2e3ba4ccb75 Mon Sep 17 00:00:00 2001 From: Archer Bowen Date: Tue, 24 Sep 2024 11:06:38 +0100 Subject: [PATCH 57/95] #2840 Addressing PR comments. --- .../simulation_components/system/services/terminal.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/simulation_components/system/services/terminal.rst b/docs/source/simulation_components/system/services/terminal.rst index de0bb026..b11d74bb 100644 --- a/docs/source/simulation_components/system/services/terminal.rst +++ b/docs/source/simulation_components/system/services/terminal.rst @@ -26,7 +26,7 @@ Key capabilities Usage """"" - - Pre-Installs on any `Node` (component with the exception of `Switches`). + - Pre-Installs on any `Node` component (with the exception of `Switches`). - Terminal Clients connect, execute commands and disconnect from remote nodes. - Ensures that users are logged in to the component before executing any commands. - Service runs on SSH port 22 by default. @@ -68,7 +68,7 @@ This is then loaded from yaml into a dictionary containing the terminal command: {"command":["file_system", "create", "file", "downloads", "cat.png", "False"]} -Which is then parsed to the ``Terminals`` Request Manager to be executed. +Which is then passed to the ``Terminals`` Request Manager to be executed. Game Layer Usage (Agents) ======================== @@ -121,7 +121,7 @@ Agents are able to use the terminal to login into remote nodes via ``SSH`` which ``NODE_SEND_REMOTE_COMMAND`` """""""""""""""""""""""""""" -After remotely login into another host, a agent can use the ``NODE_SEND_REMOTE_COMMAND`` to execute commands across the network remotely. +After remotely logging into another host, an agent can use the ``NODE_SEND_REMOTE_COMMAND`` to execute commands across the network remotely. .. code-block:: yaml From b9df2bd6a8d4e1259213bd08fd61c489fc26a845 Mon Sep 17 00:00:00 2001 From: Czar Echavez Date: Wed, 25 Sep 2024 10:50:26 +0100 Subject: [PATCH 58/95] #2445: apply PR suggestions --- CHANGELOG.md | 1 + .../observations/file_system_observations.py | 57 +++++++++---------- .../agent/observations/nic_observations.py | 38 ++++++++----- .../game/agent/observations/observations.py | 4 +- .../observations/software_observation.py | 42 +++++++++----- .../test_game_options_config.py | 18 +++--- .../test_file_system_observations.py | 6 +- .../observations/test_nic_observations.py | 12 ++-- .../test_software_observations.py | 6 +- 9 files changed, 103 insertions(+), 81 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 056742e4..c748a969 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Log observation space data by episode and step. - Added `show_history` method to Agents, allowing you to view actions taken by an agent per step. By default, `DONOTHING` actions are omitted. - New ``NODE_SEND_LOCAL_COMMAND`` action implemented which grants agents the ability to execute commands locally. (Previously limited to remote only) +- Added ability to be able to set the observation threshold for NMNE, file access and application executions ### Changed - ACL's are no longer applied to layer-2 traffic. diff --git a/src/primaite/game/agent/observations/file_system_observations.py b/src/primaite/game/agent/observations/file_system_observations.py index fe959c9f..b24b26a6 100644 --- a/src/primaite/game/agent/observations/file_system_observations.py +++ b/src/primaite/game/agent/observations/file_system_observations.py @@ -55,21 +55,35 @@ class FileObservation(AbstractObservation, identifier="FILE"): self.default_observation["num_access"] = 0 if thresholds.get("file_access") is None: - self.low_threshold = 0 - self.med_threshold = 5 - self.high_threshold = 10 + self.low_file_access_threshold = 0 + self.med_file_access_threshold = 5 + self.high_file_access_threshold = 10 else: - if self._validate_thresholds( + self._set_file_access_threshold( thresholds=[ thresholds.get("file_access")["low"], thresholds.get("file_access")["medium"], thresholds.get("file_access")["high"], - ], - threshold_identifier="file_access", - ): - self.low_threshold = thresholds.get("file_access")["low"] - self.med_threshold = thresholds.get("file_access")["medium"] - self.high_threshold = thresholds.get("file_access")["high"] + ] + ) + + def _set_file_access_threshold(self, thresholds: List[int]): + """ + Method that validates and then sets the file access threshold. + + :param: thresholds: The file access threshold to validate and set. + """ + if self._validate_thresholds( + thresholds=[ + thresholds[0], + thresholds[1], + thresholds[2], + ], + threshold_identifier="file_access", + ): + self.low_file_access_threshold = thresholds[0] + self.med_file_access_threshold = thresholds[1] + self.high_file_access_threshold = thresholds[2] def _categorise_num_access(self, num_access: int) -> int: """ @@ -78,11 +92,11 @@ class FileObservation(AbstractObservation, identifier="FILE"): :param num_access: Number of file accesses. :return: Bin number corresponding to the number of accesses. """ - if num_access > self.high_threshold: + if num_access > self.high_file_access_threshold: return 3 - elif num_access > self.med_threshold: + elif num_access > self.med_file_access_threshold: return 2 - elif num_access > self.low_threshold: + elif num_access > self.low_file_access_threshold: return 1 return 0 @@ -190,23 +204,6 @@ class FolderObservation(AbstractObservation, identifier="FOLDER"): self.file_system_requires_scan: bool = file_system_requires_scan - if thresholds.get("file_access") is None: - self.low_threshold = 0 - self.med_threshold = 5 - self.high_threshold = 10 - else: - if self._validate_thresholds( - thresholds=[ - thresholds.get("file_access")["low"], - thresholds.get("file_access")["medium"], - thresholds.get("file_access")["high"], - ], - threshold_identifier="file_access", - ): - self.low_threshold = thresholds.get("file_access")["low"] - self.med_threshold = thresholds.get("file_access")["medium"] - self.high_threshold = thresholds.get("file_access")["high"] - self.files: List[FileObservation] = files while len(self.files) < num_files: self.files.append( diff --git a/src/primaite/game/agent/observations/nic_observations.py b/src/primaite/game/agent/observations/nic_observations.py index 30ee240d..0dabd9f4 100644 --- a/src/primaite/game/agent/observations/nic_observations.py +++ b/src/primaite/game/agent/observations/nic_observations.py @@ -1,7 +1,7 @@ # © Crown-owned copyright 2024, Defence Science and Technology Laboratory UK from __future__ import annotations -from typing import ClassVar, Dict, Optional +from typing import ClassVar, Dict, List, Optional from gymnasium import spaces from gymnasium.core import ObsType @@ -55,21 +55,17 @@ class NICObservation(AbstractObservation, identifier="NETWORK_INTERFACE"): self.nmne_outbound_last_step: int = 0 if thresholds.get("nmne") is None: - self.low_threshold = 0 - self.med_threshold = 5 - self.high_threshold = 10 + self.low_nmne_threshold = 0 + self.med_nmne_threshold = 5 + self.high_nmne_threshold = 10 else: - if self._validate_thresholds( + self._set_nmne_threshold( thresholds=[ thresholds.get("nmne")["low"], thresholds.get("nmne")["medium"], thresholds.get("nmne")["high"], - ], - threshold_identifier="nmne", - ): - self.low_threshold = thresholds.get("nmne")["low"] - self.med_threshold = thresholds.get("nmne")["medium"] - self.high_threshold = thresholds.get("nmne")["high"] + ] + ) self.monitored_traffic = monitored_traffic if self.monitored_traffic: @@ -108,11 +104,11 @@ class NICObservation(AbstractObservation, identifier="NETWORK_INTERFACE"): :param nmne_count: Number of MNEs detected. :return: Bin number corresponding to the number of MNEs. Returns 0, 1, 2, or 3 based on the detected MNE count. """ - if nmne_count > self.high_threshold: + if nmne_count > self.high_nmne_threshold: return 3 - elif nmne_count > self.med_threshold: + elif nmne_count > self.med_nmne_threshold: return 2 - elif nmne_count > self.low_threshold: + elif nmne_count > self.low_nmne_threshold: return 1 return 0 @@ -126,6 +122,20 @@ class NICObservation(AbstractObservation, identifier="NETWORK_INTERFACE"): bandwidth_utilisation = traffic_value / nic_max_bandwidth return int(bandwidth_utilisation * 9) + 1 + def _set_nmne_threshold(self, thresholds: List[int]): + """ + Method that validates and then sets the NMNE threshold. + + :param: thresholds: The NMNE threshold to validate and set. + """ + if self._validate_thresholds( + thresholds=thresholds, + threshold_identifier="nmne", + ): + self.low_nmne_threshold = thresholds[0] + self.med_nmne_threshold = thresholds[1] + self.high_nmne_threshold = thresholds[2] + def observe(self, state: Dict) -> ObsType: """ Generate observation based on the current state of the simulation. diff --git a/src/primaite/game/agent/observations/observations.py b/src/primaite/game/agent/observations/observations.py index 0b209f52..7a31a26b 100644 --- a/src/primaite/game/agent/observations/observations.py +++ b/src/primaite/game/agent/observations/observations.py @@ -97,7 +97,7 @@ class AbstractObservation(ABC): if thresholds[idx] <= thresholds[idx - 1]: raise Exception( - f"{threshold_identifier} threshold ({thresholds[idx]}) " - f"is greater than or equal to ({thresholds[idx - 1]}.)" + f"{threshold_identifier} threshold ({thresholds[idx - 1]}) " + f"is greater than or equal to ({thresholds[idx]}.)" ) return True diff --git a/src/primaite/game/agent/observations/software_observation.py b/src/primaite/game/agent/observations/software_observation.py index 10ffe3fc..0318c864 100644 --- a/src/primaite/game/agent/observations/software_observation.py +++ b/src/primaite/game/agent/observations/software_observation.py @@ -1,7 +1,7 @@ # © Crown-owned copyright 2024, Defence Science and Technology Laboratory UK from __future__ import annotations -from typing import Dict, Optional +from typing import Dict, List, Optional from gymnasium import spaces from gymnasium.core import ObsType @@ -109,21 +109,35 @@ class ApplicationObservation(AbstractObservation, identifier="APPLICATION"): self.default_observation = {"operating_status": 0, "health_status": 0, "num_executions": 0} if thresholds.get("app_executions") is None: - self.low_threshold = 0 - self.med_threshold = 5 - self.high_threshold = 10 + self.low_app_execution_threshold = 0 + self.med_app_execution_threshold = 5 + self.high_app_execution_threshold = 10 else: - if self._validate_thresholds( + self._set_application_execution_thresholds( thresholds=[ thresholds.get("app_executions")["low"], thresholds.get("app_executions")["medium"], thresholds.get("app_executions")["high"], - ], - threshold_identifier="app_executions", - ): - self.low_threshold = thresholds.get("app_executions")["low"] - self.med_threshold = thresholds.get("app_executions")["medium"] - self.high_threshold = thresholds.get("app_executions")["high"] + ] + ) + + def _set_application_execution_thresholds(self, thresholds: List[int]): + """ + Method that validates and then sets the application execution threshold. + + :param: thresholds: The application execution threshold to validate and set. + """ + if self._validate_thresholds( + thresholds=[ + thresholds[0], + thresholds[1], + thresholds[2], + ], + threshold_identifier="app_executions", + ): + self.low_app_execution_threshold = thresholds[0] + self.med_app_execution_threshold = thresholds[1] + self.high_app_execution_threshold = thresholds[2] def _categorise_num_executions(self, num_executions: int) -> int: """ @@ -132,11 +146,11 @@ class ApplicationObservation(AbstractObservation, identifier="APPLICATION"): :param num_access: Number of application executions. :return: Bin number corresponding to the number of accesses. """ - if num_executions > self.high_threshold: + if num_executions > self.high_app_execution_threshold: return 3 - elif num_executions > self.med_threshold: + elif num_executions > self.med_app_execution_threshold: return 2 - elif num_executions > self.low_threshold: + elif num_executions > self.low_app_execution_threshold: return 1 return 0 diff --git a/tests/integration_tests/configuration_file_parsing/test_game_options_config.py b/tests/integration_tests/configuration_file_parsing/test_game_options_config.py index 2cb5520e..4098db7f 100644 --- a/tests/integration_tests/configuration_file_parsing/test_game_options_config.py +++ b/tests/integration_tests/configuration_file_parsing/test_game_options_config.py @@ -34,9 +34,9 @@ def test_nmne_threshold(): # get NIC observation nic_obs = game.agents["defender"].observation_manager.obs.components["NODES"].hosts[0].nics[0] - assert nic_obs.low_threshold == 5 - assert nic_obs.med_threshold == 25 - assert nic_obs.high_threshold == 100 + assert nic_obs.low_nmne_threshold == 5 + assert nic_obs.med_nmne_threshold == 25 + assert nic_obs.high_nmne_threshold == 100 def test_file_access_threshold(): @@ -47,9 +47,9 @@ def test_file_access_threshold(): # get file observation file_obs = game.agents["defender"].observation_manager.obs.components["NODES"].hosts[0].folders[0].files[0] - assert file_obs.low_threshold == 2 - assert file_obs.med_threshold == 5 - assert file_obs.high_threshold == 10 + assert file_obs.low_file_access_threshold == 2 + assert file_obs.med_file_access_threshold == 5 + assert file_obs.high_file_access_threshold == 10 def test_app_executions_threshold(): @@ -60,6 +60,6 @@ def test_app_executions_threshold(): # get application observation app_obs = game.agents["defender"].observation_manager.obs.components["NODES"].hosts[0].applications[0] - assert app_obs.low_threshold == 2 - assert app_obs.med_threshold == 3 - assert app_obs.high_threshold == 5 + assert app_obs.low_app_execution_threshold == 2 + assert app_obs.med_app_execution_threshold == 3 + assert app_obs.high_app_execution_threshold == 5 diff --git a/tests/integration_tests/game_layer/observations/test_file_system_observations.py b/tests/integration_tests/game_layer/observations/test_file_system_observations.py index cbd9f8c0..6356c297 100644 --- a/tests/integration_tests/game_layer/observations/test_file_system_observations.py +++ b/tests/integration_tests/game_layer/observations/test_file_system_observations.py @@ -53,9 +53,9 @@ def test_config_file_access_categories(simulation): thresholds={"file_access": {"low": 3, "medium": 6, "high": 9}}, ) - assert file_obs.high_threshold == 9 - assert file_obs.med_threshold == 6 - assert file_obs.low_threshold == 3 + assert file_obs.high_file_access_threshold == 9 + assert file_obs.med_file_access_threshold == 6 + assert file_obs.low_file_access_threshold == 3 with pytest.raises(Exception): # should throw an error diff --git a/tests/integration_tests/game_layer/observations/test_nic_observations.py b/tests/integration_tests/game_layer/observations/test_nic_observations.py index 9b2baf25..d01d0c8e 100644 --- a/tests/integration_tests/game_layer/observations/test_nic_observations.py +++ b/tests/integration_tests/game_layer/observations/test_nic_observations.py @@ -118,9 +118,9 @@ def test_nic_categories(simulation): nic_obs = NICObservation(where=["network", "nodes", pc.hostname, "NICs", 1], include_nmne=True) - assert nic_obs.high_threshold == 10 # default - assert nic_obs.med_threshold == 5 # default - assert nic_obs.low_threshold == 0 # default + assert nic_obs.high_nmne_threshold == 10 # default + assert nic_obs.med_nmne_threshold == 5 # default + assert nic_obs.low_nmne_threshold == 0 # default def test_config_nic_categories(simulation): @@ -131,9 +131,9 @@ def test_config_nic_categories(simulation): include_nmne=True, ) - assert nic_obs.high_threshold == 9 - assert nic_obs.med_threshold == 6 - assert nic_obs.low_threshold == 3 + assert nic_obs.high_nmne_threshold == 9 + assert nic_obs.med_nmne_threshold == 6 + assert nic_obs.low_nmne_threshold == 3 with pytest.raises(Exception): # should throw an error diff --git a/tests/integration_tests/game_layer/observations/test_software_observations.py b/tests/integration_tests/game_layer/observations/test_software_observations.py index 22374718..a0637969 100644 --- a/tests/integration_tests/game_layer/observations/test_software_observations.py +++ b/tests/integration_tests/game_layer/observations/test_software_observations.py @@ -84,9 +84,9 @@ def test_application_executions_categories(simulation): thresholds={"app_executions": {"low": 3, "medium": 6, "high": 9}}, ) - assert app_obs.high_threshold == 9 - assert app_obs.med_threshold == 6 - assert app_obs.low_threshold == 3 + assert app_obs.high_app_execution_threshold == 9 + assert app_obs.med_app_execution_threshold == 6 + assert app_obs.low_app_execution_threshold == 3 with pytest.raises(Exception): # should throw an error From 603c68acf9cc2b7ef6c247e3b8eab26040f62e8a Mon Sep 17 00:00:00 2001 From: Czar Echavez Date: Thu, 26 Sep 2024 08:51:30 +0100 Subject: [PATCH 59/95] #2445: grammar in changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c748a969..bd4b992c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Log observation space data by episode and step. - Added `show_history` method to Agents, allowing you to view actions taken by an agent per step. By default, `DONOTHING` actions are omitted. - New ``NODE_SEND_LOCAL_COMMAND`` action implemented which grants agents the ability to execute commands locally. (Previously limited to remote only) -- Added ability to be able to set the observation threshold for NMNE, file access and application executions +- Added ability to set the observation threshold for NMNE, file access and application executions ### Changed - ACL's are no longer applied to layer-2 traffic. From 17fe5cb043341f5ab52db433a059cbe36276cf77 Mon Sep 17 00:00:00 2001 From: Czar Echavez Date: Fri, 27 Sep 2024 10:47:38 +0100 Subject: [PATCH 60/95] #2897: How to guide on dev mode --- .../How-To-Use-Primaite-Dev-Mode.ipynb | 479 ++++++++++++++++++ 1 file changed, 479 insertions(+) create mode 100644 src/primaite/notebooks/How-To-Use-Primaite-Dev-Mode.ipynb diff --git a/src/primaite/notebooks/How-To-Use-Primaite-Dev-Mode.ipynb b/src/primaite/notebooks/How-To-Use-Primaite-Dev-Mode.ipynb new file mode 100644 index 00000000..8f8ec24b --- /dev/null +++ b/src/primaite/notebooks/How-To-Use-Primaite-Dev-Mode.ipynb @@ -0,0 +1,479 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# PrimAITE Developer mode\n", + "\n", + "PrimAITE has built in developer tools.\n", + "\n", + "The dev-mode is designed to help make the development of PrimAITE easier.\n", + "\n", + "`NOTE: For the purposes of the notebook, the commands are preceeded by \"!\". When running the commands, run it without the \"!\".`\n", + "\n", + "To display the available dev-mode options, run the command below:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!primaite dev-mode --help" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Save the current PRIMAITE_CONFIG to restore after the notebook runs\n", + "\n", + "from primaite import PRIMAITE_CONFIG\n", + "\n", + "temp_config = PRIMAITE_CONFIG.copy()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Dev mode options" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### enable\n", + "\n", + "Enables the dev mode for PrimAITE.\n", + "\n", + "This will enable the developer mode for PrimAITE.\n", + "\n", + "By default, when developer mode is enabled, session logs will be generated in the PRIMAITE_ROOT/sessions folder unless configured to be generated in another location." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!primaite dev-mode enable" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### disable\n", + "\n", + "Disables the dev mode for PrimAITE.\n", + "\n", + "This will disable the developer mode for PrimAITE." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!primaite dev-mode disable" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### show\n", + "\n", + "Shows if PrimAITE is running in dev mode or production mode.\n", + "\n", + "The command will also show the developer mode configuration." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!primaite dev-mode show" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### config\n", + "\n", + "Configure the PrimAITE developer mode" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!primaite dev-mode config --help" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### path\n", + "\n", + "Set the path where generated session files will be output.\n", + "\n", + "By default, this value will be in PRIMAITE_ROOT/sessions.\n", + "\n", + "To reset the path to default, run:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!primaite dev-mode config path -root\n", + "\n", + "# or\n", + "\n", + "!primaite dev-mode config path --default" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### --sys-log-level or -slevel\n", + "\n", + "Set the system log level.\n", + "\n", + "This will override the system log level in configurations and will make PrimAITE include the set log level and above.\n", + "\n", + "Available options are:\n", + "- `DEBUG`\n", + "- `INFO`\n", + "- `WARNING`\n", + "- `ERROR`\n", + "- `CRITICAL`\n", + "\n", + "Default value is `DEBUG`\n", + "\n", + "Example:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!primaite dev-mode config --sys-log-level DEBUG\n", + "\n", + "# or\n", + "\n", + "!primaite dev-mode config -slevel DEBUG" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### --agent-log-level or -alevel\n", + "\n", + "Set the agent log level.\n", + "\n", + "This will override the agent log level in configurations and will make PrimAITE include the set log level and above.\n", + "\n", + "Available options are:\n", + "- `DEBUG`\n", + "- `INFO`\n", + "- `WARNING`\n", + "- `ERROR`\n", + "- `CRITICAL`\n", + "\n", + "Default value is `DEBUG`\n", + "\n", + "Example:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!primaite dev-mode config --agent-log-level DEBUG\n", + "\n", + "# or\n", + "\n", + "!primaite dev-mode config -alevel DEBUG" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### --output-sys-logs or -sys\n", + "\n", + "If enabled, developer mode will output system logs.\n", + "\n", + "Example:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!primaite dev-mode config --output-sys-logs\n", + "\n", + "# or\n", + "\n", + "!primaite dev-mode config -sys" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To disable outputting sys logs:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!primaite dev-mode config --no-sys-logs\n", + "\n", + "# or\n", + "\n", + "!primaite dev-mode config -nsys" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### --output-agent-logs or -agent\n", + "\n", + "If enabled, developer mode will output agent action logs.\n", + "\n", + "Example:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!primaite dev-mode config --output-agent-logs\n", + "\n", + "# or\n", + "\n", + "!primaite dev-mode config -agent" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To disable outputting agent action logs:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!primaite dev-mode config --no-agent-logs\n", + "\n", + "# or\n", + "\n", + "!primaite dev-mode config -nagent" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### --output-pcap-logs or -pcap\n", + "\n", + "If enabled, developer mode will output PCAP logs.\n", + "\n", + "Example:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!primaite dev-mode config --output-pcap-logs\n", + "\n", + "# or\n", + "\n", + "!primaite dev-mode config -pcap" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To disable outputting PCAP logs:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!primaite dev-mode config --no-pcap-logs\n", + "\n", + "# or\n", + "\n", + "!primaite dev-mode config -npcap" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### --output-to-terminal or -t\n", + "\n", + "If enabled, developer mode will output logs to the terminal.\n", + "\n", + "Example:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!primaite dev-mode config --output-to-terminal\n", + "\n", + "# or\n", + "\n", + "!primaite dev-mode config -t" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To disable terminal outputs:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!primaite dev-mode config --no-terminal\n", + "\n", + "# or\n", + "\n", + "!primaite dev-mode config -nt" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Combining commands\n", + "\n", + "It is possible to combine commands to set the configuration.\n", + "\n", + "This saves having to enter multiple commands and allows for a much more efficient setting of PrimAITE developer mode configurations." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Example of setting system log level and enabling the system logging:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!primaite dev-mode config -slevel WARNING -sys" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Another example where the system log and agent action log levels are set and enabled and should be printed to terminal:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!primaite dev-mode config -slevel ERROR -sys -alevel ERROR -agent -t" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Restore PRIMAITE_CONFIG\n", + "from primaite.utils.cli.primaite_config_utils import update_primaite_application_config\n", + "\n", + "\n", + "global PRIMAITE_CONFIG\n", + "PRIMAITE_CONFIG[\"developer_mode\"] = temp_config[\"developer_mode\"]\n", + "update_primaite_application_config(config=PRIMAITE_CONFIG)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.11" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From ac921749a7f0f938505b35275ca9c7f8025c2ccb Mon Sep 17 00:00:00 2001 From: Charlie Crane Date: Mon, 30 Sep 2024 17:38:24 +0100 Subject: [PATCH 61/95] #2900 - Changes to terminal to include a last_response attribute, for use in obtaining RequestResponse from remote command executions --- .../simulator/network/hardware/base.py | 2 +- .../system/services/terminal/terminal.py | 47 +++++++++++++++---- 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/src/primaite/simulator/network/hardware/base.py b/src/primaite/simulator/network/hardware/base.py index f49d0a17..570a69b3 100644 --- a/src/primaite/simulator/network/hardware/base.py +++ b/src/primaite/simulator/network/hardware/base.py @@ -1711,7 +1711,7 @@ class Node(SimComponent): """ application_name = request[0] if self.software_manager.software.get(application_name): - self.sys_log.warning(f"Can't install {application_name}. It's already installed.") + self.sys_log.info(f"Can't install {application_name}. It's already installed.") return RequestResponse(status="success", data={"reason": "already installed"}) application_class = Application._application_registry[application_name] self.software_manager.install(application_class) diff --git a/src/primaite/simulator/system/services/terminal/terminal.py b/src/primaite/simulator/system/services/terminal/terminal.py index dc7da205..77c67460 100644 --- a/src/primaite/simulator/system/services/terminal/terminal.py +++ b/src/primaite/simulator/system/services/terminal/terminal.py @@ -135,12 +135,20 @@ class Terminal(Service): _client_connection_requests: Dict[str, Optional[Union[str, TerminalClientConnection]]] = {} """Dictionary of connect requests made to remote nodes.""" + _last_response: Optional[RequestResponse] = None + """Last response received from RequestManager, for returning remote RequestResponse.""" + def __init__(self, **kwargs): kwargs["name"] = "Terminal" kwargs["port"] = Port.SSH kwargs["protocol"] = IPProtocol.TCP super().__init__(**kwargs) + @property + def last_response(self) -> Optional[RequestResponse]: + """Public version of _last_response attribute.""" + return self._last_response + def describe_state(self) -> Dict: """ Produce a dictionary describing the current state of this object. @@ -202,12 +210,8 @@ class Terminal(Service): command: str = request[1]["command"] remote_connection = self._get_connection_from_ip(ip_address=ip_address) if remote_connection: - outcome = remote_connection.execute(command) - if outcome: - return RequestResponse( - status="success", - data={}, - ) + remote_connection.execute(command) + return self.last_response if not None else RequestResponse(status="failure", data={}) return RequestResponse( status="failure", data={}, @@ -243,7 +247,8 @@ class Terminal(Service): def execute(self, command: List[Any]) -> Optional[RequestResponse]: """Execute a passed ssh command via the request manager.""" - return self.parent.apply_request(command) + self._last_response = self.parent.apply_request(command) + return self._last_response def _get_connection_from_ip(self, ip_address: IPv4Address) -> Optional[RemoteTerminalConnection]: """Find Remote Terminal Connection from a given IP.""" @@ -423,10 +428,11 @@ class Terminal(Service): """ source_ip = kwargs["frame"].ip.src_ip_address self.sys_log.info(f"{self.name}: Received payload: {payload}. Source: {source_ip}") + self._last_response = None # Clear last response + if isinstance(payload, SSHPacket): if payload.transport_message == SSHTransportMessage.SSH_MSG_USERAUTH_REQUEST: # validate & add connection - # TODO: uncomment this as part of 2781 username = payload.user_account.username password = payload.user_account.password connection_id = self.parent.user_session_manager.remote_login( @@ -472,6 +478,9 @@ class Terminal(Service): session_id=session_id, source_ip=source_ip, ) + self._last_response: RequestResponse = RequestResponse( + status="success", data={"reason": "Login Successful"} + ) elif payload.transport_message == SSHTransportMessage.SSH_MSG_SERVICE_REQUEST: # Requesting a command to be executed @@ -483,12 +492,32 @@ class Terminal(Service): payload.connection_uuid ) remote_session.last_active_step = self.software_manager.node.user_session_manager.current_timestep - self.execute(command) + self._last_response: RequestResponse = self.execute(command) + + if self._last_response.status == "success": + transport_message = SSHTransportMessage.SSH_MSG_SERVICE_SUCCESS + else: + transport_message = SSHTransportMessage.SSH_MSG_SERVICE_FAILED + + payload: SSHPacket = SSHPacket( + payload=self._last_response, + transport_message=transport_message, + connection_message=SSHConnectionMessage.SSH_MSG_CHANNEL_DATA, + ) + self.software_manager.send_payload_to_session_manager( + payload=payload, dest_port=self.port, session_id=session_id + ) return True else: self.sys_log.error( f"{self.name}: Connection UUID:{payload.connection_uuid} is not valid. Rejecting Command." ) + elif ( + payload.transport_message == SSHTransportMessage.SSH_MSG_SERVICE_SUCCESS + or SSHTransportMessage.SSH_MSG_SERVICE_FAILED + ): + # Likely receiving command ack from remote. + self._last_response = payload.payload if isinstance(payload, dict) and payload.get("type"): if payload["type"] == "disconnect": From 3dafad71b32dc21eea918e145dbc89a232c59794 Mon Sep 17 00:00:00 2001 From: Charlie Crane Date: Tue, 1 Oct 2024 10:45:03 +0100 Subject: [PATCH 62/95] #2900 - New test to show that last_response updates as expected. Changelog updated. --- CHANGELOG.md | 1 + .../system/services/terminal/terminal.py | 2 +- .../_system/_services/test_terminal.py | 37 +++++++++++++++++++ 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bd4b992c..4a1f7919 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 or `generate_seed_value` is set to `true`. - ARP .show() method will now include the port number associated with each entry. - Added `services_requires_scan` and `applications_requires_scan` to agent observation space config to allow the agents to be able to see actual health states of services and applications without requiring scans (Default `True`, set to `False` to allow agents to see actual health state without scanning). +- Updated the `Terminal` class to provide response information when sending remote command execution. ## [3.3.0] - 2024-09-04 ### Added diff --git a/src/primaite/simulator/system/services/terminal/terminal.py b/src/primaite/simulator/system/services/terminal/terminal.py index 77c67460..ed6854f4 100644 --- a/src/primaite/simulator/system/services/terminal/terminal.py +++ b/src/primaite/simulator/system/services/terminal/terminal.py @@ -214,7 +214,7 @@ class Terminal(Service): return self.last_response if not None else RequestResponse(status="failure", data={}) return RequestResponse( status="failure", - data={}, + data={"reason": "Failed to execute command."}, ) rm.add_request( diff --git a/tests/unit_tests/_primaite/_simulator/_system/_services/test_terminal.py b/tests/unit_tests/_primaite/_simulator/_system/_services/test_terminal.py index 3c3daa61..14cc5877 100644 --- a/tests/unit_tests/_primaite/_simulator/_system/_services/test_terminal.py +++ b/tests/unit_tests/_primaite/_simulator/_system/_services/test_terminal.py @@ -6,6 +6,7 @@ import pytest from primaite.game.agent.interface import ProxyAgent from primaite.game.game import PrimaiteGame +from primaite.interface.request import RequestResponse from primaite.simulator.network.container import Network from primaite.simulator.network.hardware.nodes.host.computer import Computer from primaite.simulator.network.hardware.nodes.host.server import Server @@ -403,3 +404,39 @@ def test_terminal_connection_timeout(basic_network): assert len(computer_b.user_session_manager.remote_sessions) == 0 assert not remote_connection.is_active + + +def test_terminal_last_response_updates(basic_network): + """Test that the _last_response within Terminal correctly updates.""" + network: Network = basic_network + computer_a: Computer = network.get_node_by_hostname("node_a") + terminal_a: Terminal = computer_a.software_manager.software.get("Terminal") + computer_b: Computer = network.get_node_by_hostname("node_b") + + assert terminal_a.last_response is None + + remote_connection = terminal_a.login(username="admin", password="admin", ip_address="192.168.0.11") + + # Last response should be a successful logon + assert terminal_a.last_response == RequestResponse(status="success", data={"reason": "Login Successful"}) + + remote_connection.execute(command=["software_manager", "application", "install", "RansomwareScript"]) + + # Last response should now update following successful install + assert terminal_a.last_response == RequestResponse(status="success", data={}) + + remote_connection.execute(command=["software_manager", "application", "install", "RansomwareScript"]) + + # Last response should now update to success, but with supplied reason. + assert terminal_a.last_response == RequestResponse(status="success", data={"reason": "already installed"}) + + remote_connection.execute(command=["file_system", "create", "file", "folder123", "doggo.pdf", False]) + + # Check file was created. + assert computer_b.file_system.access_file(folder_name="folder123", file_name="doggo.pdf") + + # Last response should be confirmation of file creation. + assert terminal_a.last_response == RequestResponse( + status="success", + data={"file_name": "doggo.pdf", "folder_name": "folder123", "file_type": "PDF", "file_size": 102400}, + ) From 1e1d1524810b271cc60c5393d123d945aa4d6c1f Mon Sep 17 00:00:00 2001 From: Charlie Crane Date: Tue, 1 Oct 2024 11:02:23 +0100 Subject: [PATCH 63/95] #2900 - Updates to Terminal-Processing jupyter notebook to include a mention of last_response --- .../notebooks/Terminal-Processing.ipynb | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/primaite/notebooks/Terminal-Processing.ipynb b/src/primaite/notebooks/Terminal-Processing.ipynb index 19ce567e..2ab06a5c 100644 --- a/src/primaite/notebooks/Terminal-Processing.ipynb +++ b/src/primaite/notebooks/Terminal-Processing.ipynb @@ -167,6 +167,22 @@ "computer_b.file_system.show()" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Information about the latest response when executing a remote command can be seen by calling the `last_response` attribute within `Terminal`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(terminal_a.last_response)" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -488,7 +504,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.12" + "version": "3.10.11" } }, "nbformat": 4, From fcfea3474fe1f5701faa07ab245f60978d8246b4 Mon Sep 17 00:00:00 2001 From: Charlie Crane Date: Tue, 1 Oct 2024 11:41:42 +0100 Subject: [PATCH 64/95] #2900 - typo in test_ftp_client and expanded test_terminal_last_response_updates to include a failure scenario --- .../_system/_services/test_ftp_client.py | 2 +- .../_system/_services/test_terminal.py | 20 +++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/tests/unit_tests/_primaite/_simulator/_system/_services/test_ftp_client.py b/tests/unit_tests/_primaite/_simulator/_system/_services/test_ftp_client.py index 3ce4d8ee..99bb42ed 100644 --- a/tests/unit_tests/_primaite/_simulator/_system/_services/test_ftp_client.py +++ b/tests/unit_tests/_primaite/_simulator/_system/_services/test_ftp_client.py @@ -71,7 +71,7 @@ def test_ftp_should_not_process_commands_if_service_not_running(ftp_client): assert ftp_client_service._process_ftp_command(payload=payload).status_code is FTPStatusCode.ERROR -def test_ftp_tries_to_senf_file__that_does_not_exist(ftp_client): +def test_ftp_tries_to_send_file__that_does_not_exist(ftp_client): """Method send_file should return false if no file to send.""" assert ftp_client.file_system.get_file(folder_name="root", file_name="test.txt") is None diff --git a/tests/unit_tests/_primaite/_simulator/_system/_services/test_terminal.py b/tests/unit_tests/_primaite/_simulator/_system/_services/test_terminal.py index 14cc5877..55f89c04 100644 --- a/tests/unit_tests/_primaite/_simulator/_system/_services/test_terminal.py +++ b/tests/unit_tests/_primaite/_simulator/_system/_services/test_terminal.py @@ -440,3 +440,23 @@ def test_terminal_last_response_updates(basic_network): status="success", data={"file_name": "doggo.pdf", "folder_name": "folder123", "file_type": "PDF", "file_size": 102400}, ) + + remote_connection.execute( + command=[ + "service", + "FTPClient", + "send", + { + "dest_ip_address": "192.168.0.2", + "src_folder": "folder123", + "src_file_name": "cat.pdf", + "dest_folder": "root", + "dest_file_name": "cat.pdf", + }, + ] + ) + + assert terminal_a.last_response == RequestResponse( + status="failure", + data={"reason": "Unable to locate given file on local file system. Perhaps given options are invalid?"}, + ) From 3f94c40434d11034cd1dc5dc9e07800c03bbcec4 Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Wed, 22 Jan 2025 10:49:42 +0000 Subject: [PATCH 65/95] Fix logger inititialisation in agents --- src/primaite/game/agent/interface.py | 7 +++ src/primaite/game/game.py | 1 + .../notebooks/Training-an-SB3-Agent.ipynb | 48 ++++++++++++++++--- 3 files changed, 49 insertions(+), 7 deletions(-) diff --git a/src/primaite/game/agent/interface.py b/src/primaite/game/agent/interface.py index aac898e1..6a6bc323 100644 --- a/src/primaite/game/agent/interface.py +++ b/src/primaite/game/agent/interface.py @@ -68,6 +68,10 @@ class AbstractAgent(BaseModel, ABC): ) reward_function: RewardFunction.ConfigSchema = Field(default_factory=lambda: RewardFunction.ConfigSchema()) + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.logger: AgentLog = AgentLog(agent_name=self.config.ref) + config: "AbstractAgent.ConfigSchema" = Field(default_factory=lambda: AbstractAgent.ConfigSchema()) logger: AgentLog = AgentLog(agent_name="Abstract_Agent") @@ -81,6 +85,7 @@ class AbstractAgent(BaseModel, ABC): def __init_subclass__(cls, identifier: Optional[str] = None, **kwargs: Any) -> None: super().__init_subclass__(**kwargs) + print("cls identifier:", identifier) if identifier is None: return if identifier in cls._registry: @@ -157,6 +162,8 @@ class AbstractAgent(BaseModel, ABC): def from_config(cls, config: Dict) -> AbstractAgent: """Grab the relevant agent class and construct an instance from a config dict.""" agent_type = config["type"] + print("agent_type:", agent_type) + print("cls._registry:", cls._registry) agent_class = cls._registry[agent_type] return agent_class(config=config) diff --git a/src/primaite/game/game.py b/src/primaite/game/game.py index f59117f4..c828a462 100644 --- a/src/primaite/game/game.py +++ b/src/primaite/game/game.py @@ -493,6 +493,7 @@ class PrimaiteGame: agents_cfg = cfg.get("agents", []) for agent_cfg in agents_cfg: + print("agent_cfg:", agent_cfg) new_agent = AbstractAgent.from_config(agent_cfg) game.agents[agent_cfg["ref"]] = new_agent if isinstance(new_agent, ProxyAgent): diff --git a/src/primaite/notebooks/Training-an-SB3-Agent.ipynb b/src/primaite/notebooks/Training-an-SB3-Agent.ipynb index 2b554475..0e3245ae 100644 --- a/src/primaite/notebooks/Training-an-SB3-Agent.ipynb +++ b/src/primaite/notebooks/Training-an-SB3-Agent.ipynb @@ -29,18 +29,28 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "cls identifier: AbstractScriptedAgent\n", + "cls identifier: ProxyAgent\n" + ] + } + ], "source": [ "from primaite.game.game import PrimaiteGame\n", "from primaite.session.environment import PrimaiteGymEnv\n", + "from primaite.game.agent.scripted_agents import probabilistic_agent\n", "import yaml" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -49,7 +59,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -69,9 +79,33 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "agent_cfg: {'ref': 'client_2_green_user', 'team': 'GREEN', 'type': 'ProbabilisticAgent', 'agent_settings': {'action_probabilities': {0: 0.3, 1: 0.6, 2: 0.1}}, 'action_space': {'action_map': {0: {'action': 'do_nothing', 'options': {}}, 1: {'action': 'node_application_execute', 'options': {'node_name': 'client_2', 'application_name': 'WebBrowser'}}, 2: {'action': 'node_application_execute', 'options': {'node_name': 'client_2', 'application_name': 'DatabaseClient'}}}}, 'reward_function': {'reward_components': [{'type': 'WEBPAGE_UNAVAILABLE_PENALTY', 'weight': 0.25, 'options': {'node_hostname': 'client_2'}}, {'type': 'GREEN_ADMIN_DATABASE_UNREACHABLE_PENALTY', 'weight': 0.05, 'options': {'node_hostname': 'client_2'}}]}}\n", + "agent_type: ProbabilisticAgent\n", + "cls._registry: {'AbstractScriptedAgent': , 'ProxyAgent': }\n" + ] + }, + { + "ename": "KeyError", + "evalue": "'ProbabilisticAgent'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[4], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m gym \u001b[38;5;241m=\u001b[39m \u001b[43mPrimaiteGymEnv\u001b[49m\u001b[43m(\u001b[49m\u001b[43menv_config\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcfg\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/session/environment.py:71\u001b[0m, in \u001b[0;36mPrimaiteGymEnv.__init__\u001b[0;34m(self, env_config)\u001b[0m\n\u001b[1;32m 69\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mio \u001b[38;5;241m=\u001b[39m PrimaiteIO\u001b[38;5;241m.\u001b[39mfrom_config(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mepisode_scheduler(\u001b[38;5;241m0\u001b[39m)\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mio_settings\u001b[39m\u001b[38;5;124m\"\u001b[39m, {}))\n\u001b[1;32m 70\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Handles IO for the environment. This produces sys logs, agent logs, etc.\"\"\"\u001b[39;00m\n\u001b[0;32m---> 71\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgame: PrimaiteGame \u001b[38;5;241m=\u001b[39m \u001b[43mPrimaiteGame\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfrom_config\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mepisode_scheduler\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 72\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Current game.\"\"\"\u001b[39;00m\n\u001b[1;32m 73\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_agent_name \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mnext\u001b[39m(\u001b[38;5;28miter\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgame\u001b[38;5;241m.\u001b[39mrl_agents))\n", + "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/game/game.py:497\u001b[0m, in \u001b[0;36mPrimaiteGame.from_config\u001b[0;34m(cls, cfg)\u001b[0m\n\u001b[1;32m 495\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m agent_cfg \u001b[38;5;129;01min\u001b[39;00m agents_cfg:\n\u001b[1;32m 496\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124magent_cfg:\u001b[39m\u001b[38;5;124m\"\u001b[39m, agent_cfg)\n\u001b[0;32m--> 497\u001b[0m new_agent \u001b[38;5;241m=\u001b[39m \u001b[43mAbstractAgent\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfrom_config\u001b[49m\u001b[43m(\u001b[49m\u001b[43magent_cfg\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 498\u001b[0m game\u001b[38;5;241m.\u001b[39magents[agent_cfg[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mref\u001b[39m\u001b[38;5;124m\"\u001b[39m]] \u001b[38;5;241m=\u001b[39m new_agent\n\u001b[1;32m 499\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(new_agent, ProxyAgent):\n", + "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/game/agent/interface.py:163\u001b[0m, in \u001b[0;36mAbstractAgent.from_config\u001b[0;34m(cls, config)\u001b[0m\n\u001b[1;32m 161\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124magent_type:\u001b[39m\u001b[38;5;124m\"\u001b[39m, agent_type)\n\u001b[1;32m 162\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcls._registry:\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28mcls\u001b[39m\u001b[38;5;241m.\u001b[39m_registry)\n\u001b[0;32m--> 163\u001b[0m agent_class \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mcls\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_registry\u001b[49m\u001b[43m[\u001b[49m\u001b[43magent_type\u001b[49m\u001b[43m]\u001b[49m\n\u001b[1;32m 164\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m agent_class(config\u001b[38;5;241m=\u001b[39mconfig)\n", + "\u001b[0;31mKeyError\u001b[0m: 'ProbabilisticAgent'" + ] + } + ], "source": [ "gym = PrimaiteGymEnv(env_config=cfg)" ] @@ -191,7 +225,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.11" + "version": "3.10.12" } }, "nbformat": 4, From 0aa691752aa268880f1a9f7531286e5a159136d0 Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Wed, 22 Jan 2025 12:20:45 +0000 Subject: [PATCH 66/95] #3075: Tidy up debug print statements. --- src/primaite/game/agent/interface.py | 3 -- src/primaite/game/game.py | 2 +- .../notebooks/Training-an-SB3-Agent.ipynb | 51 ++++--------------- 3 files changed, 10 insertions(+), 46 deletions(-) diff --git a/src/primaite/game/agent/interface.py b/src/primaite/game/agent/interface.py index 6a6bc323..5dab6aac 100644 --- a/src/primaite/game/agent/interface.py +++ b/src/primaite/game/agent/interface.py @@ -85,7 +85,6 @@ class AbstractAgent(BaseModel, ABC): def __init_subclass__(cls, identifier: Optional[str] = None, **kwargs: Any) -> None: super().__init_subclass__(**kwargs) - print("cls identifier:", identifier) if identifier is None: return if identifier in cls._registry: @@ -162,8 +161,6 @@ class AbstractAgent(BaseModel, ABC): def from_config(cls, config: Dict) -> AbstractAgent: """Grab the relevant agent class and construct an instance from a config dict.""" agent_type = config["type"] - print("agent_type:", agent_type) - print("cls._registry:", cls._registry) agent_class = cls._registry[agent_type] return agent_class(config=config) diff --git a/src/primaite/game/game.py b/src/primaite/game/game.py index c828a462..d9f6fa3c 100644 --- a/src/primaite/game/game.py +++ b/src/primaite/game/game.py @@ -493,7 +493,7 @@ class PrimaiteGame: agents_cfg = cfg.get("agents", []) for agent_cfg in agents_cfg: - print("agent_cfg:", agent_cfg) + new_agent = AbstractAgent.from_config(agent_cfg) game.agents[agent_cfg["ref"]] = new_agent if isinstance(new_agent, ProxyAgent): diff --git a/src/primaite/notebooks/Training-an-SB3-Agent.ipynb b/src/primaite/notebooks/Training-an-SB3-Agent.ipynb index 0e3245ae..d3492f92 100644 --- a/src/primaite/notebooks/Training-an-SB3-Agent.ipynb +++ b/src/primaite/notebooks/Training-an-SB3-Agent.ipynb @@ -29,18 +29,9 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "cls identifier: AbstractScriptedAgent\n", - "cls identifier: ProxyAgent\n" - ] - } - ], + "outputs": [], "source": [ "from primaite.game.game import PrimaiteGame\n", "from primaite.session.environment import PrimaiteGymEnv\n", @@ -50,7 +41,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -59,7 +50,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -79,33 +70,9 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "agent_cfg: {'ref': 'client_2_green_user', 'team': 'GREEN', 'type': 'ProbabilisticAgent', 'agent_settings': {'action_probabilities': {0: 0.3, 1: 0.6, 2: 0.1}}, 'action_space': {'action_map': {0: {'action': 'do_nothing', 'options': {}}, 1: {'action': 'node_application_execute', 'options': {'node_name': 'client_2', 'application_name': 'WebBrowser'}}, 2: {'action': 'node_application_execute', 'options': {'node_name': 'client_2', 'application_name': 'DatabaseClient'}}}}, 'reward_function': {'reward_components': [{'type': 'WEBPAGE_UNAVAILABLE_PENALTY', 'weight': 0.25, 'options': {'node_hostname': 'client_2'}}, {'type': 'GREEN_ADMIN_DATABASE_UNREACHABLE_PENALTY', 'weight': 0.05, 'options': {'node_hostname': 'client_2'}}]}}\n", - "agent_type: ProbabilisticAgent\n", - "cls._registry: {'AbstractScriptedAgent': , 'ProxyAgent': }\n" - ] - }, - { - "ename": "KeyError", - "evalue": "'ProbabilisticAgent'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[4], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m gym \u001b[38;5;241m=\u001b[39m \u001b[43mPrimaiteGymEnv\u001b[49m\u001b[43m(\u001b[49m\u001b[43menv_config\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcfg\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/session/environment.py:71\u001b[0m, in \u001b[0;36mPrimaiteGymEnv.__init__\u001b[0;34m(self, env_config)\u001b[0m\n\u001b[1;32m 69\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mio \u001b[38;5;241m=\u001b[39m PrimaiteIO\u001b[38;5;241m.\u001b[39mfrom_config(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mepisode_scheduler(\u001b[38;5;241m0\u001b[39m)\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mio_settings\u001b[39m\u001b[38;5;124m\"\u001b[39m, {}))\n\u001b[1;32m 70\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Handles IO for the environment. This produces sys logs, agent logs, etc.\"\"\"\u001b[39;00m\n\u001b[0;32m---> 71\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgame: PrimaiteGame \u001b[38;5;241m=\u001b[39m \u001b[43mPrimaiteGame\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfrom_config\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mepisode_scheduler\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 72\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Current game.\"\"\"\u001b[39;00m\n\u001b[1;32m 73\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_agent_name \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mnext\u001b[39m(\u001b[38;5;28miter\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgame\u001b[38;5;241m.\u001b[39mrl_agents))\n", - "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/game/game.py:497\u001b[0m, in \u001b[0;36mPrimaiteGame.from_config\u001b[0;34m(cls, cfg)\u001b[0m\n\u001b[1;32m 495\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m agent_cfg \u001b[38;5;129;01min\u001b[39;00m agents_cfg:\n\u001b[1;32m 496\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124magent_cfg:\u001b[39m\u001b[38;5;124m\"\u001b[39m, agent_cfg)\n\u001b[0;32m--> 497\u001b[0m new_agent \u001b[38;5;241m=\u001b[39m \u001b[43mAbstractAgent\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfrom_config\u001b[49m\u001b[43m(\u001b[49m\u001b[43magent_cfg\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 498\u001b[0m game\u001b[38;5;241m.\u001b[39magents[agent_cfg[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mref\u001b[39m\u001b[38;5;124m\"\u001b[39m]] \u001b[38;5;241m=\u001b[39m new_agent\n\u001b[1;32m 499\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(new_agent, ProxyAgent):\n", - "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/game/agent/interface.py:163\u001b[0m, in \u001b[0;36mAbstractAgent.from_config\u001b[0;34m(cls, config)\u001b[0m\n\u001b[1;32m 161\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124magent_type:\u001b[39m\u001b[38;5;124m\"\u001b[39m, agent_type)\n\u001b[1;32m 162\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcls._registry:\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28mcls\u001b[39m\u001b[38;5;241m.\u001b[39m_registry)\n\u001b[0;32m--> 163\u001b[0m agent_class \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mcls\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_registry\u001b[49m\u001b[43m[\u001b[49m\u001b[43magent_type\u001b[49m\u001b[43m]\u001b[49m\n\u001b[1;32m 164\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m agent_class(config\u001b[38;5;241m=\u001b[39mconfig)\n", - "\u001b[0;31mKeyError\u001b[0m: 'ProbabilisticAgent'" - ] - } - ], + "outputs": [], "source": [ "gym = PrimaiteGymEnv(env_config=cfg)" ] @@ -134,7 +101,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ @@ -166,7 +133,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -211,7 +178,7 @@ ], "metadata": { "kernelspec": { - "display_name": ".venv", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, From 8eeba691a9755e1bc237f0687bba2e82061a8fdb Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Thu, 23 Jan 2025 15:16:49 +0000 Subject: [PATCH 67/95] #3075: Remove unnecessary print statement. --- src/primaite/game/agent/actions/node.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/primaite/game/agent/actions/node.py b/src/primaite/game/agent/actions/node.py index fbab18f0..5e1b6725 100644 --- a/src/primaite/game/agent/actions/node.py +++ b/src/primaite/game/agent/actions/node.py @@ -36,7 +36,6 @@ class NodeAbstractAction(AbstractAction, identifier="node_abstract"): @classmethod def form_request(cls, config: ConfigSchema) -> RequestFormat: """Return the action formatted as a request which can be ingested by the PrimAITE simulation.""" - print(config) return ["network", "node", config.node_name, config.verb] From f4b73057d2094f9c248ad8f3ae757501378afff3 Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Mon, 27 Jan 2025 17:08:33 +0000 Subject: [PATCH 68/95] #3075: Update notebooks to reflect extension changes. --- src/primaite/notebooks/Action-masking.ipynb | 17 +++++----- ...ommand-and-Control-E2E-Demonstration.ipynb | 16 +++++++-- ...a-Manipulation-Customising-Red-Agent.ipynb | 33 +++++++++++++------ .../Data-Manipulation-E2E-Demonstration.ipynb | 15 +++++---- .../Getting-Information-Out-Of-PrimAITE.ipynb | 6 ++-- ...ege-Escalation-and-Data-Loss-Example.ipynb | 15 +++++++-- .../Training-an-RLLIB-MARL-System.ipynb | 20 +++++++++-- .../notebooks/Training-an-RLLib-Agent.ipynb | 23 ++++++++++--- .../notebooks/Using-Episode-Schedules.ipynb | 15 ++------- src/primaite/notebooks/multi-processing.ipynb | 18 +++++----- 10 files changed, 116 insertions(+), 62 deletions(-) diff --git a/src/primaite/notebooks/Action-masking.ipynb b/src/primaite/notebooks/Action-masking.ipynb index 7fde0a49..64015080 100644 --- a/src/primaite/notebooks/Action-masking.ipynb +++ b/src/primaite/notebooks/Action-masking.ipynb @@ -22,13 +22,14 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "from primaite.session.environment import PrimaiteGymEnv\n", "from primaite.config.load import data_manipulation_config_path\n", - "from prettytable import PrettyTable" + "from prettytable import PrettyTable\n", + "from primaite.game.agent.scripted_agents import probabilistic_agent\n" ] }, { @@ -103,7 +104,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -116,7 +117,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ @@ -156,7 +157,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ @@ -167,7 +168,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "metadata": {}, "outputs": [], "source": [ @@ -204,7 +205,7 @@ ], "metadata": { "kernelspec": { - "display_name": ".venv", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -218,7 +219,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.11" + "version": "3.10.12" } }, "nbformat": 4, diff --git a/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb b/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb index 66a684de..625510b8 100644 --- a/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb +++ b/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb @@ -16,6 +16,15 @@ "execution_count": null, "metadata": {}, "outputs": [], + "source": [ + "!primaite setup" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], "source": [ "# Imports\n", "import yaml\n", @@ -27,7 +36,8 @@ "from primaite.simulator.system.applications.red_applications.c2.abstract_c2 import C2Command\n", "from primaite.simulator.system.applications.red_applications.ransomware_script import RansomwareScript\n", "from primaite.simulator.network.hardware.nodes.host.computer import Computer\n", - "from primaite.simulator.network.hardware.nodes.host.server import Server" + "from primaite.simulator.network.hardware.nodes.host.server import Server\n", + "from primaite.game.agent.scripted_agents import probabilistic_agent" ] }, { @@ -43,7 +53,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -1763,7 +1773,7 @@ ], "metadata": { "kernelspec": { - "display_name": ".venv", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, diff --git a/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb b/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb index 756fc44f..de6d3fe5 100644 --- a/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb +++ b/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb @@ -26,7 +26,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -36,12 +36,13 @@ "from primaite.game.agent.interface import AgentHistoryItem\n", "from primaite.session.environment import PrimaiteGymEnv\n", "import yaml\n", - "from pprint import pprint" + "from pprint import pprint\n", + "from primaite.game.agent.scripted_agents import probabilistic_agent, data_manipulation_bot" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -68,7 +69,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -79,7 +80,8 @@ " if red_action == 'do_nothing':\n", " red_str = 'DO NOTHING'\n", " elif red_action == 'node_application_execute':\n", - " client = \"client 1\" if red_info.parameters['node_id'] == 0 else \"client 2\"\n", + " client = \"client 1\" if red_info.parameters['node_name'] == 0 else \"client 2\"\n", + "\n", " red_str = f\"ATTACK from {client}\"\n", " return red_str" ] @@ -243,7 +245,8 @@ "outputs": [], "source": [ "change = yaml.safe_load(\"\"\"\n", - "start_settings:\n", + " possible_start_nodes: [client_1, client_2]\n", + " target_application: DataManipulationBot\n", " start_step: 25\n", " frequency: 20\n", " variance: 0\n", @@ -253,7 +256,9 @@ " cfg = yaml.safe_load(f)\n", " for agent in cfg['agents']:\n", " if agent['ref'] == \"data_manipulation_attacker\":\n", + " print(f\"{agent['agent_settings']=}\")\n", " agent['agent_settings'] = change\n", + " print(f\"{agent['agent_settings']=}\")\n", "\n", "env = PrimaiteGymEnv(env_config = cfg)\n", "env.reset()\n", @@ -310,9 +315,17 @@ "outputs": [], "source": [ "change = yaml.safe_load(\"\"\"\n", - "# TODO:\n", + " action_space:\n", + " action_map:\n", + " 0:\n", + " action: do_nothing\n", + " options: {}\n", + " 1:\n", + " action: node_application_execute\n", + " options:\n", + " node_name: client_1\n", + " application_name: DataManipulationBot\n", "\"\"\")\n", - "#TODO 2869 fix\n", "\n", "with open(data_manipulation_config_path(), 'r') as f:\n", " cfg = yaml.safe_load(f)\n", @@ -438,7 +451,7 @@ ], "metadata": { "kernelspec": { - "display_name": ".venv", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -452,7 +465,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.11" + "version": "3.10.12" } }, "nbformat": 4, diff --git a/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb b/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb index dbc6f0c1..3818bb18 100644 --- a/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb +++ b/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb @@ -382,7 +382,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": { "tags": [] }, @@ -394,7 +394,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": { "tags": [] }, @@ -405,7 +405,8 @@ "from primaite.session.environment import PrimaiteGymEnv\n", "from primaite.game.agent.interface import AgentHistoryItem\n", "import yaml\n", - "from pprint import pprint\n" + "from pprint import pprint\n", + "from primaite.game.agent.scripted_agents import probabilistic_agent, data_manipulation_bot\n" ] }, { @@ -429,9 +430,9 @@ " cfg = yaml.safe_load(f)\n", " # set success probability to 1.0 to avoid rerunning cells.\n", " cfg['simulation']['network']['nodes'][8]['applications'][0]['options']['data_manipulation_p_of_success'] = 1.0\n", - " cfg['simulation']['network']['nodes'][9]['applications'][0]['options']['data_manipulation_p_of_success'] = 1.0\n", + " cfg['simulation']['network']['nodes'][9]['applications'][1]['options']['data_manipulation_p_of_success'] = 1.0\n", " cfg['simulation']['network']['nodes'][8]['applications'][0]['options']['port_scan_p_of_success'] = 1.0\n", - " cfg['simulation']['network']['nodes'][9]['applications'][0]['options']['port_scan_p_of_success'] = 1.0\n", + " cfg['simulation']['network']['nodes'][9]['applications'][1]['options']['port_scan_p_of_success'] = 1.0\n", " # don't flatten observations so that we can see what is going on\n", " cfg['agents'][3]['agent_settings']['flatten_obs'] = False\n", "\n", @@ -450,7 +451,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -461,7 +462,7 @@ " if red_action == 'do_nothing':\n", " red_str = 'DO NOTHING'\n", " elif red_action == 'node_application_execute':\n", - " client = \"client 1\" if red_info.parameters['node_id'] == 0 else \"client 2\"\n", + " client = \"client 1\" if red_info.parameters['node_name'] == 0 else \"client 2\"\n", " red_str = f\"ATTACK from {client}\"\n", " return red_str" ] diff --git a/src/primaite/notebooks/Getting-Information-Out-Of-PrimAITE.ipynb b/src/primaite/notebooks/Getting-Information-Out-Of-PrimAITE.ipynb index f8691d7d..d7d60d76 100644 --- a/src/primaite/notebooks/Getting-Information-Out-Of-PrimAITE.ipynb +++ b/src/primaite/notebooks/Getting-Information-Out-Of-PrimAITE.ipynb @@ -32,6 +32,8 @@ "from primaite.session.environment import PrimaiteGymEnv\n", "from primaite.simulator.network.hardware.nodes.host.computer import Computer\n", "from notebook.services.config import ConfigManager\n", + "from primaite.game.agent.scripted_agents import probabilistic_agent\n", + "\n", "\n", "cm = ConfigManager().update('notebook', {'limit_output': 50}) # limit output lines to 50 - for neatness\n", "\n", @@ -157,7 +159,7 @@ ], "metadata": { "kernelspec": { - "display_name": "venv", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -171,7 +173,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.8" + "version": "3.10.12" } }, "nbformat": 4, diff --git a/src/primaite/notebooks/Privilege-Escalation-and-Data-Loss-Example.ipynb b/src/primaite/notebooks/Privilege-Escalation-and-Data-Loss-Example.ipynb index fcda4dbd..35d3813a 100644 --- a/src/primaite/notebooks/Privilege-Escalation-and-Data-Loss-Example.ipynb +++ b/src/primaite/notebooks/Privilege-Escalation-and-Data-Loss-Example.ipynb @@ -54,6 +54,15 @@ { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!primaite setup" + ] + }, + { + "cell_type": "code", + "execution_count": 2, "metadata": { "tags": [] }, @@ -68,7 +77,7 @@ "from primaite.simulator.network.hardware.nodes.host.server import Server\n", "from primaite.simulator.system.applications.database_client import DatabaseClient\n", "from primaite.simulator.system.applications.web_browser import WebBrowser\n", - "from primaite.simulator.system.services.database.database_service import DatabaseService" + "from primaite.simulator.system.services.database.database_service import DatabaseService\n" ] }, { @@ -80,7 +89,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": { "tags": [] }, @@ -103,7 +112,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": { "tags": [] }, diff --git a/src/primaite/notebooks/Training-an-RLLIB-MARL-System.ipynb b/src/primaite/notebooks/Training-an-RLLIB-MARL-System.ipynb index 76cab86a..dadb399e 100644 --- a/src/primaite/notebooks/Training-an-RLLIB-MARL-System.ipynb +++ b/src/primaite/notebooks/Training-an-RLLIB-MARL-System.ipynb @@ -40,6 +40,7 @@ "import ray\n", "from ray.rllib.algorithms.ppo import PPOConfig\n", "from primaite.session.ray_envs import PrimaiteRayMARLEnv\n", + "from primaite.game.agent.scripted_agents import probabilistic_agent\n", "\n", "with open(PRIMAITE_PATHS.user_config_path / 'example_config/data_manipulation_marl.yaml', 'r') as f:\n", " cfg = yaml.safe_load(f)\n", @@ -56,7 +57,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -102,7 +103,20 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "*** SIGTERM received at time=1737996337 on cpu 7 ***\n", + "PC: @ 0x7f3649b0fe2e (unknown) epoll_wait\n", + " @ 0x7f3649a2c520 (unknown) (unknown)\n", + "[2025-01-27 16:45:37,381 E 117142 117142] logging.cc:440: *** SIGTERM received at time=1737996337 on cpu 7 ***\n", + "[2025-01-27 16:45:37,381 E 117142 117142] logging.cc:440: PC: @ 0x7f3649b0fe2e (unknown) epoll_wait\n", + "[2025-01-27 16:45:37,381 E 117142 117142] logging.cc:440: @ 0x7f3649a2c520 (unknown) (unknown)\n" + ] + } + ], "source": [ "eval = algo.evaluate()" ] @@ -110,7 +124,7 @@ ], "metadata": { "kernelspec": { - "display_name": "venv", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, diff --git a/src/primaite/notebooks/Training-an-RLLib-Agent.ipynb b/src/primaite/notebooks/Training-an-RLLib-Agent.ipynb index 7252b046..64a9e7ab 100644 --- a/src/primaite/notebooks/Training-an-RLLib-Agent.ipynb +++ b/src/primaite/notebooks/Training-an-RLLib-Agent.ipynb @@ -32,6 +32,8 @@ "from primaite.session.ray_envs import PrimaiteRayEnv\n", "import ray\n", "from ray.rllib.algorithms.ppo import PPOConfig\n", + "from primaite.game.agent.scripted_agents import probabilistic_agent\n", + "\n", "\n", "# If you get an error saying this config file doesn't exist, you may need to run `primaite setup` in your command line\n", "# to copy the files to your user data path.\n", @@ -50,7 +52,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -96,7 +98,20 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "*** SIGTERM received at time=1737996055 on cpu 1 ***\n", + "PC: @ 0x7f6e254a6e2e (unknown) epoll_wait\n", + " @ 0x7f6e253c3520 (unknown) (unknown)\n", + "[2025-01-27 16:40:55,343 E 114171 114171] logging.cc:440: *** SIGTERM received at time=1737996055 on cpu 1 ***\n", + "[2025-01-27 16:40:55,343 E 114171 114171] logging.cc:440: PC: @ 0x7f6e254a6e2e (unknown) epoll_wait\n", + "[2025-01-27 16:40:55,344 E 114171 114171] logging.cc:440: @ 0x7f6e253c3520 (unknown) (unknown)\n" + ] + } + ], "source": [ "eval = algo.evaluate()" ] @@ -104,7 +119,7 @@ ], "metadata": { "kernelspec": { - "display_name": ".venv", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -118,7 +133,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.11" + "version": "3.10.12" } }, "nbformat": 4, diff --git a/src/primaite/notebooks/Using-Episode-Schedules.ipynb b/src/primaite/notebooks/Using-Episode-Schedules.ipynb index d08ca67b..311fe4fb 100644 --- a/src/primaite/notebooks/Using-Episode-Schedules.ipynb +++ b/src/primaite/notebooks/Using-Episode-Schedules.ipynb @@ -48,6 +48,7 @@ "from primaite.session.environment import PrimaiteGymEnv\n", "from primaite import PRIMAITE_PATHS\n", "from prettytable import PrettyTable\n", + "from primaite.game.agent.scripted_agents import probabilistic_agent, data_manipulation_bot\n", "scenario_path = PRIMAITE_PATHS.user_config_path / \"example_config/scenario_with_placeholders\"" ] }, @@ -409,21 +410,9 @@ ], "metadata": { "kernelspec": { - "display_name": "venv", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.11" } }, "nbformat": 4, diff --git a/src/primaite/notebooks/multi-processing.ipynb b/src/primaite/notebooks/multi-processing.ipynb index ad386f34..798bf3ff 100644 --- a/src/primaite/notebooks/multi-processing.ipynb +++ b/src/primaite/notebooks/multi-processing.ipynb @@ -29,7 +29,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 30, "metadata": {}, "outputs": [], "source": [ @@ -37,13 +37,13 @@ "from stable_baselines3 import PPO\n", "from stable_baselines3.common.utils import set_random_seed\n", "from stable_baselines3.common.vec_env import SubprocVecEnv\n", - "\n", - "from primaite.session.environment import PrimaiteGymEnv\n" + "from primaite.session.environment import PrimaiteGymEnv\n", + "from primaite.game.agent.scripted_agents import probabilistic_agent\n" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 31, "metadata": {}, "outputs": [], "source": [ @@ -52,7 +52,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 32, "metadata": {}, "outputs": [], "source": [ @@ -69,7 +69,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 33, "metadata": {}, "outputs": [], "source": [ @@ -90,7 +90,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 36, "metadata": {}, "outputs": [], "source": [ @@ -138,7 +138,7 @@ ], "metadata": { "kernelspec": { - "display_name": ".venv", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -152,7 +152,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.11" + "version": "3.10.12" } }, "nbformat": 4, From 4fb4c5e0f9e26b2d0c797beec562b8f35344fe74 Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Tue, 28 Jan 2025 17:18:07 +0000 Subject: [PATCH 69/95] #3075: Fix error in config file. --- src/primaite/config/_package_data/data_manipulation.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/primaite/config/_package_data/data_manipulation.yaml b/src/primaite/config/_package_data/data_manipulation.yaml index b0d5d087..fc48c8e6 100644 --- a/src/primaite/config/_package_data/data_manipulation.yaml +++ b/src/primaite/config/_package_data/data_manipulation.yaml @@ -81,7 +81,7 @@ agents: action: node_application_execute options: node_name: client_1 - application_name: WebBrowser + application_name: DatabaseClient reward_function: reward_components: From 32de95917e636c8febd08d38aa41492e7e02c606 Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Mon, 3 Feb 2025 11:33:21 +0000 Subject: [PATCH 70/95] #3075: Add repeat parameter to config file. --- .../notebooks/Data-Manipulation-E2E-Demonstration.ipynb | 3 +-- .../applications/red_applications/data_manipulation_bot.py | 2 ++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb b/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb index 3818bb18..4a34619f 100644 --- a/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb +++ b/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb @@ -405,8 +405,7 @@ "from primaite.session.environment import PrimaiteGymEnv\n", "from primaite.game.agent.interface import AgentHistoryItem\n", "import yaml\n", - "from pprint import pprint\n", - "from primaite.game.agent.scripted_agents import probabilistic_agent, data_manipulation_bot\n" + "from pprint import pprint\n" ] }, { diff --git a/src/primaite/simulator/system/applications/red_applications/data_manipulation_bot.py b/src/primaite/simulator/system/applications/red_applications/data_manipulation_bot.py index 392cdfba..79ef1e67 100644 --- a/src/primaite/simulator/system/applications/red_applications/data_manipulation_bot.py +++ b/src/primaite/simulator/system/applications/red_applications/data_manipulation_bot.py @@ -52,6 +52,7 @@ class DataManipulationBot(Application, identifier="DataManipulationBot"): payload: str = "DELETE" port_scan_p_of_success: float = 0.1 data_manipulation_p_of_success: float = 0.1 + repeat: bool = True config: "DataManipulationBot.ConfigSchema" = Field(default_factory=lambda: DataManipulationBot.ConfigSchema()) @@ -76,6 +77,7 @@ class DataManipulationBot(Application, identifier="DataManipulationBot"): self.payload = self.config.payload self.port_scan_p_of_success = self.config.port_scan_p_of_success self.data_manipulation_p_of_success = self.config.data_manipulation_p_of_success + self.repeat = self.config.repeat def describe_state(self) -> Dict: """ From d24f9872ffac89b6cf28830c073c598ef214cb2f Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Mon, 3 Feb 2025 17:06:12 +0000 Subject: [PATCH 71/95] #3075: C2C-E2E-Demo internal YAML changes. --- ...ommand-and-Control-E2E-Demonstration.ipynb | 1966 +++++++++++++++-- 1 file changed, 1747 insertions(+), 219 deletions(-) diff --git a/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb b/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb index 625510b8..2c1a94c8 100644 --- a/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb +++ b/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb @@ -13,16 +13,31 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-02-03 16:04:19,828: Performing the PrimAITE first-time setup...\n", + "2025-02-03 16:04:19,829: Building the PrimAITE app directories...\n", + "2025-02-03 16:04:19,829: Building primaite_config.yaml...\n", + "2025-02-03 16:04:19,829: Rebuilding the demo notebooks...\n", + "/home/nick/primaite/4.0.0a1-dev/notebooks/example_notebooks/Command-and-Control-E2E-Demonstration.ipynb\n", + "2025-02-03 16:04:19,831: Reset example notebook: /home/nick/primaite/4.0.0a1-dev/notebooks/example_notebooks/Command-and-Control-E2E-Demonstration.ipynb\n", + "2025-02-03 16:04:19,836: Rebuilding the example notebooks...\n", + "2025-02-03 16:04:19,840: PrimAITE setup complete!\n" + ] + } + ], "source": [ "!primaite setup" ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -36,8 +51,7 @@ "from primaite.simulator.system.applications.red_applications.c2.abstract_c2 import C2Command\n", "from primaite.simulator.system.applications.red_applications.ransomware_script import RansomwareScript\n", "from primaite.simulator.network.hardware.nodes.host.computer import Computer\n", - "from primaite.simulator.network.hardware.nodes.host.server import Server\n", - "from primaite.game.agent.scripted_agents import probabilistic_agent" + "from primaite.simulator.network.hardware.nodes.host.server import Server" ] }, { @@ -53,7 +67,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -63,24 +77,24 @@ " type: ProxyAgent\n", "\n", " action_space:\n", - " options:\n", - " nodes:\n", - " - node_name: web_server\n", - " applications:\n", - " - application_name: C2Beacon\n", - " - node_name: client_1\n", - " applications:\n", - " - application_name: C2Server\n", - " max_folders_per_node: 1\n", - " max_files_per_folder: 1\n", - " max_services_per_node: 2\n", - " max_nics_per_node: 8\n", - " max_acl_rules: 10\n", - " ip_list:\n", - " - 192.168.1.21\n", - " - 192.168.1.14\n", - " wildcard_list:\n", - " - 0.0.0.1\n", + " # options:\n", + " # nodes:\n", + " # - node_name: web_server\n", + " # applications:\n", + " # - application_name: C2Beacon\n", + " # - node_name: client_1\n", + " # applications:\n", + " # - application_name: C2Server\n", + " # max_folders_per_node: 1\n", + " # max_files_per_folder: 1\n", + " # max_services_per_node: 2\n", + " # max_nics_per_node: 8\n", + " # max_acl_rules: 10\n", + " # ip_list:\n", + " # - 192.168.1.21\n", + " # - 192.168.1.14\n", + " # wildcard_list:\n", + " # - 0.0.0.1\n", " action_map:\n", " 0:\n", " action: do_nothing\n", @@ -88,30 +102,30 @@ " 1:\n", " action: node_application_install\n", " options:\n", - " node_id: 0\n", + " node_name: web_server\n", " application_name: C2Beacon\n", " 2:\n", " action: configure_c2_beacon\n", " options:\n", - " node_id: 0\n", - " config:\n", - " c2_server_ip_address: 192.168.10.21\n", - " keep_alive_frequency:\n", - " masquerade_protocol:\n", - " masquerade_port:\n", + " node_name: web_server\n", + " # config:\n", + " c2_server_ip_address: 192.168.10.21\n", + " # keep_alive_frequency: 10\n", + " # masquerade_protocol: TCP\n", + " # masquerade_port: DNS\n", " 3:\n", " action: node_application_execute\n", " options:\n", - " node_id: 0\n", - " application_id: 0\n", + " node_name: web_server\n", + " application_name: C2Beacon\n", " 4:\n", " action: c2_server_terminal_command\n", " options:\n", - " node_id: 1\n", + " node_name: client_1\n", " ip_address:\n", - " account:\n", - " username: admin\n", - " password: admin\n", + " # account:\n", + " username: admin\n", + " password: admin\n", " commands:\n", " -\n", " - software_manager\n", @@ -121,44 +135,44 @@ " 5:\n", " action: c2_server_ransomware_configure\n", " options:\n", - " node_id: 1\n", - " config:\n", - " server_ip_address: 192.168.1.14\n", - " payload: ENCRYPT\n", + " node_name: client_1\n", + " # config:\n", + " server_ip_address: 192.168.1.14\n", + " payload: ENCRYPT\n", " 6:\n", " action: c2_server_data_exfiltrate\n", " options:\n", - " node_id: 1\n", + " node_name: client_1\n", " target_file_name: \"database.db\"\n", " target_folder_name: \"database\"\n", " exfiltration_folder_name: \"spoils\"\n", " target_ip_address: 192.168.1.14\n", - " account:\n", - " username: admin\n", - " password: admin\n", + " # account:\n", + " username: admin\n", + " password: admin\n", "\n", " 7:\n", " action: c2_server_ransomware_launch\n", " options:\n", - " node_id: 1\n", + " node_name: client_1\n", " 8:\n", " action: configure_c2_beacon\n", " options:\n", - " node_id: 0\n", - " config:\n", - " c2_server_ip_address: 192.168.10.21\n", - " keep_alive_frequency: 10\n", - " masquerade_protocol: TCP\n", - " masquerade_port: DNS\n", + " node_name: web_server\n", + " # config:\n", + " c2_server_ip_address: 192.168.10.21\n", + " # keep_alive_frequency: 10\n", + " # masquerade_protocol: TCP\n", + " # masquerade_port: DNS\n", " 9:\n", " action: configure_c2_beacon\n", " options:\n", - " node_id: 0\n", - " config:\n", - " c2_server_ip_address: 192.168.10.22\n", - " keep_alive_frequency:\n", - " masquerade_protocol:\n", - " masquerade_port:\n", + " node_name: web_server\n", + " # config:\n", + " c2_server_ip_address: 192.168.10.22\n", + " # keep_alive_frequency: 10\n", + " # masquerade_protocol: TCP\n", + " # masquerade_port: DNS\n", "\n", " reward_function:\n", " reward_components:\n", @@ -169,9 +183,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-03 16:04:24,734: PrimaiteGymEnv RNG seed = None\n" + ] + } + ], "source": [ "with open(data_manipulation_config_path()) as f:\n", " cfg = yaml.safe_load(f)\n", @@ -197,9 +219,35 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+--------------------------------------------------------------------------------------+\n", + "| client_1 Software Manager |\n", + "+---------------------+-------------+-----------------+--------------+------+----------+\n", + "| Name | Type | Operating State | Health State | Port | Protocol |\n", + "+---------------------+-------------+-----------------+--------------+------+----------+\n", + "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", + "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", + "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", + "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", + "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", + "| NMAP | Application | RUNNING | GOOD | None | none |\n", + "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", + "| UserManager | Service | RUNNING | GOOD | None | none |\n", + "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", + "| FTPClient | Service | RUNNING | GOOD | 21 | tcp |\n", + "| DataManipulationBot | Application | RUNNING | GOOD | None | none |\n", + "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", + "| C2Server | Application | RUNNING | GOOD | None | tcp |\n", + "+---------------------+-------------+-----------------+--------------+------+----------+\n" + ] + } + ], "source": [ "client_1: Computer = env.game.simulation.network.get_node_by_hostname(\"client_1\")\n", "client_1.software_manager.install(C2Server)\n", @@ -249,9 +297,34 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+-------------------------------------------------------------------------------------+\n", + "| web_server Software Manager |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| Name | Type | Operating State | Health State | Port | Protocol |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", + "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", + "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", + "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", + "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", + "| NMAP | Application | RUNNING | GOOD | None | none |\n", + "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", + "| UserManager | Service | RUNNING | GOOD | None | none |\n", + "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", + "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", + "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", + "| C2Beacon | Application | INSTALLING | UNUSED | None | tcp |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n" + ] + } + ], "source": [ "env.step(1)\n", "web_server: Computer = env.game.simulation.network.get_node_by_hostname(\"web_server\")\n", @@ -291,9 +364,41 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+-------------------------------------------------------------------------------------+\n", + "| web_server Software Manager |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| Name | Type | Operating State | Health State | Port | Protocol |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", + "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", + "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", + "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", + "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", + "| NMAP | Application | RUNNING | GOOD | None | none |\n", + "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", + "| UserManager | Service | RUNNING | GOOD | None | none |\n", + "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", + "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", + "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", + "| C2Beacon | Application | RUNNING | GOOD | None | tcp |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", + "| C2Beacon Running Status |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| False | 192.168.10.21 | 0 | 5 | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n" + ] + } + ], "source": [ "env.step(2)\n", "c2_beacon: C2Beacon = web_server.software_manager.software[\"C2Beacon\"]\n", @@ -332,18 +437,54 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "(0,\n", + " 0.0,\n", + " False,\n", + " False,\n", + " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=2, action='node_application_execute', parameters={'node_name': 'web_server', 'application_name': 'C2Beacon'}, request=['network', 'node', 'web_server', 'application', 'C2Beacon', 'execute'], response=RequestResponse(status='success', data={}), reward=0.0, reward_info={})}})" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "env.step(3)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", + "| C2Beacon Running Status |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| True | 192.168.10.21 | 1 | 5 | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "+-----------------------------------------------------------------------------------------------------+\n", + "| C2Server Running Status |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n", + "| True | 192.168.1.12 | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n" + ] + } + ], "source": [ "c2_beacon.show()\n", "c2_server.show()" @@ -406,18 +547,59 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "(0,\n", + " 0.0,\n", + " False,\n", + " False,\n", + " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=3, action='c2_server_terminal_command', parameters={'node_name': 'client_1', 'ip_address': None, 'username': 'admin', 'password': 'admin', 'commands': [['software_manager', 'application', 'install', 'RansomwareScript']]}, request=['network', 'node', 'client_1', 'application', 'C2Server', 'terminal_command', {'commands': [['software_manager', 'application', 'install', 'RansomwareScript']], 'ip_address': None, 'username': 'admin', 'password': 'admin'}], response=RequestResponse(status='success', data={0: RequestResponse(status='success', data={})}), reward=0.0, reward_info={})}})" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "env.step(4)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+--------------------------------------------------------------------------------------+\n", + "| client_1 Software Manager |\n", + "+---------------------+-------------+-----------------+--------------+------+----------+\n", + "| Name | Type | Operating State | Health State | Port | Protocol |\n", + "+---------------------+-------------+-----------------+--------------+------+----------+\n", + "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", + "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", + "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", + "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", + "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", + "| NMAP | Application | RUNNING | GOOD | None | none |\n", + "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", + "| UserManager | Service | RUNNING | GOOD | None | none |\n", + "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", + "| FTPClient | Service | RUNNING | GOOD | 21 | tcp |\n", + "| DataManipulationBot | Application | RUNNING | GOOD | None | none |\n", + "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", + "| C2Server | Application | RUNNING | GOOD | None | tcp |\n", + "+---------------------+-------------+-----------------+--------------+------+----------+\n" + ] + } + ], "source": [ "client_1.software_manager.show()" ] @@ -456,18 +638,66 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "(0,\n", + " 0.0,\n", + " False,\n", + " False,\n", + " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=4, action='c2_server_ransomware_configure', parameters={'node_name': 'client_1', 'server_ip_address': '192.168.1.14', 'payload': 'ENCRYPT'}, request=['network', 'node', 'client_1', 'application', 'C2Server', 'ransomware_configure', {'server_ip_address': '192.168.1.14', 'server_password': None, 'payload': 'ENCRYPT'}], response=RequestResponse(status='success', data={}), reward=0.0, reward_info={})}})" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "env.step(5)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+-------------------------------------------------------------------------------------+\n", + "| web_server Software Manager |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| Name | Type | Operating State | Health State | Port | Protocol |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", + "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", + "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", + "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", + "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", + "| NMAP | Application | RUNNING | GOOD | None | none |\n", + "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", + "| UserManager | Service | RUNNING | GOOD | None | none |\n", + "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", + "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", + "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", + "| C2Beacon | Application | RUNNING | GOOD | None | tcp |\n", + "| RansomwareScript | Application | RUNNING | GOOD | None | none |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "+------------------------------------+\n", + "| RansomwareScript Running Status |\n", + "+--------------------------+---------+\n", + "| Target Server IP Address | Payload |\n", + "+--------------------------+---------+\n", + "| 192.168.1.14 | ENCRYPT |\n", + "+--------------------------+---------+\n" + ] + } + ], "source": [ "ransomware_script: RansomwareScript = web_server.software_manager.software[\"RansomwareScript\"]\n", "web_server.software_manager.show()\n", @@ -513,18 +743,48 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "(0,\n", + " 0.0,\n", + " False,\n", + " False,\n", + " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=5, action='c2_server_data_exfiltrate', parameters={'node_name': 'client_1', 'target_file_name': 'database.db', 'target_folder_name': 'database', 'exfiltration_folder_name': 'spoils', 'target_ip_address': '192.168.1.14', 'username': 'admin', 'password': 'admin'}, request=['network', 'node', 'client_1', 'application', 'C2Server', 'exfiltrate', {'target_file_name': 'database.db', 'target_folder_name': 'database', 'exfiltration_folder_name': 'spoils', 'target_ip_address': '192.168.1.14', 'username': 'admin', 'password': 'admin'}], response=RequestResponse(status='success', data={}), reward=0.0, reward_info={})}})" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "env.step(6)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+--------------------------------------------------------------------------------+\n", + "| client_1 File System |\n", + "+--------------------+---------+---------------+-----------------------+---------+\n", + "| File Path | Size | Health status | Visible health status | Deleted |\n", + "+--------------------+---------+---------------+-----------------------+---------+\n", + "| root | 0 B | GOOD | NONE | False |\n", + "| spoils/database.db | 4.77 MB | GOOD | NONE | False |\n", + "+--------------------+---------+---------------+-----------------------+---------+\n" + ] + } + ], "source": [ "client_1: Computer = env.game.simulation.network.get_node_by_hostname(\"client_1\")\n", "client_1.software_manager.file_system.show(full=True)" @@ -532,9 +792,25 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+---------------------------------------------------------------------------------+\n", + "| web_server File System |\n", + "+---------------------+---------+---------------+-----------------------+---------+\n", + "| File Path | Size | Health status | Visible health status | Deleted |\n", + "+---------------------+---------+---------------+-----------------------+---------+\n", + "| primaite/index.html | 15.0 KB | GOOD | NONE | False |\n", + "| root | 0 B | GOOD | NONE | False |\n", + "| spoils/database.db | 4.77 MB | GOOD | NONE | False |\n", + "+---------------------+---------+---------------+-----------------------+---------+\n" + ] + } + ], "source": [ "web_server: Computer = env.game.simulation.network.get_node_by_hostname(\"web_server\")\n", "web_server.software_manager.file_system.show(full=True)" @@ -571,18 +847,48 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "(0,\n", + " 0.0,\n", + " False,\n", + " False,\n", + " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=6, action='c2_server_ransomware_launch', parameters={'node_name': 'client_1'}, request=['network', 'node', 'client_1', 'application', 'C2Server', 'ransomware_launch'], response=RequestResponse(status='success', data={}), reward=0.0, reward_info={})}})" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "env.step(7)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----------------------------------------------------------------------------------+\n", + "| database_server File System |\n", + "+----------------------+---------+---------------+-----------------------+---------+\n", + "| File Path | Size | Health status | Visible health status | Deleted |\n", + "+----------------------+---------+---------------+-----------------------+---------+\n", + "| database/database.db | 4.77 MB | CORRUPT | NONE | False |\n", + "| root | 0 B | GOOD | NONE | False |\n", + "+----------------------+---------+---------------+-----------------------+---------+\n" + ] + } + ], "source": [ "database_server: Server = env.game.simulation.network.get_node_by_hostname(\"database_server\")\n", "database_server.software_manager.file_system.show(full=True)" @@ -601,7 +907,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "metadata": {}, "outputs": [], "source": [ @@ -697,52 +1003,52 @@ " 1:\n", " action: node_application_remove\n", " options:\n", - " node_id: 0\n", + " node_name: web_server\n", " application_name: C2Beacon\n", " 2:\n", " action: node_shutdown\n", " options:\n", - " node_id: 0\n", + " node_name: web_server\n", " 3:\n", " action: router_acl_add_rule\n", " options:\n", " target_router: router_1\n", " position: 1\n", - " permission: 2\n", - " source_ip_id: 2\n", - " dest_ip_id: 3\n", - " source_port_id: 2\n", - " dest_port_id: 2\n", - " protocol_id: 1\n", - " source_wildcard_id: 0\n", - " dest_wildcard_id: 0\n", + " permission: DENY\n", + " src_ip: 192.168.10.21\n", + " dst_ip: 192.168.1.12\n", + " src_port: HTTP\n", + " dst_port: HTTP\n", + " protocol_name: ALL\n", + " src_wildcard: NONE\n", + " dst_wildcard: NONE\n", "\n", "\n", - " options:\n", - " nodes:\n", - " - node_name: web_server\n", - " applications:\n", - " - application_name: C2Beacon\n", + " # options:\n", + " # nodes:\n", + " # - node_name: web_server\n", + " # applications:\n", + " # - application_name: C2Beacon\n", "\n", - " - node_name: database_server\n", - " folders:\n", - " - folder_name: database\n", - " files:\n", - " - file_name: database.db\n", - " services:\n", - " - service_name: DatabaseService\n", - " - node_name: router_1\n", + " # - node_name: database_server\n", + " # folders:\n", + " # - folder_name: database\n", + " # files:\n", + " # - file_name: database.db\n", + " # services:\n", + " # - service_name: DatabaseService\n", + " # - node_name: router_1\n", "\n", - " max_folders_per_node: 2\n", - " max_files_per_folder: 2\n", - " max_services_per_node: 2\n", - " max_nics_per_node: 8\n", - " max_acl_rules: 10\n", - " ip_list:\n", - " - 192.168.10.21\n", - " - 192.168.1.12\n", - " wildcard_list:\n", - " - 0.0.0.1\n", + " # max_folders_per_node: 2\n", + " # max_files_per_folder: 2\n", + " # max_services_per_node: 2\n", + " # max_nics_per_node: 8\n", + " # max_acl_rules: 10\n", + " # ip_list:\n", + " # - 192.168.10.21\n", + " # - 192.168.1.12\n", + " # wildcard_list:\n", + " # - 0.0.0.1\n", " reward_function:\n", " reward_components:\n", " - type: DUMMY\n", @@ -755,9 +1061,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-03 16:04:26,020: PrimaiteGymEnv RNG seed = None\n" + ] + } + ], "source": [ "with open(data_manipulation_config_path()) as f:\n", " cfg = yaml.safe_load(f)\n", @@ -771,7 +1085,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 21, "metadata": {}, "outputs": [], "source": [ @@ -814,9 +1128,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 22, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-03 16:04:26,440: Resetting environment, episode 0, avg. reward: 0.0\n", + "2025-02-03 16:04:26,445: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-03/16-04-20/agent_actions/episode_0.json\n" + ] + } + ], "source": [ "# Resetting the environment and capturing the default observation space.\n", "blue_env.reset()\n", @@ -825,9 +1148,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 23, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Setting up the C2 Suite via the simulation API.\n", "\n", @@ -848,7 +1182,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 24, "metadata": {}, "outputs": [], "source": [ @@ -858,9 +1192,26 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 25, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 2\n", + "root['NODES']['HOST0']['APPLICATIONS'][1]['operating_status']: 0 -> 1\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", + "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", + "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" + ] + } + ], "source": [ "display_obs_diffs(default_obs, c2_configuration_obs, blue_env.game.step_counter)" ] @@ -880,9 +1231,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 26, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "RequestResponse(status='success', data={0: RequestResponse(status='success', data={})})" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Installing RansomwareScript via C2 Terminal Commands\n", "ransomware_install_command = {\"commands\":[[\"software_manager\", \"application\", \"install\", \"RansomwareScript\"]],\n", @@ -893,9 +1255,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 27, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "RequestResponse(status='success', data={})" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Configuring the RansomwareScript\n", "ransomware_config = {\"server_ip_address\": \"192.168.1.14\", \"payload\": \"ENCRYPT\"}\n", @@ -904,7 +1277,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 28, "metadata": {}, "outputs": [], "source": [ @@ -914,9 +1287,28 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 29, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 7\n", + "root['NODES']['HOST0']['APPLICATIONS'][1]['operating_status']: 0 -> 1\n", + "root['NODES']['HOST0']['APPLICATIONS'][2]['operating_status']: 0 -> 3\n", + "root['NODES']['HOST0']['users']['local_login']: 0 -> 1\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", + "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", + "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" + ] + } + ], "source": [ "display_obs_diffs(default_obs, c2_ransomware_obs, env.game.step_counter)" ] @@ -932,7 +1324,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 30, "metadata": {}, "outputs": [], "source": [ @@ -948,16 +1340,27 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 31, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "RequestResponse(status='success', data={})" + ] + }, + "execution_count": 31, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "c2_server.send_command(given_command=C2Command.DATA_EXFILTRATION, command_options=exfil_options)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 32, "metadata": {}, "outputs": [], "source": [ @@ -966,9 +1369,22 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 33, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 7\n", + "root['NODES']['HOST0']['APPLICATIONS'][2]['operating_status']: 3 -> 1\n", + "root['NODES']['HOST1']['users']['remote_sessions']: 0 -> 1\n" + ] + } + ], "source": [ "display_obs_diffs(c2_ransomware_obs, c2_exfil_obs, env.game.step_counter)" ] @@ -984,9 +1400,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 34, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "RequestResponse(status='success', data={})" + ] + }, + "execution_count": 34, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Configuring the RansomwareScript\n", "ransomware_config = {\"server_ip_address\": \"192.168.1.14\", \"payload\": \"ENCRYPT\"}\n", @@ -995,9 +1422,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 35, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "RequestResponse(status='success', data={})" + ] + }, + "execution_count": 35, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Waiting for the ransomware to finish installing and then launching the RansomwareScript.\n", "blue_env.step(0)\n", @@ -1006,7 +1444,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 36, "metadata": {}, "outputs": [], "source": [ @@ -1016,9 +1454,30 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 37, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 6\n", + "root['NODES']['HOST0']['APPLICATIONS'][2]['operating_status']: 3 -> 1\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['NODES']['HOST1']['users']['remote_sessions']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" + ] + } + ], "source": [ "display_obs_diffs(c2_ransomware_obs, c2_final_obs, blue_env.game.step_counter)" ] @@ -1034,7 +1493,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 38, "metadata": {}, "outputs": [], "source": [ @@ -1066,16 +1525,200 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 39, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-03 16:04:27,571: Resetting environment, episode 1, avg. reward: 0.0\n", + "2025-02-03 16:04:27,574: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-03/16-04-20/agent_actions/episode_1.json\n" + ] + }, + { + "data": { + "text/plain": [ + "({'NODES': {'HOST0': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 1,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'HOST1': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 1,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'HOST2': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 0,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'HOST3': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 0,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'ROUTER0': {'ACL': {1: {'position': 0,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 2: {'position': 1,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 3: {'position': 2,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 4: {'position': 3,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 5: {'position': 4,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 6: {'position': 5,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 7: {'position': 6,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 8: {'position': 7,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 9: {'position': 8,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 10: {'position': 9,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0}},\n", + " 'PORTS': {1: {'operating_status': 1},\n", + " 2: {'operating_status': 1},\n", + " 3: {'operating_status': 2}},\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}}},\n", + " 'LINKS': {1: {'PROTOCOLS': {'ALL': 1}},\n", + " 2: {'PROTOCOLS': {'ALL': 1}},\n", + " 3: {'PROTOCOLS': {'ALL': 0}},\n", + " 4: {'PROTOCOLS': {'ALL': 1}},\n", + " 5: {'PROTOCOLS': {'ALL': 1}},\n", + " 6: {'PROTOCOLS': {'ALL': 1}},\n", + " 7: {'PROTOCOLS': {'ALL': 1}},\n", + " 8: {'PROTOCOLS': {'ALL': 1}},\n", + " 9: {'PROTOCOLS': {'ALL': 1}},\n", + " 10: {'PROTOCOLS': {'ALL': 0}}},\n", + " 'ICS': 0},\n", + " {})" + ] + }, + "execution_count": 39, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "blue_env.reset()" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 40, "metadata": {}, "outputs": [], "source": [ @@ -1094,7 +1737,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 41, "metadata": {}, "outputs": [], "source": [ @@ -1111,9 +1754,33 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 42, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+-------------------------------------------------------------------------------------+\n", + "| web_server Software Manager |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| Name | Type | Operating State | Health State | Port | Protocol |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", + "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", + "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", + "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", + "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", + "| NMAP | Application | RUNNING | GOOD | None | none |\n", + "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", + "| UserManager | Service | RUNNING | GOOD | None | none |\n", + "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", + "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", + "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n" + ] + } + ], "source": [ "blue_env.step(0)\n", "web_server.software_manager.show()" @@ -1121,9 +1788,26 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 43, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 3\n", + "root['NODES']['HOST0']['APPLICATIONS'][1]['operating_status']: 1 -> 0\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", + "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", + "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" + ] + } + ], "source": [ "display_obs_diffs(pre_blue_action_obs, post_blue_action_obs, blue_env.game.step_counter)" ] @@ -1137,9 +1821,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 44, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "RequestResponse(status='failure', data={'Reason': 'Command sent to the C2 Beacon but no response was ever received.'})" + ] + }, + "execution_count": 44, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Attempting to install the C2 RansomwareScript\n", "ransomware_install_command = {\"commands\":[[\"software_manager\", \"application\", \"install\", \"RansomwareScript\"]],\n", @@ -1161,16 +1856,200 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 45, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-03 16:04:28,041: Resetting environment, episode 2, avg. reward: 0.0\n", + "2025-02-03 16:04:28,045: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-03/16-04-20/agent_actions/episode_2.json\n" + ] + }, + { + "data": { + "text/plain": [ + "({'NODES': {'HOST0': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 1,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'HOST1': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 1,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'HOST2': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 0,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'HOST3': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 0,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'ROUTER0': {'ACL': {1: {'position': 0,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 2: {'position': 1,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 3: {'position': 2,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 4: {'position': 3,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 5: {'position': 4,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 6: {'position': 5,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 7: {'position': 6,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 8: {'position': 7,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 9: {'position': 8,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 10: {'position': 9,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0}},\n", + " 'PORTS': {1: {'operating_status': 1},\n", + " 2: {'operating_status': 1},\n", + " 3: {'operating_status': 2}},\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}}},\n", + " 'LINKS': {1: {'PROTOCOLS': {'ALL': 1}},\n", + " 2: {'PROTOCOLS': {'ALL': 1}},\n", + " 3: {'PROTOCOLS': {'ALL': 0}},\n", + " 4: {'PROTOCOLS': {'ALL': 1}},\n", + " 5: {'PROTOCOLS': {'ALL': 1}},\n", + " 6: {'PROTOCOLS': {'ALL': 1}},\n", + " 7: {'PROTOCOLS': {'ALL': 1}},\n", + " 8: {'PROTOCOLS': {'ALL': 1}},\n", + " 9: {'PROTOCOLS': {'ALL': 1}},\n", + " 10: {'PROTOCOLS': {'ALL': 0}}},\n", + " 'ICS': 0},\n", + " {})" + ] + }, + "execution_count": 45, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "blue_env.reset()" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 46, "metadata": {}, "outputs": [], "source": [ @@ -1189,7 +2068,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 47, "metadata": {}, "outputs": [], "source": [ @@ -1206,9 +2085,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 48, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "NodeOperatingState.SHUTTING_DOWN\n" + ] + } + ], "source": [ "web_server = blue_env.game.simulation.network.get_node_by_hostname(\"web_server\")\n", "print(web_server.operating_state)" @@ -1216,18 +2103,48 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 49, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 2\n", + "root['NODES']['HOST0']['operating_status']: 1 -> 4\n", + "root['NODES']['HOST0']['APPLICATIONS'][1]['operating_status']: 1 -> 0\n", + "root['NODES']['HOST0']['NICS'][1]['nic_status']: 1 -> 0\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", + "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", + "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" + ] + } + ], "source": [ "display_obs_diffs(pre_blue_action_obs, post_blue_action_obs, blue_env.game.step_counter)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 50, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "RequestResponse(status='failure', data={'Reason': 'Command sent to the C2 Beacon but no response was ever received.'})" + ] + }, + "execution_count": 50, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Attempting to install the C2 RansomwareScript\n", "ransomware_install_command = {\"commands\":[\"software_manager\", \"application\", \"install\", \"RansomwareScript\"],\n", @@ -1251,16 +2168,200 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 51, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-03 16:04:28,560: Resetting environment, episode 3, avg. reward: 0.0\n", + "2025-02-03 16:04:28,564: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-03/16-04-20/agent_actions/episode_3.json\n" + ] + }, + { + "data": { + "text/plain": [ + "({'NODES': {'HOST0': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 1,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'HOST1': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 1,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'HOST2': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 0,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'HOST3': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 0,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'ROUTER0': {'ACL': {1: {'position': 0,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 2: {'position': 1,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 3: {'position': 2,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 4: {'position': 3,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 5: {'position': 4,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 6: {'position': 5,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 7: {'position': 6,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 8: {'position': 7,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 9: {'position': 8,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 10: {'position': 9,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0}},\n", + " 'PORTS': {1: {'operating_status': 1},\n", + " 2: {'operating_status': 1},\n", + " 3: {'operating_status': 2}},\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}}},\n", + " 'LINKS': {1: {'PROTOCOLS': {'ALL': 1}},\n", + " 2: {'PROTOCOLS': {'ALL': 1}},\n", + " 3: {'PROTOCOLS': {'ALL': 0}},\n", + " 4: {'PROTOCOLS': {'ALL': 1}},\n", + " 5: {'PROTOCOLS': {'ALL': 1}},\n", + " 6: {'PROTOCOLS': {'ALL': 1}},\n", + " 7: {'PROTOCOLS': {'ALL': 1}},\n", + " 8: {'PROTOCOLS': {'ALL': 1}},\n", + " 9: {'PROTOCOLS': {'ALL': 1}},\n", + " 10: {'PROTOCOLS': {'ALL': 0}}},\n", + " 'ICS': 0},\n", + " {})" + ] + }, + "execution_count": 51, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "blue_env.reset()" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 52, "metadata": {}, "outputs": [], "source": [ @@ -1279,7 +2380,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 53, "metadata": {}, "outputs": [], "source": [ @@ -1296,11 +2397,44 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 54, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+---------------------------------------------------------------+\n", + "| router_1 Network Interfaces |\n", + "+------+-------------------+-----------------+-------+----------+\n", + "| Port | MAC Address | Address | Speed | Status |\n", + "+------+-------------------+-----------------+-------+----------+\n", + "| 1 | dd:6e:95:d4:3f:74 | 192.168.1.1/24 | 100.0 | Enabled |\n", + "| 2 | 8b:79:07:fc:69:2c | 192.168.10.1/24 | 100.0 | Enabled |\n", + "| 3 | 1f:fd:c4:ae:7a:00 | 127.0.0.1/8 | 100.0 | Disabled |\n", + "| 4 | 7b:e3:bf:4b:76:e8 | 127.0.0.1/8 | 100.0 | Disabled |\n", + "| 5 | 4f:37:b0:6b:5d:44 | 127.0.0.1/8 | 100.0 | Disabled |\n", + "+------+-------------------+-----------------+-------+----------+\n", + "+------------------------------------------------------------------------------------------------------------------------+\n", + "| router_1 Access Control List |\n", + "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n", + "| Index | Action | Protocol | Src IP | Src Wildcard | Src Port | Dst IP | Dst Wildcard | Dst Port | Matched |\n", + "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n", + "| 1 | DENY | ANY | 192.168.10.21 | ANY | 80 | 192.168.1.12 | ANY | 80 | 0 |\n", + "| 18 | PERMIT | ANY | ANY | ANY | 5432 | ANY | ANY | 5432 | 0 |\n", + "| 19 | PERMIT | ANY | ANY | ANY | 53 | ANY | ANY | 53 | 0 |\n", + "| 20 | PERMIT | ANY | ANY | ANY | 21 | ANY | ANY | 21 | 0 |\n", + "| 21 | PERMIT | ANY | ANY | ANY | 80 | ANY | ANY | 80 | 4 |\n", + "| 22 | PERMIT | ANY | ANY | ANY | 219 | ANY | ANY | 219 | 10 |\n", + "| 23 | PERMIT | icmp | ANY | ANY | ANY | ANY | ANY | ANY | 0 |\n", + "| 24 | DENY | ANY | ANY | ANY | ANY | ANY | ANY | ANY | 0 |\n", + "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n" + ] + } + ], "source": [ "router_1: Router = blue_env.game.simulation.network.get_node_by_hostname(\"router_1\")\n", + "router_1.show()\n", "router_1.acl.show()" ] }, @@ -1313,9 +2447,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 55, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "RequestResponse(status='failure', data={'Reason': 'Command sent to the C2 Beacon but no response was ever received.'})" + ] + }, + "execution_count": 55, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "blue_env.step(0)\n", "\n", @@ -1326,9 +2471,30 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 56, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+------------------------------------------------------------------------------------------------------------------------+\n", + "| router_1 Access Control List |\n", + "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n", + "| Index | Action | Protocol | Src IP | Src Wildcard | Src Port | Dst IP | Dst Wildcard | Dst Port | Matched |\n", + "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n", + "| 1 | DENY | ANY | 192.168.10.21 | ANY | 80 | 192.168.1.12 | ANY | 80 | 2 |\n", + "| 18 | PERMIT | ANY | ANY | ANY | 5432 | ANY | ANY | 5432 | 0 |\n", + "| 19 | PERMIT | ANY | ANY | ANY | 53 | ANY | ANY | 53 | 0 |\n", + "| 20 | PERMIT | ANY | ANY | ANY | 21 | ANY | ANY | 21 | 0 |\n", + "| 21 | PERMIT | ANY | ANY | ANY | 80 | ANY | ANY | 80 | 4 |\n", + "| 22 | PERMIT | ANY | ANY | ANY | 219 | ANY | ANY | 219 | 10 |\n", + "| 23 | PERMIT | icmp | ANY | ANY | ANY | ANY | ANY | ANY | 0 |\n", + "| 24 | DENY | ANY | ANY | ANY | ANY | ANY | ANY | ANY | 0 |\n", + "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n" + ] + } + ], "source": [ "router_1.acl.show()" ] @@ -1342,18 +2508,58 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 57, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+-------------------------------------------------------------------------------------+\n", + "| web_server Software Manager |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| Name | Type | Operating State | Health State | Port | Protocol |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", + "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", + "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", + "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", + "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", + "| NMAP | Application | RUNNING | GOOD | None | none |\n", + "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", + "| UserManager | Service | RUNNING | GOOD | None | none |\n", + "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", + "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", + "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", + "| C2Beacon | Application | RUNNING | GOOD | None | tcp |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n" + ] + } + ], "source": [ "web_server.software_manager.show()" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 58, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----------------------------------------------------------------------------------+\n", + "| database_server File System |\n", + "+----------------------+---------+---------------+-----------------------+---------+\n", + "| File Path | Size | Health status | Visible health status | Deleted |\n", + "+----------------------+---------+---------------+-----------------------+---------+\n", + "| database/database.db | 4.77 MB | GOOD | NONE | False |\n", + "| root | 0 B | GOOD | NONE | False |\n", + "+----------------------+---------+---------------+-----------------------+---------+\n" + ] + } + ], "source": [ "database_server: Server = blue_env.game.simulation.network.get_node_by_hostname(\"database_server\")\n", "database_server.software_manager.file_system.show(full=True)" @@ -1361,9 +2567,33 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 59, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 3\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", + "root['NODES']['ROUTER0']['ACL'][1]['permission']: 0 -> 2\n", + "root['NODES']['ROUTER0']['ACL'][1]['source_ip_id']: 0 -> 7\n", + "root['NODES']['ROUTER0']['ACL'][1]['source_wildcard_id']: 0 -> 1\n", + "root['NODES']['ROUTER0']['ACL'][1]['source_port_id']: 0 -> 2\n", + "root['NODES']['ROUTER0']['ACL'][1]['dest_ip_id']: 0 -> 3\n", + "root['NODES']['ROUTER0']['ACL'][1]['dest_wildcard_id']: 0 -> 1\n", + "root['NODES']['ROUTER0']['ACL'][1]['dest_port_id']: 0 -> 2\n", + "root['NODES']['ROUTER0']['ACL'][1]['protocol_id']: 0 -> 1\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", + "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", + "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" + ] + } + ], "source": [ "display_obs_diffs(pre_blue_action_obs, post_blue_action_obs, blue_env.game.step_counter)" ] @@ -1427,9 +2657,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 60, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-03 16:04:29,610: PrimaiteGymEnv RNG seed = None\n" + ] + } + ], "source": [ "with open(data_manipulation_config_path()) as f:\n", " cfg = yaml.safe_load(f)\n", @@ -1450,7 +2688,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 61, "metadata": {}, "outputs": [], "source": [ @@ -1478,9 +2716,30 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 62, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", + "| C2Beacon Running Status |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| False | None | 0 | 5 | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "+-----------------------------------------------------------------------------------------------------+\n", + "| C2Server Running Status |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n", + "| False | None | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n" + ] + } + ], "source": [ "env.step(2) # Agent Action Equivalent to c2_beacon.configure(c2_server_ip_address=\"192.168.10.21\")\n", "env.step(3) # Agent action Equivalent to c2_beacon.establish()\n", @@ -1497,9 +2756,30 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 63, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", + "| C2Beacon Running Status |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| False | None | 0 | 5 | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "+-----------------------------------------------------------------------------------------------------+\n", + "| C2Server Running Status |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n", + "| False | None | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n" + ] + } + ], "source": [ "env.step(9) # Equivalent of to c2_beacon.configure(c2_server_ip_address=\"192.168.10.22\")\n", "env.step(3)\n", @@ -1517,9 +2797,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 64, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+-----------------------------------------------------------------------------------------------------+\n", + "| C2Server Running Status |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n", + "| False | None | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n" + ] + } + ], "source": [ "for i in range(6):\n", " env.step(0)\n", @@ -1542,9 +2836,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 65, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-03 16:04:30,011: PrimaiteGymEnv RNG seed = None\n" + ] + } + ], "source": [ "with open(data_manipulation_config_path()) as f:\n", " cfg = yaml.safe_load(f)\n", @@ -1560,7 +2862,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 66, "metadata": {}, "outputs": [], "source": [ @@ -1584,9 +2886,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 67, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", + "| C2Beacon Running Status |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| True | 192.168.10.21 | 0 | 5 | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n" + ] + } + ], "source": [ "c2_beacon.configure(c2_server_ip_address=\"192.168.10.21\")\n", "c2_beacon.establish()\n", @@ -1604,9 +2920,72 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 68, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 4\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 5\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 6\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 7\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 8\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 9\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 10\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 11\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 12\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 13\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" + ] + } + ], "source": [ "for i in range(10):\n", " keep_alive_obs, _, _, _, _ = blue_config_env.step(0)\n", @@ -1622,9 +3001,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 69, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", + "| C2Beacon Running Status |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| True | 192.168.10.21 | 0 | 1 | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n" + ] + } + ], "source": [ "c2_beacon.configure(c2_server_ip_address=\"192.168.10.21\", keep_alive_frequency=1)\n", "c2_beacon.establish()\n", @@ -1640,9 +3033,40 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 70, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 14\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 15\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" + ] + } + ], "source": [ "# Comparing the OBS of the default frequency to a timestep frequency of 1\n", "for i in range(2):\n", @@ -1661,9 +3085,52 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 71, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 16\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 17\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 18\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 19\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 20\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 21\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 22\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" + ] + } + ], "source": [ "c2_beacon.configure(c2_server_ip_address=\"192.168.10.21\", keep_alive_frequency=7)\n", "\n", @@ -1700,9 +3167,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 72, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-03 16:04:30,864: Resetting environment, episode 0, avg. reward: 0.0\n", + "2025-02-03 16:04:30,867: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-03/16-04-20/agent_actions/episode_0.json\n" + ] + } + ], "source": [ "blue_config_env.reset()\n", "\n", @@ -1724,9 +3200,28 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 73, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 5\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" + ] + } + ], "source": [ "# Capturing default C2 Traffic\n", "for i in range(3):\n", @@ -1744,9 +3239,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 74, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", + "| C2Beacon Running Status |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| True | 192.168.10.21 | 0 | 5 | udp | 53 |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n" + ] + } + ], "source": [ "from primaite.utils.validation.ip_protocol import PROTOCOL_LOOKUP\n", "from primaite.utils.validation.port import PORT_LOOKUP\n", @@ -1759,9 +3268,28 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 75, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 10\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 1 -> 0\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 1 -> 0\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['udp'][53]['inbound']: 0 -> 1\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['udp'][53]['outbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 1 -> 0\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 1 -> 0\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['udp'][53]['inbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['udp'][53]['outbound']: 0 -> 1\n" + ] + } + ], "source": [ "# Capturing UDP C2 Traffic\n", "for i in range(5):\n", From 454bd61fb27e2e8b8702d86c0e83806deae7cd0c Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Tue, 4 Feb 2025 10:50:53 +0000 Subject: [PATCH 72/95] #3075: Update internal YAML fragments in C2C-E2E-Demo notebook. --- ...ommand-and-Control-E2E-Demonstration.ipynb | 1802 ++--------------- 1 file changed, 137 insertions(+), 1665 deletions(-) diff --git a/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb b/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb index 2c1a94c8..c2a87e45 100644 --- a/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb +++ b/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb @@ -13,24 +13,9 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-02-03 16:04:19,828: Performing the PrimAITE first-time setup...\n", - "2025-02-03 16:04:19,829: Building the PrimAITE app directories...\n", - "2025-02-03 16:04:19,829: Building primaite_config.yaml...\n", - "2025-02-03 16:04:19,829: Rebuilding the demo notebooks...\n", - "/home/nick/primaite/4.0.0a1-dev/notebooks/example_notebooks/Command-and-Control-E2E-Demonstration.ipynb\n", - "2025-02-03 16:04:19,831: Reset example notebook: /home/nick/primaite/4.0.0a1-dev/notebooks/example_notebooks/Command-and-Control-E2E-Demonstration.ipynb\n", - "2025-02-03 16:04:19,836: Rebuilding the example notebooks...\n", - "2025-02-03 16:04:19,840: PrimAITE setup complete!\n" - ] - } - ], + "outputs": [], "source": [ "!primaite setup" ] @@ -183,17 +168,9 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-03 16:04:24,734: PrimaiteGymEnv RNG seed = None\n" - ] - } - ], + "outputs": [], "source": [ "with open(data_manipulation_config_path()) as f:\n", " cfg = yaml.safe_load(f)\n", @@ -219,35 +196,9 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+--------------------------------------------------------------------------------------+\n", - "| client_1 Software Manager |\n", - "+---------------------+-------------+-----------------+--------------+------+----------+\n", - "| Name | Type | Operating State | Health State | Port | Protocol |\n", - "+---------------------+-------------+-----------------+--------------+------+----------+\n", - "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", - "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", - "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", - "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", - "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", - "| NMAP | Application | RUNNING | GOOD | None | none |\n", - "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", - "| UserManager | Service | RUNNING | GOOD | None | none |\n", - "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", - "| FTPClient | Service | RUNNING | GOOD | 21 | tcp |\n", - "| DataManipulationBot | Application | RUNNING | GOOD | None | none |\n", - "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", - "| C2Server | Application | RUNNING | GOOD | None | tcp |\n", - "+---------------------+-------------+-----------------+--------------+------+----------+\n" - ] - } - ], + "outputs": [], "source": [ "client_1: Computer = env.game.simulation.network.get_node_by_hostname(\"client_1\")\n", "client_1.software_manager.install(C2Server)\n", @@ -297,34 +248,9 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+-------------------------------------------------------------------------------------+\n", - "| web_server Software Manager |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| Name | Type | Operating State | Health State | Port | Protocol |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", - "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", - "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", - "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", - "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", - "| NMAP | Application | RUNNING | GOOD | None | none |\n", - "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", - "| UserManager | Service | RUNNING | GOOD | None | none |\n", - "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", - "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", - "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", - "| C2Beacon | Application | INSTALLING | UNUSED | None | tcp |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n" - ] - } - ], + "outputs": [], "source": [ "env.step(1)\n", "web_server: Computer = env.game.simulation.network.get_node_by_hostname(\"web_server\")\n", @@ -364,41 +290,9 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+-------------------------------------------------------------------------------------+\n", - "| web_server Software Manager |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| Name | Type | Operating State | Health State | Port | Protocol |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", - "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", - "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", - "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", - "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", - "| NMAP | Application | RUNNING | GOOD | None | none |\n", - "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", - "| UserManager | Service | RUNNING | GOOD | None | none |\n", - "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", - "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", - "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", - "| C2Beacon | Application | RUNNING | GOOD | None | tcp |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", - "| C2Beacon Running Status |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| False | 192.168.10.21 | 0 | 5 | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n" - ] - } - ], + "outputs": [], "source": [ "env.step(2)\n", "c2_beacon: C2Beacon = web_server.software_manager.software[\"C2Beacon\"]\n", @@ -437,54 +331,18 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(0,\n", - " 0.0,\n", - " False,\n", - " False,\n", - " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=2, action='node_application_execute', parameters={'node_name': 'web_server', 'application_name': 'C2Beacon'}, request=['network', 'node', 'web_server', 'application', 'C2Beacon', 'execute'], response=RequestResponse(status='success', data={}), reward=0.0, reward_info={})}})" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "env.step(3)" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", - "| C2Beacon Running Status |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| True | 192.168.10.21 | 1 | 5 | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "+-----------------------------------------------------------------------------------------------------+\n", - "| C2Server Running Status |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n", - "| True | 192.168.1.12 | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n" - ] - } - ], + "outputs": [], "source": [ "c2_beacon.show()\n", "c2_server.show()" @@ -547,59 +405,18 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(0,\n", - " 0.0,\n", - " False,\n", - " False,\n", - " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=3, action='c2_server_terminal_command', parameters={'node_name': 'client_1', 'ip_address': None, 'username': 'admin', 'password': 'admin', 'commands': [['software_manager', 'application', 'install', 'RansomwareScript']]}, request=['network', 'node', 'client_1', 'application', 'C2Server', 'terminal_command', {'commands': [['software_manager', 'application', 'install', 'RansomwareScript']], 'ip_address': None, 'username': 'admin', 'password': 'admin'}], response=RequestResponse(status='success', data={0: RequestResponse(status='success', data={})}), reward=0.0, reward_info={})}})" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "env.step(4)" ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+--------------------------------------------------------------------------------------+\n", - "| client_1 Software Manager |\n", - "+---------------------+-------------+-----------------+--------------+------+----------+\n", - "| Name | Type | Operating State | Health State | Port | Protocol |\n", - "+---------------------+-------------+-----------------+--------------+------+----------+\n", - "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", - "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", - "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", - "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", - "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", - "| NMAP | Application | RUNNING | GOOD | None | none |\n", - "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", - "| UserManager | Service | RUNNING | GOOD | None | none |\n", - "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", - "| FTPClient | Service | RUNNING | GOOD | 21 | tcp |\n", - "| DataManipulationBot | Application | RUNNING | GOOD | None | none |\n", - "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", - "| C2Server | Application | RUNNING | GOOD | None | tcp |\n", - "+---------------------+-------------+-----------------+--------------+------+----------+\n" - ] - } - ], + "outputs": [], "source": [ "client_1.software_manager.show()" ] @@ -638,66 +455,18 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(0,\n", - " 0.0,\n", - " False,\n", - " False,\n", - " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=4, action='c2_server_ransomware_configure', parameters={'node_name': 'client_1', 'server_ip_address': '192.168.1.14', 'payload': 'ENCRYPT'}, request=['network', 'node', 'client_1', 'application', 'C2Server', 'ransomware_configure', {'server_ip_address': '192.168.1.14', 'server_password': None, 'payload': 'ENCRYPT'}], response=RequestResponse(status='success', data={}), reward=0.0, reward_info={})}})" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "env.step(5)" ] }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+-------------------------------------------------------------------------------------+\n", - "| web_server Software Manager |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| Name | Type | Operating State | Health State | Port | Protocol |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", - "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", - "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", - "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", - "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", - "| NMAP | Application | RUNNING | GOOD | None | none |\n", - "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", - "| UserManager | Service | RUNNING | GOOD | None | none |\n", - "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", - "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", - "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", - "| C2Beacon | Application | RUNNING | GOOD | None | tcp |\n", - "| RansomwareScript | Application | RUNNING | GOOD | None | none |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "+------------------------------------+\n", - "| RansomwareScript Running Status |\n", - "+--------------------------+---------+\n", - "| Target Server IP Address | Payload |\n", - "+--------------------------+---------+\n", - "| 192.168.1.14 | ENCRYPT |\n", - "+--------------------------+---------+\n" - ] - } - ], + "outputs": [], "source": [ "ransomware_script: RansomwareScript = web_server.software_manager.software[\"RansomwareScript\"]\n", "web_server.software_manager.show()\n", @@ -743,48 +512,18 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(0,\n", - " 0.0,\n", - " False,\n", - " False,\n", - " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=5, action='c2_server_data_exfiltrate', parameters={'node_name': 'client_1', 'target_file_name': 'database.db', 'target_folder_name': 'database', 'exfiltration_folder_name': 'spoils', 'target_ip_address': '192.168.1.14', 'username': 'admin', 'password': 'admin'}, request=['network', 'node', 'client_1', 'application', 'C2Server', 'exfiltrate', {'target_file_name': 'database.db', 'target_folder_name': 'database', 'exfiltration_folder_name': 'spoils', 'target_ip_address': '192.168.1.14', 'username': 'admin', 'password': 'admin'}], response=RequestResponse(status='success', data={}), reward=0.0, reward_info={})}})" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "env.step(6)" ] }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+--------------------------------------------------------------------------------+\n", - "| client_1 File System |\n", - "+--------------------+---------+---------------+-----------------------+---------+\n", - "| File Path | Size | Health status | Visible health status | Deleted |\n", - "+--------------------+---------+---------------+-----------------------+---------+\n", - "| root | 0 B | GOOD | NONE | False |\n", - "| spoils/database.db | 4.77 MB | GOOD | NONE | False |\n", - "+--------------------+---------+---------------+-----------------------+---------+\n" - ] - } - ], + "outputs": [], "source": [ "client_1: Computer = env.game.simulation.network.get_node_by_hostname(\"client_1\")\n", "client_1.software_manager.file_system.show(full=True)" @@ -792,25 +531,9 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+---------------------------------------------------------------------------------+\n", - "| web_server File System |\n", - "+---------------------+---------+---------------+-----------------------+---------+\n", - "| File Path | Size | Health status | Visible health status | Deleted |\n", - "+---------------------+---------+---------------+-----------------------+---------+\n", - "| primaite/index.html | 15.0 KB | GOOD | NONE | False |\n", - "| root | 0 B | GOOD | NONE | False |\n", - "| spoils/database.db | 4.77 MB | GOOD | NONE | False |\n", - "+---------------------+---------+---------------+-----------------------+---------+\n" - ] - } - ], + "outputs": [], "source": [ "web_server: Computer = env.game.simulation.network.get_node_by_hostname(\"web_server\")\n", "web_server.software_manager.file_system.show(full=True)" @@ -847,48 +570,18 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(0,\n", - " 0.0,\n", - " False,\n", - " False,\n", - " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=6, action='c2_server_ransomware_launch', parameters={'node_name': 'client_1'}, request=['network', 'node', 'client_1', 'application', 'C2Server', 'ransomware_launch'], response=RequestResponse(status='success', data={}), reward=0.0, reward_info={})}})" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "env.step(7)" ] }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+----------------------------------------------------------------------------------+\n", - "| database_server File System |\n", - "+----------------------+---------+---------------+-----------------------+---------+\n", - "| File Path | Size | Health status | Visible health status | Deleted |\n", - "+----------------------+---------+---------------+-----------------------+---------+\n", - "| database/database.db | 4.77 MB | CORRUPT | NONE | False |\n", - "| root | 0 B | GOOD | NONE | False |\n", - "+----------------------+---------+---------------+-----------------------+---------+\n" - ] - } - ], + "outputs": [], "source": [ "database_server: Server = env.game.simulation.network.get_node_by_hostname(\"database_server\")\n", "database_server.software_manager.file_system.show(full=True)" @@ -907,7 +600,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1025,30 +718,30 @@ "\n", "\n", " # options:\n", - " # nodes:\n", - " # - node_name: web_server\n", - " # applications:\n", - " # - application_name: C2Beacon\n", + " # # nodes:\n", + " # node_name: web_server\n", + " # # applications:\n", + " # application_name: C2Beacon\n", "\n", - " # - node_name: database_server\n", - " # folders:\n", - " # - folder_name: database\n", - " # files:\n", - " # - file_name: database.db\n", - " # services:\n", - " # - service_name: DatabaseService\n", - " # - node_name: router_1\n", + " # node_name: database_server\n", + " # folders:\n", + " # - folder_name: database\n", + " # files:\n", + " # - file_name: database.db\n", + " # services:\n", + " # - service_name: DatabaseService\n", + " # node_name: router_1\n", "\n", - " # max_folders_per_node: 2\n", - " # max_files_per_folder: 2\n", - " # max_services_per_node: 2\n", - " # max_nics_per_node: 8\n", - " # max_acl_rules: 10\n", - " # ip_list:\n", - " # - 192.168.10.21\n", - " # - 192.168.1.12\n", - " # wildcard_list:\n", - " # - 0.0.0.1\n", + " # max_folders_per_node: 2\n", + " # max_files_per_folder: 2\n", + " # max_services_per_node: 2\n", + " # max_nics_per_node: 8\n", + " # max_acl_rules: 10\n", + " # ip_list:\n", + " # - 192.168.10.21\n", + " # - 192.168.1.12\n", + " # wildcard_list:\n", + " # - 0.0.0.1\n", " reward_function:\n", " reward_components:\n", " - type: DUMMY\n", @@ -1061,17 +754,9 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-03 16:04:26,020: PrimaiteGymEnv RNG seed = None\n" - ] - } - ], + "outputs": [], "source": [ "with open(data_manipulation_config_path()) as f:\n", " cfg = yaml.safe_load(f)\n", @@ -1128,18 +813,9 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-03 16:04:26,440: Resetting environment, episode 0, avg. reward: 0.0\n", - "2025-02-03 16:04:26,445: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-03/16-04-20/agent_actions/episode_0.json\n" - ] - } - ], + "outputs": [], "source": [ "# Resetting the environment and capturing the default observation space.\n", "blue_env.reset()\n", @@ -1148,20 +824,9 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 23, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Setting up the C2 Suite via the simulation API.\n", "\n", @@ -1192,26 +857,9 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 2\n", - "root['NODES']['HOST0']['APPLICATIONS'][1]['operating_status']: 0 -> 1\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", - "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", - "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" - ] - } - ], + "outputs": [], "source": [ "display_obs_diffs(default_obs, c2_configuration_obs, blue_env.game.step_counter)" ] @@ -1231,20 +879,9 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "RequestResponse(status='success', data={0: RequestResponse(status='success', data={})})" - ] - }, - "execution_count": 26, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Installing RansomwareScript via C2 Terminal Commands\n", "ransomware_install_command = {\"commands\":[[\"software_manager\", \"application\", \"install\", \"RansomwareScript\"]],\n", @@ -1255,20 +892,9 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "RequestResponse(status='success', data={})" - ] - }, - "execution_count": 27, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Configuring the RansomwareScript\n", "ransomware_config = {\"server_ip_address\": \"192.168.1.14\", \"payload\": \"ENCRYPT\"}\n", @@ -1287,28 +913,9 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 7\n", - "root['NODES']['HOST0']['APPLICATIONS'][1]['operating_status']: 0 -> 1\n", - "root['NODES']['HOST0']['APPLICATIONS'][2]['operating_status']: 0 -> 3\n", - "root['NODES']['HOST0']['users']['local_login']: 0 -> 1\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", - "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", - "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" - ] - } - ], + "outputs": [], "source": [ "display_obs_diffs(default_obs, c2_ransomware_obs, env.game.step_counter)" ] @@ -1340,20 +947,9 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "RequestResponse(status='success', data={})" - ] - }, - "execution_count": 31, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "c2_server.send_command(given_command=C2Command.DATA_EXFILTRATION, command_options=exfil_options)" ] @@ -1369,22 +965,9 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 7\n", - "root['NODES']['HOST0']['APPLICATIONS'][2]['operating_status']: 3 -> 1\n", - "root['NODES']['HOST1']['users']['remote_sessions']: 0 -> 1\n" - ] - } - ], + "outputs": [], "source": [ "display_obs_diffs(c2_ransomware_obs, c2_exfil_obs, env.game.step_counter)" ] @@ -1400,20 +983,9 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "RequestResponse(status='success', data={})" - ] - }, - "execution_count": 34, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Configuring the RansomwareScript\n", "ransomware_config = {\"server_ip_address\": \"192.168.1.14\", \"payload\": \"ENCRYPT\"}\n", @@ -1422,20 +994,9 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "RequestResponse(status='success', data={})" - ] - }, - "execution_count": 35, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Waiting for the ransomware to finish installing and then launching the RansomwareScript.\n", "blue_env.step(0)\n", @@ -1454,30 +1015,9 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 6\n", - "root['NODES']['HOST0']['APPLICATIONS'][2]['operating_status']: 3 -> 1\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['NODES']['HOST1']['users']['remote_sessions']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" - ] - } - ], + "outputs": [], "source": [ "display_obs_diffs(c2_ransomware_obs, c2_final_obs, blue_env.game.step_counter)" ] @@ -1525,193 +1065,9 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-03 16:04:27,571: Resetting environment, episode 1, avg. reward: 0.0\n", - "2025-02-03 16:04:27,574: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-03/16-04-20/agent_actions/episode_1.json\n" - ] - }, - { - "data": { - "text/plain": [ - "({'NODES': {'HOST0': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 1,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'HOST1': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 1,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'HOST2': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 0,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'HOST3': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 0,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'ROUTER0': {'ACL': {1: {'position': 0,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 2: {'position': 1,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 3: {'position': 2,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 4: {'position': 3,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 5: {'position': 4,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 6: {'position': 5,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 7: {'position': 6,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 8: {'position': 7,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 9: {'position': 8,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 10: {'position': 9,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0}},\n", - " 'PORTS': {1: {'operating_status': 1},\n", - " 2: {'operating_status': 1},\n", - " 3: {'operating_status': 2}},\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}}},\n", - " 'LINKS': {1: {'PROTOCOLS': {'ALL': 1}},\n", - " 2: {'PROTOCOLS': {'ALL': 1}},\n", - " 3: {'PROTOCOLS': {'ALL': 0}},\n", - " 4: {'PROTOCOLS': {'ALL': 1}},\n", - " 5: {'PROTOCOLS': {'ALL': 1}},\n", - " 6: {'PROTOCOLS': {'ALL': 1}},\n", - " 7: {'PROTOCOLS': {'ALL': 1}},\n", - " 8: {'PROTOCOLS': {'ALL': 1}},\n", - " 9: {'PROTOCOLS': {'ALL': 1}},\n", - " 10: {'PROTOCOLS': {'ALL': 0}}},\n", - " 'ICS': 0},\n", - " {})" - ] - }, - "execution_count": 39, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "blue_env.reset()" ] @@ -1754,33 +1110,9 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+-------------------------------------------------------------------------------------+\n", - "| web_server Software Manager |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| Name | Type | Operating State | Health State | Port | Protocol |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", - "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", - "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", - "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", - "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", - "| NMAP | Application | RUNNING | GOOD | None | none |\n", - "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", - "| UserManager | Service | RUNNING | GOOD | None | none |\n", - "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", - "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", - "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n" - ] - } - ], + "outputs": [], "source": [ "blue_env.step(0)\n", "web_server.software_manager.show()" @@ -1788,26 +1120,9 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 3\n", - "root['NODES']['HOST0']['APPLICATIONS'][1]['operating_status']: 1 -> 0\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", - "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", - "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" - ] - } - ], + "outputs": [], "source": [ "display_obs_diffs(pre_blue_action_obs, post_blue_action_obs, blue_env.game.step_counter)" ] @@ -1821,20 +1136,9 @@ }, { "cell_type": "code", - "execution_count": 44, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "RequestResponse(status='failure', data={'Reason': 'Command sent to the C2 Beacon but no response was ever received.'})" - ] - }, - "execution_count": 44, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Attempting to install the C2 RansomwareScript\n", "ransomware_install_command = {\"commands\":[[\"software_manager\", \"application\", \"install\", \"RansomwareScript\"]],\n", @@ -1856,193 +1160,9 @@ }, { "cell_type": "code", - "execution_count": 45, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-03 16:04:28,041: Resetting environment, episode 2, avg. reward: 0.0\n", - "2025-02-03 16:04:28,045: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-03/16-04-20/agent_actions/episode_2.json\n" - ] - }, - { - "data": { - "text/plain": [ - "({'NODES': {'HOST0': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 1,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'HOST1': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 1,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'HOST2': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 0,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'HOST3': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 0,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'ROUTER0': {'ACL': {1: {'position': 0,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 2: {'position': 1,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 3: {'position': 2,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 4: {'position': 3,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 5: {'position': 4,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 6: {'position': 5,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 7: {'position': 6,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 8: {'position': 7,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 9: {'position': 8,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 10: {'position': 9,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0}},\n", - " 'PORTS': {1: {'operating_status': 1},\n", - " 2: {'operating_status': 1},\n", - " 3: {'operating_status': 2}},\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}}},\n", - " 'LINKS': {1: {'PROTOCOLS': {'ALL': 1}},\n", - " 2: {'PROTOCOLS': {'ALL': 1}},\n", - " 3: {'PROTOCOLS': {'ALL': 0}},\n", - " 4: {'PROTOCOLS': {'ALL': 1}},\n", - " 5: {'PROTOCOLS': {'ALL': 1}},\n", - " 6: {'PROTOCOLS': {'ALL': 1}},\n", - " 7: {'PROTOCOLS': {'ALL': 1}},\n", - " 8: {'PROTOCOLS': {'ALL': 1}},\n", - " 9: {'PROTOCOLS': {'ALL': 1}},\n", - " 10: {'PROTOCOLS': {'ALL': 0}}},\n", - " 'ICS': 0},\n", - " {})" - ] - }, - "execution_count": 45, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "blue_env.reset()" ] @@ -2085,17 +1205,9 @@ }, { "cell_type": "code", - "execution_count": 48, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "NodeOperatingState.SHUTTING_DOWN\n" - ] - } - ], + "outputs": [], "source": [ "web_server = blue_env.game.simulation.network.get_node_by_hostname(\"web_server\")\n", "print(web_server.operating_state)" @@ -2103,48 +1215,18 @@ }, { "cell_type": "code", - "execution_count": 49, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 2\n", - "root['NODES']['HOST0']['operating_status']: 1 -> 4\n", - "root['NODES']['HOST0']['APPLICATIONS'][1]['operating_status']: 1 -> 0\n", - "root['NODES']['HOST0']['NICS'][1]['nic_status']: 1 -> 0\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", - "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", - "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" - ] - } - ], + "outputs": [], "source": [ "display_obs_diffs(pre_blue_action_obs, post_blue_action_obs, blue_env.game.step_counter)" ] }, { "cell_type": "code", - "execution_count": 50, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "RequestResponse(status='failure', data={'Reason': 'Command sent to the C2 Beacon but no response was ever received.'})" - ] - }, - "execution_count": 50, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Attempting to install the C2 RansomwareScript\n", "ransomware_install_command = {\"commands\":[\"software_manager\", \"application\", \"install\", \"RansomwareScript\"],\n", @@ -2168,193 +1250,9 @@ }, { "cell_type": "code", - "execution_count": 51, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-03 16:04:28,560: Resetting environment, episode 3, avg. reward: 0.0\n", - "2025-02-03 16:04:28,564: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-03/16-04-20/agent_actions/episode_3.json\n" - ] - }, - { - "data": { - "text/plain": [ - "({'NODES': {'HOST0': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 1,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'HOST1': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 1,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'HOST2': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 0,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'HOST3': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 0,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'ROUTER0': {'ACL': {1: {'position': 0,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 2: {'position': 1,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 3: {'position': 2,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 4: {'position': 3,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 5: {'position': 4,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 6: {'position': 5,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 7: {'position': 6,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 8: {'position': 7,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 9: {'position': 8,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 10: {'position': 9,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0}},\n", - " 'PORTS': {1: {'operating_status': 1},\n", - " 2: {'operating_status': 1},\n", - " 3: {'operating_status': 2}},\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}}},\n", - " 'LINKS': {1: {'PROTOCOLS': {'ALL': 1}},\n", - " 2: {'PROTOCOLS': {'ALL': 1}},\n", - " 3: {'PROTOCOLS': {'ALL': 0}},\n", - " 4: {'PROTOCOLS': {'ALL': 1}},\n", - " 5: {'PROTOCOLS': {'ALL': 1}},\n", - " 6: {'PROTOCOLS': {'ALL': 1}},\n", - " 7: {'PROTOCOLS': {'ALL': 1}},\n", - " 8: {'PROTOCOLS': {'ALL': 1}},\n", - " 9: {'PROTOCOLS': {'ALL': 1}},\n", - " 10: {'PROTOCOLS': {'ALL': 0}}},\n", - " 'ICS': 0},\n", - " {})" - ] - }, - "execution_count": 51, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "blue_env.reset()" ] @@ -2397,41 +1295,9 @@ }, { "cell_type": "code", - "execution_count": 54, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+---------------------------------------------------------------+\n", - "| router_1 Network Interfaces |\n", - "+------+-------------------+-----------------+-------+----------+\n", - "| Port | MAC Address | Address | Speed | Status |\n", - "+------+-------------------+-----------------+-------+----------+\n", - "| 1 | dd:6e:95:d4:3f:74 | 192.168.1.1/24 | 100.0 | Enabled |\n", - "| 2 | 8b:79:07:fc:69:2c | 192.168.10.1/24 | 100.0 | Enabled |\n", - "| 3 | 1f:fd:c4:ae:7a:00 | 127.0.0.1/8 | 100.0 | Disabled |\n", - "| 4 | 7b:e3:bf:4b:76:e8 | 127.0.0.1/8 | 100.0 | Disabled |\n", - "| 5 | 4f:37:b0:6b:5d:44 | 127.0.0.1/8 | 100.0 | Disabled |\n", - "+------+-------------------+-----------------+-------+----------+\n", - "+------------------------------------------------------------------------------------------------------------------------+\n", - "| router_1 Access Control List |\n", - "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n", - "| Index | Action | Protocol | Src IP | Src Wildcard | Src Port | Dst IP | Dst Wildcard | Dst Port | Matched |\n", - "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n", - "| 1 | DENY | ANY | 192.168.10.21 | ANY | 80 | 192.168.1.12 | ANY | 80 | 0 |\n", - "| 18 | PERMIT | ANY | ANY | ANY | 5432 | ANY | ANY | 5432 | 0 |\n", - "| 19 | PERMIT | ANY | ANY | ANY | 53 | ANY | ANY | 53 | 0 |\n", - "| 20 | PERMIT | ANY | ANY | ANY | 21 | ANY | ANY | 21 | 0 |\n", - "| 21 | PERMIT | ANY | ANY | ANY | 80 | ANY | ANY | 80 | 4 |\n", - "| 22 | PERMIT | ANY | ANY | ANY | 219 | ANY | ANY | 219 | 10 |\n", - "| 23 | PERMIT | icmp | ANY | ANY | ANY | ANY | ANY | ANY | 0 |\n", - "| 24 | DENY | ANY | ANY | ANY | ANY | ANY | ANY | ANY | 0 |\n", - "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n" - ] - } - ], + "outputs": [], "source": [ "router_1: Router = blue_env.game.simulation.network.get_node_by_hostname(\"router_1\")\n", "router_1.show()\n", @@ -2447,20 +1313,9 @@ }, { "cell_type": "code", - "execution_count": 55, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "RequestResponse(status='failure', data={'Reason': 'Command sent to the C2 Beacon but no response was ever received.'})" - ] - }, - "execution_count": 55, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "blue_env.step(0)\n", "\n", @@ -2471,30 +1326,9 @@ }, { "cell_type": "code", - "execution_count": 56, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+------------------------------------------------------------------------------------------------------------------------+\n", - "| router_1 Access Control List |\n", - "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n", - "| Index | Action | Protocol | Src IP | Src Wildcard | Src Port | Dst IP | Dst Wildcard | Dst Port | Matched |\n", - "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n", - "| 1 | DENY | ANY | 192.168.10.21 | ANY | 80 | 192.168.1.12 | ANY | 80 | 2 |\n", - "| 18 | PERMIT | ANY | ANY | ANY | 5432 | ANY | ANY | 5432 | 0 |\n", - "| 19 | PERMIT | ANY | ANY | ANY | 53 | ANY | ANY | 53 | 0 |\n", - "| 20 | PERMIT | ANY | ANY | ANY | 21 | ANY | ANY | 21 | 0 |\n", - "| 21 | PERMIT | ANY | ANY | ANY | 80 | ANY | ANY | 80 | 4 |\n", - "| 22 | PERMIT | ANY | ANY | ANY | 219 | ANY | ANY | 219 | 10 |\n", - "| 23 | PERMIT | icmp | ANY | ANY | ANY | ANY | ANY | ANY | 0 |\n", - "| 24 | DENY | ANY | ANY | ANY | ANY | ANY | ANY | ANY | 0 |\n", - "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n" - ] - } - ], + "outputs": [], "source": [ "router_1.acl.show()" ] @@ -2508,58 +1342,18 @@ }, { "cell_type": "code", - "execution_count": 57, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+-------------------------------------------------------------------------------------+\n", - "| web_server Software Manager |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| Name | Type | Operating State | Health State | Port | Protocol |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", - "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", - "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", - "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", - "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", - "| NMAP | Application | RUNNING | GOOD | None | none |\n", - "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", - "| UserManager | Service | RUNNING | GOOD | None | none |\n", - "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", - "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", - "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", - "| C2Beacon | Application | RUNNING | GOOD | None | tcp |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n" - ] - } - ], + "outputs": [], "source": [ "web_server.software_manager.show()" ] }, { "cell_type": "code", - "execution_count": 58, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+----------------------------------------------------------------------------------+\n", - "| database_server File System |\n", - "+----------------------+---------+---------------+-----------------------+---------+\n", - "| File Path | Size | Health status | Visible health status | Deleted |\n", - "+----------------------+---------+---------------+-----------------------+---------+\n", - "| database/database.db | 4.77 MB | GOOD | NONE | False |\n", - "| root | 0 B | GOOD | NONE | False |\n", - "+----------------------+---------+---------------+-----------------------+---------+\n" - ] - } - ], + "outputs": [], "source": [ "database_server: Server = blue_env.game.simulation.network.get_node_by_hostname(\"database_server\")\n", "database_server.software_manager.file_system.show(full=True)" @@ -2567,33 +1361,9 @@ }, { "cell_type": "code", - "execution_count": 59, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 3\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", - "root['NODES']['ROUTER0']['ACL'][1]['permission']: 0 -> 2\n", - "root['NODES']['ROUTER0']['ACL'][1]['source_ip_id']: 0 -> 7\n", - "root['NODES']['ROUTER0']['ACL'][1]['source_wildcard_id']: 0 -> 1\n", - "root['NODES']['ROUTER0']['ACL'][1]['source_port_id']: 0 -> 2\n", - "root['NODES']['ROUTER0']['ACL'][1]['dest_ip_id']: 0 -> 3\n", - "root['NODES']['ROUTER0']['ACL'][1]['dest_wildcard_id']: 0 -> 1\n", - "root['NODES']['ROUTER0']['ACL'][1]['dest_port_id']: 0 -> 2\n", - "root['NODES']['ROUTER0']['ACL'][1]['protocol_id']: 0 -> 1\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", - "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", - "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" - ] - } - ], + "outputs": [], "source": [ "display_obs_diffs(pre_blue_action_obs, post_blue_action_obs, blue_env.game.step_counter)" ] @@ -2657,17 +1427,9 @@ }, { "cell_type": "code", - "execution_count": 60, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-03 16:04:29,610: PrimaiteGymEnv RNG seed = None\n" - ] - } - ], + "outputs": [], "source": [ "with open(data_manipulation_config_path()) as f:\n", " cfg = yaml.safe_load(f)\n", @@ -2716,30 +1478,9 @@ }, { "cell_type": "code", - "execution_count": 62, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", - "| C2Beacon Running Status |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| False | None | 0 | 5 | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "+-----------------------------------------------------------------------------------------------------+\n", - "| C2Server Running Status |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n", - "| False | None | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n" - ] - } - ], + "outputs": [], "source": [ "env.step(2) # Agent Action Equivalent to c2_beacon.configure(c2_server_ip_address=\"192.168.10.21\")\n", "env.step(3) # Agent action Equivalent to c2_beacon.establish()\n", @@ -2756,30 +1497,9 @@ }, { "cell_type": "code", - "execution_count": 63, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", - "| C2Beacon Running Status |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| False | None | 0 | 5 | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "+-----------------------------------------------------------------------------------------------------+\n", - "| C2Server Running Status |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n", - "| False | None | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n" - ] - } - ], + "outputs": [], "source": [ "env.step(9) # Equivalent of to c2_beacon.configure(c2_server_ip_address=\"192.168.10.22\")\n", "env.step(3)\n", @@ -2797,23 +1517,9 @@ }, { "cell_type": "code", - "execution_count": 64, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+-----------------------------------------------------------------------------------------------------+\n", - "| C2Server Running Status |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n", - "| False | None | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n" - ] - } - ], + "outputs": [], "source": [ "for i in range(6):\n", " env.step(0)\n", @@ -2836,17 +1542,9 @@ }, { "cell_type": "code", - "execution_count": 65, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-03 16:04:30,011: PrimaiteGymEnv RNG seed = None\n" - ] - } - ], + "outputs": [], "source": [ "with open(data_manipulation_config_path()) as f:\n", " cfg = yaml.safe_load(f)\n", @@ -2886,23 +1584,9 @@ }, { "cell_type": "code", - "execution_count": 67, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", - "| C2Beacon Running Status |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| True | 192.168.10.21 | 0 | 5 | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n" - ] - } - ], + "outputs": [], "source": [ "c2_beacon.configure(c2_server_ip_address=\"192.168.10.21\")\n", "c2_beacon.establish()\n", @@ -2920,72 +1604,9 @@ }, { "cell_type": "code", - "execution_count": 68, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 4\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 5\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 6\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 7\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 8\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 9\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 10\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 11\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 12\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 13\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" - ] - } - ], + "outputs": [], "source": [ "for i in range(10):\n", " keep_alive_obs, _, _, _, _ = blue_config_env.step(0)\n", @@ -3001,23 +1622,9 @@ }, { "cell_type": "code", - "execution_count": 69, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", - "| C2Beacon Running Status |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| True | 192.168.10.21 | 0 | 1 | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n" - ] - } - ], + "outputs": [], "source": [ "c2_beacon.configure(c2_server_ip_address=\"192.168.10.21\", keep_alive_frequency=1)\n", "c2_beacon.establish()\n", @@ -3033,40 +1640,9 @@ }, { "cell_type": "code", - "execution_count": 70, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 14\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 15\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" - ] - } - ], + "outputs": [], "source": [ "# Comparing the OBS of the default frequency to a timestep frequency of 1\n", "for i in range(2):\n", @@ -3085,52 +1661,9 @@ }, { "cell_type": "code", - "execution_count": 71, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 16\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 17\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 18\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 19\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 20\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 21\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 22\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" - ] - } - ], + "outputs": [], "source": [ "c2_beacon.configure(c2_server_ip_address=\"192.168.10.21\", keep_alive_frequency=7)\n", "\n", @@ -3167,18 +1700,9 @@ }, { "cell_type": "code", - "execution_count": 72, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-03 16:04:30,864: Resetting environment, episode 0, avg. reward: 0.0\n", - "2025-02-03 16:04:30,867: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-03/16-04-20/agent_actions/episode_0.json\n" - ] - } - ], + "outputs": [], "source": [ "blue_config_env.reset()\n", "\n", @@ -3200,28 +1724,9 @@ }, { "cell_type": "code", - "execution_count": 73, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 5\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" - ] - } - ], + "outputs": [], "source": [ "# Capturing default C2 Traffic\n", "for i in range(3):\n", @@ -3239,23 +1744,9 @@ }, { "cell_type": "code", - "execution_count": 74, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", - "| C2Beacon Running Status |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| True | 192.168.10.21 | 0 | 5 | udp | 53 |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n" - ] - } - ], + "outputs": [], "source": [ "from primaite.utils.validation.ip_protocol import PROTOCOL_LOOKUP\n", "from primaite.utils.validation.port import PORT_LOOKUP\n", @@ -3268,28 +1759,9 @@ }, { "cell_type": "code", - "execution_count": 75, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 10\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 1 -> 0\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 1 -> 0\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['udp'][53]['inbound']: 0 -> 1\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['udp'][53]['outbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 1 -> 0\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 1 -> 0\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['udp'][53]['inbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['udp'][53]['outbound']: 0 -> 1\n" - ] - } - ], + "outputs": [], "source": [ "# Capturing UDP C2 Traffic\n", "for i in range(5):\n", From f7c6ee3df43bc1d46af90dbac0f321a1e911e524 Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Tue, 4 Feb 2025 15:45:19 +0000 Subject: [PATCH 73/95] #3075: C2C-E2E-Demo wildcard_list changes. --- ...ommand-and-Control-E2E-Demonstration.ipynb | 1822 +++++++++++++++-- 1 file changed, 1645 insertions(+), 177 deletions(-) diff --git a/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb b/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb index c2a87e45..b8d1423f 100644 --- a/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb +++ b/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb @@ -13,9 +13,24 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-02-04 15:37:30,830: Performing the PrimAITE first-time setup...\n", + "2025-02-04 15:37:30,830: Building the PrimAITE app directories...\n", + "2025-02-04 15:37:30,830: Building primaite_config.yaml...\n", + "2025-02-04 15:37:30,830: Rebuilding the demo notebooks...\n", + "/home/nick/primaite/4.0.0a1-dev/notebooks/example_notebooks/Command-and-Control-E2E-Demonstration.ipynb\n", + "2025-02-04 15:37:30,832: Reset example notebook: /home/nick/primaite/4.0.0a1-dev/notebooks/example_notebooks/Command-and-Control-E2E-Demonstration.ipynb\n", + "2025-02-04 15:37:30,838: Rebuilding the example notebooks...\n", + "2025-02-04 15:37:30,841: PrimAITE setup complete!\n" + ] + } + ], "source": [ "!primaite setup" ] @@ -62,24 +77,6 @@ " type: ProxyAgent\n", "\n", " action_space:\n", - " # options:\n", - " # nodes:\n", - " # - node_name: web_server\n", - " # applications:\n", - " # - application_name: C2Beacon\n", - " # - node_name: client_1\n", - " # applications:\n", - " # - application_name: C2Server\n", - " # max_folders_per_node: 1\n", - " # max_files_per_folder: 1\n", - " # max_services_per_node: 2\n", - " # max_nics_per_node: 8\n", - " # max_acl_rules: 10\n", - " # ip_list:\n", - " # - 192.168.1.21\n", - " # - 192.168.1.14\n", - " # wildcard_list:\n", - " # - 0.0.0.1\n", " action_map:\n", " 0:\n", " action: do_nothing\n", @@ -93,11 +90,7 @@ " action: configure_c2_beacon\n", " options:\n", " node_name: web_server\n", - " # config:\n", " c2_server_ip_address: 192.168.10.21\n", - " # keep_alive_frequency: 10\n", - " # masquerade_protocol: TCP\n", - " # masquerade_port: DNS\n", " 3:\n", " action: node_application_execute\n", " options:\n", @@ -108,7 +101,6 @@ " options:\n", " node_name: client_1\n", " ip_address:\n", - " # account:\n", " username: admin\n", " password: admin\n", " commands:\n", @@ -121,7 +113,6 @@ " action: c2_server_ransomware_configure\n", " options:\n", " node_name: client_1\n", - " # config:\n", " server_ip_address: 192.168.1.14\n", " payload: ENCRYPT\n", " 6:\n", @@ -132,7 +123,6 @@ " target_folder_name: \"database\"\n", " exfiltration_folder_name: \"spoils\"\n", " target_ip_address: 192.168.1.14\n", - " # account:\n", " username: admin\n", " password: admin\n", "\n", @@ -144,20 +134,12 @@ " action: configure_c2_beacon\n", " options:\n", " node_name: web_server\n", - " # config:\n", " c2_server_ip_address: 192.168.10.21\n", - " # keep_alive_frequency: 10\n", - " # masquerade_protocol: TCP\n", - " # masquerade_port: DNS\n", " 9:\n", " action: configure_c2_beacon\n", " options:\n", " node_name: web_server\n", - " # config:\n", " c2_server_ip_address: 192.168.10.22\n", - " # keep_alive_frequency: 10\n", - " # masquerade_protocol: TCP\n", - " # masquerade_port: DNS\n", "\n", " reward_function:\n", " reward_components:\n", @@ -168,9 +150,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-04 15:37:35,293: PrimaiteGymEnv RNG seed = None\n" + ] + } + ], "source": [ "with open(data_manipulation_config_path()) as f:\n", " cfg = yaml.safe_load(f)\n", @@ -196,9 +186,35 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+--------------------------------------------------------------------------------------+\n", + "| client_1 Software Manager |\n", + "+---------------------+-------------+-----------------+--------------+------+----------+\n", + "| Name | Type | Operating State | Health State | Port | Protocol |\n", + "+---------------------+-------------+-----------------+--------------+------+----------+\n", + "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", + "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", + "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", + "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", + "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", + "| NMAP | Application | RUNNING | GOOD | None | none |\n", + "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", + "| UserManager | Service | RUNNING | GOOD | None | none |\n", + "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", + "| FTPClient | Service | RUNNING | GOOD | 21 | tcp |\n", + "| DataManipulationBot | Application | RUNNING | GOOD | None | none |\n", + "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", + "| C2Server | Application | RUNNING | GOOD | None | tcp |\n", + "+---------------------+-------------+-----------------+--------------+------+----------+\n" + ] + } + ], "source": [ "client_1: Computer = env.game.simulation.network.get_node_by_hostname(\"client_1\")\n", "client_1.software_manager.install(C2Server)\n", @@ -248,9 +264,34 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+-------------------------------------------------------------------------------------+\n", + "| web_server Software Manager |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| Name | Type | Operating State | Health State | Port | Protocol |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", + "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", + "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", + "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", + "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", + "| NMAP | Application | RUNNING | GOOD | None | none |\n", + "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", + "| UserManager | Service | RUNNING | GOOD | None | none |\n", + "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", + "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", + "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", + "| C2Beacon | Application | INSTALLING | UNUSED | None | tcp |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n" + ] + } + ], "source": [ "env.step(1)\n", "web_server: Computer = env.game.simulation.network.get_node_by_hostname(\"web_server\")\n", @@ -290,9 +331,41 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+-------------------------------------------------------------------------------------+\n", + "| web_server Software Manager |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| Name | Type | Operating State | Health State | Port | Protocol |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", + "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", + "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", + "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", + "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", + "| NMAP | Application | RUNNING | GOOD | None | none |\n", + "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", + "| UserManager | Service | RUNNING | GOOD | None | none |\n", + "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", + "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", + "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", + "| C2Beacon | Application | RUNNING | GOOD | None | tcp |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", + "| C2Beacon Running Status |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| False | 192.168.10.21 | 0 | 5 | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n" + ] + } + ], "source": [ "env.step(2)\n", "c2_beacon: C2Beacon = web_server.software_manager.software[\"C2Beacon\"]\n", @@ -331,18 +404,54 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "(0,\n", + " 0.0,\n", + " False,\n", + " False,\n", + " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=2, action='node_application_execute', parameters={'node_name': 'web_server', 'application_name': 'C2Beacon'}, request=['network', 'node', 'web_server', 'application', 'C2Beacon', 'execute'], response=RequestResponse(status='success', data={}), reward=0.0, reward_info={})}})" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "env.step(3)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", + "| C2Beacon Running Status |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| True | 192.168.10.21 | 1 | 5 | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "+-----------------------------------------------------------------------------------------------------+\n", + "| C2Server Running Status |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n", + "| True | 192.168.1.12 | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n" + ] + } + ], "source": [ "c2_beacon.show()\n", "c2_server.show()" @@ -405,18 +514,59 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "(0,\n", + " 0.0,\n", + " False,\n", + " False,\n", + " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=3, action='c2_server_terminal_command', parameters={'node_name': 'client_1', 'ip_address': None, 'username': 'admin', 'password': 'admin', 'commands': [['software_manager', 'application', 'install', 'RansomwareScript']]}, request=['network', 'node', 'client_1', 'application', 'C2Server', 'terminal_command', {'commands': [['software_manager', 'application', 'install', 'RansomwareScript']], 'ip_address': None, 'username': 'admin', 'password': 'admin'}], response=RequestResponse(status='success', data={0: RequestResponse(status='success', data={})}), reward=0.0, reward_info={})}})" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "env.step(4)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+--------------------------------------------------------------------------------------+\n", + "| client_1 Software Manager |\n", + "+---------------------+-------------+-----------------+--------------+------+----------+\n", + "| Name | Type | Operating State | Health State | Port | Protocol |\n", + "+---------------------+-------------+-----------------+--------------+------+----------+\n", + "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", + "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", + "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", + "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", + "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", + "| NMAP | Application | RUNNING | GOOD | None | none |\n", + "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", + "| UserManager | Service | RUNNING | GOOD | None | none |\n", + "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", + "| FTPClient | Service | RUNNING | GOOD | 21 | tcp |\n", + "| DataManipulationBot | Application | RUNNING | GOOD | None | none |\n", + "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", + "| C2Server | Application | RUNNING | GOOD | None | tcp |\n", + "+---------------------+-------------+-----------------+--------------+------+----------+\n" + ] + } + ], "source": [ "client_1.software_manager.show()" ] @@ -455,18 +605,66 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "(0,\n", + " 0.0,\n", + " False,\n", + " False,\n", + " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=4, action='c2_server_ransomware_configure', parameters={'node_name': 'client_1', 'server_ip_address': '192.168.1.14', 'payload': 'ENCRYPT'}, request=['network', 'node', 'client_1', 'application', 'C2Server', 'ransomware_configure', {'server_ip_address': '192.168.1.14', 'server_password': None, 'payload': 'ENCRYPT'}], response=RequestResponse(status='success', data={}), reward=0.0, reward_info={})}})" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "env.step(5)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+-------------------------------------------------------------------------------------+\n", + "| web_server Software Manager |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| Name | Type | Operating State | Health State | Port | Protocol |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", + "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", + "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", + "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", + "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", + "| NMAP | Application | RUNNING | GOOD | None | none |\n", + "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", + "| UserManager | Service | RUNNING | GOOD | None | none |\n", + "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", + "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", + "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", + "| C2Beacon | Application | RUNNING | GOOD | None | tcp |\n", + "| RansomwareScript | Application | RUNNING | GOOD | None | none |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "+------------------------------------+\n", + "| RansomwareScript Running Status |\n", + "+--------------------------+---------+\n", + "| Target Server IP Address | Payload |\n", + "+--------------------------+---------+\n", + "| 192.168.1.14 | ENCRYPT |\n", + "+--------------------------+---------+\n" + ] + } + ], "source": [ "ransomware_script: RansomwareScript = web_server.software_manager.software[\"RansomwareScript\"]\n", "web_server.software_manager.show()\n", @@ -512,18 +710,48 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "(0,\n", + " 0.0,\n", + " False,\n", + " False,\n", + " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=5, action='c2_server_data_exfiltrate', parameters={'node_name': 'client_1', 'target_file_name': 'database.db', 'target_folder_name': 'database', 'exfiltration_folder_name': 'spoils', 'target_ip_address': '192.168.1.14', 'username': 'admin', 'password': 'admin'}, request=['network', 'node', 'client_1', 'application', 'C2Server', 'exfiltrate', {'target_file_name': 'database.db', 'target_folder_name': 'database', 'exfiltration_folder_name': 'spoils', 'target_ip_address': '192.168.1.14', 'username': 'admin', 'password': 'admin'}], response=RequestResponse(status='success', data={}), reward=0.0, reward_info={})}})" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "env.step(6)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+--------------------------------------------------------------------------------+\n", + "| client_1 File System |\n", + "+--------------------+---------+---------------+-----------------------+---------+\n", + "| File Path | Size | Health status | Visible health status | Deleted |\n", + "+--------------------+---------+---------------+-----------------------+---------+\n", + "| root | 0 B | GOOD | NONE | False |\n", + "| spoils/database.db | 4.77 MB | GOOD | NONE | False |\n", + "+--------------------+---------+---------------+-----------------------+---------+\n" + ] + } + ], "source": [ "client_1: Computer = env.game.simulation.network.get_node_by_hostname(\"client_1\")\n", "client_1.software_manager.file_system.show(full=True)" @@ -531,9 +759,25 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+---------------------------------------------------------------------------------+\n", + "| web_server File System |\n", + "+---------------------+---------+---------------+-----------------------+---------+\n", + "| File Path | Size | Health status | Visible health status | Deleted |\n", + "+---------------------+---------+---------------+-----------------------+---------+\n", + "| primaite/index.html | 15.0 KB | GOOD | NONE | False |\n", + "| root | 0 B | GOOD | NONE | False |\n", + "| spoils/database.db | 4.77 MB | GOOD | NONE | False |\n", + "+---------------------+---------+---------------+-----------------------+---------+\n" + ] + } + ], "source": [ "web_server: Computer = env.game.simulation.network.get_node_by_hostname(\"web_server\")\n", "web_server.software_manager.file_system.show(full=True)" @@ -570,18 +814,48 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "(0,\n", + " 0.0,\n", + " False,\n", + " False,\n", + " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=6, action='c2_server_ransomware_launch', parameters={'node_name': 'client_1'}, request=['network', 'node', 'client_1', 'application', 'C2Server', 'ransomware_launch'], response=RequestResponse(status='success', data={}), reward=0.0, reward_info={})}})" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "env.step(7)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----------------------------------------------------------------------------------+\n", + "| database_server File System |\n", + "+----------------------+---------+---------------+-----------------------+---------+\n", + "| File Path | Size | Health status | Visible health status | Deleted |\n", + "+----------------------+---------+---------------+-----------------------+---------+\n", + "| database/database.db | 4.77 MB | CORRUPT | NONE | False |\n", + "| root | 0 B | GOOD | NONE | False |\n", + "+----------------------+---------+---------------+-----------------------+---------+\n" + ] + } + ], "source": [ "database_server: Server = env.game.simulation.network.get_node_by_hostname(\"database_server\")\n", "database_server.software_manager.file_system.show(full=True)" @@ -600,7 +874,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "metadata": {}, "outputs": [], "source": [ @@ -713,35 +987,9 @@ " src_port: HTTP\n", " dst_port: HTTP\n", " protocol_name: ALL\n", - " src_wildcard: NONE\n", - " dst_wildcard: NONE\n", + " src_wildcard: 0.0.0.1\n", + " dst_wildcard: 0.0.0.1\n", "\n", - "\n", - " # options:\n", - " # # nodes:\n", - " # node_name: web_server\n", - " # # applications:\n", - " # application_name: C2Beacon\n", - "\n", - " # node_name: database_server\n", - " # folders:\n", - " # - folder_name: database\n", - " # files:\n", - " # - file_name: database.db\n", - " # services:\n", - " # - service_name: DatabaseService\n", - " # node_name: router_1\n", - "\n", - " # max_folders_per_node: 2\n", - " # max_files_per_folder: 2\n", - " # max_services_per_node: 2\n", - " # max_nics_per_node: 8\n", - " # max_acl_rules: 10\n", - " # ip_list:\n", - " # - 192.168.10.21\n", - " # - 192.168.1.12\n", - " # wildcard_list:\n", - " # - 0.0.0.1\n", " reward_function:\n", " reward_components:\n", " - type: DUMMY\n", @@ -754,9 +1002,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-04 15:37:36,689: PrimaiteGymEnv RNG seed = None\n" + ] + } + ], "source": [ "with open(data_manipulation_config_path()) as f:\n", " cfg = yaml.safe_load(f)\n", @@ -813,9 +1069,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 22, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-04 15:37:37,079: Resetting environment, episode 0, avg. reward: 0.0\n", + "2025-02-04 15:37:37,082: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-04/15-37-31/agent_actions/episode_0.json\n" + ] + } + ], "source": [ "# Resetting the environment and capturing the default observation space.\n", "blue_env.reset()\n", @@ -824,9 +1089,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 23, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Setting up the C2 Suite via the simulation API.\n", "\n", @@ -857,9 +1133,26 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 25, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 2\n", + "root['NODES']['HOST0']['APPLICATIONS'][1]['operating_status']: 0 -> 1\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", + "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", + "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" + ] + } + ], "source": [ "display_obs_diffs(default_obs, c2_configuration_obs, blue_env.game.step_counter)" ] @@ -879,9 +1172,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 26, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "RequestResponse(status='success', data={0: RequestResponse(status='success', data={})})" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Installing RansomwareScript via C2 Terminal Commands\n", "ransomware_install_command = {\"commands\":[[\"software_manager\", \"application\", \"install\", \"RansomwareScript\"]],\n", @@ -892,9 +1196,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 27, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "RequestResponse(status='success', data={})" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Configuring the RansomwareScript\n", "ransomware_config = {\"server_ip_address\": \"192.168.1.14\", \"payload\": \"ENCRYPT\"}\n", @@ -913,9 +1228,28 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 29, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 7\n", + "root['NODES']['HOST0']['APPLICATIONS'][1]['operating_status']: 0 -> 1\n", + "root['NODES']['HOST0']['APPLICATIONS'][2]['operating_status']: 0 -> 3\n", + "root['NODES']['HOST0']['users']['local_login']: 0 -> 1\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", + "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", + "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" + ] + } + ], "source": [ "display_obs_diffs(default_obs, c2_ransomware_obs, env.game.step_counter)" ] @@ -947,9 +1281,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 31, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "RequestResponse(status='success', data={})" + ] + }, + "execution_count": 31, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "c2_server.send_command(given_command=C2Command.DATA_EXFILTRATION, command_options=exfil_options)" ] @@ -965,9 +1310,22 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 33, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 7\n", + "root['NODES']['HOST0']['APPLICATIONS'][2]['operating_status']: 3 -> 1\n", + "root['NODES']['HOST1']['users']['remote_sessions']: 0 -> 1\n" + ] + } + ], "source": [ "display_obs_diffs(c2_ransomware_obs, c2_exfil_obs, env.game.step_counter)" ] @@ -983,9 +1341,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 34, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "RequestResponse(status='success', data={})" + ] + }, + "execution_count": 34, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Configuring the RansomwareScript\n", "ransomware_config = {\"server_ip_address\": \"192.168.1.14\", \"payload\": \"ENCRYPT\"}\n", @@ -994,9 +1363,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 35, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "RequestResponse(status='success', data={})" + ] + }, + "execution_count": 35, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Waiting for the ransomware to finish installing and then launching the RansomwareScript.\n", "blue_env.step(0)\n", @@ -1015,9 +1395,30 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 37, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 6\n", + "root['NODES']['HOST0']['APPLICATIONS'][2]['operating_status']: 3 -> 1\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['NODES']['HOST1']['users']['remote_sessions']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" + ] + } + ], "source": [ "display_obs_diffs(c2_ransomware_obs, c2_final_obs, blue_env.game.step_counter)" ] @@ -1065,9 +1466,193 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 39, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-04 15:37:38,317: Resetting environment, episode 1, avg. reward: 0.0\n", + "2025-02-04 15:37:38,321: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-04/15-37-31/agent_actions/episode_1.json\n" + ] + }, + { + "data": { + "text/plain": [ + "({'NODES': {'HOST0': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 1,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'HOST1': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 1,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'HOST2': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 0,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'HOST3': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 0,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'ROUTER0': {'ACL': {1: {'position': 0,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 2: {'position': 1,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 3: {'position': 2,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 4: {'position': 3,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 5: {'position': 4,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 6: {'position': 5,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 7: {'position': 6,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 8: {'position': 7,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 9: {'position': 8,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 10: {'position': 9,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0}},\n", + " 'PORTS': {1: {'operating_status': 1},\n", + " 2: {'operating_status': 1},\n", + " 3: {'operating_status': 2}},\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}}},\n", + " 'LINKS': {1: {'PROTOCOLS': {'ALL': 1}},\n", + " 2: {'PROTOCOLS': {'ALL': 1}},\n", + " 3: {'PROTOCOLS': {'ALL': 0}},\n", + " 4: {'PROTOCOLS': {'ALL': 1}},\n", + " 5: {'PROTOCOLS': {'ALL': 1}},\n", + " 6: {'PROTOCOLS': {'ALL': 1}},\n", + " 7: {'PROTOCOLS': {'ALL': 1}},\n", + " 8: {'PROTOCOLS': {'ALL': 1}},\n", + " 9: {'PROTOCOLS': {'ALL': 1}},\n", + " 10: {'PROTOCOLS': {'ALL': 0}}},\n", + " 'ICS': 0},\n", + " {})" + ] + }, + "execution_count": 39, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "blue_env.reset()" ] @@ -1110,9 +1695,33 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 42, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+-------------------------------------------------------------------------------------+\n", + "| web_server Software Manager |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| Name | Type | Operating State | Health State | Port | Protocol |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", + "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", + "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", + "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", + "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", + "| NMAP | Application | RUNNING | GOOD | None | none |\n", + "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", + "| UserManager | Service | RUNNING | GOOD | None | none |\n", + "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", + "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", + "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n" + ] + } + ], "source": [ "blue_env.step(0)\n", "web_server.software_manager.show()" @@ -1120,9 +1729,26 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 43, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 3\n", + "root['NODES']['HOST0']['APPLICATIONS'][1]['operating_status']: 1 -> 0\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", + "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", + "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" + ] + } + ], "source": [ "display_obs_diffs(pre_blue_action_obs, post_blue_action_obs, blue_env.game.step_counter)" ] @@ -1136,9 +1762,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 44, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "RequestResponse(status='failure', data={'Reason': 'Command sent to the C2 Beacon but no response was ever received.'})" + ] + }, + "execution_count": 44, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Attempting to install the C2 RansomwareScript\n", "ransomware_install_command = {\"commands\":[[\"software_manager\", \"application\", \"install\", \"RansomwareScript\"]],\n", @@ -1160,9 +1797,193 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 45, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-04 15:37:38,778: Resetting environment, episode 2, avg. reward: 0.0\n", + "2025-02-04 15:37:38,781: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-04/15-37-31/agent_actions/episode_2.json\n" + ] + }, + { + "data": { + "text/plain": [ + "({'NODES': {'HOST0': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 1,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'HOST1': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 1,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'HOST2': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 0,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'HOST3': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 0,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'ROUTER0': {'ACL': {1: {'position': 0,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 2: {'position': 1,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 3: {'position': 2,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 4: {'position': 3,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 5: {'position': 4,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 6: {'position': 5,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 7: {'position': 6,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 8: {'position': 7,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 9: {'position': 8,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 10: {'position': 9,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0}},\n", + " 'PORTS': {1: {'operating_status': 1},\n", + " 2: {'operating_status': 1},\n", + " 3: {'operating_status': 2}},\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}}},\n", + " 'LINKS': {1: {'PROTOCOLS': {'ALL': 1}},\n", + " 2: {'PROTOCOLS': {'ALL': 1}},\n", + " 3: {'PROTOCOLS': {'ALL': 0}},\n", + " 4: {'PROTOCOLS': {'ALL': 1}},\n", + " 5: {'PROTOCOLS': {'ALL': 1}},\n", + " 6: {'PROTOCOLS': {'ALL': 1}},\n", + " 7: {'PROTOCOLS': {'ALL': 1}},\n", + " 8: {'PROTOCOLS': {'ALL': 1}},\n", + " 9: {'PROTOCOLS': {'ALL': 1}},\n", + " 10: {'PROTOCOLS': {'ALL': 0}}},\n", + " 'ICS': 0},\n", + " {})" + ] + }, + "execution_count": 45, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "blue_env.reset()" ] @@ -1205,9 +2026,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 48, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "NodeOperatingState.SHUTTING_DOWN\n" + ] + } + ], "source": [ "web_server = blue_env.game.simulation.network.get_node_by_hostname(\"web_server\")\n", "print(web_server.operating_state)" @@ -1215,18 +2044,48 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 49, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 2\n", + "root['NODES']['HOST0']['operating_status']: 1 -> 4\n", + "root['NODES']['HOST0']['APPLICATIONS'][1]['operating_status']: 1 -> 0\n", + "root['NODES']['HOST0']['NICS'][1]['nic_status']: 1 -> 0\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", + "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", + "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" + ] + } + ], "source": [ "display_obs_diffs(pre_blue_action_obs, post_blue_action_obs, blue_env.game.step_counter)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 50, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "RequestResponse(status='failure', data={'Reason': 'Command sent to the C2 Beacon but no response was ever received.'})" + ] + }, + "execution_count": 50, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Attempting to install the C2 RansomwareScript\n", "ransomware_install_command = {\"commands\":[\"software_manager\", \"application\", \"install\", \"RansomwareScript\"],\n", @@ -1250,9 +2109,193 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 51, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-04 15:37:39,257: Resetting environment, episode 3, avg. reward: 0.0\n", + "2025-02-04 15:37:39,260: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-04/15-37-31/agent_actions/episode_3.json\n" + ] + }, + { + "data": { + "text/plain": [ + "({'NODES': {'HOST0': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 1,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'HOST1': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 1,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'HOST2': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 0,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'HOST3': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 0,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'ROUTER0': {'ACL': {1: {'position': 0,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 2: {'position': 1,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 3: {'position': 2,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 4: {'position': 3,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 5: {'position': 4,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 6: {'position': 5,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 7: {'position': 6,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 8: {'position': 7,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 9: {'position': 8,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 10: {'position': 9,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0}},\n", + " 'PORTS': {1: {'operating_status': 1},\n", + " 2: {'operating_status': 1},\n", + " 3: {'operating_status': 2}},\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}}},\n", + " 'LINKS': {1: {'PROTOCOLS': {'ALL': 1}},\n", + " 2: {'PROTOCOLS': {'ALL': 1}},\n", + " 3: {'PROTOCOLS': {'ALL': 0}},\n", + " 4: {'PROTOCOLS': {'ALL': 1}},\n", + " 5: {'PROTOCOLS': {'ALL': 1}},\n", + " 6: {'PROTOCOLS': {'ALL': 1}},\n", + " 7: {'PROTOCOLS': {'ALL': 1}},\n", + " 8: {'PROTOCOLS': {'ALL': 1}},\n", + " 9: {'PROTOCOLS': {'ALL': 1}},\n", + " 10: {'PROTOCOLS': {'ALL': 0}}},\n", + " 'ICS': 0},\n", + " {})" + ] + }, + "execution_count": 51, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "blue_env.reset()" ] @@ -1295,12 +2338,43 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 54, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+---------------------------------------------------------------+\n", + "| router_1 Network Interfaces |\n", + "+------+-------------------+-----------------+-------+----------+\n", + "| Port | MAC Address | Address | Speed | Status |\n", + "+------+-------------------+-----------------+-------+----------+\n", + "| 1 | dd:bc:17:75:a2:c4 | 192.168.1.1/24 | 100.0 | Enabled |\n", + "| 2 | 00:f2:f5:65:51:75 | 192.168.10.1/24 | 100.0 | Enabled |\n", + "| 3 | de:7d:ac:0e:58:e1 | 127.0.0.1/8 | 100.0 | Disabled |\n", + "| 4 | 8a:b4:55:12:c7:9e | 127.0.0.1/8 | 100.0 | Disabled |\n", + "| 5 | a5:ef:c6:53:97:c4 | 127.0.0.1/8 | 100.0 | Disabled |\n", + "+------+-------------------+-----------------+-------+----------+\n", + "+------------------------------------------------------------------------------------------------------------------------+\n", + "| router_1 Access Control List |\n", + "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n", + "| Index | Action | Protocol | Src IP | Src Wildcard | Src Port | Dst IP | Dst Wildcard | Dst Port | Matched |\n", + "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n", + "| 1 | DENY | ANY | 192.168.10.21 | 0.0.0.1 | 80 | 192.168.1.12 | 0.0.0.1 | 80 | 0 |\n", + "| 18 | PERMIT | ANY | ANY | ANY | 5432 | ANY | ANY | 5432 | 0 |\n", + "| 19 | PERMIT | ANY | ANY | ANY | 53 | ANY | ANY | 53 | 0 |\n", + "| 20 | PERMIT | ANY | ANY | ANY | 21 | ANY | ANY | 21 | 0 |\n", + "| 21 | PERMIT | ANY | ANY | ANY | 80 | ANY | ANY | 80 | 4 |\n", + "| 22 | PERMIT | ANY | ANY | ANY | 219 | ANY | ANY | 219 | 10 |\n", + "| 23 | PERMIT | icmp | ANY | ANY | ANY | ANY | ANY | ANY | 0 |\n", + "| 24 | DENY | ANY | ANY | ANY | ANY | ANY | ANY | ANY | 0 |\n", + "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n" + ] + } + ], "source": [ "router_1: Router = blue_env.game.simulation.network.get_node_by_hostname(\"router_1\")\n", - "router_1.show()\n", "router_1.acl.show()" ] }, @@ -1313,9 +2387,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 55, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "RequestResponse(status='failure', data={'Reason': 'Command sent to the C2 Beacon but no response was ever received.'})" + ] + }, + "execution_count": 55, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "blue_env.step(0)\n", "\n", @@ -1326,9 +2411,30 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 56, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+------------------------------------------------------------------------------------------------------------------------+\n", + "| router_1 Access Control List |\n", + "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n", + "| Index | Action | Protocol | Src IP | Src Wildcard | Src Port | Dst IP | Dst Wildcard | Dst Port | Matched |\n", + "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n", + "| 1 | DENY | ANY | 192.168.10.21 | 0.0.0.1 | 80 | 192.168.1.12 | 0.0.0.1 | 80 | 2 |\n", + "| 18 | PERMIT | ANY | ANY | ANY | 5432 | ANY | ANY | 5432 | 0 |\n", + "| 19 | PERMIT | ANY | ANY | ANY | 53 | ANY | ANY | 53 | 0 |\n", + "| 20 | PERMIT | ANY | ANY | ANY | 21 | ANY | ANY | 21 | 0 |\n", + "| 21 | PERMIT | ANY | ANY | ANY | 80 | ANY | ANY | 80 | 4 |\n", + "| 22 | PERMIT | ANY | ANY | ANY | 219 | ANY | ANY | 219 | 10 |\n", + "| 23 | PERMIT | icmp | ANY | ANY | ANY | ANY | ANY | ANY | 0 |\n", + "| 24 | DENY | ANY | ANY | ANY | ANY | ANY | ANY | ANY | 0 |\n", + "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n" + ] + } + ], "source": [ "router_1.acl.show()" ] @@ -1342,18 +2448,58 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 57, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+-------------------------------------------------------------------------------------+\n", + "| web_server Software Manager |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| Name | Type | Operating State | Health State | Port | Protocol |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", + "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", + "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", + "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", + "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", + "| NMAP | Application | RUNNING | GOOD | None | none |\n", + "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", + "| UserManager | Service | RUNNING | GOOD | None | none |\n", + "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", + "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", + "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", + "| C2Beacon | Application | RUNNING | GOOD | None | tcp |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n" + ] + } + ], "source": [ "web_server.software_manager.show()" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 58, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----------------------------------------------------------------------------------+\n", + "| database_server File System |\n", + "+----------------------+---------+---------------+-----------------------+---------+\n", + "| File Path | Size | Health status | Visible health status | Deleted |\n", + "+----------------------+---------+---------------+-----------------------+---------+\n", + "| database/database.db | 4.77 MB | GOOD | NONE | False |\n", + "| root | 0 B | GOOD | NONE | False |\n", + "+----------------------+---------+---------------+-----------------------+---------+\n" + ] + } + ], "source": [ "database_server: Server = blue_env.game.simulation.network.get_node_by_hostname(\"database_server\")\n", "database_server.software_manager.file_system.show(full=True)" @@ -1361,9 +2507,33 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 59, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 3\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", + "root['NODES']['ROUTER0']['ACL'][1]['permission']: 0 -> 2\n", + "root['NODES']['ROUTER0']['ACL'][1]['source_ip_id']: 0 -> 7\n", + "root['NODES']['ROUTER0']['ACL'][1]['source_wildcard_id']: 0 -> 2\n", + "root['NODES']['ROUTER0']['ACL'][1]['source_port_id']: 0 -> 2\n", + "root['NODES']['ROUTER0']['ACL'][1]['dest_ip_id']: 0 -> 3\n", + "root['NODES']['ROUTER0']['ACL'][1]['dest_wildcard_id']: 0 -> 2\n", + "root['NODES']['ROUTER0']['ACL'][1]['dest_port_id']: 0 -> 2\n", + "root['NODES']['ROUTER0']['ACL'][1]['protocol_id']: 0 -> 1\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", + "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", + "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" + ] + } + ], "source": [ "display_obs_diffs(pre_blue_action_obs, post_blue_action_obs, blue_env.game.step_counter)" ] @@ -1427,9 +2597,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 60, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-04 15:37:40,175: PrimaiteGymEnv RNG seed = None\n" + ] + } + ], "source": [ "with open(data_manipulation_config_path()) as f:\n", " cfg = yaml.safe_load(f)\n", @@ -1478,9 +2656,30 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 62, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", + "| C2Beacon Running Status |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| False | None | 0 | 5 | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "+-----------------------------------------------------------------------------------------------------+\n", + "| C2Server Running Status |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n", + "| False | None | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n" + ] + } + ], "source": [ "env.step(2) # Agent Action Equivalent to c2_beacon.configure(c2_server_ip_address=\"192.168.10.21\")\n", "env.step(3) # Agent action Equivalent to c2_beacon.establish()\n", @@ -1497,9 +2696,30 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 63, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", + "| C2Beacon Running Status |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| False | None | 0 | 5 | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "+-----------------------------------------------------------------------------------------------------+\n", + "| C2Server Running Status |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n", + "| False | None | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n" + ] + } + ], "source": [ "env.step(9) # Equivalent of to c2_beacon.configure(c2_server_ip_address=\"192.168.10.22\")\n", "env.step(3)\n", @@ -1517,9 +2737,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 64, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+-----------------------------------------------------------------------------------------------------+\n", + "| C2Server Running Status |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n", + "| False | None | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n" + ] + } + ], "source": [ "for i in range(6):\n", " env.step(0)\n", @@ -1542,9 +2776,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 65, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-04 15:37:40,588: PrimaiteGymEnv RNG seed = None\n" + ] + } + ], "source": [ "with open(data_manipulation_config_path()) as f:\n", " cfg = yaml.safe_load(f)\n", @@ -1584,9 +2826,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 67, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", + "| C2Beacon Running Status |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| True | 192.168.10.21 | 0 | 5 | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n" + ] + } + ], "source": [ "c2_beacon.configure(c2_server_ip_address=\"192.168.10.21\")\n", "c2_beacon.establish()\n", @@ -1604,9 +2860,72 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 68, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 4\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 5\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 6\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 7\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 8\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 9\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 10\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 11\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 12\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 13\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" + ] + } + ], "source": [ "for i in range(10):\n", " keep_alive_obs, _, _, _, _ = blue_config_env.step(0)\n", @@ -1622,9 +2941,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 69, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", + "| C2Beacon Running Status |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| True | 192.168.10.21 | 0 | 1 | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n" + ] + } + ], "source": [ "c2_beacon.configure(c2_server_ip_address=\"192.168.10.21\", keep_alive_frequency=1)\n", "c2_beacon.establish()\n", @@ -1640,9 +2973,40 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 70, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 14\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 15\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" + ] + } + ], "source": [ "# Comparing the OBS of the default frequency to a timestep frequency of 1\n", "for i in range(2):\n", @@ -1661,9 +3025,52 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 71, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 16\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 17\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 18\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 19\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 20\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 21\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 22\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" + ] + } + ], "source": [ "c2_beacon.configure(c2_server_ip_address=\"192.168.10.21\", keep_alive_frequency=7)\n", "\n", @@ -1700,9 +3107,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 72, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-04 15:37:41,322: Resetting environment, episode 0, avg. reward: 0.0\n", + "2025-02-04 15:37:41,325: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-04/15-37-31/agent_actions/episode_0.json\n" + ] + } + ], "source": [ "blue_config_env.reset()\n", "\n", @@ -1724,9 +3140,28 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 73, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 5\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" + ] + } + ], "source": [ "# Capturing default C2 Traffic\n", "for i in range(3):\n", @@ -1744,9 +3179,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 74, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", + "| C2Beacon Running Status |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| True | 192.168.10.21 | 0 | 5 | udp | 53 |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n" + ] + } + ], "source": [ "from primaite.utils.validation.ip_protocol import PROTOCOL_LOOKUP\n", "from primaite.utils.validation.port import PORT_LOOKUP\n", @@ -1759,9 +3208,28 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 75, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 10\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 1 -> 0\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 1 -> 0\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['udp'][53]['inbound']: 0 -> 1\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['udp'][53]['outbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 1 -> 0\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 1 -> 0\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['udp'][53]['inbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['udp'][53]['outbound']: 0 -> 1\n" + ] + } + ], "source": [ "# Capturing UDP C2 Traffic\n", "for i in range(5):\n", From 3a29593f600ba3fa1b9b74806333bf7934e4303f Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Wed, 5 Feb 2025 11:50:38 +0000 Subject: [PATCH 74/95] 3075: Fix bug in notebook helper function. --- ...a-Manipulation-Customising-Red-Agent.ipynb | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb b/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb index de6d3fe5..14938c02 100644 --- a/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb +++ b/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb @@ -6,7 +6,7 @@ "source": [ "# Customising UC2 Red Agents\n", "\n", - "© Crown-owned copyright 2024, Defence Science and Technology Laboratory UK\n", + "© Crown-owned copyright 2025, Defence Science and Technology Laboratory UK\n", "\n", "This notebook will go over some examples of how red agent behaviour can be varied by changing its configuration parameters.\n", "\n", @@ -26,7 +26,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -36,13 +36,12 @@ "from primaite.game.agent.interface import AgentHistoryItem\n", "from primaite.session.environment import PrimaiteGymEnv\n", "import yaml\n", - "from pprint import pprint\n", - "from primaite.game.agent.scripted_agents import probabilistic_agent, data_manipulation_bot" + "from pprint import pprint" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -69,7 +68,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -80,9 +79,7 @@ " if red_action == 'do_nothing':\n", " red_str = 'DO NOTHING'\n", " elif red_action == 'node_application_execute':\n", - " client = \"client 1\" if red_info.parameters['node_name'] == 0 else \"client 2\"\n", - "\n", - " red_str = f\"ATTACK from {client}\"\n", + " red_str = f\"ATTACK from {red_info.parameters['node_name']}\"\n", " return red_str" ] }, @@ -245,7 +242,7 @@ "outputs": [], "source": [ "change = yaml.safe_load(\"\"\"\n", - " possible_start_nodes: [client_1, client_2]\n", + " possible_start_nodes: [client_1]\n", " target_application: DataManipulationBot\n", " start_step: 25\n", " frequency: 20\n", @@ -315,6 +312,10 @@ "outputs": [], "source": [ "change = yaml.safe_load(\"\"\"\n", + " agent_settings:\n", + " possible_start_nodes: [client_1]\n", + " target_application: DataManipulationBot\n", + "\n", " action_space:\n", " action_map:\n", " 0:\n", From a4c19608a69251c0dabff77c69fb02fab544b3c7 Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Wed, 5 Feb 2025 11:51:22 +0000 Subject: [PATCH 75/95] 3075: Pre-commit tidy up. --- docs/source/how_to_guides/extensible_nodes.rst | 2 +- src/primaite/game/game.py | 1 - src/primaite/notebooks/Action-masking.ipynb | 15 +++++++-------- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/docs/source/how_to_guides/extensible_nodes.rst b/docs/source/how_to_guides/extensible_nodes.rst index 043d0f06..4819cbb2 100644 --- a/docs/source/how_to_guides/extensible_nodes.rst +++ b/docs/source/how_to_guides/extensible_nodes.rst @@ -52,4 +52,4 @@ class Router(NetworkNode, identifier="router"): Changes to YAML file. ===================== -While effort has been made to ensure that nodes defined within configuration YAML files for use with PrimAITE 3.X remain compatible with PrimAITE v4+, it is encouraged to review for minor changes needed. +While effort has been made to ensure that nodes defined within configuration YAML files for use with PrimAITE 3.X remain compatible with PrimAITE v4+, it is encouraged to review for minor changes needed. diff --git a/src/primaite/game/game.py b/src/primaite/game/game.py index f819fa05..e9941a12 100644 --- a/src/primaite/game/game.py +++ b/src/primaite/game/game.py @@ -404,7 +404,6 @@ class PrimaiteGame: agents_cfg = cfg.get("agents", []) for agent_cfg in agents_cfg: - new_agent = AbstractAgent.from_config(agent_cfg) game.agents[agent_cfg["ref"]] = new_agent if isinstance(new_agent, ProxyAgent): diff --git a/src/primaite/notebooks/Action-masking.ipynb b/src/primaite/notebooks/Action-masking.ipynb index 64015080..74504878 100644 --- a/src/primaite/notebooks/Action-masking.ipynb +++ b/src/primaite/notebooks/Action-masking.ipynb @@ -6,7 +6,7 @@ "source": [ "# Action Masking\n", "\n", - "© Crown-owned copyright 2024, Defence Science and Technology Laboratory UK\n", + "© Crown-owned copyright 2025, Defence Science and Technology Laboratory UK\n", "\n", "PrimAITE environments support action masking. The action mask shows which of the agent's actions are applicable with the current environment state. For example, a node can only be turned on if it is currently turned off." ] @@ -22,14 +22,13 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from primaite.session.environment import PrimaiteGymEnv\n", "from primaite.config.load import data_manipulation_config_path\n", - "from prettytable import PrettyTable\n", - "from primaite.game.agent.scripted_agents import probabilistic_agent\n" + "from prettytable import PrettyTable" ] }, { @@ -104,7 +103,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -117,7 +116,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -157,7 +156,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -168,7 +167,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ From b22ba65eb69644a25ad6451d3fadfe87e5c8112d Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Wed, 5 Feb 2025 16:40:59 +0000 Subject: [PATCH 76/95] #3075: Further notebook updates. --- ...ommand-and-Control-E2E-Demonstration.ipynb | 1792 ++--------------- .../Data-Manipulation-E2E-Demonstration.ipynb | 6 +- .../Getting-Information-Out-Of-PrimAITE.ipynb | 4 +- ...ege-Escalation-and-Data-Loss-Example.ipynb | 11 +- .../notebooks/Requests-and-Responses.ipynb | 22 +- .../notebooks/Terminal-Processing.ipynb | 43 +- .../Training-an-RLLIB-MARL-System.ipynb | 31 +- .../notebooks/Training-an-RLLib-Agent.ipynb | 31 +- .../notebooks/Training-an-SB3-Agent.ipynb | 2 +- .../notebooks/Using-Episode-Schedules.ipynb | 18 +- src/primaite/notebooks/multi-processing.ipynb | 15 +- .../simulator/network/hardware/base.py | 8 +- 12 files changed, 224 insertions(+), 1759 deletions(-) diff --git a/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb b/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb index b8d1423f..d3c414d7 100644 --- a/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb +++ b/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb @@ -13,31 +13,16 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-02-04 15:37:30,830: Performing the PrimAITE first-time setup...\n", - "2025-02-04 15:37:30,830: Building the PrimAITE app directories...\n", - "2025-02-04 15:37:30,830: Building primaite_config.yaml...\n", - "2025-02-04 15:37:30,830: Rebuilding the demo notebooks...\n", - "/home/nick/primaite/4.0.0a1-dev/notebooks/example_notebooks/Command-and-Control-E2E-Demonstration.ipynb\n", - "2025-02-04 15:37:30,832: Reset example notebook: /home/nick/primaite/4.0.0a1-dev/notebooks/example_notebooks/Command-and-Control-E2E-Demonstration.ipynb\n", - "2025-02-04 15:37:30,838: Rebuilding the example notebooks...\n", - "2025-02-04 15:37:30,841: PrimAITE setup complete!\n" - ] - } - ], + "outputs": [], "source": [ "!primaite setup" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -67,7 +52,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -150,17 +135,9 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-04 15:37:35,293: PrimaiteGymEnv RNG seed = None\n" - ] - } - ], + "outputs": [], "source": [ "with open(data_manipulation_config_path()) as f:\n", " cfg = yaml.safe_load(f)\n", @@ -186,35 +163,9 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+--------------------------------------------------------------------------------------+\n", - "| client_1 Software Manager |\n", - "+---------------------+-------------+-----------------+--------------+------+----------+\n", - "| Name | Type | Operating State | Health State | Port | Protocol |\n", - "+---------------------+-------------+-----------------+--------------+------+----------+\n", - "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", - "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", - "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", - "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", - "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", - "| NMAP | Application | RUNNING | GOOD | None | none |\n", - "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", - "| UserManager | Service | RUNNING | GOOD | None | none |\n", - "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", - "| FTPClient | Service | RUNNING | GOOD | 21 | tcp |\n", - "| DataManipulationBot | Application | RUNNING | GOOD | None | none |\n", - "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", - "| C2Server | Application | RUNNING | GOOD | None | tcp |\n", - "+---------------------+-------------+-----------------+--------------+------+----------+\n" - ] - } - ], + "outputs": [], "source": [ "client_1: Computer = env.game.simulation.network.get_node_by_hostname(\"client_1\")\n", "client_1.software_manager.install(C2Server)\n", @@ -264,34 +215,9 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+-------------------------------------------------------------------------------------+\n", - "| web_server Software Manager |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| Name | Type | Operating State | Health State | Port | Protocol |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", - "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", - "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", - "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", - "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", - "| NMAP | Application | RUNNING | GOOD | None | none |\n", - "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", - "| UserManager | Service | RUNNING | GOOD | None | none |\n", - "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", - "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", - "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", - "| C2Beacon | Application | INSTALLING | UNUSED | None | tcp |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n" - ] - } - ], + "outputs": [], "source": [ "env.step(1)\n", "web_server: Computer = env.game.simulation.network.get_node_by_hostname(\"web_server\")\n", @@ -331,41 +257,9 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+-------------------------------------------------------------------------------------+\n", - "| web_server Software Manager |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| Name | Type | Operating State | Health State | Port | Protocol |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", - "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", - "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", - "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", - "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", - "| NMAP | Application | RUNNING | GOOD | None | none |\n", - "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", - "| UserManager | Service | RUNNING | GOOD | None | none |\n", - "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", - "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", - "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", - "| C2Beacon | Application | RUNNING | GOOD | None | tcp |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", - "| C2Beacon Running Status |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| False | 192.168.10.21 | 0 | 5 | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n" - ] - } - ], + "outputs": [], "source": [ "env.step(2)\n", "c2_beacon: C2Beacon = web_server.software_manager.software[\"C2Beacon\"]\n", @@ -404,54 +298,18 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(0,\n", - " 0.0,\n", - " False,\n", - " False,\n", - " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=2, action='node_application_execute', parameters={'node_name': 'web_server', 'application_name': 'C2Beacon'}, request=['network', 'node', 'web_server', 'application', 'C2Beacon', 'execute'], response=RequestResponse(status='success', data={}), reward=0.0, reward_info={})}})" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "env.step(3)" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", - "| C2Beacon Running Status |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| True | 192.168.10.21 | 1 | 5 | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "+-----------------------------------------------------------------------------------------------------+\n", - "| C2Server Running Status |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n", - "| True | 192.168.1.12 | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n" - ] - } - ], + "outputs": [], "source": [ "c2_beacon.show()\n", "c2_server.show()" @@ -514,59 +372,18 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(0,\n", - " 0.0,\n", - " False,\n", - " False,\n", - " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=3, action='c2_server_terminal_command', parameters={'node_name': 'client_1', 'ip_address': None, 'username': 'admin', 'password': 'admin', 'commands': [['software_manager', 'application', 'install', 'RansomwareScript']]}, request=['network', 'node', 'client_1', 'application', 'C2Server', 'terminal_command', {'commands': [['software_manager', 'application', 'install', 'RansomwareScript']], 'ip_address': None, 'username': 'admin', 'password': 'admin'}], response=RequestResponse(status='success', data={0: RequestResponse(status='success', data={})}), reward=0.0, reward_info={})}})" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "env.step(4)" ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+--------------------------------------------------------------------------------------+\n", - "| client_1 Software Manager |\n", - "+---------------------+-------------+-----------------+--------------+------+----------+\n", - "| Name | Type | Operating State | Health State | Port | Protocol |\n", - "+---------------------+-------------+-----------------+--------------+------+----------+\n", - "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", - "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", - "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", - "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", - "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", - "| NMAP | Application | RUNNING | GOOD | None | none |\n", - "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", - "| UserManager | Service | RUNNING | GOOD | None | none |\n", - "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", - "| FTPClient | Service | RUNNING | GOOD | 21 | tcp |\n", - "| DataManipulationBot | Application | RUNNING | GOOD | None | none |\n", - "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", - "| C2Server | Application | RUNNING | GOOD | None | tcp |\n", - "+---------------------+-------------+-----------------+--------------+------+----------+\n" - ] - } - ], + "outputs": [], "source": [ "client_1.software_manager.show()" ] @@ -605,66 +422,18 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(0,\n", - " 0.0,\n", - " False,\n", - " False,\n", - " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=4, action='c2_server_ransomware_configure', parameters={'node_name': 'client_1', 'server_ip_address': '192.168.1.14', 'payload': 'ENCRYPT'}, request=['network', 'node', 'client_1', 'application', 'C2Server', 'ransomware_configure', {'server_ip_address': '192.168.1.14', 'server_password': None, 'payload': 'ENCRYPT'}], response=RequestResponse(status='success', data={}), reward=0.0, reward_info={})}})" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "env.step(5)" ] }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+-------------------------------------------------------------------------------------+\n", - "| web_server Software Manager |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| Name | Type | Operating State | Health State | Port | Protocol |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", - "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", - "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", - "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", - "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", - "| NMAP | Application | RUNNING | GOOD | None | none |\n", - "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", - "| UserManager | Service | RUNNING | GOOD | None | none |\n", - "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", - "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", - "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", - "| C2Beacon | Application | RUNNING | GOOD | None | tcp |\n", - "| RansomwareScript | Application | RUNNING | GOOD | None | none |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "+------------------------------------+\n", - "| RansomwareScript Running Status |\n", - "+--------------------------+---------+\n", - "| Target Server IP Address | Payload |\n", - "+--------------------------+---------+\n", - "| 192.168.1.14 | ENCRYPT |\n", - "+--------------------------+---------+\n" - ] - } - ], + "outputs": [], "source": [ "ransomware_script: RansomwareScript = web_server.software_manager.software[\"RansomwareScript\"]\n", "web_server.software_manager.show()\n", @@ -710,48 +479,18 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(0,\n", - " 0.0,\n", - " False,\n", - " False,\n", - " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=5, action='c2_server_data_exfiltrate', parameters={'node_name': 'client_1', 'target_file_name': 'database.db', 'target_folder_name': 'database', 'exfiltration_folder_name': 'spoils', 'target_ip_address': '192.168.1.14', 'username': 'admin', 'password': 'admin'}, request=['network', 'node', 'client_1', 'application', 'C2Server', 'exfiltrate', {'target_file_name': 'database.db', 'target_folder_name': 'database', 'exfiltration_folder_name': 'spoils', 'target_ip_address': '192.168.1.14', 'username': 'admin', 'password': 'admin'}], response=RequestResponse(status='success', data={}), reward=0.0, reward_info={})}})" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "env.step(6)" ] }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+--------------------------------------------------------------------------------+\n", - "| client_1 File System |\n", - "+--------------------+---------+---------------+-----------------------+---------+\n", - "| File Path | Size | Health status | Visible health status | Deleted |\n", - "+--------------------+---------+---------------+-----------------------+---------+\n", - "| root | 0 B | GOOD | NONE | False |\n", - "| spoils/database.db | 4.77 MB | GOOD | NONE | False |\n", - "+--------------------+---------+---------------+-----------------------+---------+\n" - ] - } - ], + "outputs": [], "source": [ "client_1: Computer = env.game.simulation.network.get_node_by_hostname(\"client_1\")\n", "client_1.software_manager.file_system.show(full=True)" @@ -759,25 +498,9 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+---------------------------------------------------------------------------------+\n", - "| web_server File System |\n", - "+---------------------+---------+---------------+-----------------------+---------+\n", - "| File Path | Size | Health status | Visible health status | Deleted |\n", - "+---------------------+---------+---------------+-----------------------+---------+\n", - "| primaite/index.html | 15.0 KB | GOOD | NONE | False |\n", - "| root | 0 B | GOOD | NONE | False |\n", - "| spoils/database.db | 4.77 MB | GOOD | NONE | False |\n", - "+---------------------+---------+---------------+-----------------------+---------+\n" - ] - } - ], + "outputs": [], "source": [ "web_server: Computer = env.game.simulation.network.get_node_by_hostname(\"web_server\")\n", "web_server.software_manager.file_system.show(full=True)" @@ -814,48 +537,18 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(0,\n", - " 0.0,\n", - " False,\n", - " False,\n", - " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=6, action='c2_server_ransomware_launch', parameters={'node_name': 'client_1'}, request=['network', 'node', 'client_1', 'application', 'C2Server', 'ransomware_launch'], response=RequestResponse(status='success', data={}), reward=0.0, reward_info={})}})" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "env.step(7)" ] }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+----------------------------------------------------------------------------------+\n", - "| database_server File System |\n", - "+----------------------+---------+---------------+-----------------------+---------+\n", - "| File Path | Size | Health status | Visible health status | Deleted |\n", - "+----------------------+---------+---------------+-----------------------+---------+\n", - "| database/database.db | 4.77 MB | CORRUPT | NONE | False |\n", - "| root | 0 B | GOOD | NONE | False |\n", - "+----------------------+---------+---------------+-----------------------+---------+\n" - ] - } - ], + "outputs": [], "source": [ "database_server: Server = env.game.simulation.network.get_node_by_hostname(\"database_server\")\n", "database_server.software_manager.file_system.show(full=True)" @@ -874,7 +567,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1002,17 +695,9 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-04 15:37:36,689: PrimaiteGymEnv RNG seed = None\n" - ] - } - ], + "outputs": [], "source": [ "with open(data_manipulation_config_path()) as f:\n", " cfg = yaml.safe_load(f)\n", @@ -1026,7 +711,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1069,18 +754,9 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-04 15:37:37,079: Resetting environment, episode 0, avg. reward: 0.0\n", - "2025-02-04 15:37:37,082: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-04/15-37-31/agent_actions/episode_0.json\n" - ] - } - ], + "outputs": [], "source": [ "# Resetting the environment and capturing the default observation space.\n", "blue_env.reset()\n", @@ -1089,20 +765,9 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 23, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Setting up the C2 Suite via the simulation API.\n", "\n", @@ -1123,7 +788,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1133,26 +798,9 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 2\n", - "root['NODES']['HOST0']['APPLICATIONS'][1]['operating_status']: 0 -> 1\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", - "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", - "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" - ] - } - ], + "outputs": [], "source": [ "display_obs_diffs(default_obs, c2_configuration_obs, blue_env.game.step_counter)" ] @@ -1172,20 +820,9 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "RequestResponse(status='success', data={0: RequestResponse(status='success', data={})})" - ] - }, - "execution_count": 26, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Installing RansomwareScript via C2 Terminal Commands\n", "ransomware_install_command = {\"commands\":[[\"software_manager\", \"application\", \"install\", \"RansomwareScript\"]],\n", @@ -1196,20 +833,9 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "RequestResponse(status='success', data={})" - ] - }, - "execution_count": 27, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Configuring the RansomwareScript\n", "ransomware_config = {\"server_ip_address\": \"192.168.1.14\", \"payload\": \"ENCRYPT\"}\n", @@ -1218,7 +844,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1228,28 +854,9 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 7\n", - "root['NODES']['HOST0']['APPLICATIONS'][1]['operating_status']: 0 -> 1\n", - "root['NODES']['HOST0']['APPLICATIONS'][2]['operating_status']: 0 -> 3\n", - "root['NODES']['HOST0']['users']['local_login']: 0 -> 1\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", - "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", - "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" - ] - } - ], + "outputs": [], "source": [ "display_obs_diffs(default_obs, c2_ransomware_obs, env.game.step_counter)" ] @@ -1265,7 +872,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1281,27 +888,16 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "RequestResponse(status='success', data={})" - ] - }, - "execution_count": 31, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "c2_server.send_command(given_command=C2Command.DATA_EXFILTRATION, command_options=exfil_options)" ] }, { "cell_type": "code", - "execution_count": 32, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1310,22 +906,9 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 7\n", - "root['NODES']['HOST0']['APPLICATIONS'][2]['operating_status']: 3 -> 1\n", - "root['NODES']['HOST1']['users']['remote_sessions']: 0 -> 1\n" - ] - } - ], + "outputs": [], "source": [ "display_obs_diffs(c2_ransomware_obs, c2_exfil_obs, env.game.step_counter)" ] @@ -1341,20 +924,9 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "RequestResponse(status='success', data={})" - ] - }, - "execution_count": 34, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Configuring the RansomwareScript\n", "ransomware_config = {\"server_ip_address\": \"192.168.1.14\", \"payload\": \"ENCRYPT\"}\n", @@ -1363,20 +935,9 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "RequestResponse(status='success', data={})" - ] - }, - "execution_count": 35, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Waiting for the ransomware to finish installing and then launching the RansomwareScript.\n", "blue_env.step(0)\n", @@ -1385,7 +946,7 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1395,30 +956,9 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 6\n", - "root['NODES']['HOST0']['APPLICATIONS'][2]['operating_status']: 3 -> 1\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['NODES']['HOST1']['users']['remote_sessions']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" - ] - } - ], + "outputs": [], "source": [ "display_obs_diffs(c2_ransomware_obs, c2_final_obs, blue_env.game.step_counter)" ] @@ -1434,7 +974,7 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1466,200 +1006,16 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-04 15:37:38,317: Resetting environment, episode 1, avg. reward: 0.0\n", - "2025-02-04 15:37:38,321: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-04/15-37-31/agent_actions/episode_1.json\n" - ] - }, - { - "data": { - "text/plain": [ - "({'NODES': {'HOST0': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 1,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'HOST1': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 1,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'HOST2': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 0,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'HOST3': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 0,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'ROUTER0': {'ACL': {1: {'position': 0,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 2: {'position': 1,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 3: {'position': 2,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 4: {'position': 3,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 5: {'position': 4,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 6: {'position': 5,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 7: {'position': 6,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 8: {'position': 7,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 9: {'position': 8,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 10: {'position': 9,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0}},\n", - " 'PORTS': {1: {'operating_status': 1},\n", - " 2: {'operating_status': 1},\n", - " 3: {'operating_status': 2}},\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}}},\n", - " 'LINKS': {1: {'PROTOCOLS': {'ALL': 1}},\n", - " 2: {'PROTOCOLS': {'ALL': 1}},\n", - " 3: {'PROTOCOLS': {'ALL': 0}},\n", - " 4: {'PROTOCOLS': {'ALL': 1}},\n", - " 5: {'PROTOCOLS': {'ALL': 1}},\n", - " 6: {'PROTOCOLS': {'ALL': 1}},\n", - " 7: {'PROTOCOLS': {'ALL': 1}},\n", - " 8: {'PROTOCOLS': {'ALL': 1}},\n", - " 9: {'PROTOCOLS': {'ALL': 1}},\n", - " 10: {'PROTOCOLS': {'ALL': 0}}},\n", - " 'ICS': 0},\n", - " {})" - ] - }, - "execution_count": 39, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "blue_env.reset()" ] }, { "cell_type": "code", - "execution_count": 40, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1678,7 +1034,7 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1695,33 +1051,9 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+-------------------------------------------------------------------------------------+\n", - "| web_server Software Manager |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| Name | Type | Operating State | Health State | Port | Protocol |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", - "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", - "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", - "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", - "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", - "| NMAP | Application | RUNNING | GOOD | None | none |\n", - "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", - "| UserManager | Service | RUNNING | GOOD | None | none |\n", - "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", - "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", - "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n" - ] - } - ], + "outputs": [], "source": [ "blue_env.step(0)\n", "web_server.software_manager.show()" @@ -1729,26 +1061,9 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 3\n", - "root['NODES']['HOST0']['APPLICATIONS'][1]['operating_status']: 1 -> 0\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", - "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", - "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" - ] - } - ], + "outputs": [], "source": [ "display_obs_diffs(pre_blue_action_obs, post_blue_action_obs, blue_env.game.step_counter)" ] @@ -1762,20 +1077,9 @@ }, { "cell_type": "code", - "execution_count": 44, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "RequestResponse(status='failure', data={'Reason': 'Command sent to the C2 Beacon but no response was ever received.'})" - ] - }, - "execution_count": 44, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Attempting to install the C2 RansomwareScript\n", "ransomware_install_command = {\"commands\":[[\"software_manager\", \"application\", \"install\", \"RansomwareScript\"]],\n", @@ -1797,200 +1101,16 @@ }, { "cell_type": "code", - "execution_count": 45, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-04 15:37:38,778: Resetting environment, episode 2, avg. reward: 0.0\n", - "2025-02-04 15:37:38,781: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-04/15-37-31/agent_actions/episode_2.json\n" - ] - }, - { - "data": { - "text/plain": [ - "({'NODES': {'HOST0': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 1,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'HOST1': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 1,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'HOST2': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 0,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'HOST3': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 0,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'ROUTER0': {'ACL': {1: {'position': 0,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 2: {'position': 1,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 3: {'position': 2,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 4: {'position': 3,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 5: {'position': 4,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 6: {'position': 5,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 7: {'position': 6,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 8: {'position': 7,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 9: {'position': 8,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 10: {'position': 9,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0}},\n", - " 'PORTS': {1: {'operating_status': 1},\n", - " 2: {'operating_status': 1},\n", - " 3: {'operating_status': 2}},\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}}},\n", - " 'LINKS': {1: {'PROTOCOLS': {'ALL': 1}},\n", - " 2: {'PROTOCOLS': {'ALL': 1}},\n", - " 3: {'PROTOCOLS': {'ALL': 0}},\n", - " 4: {'PROTOCOLS': {'ALL': 1}},\n", - " 5: {'PROTOCOLS': {'ALL': 1}},\n", - " 6: {'PROTOCOLS': {'ALL': 1}},\n", - " 7: {'PROTOCOLS': {'ALL': 1}},\n", - " 8: {'PROTOCOLS': {'ALL': 1}},\n", - " 9: {'PROTOCOLS': {'ALL': 1}},\n", - " 10: {'PROTOCOLS': {'ALL': 0}}},\n", - " 'ICS': 0},\n", - " {})" - ] - }, - "execution_count": 45, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "blue_env.reset()" ] }, { "cell_type": "code", - "execution_count": 46, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -2009,7 +1129,7 @@ }, { "cell_type": "code", - "execution_count": 47, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -2026,17 +1146,9 @@ }, { "cell_type": "code", - "execution_count": 48, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "NodeOperatingState.SHUTTING_DOWN\n" - ] - } - ], + "outputs": [], "source": [ "web_server = blue_env.game.simulation.network.get_node_by_hostname(\"web_server\")\n", "print(web_server.operating_state)" @@ -2044,48 +1156,18 @@ }, { "cell_type": "code", - "execution_count": 49, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 2\n", - "root['NODES']['HOST0']['operating_status']: 1 -> 4\n", - "root['NODES']['HOST0']['APPLICATIONS'][1]['operating_status']: 1 -> 0\n", - "root['NODES']['HOST0']['NICS'][1]['nic_status']: 1 -> 0\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", - "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", - "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" - ] - } - ], + "outputs": [], "source": [ "display_obs_diffs(pre_blue_action_obs, post_blue_action_obs, blue_env.game.step_counter)" ] }, { "cell_type": "code", - "execution_count": 50, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "RequestResponse(status='failure', data={'Reason': 'Command sent to the C2 Beacon but no response was ever received.'})" - ] - }, - "execution_count": 50, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Attempting to install the C2 RansomwareScript\n", "ransomware_install_command = {\"commands\":[\"software_manager\", \"application\", \"install\", \"RansomwareScript\"],\n", @@ -2109,200 +1191,16 @@ }, { "cell_type": "code", - "execution_count": 51, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-04 15:37:39,257: Resetting environment, episode 3, avg. reward: 0.0\n", - "2025-02-04 15:37:39,260: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-04/15-37-31/agent_actions/episode_3.json\n" - ] - }, - { - "data": { - "text/plain": [ - "({'NODES': {'HOST0': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 1,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'HOST1': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 1,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'HOST2': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 0,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'HOST3': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 0,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'ROUTER0': {'ACL': {1: {'position': 0,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 2: {'position': 1,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 3: {'position': 2,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 4: {'position': 3,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 5: {'position': 4,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 6: {'position': 5,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 7: {'position': 6,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 8: {'position': 7,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 9: {'position': 8,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 10: {'position': 9,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0}},\n", - " 'PORTS': {1: {'operating_status': 1},\n", - " 2: {'operating_status': 1},\n", - " 3: {'operating_status': 2}},\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}}},\n", - " 'LINKS': {1: {'PROTOCOLS': {'ALL': 1}},\n", - " 2: {'PROTOCOLS': {'ALL': 1}},\n", - " 3: {'PROTOCOLS': {'ALL': 0}},\n", - " 4: {'PROTOCOLS': {'ALL': 1}},\n", - " 5: {'PROTOCOLS': {'ALL': 1}},\n", - " 6: {'PROTOCOLS': {'ALL': 1}},\n", - " 7: {'PROTOCOLS': {'ALL': 1}},\n", - " 8: {'PROTOCOLS': {'ALL': 1}},\n", - " 9: {'PROTOCOLS': {'ALL': 1}},\n", - " 10: {'PROTOCOLS': {'ALL': 0}}},\n", - " 'ICS': 0},\n", - " {})" - ] - }, - "execution_count": 51, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "blue_env.reset()" ] }, { "cell_type": "code", - "execution_count": 52, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -2321,7 +1219,7 @@ }, { "cell_type": "code", - "execution_count": 53, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -2338,41 +1236,9 @@ }, { "cell_type": "code", - "execution_count": 54, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+---------------------------------------------------------------+\n", - "| router_1 Network Interfaces |\n", - "+------+-------------------+-----------------+-------+----------+\n", - "| Port | MAC Address | Address | Speed | Status |\n", - "+------+-------------------+-----------------+-------+----------+\n", - "| 1 | dd:bc:17:75:a2:c4 | 192.168.1.1/24 | 100.0 | Enabled |\n", - "| 2 | 00:f2:f5:65:51:75 | 192.168.10.1/24 | 100.0 | Enabled |\n", - "| 3 | de:7d:ac:0e:58:e1 | 127.0.0.1/8 | 100.0 | Disabled |\n", - "| 4 | 8a:b4:55:12:c7:9e | 127.0.0.1/8 | 100.0 | Disabled |\n", - "| 5 | a5:ef:c6:53:97:c4 | 127.0.0.1/8 | 100.0 | Disabled |\n", - "+------+-------------------+-----------------+-------+----------+\n", - "+------------------------------------------------------------------------------------------------------------------------+\n", - "| router_1 Access Control List |\n", - "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n", - "| Index | Action | Protocol | Src IP | Src Wildcard | Src Port | Dst IP | Dst Wildcard | Dst Port | Matched |\n", - "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n", - "| 1 | DENY | ANY | 192.168.10.21 | 0.0.0.1 | 80 | 192.168.1.12 | 0.0.0.1 | 80 | 0 |\n", - "| 18 | PERMIT | ANY | ANY | ANY | 5432 | ANY | ANY | 5432 | 0 |\n", - "| 19 | PERMIT | ANY | ANY | ANY | 53 | ANY | ANY | 53 | 0 |\n", - "| 20 | PERMIT | ANY | ANY | ANY | 21 | ANY | ANY | 21 | 0 |\n", - "| 21 | PERMIT | ANY | ANY | ANY | 80 | ANY | ANY | 80 | 4 |\n", - "| 22 | PERMIT | ANY | ANY | ANY | 219 | ANY | ANY | 219 | 10 |\n", - "| 23 | PERMIT | icmp | ANY | ANY | ANY | ANY | ANY | ANY | 0 |\n", - "| 24 | DENY | ANY | ANY | ANY | ANY | ANY | ANY | ANY | 0 |\n", - "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n" - ] - } - ], + "outputs": [], "source": [ "router_1: Router = blue_env.game.simulation.network.get_node_by_hostname(\"router_1\")\n", "router_1.acl.show()" @@ -2387,20 +1253,9 @@ }, { "cell_type": "code", - "execution_count": 55, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "RequestResponse(status='failure', data={'Reason': 'Command sent to the C2 Beacon but no response was ever received.'})" - ] - }, - "execution_count": 55, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "blue_env.step(0)\n", "\n", @@ -2411,30 +1266,9 @@ }, { "cell_type": "code", - "execution_count": 56, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+------------------------------------------------------------------------------------------------------------------------+\n", - "| router_1 Access Control List |\n", - "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n", - "| Index | Action | Protocol | Src IP | Src Wildcard | Src Port | Dst IP | Dst Wildcard | Dst Port | Matched |\n", - "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n", - "| 1 | DENY | ANY | 192.168.10.21 | 0.0.0.1 | 80 | 192.168.1.12 | 0.0.0.1 | 80 | 2 |\n", - "| 18 | PERMIT | ANY | ANY | ANY | 5432 | ANY | ANY | 5432 | 0 |\n", - "| 19 | PERMIT | ANY | ANY | ANY | 53 | ANY | ANY | 53 | 0 |\n", - "| 20 | PERMIT | ANY | ANY | ANY | 21 | ANY | ANY | 21 | 0 |\n", - "| 21 | PERMIT | ANY | ANY | ANY | 80 | ANY | ANY | 80 | 4 |\n", - "| 22 | PERMIT | ANY | ANY | ANY | 219 | ANY | ANY | 219 | 10 |\n", - "| 23 | PERMIT | icmp | ANY | ANY | ANY | ANY | ANY | ANY | 0 |\n", - "| 24 | DENY | ANY | ANY | ANY | ANY | ANY | ANY | ANY | 0 |\n", - "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n" - ] - } - ], + "outputs": [], "source": [ "router_1.acl.show()" ] @@ -2448,58 +1282,18 @@ }, { "cell_type": "code", - "execution_count": 57, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+-------------------------------------------------------------------------------------+\n", - "| web_server Software Manager |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| Name | Type | Operating State | Health State | Port | Protocol |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", - "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", - "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", - "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", - "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", - "| NMAP | Application | RUNNING | GOOD | None | none |\n", - "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", - "| UserManager | Service | RUNNING | GOOD | None | none |\n", - "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", - "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", - "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", - "| C2Beacon | Application | RUNNING | GOOD | None | tcp |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n" - ] - } - ], + "outputs": [], "source": [ "web_server.software_manager.show()" ] }, { "cell_type": "code", - "execution_count": 58, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+----------------------------------------------------------------------------------+\n", - "| database_server File System |\n", - "+----------------------+---------+---------------+-----------------------+---------+\n", - "| File Path | Size | Health status | Visible health status | Deleted |\n", - "+----------------------+---------+---------------+-----------------------+---------+\n", - "| database/database.db | 4.77 MB | GOOD | NONE | False |\n", - "| root | 0 B | GOOD | NONE | False |\n", - "+----------------------+---------+---------------+-----------------------+---------+\n" - ] - } - ], + "outputs": [], "source": [ "database_server: Server = blue_env.game.simulation.network.get_node_by_hostname(\"database_server\")\n", "database_server.software_manager.file_system.show(full=True)" @@ -2507,33 +1301,9 @@ }, { "cell_type": "code", - "execution_count": 59, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 3\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", - "root['NODES']['ROUTER0']['ACL'][1]['permission']: 0 -> 2\n", - "root['NODES']['ROUTER0']['ACL'][1]['source_ip_id']: 0 -> 7\n", - "root['NODES']['ROUTER0']['ACL'][1]['source_wildcard_id']: 0 -> 2\n", - "root['NODES']['ROUTER0']['ACL'][1]['source_port_id']: 0 -> 2\n", - "root['NODES']['ROUTER0']['ACL'][1]['dest_ip_id']: 0 -> 3\n", - "root['NODES']['ROUTER0']['ACL'][1]['dest_wildcard_id']: 0 -> 2\n", - "root['NODES']['ROUTER0']['ACL'][1]['dest_port_id']: 0 -> 2\n", - "root['NODES']['ROUTER0']['ACL'][1]['protocol_id']: 0 -> 1\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", - "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", - "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" - ] - } - ], + "outputs": [], "source": [ "display_obs_diffs(pre_blue_action_obs, post_blue_action_obs, blue_env.game.step_counter)" ] @@ -2597,17 +1367,9 @@ }, { "cell_type": "code", - "execution_count": 60, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-04 15:37:40,175: PrimaiteGymEnv RNG seed = None\n" - ] - } - ], + "outputs": [], "source": [ "with open(data_manipulation_config_path()) as f:\n", " cfg = yaml.safe_load(f)\n", @@ -2628,7 +1390,7 @@ }, { "cell_type": "code", - "execution_count": 61, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -2656,30 +1418,9 @@ }, { "cell_type": "code", - "execution_count": 62, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", - "| C2Beacon Running Status |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| False | None | 0 | 5 | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "+-----------------------------------------------------------------------------------------------------+\n", - "| C2Server Running Status |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n", - "| False | None | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n" - ] - } - ], + "outputs": [], "source": [ "env.step(2) # Agent Action Equivalent to c2_beacon.configure(c2_server_ip_address=\"192.168.10.21\")\n", "env.step(3) # Agent action Equivalent to c2_beacon.establish()\n", @@ -2696,30 +1437,9 @@ }, { "cell_type": "code", - "execution_count": 63, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", - "| C2Beacon Running Status |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| False | None | 0 | 5 | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "+-----------------------------------------------------------------------------------------------------+\n", - "| C2Server Running Status |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n", - "| False | None | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n" - ] - } - ], + "outputs": [], "source": [ "env.step(9) # Equivalent of to c2_beacon.configure(c2_server_ip_address=\"192.168.10.22\")\n", "env.step(3)\n", @@ -2737,23 +1457,9 @@ }, { "cell_type": "code", - "execution_count": 64, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+-----------------------------------------------------------------------------------------------------+\n", - "| C2Server Running Status |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n", - "| False | None | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n" - ] - } - ], + "outputs": [], "source": [ "for i in range(6):\n", " env.step(0)\n", @@ -2776,17 +1482,9 @@ }, { "cell_type": "code", - "execution_count": 65, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-04 15:37:40,588: PrimaiteGymEnv RNG seed = None\n" - ] - } - ], + "outputs": [], "source": [ "with open(data_manipulation_config_path()) as f:\n", " cfg = yaml.safe_load(f)\n", @@ -2802,7 +1500,7 @@ }, { "cell_type": "code", - "execution_count": 66, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -2826,23 +1524,9 @@ }, { "cell_type": "code", - "execution_count": 67, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", - "| C2Beacon Running Status |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| True | 192.168.10.21 | 0 | 5 | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n" - ] - } - ], + "outputs": [], "source": [ "c2_beacon.configure(c2_server_ip_address=\"192.168.10.21\")\n", "c2_beacon.establish()\n", @@ -2860,72 +1544,9 @@ }, { "cell_type": "code", - "execution_count": 68, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 4\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 5\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 6\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 7\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 8\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 9\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 10\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 11\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 12\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 13\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" - ] - } - ], + "outputs": [], "source": [ "for i in range(10):\n", " keep_alive_obs, _, _, _, _ = blue_config_env.step(0)\n", @@ -2941,23 +1562,9 @@ }, { "cell_type": "code", - "execution_count": 69, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", - "| C2Beacon Running Status |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| True | 192.168.10.21 | 0 | 1 | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n" - ] - } - ], + "outputs": [], "source": [ "c2_beacon.configure(c2_server_ip_address=\"192.168.10.21\", keep_alive_frequency=1)\n", "c2_beacon.establish()\n", @@ -2973,40 +1580,9 @@ }, { "cell_type": "code", - "execution_count": 70, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 14\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 15\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" - ] - } - ], + "outputs": [], "source": [ "# Comparing the OBS of the default frequency to a timestep frequency of 1\n", "for i in range(2):\n", @@ -3025,52 +1601,9 @@ }, { "cell_type": "code", - "execution_count": 71, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 16\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 17\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 18\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 19\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 20\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 21\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 22\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" - ] - } - ], + "outputs": [], "source": [ "c2_beacon.configure(c2_server_ip_address=\"192.168.10.21\", keep_alive_frequency=7)\n", "\n", @@ -3107,18 +1640,9 @@ }, { "cell_type": "code", - "execution_count": 72, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-04 15:37:41,322: Resetting environment, episode 0, avg. reward: 0.0\n", - "2025-02-04 15:37:41,325: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-04/15-37-31/agent_actions/episode_0.json\n" - ] - } - ], + "outputs": [], "source": [ "blue_config_env.reset()\n", "\n", @@ -3140,28 +1664,9 @@ }, { "cell_type": "code", - "execution_count": 73, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 5\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" - ] - } - ], + "outputs": [], "source": [ "# Capturing default C2 Traffic\n", "for i in range(3):\n", @@ -3179,23 +1684,9 @@ }, { "cell_type": "code", - "execution_count": 74, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", - "| C2Beacon Running Status |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| True | 192.168.10.21 | 0 | 5 | udp | 53 |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n" - ] - } - ], + "outputs": [], "source": [ "from primaite.utils.validation.ip_protocol import PROTOCOL_LOOKUP\n", "from primaite.utils.validation.port import PORT_LOOKUP\n", @@ -3208,28 +1699,9 @@ }, { "cell_type": "code", - "execution_count": 75, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 10\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 1 -> 0\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 1 -> 0\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['udp'][53]['inbound']: 0 -> 1\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['udp'][53]['outbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 1 -> 0\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 1 -> 0\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['udp'][53]['inbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['udp'][53]['outbound']: 0 -> 1\n" - ] - } - ], + "outputs": [], "source": [ "# Capturing UDP C2 Traffic\n", "for i in range(5):\n", diff --git a/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb b/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb index 4a34619f..41b75e43 100644 --- a/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb +++ b/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb @@ -382,7 +382,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": { "tags": [] }, @@ -394,7 +394,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": { "tags": [] }, @@ -450,7 +450,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ diff --git a/src/primaite/notebooks/Getting-Information-Out-Of-PrimAITE.ipynb b/src/primaite/notebooks/Getting-Information-Out-Of-PrimAITE.ipynb index d7d60d76..c7fdafad 100644 --- a/src/primaite/notebooks/Getting-Information-Out-Of-PrimAITE.ipynb +++ b/src/primaite/notebooks/Getting-Information-Out-Of-PrimAITE.ipynb @@ -6,7 +6,7 @@ "source": [ "# Getting information out of PrimAITE\n", "\n", - "© Crown-owned copyright 2024, Defence Science and Technology Laboratory UK\n" + "© Crown-owned copyright 2025, Defence Science and Technology Laboratory UK\n" ] }, { @@ -32,8 +32,6 @@ "from primaite.session.environment import PrimaiteGymEnv\n", "from primaite.simulator.network.hardware.nodes.host.computer import Computer\n", "from notebook.services.config import ConfigManager\n", - "from primaite.game.agent.scripted_agents import probabilistic_agent\n", - "\n", "\n", "cm = ConfigManager().update('notebook', {'limit_output': 50}) # limit output lines to 50 - for neatness\n", "\n", diff --git a/src/primaite/notebooks/Privilege-Escalation-and-Data-Loss-Example.ipynb b/src/primaite/notebooks/Privilege-Escalation-and-Data-Loss-Example.ipynb index 35d3813a..cbb898ea 100644 --- a/src/primaite/notebooks/Privilege-Escalation-and-Data-Loss-Example.ipynb +++ b/src/primaite/notebooks/Privilege-Escalation-and-Data-Loss-Example.ipynb @@ -6,7 +6,7 @@ "source": [ "# Simulating Privilege Escalation and Data Loss Using SSH and ACLs Manipulation\n", "\n", - "© Crown-owned copyright 2024, Defence Science and Technology Laboratory UK\n", + "© Crown-owned copyright 2025, Defence Science and Technology Laboratory UK\n", "\n", "## Overview\n", "\n", @@ -62,7 +62,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": { "tags": [] }, @@ -77,7 +77,8 @@ "from primaite.simulator.network.hardware.nodes.host.server import Server\n", "from primaite.simulator.system.applications.database_client import DatabaseClient\n", "from primaite.simulator.system.applications.web_browser import WebBrowser\n", - "from primaite.simulator.system.services.database.database_service import DatabaseService\n" + "from primaite.simulator.system.services.database.database_service import DatabaseService\n", + "from primaite.simulator.network.hardware.nodes.network import firewall\n" ] }, { @@ -89,7 +90,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": { "tags": [] }, @@ -112,7 +113,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": { "tags": [] }, diff --git a/src/primaite/notebooks/Requests-and-Responses.ipynb b/src/primaite/notebooks/Requests-and-Responses.ipynb index 83aed07c..01a6cffa 100644 --- a/src/primaite/notebooks/Requests-and-Responses.ipynb +++ b/src/primaite/notebooks/Requests-and-Responses.ipynb @@ -6,7 +6,7 @@ "source": [ "# Requests and Responses\n", "\n", - "© Crown-owned copyright 2024, Defence Science and Technology Laboratory UK\n", + "© Crown-owned copyright 2025, Defence Science and Technology Laboratory UK\n", "\n", "Agents interact with the PrimAITE simulation via the Request system.\n" ] @@ -36,7 +36,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -47,17 +47,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "sim = Simulation()\n", "sim.network.add_node(\n", - " HostNode(\n", - " hostname=\"client\",\n", - " ip_address='10.0.0.1',\n", - " subnet_mask='255.255.255.0',\n", - " operating_state=NodeOperatingState.ON)\n", + " HostNode.from_config(\n", + " config = {\n", + " 'type': \"hostnode\",\n", + " 'hostname': \"client\",\n", + " 'ip_address': '10.0.0.1',\n", + " 'subnet_mask': '255.255.255.0',\n", + " 'operating_state': NodeOperatingState.ON,\n", + " }\n", + " )\n", ")\n", "client = sim.network.get_node_by_hostname('client')\n" ] @@ -210,7 +214,7 @@ ], "metadata": { "kernelspec": { - "display_name": "venv", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, diff --git a/src/primaite/notebooks/Terminal-Processing.ipynb b/src/primaite/notebooks/Terminal-Processing.ipynb index 9aa4e96a..48318c13 100644 --- a/src/primaite/notebooks/Terminal-Processing.ipynb +++ b/src/primaite/notebooks/Terminal-Processing.ipynb @@ -29,7 +29,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -39,12 +39,43 @@ "from primaite.simulator.system.applications.red_applications.ransomware_script import RansomwareScript\n", "from primaite.simulator.system.services.terminal.terminal import RemoteTerminalConnection\n", "\n", + "# print(dir(Computer))\n", + "# node_a = Computer.from_config(\n", + "# config = {\n", + "# \"type\": \"computer\",\n", + "# \"hostname\": \"node_a\",\n", + "# \"ip_address\": \"192.168.0.10\",\n", + "# \"subnet_mask\": \"255.255.255.0\",\n", + "# \"startup_duration\": 0,\n", + "# }\n", + "# )\n", + "# print(f\"{node_a=}\")\n", + "\n", "def basic_network() -> Network:\n", " \"\"\"Utility function for creating a default network to demonstrate Terminal functionality\"\"\"\n", " network = Network()\n", - " node_a = Computer(hostname=\"node_a\", ip_address=\"192.168.0.10\", subnet_mask=\"255.255.255.0\", start_up_duration=0)\n", + " # node_a = Computer(hostname=\"node_a\", ip_address=\"192.168.0.10\", subnet_mask=\"255.255.255.0\", start_up_duration=0)\n", + " node_a = Computer.from_config(\n", + " config = {\n", + " \"type\": \"computer\",\n", + " \"hostname\": \"node_a\",\n", + " \"ip_address\": \"192.168.0.10\",\n", + " \"subnet_mask\": \"255.255.255.0\",\n", + " \"startup_duration\": 0,\n", + " }\n", + " )\n", + " print(f\"{node_a=}\")\n", " node_a.power_on()\n", - " node_b = Computer(hostname=\"node_b\", ip_address=\"192.168.0.11\", subnet_mask=\"255.255.255.0\", start_up_duration=0)\n", + " # node_b = Computer(hostname=\"node_b\", ip_address=\"192.168.0.11\", subnet_mask=\"255.255.255.0\", start_up_duration=0)\n", + " node_b = Computer.from_config(\n", + " config = {\n", + " \"type\": \"computer\",\n", + " \"hostname\": \"node_b\",\n", + " \"ip_address\": \"192.168.0.11\",\n", + " \"subnet_mask\": \"255.255.255.0\",\n", + " \"startup_duration\": 0,\n", + " }\n", + " )\n", " node_b.power_on()\n", " network.connect(node_a.network_interface[1], node_b.network_interface[1])\n", " return network" @@ -84,7 +115,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -211,7 +242,7 @@ ], "metadata": { "kernelspec": { - "display_name": ".venv", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -225,7 +256,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.11" + "version": "3.10.12" } }, "nbformat": 4, diff --git a/src/primaite/notebooks/Training-an-RLLIB-MARL-System.ipynb b/src/primaite/notebooks/Training-an-RLLIB-MARL-System.ipynb index dadb399e..87d9c377 100644 --- a/src/primaite/notebooks/Training-an-RLLIB-MARL-System.ipynb +++ b/src/primaite/notebooks/Training-an-RLLIB-MARL-System.ipynb @@ -6,7 +6,7 @@ "source": [ "# Train a Multi agent system using RLLIB\n", "\n", - "© Crown-owned copyright 2024, Defence Science and Technology Laboratory UK\n", + "© Crown-owned copyright 2025, Defence Science and Technology Laboratory UK\n", "\n", "This notebook will demonstrate how to use the `PrimaiteRayMARLEnv` to train a very basic system with two PPO agents." ] @@ -57,7 +57,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -103,20 +103,7 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "*** SIGTERM received at time=1737996337 on cpu 7 ***\n", - "PC: @ 0x7f3649b0fe2e (unknown) epoll_wait\n", - " @ 0x7f3649a2c520 (unknown) (unknown)\n", - "[2025-01-27 16:45:37,381 E 117142 117142] logging.cc:440: *** SIGTERM received at time=1737996337 on cpu 7 ***\n", - "[2025-01-27 16:45:37,381 E 117142 117142] logging.cc:440: PC: @ 0x7f3649b0fe2e (unknown) epoll_wait\n", - "[2025-01-27 16:45:37,381 E 117142 117142] logging.cc:440: @ 0x7f3649a2c520 (unknown) (unknown)\n" - ] - } - ], + "outputs": [], "source": [ "eval = algo.evaluate()" ] @@ -127,18 +114,6 @@ "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.12" } }, "nbformat": 4, diff --git a/src/primaite/notebooks/Training-an-RLLib-Agent.ipynb b/src/primaite/notebooks/Training-an-RLLib-Agent.ipynb index 64a9e7ab..79740bca 100644 --- a/src/primaite/notebooks/Training-an-RLLib-Agent.ipynb +++ b/src/primaite/notebooks/Training-an-RLLib-Agent.ipynb @@ -6,7 +6,7 @@ "source": [ "# Train a Single agent system using RLLib\n", "\n", - "© Crown-owned copyright 2024, Defence Science and Technology Laboratory UK\n", + "© Crown-owned copyright 2025, Defence Science and Technology Laboratory UK\n", "\n", "This notebook will demonstrate how to use PrimaiteRayEnv to train a basic PPO agent." ] @@ -52,7 +52,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -98,20 +98,7 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "*** SIGTERM received at time=1737996055 on cpu 1 ***\n", - "PC: @ 0x7f6e254a6e2e (unknown) epoll_wait\n", - " @ 0x7f6e253c3520 (unknown) (unknown)\n", - "[2025-01-27 16:40:55,343 E 114171 114171] logging.cc:440: *** SIGTERM received at time=1737996055 on cpu 1 ***\n", - "[2025-01-27 16:40:55,343 E 114171 114171] logging.cc:440: PC: @ 0x7f6e254a6e2e (unknown) epoll_wait\n", - "[2025-01-27 16:40:55,344 E 114171 114171] logging.cc:440: @ 0x7f6e253c3520 (unknown) (unknown)\n" - ] - } - ], + "outputs": [], "source": [ "eval = algo.evaluate()" ] @@ -122,18 +109,6 @@ "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.12" } }, "nbformat": 4, diff --git a/src/primaite/notebooks/Training-an-SB3-Agent.ipynb b/src/primaite/notebooks/Training-an-SB3-Agent.ipynb index d3492f92..10328989 100644 --- a/src/primaite/notebooks/Training-an-SB3-Agent.ipynb +++ b/src/primaite/notebooks/Training-an-SB3-Agent.ipynb @@ -6,7 +6,7 @@ "source": [ "# Training an SB3 Agent\n", "\n", - "© Crown-owned copyright 2024, Defence Science and Technology Laboratory UK\n", + "© Crown-owned copyright 2025, Defence Science and Technology Laboratory UK\n", "\n", "This notebook will demonstrate how to use primaite to create and train a PPO agent, using a pre-defined configuration file." ] diff --git a/src/primaite/notebooks/Using-Episode-Schedules.ipynb b/src/primaite/notebooks/Using-Episode-Schedules.ipynb index 311fe4fb..44305266 100644 --- a/src/primaite/notebooks/Using-Episode-Schedules.ipynb +++ b/src/primaite/notebooks/Using-Episode-Schedules.ipynb @@ -6,7 +6,7 @@ "source": [ "# Using Episode Schedules\n", "\n", - "© Crown-owned copyright 2024, Defence Science and Technology Laboratory UK\n", + "© Crown-owned copyright 2025, Defence Science and Technology Laboratory UK\n", "\n", "PrimAITE supports the ability to use different variations on a scenario at different episodes. This can be used to increase \n", "domain randomisation to prevent overfitting, or to set up curriculum learning to train agents to perform more complicated tasks.\n", @@ -40,7 +40,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -325,7 +325,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "metadata": {}, "outputs": [], "source": [ @@ -413,6 +413,18 @@ "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" } }, "nbformat": 4, diff --git a/src/primaite/notebooks/multi-processing.ipynb b/src/primaite/notebooks/multi-processing.ipynb index 798bf3ff..e56bf362 100644 --- a/src/primaite/notebooks/multi-processing.ipynb +++ b/src/primaite/notebooks/multi-processing.ipynb @@ -6,7 +6,7 @@ "source": [ "# Simple multi-processing demonstration\n", "\n", - "© Crown-owned copyright 2024, Defence Science and Technology Laboratory UK\n", + "© Crown-owned copyright 2025, Defence Science and Technology Laboratory UK\n", "\n", "This notebook uses SubprocVecEnv from SB3." ] @@ -29,7 +29,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -37,13 +37,12 @@ "from stable_baselines3 import PPO\n", "from stable_baselines3.common.utils import set_random_seed\n", "from stable_baselines3.common.vec_env import SubprocVecEnv\n", - "from primaite.session.environment import PrimaiteGymEnv\n", - "from primaite.game.agent.scripted_agents import probabilistic_agent\n" + "from primaite.session.environment import PrimaiteGymEnv\n" ] }, { "cell_type": "code", - "execution_count": 31, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -52,7 +51,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -69,7 +68,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -90,7 +89,7 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ diff --git a/src/primaite/simulator/network/hardware/base.py b/src/primaite/simulator/network/hardware/base.py index 6543d793..cc3d4150 100644 --- a/src/primaite/simulator/network/hardware/base.py +++ b/src/primaite/simulator/network/hardware/base.py @@ -1197,7 +1197,7 @@ class UserSessionManager(Service, identifier="UserSessionManager"): """Request should take the form [username, password, remote_ip_address].""" username, password, remote_ip_address = request response = RequestResponse.from_bool(self.remote_login(username, password, remote_ip_address)) - response.data = {"remote_hostname": self.parent.hostname, "username": username} + response.data = {"remote_hostname": self.parent.config.hostname, "username": username} return response rm.add_request("remote_login", RequestType(func=_remote_login)) @@ -1230,7 +1230,7 @@ class UserSessionManager(Service, identifier="UserSessionManager"): if markdown: table.set_style(MARKDOWN) table.align = "l" - table.title = f"{self.parent.hostname} User Sessions" + table.title = f"{self.parent.config.hostname} User Sessions" def _add_session_to_table(user_session: UserSession): """ @@ -1627,9 +1627,7 @@ class Node(SimComponent, ABC): dns_server=kwargs["config"].dns_server, ) super().__init__(**kwargs) - self.operating_state = ( - NodeOperatingState.ON if not (p := kwargs["config"].operating_state) else NodeOperatingState[p.upper()] - ) + self.operating_state = NodeOperatingState.ON if not (p := kwargs["config"].operating_state) else p self._install_system_software() self.session_manager.node = self self.session_manager.software_manager = self.software_manager From 7b2a9c1d2a6d64d870de17fa0a3c8afc15641919 Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Wed, 5 Feb 2025 16:46:25 +0000 Subject: [PATCH 77/95] #3075: Pre-commit changes. --- src/primaite/notebooks/Requests-and-Responses.ipynb | 4 ++-- src/primaite/notebooks/Terminal-Processing.ipynb | 4 ++-- src/primaite/notebooks/Training-an-SB3-Agent.ipynb | 10 +++++----- src/primaite/notebooks/Using-Episode-Schedules.ipynb | 4 ++-- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/primaite/notebooks/Requests-and-Responses.ipynb b/src/primaite/notebooks/Requests-and-Responses.ipynb index 01a6cffa..48f1cf3e 100644 --- a/src/primaite/notebooks/Requests-and-Responses.ipynb +++ b/src/primaite/notebooks/Requests-and-Responses.ipynb @@ -36,7 +36,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -47,7 +47,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ diff --git a/src/primaite/notebooks/Terminal-Processing.ipynb b/src/primaite/notebooks/Terminal-Processing.ipynb index 48318c13..dc870db5 100644 --- a/src/primaite/notebooks/Terminal-Processing.ipynb +++ b/src/primaite/notebooks/Terminal-Processing.ipynb @@ -29,7 +29,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -115,7 +115,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ diff --git a/src/primaite/notebooks/Training-an-SB3-Agent.ipynb b/src/primaite/notebooks/Training-an-SB3-Agent.ipynb index 10328989..1bec0183 100644 --- a/src/primaite/notebooks/Training-an-SB3-Agent.ipynb +++ b/src/primaite/notebooks/Training-an-SB3-Agent.ipynb @@ -29,7 +29,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -41,7 +41,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -50,7 +50,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -101,7 +101,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -133,7 +133,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ diff --git a/src/primaite/notebooks/Using-Episode-Schedules.ipynb b/src/primaite/notebooks/Using-Episode-Schedules.ipynb index 44305266..4bc49a70 100644 --- a/src/primaite/notebooks/Using-Episode-Schedules.ipynb +++ b/src/primaite/notebooks/Using-Episode-Schedules.ipynb @@ -40,7 +40,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -325,7 +325,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ From 1a8c3b9471e591659eb69853a4865b1a23708ad8 Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Thu, 6 Feb 2025 16:42:26 +0000 Subject: [PATCH 78/95] #3075: Fix notebooks after test changes. --- ...ommand-and-Control-E2E-Demonstration.ipynb | 82 +++++++++---------- ...a-Manipulation-Customising-Red-Agent.ipynb | 16 ++-- .../Data-Manipulation-E2E-Demonstration.ipynb | 4 +- ...ege-Escalation-and-Data-Loss-Example.ipynb | 30 +++---- .../notebooks/Requests-and-Responses.ipynb | 13 +-- .../notebooks/Terminal-Processing.ipynb | 26 ++---- .../simulator/network/hardware/base.py | 4 +- .../network/hardware/nodes/host/host_node.py | 2 +- .../configs/basic_switched_network.yaml | 6 +- 9 files changed, 86 insertions(+), 97 deletions(-) diff --git a/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb b/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb index 17b7b79e..474dadff 100644 --- a/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb +++ b/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb @@ -59,30 +59,30 @@ "custom_c2_agent = \"\"\"\n", " - ref: CustomC2Agent\n", " team: RED\n", - " type: ProxyAgent\n", + " type: proxy-agent\n", "\n", " action_space:\n", " action_map:\n", " 0:\n", - " action: do_nothing\n", + " action: do-nothing\n", " options: {}\n", " 1:\n", - " action: node_application_install\n", + " action: node-application-install\n", " options:\n", " node_name: web_server\n", - " application_name: C2Beacon\n", + " application_name: c2-beacon\n", " 2:\n", - " action: configure_c2_beacon\n", + " action: configure-c2-beacon\n", " options:\n", " node_name: web_server\n", " c2_server_ip_address: 192.168.10.21\n", " 3:\n", - " action: node_application_execute\n", + " action: node-application-execute\n", " options:\n", " node_name: web_server\n", - " application_name: C2Beacon\n", + " application_name: c2-beacon\n", " 4:\n", - " action: c2_server_terminal_command\n", + " action: c2-server-terminal-command\n", " options:\n", " node_name: client_1\n", " ip_address:\n", @@ -93,7 +93,7 @@ " - software_manager\n", " - application\n", " - install\n", - " - RansomwareScript\n", + " - ransomware-script\n", " 5:\n", " action: c2-server-ransomware-configure\n", " options:\n", @@ -101,7 +101,7 @@ " server_ip_address: 192.168.1.14\n", " payload: ENCRYPT\n", " 6:\n", - " action: c2_server_data_exfiltrate\n", + " action: c2-server-data-exfiltrate\n", " options:\n", " node_name: client_1\n", " target_file_name: \"database.db\"\n", @@ -112,23 +112,23 @@ " password: admin\n", "\n", " 7:\n", - " action: c2_server_ransomware_launch\n", + " action: c2-server-ransomware-launch\n", " options:\n", " node_name: client_1\n", " 8:\n", - " action: configure_c2_beacon\n", + " action: configure-c2-beacon\n", " options:\n", " node_name: web_server\n", " c2_server_ip_address: 192.168.10.21\n", " 9:\n", - " action: configure_c2_beacon\n", + " action: configure-c2-beacon\n", " options:\n", " node_name: web_server\n", " c2_server_ip_address: 192.168.10.22\n", "\n", " reward_function:\n", " reward_components:\n", - " - type: DUMMY\n", + " - type: dummy\n", "\"\"\"\n", "c2_agent_yaml = yaml.safe_load(custom_c2_agent)" ] @@ -169,7 +169,7 @@ "source": [ "client_1: Computer = env.game.simulation.network.get_node_by_hostname(\"client_1\")\n", "client_1.software_manager.install(C2Server)\n", - "c2_server: C2Server = client_1.software_manager.software[\"C2Server\"]\n", + "c2_server: C2Server = client_1.software_manager.software[\"c2-server\"]\n", "c2_server.run()\n", "client_1.software_manager.show()" ] @@ -262,7 +262,7 @@ "outputs": [], "source": [ "env.step(2)\n", - "c2_beacon: C2Beacon = web_server.software_manager.software[\"C2Beacon\"]\n", + "c2_beacon: C2Beacon = web_server.software_manager.software[\"c2-beacon\"]\n", "web_server.software_manager.show()\n", "c2_beacon.show()" ] @@ -435,7 +435,7 @@ "metadata": {}, "outputs": [], "source": [ - "ransomware_script: RansomwareScript = web_server.software_manager.software[\"RansomwareScript\"]\n", + "ransomware_script: RansomwareScript = web_server.software_manager.software[\"ransomware-script\"]\n", "web_server.software_manager.show()\n", "ransomware_script.show()" ] @@ -574,20 +574,20 @@ "custom_blue_agent_yaml = \"\"\"\n", " - ref: defender\n", " team: BLUE\n", - " type: ProxyAgent\n", + " type: proxy-agent\n", "\n", " observation_space:\n", - " type: CUSTOM\n", + " type: custom\n", " options:\n", " components:\n", - " - type: NODES\n", - " label: NODES\n", + " - type: nodes\n", + " label: nodes\n", " options:\n", " hosts:\n", " - hostname: web_server\n", " applications:\n", - " - application_name: C2Beacon\n", - " - application_name: RansomwareScript\n", + " - application_name: c2-beacon\n", + " - application_name: ransomware-script\n", " folders:\n", " - folder_name: exfiltration_folder\n", " files:\n", @@ -637,7 +637,7 @@ " - UDP\n", " num_rules: 10\n", "\n", - " - type: LINKS\n", + " - type: links\n", " label: LINKS\n", " options:\n", " link_references:\n", @@ -651,26 +651,26 @@ " - switch_2:eth-1<->client_1:eth-1\n", " - switch_2:eth-2<->client_2:eth-1\n", " - switch_2:eth-7<->security_suite:eth-2\n", - " - type: \"NONE\"\n", + " - type: \"none\"\n", " label: ICS\n", " options: {}\n", "\n", " action_space:\n", " action_map:\n", " 0:\n", - " action: do_nothing\n", + " action: do-nothing\n", " options: {}\n", " 1:\n", - " action: node_application_remove\n", + " action: node-application-remove\n", " options:\n", " node_name: web_server\n", - " application_name: C2Beacon\n", + " application_name: c2-beacon\n", " 2:\n", - " action: node_shutdown\n", + " action: node-shutdown\n", " options:\n", " node_name: web_server\n", " 3:\n", - " action: router_acl_add_rule\n", + " action: router-acl-add-rule\n", " options:\n", " target_router: router_1\n", " position: 1\n", @@ -685,7 +685,7 @@ "\n", " reward_function:\n", " reward_components:\n", - " - type: DUMMY\n", + " - type: dummy\n", "\n", " agent_settings:\n", " flatten_obs: False\n", @@ -776,12 +776,12 @@ "\n", "# Installing the C2 Server.\n", "client_1.software_manager.install(C2Server)\n", - "c2_server: C2Server = client_1.software_manager.software[\"C2Server\"]\n", + "c2_server: C2Server = client_1.software_manager.software[\"c2-server\"]\n", "c2_server.run()\n", "\n", "# Installing the C2 Beacon.\n", "web_server.software_manager.install(C2Beacon)\n", - "c2_beacon: C2Beacon = web_server.software_manager.software[\"C2Beacon\"]\n", + "c2_beacon: C2Beacon = web_server.software_manager.software[\"c2-beacon\"]\n", "c2_beacon.configure(c2_server_ip_address=\"192.168.10.21\")\n", "c2_beacon.establish()" ] @@ -825,7 +825,7 @@ "outputs": [], "source": [ "# Installing RansomwareScript via C2 Terminal Commands\n", - "ransomware_install_command = {\"commands\":[[\"software_manager\", \"application\", \"install\", \"RansomwareScript\"]],\n", + "ransomware_install_command = {\"commands\":[[\"software_manager\", \"application\", \"install\", \"ransomware-script\"]],\n", " \"username\": \"admin\",\n", " \"password\": \"admin\"}\n", "c2_server.send_command(C2Command.TERMINAL, command_options=ransomware_install_command)\n" @@ -984,11 +984,11 @@ " web_server: Server = given_env.game.simulation.network.get_node_by_hostname(\"web_server\")\n", "\n", " client_1.software_manager.install(C2Server)\n", - " c2_server: C2Server = client_1.software_manager.software[\"C2Server\"]\n", + " c2_server: C2Server = client_1.software_manager.software[\"c2-server\"]\n", " c2_server.run()\n", "\n", " web_server.software_manager.install(C2Beacon)\n", - " c2_beacon: C2Beacon = web_server.software_manager.software[\"C2Beacon\"]\n", + " c2_beacon: C2Beacon = web_server.software_manager.software[\"c2-beacon\"]\n", " c2_beacon.configure(c2_server_ip_address=\"192.168.10.21\")\n", " c2_beacon.establish()\n", "\n", @@ -1086,7 +1086,7 @@ " \"username\": \"admin\",\n", " \"password\": \"admin\"}\n", "\n", - "c2_server: C2Server = client_1.software_manager.software[\"C2Server\"]\n", + "c2_server: C2Server = client_1.software_manager.software[\"c2-server\"]\n", "c2_server.send_command(C2Command.TERMINAL, command_options=ransomware_install_command)" ] }, @@ -1174,7 +1174,7 @@ " \"username\": \"admin\",\n", " \"password\": \"admin\"}\n", "\n", - "c2_server: C2Server = client_1.software_manager.software[\"C2Server\"]\n", + "c2_server: C2Server = client_1.software_manager.software[\"c2-server\"]\n", "c2_server.send_command(C2Command.TERMINAL, command_options=ransomware_install_command)" ] }, @@ -1396,16 +1396,16 @@ "source": [ "web_server: Server = c2_config_env.game.simulation.network.get_node_by_hostname(\"web_server\")\n", "web_server.software_manager.install(C2Beacon)\n", - "c2_beacon: C2Beacon = web_server.software_manager.software[\"C2Beacon\"]\n", + "c2_beacon: C2Beacon = web_server.software_manager.software[\"c2-beacon\"]\n", "\n", "client_1: Computer = c2_config_env.game.simulation.network.get_node_by_hostname(\"client_1\")\n", "client_1.software_manager.install(C2Server)\n", - "c2_server_1: C2Server = client_1.software_manager.software[\"C2Server\"]\n", + "c2_server_1: C2Server = client_1.software_manager.software[\"c2-server\"]\n", "c2_server_1.run()\n", "\n", "client_2: Computer = c2_config_env.game.simulation.network.get_node_by_hostname(\"client_2\")\n", "client_2.software_manager.install(C2Server)\n", - "c2_server_2: C2Server = client_2.software_manager.software[\"C2Server\"]\n", + "c2_server_2: C2Server = client_2.software_manager.software[\"c2-server\"]\n", "c2_server_2.run()" ] }, diff --git a/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb b/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb index 14938c02..2e0a0d40 100644 --- a/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb +++ b/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb @@ -76,9 +76,9 @@ " # parse the info dict form step output and write out what the red agent is doing\n", " red_info : AgentHistoryItem = info['agent_actions']['data_manipulation_attacker']\n", " red_action = red_info.action\n", - " if red_action == 'do_nothing':\n", + " if red_action == 'do-nothing':\n", " red_str = 'DO NOTHING'\n", - " elif red_action == 'node_application_execute':\n", + " elif red_action == 'node-application-execute':\n", " red_str = f\"ATTACK from {red_info.parameters['node_name']}\"\n", " return red_str" ] @@ -369,18 +369,18 @@ "change = yaml.safe_load(\"\"\"\n", " applications:\n", " - ref: data_manipulation_bot\n", - " type: DataManipulationBot\n", + " type: data-manipulation-bot\n", " options:\n", " port_scan_p_of_success: 1.0\n", " data_manipulation_p_of_success: 1.0\n", " payload: \"DELETE\"\n", " server_ip: 192.168.1.14\n", " - ref: client_1_web_browser\n", - " type: WebBrowser\n", + " type: web-browser\n", " options:\n", " target_url: http://arcd.com/users/\n", " - ref: client_1_database_client\n", - " type: DatabaseClient\n", + " type: database-client\n", " options:\n", " db_server_ip: 192.168.1.14\n", "\"\"\")\n", @@ -414,18 +414,18 @@ "change = yaml.safe_load(\"\"\"\n", " applications:\n", " - ref: data_manipulation_bot\n", - " type: DataManipulationBot\n", + " type: data-manipulation-bot\n", " options:\n", " port_scan_p_of_success: 0.0\n", " data_manipulation_p_of_success: 0.0\n", " payload: \"DELETE\"\n", " server_ip: 192.168.1.14\n", " - ref: client_1_web_browser\n", - " type: WebBrowser\n", + " type: web-browser\n", " options:\n", " target_url: http://arcd.com/users/\n", " - ref: client_1_database_client\n", - " type: DatabaseClient\n", + " type: database-client\n", " options:\n", " db_server_ip: 192.168.1.14\n", "\"\"\")\n", diff --git a/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb b/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb index 41b75e43..2d3f99b2 100644 --- a/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb +++ b/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb @@ -458,9 +458,9 @@ " # parse the info dict form step output and write out what the red agent is doing\n", " red_info : AgentHistoryItem = info['agent_actions']['data_manipulation_attacker']\n", " red_action = red_info.action\n", - " if red_action == 'do_nothing':\n", + " if red_action == 'do-nothing':\n", " red_str = 'DO NOTHING'\n", - " elif red_action == 'node_application_execute':\n", + " elif red_action == 'node-application-execute':\n", " client = \"client 1\" if red_info.parameters['node_name'] == 0 else \"client 2\"\n", " red_str = f\"ATTACK from {client}\"\n", " return red_str" diff --git a/src/primaite/notebooks/Privilege-Escalation-and-Data-Loss-Example.ipynb b/src/primaite/notebooks/Privilege-Escalation-and-Data-Loss-Example.ipynb index cbb898ea..deb38eea 100644 --- a/src/primaite/notebooks/Privilege-Escalation-and-Data-Loss-Example.ipynb +++ b/src/primaite/notebooks/Privilege-Escalation-and-Data-Loss-Example.ipynb @@ -120,11 +120,11 @@ "outputs": [], "source": [ "some_tech_jnr_dev_pc: Computer = game.simulation.network.get_node_by_hostname(\"some_tech_jnr_dev_pc\")\n", - "some_tech_jnr_dev_db_client: DatabaseClient = some_tech_jnr_dev_pc.software_manager.software[\"DatabaseClient\"]\n", - "some_tech_jnr_dev_web_browser: WebBrowser = some_tech_jnr_dev_pc.software_manager.software[\"WebBrowser\"]\n", + "some_tech_jnr_dev_db_client: DatabaseClient = some_tech_jnr_dev_pc.software_manager.software[\"database-client\"]\n", + "some_tech_jnr_dev_web_browser: WebBrowser = some_tech_jnr_dev_pc.software_manager.software[\"web-browser\"]\n", "some_tech_rt: Router = game.simulation.network.get_node_by_hostname(\"some_tech_rt\")\n", "some_tech_db_srv: Server = game.simulation.network.get_node_by_hostname(\"some_tech_db_srv\")\n", - "some_tech_db_service: DatabaseService = some_tech_db_srv.software_manager.software[\"DatabaseService\"]\n", + "some_tech_db_service: DatabaseService = some_tech_db_srv.software_manager.software[\"database-service\"]\n", "some_tech_storage_srv: Server = game.simulation.network.get_node_by_hostname(\"some_tech_storage_srv\")\n", "some_tech_web_srv: Server = game.simulation.network.get_node_by_hostname(\"some_tech_web_srv\")" ] @@ -211,7 +211,7 @@ "source": [ "caos_action = [\n", " \"network\", \"node\", \"some_tech_jnr_dev_pc\", \n", - " \"service\", \"Terminal\", \"node_session_remote_login\", \"admin\", \"admin\", str(some_tech_storage_srv.network_interface[1].ip_address)\n", + " \"service\", \"terminal\", \"node-session-remote-login\", \"admin\", \"admin\", str(some_tech_storage_srv.network_interface[1].ip_address)\n", "]\n", "game.simulation.apply_request(caos_action)" ] @@ -233,7 +233,7 @@ }, "outputs": [], "source": [ - "caos_action = [\"network\", \"node\", \"some_tech_jnr_dev_pc\", \"application\", \"WebBrowser\", \"execute\"]\n", + "caos_action = [\"network\", \"node\", \"some_tech_jnr_dev_pc\", \"application\", \"web-browser\", \"execute\"]\n", "game.simulation.apply_request(caos_action)" ] }, @@ -256,7 +256,7 @@ "metadata": {}, "outputs": [], "source": [ - "game.get_sim_state()[\"network\"][\"nodes\"][\"some_tech_rt\"][\"services\"][\"UserSessionManager\"][\"active_remote_sessions\"]" + "game.get_sim_state()[\"network\"][\"nodes\"][\"some_tech_rt\"][\"services\"][\"user-session-manager\"][\"active_remote_sessions\"]" ] }, { @@ -269,7 +269,7 @@ "source": [ "caos_action = [\n", " \"network\", \"node\", \"some_tech_jnr_dev_pc\", \n", - " \"service\", \"Terminal\", \"node_session_remote_login\", \"admin\", \"admin\", str(some_tech_rt.network_interface[4].ip_address)\n", + " \"service\", \"terminal\", \"node-session-remote-login\", \"admin\", \"admin\", str(some_tech_rt.network_interface[4].ip_address)\n", "]\n", "game.simulation.apply_request(caos_action)" ] @@ -282,7 +282,7 @@ }, "outputs": [], "source": [ - "game.get_sim_state()[\"network\"][\"nodes\"][\"some_tech_rt\"][\"services\"][\"UserSessionManager\"][\"active_remote_sessions\"]" + "game.get_sim_state()[\"network\"][\"nodes\"][\"some_tech_rt\"][\"services\"][\"user-session-manager\"][\"active_remote_sessions\"]" ] }, { @@ -315,7 +315,7 @@ "source": [ "caos_action = [\n", " \"network\", \"node\", \"some_tech_jnr_dev_pc\", \n", - " \"service\", \"Terminal\", \"send_remote_command\", str(some_tech_rt.network_interface[4].ip_address),\n", + " \"service\", \"terminal\", \"send_remote_command\", str(some_tech_rt.network_interface[4].ip_address),\n", " {\n", " \"command\": [\n", " \"acl\", \"add_rule\", \"PERMIT\", \"TCP\",\n", @@ -368,7 +368,7 @@ "source": [ "caos_action = [\n", " \"network\", \"node\", \"some_tech_jnr_dev_pc\", \n", - " \"service\", \"Terminal\", \"remote_logoff\", str(some_tech_rt.network_interface[4].ip_address)\n", + " \"service\", \"terminal\", \"remote_logoff\", str(some_tech_rt.network_interface[4].ip_address)\n", "]\n", "game.simulation.apply_request(caos_action)" ] @@ -386,7 +386,7 @@ "metadata": {}, "outputs": [], "source": [ - "game.get_sim_state()[\"network\"][\"nodes\"][\"some_tech_rt\"][\"services\"][\"UserSessionManager\"][\"active_remote_sessions\"]" + "game.get_sim_state()[\"network\"][\"nodes\"][\"some_tech_rt\"][\"services\"][\"user-session-manager\"][\"active_remote_sessions\"]" ] }, { @@ -406,7 +406,7 @@ "source": [ "caos_action = [\n", " \"network\", \"node\", \"some_tech_jnr_dev_pc\", \n", - " \"service\", \"Terminal\", \"node_session_remote_login\", \"admin\", \"admin\", str(some_tech_storage_srv.network_interface[1].ip_address)\n", + " \"service\", \"terminal\", \"node-session-remote-login\", \"admin\", \"admin\", str(some_tech_storage_srv.network_interface[1].ip_address)\n", "]\n", "game.simulation.apply_request(caos_action)" ] @@ -421,7 +421,7 @@ "source": [ "caos_action = [\n", " \"network\", \"node\", \"some_tech_jnr_dev_pc\", \n", - " \"service\", \"Terminal\", \"send_remote_command\", str(some_tech_storage_srv.network_interface[1].ip_address),\n", + " \"service\", \"terminal\", \"send_remote_command\", str(some_tech_storage_srv.network_interface[1].ip_address),\n", " {\n", " \"command\": [\n", " \"file_system\", \"delete\", \"file\", db_backup_folder, \"database.db\"\n", @@ -476,7 +476,7 @@ }, "outputs": [], "source": [ - "caos_action = [\"network\", \"node\", \"some_tech_jnr_dev_pc\", \"application\", \"WebBrowser\", \"execute\"]\n", + "caos_action = [\"network\", \"node\", \"some_tech_jnr_dev_pc\", \"application\", \"web-browser\", \"execute\"]\n", "game.simulation.apply_request(caos_action)" ] }, @@ -535,7 +535,7 @@ }, "outputs": [], "source": [ - "caos_action = [\"network\", \"node\", \"some_tech_jnr_dev_pc\", \"application\", \"WebBrowser\", \"execute\"]\n", + "caos_action = [\"network\", \"node\", \"some_tech_jnr_dev_pc\", \"application\", \"web-browser\", \"execute\"]\n", "game.simulation.apply_request(caos_action)" ] }, diff --git a/src/primaite/notebooks/Requests-and-Responses.ipynb b/src/primaite/notebooks/Requests-and-Responses.ipynb index 48f1cf3e..c29d41dc 100644 --- a/src/primaite/notebooks/Requests-and-Responses.ipynb +++ b/src/primaite/notebooks/Requests-and-Responses.ipynb @@ -52,14 +52,15 @@ "outputs": [], "source": [ "sim = Simulation()\n", + "\n", "sim.network.add_node(\n", " HostNode.from_config(\n", " config = {\n", - " 'type': \"hostnode\",\n", + " 'type': \"host-node\",\n", " 'hostname': \"client\",\n", " 'ip_address': '10.0.0.1',\n", " 'subnet_mask': '255.255.255.0',\n", - " 'operating_state': NodeOperatingState.ON,\n", + " 'operating_state': \"ON\",\n", " }\n", " )\n", ")\n", @@ -98,7 +99,7 @@ "outputs": [], "source": [ "response = sim.apply_request(\n", - " request=[\"network\", \"node\", \"client\", \"service\", \"DNSClient\", \"stop\"],\n", + " request=[\"network\", \"node\", \"client\", \"service\", \"dns-client\", \"stop\"],\n", " context={}\n", " )\n", "print(response)" @@ -118,7 +119,7 @@ "metadata": {}, "outputs": [], "source": [ - "print(f\"DNS Client state: {client.software_manager.software.get('DNSClient').operating_state.name}\")" + "print(f\"DNS Client state: {client.software_manager.software.get('dns-client').operating_state.name}\")" ] }, { @@ -142,7 +143,7 @@ "outputs": [], "source": [ "response = sim.apply_request(\n", - " request=[\"network\", \"node\", \"client\", \"service\", \"NonExistentApplication\", \"stop\"],\n", + " request=[\"network\", \"node\", \"client\", \"service\", \"non-existent-application\", \"stop\"],\n", " context={}\n", " )\n", "print(response)" @@ -205,7 +206,7 @@ "outputs": [], "source": [ "response = sim.apply_request(\n", - " request=[\"network\", \"node\", \"client\", \"service\", \"DNSClient\", \"start\"],\n", + " request=[\"network\", \"node\", \"client\", \"service\", \"dns-client\", \"start\"],\n", " context={}\n", " )\n", "print(response)" diff --git a/src/primaite/notebooks/Terminal-Processing.ipynb b/src/primaite/notebooks/Terminal-Processing.ipynb index dc870db5..5077732b 100644 --- a/src/primaite/notebooks/Terminal-Processing.ipynb +++ b/src/primaite/notebooks/Terminal-Processing.ipynb @@ -6,7 +6,7 @@ "source": [ "# Terminal Processing\n", "\n", - "© Crown-owned copyright 2024, Defence Science and Technology Laboratory UK" + "© Crown-owned copyright 2025, Defence Science and Technology Laboratory UK" ] }, { @@ -39,41 +39,27 @@ "from primaite.simulator.system.applications.red_applications.ransomware_script import RansomwareScript\n", "from primaite.simulator.system.services.terminal.terminal import RemoteTerminalConnection\n", "\n", - "# print(dir(Computer))\n", - "# node_a = Computer.from_config(\n", - "# config = {\n", - "# \"type\": \"computer\",\n", - "# \"hostname\": \"node_a\",\n", - "# \"ip_address\": \"192.168.0.10\",\n", - "# \"subnet_mask\": \"255.255.255.0\",\n", - "# \"startup_duration\": 0,\n", - "# }\n", - "# )\n", - "# print(f\"{node_a=}\")\n", - "\n", "def basic_network() -> Network:\n", " \"\"\"Utility function for creating a default network to demonstrate Terminal functionality\"\"\"\n", " network = Network()\n", - " # node_a = Computer(hostname=\"node_a\", ip_address=\"192.168.0.10\", subnet_mask=\"255.255.255.0\", start_up_duration=0)\n", " node_a = Computer.from_config(\n", " config = {\n", " \"type\": \"computer\",\n", " \"hostname\": \"node_a\",\n", " \"ip_address\": \"192.168.0.10\",\n", " \"subnet_mask\": \"255.255.255.0\",\n", - " \"startup_duration\": 0,\n", + " # \"startup_duration\": 0,\n", " }\n", " )\n", " print(f\"{node_a=}\")\n", " node_a.power_on()\n", - " # node_b = Computer(hostname=\"node_b\", ip_address=\"192.168.0.11\", subnet_mask=\"255.255.255.0\", start_up_duration=0)\n", " node_b = Computer.from_config(\n", " config = {\n", " \"type\": \"computer\",\n", " \"hostname\": \"node_b\",\n", " \"ip_address\": \"192.168.0.11\",\n", " \"subnet_mask\": \"255.255.255.0\",\n", - " \"startup_duration\": 0,\n", + " # \"startup_duration\": 0,\n", " }\n", " )\n", " node_b.power_on()\n", @@ -98,9 +84,9 @@ "source": [ "network: Network = basic_network()\n", "computer_a: Computer = network.get_node_by_hostname(\"node_a\")\n", - "terminal_a: Terminal = computer_a.software_manager.software.get(\"Terminal\")\n", + "terminal_a: Terminal = computer_a.software_manager.software.get(\"terminal\")\n", "computer_b: Computer = network.get_node_by_hostname(\"node_b\")\n", - "terminal_b: Terminal = computer_b.software_manager.software.get(\"Terminal\")" + "terminal_b: Terminal = computer_b.software_manager.software.get(\"terminal\")" ] }, { @@ -152,7 +138,7 @@ "metadata": {}, "outputs": [], "source": [ - "term_a_term_b_remote_connection.execute([\"software_manager\", \"application\", \"install\", \"RansomwareScript\"])" + "term_a_term_b_remote_connection.execute([\"software_manager\", \"application\", \"install\", \"ransomware-script\"])" ] }, { diff --git a/src/primaite/simulator/network/hardware/base.py b/src/primaite/simulator/network/hardware/base.py index 4712af60..4c252050 100644 --- a/src/primaite/simulator/network/hardware/base.py +++ b/src/primaite/simulator/network/hardware/base.py @@ -1632,7 +1632,9 @@ class Node(SimComponent, ABC): dns_server=kwargs["config"].dns_server, ) super().__init__(**kwargs) - self.operating_state = NodeOperatingState.ON if not (p := kwargs["config"].operating_state) else p + self.operating_state = ( + NodeOperatingState.ON if not (p := kwargs["config"].operating_state) else NodeOperatingState[p.upper()] + ) self._install_system_software() self.session_manager.node = self self.session_manager.software_manager = self.software_manager diff --git a/src/primaite/simulator/network/hardware/nodes/host/host_node.py b/src/primaite/simulator/network/hardware/nodes/host/host_node.py index 76d9167c..d38ba097 100644 --- a/src/primaite/simulator/network/hardware/nodes/host/host_node.py +++ b/src/primaite/simulator/network/hardware/nodes/host/host_node.py @@ -313,7 +313,7 @@ class HostNode(Node, discriminator="host-node"): """ SYSTEM_SOFTWARE: ClassVar[Dict] = { - "HostARP": HostARP, + "host-arp": HostARP, "icmp": ICMP, "dns-client": DNSClient, "ntp-client": NTPClient, diff --git a/tests/assets/configs/basic_switched_network.yaml b/tests/assets/configs/basic_switched_network.yaml index c9ac5f8d..4c6d2557 100644 --- a/tests/assets/configs/basic_switched_network.yaml +++ b/tests/assets/configs/basic_switched_network.yaml @@ -34,10 +34,10 @@ agents: action_space: action_map: 0: - action: do-nothing + action: do_nothing options: {} 1: - action: node-application-execute + action: node_application_execute options: node_name: client_2 application_name: web-browser @@ -109,7 +109,7 @@ agents: action_space: action_map: 0: - action: do-nothing + action: do_nothing options: {} reward_function: From 486c797d60422c6f0acfea7a2e9cf9094bf4b42c Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Mon, 17 Feb 2025 11:54:13 +0000 Subject: [PATCH 79/95] #3075: Change duplicate application_name in config file. --- src/primaite/config/_package_data/data_manipulation.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/primaite/config/_package_data/data_manipulation.yaml b/src/primaite/config/_package_data/data_manipulation.yaml index 4705c135..ceee607d 100644 --- a/src/primaite/config/_package_data/data_manipulation.yaml +++ b/src/primaite/config/_package_data/data_manipulation.yaml @@ -81,7 +81,7 @@ agents: action: node-application-execute options: node_name: client_1 - application_name: web-browser + application_name: database-client reward_function: reward_components: From 7e82de919c4287eb64d4e47b84808be2d2525f61 Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Mon, 17 Feb 2025 14:19:56 +0000 Subject: [PATCH 80/95] #3075: Markdown changes. --- ...ommand-and-Control-E2E-Demonstration.ipynb | 85 +- ...a-Manipulation-Customising-Red-Agent.ipynb | 48 +- .../Data-Manipulation-E2E-Demonstration.ipynb | 4531 ++++++++++++++++- 3 files changed, 4548 insertions(+), 116 deletions(-) diff --git a/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb b/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb index 474dadff..6dc8e077 100644 --- a/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb +++ b/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb @@ -6,7 +6,7 @@ "source": [ "# Command and Control Application Suite E2E Demonstration\n", "\n", - "© Crown-owned copyright 2024, Defence Science and Technology Laboratory UK\n", + "© Crown-owned copyright 2025, Defence Science and Technology Laboratory UK\n", "\n", "This notebook demonstrates the current implementation of the command and control (C2) server and beacon applications in primAITE." ] @@ -201,15 +201,15 @@ " nodes: # Node List\n", " - node_name: web_server\n", " applications: \n", - " - application_name: C2Beacon\n", + " - application_name: c2-beacon\n", " ...\n", " ...\n", " action_map:\n", " 1:\n", - " action: node_application_install \n", + " action: node-application-install \n", " options:\n", " node_id: 0 # Index 0 at the node list.\n", - " application_name: C2Beacon\n", + " application_name: c2-beacon\n", "```" ] }, @@ -244,7 +244,7 @@ " action_map:\n", " ...\n", " 2:\n", - " action: configure_c2_beacon\n", + " action: configure-c2-beacon\n", " options:\n", " node_id: 0 # Node Index\n", " config: # Further information about these config options can be found at the bottom of this notebook.\n", @@ -273,7 +273,7 @@ "source": [ "### **Command and Control** | C2 Beacon Actions | node_application_execute\n", "\n", - "The final action is ``node_application_execute`` which is used to establish a connection for the C2 application. This action can be called by the Red Agent via action ``3`` in it's action map. \n", + "The final action is ``node-application-execute`` which is used to establish a connection for the C2 application. This action can be called by the Red Agent via action ``3`` in it's action map. \n", "\n", "The yaml snippet below shows all the relevant agent options for this action:\n", "\n", @@ -283,13 +283,13 @@ " nodes: # Node List\n", " - node_name: web_server\n", " applications: \n", - " - application_name: C2Beacon\n", + " - application_name: c2-beacon\n", " ...\n", " ...\n", " action_map:\n", " ...\n", " 3:\n", - " action: node_application_execute\n", + " action: node-application-execute\n", " options:\n", " node_id: 0\n", " application_id: 0\n", @@ -331,34 +331,27 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### **Command and Control** | C2 Server Actions | C2_SERVER_TERMINAL_COMMAND\n", + "### **Command and Control** | C2 Server Actions | c2-server-terminal-command\n", "\n", - "The C2 Server's terminal action: ``C2_SERVER_TERMINAL_COMMAND`` is indexed at ``4`` in it's action map. \n", + "The C2 Server's terminal action: ``c2-server-terminal-command`` is indexed at ``4`` in it's action map. \n", "\n", "This action leverages the terminal service that is installed by default on all nodes to grant red agents a lot more configurability. If you're unfamiliar with terminals then it's recommended that you refer to the ``Terminal Processing`` notebook.\n", "\n", "It's worth noting that an additional benefit a red agent has when using the terminal service via the C2 Server is that you can execute multiple commands in one action. \n", "\n", - "In this notebook, the ``C2_SERVER_TERMINAL_COMMAND`` is used to install a RansomwareScript application on the ``web_server`` node.\n", + "In this notebook, the ``c2-server-terminal-command`` is used to install a RansomwareScript application on the ``web_server`` node.\n", "\n", "The yaml snippet below shows all the relevant agent options for this action:\n", "\n", "``` yaml\n", " action_space:\n", - " options:\n", - " nodes: # Node List\n", - " ...\n", - " - node_name: client_1\n", - " applications: \n", - " - application_name: C2Server\n", - " ...\n", " action_map:\n", + " ...\n", " 4:\n", - " action: C2_SERVER_TERMINAL_COMMAND\n", + " action: c2-server-terminal-command\n", " options:\n", - " node_id: 1\n", + " node_name: client_1\n", " ip_address:\n", - " account:\n", " username: admin\n", " password: admin\n", " commands:\n", @@ -366,7 +359,7 @@ " - software_manager\n", " - application\n", " - install\n", - " - RansomwareScript\n", + " - ransomware-script\n", "```" ] }, @@ -402,14 +395,8 @@ "\n", "``` yaml\n", " action_space:\n", - " options:\n", - " nodes: # Node List\n", - " ...\n", - " - node_name: client_1\n", - " applications: \n", - " - application_name: C2Server\n", - " ...\n", " action_map:\n", + " ...\n", " 5:\n", " action: c2-server-ransomware-configure\n", " options:\n", @@ -444,9 +431,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### **Command and Control** | C2 Server Actions | c2_server_data_exfiltrate\n", + "### **Command and Control** | C2 Server Actions | c2-server-data-exfiltrate\n", "\n", - "The second to last action available is the ``c2_server_data_exfiltrate`` which is indexed as action ``6`` in the action map.\n", + "The second to last action available is the ``c2-server-data-exfiltrate`` which is indexed as action ``6`` in the action map.\n", "\n", "This action can be used to exfiltrate a target file on a remote node to the C2 Beacon and the C2 Server's host file system via the ``FTP`` services.\n", "\n", @@ -454,25 +441,18 @@ "\n", "``` yaml\n", " action_space:\n", - " options:\n", - " nodes: # Node List\n", - " ...\n", - " - node_name: client_1\n", - " applications: \n", - " - application_name: C2Server\n", - " ...\n", " action_map:\n", + " ...\n", " 6:\n", - " action: c2_server_data_exfiltrate\n", + " action: c2-server-data-exfiltrate\n", " options:\n", " node_id: 1\n", " target_file_name: \"database.db\"\n", " target_folder_name: \"database\"\n", " exfiltration_folder_name: \"spoils\"\n", " target_ip_address: \"192.168.1.14\"\n", - " account:\n", - " username: \"admin\",\n", - " password: \"admin\"\n", + " username: \"admin\",\n", + " password: \"admin\"\n", "\n", "```" ] @@ -510,9 +490,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### **Command and Control** | C2 Server Actions | c2_server_ransomware_launch\n", + "### **Command and Control** | C2 Server Actions | c2-server-ransomware-launch\n", "\n", - "Finally, the last available action is for the c2_server_ransomware_launch to start the ransomware script installed on the same node as the C2 beacon.\n", + "Finally, the last available action is for the c2-server-ransomware-launch to start the ransomware script installed on the same node as the C2 beacon.\n", "\n", "This action is indexed as action ``7``.\n", "\n", @@ -520,16 +500,10 @@ "\n", "``` yaml\n", " action_space:\n", - " options:\n", - " nodes: # Node List\n", - " ...\n", - " - node_name: client_1\n", - " applications: \n", - " - application_name: C2Server\n", " ...\n", " action_map:\n", " 7:\n", - " action: c2_server_ransomware_launch\n", + " action: c2-server-ransomware-launch\n", " options:\n", " node_id: 1\n", "```\n" @@ -1337,18 +1311,15 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "As demonstrated earlier, red agents can use the ``configure_c2_beacon`` action to configure these settings mid episode through the configuration options:\n", + "As demonstrated earlier, red agents can use the ``configure-c2-beacon`` action to configure these settings mid episode through the configuration options:\n", "\n", "``` YAML\n", "...\n", - " action: configure_c2_beacon\n", + " action: configure-c2-beacon\n", " options:\n", - " node_id: 0\n", + " node_name: web_server\n", " config:\n", " c2_server_ip_address: 192.168.10.21\n", - " keep_alive_frequency: 10\n", - " masquerade_protocol: TCP\n", - " masquerade_port: DNS\n", "```" ] }, diff --git a/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb b/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb index 2e0a0d40..1eb814e3 100644 --- a/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb +++ b/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb @@ -146,42 +146,15 @@ "```yaml\n", " - ref: data_manipulation_attacker # name of agent\n", " team: RED # not used, just for human reference\n", - " type: RedDatabaseCorruptingAgent # type of agent - this lets primaite know which agent class to use\n", - "\n", - " # Since the agent does not need to react to what is happening in the environment, the observation space is empty.\n", - " observation_space:\n", - " type: UC2RedObservation\n", - " options:\n", - " nodes: {}\n", - "\n", - " action_space:\n", - " \n", - " # The agent has access to the DataManipulationBoth on clients 1 and 2.\n", - " options:\n", - " nodes:\n", - " - node_name: client_1 # The network should have a node called client_1\n", - " applications:\n", - " - application_name: DataManipulationBot # The node client_1 should have DataManipulationBot configured on it\n", - " - node_name: client_2 # The network should have a node called client_2\n", - " applications:\n", - " - application_name: DataManipulationBot # The node client_2 should have DataManipulationBot configured on it\n", - "\n", - " # not important\n", - " max_folders_per_node: 1\n", - " max_files_per_folder: 1\n", - " max_services_per_node: 1\n", - "\n", - " # red agent does not need a reward function\n", - " reward_function:\n", - " reward_components:\n", - " - type: DUMMY\n", + " type: red-database-corrupting-agent # type of agent - this lets primaite know which agent class to use\n", "\n", " # These actions are passed to the RedDatabaseCorruptingAgent init method, they dictate the schedule of attacks\n", " agent_settings:\n", - " start_settings:\n", - " start_step: 25 # first attack at step 25\n", - " frequency: 20 # attacks will happen every 20 steps (on average)\n", - " variance: 5 # the timing of attacks will vary by up to 5 steps earlier or later\n", + " possible_start_nodes: [client_1, client_2] # List of clients the attack can start from\n", + " target_application: data-manipulation-bot\n", + " start_step: 25 # first attack at step 25\n", + " frequency: 20 # attacks will happen every 20 steps (on average)\n", + " variance: 5 # the timing of attacks will vary by up to 5 steps earlier or later\n", "```" ] }, @@ -201,8 +174,7 @@ "simulation:\n", " network:\n", " nodes:\n", - " - ref: client_1\n", - " hostname: client_1\n", + " - hostname: client_1\n", " type: computer\n", " ip_address: 192.168.10.21\n", " subnet_mask: 255.255.255.0\n", @@ -210,15 +182,13 @@ " \n", " # \n", " applications:\n", - " - ref: data_manipulation_bot\n", - " type: DataManipulationBot\n", + " - type: data-manipulation-bot\n", " options:\n", " port_scan_p_of_success: 0.8 # Probability that port scan is successful\n", " data_manipulation_p_of_success: 0.8 # Probability that SQL attack is successful\n", " payload: \"DELETE\" # The SQL query which causes the attack (this has to be DELETE)\n", " server_ip: 192.168.1.14 # IP address of server hosting the database\n", - " - ref: client_1_database_client\n", - " type: DatabaseClient # Database client must be installed in order for DataManipulationBot to function\n", + " - type: database-client # Database client must be installed in order for DataManipulationBot to function\n", " options:\n", " db_server_ip: 192.168.1.14 # IP address of server hosting the database\n", "```" diff --git a/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb b/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb index 2d3f99b2..f3e3fb98 100644 --- a/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb +++ b/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb @@ -6,7 +6,7 @@ "source": [ "# Data Manipulation Scenario\n", "\n", - "© Crown-owned copyright 2024, Defence Science and Technology Laboratory UK" + "© Crown-owned copyright 2025, Defence Science and Technology Laboratory UK" ] }, { @@ -373,16 +373,29 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-02-17 11:15:30,959: Performing the PrimAITE first-time setup...\n", + "2025-02-17 11:15:30,960: Building the PrimAITE app directories...\n", + "2025-02-17 11:15:30,960: Building primaite_config.yaml...\n", + "2025-02-17 11:15:30,960: Rebuilding the demo notebooks...\n", + "2025-02-17 11:15:30,967: Rebuilding the example notebooks...\n", + "2025-02-17 11:15:30,970: PrimAITE setup complete!\n" + ] + } + ], "source": [ "!primaite setup" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": { "tags": [] }, @@ -394,7 +407,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": { "tags": [] }, @@ -420,9 +433,254 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-17 11:15:35,108: PrimaiteGymEnv RNG seed = None\n", + "2025-02-17 11:15:35,109: Resetting environment, episode 0, avg. reward: 0.0\n", + "2025-02-17 11:15:35,111: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-17/11-15-31/agent_actions/episode_0.json\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "env created successfully\n", + "{'ICS': 0,\n", + " 'LINKS': {1: {'PROTOCOLS': {'ALL': 0}},\n", + " 2: {'PROTOCOLS': {'ALL': 0}},\n", + " 3: {'PROTOCOLS': {'ALL': 0}},\n", + " 4: {'PROTOCOLS': {'ALL': 0}},\n", + " 5: {'PROTOCOLS': {'ALL': 0}},\n", + " 6: {'PROTOCOLS': {'ALL': 0}},\n", + " 7: {'PROTOCOLS': {'ALL': 0}},\n", + " 8: {'PROTOCOLS': {'ALL': 0}},\n", + " 9: {'PROTOCOLS': {'ALL': 0}},\n", + " 10: {'PROTOCOLS': {'ALL': 0}}},\n", + " 'NODES': {'HOST0': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0,\n", + " 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0,\n", + " 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0,\n", + " 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0,\n", + " 'outbound': 0}}},\n", + " 'nic_status': 0}},\n", + " 'SERVICES': {1: {'health_status': 0,\n", + " 'operating_status': 0}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'HOST1': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0,\n", + " 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0,\n", + " 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0,\n", + " 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0,\n", + " 'outbound': 0}}},\n", + " 'nic_status': 0}},\n", + " 'SERVICES': {1: {'health_status': 0,\n", + " 'operating_status': 1}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'HOST2': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0,\n", + " 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0,\n", + " 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0,\n", + " 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0,\n", + " 'outbound': 0}}},\n", + " 'nic_status': 0}},\n", + " 'SERVICES': {1: {'health_status': 0,\n", + " 'operating_status': 0}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'HOST3': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0,\n", + " 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0,\n", + " 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0,\n", + " 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0,\n", + " 'outbound': 0}}},\n", + " 'nic_status': 0}},\n", + " 'SERVICES': {1: {'health_status': 0,\n", + " 'operating_status': 0}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'HOST4': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0,\n", + " 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0,\n", + " 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0,\n", + " 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0,\n", + " 'outbound': 0}}},\n", + " 'nic_status': 1}},\n", + " 'SERVICES': {1: {'health_status': 0,\n", + " 'operating_status': 0}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'HOST5': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0,\n", + " 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0,\n", + " 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0,\n", + " 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0,\n", + " 'outbound': 0}}},\n", + " 'nic_status': 0}},\n", + " 'SERVICES': {1: {'health_status': 0,\n", + " 'operating_status': 0}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'HOST6': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0,\n", + " 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0,\n", + " 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0,\n", + " 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0,\n", + " 'outbound': 0}}},\n", + " 'nic_status': 0}},\n", + " 'SERVICES': {1: {'health_status': 0,\n", + " 'operating_status': 0}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'ROUTER0': {'ACL': {1: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 0,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 2: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 1,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 3: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 2,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 4: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 3,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 5: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 4,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 6: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 5,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 7: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 6,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 8: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 7,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 9: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 8,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 10: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 9,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0}},\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}}}}\n" + ] + } + ], "source": [ "# create the env\n", "with open(data_manipulation_config_path(), 'r') as f:\n", @@ -450,7 +708,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -468,9 +726,51 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "step: 1, Red action: DO NOTHING, Blue reward:0.65\n", + "step: 2, Red action: DO NOTHING, Blue reward:0.65\n", + "step: 3, Red action: DO NOTHING, Blue reward:0.65\n", + "step: 4, Red action: DO NOTHING, Blue reward:0.90\n", + "step: 5, Red action: DO NOTHING, Blue reward:0.90\n", + "step: 6, Red action: DO NOTHING, Blue reward:0.90\n", + "step: 7, Red action: DO NOTHING, Blue reward:0.90\n", + "step: 8, Red action: DO NOTHING, Blue reward:0.90\n", + "step: 9, Red action: DO NOTHING, Blue reward:0.90\n", + "step: 10, Red action: DO NOTHING, Blue reward:0.90\n", + "step: 11, Red action: DO NOTHING, Blue reward:0.90\n", + "step: 12, Red action: DO NOTHING, Blue reward:0.90\n", + "step: 13, Red action: DO NOTHING, Blue reward:0.90\n", + "step: 14, Red action: DO NOTHING, Blue reward:0.90\n", + "step: 15, Red action: DO NOTHING, Blue reward:0.90\n", + "step: 16, Red action: DO NOTHING, Blue reward:0.90\n", + "step: 17, Red action: DO NOTHING, Blue reward:0.95\n", + "step: 18, Red action: DO NOTHING, Blue reward:0.95\n", + "step: 19, Red action: DO NOTHING, Blue reward:0.95\n", + "step: 20, Red action: DO NOTHING, Blue reward:0.95\n", + "step: 21, Red action: DO NOTHING, Blue reward:0.95\n", + "step: 22, Red action: DO NOTHING, Blue reward:0.95\n", + "step: 23, Red action: DO NOTHING, Blue reward:0.95\n", + "step: 24, Red action: DO NOTHING, Blue reward:0.95\n", + "step: 25, Red action: DO NOTHING, Blue reward:0.95\n", + "step: 26, Red action: ATTACK from client 2, Blue reward:0.15\n", + "step: 27, Red action: DO NOTHING, Blue reward:-0.35\n", + "step: 28, Red action: DO NOTHING, Blue reward:-0.85\n", + "step: 29, Red action: DO NOTHING, Blue reward:-0.85\n", + "step: 30, Red action: DO NOTHING, Blue reward:-0.85\n", + "step: 31, Red action: DO NOTHING, Blue reward:-0.85\n", + "step: 32, Red action: DO NOTHING, Blue reward:-0.85\n", + "step: 33, Red action: DO NOTHING, Blue reward:-0.85\n", + "step: 34, Red action: DO NOTHING, Blue reward:-0.85\n", + "step: 35, Red action: DO NOTHING, Blue reward:-0.85\n" + ] + } + ], "source": [ "for step in range(35):\n", " obs, reward, terminated, truncated, info = env.step(0)\n", @@ -486,9 +786,198 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'HOST0': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 0}},\n", + " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'HOST1': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 0}},\n", + " 'SERVICES': {1: {'health_status': 0, 'operating_status': 1}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'HOST2': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 0}},\n", + " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'HOST3': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 0}},\n", + " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'HOST4': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 1}},\n", + " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'HOST5': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 0}},\n", + " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'HOST6': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 0}},\n", + " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'ROUTER0': {'ACL': {1: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 0,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 2: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 1,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 3: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 2,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 4: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 3,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 5: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 4,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 6: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 5,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 7: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 6,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 8: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 7,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 9: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 8,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 10: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 9,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0}},\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}}}\n" + ] + } + ], "source": [ "pprint(obs['NODES'])" ] @@ -502,9 +991,198 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'HOST0': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 0}},\n", + " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'HOST1': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 0}},\n", + " 'SERVICES': {1: {'health_status': 3, 'operating_status': 1}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'HOST2': {'FOLDERS': {1: {'FILES': {1: {'health_status': 2}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 0}},\n", + " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'HOST3': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 0}},\n", + " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'HOST4': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 1}},\n", + " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'HOST5': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 0}},\n", + " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'HOST6': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 0}},\n", + " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'ROUTER0': {'ACL': {1: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 0,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 2: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 1,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 3: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 2,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 4: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 3,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 5: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 4,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 6: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 5,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 7: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 6,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 8: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 7,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 9: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 8,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 10: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 9,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0}},\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}}}\n" + ] + } + ], "source": [ "obs, reward, terminated, truncated, info = env.step(9) # scan database file\n", "obs, reward, terminated, truncated, info = env.step(1) # scan webapp service\n", @@ -536,9 +1214,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "step: 38\n", + "Red action: do-nothing\n", + "Green action: do-nothing\n", + "Green action: node-application-execute\n", + "Blue reward:-0.8500000000000001\n" + ] + } + ], "source": [ "obs, reward, terminated, truncated, info = env.step(13) # patch the database\n", "print(f\"step: {env.game.step_counter}\")\n", @@ -561,9 +1251,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "step: 39\n", + "Red action: do-nothing\n", + "Green action: timestep=38 action='node-application-execute' parameters={'node_name': 'client_2', 'application_name': 'web-browser'} request=['network', 'node', 'client_2', 'application', 'web-browser', 'execute'] response=RequestResponse(status='failure', data={}) reward=-0.2 reward_info={'connection_attempt_status': 'n/a'}\n", + "Green action: timestep=38 action='node-application-execute' parameters={'node_name': 'client_1', 'application_name': 'web-browser'} request=['network', 'node', 'client_1', 'application', 'web-browser', 'execute'] response=RequestResponse(status='failure', data={}) reward=-0.25 reward_info={'connection_attempt_status': 'n/a'}\n", + "Blue reward:-0.05\n" + ] + } + ], "source": [ "obs, reward, terminated, truncated, info = env.step(0) # do nothing\n", "print(f\"step: {env.game.step_counter}\")\n", @@ -586,7 +1288,3796 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "step: 40, Red action: do-nothing, Blue reward:-0.05\n", + "step: 41, Red action: do-nothing, Blue reward:-0.05\n", + "step: 42, Red action: do-nothing, Blue reward:-0.05\n", + "step: 43, Red action: do-nothing, Blue reward:0.95\n", + "step: 44, Red action: do-nothing, Blue reward:0.95\n", + "step: 45, Red action: do-nothing, Blue reward:0.95\n", + "step: 46, Red action: do-nothing, Blue reward:0.95\n", + "step: 47, Red action: do-nothing, Blue reward:0.95\n", + "step: 48, Red action: do-nothing, Blue reward:0.95\n", + "step: 49, Red action: do-nothing, Blue reward:0.95\n", + "step: 50, Red action: do-nothing, Blue reward:0.95\n", + "step: 51, Red action: node-application-execute, Blue reward:0.95\n", + "step: 52, Red action: do-nothing, Blue reward:0.95\n", + "step: 53, Red action: do-nothing, Blue reward:0.85\n", + "step: 54, Red action: do-nothing, Blue reward:0.85\n", + "step: 55, Red action: do-nothing, Blue reward:0.85\n", + "step: 56, Red action: do-nothing, Blue reward:0.85\n", + "step: 57, Red action: do-nothing, Blue reward:0.85\n", + "step: 58, Red action: do-nothing, Blue reward:0.85\n", + "step: 59, Red action: do-nothing, Blue reward:0.85\n", + "step: 60, Red action: do-nothing, Blue reward:0.85\n", + "step: 61, Red action: do-nothing, Blue reward:0.85\n", + "step: 62, Red action: do-nothing, Blue reward:0.85\n", + "step: 63, Red action: do-nothing, Blue reward:0.85\n", + "step: 64, Red action: do-nothing, Blue reward:0.85\n", + "step: 65, Red action: do-nothing, Blue reward:0.85\n", + "step: 66, Red action: do-nothing, Blue reward:0.85\n", + "step: 67, Red action: do-nothing, Blue reward:0.85\n", + "step: 68, Red action: do-nothing, Blue reward:0.85\n", + "step: 69, Red action: do-nothing, Blue reward:0.85\n", + "step: 70, Red action: do-nothing, Blue reward:0.85\n", + "step: 71, Red action: do-nothing, Blue reward:0.85\n", + "step: 72, Red action: do-nothing, Blue reward:0.85\n", + "step: 73, Red action: do-nothing, Blue reward:0.85\n", + "step: 74, Red action: do-nothing, Blue reward:0.85\n", + "step: 75, Red action: node-application-execute, Blue reward:0.85\n", + "step: 76, Red action: do-nothing, Blue reward:0.85\n", + "step: 77, Red action: do-nothing, Blue reward:0.85\n", + "step: 78, Red action: do-nothing, Blue reward:0.85\n", + "step: 79, Red action: do-nothing, Blue reward:0.85\n", + "step: 80, Red action: do-nothing, Blue reward:0.85\n", + "step: 81, Red action: do-nothing, Blue reward:0.85\n", + "step: 82, Red action: do-nothing, Blue reward:0.85\n", + "step: 83, Red action: do-nothing, Blue reward:0.85\n", + "step: 84, Red action: do-nothing, Blue reward:0.85\n", + "step: 85, Red action: do-nothing, Blue reward:0.85\n", + "step: 86, Red action: do-nothing, Blue reward:0.85\n", + "step: 87, Red action: do-nothing, Blue reward:0.85\n", + "step: 88, Red action: do-nothing, Blue reward:0.85\n", + "step: 89, Red action: do-nothing, Blue reward:0.85\n", + "step: 90, Red action: do-nothing, Blue reward:0.85\n", + "step: 91, Red action: do-nothing, Blue reward:0.85\n", + "step: 92, Red action: do-nothing, Blue reward:0.85\n", + "step: 93, Red action: do-nothing, Blue reward:0.85\n", + "step: 94, Red action: do-nothing, Blue reward:0.85\n", + "step: 95, Red action: do-nothing, Blue reward:0.85\n", + "step: 96, Red action: do-nothing, Blue reward:0.85\n", + "step: 97, Red action: node-application-execute, Blue reward:0.85\n", + "step: 98, Red action: do-nothing, Blue reward:0.85\n", + "step: 99, Red action: do-nothing, Blue reward:0.85\n", + "step: 100, Red action: do-nothing, Blue reward:0.85\n", + "step: 101, Red action: do-nothing, Blue reward:0.85\n", + "step: 102, Red action: do-nothing, Blue reward:0.85\n", + "step: 103, Red action: do-nothing, Blue reward:0.85\n", + "step: 104, Red action: do-nothing, Blue reward:0.85\n", + "step: 105, Red action: do-nothing, Blue reward:0.85\n", + "step: 106, Red action: do-nothing, Blue reward:0.85\n", + "step: 107, Red action: do-nothing, Blue reward:0.85\n", + "step: 108, Red action: do-nothing, Blue reward:0.85\n", + "step: 109, Red action: do-nothing, Blue reward:0.85\n", + "step: 110, Red action: do-nothing, Blue reward:0.85\n", + "step: 111, Red action: do-nothing, Blue reward:0.85\n", + "step: 112, Red action: do-nothing, Blue reward:0.85\n", + "step: 113, Red action: node-application-execute, Blue reward:0.85\n", + "step: 114, Red action: do-nothing, Blue reward:0.85\n", + "step: 115, Red action: do-nothing, Blue reward:0.85\n", + "step: 116, Red action: do-nothing, Blue reward:0.85\n", + "step: 117, Red action: do-nothing, Blue reward:0.85\n", + "step: 118, Red action: do-nothing, Blue reward:0.85\n", + "step: 119, Red action: do-nothing, Blue reward:0.85\n", + "step: 120, Red action: do-nothing, Blue reward:0.85\n", + "step: 121, Red action: do-nothing, Blue reward:0.85\n", + "step: 122, Red action: do-nothing, Blue reward:0.85\n", + "step: 123, Red action: do-nothing, Blue reward:0.85\n", + "step: 124, Red action: do-nothing, Blue reward:0.85\n", + "step: 125, Red action: do-nothing, Blue reward:0.85\n", + "step: 126, Red action: do-nothing, Blue reward:0.85\n", + "step: 127, Red action: do-nothing, Blue reward:0.85\n", + "step: 128, Red action: do-nothing, Blue reward:0.85\n", + "step: 129, Red action: do-nothing, Blue reward:0.85\n", + "step: 130, Red action: do-nothing, Blue reward:0.85\n", + "step: 131, Red action: do-nothing, Blue reward:0.85\n", + "step: 132, Red action: node-application-execute, Blue reward:0.85\n", + "step: 133, Red action: do-nothing, Blue reward:0.85\n", + "step: 134, Red action: do-nothing, Blue reward:0.85\n", + "step: 135, Red action: do-nothing, Blue reward:0.85\n", + "step: 136, Red action: do-nothing, Blue reward:0.85\n", + "step: 137, Red action: do-nothing, Blue reward:0.85\n", + "step: 138, Red action: do-nothing, Blue reward:0.85\n", + "step: 139, Red action: do-nothing, Blue reward:0.85\n", + "step: 140, Red action: do-nothing, Blue reward:0.85\n", + "step: 141, Red action: do-nothing, Blue reward:0.85\n", + "step: 142, Red action: do-nothing, Blue reward:0.85\n", + "step: 143, Red action: do-nothing, Blue reward:0.85\n", + "step: 144, Red action: do-nothing, Blue reward:0.85\n", + "step: 145, Red action: do-nothing, Blue reward:0.85\n", + "step: 146, Red action: do-nothing, Blue reward:0.85\n", + "step: 147, Red action: do-nothing, Blue reward:0.85\n", + "step: 148, Red action: do-nothing, Blue reward:0.85\n", + "step: 149, Red action: do-nothing, Blue reward:0.85\n", + "step: 150, Red action: do-nothing, Blue reward:0.85\n", + "step: 151, Red action: do-nothing, Blue reward:0.85\n", + "step: 152, Red action: do-nothing, Blue reward:0.85\n", + "step: 153, Red action: do-nothing, Blue reward:0.85\n", + "step: 154, Red action: do-nothing, Blue reward:0.85\n", + "step: 155, Red action: do-nothing, Blue reward:0.85\n", + "step: 156, Red action: do-nothing, Blue reward:0.85\n", + "step: 157, Red action: node-application-execute, Blue reward:0.85\n", + "step: 158, Red action: do-nothing, Blue reward:0.85\n", + "step: 159, Red action: do-nothing, Blue reward:0.85\n", + "step: 160, Red action: do-nothing, Blue reward:0.85\n", + "step: 161, Red action: do-nothing, Blue reward:0.85\n", + "step: 162, Red action: do-nothing, Blue reward:0.85\n", + "step: 163, Red action: do-nothing, Blue reward:0.85\n", + "step: 164, Red action: do-nothing, Blue reward:0.85\n", + "step: 165, Red action: do-nothing, Blue reward:0.85\n", + "step: 166, Red action: do-nothing, Blue reward:0.85\n", + "step: 167, Red action: do-nothing, Blue reward:0.85\n", + "step: 168, Red action: do-nothing, Blue reward:0.85\n", + "step: 169, Red action: do-nothing, Blue reward:0.85\n", + "step: 170, Red action: do-nothing, Blue reward:0.85\n", + "step: 171, Red action: do-nothing, Blue reward:0.85\n", + "step: 172, Red action: do-nothing, Blue reward:0.85\n", + "step: 173, Red action: do-nothing, Blue reward:0.85\n", + "step: 174, Red action: do-nothing, Blue reward:0.85\n", + "step: 175, Red action: do-nothing, Blue reward:0.85\n", + "step: 176, Red action: do-nothing, Blue reward:0.85\n", + "step: 177, Red action: do-nothing, Blue reward:0.85\n", + "step: 178, Red action: node-application-execute, Blue reward:0.85\n", + "step: 179, Red action: do-nothing, Blue reward:0.85\n", + "step: 180, Red action: do-nothing, Blue reward:0.85\n", + "step: 181, Red action: do-nothing, Blue reward:0.85\n", + "step: 182, Red action: do-nothing, Blue reward:0.85\n", + "step: 183, Red action: do-nothing, Blue reward:0.85\n", + "step: 184, Red action: do-nothing, Blue reward:0.85\n", + "step: 185, Red action: do-nothing, Blue reward:0.85\n", + "step: 186, Red action: do-nothing, Blue reward:0.85\n", + "step: 187, Red action: do-nothing, Blue reward:0.85\n", + "step: 188, Red action: do-nothing, Blue reward:0.85\n", + "step: 189, Red action: do-nothing, Blue reward:0.85\n", + "step: 190, Red action: do-nothing, Blue reward:0.85\n", + "step: 191, Red action: do-nothing, Blue reward:0.85\n", + "step: 192, Red action: do-nothing, Blue reward:0.85\n", + "step: 193, Red action: do-nothing, Blue reward:0.85\n", + "step: 194, Red action: do-nothing, Blue reward:0.85\n", + "step: 195, Red action: do-nothing, Blue reward:0.85\n", + "step: 196, Red action: do-nothing, Blue reward:0.85\n", + "step: 197, Red action: do-nothing, Blue reward:0.85\n", + "step: 198, Red action: node-application-execute, Blue reward:0.85\n", + "step: 199, Red action: do-nothing, Blue reward:0.85\n", + "step: 200, Red action: do-nothing, Blue reward:0.85\n", + "step: 201, Red action: do-nothing, Blue reward:0.85\n", + "step: 202, Red action: do-nothing, Blue reward:0.85\n", + "step: 203, Red action: do-nothing, Blue reward:0.85\n", + "step: 204, Red action: do-nothing, Blue reward:0.85\n", + "step: 205, Red action: do-nothing, Blue reward:0.85\n", + "step: 206, Red action: do-nothing, Blue reward:0.85\n", + "step: 207, Red action: do-nothing, Blue reward:0.85\n", + "step: 208, Red action: do-nothing, Blue reward:0.85\n", + "step: 209, Red action: do-nothing, Blue reward:0.85\n", + "step: 210, Red action: do-nothing, Blue reward:0.85\n", + "step: 211, Red action: do-nothing, Blue reward:0.85\n", + "step: 212, Red action: do-nothing, Blue reward:0.85\n", + "step: 213, Red action: do-nothing, Blue reward:0.85\n", + "step: 214, Red action: do-nothing, Blue reward:0.85\n", + "step: 215, Red action: do-nothing, Blue reward:0.85\n", + "step: 216, Red action: do-nothing, Blue reward:0.85\n", + "step: 217, Red action: do-nothing, Blue reward:0.85\n", + "step: 218, Red action: node-application-execute, Blue reward:0.85\n", + "step: 219, Red action: do-nothing, Blue reward:0.85\n", + "step: 220, Red action: do-nothing, Blue reward:0.85\n", + "step: 221, Red action: do-nothing, Blue reward:0.85\n", + "step: 222, Red action: do-nothing, Blue reward:0.85\n", + "step: 223, Red action: do-nothing, Blue reward:0.85\n", + "step: 224, Red action: do-nothing, Blue reward:0.85\n", + "step: 225, Red action: do-nothing, Blue reward:0.85\n", + "step: 226, Red action: do-nothing, Blue reward:0.85\n", + "step: 227, Red action: do-nothing, Blue reward:0.85\n", + "step: 228, Red action: do-nothing, Blue reward:0.85\n", + "step: 229, Red action: do-nothing, Blue reward:0.85\n", + "step: 230, Red action: do-nothing, Blue reward:0.85\n", + "step: 231, Red action: do-nothing, Blue reward:0.85\n", + "step: 232, Red action: do-nothing, Blue reward:0.85\n", + "step: 233, Red action: do-nothing, Blue reward:0.85\n", + "step: 234, Red action: do-nothing, Blue reward:0.85\n", + "step: 235, Red action: do-nothing, Blue reward:0.85\n", + "step: 236, Red action: do-nothing, Blue reward:0.85\n", + "step: 237, Red action: do-nothing, Blue reward:0.85\n", + "step: 238, Red action: do-nothing, Blue reward:0.85\n", + "step: 239, Red action: do-nothing, Blue reward:0.85\n", + "step: 240, Red action: do-nothing, Blue reward:0.85\n", + "step: 241, Red action: node-application-execute, Blue reward:0.85\n", + "step: 242, Red action: do-nothing, Blue reward:0.85\n", + "step: 243, Red action: do-nothing, Blue reward:0.85\n", + "step: 244, Red action: do-nothing, Blue reward:0.85\n", + "step: 245, Red action: do-nothing, Blue reward:0.85\n", + "step: 246, Red action: do-nothing, Blue reward:0.85\n", + "step: 247, Red action: do-nothing, Blue reward:0.85\n", + "step: 248, Red action: do-nothing, Blue reward:0.85\n", + "step: 249, Red action: do-nothing, Blue reward:0.85\n", + "step: 250, Red action: do-nothing, Blue reward:0.85\n", + "step: 251, Red action: do-nothing, Blue reward:0.85\n", + "step: 252, Red action: do-nothing, Blue reward:0.85\n", + "step: 253, Red action: do-nothing, Blue reward:0.85\n", + "step: 254, Red action: do-nothing, Blue reward:0.85\n", + "step: 255, Red action: do-nothing, Blue reward:0.85\n", + "step: 256, Red action: do-nothing, Blue reward:0.85\n", + "step: 257, Red action: do-nothing, Blue reward:0.85\n", + "step: 258, Red action: do-nothing, Blue reward:0.85\n", + "step: 259, Red action: do-nothing, Blue reward:0.85\n", + "step: 260, Red action: do-nothing, Blue reward:0.85\n", + "step: 261, Red action: node-application-execute, Blue reward:0.85\n", + "step: 262, Red action: do-nothing, Blue reward:0.85\n", + "step: 263, Red action: do-nothing, Blue reward:0.85\n", + "step: 264, Red action: do-nothing, Blue reward:0.85\n", + "step: 265, Red action: do-nothing, Blue reward:0.85\n", + "step: 266, Red action: do-nothing, Blue reward:0.85\n", + "step: 267, Red action: do-nothing, Blue reward:0.85\n", + "step: 268, Red action: do-nothing, Blue reward:0.85\n", + "step: 269, Red action: do-nothing, Blue reward:0.85\n", + "step: 270, Red action: do-nothing, Blue reward:0.85\n", + "step: 271, Red action: do-nothing, Blue reward:0.85\n", + "step: 272, Red action: do-nothing, Blue reward:0.85\n", + "step: 273, Red action: do-nothing, Blue reward:0.85\n", + "step: 274, Red action: do-nothing, Blue reward:0.85\n", + "step: 275, Red action: do-nothing, Blue reward:0.85\n", + "step: 276, Red action: do-nothing, Blue reward:0.85\n", + "step: 277, Red action: do-nothing, Blue reward:0.85\n", + "step: 278, Red action: do-nothing, Blue reward:0.85\n", + "step: 279, Red action: do-nothing, Blue reward:0.85\n", + "step: 280, Red action: do-nothing, Blue reward:0.85\n", + "step: 281, Red action: node-application-execute, Blue reward:0.85\n", + "step: 282, Red action: do-nothing, Blue reward:0.85\n", + "step: 283, Red action: do-nothing, Blue reward:0.85\n", + "step: 284, Red action: do-nothing, Blue reward:0.85\n", + "step: 285, Red action: do-nothing, Blue reward:0.85\n", + "step: 286, Red action: do-nothing, Blue reward:0.85\n", + "step: 287, Red action: do-nothing, Blue reward:0.85\n", + "step: 288, Red action: do-nothing, Blue reward:0.85\n", + "step: 289, Red action: do-nothing, Blue reward:0.85\n", + "step: 290, Red action: do-nothing, Blue reward:0.85\n", + "step: 291, Red action: do-nothing, Blue reward:0.85\n", + "step: 292, Red action: do-nothing, Blue reward:0.85\n", + "step: 293, Red action: do-nothing, Blue reward:0.85\n", + "step: 294, Red action: do-nothing, Blue reward:0.85\n", + "step: 295, Red action: do-nothing, Blue reward:0.85\n", + "step: 296, Red action: do-nothing, Blue reward:0.85\n", + "step: 297, Red action: do-nothing, Blue reward:0.85\n", + "step: 298, Red action: do-nothing, Blue reward:0.85\n", + "step: 299, Red action: do-nothing, Blue reward:0.85\n", + "step: 300, Red action: do-nothing, Blue reward:0.85\n", + "step: 301, Red action: do-nothing, Blue reward:0.85\n", + "step: 302, Red action: do-nothing, Blue reward:0.85\n", + "step: 303, Red action: do-nothing, Blue reward:0.85\n", + "step: 304, Red action: node-application-execute, Blue reward:0.85\n", + "step: 305, Red action: do-nothing, Blue reward:0.85\n", + "step: 306, Red action: do-nothing, Blue reward:0.85\n", + "step: 307, Red action: do-nothing, Blue reward:0.85\n", + "step: 308, Red action: do-nothing, Blue reward:0.85\n", + "step: 309, Red action: do-nothing, Blue reward:0.85\n", + "step: 310, Red action: do-nothing, Blue reward:0.85\n", + "step: 311, Red action: do-nothing, Blue reward:0.85\n", + "step: 312, Red action: do-nothing, Blue reward:0.85\n", + "step: 313, Red action: do-nothing, Blue reward:0.85\n", + "step: 314, Red action: do-nothing, Blue reward:0.85\n", + "step: 315, Red action: do-nothing, Blue reward:0.85\n", + "step: 316, Red action: do-nothing, Blue reward:0.85\n", + "step: 317, Red action: do-nothing, Blue reward:0.85\n", + "step: 318, Red action: do-nothing, Blue reward:0.85\n", + "step: 319, Red action: do-nothing, Blue reward:0.85\n", + "step: 320, Red action: do-nothing, Blue reward:0.85\n", + "step: 321, Red action: do-nothing, Blue reward:0.85\n", + "step: 322, Red action: do-nothing, Blue reward:0.85\n", + "step: 323, Red action: do-nothing, Blue reward:0.85\n", + "step: 324, Red action: do-nothing, Blue reward:0.85\n", + "step: 325, Red action: node-application-execute, Blue reward:0.85\n", + "step: 326, Red action: do-nothing, Blue reward:0.85\n", + "step: 327, Red action: do-nothing, Blue reward:0.85\n", + "step: 328, Red action: do-nothing, Blue reward:0.85\n", + "step: 329, Red action: do-nothing, Blue reward:0.85\n", + "step: 330, Red action: do-nothing, Blue reward:0.85\n", + "step: 331, Red action: do-nothing, Blue reward:0.85\n", + "step: 332, Red action: do-nothing, Blue reward:0.85\n", + "step: 333, Red action: do-nothing, Blue reward:0.85\n", + "step: 334, Red action: do-nothing, Blue reward:0.85\n", + "step: 335, Red action: do-nothing, Blue reward:0.85\n", + "step: 336, Red action: do-nothing, Blue reward:0.85\n", + "step: 337, Red action: do-nothing, Blue reward:0.85\n", + "step: 338, Red action: do-nothing, Blue reward:0.85\n", + "step: 339, Red action: do-nothing, Blue reward:0.85\n", + "step: 340, Red action: do-nothing, Blue reward:0.85\n", + "step: 341, Red action: do-nothing, Blue reward:0.85\n", + "step: 342, Red action: do-nothing, Blue reward:0.85\n", + "step: 343, Red action: do-nothing, Blue reward:0.85\n", + "step: 344, Red action: do-nothing, Blue reward:0.85\n", + "step: 345, Red action: do-nothing, Blue reward:0.85\n", + "step: 346, Red action: do-nothing, Blue reward:0.85\n", + "step: 347, Red action: do-nothing, Blue reward:0.85\n", + "step: 348, Red action: do-nothing, Blue reward:0.85\n", + "step: 349, Red action: node-application-execute, Blue reward:0.85\n", + "step: 350, Red action: do-nothing, Blue reward:0.85\n", + "step: 351, Red action: do-nothing, Blue reward:0.85\n", + "step: 352, Red action: do-nothing, Blue reward:0.85\n", + "step: 353, Red action: do-nothing, Blue reward:0.85\n", + "step: 354, Red action: do-nothing, Blue reward:0.85\n", + "step: 355, Red action: do-nothing, Blue reward:0.85\n", + "step: 356, Red action: do-nothing, Blue reward:0.85\n", + "step: 357, Red action: do-nothing, Blue reward:0.85\n", + "step: 358, Red action: do-nothing, Blue reward:0.85\n", + "step: 359, Red action: do-nothing, Blue reward:0.85\n", + "step: 360, Red action: do-nothing, Blue reward:0.85\n", + "step: 361, Red action: do-nothing, Blue reward:0.85\n", + "step: 362, Red action: do-nothing, Blue reward:0.85\n", + "step: 363, Red action: do-nothing, Blue reward:0.85\n", + "step: 364, Red action: do-nothing, Blue reward:0.85\n", + "step: 365, Red action: do-nothing, Blue reward:0.85\n", + "step: 366, Red action: do-nothing, Blue reward:0.85\n", + "step: 367, Red action: do-nothing, Blue reward:0.85\n", + "step: 368, Red action: do-nothing, Blue reward:0.85\n", + "step: 369, Red action: node-application-execute, Blue reward:0.85\n", + "step: 370, Red action: do-nothing, Blue reward:0.85\n", + "step: 371, Red action: do-nothing, Blue reward:0.85\n", + "step: 372, Red action: do-nothing, Blue reward:0.85\n", + "step: 373, Red action: do-nothing, Blue reward:0.85\n", + "step: 374, Red action: do-nothing, Blue reward:0.85\n", + "step: 375, Red action: do-nothing, Blue reward:0.85\n", + "step: 376, Red action: do-nothing, Blue reward:0.85\n", + "step: 377, Red action: do-nothing, Blue reward:0.85\n", + "step: 378, Red action: do-nothing, Blue reward:0.85\n", + "step: 379, Red action: do-nothing, Blue reward:0.85\n", + "step: 380, Red action: do-nothing, Blue reward:0.85\n", + "step: 381, Red action: do-nothing, Blue reward:0.85\n", + "step: 382, Red action: do-nothing, Blue reward:0.85\n", + "step: 383, Red action: do-nothing, Blue reward:0.85\n", + "step: 384, Red action: do-nothing, Blue reward:0.85\n", + "step: 385, Red action: do-nothing, Blue reward:0.85\n", + "step: 386, Red action: do-nothing, Blue reward:0.85\n", + "step: 387, Red action: do-nothing, Blue reward:0.85\n", + "step: 388, Red action: do-nothing, Blue reward:0.85\n", + "step: 389, Red action: do-nothing, Blue reward:0.85\n", + "step: 390, Red action: do-nothing, Blue reward:0.85\n", + "step: 391, Red action: do-nothing, Blue reward:0.85\n", + "step: 392, Red action: do-nothing, Blue reward:0.85\n", + "step: 393, Red action: do-nothing, Blue reward:0.85\n", + "step: 394, Red action: node-application-execute, Blue reward:0.85\n", + "step: 395, Red action: do-nothing, Blue reward:0.85\n", + "step: 396, Red action: do-nothing, Blue reward:0.85\n", + "step: 397, Red action: do-nothing, Blue reward:0.85\n", + "step: 398, Red action: do-nothing, Blue reward:0.85\n", + "step: 399, Red action: do-nothing, Blue reward:0.85\n", + "step: 400, Red action: do-nothing, Blue reward:0.85\n", + "step: 401, Red action: do-nothing, Blue reward:0.85\n", + "step: 402, Red action: do-nothing, Blue reward:0.85\n", + "step: 403, Red action: do-nothing, Blue reward:0.85\n", + "step: 404, Red action: do-nothing, Blue reward:0.85\n", + "step: 405, Red action: do-nothing, Blue reward:0.85\n", + "step: 406, Red action: do-nothing, Blue reward:0.85\n", + "step: 407, Red action: do-nothing, Blue reward:0.85\n", + "step: 408, Red action: do-nothing, Blue reward:0.85\n", + "step: 409, Red action: do-nothing, Blue reward:0.85\n", + "step: 410, Red action: do-nothing, Blue reward:0.85\n", + "step: 411, Red action: do-nothing, Blue reward:0.85\n", + "step: 412, Red action: do-nothing, Blue reward:0.85\n", + "step: 413, Red action: do-nothing, Blue reward:0.85\n", + "step: 414, Red action: do-nothing, Blue reward:0.85\n", + "step: 415, Red action: do-nothing, Blue reward:0.85\n", + "step: 416, Red action: do-nothing, Blue reward:0.85\n", + "step: 417, Red action: do-nothing, Blue reward:0.85\n", + "step: 418, Red action: node-application-execute, Blue reward:0.85\n", + "step: 419, Red action: do-nothing, Blue reward:0.85\n", + "step: 420, Red action: do-nothing, Blue reward:0.85\n", + "step: 421, Red action: do-nothing, Blue reward:0.85\n", + "step: 422, Red action: do-nothing, Blue reward:0.85\n", + "step: 423, Red action: do-nothing, Blue reward:0.85\n", + "step: 424, Red action: do-nothing, Blue reward:0.85\n", + "step: 425, Red action: do-nothing, Blue reward:0.85\n", + "step: 426, Red action: do-nothing, Blue reward:0.85\n", + "step: 427, Red action: do-nothing, Blue reward:0.85\n", + "step: 428, Red action: do-nothing, Blue reward:0.85\n", + "step: 429, Red action: do-nothing, Blue reward:0.85\n", + "step: 430, Red action: do-nothing, Blue reward:0.85\n", + "step: 431, Red action: do-nothing, Blue reward:0.85\n", + "step: 432, Red action: do-nothing, Blue reward:0.85\n", + "step: 433, Red action: do-nothing, Blue reward:0.85\n", + "step: 434, Red action: do-nothing, Blue reward:0.85\n", + "step: 435, Red action: do-nothing, Blue reward:0.85\n", + "step: 436, Red action: do-nothing, Blue reward:0.85\n", + "step: 437, Red action: do-nothing, Blue reward:0.85\n", + "step: 438, Red action: do-nothing, Blue reward:0.85\n", + "step: 439, Red action: do-nothing, Blue reward:0.85\n", + "step: 440, Red action: do-nothing, Blue reward:0.85\n", + "step: 441, Red action: do-nothing, Blue reward:0.85\n", + "step: 442, Red action: do-nothing, Blue reward:0.85\n", + "step: 443, Red action: node-application-execute, Blue reward:0.85\n", + "step: 444, Red action: do-nothing, Blue reward:0.85\n", + "step: 445, Red action: do-nothing, Blue reward:0.85\n", + "step: 446, Red action: do-nothing, Blue reward:0.85\n", + "step: 447, Red action: do-nothing, Blue reward:0.85\n", + "step: 448, Red action: do-nothing, Blue reward:0.85\n", + "step: 449, Red action: do-nothing, Blue reward:0.85\n", + "step: 450, Red action: do-nothing, Blue reward:0.85\n", + "step: 451, Red action: do-nothing, Blue reward:0.85\n", + "step: 452, Red action: do-nothing, Blue reward:0.85\n", + "step: 453, Red action: do-nothing, Blue reward:0.85\n", + "step: 454, Red action: do-nothing, Blue reward:0.85\n", + "step: 455, Red action: do-nothing, Blue reward:0.85\n", + "step: 456, Red action: do-nothing, Blue reward:0.85\n", + "step: 457, Red action: do-nothing, Blue reward:0.85\n", + "step: 458, Red action: do-nothing, Blue reward:0.85\n", + "step: 459, Red action: do-nothing, Blue reward:0.85\n", + "step: 460, Red action: do-nothing, Blue reward:0.85\n", + "step: 461, Red action: node-application-execute, Blue reward:0.85\n", + "step: 462, Red action: do-nothing, Blue reward:0.85\n", + "step: 463, Red action: do-nothing, Blue reward:0.85\n", + "step: 464, Red action: do-nothing, Blue reward:0.85\n", + "step: 465, Red action: do-nothing, Blue reward:0.85\n", + "step: 466, Red action: do-nothing, Blue reward:0.85\n", + "step: 467, Red action: do-nothing, Blue reward:0.85\n", + "step: 468, Red action: do-nothing, Blue reward:0.85\n", + "step: 469, Red action: do-nothing, Blue reward:0.85\n", + "step: 470, Red action: do-nothing, Blue reward:0.85\n", + "step: 471, Red action: do-nothing, Blue reward:0.85\n", + "step: 472, Red action: do-nothing, Blue reward:0.85\n", + "step: 473, Red action: do-nothing, Blue reward:0.85\n", + "step: 474, Red action: do-nothing, Blue reward:0.85\n", + "step: 475, Red action: do-nothing, Blue reward:0.85\n", + "step: 476, Red action: do-nothing, Blue reward:0.85\n", + "step: 477, Red action: do-nothing, Blue reward:0.85\n", + "step: 478, Red action: do-nothing, Blue reward:0.85\n", + "step: 479, Red action: do-nothing, Blue reward:0.85\n", + "step: 480, Red action: node-application-execute, Blue reward:0.85\n", + "step: 481, Red action: do-nothing, Blue reward:0.85\n", + "step: 482, Red action: do-nothing, Blue reward:0.85\n", + "step: 483, Red action: do-nothing, Blue reward:0.85\n", + "step: 484, Red action: do-nothing, Blue reward:0.85\n", + "step: 485, Red action: do-nothing, Blue reward:0.85\n", + "step: 486, Red action: do-nothing, Blue reward:0.85\n", + "step: 487, Red action: do-nothing, Blue reward:0.85\n", + "step: 488, Red action: do-nothing, Blue reward:0.85\n", + "step: 489, Red action: do-nothing, Blue reward:0.85\n", + "step: 490, Red action: do-nothing, Blue reward:0.85\n", + "step: 491, Red action: do-nothing, Blue reward:0.85\n", + "step: 492, Red action: do-nothing, Blue reward:0.85\n", + "step: 493, Red action: do-nothing, Blue reward:0.85\n", + "step: 494, Red action: do-nothing, Blue reward:0.85\n", + "step: 495, Red action: do-nothing, Blue reward:0.85\n", + "step: 496, Red action: do-nothing, Blue reward:0.85\n", + "step: 497, Red action: do-nothing, Blue reward:0.85\n", + "step: 498, Red action: do-nothing, Blue reward:0.85\n", + "step: 499, Red action: do-nothing, Blue reward:0.85\n", + "step: 500, Red action: do-nothing, Blue reward:0.85\n", + "step: 501, Red action: do-nothing, Blue reward:0.85\n", + "step: 502, Red action: do-nothing, Blue reward:0.85\n", + "step: 503, Red action: do-nothing, Blue reward:0.85\n", + "step: 504, Red action: node-application-execute, Blue reward:0.85\n", + "step: 505, Red action: do-nothing, Blue reward:0.85\n", + "step: 506, Red action: do-nothing, Blue reward:0.85\n", + "step: 507, Red action: do-nothing, Blue reward:0.85\n", + "step: 508, Red action: do-nothing, Blue reward:0.85\n", + "step: 509, Red action: do-nothing, Blue reward:0.85\n", + "step: 510, Red action: do-nothing, Blue reward:0.85\n", + "step: 511, Red action: do-nothing, Blue reward:0.85\n", + "step: 512, Red action: do-nothing, Blue reward:0.85\n", + "step: 513, Red action: do-nothing, Blue reward:0.85\n", + "step: 514, Red action: do-nothing, Blue reward:0.85\n", + "step: 515, Red action: do-nothing, Blue reward:0.85\n", + "step: 516, Red action: do-nothing, Blue reward:0.85\n", + "step: 517, Red action: do-nothing, Blue reward:0.85\n", + "step: 518, Red action: do-nothing, Blue reward:0.85\n", + "step: 519, Red action: do-nothing, Blue reward:0.85\n", + "step: 520, Red action: do-nothing, Blue reward:0.85\n", + "step: 521, Red action: do-nothing, Blue reward:0.85\n", + "step: 522, Red action: do-nothing, Blue reward:0.85\n", + "step: 523, Red action: do-nothing, Blue reward:0.85\n", + "step: 524, Red action: do-nothing, Blue reward:0.85\n", + "step: 525, Red action: do-nothing, Blue reward:0.85\n", + "step: 526, Red action: do-nothing, Blue reward:0.85\n", + "step: 527, Red action: node-application-execute, Blue reward:0.85\n", + "step: 528, Red action: do-nothing, Blue reward:0.85\n", + "step: 529, Red action: do-nothing, Blue reward:0.85\n", + "step: 530, Red action: do-nothing, Blue reward:0.85\n", + "step: 531, Red action: do-nothing, Blue reward:0.85\n", + "step: 532, Red action: do-nothing, Blue reward:0.85\n", + "step: 533, Red action: do-nothing, Blue reward:0.85\n", + "step: 534, Red action: do-nothing, Blue reward:0.85\n", + "step: 535, Red action: do-nothing, Blue reward:0.85\n", + "step: 536, Red action: do-nothing, Blue reward:0.85\n", + "step: 537, Red action: do-nothing, Blue reward:0.85\n", + "step: 538, Red action: do-nothing, Blue reward:0.85\n", + "step: 539, Red action: do-nothing, Blue reward:0.85\n", + "step: 540, Red action: do-nothing, Blue reward:0.85\n", + "step: 541, Red action: do-nothing, Blue reward:0.85\n", + "step: 542, Red action: do-nothing, Blue reward:0.85\n", + "step: 543, Red action: do-nothing, Blue reward:0.85\n", + "step: 544, Red action: do-nothing, Blue reward:0.85\n", + "step: 545, Red action: do-nothing, Blue reward:0.85\n", + "step: 546, Red action: do-nothing, Blue reward:0.85\n", + "step: 547, Red action: do-nothing, Blue reward:0.85\n", + "step: 548, Red action: do-nothing, Blue reward:0.85\n", + "step: 549, Red action: node-application-execute, Blue reward:0.85\n", + "step: 550, Red action: do-nothing, Blue reward:0.85\n", + "step: 551, Red action: do-nothing, Blue reward:0.85\n", + "step: 552, Red action: do-nothing, Blue reward:0.85\n", + "step: 553, Red action: do-nothing, Blue reward:0.85\n", + "step: 554, Red action: do-nothing, Blue reward:0.85\n", + "step: 555, Red action: do-nothing, Blue reward:0.85\n", + "step: 556, Red action: do-nothing, Blue reward:0.85\n", + "step: 557, Red action: do-nothing, Blue reward:0.85\n", + "step: 558, Red action: do-nothing, Blue reward:0.85\n", + "step: 559, Red action: do-nothing, Blue reward:0.85\n", + "step: 560, Red action: do-nothing, Blue reward:0.85\n", + "step: 561, Red action: do-nothing, Blue reward:0.85\n", + "step: 562, Red action: do-nothing, Blue reward:0.85\n", + "step: 563, Red action: do-nothing, Blue reward:0.85\n", + "step: 564, Red action: do-nothing, Blue reward:0.85\n", + "step: 565, Red action: node-application-execute, Blue reward:0.85\n", + "step: 566, Red action: do-nothing, Blue reward:0.85\n", + "step: 567, Red action: do-nothing, Blue reward:0.85\n", + "step: 568, Red action: do-nothing, Blue reward:0.85\n", + "step: 569, Red action: do-nothing, Blue reward:0.85\n", + "step: 570, Red action: do-nothing, Blue reward:0.85\n", + "step: 571, Red action: do-nothing, Blue reward:0.85\n", + "step: 572, Red action: do-nothing, Blue reward:0.85\n", + "step: 573, Red action: do-nothing, Blue reward:0.85\n", + "step: 574, Red action: do-nothing, Blue reward:0.85\n", + "step: 575, Red action: do-nothing, Blue reward:0.85\n", + "step: 576, Red action: do-nothing, Blue reward:0.85\n", + "step: 577, Red action: do-nothing, Blue reward:0.85\n", + "step: 578, Red action: do-nothing, Blue reward:0.85\n", + "step: 579, Red action: do-nothing, Blue reward:0.85\n", + "step: 580, Red action: do-nothing, Blue reward:0.85\n", + "step: 581, Red action: do-nothing, Blue reward:0.85\n", + "step: 582, Red action: do-nothing, Blue reward:0.85\n", + "step: 583, Red action: do-nothing, Blue reward:0.85\n", + "step: 584, Red action: do-nothing, Blue reward:0.85\n", + "step: 585, Red action: do-nothing, Blue reward:0.85\n", + "step: 586, Red action: do-nothing, Blue reward:0.85\n", + "step: 587, Red action: do-nothing, Blue reward:0.85\n", + "step: 588, Red action: do-nothing, Blue reward:0.85\n", + "step: 589, Red action: node-application-execute, Blue reward:0.85\n", + "step: 590, Red action: do-nothing, Blue reward:0.85\n", + "step: 591, Red action: do-nothing, Blue reward:0.85\n", + "step: 592, Red action: do-nothing, Blue reward:0.85\n", + "step: 593, Red action: do-nothing, Blue reward:0.85\n", + "step: 594, Red action: do-nothing, Blue reward:0.85\n", + "step: 595, Red action: do-nothing, Blue reward:0.85\n", + "step: 596, Red action: do-nothing, Blue reward:0.85\n", + "step: 597, Red action: do-nothing, Blue reward:0.85\n", + "step: 598, Red action: do-nothing, Blue reward:0.85\n", + "step: 599, Red action: do-nothing, Blue reward:0.85\n", + "step: 600, Red action: do-nothing, Blue reward:0.85\n", + "step: 601, Red action: do-nothing, Blue reward:0.85\n", + "step: 602, Red action: do-nothing, Blue reward:0.85\n", + "step: 603, Red action: do-nothing, Blue reward:0.85\n", + "step: 604, Red action: do-nothing, Blue reward:0.85\n", + "step: 605, Red action: do-nothing, Blue reward:0.85\n", + "step: 606, Red action: do-nothing, Blue reward:0.85\n", + "step: 607, Red action: do-nothing, Blue reward:0.85\n", + "step: 608, Red action: do-nothing, Blue reward:0.85\n", + "step: 609, Red action: do-nothing, Blue reward:0.85\n", + "step: 610, Red action: node-application-execute, Blue reward:0.85\n", + "step: 611, Red action: do-nothing, Blue reward:0.85\n", + "step: 612, Red action: do-nothing, Blue reward:0.85\n", + "step: 613, Red action: do-nothing, Blue reward:0.85\n", + "step: 614, Red action: do-nothing, Blue reward:0.85\n", + "step: 615, Red action: do-nothing, Blue reward:0.85\n", + "step: 616, Red action: do-nothing, Blue reward:0.85\n", + "step: 617, Red action: do-nothing, Blue reward:0.85\n", + "step: 618, Red action: do-nothing, Blue reward:0.85\n", + "step: 619, Red action: do-nothing, Blue reward:0.85\n", + "step: 620, Red action: do-nothing, Blue reward:0.85\n", + "step: 621, Red action: do-nothing, Blue reward:0.85\n", + "step: 622, Red action: do-nothing, Blue reward:0.85\n", + "step: 623, Red action: do-nothing, Blue reward:0.85\n", + "step: 624, Red action: do-nothing, Blue reward:0.85\n", + "step: 625, Red action: do-nothing, Blue reward:0.85\n", + "step: 626, Red action: node-application-execute, Blue reward:0.85\n", + "step: 627, Red action: do-nothing, Blue reward:0.85\n", + "step: 628, Red action: do-nothing, Blue reward:0.85\n", + "step: 629, Red action: do-nothing, Blue reward:0.85\n", + "step: 630, Red action: do-nothing, Blue reward:0.85\n", + "step: 631, Red action: do-nothing, Blue reward:0.85\n", + "step: 632, Red action: do-nothing, Blue reward:0.85\n", + "step: 633, Red action: do-nothing, Blue reward:0.85\n", + "step: 634, Red action: do-nothing, Blue reward:0.85\n", + "step: 635, Red action: do-nothing, Blue reward:0.85\n", + "step: 636, Red action: do-nothing, Blue reward:0.85\n", + "step: 637, Red action: do-nothing, Blue reward:0.85\n", + "step: 638, Red action: do-nothing, Blue reward:0.85\n", + "step: 639, Red action: do-nothing, Blue reward:0.85\n", + "step: 640, Red action: do-nothing, Blue reward:0.85\n", + "step: 641, Red action: do-nothing, Blue reward:0.85\n", + "step: 642, Red action: do-nothing, Blue reward:0.85\n", + "step: 643, Red action: do-nothing, Blue reward:0.85\n", + "step: 644, Red action: node-application-execute, Blue reward:0.85\n", + "step: 645, Red action: do-nothing, Blue reward:0.85\n", + "step: 646, Red action: do-nothing, Blue reward:0.85\n", + "step: 647, Red action: do-nothing, Blue reward:0.85\n", + "step: 648, Red action: do-nothing, Blue reward:0.85\n", + "step: 649, Red action: do-nothing, Blue reward:0.85\n", + "step: 650, Red action: do-nothing, Blue reward:0.85\n", + "step: 651, Red action: do-nothing, Blue reward:0.85\n", + "step: 652, Red action: do-nothing, Blue reward:0.85\n", + "step: 653, Red action: do-nothing, Blue reward:0.85\n", + "step: 654, Red action: do-nothing, Blue reward:0.85\n", + "step: 655, Red action: do-nothing, Blue reward:0.85\n", + "step: 656, Red action: do-nothing, Blue reward:0.85\n", + "step: 657, Red action: do-nothing, Blue reward:0.85\n", + "step: 658, Red action: do-nothing, Blue reward:0.85\n", + "step: 659, Red action: do-nothing, Blue reward:0.85\n", + "step: 660, Red action: do-nothing, Blue reward:0.85\n", + "step: 661, Red action: do-nothing, Blue reward:0.85\n", + "step: 662, Red action: do-nothing, Blue reward:0.85\n", + "step: 663, Red action: do-nothing, Blue reward:0.85\n", + "step: 664, Red action: do-nothing, Blue reward:0.85\n", + "step: 665, Red action: do-nothing, Blue reward:0.85\n", + "step: 666, Red action: do-nothing, Blue reward:0.85\n", + "step: 667, Red action: node-application-execute, Blue reward:0.85\n", + "step: 668, Red action: do-nothing, Blue reward:0.85\n", + "step: 669, Red action: do-nothing, Blue reward:0.85\n", + "step: 670, Red action: do-nothing, Blue reward:0.85\n", + "step: 671, Red action: do-nothing, Blue reward:0.85\n", + "step: 672, Red action: do-nothing, Blue reward:0.85\n", + "step: 673, Red action: do-nothing, Blue reward:0.85\n", + "step: 674, Red action: do-nothing, Blue reward:0.85\n", + "step: 675, Red action: do-nothing, Blue reward:0.85\n", + "step: 676, Red action: do-nothing, Blue reward:0.85\n", + "step: 677, Red action: do-nothing, Blue reward:0.85\n", + "step: 678, Red action: do-nothing, Blue reward:0.85\n", + "step: 679, Red action: do-nothing, Blue reward:0.85\n", + "step: 680, Red action: do-nothing, Blue reward:0.85\n", + "step: 681, Red action: do-nothing, Blue reward:0.85\n", + "step: 682, Red action: do-nothing, Blue reward:0.85\n", + "step: 683, Red action: do-nothing, Blue reward:0.85\n", + "step: 684, Red action: do-nothing, Blue reward:0.85\n", + "step: 685, Red action: node-application-execute, Blue reward:0.85\n", + "step: 686, Red action: do-nothing, Blue reward:0.85\n", + "step: 687, Red action: do-nothing, Blue reward:0.85\n", + "step: 688, Red action: do-nothing, Blue reward:0.85\n", + "step: 689, Red action: do-nothing, Blue reward:0.85\n", + "step: 690, Red action: do-nothing, Blue reward:0.85\n", + "step: 691, Red action: do-nothing, Blue reward:0.85\n", + "step: 692, Red action: do-nothing, Blue reward:0.85\n", + "step: 693, Red action: do-nothing, Blue reward:0.85\n", + "step: 694, Red action: do-nothing, Blue reward:0.85\n", + "step: 695, Red action: do-nothing, Blue reward:0.85\n", + "step: 696, Red action: do-nothing, Blue reward:0.85\n", + "step: 697, Red action: do-nothing, Blue reward:0.85\n", + "step: 698, Red action: do-nothing, Blue reward:0.85\n", + "step: 699, Red action: do-nothing, Blue reward:0.85\n", + "step: 700, Red action: do-nothing, Blue reward:0.85\n", + "step: 701, Red action: node-application-execute, Blue reward:0.85\n", + "step: 702, Red action: do-nothing, Blue reward:0.85\n", + "step: 703, Red action: do-nothing, Blue reward:0.85\n", + "step: 704, Red action: do-nothing, Blue reward:0.85\n", + "step: 705, Red action: do-nothing, Blue reward:0.85\n", + "step: 706, Red action: do-nothing, Blue reward:0.85\n", + "step: 707, Red action: do-nothing, Blue reward:0.85\n", + "step: 708, Red action: do-nothing, Blue reward:0.85\n", + "step: 709, Red action: do-nothing, Blue reward:0.85\n", + "step: 710, Red action: do-nothing, Blue reward:0.85\n", + "step: 711, Red action: do-nothing, Blue reward:0.85\n", + "step: 712, Red action: do-nothing, Blue reward:0.85\n", + "step: 713, Red action: do-nothing, Blue reward:0.85\n", + "step: 714, Red action: do-nothing, Blue reward:0.85\n", + "step: 715, Red action: do-nothing, Blue reward:0.85\n", + "step: 716, Red action: do-nothing, Blue reward:0.85\n", + "step: 717, Red action: do-nothing, Blue reward:0.85\n", + "step: 718, Red action: node-application-execute, Blue reward:0.85\n", + "step: 719, Red action: do-nothing, Blue reward:0.85\n", + "step: 720, Red action: do-nothing, Blue reward:0.85\n", + "step: 721, Red action: do-nothing, Blue reward:0.85\n", + "step: 722, Red action: do-nothing, Blue reward:0.85\n", + "step: 723, Red action: do-nothing, Blue reward:0.85\n", + "step: 724, Red action: do-nothing, Blue reward:0.85\n", + "step: 725, Red action: do-nothing, Blue reward:0.85\n", + "step: 726, Red action: do-nothing, Blue reward:0.85\n", + "step: 727, Red action: do-nothing, Blue reward:0.85\n", + "step: 728, Red action: do-nothing, Blue reward:0.85\n", + "step: 729, Red action: do-nothing, Blue reward:0.85\n", + "step: 730, Red action: do-nothing, Blue reward:0.85\n", + "step: 731, Red action: do-nothing, Blue reward:0.85\n", + "step: 732, Red action: do-nothing, Blue reward:0.85\n", + "step: 733, Red action: do-nothing, Blue reward:0.85\n", + "step: 734, Red action: do-nothing, Blue reward:0.85\n", + "step: 735, Red action: do-nothing, Blue reward:0.85\n", + "step: 736, Red action: do-nothing, Blue reward:0.85\n", + "step: 737, Red action: do-nothing, Blue reward:0.85\n", + "step: 738, Red action: do-nothing, Blue reward:0.85\n", + "step: 739, Red action: do-nothing, Blue reward:0.85\n", + "step: 740, Red action: do-nothing, Blue reward:0.85\n", + "step: 741, Red action: do-nothing, Blue reward:0.85\n", + "step: 742, Red action: do-nothing, Blue reward:0.85\n", + "step: 743, Red action: node-application-execute, Blue reward:0.85\n", + "step: 744, Red action: do-nothing, Blue reward:0.85\n", + "step: 745, Red action: do-nothing, Blue reward:0.85\n", + "step: 746, Red action: do-nothing, Blue reward:0.85\n", + "step: 747, Red action: do-nothing, Blue reward:0.85\n", + "step: 748, Red action: do-nothing, Blue reward:0.85\n", + "step: 749, Red action: do-nothing, Blue reward:0.85\n", + "step: 750, Red action: do-nothing, Blue reward:0.85\n", + "step: 751, Red action: do-nothing, Blue reward:0.85\n", + "step: 752, Red action: do-nothing, Blue reward:0.85\n", + "step: 753, Red action: do-nothing, Blue reward:0.85\n", + "step: 754, Red action: do-nothing, Blue reward:0.85\n", + "step: 755, Red action: do-nothing, Blue reward:0.85\n", + "step: 756, Red action: do-nothing, Blue reward:0.85\n", + "step: 757, Red action: do-nothing, Blue reward:0.85\n", + "step: 758, Red action: do-nothing, Blue reward:0.85\n", + "step: 759, Red action: do-nothing, Blue reward:0.85\n", + "step: 760, Red action: node-application-execute, Blue reward:0.85\n", + "step: 761, Red action: do-nothing, Blue reward:0.85\n", + "step: 762, Red action: do-nothing, Blue reward:0.85\n", + "step: 763, Red action: do-nothing, Blue reward:0.85\n", + "step: 764, Red action: do-nothing, Blue reward:0.85\n", + "step: 765, Red action: do-nothing, Blue reward:0.85\n", + "step: 766, Red action: do-nothing, Blue reward:0.85\n", + "step: 767, Red action: do-nothing, Blue reward:0.85\n", + "step: 768, Red action: do-nothing, Blue reward:0.85\n", + "step: 769, Red action: do-nothing, Blue reward:0.85\n", + "step: 770, Red action: do-nothing, Blue reward:0.85\n", + "step: 771, Red action: do-nothing, Blue reward:0.85\n", + "step: 772, Red action: do-nothing, Blue reward:0.85\n", + "step: 773, Red action: do-nothing, Blue reward:0.85\n", + "step: 774, Red action: do-nothing, Blue reward:0.85\n", + "step: 775, Red action: do-nothing, Blue reward:0.85\n", + "step: 776, Red action: do-nothing, Blue reward:0.85\n", + "step: 777, Red action: do-nothing, Blue reward:0.85\n", + "step: 778, Red action: do-nothing, Blue reward:0.85\n", + "step: 779, Red action: do-nothing, Blue reward:0.85\n", + "step: 780, Red action: do-nothing, Blue reward:0.85\n", + "step: 781, Red action: do-nothing, Blue reward:0.85\n", + "step: 782, Red action: do-nothing, Blue reward:0.85\n", + "step: 783, Red action: node-application-execute, Blue reward:0.85\n", + "step: 784, Red action: do-nothing, Blue reward:0.85\n", + "step: 785, Red action: do-nothing, Blue reward:0.85\n", + "step: 786, Red action: do-nothing, Blue reward:0.85\n", + "step: 787, Red action: do-nothing, Blue reward:0.85\n", + "step: 788, Red action: do-nothing, Blue reward:0.85\n", + "step: 789, Red action: do-nothing, Blue reward:0.85\n", + "step: 790, Red action: do-nothing, Blue reward:0.85\n", + "step: 791, Red action: do-nothing, Blue reward:0.85\n", + "step: 792, Red action: do-nothing, Blue reward:0.85\n", + "step: 793, Red action: do-nothing, Blue reward:0.85\n", + "step: 794, Red action: do-nothing, Blue reward:0.85\n", + "step: 795, Red action: do-nothing, Blue reward:0.85\n", + "step: 796, Red action: do-nothing, Blue reward:0.85\n", + "step: 797, Red action: do-nothing, Blue reward:0.85\n", + "step: 798, Red action: do-nothing, Blue reward:0.85\n", + "step: 799, Red action: do-nothing, Blue reward:0.85\n", + "step: 800, Red action: do-nothing, Blue reward:0.85\n", + "step: 801, Red action: do-nothing, Blue reward:0.85\n", + "step: 802, Red action: do-nothing, Blue reward:0.85\n", + "step: 803, Red action: node-application-execute, Blue reward:0.85\n", + "step: 804, Red action: do-nothing, Blue reward:0.85\n", + "step: 805, Red action: do-nothing, Blue reward:0.85\n", + "step: 806, Red action: do-nothing, Blue reward:0.85\n", + "step: 807, Red action: do-nothing, Blue reward:0.85\n", + "step: 808, Red action: do-nothing, Blue reward:0.85\n", + "step: 809, Red action: do-nothing, Blue reward:0.85\n", + "step: 810, Red action: do-nothing, Blue reward:0.85\n", + "step: 811, Red action: do-nothing, Blue reward:0.85\n", + "step: 812, Red action: do-nothing, Blue reward:0.85\n", + "step: 813, Red action: do-nothing, Blue reward:0.85\n", + "step: 814, Red action: do-nothing, Blue reward:0.85\n", + "step: 815, Red action: do-nothing, Blue reward:0.85\n", + "step: 816, Red action: do-nothing, Blue reward:0.85\n", + "step: 817, Red action: do-nothing, Blue reward:0.85\n", + "step: 818, Red action: do-nothing, Blue reward:0.85\n", + "step: 819, Red action: do-nothing, Blue reward:0.85\n", + "step: 820, Red action: do-nothing, Blue reward:0.85\n", + "step: 821, Red action: do-nothing, Blue reward:0.85\n", + "step: 822, Red action: do-nothing, Blue reward:0.85\n", + "step: 823, Red action: do-nothing, Blue reward:0.85\n", + "step: 824, Red action: do-nothing, Blue reward:0.85\n", + "step: 825, Red action: do-nothing, Blue reward:0.85\n", + "step: 826, Red action: do-nothing, Blue reward:0.85\n", + "step: 827, Red action: node-application-execute, Blue reward:0.85\n", + "step: 828, Red action: do-nothing, Blue reward:0.85\n", + "step: 829, Red action: do-nothing, Blue reward:0.85\n", + "step: 830, Red action: do-nothing, Blue reward:0.85\n", + "step: 831, Red action: do-nothing, Blue reward:0.85\n", + "step: 832, Red action: do-nothing, Blue reward:0.85\n", + "step: 833, Red action: do-nothing, Blue reward:0.85\n", + "step: 834, Red action: do-nothing, Blue reward:0.85\n", + "step: 835, Red action: do-nothing, Blue reward:0.85\n", + "step: 836, Red action: do-nothing, Blue reward:0.85\n", + "step: 837, Red action: do-nothing, Blue reward:0.85\n", + "step: 838, Red action: do-nothing, Blue reward:0.85\n", + "step: 839, Red action: do-nothing, Blue reward:0.85\n", + "step: 840, Red action: do-nothing, Blue reward:0.85\n", + "step: 841, Red action: do-nothing, Blue reward:0.85\n", + "step: 842, Red action: do-nothing, Blue reward:0.85\n", + "step: 843, Red action: do-nothing, Blue reward:0.85\n", + "step: 844, Red action: do-nothing, Blue reward:0.85\n", + "step: 845, Red action: do-nothing, Blue reward:0.85\n", + "step: 846, Red action: do-nothing, Blue reward:0.85\n", + "step: 847, Red action: node-application-execute, Blue reward:0.85\n", + "step: 848, Red action: do-nothing, Blue reward:0.85\n", + "step: 849, Red action: do-nothing, Blue reward:0.85\n", + "step: 850, Red action: do-nothing, Blue reward:0.85\n", + "step: 851, Red action: do-nothing, Blue reward:0.85\n", + "step: 852, Red action: do-nothing, Blue reward:0.85\n", + "step: 853, Red action: do-nothing, Blue reward:0.85\n", + "step: 854, Red action: do-nothing, Blue reward:0.85\n", + "step: 855, Red action: do-nothing, Blue reward:0.85\n", + "step: 856, Red action: do-nothing, Blue reward:0.85\n", + "step: 857, Red action: do-nothing, Blue reward:0.85\n", + "step: 858, Red action: do-nothing, Blue reward:0.85\n", + "step: 859, Red action: do-nothing, Blue reward:0.85\n", + "step: 860, Red action: do-nothing, Blue reward:0.85\n", + "step: 861, Red action: do-nothing, Blue reward:0.85\n", + "step: 862, Red action: do-nothing, Blue reward:0.85\n", + "step: 863, Red action: do-nothing, Blue reward:0.85\n", + "step: 864, Red action: do-nothing, Blue reward:0.85\n", + "step: 865, Red action: do-nothing, Blue reward:0.85\n", + "step: 866, Red action: do-nothing, Blue reward:0.85\n", + "step: 867, Red action: do-nothing, Blue reward:0.85\n", + "step: 868, Red action: do-nothing, Blue reward:0.85\n", + "step: 869, Red action: do-nothing, Blue reward:0.85\n", + "step: 870, Red action: do-nothing, Blue reward:0.85\n", + "step: 871, Red action: node-application-execute, Blue reward:0.85\n", + "step: 872, Red action: do-nothing, Blue reward:0.85\n", + "step: 873, Red action: do-nothing, Blue reward:0.85\n", + "step: 874, Red action: do-nothing, Blue reward:0.85\n", + "step: 875, Red action: do-nothing, Blue reward:0.85\n", + "step: 876, Red action: do-nothing, Blue reward:0.85\n", + "step: 877, Red action: do-nothing, Blue reward:0.85\n", + "step: 878, Red action: do-nothing, Blue reward:0.85\n", + "step: 879, Red action: do-nothing, Blue reward:0.85\n", + "step: 880, Red action: do-nothing, Blue reward:0.85\n", + "step: 881, Red action: do-nothing, Blue reward:0.85\n", + "step: 882, Red action: do-nothing, Blue reward:0.85\n", + "step: 883, Red action: do-nothing, Blue reward:0.85\n", + "step: 884, Red action: do-nothing, Blue reward:0.85\n", + "step: 885, Red action: do-nothing, Blue reward:0.85\n", + "step: 886, Red action: do-nothing, Blue reward:0.85\n", + "step: 887, Red action: do-nothing, Blue reward:0.85\n", + "step: 888, Red action: do-nothing, Blue reward:0.85\n", + "step: 889, Red action: do-nothing, Blue reward:0.85\n", + "step: 890, Red action: do-nothing, Blue reward:0.85\n", + "step: 891, Red action: node-application-execute, Blue reward:0.85\n", + "step: 892, Red action: do-nothing, Blue reward:0.85\n", + "step: 893, Red action: do-nothing, Blue reward:0.85\n", + "step: 894, Red action: do-nothing, Blue reward:0.85\n", + "step: 895, Red action: do-nothing, Blue reward:0.85\n", + "step: 896, Red action: do-nothing, Blue reward:0.85\n", + "step: 897, Red action: do-nothing, Blue reward:0.85\n", + "step: 898, Red action: do-nothing, Blue reward:0.85\n", + "step: 899, Red action: do-nothing, Blue reward:0.85\n", + "step: 900, Red action: do-nothing, Blue reward:0.85\n", + "step: 901, Red action: do-nothing, Blue reward:0.85\n", + "step: 902, Red action: do-nothing, Blue reward:0.85\n", + "step: 903, Red action: do-nothing, Blue reward:0.85\n", + "step: 904, Red action: do-nothing, Blue reward:0.85\n", + "step: 905, Red action: do-nothing, Blue reward:0.85\n", + "step: 906, Red action: node-application-execute, Blue reward:0.85\n", + "step: 907, Red action: do-nothing, Blue reward:0.85\n", + "step: 908, Red action: do-nothing, Blue reward:0.85\n", + "step: 909, Red action: do-nothing, Blue reward:0.85\n", + "step: 910, Red action: do-nothing, Blue reward:0.85\n", + "step: 911, Red action: do-nothing, Blue reward:0.85\n", + "step: 912, Red action: do-nothing, Blue reward:0.85\n", + "step: 913, Red action: do-nothing, Blue reward:0.85\n", + "step: 914, Red action: do-nothing, Blue reward:0.85\n", + "step: 915, Red action: do-nothing, Blue reward:0.85\n", + "step: 916, Red action: do-nothing, Blue reward:0.85\n", + "step: 917, Red action: do-nothing, Blue reward:0.85\n", + "step: 918, Red action: do-nothing, Blue reward:0.85\n", + "step: 919, Red action: do-nothing, Blue reward:0.85\n", + "step: 920, Red action: do-nothing, Blue reward:0.85\n", + "step: 921, Red action: do-nothing, Blue reward:0.85\n", + "step: 922, Red action: do-nothing, Blue reward:0.85\n", + "step: 923, Red action: do-nothing, Blue reward:0.85\n", + "step: 924, Red action: do-nothing, Blue reward:0.85\n", + "step: 925, Red action: do-nothing, Blue reward:0.85\n", + "step: 926, Red action: do-nothing, Blue reward:0.85\n", + "step: 927, Red action: do-nothing, Blue reward:0.85\n", + "step: 928, Red action: do-nothing, Blue reward:0.85\n", + "step: 929, Red action: do-nothing, Blue reward:0.85\n", + "step: 930, Red action: node-application-execute, Blue reward:0.85\n", + "step: 931, Red action: do-nothing, Blue reward:0.85\n", + "step: 932, Red action: do-nothing, Blue reward:0.85\n", + "step: 933, Red action: do-nothing, Blue reward:0.85\n", + "step: 934, Red action: do-nothing, Blue reward:0.85\n", + "step: 935, Red action: do-nothing, Blue reward:0.85\n", + "step: 936, Red action: do-nothing, Blue reward:0.85\n", + "step: 937, Red action: do-nothing, Blue reward:0.85\n", + "step: 938, Red action: do-nothing, Blue reward:0.85\n", + "step: 939, Red action: do-nothing, Blue reward:0.85\n", + "step: 940, Red action: do-nothing, Blue reward:0.85\n", + "step: 941, Red action: do-nothing, Blue reward:0.85\n", + "step: 942, Red action: do-nothing, Blue reward:0.85\n", + "step: 943, Red action: do-nothing, Blue reward:0.85\n", + "step: 944, Red action: do-nothing, Blue reward:0.85\n", + "step: 945, Red action: do-nothing, Blue reward:0.85\n", + "step: 946, Red action: do-nothing, Blue reward:0.85\n", + "step: 947, Red action: node-application-execute, Blue reward:0.85\n", + "step: 948, Red action: do-nothing, Blue reward:0.85\n", + "step: 949, Red action: do-nothing, Blue reward:0.85\n", + "step: 950, Red action: do-nothing, Blue reward:0.85\n", + "step: 951, Red action: do-nothing, Blue reward:0.85\n", + "step: 952, Red action: do-nothing, Blue reward:0.85\n", + "step: 953, Red action: do-nothing, Blue reward:0.85\n", + "step: 954, Red action: do-nothing, Blue reward:0.85\n", + "step: 955, Red action: do-nothing, Blue reward:0.85\n", + "step: 956, Red action: do-nothing, Blue reward:0.85\n", + "step: 957, Red action: do-nothing, Blue reward:0.85\n", + "step: 958, Red action: do-nothing, Blue reward:0.85\n", + "step: 959, Red action: do-nothing, Blue reward:0.85\n", + "step: 960, Red action: do-nothing, Blue reward:0.85\n", + "step: 961, Red action: do-nothing, Blue reward:0.85\n", + "step: 962, Red action: do-nothing, Blue reward:0.85\n", + "step: 963, Red action: node-application-execute, Blue reward:0.85\n", + "step: 964, Red action: do-nothing, Blue reward:0.85\n", + "step: 965, Red action: do-nothing, Blue reward:0.85\n", + "step: 966, Red action: do-nothing, Blue reward:0.85\n", + "step: 967, Red action: do-nothing, Blue reward:0.85\n", + "step: 968, Red action: do-nothing, Blue reward:0.85\n", + "step: 969, Red action: do-nothing, Blue reward:0.85\n", + "step: 970, Red action: do-nothing, Blue reward:0.85\n", + "step: 971, Red action: do-nothing, Blue reward:0.85\n", + "step: 972, Red action: do-nothing, Blue reward:0.85\n", + "step: 973, Red action: do-nothing, Blue reward:0.85\n", + "step: 974, Red action: do-nothing, Blue reward:0.85\n", + "step: 975, Red action: do-nothing, Blue reward:0.85\n", + "step: 976, Red action: do-nothing, Blue reward:0.85\n", + "step: 977, Red action: do-nothing, Blue reward:0.85\n", + "step: 978, Red action: do-nothing, Blue reward:0.85\n", + "step: 979, Red action: do-nothing, Blue reward:0.85\n", + "step: 980, Red action: do-nothing, Blue reward:0.85\n", + "step: 981, Red action: do-nothing, Blue reward:0.85\n", + "step: 982, Red action: do-nothing, Blue reward:0.85\n", + "step: 983, Red action: do-nothing, Blue reward:0.85\n", + "step: 984, Red action: do-nothing, Blue reward:0.85\n", + "step: 985, Red action: do-nothing, Blue reward:0.85\n", + "step: 986, Red action: do-nothing, Blue reward:0.85\n", + "step: 987, Red action: do-nothing, Blue reward:0.85\n", + "step: 988, Red action: node-application-execute, Blue reward:0.85\n", + "step: 989, Red action: do-nothing, Blue reward:0.85\n", + "step: 990, Red action: do-nothing, Blue reward:0.85\n", + "step: 991, Red action: do-nothing, Blue reward:0.85\n", + "step: 992, Red action: do-nothing, Blue reward:0.85\n", + "step: 993, Red action: do-nothing, Blue reward:0.85\n", + "step: 994, Red action: do-nothing, Blue reward:0.85\n", + "step: 995, Red action: do-nothing, Blue reward:0.85\n", + "step: 996, Red action: do-nothing, Blue reward:0.85\n", + "step: 997, Red action: do-nothing, Blue reward:0.85\n", + "step: 998, Red action: do-nothing, Blue reward:0.85\n", + "step: 999, Red action: do-nothing, Blue reward:0.85\n", + "step: 1000, Red action: do-nothing, Blue reward:0.85\n", + "step: 1001, Red action: do-nothing, Blue reward:0.85\n", + "step: 1002, Red action: do-nothing, Blue reward:0.85\n", + "step: 1003, Red action: do-nothing, Blue reward:0.85\n", + "step: 1004, Red action: do-nothing, Blue reward:0.85\n", + "step: 1005, Red action: do-nothing, Blue reward:0.85\n", + "step: 1006, Red action: do-nothing, Blue reward:0.85\n", + "step: 1007, Red action: do-nothing, Blue reward:0.85\n", + "step: 1008, Red action: do-nothing, Blue reward:0.85\n", + "step: 1009, Red action: do-nothing, Blue reward:0.85\n", + "step: 1010, Red action: do-nothing, Blue reward:0.85\n", + "step: 1011, Red action: do-nothing, Blue reward:0.85\n", + "step: 1012, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1013, Red action: do-nothing, Blue reward:0.85\n", + "step: 1014, Red action: do-nothing, Blue reward:0.85\n", + "step: 1015, Red action: do-nothing, Blue reward:0.85\n", + "step: 1016, Red action: do-nothing, Blue reward:0.85\n", + "step: 1017, Red action: do-nothing, Blue reward:0.85\n", + "step: 1018, Red action: do-nothing, Blue reward:0.85\n", + "step: 1019, Red action: do-nothing, Blue reward:0.85\n", + "step: 1020, Red action: do-nothing, Blue reward:0.85\n", + "step: 1021, Red action: do-nothing, Blue reward:0.85\n", + "step: 1022, Red action: do-nothing, Blue reward:0.85\n", + "step: 1023, Red action: do-nothing, Blue reward:0.85\n", + "step: 1024, Red action: do-nothing, Blue reward:0.85\n", + "step: 1025, Red action: do-nothing, Blue reward:0.85\n", + "step: 1026, Red action: do-nothing, Blue reward:0.85\n", + "step: 1027, Red action: do-nothing, Blue reward:0.85\n", + "step: 1028, Red action: do-nothing, Blue reward:0.85\n", + "step: 1029, Red action: do-nothing, Blue reward:0.85\n", + "step: 1030, Red action: do-nothing, Blue reward:0.85\n", + "step: 1031, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1032, Red action: do-nothing, Blue reward:0.85\n", + "step: 1033, Red action: do-nothing, Blue reward:0.85\n", + "step: 1034, Red action: do-nothing, Blue reward:0.85\n", + "step: 1035, Red action: do-nothing, Blue reward:0.85\n", + "step: 1036, Red action: do-nothing, Blue reward:0.85\n", + "step: 1037, Red action: do-nothing, Blue reward:0.85\n", + "step: 1038, Red action: do-nothing, Blue reward:0.85\n", + "step: 1039, Red action: do-nothing, Blue reward:0.85\n", + "step: 1040, Red action: do-nothing, Blue reward:0.85\n", + "step: 1041, Red action: do-nothing, Blue reward:0.85\n", + "step: 1042, Red action: do-nothing, Blue reward:0.85\n", + "step: 1043, Red action: do-nothing, Blue reward:0.85\n", + "step: 1044, Red action: do-nothing, Blue reward:0.85\n", + "step: 1045, Red action: do-nothing, Blue reward:0.85\n", + "step: 1046, Red action: do-nothing, Blue reward:0.85\n", + "step: 1047, Red action: do-nothing, Blue reward:0.85\n", + "step: 1048, Red action: do-nothing, Blue reward:0.85\n", + "step: 1049, Red action: do-nothing, Blue reward:0.85\n", + "step: 1050, Red action: do-nothing, Blue reward:0.85\n", + "step: 1051, Red action: do-nothing, Blue reward:0.85\n", + "step: 1052, Red action: do-nothing, Blue reward:0.85\n", + "step: 1053, Red action: do-nothing, Blue reward:0.85\n", + "step: 1054, Red action: do-nothing, Blue reward:0.85\n", + "step: 1055, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1056, Red action: do-nothing, Blue reward:0.85\n", + "step: 1057, Red action: do-nothing, Blue reward:0.85\n", + "step: 1058, Red action: do-nothing, Blue reward:0.85\n", + "step: 1059, Red action: do-nothing, Blue reward:0.85\n", + "step: 1060, Red action: do-nothing, Blue reward:0.85\n", + "step: 1061, Red action: do-nothing, Blue reward:0.85\n", + "step: 1062, Red action: do-nothing, Blue reward:0.85\n", + "step: 1063, Red action: do-nothing, Blue reward:0.85\n", + "step: 1064, Red action: do-nothing, Blue reward:0.85\n", + "step: 1065, Red action: do-nothing, Blue reward:0.85\n", + "step: 1066, Red action: do-nothing, Blue reward:0.85\n", + "step: 1067, Red action: do-nothing, Blue reward:0.85\n", + "step: 1068, Red action: do-nothing, Blue reward:0.85\n", + "step: 1069, Red action: do-nothing, Blue reward:0.85\n", + "step: 1070, Red action: do-nothing, Blue reward:0.85\n", + "step: 1071, Red action: do-nothing, Blue reward:0.85\n", + "step: 1072, Red action: do-nothing, Blue reward:0.85\n", + "step: 1073, Red action: do-nothing, Blue reward:0.85\n", + "step: 1074, Red action: do-nothing, Blue reward:0.85\n", + "step: 1075, Red action: do-nothing, Blue reward:0.85\n", + "step: 1076, Red action: do-nothing, Blue reward:0.85\n", + "step: 1077, Red action: do-nothing, Blue reward:0.85\n", + "step: 1078, Red action: do-nothing, Blue reward:0.85\n", + "step: 1079, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1080, Red action: do-nothing, Blue reward:0.85\n", + "step: 1081, Red action: do-nothing, Blue reward:0.85\n", + "step: 1082, Red action: do-nothing, Blue reward:0.85\n", + "step: 1083, Red action: do-nothing, Blue reward:0.85\n", + "step: 1084, Red action: do-nothing, Blue reward:0.85\n", + "step: 1085, Red action: do-nothing, Blue reward:0.85\n", + "step: 1086, Red action: do-nothing, Blue reward:0.85\n", + "step: 1087, Red action: do-nothing, Blue reward:0.85\n", + "step: 1088, Red action: do-nothing, Blue reward:0.85\n", + "step: 1089, Red action: do-nothing, Blue reward:0.85\n", + "step: 1090, Red action: do-nothing, Blue reward:0.85\n", + "step: 1091, Red action: do-nothing, Blue reward:0.85\n", + "step: 1092, Red action: do-nothing, Blue reward:0.85\n", + "step: 1093, Red action: do-nothing, Blue reward:0.85\n", + "step: 1094, Red action: do-nothing, Blue reward:0.85\n", + "step: 1095, Red action: do-nothing, Blue reward:0.85\n", + "step: 1096, Red action: do-nothing, Blue reward:0.85\n", + "step: 1097, Red action: do-nothing, Blue reward:0.85\n", + "step: 1098, Red action: do-nothing, Blue reward:0.85\n", + "step: 1099, Red action: do-nothing, Blue reward:0.85\n", + "step: 1100, Red action: do-nothing, Blue reward:0.85\n", + "step: 1101, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1102, Red action: do-nothing, Blue reward:0.85\n", + "step: 1103, Red action: do-nothing, Blue reward:0.85\n", + "step: 1104, Red action: do-nothing, Blue reward:0.85\n", + "step: 1105, Red action: do-nothing, Blue reward:0.85\n", + "step: 1106, Red action: do-nothing, Blue reward:0.85\n", + "step: 1107, Red action: do-nothing, Blue reward:0.85\n", + "step: 1108, Red action: do-nothing, Blue reward:0.85\n", + "step: 1109, Red action: do-nothing, Blue reward:0.85\n", + "step: 1110, Red action: do-nothing, Blue reward:0.85\n", + "step: 1111, Red action: do-nothing, Blue reward:0.85\n", + "step: 1112, Red action: do-nothing, Blue reward:0.85\n", + "step: 1113, Red action: do-nothing, Blue reward:0.85\n", + "step: 1114, Red action: do-nothing, Blue reward:0.85\n", + "step: 1115, Red action: do-nothing, Blue reward:0.85\n", + "step: 1116, Red action: do-nothing, Blue reward:0.85\n", + "step: 1117, Red action: do-nothing, Blue reward:0.85\n", + "step: 1118, Red action: do-nothing, Blue reward:0.85\n", + "step: 1119, Red action: do-nothing, Blue reward:0.85\n", + "step: 1120, Red action: do-nothing, Blue reward:0.85\n", + "step: 1121, Red action: do-nothing, Blue reward:0.85\n", + "step: 1122, Red action: do-nothing, Blue reward:0.85\n", + "step: 1123, Red action: do-nothing, Blue reward:0.85\n", + "step: 1124, Red action: do-nothing, Blue reward:0.85\n", + "step: 1125, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1126, Red action: do-nothing, Blue reward:0.85\n", + "step: 1127, Red action: do-nothing, Blue reward:0.85\n", + "step: 1128, Red action: do-nothing, Blue reward:0.85\n", + "step: 1129, Red action: do-nothing, Blue reward:0.85\n", + "step: 1130, Red action: do-nothing, Blue reward:0.85\n", + "step: 1131, Red action: do-nothing, Blue reward:0.85\n", + "step: 1132, Red action: do-nothing, Blue reward:0.85\n", + "step: 1133, Red action: do-nothing, Blue reward:0.85\n", + "step: 1134, Red action: do-nothing, Blue reward:0.85\n", + "step: 1135, Red action: do-nothing, Blue reward:0.85\n", + "step: 1136, Red action: do-nothing, Blue reward:0.85\n", + "step: 1137, Red action: do-nothing, Blue reward:0.85\n", + "step: 1138, Red action: do-nothing, Blue reward:0.85\n", + "step: 1139, Red action: do-nothing, Blue reward:0.85\n", + "step: 1140, Red action: do-nothing, Blue reward:0.85\n", + "step: 1141, Red action: do-nothing, Blue reward:0.85\n", + "step: 1142, Red action: do-nothing, Blue reward:0.85\n", + "step: 1143, Red action: do-nothing, Blue reward:0.85\n", + "step: 1144, Red action: do-nothing, Blue reward:0.85\n", + "step: 1145, Red action: do-nothing, Blue reward:0.85\n", + "step: 1146, Red action: do-nothing, Blue reward:0.85\n", + "step: 1147, Red action: do-nothing, Blue reward:0.85\n", + "step: 1148, Red action: do-nothing, Blue reward:0.85\n", + "step: 1149, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1150, Red action: do-nothing, Blue reward:0.85\n", + "step: 1151, Red action: do-nothing, Blue reward:0.85\n", + "step: 1152, Red action: do-nothing, Blue reward:0.85\n", + "step: 1153, Red action: do-nothing, Blue reward:0.85\n", + "step: 1154, Red action: do-nothing, Blue reward:0.85\n", + "step: 1155, Red action: do-nothing, Blue reward:0.85\n", + "step: 1156, Red action: do-nothing, Blue reward:0.85\n", + "step: 1157, Red action: do-nothing, Blue reward:0.85\n", + "step: 1158, Red action: do-nothing, Blue reward:0.85\n", + "step: 1159, Red action: do-nothing, Blue reward:0.85\n", + "step: 1160, Red action: do-nothing, Blue reward:0.85\n", + "step: 1161, Red action: do-nothing, Blue reward:0.85\n", + "step: 1162, Red action: do-nothing, Blue reward:0.85\n", + "step: 1163, Red action: do-nothing, Blue reward:0.85\n", + "step: 1164, Red action: do-nothing, Blue reward:0.85\n", + "step: 1165, Red action: do-nothing, Blue reward:0.85\n", + "step: 1166, Red action: do-nothing, Blue reward:0.85\n", + "step: 1167, Red action: do-nothing, Blue reward:0.85\n", + "step: 1168, Red action: do-nothing, Blue reward:0.85\n", + "step: 1169, Red action: do-nothing, Blue reward:0.85\n", + "step: 1170, Red action: do-nothing, Blue reward:0.85\n", + "step: 1171, Red action: do-nothing, Blue reward:0.85\n", + "step: 1172, Red action: do-nothing, Blue reward:0.85\n", + "step: 1173, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1174, Red action: do-nothing, Blue reward:0.85\n", + "step: 1175, Red action: do-nothing, Blue reward:0.85\n", + "step: 1176, Red action: do-nothing, Blue reward:0.85\n", + "step: 1177, Red action: do-nothing, Blue reward:0.85\n", + "step: 1178, Red action: do-nothing, Blue reward:0.85\n", + "step: 1179, Red action: do-nothing, Blue reward:0.85\n", + "step: 1180, Red action: do-nothing, Blue reward:0.85\n", + "step: 1181, Red action: do-nothing, Blue reward:0.85\n", + "step: 1182, Red action: do-nothing, Blue reward:0.85\n", + "step: 1183, Red action: do-nothing, Blue reward:0.85\n", + "step: 1184, Red action: do-nothing, Blue reward:0.85\n", + "step: 1185, Red action: do-nothing, Blue reward:0.85\n", + "step: 1186, Red action: do-nothing, Blue reward:0.85\n", + "step: 1187, Red action: do-nothing, Blue reward:0.85\n", + "step: 1188, Red action: do-nothing, Blue reward:0.85\n", + "step: 1189, Red action: do-nothing, Blue reward:0.85\n", + "step: 1190, Red action: do-nothing, Blue reward:0.85\n", + "step: 1191, Red action: do-nothing, Blue reward:0.85\n", + "step: 1192, Red action: do-nothing, Blue reward:0.85\n", + "step: 1193, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1194, Red action: do-nothing, Blue reward:0.85\n", + "step: 1195, Red action: do-nothing, Blue reward:0.85\n", + "step: 1196, Red action: do-nothing, Blue reward:0.85\n", + "step: 1197, Red action: do-nothing, Blue reward:0.85\n", + "step: 1198, Red action: do-nothing, Blue reward:0.85\n", + "step: 1199, Red action: do-nothing, Blue reward:0.85\n", + "step: 1200, Red action: do-nothing, Blue reward:0.85\n", + "step: 1201, Red action: do-nothing, Blue reward:0.85\n", + "step: 1202, Red action: do-nothing, Blue reward:0.85\n", + "step: 1203, Red action: do-nothing, Blue reward:0.85\n", + "step: 1204, Red action: do-nothing, Blue reward:0.85\n", + "step: 1205, Red action: do-nothing, Blue reward:0.85\n", + "step: 1206, Red action: do-nothing, Blue reward:0.85\n", + "step: 1207, Red action: do-nothing, Blue reward:0.85\n", + "step: 1208, Red action: do-nothing, Blue reward:0.85\n", + "step: 1209, Red action: do-nothing, Blue reward:0.85\n", + "step: 1210, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1211, Red action: do-nothing, Blue reward:0.85\n", + "step: 1212, Red action: do-nothing, Blue reward:0.85\n", + "step: 1213, Red action: do-nothing, Blue reward:0.85\n", + "step: 1214, Red action: do-nothing, Blue reward:0.85\n", + "step: 1215, Red action: do-nothing, Blue reward:0.85\n", + "step: 1216, Red action: do-nothing, Blue reward:0.85\n", + "step: 1217, Red action: do-nothing, Blue reward:0.85\n", + "step: 1218, Red action: do-nothing, Blue reward:0.85\n", + "step: 1219, Red action: do-nothing, Blue reward:0.85\n", + "step: 1220, Red action: do-nothing, Blue reward:0.85\n", + "step: 1221, Red action: do-nothing, Blue reward:0.85\n", + "step: 1222, Red action: do-nothing, Blue reward:0.85\n", + "step: 1223, Red action: do-nothing, Blue reward:0.85\n", + "step: 1224, Red action: do-nothing, Blue reward:0.85\n", + "step: 1225, Red action: do-nothing, Blue reward:0.85\n", + "step: 1226, Red action: do-nothing, Blue reward:0.85\n", + "step: 1227, Red action: do-nothing, Blue reward:0.85\n", + "step: 1228, Red action: do-nothing, Blue reward:0.85\n", + "step: 1229, Red action: do-nothing, Blue reward:0.85\n", + "step: 1230, Red action: do-nothing, Blue reward:0.85\n", + "step: 1231, Red action: do-nothing, Blue reward:0.85\n", + "step: 1232, Red action: do-nothing, Blue reward:0.85\n", + "step: 1233, Red action: do-nothing, Blue reward:0.85\n", + "step: 1234, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1235, Red action: do-nothing, Blue reward:0.85\n", + "step: 1236, Red action: do-nothing, Blue reward:0.85\n", + "step: 1237, Red action: do-nothing, Blue reward:0.85\n", + "step: 1238, Red action: do-nothing, Blue reward:0.85\n", + "step: 1239, Red action: do-nothing, Blue reward:0.85\n", + "step: 1240, Red action: do-nothing, Blue reward:0.85\n", + "step: 1241, Red action: do-nothing, Blue reward:0.85\n", + "step: 1242, Red action: do-nothing, Blue reward:0.85\n", + "step: 1243, Red action: do-nothing, Blue reward:0.85\n", + "step: 1244, Red action: do-nothing, Blue reward:0.85\n", + "step: 1245, Red action: do-nothing, Blue reward:0.85\n", + "step: 1246, Red action: do-nothing, Blue reward:0.85\n", + "step: 1247, Red action: do-nothing, Blue reward:0.85\n", + "step: 1248, Red action: do-nothing, Blue reward:0.85\n", + "step: 1249, Red action: do-nothing, Blue reward:0.85\n", + "step: 1250, Red action: do-nothing, Blue reward:0.85\n", + "step: 1251, Red action: do-nothing, Blue reward:0.85\n", + "step: 1252, Red action: do-nothing, Blue reward:0.85\n", + "step: 1253, Red action: do-nothing, Blue reward:0.85\n", + "step: 1254, Red action: do-nothing, Blue reward:0.85\n", + "step: 1255, Red action: do-nothing, Blue reward:0.85\n", + "step: 1256, Red action: do-nothing, Blue reward:0.85\n", + "step: 1257, Red action: do-nothing, Blue reward:0.85\n", + "step: 1258, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1259, Red action: do-nothing, Blue reward:0.85\n", + "step: 1260, Red action: do-nothing, Blue reward:0.85\n", + "step: 1261, Red action: do-nothing, Blue reward:0.85\n", + "step: 1262, Red action: do-nothing, Blue reward:0.85\n", + "step: 1263, Red action: do-nothing, Blue reward:0.85\n", + "step: 1264, Red action: do-nothing, Blue reward:0.85\n", + "step: 1265, Red action: do-nothing, Blue reward:0.85\n", + "step: 1266, Red action: do-nothing, Blue reward:0.85\n", + "step: 1267, Red action: do-nothing, Blue reward:0.85\n", + "step: 1268, Red action: do-nothing, Blue reward:0.85\n", + "step: 1269, Red action: do-nothing, Blue reward:0.85\n", + "step: 1270, Red action: do-nothing, Blue reward:0.85\n", + "step: 1271, Red action: do-nothing, Blue reward:0.85\n", + "step: 1272, Red action: do-nothing, Blue reward:0.85\n", + "step: 1273, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1274, Red action: do-nothing, Blue reward:0.85\n", + "step: 1275, Red action: do-nothing, Blue reward:0.85\n", + "step: 1276, Red action: do-nothing, Blue reward:0.85\n", + "step: 1277, Red action: do-nothing, Blue reward:0.85\n", + "step: 1278, Red action: do-nothing, Blue reward:0.85\n", + "step: 1279, Red action: do-nothing, Blue reward:0.85\n", + "step: 1280, Red action: do-nothing, Blue reward:0.85\n", + "step: 1281, Red action: do-nothing, Blue reward:0.85\n", + "step: 1282, Red action: do-nothing, Blue reward:0.85\n", + "step: 1283, Red action: do-nothing, Blue reward:0.85\n", + "step: 1284, Red action: do-nothing, Blue reward:0.85\n", + "step: 1285, Red action: do-nothing, Blue reward:0.85\n", + "step: 1286, Red action: do-nothing, Blue reward:0.85\n", + "step: 1287, Red action: do-nothing, Blue reward:0.85\n", + "step: 1288, Red action: do-nothing, Blue reward:0.85\n", + "step: 1289, Red action: do-nothing, Blue reward:0.85\n", + "step: 1290, Red action: do-nothing, Blue reward:0.85\n", + "step: 1291, Red action: do-nothing, Blue reward:0.85\n", + "step: 1292, Red action: do-nothing, Blue reward:0.85\n", + "step: 1293, Red action: do-nothing, Blue reward:0.85\n", + "step: 1294, Red action: do-nothing, Blue reward:0.85\n", + "step: 1295, Red action: do-nothing, Blue reward:0.85\n", + "step: 1296, Red action: do-nothing, Blue reward:0.85\n", + "step: 1297, Red action: do-nothing, Blue reward:0.85\n", + "step: 1298, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1299, Red action: do-nothing, Blue reward:0.85\n", + "step: 1300, Red action: do-nothing, Blue reward:0.85\n", + "step: 1301, Red action: do-nothing, Blue reward:0.85\n", + "step: 1302, Red action: do-nothing, Blue reward:0.85\n", + "step: 1303, Red action: do-nothing, Blue reward:0.85\n", + "step: 1304, Red action: do-nothing, Blue reward:0.85\n", + "step: 1305, Red action: do-nothing, Blue reward:0.85\n", + "step: 1306, Red action: do-nothing, Blue reward:0.85\n", + "step: 1307, Red action: do-nothing, Blue reward:0.85\n", + "step: 1308, Red action: do-nothing, Blue reward:0.85\n", + "step: 1309, Red action: do-nothing, Blue reward:0.85\n", + "step: 1310, Red action: do-nothing, Blue reward:0.85\n", + "step: 1311, Red action: do-nothing, Blue reward:0.85\n", + "step: 1312, Red action: do-nothing, Blue reward:0.85\n", + "step: 1313, Red action: do-nothing, Blue reward:0.85\n", + "step: 1314, Red action: do-nothing, Blue reward:0.85\n", + "step: 1315, Red action: do-nothing, Blue reward:0.85\n", + "step: 1316, Red action: do-nothing, Blue reward:0.85\n", + "step: 1317, Red action: do-nothing, Blue reward:0.85\n", + "step: 1318, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1319, Red action: do-nothing, Blue reward:0.85\n", + "step: 1320, Red action: do-nothing, Blue reward:0.85\n", + "step: 1321, Red action: do-nothing, Blue reward:0.85\n", + "step: 1322, Red action: do-nothing, Blue reward:0.85\n", + "step: 1323, Red action: do-nothing, Blue reward:0.85\n", + "step: 1324, Red action: do-nothing, Blue reward:0.85\n", + "step: 1325, Red action: do-nothing, Blue reward:0.85\n", + "step: 1326, Red action: do-nothing, Blue reward:0.85\n", + "step: 1327, Red action: do-nothing, Blue reward:0.85\n", + "step: 1328, Red action: do-nothing, Blue reward:0.85\n", + "step: 1329, Red action: do-nothing, Blue reward:0.85\n", + "step: 1330, Red action: do-nothing, Blue reward:0.85\n", + "step: 1331, Red action: do-nothing, Blue reward:0.85\n", + "step: 1332, Red action: do-nothing, Blue reward:0.85\n", + "step: 1333, Red action: do-nothing, Blue reward:0.85\n", + "step: 1334, Red action: do-nothing, Blue reward:0.85\n", + "step: 1335, Red action: do-nothing, Blue reward:0.85\n", + "step: 1336, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1337, Red action: do-nothing, Blue reward:0.85\n", + "step: 1338, Red action: do-nothing, Blue reward:0.85\n", + "step: 1339, Red action: do-nothing, Blue reward:0.85\n", + "step: 1340, Red action: do-nothing, Blue reward:0.85\n", + "step: 1341, Red action: do-nothing, Blue reward:0.85\n", + "step: 1342, Red action: do-nothing, Blue reward:0.85\n", + "step: 1343, Red action: do-nothing, Blue reward:0.85\n", + "step: 1344, Red action: do-nothing, Blue reward:0.85\n", + "step: 1345, Red action: do-nothing, Blue reward:0.85\n", + "step: 1346, Red action: do-nothing, Blue reward:0.85\n", + "step: 1347, Red action: do-nothing, Blue reward:0.85\n", + "step: 1348, Red action: do-nothing, Blue reward:0.85\n", + "step: 1349, Red action: do-nothing, Blue reward:0.85\n", + "step: 1350, Red action: do-nothing, Blue reward:0.85\n", + "step: 1351, Red action: do-nothing, Blue reward:0.85\n", + "step: 1352, Red action: do-nothing, Blue reward:0.85\n", + "step: 1353, Red action: do-nothing, Blue reward:0.85\n", + "step: 1354, Red action: do-nothing, Blue reward:0.85\n", + "step: 1355, Red action: do-nothing, Blue reward:0.85\n", + "step: 1356, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1357, Red action: do-nothing, Blue reward:0.85\n", + "step: 1358, Red action: do-nothing, Blue reward:0.85\n", + "step: 1359, Red action: do-nothing, Blue reward:0.85\n", + "step: 1360, Red action: do-nothing, Blue reward:0.85\n", + "step: 1361, Red action: do-nothing, Blue reward:0.85\n", + "step: 1362, Red action: do-nothing, Blue reward:0.85\n", + "step: 1363, Red action: do-nothing, Blue reward:0.85\n", + "step: 1364, Red action: do-nothing, Blue reward:0.85\n", + "step: 1365, Red action: do-nothing, Blue reward:0.85\n", + "step: 1366, Red action: do-nothing, Blue reward:0.85\n", + "step: 1367, Red action: do-nothing, Blue reward:0.85\n", + "step: 1368, Red action: do-nothing, Blue reward:0.85\n", + "step: 1369, Red action: do-nothing, Blue reward:0.85\n", + "step: 1370, Red action: do-nothing, Blue reward:0.85\n", + "step: 1371, Red action: do-nothing, Blue reward:0.85\n", + "step: 1372, Red action: do-nothing, Blue reward:0.85\n", + "step: 1373, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1374, Red action: do-nothing, Blue reward:0.85\n", + "step: 1375, Red action: do-nothing, Blue reward:0.85\n", + "step: 1376, Red action: do-nothing, Blue reward:0.85\n", + "step: 1377, Red action: do-nothing, Blue reward:0.85\n", + "step: 1378, Red action: do-nothing, Blue reward:0.85\n", + "step: 1379, Red action: do-nothing, Blue reward:0.85\n", + "step: 1380, Red action: do-nothing, Blue reward:0.85\n", + "step: 1381, Red action: do-nothing, Blue reward:0.85\n", + "step: 1382, Red action: do-nothing, Blue reward:0.85\n", + "step: 1383, Red action: do-nothing, Blue reward:0.85\n", + "step: 1384, Red action: do-nothing, Blue reward:0.85\n", + "step: 1385, Red action: do-nothing, Blue reward:0.85\n", + "step: 1386, Red action: do-nothing, Blue reward:0.85\n", + "step: 1387, Red action: do-nothing, Blue reward:0.85\n", + "step: 1388, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1389, Red action: do-nothing, Blue reward:0.85\n", + "step: 1390, Red action: do-nothing, Blue reward:0.85\n", + "step: 1391, Red action: do-nothing, Blue reward:0.85\n", + "step: 1392, Red action: do-nothing, Blue reward:0.85\n", + "step: 1393, Red action: do-nothing, Blue reward:0.85\n", + "step: 1394, Red action: do-nothing, Blue reward:0.85\n", + "step: 1395, Red action: do-nothing, Blue reward:0.85\n", + "step: 1396, Red action: do-nothing, Blue reward:0.85\n", + "step: 1397, Red action: do-nothing, Blue reward:0.85\n", + "step: 1398, Red action: do-nothing, Blue reward:0.85\n", + "step: 1399, Red action: do-nothing, Blue reward:0.85\n", + "step: 1400, Red action: do-nothing, Blue reward:0.85\n", + "step: 1401, Red action: do-nothing, Blue reward:0.85\n", + "step: 1402, Red action: do-nothing, Blue reward:0.85\n", + "step: 1403, Red action: do-nothing, Blue reward:0.85\n", + "step: 1404, Red action: do-nothing, Blue reward:0.85\n", + "step: 1405, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1406, Red action: do-nothing, Blue reward:0.85\n", + "step: 1407, Red action: do-nothing, Blue reward:0.85\n", + "step: 1408, Red action: do-nothing, Blue reward:0.85\n", + "step: 1409, Red action: do-nothing, Blue reward:0.85\n", + "step: 1410, Red action: do-nothing, Blue reward:0.85\n", + "step: 1411, Red action: do-nothing, Blue reward:0.85\n", + "step: 1412, Red action: do-nothing, Blue reward:0.85\n", + "step: 1413, Red action: do-nothing, Blue reward:0.85\n", + "step: 1414, Red action: do-nothing, Blue reward:0.85\n", + "step: 1415, Red action: do-nothing, Blue reward:0.85\n", + "step: 1416, Red action: do-nothing, Blue reward:0.85\n", + "step: 1417, Red action: do-nothing, Blue reward:0.85\n", + "step: 1418, Red action: do-nothing, Blue reward:0.85\n", + "step: 1419, Red action: do-nothing, Blue reward:0.85\n", + "step: 1420, Red action: do-nothing, Blue reward:0.85\n", + "step: 1421, Red action: do-nothing, Blue reward:0.85\n", + "step: 1422, Red action: do-nothing, Blue reward:0.85\n", + "step: 1423, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1424, Red action: do-nothing, Blue reward:0.85\n", + "step: 1425, Red action: do-nothing, Blue reward:0.85\n", + "step: 1426, Red action: do-nothing, Blue reward:0.85\n", + "step: 1427, Red action: do-nothing, Blue reward:0.85\n", + "step: 1428, Red action: do-nothing, Blue reward:0.85\n", + "step: 1429, Red action: do-nothing, Blue reward:0.85\n", + "step: 1430, Red action: do-nothing, Blue reward:0.85\n", + "step: 1431, Red action: do-nothing, Blue reward:0.85\n", + "step: 1432, Red action: do-nothing, Blue reward:0.85\n", + "step: 1433, Red action: do-nothing, Blue reward:0.85\n", + "step: 1434, Red action: do-nothing, Blue reward:0.85\n", + "step: 1435, Red action: do-nothing, Blue reward:0.85\n", + "step: 1436, Red action: do-nothing, Blue reward:0.85\n", + "step: 1437, Red action: do-nothing, Blue reward:0.85\n", + "step: 1438, Red action: do-nothing, Blue reward:0.85\n", + "step: 1439, Red action: do-nothing, Blue reward:0.85\n", + "step: 1440, Red action: do-nothing, Blue reward:0.85\n", + "step: 1441, Red action: do-nothing, Blue reward:0.85\n", + "step: 1442, Red action: do-nothing, Blue reward:0.85\n", + "step: 1443, Red action: do-nothing, Blue reward:0.85\n", + "step: 1444, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1445, Red action: do-nothing, Blue reward:0.85\n", + "step: 1446, Red action: do-nothing, Blue reward:0.85\n", + "step: 1447, Red action: do-nothing, Blue reward:0.85\n", + "step: 1448, Red action: do-nothing, Blue reward:0.85\n", + "step: 1449, Red action: do-nothing, Blue reward:0.85\n", + "step: 1450, Red action: do-nothing, Blue reward:0.85\n", + "step: 1451, Red action: do-nothing, Blue reward:0.85\n", + "step: 1452, Red action: do-nothing, Blue reward:0.85\n", + "step: 1453, Red action: do-nothing, Blue reward:0.85\n", + "step: 1454, Red action: do-nothing, Blue reward:0.85\n", + "step: 1455, Red action: do-nothing, Blue reward:0.85\n", + "step: 1456, Red action: do-nothing, Blue reward:0.85\n", + "step: 1457, Red action: do-nothing, Blue reward:0.85\n", + "step: 1458, Red action: do-nothing, Blue reward:0.85\n", + "step: 1459, Red action: do-nothing, Blue reward:0.85\n", + "step: 1460, Red action: do-nothing, Blue reward:0.85\n", + "step: 1461, Red action: do-nothing, Blue reward:0.85\n", + "step: 1462, Red action: do-nothing, Blue reward:0.85\n", + "step: 1463, Red action: do-nothing, Blue reward:0.85\n", + "step: 1464, Red action: do-nothing, Blue reward:0.85\n", + "step: 1465, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1466, Red action: do-nothing, Blue reward:0.85\n", + "step: 1467, Red action: do-nothing, Blue reward:0.85\n", + "step: 1468, Red action: do-nothing, Blue reward:0.85\n", + "step: 1469, Red action: do-nothing, Blue reward:0.85\n", + "step: 1470, Red action: do-nothing, Blue reward:0.85\n", + "step: 1471, Red action: do-nothing, Blue reward:0.85\n", + "step: 1472, Red action: do-nothing, Blue reward:0.85\n", + "step: 1473, Red action: do-nothing, Blue reward:0.85\n", + "step: 1474, Red action: do-nothing, Blue reward:0.85\n", + "step: 1475, Red action: do-nothing, Blue reward:0.85\n", + "step: 1476, Red action: do-nothing, Blue reward:0.85\n", + "step: 1477, Red action: do-nothing, Blue reward:0.85\n", + "step: 1478, Red action: do-nothing, Blue reward:0.85\n", + "step: 1479, Red action: do-nothing, Blue reward:0.85\n", + "step: 1480, Red action: do-nothing, Blue reward:0.85\n", + "step: 1481, Red action: do-nothing, Blue reward:0.85\n", + "step: 1482, Red action: do-nothing, Blue reward:0.85\n", + "step: 1483, Red action: do-nothing, Blue reward:0.85\n", + "step: 1484, Red action: do-nothing, Blue reward:0.85\n", + "step: 1485, Red action: do-nothing, Blue reward:0.85\n", + "step: 1486, Red action: do-nothing, Blue reward:0.85\n", + "step: 1487, Red action: do-nothing, Blue reward:0.85\n", + "step: 1488, Red action: do-nothing, Blue reward:0.85\n", + "step: 1489, Red action: do-nothing, Blue reward:0.85\n", + "step: 1490, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1491, Red action: do-nothing, Blue reward:0.85\n", + "step: 1492, Red action: do-nothing, Blue reward:0.85\n", + "step: 1493, Red action: do-nothing, Blue reward:0.85\n", + "step: 1494, Red action: do-nothing, Blue reward:0.85\n", + "step: 1495, Red action: do-nothing, Blue reward:0.85\n", + "step: 1496, Red action: do-nothing, Blue reward:0.85\n", + "step: 1497, Red action: do-nothing, Blue reward:0.85\n", + "step: 1498, Red action: do-nothing, Blue reward:0.85\n", + "step: 1499, Red action: do-nothing, Blue reward:0.85\n", + "step: 1500, Red action: do-nothing, Blue reward:0.85\n", + "step: 1501, Red action: do-nothing, Blue reward:0.85\n", + "step: 1502, Red action: do-nothing, Blue reward:0.85\n", + "step: 1503, Red action: do-nothing, Blue reward:0.85\n", + "step: 1504, Red action: do-nothing, Blue reward:0.85\n", + "step: 1505, Red action: do-nothing, Blue reward:0.85\n", + "step: 1506, Red action: do-nothing, Blue reward:0.85\n", + "step: 1507, Red action: do-nothing, Blue reward:0.85\n", + "step: 1508, Red action: do-nothing, Blue reward:0.85\n", + "step: 1509, Red action: do-nothing, Blue reward:0.85\n", + "step: 1510, Red action: do-nothing, Blue reward:0.85\n", + "step: 1511, Red action: do-nothing, Blue reward:0.85\n", + "step: 1512, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1513, Red action: do-nothing, Blue reward:0.85\n", + "step: 1514, Red action: do-nothing, Blue reward:0.85\n", + "step: 1515, Red action: do-nothing, Blue reward:0.85\n", + "step: 1516, Red action: do-nothing, Blue reward:0.85\n", + "step: 1517, Red action: do-nothing, Blue reward:0.85\n", + "step: 1518, Red action: do-nothing, Blue reward:0.85\n", + "step: 1519, Red action: do-nothing, Blue reward:0.85\n", + "step: 1520, Red action: do-nothing, Blue reward:0.85\n", + "step: 1521, Red action: do-nothing, Blue reward:0.85\n", + "step: 1522, Red action: do-nothing, Blue reward:0.85\n", + "step: 1523, Red action: do-nothing, Blue reward:0.85\n", + "step: 1524, Red action: do-nothing, Blue reward:0.85\n", + "step: 1525, Red action: do-nothing, Blue reward:0.85\n", + "step: 1526, Red action: do-nothing, Blue reward:0.85\n", + "step: 1527, Red action: do-nothing, Blue reward:0.85\n", + "step: 1528, Red action: do-nothing, Blue reward:0.85\n", + "step: 1529, Red action: do-nothing, Blue reward:0.85\n", + "step: 1530, Red action: do-nothing, Blue reward:0.85\n", + "step: 1531, Red action: do-nothing, Blue reward:0.85\n", + "step: 1532, Red action: do-nothing, Blue reward:0.85\n", + "step: 1533, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1534, Red action: do-nothing, Blue reward:0.85\n", + "step: 1535, Red action: do-nothing, Blue reward:0.85\n", + "step: 1536, Red action: do-nothing, Blue reward:0.85\n", + "step: 1537, Red action: do-nothing, Blue reward:0.85\n", + "step: 1538, Red action: do-nothing, Blue reward:0.85\n", + "step: 1539, Red action: do-nothing, Blue reward:0.85\n", + "step: 1540, Red action: do-nothing, Blue reward:0.85\n", + "step: 1541, Red action: do-nothing, Blue reward:0.85\n", + "step: 1542, Red action: do-nothing, Blue reward:0.85\n", + "step: 1543, Red action: do-nothing, Blue reward:0.85\n", + "step: 1544, Red action: do-nothing, Blue reward:0.85\n", + "step: 1545, Red action: do-nothing, Blue reward:0.85\n", + "step: 1546, Red action: do-nothing, Blue reward:0.85\n", + "step: 1547, Red action: do-nothing, Blue reward:0.85\n", + "step: 1548, Red action: do-nothing, Blue reward:0.85\n", + "step: 1549, Red action: do-nothing, Blue reward:0.85\n", + "step: 1550, Red action: do-nothing, Blue reward:0.85\n", + "step: 1551, Red action: do-nothing, Blue reward:0.85\n", + "step: 1552, Red action: do-nothing, Blue reward:0.85\n", + "step: 1553, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1554, Red action: do-nothing, Blue reward:0.85\n", + "step: 1555, Red action: do-nothing, Blue reward:0.85\n", + "step: 1556, Red action: do-nothing, Blue reward:0.85\n", + "step: 1557, Red action: do-nothing, Blue reward:0.85\n", + "step: 1558, Red action: do-nothing, Blue reward:0.85\n", + "step: 1559, Red action: do-nothing, Blue reward:0.85\n", + "step: 1560, Red action: do-nothing, Blue reward:0.85\n", + "step: 1561, Red action: do-nothing, Blue reward:0.85\n", + "step: 1562, Red action: do-nothing, Blue reward:0.85\n", + "step: 1563, Red action: do-nothing, Blue reward:0.85\n", + "step: 1564, Red action: do-nothing, Blue reward:0.85\n", + "step: 1565, Red action: do-nothing, Blue reward:0.85\n", + "step: 1566, Red action: do-nothing, Blue reward:0.85\n", + "step: 1567, Red action: do-nothing, Blue reward:0.85\n", + "step: 1568, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1569, Red action: do-nothing, Blue reward:0.85\n", + "step: 1570, Red action: do-nothing, Blue reward:0.85\n", + "step: 1571, Red action: do-nothing, Blue reward:0.85\n", + "step: 1572, Red action: do-nothing, Blue reward:0.85\n", + "step: 1573, Red action: do-nothing, Blue reward:0.85\n", + "step: 1574, Red action: do-nothing, Blue reward:0.85\n", + "step: 1575, Red action: do-nothing, Blue reward:0.85\n", + "step: 1576, Red action: do-nothing, Blue reward:0.85\n", + "step: 1577, Red action: do-nothing, Blue reward:0.85\n", + "step: 1578, Red action: do-nothing, Blue reward:0.85\n", + "step: 1579, Red action: do-nothing, Blue reward:0.85\n", + "step: 1580, Red action: do-nothing, Blue reward:0.85\n", + "step: 1581, Red action: do-nothing, Blue reward:0.85\n", + "step: 1582, Red action: do-nothing, Blue reward:0.85\n", + "step: 1583, Red action: do-nothing, Blue reward:0.85\n", + "step: 1584, Red action: do-nothing, Blue reward:0.85\n", + "step: 1585, Red action: do-nothing, Blue reward:0.85\n", + "step: 1586, Red action: do-nothing, Blue reward:0.85\n", + "step: 1587, Red action: do-nothing, Blue reward:0.85\n", + "step: 1588, Red action: do-nothing, Blue reward:0.85\n", + "step: 1589, Red action: do-nothing, Blue reward:0.85\n", + "step: 1590, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1591, Red action: do-nothing, Blue reward:0.85\n", + "step: 1592, Red action: do-nothing, Blue reward:0.85\n", + "step: 1593, Red action: do-nothing, Blue reward:0.85\n", + "step: 1594, Red action: do-nothing, Blue reward:0.85\n", + "step: 1595, Red action: do-nothing, Blue reward:0.85\n", + "step: 1596, Red action: do-nothing, Blue reward:0.85\n", + "step: 1597, Red action: do-nothing, Blue reward:0.85\n", + "step: 1598, Red action: do-nothing, Blue reward:0.85\n", + "step: 1599, Red action: do-nothing, Blue reward:0.85\n", + "step: 1600, Red action: do-nothing, Blue reward:0.85\n", + "step: 1601, Red action: do-nothing, Blue reward:0.85\n", + "step: 1602, Red action: do-nothing, Blue reward:0.85\n", + "step: 1603, Red action: do-nothing, Blue reward:0.85\n", + "step: 1604, Red action: do-nothing, Blue reward:0.85\n", + "step: 1605, Red action: do-nothing, Blue reward:0.85\n", + "step: 1606, Red action: do-nothing, Blue reward:0.85\n", + "step: 1607, Red action: do-nothing, Blue reward:0.85\n", + "step: 1608, Red action: do-nothing, Blue reward:0.85\n", + "step: 1609, Red action: do-nothing, Blue reward:0.85\n", + "step: 1610, Red action: do-nothing, Blue reward:0.85\n", + "step: 1611, Red action: do-nothing, Blue reward:0.85\n", + "step: 1612, Red action: do-nothing, Blue reward:0.85\n", + "step: 1613, Red action: do-nothing, Blue reward:0.85\n", + "step: 1614, Red action: do-nothing, Blue reward:0.85\n", + "step: 1615, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1616, Red action: do-nothing, Blue reward:0.85\n", + "step: 1617, Red action: do-nothing, Blue reward:0.85\n", + "step: 1618, Red action: do-nothing, Blue reward:0.85\n", + "step: 1619, Red action: do-nothing, Blue reward:0.85\n", + "step: 1620, Red action: do-nothing, Blue reward:0.85\n", + "step: 1621, Red action: do-nothing, Blue reward:0.85\n", + "step: 1622, Red action: do-nothing, Blue reward:0.85\n", + "step: 1623, Red action: do-nothing, Blue reward:0.85\n", + "step: 1624, Red action: do-nothing, Blue reward:0.85\n", + "step: 1625, Red action: do-nothing, Blue reward:0.85\n", + "step: 1626, Red action: do-nothing, Blue reward:0.85\n", + "step: 1627, Red action: do-nothing, Blue reward:0.85\n", + "step: 1628, Red action: do-nothing, Blue reward:0.85\n", + "step: 1629, Red action: do-nothing, Blue reward:0.85\n", + "step: 1630, Red action: do-nothing, Blue reward:0.85\n", + "step: 1631, Red action: do-nothing, Blue reward:0.85\n", + "step: 1632, Red action: do-nothing, Blue reward:0.85\n", + "step: 1633, Red action: do-nothing, Blue reward:0.85\n", + "step: 1634, Red action: do-nothing, Blue reward:0.85\n", + "step: 1635, Red action: do-nothing, Blue reward:0.85\n", + "step: 1636, Red action: do-nothing, Blue reward:0.85\n", + "step: 1637, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1638, Red action: do-nothing, Blue reward:0.85\n", + "step: 1639, Red action: do-nothing, Blue reward:0.85\n", + "step: 1640, Red action: do-nothing, Blue reward:0.85\n", + "step: 1641, Red action: do-nothing, Blue reward:0.85\n", + "step: 1642, Red action: do-nothing, Blue reward:0.85\n", + "step: 1643, Red action: do-nothing, Blue reward:0.85\n", + "step: 1644, Red action: do-nothing, Blue reward:0.85\n", + "step: 1645, Red action: do-nothing, Blue reward:0.85\n", + "step: 1646, Red action: do-nothing, Blue reward:0.85\n", + "step: 1647, Red action: do-nothing, Blue reward:0.85\n", + "step: 1648, Red action: do-nothing, Blue reward:0.85\n", + "step: 1649, Red action: do-nothing, Blue reward:0.85\n", + "step: 1650, Red action: do-nothing, Blue reward:0.85\n", + "step: 1651, Red action: do-nothing, Blue reward:0.85\n", + "step: 1652, Red action: do-nothing, Blue reward:0.85\n", + "step: 1653, Red action: do-nothing, Blue reward:0.85\n", + "step: 1654, Red action: do-nothing, Blue reward:0.85\n", + "step: 1655, Red action: do-nothing, Blue reward:0.85\n", + "step: 1656, Red action: do-nothing, Blue reward:0.85\n", + "step: 1657, Red action: do-nothing, Blue reward:0.85\n", + "step: 1658, Red action: do-nothing, Blue reward:0.85\n", + "step: 1659, Red action: do-nothing, Blue reward:0.85\n", + "step: 1660, Red action: do-nothing, Blue reward:0.85\n", + "step: 1661, Red action: do-nothing, Blue reward:0.85\n", + "step: 1662, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1663, Red action: do-nothing, Blue reward:0.85\n", + "step: 1664, Red action: do-nothing, Blue reward:0.85\n", + "step: 1665, Red action: do-nothing, Blue reward:0.85\n", + "step: 1666, Red action: do-nothing, Blue reward:0.85\n", + "step: 1667, Red action: do-nothing, Blue reward:0.85\n", + "step: 1668, Red action: do-nothing, Blue reward:0.85\n", + "step: 1669, Red action: do-nothing, Blue reward:0.85\n", + "step: 1670, Red action: do-nothing, Blue reward:0.85\n", + "step: 1671, Red action: do-nothing, Blue reward:0.85\n", + "step: 1672, Red action: do-nothing, Blue reward:0.85\n", + "step: 1673, Red action: do-nothing, Blue reward:0.85\n", + "step: 1674, Red action: do-nothing, Blue reward:0.85\n", + "step: 1675, Red action: do-nothing, Blue reward:0.85\n", + "step: 1676, Red action: do-nothing, Blue reward:0.85\n", + "step: 1677, Red action: do-nothing, Blue reward:0.85\n", + "step: 1678, Red action: do-nothing, Blue reward:0.85\n", + "step: 1679, Red action: do-nothing, Blue reward:0.85\n", + "step: 1680, Red action: do-nothing, Blue reward:0.85\n", + "step: 1681, Red action: do-nothing, Blue reward:0.85\n", + "step: 1682, Red action: do-nothing, Blue reward:0.85\n", + "step: 1683, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1684, Red action: do-nothing, Blue reward:0.85\n", + "step: 1685, Red action: do-nothing, Blue reward:0.85\n", + "step: 1686, Red action: do-nothing, Blue reward:0.85\n", + "step: 1687, Red action: do-nothing, Blue reward:0.85\n", + "step: 1688, Red action: do-nothing, Blue reward:0.85\n", + "step: 1689, Red action: do-nothing, Blue reward:0.85\n", + "step: 1690, Red action: do-nothing, Blue reward:0.85\n", + "step: 1691, Red action: do-nothing, Blue reward:0.85\n", + "step: 1692, Red action: do-nothing, Blue reward:0.85\n", + "step: 1693, Red action: do-nothing, Blue reward:0.85\n", + "step: 1694, Red action: do-nothing, Blue reward:0.85\n", + "step: 1695, Red action: do-nothing, Blue reward:0.85\n", + "step: 1696, Red action: do-nothing, Blue reward:0.85\n", + "step: 1697, Red action: do-nothing, Blue reward:0.85\n", + "step: 1698, Red action: do-nothing, Blue reward:0.85\n", + "step: 1699, Red action: do-nothing, Blue reward:0.85\n", + "step: 1700, Red action: do-nothing, Blue reward:0.85\n", + "step: 1701, Red action: do-nothing, Blue reward:0.85\n", + "step: 1702, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1703, Red action: do-nothing, Blue reward:0.85\n", + "step: 1704, Red action: do-nothing, Blue reward:0.85\n", + "step: 1705, Red action: do-nothing, Blue reward:0.85\n", + "step: 1706, Red action: do-nothing, Blue reward:0.85\n", + "step: 1707, Red action: do-nothing, Blue reward:0.85\n", + "step: 1708, Red action: do-nothing, Blue reward:0.85\n", + "step: 1709, Red action: do-nothing, Blue reward:0.85\n", + "step: 1710, Red action: do-nothing, Blue reward:0.85\n", + "step: 1711, Red action: do-nothing, Blue reward:0.85\n", + "step: 1712, Red action: do-nothing, Blue reward:0.85\n", + "step: 1713, Red action: do-nothing, Blue reward:0.85\n", + "step: 1714, Red action: do-nothing, Blue reward:0.85\n", + "step: 1715, Red action: do-nothing, Blue reward:0.85\n", + "step: 1716, Red action: do-nothing, Blue reward:0.85\n", + "step: 1717, Red action: do-nothing, Blue reward:0.85\n", + "step: 1718, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1719, Red action: do-nothing, Blue reward:0.85\n", + "step: 1720, Red action: do-nothing, Blue reward:0.85\n", + "step: 1721, Red action: do-nothing, Blue reward:0.85\n", + "step: 1722, Red action: do-nothing, Blue reward:0.85\n", + "step: 1723, Red action: do-nothing, Blue reward:0.85\n", + "step: 1724, Red action: do-nothing, Blue reward:0.85\n", + "step: 1725, Red action: do-nothing, Blue reward:0.85\n", + "step: 1726, Red action: do-nothing, Blue reward:0.85\n", + "step: 1727, Red action: do-nothing, Blue reward:0.85\n", + "step: 1728, Red action: do-nothing, Blue reward:0.85\n", + "step: 1729, Red action: do-nothing, Blue reward:0.85\n", + "step: 1730, Red action: do-nothing, Blue reward:0.85\n", + "step: 1731, Red action: do-nothing, Blue reward:0.85\n", + "step: 1732, Red action: do-nothing, Blue reward:0.85\n", + "step: 1733, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1734, Red action: do-nothing, Blue reward:0.85\n", + "step: 1735, Red action: do-nothing, Blue reward:0.85\n", + "step: 1736, Red action: do-nothing, Blue reward:0.85\n", + "step: 1737, Red action: do-nothing, Blue reward:0.85\n", + "step: 1738, Red action: do-nothing, Blue reward:0.85\n", + "step: 1739, Red action: do-nothing, Blue reward:0.85\n", + "step: 1740, Red action: do-nothing, Blue reward:0.85\n", + "step: 1741, Red action: do-nothing, Blue reward:0.85\n", + "step: 1742, Red action: do-nothing, Blue reward:0.85\n", + "step: 1743, Red action: do-nothing, Blue reward:0.85\n", + "step: 1744, Red action: do-nothing, Blue reward:0.85\n", + "step: 1745, Red action: do-nothing, Blue reward:0.85\n", + "step: 1746, Red action: do-nothing, Blue reward:0.85\n", + "step: 1747, Red action: do-nothing, Blue reward:0.85\n", + "step: 1748, Red action: do-nothing, Blue reward:0.85\n", + "step: 1749, Red action: do-nothing, Blue reward:0.85\n", + "step: 1750, Red action: do-nothing, Blue reward:0.85\n", + "step: 1751, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1752, Red action: do-nothing, Blue reward:0.85\n", + "step: 1753, Red action: do-nothing, Blue reward:0.85\n", + "step: 1754, Red action: do-nothing, Blue reward:0.85\n", + "step: 1755, Red action: do-nothing, Blue reward:0.85\n", + "step: 1756, Red action: do-nothing, Blue reward:0.85\n", + "step: 1757, Red action: do-nothing, Blue reward:0.85\n", + "step: 1758, Red action: do-nothing, Blue reward:0.85\n", + "step: 1759, Red action: do-nothing, Blue reward:0.85\n", + "step: 1760, Red action: do-nothing, Blue reward:0.85\n", + "step: 1761, Red action: do-nothing, Blue reward:0.85\n", + "step: 1762, Red action: do-nothing, Blue reward:0.85\n", + "step: 1763, Red action: do-nothing, Blue reward:0.85\n", + "step: 1764, Red action: do-nothing, Blue reward:0.85\n", + "step: 1765, Red action: do-nothing, Blue reward:0.85\n", + "step: 1766, Red action: do-nothing, Blue reward:0.85\n", + "step: 1767, Red action: do-nothing, Blue reward:0.85\n", + "step: 1768, Red action: do-nothing, Blue reward:0.85\n", + "step: 1769, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1770, Red action: do-nothing, Blue reward:0.85\n", + "step: 1771, Red action: do-nothing, Blue reward:0.85\n", + "step: 1772, Red action: do-nothing, Blue reward:0.85\n", + "step: 1773, Red action: do-nothing, Blue reward:0.85\n", + "step: 1774, Red action: do-nothing, Blue reward:0.85\n", + "step: 1775, Red action: do-nothing, Blue reward:0.85\n", + "step: 1776, Red action: do-nothing, Blue reward:0.85\n", + "step: 1777, Red action: do-nothing, Blue reward:0.85\n", + "step: 1778, Red action: do-nothing, Blue reward:0.85\n", + "step: 1779, Red action: do-nothing, Blue reward:0.85\n", + "step: 1780, Red action: do-nothing, Blue reward:0.85\n", + "step: 1781, Red action: do-nothing, Blue reward:0.85\n", + "step: 1782, Red action: do-nothing, Blue reward:0.85\n", + "step: 1783, Red action: do-nothing, Blue reward:0.85\n", + "step: 1784, Red action: do-nothing, Blue reward:0.85\n", + "step: 1785, Red action: do-nothing, Blue reward:0.85\n", + "step: 1786, Red action: do-nothing, Blue reward:0.85\n", + "step: 1787, Red action: do-nothing, Blue reward:0.85\n", + "step: 1788, Red action: do-nothing, Blue reward:0.85\n", + "step: 1789, Red action: do-nothing, Blue reward:0.85\n", + "step: 1790, Red action: do-nothing, Blue reward:0.85\n", + "step: 1791, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1792, Red action: do-nothing, Blue reward:0.85\n", + "step: 1793, Red action: do-nothing, Blue reward:0.85\n", + "step: 1794, Red action: do-nothing, Blue reward:0.85\n", + "step: 1795, Red action: do-nothing, Blue reward:0.85\n", + "step: 1796, Red action: do-nothing, Blue reward:0.85\n", + "step: 1797, Red action: do-nothing, Blue reward:0.85\n", + "step: 1798, Red action: do-nothing, Blue reward:0.85\n", + "step: 1799, Red action: do-nothing, Blue reward:0.85\n", + "step: 1800, Red action: do-nothing, Blue reward:0.85\n", + "step: 1801, Red action: do-nothing, Blue reward:0.85\n", + "step: 1802, Red action: do-nothing, Blue reward:0.85\n", + "step: 1803, Red action: do-nothing, Blue reward:0.85\n", + "step: 1804, Red action: do-nothing, Blue reward:0.85\n", + "step: 1805, Red action: do-nothing, Blue reward:0.85\n", + "step: 1806, Red action: do-nothing, Blue reward:0.85\n", + "step: 1807, Red action: do-nothing, Blue reward:0.85\n", + "step: 1808, Red action: do-nothing, Blue reward:0.85\n", + "step: 1809, Red action: do-nothing, Blue reward:0.85\n", + "step: 1810, Red action: do-nothing, Blue reward:0.85\n", + "step: 1811, Red action: do-nothing, Blue reward:0.85\n", + "step: 1812, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1813, Red action: do-nothing, Blue reward:0.85\n", + "step: 1814, Red action: do-nothing, Blue reward:0.85\n", + "step: 1815, Red action: do-nothing, Blue reward:0.85\n", + "step: 1816, Red action: do-nothing, Blue reward:0.85\n", + "step: 1817, Red action: do-nothing, Blue reward:0.85\n", + "step: 1818, Red action: do-nothing, Blue reward:0.85\n", + "step: 1819, Red action: do-nothing, Blue reward:0.85\n", + "step: 1820, Red action: do-nothing, Blue reward:0.85\n", + "step: 1821, Red action: do-nothing, Blue reward:0.85\n", + "step: 1822, Red action: do-nothing, Blue reward:0.85\n", + "step: 1823, Red action: do-nothing, Blue reward:0.85\n", + "step: 1824, Red action: do-nothing, Blue reward:0.85\n", + "step: 1825, Red action: do-nothing, Blue reward:0.85\n", + "step: 1826, Red action: do-nothing, Blue reward:0.85\n", + "step: 1827, Red action: do-nothing, Blue reward:0.85\n", + "step: 1828, Red action: do-nothing, Blue reward:0.85\n", + "step: 1829, Red action: do-nothing, Blue reward:0.85\n", + "step: 1830, Red action: do-nothing, Blue reward:0.85\n", + "step: 1831, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1832, Red action: do-nothing, Blue reward:0.85\n", + "step: 1833, Red action: do-nothing, Blue reward:0.85\n", + "step: 1834, Red action: do-nothing, Blue reward:0.85\n", + "step: 1835, Red action: do-nothing, Blue reward:0.85\n", + "step: 1836, Red action: do-nothing, Blue reward:0.85\n", + "step: 1837, Red action: do-nothing, Blue reward:0.85\n", + "step: 1838, Red action: do-nothing, Blue reward:0.85\n", + "step: 1839, Red action: do-nothing, Blue reward:0.85\n", + "step: 1840, Red action: do-nothing, Blue reward:0.85\n", + "step: 1841, Red action: do-nothing, Blue reward:0.85\n", + "step: 1842, Red action: do-nothing, Blue reward:0.85\n", + "step: 1843, Red action: do-nothing, Blue reward:0.85\n", + "step: 1844, Red action: do-nothing, Blue reward:0.85\n", + "step: 1845, Red action: do-nothing, Blue reward:0.85\n", + "step: 1846, Red action: do-nothing, Blue reward:0.85\n", + "step: 1847, Red action: do-nothing, Blue reward:0.85\n", + "step: 1848, Red action: do-nothing, Blue reward:0.85\n", + "step: 1849, Red action: do-nothing, Blue reward:0.85\n", + "step: 1850, Red action: do-nothing, Blue reward:0.85\n", + "step: 1851, Red action: do-nothing, Blue reward:0.85\n", + "step: 1852, Red action: do-nothing, Blue reward:0.85\n", + "step: 1853, Red action: do-nothing, Blue reward:0.85\n", + "step: 1854, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1855, Red action: do-nothing, Blue reward:0.85\n", + "step: 1856, Red action: do-nothing, Blue reward:0.85\n", + "step: 1857, Red action: do-nothing, Blue reward:0.85\n", + "step: 1858, Red action: do-nothing, Blue reward:0.85\n", + "step: 1859, Red action: do-nothing, Blue reward:0.85\n", + "step: 1860, Red action: do-nothing, Blue reward:0.85\n", + "step: 1861, Red action: do-nothing, Blue reward:0.85\n", + "step: 1862, Red action: do-nothing, Blue reward:0.85\n", + "step: 1863, Red action: do-nothing, Blue reward:0.85\n", + "step: 1864, Red action: do-nothing, Blue reward:0.85\n", + "step: 1865, Red action: do-nothing, Blue reward:0.85\n", + "step: 1866, Red action: do-nothing, Blue reward:0.85\n", + "step: 1867, Red action: do-nothing, Blue reward:0.85\n", + "step: 1868, Red action: do-nothing, Blue reward:0.85\n", + "step: 1869, Red action: do-nothing, Blue reward:0.85\n", + "step: 1870, Red action: do-nothing, Blue reward:0.85\n", + "step: 1871, Red action: do-nothing, Blue reward:0.85\n", + "step: 1872, Red action: do-nothing, Blue reward:0.85\n", + "step: 1873, Red action: do-nothing, Blue reward:0.85\n", + "step: 1874, Red action: do-nothing, Blue reward:0.85\n", + "step: 1875, Red action: do-nothing, Blue reward:0.85\n", + "step: 1876, Red action: do-nothing, Blue reward:0.85\n", + "step: 1877, Red action: do-nothing, Blue reward:0.85\n", + "step: 1878, Red action: do-nothing, Blue reward:0.85\n", + "step: 1879, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1880, Red action: do-nothing, Blue reward:0.85\n", + "step: 1881, Red action: do-nothing, Blue reward:0.85\n", + "step: 1882, Red action: do-nothing, Blue reward:0.85\n", + "step: 1883, Red action: do-nothing, Blue reward:0.85\n", + "step: 1884, Red action: do-nothing, Blue reward:0.85\n", + "step: 1885, Red action: do-nothing, Blue reward:0.85\n", + "step: 1886, Red action: do-nothing, Blue reward:0.85\n", + "step: 1887, Red action: do-nothing, Blue reward:0.85\n", + "step: 1888, Red action: do-nothing, Blue reward:0.85\n", + "step: 1889, Red action: do-nothing, Blue reward:0.85\n", + "step: 1890, Red action: do-nothing, Blue reward:0.85\n", + "step: 1891, Red action: do-nothing, Blue reward:0.85\n", + "step: 1892, Red action: do-nothing, Blue reward:0.85\n", + "step: 1893, Red action: do-nothing, Blue reward:0.85\n", + "step: 1894, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1895, Red action: do-nothing, Blue reward:0.85\n", + "step: 1896, Red action: do-nothing, Blue reward:0.85\n", + "step: 1897, Red action: do-nothing, Blue reward:0.85\n", + "step: 1898, Red action: do-nothing, Blue reward:0.85\n", + "step: 1899, Red action: do-nothing, Blue reward:0.85\n", + "step: 1900, Red action: do-nothing, Blue reward:0.85\n", + "step: 1901, Red action: do-nothing, Blue reward:0.85\n", + "step: 1902, Red action: do-nothing, Blue reward:0.85\n", + "step: 1903, Red action: do-nothing, Blue reward:0.85\n", + "step: 1904, Red action: do-nothing, Blue reward:0.85\n", + "step: 1905, Red action: do-nothing, Blue reward:0.85\n", + "step: 1906, Red action: do-nothing, Blue reward:0.85\n", + "step: 1907, Red action: do-nothing, Blue reward:0.85\n", + "step: 1908, Red action: do-nothing, Blue reward:0.85\n", + "step: 1909, Red action: do-nothing, Blue reward:0.85\n", + "step: 1910, Red action: do-nothing, Blue reward:0.85\n", + "step: 1911, Red action: do-nothing, Blue reward:0.85\n", + "step: 1912, Red action: do-nothing, Blue reward:0.85\n", + "step: 1913, Red action: do-nothing, Blue reward:0.85\n", + "step: 1914, Red action: do-nothing, Blue reward:0.85\n", + "step: 1915, Red action: do-nothing, Blue reward:0.85\n", + "step: 1916, Red action: do-nothing, Blue reward:0.85\n", + "step: 1917, Red action: do-nothing, Blue reward:0.85\n", + "step: 1918, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1919, Red action: do-nothing, Blue reward:0.85\n", + "step: 1920, Red action: do-nothing, Blue reward:0.85\n", + "step: 1921, Red action: do-nothing, Blue reward:0.85\n", + "step: 1922, Red action: do-nothing, Blue reward:0.85\n", + "step: 1923, Red action: do-nothing, Blue reward:0.85\n", + "step: 1924, Red action: do-nothing, Blue reward:0.85\n", + "step: 1925, Red action: do-nothing, Blue reward:0.85\n", + "step: 1926, Red action: do-nothing, Blue reward:0.85\n", + "step: 1927, Red action: do-nothing, Blue reward:0.85\n", + "step: 1928, Red action: do-nothing, Blue reward:0.85\n", + "step: 1929, Red action: do-nothing, Blue reward:0.85\n", + "step: 1930, Red action: do-nothing, Blue reward:0.85\n", + "step: 1931, Red action: do-nothing, Blue reward:0.85\n", + "step: 1932, Red action: do-nothing, Blue reward:0.85\n", + "step: 1933, Red action: do-nothing, Blue reward:0.85\n", + "step: 1934, Red action: do-nothing, Blue reward:0.85\n", + "step: 1935, Red action: do-nothing, Blue reward:0.85\n", + "step: 1936, Red action: do-nothing, Blue reward:0.85\n", + "step: 1937, Red action: do-nothing, Blue reward:0.85\n", + "step: 1938, Red action: do-nothing, Blue reward:0.85\n", + "step: 1939, Red action: do-nothing, Blue reward:0.85\n", + "step: 1940, Red action: do-nothing, Blue reward:0.85\n", + "step: 1941, Red action: do-nothing, Blue reward:0.85\n", + "step: 1942, Red action: do-nothing, Blue reward:0.85\n", + "step: 1943, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1944, Red action: do-nothing, Blue reward:0.85\n", + "step: 1945, Red action: do-nothing, Blue reward:0.85\n", + "step: 1946, Red action: do-nothing, Blue reward:0.85\n", + "step: 1947, Red action: do-nothing, Blue reward:0.85\n", + "step: 1948, Red action: do-nothing, Blue reward:0.85\n", + "step: 1949, Red action: do-nothing, Blue reward:0.85\n", + "step: 1950, Red action: do-nothing, Blue reward:0.85\n", + "step: 1951, Red action: do-nothing, Blue reward:0.85\n", + "step: 1952, Red action: do-nothing, Blue reward:0.85\n", + "step: 1953, Red action: do-nothing, Blue reward:0.85\n", + "step: 1954, Red action: do-nothing, Blue reward:0.85\n", + "step: 1955, Red action: do-nothing, Blue reward:0.85\n", + "step: 1956, Red action: do-nothing, Blue reward:0.85\n", + "step: 1957, Red action: do-nothing, Blue reward:0.85\n", + "step: 1958, Red action: do-nothing, Blue reward:0.85\n", + "step: 1959, Red action: do-nothing, Blue reward:0.85\n", + "step: 1960, Red action: do-nothing, Blue reward:0.85\n", + "step: 1961, Red action: do-nothing, Blue reward:0.85\n", + "step: 1962, Red action: do-nothing, Blue reward:0.85\n", + "step: 1963, Red action: do-nothing, Blue reward:0.85\n", + "step: 1964, Red action: do-nothing, Blue reward:0.85\n", + "step: 1965, Red action: do-nothing, Blue reward:0.85\n", + "step: 1966, Red action: do-nothing, Blue reward:0.85\n", + "step: 1967, Red action: do-nothing, Blue reward:0.85\n", + "step: 1968, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1969, Red action: do-nothing, Blue reward:0.85\n", + "step: 1970, Red action: do-nothing, Blue reward:0.85\n", + "step: 1971, Red action: do-nothing, Blue reward:0.85\n", + "step: 1972, Red action: do-nothing, Blue reward:0.85\n", + "step: 1973, Red action: do-nothing, Blue reward:0.85\n", + "step: 1974, Red action: do-nothing, Blue reward:0.85\n", + "step: 1975, Red action: do-nothing, Blue reward:0.85\n", + "step: 1976, Red action: do-nothing, Blue reward:0.85\n", + "step: 1977, Red action: do-nothing, Blue reward:0.85\n", + "step: 1978, Red action: do-nothing, Blue reward:0.85\n", + "step: 1979, Red action: do-nothing, Blue reward:0.85\n", + "step: 1980, Red action: do-nothing, Blue reward:0.85\n", + "step: 1981, Red action: do-nothing, Blue reward:0.85\n", + "step: 1982, Red action: do-nothing, Blue reward:0.85\n", + "step: 1983, Red action: do-nothing, Blue reward:0.85\n", + "step: 1984, Red action: do-nothing, Blue reward:0.85\n", + "step: 1985, Red action: do-nothing, Blue reward:0.85\n", + "step: 1986, Red action: do-nothing, Blue reward:0.85\n", + "step: 1987, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1988, Red action: do-nothing, Blue reward:0.85\n", + "step: 1989, Red action: do-nothing, Blue reward:0.85\n", + "step: 1990, Red action: do-nothing, Blue reward:0.85\n", + "step: 1991, Red action: do-nothing, Blue reward:0.85\n", + "step: 1992, Red action: do-nothing, Blue reward:0.85\n", + "step: 1993, Red action: do-nothing, Blue reward:0.85\n", + "step: 1994, Red action: do-nothing, Blue reward:0.85\n", + "step: 1995, Red action: do-nothing, Blue reward:0.85\n", + "step: 1996, Red action: do-nothing, Blue reward:0.85\n", + "step: 1997, Red action: do-nothing, Blue reward:0.85\n", + "step: 1998, Red action: do-nothing, Blue reward:0.85\n", + "step: 1999, Red action: do-nothing, Blue reward:0.85\n", + "step: 2000, Red action: do-nothing, Blue reward:0.85\n", + "step: 2001, Red action: do-nothing, Blue reward:0.85\n", + "step: 2002, Red action: do-nothing, Blue reward:0.85\n", + "step: 2003, Red action: do-nothing, Blue reward:0.85\n", + "step: 2004, Red action: do-nothing, Blue reward:0.85\n", + "step: 2005, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2006, Red action: do-nothing, Blue reward:0.85\n", + "step: 2007, Red action: do-nothing, Blue reward:0.85\n", + "step: 2008, Red action: do-nothing, Blue reward:0.85\n", + "step: 2009, Red action: do-nothing, Blue reward:0.85\n", + "step: 2010, Red action: do-nothing, Blue reward:0.85\n", + "step: 2011, Red action: do-nothing, Blue reward:0.85\n", + "step: 2012, Red action: do-nothing, Blue reward:0.85\n", + "step: 2013, Red action: do-nothing, Blue reward:0.85\n", + "step: 2014, Red action: do-nothing, Blue reward:0.85\n", + "step: 2015, Red action: do-nothing, Blue reward:0.85\n", + "step: 2016, Red action: do-nothing, Blue reward:0.85\n", + "step: 2017, Red action: do-nothing, Blue reward:0.85\n", + "step: 2018, Red action: do-nothing, Blue reward:0.85\n", + "step: 2019, Red action: do-nothing, Blue reward:0.85\n", + "step: 2020, Red action: do-nothing, Blue reward:0.85\n", + "step: 2021, Red action: do-nothing, Blue reward:0.85\n", + "step: 2022, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2023, Red action: do-nothing, Blue reward:0.85\n", + "step: 2024, Red action: do-nothing, Blue reward:0.85\n", + "step: 2025, Red action: do-nothing, Blue reward:0.85\n", + "step: 2026, Red action: do-nothing, Blue reward:0.85\n", + "step: 2027, Red action: do-nothing, Blue reward:0.85\n", + "step: 2028, Red action: do-nothing, Blue reward:0.85\n", + "step: 2029, Red action: do-nothing, Blue reward:0.85\n", + "step: 2030, Red action: do-nothing, Blue reward:0.85\n", + "step: 2031, Red action: do-nothing, Blue reward:0.85\n", + "step: 2032, Red action: do-nothing, Blue reward:0.85\n", + "step: 2033, Red action: do-nothing, Blue reward:0.85\n", + "step: 2034, Red action: do-nothing, Blue reward:0.85\n", + "step: 2035, Red action: do-nothing, Blue reward:0.85\n", + "step: 2036, Red action: do-nothing, Blue reward:0.85\n", + "step: 2037, Red action: do-nothing, Blue reward:0.85\n", + "step: 2038, Red action: do-nothing, Blue reward:0.85\n", + "step: 2039, Red action: do-nothing, Blue reward:0.85\n", + "step: 2040, Red action: do-nothing, Blue reward:0.85\n", + "step: 2041, Red action: do-nothing, Blue reward:0.85\n", + "step: 2042, Red action: do-nothing, Blue reward:0.85\n", + "step: 2043, Red action: do-nothing, Blue reward:0.85\n", + "step: 2044, Red action: do-nothing, Blue reward:0.85\n", + "step: 2045, Red action: do-nothing, Blue reward:0.85\n", + "step: 2046, Red action: do-nothing, Blue reward:0.85\n", + "step: 2047, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2048, Red action: do-nothing, Blue reward:0.85\n", + "step: 2049, Red action: do-nothing, Blue reward:0.85\n", + "step: 2050, Red action: do-nothing, Blue reward:0.85\n", + "step: 2051, Red action: do-nothing, Blue reward:0.85\n", + "step: 2052, Red action: do-nothing, Blue reward:0.85\n", + "step: 2053, Red action: do-nothing, Blue reward:0.85\n", + "step: 2054, Red action: do-nothing, Blue reward:0.85\n", + "step: 2055, Red action: do-nothing, Blue reward:0.85\n", + "step: 2056, Red action: do-nothing, Blue reward:0.85\n", + "step: 2057, Red action: do-nothing, Blue reward:0.85\n", + "step: 2058, Red action: do-nothing, Blue reward:0.85\n", + "step: 2059, Red action: do-nothing, Blue reward:0.85\n", + "step: 2060, Red action: do-nothing, Blue reward:0.85\n", + "step: 2061, Red action: do-nothing, Blue reward:0.85\n", + "step: 2062, Red action: do-nothing, Blue reward:0.85\n", + "step: 2063, Red action: do-nothing, Blue reward:0.85\n", + "step: 2064, Red action: do-nothing, Blue reward:0.85\n", + "step: 2065, Red action: do-nothing, Blue reward:0.85\n", + "step: 2066, Red action: do-nothing, Blue reward:0.85\n", + "step: 2067, Red action: do-nothing, Blue reward:0.85\n", + "step: 2068, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2069, Red action: do-nothing, Blue reward:0.85\n", + "step: 2070, Red action: do-nothing, Blue reward:0.85\n", + "step: 2071, Red action: do-nothing, Blue reward:0.85\n", + "step: 2072, Red action: do-nothing, Blue reward:0.85\n", + "step: 2073, Red action: do-nothing, Blue reward:0.85\n", + "step: 2074, Red action: do-nothing, Blue reward:0.85\n", + "step: 2075, Red action: do-nothing, Blue reward:0.85\n", + "step: 2076, Red action: do-nothing, Blue reward:0.85\n", + "step: 2077, Red action: do-nothing, Blue reward:0.85\n", + "step: 2078, Red action: do-nothing, Blue reward:0.85\n", + "step: 2079, Red action: do-nothing, Blue reward:0.85\n", + "step: 2080, Red action: do-nothing, Blue reward:0.85\n", + "step: 2081, Red action: do-nothing, Blue reward:0.85\n", + "step: 2082, Red action: do-nothing, Blue reward:0.85\n", + "step: 2083, Red action: do-nothing, Blue reward:0.85\n", + "step: 2084, Red action: do-nothing, Blue reward:0.85\n", + "step: 2085, Red action: do-nothing, Blue reward:0.85\n", + "step: 2086, Red action: do-nothing, Blue reward:0.85\n", + "step: 2087, Red action: do-nothing, Blue reward:0.85\n", + "step: 2088, Red action: do-nothing, Blue reward:0.85\n", + "step: 2089, Red action: do-nothing, Blue reward:0.85\n", + "step: 2090, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2091, Red action: do-nothing, Blue reward:0.85\n", + "step: 2092, Red action: do-nothing, Blue reward:0.85\n", + "step: 2093, Red action: do-nothing, Blue reward:0.85\n", + "step: 2094, Red action: do-nothing, Blue reward:0.85\n", + "step: 2095, Red action: do-nothing, Blue reward:0.85\n", + "step: 2096, Red action: do-nothing, Blue reward:0.85\n", + "step: 2097, Red action: do-nothing, Blue reward:0.85\n", + "step: 2098, Red action: do-nothing, Blue reward:0.85\n", + "step: 2099, Red action: do-nothing, Blue reward:0.85\n", + "step: 2100, Red action: do-nothing, Blue reward:0.85\n", + "step: 2101, Red action: do-nothing, Blue reward:0.85\n", + "step: 2102, Red action: do-nothing, Blue reward:0.85\n", + "step: 2103, Red action: do-nothing, Blue reward:0.85\n", + "step: 2104, Red action: do-nothing, Blue reward:0.85\n", + "step: 2105, Red action: do-nothing, Blue reward:0.85\n", + "step: 2106, Red action: do-nothing, Blue reward:0.85\n", + "step: 2107, Red action: do-nothing, Blue reward:0.85\n", + "step: 2108, Red action: do-nothing, Blue reward:0.85\n", + "step: 2109, Red action: do-nothing, Blue reward:0.85\n", + "step: 2110, Red action: do-nothing, Blue reward:0.85\n", + "step: 2111, Red action: do-nothing, Blue reward:0.85\n", + "step: 2112, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2113, Red action: do-nothing, Blue reward:0.85\n", + "step: 2114, Red action: do-nothing, Blue reward:0.85\n", + "step: 2115, Red action: do-nothing, Blue reward:0.85\n", + "step: 2116, Red action: do-nothing, Blue reward:0.85\n", + "step: 2117, Red action: do-nothing, Blue reward:0.85\n", + "step: 2118, Red action: do-nothing, Blue reward:0.85\n", + "step: 2119, Red action: do-nothing, Blue reward:0.85\n", + "step: 2120, Red action: do-nothing, Blue reward:0.85\n", + "step: 2121, Red action: do-nothing, Blue reward:0.85\n", + "step: 2122, Red action: do-nothing, Blue reward:0.85\n", + "step: 2123, Red action: do-nothing, Blue reward:0.85\n", + "step: 2124, Red action: do-nothing, Blue reward:0.85\n", + "step: 2125, Red action: do-nothing, Blue reward:0.85\n", + "step: 2126, Red action: do-nothing, Blue reward:0.85\n", + "step: 2127, Red action: do-nothing, Blue reward:0.85\n", + "step: 2128, Red action: do-nothing, Blue reward:0.85\n", + "step: 2129, Red action: do-nothing, Blue reward:0.85\n", + "step: 2130, Red action: do-nothing, Blue reward:0.85\n", + "step: 2131, Red action: do-nothing, Blue reward:0.85\n", + "step: 2132, Red action: do-nothing, Blue reward:0.85\n", + "step: 2133, Red action: do-nothing, Blue reward:0.85\n", + "step: 2134, Red action: do-nothing, Blue reward:0.85\n", + "step: 2135, Red action: do-nothing, Blue reward:0.85\n", + "step: 2136, Red action: do-nothing, Blue reward:0.85\n", + "step: 2137, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2138, Red action: do-nothing, Blue reward:0.85\n", + "step: 2139, Red action: do-nothing, Blue reward:0.85\n", + "step: 2140, Red action: do-nothing, Blue reward:0.85\n", + "step: 2141, Red action: do-nothing, Blue reward:0.85\n", + "step: 2142, Red action: do-nothing, Blue reward:0.85\n", + "step: 2143, Red action: do-nothing, Blue reward:0.85\n", + "step: 2144, Red action: do-nothing, Blue reward:0.85\n", + "step: 2145, Red action: do-nothing, Blue reward:0.85\n", + "step: 2146, Red action: do-nothing, Blue reward:0.85\n", + "step: 2147, Red action: do-nothing, Blue reward:0.85\n", + "step: 2148, Red action: do-nothing, Blue reward:0.85\n", + "step: 2149, Red action: do-nothing, Blue reward:0.85\n", + "step: 2150, Red action: do-nothing, Blue reward:0.85\n", + "step: 2151, Red action: do-nothing, Blue reward:0.85\n", + "step: 2152, Red action: do-nothing, Blue reward:0.85\n", + "step: 2153, Red action: do-nothing, Blue reward:0.85\n", + "step: 2154, Red action: do-nothing, Blue reward:0.85\n", + "step: 2155, Red action: do-nothing, Blue reward:0.85\n", + "step: 2156, Red action: do-nothing, Blue reward:0.85\n", + "step: 2157, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2158, Red action: do-nothing, Blue reward:0.85\n", + "step: 2159, Red action: do-nothing, Blue reward:0.85\n", + "step: 2160, Red action: do-nothing, Blue reward:0.85\n", + "step: 2161, Red action: do-nothing, Blue reward:0.85\n", + "step: 2162, Red action: do-nothing, Blue reward:0.85\n", + "step: 2163, Red action: do-nothing, Blue reward:0.85\n", + "step: 2164, Red action: do-nothing, Blue reward:0.85\n", + "step: 2165, Red action: do-nothing, Blue reward:0.85\n", + "step: 2166, Red action: do-nothing, Blue reward:0.85\n", + "step: 2167, Red action: do-nothing, Blue reward:0.85\n", + "step: 2168, Red action: do-nothing, Blue reward:0.85\n", + "step: 2169, Red action: do-nothing, Blue reward:0.85\n", + "step: 2170, Red action: do-nothing, Blue reward:0.85\n", + "step: 2171, Red action: do-nothing, Blue reward:0.85\n", + "step: 2172, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2173, Red action: do-nothing, Blue reward:0.85\n", + "step: 2174, Red action: do-nothing, Blue reward:0.85\n", + "step: 2175, Red action: do-nothing, Blue reward:0.85\n", + "step: 2176, Red action: do-nothing, Blue reward:0.85\n", + "step: 2177, Red action: do-nothing, Blue reward:0.85\n", + "step: 2178, Red action: do-nothing, Blue reward:0.85\n", + "step: 2179, Red action: do-nothing, Blue reward:0.85\n", + "step: 2180, Red action: do-nothing, Blue reward:0.85\n", + "step: 2181, Red action: do-nothing, Blue reward:0.85\n", + "step: 2182, Red action: do-nothing, Blue reward:0.85\n", + "step: 2183, Red action: do-nothing, Blue reward:0.85\n", + "step: 2184, Red action: do-nothing, Blue reward:0.85\n", + "step: 2185, Red action: do-nothing, Blue reward:0.85\n", + "step: 2186, Red action: do-nothing, Blue reward:0.85\n", + "step: 2187, Red action: do-nothing, Blue reward:0.85\n", + "step: 2188, Red action: do-nothing, Blue reward:0.85\n", + "step: 2189, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2190, Red action: do-nothing, Blue reward:0.85\n", + "step: 2191, Red action: do-nothing, Blue reward:0.85\n", + "step: 2192, Red action: do-nothing, Blue reward:0.85\n", + "step: 2193, Red action: do-nothing, Blue reward:0.85\n", + "step: 2194, Red action: do-nothing, Blue reward:0.85\n", + "step: 2195, Red action: do-nothing, Blue reward:0.85\n", + "step: 2196, Red action: do-nothing, Blue reward:0.85\n", + "step: 2197, Red action: do-nothing, Blue reward:0.85\n", + "step: 2198, Red action: do-nothing, Blue reward:0.85\n", + "step: 2199, Red action: do-nothing, Blue reward:0.85\n", + "step: 2200, Red action: do-nothing, Blue reward:0.85\n", + "step: 2201, Red action: do-nothing, Blue reward:0.85\n", + "step: 2202, Red action: do-nothing, Blue reward:0.85\n", + "step: 2203, Red action: do-nothing, Blue reward:0.85\n", + "step: 2204, Red action: do-nothing, Blue reward:0.85\n", + "step: 2205, Red action: do-nothing, Blue reward:0.85\n", + "step: 2206, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2207, Red action: do-nothing, Blue reward:0.85\n", + "step: 2208, Red action: do-nothing, Blue reward:0.85\n", + "step: 2209, Red action: do-nothing, Blue reward:0.85\n", + "step: 2210, Red action: do-nothing, Blue reward:0.85\n", + "step: 2211, Red action: do-nothing, Blue reward:0.85\n", + "step: 2212, Red action: do-nothing, Blue reward:0.85\n", + "step: 2213, Red action: do-nothing, Blue reward:0.85\n", + "step: 2214, Red action: do-nothing, Blue reward:0.85\n", + "step: 2215, Red action: do-nothing, Blue reward:0.85\n", + "step: 2216, Red action: do-nothing, Blue reward:0.85\n", + "step: 2217, Red action: do-nothing, Blue reward:0.85\n", + "step: 2218, Red action: do-nothing, Blue reward:0.85\n", + "step: 2219, Red action: do-nothing, Blue reward:0.85\n", + "step: 2220, Red action: do-nothing, Blue reward:0.85\n", + "step: 2221, Red action: do-nothing, Blue reward:0.85\n", + "step: 2222, Red action: do-nothing, Blue reward:0.85\n", + "step: 2223, Red action: do-nothing, Blue reward:0.85\n", + "step: 2224, Red action: do-nothing, Blue reward:0.85\n", + "step: 2225, Red action: do-nothing, Blue reward:0.85\n", + "step: 2226, Red action: do-nothing, Blue reward:0.85\n", + "step: 2227, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2228, Red action: do-nothing, Blue reward:0.85\n", + "step: 2229, Red action: do-nothing, Blue reward:0.85\n", + "step: 2230, Red action: do-nothing, Blue reward:0.85\n", + "step: 2231, Red action: do-nothing, Blue reward:0.85\n", + "step: 2232, Red action: do-nothing, Blue reward:0.85\n", + "step: 2233, Red action: do-nothing, Blue reward:0.85\n", + "step: 2234, Red action: do-nothing, Blue reward:0.85\n", + "step: 2235, Red action: do-nothing, Blue reward:0.85\n", + "step: 2236, Red action: do-nothing, Blue reward:0.85\n", + "step: 2237, Red action: do-nothing, Blue reward:0.85\n", + "step: 2238, Red action: do-nothing, Blue reward:0.85\n", + "step: 2239, Red action: do-nothing, Blue reward:0.85\n", + "step: 2240, Red action: do-nothing, Blue reward:0.85\n", + "step: 2241, Red action: do-nothing, Blue reward:0.85\n", + "step: 2242, Red action: do-nothing, Blue reward:0.85\n", + "step: 2243, Red action: do-nothing, Blue reward:0.85\n", + "step: 2244, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2245, Red action: do-nothing, Blue reward:0.85\n", + "step: 2246, Red action: do-nothing, Blue reward:0.85\n", + "step: 2247, Red action: do-nothing, Blue reward:0.85\n", + "step: 2248, Red action: do-nothing, Blue reward:0.85\n", + "step: 2249, Red action: do-nothing, Blue reward:0.85\n", + "step: 2250, Red action: do-nothing, Blue reward:0.85\n", + "step: 2251, Red action: do-nothing, Blue reward:0.85\n", + "step: 2252, Red action: do-nothing, Blue reward:0.85\n", + "step: 2253, Red action: do-nothing, Blue reward:0.85\n", + "step: 2254, Red action: do-nothing, Blue reward:0.85\n", + "step: 2255, Red action: do-nothing, Blue reward:0.85\n", + "step: 2256, Red action: do-nothing, Blue reward:0.85\n", + "step: 2257, Red action: do-nothing, Blue reward:0.85\n", + "step: 2258, Red action: do-nothing, Blue reward:0.85\n", + "step: 2259, Red action: do-nothing, Blue reward:0.85\n", + "step: 2260, Red action: do-nothing, Blue reward:0.85\n", + "step: 2261, Red action: do-nothing, Blue reward:0.85\n", + "step: 2262, Red action: do-nothing, Blue reward:0.85\n", + "step: 2263, Red action: do-nothing, Blue reward:0.85\n", + "step: 2264, Red action: do-nothing, Blue reward:0.85\n", + "step: 2265, Red action: do-nothing, Blue reward:0.85\n", + "step: 2266, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2267, Red action: do-nothing, Blue reward:0.85\n", + "step: 2268, Red action: do-nothing, Blue reward:0.85\n", + "step: 2269, Red action: do-nothing, Blue reward:0.85\n", + "step: 2270, Red action: do-nothing, Blue reward:0.85\n", + "step: 2271, Red action: do-nothing, Blue reward:0.85\n", + "step: 2272, Red action: do-nothing, Blue reward:0.85\n", + "step: 2273, Red action: do-nothing, Blue reward:0.85\n", + "step: 2274, Red action: do-nothing, Blue reward:0.85\n", + "step: 2275, Red action: do-nothing, Blue reward:0.85\n", + "step: 2276, Red action: do-nothing, Blue reward:0.85\n", + "step: 2277, Red action: do-nothing, Blue reward:0.85\n", + "step: 2278, Red action: do-nothing, Blue reward:0.85\n", + "step: 2279, Red action: do-nothing, Blue reward:0.85\n", + "step: 2280, Red action: do-nothing, Blue reward:0.85\n", + "step: 2281, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2282, Red action: do-nothing, Blue reward:0.85\n", + "step: 2283, Red action: do-nothing, Blue reward:0.85\n", + "step: 2284, Red action: do-nothing, Blue reward:0.85\n", + "step: 2285, Red action: do-nothing, Blue reward:0.85\n", + "step: 2286, Red action: do-nothing, Blue reward:0.85\n", + "step: 2287, Red action: do-nothing, Blue reward:0.85\n", + "step: 2288, Red action: do-nothing, Blue reward:0.85\n", + "step: 2289, Red action: do-nothing, Blue reward:0.85\n", + "step: 2290, Red action: do-nothing, Blue reward:0.85\n", + "step: 2291, Red action: do-nothing, Blue reward:0.85\n", + "step: 2292, Red action: do-nothing, Blue reward:0.85\n", + "step: 2293, Red action: do-nothing, Blue reward:0.85\n", + "step: 2294, Red action: do-nothing, Blue reward:0.85\n", + "step: 2295, Red action: do-nothing, Blue reward:0.85\n", + "step: 2296, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2297, Red action: do-nothing, Blue reward:0.85\n", + "step: 2298, Red action: do-nothing, Blue reward:0.85\n", + "step: 2299, Red action: do-nothing, Blue reward:0.85\n", + "step: 2300, Red action: do-nothing, Blue reward:0.85\n", + "step: 2301, Red action: do-nothing, Blue reward:0.85\n", + "step: 2302, Red action: do-nothing, Blue reward:0.85\n", + "step: 2303, Red action: do-nothing, Blue reward:0.85\n", + "step: 2304, Red action: do-nothing, Blue reward:0.85\n", + "step: 2305, Red action: do-nothing, Blue reward:0.85\n", + "step: 2306, Red action: do-nothing, Blue reward:0.85\n", + "step: 2307, Red action: do-nothing, Blue reward:0.85\n", + "step: 2308, Red action: do-nothing, Blue reward:0.85\n", + "step: 2309, Red action: do-nothing, Blue reward:0.85\n", + "step: 2310, Red action: do-nothing, Blue reward:0.85\n", + "step: 2311, Red action: do-nothing, Blue reward:0.85\n", + "step: 2312, Red action: do-nothing, Blue reward:0.85\n", + "step: 2313, Red action: do-nothing, Blue reward:0.85\n", + "step: 2314, Red action: do-nothing, Blue reward:0.85\n", + "step: 2315, Red action: do-nothing, Blue reward:0.85\n", + "step: 2316, Red action: do-nothing, Blue reward:0.85\n", + "step: 2317, Red action: do-nothing, Blue reward:0.85\n", + "step: 2318, Red action: do-nothing, Blue reward:0.85\n", + "step: 2319, Red action: do-nothing, Blue reward:0.85\n", + "step: 2320, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2321, Red action: do-nothing, Blue reward:0.85\n", + "step: 2322, Red action: do-nothing, Blue reward:0.85\n", + "step: 2323, Red action: do-nothing, Blue reward:0.85\n", + "step: 2324, Red action: do-nothing, Blue reward:0.85\n", + "step: 2325, Red action: do-nothing, Blue reward:0.85\n", + "step: 2326, Red action: do-nothing, Blue reward:0.85\n", + "step: 2327, Red action: do-nothing, Blue reward:0.85\n", + "step: 2328, Red action: do-nothing, Blue reward:0.85\n", + "step: 2329, Red action: do-nothing, Blue reward:0.85\n", + "step: 2330, Red action: do-nothing, Blue reward:0.85\n", + "step: 2331, Red action: do-nothing, Blue reward:0.85\n", + "step: 2332, Red action: do-nothing, Blue reward:0.85\n", + "step: 2333, Red action: do-nothing, Blue reward:0.85\n", + "step: 2334, Red action: do-nothing, Blue reward:0.85\n", + "step: 2335, Red action: do-nothing, Blue reward:0.85\n", + "step: 2336, Red action: do-nothing, Blue reward:0.85\n", + "step: 2337, Red action: do-nothing, Blue reward:0.85\n", + "step: 2338, Red action: do-nothing, Blue reward:0.85\n", + "step: 2339, Red action: do-nothing, Blue reward:0.85\n", + "step: 2340, Red action: do-nothing, Blue reward:0.85\n", + "step: 2341, Red action: do-nothing, Blue reward:0.85\n", + "step: 2342, Red action: do-nothing, Blue reward:0.85\n", + "step: 2343, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2344, Red action: do-nothing, Blue reward:0.85\n", + "step: 2345, Red action: do-nothing, Blue reward:0.85\n", + "step: 2346, Red action: do-nothing, Blue reward:0.85\n", + "step: 2347, Red action: do-nothing, Blue reward:0.85\n", + "step: 2348, Red action: do-nothing, Blue reward:0.85\n", + "step: 2349, Red action: do-nothing, Blue reward:0.85\n", + "step: 2350, Red action: do-nothing, Blue reward:0.85\n", + "step: 2351, Red action: do-nothing, Blue reward:0.85\n", + "step: 2352, Red action: do-nothing, Blue reward:0.85\n", + "step: 2353, Red action: do-nothing, Blue reward:0.85\n", + "step: 2354, Red action: do-nothing, Blue reward:0.85\n", + "step: 2355, Red action: do-nothing, Blue reward:0.85\n", + "step: 2356, Red action: do-nothing, Blue reward:0.85\n", + "step: 2357, Red action: do-nothing, Blue reward:0.85\n", + "step: 2358, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2359, Red action: do-nothing, Blue reward:0.85\n", + "step: 2360, Red action: do-nothing, Blue reward:0.85\n", + "step: 2361, Red action: do-nothing, Blue reward:0.85\n", + "step: 2362, Red action: do-nothing, Blue reward:0.85\n", + "step: 2363, Red action: do-nothing, Blue reward:0.85\n", + "step: 2364, Red action: do-nothing, Blue reward:0.85\n", + "step: 2365, Red action: do-nothing, Blue reward:0.85\n", + "step: 2366, Red action: do-nothing, Blue reward:0.85\n", + "step: 2367, Red action: do-nothing, Blue reward:0.85\n", + "step: 2368, Red action: do-nothing, Blue reward:0.85\n", + "step: 2369, Red action: do-nothing, Blue reward:0.85\n", + "step: 2370, Red action: do-nothing, Blue reward:0.85\n", + "step: 2371, Red action: do-nothing, Blue reward:0.85\n", + "step: 2372, Red action: do-nothing, Blue reward:0.85\n", + "step: 2373, Red action: do-nothing, Blue reward:0.85\n", + "step: 2374, Red action: do-nothing, Blue reward:0.85\n", + "step: 2375, Red action: do-nothing, Blue reward:0.85\n", + "step: 2376, Red action: do-nothing, Blue reward:0.85\n", + "step: 2377, Red action: do-nothing, Blue reward:0.85\n", + "step: 2378, Red action: do-nothing, Blue reward:0.85\n", + "step: 2379, Red action: do-nothing, Blue reward:0.85\n", + "step: 2380, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2381, Red action: do-nothing, Blue reward:0.85\n", + "step: 2382, Red action: do-nothing, Blue reward:0.85\n", + "step: 2383, Red action: do-nothing, Blue reward:0.85\n", + "step: 2384, Red action: do-nothing, Blue reward:0.85\n", + "step: 2385, Red action: do-nothing, Blue reward:0.85\n", + "step: 2386, Red action: do-nothing, Blue reward:0.85\n", + "step: 2387, Red action: do-nothing, Blue reward:0.85\n", + "step: 2388, Red action: do-nothing, Blue reward:0.85\n", + "step: 2389, Red action: do-nothing, Blue reward:0.85\n", + "step: 2390, Red action: do-nothing, Blue reward:0.85\n", + "step: 2391, Red action: do-nothing, Blue reward:0.85\n", + "step: 2392, Red action: do-nothing, Blue reward:0.85\n", + "step: 2393, Red action: do-nothing, Blue reward:0.85\n", + "step: 2394, Red action: do-nothing, Blue reward:0.85\n", + "step: 2395, Red action: do-nothing, Blue reward:0.85\n", + "step: 2396, Red action: do-nothing, Blue reward:0.85\n", + "step: 2397, Red action: do-nothing, Blue reward:0.85\n", + "step: 2398, Red action: do-nothing, Blue reward:0.85\n", + "step: 2399, Red action: do-nothing, Blue reward:0.85\n", + "step: 2400, Red action: do-nothing, Blue reward:0.85\n", + "step: 2401, Red action: do-nothing, Blue reward:0.85\n", + "step: 2402, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2403, Red action: do-nothing, Blue reward:0.85\n", + "step: 2404, Red action: do-nothing, Blue reward:0.85\n", + "step: 2405, Red action: do-nothing, Blue reward:0.85\n", + "step: 2406, Red action: do-nothing, Blue reward:0.85\n", + "step: 2407, Red action: do-nothing, Blue reward:0.85\n", + "step: 2408, Red action: do-nothing, Blue reward:0.85\n", + "step: 2409, Red action: do-nothing, Blue reward:0.85\n", + "step: 2410, Red action: do-nothing, Blue reward:0.85\n", + "step: 2411, Red action: do-nothing, Blue reward:0.85\n", + "step: 2412, Red action: do-nothing, Blue reward:0.85\n", + "step: 2413, Red action: do-nothing, Blue reward:0.85\n", + "step: 2414, Red action: do-nothing, Blue reward:0.85\n", + "step: 2415, Red action: do-nothing, Blue reward:0.85\n", + "step: 2416, Red action: do-nothing, Blue reward:0.85\n", + "step: 2417, Red action: do-nothing, Blue reward:0.85\n", + "step: 2418, Red action: do-nothing, Blue reward:0.85\n", + "step: 2419, Red action: do-nothing, Blue reward:0.85\n", + "step: 2420, Red action: do-nothing, Blue reward:0.85\n", + "step: 2421, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2422, Red action: do-nothing, Blue reward:0.85\n", + "step: 2423, Red action: do-nothing, Blue reward:0.85\n", + "step: 2424, Red action: do-nothing, Blue reward:0.85\n", + "step: 2425, Red action: do-nothing, Blue reward:0.85\n", + "step: 2426, Red action: do-nothing, Blue reward:0.85\n", + "step: 2427, Red action: do-nothing, Blue reward:0.85\n", + "step: 2428, Red action: do-nothing, Blue reward:0.85\n", + "step: 2429, Red action: do-nothing, Blue reward:0.85\n", + "step: 2430, Red action: do-nothing, Blue reward:0.85\n", + "step: 2431, Red action: do-nothing, Blue reward:0.85\n", + "step: 2432, Red action: do-nothing, Blue reward:0.85\n", + "step: 2433, Red action: do-nothing, Blue reward:0.85\n", + "step: 2434, Red action: do-nothing, Blue reward:0.85\n", + "step: 2435, Red action: do-nothing, Blue reward:0.85\n", + "step: 2436, Red action: do-nothing, Blue reward:0.85\n", + "step: 2437, Red action: do-nothing, Blue reward:0.85\n", + "step: 2438, Red action: do-nothing, Blue reward:0.85\n", + "step: 2439, Red action: do-nothing, Blue reward:0.85\n", + "step: 2440, Red action: do-nothing, Blue reward:0.85\n", + "step: 2441, Red action: do-nothing, Blue reward:0.85\n", + "step: 2442, Red action: do-nothing, Blue reward:0.85\n", + "step: 2443, Red action: do-nothing, Blue reward:0.85\n", + "step: 2444, Red action: do-nothing, Blue reward:0.85\n", + "step: 2445, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2446, Red action: do-nothing, Blue reward:0.85\n", + "step: 2447, Red action: do-nothing, Blue reward:0.85\n", + "step: 2448, Red action: do-nothing, Blue reward:0.85\n", + "step: 2449, Red action: do-nothing, Blue reward:0.85\n", + "step: 2450, Red action: do-nothing, Blue reward:0.85\n", + "step: 2451, Red action: do-nothing, Blue reward:0.85\n", + "step: 2452, Red action: do-nothing, Blue reward:0.85\n", + "step: 2453, Red action: do-nothing, Blue reward:0.85\n", + "step: 2454, Red action: do-nothing, Blue reward:0.85\n", + "step: 2455, Red action: do-nothing, Blue reward:0.85\n", + "step: 2456, Red action: do-nothing, Blue reward:0.85\n", + "step: 2457, Red action: do-nothing, Blue reward:0.85\n", + "step: 2458, Red action: do-nothing, Blue reward:0.85\n", + "step: 2459, Red action: do-nothing, Blue reward:0.85\n", + "step: 2460, Red action: do-nothing, Blue reward:0.85\n", + "step: 2461, Red action: do-nothing, Blue reward:0.85\n", + "step: 2462, Red action: do-nothing, Blue reward:0.85\n", + "step: 2463, Red action: do-nothing, Blue reward:0.85\n", + "step: 2464, Red action: do-nothing, Blue reward:0.85\n", + "step: 2465, Red action: do-nothing, Blue reward:0.85\n", + "step: 2466, Red action: do-nothing, Blue reward:0.85\n", + "step: 2467, Red action: do-nothing, Blue reward:0.85\n", + "step: 2468, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2469, Red action: do-nothing, Blue reward:0.85\n", + "step: 2470, Red action: do-nothing, Blue reward:0.85\n", + "step: 2471, Red action: do-nothing, Blue reward:0.85\n", + "step: 2472, Red action: do-nothing, Blue reward:0.85\n", + "step: 2473, Red action: do-nothing, Blue reward:0.85\n", + "step: 2474, Red action: do-nothing, Blue reward:0.85\n", + "step: 2475, Red action: do-nothing, Blue reward:0.85\n", + "step: 2476, Red action: do-nothing, Blue reward:0.85\n", + "step: 2477, Red action: do-nothing, Blue reward:0.85\n", + "step: 2478, Red action: do-nothing, Blue reward:0.85\n", + "step: 2479, Red action: do-nothing, Blue reward:0.85\n", + "step: 2480, Red action: do-nothing, Blue reward:0.85\n", + "step: 2481, Red action: do-nothing, Blue reward:0.85\n", + "step: 2482, Red action: do-nothing, Blue reward:0.85\n", + "step: 2483, Red action: do-nothing, Blue reward:0.85\n", + "step: 2484, Red action: do-nothing, Blue reward:0.85\n", + "step: 2485, Red action: do-nothing, Blue reward:0.85\n", + "step: 2486, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2487, Red action: do-nothing, Blue reward:0.85\n", + "step: 2488, Red action: do-nothing, Blue reward:0.85\n", + "step: 2489, Red action: do-nothing, Blue reward:0.85\n", + "step: 2490, Red action: do-nothing, Blue reward:0.85\n", + "step: 2491, Red action: do-nothing, Blue reward:0.85\n", + "step: 2492, Red action: do-nothing, Blue reward:0.85\n", + "step: 2493, Red action: do-nothing, Blue reward:0.85\n", + "step: 2494, Red action: do-nothing, Blue reward:0.85\n", + "step: 2495, Red action: do-nothing, Blue reward:0.85\n", + "step: 2496, Red action: do-nothing, Blue reward:0.85\n", + "step: 2497, Red action: do-nothing, Blue reward:0.85\n", + "step: 2498, Red action: do-nothing, Blue reward:0.85\n", + "step: 2499, Red action: do-nothing, Blue reward:0.85\n", + "step: 2500, Red action: do-nothing, Blue reward:0.85\n", + "step: 2501, Red action: do-nothing, Blue reward:0.85\n", + "step: 2502, Red action: do-nothing, Blue reward:0.85\n", + "step: 2503, Red action: do-nothing, Blue reward:0.85\n", + "step: 2504, Red action: do-nothing, Blue reward:0.85\n", + "step: 2505, Red action: do-nothing, Blue reward:0.85\n", + "step: 2506, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2507, Red action: do-nothing, Blue reward:0.85\n", + "step: 2508, Red action: do-nothing, Blue reward:0.85\n", + "step: 2509, Red action: do-nothing, Blue reward:0.85\n", + "step: 2510, Red action: do-nothing, Blue reward:0.85\n", + "step: 2511, Red action: do-nothing, Blue reward:0.85\n", + "step: 2512, Red action: do-nothing, Blue reward:0.85\n", + "step: 2513, Red action: do-nothing, Blue reward:0.85\n", + "step: 2514, Red action: do-nothing, Blue reward:0.85\n", + "step: 2515, Red action: do-nothing, Blue reward:0.85\n", + "step: 2516, Red action: do-nothing, Blue reward:0.85\n", + "step: 2517, Red action: do-nothing, Blue reward:0.85\n", + "step: 2518, Red action: do-nothing, Blue reward:0.85\n", + "step: 2519, Red action: do-nothing, Blue reward:0.85\n", + "step: 2520, Red action: do-nothing, Blue reward:0.85\n", + "step: 2521, Red action: do-nothing, Blue reward:0.85\n", + "step: 2522, Red action: do-nothing, Blue reward:0.85\n", + "step: 2523, Red action: do-nothing, Blue reward:0.85\n", + "step: 2524, Red action: do-nothing, Blue reward:0.85\n", + "step: 2525, Red action: do-nothing, Blue reward:0.85\n", + "step: 2526, Red action: do-nothing, Blue reward:0.85\n", + "step: 2527, Red action: do-nothing, Blue reward:0.85\n", + "step: 2528, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2529, Red action: do-nothing, Blue reward:0.85\n", + "step: 2530, Red action: do-nothing, Blue reward:0.85\n", + "step: 2531, Red action: do-nothing, Blue reward:0.85\n", + "step: 2532, Red action: do-nothing, Blue reward:0.85\n", + "step: 2533, Red action: do-nothing, Blue reward:0.85\n", + "step: 2534, Red action: do-nothing, Blue reward:0.85\n", + "step: 2535, Red action: do-nothing, Blue reward:0.85\n", + "step: 2536, Red action: do-nothing, Blue reward:0.85\n", + "step: 2537, Red action: do-nothing, Blue reward:0.85\n", + "step: 2538, Red action: do-nothing, Blue reward:0.85\n", + "step: 2539, Red action: do-nothing, Blue reward:0.85\n", + "step: 2540, Red action: do-nothing, Blue reward:0.85\n", + "step: 2541, Red action: do-nothing, Blue reward:0.85\n", + "step: 2542, Red action: do-nothing, Blue reward:0.85\n", + "step: 2543, Red action: do-nothing, Blue reward:0.85\n", + "step: 2544, Red action: do-nothing, Blue reward:0.85\n", + "step: 2545, Red action: do-nothing, Blue reward:0.85\n", + "step: 2546, Red action: do-nothing, Blue reward:0.85\n", + "step: 2547, Red action: do-nothing, Blue reward:0.85\n", + "step: 2548, Red action: do-nothing, Blue reward:0.85\n", + "step: 2549, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2550, Red action: do-nothing, Blue reward:0.85\n", + "step: 2551, Red action: do-nothing, Blue reward:0.85\n", + "step: 2552, Red action: do-nothing, Blue reward:0.85\n", + "step: 2553, Red action: do-nothing, Blue reward:0.85\n", + "step: 2554, Red action: do-nothing, Blue reward:0.85\n", + "step: 2555, Red action: do-nothing, Blue reward:0.85\n", + "step: 2556, Red action: do-nothing, Blue reward:0.85\n", + "step: 2557, Red action: do-nothing, Blue reward:0.85\n", + "step: 2558, Red action: do-nothing, Blue reward:0.85\n", + "step: 2559, Red action: do-nothing, Blue reward:0.85\n", + "step: 2560, Red action: do-nothing, Blue reward:0.85\n", + "step: 2561, Red action: do-nothing, Blue reward:0.85\n", + "step: 2562, Red action: do-nothing, Blue reward:0.85\n", + "step: 2563, Red action: do-nothing, Blue reward:0.85\n", + "step: 2564, Red action: do-nothing, Blue reward:0.85\n", + "step: 2565, Red action: do-nothing, Blue reward:0.85\n", + "step: 2566, Red action: do-nothing, Blue reward:0.85\n", + "step: 2567, Red action: do-nothing, Blue reward:0.85\n", + "step: 2568, Red action: do-nothing, Blue reward:0.85\n", + "step: 2569, Red action: do-nothing, Blue reward:0.85\n", + "step: 2570, Red action: do-nothing, Blue reward:0.85\n", + "step: 2571, Red action: do-nothing, Blue reward:0.85\n", + "step: 2572, Red action: do-nothing, Blue reward:0.85\n", + "step: 2573, Red action: do-nothing, Blue reward:0.85\n", + "step: 2574, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2575, Red action: do-nothing, Blue reward:0.85\n", + "step: 2576, Red action: do-nothing, Blue reward:0.85\n", + "step: 2577, Red action: do-nothing, Blue reward:0.85\n", + "step: 2578, Red action: do-nothing, Blue reward:0.85\n", + "step: 2579, Red action: do-nothing, Blue reward:0.85\n", + "step: 2580, Red action: do-nothing, Blue reward:0.85\n", + "step: 2581, Red action: do-nothing, Blue reward:0.85\n", + "step: 2582, Red action: do-nothing, Blue reward:0.85\n", + "step: 2583, Red action: do-nothing, Blue reward:0.85\n", + "step: 2584, Red action: do-nothing, Blue reward:0.85\n", + "step: 2585, Red action: do-nothing, Blue reward:0.85\n", + "step: 2586, Red action: do-nothing, Blue reward:0.85\n", + "step: 2587, Red action: do-nothing, Blue reward:0.85\n", + "step: 2588, Red action: do-nothing, Blue reward:0.85\n", + "step: 2589, Red action: do-nothing, Blue reward:0.85\n", + "step: 2590, Red action: do-nothing, Blue reward:0.85\n", + "step: 2591, Red action: do-nothing, Blue reward:0.85\n", + "step: 2592, Red action: do-nothing, Blue reward:0.85\n", + "step: 2593, Red action: do-nothing, Blue reward:0.85\n", + "step: 2594, Red action: do-nothing, Blue reward:0.85\n", + "step: 2595, Red action: do-nothing, Blue reward:0.85\n", + "step: 2596, Red action: do-nothing, Blue reward:0.85\n", + "step: 2597, Red action: do-nothing, Blue reward:0.85\n", + "step: 2598, Red action: do-nothing, Blue reward:0.85\n", + "step: 2599, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2600, Red action: do-nothing, Blue reward:0.85\n", + "step: 2601, Red action: do-nothing, Blue reward:0.85\n", + "step: 2602, Red action: do-nothing, Blue reward:0.85\n", + "step: 2603, Red action: do-nothing, Blue reward:0.85\n", + "step: 2604, Red action: do-nothing, Blue reward:0.85\n", + "step: 2605, Red action: do-nothing, Blue reward:0.85\n", + "step: 2606, Red action: do-nothing, Blue reward:0.85\n", + "step: 2607, Red action: do-nothing, Blue reward:0.85\n", + "step: 2608, Red action: do-nothing, Blue reward:0.85\n", + "step: 2609, Red action: do-nothing, Blue reward:0.85\n", + "step: 2610, Red action: do-nothing, Blue reward:0.85\n", + "step: 2611, Red action: do-nothing, Blue reward:0.85\n", + "step: 2612, Red action: do-nothing, Blue reward:0.85\n", + "step: 2613, Red action: do-nothing, Blue reward:0.85\n", + "step: 2614, Red action: do-nothing, Blue reward:0.85\n", + "step: 2615, Red action: do-nothing, Blue reward:0.85\n", + "step: 2616, Red action: do-nothing, Blue reward:0.85\n", + "step: 2617, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2618, Red action: do-nothing, Blue reward:0.85\n", + "step: 2619, Red action: do-nothing, Blue reward:0.85\n", + "step: 2620, Red action: do-nothing, Blue reward:0.85\n", + "step: 2621, Red action: do-nothing, Blue reward:0.85\n", + "step: 2622, Red action: do-nothing, Blue reward:0.85\n", + "step: 2623, Red action: do-nothing, Blue reward:0.85\n", + "step: 2624, Red action: do-nothing, Blue reward:0.85\n", + "step: 2625, Red action: do-nothing, Blue reward:0.85\n", + "step: 2626, Red action: do-nothing, Blue reward:0.85\n", + "step: 2627, Red action: do-nothing, Blue reward:0.85\n", + "step: 2628, Red action: do-nothing, Blue reward:0.85\n", + "step: 2629, Red action: do-nothing, Blue reward:0.85\n", + "step: 2630, Red action: do-nothing, Blue reward:0.85\n", + "step: 2631, Red action: do-nothing, Blue reward:0.85\n", + "step: 2632, Red action: do-nothing, Blue reward:0.85\n", + "step: 2633, Red action: do-nothing, Blue reward:0.85\n", + "step: 2634, Red action: do-nothing, Blue reward:0.85\n", + "step: 2635, Red action: do-nothing, Blue reward:0.85\n", + "step: 2636, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2637, Red action: do-nothing, Blue reward:0.85\n", + "step: 2638, Red action: do-nothing, Blue reward:0.85\n", + "step: 2639, Red action: do-nothing, Blue reward:0.85\n", + "step: 2640, Red action: do-nothing, Blue reward:0.85\n", + "step: 2641, Red action: do-nothing, Blue reward:0.85\n", + "step: 2642, Red action: do-nothing, Blue reward:0.85\n", + "step: 2643, Red action: do-nothing, Blue reward:0.85\n", + "step: 2644, Red action: do-nothing, Blue reward:0.85\n", + "step: 2645, Red action: do-nothing, Blue reward:0.85\n", + "step: 2646, Red action: do-nothing, Blue reward:0.85\n", + "step: 2647, Red action: do-nothing, Blue reward:0.85\n", + "step: 2648, Red action: do-nothing, Blue reward:0.85\n", + "step: 2649, Red action: do-nothing, Blue reward:0.85\n", + "step: 2650, Red action: do-nothing, Blue reward:0.85\n", + "step: 2651, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2652, Red action: do-nothing, Blue reward:0.85\n", + "step: 2653, Red action: do-nothing, Blue reward:0.85\n", + "step: 2654, Red action: do-nothing, Blue reward:0.85\n", + "step: 2655, Red action: do-nothing, Blue reward:0.85\n", + "step: 2656, Red action: do-nothing, Blue reward:0.85\n", + "step: 2657, Red action: do-nothing, Blue reward:0.85\n", + "step: 2658, Red action: do-nothing, Blue reward:0.85\n", + "step: 2659, Red action: do-nothing, Blue reward:0.85\n", + "step: 2660, Red action: do-nothing, Blue reward:0.85\n", + "step: 2661, Red action: do-nothing, Blue reward:0.85\n", + "step: 2662, Red action: do-nothing, Blue reward:0.85\n", + "step: 2663, Red action: do-nothing, Blue reward:0.85\n", + "step: 2664, Red action: do-nothing, Blue reward:0.85\n", + "step: 2665, Red action: do-nothing, Blue reward:0.85\n", + "step: 2666, Red action: do-nothing, Blue reward:0.85\n", + "step: 2667, Red action: do-nothing, Blue reward:0.85\n", + "step: 2668, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2669, Red action: do-nothing, Blue reward:0.85\n", + "step: 2670, Red action: do-nothing, Blue reward:0.85\n", + "step: 2671, Red action: do-nothing, Blue reward:0.85\n", + "step: 2672, Red action: do-nothing, Blue reward:0.85\n", + "step: 2673, Red action: do-nothing, Blue reward:0.85\n", + "step: 2674, Red action: do-nothing, Blue reward:0.85\n", + "step: 2675, Red action: do-nothing, Blue reward:0.85\n", + "step: 2676, Red action: do-nothing, Blue reward:0.85\n", + "step: 2677, Red action: do-nothing, Blue reward:0.85\n", + "step: 2678, Red action: do-nothing, Blue reward:0.85\n", + "step: 2679, Red action: do-nothing, Blue reward:0.85\n", + "step: 2680, Red action: do-nothing, Blue reward:0.85\n", + "step: 2681, Red action: do-nothing, Blue reward:0.85\n", + "step: 2682, Red action: do-nothing, Blue reward:0.85\n", + "step: 2683, Red action: do-nothing, Blue reward:0.85\n", + "step: 2684, Red action: do-nothing, Blue reward:0.85\n", + "step: 2685, Red action: do-nothing, Blue reward:0.85\n", + "step: 2686, Red action: do-nothing, Blue reward:0.85\n", + "step: 2687, Red action: do-nothing, Blue reward:0.85\n", + "step: 2688, Red action: do-nothing, Blue reward:0.85\n", + "step: 2689, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2690, Red action: do-nothing, Blue reward:0.85\n", + "step: 2691, Red action: do-nothing, Blue reward:0.85\n", + "step: 2692, Red action: do-nothing, Blue reward:0.85\n", + "step: 2693, Red action: do-nothing, Blue reward:0.85\n", + "step: 2694, Red action: do-nothing, Blue reward:0.85\n", + "step: 2695, Red action: do-nothing, Blue reward:0.85\n", + "step: 2696, Red action: do-nothing, Blue reward:0.85\n", + "step: 2697, Red action: do-nothing, Blue reward:0.85\n", + "step: 2698, Red action: do-nothing, Blue reward:0.85\n", + "step: 2699, Red action: do-nothing, Blue reward:0.85\n", + "step: 2700, Red action: do-nothing, Blue reward:0.85\n", + "step: 2701, Red action: do-nothing, Blue reward:0.85\n", + "step: 2702, Red action: do-nothing, Blue reward:0.85\n", + "step: 2703, Red action: do-nothing, Blue reward:0.85\n", + "step: 2704, Red action: do-nothing, Blue reward:0.85\n", + "step: 2705, Red action: do-nothing, Blue reward:0.85\n", + "step: 2706, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2707, Red action: do-nothing, Blue reward:0.85\n", + "step: 2708, Red action: do-nothing, Blue reward:0.85\n", + "step: 2709, Red action: do-nothing, Blue reward:0.85\n", + "step: 2710, Red action: do-nothing, Blue reward:0.85\n", + "step: 2711, Red action: do-nothing, Blue reward:0.85\n", + "step: 2712, Red action: do-nothing, Blue reward:0.85\n", + "step: 2713, Red action: do-nothing, Blue reward:0.85\n", + "step: 2714, Red action: do-nothing, Blue reward:0.85\n", + "step: 2715, Red action: do-nothing, Blue reward:0.85\n", + "step: 2716, Red action: do-nothing, Blue reward:0.85\n", + "step: 2717, Red action: do-nothing, Blue reward:0.85\n", + "step: 2718, Red action: do-nothing, Blue reward:0.85\n", + "step: 2719, Red action: do-nothing, Blue reward:0.85\n", + "step: 2720, Red action: do-nothing, Blue reward:0.85\n", + "step: 2721, Red action: do-nothing, Blue reward:0.85\n", + "step: 2722, Red action: do-nothing, Blue reward:0.85\n", + "step: 2723, Red action: do-nothing, Blue reward:0.85\n", + "step: 2724, Red action: do-nothing, Blue reward:0.85\n", + "step: 2725, Red action: do-nothing, Blue reward:0.85\n", + "step: 2726, Red action: do-nothing, Blue reward:0.85\n", + "step: 2727, Red action: do-nothing, Blue reward:0.85\n", + "step: 2728, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2729, Red action: do-nothing, Blue reward:0.85\n", + "step: 2730, Red action: do-nothing, Blue reward:0.85\n", + "step: 2731, Red action: do-nothing, Blue reward:0.85\n", + "step: 2732, Red action: do-nothing, Blue reward:0.85\n", + "step: 2733, Red action: do-nothing, Blue reward:0.85\n", + "step: 2734, Red action: do-nothing, Blue reward:0.85\n", + "step: 2735, Red action: do-nothing, Blue reward:0.85\n", + "step: 2736, Red action: do-nothing, Blue reward:0.85\n", + "step: 2737, Red action: do-nothing, Blue reward:0.85\n", + "step: 2738, Red action: do-nothing, Blue reward:0.85\n", + "step: 2739, Red action: do-nothing, Blue reward:0.85\n", + "step: 2740, Red action: do-nothing, Blue reward:0.85\n", + "step: 2741, Red action: do-nothing, Blue reward:0.85\n", + "step: 2742, Red action: do-nothing, Blue reward:0.85\n", + "step: 2743, Red action: do-nothing, Blue reward:0.85\n", + "step: 2744, Red action: do-nothing, Blue reward:0.85\n", + "step: 2745, Red action: do-nothing, Blue reward:0.85\n", + "step: 2746, Red action: do-nothing, Blue reward:0.85\n", + "step: 2747, Red action: do-nothing, Blue reward:0.85\n", + "step: 2748, Red action: do-nothing, Blue reward:0.85\n", + "step: 2749, Red action: do-nothing, Blue reward:0.85\n", + "step: 2750, Red action: do-nothing, Blue reward:0.85\n", + "step: 2751, Red action: do-nothing, Blue reward:0.85\n", + "step: 2752, Red action: do-nothing, Blue reward:0.85\n", + "step: 2753, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2754, Red action: do-nothing, Blue reward:0.85\n", + "step: 2755, Red action: do-nothing, Blue reward:0.85\n", + "step: 2756, Red action: do-nothing, Blue reward:0.85\n", + "step: 2757, Red action: do-nothing, Blue reward:0.85\n", + "step: 2758, Red action: do-nothing, Blue reward:0.85\n", + "step: 2759, Red action: do-nothing, Blue reward:0.85\n", + "step: 2760, Red action: do-nothing, Blue reward:0.85\n", + "step: 2761, Red action: do-nothing, Blue reward:0.85\n", + "step: 2762, Red action: do-nothing, Blue reward:0.85\n", + "step: 2763, Red action: do-nothing, Blue reward:0.85\n", + "step: 2764, Red action: do-nothing, Blue reward:0.85\n", + "step: 2765, Red action: do-nothing, Blue reward:0.85\n", + "step: 2766, Red action: do-nothing, Blue reward:0.85\n", + "step: 2767, Red action: do-nothing, Blue reward:0.85\n", + "step: 2768, Red action: do-nothing, Blue reward:0.85\n", + "step: 2769, Red action: do-nothing, Blue reward:0.85\n", + "step: 2770, Red action: do-nothing, Blue reward:0.85\n", + "step: 2771, Red action: do-nothing, Blue reward:0.85\n", + "step: 2772, Red action: do-nothing, Blue reward:0.85\n", + "step: 2773, Red action: do-nothing, Blue reward:0.85\n", + "step: 2774, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2775, Red action: do-nothing, Blue reward:0.85\n", + "step: 2776, Red action: do-nothing, Blue reward:0.85\n", + "step: 2777, Red action: do-nothing, Blue reward:0.85\n", + "step: 2778, Red action: do-nothing, Blue reward:0.85\n", + "step: 2779, Red action: do-nothing, Blue reward:0.85\n", + "step: 2780, Red action: do-nothing, Blue reward:0.85\n", + "step: 2781, Red action: do-nothing, Blue reward:0.85\n", + "step: 2782, Red action: do-nothing, Blue reward:0.85\n", + "step: 2783, Red action: do-nothing, Blue reward:0.85\n", + "step: 2784, Red action: do-nothing, Blue reward:0.85\n", + "step: 2785, Red action: do-nothing, Blue reward:0.85\n", + "step: 2786, Red action: do-nothing, Blue reward:0.85\n", + "step: 2787, Red action: do-nothing, Blue reward:0.85\n", + "step: 2788, Red action: do-nothing, Blue reward:0.85\n", + "step: 2789, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2790, Red action: do-nothing, Blue reward:0.85\n", + "step: 2791, Red action: do-nothing, Blue reward:0.85\n", + "step: 2792, Red action: do-nothing, Blue reward:0.85\n", + "step: 2793, Red action: do-nothing, Blue reward:0.85\n", + "step: 2794, Red action: do-nothing, Blue reward:0.85\n", + "step: 2795, Red action: do-nothing, Blue reward:0.85\n", + "step: 2796, Red action: do-nothing, Blue reward:0.85\n", + "step: 2797, Red action: do-nothing, Blue reward:0.85\n", + "step: 2798, Red action: do-nothing, Blue reward:0.85\n", + "step: 2799, Red action: do-nothing, Blue reward:0.85\n", + "step: 2800, Red action: do-nothing, Blue reward:0.85\n", + "step: 2801, Red action: do-nothing, Blue reward:0.85\n", + "step: 2802, Red action: do-nothing, Blue reward:0.85\n", + "step: 2803, Red action: do-nothing, Blue reward:0.85\n", + "step: 2804, Red action: do-nothing, Blue reward:0.85\n", + "step: 2805, Red action: do-nothing, Blue reward:0.85\n", + "step: 2806, Red action: do-nothing, Blue reward:0.85\n", + "step: 2807, Red action: do-nothing, Blue reward:0.85\n", + "step: 2808, Red action: do-nothing, Blue reward:0.85\n", + "step: 2809, Red action: do-nothing, Blue reward:0.85\n", + "step: 2810, Red action: do-nothing, Blue reward:0.85\n", + "step: 2811, Red action: do-nothing, Blue reward:0.85\n", + "step: 2812, Red action: do-nothing, Blue reward:0.85\n", + "step: 2813, Red action: do-nothing, Blue reward:0.85\n", + "step: 2814, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2815, Red action: do-nothing, Blue reward:0.85\n", + "step: 2816, Red action: do-nothing, Blue reward:0.85\n", + "step: 2817, Red action: do-nothing, Blue reward:0.85\n", + "step: 2818, Red action: do-nothing, Blue reward:0.85\n", + "step: 2819, Red action: do-nothing, Blue reward:0.85\n", + "step: 2820, Red action: do-nothing, Blue reward:0.85\n", + "step: 2821, Red action: do-nothing, Blue reward:0.85\n", + "step: 2822, Red action: do-nothing, Blue reward:0.85\n", + "step: 2823, Red action: do-nothing, Blue reward:0.85\n", + "step: 2824, Red action: do-nothing, Blue reward:0.85\n", + "step: 2825, Red action: do-nothing, Blue reward:0.85\n", + "step: 2826, Red action: do-nothing, Blue reward:0.85\n", + "step: 2827, Red action: do-nothing, Blue reward:0.85\n", + "step: 2828, Red action: do-nothing, Blue reward:0.85\n", + "step: 2829, Red action: do-nothing, Blue reward:0.85\n", + "step: 2830, Red action: do-nothing, Blue reward:0.85\n", + "step: 2831, Red action: do-nothing, Blue reward:0.85\n", + "step: 2832, Red action: do-nothing, Blue reward:0.85\n", + "step: 2833, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2834, Red action: do-nothing, Blue reward:0.85\n", + "step: 2835, Red action: do-nothing, Blue reward:0.85\n", + "step: 2836, Red action: do-nothing, Blue reward:0.85\n", + "step: 2837, Red action: do-nothing, Blue reward:0.85\n", + "step: 2838, Red action: do-nothing, Blue reward:0.85\n", + "step: 2839, Red action: do-nothing, Blue reward:0.85\n", + "step: 2840, Red action: do-nothing, Blue reward:0.85\n", + "step: 2841, Red action: do-nothing, Blue reward:0.85\n", + "step: 2842, Red action: do-nothing, Blue reward:0.85\n", + "step: 2843, Red action: do-nothing, Blue reward:0.85\n", + "step: 2844, Red action: do-nothing, Blue reward:0.85\n", + "step: 2845, Red action: do-nothing, Blue reward:0.85\n", + "step: 2846, Red action: do-nothing, Blue reward:0.85\n", + "step: 2847, Red action: do-nothing, Blue reward:0.85\n", + "step: 2848, Red action: do-nothing, Blue reward:0.85\n", + "step: 2849, Red action: do-nothing, Blue reward:0.85\n", + "step: 2850, Red action: do-nothing, Blue reward:0.85\n", + "step: 2851, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2852, Red action: do-nothing, Blue reward:0.85\n", + "step: 2853, Red action: do-nothing, Blue reward:0.85\n", + "step: 2854, Red action: do-nothing, Blue reward:0.85\n", + "step: 2855, Red action: do-nothing, Blue reward:0.85\n", + "step: 2856, Red action: do-nothing, Blue reward:0.85\n", + "step: 2857, Red action: do-nothing, Blue reward:0.85\n", + "step: 2858, Red action: do-nothing, Blue reward:0.85\n", + "step: 2859, Red action: do-nothing, Blue reward:0.85\n", + "step: 2860, Red action: do-nothing, Blue reward:0.85\n", + "step: 2861, Red action: do-nothing, Blue reward:0.85\n", + "step: 2862, Red action: do-nothing, Blue reward:0.85\n", + "step: 2863, Red action: do-nothing, Blue reward:0.85\n", + "step: 2864, Red action: do-nothing, Blue reward:0.85\n", + "step: 2865, Red action: do-nothing, Blue reward:0.85\n", + "step: 2866, Red action: do-nothing, Blue reward:0.85\n", + "step: 2867, Red action: do-nothing, Blue reward:0.85\n", + "step: 2868, Red action: do-nothing, Blue reward:0.85\n", + "step: 2869, Red action: do-nothing, Blue reward:0.85\n", + "step: 2870, Red action: do-nothing, Blue reward:0.85\n", + "step: 2871, Red action: do-nothing, Blue reward:0.85\n", + "step: 2872, Red action: do-nothing, Blue reward:0.85\n", + "step: 2873, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2874, Red action: do-nothing, Blue reward:0.85\n", + "step: 2875, Red action: do-nothing, Blue reward:0.85\n", + "step: 2876, Red action: do-nothing, Blue reward:0.85\n", + "step: 2877, Red action: do-nothing, Blue reward:0.85\n", + "step: 2878, Red action: do-nothing, Blue reward:0.85\n", + "step: 2879, Red action: do-nothing, Blue reward:0.85\n", + "step: 2880, Red action: do-nothing, Blue reward:0.85\n", + "step: 2881, Red action: do-nothing, Blue reward:0.85\n", + "step: 2882, Red action: do-nothing, Blue reward:0.85\n", + "step: 2883, Red action: do-nothing, Blue reward:0.85\n", + "step: 2884, Red action: do-nothing, Blue reward:0.85\n", + "step: 2885, Red action: do-nothing, Blue reward:0.85\n", + "step: 2886, Red action: do-nothing, Blue reward:0.85\n", + "step: 2887, Red action: do-nothing, Blue reward:0.85\n", + "step: 2888, Red action: do-nothing, Blue reward:0.85\n", + "step: 2889, Red action: do-nothing, Blue reward:0.85\n", + "step: 2890, Red action: do-nothing, Blue reward:0.85\n", + "step: 2891, Red action: do-nothing, Blue reward:0.85\n", + "step: 2892, Red action: do-nothing, Blue reward:0.85\n", + "step: 2893, Red action: do-nothing, Blue reward:0.85\n", + "step: 2894, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2895, Red action: do-nothing, Blue reward:0.85\n", + "step: 2896, Red action: do-nothing, Blue reward:0.85\n", + "step: 2897, Red action: do-nothing, Blue reward:0.85\n", + "step: 2898, Red action: do-nothing, Blue reward:0.85\n", + "step: 2899, Red action: do-nothing, Blue reward:0.85\n", + "step: 2900, Red action: do-nothing, Blue reward:0.85\n", + "step: 2901, Red action: do-nothing, Blue reward:0.85\n", + "step: 2902, Red action: do-nothing, Blue reward:0.85\n", + "step: 2903, Red action: do-nothing, Blue reward:0.85\n", + "step: 2904, Red action: do-nothing, Blue reward:0.85\n", + "step: 2905, Red action: do-nothing, Blue reward:0.85\n", + "step: 2906, Red action: do-nothing, Blue reward:0.85\n", + "step: 2907, Red action: do-nothing, Blue reward:0.85\n", + "step: 2908, Red action: do-nothing, Blue reward:0.85\n", + "step: 2909, Red action: do-nothing, Blue reward:0.85\n", + "step: 2910, Red action: do-nothing, Blue reward:0.85\n", + "step: 2911, Red action: do-nothing, Blue reward:0.85\n", + "step: 2912, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2913, Red action: do-nothing, Blue reward:0.85\n", + "step: 2914, Red action: do-nothing, Blue reward:0.85\n", + "step: 2915, Red action: do-nothing, Blue reward:0.85\n", + "step: 2916, Red action: do-nothing, Blue reward:0.85\n", + "step: 2917, Red action: do-nothing, Blue reward:0.85\n", + "step: 2918, Red action: do-nothing, Blue reward:0.85\n", + "step: 2919, Red action: do-nothing, Blue reward:0.85\n", + "step: 2920, Red action: do-nothing, Blue reward:0.85\n", + "step: 2921, Red action: do-nothing, Blue reward:0.85\n", + "step: 2922, Red action: do-nothing, Blue reward:0.85\n", + "step: 2923, Red action: do-nothing, Blue reward:0.85\n", + "step: 2924, Red action: do-nothing, Blue reward:0.85\n", + "step: 2925, Red action: do-nothing, Blue reward:0.85\n", + "step: 2926, Red action: do-nothing, Blue reward:0.85\n", + "step: 2927, Red action: do-nothing, Blue reward:0.85\n", + "step: 2928, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2929, Red action: do-nothing, Blue reward:0.85\n", + "step: 2930, Red action: do-nothing, Blue reward:0.85\n", + "step: 2931, Red action: do-nothing, Blue reward:0.85\n", + "step: 2932, Red action: do-nothing, Blue reward:0.85\n", + "step: 2933, Red action: do-nothing, Blue reward:0.85\n", + "step: 2934, Red action: do-nothing, Blue reward:0.85\n", + "step: 2935, Red action: do-nothing, Blue reward:0.85\n", + "step: 2936, Red action: do-nothing, Blue reward:0.85\n", + "step: 2937, Red action: do-nothing, Blue reward:0.85\n", + "step: 2938, Red action: do-nothing, Blue reward:0.85\n", + "step: 2939, Red action: do-nothing, Blue reward:0.85\n", + "step: 2940, Red action: do-nothing, Blue reward:0.85\n", + "step: 2941, Red action: do-nothing, Blue reward:0.85\n", + "step: 2942, Red action: do-nothing, Blue reward:0.85\n", + "step: 2943, Red action: do-nothing, Blue reward:0.85\n", + "step: 2944, Red action: do-nothing, Blue reward:0.85\n", + "step: 2945, Red action: do-nothing, Blue reward:0.85\n", + "step: 2946, Red action: do-nothing, Blue reward:0.85\n", + "step: 2947, Red action: do-nothing, Blue reward:0.85\n", + "step: 2948, Red action: do-nothing, Blue reward:0.85\n", + "step: 2949, Red action: do-nothing, Blue reward:0.85\n", + "step: 2950, Red action: do-nothing, Blue reward:0.85\n", + "step: 2951, Red action: do-nothing, Blue reward:0.85\n", + "step: 2952, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2953, Red action: do-nothing, Blue reward:0.85\n", + "step: 2954, Red action: do-nothing, Blue reward:0.85\n", + "step: 2955, Red action: do-nothing, Blue reward:0.85\n", + "step: 2956, Red action: do-nothing, Blue reward:0.85\n", + "step: 2957, Red action: do-nothing, Blue reward:0.85\n", + "step: 2958, Red action: do-nothing, Blue reward:0.85\n", + "step: 2959, Red action: do-nothing, Blue reward:0.85\n", + "step: 2960, Red action: do-nothing, Blue reward:0.85\n", + "step: 2961, Red action: do-nothing, Blue reward:0.85\n", + "step: 2962, Red action: do-nothing, Blue reward:0.85\n", + "step: 2963, Red action: do-nothing, Blue reward:0.85\n", + "step: 2964, Red action: do-nothing, Blue reward:0.85\n", + "step: 2965, Red action: do-nothing, Blue reward:0.85\n", + "step: 2966, Red action: do-nothing, Blue reward:0.85\n", + "step: 2967, Red action: do-nothing, Blue reward:0.85\n", + "step: 2968, Red action: do-nothing, Blue reward:0.85\n", + "step: 2969, Red action: do-nothing, Blue reward:0.85\n", + "step: 2970, Red action: do-nothing, Blue reward:0.85\n", + "step: 2971, Red action: do-nothing, Blue reward:0.85\n", + "step: 2972, Red action: do-nothing, Blue reward:0.85\n", + "step: 2973, Red action: do-nothing, Blue reward:0.85\n", + "step: 2974, Red action: do-nothing, Blue reward:0.85\n", + "step: 2975, Red action: do-nothing, Blue reward:0.85\n", + "step: 2976, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2977, Red action: do-nothing, Blue reward:0.85\n", + "step: 2978, Red action: do-nothing, Blue reward:0.85\n", + "step: 2979, Red action: do-nothing, Blue reward:0.85\n", + "step: 2980, Red action: do-nothing, Blue reward:0.85\n", + "step: 2981, Red action: do-nothing, Blue reward:0.85\n", + "step: 2982, Red action: do-nothing, Blue reward:0.85\n", + "step: 2983, Red action: do-nothing, Blue reward:0.85\n", + "step: 2984, Red action: do-nothing, Blue reward:0.85\n", + "step: 2985, Red action: do-nothing, Blue reward:0.85\n", + "step: 2986, Red action: do-nothing, Blue reward:0.85\n", + "step: 2987, Red action: do-nothing, Blue reward:0.85\n", + "step: 2988, Red action: do-nothing, Blue reward:0.85\n", + "step: 2989, Red action: do-nothing, Blue reward:0.85\n", + "step: 2990, Red action: do-nothing, Blue reward:0.85\n", + "step: 2991, Red action: do-nothing, Blue reward:0.85\n", + "step: 2992, Red action: do-nothing, Blue reward:0.85\n", + "step: 2993, Red action: do-nothing, Blue reward:0.85\n", + "step: 2994, Red action: do-nothing, Blue reward:0.85\n", + "step: 2995, Red action: do-nothing, Blue reward:0.85\n", + "step: 2996, Red action: do-nothing, Blue reward:0.85\n", + "step: 2997, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2998, Red action: do-nothing, Blue reward:0.85\n", + "step: 2999, Red action: do-nothing, Blue reward:0.85\n", + "step: 3000, Red action: do-nothing, Blue reward:0.85\n", + "step: 3001, Red action: do-nothing, Blue reward:0.85\n", + "step: 3002, Red action: do-nothing, Blue reward:0.85\n", + "step: 3003, Red action: do-nothing, Blue reward:0.85\n", + "step: 3004, Red action: do-nothing, Blue reward:0.85\n", + "step: 3005, Red action: do-nothing, Blue reward:0.85\n", + "step: 3006, Red action: do-nothing, Blue reward:0.85\n", + "step: 3007, Red action: do-nothing, Blue reward:0.85\n", + "step: 3008, Red action: do-nothing, Blue reward:0.85\n", + "step: 3009, Red action: do-nothing, Blue reward:0.85\n", + "step: 3010, Red action: do-nothing, Blue reward:0.85\n", + "step: 3011, Red action: do-nothing, Blue reward:0.85\n", + "step: 3012, Red action: do-nothing, Blue reward:0.85\n", + "step: 3013, Red action: do-nothing, Blue reward:0.85\n", + "step: 3014, Red action: do-nothing, Blue reward:0.85\n", + "step: 3015, Red action: do-nothing, Blue reward:0.85\n", + "step: 3016, Red action: do-nothing, Blue reward:0.85\n", + "step: 3017, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3018, Red action: do-nothing, Blue reward:0.85\n", + "step: 3019, Red action: do-nothing, Blue reward:0.85\n", + "step: 3020, Red action: do-nothing, Blue reward:0.85\n", + "step: 3021, Red action: do-nothing, Blue reward:0.85\n", + "step: 3022, Red action: do-nothing, Blue reward:0.85\n", + "step: 3023, Red action: do-nothing, Blue reward:0.85\n", + "step: 3024, Red action: do-nothing, Blue reward:0.85\n", + "step: 3025, Red action: do-nothing, Blue reward:0.85\n", + "step: 3026, Red action: do-nothing, Blue reward:0.85\n", + "step: 3027, Red action: do-nothing, Blue reward:0.85\n", + "step: 3028, Red action: do-nothing, Blue reward:0.85\n", + "step: 3029, Red action: do-nothing, Blue reward:0.85\n", + "step: 3030, Red action: do-nothing, Blue reward:0.85\n", + "step: 3031, Red action: do-nothing, Blue reward:0.85\n", + "step: 3032, Red action: do-nothing, Blue reward:0.85\n", + "step: 3033, Red action: do-nothing, Blue reward:0.85\n", + "step: 3034, Red action: do-nothing, Blue reward:0.85\n", + "step: 3035, Red action: do-nothing, Blue reward:0.85\n", + "step: 3036, Red action: do-nothing, Blue reward:0.85\n", + "step: 3037, Red action: do-nothing, Blue reward:0.85\n", + "step: 3038, Red action: do-nothing, Blue reward:0.85\n", + "step: 3039, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3040, Red action: do-nothing, Blue reward:0.85\n", + "step: 3041, Red action: do-nothing, Blue reward:0.85\n", + "step: 3042, Red action: do-nothing, Blue reward:0.85\n", + "step: 3043, Red action: do-nothing, Blue reward:0.85\n", + "step: 3044, Red action: do-nothing, Blue reward:0.85\n", + "step: 3045, Red action: do-nothing, Blue reward:0.85\n", + "step: 3046, Red action: do-nothing, Blue reward:0.85\n", + "step: 3047, Red action: do-nothing, Blue reward:0.85\n", + "step: 3048, Red action: do-nothing, Blue reward:0.85\n", + "step: 3049, Red action: do-nothing, Blue reward:0.85\n", + "step: 3050, Red action: do-nothing, Blue reward:0.85\n", + "step: 3051, Red action: do-nothing, Blue reward:0.85\n", + "step: 3052, Red action: do-nothing, Blue reward:0.85\n", + "step: 3053, Red action: do-nothing, Blue reward:0.85\n", + "step: 3054, Red action: do-nothing, Blue reward:0.85\n", + "step: 3055, Red action: do-nothing, Blue reward:0.85\n", + "step: 3056, Red action: do-nothing, Blue reward:0.85\n", + "step: 3057, Red action: do-nothing, Blue reward:0.85\n", + "step: 3058, Red action: do-nothing, Blue reward:0.85\n", + "step: 3059, Red action: do-nothing, Blue reward:0.85\n", + "step: 3060, Red action: do-nothing, Blue reward:0.85\n", + "step: 3061, Red action: do-nothing, Blue reward:0.85\n", + "step: 3062, Red action: do-nothing, Blue reward:0.85\n", + "step: 3063, Red action: do-nothing, Blue reward:0.85\n", + "step: 3064, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3065, Red action: do-nothing, Blue reward:0.85\n", + "step: 3066, Red action: do-nothing, Blue reward:0.85\n", + "step: 3067, Red action: do-nothing, Blue reward:0.85\n", + "step: 3068, Red action: do-nothing, Blue reward:0.85\n", + "step: 3069, Red action: do-nothing, Blue reward:0.85\n", + "step: 3070, Red action: do-nothing, Blue reward:0.85\n", + "step: 3071, Red action: do-nothing, Blue reward:0.85\n", + "step: 3072, Red action: do-nothing, Blue reward:0.85\n", + "step: 3073, Red action: do-nothing, Blue reward:0.85\n", + "step: 3074, Red action: do-nothing, Blue reward:0.85\n", + "step: 3075, Red action: do-nothing, Blue reward:0.85\n", + "step: 3076, Red action: do-nothing, Blue reward:0.85\n", + "step: 3077, Red action: do-nothing, Blue reward:0.85\n", + "step: 3078, Red action: do-nothing, Blue reward:0.85\n", + "step: 3079, Red action: do-nothing, Blue reward:0.85\n", + "step: 3080, Red action: do-nothing, Blue reward:0.85\n", + "step: 3081, Red action: do-nothing, Blue reward:0.85\n", + "step: 3082, Red action: do-nothing, Blue reward:0.85\n", + "step: 3083, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3084, Red action: do-nothing, Blue reward:0.85\n", + "step: 3085, Red action: do-nothing, Blue reward:0.85\n", + "step: 3086, Red action: do-nothing, Blue reward:0.85\n", + "step: 3087, Red action: do-nothing, Blue reward:0.85\n", + "step: 3088, Red action: do-nothing, Blue reward:0.85\n", + "step: 3089, Red action: do-nothing, Blue reward:0.85\n", + "step: 3090, Red action: do-nothing, Blue reward:0.85\n", + "step: 3091, Red action: do-nothing, Blue reward:0.85\n", + "step: 3092, Red action: do-nothing, Blue reward:0.85\n", + "step: 3093, Red action: do-nothing, Blue reward:0.85\n", + "step: 3094, Red action: do-nothing, Blue reward:0.85\n", + "step: 3095, Red action: do-nothing, Blue reward:0.85\n", + "step: 3096, Red action: do-nothing, Blue reward:0.85\n", + "step: 3097, Red action: do-nothing, Blue reward:0.85\n", + "step: 3098, Red action: do-nothing, Blue reward:0.85\n", + "step: 3099, Red action: do-nothing, Blue reward:0.85\n", + "step: 3100, Red action: do-nothing, Blue reward:0.85\n", + "step: 3101, Red action: do-nothing, Blue reward:0.85\n", + "step: 3102, Red action: do-nothing, Blue reward:0.85\n", + "step: 3103, Red action: do-nothing, Blue reward:0.85\n", + "step: 3104, Red action: do-nothing, Blue reward:0.85\n", + "step: 3105, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3106, Red action: do-nothing, Blue reward:0.85\n", + "step: 3107, Red action: do-nothing, Blue reward:0.85\n", + "step: 3108, Red action: do-nothing, Blue reward:0.85\n", + "step: 3109, Red action: do-nothing, Blue reward:0.85\n", + "step: 3110, Red action: do-nothing, Blue reward:0.85\n", + "step: 3111, Red action: do-nothing, Blue reward:0.85\n", + "step: 3112, Red action: do-nothing, Blue reward:0.85\n", + "step: 3113, Red action: do-nothing, Blue reward:0.85\n", + "step: 3114, Red action: do-nothing, Blue reward:0.85\n", + "step: 3115, Red action: do-nothing, Blue reward:0.85\n", + "step: 3116, Red action: do-nothing, Blue reward:0.85\n", + "step: 3117, Red action: do-nothing, Blue reward:0.85\n", + "step: 3118, Red action: do-nothing, Blue reward:0.85\n", + "step: 3119, Red action: do-nothing, Blue reward:0.85\n", + "step: 3120, Red action: do-nothing, Blue reward:0.85\n", + "step: 3121, Red action: do-nothing, Blue reward:0.85\n", + "step: 3122, Red action: do-nothing, Blue reward:0.85\n", + "step: 3123, Red action: do-nothing, Blue reward:0.85\n", + "step: 3124, Red action: do-nothing, Blue reward:0.85\n", + "step: 3125, Red action: do-nothing, Blue reward:0.85\n", + "step: 3126, Red action: do-nothing, Blue reward:0.85\n", + "step: 3127, Red action: do-nothing, Blue reward:0.85\n", + "step: 3128, Red action: do-nothing, Blue reward:0.85\n", + "step: 3129, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3130, Red action: do-nothing, Blue reward:0.85\n", + "step: 3131, Red action: do-nothing, Blue reward:0.85\n", + "step: 3132, Red action: do-nothing, Blue reward:0.85\n", + "step: 3133, Red action: do-nothing, Blue reward:0.85\n", + "step: 3134, Red action: do-nothing, Blue reward:0.85\n", + "step: 3135, Red action: do-nothing, Blue reward:0.85\n", + "step: 3136, Red action: do-nothing, Blue reward:0.85\n", + "step: 3137, Red action: do-nothing, Blue reward:0.85\n", + "step: 3138, Red action: do-nothing, Blue reward:0.85\n", + "step: 3139, Red action: do-nothing, Blue reward:0.85\n", + "step: 3140, Red action: do-nothing, Blue reward:0.85\n", + "step: 3141, Red action: do-nothing, Blue reward:0.85\n", + "step: 3142, Red action: do-nothing, Blue reward:0.85\n", + "step: 3143, Red action: do-nothing, Blue reward:0.85\n", + "step: 3144, Red action: do-nothing, Blue reward:0.85\n", + "step: 3145, Red action: do-nothing, Blue reward:0.85\n", + "step: 3146, Red action: do-nothing, Blue reward:0.85\n", + "step: 3147, Red action: do-nothing, Blue reward:0.85\n", + "step: 3148, Red action: do-nothing, Blue reward:0.85\n", + "step: 3149, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3150, Red action: do-nothing, Blue reward:0.85\n", + "step: 3151, Red action: do-nothing, Blue reward:0.85\n", + "step: 3152, Red action: do-nothing, Blue reward:0.85\n", + "step: 3153, Red action: do-nothing, Blue reward:0.85\n", + "step: 3154, Red action: do-nothing, Blue reward:0.85\n", + "step: 3155, Red action: do-nothing, Blue reward:0.85\n", + "step: 3156, Red action: do-nothing, Blue reward:0.85\n", + "step: 3157, Red action: do-nothing, Blue reward:0.85\n", + "step: 3158, Red action: do-nothing, Blue reward:0.85\n", + "step: 3159, Red action: do-nothing, Blue reward:0.85\n", + "step: 3160, Red action: do-nothing, Blue reward:0.85\n", + "step: 3161, Red action: do-nothing, Blue reward:0.85\n", + "step: 3162, Red action: do-nothing, Blue reward:0.85\n", + "step: 3163, Red action: do-nothing, Blue reward:0.85\n", + "step: 3164, Red action: do-nothing, Blue reward:0.85\n", + "step: 3165, Red action: do-nothing, Blue reward:0.85\n", + "step: 3166, Red action: do-nothing, Blue reward:0.85\n", + "step: 3167, Red action: do-nothing, Blue reward:0.85\n", + "step: 3168, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3169, Red action: do-nothing, Blue reward:0.85\n", + "step: 3170, Red action: do-nothing, Blue reward:0.85\n", + "step: 3171, Red action: do-nothing, Blue reward:0.85\n", + "step: 3172, Red action: do-nothing, Blue reward:0.85\n", + "step: 3173, Red action: do-nothing, Blue reward:0.85\n", + "step: 3174, Red action: do-nothing, Blue reward:0.85\n", + "step: 3175, Red action: do-nothing, Blue reward:0.85\n", + "step: 3176, Red action: do-nothing, Blue reward:0.85\n", + "step: 3177, Red action: do-nothing, Blue reward:0.85\n", + "step: 3178, Red action: do-nothing, Blue reward:0.85\n", + "step: 3179, Red action: do-nothing, Blue reward:0.85\n", + "step: 3180, Red action: do-nothing, Blue reward:0.85\n", + "step: 3181, Red action: do-nothing, Blue reward:0.85\n", + "step: 3182, Red action: do-nothing, Blue reward:0.85\n", + "step: 3183, Red action: do-nothing, Blue reward:0.85\n", + "step: 3184, Red action: do-nothing, Blue reward:0.85\n", + "step: 3185, Red action: do-nothing, Blue reward:0.85\n", + "step: 3186, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3187, Red action: do-nothing, Blue reward:0.85\n", + "step: 3188, Red action: do-nothing, Blue reward:0.85\n", + "step: 3189, Red action: do-nothing, Blue reward:0.85\n", + "step: 3190, Red action: do-nothing, Blue reward:0.85\n", + "step: 3191, Red action: do-nothing, Blue reward:0.85\n", + "step: 3192, Red action: do-nothing, Blue reward:0.85\n", + "step: 3193, Red action: do-nothing, Blue reward:0.85\n", + "step: 3194, Red action: do-nothing, Blue reward:0.85\n", + "step: 3195, Red action: do-nothing, Blue reward:0.85\n", + "step: 3196, Red action: do-nothing, Blue reward:0.85\n", + "step: 3197, Red action: do-nothing, Blue reward:0.85\n", + "step: 3198, Red action: do-nothing, Blue reward:0.85\n", + "step: 3199, Red action: do-nothing, Blue reward:0.85\n", + "step: 3200, Red action: do-nothing, Blue reward:0.85\n", + "step: 3201, Red action: do-nothing, Blue reward:0.85\n", + "step: 3202, Red action: do-nothing, Blue reward:0.85\n", + "step: 3203, Red action: do-nothing, Blue reward:0.85\n", + "step: 3204, Red action: do-nothing, Blue reward:0.85\n", + "step: 3205, Red action: do-nothing, Blue reward:0.85\n", + "step: 3206, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3207, Red action: do-nothing, Blue reward:0.85\n", + "step: 3208, Red action: do-nothing, Blue reward:0.85\n", + "step: 3209, Red action: do-nothing, Blue reward:0.85\n", + "step: 3210, Red action: do-nothing, Blue reward:0.85\n", + "step: 3211, Red action: do-nothing, Blue reward:0.85\n", + "step: 3212, Red action: do-nothing, Blue reward:0.85\n", + "step: 3213, Red action: do-nothing, Blue reward:0.85\n", + "step: 3214, Red action: do-nothing, Blue reward:0.85\n", + "step: 3215, Red action: do-nothing, Blue reward:0.85\n", + "step: 3216, Red action: do-nothing, Blue reward:0.85\n", + "step: 3217, Red action: do-nothing, Blue reward:0.85\n", + "step: 3218, Red action: do-nothing, Blue reward:0.85\n", + "step: 3219, Red action: do-nothing, Blue reward:0.85\n", + "step: 3220, Red action: do-nothing, Blue reward:0.85\n", + "step: 3221, Red action: do-nothing, Blue reward:0.85\n", + "step: 3222, Red action: do-nothing, Blue reward:0.85\n", + "step: 3223, Red action: do-nothing, Blue reward:0.85\n", + "step: 3224, Red action: do-nothing, Blue reward:0.85\n", + "step: 3225, Red action: do-nothing, Blue reward:0.85\n", + "step: 3226, Red action: do-nothing, Blue reward:0.85\n", + "step: 3227, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3228, Red action: do-nothing, Blue reward:0.85\n", + "step: 3229, Red action: do-nothing, Blue reward:0.85\n", + "step: 3230, Red action: do-nothing, Blue reward:0.85\n", + "step: 3231, Red action: do-nothing, Blue reward:0.85\n", + "step: 3232, Red action: do-nothing, Blue reward:0.85\n", + "step: 3233, Red action: do-nothing, Blue reward:0.85\n", + "step: 3234, Red action: do-nothing, Blue reward:0.85\n", + "step: 3235, Red action: do-nothing, Blue reward:0.85\n", + "step: 3236, Red action: do-nothing, Blue reward:0.85\n", + "step: 3237, Red action: do-nothing, Blue reward:0.85\n", + "step: 3238, Red action: do-nothing, Blue reward:0.85\n", + "step: 3239, Red action: do-nothing, Blue reward:0.85\n", + "step: 3240, Red action: do-nothing, Blue reward:0.85\n", + "step: 3241, Red action: do-nothing, Blue reward:0.85\n", + "step: 3242, Red action: do-nothing, Blue reward:0.85\n", + "step: 3243, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3244, Red action: do-nothing, Blue reward:0.85\n", + "step: 3245, Red action: do-nothing, Blue reward:0.85\n", + "step: 3246, Red action: do-nothing, Blue reward:0.85\n", + "step: 3247, Red action: do-nothing, Blue reward:0.85\n", + "step: 3248, Red action: do-nothing, Blue reward:0.85\n", + "step: 3249, Red action: do-nothing, Blue reward:0.85\n", + "step: 3250, Red action: do-nothing, Blue reward:0.85\n", + "step: 3251, Red action: do-nothing, Blue reward:0.85\n", + "step: 3252, Red action: do-nothing, Blue reward:0.85\n", + "step: 3253, Red action: do-nothing, Blue reward:0.85\n", + "step: 3254, Red action: do-nothing, Blue reward:0.85\n", + "step: 3255, Red action: do-nothing, Blue reward:0.85\n", + "step: 3256, Red action: do-nothing, Blue reward:0.85\n", + "step: 3257, Red action: do-nothing, Blue reward:0.85\n", + "step: 3258, Red action: do-nothing, Blue reward:0.85\n", + "step: 3259, Red action: do-nothing, Blue reward:0.85\n", + "step: 3260, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3261, Red action: do-nothing, Blue reward:0.85\n", + "step: 3262, Red action: do-nothing, Blue reward:0.85\n", + "step: 3263, Red action: do-nothing, Blue reward:0.85\n", + "step: 3264, Red action: do-nothing, Blue reward:0.85\n", + "step: 3265, Red action: do-nothing, Blue reward:0.85\n", + "step: 3266, Red action: do-nothing, Blue reward:0.85\n", + "step: 3267, Red action: do-nothing, Blue reward:0.85\n", + "step: 3268, Red action: do-nothing, Blue reward:0.85\n", + "step: 3269, Red action: do-nothing, Blue reward:0.85\n", + "step: 3270, Red action: do-nothing, Blue reward:0.85\n", + "step: 3271, Red action: do-nothing, Blue reward:0.85\n", + "step: 3272, Red action: do-nothing, Blue reward:0.85\n", + "step: 3273, Red action: do-nothing, Blue reward:0.85\n", + "step: 3274, Red action: do-nothing, Blue reward:0.85\n", + "step: 3275, Red action: do-nothing, Blue reward:0.85\n", + "step: 3276, Red action: do-nothing, Blue reward:0.85\n", + "step: 3277, Red action: do-nothing, Blue reward:0.85\n", + "step: 3278, Red action: do-nothing, Blue reward:0.85\n", + "step: 3279, Red action: do-nothing, Blue reward:0.85\n", + "step: 3280, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3281, Red action: do-nothing, Blue reward:0.85\n", + "step: 3282, Red action: do-nothing, Blue reward:0.85\n", + "step: 3283, Red action: do-nothing, Blue reward:0.85\n", + "step: 3284, Red action: do-nothing, Blue reward:0.85\n", + "step: 3285, Red action: do-nothing, Blue reward:0.85\n", + "step: 3286, Red action: do-nothing, Blue reward:0.85\n", + "step: 3287, Red action: do-nothing, Blue reward:0.85\n", + "step: 3288, Red action: do-nothing, Blue reward:0.85\n", + "step: 3289, Red action: do-nothing, Blue reward:0.85\n", + "step: 3290, Red action: do-nothing, Blue reward:0.85\n", + "step: 3291, Red action: do-nothing, Blue reward:0.85\n", + "step: 3292, Red action: do-nothing, Blue reward:0.85\n", + "step: 3293, Red action: do-nothing, Blue reward:0.85\n", + "step: 3294, Red action: do-nothing, Blue reward:0.85\n", + "step: 3295, Red action: do-nothing, Blue reward:0.85\n", + "step: 3296, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3297, Red action: do-nothing, Blue reward:0.85\n", + "step: 3298, Red action: do-nothing, Blue reward:0.85\n", + "step: 3299, Red action: do-nothing, Blue reward:0.85\n", + "step: 3300, Red action: do-nothing, Blue reward:0.85\n", + "step: 3301, Red action: do-nothing, Blue reward:0.85\n", + "step: 3302, Red action: do-nothing, Blue reward:0.85\n", + "step: 3303, Red action: do-nothing, Blue reward:0.85\n", + "step: 3304, Red action: do-nothing, Blue reward:0.85\n", + "step: 3305, Red action: do-nothing, Blue reward:0.85\n", + "step: 3306, Red action: do-nothing, Blue reward:0.85\n", + "step: 3307, Red action: do-nothing, Blue reward:0.85\n", + "step: 3308, Red action: do-nothing, Blue reward:0.85\n", + "step: 3309, Red action: do-nothing, Blue reward:0.85\n", + "step: 3310, Red action: do-nothing, Blue reward:0.85\n", + "step: 3311, Red action: do-nothing, Blue reward:0.85\n", + "step: 3312, Red action: do-nothing, Blue reward:0.85\n", + "step: 3313, Red action: do-nothing, Blue reward:0.85\n", + "step: 3314, Red action: do-nothing, Blue reward:0.85\n", + "step: 3315, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3316, Red action: do-nothing, Blue reward:0.85\n", + "step: 3317, Red action: do-nothing, Blue reward:0.85\n", + "step: 3318, Red action: do-nothing, Blue reward:0.85\n", + "step: 3319, Red action: do-nothing, Blue reward:0.85\n", + "step: 3320, Red action: do-nothing, Blue reward:0.85\n", + "step: 3321, Red action: do-nothing, Blue reward:0.85\n", + "step: 3322, Red action: do-nothing, Blue reward:0.85\n", + "step: 3323, Red action: do-nothing, Blue reward:0.85\n", + "step: 3324, Red action: do-nothing, Blue reward:0.85\n", + "step: 3325, Red action: do-nothing, Blue reward:0.85\n", + "step: 3326, Red action: do-nothing, Blue reward:0.85\n", + "step: 3327, Red action: do-nothing, Blue reward:0.85\n", + "step: 3328, Red action: do-nothing, Blue reward:0.85\n", + "step: 3329, Red action: do-nothing, Blue reward:0.85\n", + "step: 3330, Red action: do-nothing, Blue reward:0.85\n", + "step: 3331, Red action: do-nothing, Blue reward:0.85\n", + "step: 3332, Red action: do-nothing, Blue reward:0.85\n", + "step: 3333, Red action: do-nothing, Blue reward:0.85\n", + "step: 3334, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3335, Red action: do-nothing, Blue reward:0.85\n", + "step: 3336, Red action: do-nothing, Blue reward:0.85\n", + "step: 3337, Red action: do-nothing, Blue reward:0.85\n", + "step: 3338, Red action: do-nothing, Blue reward:0.85\n", + "step: 3339, Red action: do-nothing, Blue reward:0.85\n", + "step: 3340, Red action: do-nothing, Blue reward:0.85\n", + "step: 3341, Red action: do-nothing, Blue reward:0.85\n", + "step: 3342, Red action: do-nothing, Blue reward:0.85\n", + "step: 3343, Red action: do-nothing, Blue reward:0.85\n", + "step: 3344, Red action: do-nothing, Blue reward:0.85\n", + "step: 3345, Red action: do-nothing, Blue reward:0.85\n", + "step: 3346, Red action: do-nothing, Blue reward:0.85\n", + "step: 3347, Red action: do-nothing, Blue reward:0.85\n", + "step: 3348, Red action: do-nothing, Blue reward:0.85\n", + "step: 3349, Red action: do-nothing, Blue reward:0.85\n", + "step: 3350, Red action: do-nothing, Blue reward:0.85\n", + "step: 3351, Red action: do-nothing, Blue reward:0.85\n", + "step: 3352, Red action: do-nothing, Blue reward:0.85\n", + "step: 3353, Red action: do-nothing, Blue reward:0.85\n", + "step: 3354, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3355, Red action: do-nothing, Blue reward:0.85\n", + "step: 3356, Red action: do-nothing, Blue reward:0.85\n", + "step: 3357, Red action: do-nothing, Blue reward:0.85\n", + "step: 3358, Red action: do-nothing, Blue reward:0.85\n", + "step: 3359, Red action: do-nothing, Blue reward:0.85\n", + "step: 3360, Red action: do-nothing, Blue reward:0.85\n", + "step: 3361, Red action: do-nothing, Blue reward:0.85\n", + "step: 3362, Red action: do-nothing, Blue reward:0.85\n", + "step: 3363, Red action: do-nothing, Blue reward:0.85\n", + "step: 3364, Red action: do-nothing, Blue reward:0.85\n", + "step: 3365, Red action: do-nothing, Blue reward:0.85\n", + "step: 3366, Red action: do-nothing, Blue reward:0.85\n", + "step: 3367, Red action: do-nothing, Blue reward:0.85\n", + "step: 3368, Red action: do-nothing, Blue reward:0.85\n", + "step: 3369, Red action: do-nothing, Blue reward:0.85\n", + "step: 3370, Red action: do-nothing, Blue reward:0.85\n", + "step: 3371, Red action: do-nothing, Blue reward:0.85\n", + "step: 3372, Red action: do-nothing, Blue reward:0.85\n", + "step: 3373, Red action: do-nothing, Blue reward:0.85\n", + "step: 3374, Red action: do-nothing, Blue reward:0.85\n", + "step: 3375, Red action: do-nothing, Blue reward:0.85\n", + "step: 3376, Red action: do-nothing, Blue reward:0.85\n", + "step: 3377, Red action: do-nothing, Blue reward:0.85\n", + "step: 3378, Red action: do-nothing, Blue reward:0.85\n", + "step: 3379, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3380, Red action: do-nothing, Blue reward:0.85\n", + "step: 3381, Red action: do-nothing, Blue reward:0.85\n", + "step: 3382, Red action: do-nothing, Blue reward:0.85\n", + "step: 3383, Red action: do-nothing, Blue reward:0.85\n", + "step: 3384, Red action: do-nothing, Blue reward:0.85\n", + "step: 3385, Red action: do-nothing, Blue reward:0.85\n", + "step: 3386, Red action: do-nothing, Blue reward:0.85\n", + "step: 3387, Red action: do-nothing, Blue reward:0.85\n", + "step: 3388, Red action: do-nothing, Blue reward:0.85\n", + "step: 3389, Red action: do-nothing, Blue reward:0.85\n", + "step: 3390, Red action: do-nothing, Blue reward:0.85\n", + "step: 3391, Red action: do-nothing, Blue reward:0.85\n", + "step: 3392, Red action: do-nothing, Blue reward:0.85\n", + "step: 3393, Red action: do-nothing, Blue reward:0.85\n", + "step: 3394, Red action: do-nothing, Blue reward:0.85\n", + "step: 3395, Red action: do-nothing, Blue reward:0.85\n", + "step: 3396, Red action: do-nothing, Blue reward:0.85\n", + "step: 3397, Red action: do-nothing, Blue reward:0.85\n", + "step: 3398, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3399, Red action: do-nothing, Blue reward:0.85\n", + "step: 3400, Red action: do-nothing, Blue reward:0.85\n", + "step: 3401, Red action: do-nothing, Blue reward:0.85\n", + "step: 3402, Red action: do-nothing, Blue reward:0.85\n", + "step: 3403, Red action: do-nothing, Blue reward:0.85\n", + "step: 3404, Red action: do-nothing, Blue reward:0.85\n", + "step: 3405, Red action: do-nothing, Blue reward:0.85\n", + "step: 3406, Red action: do-nothing, Blue reward:0.85\n", + "step: 3407, Red action: do-nothing, Blue reward:0.85\n", + "step: 3408, Red action: do-nothing, Blue reward:0.85\n", + "step: 3409, Red action: do-nothing, Blue reward:0.85\n", + "step: 3410, Red action: do-nothing, Blue reward:0.85\n", + "step: 3411, Red action: do-nothing, Blue reward:0.85\n", + "step: 3412, Red action: do-nothing, Blue reward:0.85\n", + "step: 3413, Red action: do-nothing, Blue reward:0.85\n", + "step: 3414, Red action: do-nothing, Blue reward:0.85\n", + "step: 3415, Red action: do-nothing, Blue reward:0.85\n", + "step: 3416, Red action: do-nothing, Blue reward:0.85\n", + "step: 3417, Red action: do-nothing, Blue reward:0.85\n", + "step: 3418, Red action: do-nothing, Blue reward:0.85\n", + "step: 3419, Red action: do-nothing, Blue reward:0.85\n", + "step: 3420, Red action: do-nothing, Blue reward:0.85\n", + "step: 3421, Red action: do-nothing, Blue reward:0.85\n", + "step: 3422, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3423, Red action: do-nothing, Blue reward:0.85\n", + "step: 3424, Red action: do-nothing, Blue reward:0.85\n", + "step: 3425, Red action: do-nothing, Blue reward:0.85\n", + "step: 3426, Red action: do-nothing, Blue reward:0.85\n", + "step: 3427, Red action: do-nothing, Blue reward:0.85\n", + "step: 3428, Red action: do-nothing, Blue reward:0.85\n", + "step: 3429, Red action: do-nothing, Blue reward:0.85\n", + "step: 3430, Red action: do-nothing, Blue reward:0.85\n", + "step: 3431, Red action: do-nothing, Blue reward:0.85\n", + "step: 3432, Red action: do-nothing, Blue reward:0.85\n", + "step: 3433, Red action: do-nothing, Blue reward:0.85\n", + "step: 3434, Red action: do-nothing, Blue reward:0.85\n", + "step: 3435, Red action: do-nothing, Blue reward:0.85\n", + "step: 3436, Red action: do-nothing, Blue reward:0.85\n", + "step: 3437, Red action: do-nothing, Blue reward:0.85\n", + "step: 3438, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3439, Red action: do-nothing, Blue reward:0.85\n", + "step: 3440, Red action: do-nothing, Blue reward:0.85\n", + "step: 3441, Red action: do-nothing, Blue reward:0.85\n", + "step: 3442, Red action: do-nothing, Blue reward:0.85\n", + "step: 3443, Red action: do-nothing, Blue reward:0.85\n", + "step: 3444, Red action: do-nothing, Blue reward:0.85\n", + "step: 3445, Red action: do-nothing, Blue reward:0.85\n", + "step: 3446, Red action: do-nothing, Blue reward:0.85\n", + "step: 3447, Red action: do-nothing, Blue reward:0.85\n", + "step: 3448, Red action: do-nothing, Blue reward:0.85\n", + "step: 3449, Red action: do-nothing, Blue reward:0.85\n", + "step: 3450, Red action: do-nothing, Blue reward:0.85\n", + "step: 3451, Red action: do-nothing, Blue reward:0.85\n", + "step: 3452, Red action: do-nothing, Blue reward:0.85\n", + "step: 3453, Red action: do-nothing, Blue reward:0.85\n", + "step: 3454, Red action: do-nothing, Blue reward:0.85\n", + "step: 3455, Red action: do-nothing, Blue reward:0.85\n", + "step: 3456, Red action: do-nothing, Blue reward:0.85\n", + "step: 3457, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3458, Red action: do-nothing, Blue reward:0.85\n", + "step: 3459, Red action: do-nothing, Blue reward:0.85\n", + "step: 3460, Red action: do-nothing, Blue reward:0.85\n", + "step: 3461, Red action: do-nothing, Blue reward:0.85\n", + "step: 3462, Red action: do-nothing, Blue reward:0.85\n", + "step: 3463, Red action: do-nothing, Blue reward:0.85\n", + "step: 3464, Red action: do-nothing, Blue reward:0.85\n", + "step: 3465, Red action: do-nothing, Blue reward:0.85\n", + "step: 3466, Red action: do-nothing, Blue reward:0.85\n", + "step: 3467, Red action: do-nothing, Blue reward:0.85\n", + "step: 3468, Red action: do-nothing, Blue reward:0.85\n", + "step: 3469, Red action: do-nothing, Blue reward:0.85\n", + "step: 3470, Red action: do-nothing, Blue reward:0.85\n", + "step: 3471, Red action: do-nothing, Blue reward:0.85\n", + "step: 3472, Red action: do-nothing, Blue reward:0.85\n", + "step: 3473, Red action: do-nothing, Blue reward:0.85\n", + "step: 3474, Red action: do-nothing, Blue reward:0.85\n", + "step: 3475, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3476, Red action: do-nothing, Blue reward:0.85\n", + "step: 3477, Red action: do-nothing, Blue reward:0.85\n", + "step: 3478, Red action: do-nothing, Blue reward:0.85\n", + "step: 3479, Red action: do-nothing, Blue reward:0.85\n", + "step: 3480, Red action: do-nothing, Blue reward:0.85\n", + "step: 3481, Red action: do-nothing, Blue reward:0.85\n", + "step: 3482, Red action: do-nothing, Blue reward:0.85\n", + "step: 3483, Red action: do-nothing, Blue reward:0.85\n", + "step: 3484, Red action: do-nothing, Blue reward:0.85\n", + "step: 3485, Red action: do-nothing, Blue reward:0.85\n", + "step: 3486, Red action: do-nothing, Blue reward:0.85\n", + "step: 3487, Red action: do-nothing, Blue reward:0.85\n", + "step: 3488, Red action: do-nothing, Blue reward:0.85\n", + "step: 3489, Red action: do-nothing, Blue reward:0.85\n", + "step: 3490, Red action: do-nothing, Blue reward:0.85\n", + "step: 3491, Red action: do-nothing, Blue reward:0.85\n", + "step: 3492, Red action: do-nothing, Blue reward:0.85\n", + "step: 3493, Red action: do-nothing, Blue reward:0.85\n", + "step: 3494, Red action: do-nothing, Blue reward:0.85\n", + "step: 3495, Red action: do-nothing, Blue reward:0.85\n", + "step: 3496, Red action: do-nothing, Blue reward:0.85\n", + "step: 3497, Red action: do-nothing, Blue reward:0.85\n", + "step: 3498, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3499, Red action: do-nothing, Blue reward:0.85\n", + "step: 3500, Red action: do-nothing, Blue reward:0.85\n", + "step: 3501, Red action: do-nothing, Blue reward:0.85\n", + "step: 3502, Red action: do-nothing, Blue reward:0.85\n", + "step: 3503, Red action: do-nothing, Blue reward:0.85\n", + "step: 3504, Red action: do-nothing, Blue reward:0.85\n", + "step: 3505, Red action: do-nothing, Blue reward:0.85\n", + "step: 3506, Red action: do-nothing, Blue reward:0.85\n", + "step: 3507, Red action: do-nothing, Blue reward:0.85\n", + "step: 3508, Red action: do-nothing, Blue reward:0.85\n", + "step: 3509, Red action: do-nothing, Blue reward:0.85\n", + "step: 3510, Red action: do-nothing, Blue reward:0.85\n", + "step: 3511, Red action: do-nothing, Blue reward:0.85\n", + "step: 3512, Red action: do-nothing, Blue reward:0.85\n", + "step: 3513, Red action: do-nothing, Blue reward:0.85\n", + "step: 3514, Red action: do-nothing, Blue reward:0.85\n", + "step: 3515, Red action: do-nothing, Blue reward:0.85\n", + "step: 3516, Red action: do-nothing, Blue reward:0.85\n", + "step: 3517, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3518, Red action: do-nothing, Blue reward:0.85\n", + "step: 3519, Red action: do-nothing, Blue reward:0.85\n", + "step: 3520, Red action: do-nothing, Blue reward:0.85\n", + "step: 3521, Red action: do-nothing, Blue reward:0.85\n", + "step: 3522, Red action: do-nothing, Blue reward:0.85\n", + "step: 3523, Red action: do-nothing, Blue reward:0.85\n", + "step: 3524, Red action: do-nothing, Blue reward:0.85\n", + "step: 3525, Red action: do-nothing, Blue reward:0.85\n", + "step: 3526, Red action: do-nothing, Blue reward:0.85\n", + "step: 3527, Red action: do-nothing, Blue reward:0.85\n", + "step: 3528, Red action: do-nothing, Blue reward:0.85\n", + "step: 3529, Red action: do-nothing, Blue reward:0.85\n", + "step: 3530, Red action: do-nothing, Blue reward:0.85\n", + "step: 3531, Red action: do-nothing, Blue reward:0.85\n", + "step: 3532, Red action: do-nothing, Blue reward:0.85\n", + "step: 3533, Red action: do-nothing, Blue reward:0.85\n", + "step: 3534, Red action: do-nothing, Blue reward:0.85\n", + "step: 3535, Red action: do-nothing, Blue reward:0.85\n", + "step: 3536, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3537, Red action: do-nothing, Blue reward:0.85\n", + "step: 3538, Red action: do-nothing, Blue reward:0.85\n", + "step: 3539, Red action: do-nothing, Blue reward:0.85\n", + "step: 3540, Red action: do-nothing, Blue reward:0.85\n", + "step: 3541, Red action: do-nothing, Blue reward:0.85\n", + "step: 3542, Red action: do-nothing, Blue reward:0.85\n", + "step: 3543, Red action: do-nothing, Blue reward:0.85\n", + "step: 3544, Red action: do-nothing, Blue reward:0.85\n", + "step: 3545, Red action: do-nothing, Blue reward:0.85\n", + "step: 3546, Red action: do-nothing, Blue reward:0.85\n", + "step: 3547, Red action: do-nothing, Blue reward:0.85\n", + "step: 3548, Red action: do-nothing, Blue reward:0.85\n", + "step: 3549, Red action: do-nothing, Blue reward:0.85\n", + "step: 3550, Red action: do-nothing, Blue reward:0.85\n", + "step: 3551, Red action: do-nothing, Blue reward:0.85\n", + "step: 3552, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3553, Red action: do-nothing, Blue reward:0.85\n", + "step: 3554, Red action: do-nothing, Blue reward:0.85\n", + "step: 3555, Red action: do-nothing, Blue reward:0.85\n", + "step: 3556, Red action: do-nothing, Blue reward:0.85\n", + "step: 3557, Red action: do-nothing, Blue reward:0.85\n", + "step: 3558, Red action: do-nothing, Blue reward:0.85\n", + "step: 3559, Red action: do-nothing, Blue reward:0.85\n", + "step: 3560, Red action: do-nothing, Blue reward:0.85\n", + "step: 3561, Red action: do-nothing, Blue reward:0.85\n", + "step: 3562, Red action: do-nothing, Blue reward:0.85\n", + "step: 3563, Red action: do-nothing, Blue reward:0.85\n", + "step: 3564, Red action: do-nothing, Blue reward:0.85\n", + "step: 3565, Red action: do-nothing, Blue reward:0.85\n", + "step: 3566, Red action: do-nothing, Blue reward:0.85\n", + "step: 3567, Red action: do-nothing, Blue reward:0.85\n", + "step: 3568, Red action: do-nothing, Blue reward:0.85\n", + "step: 3569, Red action: do-nothing, Blue reward:0.85\n", + "step: 3570, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3571, Red action: do-nothing, Blue reward:0.85\n", + "step: 3572, Red action: do-nothing, Blue reward:0.85\n", + "step: 3573, Red action: do-nothing, Blue reward:0.85\n", + "step: 3574, Red action: do-nothing, Blue reward:0.85\n", + "step: 3575, Red action: do-nothing, Blue reward:0.85\n", + "step: 3576, Red action: do-nothing, Blue reward:0.85\n", + "step: 3577, Red action: do-nothing, Blue reward:0.85\n", + "step: 3578, Red action: do-nothing, Blue reward:0.85\n", + "step: 3579, Red action: do-nothing, Blue reward:0.85\n", + "step: 3580, Red action: do-nothing, Blue reward:0.85\n", + "step: 3581, Red action: do-nothing, Blue reward:0.85\n", + "step: 3582, Red action: do-nothing, Blue reward:0.85\n", + "step: 3583, Red action: do-nothing, Blue reward:0.85\n", + "step: 3584, Red action: do-nothing, Blue reward:0.85\n", + "step: 3585, Red action: do-nothing, Blue reward:0.85\n", + "step: 3586, Red action: do-nothing, Blue reward:0.85\n", + "step: 3587, Red action: do-nothing, Blue reward:0.85\n", + "step: 3588, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3589, Red action: do-nothing, Blue reward:0.85\n", + "step: 3590, Red action: do-nothing, Blue reward:0.85\n", + "step: 3591, Red action: do-nothing, Blue reward:0.85\n", + "step: 3592, Red action: do-nothing, Blue reward:0.85\n", + "step: 3593, Red action: do-nothing, Blue reward:0.85\n", + "step: 3594, Red action: do-nothing, Blue reward:0.85\n", + "step: 3595, Red action: do-nothing, Blue reward:0.85\n", + "step: 3596, Red action: do-nothing, Blue reward:0.85\n", + "step: 3597, Red action: do-nothing, Blue reward:0.85\n", + "step: 3598, Red action: do-nothing, Blue reward:0.85\n", + "step: 3599, Red action: do-nothing, Blue reward:0.85\n", + "step: 3600, Red action: do-nothing, Blue reward:0.85\n", + "step: 3601, Red action: do-nothing, Blue reward:0.85\n", + "step: 3602, Red action: do-nothing, Blue reward:0.85\n", + "step: 3603, Red action: do-nothing, Blue reward:0.85\n", + "step: 3604, Red action: do-nothing, Blue reward:0.85\n", + "step: 3605, Red action: do-nothing, Blue reward:0.85\n", + "step: 3606, Red action: do-nothing, Blue reward:0.85\n", + "step: 3607, Red action: do-nothing, Blue reward:0.85\n", + "step: 3608, Red action: do-nothing, Blue reward:0.85\n", + "step: 3609, Red action: do-nothing, Blue reward:0.85\n", + "step: 3610, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3611, Red action: do-nothing, Blue reward:0.85\n", + "step: 3612, Red action: do-nothing, Blue reward:0.85\n", + "step: 3613, Red action: do-nothing, Blue reward:0.85\n", + "step: 3614, Red action: do-nothing, Blue reward:0.85\n", + "step: 3615, Red action: do-nothing, Blue reward:0.85\n", + "step: 3616, Red action: do-nothing, Blue reward:0.85\n", + "step: 3617, Red action: do-nothing, Blue reward:0.85\n", + "step: 3618, Red action: do-nothing, Blue reward:0.85\n", + "step: 3619, Red action: do-nothing, Blue reward:0.85\n", + "step: 3620, Red action: do-nothing, Blue reward:0.85\n", + "step: 3621, Red action: do-nothing, Blue reward:0.85\n", + "step: 3622, Red action: do-nothing, Blue reward:0.85\n", + "step: 3623, Red action: do-nothing, Blue reward:0.85\n", + "step: 3624, Red action: do-nothing, Blue reward:0.85\n", + "step: 3625, Red action: do-nothing, Blue reward:0.85\n", + "step: 3626, Red action: do-nothing, Blue reward:0.85\n", + "step: 3627, Red action: do-nothing, Blue reward:0.85\n", + "step: 3628, Red action: do-nothing, Blue reward:0.85\n", + "step: 3629, Red action: do-nothing, Blue reward:0.85\n", + "step: 3630, Red action: do-nothing, Blue reward:0.85\n", + "step: 3631, Red action: do-nothing, Blue reward:0.85\n", + "step: 3632, Red action: do-nothing, Blue reward:0.85\n", + "step: 3633, Red action: do-nothing, Blue reward:0.85\n", + "step: 3634, Red action: do-nothing, Blue reward:0.85\n", + "step: 3635, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3636, Red action: do-nothing, Blue reward:0.85\n", + "step: 3637, Red action: do-nothing, Blue reward:0.85\n", + "step: 3638, Red action: do-nothing, Blue reward:0.85\n", + "step: 3639, Red action: do-nothing, Blue reward:0.85\n", + "step: 3640, Red action: do-nothing, Blue reward:0.85\n", + "step: 3641, Red action: do-nothing, Blue reward:0.85\n", + "step: 3642, Red action: do-nothing, Blue reward:0.85\n", + "step: 3643, Red action: do-nothing, Blue reward:0.85\n", + "step: 3644, Red action: do-nothing, Blue reward:0.85\n", + "step: 3645, Red action: do-nothing, Blue reward:0.85\n", + "step: 3646, Red action: do-nothing, Blue reward:0.85\n", + "step: 3647, Red action: do-nothing, Blue reward:0.85\n", + "step: 3648, Red action: do-nothing, Blue reward:0.85\n", + "step: 3649, Red action: do-nothing, Blue reward:0.85\n", + "step: 3650, Red action: do-nothing, Blue reward:0.85\n", + "step: 3651, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3652, Red action: do-nothing, Blue reward:0.85\n", + "step: 3653, Red action: do-nothing, Blue reward:0.85\n", + "step: 3654, Red action: do-nothing, Blue reward:0.85\n", + "step: 3655, Red action: do-nothing, Blue reward:0.85\n", + "step: 3656, Red action: do-nothing, Blue reward:0.85\n", + "step: 3657, Red action: do-nothing, Blue reward:0.85\n", + "step: 3658, Red action: do-nothing, Blue reward:0.85\n", + "step: 3659, Red action: do-nothing, Blue reward:0.85\n", + "step: 3660, Red action: do-nothing, Blue reward:0.85\n", + "step: 3661, Red action: do-nothing, Blue reward:0.85\n", + "step: 3662, Red action: do-nothing, Blue reward:0.85\n", + "step: 3663, Red action: do-nothing, Blue reward:0.85\n", + "step: 3664, Red action: do-nothing, Blue reward:0.85\n", + "step: 3665, Red action: do-nothing, Blue reward:0.85\n", + "step: 3666, Red action: do-nothing, Blue reward:0.85\n", + "step: 3667, Red action: do-nothing, Blue reward:0.85\n", + "step: 3668, Red action: do-nothing, Blue reward:0.85\n", + "step: 3669, Red action: do-nothing, Blue reward:0.85\n", + "step: 3670, Red action: do-nothing, Blue reward:0.85\n", + "step: 3671, Red action: do-nothing, Blue reward:0.85\n", + "step: 3672, Red action: do-nothing, Blue reward:0.85\n", + "step: 3673, Red action: do-nothing, Blue reward:0.85\n", + "step: 3674, Red action: do-nothing, Blue reward:0.85\n", + "step: 3675, Red action: do-nothing, Blue reward:0.85\n", + "step: 3676, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3677, Red action: do-nothing, Blue reward:0.85\n", + "step: 3678, Red action: do-nothing, Blue reward:0.85\n", + "step: 3679, Red action: do-nothing, Blue reward:0.85\n", + "step: 3680, Red action: do-nothing, Blue reward:0.85\n", + "step: 3681, Red action: do-nothing, Blue reward:0.85\n", + "step: 3682, Red action: do-nothing, Blue reward:0.85\n", + "step: 3683, Red action: do-nothing, Blue reward:0.85\n", + "step: 3684, Red action: do-nothing, Blue reward:0.85\n", + "step: 3685, Red action: do-nothing, Blue reward:0.85\n", + "step: 3686, Red action: do-nothing, Blue reward:0.85\n", + "step: 3687, Red action: do-nothing, Blue reward:0.85\n", + "step: 3688, Red action: do-nothing, Blue reward:0.85\n", + "step: 3689, Red action: do-nothing, Blue reward:0.85\n", + "step: 3690, Red action: do-nothing, Blue reward:0.85\n", + "step: 3691, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3692, Red action: do-nothing, Blue reward:0.85\n", + "step: 3693, Red action: do-nothing, Blue reward:0.85\n", + "step: 3694, Red action: do-nothing, Blue reward:0.85\n", + "step: 3695, Red action: do-nothing, Blue reward:0.85\n", + "step: 3696, Red action: do-nothing, Blue reward:0.85\n", + "step: 3697, Red action: do-nothing, Blue reward:0.85\n", + "step: 3698, Red action: do-nothing, Blue reward:0.85\n", + "step: 3699, Red action: do-nothing, Blue reward:0.85\n", + "step: 3700, Red action: do-nothing, Blue reward:0.85\n", + "step: 3701, Red action: do-nothing, Blue reward:0.85\n", + "step: 3702, Red action: do-nothing, Blue reward:0.85\n", + "step: 3703, Red action: do-nothing, Blue reward:0.85\n", + "step: 3704, Red action: do-nothing, Blue reward:0.85\n", + "step: 3705, Red action: do-nothing, Blue reward:0.85\n", + "step: 3706, Red action: do-nothing, Blue reward:0.85\n", + "step: 3707, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3708, Red action: do-nothing, Blue reward:0.85\n", + "step: 3709, Red action: do-nothing, Blue reward:0.85\n", + "step: 3710, Red action: do-nothing, Blue reward:0.85\n", + "step: 3711, Red action: do-nothing, Blue reward:0.85\n", + "step: 3712, Red action: do-nothing, Blue reward:0.85\n", + "step: 3713, Red action: do-nothing, Blue reward:0.85\n", + "step: 3714, Red action: do-nothing, Blue reward:0.85\n", + "step: 3715, Red action: do-nothing, Blue reward:0.85\n", + "step: 3716, Red action: do-nothing, Blue reward:0.85\n", + "step: 3717, Red action: do-nothing, Blue reward:0.85\n", + "step: 3718, Red action: do-nothing, Blue reward:0.85\n", + "step: 3719, Red action: do-nothing, Blue reward:0.85\n", + "step: 3720, Red action: do-nothing, Blue reward:0.85\n", + "step: 3721, Red action: do-nothing, Blue reward:0.85\n", + "step: 3722, Red action: do-nothing, Blue reward:0.85\n", + "step: 3723, Red action: do-nothing, Blue reward:0.85\n", + "step: 3724, Red action: do-nothing, Blue reward:0.85\n", + "step: 3725, Red action: do-nothing, Blue reward:0.85\n", + "step: 3726, Red action: do-nothing, Blue reward:0.85\n", + "step: 3727, Red action: do-nothing, Blue reward:0.85\n", + "step: 3728, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3729, Red action: do-nothing, Blue reward:0.85\n", + "step: 3730, Red action: do-nothing, Blue reward:0.85\n", + "step: 3731, Red action: do-nothing, Blue reward:0.85\n", + "step: 3732, Red action: do-nothing, Blue reward:0.85\n", + "step: 3733, Red action: do-nothing, Blue reward:0.85\n", + "step: 3734, Red action: do-nothing, Blue reward:0.85\n", + "step: 3735, Red action: do-nothing, Blue reward:0.85\n", + "step: 3736, Red action: do-nothing, Blue reward:0.85\n", + "step: 3737, Red action: do-nothing, Blue reward:0.85\n", + "step: 3738, Red action: do-nothing, Blue reward:0.85\n", + "step: 3739, Red action: do-nothing, Blue reward:0.85\n", + "step: 3740, Red action: do-nothing, Blue reward:0.85\n", + "step: 3741, Red action: do-nothing, Blue reward:0.85\n", + "step: 3742, Red action: do-nothing, Blue reward:0.85\n", + "step: 3743, Red action: do-nothing, Blue reward:0.85\n", + "step: 3744, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3745, Red action: do-nothing, Blue reward:0.85\n", + "step: 3746, Red action: do-nothing, Blue reward:0.85\n", + "step: 3747, Red action: do-nothing, Blue reward:0.85\n", + "step: 3748, Red action: do-nothing, Blue reward:0.85\n", + "step: 3749, Red action: do-nothing, Blue reward:0.85\n", + "step: 3750, Red action: do-nothing, Blue reward:0.85\n", + "step: 3751, Red action: do-nothing, Blue reward:0.85\n", + "step: 3752, Red action: do-nothing, Blue reward:0.85\n", + "step: 3753, Red action: do-nothing, Blue reward:0.85\n", + "step: 3754, Red action: do-nothing, Blue reward:0.85\n", + "step: 3755, Red action: do-nothing, Blue reward:0.85\n", + "step: 3756, Red action: do-nothing, Blue reward:0.85\n", + "step: 3757, Red action: do-nothing, Blue reward:0.85\n", + "step: 3758, Red action: do-nothing, Blue reward:0.85\n", + "step: 3759, Red action: do-nothing, Blue reward:0.85\n", + "step: 3760, Red action: do-nothing, Blue reward:0.85\n", + "step: 3761, Red action: do-nothing, Blue reward:0.85\n", + "step: 3762, Red action: do-nothing, Blue reward:0.85\n", + "step: 3763, Red action: do-nothing, Blue reward:0.85\n", + "step: 3764, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3765, Red action: do-nothing, Blue reward:0.85\n", + "step: 3766, Red action: do-nothing, Blue reward:0.85\n", + "step: 3767, Red action: do-nothing, Blue reward:0.85\n", + "step: 3768, Red action: do-nothing, Blue reward:0.85\n", + "step: 3769, Red action: do-nothing, Blue reward:0.85\n", + "step: 3770, Red action: do-nothing, Blue reward:0.85\n", + "step: 3771, Red action: do-nothing, Blue reward:0.85\n", + "step: 3772, Red action: do-nothing, Blue reward:0.85\n", + "step: 3773, Red action: do-nothing, Blue reward:0.85\n", + "step: 3774, Red action: do-nothing, Blue reward:0.85\n", + "step: 3775, Red action: do-nothing, Blue reward:0.85\n", + "step: 3776, Red action: do-nothing, Blue reward:0.85\n", + "step: 3777, Red action: do-nothing, Blue reward:0.85\n", + "step: 3778, Red action: do-nothing, Blue reward:0.85\n", + "step: 3779, Red action: do-nothing, Blue reward:0.85\n", + "step: 3780, Red action: do-nothing, Blue reward:0.85\n", + "step: 3781, Red action: do-nothing, Blue reward:0.85\n", + "step: 3782, Red action: do-nothing, Blue reward:0.85\n", + "step: 3783, Red action: do-nothing, Blue reward:0.85\n", + "step: 3784, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3785, Red action: do-nothing, Blue reward:0.85\n", + "step: 3786, Red action: do-nothing, Blue reward:0.85\n", + "step: 3787, Red action: do-nothing, Blue reward:0.85\n", + "step: 3788, Red action: do-nothing, Blue reward:0.85\n", + "step: 3789, Red action: do-nothing, Blue reward:0.85\n", + "step: 3790, Red action: do-nothing, Blue reward:0.85\n", + "step: 3791, Red action: do-nothing, Blue reward:0.85\n", + "step: 3792, Red action: do-nothing, Blue reward:0.85\n", + "step: 3793, Red action: do-nothing, Blue reward:0.85\n", + "step: 3794, Red action: do-nothing, Blue reward:0.85\n", + "step: 3795, Red action: do-nothing, Blue reward:0.85\n", + "step: 3796, Red action: do-nothing, Blue reward:0.85\n", + "step: 3797, Red action: do-nothing, Blue reward:0.85\n", + "step: 3798, Red action: do-nothing, Blue reward:0.85\n", + "step: 3799, Red action: do-nothing, Blue reward:0.85\n", + "step: 3800, Red action: do-nothing, Blue reward:0.85\n", + "step: 3801, Red action: do-nothing, Blue reward:0.85\n", + "step: 3802, Red action: do-nothing, Blue reward:0.85\n" + ] + }, + { + "ename": "KeyboardInterrupt", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[11], line 11\u001b[0m\n\u001b[1;32m 8\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mstep: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00menv\u001b[38;5;241m.\u001b[39mgame\u001b[38;5;241m.\u001b[39mstep_counter\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m, Red action: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00minfo[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124magent_actions\u001b[39m\u001b[38;5;124m'\u001b[39m][\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mdata_manipulation_attacker\u001b[39m\u001b[38;5;124m'\u001b[39m]\u001b[38;5;241m.\u001b[39maction\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m, Blue reward:\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mreward\u001b[38;5;132;01m:\u001b[39;00m\u001b[38;5;124m.2f\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m )\n\u001b[1;32m 10\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28mabs\u001b[39m(reward \u001b[38;5;241m-\u001b[39m \u001b[38;5;241m0.8\u001b[39m) \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m1e-5\u001b[39m:\n\u001b[0;32m---> 11\u001b[0m obs, reward, terminated, truncated, info \u001b[38;5;241m=\u001b[39m \u001b[43menv\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstep\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m)\u001b[49m \u001b[38;5;66;03m# do nothing\u001b[39;00m\n\u001b[1;32m 12\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mstep: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00menv\u001b[38;5;241m.\u001b[39mgame\u001b[38;5;241m.\u001b[39mstep_counter\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m, Red action: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00minfo[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124magent_actions\u001b[39m\u001b[38;5;124m'\u001b[39m][\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mdata_manipulation_attacker\u001b[39m\u001b[38;5;124m'\u001b[39m]\u001b[38;5;241m.\u001b[39maction\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m, Blue reward:\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mreward\u001b[38;5;132;01m:\u001b[39;00m\u001b[38;5;124m.2f\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m )\n\u001b[1;32m 13\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m env\u001b[38;5;241m.\u001b[39mgame\u001b[38;5;241m.\u001b[39mstep_counter \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m10000\u001b[39m:\n", + "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/session/environment.py:111\u001b[0m, in \u001b[0;36mPrimaiteGymEnv.step\u001b[0;34m(self, action)\u001b[0m\n\u001b[1;32m 109\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgame\u001b[38;5;241m.\u001b[39mapply_agent_actions()\n\u001b[1;32m 110\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgame\u001b[38;5;241m.\u001b[39madvance_timestep()\n\u001b[0;32m--> 111\u001b[0m state \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgame\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_sim_state\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 112\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgame\u001b[38;5;241m.\u001b[39mupdate_agents(state)\n\u001b[1;32m 114\u001b[0m next_obs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_get_obs() \u001b[38;5;66;03m# this doesn't update observation, just gets the current observation\u001b[39;00m\n", + "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/game/game.py:151\u001b[0m, in \u001b[0;36mPrimaiteGame.get_sim_state\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 149\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mget_sim_state\u001b[39m(\u001b[38;5;28mself\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Dict:\n\u001b[1;32m 150\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"Get the current state of the simulation.\"\"\"\u001b[39;00m\n\u001b[0;32m--> 151\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msimulation\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdescribe_state\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/simulator/sim_container.py:57\u001b[0m, in \u001b[0;36mSimulation.describe_state\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 46\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 47\u001b[0m \u001b[38;5;124;03mProduce a dictionary describing the current state of this object.\u001b[39;00m\n\u001b[1;32m 48\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 52\u001b[0m \u001b[38;5;124;03m:rtype: Dict\u001b[39;00m\n\u001b[1;32m 53\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 54\u001b[0m state \u001b[38;5;241m=\u001b[39m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39mdescribe_state()\n\u001b[1;32m 55\u001b[0m state\u001b[38;5;241m.\u001b[39mupdate(\n\u001b[1;32m 56\u001b[0m {\n\u001b[0;32m---> 57\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnetwork\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mnetwork\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdescribe_state\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m,\n\u001b[1;32m 58\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdomain\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdomain\u001b[38;5;241m.\u001b[39mdescribe_state(),\n\u001b[1;32m 59\u001b[0m }\n\u001b[1;32m 60\u001b[0m )\n\u001b[1;32m 61\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m state\n", + "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/simulator/network/container.py:260\u001b[0m, in \u001b[0;36mNetwork.describe_state\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 252\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 253\u001b[0m \u001b[38;5;124;03mProduce a dictionary describing the current state of the Network.\u001b[39;00m\n\u001b[1;32m 254\u001b[0m \n\u001b[1;32m 255\u001b[0m \u001b[38;5;124;03m:return: A dictionary capturing the current state of the Network and its child objects.\u001b[39;00m\n\u001b[1;32m 256\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 257\u001b[0m state \u001b[38;5;241m=\u001b[39m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39mdescribe_state()\n\u001b[1;32m 258\u001b[0m state\u001b[38;5;241m.\u001b[39mupdate(\n\u001b[1;32m 259\u001b[0m {\n\u001b[0;32m--> 260\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnodes\u001b[39m\u001b[38;5;124m\"\u001b[39m: {node\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mhostname: node\u001b[38;5;241m.\u001b[39mdescribe_state() \u001b[38;5;28;01mfor\u001b[39;00m node \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnodes\u001b[38;5;241m.\u001b[39mvalues()},\n\u001b[1;32m 261\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mlinks\u001b[39m\u001b[38;5;124m\"\u001b[39m: {},\n\u001b[1;32m 262\u001b[0m }\n\u001b[1;32m 263\u001b[0m )\n\u001b[1;32m 264\u001b[0m \u001b[38;5;66;03m# Update the links one-by-one. The key is a 4-tuple of `hostname_a, port_a, hostname_b, port_b`\u001b[39;00m\n\u001b[1;32m 265\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m _, link \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlinks\u001b[38;5;241m.\u001b[39mitems():\n", + "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/simulator/network/container.py:260\u001b[0m, in \u001b[0;36m\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 252\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 253\u001b[0m \u001b[38;5;124;03mProduce a dictionary describing the current state of the Network.\u001b[39;00m\n\u001b[1;32m 254\u001b[0m \n\u001b[1;32m 255\u001b[0m \u001b[38;5;124;03m:return: A dictionary capturing the current state of the Network and its child objects.\u001b[39;00m\n\u001b[1;32m 256\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 257\u001b[0m state \u001b[38;5;241m=\u001b[39m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39mdescribe_state()\n\u001b[1;32m 258\u001b[0m state\u001b[38;5;241m.\u001b[39mupdate(\n\u001b[1;32m 259\u001b[0m {\n\u001b[0;32m--> 260\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnodes\u001b[39m\u001b[38;5;124m\"\u001b[39m: {node\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mhostname: \u001b[43mnode\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdescribe_state\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mfor\u001b[39;00m node \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnodes\u001b[38;5;241m.\u001b[39mvalues()},\n\u001b[1;32m 261\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mlinks\u001b[39m\u001b[38;5;124m\"\u001b[39m: {},\n\u001b[1;32m 262\u001b[0m }\n\u001b[1;32m 263\u001b[0m )\n\u001b[1;32m 264\u001b[0m \u001b[38;5;66;03m# Update the links one-by-one. The key is a 4-tuple of `hostname_a, port_a, hostname_b, port_b`\u001b[39;00m\n\u001b[1;32m 265\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m _, link \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlinks\u001b[38;5;241m.\u001b[39mitems():\n", + "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/simulator/network/hardware/base.py:1900\u001b[0m, in \u001b[0;36mNode.describe_state\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1881\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 1882\u001b[0m \u001b[38;5;124;03mProduce a dictionary describing the current state of this object.\u001b[39;00m\n\u001b[1;32m 1883\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1887\u001b[0m \u001b[38;5;124;03m:rtype: Dict\u001b[39;00m\n\u001b[1;32m 1888\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 1889\u001b[0m state \u001b[38;5;241m=\u001b[39m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39mdescribe_state()\n\u001b[1;32m 1890\u001b[0m state\u001b[38;5;241m.\u001b[39mupdate(\n\u001b[1;32m 1891\u001b[0m {\n\u001b[1;32m 1892\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhostname\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mhostname,\n\u001b[1;32m 1893\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moperating_state\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39moperating_state\u001b[38;5;241m.\u001b[39mvalue,\n\u001b[1;32m 1894\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mNICs\u001b[39m\u001b[38;5;124m\"\u001b[39m: {\n\u001b[1;32m 1895\u001b[0m eth_num: network_interface\u001b[38;5;241m.\u001b[39mdescribe_state()\n\u001b[1;32m 1896\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m eth_num, network_interface \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnetwork_interface\u001b[38;5;241m.\u001b[39mitems()\n\u001b[1;32m 1897\u001b[0m },\n\u001b[1;32m 1898\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfile_system\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfile_system\u001b[38;5;241m.\u001b[39mdescribe_state(),\n\u001b[1;32m 1899\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mapplications\u001b[39m\u001b[38;5;124m\"\u001b[39m: {app\u001b[38;5;241m.\u001b[39mname: app\u001b[38;5;241m.\u001b[39mdescribe_state() \u001b[38;5;28;01mfor\u001b[39;00m app \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mapplications\u001b[38;5;241m.\u001b[39mvalues()},\n\u001b[0;32m-> 1900\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mservices\u001b[39m\u001b[38;5;124m\"\u001b[39m: {svc\u001b[38;5;241m.\u001b[39mname: svc\u001b[38;5;241m.\u001b[39mdescribe_state() \u001b[38;5;28;01mfor\u001b[39;00m svc \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mservices\u001b[38;5;241m.\u001b[39mvalues()},\n\u001b[1;32m 1901\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mprocess\u001b[39m\u001b[38;5;124m\"\u001b[39m: {proc\u001b[38;5;241m.\u001b[39mname: proc\u001b[38;5;241m.\u001b[39mdescribe_state() \u001b[38;5;28;01mfor\u001b[39;00m proc \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprocesses\u001b[38;5;241m.\u001b[39mvalues()},\n\u001b[1;32m 1902\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mrevealed_to_red\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mrevealed_to_red,\n\u001b[1;32m 1903\u001b[0m }\n\u001b[1;32m 1904\u001b[0m )\n\u001b[1;32m 1905\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m state\n", + "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/simulator/network/hardware/base.py:1900\u001b[0m, in \u001b[0;36m\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 1881\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 1882\u001b[0m \u001b[38;5;124;03mProduce a dictionary describing the current state of this object.\u001b[39;00m\n\u001b[1;32m 1883\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1887\u001b[0m \u001b[38;5;124;03m:rtype: Dict\u001b[39;00m\n\u001b[1;32m 1888\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 1889\u001b[0m state \u001b[38;5;241m=\u001b[39m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39mdescribe_state()\n\u001b[1;32m 1890\u001b[0m state\u001b[38;5;241m.\u001b[39mupdate(\n\u001b[1;32m 1891\u001b[0m {\n\u001b[1;32m 1892\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhostname\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mhostname,\n\u001b[1;32m 1893\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moperating_state\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39moperating_state\u001b[38;5;241m.\u001b[39mvalue,\n\u001b[1;32m 1894\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mNICs\u001b[39m\u001b[38;5;124m\"\u001b[39m: {\n\u001b[1;32m 1895\u001b[0m eth_num: network_interface\u001b[38;5;241m.\u001b[39mdescribe_state()\n\u001b[1;32m 1896\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m eth_num, network_interface \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnetwork_interface\u001b[38;5;241m.\u001b[39mitems()\n\u001b[1;32m 1897\u001b[0m },\n\u001b[1;32m 1898\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfile_system\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfile_system\u001b[38;5;241m.\u001b[39mdescribe_state(),\n\u001b[1;32m 1899\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mapplications\u001b[39m\u001b[38;5;124m\"\u001b[39m: {app\u001b[38;5;241m.\u001b[39mname: app\u001b[38;5;241m.\u001b[39mdescribe_state() \u001b[38;5;28;01mfor\u001b[39;00m app \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mapplications\u001b[38;5;241m.\u001b[39mvalues()},\n\u001b[0;32m-> 1900\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mservices\u001b[39m\u001b[38;5;124m\"\u001b[39m: {svc\u001b[38;5;241m.\u001b[39mname: \u001b[43msvc\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdescribe_state\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mfor\u001b[39;00m svc \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mservices\u001b[38;5;241m.\u001b[39mvalues()},\n\u001b[1;32m 1901\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mprocess\u001b[39m\u001b[38;5;124m\"\u001b[39m: {proc\u001b[38;5;241m.\u001b[39mname: proc\u001b[38;5;241m.\u001b[39mdescribe_state() \u001b[38;5;28;01mfor\u001b[39;00m proc \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprocesses\u001b[38;5;241m.\u001b[39mvalues()},\n\u001b[1;32m 1902\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mrevealed_to_red\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mrevealed_to_red,\n\u001b[1;32m 1903\u001b[0m }\n\u001b[1;32m 1904\u001b[0m )\n\u001b[1;32m 1905\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m state\n", + "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/simulator/system/services/ntp/ntp_client.py:61\u001b[0m, in \u001b[0;36mNTPClient.describe_state\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 50\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdescribe_state\u001b[39m(\u001b[38;5;28mself\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Dict:\n\u001b[1;32m 51\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 52\u001b[0m \u001b[38;5;124;03m Describes the current state of the software.\u001b[39;00m\n\u001b[1;32m 53\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 59\u001b[0m \u001b[38;5;124;03m :rtype: Dict\u001b[39;00m\n\u001b[1;32m 60\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m---> 61\u001b[0m state \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241m.\u001b[39mdescribe_state()\n\u001b[1;32m 62\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m state\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + ] + } + ], "source": [ "env.step(13) # Patch the database\n", "print(f\"step: {env.game.step_counter}, Red action: {info['agent_actions']['data_manipulation_attacker'].action}, Blue reward:{reward:.2f}\" )\n", @@ -600,7 +5091,7 @@ "while abs(reward - 0.8) > 1e-5:\n", " obs, reward, terminated, truncated, info = env.step(0) # do nothing\n", " print(f\"step: {env.game.step_counter}, Red action: {info['agent_actions']['data_manipulation_attacker'].action}, Blue reward:{reward:.2f}\" )\n", - " if env.game.step_counter > 10000:\n", + " if env.game.step_counter > 2000:\n", " break # make sure there's no infinite loop if something went wrong" ] }, From 7dafec85176130dc3c9ce66bf4dbe8ec3b2faa7e Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Mon, 17 Feb 2025 15:01:02 +0000 Subject: [PATCH 81/95] #3075: Test fix --- .../game_layer/actions/test_terminal_actions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration_tests/game_layer/actions/test_terminal_actions.py b/tests/integration_tests/game_layer/actions/test_terminal_actions.py index c39d8263..af9c7b74 100644 --- a/tests/integration_tests/game_layer/actions/test_terminal_actions.py +++ b/tests/integration_tests/game_layer/actions/test_terminal_actions.py @@ -106,7 +106,7 @@ def test_remote_login_change_password(game_and_agent_fixture: Tuple[PrimaiteGame "username": "user123", "current_password": "password", "new_password": "different_password", - "remote_ip": str(server_1.network_interface[1].ip_address), + # "remote_ip": str(server_1.network_interface[1].ip_address), }, ) agent.store_action(action) @@ -146,7 +146,7 @@ def test_change_password_logs_out_user(game_and_agent_fixture: Tuple[PrimaiteGam "username": "user123", "current_password": "password", "new_password": "different_password", - "remote_ip": str(server_1.network_interface[1].ip_address), + # "remote_ip": str(server_1.network_interface[1].ip_address), }, ) agent.store_action(action) From 5f076ba225bf239a61377478949e8b474b7504d7 Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Mon, 17 Feb 2025 15:12:12 +0000 Subject: [PATCH 82/95] #3075: Fix pre-commit errors --- .../Data-Manipulation-E2E-Demonstration.ipynb | 4527 +---------------- 1 file changed, 18 insertions(+), 4509 deletions(-) diff --git a/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb b/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb index f3e3fb98..31e1f9d3 100644 --- a/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb +++ b/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb @@ -373,29 +373,16 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-02-17 11:15:30,959: Performing the PrimAITE first-time setup...\n", - "2025-02-17 11:15:30,960: Building the PrimAITE app directories...\n", - "2025-02-17 11:15:30,960: Building primaite_config.yaml...\n", - "2025-02-17 11:15:30,960: Rebuilding the demo notebooks...\n", - "2025-02-17 11:15:30,967: Rebuilding the example notebooks...\n", - "2025-02-17 11:15:30,970: PrimAITE setup complete!\n" - ] - } - ], + "outputs": [], "source": [ "!primaite setup" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": { "tags": [] }, @@ -407,7 +394,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": { "tags": [] }, @@ -433,254 +420,9 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-17 11:15:35,108: PrimaiteGymEnv RNG seed = None\n", - "2025-02-17 11:15:35,109: Resetting environment, episode 0, avg. reward: 0.0\n", - "2025-02-17 11:15:35,111: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-17/11-15-31/agent_actions/episode_0.json\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "env created successfully\n", - "{'ICS': 0,\n", - " 'LINKS': {1: {'PROTOCOLS': {'ALL': 0}},\n", - " 2: {'PROTOCOLS': {'ALL': 0}},\n", - " 3: {'PROTOCOLS': {'ALL': 0}},\n", - " 4: {'PROTOCOLS': {'ALL': 0}},\n", - " 5: {'PROTOCOLS': {'ALL': 0}},\n", - " 6: {'PROTOCOLS': {'ALL': 0}},\n", - " 7: {'PROTOCOLS': {'ALL': 0}},\n", - " 8: {'PROTOCOLS': {'ALL': 0}},\n", - " 9: {'PROTOCOLS': {'ALL': 0}},\n", - " 10: {'PROTOCOLS': {'ALL': 0}}},\n", - " 'NODES': {'HOST0': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0,\n", - " 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0,\n", - " 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0,\n", - " 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0,\n", - " 'outbound': 0}}},\n", - " 'nic_status': 0}},\n", - " 'SERVICES': {1: {'health_status': 0,\n", - " 'operating_status': 0}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'HOST1': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0,\n", - " 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0,\n", - " 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0,\n", - " 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0,\n", - " 'outbound': 0}}},\n", - " 'nic_status': 0}},\n", - " 'SERVICES': {1: {'health_status': 0,\n", - " 'operating_status': 1}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'HOST2': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0,\n", - " 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0,\n", - " 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0,\n", - " 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0,\n", - " 'outbound': 0}}},\n", - " 'nic_status': 0}},\n", - " 'SERVICES': {1: {'health_status': 0,\n", - " 'operating_status': 0}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'HOST3': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0,\n", - " 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0,\n", - " 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0,\n", - " 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0,\n", - " 'outbound': 0}}},\n", - " 'nic_status': 0}},\n", - " 'SERVICES': {1: {'health_status': 0,\n", - " 'operating_status': 0}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'HOST4': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0,\n", - " 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0,\n", - " 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0,\n", - " 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0,\n", - " 'outbound': 0}}},\n", - " 'nic_status': 1}},\n", - " 'SERVICES': {1: {'health_status': 0,\n", - " 'operating_status': 0}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'HOST5': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0,\n", - " 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0,\n", - " 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0,\n", - " 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0,\n", - " 'outbound': 0}}},\n", - " 'nic_status': 0}},\n", - " 'SERVICES': {1: {'health_status': 0,\n", - " 'operating_status': 0}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'HOST6': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0,\n", - " 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0,\n", - " 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0,\n", - " 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0,\n", - " 'outbound': 0}}},\n", - " 'nic_status': 0}},\n", - " 'SERVICES': {1: {'health_status': 0,\n", - " 'operating_status': 0}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'ROUTER0': {'ACL': {1: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 0,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 2: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 1,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 3: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 2,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 4: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 3,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 5: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 4,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 6: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 5,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 7: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 6,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 8: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 7,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 9: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 8,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 10: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 9,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0}},\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}}}}\n" - ] - } - ], + "outputs": [], "source": [ "# create the env\n", "with open(data_manipulation_config_path(), 'r') as f:\n", @@ -708,7 +450,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -726,51 +468,9 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "step: 1, Red action: DO NOTHING, Blue reward:0.65\n", - "step: 2, Red action: DO NOTHING, Blue reward:0.65\n", - "step: 3, Red action: DO NOTHING, Blue reward:0.65\n", - "step: 4, Red action: DO NOTHING, Blue reward:0.90\n", - "step: 5, Red action: DO NOTHING, Blue reward:0.90\n", - "step: 6, Red action: DO NOTHING, Blue reward:0.90\n", - "step: 7, Red action: DO NOTHING, Blue reward:0.90\n", - "step: 8, Red action: DO NOTHING, Blue reward:0.90\n", - "step: 9, Red action: DO NOTHING, Blue reward:0.90\n", - "step: 10, Red action: DO NOTHING, Blue reward:0.90\n", - "step: 11, Red action: DO NOTHING, Blue reward:0.90\n", - "step: 12, Red action: DO NOTHING, Blue reward:0.90\n", - "step: 13, Red action: DO NOTHING, Blue reward:0.90\n", - "step: 14, Red action: DO NOTHING, Blue reward:0.90\n", - "step: 15, Red action: DO NOTHING, Blue reward:0.90\n", - "step: 16, Red action: DO NOTHING, Blue reward:0.90\n", - "step: 17, Red action: DO NOTHING, Blue reward:0.95\n", - "step: 18, Red action: DO NOTHING, Blue reward:0.95\n", - "step: 19, Red action: DO NOTHING, Blue reward:0.95\n", - "step: 20, Red action: DO NOTHING, Blue reward:0.95\n", - "step: 21, Red action: DO NOTHING, Blue reward:0.95\n", - "step: 22, Red action: DO NOTHING, Blue reward:0.95\n", - "step: 23, Red action: DO NOTHING, Blue reward:0.95\n", - "step: 24, Red action: DO NOTHING, Blue reward:0.95\n", - "step: 25, Red action: DO NOTHING, Blue reward:0.95\n", - "step: 26, Red action: ATTACK from client 2, Blue reward:0.15\n", - "step: 27, Red action: DO NOTHING, Blue reward:-0.35\n", - "step: 28, Red action: DO NOTHING, Blue reward:-0.85\n", - "step: 29, Red action: DO NOTHING, Blue reward:-0.85\n", - "step: 30, Red action: DO NOTHING, Blue reward:-0.85\n", - "step: 31, Red action: DO NOTHING, Blue reward:-0.85\n", - "step: 32, Red action: DO NOTHING, Blue reward:-0.85\n", - "step: 33, Red action: DO NOTHING, Blue reward:-0.85\n", - "step: 34, Red action: DO NOTHING, Blue reward:-0.85\n", - "step: 35, Red action: DO NOTHING, Blue reward:-0.85\n" - ] - } - ], + "outputs": [], "source": [ "for step in range(35):\n", " obs, reward, terminated, truncated, info = env.step(0)\n", @@ -786,198 +486,9 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'HOST0': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 0}},\n", - " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'HOST1': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 0}},\n", - " 'SERVICES': {1: {'health_status': 0, 'operating_status': 1}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'HOST2': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 0}},\n", - " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'HOST3': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 0}},\n", - " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'HOST4': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 1}},\n", - " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'HOST5': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 0}},\n", - " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'HOST6': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 0}},\n", - " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'ROUTER0': {'ACL': {1: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 0,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 2: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 1,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 3: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 2,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 4: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 3,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 5: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 4,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 6: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 5,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 7: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 6,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 8: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 7,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 9: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 8,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 10: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 9,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0}},\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}}}\n" - ] - } - ], + "outputs": [], "source": [ "pprint(obs['NODES'])" ] @@ -991,198 +502,9 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'HOST0': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 0}},\n", - " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'HOST1': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 0}},\n", - " 'SERVICES': {1: {'health_status': 3, 'operating_status': 1}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'HOST2': {'FOLDERS': {1: {'FILES': {1: {'health_status': 2}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 0}},\n", - " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'HOST3': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 0}},\n", - " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'HOST4': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 1}},\n", - " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'HOST5': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 0}},\n", - " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'HOST6': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 0}},\n", - " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'ROUTER0': {'ACL': {1: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 0,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 2: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 1,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 3: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 2,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 4: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 3,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 5: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 4,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 6: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 5,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 7: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 6,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 8: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 7,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 9: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 8,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 10: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 9,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0}},\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}}}\n" - ] - } - ], + "outputs": [], "source": [ "obs, reward, terminated, truncated, info = env.step(9) # scan database file\n", "obs, reward, terminated, truncated, info = env.step(1) # scan webapp service\n", @@ -1214,21 +536,9 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "step: 38\n", - "Red action: do-nothing\n", - "Green action: do-nothing\n", - "Green action: node-application-execute\n", - "Blue reward:-0.8500000000000001\n" - ] - } - ], + "outputs": [], "source": [ "obs, reward, terminated, truncated, info = env.step(13) # patch the database\n", "print(f\"step: {env.game.step_counter}\")\n", @@ -1251,21 +561,9 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "step: 39\n", - "Red action: do-nothing\n", - "Green action: timestep=38 action='node-application-execute' parameters={'node_name': 'client_2', 'application_name': 'web-browser'} request=['network', 'node', 'client_2', 'application', 'web-browser', 'execute'] response=RequestResponse(status='failure', data={}) reward=-0.2 reward_info={'connection_attempt_status': 'n/a'}\n", - "Green action: timestep=38 action='node-application-execute' parameters={'node_name': 'client_1', 'application_name': 'web-browser'} request=['network', 'node', 'client_1', 'application', 'web-browser', 'execute'] response=RequestResponse(status='failure', data={}) reward=-0.25 reward_info={'connection_attempt_status': 'n/a'}\n", - "Blue reward:-0.05\n" - ] - } - ], + "outputs": [], "source": [ "obs, reward, terminated, truncated, info = env.step(0) # do nothing\n", "print(f\"step: {env.game.step_counter}\")\n", @@ -1288,3796 +586,7 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "step: 40, Red action: do-nothing, Blue reward:-0.05\n", - "step: 41, Red action: do-nothing, Blue reward:-0.05\n", - "step: 42, Red action: do-nothing, Blue reward:-0.05\n", - "step: 43, Red action: do-nothing, Blue reward:0.95\n", - "step: 44, Red action: do-nothing, Blue reward:0.95\n", - "step: 45, Red action: do-nothing, Blue reward:0.95\n", - "step: 46, Red action: do-nothing, Blue reward:0.95\n", - "step: 47, Red action: do-nothing, Blue reward:0.95\n", - "step: 48, Red action: do-nothing, Blue reward:0.95\n", - "step: 49, Red action: do-nothing, Blue reward:0.95\n", - "step: 50, Red action: do-nothing, Blue reward:0.95\n", - "step: 51, Red action: node-application-execute, Blue reward:0.95\n", - "step: 52, Red action: do-nothing, Blue reward:0.95\n", - "step: 53, Red action: do-nothing, Blue reward:0.85\n", - "step: 54, Red action: do-nothing, Blue reward:0.85\n", - "step: 55, Red action: do-nothing, Blue reward:0.85\n", - "step: 56, Red action: do-nothing, Blue reward:0.85\n", - "step: 57, Red action: do-nothing, Blue reward:0.85\n", - "step: 58, Red action: do-nothing, Blue reward:0.85\n", - "step: 59, Red action: do-nothing, Blue reward:0.85\n", - "step: 60, Red action: do-nothing, Blue reward:0.85\n", - "step: 61, Red action: do-nothing, Blue reward:0.85\n", - "step: 62, Red action: do-nothing, Blue reward:0.85\n", - "step: 63, Red action: do-nothing, Blue reward:0.85\n", - "step: 64, Red action: do-nothing, Blue reward:0.85\n", - "step: 65, Red action: do-nothing, Blue reward:0.85\n", - "step: 66, Red action: do-nothing, Blue reward:0.85\n", - "step: 67, Red action: do-nothing, Blue reward:0.85\n", - "step: 68, Red action: do-nothing, Blue reward:0.85\n", - "step: 69, Red action: do-nothing, Blue reward:0.85\n", - "step: 70, Red action: do-nothing, Blue reward:0.85\n", - "step: 71, Red action: do-nothing, Blue reward:0.85\n", - "step: 72, Red action: do-nothing, Blue reward:0.85\n", - "step: 73, Red action: do-nothing, Blue reward:0.85\n", - "step: 74, Red action: do-nothing, Blue reward:0.85\n", - "step: 75, Red action: node-application-execute, Blue reward:0.85\n", - "step: 76, Red action: do-nothing, Blue reward:0.85\n", - "step: 77, Red action: do-nothing, Blue reward:0.85\n", - "step: 78, Red action: do-nothing, Blue reward:0.85\n", - "step: 79, Red action: do-nothing, Blue reward:0.85\n", - "step: 80, Red action: do-nothing, Blue reward:0.85\n", - "step: 81, Red action: do-nothing, Blue reward:0.85\n", - "step: 82, Red action: do-nothing, Blue reward:0.85\n", - "step: 83, Red action: do-nothing, Blue reward:0.85\n", - "step: 84, Red action: do-nothing, Blue reward:0.85\n", - "step: 85, Red action: do-nothing, Blue reward:0.85\n", - "step: 86, Red action: do-nothing, Blue reward:0.85\n", - "step: 87, Red action: do-nothing, Blue reward:0.85\n", - "step: 88, Red action: do-nothing, Blue reward:0.85\n", - "step: 89, Red action: do-nothing, Blue reward:0.85\n", - "step: 90, Red action: do-nothing, Blue reward:0.85\n", - "step: 91, Red action: do-nothing, Blue reward:0.85\n", - "step: 92, Red action: do-nothing, Blue reward:0.85\n", - "step: 93, Red action: do-nothing, Blue reward:0.85\n", - "step: 94, Red action: do-nothing, Blue reward:0.85\n", - "step: 95, Red action: do-nothing, Blue reward:0.85\n", - "step: 96, Red action: do-nothing, Blue reward:0.85\n", - "step: 97, Red action: node-application-execute, Blue reward:0.85\n", - "step: 98, Red action: do-nothing, Blue reward:0.85\n", - "step: 99, Red action: do-nothing, Blue reward:0.85\n", - "step: 100, Red action: do-nothing, Blue reward:0.85\n", - "step: 101, Red action: do-nothing, Blue reward:0.85\n", - "step: 102, Red action: do-nothing, Blue reward:0.85\n", - "step: 103, Red action: do-nothing, Blue reward:0.85\n", - "step: 104, Red action: do-nothing, Blue reward:0.85\n", - "step: 105, Red action: do-nothing, Blue reward:0.85\n", - "step: 106, Red action: do-nothing, Blue reward:0.85\n", - "step: 107, Red action: do-nothing, Blue reward:0.85\n", - "step: 108, Red action: do-nothing, Blue reward:0.85\n", - "step: 109, Red action: do-nothing, Blue reward:0.85\n", - "step: 110, Red action: do-nothing, Blue reward:0.85\n", - "step: 111, Red action: do-nothing, Blue reward:0.85\n", - "step: 112, Red action: do-nothing, Blue reward:0.85\n", - "step: 113, Red action: node-application-execute, Blue reward:0.85\n", - "step: 114, Red action: do-nothing, Blue reward:0.85\n", - "step: 115, Red action: do-nothing, Blue reward:0.85\n", - "step: 116, Red action: do-nothing, Blue reward:0.85\n", - "step: 117, Red action: do-nothing, Blue reward:0.85\n", - "step: 118, Red action: do-nothing, Blue reward:0.85\n", - "step: 119, Red action: do-nothing, Blue reward:0.85\n", - "step: 120, Red action: do-nothing, Blue reward:0.85\n", - "step: 121, Red action: do-nothing, Blue reward:0.85\n", - "step: 122, Red action: do-nothing, Blue reward:0.85\n", - "step: 123, Red action: do-nothing, Blue reward:0.85\n", - "step: 124, Red action: do-nothing, Blue reward:0.85\n", - "step: 125, Red action: do-nothing, Blue reward:0.85\n", - "step: 126, Red action: do-nothing, Blue reward:0.85\n", - "step: 127, Red action: do-nothing, Blue reward:0.85\n", - "step: 128, Red action: do-nothing, Blue reward:0.85\n", - "step: 129, Red action: do-nothing, Blue reward:0.85\n", - "step: 130, Red action: do-nothing, Blue reward:0.85\n", - "step: 131, Red action: do-nothing, Blue reward:0.85\n", - "step: 132, Red action: node-application-execute, Blue reward:0.85\n", - "step: 133, Red action: do-nothing, Blue reward:0.85\n", - "step: 134, Red action: do-nothing, Blue reward:0.85\n", - "step: 135, Red action: do-nothing, Blue reward:0.85\n", - "step: 136, Red action: do-nothing, Blue reward:0.85\n", - "step: 137, Red action: do-nothing, Blue reward:0.85\n", - "step: 138, Red action: do-nothing, Blue reward:0.85\n", - "step: 139, Red action: do-nothing, Blue reward:0.85\n", - "step: 140, Red action: do-nothing, Blue reward:0.85\n", - "step: 141, Red action: do-nothing, Blue reward:0.85\n", - "step: 142, Red action: do-nothing, Blue reward:0.85\n", - "step: 143, Red action: do-nothing, Blue reward:0.85\n", - "step: 144, Red action: do-nothing, Blue reward:0.85\n", - "step: 145, Red action: do-nothing, Blue reward:0.85\n", - "step: 146, Red action: do-nothing, Blue reward:0.85\n", - "step: 147, Red action: do-nothing, Blue reward:0.85\n", - "step: 148, Red action: do-nothing, Blue reward:0.85\n", - "step: 149, Red action: do-nothing, Blue reward:0.85\n", - "step: 150, Red action: do-nothing, Blue reward:0.85\n", - "step: 151, Red action: do-nothing, Blue reward:0.85\n", - "step: 152, Red action: do-nothing, Blue reward:0.85\n", - "step: 153, Red action: do-nothing, Blue reward:0.85\n", - "step: 154, Red action: do-nothing, Blue reward:0.85\n", - "step: 155, Red action: do-nothing, Blue reward:0.85\n", - "step: 156, Red action: do-nothing, Blue reward:0.85\n", - "step: 157, Red action: node-application-execute, Blue reward:0.85\n", - "step: 158, Red action: do-nothing, Blue reward:0.85\n", - "step: 159, Red action: do-nothing, Blue reward:0.85\n", - "step: 160, Red action: do-nothing, Blue reward:0.85\n", - "step: 161, Red action: do-nothing, Blue reward:0.85\n", - "step: 162, Red action: do-nothing, Blue reward:0.85\n", - "step: 163, Red action: do-nothing, Blue reward:0.85\n", - "step: 164, Red action: do-nothing, Blue reward:0.85\n", - "step: 165, Red action: do-nothing, Blue reward:0.85\n", - "step: 166, Red action: do-nothing, Blue reward:0.85\n", - "step: 167, Red action: do-nothing, Blue reward:0.85\n", - "step: 168, Red action: do-nothing, Blue reward:0.85\n", - "step: 169, Red action: do-nothing, Blue reward:0.85\n", - "step: 170, Red action: do-nothing, Blue reward:0.85\n", - "step: 171, Red action: do-nothing, Blue reward:0.85\n", - "step: 172, Red action: do-nothing, Blue reward:0.85\n", - "step: 173, Red action: do-nothing, Blue reward:0.85\n", - "step: 174, Red action: do-nothing, Blue reward:0.85\n", - "step: 175, Red action: do-nothing, Blue reward:0.85\n", - "step: 176, Red action: do-nothing, Blue reward:0.85\n", - "step: 177, Red action: do-nothing, Blue reward:0.85\n", - "step: 178, Red action: node-application-execute, Blue reward:0.85\n", - "step: 179, Red action: do-nothing, Blue reward:0.85\n", - "step: 180, Red action: do-nothing, Blue reward:0.85\n", - "step: 181, Red action: do-nothing, Blue reward:0.85\n", - "step: 182, Red action: do-nothing, Blue reward:0.85\n", - "step: 183, Red action: do-nothing, Blue reward:0.85\n", - "step: 184, Red action: do-nothing, Blue reward:0.85\n", - "step: 185, Red action: do-nothing, Blue reward:0.85\n", - "step: 186, Red action: do-nothing, Blue reward:0.85\n", - "step: 187, Red action: do-nothing, Blue reward:0.85\n", - "step: 188, Red action: do-nothing, Blue reward:0.85\n", - "step: 189, Red action: do-nothing, Blue reward:0.85\n", - "step: 190, Red action: do-nothing, Blue reward:0.85\n", - "step: 191, Red action: do-nothing, Blue reward:0.85\n", - "step: 192, Red action: do-nothing, Blue reward:0.85\n", - "step: 193, Red action: do-nothing, Blue reward:0.85\n", - "step: 194, Red action: do-nothing, Blue reward:0.85\n", - "step: 195, Red action: do-nothing, Blue reward:0.85\n", - "step: 196, Red action: do-nothing, Blue reward:0.85\n", - "step: 197, Red action: do-nothing, Blue reward:0.85\n", - "step: 198, Red action: node-application-execute, Blue reward:0.85\n", - "step: 199, Red action: do-nothing, Blue reward:0.85\n", - "step: 200, Red action: do-nothing, Blue reward:0.85\n", - "step: 201, Red action: do-nothing, Blue reward:0.85\n", - "step: 202, Red action: do-nothing, Blue reward:0.85\n", - "step: 203, Red action: do-nothing, Blue reward:0.85\n", - "step: 204, Red action: do-nothing, Blue reward:0.85\n", - "step: 205, Red action: do-nothing, Blue reward:0.85\n", - "step: 206, Red action: do-nothing, Blue reward:0.85\n", - "step: 207, Red action: do-nothing, Blue reward:0.85\n", - "step: 208, Red action: do-nothing, Blue reward:0.85\n", - "step: 209, Red action: do-nothing, Blue reward:0.85\n", - "step: 210, Red action: do-nothing, Blue reward:0.85\n", - "step: 211, Red action: do-nothing, Blue reward:0.85\n", - "step: 212, Red action: do-nothing, Blue reward:0.85\n", - "step: 213, Red action: do-nothing, Blue reward:0.85\n", - "step: 214, Red action: do-nothing, Blue reward:0.85\n", - "step: 215, Red action: do-nothing, Blue reward:0.85\n", - "step: 216, Red action: do-nothing, Blue reward:0.85\n", - "step: 217, Red action: do-nothing, Blue reward:0.85\n", - "step: 218, Red action: node-application-execute, Blue reward:0.85\n", - "step: 219, Red action: do-nothing, Blue reward:0.85\n", - "step: 220, Red action: do-nothing, Blue reward:0.85\n", - "step: 221, Red action: do-nothing, Blue reward:0.85\n", - "step: 222, Red action: do-nothing, Blue reward:0.85\n", - "step: 223, Red action: do-nothing, Blue reward:0.85\n", - "step: 224, Red action: do-nothing, Blue reward:0.85\n", - "step: 225, Red action: do-nothing, Blue reward:0.85\n", - "step: 226, Red action: do-nothing, Blue reward:0.85\n", - "step: 227, Red action: do-nothing, Blue reward:0.85\n", - "step: 228, Red action: do-nothing, Blue reward:0.85\n", - "step: 229, Red action: do-nothing, Blue reward:0.85\n", - "step: 230, Red action: do-nothing, Blue reward:0.85\n", - "step: 231, Red action: do-nothing, Blue reward:0.85\n", - "step: 232, Red action: do-nothing, Blue reward:0.85\n", - "step: 233, Red action: do-nothing, Blue reward:0.85\n", - "step: 234, Red action: do-nothing, Blue reward:0.85\n", - "step: 235, Red action: do-nothing, Blue reward:0.85\n", - "step: 236, Red action: do-nothing, Blue reward:0.85\n", - "step: 237, Red action: do-nothing, Blue reward:0.85\n", - "step: 238, Red action: do-nothing, Blue reward:0.85\n", - "step: 239, Red action: do-nothing, Blue reward:0.85\n", - "step: 240, Red action: do-nothing, Blue reward:0.85\n", - "step: 241, Red action: node-application-execute, Blue reward:0.85\n", - "step: 242, Red action: do-nothing, Blue reward:0.85\n", - "step: 243, Red action: do-nothing, Blue reward:0.85\n", - "step: 244, Red action: do-nothing, Blue reward:0.85\n", - "step: 245, Red action: do-nothing, Blue reward:0.85\n", - "step: 246, Red action: do-nothing, Blue reward:0.85\n", - "step: 247, Red action: do-nothing, Blue reward:0.85\n", - "step: 248, Red action: do-nothing, Blue reward:0.85\n", - "step: 249, Red action: do-nothing, Blue reward:0.85\n", - "step: 250, Red action: do-nothing, Blue reward:0.85\n", - "step: 251, Red action: do-nothing, Blue reward:0.85\n", - "step: 252, Red action: do-nothing, Blue reward:0.85\n", - "step: 253, Red action: do-nothing, Blue reward:0.85\n", - "step: 254, Red action: do-nothing, Blue reward:0.85\n", - "step: 255, Red action: do-nothing, Blue reward:0.85\n", - "step: 256, Red action: do-nothing, Blue reward:0.85\n", - "step: 257, Red action: do-nothing, Blue reward:0.85\n", - "step: 258, Red action: do-nothing, Blue reward:0.85\n", - "step: 259, Red action: do-nothing, Blue reward:0.85\n", - "step: 260, Red action: do-nothing, Blue reward:0.85\n", - "step: 261, Red action: node-application-execute, Blue reward:0.85\n", - "step: 262, Red action: do-nothing, Blue reward:0.85\n", - "step: 263, Red action: do-nothing, Blue reward:0.85\n", - "step: 264, Red action: do-nothing, Blue reward:0.85\n", - "step: 265, Red action: do-nothing, Blue reward:0.85\n", - "step: 266, Red action: do-nothing, Blue reward:0.85\n", - "step: 267, Red action: do-nothing, Blue reward:0.85\n", - "step: 268, Red action: do-nothing, Blue reward:0.85\n", - "step: 269, Red action: do-nothing, Blue reward:0.85\n", - "step: 270, Red action: do-nothing, Blue reward:0.85\n", - "step: 271, Red action: do-nothing, Blue reward:0.85\n", - "step: 272, Red action: do-nothing, Blue reward:0.85\n", - "step: 273, Red action: do-nothing, Blue reward:0.85\n", - "step: 274, Red action: do-nothing, Blue reward:0.85\n", - "step: 275, Red action: do-nothing, Blue reward:0.85\n", - "step: 276, Red action: do-nothing, Blue reward:0.85\n", - "step: 277, Red action: do-nothing, Blue reward:0.85\n", - "step: 278, Red action: do-nothing, Blue reward:0.85\n", - "step: 279, Red action: do-nothing, Blue reward:0.85\n", - "step: 280, Red action: do-nothing, Blue reward:0.85\n", - "step: 281, Red action: node-application-execute, Blue reward:0.85\n", - "step: 282, Red action: do-nothing, Blue reward:0.85\n", - "step: 283, Red action: do-nothing, Blue reward:0.85\n", - "step: 284, Red action: do-nothing, Blue reward:0.85\n", - "step: 285, Red action: do-nothing, Blue reward:0.85\n", - "step: 286, Red action: do-nothing, Blue reward:0.85\n", - "step: 287, Red action: do-nothing, Blue reward:0.85\n", - "step: 288, Red action: do-nothing, Blue reward:0.85\n", - "step: 289, Red action: do-nothing, Blue reward:0.85\n", - "step: 290, Red action: do-nothing, Blue reward:0.85\n", - "step: 291, Red action: do-nothing, Blue reward:0.85\n", - "step: 292, Red action: do-nothing, Blue reward:0.85\n", - "step: 293, Red action: do-nothing, Blue reward:0.85\n", - "step: 294, Red action: do-nothing, Blue reward:0.85\n", - "step: 295, Red action: do-nothing, Blue reward:0.85\n", - "step: 296, Red action: do-nothing, Blue reward:0.85\n", - "step: 297, Red action: do-nothing, Blue reward:0.85\n", - "step: 298, Red action: do-nothing, Blue reward:0.85\n", - "step: 299, Red action: do-nothing, Blue reward:0.85\n", - "step: 300, Red action: do-nothing, Blue reward:0.85\n", - "step: 301, Red action: do-nothing, Blue reward:0.85\n", - "step: 302, Red action: do-nothing, Blue reward:0.85\n", - "step: 303, Red action: do-nothing, Blue reward:0.85\n", - "step: 304, Red action: node-application-execute, Blue reward:0.85\n", - "step: 305, Red action: do-nothing, Blue reward:0.85\n", - "step: 306, Red action: do-nothing, Blue reward:0.85\n", - "step: 307, Red action: do-nothing, Blue reward:0.85\n", - "step: 308, Red action: do-nothing, Blue reward:0.85\n", - "step: 309, Red action: do-nothing, Blue reward:0.85\n", - "step: 310, Red action: do-nothing, Blue reward:0.85\n", - "step: 311, Red action: do-nothing, Blue reward:0.85\n", - "step: 312, Red action: do-nothing, Blue reward:0.85\n", - "step: 313, Red action: do-nothing, Blue reward:0.85\n", - "step: 314, Red action: do-nothing, Blue reward:0.85\n", - "step: 315, Red action: do-nothing, Blue reward:0.85\n", - "step: 316, Red action: do-nothing, Blue reward:0.85\n", - "step: 317, Red action: do-nothing, Blue reward:0.85\n", - "step: 318, Red action: do-nothing, Blue reward:0.85\n", - "step: 319, Red action: do-nothing, Blue reward:0.85\n", - "step: 320, Red action: do-nothing, Blue reward:0.85\n", - "step: 321, Red action: do-nothing, Blue reward:0.85\n", - "step: 322, Red action: do-nothing, Blue reward:0.85\n", - "step: 323, Red action: do-nothing, Blue reward:0.85\n", - "step: 324, Red action: do-nothing, Blue reward:0.85\n", - "step: 325, Red action: node-application-execute, Blue reward:0.85\n", - "step: 326, Red action: do-nothing, Blue reward:0.85\n", - "step: 327, Red action: do-nothing, Blue reward:0.85\n", - "step: 328, Red action: do-nothing, Blue reward:0.85\n", - "step: 329, Red action: do-nothing, Blue reward:0.85\n", - "step: 330, Red action: do-nothing, Blue reward:0.85\n", - "step: 331, Red action: do-nothing, Blue reward:0.85\n", - "step: 332, Red action: do-nothing, Blue reward:0.85\n", - "step: 333, Red action: do-nothing, Blue reward:0.85\n", - "step: 334, Red action: do-nothing, Blue reward:0.85\n", - "step: 335, Red action: do-nothing, Blue reward:0.85\n", - "step: 336, Red action: do-nothing, Blue reward:0.85\n", - "step: 337, Red action: do-nothing, Blue reward:0.85\n", - "step: 338, Red action: do-nothing, Blue reward:0.85\n", - "step: 339, Red action: do-nothing, Blue reward:0.85\n", - "step: 340, Red action: do-nothing, Blue reward:0.85\n", - "step: 341, Red action: do-nothing, Blue reward:0.85\n", - "step: 342, Red action: do-nothing, Blue reward:0.85\n", - "step: 343, Red action: do-nothing, Blue reward:0.85\n", - "step: 344, Red action: do-nothing, Blue reward:0.85\n", - "step: 345, Red action: do-nothing, Blue reward:0.85\n", - "step: 346, Red action: do-nothing, Blue reward:0.85\n", - "step: 347, Red action: do-nothing, Blue reward:0.85\n", - "step: 348, Red action: do-nothing, Blue reward:0.85\n", - "step: 349, Red action: node-application-execute, Blue reward:0.85\n", - "step: 350, Red action: do-nothing, Blue reward:0.85\n", - "step: 351, Red action: do-nothing, Blue reward:0.85\n", - "step: 352, Red action: do-nothing, Blue reward:0.85\n", - "step: 353, Red action: do-nothing, Blue reward:0.85\n", - "step: 354, Red action: do-nothing, Blue reward:0.85\n", - "step: 355, Red action: do-nothing, Blue reward:0.85\n", - "step: 356, Red action: do-nothing, Blue reward:0.85\n", - "step: 357, Red action: do-nothing, Blue reward:0.85\n", - "step: 358, Red action: do-nothing, Blue reward:0.85\n", - "step: 359, Red action: do-nothing, Blue reward:0.85\n", - "step: 360, Red action: do-nothing, Blue reward:0.85\n", - "step: 361, Red action: do-nothing, Blue reward:0.85\n", - "step: 362, Red action: do-nothing, Blue reward:0.85\n", - "step: 363, Red action: do-nothing, Blue reward:0.85\n", - "step: 364, Red action: do-nothing, Blue reward:0.85\n", - "step: 365, Red action: do-nothing, Blue reward:0.85\n", - "step: 366, Red action: do-nothing, Blue reward:0.85\n", - "step: 367, Red action: do-nothing, Blue reward:0.85\n", - "step: 368, Red action: do-nothing, Blue reward:0.85\n", - "step: 369, Red action: node-application-execute, Blue reward:0.85\n", - "step: 370, Red action: do-nothing, Blue reward:0.85\n", - "step: 371, Red action: do-nothing, Blue reward:0.85\n", - "step: 372, Red action: do-nothing, Blue reward:0.85\n", - "step: 373, Red action: do-nothing, Blue reward:0.85\n", - "step: 374, Red action: do-nothing, Blue reward:0.85\n", - "step: 375, Red action: do-nothing, Blue reward:0.85\n", - "step: 376, Red action: do-nothing, Blue reward:0.85\n", - "step: 377, Red action: do-nothing, Blue reward:0.85\n", - "step: 378, Red action: do-nothing, Blue reward:0.85\n", - "step: 379, Red action: do-nothing, Blue reward:0.85\n", - "step: 380, Red action: do-nothing, Blue reward:0.85\n", - "step: 381, Red action: do-nothing, Blue reward:0.85\n", - "step: 382, Red action: do-nothing, Blue reward:0.85\n", - "step: 383, Red action: do-nothing, Blue reward:0.85\n", - "step: 384, Red action: do-nothing, Blue reward:0.85\n", - "step: 385, Red action: do-nothing, Blue reward:0.85\n", - "step: 386, Red action: do-nothing, Blue reward:0.85\n", - "step: 387, Red action: do-nothing, Blue reward:0.85\n", - "step: 388, Red action: do-nothing, Blue reward:0.85\n", - "step: 389, Red action: do-nothing, Blue reward:0.85\n", - "step: 390, Red action: do-nothing, Blue reward:0.85\n", - "step: 391, Red action: do-nothing, Blue reward:0.85\n", - "step: 392, Red action: do-nothing, Blue reward:0.85\n", - "step: 393, Red action: do-nothing, Blue reward:0.85\n", - "step: 394, Red action: node-application-execute, Blue reward:0.85\n", - "step: 395, Red action: do-nothing, Blue reward:0.85\n", - "step: 396, Red action: do-nothing, Blue reward:0.85\n", - "step: 397, Red action: do-nothing, Blue reward:0.85\n", - "step: 398, Red action: do-nothing, Blue reward:0.85\n", - "step: 399, Red action: do-nothing, Blue reward:0.85\n", - "step: 400, Red action: do-nothing, Blue reward:0.85\n", - "step: 401, Red action: do-nothing, Blue reward:0.85\n", - "step: 402, Red action: do-nothing, Blue reward:0.85\n", - "step: 403, Red action: do-nothing, Blue reward:0.85\n", - "step: 404, Red action: do-nothing, Blue reward:0.85\n", - "step: 405, Red action: do-nothing, Blue reward:0.85\n", - "step: 406, Red action: do-nothing, Blue reward:0.85\n", - "step: 407, Red action: do-nothing, Blue reward:0.85\n", - "step: 408, Red action: do-nothing, Blue reward:0.85\n", - "step: 409, Red action: do-nothing, Blue reward:0.85\n", - "step: 410, Red action: do-nothing, Blue reward:0.85\n", - "step: 411, Red action: do-nothing, Blue reward:0.85\n", - "step: 412, Red action: do-nothing, Blue reward:0.85\n", - "step: 413, Red action: do-nothing, Blue reward:0.85\n", - "step: 414, Red action: do-nothing, Blue reward:0.85\n", - "step: 415, Red action: do-nothing, Blue reward:0.85\n", - "step: 416, Red action: do-nothing, Blue reward:0.85\n", - "step: 417, Red action: do-nothing, Blue reward:0.85\n", - "step: 418, Red action: node-application-execute, Blue reward:0.85\n", - "step: 419, Red action: do-nothing, Blue reward:0.85\n", - "step: 420, Red action: do-nothing, Blue reward:0.85\n", - "step: 421, Red action: do-nothing, Blue reward:0.85\n", - "step: 422, Red action: do-nothing, Blue reward:0.85\n", - "step: 423, Red action: do-nothing, Blue reward:0.85\n", - "step: 424, Red action: do-nothing, Blue reward:0.85\n", - "step: 425, Red action: do-nothing, Blue reward:0.85\n", - "step: 426, Red action: do-nothing, Blue reward:0.85\n", - "step: 427, Red action: do-nothing, Blue reward:0.85\n", - "step: 428, Red action: do-nothing, Blue reward:0.85\n", - "step: 429, Red action: do-nothing, Blue reward:0.85\n", - "step: 430, Red action: do-nothing, Blue reward:0.85\n", - "step: 431, Red action: do-nothing, Blue reward:0.85\n", - "step: 432, Red action: do-nothing, Blue reward:0.85\n", - "step: 433, Red action: do-nothing, Blue reward:0.85\n", - "step: 434, Red action: do-nothing, Blue reward:0.85\n", - "step: 435, Red action: do-nothing, Blue reward:0.85\n", - "step: 436, Red action: do-nothing, Blue reward:0.85\n", - "step: 437, Red action: do-nothing, Blue reward:0.85\n", - "step: 438, Red action: do-nothing, Blue reward:0.85\n", - "step: 439, Red action: do-nothing, Blue reward:0.85\n", - "step: 440, Red action: do-nothing, Blue reward:0.85\n", - "step: 441, Red action: do-nothing, Blue reward:0.85\n", - "step: 442, Red action: do-nothing, Blue reward:0.85\n", - "step: 443, Red action: node-application-execute, Blue reward:0.85\n", - "step: 444, Red action: do-nothing, Blue reward:0.85\n", - "step: 445, Red action: do-nothing, Blue reward:0.85\n", - "step: 446, Red action: do-nothing, Blue reward:0.85\n", - "step: 447, Red action: do-nothing, Blue reward:0.85\n", - "step: 448, Red action: do-nothing, Blue reward:0.85\n", - "step: 449, Red action: do-nothing, Blue reward:0.85\n", - "step: 450, Red action: do-nothing, Blue reward:0.85\n", - "step: 451, Red action: do-nothing, Blue reward:0.85\n", - "step: 452, Red action: do-nothing, Blue reward:0.85\n", - "step: 453, Red action: do-nothing, Blue reward:0.85\n", - "step: 454, Red action: do-nothing, Blue reward:0.85\n", - "step: 455, Red action: do-nothing, Blue reward:0.85\n", - "step: 456, Red action: do-nothing, Blue reward:0.85\n", - "step: 457, Red action: do-nothing, Blue reward:0.85\n", - "step: 458, Red action: do-nothing, Blue reward:0.85\n", - "step: 459, Red action: do-nothing, Blue reward:0.85\n", - "step: 460, Red action: do-nothing, Blue reward:0.85\n", - "step: 461, Red action: node-application-execute, Blue reward:0.85\n", - "step: 462, Red action: do-nothing, Blue reward:0.85\n", - "step: 463, Red action: do-nothing, Blue reward:0.85\n", - "step: 464, Red action: do-nothing, Blue reward:0.85\n", - "step: 465, Red action: do-nothing, Blue reward:0.85\n", - "step: 466, Red action: do-nothing, Blue reward:0.85\n", - "step: 467, Red action: do-nothing, Blue reward:0.85\n", - "step: 468, Red action: do-nothing, Blue reward:0.85\n", - "step: 469, Red action: do-nothing, Blue reward:0.85\n", - "step: 470, Red action: do-nothing, Blue reward:0.85\n", - "step: 471, Red action: do-nothing, Blue reward:0.85\n", - "step: 472, Red action: do-nothing, Blue reward:0.85\n", - "step: 473, Red action: do-nothing, Blue reward:0.85\n", - "step: 474, Red action: do-nothing, Blue reward:0.85\n", - "step: 475, Red action: do-nothing, Blue reward:0.85\n", - "step: 476, Red action: do-nothing, Blue reward:0.85\n", - "step: 477, Red action: do-nothing, Blue reward:0.85\n", - "step: 478, Red action: do-nothing, Blue reward:0.85\n", - "step: 479, Red action: do-nothing, Blue reward:0.85\n", - "step: 480, Red action: node-application-execute, Blue reward:0.85\n", - "step: 481, Red action: do-nothing, Blue reward:0.85\n", - "step: 482, Red action: do-nothing, Blue reward:0.85\n", - "step: 483, Red action: do-nothing, Blue reward:0.85\n", - "step: 484, Red action: do-nothing, Blue reward:0.85\n", - "step: 485, Red action: do-nothing, Blue reward:0.85\n", - "step: 486, Red action: do-nothing, Blue reward:0.85\n", - "step: 487, Red action: do-nothing, Blue reward:0.85\n", - "step: 488, Red action: do-nothing, Blue reward:0.85\n", - "step: 489, Red action: do-nothing, Blue reward:0.85\n", - "step: 490, Red action: do-nothing, Blue reward:0.85\n", - "step: 491, Red action: do-nothing, Blue reward:0.85\n", - "step: 492, Red action: do-nothing, Blue reward:0.85\n", - "step: 493, Red action: do-nothing, Blue reward:0.85\n", - "step: 494, Red action: do-nothing, Blue reward:0.85\n", - "step: 495, Red action: do-nothing, Blue reward:0.85\n", - "step: 496, Red action: do-nothing, Blue reward:0.85\n", - "step: 497, Red action: do-nothing, Blue reward:0.85\n", - "step: 498, Red action: do-nothing, Blue reward:0.85\n", - "step: 499, Red action: do-nothing, Blue reward:0.85\n", - "step: 500, Red action: do-nothing, Blue reward:0.85\n", - "step: 501, Red action: do-nothing, Blue reward:0.85\n", - "step: 502, Red action: do-nothing, Blue reward:0.85\n", - "step: 503, Red action: do-nothing, Blue reward:0.85\n", - "step: 504, Red action: node-application-execute, Blue reward:0.85\n", - "step: 505, Red action: do-nothing, Blue reward:0.85\n", - "step: 506, Red action: do-nothing, Blue reward:0.85\n", - "step: 507, Red action: do-nothing, Blue reward:0.85\n", - "step: 508, Red action: do-nothing, Blue reward:0.85\n", - "step: 509, Red action: do-nothing, Blue reward:0.85\n", - "step: 510, Red action: do-nothing, Blue reward:0.85\n", - "step: 511, Red action: do-nothing, Blue reward:0.85\n", - "step: 512, Red action: do-nothing, Blue reward:0.85\n", - "step: 513, Red action: do-nothing, Blue reward:0.85\n", - "step: 514, Red action: do-nothing, Blue reward:0.85\n", - "step: 515, Red action: do-nothing, Blue reward:0.85\n", - "step: 516, Red action: do-nothing, Blue reward:0.85\n", - "step: 517, Red action: do-nothing, Blue reward:0.85\n", - "step: 518, Red action: do-nothing, Blue reward:0.85\n", - "step: 519, Red action: do-nothing, Blue reward:0.85\n", - "step: 520, Red action: do-nothing, Blue reward:0.85\n", - "step: 521, Red action: do-nothing, Blue reward:0.85\n", - "step: 522, Red action: do-nothing, Blue reward:0.85\n", - "step: 523, Red action: do-nothing, Blue reward:0.85\n", - "step: 524, Red action: do-nothing, Blue reward:0.85\n", - "step: 525, Red action: do-nothing, Blue reward:0.85\n", - "step: 526, Red action: do-nothing, Blue reward:0.85\n", - "step: 527, Red action: node-application-execute, Blue reward:0.85\n", - "step: 528, Red action: do-nothing, Blue reward:0.85\n", - "step: 529, Red action: do-nothing, Blue reward:0.85\n", - "step: 530, Red action: do-nothing, Blue reward:0.85\n", - "step: 531, Red action: do-nothing, Blue reward:0.85\n", - "step: 532, Red action: do-nothing, Blue reward:0.85\n", - "step: 533, Red action: do-nothing, Blue reward:0.85\n", - "step: 534, Red action: do-nothing, Blue reward:0.85\n", - "step: 535, Red action: do-nothing, Blue reward:0.85\n", - "step: 536, Red action: do-nothing, Blue reward:0.85\n", - "step: 537, Red action: do-nothing, Blue reward:0.85\n", - "step: 538, Red action: do-nothing, Blue reward:0.85\n", - "step: 539, Red action: do-nothing, Blue reward:0.85\n", - "step: 540, Red action: do-nothing, Blue reward:0.85\n", - "step: 541, Red action: do-nothing, Blue reward:0.85\n", - "step: 542, Red action: do-nothing, Blue reward:0.85\n", - "step: 543, Red action: do-nothing, Blue reward:0.85\n", - "step: 544, Red action: do-nothing, Blue reward:0.85\n", - "step: 545, Red action: do-nothing, Blue reward:0.85\n", - "step: 546, Red action: do-nothing, Blue reward:0.85\n", - "step: 547, Red action: do-nothing, Blue reward:0.85\n", - "step: 548, Red action: do-nothing, Blue reward:0.85\n", - "step: 549, Red action: node-application-execute, Blue reward:0.85\n", - "step: 550, Red action: do-nothing, Blue reward:0.85\n", - "step: 551, Red action: do-nothing, Blue reward:0.85\n", - "step: 552, Red action: do-nothing, Blue reward:0.85\n", - "step: 553, Red action: do-nothing, Blue reward:0.85\n", - "step: 554, Red action: do-nothing, Blue reward:0.85\n", - "step: 555, Red action: do-nothing, Blue reward:0.85\n", - "step: 556, Red action: do-nothing, Blue reward:0.85\n", - "step: 557, Red action: do-nothing, Blue reward:0.85\n", - "step: 558, Red action: do-nothing, Blue reward:0.85\n", - "step: 559, Red action: do-nothing, Blue reward:0.85\n", - "step: 560, Red action: do-nothing, Blue reward:0.85\n", - "step: 561, Red action: do-nothing, Blue reward:0.85\n", - "step: 562, Red action: do-nothing, Blue reward:0.85\n", - "step: 563, Red action: do-nothing, Blue reward:0.85\n", - "step: 564, Red action: do-nothing, Blue reward:0.85\n", - "step: 565, Red action: node-application-execute, Blue reward:0.85\n", - "step: 566, Red action: do-nothing, Blue reward:0.85\n", - "step: 567, Red action: do-nothing, Blue reward:0.85\n", - "step: 568, Red action: do-nothing, Blue reward:0.85\n", - "step: 569, Red action: do-nothing, Blue reward:0.85\n", - "step: 570, Red action: do-nothing, Blue reward:0.85\n", - "step: 571, Red action: do-nothing, Blue reward:0.85\n", - "step: 572, Red action: do-nothing, Blue reward:0.85\n", - "step: 573, Red action: do-nothing, Blue reward:0.85\n", - "step: 574, Red action: do-nothing, Blue reward:0.85\n", - "step: 575, Red action: do-nothing, Blue reward:0.85\n", - "step: 576, Red action: do-nothing, Blue reward:0.85\n", - "step: 577, Red action: do-nothing, Blue reward:0.85\n", - "step: 578, Red action: do-nothing, Blue reward:0.85\n", - "step: 579, Red action: do-nothing, Blue reward:0.85\n", - "step: 580, Red action: do-nothing, Blue reward:0.85\n", - "step: 581, Red action: do-nothing, Blue reward:0.85\n", - "step: 582, Red action: do-nothing, Blue reward:0.85\n", - "step: 583, Red action: do-nothing, Blue reward:0.85\n", - "step: 584, Red action: do-nothing, Blue reward:0.85\n", - "step: 585, Red action: do-nothing, Blue reward:0.85\n", - "step: 586, Red action: do-nothing, Blue reward:0.85\n", - "step: 587, Red action: do-nothing, Blue reward:0.85\n", - "step: 588, Red action: do-nothing, Blue reward:0.85\n", - "step: 589, Red action: node-application-execute, Blue reward:0.85\n", - "step: 590, Red action: do-nothing, Blue reward:0.85\n", - "step: 591, Red action: do-nothing, Blue reward:0.85\n", - "step: 592, Red action: do-nothing, Blue reward:0.85\n", - "step: 593, Red action: do-nothing, Blue reward:0.85\n", - "step: 594, Red action: do-nothing, Blue reward:0.85\n", - "step: 595, Red action: do-nothing, Blue reward:0.85\n", - "step: 596, Red action: do-nothing, Blue reward:0.85\n", - "step: 597, Red action: do-nothing, Blue reward:0.85\n", - "step: 598, Red action: do-nothing, Blue reward:0.85\n", - "step: 599, Red action: do-nothing, Blue reward:0.85\n", - "step: 600, Red action: do-nothing, Blue reward:0.85\n", - "step: 601, Red action: do-nothing, Blue reward:0.85\n", - "step: 602, Red action: do-nothing, Blue reward:0.85\n", - "step: 603, Red action: do-nothing, Blue reward:0.85\n", - "step: 604, Red action: do-nothing, Blue reward:0.85\n", - "step: 605, Red action: do-nothing, Blue reward:0.85\n", - "step: 606, Red action: do-nothing, Blue reward:0.85\n", - "step: 607, Red action: do-nothing, Blue reward:0.85\n", - "step: 608, Red action: do-nothing, Blue reward:0.85\n", - "step: 609, Red action: do-nothing, Blue reward:0.85\n", - "step: 610, Red action: node-application-execute, Blue reward:0.85\n", - "step: 611, Red action: do-nothing, Blue reward:0.85\n", - "step: 612, Red action: do-nothing, Blue reward:0.85\n", - "step: 613, Red action: do-nothing, Blue reward:0.85\n", - "step: 614, Red action: do-nothing, Blue reward:0.85\n", - "step: 615, Red action: do-nothing, Blue reward:0.85\n", - "step: 616, Red action: do-nothing, Blue reward:0.85\n", - "step: 617, Red action: do-nothing, Blue reward:0.85\n", - "step: 618, Red action: do-nothing, Blue reward:0.85\n", - "step: 619, Red action: do-nothing, Blue reward:0.85\n", - "step: 620, Red action: do-nothing, Blue reward:0.85\n", - "step: 621, Red action: do-nothing, Blue reward:0.85\n", - "step: 622, Red action: do-nothing, Blue reward:0.85\n", - "step: 623, Red action: do-nothing, Blue reward:0.85\n", - "step: 624, Red action: do-nothing, Blue reward:0.85\n", - "step: 625, Red action: do-nothing, Blue reward:0.85\n", - "step: 626, Red action: node-application-execute, Blue reward:0.85\n", - "step: 627, Red action: do-nothing, Blue reward:0.85\n", - "step: 628, Red action: do-nothing, Blue reward:0.85\n", - "step: 629, Red action: do-nothing, Blue reward:0.85\n", - "step: 630, Red action: do-nothing, Blue reward:0.85\n", - "step: 631, Red action: do-nothing, Blue reward:0.85\n", - "step: 632, Red action: do-nothing, Blue reward:0.85\n", - "step: 633, Red action: do-nothing, Blue reward:0.85\n", - "step: 634, Red action: do-nothing, Blue reward:0.85\n", - "step: 635, Red action: do-nothing, Blue reward:0.85\n", - "step: 636, Red action: do-nothing, Blue reward:0.85\n", - "step: 637, Red action: do-nothing, Blue reward:0.85\n", - "step: 638, Red action: do-nothing, Blue reward:0.85\n", - "step: 639, Red action: do-nothing, Blue reward:0.85\n", - "step: 640, Red action: do-nothing, Blue reward:0.85\n", - "step: 641, Red action: do-nothing, Blue reward:0.85\n", - "step: 642, Red action: do-nothing, Blue reward:0.85\n", - "step: 643, Red action: do-nothing, Blue reward:0.85\n", - "step: 644, Red action: node-application-execute, Blue reward:0.85\n", - "step: 645, Red action: do-nothing, Blue reward:0.85\n", - "step: 646, Red action: do-nothing, Blue reward:0.85\n", - "step: 647, Red action: do-nothing, Blue reward:0.85\n", - "step: 648, Red action: do-nothing, Blue reward:0.85\n", - "step: 649, Red action: do-nothing, Blue reward:0.85\n", - "step: 650, Red action: do-nothing, Blue reward:0.85\n", - "step: 651, Red action: do-nothing, Blue reward:0.85\n", - "step: 652, Red action: do-nothing, Blue reward:0.85\n", - "step: 653, Red action: do-nothing, Blue reward:0.85\n", - "step: 654, Red action: do-nothing, Blue reward:0.85\n", - "step: 655, Red action: do-nothing, Blue reward:0.85\n", - "step: 656, Red action: do-nothing, Blue reward:0.85\n", - "step: 657, Red action: do-nothing, Blue reward:0.85\n", - "step: 658, Red action: do-nothing, Blue reward:0.85\n", - "step: 659, Red action: do-nothing, Blue reward:0.85\n", - "step: 660, Red action: do-nothing, Blue reward:0.85\n", - "step: 661, Red action: do-nothing, Blue reward:0.85\n", - "step: 662, Red action: do-nothing, Blue reward:0.85\n", - "step: 663, Red action: do-nothing, Blue reward:0.85\n", - "step: 664, Red action: do-nothing, Blue reward:0.85\n", - "step: 665, Red action: do-nothing, Blue reward:0.85\n", - "step: 666, Red action: do-nothing, Blue reward:0.85\n", - "step: 667, Red action: node-application-execute, Blue reward:0.85\n", - "step: 668, Red action: do-nothing, Blue reward:0.85\n", - "step: 669, Red action: do-nothing, Blue reward:0.85\n", - "step: 670, Red action: do-nothing, Blue reward:0.85\n", - "step: 671, Red action: do-nothing, Blue reward:0.85\n", - "step: 672, Red action: do-nothing, Blue reward:0.85\n", - "step: 673, Red action: do-nothing, Blue reward:0.85\n", - "step: 674, Red action: do-nothing, Blue reward:0.85\n", - "step: 675, Red action: do-nothing, Blue reward:0.85\n", - "step: 676, Red action: do-nothing, Blue reward:0.85\n", - "step: 677, Red action: do-nothing, Blue reward:0.85\n", - "step: 678, Red action: do-nothing, Blue reward:0.85\n", - "step: 679, Red action: do-nothing, Blue reward:0.85\n", - "step: 680, Red action: do-nothing, Blue reward:0.85\n", - "step: 681, Red action: do-nothing, Blue reward:0.85\n", - "step: 682, Red action: do-nothing, Blue reward:0.85\n", - "step: 683, Red action: do-nothing, Blue reward:0.85\n", - "step: 684, Red action: do-nothing, Blue reward:0.85\n", - "step: 685, Red action: node-application-execute, Blue reward:0.85\n", - "step: 686, Red action: do-nothing, Blue reward:0.85\n", - "step: 687, Red action: do-nothing, Blue reward:0.85\n", - "step: 688, Red action: do-nothing, Blue reward:0.85\n", - "step: 689, Red action: do-nothing, Blue reward:0.85\n", - "step: 690, Red action: do-nothing, Blue reward:0.85\n", - "step: 691, Red action: do-nothing, Blue reward:0.85\n", - "step: 692, Red action: do-nothing, Blue reward:0.85\n", - "step: 693, Red action: do-nothing, Blue reward:0.85\n", - "step: 694, Red action: do-nothing, Blue reward:0.85\n", - "step: 695, Red action: do-nothing, Blue reward:0.85\n", - "step: 696, Red action: do-nothing, Blue reward:0.85\n", - "step: 697, Red action: do-nothing, Blue reward:0.85\n", - "step: 698, Red action: do-nothing, Blue reward:0.85\n", - "step: 699, Red action: do-nothing, Blue reward:0.85\n", - "step: 700, Red action: do-nothing, Blue reward:0.85\n", - "step: 701, Red action: node-application-execute, Blue reward:0.85\n", - "step: 702, Red action: do-nothing, Blue reward:0.85\n", - "step: 703, Red action: do-nothing, Blue reward:0.85\n", - "step: 704, Red action: do-nothing, Blue reward:0.85\n", - "step: 705, Red action: do-nothing, Blue reward:0.85\n", - "step: 706, Red action: do-nothing, Blue reward:0.85\n", - "step: 707, Red action: do-nothing, Blue reward:0.85\n", - "step: 708, Red action: do-nothing, Blue reward:0.85\n", - "step: 709, Red action: do-nothing, Blue reward:0.85\n", - "step: 710, Red action: do-nothing, Blue reward:0.85\n", - "step: 711, Red action: do-nothing, Blue reward:0.85\n", - "step: 712, Red action: do-nothing, Blue reward:0.85\n", - "step: 713, Red action: do-nothing, Blue reward:0.85\n", - "step: 714, Red action: do-nothing, Blue reward:0.85\n", - "step: 715, Red action: do-nothing, Blue reward:0.85\n", - "step: 716, Red action: do-nothing, Blue reward:0.85\n", - "step: 717, Red action: do-nothing, Blue reward:0.85\n", - "step: 718, Red action: node-application-execute, Blue reward:0.85\n", - "step: 719, Red action: do-nothing, Blue reward:0.85\n", - "step: 720, Red action: do-nothing, Blue reward:0.85\n", - "step: 721, Red action: do-nothing, Blue reward:0.85\n", - "step: 722, Red action: do-nothing, Blue reward:0.85\n", - "step: 723, Red action: do-nothing, Blue reward:0.85\n", - "step: 724, Red action: do-nothing, Blue reward:0.85\n", - "step: 725, Red action: do-nothing, Blue reward:0.85\n", - "step: 726, Red action: do-nothing, Blue reward:0.85\n", - "step: 727, Red action: do-nothing, Blue reward:0.85\n", - "step: 728, Red action: do-nothing, Blue reward:0.85\n", - "step: 729, Red action: do-nothing, Blue reward:0.85\n", - "step: 730, Red action: do-nothing, Blue reward:0.85\n", - "step: 731, Red action: do-nothing, Blue reward:0.85\n", - "step: 732, Red action: do-nothing, Blue reward:0.85\n", - "step: 733, Red action: do-nothing, Blue reward:0.85\n", - "step: 734, Red action: do-nothing, Blue reward:0.85\n", - "step: 735, Red action: do-nothing, Blue reward:0.85\n", - "step: 736, Red action: do-nothing, Blue reward:0.85\n", - "step: 737, Red action: do-nothing, Blue reward:0.85\n", - "step: 738, Red action: do-nothing, Blue reward:0.85\n", - "step: 739, Red action: do-nothing, Blue reward:0.85\n", - "step: 740, Red action: do-nothing, Blue reward:0.85\n", - "step: 741, Red action: do-nothing, Blue reward:0.85\n", - "step: 742, Red action: do-nothing, Blue reward:0.85\n", - "step: 743, Red action: node-application-execute, Blue reward:0.85\n", - "step: 744, Red action: do-nothing, Blue reward:0.85\n", - "step: 745, Red action: do-nothing, Blue reward:0.85\n", - "step: 746, Red action: do-nothing, Blue reward:0.85\n", - "step: 747, Red action: do-nothing, Blue reward:0.85\n", - "step: 748, Red action: do-nothing, Blue reward:0.85\n", - "step: 749, Red action: do-nothing, Blue reward:0.85\n", - "step: 750, Red action: do-nothing, Blue reward:0.85\n", - "step: 751, Red action: do-nothing, Blue reward:0.85\n", - "step: 752, Red action: do-nothing, Blue reward:0.85\n", - "step: 753, Red action: do-nothing, Blue reward:0.85\n", - "step: 754, Red action: do-nothing, Blue reward:0.85\n", - "step: 755, Red action: do-nothing, Blue reward:0.85\n", - "step: 756, Red action: do-nothing, Blue reward:0.85\n", - "step: 757, Red action: do-nothing, Blue reward:0.85\n", - "step: 758, Red action: do-nothing, Blue reward:0.85\n", - "step: 759, Red action: do-nothing, Blue reward:0.85\n", - "step: 760, Red action: node-application-execute, Blue reward:0.85\n", - "step: 761, Red action: do-nothing, Blue reward:0.85\n", - "step: 762, Red action: do-nothing, Blue reward:0.85\n", - "step: 763, Red action: do-nothing, Blue reward:0.85\n", - "step: 764, Red action: do-nothing, Blue reward:0.85\n", - "step: 765, Red action: do-nothing, Blue reward:0.85\n", - "step: 766, Red action: do-nothing, Blue reward:0.85\n", - "step: 767, Red action: do-nothing, Blue reward:0.85\n", - "step: 768, Red action: do-nothing, Blue reward:0.85\n", - "step: 769, Red action: do-nothing, Blue reward:0.85\n", - "step: 770, Red action: do-nothing, Blue reward:0.85\n", - "step: 771, Red action: do-nothing, Blue reward:0.85\n", - "step: 772, Red action: do-nothing, Blue reward:0.85\n", - "step: 773, Red action: do-nothing, Blue reward:0.85\n", - "step: 774, Red action: do-nothing, Blue reward:0.85\n", - "step: 775, Red action: do-nothing, Blue reward:0.85\n", - "step: 776, Red action: do-nothing, Blue reward:0.85\n", - "step: 777, Red action: do-nothing, Blue reward:0.85\n", - "step: 778, Red action: do-nothing, Blue reward:0.85\n", - "step: 779, Red action: do-nothing, Blue reward:0.85\n", - "step: 780, Red action: do-nothing, Blue reward:0.85\n", - "step: 781, Red action: do-nothing, Blue reward:0.85\n", - "step: 782, Red action: do-nothing, Blue reward:0.85\n", - "step: 783, Red action: node-application-execute, Blue reward:0.85\n", - "step: 784, Red action: do-nothing, Blue reward:0.85\n", - "step: 785, Red action: do-nothing, Blue reward:0.85\n", - "step: 786, Red action: do-nothing, Blue reward:0.85\n", - "step: 787, Red action: do-nothing, Blue reward:0.85\n", - "step: 788, Red action: do-nothing, Blue reward:0.85\n", - "step: 789, Red action: do-nothing, Blue reward:0.85\n", - "step: 790, Red action: do-nothing, Blue reward:0.85\n", - "step: 791, Red action: do-nothing, Blue reward:0.85\n", - "step: 792, Red action: do-nothing, Blue reward:0.85\n", - "step: 793, Red action: do-nothing, Blue reward:0.85\n", - "step: 794, Red action: do-nothing, Blue reward:0.85\n", - "step: 795, Red action: do-nothing, Blue reward:0.85\n", - "step: 796, Red action: do-nothing, Blue reward:0.85\n", - "step: 797, Red action: do-nothing, Blue reward:0.85\n", - "step: 798, Red action: do-nothing, Blue reward:0.85\n", - "step: 799, Red action: do-nothing, Blue reward:0.85\n", - "step: 800, Red action: do-nothing, Blue reward:0.85\n", - "step: 801, Red action: do-nothing, Blue reward:0.85\n", - "step: 802, Red action: do-nothing, Blue reward:0.85\n", - "step: 803, Red action: node-application-execute, Blue reward:0.85\n", - "step: 804, Red action: do-nothing, Blue reward:0.85\n", - "step: 805, Red action: do-nothing, Blue reward:0.85\n", - "step: 806, Red action: do-nothing, Blue reward:0.85\n", - "step: 807, Red action: do-nothing, Blue reward:0.85\n", - "step: 808, Red action: do-nothing, Blue reward:0.85\n", - "step: 809, Red action: do-nothing, Blue reward:0.85\n", - "step: 810, Red action: do-nothing, Blue reward:0.85\n", - "step: 811, Red action: do-nothing, Blue reward:0.85\n", - "step: 812, Red action: do-nothing, Blue reward:0.85\n", - "step: 813, Red action: do-nothing, Blue reward:0.85\n", - "step: 814, Red action: do-nothing, Blue reward:0.85\n", - "step: 815, Red action: do-nothing, Blue reward:0.85\n", - "step: 816, Red action: do-nothing, Blue reward:0.85\n", - "step: 817, Red action: do-nothing, Blue reward:0.85\n", - "step: 818, Red action: do-nothing, Blue reward:0.85\n", - "step: 819, Red action: do-nothing, Blue reward:0.85\n", - "step: 820, Red action: do-nothing, Blue reward:0.85\n", - "step: 821, Red action: do-nothing, Blue reward:0.85\n", - "step: 822, Red action: do-nothing, Blue reward:0.85\n", - "step: 823, Red action: do-nothing, Blue reward:0.85\n", - "step: 824, Red action: do-nothing, Blue reward:0.85\n", - "step: 825, Red action: do-nothing, Blue reward:0.85\n", - "step: 826, Red action: do-nothing, Blue reward:0.85\n", - "step: 827, Red action: node-application-execute, Blue reward:0.85\n", - "step: 828, Red action: do-nothing, Blue reward:0.85\n", - "step: 829, Red action: do-nothing, Blue reward:0.85\n", - "step: 830, Red action: do-nothing, Blue reward:0.85\n", - "step: 831, Red action: do-nothing, Blue reward:0.85\n", - "step: 832, Red action: do-nothing, Blue reward:0.85\n", - "step: 833, Red action: do-nothing, Blue reward:0.85\n", - "step: 834, Red action: do-nothing, Blue reward:0.85\n", - "step: 835, Red action: do-nothing, Blue reward:0.85\n", - "step: 836, Red action: do-nothing, Blue reward:0.85\n", - "step: 837, Red action: do-nothing, Blue reward:0.85\n", - "step: 838, Red action: do-nothing, Blue reward:0.85\n", - "step: 839, Red action: do-nothing, Blue reward:0.85\n", - "step: 840, Red action: do-nothing, Blue reward:0.85\n", - "step: 841, Red action: do-nothing, Blue reward:0.85\n", - "step: 842, Red action: do-nothing, Blue reward:0.85\n", - "step: 843, Red action: do-nothing, Blue reward:0.85\n", - "step: 844, Red action: do-nothing, Blue reward:0.85\n", - "step: 845, Red action: do-nothing, Blue reward:0.85\n", - "step: 846, Red action: do-nothing, Blue reward:0.85\n", - "step: 847, Red action: node-application-execute, Blue reward:0.85\n", - "step: 848, Red action: do-nothing, Blue reward:0.85\n", - "step: 849, Red action: do-nothing, Blue reward:0.85\n", - "step: 850, Red action: do-nothing, Blue reward:0.85\n", - "step: 851, Red action: do-nothing, Blue reward:0.85\n", - "step: 852, Red action: do-nothing, Blue reward:0.85\n", - "step: 853, Red action: do-nothing, Blue reward:0.85\n", - "step: 854, Red action: do-nothing, Blue reward:0.85\n", - "step: 855, Red action: do-nothing, Blue reward:0.85\n", - "step: 856, Red action: do-nothing, Blue reward:0.85\n", - "step: 857, Red action: do-nothing, Blue reward:0.85\n", - "step: 858, Red action: do-nothing, Blue reward:0.85\n", - "step: 859, Red action: do-nothing, Blue reward:0.85\n", - "step: 860, Red action: do-nothing, Blue reward:0.85\n", - "step: 861, Red action: do-nothing, Blue reward:0.85\n", - "step: 862, Red action: do-nothing, Blue reward:0.85\n", - "step: 863, Red action: do-nothing, Blue reward:0.85\n", - "step: 864, Red action: do-nothing, Blue reward:0.85\n", - "step: 865, Red action: do-nothing, Blue reward:0.85\n", - "step: 866, Red action: do-nothing, Blue reward:0.85\n", - "step: 867, Red action: do-nothing, Blue reward:0.85\n", - "step: 868, Red action: do-nothing, Blue reward:0.85\n", - "step: 869, Red action: do-nothing, Blue reward:0.85\n", - "step: 870, Red action: do-nothing, Blue reward:0.85\n", - "step: 871, Red action: node-application-execute, Blue reward:0.85\n", - "step: 872, Red action: do-nothing, Blue reward:0.85\n", - "step: 873, Red action: do-nothing, Blue reward:0.85\n", - "step: 874, Red action: do-nothing, Blue reward:0.85\n", - "step: 875, Red action: do-nothing, Blue reward:0.85\n", - "step: 876, Red action: do-nothing, Blue reward:0.85\n", - "step: 877, Red action: do-nothing, Blue reward:0.85\n", - "step: 878, Red action: do-nothing, Blue reward:0.85\n", - "step: 879, Red action: do-nothing, Blue reward:0.85\n", - "step: 880, Red action: do-nothing, Blue reward:0.85\n", - "step: 881, Red action: do-nothing, Blue reward:0.85\n", - "step: 882, Red action: do-nothing, Blue reward:0.85\n", - "step: 883, Red action: do-nothing, Blue reward:0.85\n", - "step: 884, Red action: do-nothing, Blue reward:0.85\n", - "step: 885, Red action: do-nothing, Blue reward:0.85\n", - "step: 886, Red action: do-nothing, Blue reward:0.85\n", - "step: 887, Red action: do-nothing, Blue reward:0.85\n", - "step: 888, Red action: do-nothing, Blue reward:0.85\n", - "step: 889, Red action: do-nothing, Blue reward:0.85\n", - "step: 890, Red action: do-nothing, Blue reward:0.85\n", - "step: 891, Red action: node-application-execute, Blue reward:0.85\n", - "step: 892, Red action: do-nothing, Blue reward:0.85\n", - "step: 893, Red action: do-nothing, Blue reward:0.85\n", - "step: 894, Red action: do-nothing, Blue reward:0.85\n", - "step: 895, Red action: do-nothing, Blue reward:0.85\n", - "step: 896, Red action: do-nothing, Blue reward:0.85\n", - "step: 897, Red action: do-nothing, Blue reward:0.85\n", - "step: 898, Red action: do-nothing, Blue reward:0.85\n", - "step: 899, Red action: do-nothing, Blue reward:0.85\n", - "step: 900, Red action: do-nothing, Blue reward:0.85\n", - "step: 901, Red action: do-nothing, Blue reward:0.85\n", - "step: 902, Red action: do-nothing, Blue reward:0.85\n", - "step: 903, Red action: do-nothing, Blue reward:0.85\n", - "step: 904, Red action: do-nothing, Blue reward:0.85\n", - "step: 905, Red action: do-nothing, Blue reward:0.85\n", - "step: 906, Red action: node-application-execute, Blue reward:0.85\n", - "step: 907, Red action: do-nothing, Blue reward:0.85\n", - "step: 908, Red action: do-nothing, Blue reward:0.85\n", - "step: 909, Red action: do-nothing, Blue reward:0.85\n", - "step: 910, Red action: do-nothing, Blue reward:0.85\n", - "step: 911, Red action: do-nothing, Blue reward:0.85\n", - "step: 912, Red action: do-nothing, Blue reward:0.85\n", - "step: 913, Red action: do-nothing, Blue reward:0.85\n", - "step: 914, Red action: do-nothing, Blue reward:0.85\n", - "step: 915, Red action: do-nothing, Blue reward:0.85\n", - "step: 916, Red action: do-nothing, Blue reward:0.85\n", - "step: 917, Red action: do-nothing, Blue reward:0.85\n", - "step: 918, Red action: do-nothing, Blue reward:0.85\n", - "step: 919, Red action: do-nothing, Blue reward:0.85\n", - "step: 920, Red action: do-nothing, Blue reward:0.85\n", - "step: 921, Red action: do-nothing, Blue reward:0.85\n", - "step: 922, Red action: do-nothing, Blue reward:0.85\n", - "step: 923, Red action: do-nothing, Blue reward:0.85\n", - "step: 924, Red action: do-nothing, Blue reward:0.85\n", - "step: 925, Red action: do-nothing, Blue reward:0.85\n", - "step: 926, Red action: do-nothing, Blue reward:0.85\n", - "step: 927, Red action: do-nothing, Blue reward:0.85\n", - "step: 928, Red action: do-nothing, Blue reward:0.85\n", - "step: 929, Red action: do-nothing, Blue reward:0.85\n", - "step: 930, Red action: node-application-execute, Blue reward:0.85\n", - "step: 931, Red action: do-nothing, Blue reward:0.85\n", - "step: 932, Red action: do-nothing, Blue reward:0.85\n", - "step: 933, Red action: do-nothing, Blue reward:0.85\n", - "step: 934, Red action: do-nothing, Blue reward:0.85\n", - "step: 935, Red action: do-nothing, Blue reward:0.85\n", - "step: 936, Red action: do-nothing, Blue reward:0.85\n", - "step: 937, Red action: do-nothing, Blue reward:0.85\n", - "step: 938, Red action: do-nothing, Blue reward:0.85\n", - "step: 939, Red action: do-nothing, Blue reward:0.85\n", - "step: 940, Red action: do-nothing, Blue reward:0.85\n", - "step: 941, Red action: do-nothing, Blue reward:0.85\n", - "step: 942, Red action: do-nothing, Blue reward:0.85\n", - "step: 943, Red action: do-nothing, Blue reward:0.85\n", - "step: 944, Red action: do-nothing, Blue reward:0.85\n", - "step: 945, Red action: do-nothing, Blue reward:0.85\n", - "step: 946, Red action: do-nothing, Blue reward:0.85\n", - "step: 947, Red action: node-application-execute, Blue reward:0.85\n", - "step: 948, Red action: do-nothing, Blue reward:0.85\n", - "step: 949, Red action: do-nothing, Blue reward:0.85\n", - "step: 950, Red action: do-nothing, Blue reward:0.85\n", - "step: 951, Red action: do-nothing, Blue reward:0.85\n", - "step: 952, Red action: do-nothing, Blue reward:0.85\n", - "step: 953, Red action: do-nothing, Blue reward:0.85\n", - "step: 954, Red action: do-nothing, Blue reward:0.85\n", - "step: 955, Red action: do-nothing, Blue reward:0.85\n", - "step: 956, Red action: do-nothing, Blue reward:0.85\n", - "step: 957, Red action: do-nothing, Blue reward:0.85\n", - "step: 958, Red action: do-nothing, Blue reward:0.85\n", - "step: 959, Red action: do-nothing, Blue reward:0.85\n", - "step: 960, Red action: do-nothing, Blue reward:0.85\n", - "step: 961, Red action: do-nothing, Blue reward:0.85\n", - "step: 962, Red action: do-nothing, Blue reward:0.85\n", - "step: 963, Red action: node-application-execute, Blue reward:0.85\n", - "step: 964, Red action: do-nothing, Blue reward:0.85\n", - "step: 965, Red action: do-nothing, Blue reward:0.85\n", - "step: 966, Red action: do-nothing, Blue reward:0.85\n", - "step: 967, Red action: do-nothing, Blue reward:0.85\n", - "step: 968, Red action: do-nothing, Blue reward:0.85\n", - "step: 969, Red action: do-nothing, Blue reward:0.85\n", - "step: 970, Red action: do-nothing, Blue reward:0.85\n", - "step: 971, Red action: do-nothing, Blue reward:0.85\n", - "step: 972, Red action: do-nothing, Blue reward:0.85\n", - "step: 973, Red action: do-nothing, Blue reward:0.85\n", - "step: 974, Red action: do-nothing, Blue reward:0.85\n", - "step: 975, Red action: do-nothing, Blue reward:0.85\n", - "step: 976, Red action: do-nothing, Blue reward:0.85\n", - "step: 977, Red action: do-nothing, Blue reward:0.85\n", - "step: 978, Red action: do-nothing, Blue reward:0.85\n", - "step: 979, Red action: do-nothing, Blue reward:0.85\n", - "step: 980, Red action: do-nothing, Blue reward:0.85\n", - "step: 981, Red action: do-nothing, Blue reward:0.85\n", - "step: 982, Red action: do-nothing, Blue reward:0.85\n", - "step: 983, Red action: do-nothing, Blue reward:0.85\n", - "step: 984, Red action: do-nothing, Blue reward:0.85\n", - "step: 985, Red action: do-nothing, Blue reward:0.85\n", - "step: 986, Red action: do-nothing, Blue reward:0.85\n", - "step: 987, Red action: do-nothing, Blue reward:0.85\n", - "step: 988, Red action: node-application-execute, Blue reward:0.85\n", - "step: 989, Red action: do-nothing, Blue reward:0.85\n", - "step: 990, Red action: do-nothing, Blue reward:0.85\n", - "step: 991, Red action: do-nothing, Blue reward:0.85\n", - "step: 992, Red action: do-nothing, Blue reward:0.85\n", - "step: 993, Red action: do-nothing, Blue reward:0.85\n", - "step: 994, Red action: do-nothing, Blue reward:0.85\n", - "step: 995, Red action: do-nothing, Blue reward:0.85\n", - "step: 996, Red action: do-nothing, Blue reward:0.85\n", - "step: 997, Red action: do-nothing, Blue reward:0.85\n", - "step: 998, Red action: do-nothing, Blue reward:0.85\n", - "step: 999, Red action: do-nothing, Blue reward:0.85\n", - "step: 1000, Red action: do-nothing, Blue reward:0.85\n", - "step: 1001, Red action: do-nothing, Blue reward:0.85\n", - "step: 1002, Red action: do-nothing, Blue reward:0.85\n", - "step: 1003, Red action: do-nothing, Blue reward:0.85\n", - "step: 1004, Red action: do-nothing, Blue reward:0.85\n", - "step: 1005, Red action: do-nothing, Blue reward:0.85\n", - "step: 1006, Red action: do-nothing, Blue reward:0.85\n", - "step: 1007, Red action: do-nothing, Blue reward:0.85\n", - "step: 1008, Red action: do-nothing, Blue reward:0.85\n", - "step: 1009, Red action: do-nothing, Blue reward:0.85\n", - "step: 1010, Red action: do-nothing, Blue reward:0.85\n", - "step: 1011, Red action: do-nothing, Blue reward:0.85\n", - "step: 1012, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1013, Red action: do-nothing, Blue reward:0.85\n", - "step: 1014, Red action: do-nothing, Blue reward:0.85\n", - "step: 1015, Red action: do-nothing, Blue reward:0.85\n", - "step: 1016, Red action: do-nothing, Blue reward:0.85\n", - "step: 1017, Red action: do-nothing, Blue reward:0.85\n", - "step: 1018, Red action: do-nothing, Blue reward:0.85\n", - "step: 1019, Red action: do-nothing, Blue reward:0.85\n", - "step: 1020, Red action: do-nothing, Blue reward:0.85\n", - "step: 1021, Red action: do-nothing, Blue reward:0.85\n", - "step: 1022, Red action: do-nothing, Blue reward:0.85\n", - "step: 1023, Red action: do-nothing, Blue reward:0.85\n", - "step: 1024, Red action: do-nothing, Blue reward:0.85\n", - "step: 1025, Red action: do-nothing, Blue reward:0.85\n", - "step: 1026, Red action: do-nothing, Blue reward:0.85\n", - "step: 1027, Red action: do-nothing, Blue reward:0.85\n", - "step: 1028, Red action: do-nothing, Blue reward:0.85\n", - "step: 1029, Red action: do-nothing, Blue reward:0.85\n", - "step: 1030, Red action: do-nothing, Blue reward:0.85\n", - "step: 1031, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1032, Red action: do-nothing, Blue reward:0.85\n", - "step: 1033, Red action: do-nothing, Blue reward:0.85\n", - "step: 1034, Red action: do-nothing, Blue reward:0.85\n", - "step: 1035, Red action: do-nothing, Blue reward:0.85\n", - "step: 1036, Red action: do-nothing, Blue reward:0.85\n", - "step: 1037, Red action: do-nothing, Blue reward:0.85\n", - "step: 1038, Red action: do-nothing, Blue reward:0.85\n", - "step: 1039, Red action: do-nothing, Blue reward:0.85\n", - "step: 1040, Red action: do-nothing, Blue reward:0.85\n", - "step: 1041, Red action: do-nothing, Blue reward:0.85\n", - "step: 1042, Red action: do-nothing, Blue reward:0.85\n", - "step: 1043, Red action: do-nothing, Blue reward:0.85\n", - "step: 1044, Red action: do-nothing, Blue reward:0.85\n", - "step: 1045, Red action: do-nothing, Blue reward:0.85\n", - "step: 1046, Red action: do-nothing, Blue reward:0.85\n", - "step: 1047, Red action: do-nothing, Blue reward:0.85\n", - "step: 1048, Red action: do-nothing, Blue reward:0.85\n", - "step: 1049, Red action: do-nothing, Blue reward:0.85\n", - "step: 1050, Red action: do-nothing, Blue reward:0.85\n", - "step: 1051, Red action: do-nothing, Blue reward:0.85\n", - "step: 1052, Red action: do-nothing, Blue reward:0.85\n", - "step: 1053, Red action: do-nothing, Blue reward:0.85\n", - "step: 1054, Red action: do-nothing, Blue reward:0.85\n", - "step: 1055, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1056, Red action: do-nothing, Blue reward:0.85\n", - "step: 1057, Red action: do-nothing, Blue reward:0.85\n", - "step: 1058, Red action: do-nothing, Blue reward:0.85\n", - "step: 1059, Red action: do-nothing, Blue reward:0.85\n", - "step: 1060, Red action: do-nothing, Blue reward:0.85\n", - "step: 1061, Red action: do-nothing, Blue reward:0.85\n", - "step: 1062, Red action: do-nothing, Blue reward:0.85\n", - "step: 1063, Red action: do-nothing, Blue reward:0.85\n", - "step: 1064, Red action: do-nothing, Blue reward:0.85\n", - "step: 1065, Red action: do-nothing, Blue reward:0.85\n", - "step: 1066, Red action: do-nothing, Blue reward:0.85\n", - "step: 1067, Red action: do-nothing, Blue reward:0.85\n", - "step: 1068, Red action: do-nothing, Blue reward:0.85\n", - "step: 1069, Red action: do-nothing, Blue reward:0.85\n", - "step: 1070, Red action: do-nothing, Blue reward:0.85\n", - "step: 1071, Red action: do-nothing, Blue reward:0.85\n", - "step: 1072, Red action: do-nothing, Blue reward:0.85\n", - "step: 1073, Red action: do-nothing, Blue reward:0.85\n", - "step: 1074, Red action: do-nothing, Blue reward:0.85\n", - "step: 1075, Red action: do-nothing, Blue reward:0.85\n", - "step: 1076, Red action: do-nothing, Blue reward:0.85\n", - "step: 1077, Red action: do-nothing, Blue reward:0.85\n", - "step: 1078, Red action: do-nothing, Blue reward:0.85\n", - "step: 1079, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1080, Red action: do-nothing, Blue reward:0.85\n", - "step: 1081, Red action: do-nothing, Blue reward:0.85\n", - "step: 1082, Red action: do-nothing, Blue reward:0.85\n", - "step: 1083, Red action: do-nothing, Blue reward:0.85\n", - "step: 1084, Red action: do-nothing, Blue reward:0.85\n", - "step: 1085, Red action: do-nothing, Blue reward:0.85\n", - "step: 1086, Red action: do-nothing, Blue reward:0.85\n", - "step: 1087, Red action: do-nothing, Blue reward:0.85\n", - "step: 1088, Red action: do-nothing, Blue reward:0.85\n", - "step: 1089, Red action: do-nothing, Blue reward:0.85\n", - "step: 1090, Red action: do-nothing, Blue reward:0.85\n", - "step: 1091, Red action: do-nothing, Blue reward:0.85\n", - "step: 1092, Red action: do-nothing, Blue reward:0.85\n", - "step: 1093, Red action: do-nothing, Blue reward:0.85\n", - "step: 1094, Red action: do-nothing, Blue reward:0.85\n", - "step: 1095, Red action: do-nothing, Blue reward:0.85\n", - "step: 1096, Red action: do-nothing, Blue reward:0.85\n", - "step: 1097, Red action: do-nothing, Blue reward:0.85\n", - "step: 1098, Red action: do-nothing, Blue reward:0.85\n", - "step: 1099, Red action: do-nothing, Blue reward:0.85\n", - "step: 1100, Red action: do-nothing, Blue reward:0.85\n", - "step: 1101, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1102, Red action: do-nothing, Blue reward:0.85\n", - "step: 1103, Red action: do-nothing, Blue reward:0.85\n", - "step: 1104, Red action: do-nothing, Blue reward:0.85\n", - "step: 1105, Red action: do-nothing, Blue reward:0.85\n", - "step: 1106, Red action: do-nothing, Blue reward:0.85\n", - "step: 1107, Red action: do-nothing, Blue reward:0.85\n", - "step: 1108, Red action: do-nothing, Blue reward:0.85\n", - "step: 1109, Red action: do-nothing, Blue reward:0.85\n", - "step: 1110, Red action: do-nothing, Blue reward:0.85\n", - "step: 1111, Red action: do-nothing, Blue reward:0.85\n", - "step: 1112, Red action: do-nothing, Blue reward:0.85\n", - "step: 1113, Red action: do-nothing, Blue reward:0.85\n", - "step: 1114, Red action: do-nothing, Blue reward:0.85\n", - "step: 1115, Red action: do-nothing, Blue reward:0.85\n", - "step: 1116, Red action: do-nothing, Blue reward:0.85\n", - "step: 1117, Red action: do-nothing, Blue reward:0.85\n", - "step: 1118, Red action: do-nothing, Blue reward:0.85\n", - "step: 1119, Red action: do-nothing, Blue reward:0.85\n", - "step: 1120, Red action: do-nothing, Blue reward:0.85\n", - "step: 1121, Red action: do-nothing, Blue reward:0.85\n", - "step: 1122, Red action: do-nothing, Blue reward:0.85\n", - "step: 1123, Red action: do-nothing, Blue reward:0.85\n", - "step: 1124, Red action: do-nothing, Blue reward:0.85\n", - "step: 1125, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1126, Red action: do-nothing, Blue reward:0.85\n", - "step: 1127, Red action: do-nothing, Blue reward:0.85\n", - "step: 1128, Red action: do-nothing, Blue reward:0.85\n", - "step: 1129, Red action: do-nothing, Blue reward:0.85\n", - "step: 1130, Red action: do-nothing, Blue reward:0.85\n", - "step: 1131, Red action: do-nothing, Blue reward:0.85\n", - "step: 1132, Red action: do-nothing, Blue reward:0.85\n", - "step: 1133, Red action: do-nothing, Blue reward:0.85\n", - "step: 1134, Red action: do-nothing, Blue reward:0.85\n", - "step: 1135, Red action: do-nothing, Blue reward:0.85\n", - "step: 1136, Red action: do-nothing, Blue reward:0.85\n", - "step: 1137, Red action: do-nothing, Blue reward:0.85\n", - "step: 1138, Red action: do-nothing, Blue reward:0.85\n", - "step: 1139, Red action: do-nothing, Blue reward:0.85\n", - "step: 1140, Red action: do-nothing, Blue reward:0.85\n", - "step: 1141, Red action: do-nothing, Blue reward:0.85\n", - "step: 1142, Red action: do-nothing, Blue reward:0.85\n", - "step: 1143, Red action: do-nothing, Blue reward:0.85\n", - "step: 1144, Red action: do-nothing, Blue reward:0.85\n", - "step: 1145, Red action: do-nothing, Blue reward:0.85\n", - "step: 1146, Red action: do-nothing, Blue reward:0.85\n", - "step: 1147, Red action: do-nothing, Blue reward:0.85\n", - "step: 1148, Red action: do-nothing, Blue reward:0.85\n", - "step: 1149, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1150, Red action: do-nothing, Blue reward:0.85\n", - "step: 1151, Red action: do-nothing, Blue reward:0.85\n", - "step: 1152, Red action: do-nothing, Blue reward:0.85\n", - "step: 1153, Red action: do-nothing, Blue reward:0.85\n", - "step: 1154, Red action: do-nothing, Blue reward:0.85\n", - "step: 1155, Red action: do-nothing, Blue reward:0.85\n", - "step: 1156, Red action: do-nothing, Blue reward:0.85\n", - "step: 1157, Red action: do-nothing, Blue reward:0.85\n", - "step: 1158, Red action: do-nothing, Blue reward:0.85\n", - "step: 1159, Red action: do-nothing, Blue reward:0.85\n", - "step: 1160, Red action: do-nothing, Blue reward:0.85\n", - "step: 1161, Red action: do-nothing, Blue reward:0.85\n", - "step: 1162, Red action: do-nothing, Blue reward:0.85\n", - "step: 1163, Red action: do-nothing, Blue reward:0.85\n", - "step: 1164, Red action: do-nothing, Blue reward:0.85\n", - "step: 1165, Red action: do-nothing, Blue reward:0.85\n", - "step: 1166, Red action: do-nothing, Blue reward:0.85\n", - "step: 1167, Red action: do-nothing, Blue reward:0.85\n", - "step: 1168, Red action: do-nothing, Blue reward:0.85\n", - "step: 1169, Red action: do-nothing, Blue reward:0.85\n", - "step: 1170, Red action: do-nothing, Blue reward:0.85\n", - "step: 1171, Red action: do-nothing, Blue reward:0.85\n", - "step: 1172, Red action: do-nothing, Blue reward:0.85\n", - "step: 1173, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1174, Red action: do-nothing, Blue reward:0.85\n", - "step: 1175, Red action: do-nothing, Blue reward:0.85\n", - "step: 1176, Red action: do-nothing, Blue reward:0.85\n", - "step: 1177, Red action: do-nothing, Blue reward:0.85\n", - "step: 1178, Red action: do-nothing, Blue reward:0.85\n", - "step: 1179, Red action: do-nothing, Blue reward:0.85\n", - "step: 1180, Red action: do-nothing, Blue reward:0.85\n", - "step: 1181, Red action: do-nothing, Blue reward:0.85\n", - "step: 1182, Red action: do-nothing, Blue reward:0.85\n", - "step: 1183, Red action: do-nothing, Blue reward:0.85\n", - "step: 1184, Red action: do-nothing, Blue reward:0.85\n", - "step: 1185, Red action: do-nothing, Blue reward:0.85\n", - "step: 1186, Red action: do-nothing, Blue reward:0.85\n", - "step: 1187, Red action: do-nothing, Blue reward:0.85\n", - "step: 1188, Red action: do-nothing, Blue reward:0.85\n", - "step: 1189, Red action: do-nothing, Blue reward:0.85\n", - "step: 1190, Red action: do-nothing, Blue reward:0.85\n", - "step: 1191, Red action: do-nothing, Blue reward:0.85\n", - "step: 1192, Red action: do-nothing, Blue reward:0.85\n", - "step: 1193, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1194, Red action: do-nothing, Blue reward:0.85\n", - "step: 1195, Red action: do-nothing, Blue reward:0.85\n", - "step: 1196, Red action: do-nothing, Blue reward:0.85\n", - "step: 1197, Red action: do-nothing, Blue reward:0.85\n", - "step: 1198, Red action: do-nothing, Blue reward:0.85\n", - "step: 1199, Red action: do-nothing, Blue reward:0.85\n", - "step: 1200, Red action: do-nothing, Blue reward:0.85\n", - "step: 1201, Red action: do-nothing, Blue reward:0.85\n", - "step: 1202, Red action: do-nothing, Blue reward:0.85\n", - "step: 1203, Red action: do-nothing, Blue reward:0.85\n", - "step: 1204, Red action: do-nothing, Blue reward:0.85\n", - "step: 1205, Red action: do-nothing, Blue reward:0.85\n", - "step: 1206, Red action: do-nothing, Blue reward:0.85\n", - "step: 1207, Red action: do-nothing, Blue reward:0.85\n", - "step: 1208, Red action: do-nothing, Blue reward:0.85\n", - "step: 1209, Red action: do-nothing, Blue reward:0.85\n", - "step: 1210, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1211, Red action: do-nothing, Blue reward:0.85\n", - "step: 1212, Red action: do-nothing, Blue reward:0.85\n", - "step: 1213, Red action: do-nothing, Blue reward:0.85\n", - "step: 1214, Red action: do-nothing, Blue reward:0.85\n", - "step: 1215, Red action: do-nothing, Blue reward:0.85\n", - "step: 1216, Red action: do-nothing, Blue reward:0.85\n", - "step: 1217, Red action: do-nothing, Blue reward:0.85\n", - "step: 1218, Red action: do-nothing, Blue reward:0.85\n", - "step: 1219, Red action: do-nothing, Blue reward:0.85\n", - "step: 1220, Red action: do-nothing, Blue reward:0.85\n", - "step: 1221, Red action: do-nothing, Blue reward:0.85\n", - "step: 1222, Red action: do-nothing, Blue reward:0.85\n", - "step: 1223, Red action: do-nothing, Blue reward:0.85\n", - "step: 1224, Red action: do-nothing, Blue reward:0.85\n", - "step: 1225, Red action: do-nothing, Blue reward:0.85\n", - "step: 1226, Red action: do-nothing, Blue reward:0.85\n", - "step: 1227, Red action: do-nothing, Blue reward:0.85\n", - "step: 1228, Red action: do-nothing, Blue reward:0.85\n", - "step: 1229, Red action: do-nothing, Blue reward:0.85\n", - "step: 1230, Red action: do-nothing, Blue reward:0.85\n", - "step: 1231, Red action: do-nothing, Blue reward:0.85\n", - "step: 1232, Red action: do-nothing, Blue reward:0.85\n", - "step: 1233, Red action: do-nothing, Blue reward:0.85\n", - "step: 1234, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1235, Red action: do-nothing, Blue reward:0.85\n", - "step: 1236, Red action: do-nothing, Blue reward:0.85\n", - "step: 1237, Red action: do-nothing, Blue reward:0.85\n", - "step: 1238, Red action: do-nothing, Blue reward:0.85\n", - "step: 1239, Red action: do-nothing, Blue reward:0.85\n", - "step: 1240, Red action: do-nothing, Blue reward:0.85\n", - "step: 1241, Red action: do-nothing, Blue reward:0.85\n", - "step: 1242, Red action: do-nothing, Blue reward:0.85\n", - "step: 1243, Red action: do-nothing, Blue reward:0.85\n", - "step: 1244, Red action: do-nothing, Blue reward:0.85\n", - "step: 1245, Red action: do-nothing, Blue reward:0.85\n", - "step: 1246, Red action: do-nothing, Blue reward:0.85\n", - "step: 1247, Red action: do-nothing, Blue reward:0.85\n", - "step: 1248, Red action: do-nothing, Blue reward:0.85\n", - "step: 1249, Red action: do-nothing, Blue reward:0.85\n", - "step: 1250, Red action: do-nothing, Blue reward:0.85\n", - "step: 1251, Red action: do-nothing, Blue reward:0.85\n", - "step: 1252, Red action: do-nothing, Blue reward:0.85\n", - "step: 1253, Red action: do-nothing, Blue reward:0.85\n", - "step: 1254, Red action: do-nothing, Blue reward:0.85\n", - "step: 1255, Red action: do-nothing, Blue reward:0.85\n", - "step: 1256, Red action: do-nothing, Blue reward:0.85\n", - "step: 1257, Red action: do-nothing, Blue reward:0.85\n", - "step: 1258, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1259, Red action: do-nothing, Blue reward:0.85\n", - "step: 1260, Red action: do-nothing, Blue reward:0.85\n", - "step: 1261, Red action: do-nothing, Blue reward:0.85\n", - "step: 1262, Red action: do-nothing, Blue reward:0.85\n", - "step: 1263, Red action: do-nothing, Blue reward:0.85\n", - "step: 1264, Red action: do-nothing, Blue reward:0.85\n", - "step: 1265, Red action: do-nothing, Blue reward:0.85\n", - "step: 1266, Red action: do-nothing, Blue reward:0.85\n", - "step: 1267, Red action: do-nothing, Blue reward:0.85\n", - "step: 1268, Red action: do-nothing, Blue reward:0.85\n", - "step: 1269, Red action: do-nothing, Blue reward:0.85\n", - "step: 1270, Red action: do-nothing, Blue reward:0.85\n", - "step: 1271, Red action: do-nothing, Blue reward:0.85\n", - "step: 1272, Red action: do-nothing, Blue reward:0.85\n", - "step: 1273, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1274, Red action: do-nothing, Blue reward:0.85\n", - "step: 1275, Red action: do-nothing, Blue reward:0.85\n", - "step: 1276, Red action: do-nothing, Blue reward:0.85\n", - "step: 1277, Red action: do-nothing, Blue reward:0.85\n", - "step: 1278, Red action: do-nothing, Blue reward:0.85\n", - "step: 1279, Red action: do-nothing, Blue reward:0.85\n", - "step: 1280, Red action: do-nothing, Blue reward:0.85\n", - "step: 1281, Red action: do-nothing, Blue reward:0.85\n", - "step: 1282, Red action: do-nothing, Blue reward:0.85\n", - "step: 1283, Red action: do-nothing, Blue reward:0.85\n", - "step: 1284, Red action: do-nothing, Blue reward:0.85\n", - "step: 1285, Red action: do-nothing, Blue reward:0.85\n", - "step: 1286, Red action: do-nothing, Blue reward:0.85\n", - "step: 1287, Red action: do-nothing, Blue reward:0.85\n", - "step: 1288, Red action: do-nothing, Blue reward:0.85\n", - "step: 1289, Red action: do-nothing, Blue reward:0.85\n", - "step: 1290, Red action: do-nothing, Blue reward:0.85\n", - "step: 1291, Red action: do-nothing, Blue reward:0.85\n", - "step: 1292, Red action: do-nothing, Blue reward:0.85\n", - "step: 1293, Red action: do-nothing, Blue reward:0.85\n", - "step: 1294, Red action: do-nothing, Blue reward:0.85\n", - "step: 1295, Red action: do-nothing, Blue reward:0.85\n", - "step: 1296, Red action: do-nothing, Blue reward:0.85\n", - "step: 1297, Red action: do-nothing, Blue reward:0.85\n", - "step: 1298, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1299, Red action: do-nothing, Blue reward:0.85\n", - "step: 1300, Red action: do-nothing, Blue reward:0.85\n", - "step: 1301, Red action: do-nothing, Blue reward:0.85\n", - "step: 1302, Red action: do-nothing, Blue reward:0.85\n", - "step: 1303, Red action: do-nothing, Blue reward:0.85\n", - "step: 1304, Red action: do-nothing, Blue reward:0.85\n", - "step: 1305, Red action: do-nothing, Blue reward:0.85\n", - "step: 1306, Red action: do-nothing, Blue reward:0.85\n", - "step: 1307, Red action: do-nothing, Blue reward:0.85\n", - "step: 1308, Red action: do-nothing, Blue reward:0.85\n", - "step: 1309, Red action: do-nothing, Blue reward:0.85\n", - "step: 1310, Red action: do-nothing, Blue reward:0.85\n", - "step: 1311, Red action: do-nothing, Blue reward:0.85\n", - "step: 1312, Red action: do-nothing, Blue reward:0.85\n", - "step: 1313, Red action: do-nothing, Blue reward:0.85\n", - "step: 1314, Red action: do-nothing, Blue reward:0.85\n", - "step: 1315, Red action: do-nothing, Blue reward:0.85\n", - "step: 1316, Red action: do-nothing, Blue reward:0.85\n", - "step: 1317, Red action: do-nothing, Blue reward:0.85\n", - "step: 1318, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1319, Red action: do-nothing, Blue reward:0.85\n", - "step: 1320, Red action: do-nothing, Blue reward:0.85\n", - "step: 1321, Red action: do-nothing, Blue reward:0.85\n", - "step: 1322, Red action: do-nothing, Blue reward:0.85\n", - "step: 1323, Red action: do-nothing, Blue reward:0.85\n", - "step: 1324, Red action: do-nothing, Blue reward:0.85\n", - "step: 1325, Red action: do-nothing, Blue reward:0.85\n", - "step: 1326, Red action: do-nothing, Blue reward:0.85\n", - "step: 1327, Red action: do-nothing, Blue reward:0.85\n", - "step: 1328, Red action: do-nothing, Blue reward:0.85\n", - "step: 1329, Red action: do-nothing, Blue reward:0.85\n", - "step: 1330, Red action: do-nothing, Blue reward:0.85\n", - "step: 1331, Red action: do-nothing, Blue reward:0.85\n", - "step: 1332, Red action: do-nothing, Blue reward:0.85\n", - "step: 1333, Red action: do-nothing, Blue reward:0.85\n", - "step: 1334, Red action: do-nothing, Blue reward:0.85\n", - "step: 1335, Red action: do-nothing, Blue reward:0.85\n", - "step: 1336, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1337, Red action: do-nothing, Blue reward:0.85\n", - "step: 1338, Red action: do-nothing, Blue reward:0.85\n", - "step: 1339, Red action: do-nothing, Blue reward:0.85\n", - "step: 1340, Red action: do-nothing, Blue reward:0.85\n", - "step: 1341, Red action: do-nothing, Blue reward:0.85\n", - "step: 1342, Red action: do-nothing, Blue reward:0.85\n", - "step: 1343, Red action: do-nothing, Blue reward:0.85\n", - "step: 1344, Red action: do-nothing, Blue reward:0.85\n", - "step: 1345, Red action: do-nothing, Blue reward:0.85\n", - "step: 1346, Red action: do-nothing, Blue reward:0.85\n", - "step: 1347, Red action: do-nothing, Blue reward:0.85\n", - "step: 1348, Red action: do-nothing, Blue reward:0.85\n", - "step: 1349, Red action: do-nothing, Blue reward:0.85\n", - "step: 1350, Red action: do-nothing, Blue reward:0.85\n", - "step: 1351, Red action: do-nothing, Blue reward:0.85\n", - "step: 1352, Red action: do-nothing, Blue reward:0.85\n", - "step: 1353, Red action: do-nothing, Blue reward:0.85\n", - "step: 1354, Red action: do-nothing, Blue reward:0.85\n", - "step: 1355, Red action: do-nothing, Blue reward:0.85\n", - "step: 1356, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1357, Red action: do-nothing, Blue reward:0.85\n", - "step: 1358, Red action: do-nothing, Blue reward:0.85\n", - "step: 1359, Red action: do-nothing, Blue reward:0.85\n", - "step: 1360, Red action: do-nothing, Blue reward:0.85\n", - "step: 1361, Red action: do-nothing, Blue reward:0.85\n", - "step: 1362, Red action: do-nothing, Blue reward:0.85\n", - "step: 1363, Red action: do-nothing, Blue reward:0.85\n", - "step: 1364, Red action: do-nothing, Blue reward:0.85\n", - "step: 1365, Red action: do-nothing, Blue reward:0.85\n", - "step: 1366, Red action: do-nothing, Blue reward:0.85\n", - "step: 1367, Red action: do-nothing, Blue reward:0.85\n", - "step: 1368, Red action: do-nothing, Blue reward:0.85\n", - "step: 1369, Red action: do-nothing, Blue reward:0.85\n", - "step: 1370, Red action: do-nothing, Blue reward:0.85\n", - "step: 1371, Red action: do-nothing, Blue reward:0.85\n", - "step: 1372, Red action: do-nothing, Blue reward:0.85\n", - "step: 1373, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1374, Red action: do-nothing, Blue reward:0.85\n", - "step: 1375, Red action: do-nothing, Blue reward:0.85\n", - "step: 1376, Red action: do-nothing, Blue reward:0.85\n", - "step: 1377, Red action: do-nothing, Blue reward:0.85\n", - "step: 1378, Red action: do-nothing, Blue reward:0.85\n", - "step: 1379, Red action: do-nothing, Blue reward:0.85\n", - "step: 1380, Red action: do-nothing, Blue reward:0.85\n", - "step: 1381, Red action: do-nothing, Blue reward:0.85\n", - "step: 1382, Red action: do-nothing, Blue reward:0.85\n", - "step: 1383, Red action: do-nothing, Blue reward:0.85\n", - "step: 1384, Red action: do-nothing, Blue reward:0.85\n", - "step: 1385, Red action: do-nothing, Blue reward:0.85\n", - "step: 1386, Red action: do-nothing, Blue reward:0.85\n", - "step: 1387, Red action: do-nothing, Blue reward:0.85\n", - "step: 1388, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1389, Red action: do-nothing, Blue reward:0.85\n", - "step: 1390, Red action: do-nothing, Blue reward:0.85\n", - "step: 1391, Red action: do-nothing, Blue reward:0.85\n", - "step: 1392, Red action: do-nothing, Blue reward:0.85\n", - "step: 1393, Red action: do-nothing, Blue reward:0.85\n", - "step: 1394, Red action: do-nothing, Blue reward:0.85\n", - "step: 1395, Red action: do-nothing, Blue reward:0.85\n", - "step: 1396, Red action: do-nothing, Blue reward:0.85\n", - "step: 1397, Red action: do-nothing, Blue reward:0.85\n", - "step: 1398, Red action: do-nothing, Blue reward:0.85\n", - "step: 1399, Red action: do-nothing, Blue reward:0.85\n", - "step: 1400, Red action: do-nothing, Blue reward:0.85\n", - "step: 1401, Red action: do-nothing, Blue reward:0.85\n", - "step: 1402, Red action: do-nothing, Blue reward:0.85\n", - "step: 1403, Red action: do-nothing, Blue reward:0.85\n", - "step: 1404, Red action: do-nothing, Blue reward:0.85\n", - "step: 1405, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1406, Red action: do-nothing, Blue reward:0.85\n", - "step: 1407, Red action: do-nothing, Blue reward:0.85\n", - "step: 1408, Red action: do-nothing, Blue reward:0.85\n", - "step: 1409, Red action: do-nothing, Blue reward:0.85\n", - "step: 1410, Red action: do-nothing, Blue reward:0.85\n", - "step: 1411, Red action: do-nothing, Blue reward:0.85\n", - "step: 1412, Red action: do-nothing, Blue reward:0.85\n", - "step: 1413, Red action: do-nothing, Blue reward:0.85\n", - "step: 1414, Red action: do-nothing, Blue reward:0.85\n", - "step: 1415, Red action: do-nothing, Blue reward:0.85\n", - "step: 1416, Red action: do-nothing, Blue reward:0.85\n", - "step: 1417, Red action: do-nothing, Blue reward:0.85\n", - "step: 1418, Red action: do-nothing, Blue reward:0.85\n", - "step: 1419, Red action: do-nothing, Blue reward:0.85\n", - "step: 1420, Red action: do-nothing, Blue reward:0.85\n", - "step: 1421, Red action: do-nothing, Blue reward:0.85\n", - "step: 1422, Red action: do-nothing, Blue reward:0.85\n", - "step: 1423, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1424, Red action: do-nothing, Blue reward:0.85\n", - "step: 1425, Red action: do-nothing, Blue reward:0.85\n", - "step: 1426, Red action: do-nothing, Blue reward:0.85\n", - "step: 1427, Red action: do-nothing, Blue reward:0.85\n", - "step: 1428, Red action: do-nothing, Blue reward:0.85\n", - "step: 1429, Red action: do-nothing, Blue reward:0.85\n", - "step: 1430, Red action: do-nothing, Blue reward:0.85\n", - "step: 1431, Red action: do-nothing, Blue reward:0.85\n", - "step: 1432, Red action: do-nothing, Blue reward:0.85\n", - "step: 1433, Red action: do-nothing, Blue reward:0.85\n", - "step: 1434, Red action: do-nothing, Blue reward:0.85\n", - "step: 1435, Red action: do-nothing, Blue reward:0.85\n", - "step: 1436, Red action: do-nothing, Blue reward:0.85\n", - "step: 1437, Red action: do-nothing, Blue reward:0.85\n", - "step: 1438, Red action: do-nothing, Blue reward:0.85\n", - "step: 1439, Red action: do-nothing, Blue reward:0.85\n", - "step: 1440, Red action: do-nothing, Blue reward:0.85\n", - "step: 1441, Red action: do-nothing, Blue reward:0.85\n", - "step: 1442, Red action: do-nothing, Blue reward:0.85\n", - "step: 1443, Red action: do-nothing, Blue reward:0.85\n", - "step: 1444, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1445, Red action: do-nothing, Blue reward:0.85\n", - "step: 1446, Red action: do-nothing, Blue reward:0.85\n", - "step: 1447, Red action: do-nothing, Blue reward:0.85\n", - "step: 1448, Red action: do-nothing, Blue reward:0.85\n", - "step: 1449, Red action: do-nothing, Blue reward:0.85\n", - "step: 1450, Red action: do-nothing, Blue reward:0.85\n", - "step: 1451, Red action: do-nothing, Blue reward:0.85\n", - "step: 1452, Red action: do-nothing, Blue reward:0.85\n", - "step: 1453, Red action: do-nothing, Blue reward:0.85\n", - "step: 1454, Red action: do-nothing, Blue reward:0.85\n", - "step: 1455, Red action: do-nothing, Blue reward:0.85\n", - "step: 1456, Red action: do-nothing, Blue reward:0.85\n", - "step: 1457, Red action: do-nothing, Blue reward:0.85\n", - "step: 1458, Red action: do-nothing, Blue reward:0.85\n", - "step: 1459, Red action: do-nothing, Blue reward:0.85\n", - "step: 1460, Red action: do-nothing, Blue reward:0.85\n", - "step: 1461, Red action: do-nothing, Blue reward:0.85\n", - "step: 1462, Red action: do-nothing, Blue reward:0.85\n", - "step: 1463, Red action: do-nothing, Blue reward:0.85\n", - "step: 1464, Red action: do-nothing, Blue reward:0.85\n", - "step: 1465, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1466, Red action: do-nothing, Blue reward:0.85\n", - "step: 1467, Red action: do-nothing, Blue reward:0.85\n", - "step: 1468, Red action: do-nothing, Blue reward:0.85\n", - "step: 1469, Red action: do-nothing, Blue reward:0.85\n", - "step: 1470, Red action: do-nothing, Blue reward:0.85\n", - "step: 1471, Red action: do-nothing, Blue reward:0.85\n", - "step: 1472, Red action: do-nothing, Blue reward:0.85\n", - "step: 1473, Red action: do-nothing, Blue reward:0.85\n", - "step: 1474, Red action: do-nothing, Blue reward:0.85\n", - "step: 1475, Red action: do-nothing, Blue reward:0.85\n", - "step: 1476, Red action: do-nothing, Blue reward:0.85\n", - "step: 1477, Red action: do-nothing, Blue reward:0.85\n", - "step: 1478, Red action: do-nothing, Blue reward:0.85\n", - "step: 1479, Red action: do-nothing, Blue reward:0.85\n", - "step: 1480, Red action: do-nothing, Blue reward:0.85\n", - "step: 1481, Red action: do-nothing, Blue reward:0.85\n", - "step: 1482, Red action: do-nothing, Blue reward:0.85\n", - "step: 1483, Red action: do-nothing, Blue reward:0.85\n", - "step: 1484, Red action: do-nothing, Blue reward:0.85\n", - "step: 1485, Red action: do-nothing, Blue reward:0.85\n", - "step: 1486, Red action: do-nothing, Blue reward:0.85\n", - "step: 1487, Red action: do-nothing, Blue reward:0.85\n", - "step: 1488, Red action: do-nothing, Blue reward:0.85\n", - "step: 1489, Red action: do-nothing, Blue reward:0.85\n", - "step: 1490, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1491, Red action: do-nothing, Blue reward:0.85\n", - "step: 1492, Red action: do-nothing, Blue reward:0.85\n", - "step: 1493, Red action: do-nothing, Blue reward:0.85\n", - "step: 1494, Red action: do-nothing, Blue reward:0.85\n", - "step: 1495, Red action: do-nothing, Blue reward:0.85\n", - "step: 1496, Red action: do-nothing, Blue reward:0.85\n", - "step: 1497, Red action: do-nothing, Blue reward:0.85\n", - "step: 1498, Red action: do-nothing, Blue reward:0.85\n", - "step: 1499, Red action: do-nothing, Blue reward:0.85\n", - "step: 1500, Red action: do-nothing, Blue reward:0.85\n", - "step: 1501, Red action: do-nothing, Blue reward:0.85\n", - "step: 1502, Red action: do-nothing, Blue reward:0.85\n", - "step: 1503, Red action: do-nothing, Blue reward:0.85\n", - "step: 1504, Red action: do-nothing, Blue reward:0.85\n", - "step: 1505, Red action: do-nothing, Blue reward:0.85\n", - "step: 1506, Red action: do-nothing, Blue reward:0.85\n", - "step: 1507, Red action: do-nothing, Blue reward:0.85\n", - "step: 1508, Red action: do-nothing, Blue reward:0.85\n", - "step: 1509, Red action: do-nothing, Blue reward:0.85\n", - "step: 1510, Red action: do-nothing, Blue reward:0.85\n", - "step: 1511, Red action: do-nothing, Blue reward:0.85\n", - "step: 1512, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1513, Red action: do-nothing, Blue reward:0.85\n", - "step: 1514, Red action: do-nothing, Blue reward:0.85\n", - "step: 1515, Red action: do-nothing, Blue reward:0.85\n", - "step: 1516, Red action: do-nothing, Blue reward:0.85\n", - "step: 1517, Red action: do-nothing, Blue reward:0.85\n", - "step: 1518, Red action: do-nothing, Blue reward:0.85\n", - "step: 1519, Red action: do-nothing, Blue reward:0.85\n", - "step: 1520, Red action: do-nothing, Blue reward:0.85\n", - "step: 1521, Red action: do-nothing, Blue reward:0.85\n", - "step: 1522, Red action: do-nothing, Blue reward:0.85\n", - "step: 1523, Red action: do-nothing, Blue reward:0.85\n", - "step: 1524, Red action: do-nothing, Blue reward:0.85\n", - "step: 1525, Red action: do-nothing, Blue reward:0.85\n", - "step: 1526, Red action: do-nothing, Blue reward:0.85\n", - "step: 1527, Red action: do-nothing, Blue reward:0.85\n", - "step: 1528, Red action: do-nothing, Blue reward:0.85\n", - "step: 1529, Red action: do-nothing, Blue reward:0.85\n", - "step: 1530, Red action: do-nothing, Blue reward:0.85\n", - "step: 1531, Red action: do-nothing, Blue reward:0.85\n", - "step: 1532, Red action: do-nothing, Blue reward:0.85\n", - "step: 1533, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1534, Red action: do-nothing, Blue reward:0.85\n", - "step: 1535, Red action: do-nothing, Blue reward:0.85\n", - "step: 1536, Red action: do-nothing, Blue reward:0.85\n", - "step: 1537, Red action: do-nothing, Blue reward:0.85\n", - "step: 1538, Red action: do-nothing, Blue reward:0.85\n", - "step: 1539, Red action: do-nothing, Blue reward:0.85\n", - "step: 1540, Red action: do-nothing, Blue reward:0.85\n", - "step: 1541, Red action: do-nothing, Blue reward:0.85\n", - "step: 1542, Red action: do-nothing, Blue reward:0.85\n", - "step: 1543, Red action: do-nothing, Blue reward:0.85\n", - "step: 1544, Red action: do-nothing, Blue reward:0.85\n", - "step: 1545, Red action: do-nothing, Blue reward:0.85\n", - "step: 1546, Red action: do-nothing, Blue reward:0.85\n", - "step: 1547, Red action: do-nothing, Blue reward:0.85\n", - "step: 1548, Red action: do-nothing, Blue reward:0.85\n", - "step: 1549, Red action: do-nothing, Blue reward:0.85\n", - "step: 1550, Red action: do-nothing, Blue reward:0.85\n", - "step: 1551, Red action: do-nothing, Blue reward:0.85\n", - "step: 1552, Red action: do-nothing, Blue reward:0.85\n", - "step: 1553, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1554, Red action: do-nothing, Blue reward:0.85\n", - "step: 1555, Red action: do-nothing, Blue reward:0.85\n", - "step: 1556, Red action: do-nothing, Blue reward:0.85\n", - "step: 1557, Red action: do-nothing, Blue reward:0.85\n", - "step: 1558, Red action: do-nothing, Blue reward:0.85\n", - "step: 1559, Red action: do-nothing, Blue reward:0.85\n", - "step: 1560, Red action: do-nothing, Blue reward:0.85\n", - "step: 1561, Red action: do-nothing, Blue reward:0.85\n", - "step: 1562, Red action: do-nothing, Blue reward:0.85\n", - "step: 1563, Red action: do-nothing, Blue reward:0.85\n", - "step: 1564, Red action: do-nothing, Blue reward:0.85\n", - "step: 1565, Red action: do-nothing, Blue reward:0.85\n", - "step: 1566, Red action: do-nothing, Blue reward:0.85\n", - "step: 1567, Red action: do-nothing, Blue reward:0.85\n", - "step: 1568, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1569, Red action: do-nothing, Blue reward:0.85\n", - "step: 1570, Red action: do-nothing, Blue reward:0.85\n", - "step: 1571, Red action: do-nothing, Blue reward:0.85\n", - "step: 1572, Red action: do-nothing, Blue reward:0.85\n", - "step: 1573, Red action: do-nothing, Blue reward:0.85\n", - "step: 1574, Red action: do-nothing, Blue reward:0.85\n", - "step: 1575, Red action: do-nothing, Blue reward:0.85\n", - "step: 1576, Red action: do-nothing, Blue reward:0.85\n", - "step: 1577, Red action: do-nothing, Blue reward:0.85\n", - "step: 1578, Red action: do-nothing, Blue reward:0.85\n", - "step: 1579, Red action: do-nothing, Blue reward:0.85\n", - "step: 1580, Red action: do-nothing, Blue reward:0.85\n", - "step: 1581, Red action: do-nothing, Blue reward:0.85\n", - "step: 1582, Red action: do-nothing, Blue reward:0.85\n", - "step: 1583, Red action: do-nothing, Blue reward:0.85\n", - "step: 1584, Red action: do-nothing, Blue reward:0.85\n", - "step: 1585, Red action: do-nothing, Blue reward:0.85\n", - "step: 1586, Red action: do-nothing, Blue reward:0.85\n", - "step: 1587, Red action: do-nothing, Blue reward:0.85\n", - "step: 1588, Red action: do-nothing, Blue reward:0.85\n", - "step: 1589, Red action: do-nothing, Blue reward:0.85\n", - "step: 1590, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1591, Red action: do-nothing, Blue reward:0.85\n", - "step: 1592, Red action: do-nothing, Blue reward:0.85\n", - "step: 1593, Red action: do-nothing, Blue reward:0.85\n", - "step: 1594, Red action: do-nothing, Blue reward:0.85\n", - "step: 1595, Red action: do-nothing, Blue reward:0.85\n", - "step: 1596, Red action: do-nothing, Blue reward:0.85\n", - "step: 1597, Red action: do-nothing, Blue reward:0.85\n", - "step: 1598, Red action: do-nothing, Blue reward:0.85\n", - "step: 1599, Red action: do-nothing, Blue reward:0.85\n", - "step: 1600, Red action: do-nothing, Blue reward:0.85\n", - "step: 1601, Red action: do-nothing, Blue reward:0.85\n", - "step: 1602, Red action: do-nothing, Blue reward:0.85\n", - "step: 1603, Red action: do-nothing, Blue reward:0.85\n", - "step: 1604, Red action: do-nothing, Blue reward:0.85\n", - "step: 1605, Red action: do-nothing, Blue reward:0.85\n", - "step: 1606, Red action: do-nothing, Blue reward:0.85\n", - "step: 1607, Red action: do-nothing, Blue reward:0.85\n", - "step: 1608, Red action: do-nothing, Blue reward:0.85\n", - "step: 1609, Red action: do-nothing, Blue reward:0.85\n", - "step: 1610, Red action: do-nothing, Blue reward:0.85\n", - "step: 1611, Red action: do-nothing, Blue reward:0.85\n", - "step: 1612, Red action: do-nothing, Blue reward:0.85\n", - "step: 1613, Red action: do-nothing, Blue reward:0.85\n", - "step: 1614, Red action: do-nothing, Blue reward:0.85\n", - "step: 1615, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1616, Red action: do-nothing, Blue reward:0.85\n", - "step: 1617, Red action: do-nothing, Blue reward:0.85\n", - "step: 1618, Red action: do-nothing, Blue reward:0.85\n", - "step: 1619, Red action: do-nothing, Blue reward:0.85\n", - "step: 1620, Red action: do-nothing, Blue reward:0.85\n", - "step: 1621, Red action: do-nothing, Blue reward:0.85\n", - "step: 1622, Red action: do-nothing, Blue reward:0.85\n", - "step: 1623, Red action: do-nothing, Blue reward:0.85\n", - "step: 1624, Red action: do-nothing, Blue reward:0.85\n", - "step: 1625, Red action: do-nothing, Blue reward:0.85\n", - "step: 1626, Red action: do-nothing, Blue reward:0.85\n", - "step: 1627, Red action: do-nothing, Blue reward:0.85\n", - "step: 1628, Red action: do-nothing, Blue reward:0.85\n", - "step: 1629, Red action: do-nothing, Blue reward:0.85\n", - "step: 1630, Red action: do-nothing, Blue reward:0.85\n", - "step: 1631, Red action: do-nothing, Blue reward:0.85\n", - "step: 1632, Red action: do-nothing, Blue reward:0.85\n", - "step: 1633, Red action: do-nothing, Blue reward:0.85\n", - "step: 1634, Red action: do-nothing, Blue reward:0.85\n", - "step: 1635, Red action: do-nothing, Blue reward:0.85\n", - "step: 1636, Red action: do-nothing, Blue reward:0.85\n", - "step: 1637, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1638, Red action: do-nothing, Blue reward:0.85\n", - "step: 1639, Red action: do-nothing, Blue reward:0.85\n", - "step: 1640, Red action: do-nothing, Blue reward:0.85\n", - "step: 1641, Red action: do-nothing, Blue reward:0.85\n", - "step: 1642, Red action: do-nothing, Blue reward:0.85\n", - "step: 1643, Red action: do-nothing, Blue reward:0.85\n", - "step: 1644, Red action: do-nothing, Blue reward:0.85\n", - "step: 1645, Red action: do-nothing, Blue reward:0.85\n", - "step: 1646, Red action: do-nothing, Blue reward:0.85\n", - "step: 1647, Red action: do-nothing, Blue reward:0.85\n", - "step: 1648, Red action: do-nothing, Blue reward:0.85\n", - "step: 1649, Red action: do-nothing, Blue reward:0.85\n", - "step: 1650, Red action: do-nothing, Blue reward:0.85\n", - "step: 1651, Red action: do-nothing, Blue reward:0.85\n", - "step: 1652, Red action: do-nothing, Blue reward:0.85\n", - "step: 1653, Red action: do-nothing, Blue reward:0.85\n", - "step: 1654, Red action: do-nothing, Blue reward:0.85\n", - "step: 1655, Red action: do-nothing, Blue reward:0.85\n", - "step: 1656, Red action: do-nothing, Blue reward:0.85\n", - "step: 1657, Red action: do-nothing, Blue reward:0.85\n", - "step: 1658, Red action: do-nothing, Blue reward:0.85\n", - "step: 1659, Red action: do-nothing, Blue reward:0.85\n", - "step: 1660, Red action: do-nothing, Blue reward:0.85\n", - "step: 1661, Red action: do-nothing, Blue reward:0.85\n", - "step: 1662, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1663, Red action: do-nothing, Blue reward:0.85\n", - "step: 1664, Red action: do-nothing, Blue reward:0.85\n", - "step: 1665, Red action: do-nothing, Blue reward:0.85\n", - "step: 1666, Red action: do-nothing, Blue reward:0.85\n", - "step: 1667, Red action: do-nothing, Blue reward:0.85\n", - "step: 1668, Red action: do-nothing, Blue reward:0.85\n", - "step: 1669, Red action: do-nothing, Blue reward:0.85\n", - "step: 1670, Red action: do-nothing, Blue reward:0.85\n", - "step: 1671, Red action: do-nothing, Blue reward:0.85\n", - "step: 1672, Red action: do-nothing, Blue reward:0.85\n", - "step: 1673, Red action: do-nothing, Blue reward:0.85\n", - "step: 1674, Red action: do-nothing, Blue reward:0.85\n", - "step: 1675, Red action: do-nothing, Blue reward:0.85\n", - "step: 1676, Red action: do-nothing, Blue reward:0.85\n", - "step: 1677, Red action: do-nothing, Blue reward:0.85\n", - "step: 1678, Red action: do-nothing, Blue reward:0.85\n", - "step: 1679, Red action: do-nothing, Blue reward:0.85\n", - "step: 1680, Red action: do-nothing, Blue reward:0.85\n", - "step: 1681, Red action: do-nothing, Blue reward:0.85\n", - "step: 1682, Red action: do-nothing, Blue reward:0.85\n", - "step: 1683, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1684, Red action: do-nothing, Blue reward:0.85\n", - "step: 1685, Red action: do-nothing, Blue reward:0.85\n", - "step: 1686, Red action: do-nothing, Blue reward:0.85\n", - "step: 1687, Red action: do-nothing, Blue reward:0.85\n", - "step: 1688, Red action: do-nothing, Blue reward:0.85\n", - "step: 1689, Red action: do-nothing, Blue reward:0.85\n", - "step: 1690, Red action: do-nothing, Blue reward:0.85\n", - "step: 1691, Red action: do-nothing, Blue reward:0.85\n", - "step: 1692, Red action: do-nothing, Blue reward:0.85\n", - "step: 1693, Red action: do-nothing, Blue reward:0.85\n", - "step: 1694, Red action: do-nothing, Blue reward:0.85\n", - "step: 1695, Red action: do-nothing, Blue reward:0.85\n", - "step: 1696, Red action: do-nothing, Blue reward:0.85\n", - "step: 1697, Red action: do-nothing, Blue reward:0.85\n", - "step: 1698, Red action: do-nothing, Blue reward:0.85\n", - "step: 1699, Red action: do-nothing, Blue reward:0.85\n", - "step: 1700, Red action: do-nothing, Blue reward:0.85\n", - "step: 1701, Red action: do-nothing, Blue reward:0.85\n", - "step: 1702, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1703, Red action: do-nothing, Blue reward:0.85\n", - "step: 1704, Red action: do-nothing, Blue reward:0.85\n", - "step: 1705, Red action: do-nothing, Blue reward:0.85\n", - "step: 1706, Red action: do-nothing, Blue reward:0.85\n", - "step: 1707, Red action: do-nothing, Blue reward:0.85\n", - "step: 1708, Red action: do-nothing, Blue reward:0.85\n", - "step: 1709, Red action: do-nothing, Blue reward:0.85\n", - "step: 1710, Red action: do-nothing, Blue reward:0.85\n", - "step: 1711, Red action: do-nothing, Blue reward:0.85\n", - "step: 1712, Red action: do-nothing, Blue reward:0.85\n", - "step: 1713, Red action: do-nothing, Blue reward:0.85\n", - "step: 1714, Red action: do-nothing, Blue reward:0.85\n", - "step: 1715, Red action: do-nothing, Blue reward:0.85\n", - "step: 1716, Red action: do-nothing, Blue reward:0.85\n", - "step: 1717, Red action: do-nothing, Blue reward:0.85\n", - "step: 1718, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1719, Red action: do-nothing, Blue reward:0.85\n", - "step: 1720, Red action: do-nothing, Blue reward:0.85\n", - "step: 1721, Red action: do-nothing, Blue reward:0.85\n", - "step: 1722, Red action: do-nothing, Blue reward:0.85\n", - "step: 1723, Red action: do-nothing, Blue reward:0.85\n", - "step: 1724, Red action: do-nothing, Blue reward:0.85\n", - "step: 1725, Red action: do-nothing, Blue reward:0.85\n", - "step: 1726, Red action: do-nothing, Blue reward:0.85\n", - "step: 1727, Red action: do-nothing, Blue reward:0.85\n", - "step: 1728, Red action: do-nothing, Blue reward:0.85\n", - "step: 1729, Red action: do-nothing, Blue reward:0.85\n", - "step: 1730, Red action: do-nothing, Blue reward:0.85\n", - "step: 1731, Red action: do-nothing, Blue reward:0.85\n", - "step: 1732, Red action: do-nothing, Blue reward:0.85\n", - "step: 1733, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1734, Red action: do-nothing, Blue reward:0.85\n", - "step: 1735, Red action: do-nothing, Blue reward:0.85\n", - "step: 1736, Red action: do-nothing, Blue reward:0.85\n", - "step: 1737, Red action: do-nothing, Blue reward:0.85\n", - "step: 1738, Red action: do-nothing, Blue reward:0.85\n", - "step: 1739, Red action: do-nothing, Blue reward:0.85\n", - "step: 1740, Red action: do-nothing, Blue reward:0.85\n", - "step: 1741, Red action: do-nothing, Blue reward:0.85\n", - "step: 1742, Red action: do-nothing, Blue reward:0.85\n", - "step: 1743, Red action: do-nothing, Blue reward:0.85\n", - "step: 1744, Red action: do-nothing, Blue reward:0.85\n", - "step: 1745, Red action: do-nothing, Blue reward:0.85\n", - "step: 1746, Red action: do-nothing, Blue reward:0.85\n", - "step: 1747, Red action: do-nothing, Blue reward:0.85\n", - "step: 1748, Red action: do-nothing, Blue reward:0.85\n", - "step: 1749, Red action: do-nothing, Blue reward:0.85\n", - "step: 1750, Red action: do-nothing, Blue reward:0.85\n", - "step: 1751, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1752, Red action: do-nothing, Blue reward:0.85\n", - "step: 1753, Red action: do-nothing, Blue reward:0.85\n", - "step: 1754, Red action: do-nothing, Blue reward:0.85\n", - "step: 1755, Red action: do-nothing, Blue reward:0.85\n", - "step: 1756, Red action: do-nothing, Blue reward:0.85\n", - "step: 1757, Red action: do-nothing, Blue reward:0.85\n", - "step: 1758, Red action: do-nothing, Blue reward:0.85\n", - "step: 1759, Red action: do-nothing, Blue reward:0.85\n", - "step: 1760, Red action: do-nothing, Blue reward:0.85\n", - "step: 1761, Red action: do-nothing, Blue reward:0.85\n", - "step: 1762, Red action: do-nothing, Blue reward:0.85\n", - "step: 1763, Red action: do-nothing, Blue reward:0.85\n", - "step: 1764, Red action: do-nothing, Blue reward:0.85\n", - "step: 1765, Red action: do-nothing, Blue reward:0.85\n", - "step: 1766, Red action: do-nothing, Blue reward:0.85\n", - "step: 1767, Red action: do-nothing, Blue reward:0.85\n", - "step: 1768, Red action: do-nothing, Blue reward:0.85\n", - "step: 1769, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1770, Red action: do-nothing, Blue reward:0.85\n", - "step: 1771, Red action: do-nothing, Blue reward:0.85\n", - "step: 1772, Red action: do-nothing, Blue reward:0.85\n", - "step: 1773, Red action: do-nothing, Blue reward:0.85\n", - "step: 1774, Red action: do-nothing, Blue reward:0.85\n", - "step: 1775, Red action: do-nothing, Blue reward:0.85\n", - "step: 1776, Red action: do-nothing, Blue reward:0.85\n", - "step: 1777, Red action: do-nothing, Blue reward:0.85\n", - "step: 1778, Red action: do-nothing, Blue reward:0.85\n", - "step: 1779, Red action: do-nothing, Blue reward:0.85\n", - "step: 1780, Red action: do-nothing, Blue reward:0.85\n", - "step: 1781, Red action: do-nothing, Blue reward:0.85\n", - "step: 1782, Red action: do-nothing, Blue reward:0.85\n", - "step: 1783, Red action: do-nothing, Blue reward:0.85\n", - "step: 1784, Red action: do-nothing, Blue reward:0.85\n", - "step: 1785, Red action: do-nothing, Blue reward:0.85\n", - "step: 1786, Red action: do-nothing, Blue reward:0.85\n", - "step: 1787, Red action: do-nothing, Blue reward:0.85\n", - "step: 1788, Red action: do-nothing, Blue reward:0.85\n", - "step: 1789, Red action: do-nothing, Blue reward:0.85\n", - "step: 1790, Red action: do-nothing, Blue reward:0.85\n", - "step: 1791, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1792, Red action: do-nothing, Blue reward:0.85\n", - "step: 1793, Red action: do-nothing, Blue reward:0.85\n", - "step: 1794, Red action: do-nothing, Blue reward:0.85\n", - "step: 1795, Red action: do-nothing, Blue reward:0.85\n", - "step: 1796, Red action: do-nothing, Blue reward:0.85\n", - "step: 1797, Red action: do-nothing, Blue reward:0.85\n", - "step: 1798, Red action: do-nothing, Blue reward:0.85\n", - "step: 1799, Red action: do-nothing, Blue reward:0.85\n", - "step: 1800, Red action: do-nothing, Blue reward:0.85\n", - "step: 1801, Red action: do-nothing, Blue reward:0.85\n", - "step: 1802, Red action: do-nothing, Blue reward:0.85\n", - "step: 1803, Red action: do-nothing, Blue reward:0.85\n", - "step: 1804, Red action: do-nothing, Blue reward:0.85\n", - "step: 1805, Red action: do-nothing, Blue reward:0.85\n", - "step: 1806, Red action: do-nothing, Blue reward:0.85\n", - "step: 1807, Red action: do-nothing, Blue reward:0.85\n", - "step: 1808, Red action: do-nothing, Blue reward:0.85\n", - "step: 1809, Red action: do-nothing, Blue reward:0.85\n", - "step: 1810, Red action: do-nothing, Blue reward:0.85\n", - "step: 1811, Red action: do-nothing, Blue reward:0.85\n", - "step: 1812, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1813, Red action: do-nothing, Blue reward:0.85\n", - "step: 1814, Red action: do-nothing, Blue reward:0.85\n", - "step: 1815, Red action: do-nothing, Blue reward:0.85\n", - "step: 1816, Red action: do-nothing, Blue reward:0.85\n", - "step: 1817, Red action: do-nothing, Blue reward:0.85\n", - "step: 1818, Red action: do-nothing, Blue reward:0.85\n", - "step: 1819, Red action: do-nothing, Blue reward:0.85\n", - "step: 1820, Red action: do-nothing, Blue reward:0.85\n", - "step: 1821, Red action: do-nothing, Blue reward:0.85\n", - "step: 1822, Red action: do-nothing, Blue reward:0.85\n", - "step: 1823, Red action: do-nothing, Blue reward:0.85\n", - "step: 1824, Red action: do-nothing, Blue reward:0.85\n", - "step: 1825, Red action: do-nothing, Blue reward:0.85\n", - "step: 1826, Red action: do-nothing, Blue reward:0.85\n", - "step: 1827, Red action: do-nothing, Blue reward:0.85\n", - "step: 1828, Red action: do-nothing, Blue reward:0.85\n", - "step: 1829, Red action: do-nothing, Blue reward:0.85\n", - "step: 1830, Red action: do-nothing, Blue reward:0.85\n", - "step: 1831, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1832, Red action: do-nothing, Blue reward:0.85\n", - "step: 1833, Red action: do-nothing, Blue reward:0.85\n", - "step: 1834, Red action: do-nothing, Blue reward:0.85\n", - "step: 1835, Red action: do-nothing, Blue reward:0.85\n", - "step: 1836, Red action: do-nothing, Blue reward:0.85\n", - "step: 1837, Red action: do-nothing, Blue reward:0.85\n", - "step: 1838, Red action: do-nothing, Blue reward:0.85\n", - "step: 1839, Red action: do-nothing, Blue reward:0.85\n", - "step: 1840, Red action: do-nothing, Blue reward:0.85\n", - "step: 1841, Red action: do-nothing, Blue reward:0.85\n", - "step: 1842, Red action: do-nothing, Blue reward:0.85\n", - "step: 1843, Red action: do-nothing, Blue reward:0.85\n", - "step: 1844, Red action: do-nothing, Blue reward:0.85\n", - "step: 1845, Red action: do-nothing, Blue reward:0.85\n", - "step: 1846, Red action: do-nothing, Blue reward:0.85\n", - "step: 1847, Red action: do-nothing, Blue reward:0.85\n", - "step: 1848, Red action: do-nothing, Blue reward:0.85\n", - "step: 1849, Red action: do-nothing, Blue reward:0.85\n", - "step: 1850, Red action: do-nothing, Blue reward:0.85\n", - "step: 1851, Red action: do-nothing, Blue reward:0.85\n", - "step: 1852, Red action: do-nothing, Blue reward:0.85\n", - "step: 1853, Red action: do-nothing, Blue reward:0.85\n", - "step: 1854, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1855, Red action: do-nothing, Blue reward:0.85\n", - "step: 1856, Red action: do-nothing, Blue reward:0.85\n", - "step: 1857, Red action: do-nothing, Blue reward:0.85\n", - "step: 1858, Red action: do-nothing, Blue reward:0.85\n", - "step: 1859, Red action: do-nothing, Blue reward:0.85\n", - "step: 1860, Red action: do-nothing, Blue reward:0.85\n", - "step: 1861, Red action: do-nothing, Blue reward:0.85\n", - "step: 1862, Red action: do-nothing, Blue reward:0.85\n", - "step: 1863, Red action: do-nothing, Blue reward:0.85\n", - "step: 1864, Red action: do-nothing, Blue reward:0.85\n", - "step: 1865, Red action: do-nothing, Blue reward:0.85\n", - "step: 1866, Red action: do-nothing, Blue reward:0.85\n", - "step: 1867, Red action: do-nothing, Blue reward:0.85\n", - "step: 1868, Red action: do-nothing, Blue reward:0.85\n", - "step: 1869, Red action: do-nothing, Blue reward:0.85\n", - "step: 1870, Red action: do-nothing, Blue reward:0.85\n", - "step: 1871, Red action: do-nothing, Blue reward:0.85\n", - "step: 1872, Red action: do-nothing, Blue reward:0.85\n", - "step: 1873, Red action: do-nothing, Blue reward:0.85\n", - "step: 1874, Red action: do-nothing, Blue reward:0.85\n", - "step: 1875, Red action: do-nothing, Blue reward:0.85\n", - "step: 1876, Red action: do-nothing, Blue reward:0.85\n", - "step: 1877, Red action: do-nothing, Blue reward:0.85\n", - "step: 1878, Red action: do-nothing, Blue reward:0.85\n", - "step: 1879, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1880, Red action: do-nothing, Blue reward:0.85\n", - "step: 1881, Red action: do-nothing, Blue reward:0.85\n", - "step: 1882, Red action: do-nothing, Blue reward:0.85\n", - "step: 1883, Red action: do-nothing, Blue reward:0.85\n", - "step: 1884, Red action: do-nothing, Blue reward:0.85\n", - "step: 1885, Red action: do-nothing, Blue reward:0.85\n", - "step: 1886, Red action: do-nothing, Blue reward:0.85\n", - "step: 1887, Red action: do-nothing, Blue reward:0.85\n", - "step: 1888, Red action: do-nothing, Blue reward:0.85\n", - "step: 1889, Red action: do-nothing, Blue reward:0.85\n", - "step: 1890, Red action: do-nothing, Blue reward:0.85\n", - "step: 1891, Red action: do-nothing, Blue reward:0.85\n", - "step: 1892, Red action: do-nothing, Blue reward:0.85\n", - "step: 1893, Red action: do-nothing, Blue reward:0.85\n", - "step: 1894, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1895, Red action: do-nothing, Blue reward:0.85\n", - "step: 1896, Red action: do-nothing, Blue reward:0.85\n", - "step: 1897, Red action: do-nothing, Blue reward:0.85\n", - "step: 1898, Red action: do-nothing, Blue reward:0.85\n", - "step: 1899, Red action: do-nothing, Blue reward:0.85\n", - "step: 1900, Red action: do-nothing, Blue reward:0.85\n", - "step: 1901, Red action: do-nothing, Blue reward:0.85\n", - "step: 1902, Red action: do-nothing, Blue reward:0.85\n", - "step: 1903, Red action: do-nothing, Blue reward:0.85\n", - "step: 1904, Red action: do-nothing, Blue reward:0.85\n", - "step: 1905, Red action: do-nothing, Blue reward:0.85\n", - "step: 1906, Red action: do-nothing, Blue reward:0.85\n", - "step: 1907, Red action: do-nothing, Blue reward:0.85\n", - "step: 1908, Red action: do-nothing, Blue reward:0.85\n", - "step: 1909, Red action: do-nothing, Blue reward:0.85\n", - "step: 1910, Red action: do-nothing, Blue reward:0.85\n", - "step: 1911, Red action: do-nothing, Blue reward:0.85\n", - "step: 1912, Red action: do-nothing, Blue reward:0.85\n", - "step: 1913, Red action: do-nothing, Blue reward:0.85\n", - "step: 1914, Red action: do-nothing, Blue reward:0.85\n", - "step: 1915, Red action: do-nothing, Blue reward:0.85\n", - "step: 1916, Red action: do-nothing, Blue reward:0.85\n", - "step: 1917, Red action: do-nothing, Blue reward:0.85\n", - "step: 1918, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1919, Red action: do-nothing, Blue reward:0.85\n", - "step: 1920, Red action: do-nothing, Blue reward:0.85\n", - "step: 1921, Red action: do-nothing, Blue reward:0.85\n", - "step: 1922, Red action: do-nothing, Blue reward:0.85\n", - "step: 1923, Red action: do-nothing, Blue reward:0.85\n", - "step: 1924, Red action: do-nothing, Blue reward:0.85\n", - "step: 1925, Red action: do-nothing, Blue reward:0.85\n", - "step: 1926, Red action: do-nothing, Blue reward:0.85\n", - "step: 1927, Red action: do-nothing, Blue reward:0.85\n", - "step: 1928, Red action: do-nothing, Blue reward:0.85\n", - "step: 1929, Red action: do-nothing, Blue reward:0.85\n", - "step: 1930, Red action: do-nothing, Blue reward:0.85\n", - "step: 1931, Red action: do-nothing, Blue reward:0.85\n", - "step: 1932, Red action: do-nothing, Blue reward:0.85\n", - "step: 1933, Red action: do-nothing, Blue reward:0.85\n", - "step: 1934, Red action: do-nothing, Blue reward:0.85\n", - "step: 1935, Red action: do-nothing, Blue reward:0.85\n", - "step: 1936, Red action: do-nothing, Blue reward:0.85\n", - "step: 1937, Red action: do-nothing, Blue reward:0.85\n", - "step: 1938, Red action: do-nothing, Blue reward:0.85\n", - "step: 1939, Red action: do-nothing, Blue reward:0.85\n", - "step: 1940, Red action: do-nothing, Blue reward:0.85\n", - "step: 1941, Red action: do-nothing, Blue reward:0.85\n", - "step: 1942, Red action: do-nothing, Blue reward:0.85\n", - "step: 1943, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1944, Red action: do-nothing, Blue reward:0.85\n", - "step: 1945, Red action: do-nothing, Blue reward:0.85\n", - "step: 1946, Red action: do-nothing, Blue reward:0.85\n", - "step: 1947, Red action: do-nothing, Blue reward:0.85\n", - "step: 1948, Red action: do-nothing, Blue reward:0.85\n", - "step: 1949, Red action: do-nothing, Blue reward:0.85\n", - "step: 1950, Red action: do-nothing, Blue reward:0.85\n", - "step: 1951, Red action: do-nothing, Blue reward:0.85\n", - "step: 1952, Red action: do-nothing, Blue reward:0.85\n", - "step: 1953, Red action: do-nothing, Blue reward:0.85\n", - "step: 1954, Red action: do-nothing, Blue reward:0.85\n", - "step: 1955, Red action: do-nothing, Blue reward:0.85\n", - "step: 1956, Red action: do-nothing, Blue reward:0.85\n", - "step: 1957, Red action: do-nothing, Blue reward:0.85\n", - "step: 1958, Red action: do-nothing, Blue reward:0.85\n", - "step: 1959, Red action: do-nothing, Blue reward:0.85\n", - "step: 1960, Red action: do-nothing, Blue reward:0.85\n", - "step: 1961, Red action: do-nothing, Blue reward:0.85\n", - "step: 1962, Red action: do-nothing, Blue reward:0.85\n", - "step: 1963, Red action: do-nothing, Blue reward:0.85\n", - "step: 1964, Red action: do-nothing, Blue reward:0.85\n", - "step: 1965, Red action: do-nothing, Blue reward:0.85\n", - "step: 1966, Red action: do-nothing, Blue reward:0.85\n", - "step: 1967, Red action: do-nothing, Blue reward:0.85\n", - "step: 1968, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1969, Red action: do-nothing, Blue reward:0.85\n", - "step: 1970, Red action: do-nothing, Blue reward:0.85\n", - "step: 1971, Red action: do-nothing, Blue reward:0.85\n", - "step: 1972, Red action: do-nothing, Blue reward:0.85\n", - "step: 1973, Red action: do-nothing, Blue reward:0.85\n", - "step: 1974, Red action: do-nothing, Blue reward:0.85\n", - "step: 1975, Red action: do-nothing, Blue reward:0.85\n", - "step: 1976, Red action: do-nothing, Blue reward:0.85\n", - "step: 1977, Red action: do-nothing, Blue reward:0.85\n", - "step: 1978, Red action: do-nothing, Blue reward:0.85\n", - "step: 1979, Red action: do-nothing, Blue reward:0.85\n", - "step: 1980, Red action: do-nothing, Blue reward:0.85\n", - "step: 1981, Red action: do-nothing, Blue reward:0.85\n", - "step: 1982, Red action: do-nothing, Blue reward:0.85\n", - "step: 1983, Red action: do-nothing, Blue reward:0.85\n", - "step: 1984, Red action: do-nothing, Blue reward:0.85\n", - "step: 1985, Red action: do-nothing, Blue reward:0.85\n", - "step: 1986, Red action: do-nothing, Blue reward:0.85\n", - "step: 1987, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1988, Red action: do-nothing, Blue reward:0.85\n", - "step: 1989, Red action: do-nothing, Blue reward:0.85\n", - "step: 1990, Red action: do-nothing, Blue reward:0.85\n", - "step: 1991, Red action: do-nothing, Blue reward:0.85\n", - "step: 1992, Red action: do-nothing, Blue reward:0.85\n", - "step: 1993, Red action: do-nothing, Blue reward:0.85\n", - "step: 1994, Red action: do-nothing, Blue reward:0.85\n", - "step: 1995, Red action: do-nothing, Blue reward:0.85\n", - "step: 1996, Red action: do-nothing, Blue reward:0.85\n", - "step: 1997, Red action: do-nothing, Blue reward:0.85\n", - "step: 1998, Red action: do-nothing, Blue reward:0.85\n", - "step: 1999, Red action: do-nothing, Blue reward:0.85\n", - "step: 2000, Red action: do-nothing, Blue reward:0.85\n", - "step: 2001, Red action: do-nothing, Blue reward:0.85\n", - "step: 2002, Red action: do-nothing, Blue reward:0.85\n", - "step: 2003, Red action: do-nothing, Blue reward:0.85\n", - "step: 2004, Red action: do-nothing, Blue reward:0.85\n", - "step: 2005, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2006, Red action: do-nothing, Blue reward:0.85\n", - "step: 2007, Red action: do-nothing, Blue reward:0.85\n", - "step: 2008, Red action: do-nothing, Blue reward:0.85\n", - "step: 2009, Red action: do-nothing, Blue reward:0.85\n", - "step: 2010, Red action: do-nothing, Blue reward:0.85\n", - "step: 2011, Red action: do-nothing, Blue reward:0.85\n", - "step: 2012, Red action: do-nothing, Blue reward:0.85\n", - "step: 2013, Red action: do-nothing, Blue reward:0.85\n", - "step: 2014, Red action: do-nothing, Blue reward:0.85\n", - "step: 2015, Red action: do-nothing, Blue reward:0.85\n", - "step: 2016, Red action: do-nothing, Blue reward:0.85\n", - "step: 2017, Red action: do-nothing, Blue reward:0.85\n", - "step: 2018, Red action: do-nothing, Blue reward:0.85\n", - "step: 2019, Red action: do-nothing, Blue reward:0.85\n", - "step: 2020, Red action: do-nothing, Blue reward:0.85\n", - "step: 2021, Red action: do-nothing, Blue reward:0.85\n", - "step: 2022, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2023, Red action: do-nothing, Blue reward:0.85\n", - "step: 2024, Red action: do-nothing, Blue reward:0.85\n", - "step: 2025, Red action: do-nothing, Blue reward:0.85\n", - "step: 2026, Red action: do-nothing, Blue reward:0.85\n", - "step: 2027, Red action: do-nothing, Blue reward:0.85\n", - "step: 2028, Red action: do-nothing, Blue reward:0.85\n", - "step: 2029, Red action: do-nothing, Blue reward:0.85\n", - "step: 2030, Red action: do-nothing, Blue reward:0.85\n", - "step: 2031, Red action: do-nothing, Blue reward:0.85\n", - "step: 2032, Red action: do-nothing, Blue reward:0.85\n", - "step: 2033, Red action: do-nothing, Blue reward:0.85\n", - "step: 2034, Red action: do-nothing, Blue reward:0.85\n", - "step: 2035, Red action: do-nothing, Blue reward:0.85\n", - "step: 2036, Red action: do-nothing, Blue reward:0.85\n", - "step: 2037, Red action: do-nothing, Blue reward:0.85\n", - "step: 2038, Red action: do-nothing, Blue reward:0.85\n", - "step: 2039, Red action: do-nothing, Blue reward:0.85\n", - "step: 2040, Red action: do-nothing, Blue reward:0.85\n", - "step: 2041, Red action: do-nothing, Blue reward:0.85\n", - "step: 2042, Red action: do-nothing, Blue reward:0.85\n", - "step: 2043, Red action: do-nothing, Blue reward:0.85\n", - "step: 2044, Red action: do-nothing, Blue reward:0.85\n", - "step: 2045, Red action: do-nothing, Blue reward:0.85\n", - "step: 2046, Red action: do-nothing, Blue reward:0.85\n", - "step: 2047, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2048, Red action: do-nothing, Blue reward:0.85\n", - "step: 2049, Red action: do-nothing, Blue reward:0.85\n", - "step: 2050, Red action: do-nothing, Blue reward:0.85\n", - "step: 2051, Red action: do-nothing, Blue reward:0.85\n", - "step: 2052, Red action: do-nothing, Blue reward:0.85\n", - "step: 2053, Red action: do-nothing, Blue reward:0.85\n", - "step: 2054, Red action: do-nothing, Blue reward:0.85\n", - "step: 2055, Red action: do-nothing, Blue reward:0.85\n", - "step: 2056, Red action: do-nothing, Blue reward:0.85\n", - "step: 2057, Red action: do-nothing, Blue reward:0.85\n", - "step: 2058, Red action: do-nothing, Blue reward:0.85\n", - "step: 2059, Red action: do-nothing, Blue reward:0.85\n", - "step: 2060, Red action: do-nothing, Blue reward:0.85\n", - "step: 2061, Red action: do-nothing, Blue reward:0.85\n", - "step: 2062, Red action: do-nothing, Blue reward:0.85\n", - "step: 2063, Red action: do-nothing, Blue reward:0.85\n", - "step: 2064, Red action: do-nothing, Blue reward:0.85\n", - "step: 2065, Red action: do-nothing, Blue reward:0.85\n", - "step: 2066, Red action: do-nothing, Blue reward:0.85\n", - "step: 2067, Red action: do-nothing, Blue reward:0.85\n", - "step: 2068, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2069, Red action: do-nothing, Blue reward:0.85\n", - "step: 2070, Red action: do-nothing, Blue reward:0.85\n", - "step: 2071, Red action: do-nothing, Blue reward:0.85\n", - "step: 2072, Red action: do-nothing, Blue reward:0.85\n", - "step: 2073, Red action: do-nothing, Blue reward:0.85\n", - "step: 2074, Red action: do-nothing, Blue reward:0.85\n", - "step: 2075, Red action: do-nothing, Blue reward:0.85\n", - "step: 2076, Red action: do-nothing, Blue reward:0.85\n", - "step: 2077, Red action: do-nothing, Blue reward:0.85\n", - "step: 2078, Red action: do-nothing, Blue reward:0.85\n", - "step: 2079, Red action: do-nothing, Blue reward:0.85\n", - "step: 2080, Red action: do-nothing, Blue reward:0.85\n", - "step: 2081, Red action: do-nothing, Blue reward:0.85\n", - "step: 2082, Red action: do-nothing, Blue reward:0.85\n", - "step: 2083, Red action: do-nothing, Blue reward:0.85\n", - "step: 2084, Red action: do-nothing, Blue reward:0.85\n", - "step: 2085, Red action: do-nothing, Blue reward:0.85\n", - "step: 2086, Red action: do-nothing, Blue reward:0.85\n", - "step: 2087, Red action: do-nothing, Blue reward:0.85\n", - "step: 2088, Red action: do-nothing, Blue reward:0.85\n", - "step: 2089, Red action: do-nothing, Blue reward:0.85\n", - "step: 2090, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2091, Red action: do-nothing, Blue reward:0.85\n", - "step: 2092, Red action: do-nothing, Blue reward:0.85\n", - "step: 2093, Red action: do-nothing, Blue reward:0.85\n", - "step: 2094, Red action: do-nothing, Blue reward:0.85\n", - "step: 2095, Red action: do-nothing, Blue reward:0.85\n", - "step: 2096, Red action: do-nothing, Blue reward:0.85\n", - "step: 2097, Red action: do-nothing, Blue reward:0.85\n", - "step: 2098, Red action: do-nothing, Blue reward:0.85\n", - "step: 2099, Red action: do-nothing, Blue reward:0.85\n", - "step: 2100, Red action: do-nothing, Blue reward:0.85\n", - "step: 2101, Red action: do-nothing, Blue reward:0.85\n", - "step: 2102, Red action: do-nothing, Blue reward:0.85\n", - "step: 2103, Red action: do-nothing, Blue reward:0.85\n", - "step: 2104, Red action: do-nothing, Blue reward:0.85\n", - "step: 2105, Red action: do-nothing, Blue reward:0.85\n", - "step: 2106, Red action: do-nothing, Blue reward:0.85\n", - "step: 2107, Red action: do-nothing, Blue reward:0.85\n", - "step: 2108, Red action: do-nothing, Blue reward:0.85\n", - "step: 2109, Red action: do-nothing, Blue reward:0.85\n", - "step: 2110, Red action: do-nothing, Blue reward:0.85\n", - "step: 2111, Red action: do-nothing, Blue reward:0.85\n", - "step: 2112, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2113, Red action: do-nothing, Blue reward:0.85\n", - "step: 2114, Red action: do-nothing, Blue reward:0.85\n", - "step: 2115, Red action: do-nothing, Blue reward:0.85\n", - "step: 2116, Red action: do-nothing, Blue reward:0.85\n", - "step: 2117, Red action: do-nothing, Blue reward:0.85\n", - "step: 2118, Red action: do-nothing, Blue reward:0.85\n", - "step: 2119, Red action: do-nothing, Blue reward:0.85\n", - "step: 2120, Red action: do-nothing, Blue reward:0.85\n", - "step: 2121, Red action: do-nothing, Blue reward:0.85\n", - "step: 2122, Red action: do-nothing, Blue reward:0.85\n", - "step: 2123, Red action: do-nothing, Blue reward:0.85\n", - "step: 2124, Red action: do-nothing, Blue reward:0.85\n", - "step: 2125, Red action: do-nothing, Blue reward:0.85\n", - "step: 2126, Red action: do-nothing, Blue reward:0.85\n", - "step: 2127, Red action: do-nothing, Blue reward:0.85\n", - "step: 2128, Red action: do-nothing, Blue reward:0.85\n", - "step: 2129, Red action: do-nothing, Blue reward:0.85\n", - "step: 2130, Red action: do-nothing, Blue reward:0.85\n", - "step: 2131, Red action: do-nothing, Blue reward:0.85\n", - "step: 2132, Red action: do-nothing, Blue reward:0.85\n", - "step: 2133, Red action: do-nothing, Blue reward:0.85\n", - "step: 2134, Red action: do-nothing, Blue reward:0.85\n", - "step: 2135, Red action: do-nothing, Blue reward:0.85\n", - "step: 2136, Red action: do-nothing, Blue reward:0.85\n", - "step: 2137, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2138, Red action: do-nothing, Blue reward:0.85\n", - "step: 2139, Red action: do-nothing, Blue reward:0.85\n", - "step: 2140, Red action: do-nothing, Blue reward:0.85\n", - "step: 2141, Red action: do-nothing, Blue reward:0.85\n", - "step: 2142, Red action: do-nothing, Blue reward:0.85\n", - "step: 2143, Red action: do-nothing, Blue reward:0.85\n", - "step: 2144, Red action: do-nothing, Blue reward:0.85\n", - "step: 2145, Red action: do-nothing, Blue reward:0.85\n", - "step: 2146, Red action: do-nothing, Blue reward:0.85\n", - "step: 2147, Red action: do-nothing, Blue reward:0.85\n", - "step: 2148, Red action: do-nothing, Blue reward:0.85\n", - "step: 2149, Red action: do-nothing, Blue reward:0.85\n", - "step: 2150, Red action: do-nothing, Blue reward:0.85\n", - "step: 2151, Red action: do-nothing, Blue reward:0.85\n", - "step: 2152, Red action: do-nothing, Blue reward:0.85\n", - "step: 2153, Red action: do-nothing, Blue reward:0.85\n", - "step: 2154, Red action: do-nothing, Blue reward:0.85\n", - "step: 2155, Red action: do-nothing, Blue reward:0.85\n", - "step: 2156, Red action: do-nothing, Blue reward:0.85\n", - "step: 2157, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2158, Red action: do-nothing, Blue reward:0.85\n", - "step: 2159, Red action: do-nothing, Blue reward:0.85\n", - "step: 2160, Red action: do-nothing, Blue reward:0.85\n", - "step: 2161, Red action: do-nothing, Blue reward:0.85\n", - "step: 2162, Red action: do-nothing, Blue reward:0.85\n", - "step: 2163, Red action: do-nothing, Blue reward:0.85\n", - "step: 2164, Red action: do-nothing, Blue reward:0.85\n", - "step: 2165, Red action: do-nothing, Blue reward:0.85\n", - "step: 2166, Red action: do-nothing, Blue reward:0.85\n", - "step: 2167, Red action: do-nothing, Blue reward:0.85\n", - "step: 2168, Red action: do-nothing, Blue reward:0.85\n", - "step: 2169, Red action: do-nothing, Blue reward:0.85\n", - "step: 2170, Red action: do-nothing, Blue reward:0.85\n", - "step: 2171, Red action: do-nothing, Blue reward:0.85\n", - "step: 2172, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2173, Red action: do-nothing, Blue reward:0.85\n", - "step: 2174, Red action: do-nothing, Blue reward:0.85\n", - "step: 2175, Red action: do-nothing, Blue reward:0.85\n", - "step: 2176, Red action: do-nothing, Blue reward:0.85\n", - "step: 2177, Red action: do-nothing, Blue reward:0.85\n", - "step: 2178, Red action: do-nothing, Blue reward:0.85\n", - "step: 2179, Red action: do-nothing, Blue reward:0.85\n", - "step: 2180, Red action: do-nothing, Blue reward:0.85\n", - "step: 2181, Red action: do-nothing, Blue reward:0.85\n", - "step: 2182, Red action: do-nothing, Blue reward:0.85\n", - "step: 2183, Red action: do-nothing, Blue reward:0.85\n", - "step: 2184, Red action: do-nothing, Blue reward:0.85\n", - "step: 2185, Red action: do-nothing, Blue reward:0.85\n", - "step: 2186, Red action: do-nothing, Blue reward:0.85\n", - "step: 2187, Red action: do-nothing, Blue reward:0.85\n", - "step: 2188, Red action: do-nothing, Blue reward:0.85\n", - "step: 2189, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2190, Red action: do-nothing, Blue reward:0.85\n", - "step: 2191, Red action: do-nothing, Blue reward:0.85\n", - "step: 2192, Red action: do-nothing, Blue reward:0.85\n", - "step: 2193, Red action: do-nothing, Blue reward:0.85\n", - "step: 2194, Red action: do-nothing, Blue reward:0.85\n", - "step: 2195, Red action: do-nothing, Blue reward:0.85\n", - "step: 2196, Red action: do-nothing, Blue reward:0.85\n", - "step: 2197, Red action: do-nothing, Blue reward:0.85\n", - "step: 2198, Red action: do-nothing, Blue reward:0.85\n", - "step: 2199, Red action: do-nothing, Blue reward:0.85\n", - "step: 2200, Red action: do-nothing, Blue reward:0.85\n", - "step: 2201, Red action: do-nothing, Blue reward:0.85\n", - "step: 2202, Red action: do-nothing, Blue reward:0.85\n", - "step: 2203, Red action: do-nothing, Blue reward:0.85\n", - "step: 2204, Red action: do-nothing, Blue reward:0.85\n", - "step: 2205, Red action: do-nothing, Blue reward:0.85\n", - "step: 2206, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2207, Red action: do-nothing, Blue reward:0.85\n", - "step: 2208, Red action: do-nothing, Blue reward:0.85\n", - "step: 2209, Red action: do-nothing, Blue reward:0.85\n", - "step: 2210, Red action: do-nothing, Blue reward:0.85\n", - "step: 2211, Red action: do-nothing, Blue reward:0.85\n", - "step: 2212, Red action: do-nothing, Blue reward:0.85\n", - "step: 2213, Red action: do-nothing, Blue reward:0.85\n", - "step: 2214, Red action: do-nothing, Blue reward:0.85\n", - "step: 2215, Red action: do-nothing, Blue reward:0.85\n", - "step: 2216, Red action: do-nothing, Blue reward:0.85\n", - "step: 2217, Red action: do-nothing, Blue reward:0.85\n", - "step: 2218, Red action: do-nothing, Blue reward:0.85\n", - "step: 2219, Red action: do-nothing, Blue reward:0.85\n", - "step: 2220, Red action: do-nothing, Blue reward:0.85\n", - "step: 2221, Red action: do-nothing, Blue reward:0.85\n", - "step: 2222, Red action: do-nothing, Blue reward:0.85\n", - "step: 2223, Red action: do-nothing, Blue reward:0.85\n", - "step: 2224, Red action: do-nothing, Blue reward:0.85\n", - "step: 2225, Red action: do-nothing, Blue reward:0.85\n", - "step: 2226, Red action: do-nothing, Blue reward:0.85\n", - "step: 2227, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2228, Red action: do-nothing, Blue reward:0.85\n", - "step: 2229, Red action: do-nothing, Blue reward:0.85\n", - "step: 2230, Red action: do-nothing, Blue reward:0.85\n", - "step: 2231, Red action: do-nothing, Blue reward:0.85\n", - "step: 2232, Red action: do-nothing, Blue reward:0.85\n", - "step: 2233, Red action: do-nothing, Blue reward:0.85\n", - "step: 2234, Red action: do-nothing, Blue reward:0.85\n", - "step: 2235, Red action: do-nothing, Blue reward:0.85\n", - "step: 2236, Red action: do-nothing, Blue reward:0.85\n", - "step: 2237, Red action: do-nothing, Blue reward:0.85\n", - "step: 2238, Red action: do-nothing, Blue reward:0.85\n", - "step: 2239, Red action: do-nothing, Blue reward:0.85\n", - "step: 2240, Red action: do-nothing, Blue reward:0.85\n", - "step: 2241, Red action: do-nothing, Blue reward:0.85\n", - "step: 2242, Red action: do-nothing, Blue reward:0.85\n", - "step: 2243, Red action: do-nothing, Blue reward:0.85\n", - "step: 2244, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2245, Red action: do-nothing, Blue reward:0.85\n", - "step: 2246, Red action: do-nothing, Blue reward:0.85\n", - "step: 2247, Red action: do-nothing, Blue reward:0.85\n", - "step: 2248, Red action: do-nothing, Blue reward:0.85\n", - "step: 2249, Red action: do-nothing, Blue reward:0.85\n", - "step: 2250, Red action: do-nothing, Blue reward:0.85\n", - "step: 2251, Red action: do-nothing, Blue reward:0.85\n", - "step: 2252, Red action: do-nothing, Blue reward:0.85\n", - "step: 2253, Red action: do-nothing, Blue reward:0.85\n", - "step: 2254, Red action: do-nothing, Blue reward:0.85\n", - "step: 2255, Red action: do-nothing, Blue reward:0.85\n", - "step: 2256, Red action: do-nothing, Blue reward:0.85\n", - "step: 2257, Red action: do-nothing, Blue reward:0.85\n", - "step: 2258, Red action: do-nothing, Blue reward:0.85\n", - "step: 2259, Red action: do-nothing, Blue reward:0.85\n", - "step: 2260, Red action: do-nothing, Blue reward:0.85\n", - "step: 2261, Red action: do-nothing, Blue reward:0.85\n", - "step: 2262, Red action: do-nothing, Blue reward:0.85\n", - "step: 2263, Red action: do-nothing, Blue reward:0.85\n", - "step: 2264, Red action: do-nothing, Blue reward:0.85\n", - "step: 2265, Red action: do-nothing, Blue reward:0.85\n", - "step: 2266, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2267, Red action: do-nothing, Blue reward:0.85\n", - "step: 2268, Red action: do-nothing, Blue reward:0.85\n", - "step: 2269, Red action: do-nothing, Blue reward:0.85\n", - "step: 2270, Red action: do-nothing, Blue reward:0.85\n", - "step: 2271, Red action: do-nothing, Blue reward:0.85\n", - "step: 2272, Red action: do-nothing, Blue reward:0.85\n", - "step: 2273, Red action: do-nothing, Blue reward:0.85\n", - "step: 2274, Red action: do-nothing, Blue reward:0.85\n", - "step: 2275, Red action: do-nothing, Blue reward:0.85\n", - "step: 2276, Red action: do-nothing, Blue reward:0.85\n", - "step: 2277, Red action: do-nothing, Blue reward:0.85\n", - "step: 2278, Red action: do-nothing, Blue reward:0.85\n", - "step: 2279, Red action: do-nothing, Blue reward:0.85\n", - "step: 2280, Red action: do-nothing, Blue reward:0.85\n", - "step: 2281, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2282, Red action: do-nothing, Blue reward:0.85\n", - "step: 2283, Red action: do-nothing, Blue reward:0.85\n", - "step: 2284, Red action: do-nothing, Blue reward:0.85\n", - "step: 2285, Red action: do-nothing, Blue reward:0.85\n", - "step: 2286, Red action: do-nothing, Blue reward:0.85\n", - "step: 2287, Red action: do-nothing, Blue reward:0.85\n", - "step: 2288, Red action: do-nothing, Blue reward:0.85\n", - "step: 2289, Red action: do-nothing, Blue reward:0.85\n", - "step: 2290, Red action: do-nothing, Blue reward:0.85\n", - "step: 2291, Red action: do-nothing, Blue reward:0.85\n", - "step: 2292, Red action: do-nothing, Blue reward:0.85\n", - "step: 2293, Red action: do-nothing, Blue reward:0.85\n", - "step: 2294, Red action: do-nothing, Blue reward:0.85\n", - "step: 2295, Red action: do-nothing, Blue reward:0.85\n", - "step: 2296, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2297, Red action: do-nothing, Blue reward:0.85\n", - "step: 2298, Red action: do-nothing, Blue reward:0.85\n", - "step: 2299, Red action: do-nothing, Blue reward:0.85\n", - "step: 2300, Red action: do-nothing, Blue reward:0.85\n", - "step: 2301, Red action: do-nothing, Blue reward:0.85\n", - "step: 2302, Red action: do-nothing, Blue reward:0.85\n", - "step: 2303, Red action: do-nothing, Blue reward:0.85\n", - "step: 2304, Red action: do-nothing, Blue reward:0.85\n", - "step: 2305, Red action: do-nothing, Blue reward:0.85\n", - "step: 2306, Red action: do-nothing, Blue reward:0.85\n", - "step: 2307, Red action: do-nothing, Blue reward:0.85\n", - "step: 2308, Red action: do-nothing, Blue reward:0.85\n", - "step: 2309, Red action: do-nothing, Blue reward:0.85\n", - "step: 2310, Red action: do-nothing, Blue reward:0.85\n", - "step: 2311, Red action: do-nothing, Blue reward:0.85\n", - "step: 2312, Red action: do-nothing, Blue reward:0.85\n", - "step: 2313, Red action: do-nothing, Blue reward:0.85\n", - "step: 2314, Red action: do-nothing, Blue reward:0.85\n", - "step: 2315, Red action: do-nothing, Blue reward:0.85\n", - "step: 2316, Red action: do-nothing, Blue reward:0.85\n", - "step: 2317, Red action: do-nothing, Blue reward:0.85\n", - "step: 2318, Red action: do-nothing, Blue reward:0.85\n", - "step: 2319, Red action: do-nothing, Blue reward:0.85\n", - "step: 2320, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2321, Red action: do-nothing, Blue reward:0.85\n", - "step: 2322, Red action: do-nothing, Blue reward:0.85\n", - "step: 2323, Red action: do-nothing, Blue reward:0.85\n", - "step: 2324, Red action: do-nothing, Blue reward:0.85\n", - "step: 2325, Red action: do-nothing, Blue reward:0.85\n", - "step: 2326, Red action: do-nothing, Blue reward:0.85\n", - "step: 2327, Red action: do-nothing, Blue reward:0.85\n", - "step: 2328, Red action: do-nothing, Blue reward:0.85\n", - "step: 2329, Red action: do-nothing, Blue reward:0.85\n", - "step: 2330, Red action: do-nothing, Blue reward:0.85\n", - "step: 2331, Red action: do-nothing, Blue reward:0.85\n", - "step: 2332, Red action: do-nothing, Blue reward:0.85\n", - "step: 2333, Red action: do-nothing, Blue reward:0.85\n", - "step: 2334, Red action: do-nothing, Blue reward:0.85\n", - "step: 2335, Red action: do-nothing, Blue reward:0.85\n", - "step: 2336, Red action: do-nothing, Blue reward:0.85\n", - "step: 2337, Red action: do-nothing, Blue reward:0.85\n", - "step: 2338, Red action: do-nothing, Blue reward:0.85\n", - "step: 2339, Red action: do-nothing, Blue reward:0.85\n", - "step: 2340, Red action: do-nothing, Blue reward:0.85\n", - "step: 2341, Red action: do-nothing, Blue reward:0.85\n", - "step: 2342, Red action: do-nothing, Blue reward:0.85\n", - "step: 2343, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2344, Red action: do-nothing, Blue reward:0.85\n", - "step: 2345, Red action: do-nothing, Blue reward:0.85\n", - "step: 2346, Red action: do-nothing, Blue reward:0.85\n", - "step: 2347, Red action: do-nothing, Blue reward:0.85\n", - "step: 2348, Red action: do-nothing, Blue reward:0.85\n", - "step: 2349, Red action: do-nothing, Blue reward:0.85\n", - "step: 2350, Red action: do-nothing, Blue reward:0.85\n", - "step: 2351, Red action: do-nothing, Blue reward:0.85\n", - "step: 2352, Red action: do-nothing, Blue reward:0.85\n", - "step: 2353, Red action: do-nothing, Blue reward:0.85\n", - "step: 2354, Red action: do-nothing, Blue reward:0.85\n", - "step: 2355, Red action: do-nothing, Blue reward:0.85\n", - "step: 2356, Red action: do-nothing, Blue reward:0.85\n", - "step: 2357, Red action: do-nothing, Blue reward:0.85\n", - "step: 2358, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2359, Red action: do-nothing, Blue reward:0.85\n", - "step: 2360, Red action: do-nothing, Blue reward:0.85\n", - "step: 2361, Red action: do-nothing, Blue reward:0.85\n", - "step: 2362, Red action: do-nothing, Blue reward:0.85\n", - "step: 2363, Red action: do-nothing, Blue reward:0.85\n", - "step: 2364, Red action: do-nothing, Blue reward:0.85\n", - "step: 2365, Red action: do-nothing, Blue reward:0.85\n", - "step: 2366, Red action: do-nothing, Blue reward:0.85\n", - "step: 2367, Red action: do-nothing, Blue reward:0.85\n", - "step: 2368, Red action: do-nothing, Blue reward:0.85\n", - "step: 2369, Red action: do-nothing, Blue reward:0.85\n", - "step: 2370, Red action: do-nothing, Blue reward:0.85\n", - "step: 2371, Red action: do-nothing, Blue reward:0.85\n", - "step: 2372, Red action: do-nothing, Blue reward:0.85\n", - "step: 2373, Red action: do-nothing, Blue reward:0.85\n", - "step: 2374, Red action: do-nothing, Blue reward:0.85\n", - "step: 2375, Red action: do-nothing, Blue reward:0.85\n", - "step: 2376, Red action: do-nothing, Blue reward:0.85\n", - "step: 2377, Red action: do-nothing, Blue reward:0.85\n", - "step: 2378, Red action: do-nothing, Blue reward:0.85\n", - "step: 2379, Red action: do-nothing, Blue reward:0.85\n", - "step: 2380, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2381, Red action: do-nothing, Blue reward:0.85\n", - "step: 2382, Red action: do-nothing, Blue reward:0.85\n", - "step: 2383, Red action: do-nothing, Blue reward:0.85\n", - "step: 2384, Red action: do-nothing, Blue reward:0.85\n", - "step: 2385, Red action: do-nothing, Blue reward:0.85\n", - "step: 2386, Red action: do-nothing, Blue reward:0.85\n", - "step: 2387, Red action: do-nothing, Blue reward:0.85\n", - "step: 2388, Red action: do-nothing, Blue reward:0.85\n", - "step: 2389, Red action: do-nothing, Blue reward:0.85\n", - "step: 2390, Red action: do-nothing, Blue reward:0.85\n", - "step: 2391, Red action: do-nothing, Blue reward:0.85\n", - "step: 2392, Red action: do-nothing, Blue reward:0.85\n", - "step: 2393, Red action: do-nothing, Blue reward:0.85\n", - "step: 2394, Red action: do-nothing, Blue reward:0.85\n", - "step: 2395, Red action: do-nothing, Blue reward:0.85\n", - "step: 2396, Red action: do-nothing, Blue reward:0.85\n", - "step: 2397, Red action: do-nothing, Blue reward:0.85\n", - "step: 2398, Red action: do-nothing, Blue reward:0.85\n", - "step: 2399, Red action: do-nothing, Blue reward:0.85\n", - "step: 2400, Red action: do-nothing, Blue reward:0.85\n", - "step: 2401, Red action: do-nothing, Blue reward:0.85\n", - "step: 2402, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2403, Red action: do-nothing, Blue reward:0.85\n", - "step: 2404, Red action: do-nothing, Blue reward:0.85\n", - "step: 2405, Red action: do-nothing, Blue reward:0.85\n", - "step: 2406, Red action: do-nothing, Blue reward:0.85\n", - "step: 2407, Red action: do-nothing, Blue reward:0.85\n", - "step: 2408, Red action: do-nothing, Blue reward:0.85\n", - "step: 2409, Red action: do-nothing, Blue reward:0.85\n", - "step: 2410, Red action: do-nothing, Blue reward:0.85\n", - "step: 2411, Red action: do-nothing, Blue reward:0.85\n", - "step: 2412, Red action: do-nothing, Blue reward:0.85\n", - "step: 2413, Red action: do-nothing, Blue reward:0.85\n", - "step: 2414, Red action: do-nothing, Blue reward:0.85\n", - "step: 2415, Red action: do-nothing, Blue reward:0.85\n", - "step: 2416, Red action: do-nothing, Blue reward:0.85\n", - "step: 2417, Red action: do-nothing, Blue reward:0.85\n", - "step: 2418, Red action: do-nothing, Blue reward:0.85\n", - "step: 2419, Red action: do-nothing, Blue reward:0.85\n", - "step: 2420, Red action: do-nothing, Blue reward:0.85\n", - "step: 2421, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2422, Red action: do-nothing, Blue reward:0.85\n", - "step: 2423, Red action: do-nothing, Blue reward:0.85\n", - "step: 2424, Red action: do-nothing, Blue reward:0.85\n", - "step: 2425, Red action: do-nothing, Blue reward:0.85\n", - "step: 2426, Red action: do-nothing, Blue reward:0.85\n", - "step: 2427, Red action: do-nothing, Blue reward:0.85\n", - "step: 2428, Red action: do-nothing, Blue reward:0.85\n", - "step: 2429, Red action: do-nothing, Blue reward:0.85\n", - "step: 2430, Red action: do-nothing, Blue reward:0.85\n", - "step: 2431, Red action: do-nothing, Blue reward:0.85\n", - "step: 2432, Red action: do-nothing, Blue reward:0.85\n", - "step: 2433, Red action: do-nothing, Blue reward:0.85\n", - "step: 2434, Red action: do-nothing, Blue reward:0.85\n", - "step: 2435, Red action: do-nothing, Blue reward:0.85\n", - "step: 2436, Red action: do-nothing, Blue reward:0.85\n", - "step: 2437, Red action: do-nothing, Blue reward:0.85\n", - "step: 2438, Red action: do-nothing, Blue reward:0.85\n", - "step: 2439, Red action: do-nothing, Blue reward:0.85\n", - "step: 2440, Red action: do-nothing, Blue reward:0.85\n", - "step: 2441, Red action: do-nothing, Blue reward:0.85\n", - "step: 2442, Red action: do-nothing, Blue reward:0.85\n", - "step: 2443, Red action: do-nothing, Blue reward:0.85\n", - "step: 2444, Red action: do-nothing, Blue reward:0.85\n", - "step: 2445, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2446, Red action: do-nothing, Blue reward:0.85\n", - "step: 2447, Red action: do-nothing, Blue reward:0.85\n", - "step: 2448, Red action: do-nothing, Blue reward:0.85\n", - "step: 2449, Red action: do-nothing, Blue reward:0.85\n", - "step: 2450, Red action: do-nothing, Blue reward:0.85\n", - "step: 2451, Red action: do-nothing, Blue reward:0.85\n", - "step: 2452, Red action: do-nothing, Blue reward:0.85\n", - "step: 2453, Red action: do-nothing, Blue reward:0.85\n", - "step: 2454, Red action: do-nothing, Blue reward:0.85\n", - "step: 2455, Red action: do-nothing, Blue reward:0.85\n", - "step: 2456, Red action: do-nothing, Blue reward:0.85\n", - "step: 2457, Red action: do-nothing, Blue reward:0.85\n", - "step: 2458, Red action: do-nothing, Blue reward:0.85\n", - "step: 2459, Red action: do-nothing, Blue reward:0.85\n", - "step: 2460, Red action: do-nothing, Blue reward:0.85\n", - "step: 2461, Red action: do-nothing, Blue reward:0.85\n", - "step: 2462, Red action: do-nothing, Blue reward:0.85\n", - "step: 2463, Red action: do-nothing, Blue reward:0.85\n", - "step: 2464, Red action: do-nothing, Blue reward:0.85\n", - "step: 2465, Red action: do-nothing, Blue reward:0.85\n", - "step: 2466, Red action: do-nothing, Blue reward:0.85\n", - "step: 2467, Red action: do-nothing, Blue reward:0.85\n", - "step: 2468, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2469, Red action: do-nothing, Blue reward:0.85\n", - "step: 2470, Red action: do-nothing, Blue reward:0.85\n", - "step: 2471, Red action: do-nothing, Blue reward:0.85\n", - "step: 2472, Red action: do-nothing, Blue reward:0.85\n", - "step: 2473, Red action: do-nothing, Blue reward:0.85\n", - "step: 2474, Red action: do-nothing, Blue reward:0.85\n", - "step: 2475, Red action: do-nothing, Blue reward:0.85\n", - "step: 2476, Red action: do-nothing, Blue reward:0.85\n", - "step: 2477, Red action: do-nothing, Blue reward:0.85\n", - "step: 2478, Red action: do-nothing, Blue reward:0.85\n", - "step: 2479, Red action: do-nothing, Blue reward:0.85\n", - "step: 2480, Red action: do-nothing, Blue reward:0.85\n", - "step: 2481, Red action: do-nothing, Blue reward:0.85\n", - "step: 2482, Red action: do-nothing, Blue reward:0.85\n", - "step: 2483, Red action: do-nothing, Blue reward:0.85\n", - "step: 2484, Red action: do-nothing, Blue reward:0.85\n", - "step: 2485, Red action: do-nothing, Blue reward:0.85\n", - "step: 2486, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2487, Red action: do-nothing, Blue reward:0.85\n", - "step: 2488, Red action: do-nothing, Blue reward:0.85\n", - "step: 2489, Red action: do-nothing, Blue reward:0.85\n", - "step: 2490, Red action: do-nothing, Blue reward:0.85\n", - "step: 2491, Red action: do-nothing, Blue reward:0.85\n", - "step: 2492, Red action: do-nothing, Blue reward:0.85\n", - "step: 2493, Red action: do-nothing, Blue reward:0.85\n", - "step: 2494, Red action: do-nothing, Blue reward:0.85\n", - "step: 2495, Red action: do-nothing, Blue reward:0.85\n", - "step: 2496, Red action: do-nothing, Blue reward:0.85\n", - "step: 2497, Red action: do-nothing, Blue reward:0.85\n", - "step: 2498, Red action: do-nothing, Blue reward:0.85\n", - "step: 2499, Red action: do-nothing, Blue reward:0.85\n", - "step: 2500, Red action: do-nothing, Blue reward:0.85\n", - "step: 2501, Red action: do-nothing, Blue reward:0.85\n", - "step: 2502, Red action: do-nothing, Blue reward:0.85\n", - "step: 2503, Red action: do-nothing, Blue reward:0.85\n", - "step: 2504, Red action: do-nothing, Blue reward:0.85\n", - "step: 2505, Red action: do-nothing, Blue reward:0.85\n", - "step: 2506, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2507, Red action: do-nothing, Blue reward:0.85\n", - "step: 2508, Red action: do-nothing, Blue reward:0.85\n", - "step: 2509, Red action: do-nothing, Blue reward:0.85\n", - "step: 2510, Red action: do-nothing, Blue reward:0.85\n", - "step: 2511, Red action: do-nothing, Blue reward:0.85\n", - "step: 2512, Red action: do-nothing, Blue reward:0.85\n", - "step: 2513, Red action: do-nothing, Blue reward:0.85\n", - "step: 2514, Red action: do-nothing, Blue reward:0.85\n", - "step: 2515, Red action: do-nothing, Blue reward:0.85\n", - "step: 2516, Red action: do-nothing, Blue reward:0.85\n", - "step: 2517, Red action: do-nothing, Blue reward:0.85\n", - "step: 2518, Red action: do-nothing, Blue reward:0.85\n", - "step: 2519, Red action: do-nothing, Blue reward:0.85\n", - "step: 2520, Red action: do-nothing, Blue reward:0.85\n", - "step: 2521, Red action: do-nothing, Blue reward:0.85\n", - "step: 2522, Red action: do-nothing, Blue reward:0.85\n", - "step: 2523, Red action: do-nothing, Blue reward:0.85\n", - "step: 2524, Red action: do-nothing, Blue reward:0.85\n", - "step: 2525, Red action: do-nothing, Blue reward:0.85\n", - "step: 2526, Red action: do-nothing, Blue reward:0.85\n", - "step: 2527, Red action: do-nothing, Blue reward:0.85\n", - "step: 2528, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2529, Red action: do-nothing, Blue reward:0.85\n", - "step: 2530, Red action: do-nothing, Blue reward:0.85\n", - "step: 2531, Red action: do-nothing, Blue reward:0.85\n", - "step: 2532, Red action: do-nothing, Blue reward:0.85\n", - "step: 2533, Red action: do-nothing, Blue reward:0.85\n", - "step: 2534, Red action: do-nothing, Blue reward:0.85\n", - "step: 2535, Red action: do-nothing, Blue reward:0.85\n", - "step: 2536, Red action: do-nothing, Blue reward:0.85\n", - "step: 2537, Red action: do-nothing, Blue reward:0.85\n", - "step: 2538, Red action: do-nothing, Blue reward:0.85\n", - "step: 2539, Red action: do-nothing, Blue reward:0.85\n", - "step: 2540, Red action: do-nothing, Blue reward:0.85\n", - "step: 2541, Red action: do-nothing, Blue reward:0.85\n", - "step: 2542, Red action: do-nothing, Blue reward:0.85\n", - "step: 2543, Red action: do-nothing, Blue reward:0.85\n", - "step: 2544, Red action: do-nothing, Blue reward:0.85\n", - "step: 2545, Red action: do-nothing, Blue reward:0.85\n", - "step: 2546, Red action: do-nothing, Blue reward:0.85\n", - "step: 2547, Red action: do-nothing, Blue reward:0.85\n", - "step: 2548, Red action: do-nothing, Blue reward:0.85\n", - "step: 2549, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2550, Red action: do-nothing, Blue reward:0.85\n", - "step: 2551, Red action: do-nothing, Blue reward:0.85\n", - "step: 2552, Red action: do-nothing, Blue reward:0.85\n", - "step: 2553, Red action: do-nothing, Blue reward:0.85\n", - "step: 2554, Red action: do-nothing, Blue reward:0.85\n", - "step: 2555, Red action: do-nothing, Blue reward:0.85\n", - "step: 2556, Red action: do-nothing, Blue reward:0.85\n", - "step: 2557, Red action: do-nothing, Blue reward:0.85\n", - "step: 2558, Red action: do-nothing, Blue reward:0.85\n", - "step: 2559, Red action: do-nothing, Blue reward:0.85\n", - "step: 2560, Red action: do-nothing, Blue reward:0.85\n", - "step: 2561, Red action: do-nothing, Blue reward:0.85\n", - "step: 2562, Red action: do-nothing, Blue reward:0.85\n", - "step: 2563, Red action: do-nothing, Blue reward:0.85\n", - "step: 2564, Red action: do-nothing, Blue reward:0.85\n", - "step: 2565, Red action: do-nothing, Blue reward:0.85\n", - "step: 2566, Red action: do-nothing, Blue reward:0.85\n", - "step: 2567, Red action: do-nothing, Blue reward:0.85\n", - "step: 2568, Red action: do-nothing, Blue reward:0.85\n", - "step: 2569, Red action: do-nothing, Blue reward:0.85\n", - "step: 2570, Red action: do-nothing, Blue reward:0.85\n", - "step: 2571, Red action: do-nothing, Blue reward:0.85\n", - "step: 2572, Red action: do-nothing, Blue reward:0.85\n", - "step: 2573, Red action: do-nothing, Blue reward:0.85\n", - "step: 2574, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2575, Red action: do-nothing, Blue reward:0.85\n", - "step: 2576, Red action: do-nothing, Blue reward:0.85\n", - "step: 2577, Red action: do-nothing, Blue reward:0.85\n", - "step: 2578, Red action: do-nothing, Blue reward:0.85\n", - "step: 2579, Red action: do-nothing, Blue reward:0.85\n", - "step: 2580, Red action: do-nothing, Blue reward:0.85\n", - "step: 2581, Red action: do-nothing, Blue reward:0.85\n", - "step: 2582, Red action: do-nothing, Blue reward:0.85\n", - "step: 2583, Red action: do-nothing, Blue reward:0.85\n", - "step: 2584, Red action: do-nothing, Blue reward:0.85\n", - "step: 2585, Red action: do-nothing, Blue reward:0.85\n", - "step: 2586, Red action: do-nothing, Blue reward:0.85\n", - "step: 2587, Red action: do-nothing, Blue reward:0.85\n", - "step: 2588, Red action: do-nothing, Blue reward:0.85\n", - "step: 2589, Red action: do-nothing, Blue reward:0.85\n", - "step: 2590, Red action: do-nothing, Blue reward:0.85\n", - "step: 2591, Red action: do-nothing, Blue reward:0.85\n", - "step: 2592, Red action: do-nothing, Blue reward:0.85\n", - "step: 2593, Red action: do-nothing, Blue reward:0.85\n", - "step: 2594, Red action: do-nothing, Blue reward:0.85\n", - "step: 2595, Red action: do-nothing, Blue reward:0.85\n", - "step: 2596, Red action: do-nothing, Blue reward:0.85\n", - "step: 2597, Red action: do-nothing, Blue reward:0.85\n", - "step: 2598, Red action: do-nothing, Blue reward:0.85\n", - "step: 2599, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2600, Red action: do-nothing, Blue reward:0.85\n", - "step: 2601, Red action: do-nothing, Blue reward:0.85\n", - "step: 2602, Red action: do-nothing, Blue reward:0.85\n", - "step: 2603, Red action: do-nothing, Blue reward:0.85\n", - "step: 2604, Red action: do-nothing, Blue reward:0.85\n", - "step: 2605, Red action: do-nothing, Blue reward:0.85\n", - "step: 2606, Red action: do-nothing, Blue reward:0.85\n", - "step: 2607, Red action: do-nothing, Blue reward:0.85\n", - "step: 2608, Red action: do-nothing, Blue reward:0.85\n", - "step: 2609, Red action: do-nothing, Blue reward:0.85\n", - "step: 2610, Red action: do-nothing, Blue reward:0.85\n", - "step: 2611, Red action: do-nothing, Blue reward:0.85\n", - "step: 2612, Red action: do-nothing, Blue reward:0.85\n", - "step: 2613, Red action: do-nothing, Blue reward:0.85\n", - "step: 2614, Red action: do-nothing, Blue reward:0.85\n", - "step: 2615, Red action: do-nothing, Blue reward:0.85\n", - "step: 2616, Red action: do-nothing, Blue reward:0.85\n", - "step: 2617, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2618, Red action: do-nothing, Blue reward:0.85\n", - "step: 2619, Red action: do-nothing, Blue reward:0.85\n", - "step: 2620, Red action: do-nothing, Blue reward:0.85\n", - "step: 2621, Red action: do-nothing, Blue reward:0.85\n", - "step: 2622, Red action: do-nothing, Blue reward:0.85\n", - "step: 2623, Red action: do-nothing, Blue reward:0.85\n", - "step: 2624, Red action: do-nothing, Blue reward:0.85\n", - "step: 2625, Red action: do-nothing, Blue reward:0.85\n", - "step: 2626, Red action: do-nothing, Blue reward:0.85\n", - "step: 2627, Red action: do-nothing, Blue reward:0.85\n", - "step: 2628, Red action: do-nothing, Blue reward:0.85\n", - "step: 2629, Red action: do-nothing, Blue reward:0.85\n", - "step: 2630, Red action: do-nothing, Blue reward:0.85\n", - "step: 2631, Red action: do-nothing, Blue reward:0.85\n", - "step: 2632, Red action: do-nothing, Blue reward:0.85\n", - "step: 2633, Red action: do-nothing, Blue reward:0.85\n", - "step: 2634, Red action: do-nothing, Blue reward:0.85\n", - "step: 2635, Red action: do-nothing, Blue reward:0.85\n", - "step: 2636, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2637, Red action: do-nothing, Blue reward:0.85\n", - "step: 2638, Red action: do-nothing, Blue reward:0.85\n", - "step: 2639, Red action: do-nothing, Blue reward:0.85\n", - "step: 2640, Red action: do-nothing, Blue reward:0.85\n", - "step: 2641, Red action: do-nothing, Blue reward:0.85\n", - "step: 2642, Red action: do-nothing, Blue reward:0.85\n", - "step: 2643, Red action: do-nothing, Blue reward:0.85\n", - "step: 2644, Red action: do-nothing, Blue reward:0.85\n", - "step: 2645, Red action: do-nothing, Blue reward:0.85\n", - "step: 2646, Red action: do-nothing, Blue reward:0.85\n", - "step: 2647, Red action: do-nothing, Blue reward:0.85\n", - "step: 2648, Red action: do-nothing, Blue reward:0.85\n", - "step: 2649, Red action: do-nothing, Blue reward:0.85\n", - "step: 2650, Red action: do-nothing, Blue reward:0.85\n", - "step: 2651, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2652, Red action: do-nothing, Blue reward:0.85\n", - "step: 2653, Red action: do-nothing, Blue reward:0.85\n", - "step: 2654, Red action: do-nothing, Blue reward:0.85\n", - "step: 2655, Red action: do-nothing, Blue reward:0.85\n", - "step: 2656, Red action: do-nothing, Blue reward:0.85\n", - "step: 2657, Red action: do-nothing, Blue reward:0.85\n", - "step: 2658, Red action: do-nothing, Blue reward:0.85\n", - "step: 2659, Red action: do-nothing, Blue reward:0.85\n", - "step: 2660, Red action: do-nothing, Blue reward:0.85\n", - "step: 2661, Red action: do-nothing, Blue reward:0.85\n", - "step: 2662, Red action: do-nothing, Blue reward:0.85\n", - "step: 2663, Red action: do-nothing, Blue reward:0.85\n", - "step: 2664, Red action: do-nothing, Blue reward:0.85\n", - "step: 2665, Red action: do-nothing, Blue reward:0.85\n", - "step: 2666, Red action: do-nothing, Blue reward:0.85\n", - "step: 2667, Red action: do-nothing, Blue reward:0.85\n", - "step: 2668, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2669, Red action: do-nothing, Blue reward:0.85\n", - "step: 2670, Red action: do-nothing, Blue reward:0.85\n", - "step: 2671, Red action: do-nothing, Blue reward:0.85\n", - "step: 2672, Red action: do-nothing, Blue reward:0.85\n", - "step: 2673, Red action: do-nothing, Blue reward:0.85\n", - "step: 2674, Red action: do-nothing, Blue reward:0.85\n", - "step: 2675, Red action: do-nothing, Blue reward:0.85\n", - "step: 2676, Red action: do-nothing, Blue reward:0.85\n", - "step: 2677, Red action: do-nothing, Blue reward:0.85\n", - "step: 2678, Red action: do-nothing, Blue reward:0.85\n", - "step: 2679, Red action: do-nothing, Blue reward:0.85\n", - "step: 2680, Red action: do-nothing, Blue reward:0.85\n", - "step: 2681, Red action: do-nothing, Blue reward:0.85\n", - "step: 2682, Red action: do-nothing, Blue reward:0.85\n", - "step: 2683, Red action: do-nothing, Blue reward:0.85\n", - "step: 2684, Red action: do-nothing, Blue reward:0.85\n", - "step: 2685, Red action: do-nothing, Blue reward:0.85\n", - "step: 2686, Red action: do-nothing, Blue reward:0.85\n", - "step: 2687, Red action: do-nothing, Blue reward:0.85\n", - "step: 2688, Red action: do-nothing, Blue reward:0.85\n", - "step: 2689, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2690, Red action: do-nothing, Blue reward:0.85\n", - "step: 2691, Red action: do-nothing, Blue reward:0.85\n", - "step: 2692, Red action: do-nothing, Blue reward:0.85\n", - "step: 2693, Red action: do-nothing, Blue reward:0.85\n", - "step: 2694, Red action: do-nothing, Blue reward:0.85\n", - "step: 2695, Red action: do-nothing, Blue reward:0.85\n", - "step: 2696, Red action: do-nothing, Blue reward:0.85\n", - "step: 2697, Red action: do-nothing, Blue reward:0.85\n", - "step: 2698, Red action: do-nothing, Blue reward:0.85\n", - "step: 2699, Red action: do-nothing, Blue reward:0.85\n", - "step: 2700, Red action: do-nothing, Blue reward:0.85\n", - "step: 2701, Red action: do-nothing, Blue reward:0.85\n", - "step: 2702, Red action: do-nothing, Blue reward:0.85\n", - "step: 2703, Red action: do-nothing, Blue reward:0.85\n", - "step: 2704, Red action: do-nothing, Blue reward:0.85\n", - "step: 2705, Red action: do-nothing, Blue reward:0.85\n", - "step: 2706, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2707, Red action: do-nothing, Blue reward:0.85\n", - "step: 2708, Red action: do-nothing, Blue reward:0.85\n", - "step: 2709, Red action: do-nothing, Blue reward:0.85\n", - "step: 2710, Red action: do-nothing, Blue reward:0.85\n", - "step: 2711, Red action: do-nothing, Blue reward:0.85\n", - "step: 2712, Red action: do-nothing, Blue reward:0.85\n", - "step: 2713, Red action: do-nothing, Blue reward:0.85\n", - "step: 2714, Red action: do-nothing, Blue reward:0.85\n", - "step: 2715, Red action: do-nothing, Blue reward:0.85\n", - "step: 2716, Red action: do-nothing, Blue reward:0.85\n", - "step: 2717, Red action: do-nothing, Blue reward:0.85\n", - "step: 2718, Red action: do-nothing, Blue reward:0.85\n", - "step: 2719, Red action: do-nothing, Blue reward:0.85\n", - "step: 2720, Red action: do-nothing, Blue reward:0.85\n", - "step: 2721, Red action: do-nothing, Blue reward:0.85\n", - "step: 2722, Red action: do-nothing, Blue reward:0.85\n", - "step: 2723, Red action: do-nothing, Blue reward:0.85\n", - "step: 2724, Red action: do-nothing, Blue reward:0.85\n", - "step: 2725, Red action: do-nothing, Blue reward:0.85\n", - "step: 2726, Red action: do-nothing, Blue reward:0.85\n", - "step: 2727, Red action: do-nothing, Blue reward:0.85\n", - "step: 2728, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2729, Red action: do-nothing, Blue reward:0.85\n", - "step: 2730, Red action: do-nothing, Blue reward:0.85\n", - "step: 2731, Red action: do-nothing, Blue reward:0.85\n", - "step: 2732, Red action: do-nothing, Blue reward:0.85\n", - "step: 2733, Red action: do-nothing, Blue reward:0.85\n", - "step: 2734, Red action: do-nothing, Blue reward:0.85\n", - "step: 2735, Red action: do-nothing, Blue reward:0.85\n", - "step: 2736, Red action: do-nothing, Blue reward:0.85\n", - "step: 2737, Red action: do-nothing, Blue reward:0.85\n", - "step: 2738, Red action: do-nothing, Blue reward:0.85\n", - "step: 2739, Red action: do-nothing, Blue reward:0.85\n", - "step: 2740, Red action: do-nothing, Blue reward:0.85\n", - "step: 2741, Red action: do-nothing, Blue reward:0.85\n", - "step: 2742, Red action: do-nothing, Blue reward:0.85\n", - "step: 2743, Red action: do-nothing, Blue reward:0.85\n", - "step: 2744, Red action: do-nothing, Blue reward:0.85\n", - "step: 2745, Red action: do-nothing, Blue reward:0.85\n", - "step: 2746, Red action: do-nothing, Blue reward:0.85\n", - "step: 2747, Red action: do-nothing, Blue reward:0.85\n", - "step: 2748, Red action: do-nothing, Blue reward:0.85\n", - "step: 2749, Red action: do-nothing, Blue reward:0.85\n", - "step: 2750, Red action: do-nothing, Blue reward:0.85\n", - "step: 2751, Red action: do-nothing, Blue reward:0.85\n", - "step: 2752, Red action: do-nothing, Blue reward:0.85\n", - "step: 2753, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2754, Red action: do-nothing, Blue reward:0.85\n", - "step: 2755, Red action: do-nothing, Blue reward:0.85\n", - "step: 2756, Red action: do-nothing, Blue reward:0.85\n", - "step: 2757, Red action: do-nothing, Blue reward:0.85\n", - "step: 2758, Red action: do-nothing, Blue reward:0.85\n", - "step: 2759, Red action: do-nothing, Blue reward:0.85\n", - "step: 2760, Red action: do-nothing, Blue reward:0.85\n", - "step: 2761, Red action: do-nothing, Blue reward:0.85\n", - "step: 2762, Red action: do-nothing, Blue reward:0.85\n", - "step: 2763, Red action: do-nothing, Blue reward:0.85\n", - "step: 2764, Red action: do-nothing, Blue reward:0.85\n", - "step: 2765, Red action: do-nothing, Blue reward:0.85\n", - "step: 2766, Red action: do-nothing, Blue reward:0.85\n", - "step: 2767, Red action: do-nothing, Blue reward:0.85\n", - "step: 2768, Red action: do-nothing, Blue reward:0.85\n", - "step: 2769, Red action: do-nothing, Blue reward:0.85\n", - "step: 2770, Red action: do-nothing, Blue reward:0.85\n", - "step: 2771, Red action: do-nothing, Blue reward:0.85\n", - "step: 2772, Red action: do-nothing, Blue reward:0.85\n", - "step: 2773, Red action: do-nothing, Blue reward:0.85\n", - "step: 2774, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2775, Red action: do-nothing, Blue reward:0.85\n", - "step: 2776, Red action: do-nothing, Blue reward:0.85\n", - "step: 2777, Red action: do-nothing, Blue reward:0.85\n", - "step: 2778, Red action: do-nothing, Blue reward:0.85\n", - "step: 2779, Red action: do-nothing, Blue reward:0.85\n", - "step: 2780, Red action: do-nothing, Blue reward:0.85\n", - "step: 2781, Red action: do-nothing, Blue reward:0.85\n", - "step: 2782, Red action: do-nothing, Blue reward:0.85\n", - "step: 2783, Red action: do-nothing, Blue reward:0.85\n", - "step: 2784, Red action: do-nothing, Blue reward:0.85\n", - "step: 2785, Red action: do-nothing, Blue reward:0.85\n", - "step: 2786, Red action: do-nothing, Blue reward:0.85\n", - "step: 2787, Red action: do-nothing, Blue reward:0.85\n", - "step: 2788, Red action: do-nothing, Blue reward:0.85\n", - "step: 2789, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2790, Red action: do-nothing, Blue reward:0.85\n", - "step: 2791, Red action: do-nothing, Blue reward:0.85\n", - "step: 2792, Red action: do-nothing, Blue reward:0.85\n", - "step: 2793, Red action: do-nothing, Blue reward:0.85\n", - "step: 2794, Red action: do-nothing, Blue reward:0.85\n", - "step: 2795, Red action: do-nothing, Blue reward:0.85\n", - "step: 2796, Red action: do-nothing, Blue reward:0.85\n", - "step: 2797, Red action: do-nothing, Blue reward:0.85\n", - "step: 2798, Red action: do-nothing, Blue reward:0.85\n", - "step: 2799, Red action: do-nothing, Blue reward:0.85\n", - "step: 2800, Red action: do-nothing, Blue reward:0.85\n", - "step: 2801, Red action: do-nothing, Blue reward:0.85\n", - "step: 2802, Red action: do-nothing, Blue reward:0.85\n", - "step: 2803, Red action: do-nothing, Blue reward:0.85\n", - "step: 2804, Red action: do-nothing, Blue reward:0.85\n", - "step: 2805, Red action: do-nothing, Blue reward:0.85\n", - "step: 2806, Red action: do-nothing, Blue reward:0.85\n", - "step: 2807, Red action: do-nothing, Blue reward:0.85\n", - "step: 2808, Red action: do-nothing, Blue reward:0.85\n", - "step: 2809, Red action: do-nothing, Blue reward:0.85\n", - "step: 2810, Red action: do-nothing, Blue reward:0.85\n", - "step: 2811, Red action: do-nothing, Blue reward:0.85\n", - "step: 2812, Red action: do-nothing, Blue reward:0.85\n", - "step: 2813, Red action: do-nothing, Blue reward:0.85\n", - "step: 2814, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2815, Red action: do-nothing, Blue reward:0.85\n", - "step: 2816, Red action: do-nothing, Blue reward:0.85\n", - "step: 2817, Red action: do-nothing, Blue reward:0.85\n", - "step: 2818, Red action: do-nothing, Blue reward:0.85\n", - "step: 2819, Red action: do-nothing, Blue reward:0.85\n", - "step: 2820, Red action: do-nothing, Blue reward:0.85\n", - "step: 2821, Red action: do-nothing, Blue reward:0.85\n", - "step: 2822, Red action: do-nothing, Blue reward:0.85\n", - "step: 2823, Red action: do-nothing, Blue reward:0.85\n", - "step: 2824, Red action: do-nothing, Blue reward:0.85\n", - "step: 2825, Red action: do-nothing, Blue reward:0.85\n", - "step: 2826, Red action: do-nothing, Blue reward:0.85\n", - "step: 2827, Red action: do-nothing, Blue reward:0.85\n", - "step: 2828, Red action: do-nothing, Blue reward:0.85\n", - "step: 2829, Red action: do-nothing, Blue reward:0.85\n", - "step: 2830, Red action: do-nothing, Blue reward:0.85\n", - "step: 2831, Red action: do-nothing, Blue reward:0.85\n", - "step: 2832, Red action: do-nothing, Blue reward:0.85\n", - "step: 2833, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2834, Red action: do-nothing, Blue reward:0.85\n", - "step: 2835, Red action: do-nothing, Blue reward:0.85\n", - "step: 2836, Red action: do-nothing, Blue reward:0.85\n", - "step: 2837, Red action: do-nothing, Blue reward:0.85\n", - "step: 2838, Red action: do-nothing, Blue reward:0.85\n", - "step: 2839, Red action: do-nothing, Blue reward:0.85\n", - "step: 2840, Red action: do-nothing, Blue reward:0.85\n", - "step: 2841, Red action: do-nothing, Blue reward:0.85\n", - "step: 2842, Red action: do-nothing, Blue reward:0.85\n", - "step: 2843, Red action: do-nothing, Blue reward:0.85\n", - "step: 2844, Red action: do-nothing, Blue reward:0.85\n", - "step: 2845, Red action: do-nothing, Blue reward:0.85\n", - "step: 2846, Red action: do-nothing, Blue reward:0.85\n", - "step: 2847, Red action: do-nothing, Blue reward:0.85\n", - "step: 2848, Red action: do-nothing, Blue reward:0.85\n", - "step: 2849, Red action: do-nothing, Blue reward:0.85\n", - "step: 2850, Red action: do-nothing, Blue reward:0.85\n", - "step: 2851, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2852, Red action: do-nothing, Blue reward:0.85\n", - "step: 2853, Red action: do-nothing, Blue reward:0.85\n", - "step: 2854, Red action: do-nothing, Blue reward:0.85\n", - "step: 2855, Red action: do-nothing, Blue reward:0.85\n", - "step: 2856, Red action: do-nothing, Blue reward:0.85\n", - "step: 2857, Red action: do-nothing, Blue reward:0.85\n", - "step: 2858, Red action: do-nothing, Blue reward:0.85\n", - "step: 2859, Red action: do-nothing, Blue reward:0.85\n", - "step: 2860, Red action: do-nothing, Blue reward:0.85\n", - "step: 2861, Red action: do-nothing, Blue reward:0.85\n", - "step: 2862, Red action: do-nothing, Blue reward:0.85\n", - "step: 2863, Red action: do-nothing, Blue reward:0.85\n", - "step: 2864, Red action: do-nothing, Blue reward:0.85\n", - "step: 2865, Red action: do-nothing, Blue reward:0.85\n", - "step: 2866, Red action: do-nothing, Blue reward:0.85\n", - "step: 2867, Red action: do-nothing, Blue reward:0.85\n", - "step: 2868, Red action: do-nothing, Blue reward:0.85\n", - "step: 2869, Red action: do-nothing, Blue reward:0.85\n", - "step: 2870, Red action: do-nothing, Blue reward:0.85\n", - "step: 2871, Red action: do-nothing, Blue reward:0.85\n", - "step: 2872, Red action: do-nothing, Blue reward:0.85\n", - "step: 2873, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2874, Red action: do-nothing, Blue reward:0.85\n", - "step: 2875, Red action: do-nothing, Blue reward:0.85\n", - "step: 2876, Red action: do-nothing, Blue reward:0.85\n", - "step: 2877, Red action: do-nothing, Blue reward:0.85\n", - "step: 2878, Red action: do-nothing, Blue reward:0.85\n", - "step: 2879, Red action: do-nothing, Blue reward:0.85\n", - "step: 2880, Red action: do-nothing, Blue reward:0.85\n", - "step: 2881, Red action: do-nothing, Blue reward:0.85\n", - "step: 2882, Red action: do-nothing, Blue reward:0.85\n", - "step: 2883, Red action: do-nothing, Blue reward:0.85\n", - "step: 2884, Red action: do-nothing, Blue reward:0.85\n", - "step: 2885, Red action: do-nothing, Blue reward:0.85\n", - "step: 2886, Red action: do-nothing, Blue reward:0.85\n", - "step: 2887, Red action: do-nothing, Blue reward:0.85\n", - "step: 2888, Red action: do-nothing, Blue reward:0.85\n", - "step: 2889, Red action: do-nothing, Blue reward:0.85\n", - "step: 2890, Red action: do-nothing, Blue reward:0.85\n", - "step: 2891, Red action: do-nothing, Blue reward:0.85\n", - "step: 2892, Red action: do-nothing, Blue reward:0.85\n", - "step: 2893, Red action: do-nothing, Blue reward:0.85\n", - "step: 2894, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2895, Red action: do-nothing, Blue reward:0.85\n", - "step: 2896, Red action: do-nothing, Blue reward:0.85\n", - "step: 2897, Red action: do-nothing, Blue reward:0.85\n", - "step: 2898, Red action: do-nothing, Blue reward:0.85\n", - "step: 2899, Red action: do-nothing, Blue reward:0.85\n", - "step: 2900, Red action: do-nothing, Blue reward:0.85\n", - "step: 2901, Red action: do-nothing, Blue reward:0.85\n", - "step: 2902, Red action: do-nothing, Blue reward:0.85\n", - "step: 2903, Red action: do-nothing, Blue reward:0.85\n", - "step: 2904, Red action: do-nothing, Blue reward:0.85\n", - "step: 2905, Red action: do-nothing, Blue reward:0.85\n", - "step: 2906, Red action: do-nothing, Blue reward:0.85\n", - "step: 2907, Red action: do-nothing, Blue reward:0.85\n", - "step: 2908, Red action: do-nothing, Blue reward:0.85\n", - "step: 2909, Red action: do-nothing, Blue reward:0.85\n", - "step: 2910, Red action: do-nothing, Blue reward:0.85\n", - "step: 2911, Red action: do-nothing, Blue reward:0.85\n", - "step: 2912, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2913, Red action: do-nothing, Blue reward:0.85\n", - "step: 2914, Red action: do-nothing, Blue reward:0.85\n", - "step: 2915, Red action: do-nothing, Blue reward:0.85\n", - "step: 2916, Red action: do-nothing, Blue reward:0.85\n", - "step: 2917, Red action: do-nothing, Blue reward:0.85\n", - "step: 2918, Red action: do-nothing, Blue reward:0.85\n", - "step: 2919, Red action: do-nothing, Blue reward:0.85\n", - "step: 2920, Red action: do-nothing, Blue reward:0.85\n", - "step: 2921, Red action: do-nothing, Blue reward:0.85\n", - "step: 2922, Red action: do-nothing, Blue reward:0.85\n", - "step: 2923, Red action: do-nothing, Blue reward:0.85\n", - "step: 2924, Red action: do-nothing, Blue reward:0.85\n", - "step: 2925, Red action: do-nothing, Blue reward:0.85\n", - "step: 2926, Red action: do-nothing, Blue reward:0.85\n", - "step: 2927, Red action: do-nothing, Blue reward:0.85\n", - "step: 2928, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2929, Red action: do-nothing, Blue reward:0.85\n", - "step: 2930, Red action: do-nothing, Blue reward:0.85\n", - "step: 2931, Red action: do-nothing, Blue reward:0.85\n", - "step: 2932, Red action: do-nothing, Blue reward:0.85\n", - "step: 2933, Red action: do-nothing, Blue reward:0.85\n", - "step: 2934, Red action: do-nothing, Blue reward:0.85\n", - "step: 2935, Red action: do-nothing, Blue reward:0.85\n", - "step: 2936, Red action: do-nothing, Blue reward:0.85\n", - "step: 2937, Red action: do-nothing, Blue reward:0.85\n", - "step: 2938, Red action: do-nothing, Blue reward:0.85\n", - "step: 2939, Red action: do-nothing, Blue reward:0.85\n", - "step: 2940, Red action: do-nothing, Blue reward:0.85\n", - "step: 2941, Red action: do-nothing, Blue reward:0.85\n", - "step: 2942, Red action: do-nothing, Blue reward:0.85\n", - "step: 2943, Red action: do-nothing, Blue reward:0.85\n", - "step: 2944, Red action: do-nothing, Blue reward:0.85\n", - "step: 2945, Red action: do-nothing, Blue reward:0.85\n", - "step: 2946, Red action: do-nothing, Blue reward:0.85\n", - "step: 2947, Red action: do-nothing, Blue reward:0.85\n", - "step: 2948, Red action: do-nothing, Blue reward:0.85\n", - "step: 2949, Red action: do-nothing, Blue reward:0.85\n", - "step: 2950, Red action: do-nothing, Blue reward:0.85\n", - "step: 2951, Red action: do-nothing, Blue reward:0.85\n", - "step: 2952, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2953, Red action: do-nothing, Blue reward:0.85\n", - "step: 2954, Red action: do-nothing, Blue reward:0.85\n", - "step: 2955, Red action: do-nothing, Blue reward:0.85\n", - "step: 2956, Red action: do-nothing, Blue reward:0.85\n", - "step: 2957, Red action: do-nothing, Blue reward:0.85\n", - "step: 2958, Red action: do-nothing, Blue reward:0.85\n", - "step: 2959, Red action: do-nothing, Blue reward:0.85\n", - "step: 2960, Red action: do-nothing, Blue reward:0.85\n", - "step: 2961, Red action: do-nothing, Blue reward:0.85\n", - "step: 2962, Red action: do-nothing, Blue reward:0.85\n", - "step: 2963, Red action: do-nothing, Blue reward:0.85\n", - "step: 2964, Red action: do-nothing, Blue reward:0.85\n", - "step: 2965, Red action: do-nothing, Blue reward:0.85\n", - "step: 2966, Red action: do-nothing, Blue reward:0.85\n", - "step: 2967, Red action: do-nothing, Blue reward:0.85\n", - "step: 2968, Red action: do-nothing, Blue reward:0.85\n", - "step: 2969, Red action: do-nothing, Blue reward:0.85\n", - "step: 2970, Red action: do-nothing, Blue reward:0.85\n", - "step: 2971, Red action: do-nothing, Blue reward:0.85\n", - "step: 2972, Red action: do-nothing, Blue reward:0.85\n", - "step: 2973, Red action: do-nothing, Blue reward:0.85\n", - "step: 2974, Red action: do-nothing, Blue reward:0.85\n", - "step: 2975, Red action: do-nothing, Blue reward:0.85\n", - "step: 2976, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2977, Red action: do-nothing, Blue reward:0.85\n", - "step: 2978, Red action: do-nothing, Blue reward:0.85\n", - "step: 2979, Red action: do-nothing, Blue reward:0.85\n", - "step: 2980, Red action: do-nothing, Blue reward:0.85\n", - "step: 2981, Red action: do-nothing, Blue reward:0.85\n", - "step: 2982, Red action: do-nothing, Blue reward:0.85\n", - "step: 2983, Red action: do-nothing, Blue reward:0.85\n", - "step: 2984, Red action: do-nothing, Blue reward:0.85\n", - "step: 2985, Red action: do-nothing, Blue reward:0.85\n", - "step: 2986, Red action: do-nothing, Blue reward:0.85\n", - "step: 2987, Red action: do-nothing, Blue reward:0.85\n", - "step: 2988, Red action: do-nothing, Blue reward:0.85\n", - "step: 2989, Red action: do-nothing, Blue reward:0.85\n", - "step: 2990, Red action: do-nothing, Blue reward:0.85\n", - "step: 2991, Red action: do-nothing, Blue reward:0.85\n", - "step: 2992, Red action: do-nothing, Blue reward:0.85\n", - "step: 2993, Red action: do-nothing, Blue reward:0.85\n", - "step: 2994, Red action: do-nothing, Blue reward:0.85\n", - "step: 2995, Red action: do-nothing, Blue reward:0.85\n", - "step: 2996, Red action: do-nothing, Blue reward:0.85\n", - "step: 2997, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2998, Red action: do-nothing, Blue reward:0.85\n", - "step: 2999, Red action: do-nothing, Blue reward:0.85\n", - "step: 3000, Red action: do-nothing, Blue reward:0.85\n", - "step: 3001, Red action: do-nothing, Blue reward:0.85\n", - "step: 3002, Red action: do-nothing, Blue reward:0.85\n", - "step: 3003, Red action: do-nothing, Blue reward:0.85\n", - "step: 3004, Red action: do-nothing, Blue reward:0.85\n", - "step: 3005, Red action: do-nothing, Blue reward:0.85\n", - "step: 3006, Red action: do-nothing, Blue reward:0.85\n", - "step: 3007, Red action: do-nothing, Blue reward:0.85\n", - "step: 3008, Red action: do-nothing, Blue reward:0.85\n", - "step: 3009, Red action: do-nothing, Blue reward:0.85\n", - "step: 3010, Red action: do-nothing, Blue reward:0.85\n", - "step: 3011, Red action: do-nothing, Blue reward:0.85\n", - "step: 3012, Red action: do-nothing, Blue reward:0.85\n", - "step: 3013, Red action: do-nothing, Blue reward:0.85\n", - "step: 3014, Red action: do-nothing, Blue reward:0.85\n", - "step: 3015, Red action: do-nothing, Blue reward:0.85\n", - "step: 3016, Red action: do-nothing, Blue reward:0.85\n", - "step: 3017, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3018, Red action: do-nothing, Blue reward:0.85\n", - "step: 3019, Red action: do-nothing, Blue reward:0.85\n", - "step: 3020, Red action: do-nothing, Blue reward:0.85\n", - "step: 3021, Red action: do-nothing, Blue reward:0.85\n", - "step: 3022, Red action: do-nothing, Blue reward:0.85\n", - "step: 3023, Red action: do-nothing, Blue reward:0.85\n", - "step: 3024, Red action: do-nothing, Blue reward:0.85\n", - "step: 3025, Red action: do-nothing, Blue reward:0.85\n", - "step: 3026, Red action: do-nothing, Blue reward:0.85\n", - "step: 3027, Red action: do-nothing, Blue reward:0.85\n", - "step: 3028, Red action: do-nothing, Blue reward:0.85\n", - "step: 3029, Red action: do-nothing, Blue reward:0.85\n", - "step: 3030, Red action: do-nothing, Blue reward:0.85\n", - "step: 3031, Red action: do-nothing, Blue reward:0.85\n", - "step: 3032, Red action: do-nothing, Blue reward:0.85\n", - "step: 3033, Red action: do-nothing, Blue reward:0.85\n", - "step: 3034, Red action: do-nothing, Blue reward:0.85\n", - "step: 3035, Red action: do-nothing, Blue reward:0.85\n", - "step: 3036, Red action: do-nothing, Blue reward:0.85\n", - "step: 3037, Red action: do-nothing, Blue reward:0.85\n", - "step: 3038, Red action: do-nothing, Blue reward:0.85\n", - "step: 3039, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3040, Red action: do-nothing, Blue reward:0.85\n", - "step: 3041, Red action: do-nothing, Blue reward:0.85\n", - "step: 3042, Red action: do-nothing, Blue reward:0.85\n", - "step: 3043, Red action: do-nothing, Blue reward:0.85\n", - "step: 3044, Red action: do-nothing, Blue reward:0.85\n", - "step: 3045, Red action: do-nothing, Blue reward:0.85\n", - "step: 3046, Red action: do-nothing, Blue reward:0.85\n", - "step: 3047, Red action: do-nothing, Blue reward:0.85\n", - "step: 3048, Red action: do-nothing, Blue reward:0.85\n", - "step: 3049, Red action: do-nothing, Blue reward:0.85\n", - "step: 3050, Red action: do-nothing, Blue reward:0.85\n", - "step: 3051, Red action: do-nothing, Blue reward:0.85\n", - "step: 3052, Red action: do-nothing, Blue reward:0.85\n", - "step: 3053, Red action: do-nothing, Blue reward:0.85\n", - "step: 3054, Red action: do-nothing, Blue reward:0.85\n", - "step: 3055, Red action: do-nothing, Blue reward:0.85\n", - "step: 3056, Red action: do-nothing, Blue reward:0.85\n", - "step: 3057, Red action: do-nothing, Blue reward:0.85\n", - "step: 3058, Red action: do-nothing, Blue reward:0.85\n", - "step: 3059, Red action: do-nothing, Blue reward:0.85\n", - "step: 3060, Red action: do-nothing, Blue reward:0.85\n", - "step: 3061, Red action: do-nothing, Blue reward:0.85\n", - "step: 3062, Red action: do-nothing, Blue reward:0.85\n", - "step: 3063, Red action: do-nothing, Blue reward:0.85\n", - "step: 3064, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3065, Red action: do-nothing, Blue reward:0.85\n", - "step: 3066, Red action: do-nothing, Blue reward:0.85\n", - "step: 3067, Red action: do-nothing, Blue reward:0.85\n", - "step: 3068, Red action: do-nothing, Blue reward:0.85\n", - "step: 3069, Red action: do-nothing, Blue reward:0.85\n", - "step: 3070, Red action: do-nothing, Blue reward:0.85\n", - "step: 3071, Red action: do-nothing, Blue reward:0.85\n", - "step: 3072, Red action: do-nothing, Blue reward:0.85\n", - "step: 3073, Red action: do-nothing, Blue reward:0.85\n", - "step: 3074, Red action: do-nothing, Blue reward:0.85\n", - "step: 3075, Red action: do-nothing, Blue reward:0.85\n", - "step: 3076, Red action: do-nothing, Blue reward:0.85\n", - "step: 3077, Red action: do-nothing, Blue reward:0.85\n", - "step: 3078, Red action: do-nothing, Blue reward:0.85\n", - "step: 3079, Red action: do-nothing, Blue reward:0.85\n", - "step: 3080, Red action: do-nothing, Blue reward:0.85\n", - "step: 3081, Red action: do-nothing, Blue reward:0.85\n", - "step: 3082, Red action: do-nothing, Blue reward:0.85\n", - "step: 3083, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3084, Red action: do-nothing, Blue reward:0.85\n", - "step: 3085, Red action: do-nothing, Blue reward:0.85\n", - "step: 3086, Red action: do-nothing, Blue reward:0.85\n", - "step: 3087, Red action: do-nothing, Blue reward:0.85\n", - "step: 3088, Red action: do-nothing, Blue reward:0.85\n", - "step: 3089, Red action: do-nothing, Blue reward:0.85\n", - "step: 3090, Red action: do-nothing, Blue reward:0.85\n", - "step: 3091, Red action: do-nothing, Blue reward:0.85\n", - "step: 3092, Red action: do-nothing, Blue reward:0.85\n", - "step: 3093, Red action: do-nothing, Blue reward:0.85\n", - "step: 3094, Red action: do-nothing, Blue reward:0.85\n", - "step: 3095, Red action: do-nothing, Blue reward:0.85\n", - "step: 3096, Red action: do-nothing, Blue reward:0.85\n", - "step: 3097, Red action: do-nothing, Blue reward:0.85\n", - "step: 3098, Red action: do-nothing, Blue reward:0.85\n", - "step: 3099, Red action: do-nothing, Blue reward:0.85\n", - "step: 3100, Red action: do-nothing, Blue reward:0.85\n", - "step: 3101, Red action: do-nothing, Blue reward:0.85\n", - "step: 3102, Red action: do-nothing, Blue reward:0.85\n", - "step: 3103, Red action: do-nothing, Blue reward:0.85\n", - "step: 3104, Red action: do-nothing, Blue reward:0.85\n", - "step: 3105, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3106, Red action: do-nothing, Blue reward:0.85\n", - "step: 3107, Red action: do-nothing, Blue reward:0.85\n", - "step: 3108, Red action: do-nothing, Blue reward:0.85\n", - "step: 3109, Red action: do-nothing, Blue reward:0.85\n", - "step: 3110, Red action: do-nothing, Blue reward:0.85\n", - "step: 3111, Red action: do-nothing, Blue reward:0.85\n", - "step: 3112, Red action: do-nothing, Blue reward:0.85\n", - "step: 3113, Red action: do-nothing, Blue reward:0.85\n", - "step: 3114, Red action: do-nothing, Blue reward:0.85\n", - "step: 3115, Red action: do-nothing, Blue reward:0.85\n", - "step: 3116, Red action: do-nothing, Blue reward:0.85\n", - "step: 3117, Red action: do-nothing, Blue reward:0.85\n", - "step: 3118, Red action: do-nothing, Blue reward:0.85\n", - "step: 3119, Red action: do-nothing, Blue reward:0.85\n", - "step: 3120, Red action: do-nothing, Blue reward:0.85\n", - "step: 3121, Red action: do-nothing, Blue reward:0.85\n", - "step: 3122, Red action: do-nothing, Blue reward:0.85\n", - "step: 3123, Red action: do-nothing, Blue reward:0.85\n", - "step: 3124, Red action: do-nothing, Blue reward:0.85\n", - "step: 3125, Red action: do-nothing, Blue reward:0.85\n", - "step: 3126, Red action: do-nothing, Blue reward:0.85\n", - "step: 3127, Red action: do-nothing, Blue reward:0.85\n", - "step: 3128, Red action: do-nothing, Blue reward:0.85\n", - "step: 3129, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3130, Red action: do-nothing, Blue reward:0.85\n", - "step: 3131, Red action: do-nothing, Blue reward:0.85\n", - "step: 3132, Red action: do-nothing, Blue reward:0.85\n", - "step: 3133, Red action: do-nothing, Blue reward:0.85\n", - "step: 3134, Red action: do-nothing, Blue reward:0.85\n", - "step: 3135, Red action: do-nothing, Blue reward:0.85\n", - "step: 3136, Red action: do-nothing, Blue reward:0.85\n", - "step: 3137, Red action: do-nothing, Blue reward:0.85\n", - "step: 3138, Red action: do-nothing, Blue reward:0.85\n", - "step: 3139, Red action: do-nothing, Blue reward:0.85\n", - "step: 3140, Red action: do-nothing, Blue reward:0.85\n", - "step: 3141, Red action: do-nothing, Blue reward:0.85\n", - "step: 3142, Red action: do-nothing, Blue reward:0.85\n", - "step: 3143, Red action: do-nothing, Blue reward:0.85\n", - "step: 3144, Red action: do-nothing, Blue reward:0.85\n", - "step: 3145, Red action: do-nothing, Blue reward:0.85\n", - "step: 3146, Red action: do-nothing, Blue reward:0.85\n", - "step: 3147, Red action: do-nothing, Blue reward:0.85\n", - "step: 3148, Red action: do-nothing, Blue reward:0.85\n", - "step: 3149, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3150, Red action: do-nothing, Blue reward:0.85\n", - "step: 3151, Red action: do-nothing, Blue reward:0.85\n", - "step: 3152, Red action: do-nothing, Blue reward:0.85\n", - "step: 3153, Red action: do-nothing, Blue reward:0.85\n", - "step: 3154, Red action: do-nothing, Blue reward:0.85\n", - "step: 3155, Red action: do-nothing, Blue reward:0.85\n", - "step: 3156, Red action: do-nothing, Blue reward:0.85\n", - "step: 3157, Red action: do-nothing, Blue reward:0.85\n", - "step: 3158, Red action: do-nothing, Blue reward:0.85\n", - "step: 3159, Red action: do-nothing, Blue reward:0.85\n", - "step: 3160, Red action: do-nothing, Blue reward:0.85\n", - "step: 3161, Red action: do-nothing, Blue reward:0.85\n", - "step: 3162, Red action: do-nothing, Blue reward:0.85\n", - "step: 3163, Red action: do-nothing, Blue reward:0.85\n", - "step: 3164, Red action: do-nothing, Blue reward:0.85\n", - "step: 3165, Red action: do-nothing, Blue reward:0.85\n", - "step: 3166, Red action: do-nothing, Blue reward:0.85\n", - "step: 3167, Red action: do-nothing, Blue reward:0.85\n", - "step: 3168, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3169, Red action: do-nothing, Blue reward:0.85\n", - "step: 3170, Red action: do-nothing, Blue reward:0.85\n", - "step: 3171, Red action: do-nothing, Blue reward:0.85\n", - "step: 3172, Red action: do-nothing, Blue reward:0.85\n", - "step: 3173, Red action: do-nothing, Blue reward:0.85\n", - "step: 3174, Red action: do-nothing, Blue reward:0.85\n", - "step: 3175, Red action: do-nothing, Blue reward:0.85\n", - "step: 3176, Red action: do-nothing, Blue reward:0.85\n", - "step: 3177, Red action: do-nothing, Blue reward:0.85\n", - "step: 3178, Red action: do-nothing, Blue reward:0.85\n", - "step: 3179, Red action: do-nothing, Blue reward:0.85\n", - "step: 3180, Red action: do-nothing, Blue reward:0.85\n", - "step: 3181, Red action: do-nothing, Blue reward:0.85\n", - "step: 3182, Red action: do-nothing, Blue reward:0.85\n", - "step: 3183, Red action: do-nothing, Blue reward:0.85\n", - "step: 3184, Red action: do-nothing, Blue reward:0.85\n", - "step: 3185, Red action: do-nothing, Blue reward:0.85\n", - "step: 3186, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3187, Red action: do-nothing, Blue reward:0.85\n", - "step: 3188, Red action: do-nothing, Blue reward:0.85\n", - "step: 3189, Red action: do-nothing, Blue reward:0.85\n", - "step: 3190, Red action: do-nothing, Blue reward:0.85\n", - "step: 3191, Red action: do-nothing, Blue reward:0.85\n", - "step: 3192, Red action: do-nothing, Blue reward:0.85\n", - "step: 3193, Red action: do-nothing, Blue reward:0.85\n", - "step: 3194, Red action: do-nothing, Blue reward:0.85\n", - "step: 3195, Red action: do-nothing, Blue reward:0.85\n", - "step: 3196, Red action: do-nothing, Blue reward:0.85\n", - "step: 3197, Red action: do-nothing, Blue reward:0.85\n", - "step: 3198, Red action: do-nothing, Blue reward:0.85\n", - "step: 3199, Red action: do-nothing, Blue reward:0.85\n", - "step: 3200, Red action: do-nothing, Blue reward:0.85\n", - "step: 3201, Red action: do-nothing, Blue reward:0.85\n", - "step: 3202, Red action: do-nothing, Blue reward:0.85\n", - "step: 3203, Red action: do-nothing, Blue reward:0.85\n", - "step: 3204, Red action: do-nothing, Blue reward:0.85\n", - "step: 3205, Red action: do-nothing, Blue reward:0.85\n", - "step: 3206, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3207, Red action: do-nothing, Blue reward:0.85\n", - "step: 3208, Red action: do-nothing, Blue reward:0.85\n", - "step: 3209, Red action: do-nothing, Blue reward:0.85\n", - "step: 3210, Red action: do-nothing, Blue reward:0.85\n", - "step: 3211, Red action: do-nothing, Blue reward:0.85\n", - "step: 3212, Red action: do-nothing, Blue reward:0.85\n", - "step: 3213, Red action: do-nothing, Blue reward:0.85\n", - "step: 3214, Red action: do-nothing, Blue reward:0.85\n", - "step: 3215, Red action: do-nothing, Blue reward:0.85\n", - "step: 3216, Red action: do-nothing, Blue reward:0.85\n", - "step: 3217, Red action: do-nothing, Blue reward:0.85\n", - "step: 3218, Red action: do-nothing, Blue reward:0.85\n", - "step: 3219, Red action: do-nothing, Blue reward:0.85\n", - "step: 3220, Red action: do-nothing, Blue reward:0.85\n", - "step: 3221, Red action: do-nothing, Blue reward:0.85\n", - "step: 3222, Red action: do-nothing, Blue reward:0.85\n", - "step: 3223, Red action: do-nothing, Blue reward:0.85\n", - "step: 3224, Red action: do-nothing, Blue reward:0.85\n", - "step: 3225, Red action: do-nothing, Blue reward:0.85\n", - "step: 3226, Red action: do-nothing, Blue reward:0.85\n", - "step: 3227, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3228, Red action: do-nothing, Blue reward:0.85\n", - "step: 3229, Red action: do-nothing, Blue reward:0.85\n", - "step: 3230, Red action: do-nothing, Blue reward:0.85\n", - "step: 3231, Red action: do-nothing, Blue reward:0.85\n", - "step: 3232, Red action: do-nothing, Blue reward:0.85\n", - "step: 3233, Red action: do-nothing, Blue reward:0.85\n", - "step: 3234, Red action: do-nothing, Blue reward:0.85\n", - "step: 3235, Red action: do-nothing, Blue reward:0.85\n", - "step: 3236, Red action: do-nothing, Blue reward:0.85\n", - "step: 3237, Red action: do-nothing, Blue reward:0.85\n", - "step: 3238, Red action: do-nothing, Blue reward:0.85\n", - "step: 3239, Red action: do-nothing, Blue reward:0.85\n", - "step: 3240, Red action: do-nothing, Blue reward:0.85\n", - "step: 3241, Red action: do-nothing, Blue reward:0.85\n", - "step: 3242, Red action: do-nothing, Blue reward:0.85\n", - "step: 3243, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3244, Red action: do-nothing, Blue reward:0.85\n", - "step: 3245, Red action: do-nothing, Blue reward:0.85\n", - "step: 3246, Red action: do-nothing, Blue reward:0.85\n", - "step: 3247, Red action: do-nothing, Blue reward:0.85\n", - "step: 3248, Red action: do-nothing, Blue reward:0.85\n", - "step: 3249, Red action: do-nothing, Blue reward:0.85\n", - "step: 3250, Red action: do-nothing, Blue reward:0.85\n", - "step: 3251, Red action: do-nothing, Blue reward:0.85\n", - "step: 3252, Red action: do-nothing, Blue reward:0.85\n", - "step: 3253, Red action: do-nothing, Blue reward:0.85\n", - "step: 3254, Red action: do-nothing, Blue reward:0.85\n", - "step: 3255, Red action: do-nothing, Blue reward:0.85\n", - "step: 3256, Red action: do-nothing, Blue reward:0.85\n", - "step: 3257, Red action: do-nothing, Blue reward:0.85\n", - "step: 3258, Red action: do-nothing, Blue reward:0.85\n", - "step: 3259, Red action: do-nothing, Blue reward:0.85\n", - "step: 3260, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3261, Red action: do-nothing, Blue reward:0.85\n", - "step: 3262, Red action: do-nothing, Blue reward:0.85\n", - "step: 3263, Red action: do-nothing, Blue reward:0.85\n", - "step: 3264, Red action: do-nothing, Blue reward:0.85\n", - "step: 3265, Red action: do-nothing, Blue reward:0.85\n", - "step: 3266, Red action: do-nothing, Blue reward:0.85\n", - "step: 3267, Red action: do-nothing, Blue reward:0.85\n", - "step: 3268, Red action: do-nothing, Blue reward:0.85\n", - "step: 3269, Red action: do-nothing, Blue reward:0.85\n", - "step: 3270, Red action: do-nothing, Blue reward:0.85\n", - "step: 3271, Red action: do-nothing, Blue reward:0.85\n", - "step: 3272, Red action: do-nothing, Blue reward:0.85\n", - "step: 3273, Red action: do-nothing, Blue reward:0.85\n", - "step: 3274, Red action: do-nothing, Blue reward:0.85\n", - "step: 3275, Red action: do-nothing, Blue reward:0.85\n", - "step: 3276, Red action: do-nothing, Blue reward:0.85\n", - "step: 3277, Red action: do-nothing, Blue reward:0.85\n", - "step: 3278, Red action: do-nothing, Blue reward:0.85\n", - "step: 3279, Red action: do-nothing, Blue reward:0.85\n", - "step: 3280, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3281, Red action: do-nothing, Blue reward:0.85\n", - "step: 3282, Red action: do-nothing, Blue reward:0.85\n", - "step: 3283, Red action: do-nothing, Blue reward:0.85\n", - "step: 3284, Red action: do-nothing, Blue reward:0.85\n", - "step: 3285, Red action: do-nothing, Blue reward:0.85\n", - "step: 3286, Red action: do-nothing, Blue reward:0.85\n", - "step: 3287, Red action: do-nothing, Blue reward:0.85\n", - "step: 3288, Red action: do-nothing, Blue reward:0.85\n", - "step: 3289, Red action: do-nothing, Blue reward:0.85\n", - "step: 3290, Red action: do-nothing, Blue reward:0.85\n", - "step: 3291, Red action: do-nothing, Blue reward:0.85\n", - "step: 3292, Red action: do-nothing, Blue reward:0.85\n", - "step: 3293, Red action: do-nothing, Blue reward:0.85\n", - "step: 3294, Red action: do-nothing, Blue reward:0.85\n", - "step: 3295, Red action: do-nothing, Blue reward:0.85\n", - "step: 3296, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3297, Red action: do-nothing, Blue reward:0.85\n", - "step: 3298, Red action: do-nothing, Blue reward:0.85\n", - "step: 3299, Red action: do-nothing, Blue reward:0.85\n", - "step: 3300, Red action: do-nothing, Blue reward:0.85\n", - "step: 3301, Red action: do-nothing, Blue reward:0.85\n", - "step: 3302, Red action: do-nothing, Blue reward:0.85\n", - "step: 3303, Red action: do-nothing, Blue reward:0.85\n", - "step: 3304, Red action: do-nothing, Blue reward:0.85\n", - "step: 3305, Red action: do-nothing, Blue reward:0.85\n", - "step: 3306, Red action: do-nothing, Blue reward:0.85\n", - "step: 3307, Red action: do-nothing, Blue reward:0.85\n", - "step: 3308, Red action: do-nothing, Blue reward:0.85\n", - "step: 3309, Red action: do-nothing, Blue reward:0.85\n", - "step: 3310, Red action: do-nothing, Blue reward:0.85\n", - "step: 3311, Red action: do-nothing, Blue reward:0.85\n", - "step: 3312, Red action: do-nothing, Blue reward:0.85\n", - "step: 3313, Red action: do-nothing, Blue reward:0.85\n", - "step: 3314, Red action: do-nothing, Blue reward:0.85\n", - "step: 3315, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3316, Red action: do-nothing, Blue reward:0.85\n", - "step: 3317, Red action: do-nothing, Blue reward:0.85\n", - "step: 3318, Red action: do-nothing, Blue reward:0.85\n", - "step: 3319, Red action: do-nothing, Blue reward:0.85\n", - "step: 3320, Red action: do-nothing, Blue reward:0.85\n", - "step: 3321, Red action: do-nothing, Blue reward:0.85\n", - "step: 3322, Red action: do-nothing, Blue reward:0.85\n", - "step: 3323, Red action: do-nothing, Blue reward:0.85\n", - "step: 3324, Red action: do-nothing, Blue reward:0.85\n", - "step: 3325, Red action: do-nothing, Blue reward:0.85\n", - "step: 3326, Red action: do-nothing, Blue reward:0.85\n", - "step: 3327, Red action: do-nothing, Blue reward:0.85\n", - "step: 3328, Red action: do-nothing, Blue reward:0.85\n", - "step: 3329, Red action: do-nothing, Blue reward:0.85\n", - "step: 3330, Red action: do-nothing, Blue reward:0.85\n", - "step: 3331, Red action: do-nothing, Blue reward:0.85\n", - "step: 3332, Red action: do-nothing, Blue reward:0.85\n", - "step: 3333, Red action: do-nothing, Blue reward:0.85\n", - "step: 3334, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3335, Red action: do-nothing, Blue reward:0.85\n", - "step: 3336, Red action: do-nothing, Blue reward:0.85\n", - "step: 3337, Red action: do-nothing, Blue reward:0.85\n", - "step: 3338, Red action: do-nothing, Blue reward:0.85\n", - "step: 3339, Red action: do-nothing, Blue reward:0.85\n", - "step: 3340, Red action: do-nothing, Blue reward:0.85\n", - "step: 3341, Red action: do-nothing, Blue reward:0.85\n", - "step: 3342, Red action: do-nothing, Blue reward:0.85\n", - "step: 3343, Red action: do-nothing, Blue reward:0.85\n", - "step: 3344, Red action: do-nothing, Blue reward:0.85\n", - "step: 3345, Red action: do-nothing, Blue reward:0.85\n", - "step: 3346, Red action: do-nothing, Blue reward:0.85\n", - "step: 3347, Red action: do-nothing, Blue reward:0.85\n", - "step: 3348, Red action: do-nothing, Blue reward:0.85\n", - "step: 3349, Red action: do-nothing, Blue reward:0.85\n", - "step: 3350, Red action: do-nothing, Blue reward:0.85\n", - "step: 3351, Red action: do-nothing, Blue reward:0.85\n", - "step: 3352, Red action: do-nothing, Blue reward:0.85\n", - "step: 3353, Red action: do-nothing, Blue reward:0.85\n", - "step: 3354, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3355, Red action: do-nothing, Blue reward:0.85\n", - "step: 3356, Red action: do-nothing, Blue reward:0.85\n", - "step: 3357, Red action: do-nothing, Blue reward:0.85\n", - "step: 3358, Red action: do-nothing, Blue reward:0.85\n", - "step: 3359, Red action: do-nothing, Blue reward:0.85\n", - "step: 3360, Red action: do-nothing, Blue reward:0.85\n", - "step: 3361, Red action: do-nothing, Blue reward:0.85\n", - "step: 3362, Red action: do-nothing, Blue reward:0.85\n", - "step: 3363, Red action: do-nothing, Blue reward:0.85\n", - "step: 3364, Red action: do-nothing, Blue reward:0.85\n", - "step: 3365, Red action: do-nothing, Blue reward:0.85\n", - "step: 3366, Red action: do-nothing, Blue reward:0.85\n", - "step: 3367, Red action: do-nothing, Blue reward:0.85\n", - "step: 3368, Red action: do-nothing, Blue reward:0.85\n", - "step: 3369, Red action: do-nothing, Blue reward:0.85\n", - "step: 3370, Red action: do-nothing, Blue reward:0.85\n", - "step: 3371, Red action: do-nothing, Blue reward:0.85\n", - "step: 3372, Red action: do-nothing, Blue reward:0.85\n", - "step: 3373, Red action: do-nothing, Blue reward:0.85\n", - "step: 3374, Red action: do-nothing, Blue reward:0.85\n", - "step: 3375, Red action: do-nothing, Blue reward:0.85\n", - "step: 3376, Red action: do-nothing, Blue reward:0.85\n", - "step: 3377, Red action: do-nothing, Blue reward:0.85\n", - "step: 3378, Red action: do-nothing, Blue reward:0.85\n", - "step: 3379, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3380, Red action: do-nothing, Blue reward:0.85\n", - "step: 3381, Red action: do-nothing, Blue reward:0.85\n", - "step: 3382, Red action: do-nothing, Blue reward:0.85\n", - "step: 3383, Red action: do-nothing, Blue reward:0.85\n", - "step: 3384, Red action: do-nothing, Blue reward:0.85\n", - "step: 3385, Red action: do-nothing, Blue reward:0.85\n", - "step: 3386, Red action: do-nothing, Blue reward:0.85\n", - "step: 3387, Red action: do-nothing, Blue reward:0.85\n", - "step: 3388, Red action: do-nothing, Blue reward:0.85\n", - "step: 3389, Red action: do-nothing, Blue reward:0.85\n", - "step: 3390, Red action: do-nothing, Blue reward:0.85\n", - "step: 3391, Red action: do-nothing, Blue reward:0.85\n", - "step: 3392, Red action: do-nothing, Blue reward:0.85\n", - "step: 3393, Red action: do-nothing, Blue reward:0.85\n", - "step: 3394, Red action: do-nothing, Blue reward:0.85\n", - "step: 3395, Red action: do-nothing, Blue reward:0.85\n", - "step: 3396, Red action: do-nothing, Blue reward:0.85\n", - "step: 3397, Red action: do-nothing, Blue reward:0.85\n", - "step: 3398, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3399, Red action: do-nothing, Blue reward:0.85\n", - "step: 3400, Red action: do-nothing, Blue reward:0.85\n", - "step: 3401, Red action: do-nothing, Blue reward:0.85\n", - "step: 3402, Red action: do-nothing, Blue reward:0.85\n", - "step: 3403, Red action: do-nothing, Blue reward:0.85\n", - "step: 3404, Red action: do-nothing, Blue reward:0.85\n", - "step: 3405, Red action: do-nothing, Blue reward:0.85\n", - "step: 3406, Red action: do-nothing, Blue reward:0.85\n", - "step: 3407, Red action: do-nothing, Blue reward:0.85\n", - "step: 3408, Red action: do-nothing, Blue reward:0.85\n", - "step: 3409, Red action: do-nothing, Blue reward:0.85\n", - "step: 3410, Red action: do-nothing, Blue reward:0.85\n", - "step: 3411, Red action: do-nothing, Blue reward:0.85\n", - "step: 3412, Red action: do-nothing, Blue reward:0.85\n", - "step: 3413, Red action: do-nothing, Blue reward:0.85\n", - "step: 3414, Red action: do-nothing, Blue reward:0.85\n", - "step: 3415, Red action: do-nothing, Blue reward:0.85\n", - "step: 3416, Red action: do-nothing, Blue reward:0.85\n", - "step: 3417, Red action: do-nothing, Blue reward:0.85\n", - "step: 3418, Red action: do-nothing, Blue reward:0.85\n", - "step: 3419, Red action: do-nothing, Blue reward:0.85\n", - "step: 3420, Red action: do-nothing, Blue reward:0.85\n", - "step: 3421, Red action: do-nothing, Blue reward:0.85\n", - "step: 3422, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3423, Red action: do-nothing, Blue reward:0.85\n", - "step: 3424, Red action: do-nothing, Blue reward:0.85\n", - "step: 3425, Red action: do-nothing, Blue reward:0.85\n", - "step: 3426, Red action: do-nothing, Blue reward:0.85\n", - "step: 3427, Red action: do-nothing, Blue reward:0.85\n", - "step: 3428, Red action: do-nothing, Blue reward:0.85\n", - "step: 3429, Red action: do-nothing, Blue reward:0.85\n", - "step: 3430, Red action: do-nothing, Blue reward:0.85\n", - "step: 3431, Red action: do-nothing, Blue reward:0.85\n", - "step: 3432, Red action: do-nothing, Blue reward:0.85\n", - "step: 3433, Red action: do-nothing, Blue reward:0.85\n", - "step: 3434, Red action: do-nothing, Blue reward:0.85\n", - "step: 3435, Red action: do-nothing, Blue reward:0.85\n", - "step: 3436, Red action: do-nothing, Blue reward:0.85\n", - "step: 3437, Red action: do-nothing, Blue reward:0.85\n", - "step: 3438, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3439, Red action: do-nothing, Blue reward:0.85\n", - "step: 3440, Red action: do-nothing, Blue reward:0.85\n", - "step: 3441, Red action: do-nothing, Blue reward:0.85\n", - "step: 3442, Red action: do-nothing, Blue reward:0.85\n", - "step: 3443, Red action: do-nothing, Blue reward:0.85\n", - "step: 3444, Red action: do-nothing, Blue reward:0.85\n", - "step: 3445, Red action: do-nothing, Blue reward:0.85\n", - "step: 3446, Red action: do-nothing, Blue reward:0.85\n", - "step: 3447, Red action: do-nothing, Blue reward:0.85\n", - "step: 3448, Red action: do-nothing, Blue reward:0.85\n", - "step: 3449, Red action: do-nothing, Blue reward:0.85\n", - "step: 3450, Red action: do-nothing, Blue reward:0.85\n", - "step: 3451, Red action: do-nothing, Blue reward:0.85\n", - "step: 3452, Red action: do-nothing, Blue reward:0.85\n", - "step: 3453, Red action: do-nothing, Blue reward:0.85\n", - "step: 3454, Red action: do-nothing, Blue reward:0.85\n", - "step: 3455, Red action: do-nothing, Blue reward:0.85\n", - "step: 3456, Red action: do-nothing, Blue reward:0.85\n", - "step: 3457, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3458, Red action: do-nothing, Blue reward:0.85\n", - "step: 3459, Red action: do-nothing, Blue reward:0.85\n", - "step: 3460, Red action: do-nothing, Blue reward:0.85\n", - "step: 3461, Red action: do-nothing, Blue reward:0.85\n", - "step: 3462, Red action: do-nothing, Blue reward:0.85\n", - "step: 3463, Red action: do-nothing, Blue reward:0.85\n", - "step: 3464, Red action: do-nothing, Blue reward:0.85\n", - "step: 3465, Red action: do-nothing, Blue reward:0.85\n", - "step: 3466, Red action: do-nothing, Blue reward:0.85\n", - "step: 3467, Red action: do-nothing, Blue reward:0.85\n", - "step: 3468, Red action: do-nothing, Blue reward:0.85\n", - "step: 3469, Red action: do-nothing, Blue reward:0.85\n", - "step: 3470, Red action: do-nothing, Blue reward:0.85\n", - "step: 3471, Red action: do-nothing, Blue reward:0.85\n", - "step: 3472, Red action: do-nothing, Blue reward:0.85\n", - "step: 3473, Red action: do-nothing, Blue reward:0.85\n", - "step: 3474, Red action: do-nothing, Blue reward:0.85\n", - "step: 3475, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3476, Red action: do-nothing, Blue reward:0.85\n", - "step: 3477, Red action: do-nothing, Blue reward:0.85\n", - "step: 3478, Red action: do-nothing, Blue reward:0.85\n", - "step: 3479, Red action: do-nothing, Blue reward:0.85\n", - "step: 3480, Red action: do-nothing, Blue reward:0.85\n", - "step: 3481, Red action: do-nothing, Blue reward:0.85\n", - "step: 3482, Red action: do-nothing, Blue reward:0.85\n", - "step: 3483, Red action: do-nothing, Blue reward:0.85\n", - "step: 3484, Red action: do-nothing, Blue reward:0.85\n", - "step: 3485, Red action: do-nothing, Blue reward:0.85\n", - "step: 3486, Red action: do-nothing, Blue reward:0.85\n", - "step: 3487, Red action: do-nothing, Blue reward:0.85\n", - "step: 3488, Red action: do-nothing, Blue reward:0.85\n", - "step: 3489, Red action: do-nothing, Blue reward:0.85\n", - "step: 3490, Red action: do-nothing, Blue reward:0.85\n", - "step: 3491, Red action: do-nothing, Blue reward:0.85\n", - "step: 3492, Red action: do-nothing, Blue reward:0.85\n", - "step: 3493, Red action: do-nothing, Blue reward:0.85\n", - "step: 3494, Red action: do-nothing, Blue reward:0.85\n", - "step: 3495, Red action: do-nothing, Blue reward:0.85\n", - "step: 3496, Red action: do-nothing, Blue reward:0.85\n", - "step: 3497, Red action: do-nothing, Blue reward:0.85\n", - "step: 3498, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3499, Red action: do-nothing, Blue reward:0.85\n", - "step: 3500, Red action: do-nothing, Blue reward:0.85\n", - "step: 3501, Red action: do-nothing, Blue reward:0.85\n", - "step: 3502, Red action: do-nothing, Blue reward:0.85\n", - "step: 3503, Red action: do-nothing, Blue reward:0.85\n", - "step: 3504, Red action: do-nothing, Blue reward:0.85\n", - "step: 3505, Red action: do-nothing, Blue reward:0.85\n", - "step: 3506, Red action: do-nothing, Blue reward:0.85\n", - "step: 3507, Red action: do-nothing, Blue reward:0.85\n", - "step: 3508, Red action: do-nothing, Blue reward:0.85\n", - "step: 3509, Red action: do-nothing, Blue reward:0.85\n", - "step: 3510, Red action: do-nothing, Blue reward:0.85\n", - "step: 3511, Red action: do-nothing, Blue reward:0.85\n", - "step: 3512, Red action: do-nothing, Blue reward:0.85\n", - "step: 3513, Red action: do-nothing, Blue reward:0.85\n", - "step: 3514, Red action: do-nothing, Blue reward:0.85\n", - "step: 3515, Red action: do-nothing, Blue reward:0.85\n", - "step: 3516, Red action: do-nothing, Blue reward:0.85\n", - "step: 3517, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3518, Red action: do-nothing, Blue reward:0.85\n", - "step: 3519, Red action: do-nothing, Blue reward:0.85\n", - "step: 3520, Red action: do-nothing, Blue reward:0.85\n", - "step: 3521, Red action: do-nothing, Blue reward:0.85\n", - "step: 3522, Red action: do-nothing, Blue reward:0.85\n", - "step: 3523, Red action: do-nothing, Blue reward:0.85\n", - "step: 3524, Red action: do-nothing, Blue reward:0.85\n", - "step: 3525, Red action: do-nothing, Blue reward:0.85\n", - "step: 3526, Red action: do-nothing, Blue reward:0.85\n", - "step: 3527, Red action: do-nothing, Blue reward:0.85\n", - "step: 3528, Red action: do-nothing, Blue reward:0.85\n", - "step: 3529, Red action: do-nothing, Blue reward:0.85\n", - "step: 3530, Red action: do-nothing, Blue reward:0.85\n", - "step: 3531, Red action: do-nothing, Blue reward:0.85\n", - "step: 3532, Red action: do-nothing, Blue reward:0.85\n", - "step: 3533, Red action: do-nothing, Blue reward:0.85\n", - "step: 3534, Red action: do-nothing, Blue reward:0.85\n", - "step: 3535, Red action: do-nothing, Blue reward:0.85\n", - "step: 3536, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3537, Red action: do-nothing, Blue reward:0.85\n", - "step: 3538, Red action: do-nothing, Blue reward:0.85\n", - "step: 3539, Red action: do-nothing, Blue reward:0.85\n", - "step: 3540, Red action: do-nothing, Blue reward:0.85\n", - "step: 3541, Red action: do-nothing, Blue reward:0.85\n", - "step: 3542, Red action: do-nothing, Blue reward:0.85\n", - "step: 3543, Red action: do-nothing, Blue reward:0.85\n", - "step: 3544, Red action: do-nothing, Blue reward:0.85\n", - "step: 3545, Red action: do-nothing, Blue reward:0.85\n", - "step: 3546, Red action: do-nothing, Blue reward:0.85\n", - "step: 3547, Red action: do-nothing, Blue reward:0.85\n", - "step: 3548, Red action: do-nothing, Blue reward:0.85\n", - "step: 3549, Red action: do-nothing, Blue reward:0.85\n", - "step: 3550, Red action: do-nothing, Blue reward:0.85\n", - "step: 3551, Red action: do-nothing, Blue reward:0.85\n", - "step: 3552, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3553, Red action: do-nothing, Blue reward:0.85\n", - "step: 3554, Red action: do-nothing, Blue reward:0.85\n", - "step: 3555, Red action: do-nothing, Blue reward:0.85\n", - "step: 3556, Red action: do-nothing, Blue reward:0.85\n", - "step: 3557, Red action: do-nothing, Blue reward:0.85\n", - "step: 3558, Red action: do-nothing, Blue reward:0.85\n", - "step: 3559, Red action: do-nothing, Blue reward:0.85\n", - "step: 3560, Red action: do-nothing, Blue reward:0.85\n", - "step: 3561, Red action: do-nothing, Blue reward:0.85\n", - "step: 3562, Red action: do-nothing, Blue reward:0.85\n", - "step: 3563, Red action: do-nothing, Blue reward:0.85\n", - "step: 3564, Red action: do-nothing, Blue reward:0.85\n", - "step: 3565, Red action: do-nothing, Blue reward:0.85\n", - "step: 3566, Red action: do-nothing, Blue reward:0.85\n", - "step: 3567, Red action: do-nothing, Blue reward:0.85\n", - "step: 3568, Red action: do-nothing, Blue reward:0.85\n", - "step: 3569, Red action: do-nothing, Blue reward:0.85\n", - "step: 3570, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3571, Red action: do-nothing, Blue reward:0.85\n", - "step: 3572, Red action: do-nothing, Blue reward:0.85\n", - "step: 3573, Red action: do-nothing, Blue reward:0.85\n", - "step: 3574, Red action: do-nothing, Blue reward:0.85\n", - "step: 3575, Red action: do-nothing, Blue reward:0.85\n", - "step: 3576, Red action: do-nothing, Blue reward:0.85\n", - "step: 3577, Red action: do-nothing, Blue reward:0.85\n", - "step: 3578, Red action: do-nothing, Blue reward:0.85\n", - "step: 3579, Red action: do-nothing, Blue reward:0.85\n", - "step: 3580, Red action: do-nothing, Blue reward:0.85\n", - "step: 3581, Red action: do-nothing, Blue reward:0.85\n", - "step: 3582, Red action: do-nothing, Blue reward:0.85\n", - "step: 3583, Red action: do-nothing, Blue reward:0.85\n", - "step: 3584, Red action: do-nothing, Blue reward:0.85\n", - "step: 3585, Red action: do-nothing, Blue reward:0.85\n", - "step: 3586, Red action: do-nothing, Blue reward:0.85\n", - "step: 3587, Red action: do-nothing, Blue reward:0.85\n", - "step: 3588, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3589, Red action: do-nothing, Blue reward:0.85\n", - "step: 3590, Red action: do-nothing, Blue reward:0.85\n", - "step: 3591, Red action: do-nothing, Blue reward:0.85\n", - "step: 3592, Red action: do-nothing, Blue reward:0.85\n", - "step: 3593, Red action: do-nothing, Blue reward:0.85\n", - "step: 3594, Red action: do-nothing, Blue reward:0.85\n", - "step: 3595, Red action: do-nothing, Blue reward:0.85\n", - "step: 3596, Red action: do-nothing, Blue reward:0.85\n", - "step: 3597, Red action: do-nothing, Blue reward:0.85\n", - "step: 3598, Red action: do-nothing, Blue reward:0.85\n", - "step: 3599, Red action: do-nothing, Blue reward:0.85\n", - "step: 3600, Red action: do-nothing, Blue reward:0.85\n", - "step: 3601, Red action: do-nothing, Blue reward:0.85\n", - "step: 3602, Red action: do-nothing, Blue reward:0.85\n", - "step: 3603, Red action: do-nothing, Blue reward:0.85\n", - "step: 3604, Red action: do-nothing, Blue reward:0.85\n", - "step: 3605, Red action: do-nothing, Blue reward:0.85\n", - "step: 3606, Red action: do-nothing, Blue reward:0.85\n", - "step: 3607, Red action: do-nothing, Blue reward:0.85\n", - "step: 3608, Red action: do-nothing, Blue reward:0.85\n", - "step: 3609, Red action: do-nothing, Blue reward:0.85\n", - "step: 3610, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3611, Red action: do-nothing, Blue reward:0.85\n", - "step: 3612, Red action: do-nothing, Blue reward:0.85\n", - "step: 3613, Red action: do-nothing, Blue reward:0.85\n", - "step: 3614, Red action: do-nothing, Blue reward:0.85\n", - "step: 3615, Red action: do-nothing, Blue reward:0.85\n", - "step: 3616, Red action: do-nothing, Blue reward:0.85\n", - "step: 3617, Red action: do-nothing, Blue reward:0.85\n", - "step: 3618, Red action: do-nothing, Blue reward:0.85\n", - "step: 3619, Red action: do-nothing, Blue reward:0.85\n", - "step: 3620, Red action: do-nothing, Blue reward:0.85\n", - "step: 3621, Red action: do-nothing, Blue reward:0.85\n", - "step: 3622, Red action: do-nothing, Blue reward:0.85\n", - "step: 3623, Red action: do-nothing, Blue reward:0.85\n", - "step: 3624, Red action: do-nothing, Blue reward:0.85\n", - "step: 3625, Red action: do-nothing, Blue reward:0.85\n", - "step: 3626, Red action: do-nothing, Blue reward:0.85\n", - "step: 3627, Red action: do-nothing, Blue reward:0.85\n", - "step: 3628, Red action: do-nothing, Blue reward:0.85\n", - "step: 3629, Red action: do-nothing, Blue reward:0.85\n", - "step: 3630, Red action: do-nothing, Blue reward:0.85\n", - "step: 3631, Red action: do-nothing, Blue reward:0.85\n", - "step: 3632, Red action: do-nothing, Blue reward:0.85\n", - "step: 3633, Red action: do-nothing, Blue reward:0.85\n", - "step: 3634, Red action: do-nothing, Blue reward:0.85\n", - "step: 3635, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3636, Red action: do-nothing, Blue reward:0.85\n", - "step: 3637, Red action: do-nothing, Blue reward:0.85\n", - "step: 3638, Red action: do-nothing, Blue reward:0.85\n", - "step: 3639, Red action: do-nothing, Blue reward:0.85\n", - "step: 3640, Red action: do-nothing, Blue reward:0.85\n", - "step: 3641, Red action: do-nothing, Blue reward:0.85\n", - "step: 3642, Red action: do-nothing, Blue reward:0.85\n", - "step: 3643, Red action: do-nothing, Blue reward:0.85\n", - "step: 3644, Red action: do-nothing, Blue reward:0.85\n", - "step: 3645, Red action: do-nothing, Blue reward:0.85\n", - "step: 3646, Red action: do-nothing, Blue reward:0.85\n", - "step: 3647, Red action: do-nothing, Blue reward:0.85\n", - "step: 3648, Red action: do-nothing, Blue reward:0.85\n", - "step: 3649, Red action: do-nothing, Blue reward:0.85\n", - "step: 3650, Red action: do-nothing, Blue reward:0.85\n", - "step: 3651, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3652, Red action: do-nothing, Blue reward:0.85\n", - "step: 3653, Red action: do-nothing, Blue reward:0.85\n", - "step: 3654, Red action: do-nothing, Blue reward:0.85\n", - "step: 3655, Red action: do-nothing, Blue reward:0.85\n", - "step: 3656, Red action: do-nothing, Blue reward:0.85\n", - "step: 3657, Red action: do-nothing, Blue reward:0.85\n", - "step: 3658, Red action: do-nothing, Blue reward:0.85\n", - "step: 3659, Red action: do-nothing, Blue reward:0.85\n", - "step: 3660, Red action: do-nothing, Blue reward:0.85\n", - "step: 3661, Red action: do-nothing, Blue reward:0.85\n", - "step: 3662, Red action: do-nothing, Blue reward:0.85\n", - "step: 3663, Red action: do-nothing, Blue reward:0.85\n", - "step: 3664, Red action: do-nothing, Blue reward:0.85\n", - "step: 3665, Red action: do-nothing, Blue reward:0.85\n", - "step: 3666, Red action: do-nothing, Blue reward:0.85\n", - "step: 3667, Red action: do-nothing, Blue reward:0.85\n", - "step: 3668, Red action: do-nothing, Blue reward:0.85\n", - "step: 3669, Red action: do-nothing, Blue reward:0.85\n", - "step: 3670, Red action: do-nothing, Blue reward:0.85\n", - "step: 3671, Red action: do-nothing, Blue reward:0.85\n", - "step: 3672, Red action: do-nothing, Blue reward:0.85\n", - "step: 3673, Red action: do-nothing, Blue reward:0.85\n", - "step: 3674, Red action: do-nothing, Blue reward:0.85\n", - "step: 3675, Red action: do-nothing, Blue reward:0.85\n", - "step: 3676, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3677, Red action: do-nothing, Blue reward:0.85\n", - "step: 3678, Red action: do-nothing, Blue reward:0.85\n", - "step: 3679, Red action: do-nothing, Blue reward:0.85\n", - "step: 3680, Red action: do-nothing, Blue reward:0.85\n", - "step: 3681, Red action: do-nothing, Blue reward:0.85\n", - "step: 3682, Red action: do-nothing, Blue reward:0.85\n", - "step: 3683, Red action: do-nothing, Blue reward:0.85\n", - "step: 3684, Red action: do-nothing, Blue reward:0.85\n", - "step: 3685, Red action: do-nothing, Blue reward:0.85\n", - "step: 3686, Red action: do-nothing, Blue reward:0.85\n", - "step: 3687, Red action: do-nothing, Blue reward:0.85\n", - "step: 3688, Red action: do-nothing, Blue reward:0.85\n", - "step: 3689, Red action: do-nothing, Blue reward:0.85\n", - "step: 3690, Red action: do-nothing, Blue reward:0.85\n", - "step: 3691, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3692, Red action: do-nothing, Blue reward:0.85\n", - "step: 3693, Red action: do-nothing, Blue reward:0.85\n", - "step: 3694, Red action: do-nothing, Blue reward:0.85\n", - "step: 3695, Red action: do-nothing, Blue reward:0.85\n", - "step: 3696, Red action: do-nothing, Blue reward:0.85\n", - "step: 3697, Red action: do-nothing, Blue reward:0.85\n", - "step: 3698, Red action: do-nothing, Blue reward:0.85\n", - "step: 3699, Red action: do-nothing, Blue reward:0.85\n", - "step: 3700, Red action: do-nothing, Blue reward:0.85\n", - "step: 3701, Red action: do-nothing, Blue reward:0.85\n", - "step: 3702, Red action: do-nothing, Blue reward:0.85\n", - "step: 3703, Red action: do-nothing, Blue reward:0.85\n", - "step: 3704, Red action: do-nothing, Blue reward:0.85\n", - "step: 3705, Red action: do-nothing, Blue reward:0.85\n", - "step: 3706, Red action: do-nothing, Blue reward:0.85\n", - "step: 3707, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3708, Red action: do-nothing, Blue reward:0.85\n", - "step: 3709, Red action: do-nothing, Blue reward:0.85\n", - "step: 3710, Red action: do-nothing, Blue reward:0.85\n", - "step: 3711, Red action: do-nothing, Blue reward:0.85\n", - "step: 3712, Red action: do-nothing, Blue reward:0.85\n", - "step: 3713, Red action: do-nothing, Blue reward:0.85\n", - "step: 3714, Red action: do-nothing, Blue reward:0.85\n", - "step: 3715, Red action: do-nothing, Blue reward:0.85\n", - "step: 3716, Red action: do-nothing, Blue reward:0.85\n", - "step: 3717, Red action: do-nothing, Blue reward:0.85\n", - "step: 3718, Red action: do-nothing, Blue reward:0.85\n", - "step: 3719, Red action: do-nothing, Blue reward:0.85\n", - "step: 3720, Red action: do-nothing, Blue reward:0.85\n", - "step: 3721, Red action: do-nothing, Blue reward:0.85\n", - "step: 3722, Red action: do-nothing, Blue reward:0.85\n", - "step: 3723, Red action: do-nothing, Blue reward:0.85\n", - "step: 3724, Red action: do-nothing, Blue reward:0.85\n", - "step: 3725, Red action: do-nothing, Blue reward:0.85\n", - "step: 3726, Red action: do-nothing, Blue reward:0.85\n", - "step: 3727, Red action: do-nothing, Blue reward:0.85\n", - "step: 3728, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3729, Red action: do-nothing, Blue reward:0.85\n", - "step: 3730, Red action: do-nothing, Blue reward:0.85\n", - "step: 3731, Red action: do-nothing, Blue reward:0.85\n", - "step: 3732, Red action: do-nothing, Blue reward:0.85\n", - "step: 3733, Red action: do-nothing, Blue reward:0.85\n", - "step: 3734, Red action: do-nothing, Blue reward:0.85\n", - "step: 3735, Red action: do-nothing, Blue reward:0.85\n", - "step: 3736, Red action: do-nothing, Blue reward:0.85\n", - "step: 3737, Red action: do-nothing, Blue reward:0.85\n", - "step: 3738, Red action: do-nothing, Blue reward:0.85\n", - "step: 3739, Red action: do-nothing, Blue reward:0.85\n", - "step: 3740, Red action: do-nothing, Blue reward:0.85\n", - "step: 3741, Red action: do-nothing, Blue reward:0.85\n", - "step: 3742, Red action: do-nothing, Blue reward:0.85\n", - "step: 3743, Red action: do-nothing, Blue reward:0.85\n", - "step: 3744, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3745, Red action: do-nothing, Blue reward:0.85\n", - "step: 3746, Red action: do-nothing, Blue reward:0.85\n", - "step: 3747, Red action: do-nothing, Blue reward:0.85\n", - "step: 3748, Red action: do-nothing, Blue reward:0.85\n", - "step: 3749, Red action: do-nothing, Blue reward:0.85\n", - "step: 3750, Red action: do-nothing, Blue reward:0.85\n", - "step: 3751, Red action: do-nothing, Blue reward:0.85\n", - "step: 3752, Red action: do-nothing, Blue reward:0.85\n", - "step: 3753, Red action: do-nothing, Blue reward:0.85\n", - "step: 3754, Red action: do-nothing, Blue reward:0.85\n", - "step: 3755, Red action: do-nothing, Blue reward:0.85\n", - "step: 3756, Red action: do-nothing, Blue reward:0.85\n", - "step: 3757, Red action: do-nothing, Blue reward:0.85\n", - "step: 3758, Red action: do-nothing, Blue reward:0.85\n", - "step: 3759, Red action: do-nothing, Blue reward:0.85\n", - "step: 3760, Red action: do-nothing, Blue reward:0.85\n", - "step: 3761, Red action: do-nothing, Blue reward:0.85\n", - "step: 3762, Red action: do-nothing, Blue reward:0.85\n", - "step: 3763, Red action: do-nothing, Blue reward:0.85\n", - "step: 3764, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3765, Red action: do-nothing, Blue reward:0.85\n", - "step: 3766, Red action: do-nothing, Blue reward:0.85\n", - "step: 3767, Red action: do-nothing, Blue reward:0.85\n", - "step: 3768, Red action: do-nothing, Blue reward:0.85\n", - "step: 3769, Red action: do-nothing, Blue reward:0.85\n", - "step: 3770, Red action: do-nothing, Blue reward:0.85\n", - "step: 3771, Red action: do-nothing, Blue reward:0.85\n", - "step: 3772, Red action: do-nothing, Blue reward:0.85\n", - "step: 3773, Red action: do-nothing, Blue reward:0.85\n", - "step: 3774, Red action: do-nothing, Blue reward:0.85\n", - "step: 3775, Red action: do-nothing, Blue reward:0.85\n", - "step: 3776, Red action: do-nothing, Blue reward:0.85\n", - "step: 3777, Red action: do-nothing, Blue reward:0.85\n", - "step: 3778, Red action: do-nothing, Blue reward:0.85\n", - "step: 3779, Red action: do-nothing, Blue reward:0.85\n", - "step: 3780, Red action: do-nothing, Blue reward:0.85\n", - "step: 3781, Red action: do-nothing, Blue reward:0.85\n", - "step: 3782, Red action: do-nothing, Blue reward:0.85\n", - "step: 3783, Red action: do-nothing, Blue reward:0.85\n", - "step: 3784, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3785, Red action: do-nothing, Blue reward:0.85\n", - "step: 3786, Red action: do-nothing, Blue reward:0.85\n", - "step: 3787, Red action: do-nothing, Blue reward:0.85\n", - "step: 3788, Red action: do-nothing, Blue reward:0.85\n", - "step: 3789, Red action: do-nothing, Blue reward:0.85\n", - "step: 3790, Red action: do-nothing, Blue reward:0.85\n", - "step: 3791, Red action: do-nothing, Blue reward:0.85\n", - "step: 3792, Red action: do-nothing, Blue reward:0.85\n", - "step: 3793, Red action: do-nothing, Blue reward:0.85\n", - "step: 3794, Red action: do-nothing, Blue reward:0.85\n", - "step: 3795, Red action: do-nothing, Blue reward:0.85\n", - "step: 3796, Red action: do-nothing, Blue reward:0.85\n", - "step: 3797, Red action: do-nothing, Blue reward:0.85\n", - "step: 3798, Red action: do-nothing, Blue reward:0.85\n", - "step: 3799, Red action: do-nothing, Blue reward:0.85\n", - "step: 3800, Red action: do-nothing, Blue reward:0.85\n", - "step: 3801, Red action: do-nothing, Blue reward:0.85\n", - "step: 3802, Red action: do-nothing, Blue reward:0.85\n" - ] - }, - { - "ename": "KeyboardInterrupt", - "evalue": "", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[11], line 11\u001b[0m\n\u001b[1;32m 8\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mstep: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00menv\u001b[38;5;241m.\u001b[39mgame\u001b[38;5;241m.\u001b[39mstep_counter\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m, Red action: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00minfo[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124magent_actions\u001b[39m\u001b[38;5;124m'\u001b[39m][\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mdata_manipulation_attacker\u001b[39m\u001b[38;5;124m'\u001b[39m]\u001b[38;5;241m.\u001b[39maction\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m, Blue reward:\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mreward\u001b[38;5;132;01m:\u001b[39;00m\u001b[38;5;124m.2f\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m )\n\u001b[1;32m 10\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28mabs\u001b[39m(reward \u001b[38;5;241m-\u001b[39m \u001b[38;5;241m0.8\u001b[39m) \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m1e-5\u001b[39m:\n\u001b[0;32m---> 11\u001b[0m obs, reward, terminated, truncated, info \u001b[38;5;241m=\u001b[39m \u001b[43menv\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstep\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m)\u001b[49m \u001b[38;5;66;03m# do nothing\u001b[39;00m\n\u001b[1;32m 12\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mstep: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00menv\u001b[38;5;241m.\u001b[39mgame\u001b[38;5;241m.\u001b[39mstep_counter\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m, Red action: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00minfo[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124magent_actions\u001b[39m\u001b[38;5;124m'\u001b[39m][\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mdata_manipulation_attacker\u001b[39m\u001b[38;5;124m'\u001b[39m]\u001b[38;5;241m.\u001b[39maction\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m, Blue reward:\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mreward\u001b[38;5;132;01m:\u001b[39;00m\u001b[38;5;124m.2f\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m )\n\u001b[1;32m 13\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m env\u001b[38;5;241m.\u001b[39mgame\u001b[38;5;241m.\u001b[39mstep_counter \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m10000\u001b[39m:\n", - "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/session/environment.py:111\u001b[0m, in \u001b[0;36mPrimaiteGymEnv.step\u001b[0;34m(self, action)\u001b[0m\n\u001b[1;32m 109\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgame\u001b[38;5;241m.\u001b[39mapply_agent_actions()\n\u001b[1;32m 110\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgame\u001b[38;5;241m.\u001b[39madvance_timestep()\n\u001b[0;32m--> 111\u001b[0m state \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgame\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_sim_state\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 112\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgame\u001b[38;5;241m.\u001b[39mupdate_agents(state)\n\u001b[1;32m 114\u001b[0m next_obs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_get_obs() \u001b[38;5;66;03m# this doesn't update observation, just gets the current observation\u001b[39;00m\n", - "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/game/game.py:151\u001b[0m, in \u001b[0;36mPrimaiteGame.get_sim_state\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 149\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mget_sim_state\u001b[39m(\u001b[38;5;28mself\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Dict:\n\u001b[1;32m 150\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"Get the current state of the simulation.\"\"\"\u001b[39;00m\n\u001b[0;32m--> 151\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msimulation\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdescribe_state\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/simulator/sim_container.py:57\u001b[0m, in \u001b[0;36mSimulation.describe_state\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 46\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 47\u001b[0m \u001b[38;5;124;03mProduce a dictionary describing the current state of this object.\u001b[39;00m\n\u001b[1;32m 48\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 52\u001b[0m \u001b[38;5;124;03m:rtype: Dict\u001b[39;00m\n\u001b[1;32m 53\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 54\u001b[0m state \u001b[38;5;241m=\u001b[39m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39mdescribe_state()\n\u001b[1;32m 55\u001b[0m state\u001b[38;5;241m.\u001b[39mupdate(\n\u001b[1;32m 56\u001b[0m {\n\u001b[0;32m---> 57\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnetwork\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mnetwork\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdescribe_state\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m,\n\u001b[1;32m 58\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdomain\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdomain\u001b[38;5;241m.\u001b[39mdescribe_state(),\n\u001b[1;32m 59\u001b[0m }\n\u001b[1;32m 60\u001b[0m )\n\u001b[1;32m 61\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m state\n", - "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/simulator/network/container.py:260\u001b[0m, in \u001b[0;36mNetwork.describe_state\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 252\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 253\u001b[0m \u001b[38;5;124;03mProduce a dictionary describing the current state of the Network.\u001b[39;00m\n\u001b[1;32m 254\u001b[0m \n\u001b[1;32m 255\u001b[0m \u001b[38;5;124;03m:return: A dictionary capturing the current state of the Network and its child objects.\u001b[39;00m\n\u001b[1;32m 256\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 257\u001b[0m state \u001b[38;5;241m=\u001b[39m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39mdescribe_state()\n\u001b[1;32m 258\u001b[0m state\u001b[38;5;241m.\u001b[39mupdate(\n\u001b[1;32m 259\u001b[0m {\n\u001b[0;32m--> 260\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnodes\u001b[39m\u001b[38;5;124m\"\u001b[39m: {node\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mhostname: node\u001b[38;5;241m.\u001b[39mdescribe_state() \u001b[38;5;28;01mfor\u001b[39;00m node \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnodes\u001b[38;5;241m.\u001b[39mvalues()},\n\u001b[1;32m 261\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mlinks\u001b[39m\u001b[38;5;124m\"\u001b[39m: {},\n\u001b[1;32m 262\u001b[0m }\n\u001b[1;32m 263\u001b[0m )\n\u001b[1;32m 264\u001b[0m \u001b[38;5;66;03m# Update the links one-by-one. The key is a 4-tuple of `hostname_a, port_a, hostname_b, port_b`\u001b[39;00m\n\u001b[1;32m 265\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m _, link \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlinks\u001b[38;5;241m.\u001b[39mitems():\n", - "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/simulator/network/container.py:260\u001b[0m, in \u001b[0;36m\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 252\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 253\u001b[0m \u001b[38;5;124;03mProduce a dictionary describing the current state of the Network.\u001b[39;00m\n\u001b[1;32m 254\u001b[0m \n\u001b[1;32m 255\u001b[0m \u001b[38;5;124;03m:return: A dictionary capturing the current state of the Network and its child objects.\u001b[39;00m\n\u001b[1;32m 256\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 257\u001b[0m state \u001b[38;5;241m=\u001b[39m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39mdescribe_state()\n\u001b[1;32m 258\u001b[0m state\u001b[38;5;241m.\u001b[39mupdate(\n\u001b[1;32m 259\u001b[0m {\n\u001b[0;32m--> 260\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnodes\u001b[39m\u001b[38;5;124m\"\u001b[39m: {node\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mhostname: \u001b[43mnode\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdescribe_state\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mfor\u001b[39;00m node \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnodes\u001b[38;5;241m.\u001b[39mvalues()},\n\u001b[1;32m 261\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mlinks\u001b[39m\u001b[38;5;124m\"\u001b[39m: {},\n\u001b[1;32m 262\u001b[0m }\n\u001b[1;32m 263\u001b[0m )\n\u001b[1;32m 264\u001b[0m \u001b[38;5;66;03m# Update the links one-by-one. The key is a 4-tuple of `hostname_a, port_a, hostname_b, port_b`\u001b[39;00m\n\u001b[1;32m 265\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m _, link \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlinks\u001b[38;5;241m.\u001b[39mitems():\n", - "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/simulator/network/hardware/base.py:1900\u001b[0m, in \u001b[0;36mNode.describe_state\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1881\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 1882\u001b[0m \u001b[38;5;124;03mProduce a dictionary describing the current state of this object.\u001b[39;00m\n\u001b[1;32m 1883\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1887\u001b[0m \u001b[38;5;124;03m:rtype: Dict\u001b[39;00m\n\u001b[1;32m 1888\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 1889\u001b[0m state \u001b[38;5;241m=\u001b[39m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39mdescribe_state()\n\u001b[1;32m 1890\u001b[0m state\u001b[38;5;241m.\u001b[39mupdate(\n\u001b[1;32m 1891\u001b[0m {\n\u001b[1;32m 1892\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhostname\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mhostname,\n\u001b[1;32m 1893\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moperating_state\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39moperating_state\u001b[38;5;241m.\u001b[39mvalue,\n\u001b[1;32m 1894\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mNICs\u001b[39m\u001b[38;5;124m\"\u001b[39m: {\n\u001b[1;32m 1895\u001b[0m eth_num: network_interface\u001b[38;5;241m.\u001b[39mdescribe_state()\n\u001b[1;32m 1896\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m eth_num, network_interface \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnetwork_interface\u001b[38;5;241m.\u001b[39mitems()\n\u001b[1;32m 1897\u001b[0m },\n\u001b[1;32m 1898\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfile_system\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfile_system\u001b[38;5;241m.\u001b[39mdescribe_state(),\n\u001b[1;32m 1899\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mapplications\u001b[39m\u001b[38;5;124m\"\u001b[39m: {app\u001b[38;5;241m.\u001b[39mname: app\u001b[38;5;241m.\u001b[39mdescribe_state() \u001b[38;5;28;01mfor\u001b[39;00m app \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mapplications\u001b[38;5;241m.\u001b[39mvalues()},\n\u001b[0;32m-> 1900\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mservices\u001b[39m\u001b[38;5;124m\"\u001b[39m: {svc\u001b[38;5;241m.\u001b[39mname: svc\u001b[38;5;241m.\u001b[39mdescribe_state() \u001b[38;5;28;01mfor\u001b[39;00m svc \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mservices\u001b[38;5;241m.\u001b[39mvalues()},\n\u001b[1;32m 1901\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mprocess\u001b[39m\u001b[38;5;124m\"\u001b[39m: {proc\u001b[38;5;241m.\u001b[39mname: proc\u001b[38;5;241m.\u001b[39mdescribe_state() \u001b[38;5;28;01mfor\u001b[39;00m proc \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprocesses\u001b[38;5;241m.\u001b[39mvalues()},\n\u001b[1;32m 1902\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mrevealed_to_red\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mrevealed_to_red,\n\u001b[1;32m 1903\u001b[0m }\n\u001b[1;32m 1904\u001b[0m )\n\u001b[1;32m 1905\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m state\n", - "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/simulator/network/hardware/base.py:1900\u001b[0m, in \u001b[0;36m\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 1881\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 1882\u001b[0m \u001b[38;5;124;03mProduce a dictionary describing the current state of this object.\u001b[39;00m\n\u001b[1;32m 1883\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1887\u001b[0m \u001b[38;5;124;03m:rtype: Dict\u001b[39;00m\n\u001b[1;32m 1888\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 1889\u001b[0m state \u001b[38;5;241m=\u001b[39m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39mdescribe_state()\n\u001b[1;32m 1890\u001b[0m state\u001b[38;5;241m.\u001b[39mupdate(\n\u001b[1;32m 1891\u001b[0m {\n\u001b[1;32m 1892\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhostname\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mhostname,\n\u001b[1;32m 1893\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moperating_state\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39moperating_state\u001b[38;5;241m.\u001b[39mvalue,\n\u001b[1;32m 1894\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mNICs\u001b[39m\u001b[38;5;124m\"\u001b[39m: {\n\u001b[1;32m 1895\u001b[0m eth_num: network_interface\u001b[38;5;241m.\u001b[39mdescribe_state()\n\u001b[1;32m 1896\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m eth_num, network_interface \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnetwork_interface\u001b[38;5;241m.\u001b[39mitems()\n\u001b[1;32m 1897\u001b[0m },\n\u001b[1;32m 1898\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfile_system\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfile_system\u001b[38;5;241m.\u001b[39mdescribe_state(),\n\u001b[1;32m 1899\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mapplications\u001b[39m\u001b[38;5;124m\"\u001b[39m: {app\u001b[38;5;241m.\u001b[39mname: app\u001b[38;5;241m.\u001b[39mdescribe_state() \u001b[38;5;28;01mfor\u001b[39;00m app \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mapplications\u001b[38;5;241m.\u001b[39mvalues()},\n\u001b[0;32m-> 1900\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mservices\u001b[39m\u001b[38;5;124m\"\u001b[39m: {svc\u001b[38;5;241m.\u001b[39mname: \u001b[43msvc\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdescribe_state\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mfor\u001b[39;00m svc \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mservices\u001b[38;5;241m.\u001b[39mvalues()},\n\u001b[1;32m 1901\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mprocess\u001b[39m\u001b[38;5;124m\"\u001b[39m: {proc\u001b[38;5;241m.\u001b[39mname: proc\u001b[38;5;241m.\u001b[39mdescribe_state() \u001b[38;5;28;01mfor\u001b[39;00m proc \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprocesses\u001b[38;5;241m.\u001b[39mvalues()},\n\u001b[1;32m 1902\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mrevealed_to_red\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mrevealed_to_red,\n\u001b[1;32m 1903\u001b[0m }\n\u001b[1;32m 1904\u001b[0m )\n\u001b[1;32m 1905\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m state\n", - "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/simulator/system/services/ntp/ntp_client.py:61\u001b[0m, in \u001b[0;36mNTPClient.describe_state\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 50\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdescribe_state\u001b[39m(\u001b[38;5;28mself\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Dict:\n\u001b[1;32m 51\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 52\u001b[0m \u001b[38;5;124;03m Describes the current state of the software.\u001b[39;00m\n\u001b[1;32m 53\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 59\u001b[0m \u001b[38;5;124;03m :rtype: Dict\u001b[39;00m\n\u001b[1;32m 60\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m---> 61\u001b[0m state \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241m.\u001b[39mdescribe_state()\n\u001b[1;32m 62\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m state\n", - "\u001b[0;31mKeyboardInterrupt\u001b[0m: " - ] - } - ], + "outputs": [], "source": [ "env.step(13) # Patch the database\n", "print(f\"step: {env.game.step_counter}, Red action: {info['agent_actions']['data_manipulation_attacker'].action}, Blue reward:{reward:.2f}\" )\n", From 8c7f8cd0ecb257df594825201dce6b598f35f5cf Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Tue, 18 Feb 2025 11:36:22 +0000 Subject: [PATCH 83/95] #3075: Fix final two demo notebooks --- .../create-simulation_demo.ipynb | 49 ++++++++++++++++--- .../network_simulator_demo.ipynb | 16 +----- src/primaite/simulator/network/networks.py | 2 +- 3 files changed, 45 insertions(+), 22 deletions(-) diff --git a/src/primaite/simulator/_package_data/create-simulation_demo.ipynb b/src/primaite/simulator/_package_data/create-simulation_demo.ipynb index 690e7856..46f03be6 100644 --- a/src/primaite/simulator/_package_data/create-simulation_demo.ipynb +++ b/src/primaite/simulator/_package_data/create-simulation_demo.ipynb @@ -6,7 +6,7 @@ "source": [ "# Build a simulation using the Python API\n", "\n", - "© Crown-owned copyright 2024, Defence Science and Technology Laboratory UK\n", + "© Crown-owned copyright 2025, Defence Science and Technology Laboratory UK\n", "\n", "Currently, this notebook manipulates the simulation by directly placing objects inside of the attributes of the network and domain. It should be refactored when proper methods exist for adding these objects." ] @@ -70,9 +70,23 @@ "metadata": {}, "outputs": [], "source": [ - "my_pc = Computer(hostname=\"Computer\", ip_address=\"192.168.1.10\", subnet_mask=\"255.255.255.0\")\n", + "my_pc = Computer.from_config(\n", + " config={\n", + " \"type\": \"computer\",\n", + " \"hostname\":\"pc_1\",\n", + " \"ip_address\":\"192.168.1.10\",\n", + " \"subnet_mask\":\"255.255.255.0\",\n", + " }\n", + " )\n", "net.add_node(my_pc)\n", - "my_server = Server(hostname=\"Server\", ip_address=\"192.168.1.11\", subnet_mask=\"255.255.255.0\")\n", + "my_server = Server.from_config(\n", + " config={\n", + " \"type\": \"server\",\n", + " \"hostname\":\"Server\",\n", + " \"ip_address\":\"192.168.1.11\",\n", + " \"subnet_mask\":\"255.255.255.0\"\n", + " }\n", + ")\n", "net.add_node(my_server)\n" ] }, @@ -99,7 +113,13 @@ "metadata": {}, "outputs": [], "source": [ - "my_switch = Switch(hostname=\"switch1\", num_ports=12)\n", + "my_switch = Switch.from_config(\n", + " config = {\n", + " \"type\":\"switch\",\n", + " \"hostname\":\"switch1\",\n", + " \"num_ports\":12\n", + " }\n", + ")\n", "net.add_node(my_switch)\n", "\n", "pc_nic = NIC(ip_address=\"130.1.1.1\", gateway=\"130.1.1.255\", subnet_mask=\"255.255.255.0\")\n", @@ -163,16 +183,30 @@ "metadata": {}, "outputs": [], "source": [ + "from pydantic import Field\n", + "\n", "from pathlib import Path\n", "from primaite.simulator.system.applications.application import Application, ApplicationOperatingState\n", "from primaite.simulator.system.software import SoftwareHealthState, SoftwareCriticality\n", "from primaite.simulator.file_system.file_system import FileSystem\n", "from primaite.utils.validation.ip_protocol import PROTOCOL_LOOKUP\n", "from primaite.utils.validation.port import PORT_LOOKUP\n", + "from primaite.simulator.system.core.sys_log import SysLog\n", "\n", "\n", "# no applications exist yet so we will create our own.\n", "class MSPaint(Application, discriminator=\"MSPaint\"):\n", + " class ConfigSchema(Application.ConfigSchema):\n", + " type: str = \"MSPaint\"\n", + "\n", + " config: ConfigSchema = Field(default_factory=lambda: MSPaint.ConfigSchema())\n", + "\n", + " def __init__(self, **kwargs):\n", + " kwargs[\"name\"] = \"MSPaint\"\n", + " kwargs[\"port\"] = PORT_LOOKUP[\"HTTP\"]\n", + " kwargs[\"protocol\"] = PROTOCOL_LOOKUP[\"NONE\"]\n", + " super().__init__(**kwargs)\n", + "\n", " def describe_state(self):\n", " return super().describe_state()" ] @@ -183,7 +217,8 @@ "metadata": {}, "outputs": [], "source": [ - "mspaint = MSPaint(name = \"mspaint\", health_state_actual=SoftwareHealthState.GOOD, health_state_visible=SoftwareHealthState.GOOD, criticality=SoftwareCriticality.MEDIUM, port=PORT_LOOKUP[\"HTTP\"], protocol = PROTOCOL_LOOKUP[\"NONE\"],operating_state=ApplicationOperatingState.RUNNING,execution_control_status='manual', file_system=FileSystem(sys_log=SysLog(hostname=\"Test\"), sim_root=Path(__name__).parent),)" + "my_pc.software_manager.install(MSPaint)\n", + "mspaint = my_pc.software_manager.software.get(\"MSPaint\")" ] }, { @@ -250,7 +285,7 @@ ], "metadata": { "kernelspec": { - "display_name": ".venv", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -264,7 +299,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.11" + "version": "3.10.12" } }, "nbformat": 4, diff --git a/src/primaite/simulator/_package_data/network_simulator_demo.ipynb b/src/primaite/simulator/_package_data/network_simulator_demo.ipynb index 58c07fe5..be930ac0 100644 --- a/src/primaite/simulator/_package_data/network_simulator_demo.ipynb +++ b/src/primaite/simulator/_package_data/network_simulator_demo.ipynb @@ -7,7 +7,7 @@ "source": [ "# PrimAITE Router Simulation Demo\n", "\n", - "© Crown-owned copyright 2024, Defence Science and Technology Laboratory UK\n", + "© Crown-owned copyright 2025, Defence Science and Technology Laboratory UK\n", "\n", "This demo uses a modified version of the ARCD Use Case 2 Network (seen below) to demonstrate the capabilities of the Network simulator in PrimAITE." ] @@ -650,21 +650,9 @@ ], "metadata": { "kernelspec": { - "display_name": ".venv", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.11" } }, "nbformat": 4, diff --git a/src/primaite/simulator/network/networks.py b/src/primaite/simulator/network/networks.py index 5e677679..5d558e80 100644 --- a/src/primaite/simulator/network/networks.py +++ b/src/primaite/simulator/network/networks.py @@ -295,7 +295,7 @@ def arcd_uc2_network() -> Network: # Security Suite security_suite_cfg = { "type": "server", - "hostname": "backup_server", + "hostname": "security_suite", "ip_address": "192.168.1.110", "subnet_mask": "255.255.255.0", "default_gateway": "192.168.1.1", From 72f80e55da0a17766975bf1d73aaff75786ec303 Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Tue, 18 Feb 2025 11:51:35 +0000 Subject: [PATCH 84/95] #3075: Move demo notebook to correct folder. --- .../_package_data => notebooks}/create-simulation_demo.ipynb | 0 .../_package_data => notebooks}/network_simulator_demo.ipynb | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename src/primaite/{simulator/_package_data => notebooks}/create-simulation_demo.ipynb (100%) rename src/primaite/{simulator/_package_data => notebooks}/network_simulator_demo.ipynb (100%) diff --git a/src/primaite/simulator/_package_data/create-simulation_demo.ipynb b/src/primaite/notebooks/create-simulation_demo.ipynb similarity index 100% rename from src/primaite/simulator/_package_data/create-simulation_demo.ipynb rename to src/primaite/notebooks/create-simulation_demo.ipynb diff --git a/src/primaite/simulator/_package_data/network_simulator_demo.ipynb b/src/primaite/notebooks/network_simulator_demo.ipynb similarity index 100% rename from src/primaite/simulator/_package_data/network_simulator_demo.ipynb rename to src/primaite/notebooks/network_simulator_demo.ipynb From 90d7af3ff9fb0bc5fd60515b3d30752d7da27493 Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Tue, 18 Feb 2025 12:42:33 +0000 Subject: [PATCH 85/95] #3075: Remove references to src/primaite/simulator/_package_data after notebook move. --- .azure/azure-ci-build-pipeline.yaml | 4 ---- MANIFEST.in | 1 - 2 files changed, 5 deletions(-) diff --git a/.azure/azure-ci-build-pipeline.yaml b/.azure/azure-ci-build-pipeline.yaml index 624c9ca4..20c5b239 100644 --- a/.azure/azure-ci-build-pipeline.yaml +++ b/.azure/azure-ci-build-pipeline.yaml @@ -113,8 +113,6 @@ stages: - script: | pytest --nbmake -n=auto src/primaite/notebooks --junit-xml=./notebook-tests/notebooks.xml notebooks_exit_code=$? - pytest --nbmake -n=auto src/primaite/simulator/_package_data --junit-xml=./notebook-tests/package-notebooks.xml - package_notebooks_exit_code=$? # Fail step if either of these do not have exit code 0 if [ $notebooks_exit_code -ne 0 ] || [ $package_notebooks_exit_code -ne 0 ]; then exit 1 @@ -126,8 +124,6 @@ stages: - script: | pytest --nbmake -n=auto src/primaite/notebooks --junit-xml=./notebook-tests/notebooks.xml set notebooks_exit_code=%ERRORLEVEL% - pytest --nbmake -n=auto src/primaite/simulator/_package_data --junit-xml=./notebook-tests/package-notebooks.xml - set package_notebooks_exit_code=%ERRORLEVEL% rem Fail step if either of these do not have exit code 0 if %notebooks_exit_code% NEQ 0 exit /b 1 if %package_notebooks_exit_code% NEQ 0 exit /b 1 diff --git a/MANIFEST.in b/MANIFEST.in index 2ac7b306..51ae4ddf 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,2 @@ include src/primaite/setup/_package_data/primaite_config.yaml include src/primaite/config/_package_data/*.yaml -include src/primaite/simulator/_package_data/*.ipynb From f5cfaeedab633498274bbc05e34679dea19fec7a Mon Sep 17 00:00:00 2001 From: Archer Bowen Date: Fri, 21 Feb 2025 14:57:48 +0000 Subject: [PATCH 86/95] #3075 Cleaned up yaml snippets that were rendering as a strings. It seems that adding '...' to yaml snippets seems to end up causing the snippets to render as strings rather than actual yaml. --- ...ommand-and-Control-E2E-Demonstration.ipynb | 66 ++++++++----------- 1 file changed, 28 insertions(+), 38 deletions(-) diff --git a/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb b/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb index 6dc8e077..f187c8d5 100644 --- a/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb +++ b/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb @@ -120,6 +120,9 @@ " options:\n", " node_name: web_server\n", " c2_server_ip_address: 192.168.10.21\n", + " keep_alive_frequency: 10\n", + " masquerade_protocol: tcp\n", + " masquerade_port: dns\n", " 9:\n", " action: configure-c2-beacon\n", " options:\n", @@ -235,23 +238,18 @@ "The yaml snippet below shows all the relevant agent options for this action:\n", "\n", "```yaml\n", + "\n", " action_space:\n", - " options:\n", - " nodes: # Node List\n", - " - node_name: web_server\n", - " ...\n", - " ...\n", " action_map:\n", - " ...\n", " 2:\n", " action: configure-c2-beacon\n", " options:\n", - " node_id: 0 # Node Index\n", - " config: # Further information about these config options can be found at the bottom of this notebook.\n", - " c2_server_ip_address: 192.168.10.21\n", - " keep_alive_frequency:\n", - " masquerade_protocol:\n", - " masquerade_port:\n", + " node_name: web_server\n", + " c2_server_ip_address: 192.168.10.21 # Further information about these config options can be found at the bottom of this notebook.\n", + " keep_alive_frequency:\n", + " masquerade_protocol:\n", + " masquerade_port:\n", + "\n", "```" ] }, @@ -279,20 +277,12 @@ "\n", "```yaml\n", " action_space:\n", - " options:\n", - " nodes: # Node List\n", - " - node_name: web_server\n", - " applications: \n", - " - application_name: c2-beacon\n", - " ...\n", - " ...\n", " action_map:\n", - " ...\n", " 3:\n", " action: node-application-execute\n", " options:\n", - " node_id: 0\n", - " application_id: 0\n", + " node_name: web_server\n", + " application_name: c2-beacon\n", "```" ] }, @@ -346,7 +336,6 @@ "``` yaml\n", " action_space:\n", " action_map:\n", - " ...\n", " 4:\n", " action: c2-server-terminal-command\n", " options:\n", @@ -396,14 +385,12 @@ "``` yaml\n", " action_space:\n", " action_map:\n", - " ...\n", " 5:\n", " action: c2-server-ransomware-configure\n", " options:\n", - " node_id: 1\n", - " config:\n", - " server_ip_address: 192.168.1.14\n", - " payload: ENCRYPT\n", + " node_name: client_1\n", + " server_ip_address: 192.168.1.14\n", + " payload: ENCRYPT\n", "```\n" ] }, @@ -442,7 +429,6 @@ "``` yaml\n", " action_space:\n", " action_map:\n", - " ...\n", " 6:\n", " action: c2-server-data-exfiltrate\n", " options:\n", @@ -451,7 +437,7 @@ " target_folder_name: \"database\"\n", " exfiltration_folder_name: \"spoils\"\n", " target_ip_address: \"192.168.1.14\"\n", - " username: \"admin\",\n", + " username: \"admin\"\n", " password: \"admin\"\n", "\n", "```" @@ -500,7 +486,6 @@ "\n", "``` yaml\n", " action_space:\n", - " ...\n", " action_map:\n", " 7:\n", " action: c2-server-ransomware-launch\n", @@ -1313,13 +1298,18 @@ "source": [ "As demonstrated earlier, red agents can use the ``configure-c2-beacon`` action to configure these settings mid episode through the configuration options:\n", "\n", - "``` YAML\n", - "...\n", - " action: configure-c2-beacon\n", - " options:\n", - " node_name: web_server\n", - " config:\n", + "```YAML\n", + "\n", + " action_space:\n", + " action_map:\n", + " 8:\n", + " action: configure-c2-beacon\n", + " options:\n", + " node_name: web_server\n", " c2_server_ip_address: 192.168.10.21\n", + " keep_alive_frequency: 10\n", + " masquerade_protocol: tcp\n", + " masquerade_port: dns\n", "```" ] }, @@ -1684,7 +1674,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": ".venv", "language": "python", "name": "python3" }, From 892cdb82b91d621b92328dc0b2014348a08aac94 Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Tue, 25 Feb 2025 14:25:27 +0000 Subject: [PATCH 87/95] #3075: Code review changes. --- .azure/azure-ci-build-pipeline.yaml | 7 +++---- .../Data-Manipulation-Customising-Red-Agent.ipynb | 4 ++-- tests/assets/configs/basic_switched_network.yaml | 4 ++-- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.azure/azure-ci-build-pipeline.yaml b/.azure/azure-ci-build-pipeline.yaml index 20c5b239..b6f24777 100644 --- a/.azure/azure-ci-build-pipeline.yaml +++ b/.azure/azure-ci-build-pipeline.yaml @@ -113,8 +113,8 @@ stages: - script: | pytest --nbmake -n=auto src/primaite/notebooks --junit-xml=./notebook-tests/notebooks.xml notebooks_exit_code=$? - # Fail step if either of these do not have exit code 0 - if [ $notebooks_exit_code -ne 0 ] || [ $package_notebooks_exit_code -ne 0 ]; then + # Fail step if exit code not equal to 0 + if [ $notebooks_exit_code -ne 0 ]; then exit 1 fi displayName: 'Run notebooks on Linux and macOS' @@ -124,9 +124,8 @@ stages: - script: | pytest --nbmake -n=auto src/primaite/notebooks --junit-xml=./notebook-tests/notebooks.xml set notebooks_exit_code=%ERRORLEVEL% - rem Fail step if either of these do not have exit code 0 + rem Fail step if exit code not equal to 0 if %notebooks_exit_code% NEQ 0 exit /b 1 - if %package_notebooks_exit_code% NEQ 0 exit /b 1 displayName: 'Run notebooks on Windows' condition: eq(variables['Agent.OS'], 'Windows_NT') diff --git a/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb b/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb index 1eb814e3..5982407b 100644 --- a/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb +++ b/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb @@ -289,10 +289,10 @@ " action_space:\n", " action_map:\n", " 0:\n", - " action: do_nothing\n", + " action: do-nothing\n", " options: {}\n", " 1:\n", - " action: node_application_execute\n", + " action: node-application-execute\n", " options:\n", " node_name: client_1\n", " application_name: DataManipulationBot\n", diff --git a/tests/assets/configs/basic_switched_network.yaml b/tests/assets/configs/basic_switched_network.yaml index 2af48ea2..7339c34c 100644 --- a/tests/assets/configs/basic_switched_network.yaml +++ b/tests/assets/configs/basic_switched_network.yaml @@ -37,10 +37,10 @@ agents: action_space: action_map: 0: - action: do_nothing + action: do-nothing options: {} 1: - action: node_application_execute + action: node-application-execute options: node_name: client_2 application_name: web-browser From 1605b0989962a9c00c44cd63dc470a17b432e35e Mon Sep 17 00:00:00 2001 From: Marek Wolan Date: Wed, 26 Feb 2025 10:49:01 +0000 Subject: [PATCH 88/95] Apply suggestions from code review --- tests/assets/configs/basic_switched_network.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/assets/configs/basic_switched_network.yaml b/tests/assets/configs/basic_switched_network.yaml index 7339c34c..b57ed3e0 100644 --- a/tests/assets/configs/basic_switched_network.yaml +++ b/tests/assets/configs/basic_switched_network.yaml @@ -112,7 +112,7 @@ agents: action_space: action_map: 0: - action: do_nothing + action: do-nothing options: {} reward_function: From b4b0f99c23baaefc4cf9750cad1f28c8edba23c8 Mon Sep 17 00:00:00 2001 From: Marek Wolan Date: Wed, 26 Feb 2025 17:57:23 +0000 Subject: [PATCH 89/95] Fix mismerge of agent show_history method --- src/primaite/game/agent/interface.py | 57 +++++----------------------- 1 file changed, 9 insertions(+), 48 deletions(-) diff --git a/src/primaite/game/agent/interface.py b/src/primaite/game/agent/interface.py index 1fef14ef..a6e9739f 100644 --- a/src/primaite/game/agent/interface.py +++ b/src/primaite/game/agent/interface.py @@ -107,66 +107,27 @@ class AbstractAgent(BaseModel, ABC): self.reward_function = RewardFunction(config=self.config.reward_function) return super().model_post_init(__context) - def add_agent_action(self, item: AgentHistoryItem, table: PrettyTable) -> PrettyTable: - """Update the given table with information from given AgentHistoryItem.""" - node, application = "unknown", "unknown" - if (node_id := item.parameters.get("node_id")) is not None: - node = self.action_manager.node_names[node_id] - if (application_id := item.parameters.get("application_id")) is not None: - application = self.action_manager.application_names[node_id][application_id] - if (application_name := item.parameters.get("application_name")) is not None: - application = application_name - table.add_row([item.timestep, item.action, node, application, item.response.status]) - return table - def show_history(self, ignored_actions: Optional[list] = None): """ - Print an agent action provided it's not the DONOTHING action. + Print an agent action provided it's not the do-nothing action. :param ignored_actions: OPTIONAL: List of actions to be ignored when displaying the history. - If not provided, defaults to ignore DONOTHING actions. + If not provided, defaults to ignore do-nothing actions. """ if not ignored_actions: - ignored_actions = ["DONOTHING"] + ignored_actions = ["do-nothing"] table = PrettyTable() - table.field_names = ["Step", "Action", "Node", "Application", "Response"] - print(f"Actions for '{self.agent_name}':") + table.field_names = ["Step", "Action", "Params", "Response", "Response Data"] + print(f"Actions for '{self.config.ref}':") for item in self.history: if item.action in ignored_actions: pass else: - table = self.add_agent_action(item=item, table=table) - print(table) + # format dict by putting each key-value entry on a separate line and putting a blank line on the end. + param_string = "\n".join([*[f"{k}: {v:.30}" for k, v in item.parameters.items()], ""]) + data_string = "\n".join([*[f"{k}: {v:.30}" for k, v in item.response.data], ""]) - def add_agent_action(self, item: AgentHistoryItem, table: PrettyTable) -> PrettyTable: - """Update the given table with information from given AgentHistoryItem.""" - node, application = "unknown", "unknown" - if (node_id := item.parameters.get("node_id")) is not None: - node = self.action_manager.node_names[node_id] - if (application_id := item.parameters.get("application_id")) is not None: - application = self.action_manager.application_names[node_id][application_id] - if (application_name := item.parameters.get("application_name")) is not None: - application = application_name - table.add_row([item.timestep, item.action, node, application, item.response.status]) - return table - - def show_history(self, ignored_actions: Optional[list] = None): - """ - Print an agent action provided it's not the DONOTHING action. - - :param ignored_actions: OPTIONAL: List of actions to be ignored when displaying the history. - If not provided, defaults to ignore DONOTHING actions. - """ - if not ignored_actions: - ignored_actions = ["DONOTHING"] - table = PrettyTable() - table.field_names = ["Step", "Action", "Node", "Application", "Response"] - print(f"Actions for '{self.agent_name}':") - for item in self.history: - if item.action in ignored_actions: - pass - else: - table = self.add_agent_action(item=item, table=table) + table.add_row([item.timestep, item.action, param_string, item.response.status, data_string]) print(table) def update_observation(self, state: Dict) -> ObsType: From 8c399c4f61052c7bd55e0d330ee0e3542e61a0d5 Mon Sep 17 00:00:00 2001 From: Marek Wolan Date: Wed, 26 Feb 2025 18:11:42 +0000 Subject: [PATCH 90/95] Fix mismerge of c2 e2e notebook --- ...ommand-and-Control-E2E-Demonstration.ipynb | 157 +++++------------- 1 file changed, 39 insertions(+), 118 deletions(-) diff --git a/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb b/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb index 882c3429..f187c8d5 100644 --- a/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb +++ b/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb @@ -59,7 +59,7 @@ "custom_c2_agent = \"\"\"\n", " - ref: CustomC2Agent\n", " team: RED\n", - " type: ProxyAgent\n", + " type: proxy-agent\n", "\n", " action_space:\n", " action_map:\n", @@ -74,12 +74,8 @@ " 2:\n", " action: configure-c2-beacon\n", " options:\n", - " node_id: 0\n", - " config:\n", - " c2_server_ip_address: 192.168.10.21\n", - " keep_alive_frequency:\n", - " masquerade_protocol:\n", - " masquerade_port:\n", + " node_name: web_server\n", + " c2_server_ip_address: 192.168.10.21\n", " 3:\n", " action: node-application-execute\n", " options:\n", @@ -101,10 +97,9 @@ " 5:\n", " action: c2-server-ransomware-configure\n", " options:\n", - " node_id: 1\n", - " config:\n", - " server_ip_address: 192.168.1.14\n", - " payload: ENCRYPT\n", + " node_name: client_1\n", + " server_ip_address: 192.168.1.14\n", + " payload: ENCRYPT\n", " 6:\n", " action: c2-server-data-exfiltrate\n", " options:\n", @@ -123,25 +118,20 @@ " 8:\n", " action: configure-c2-beacon\n", " options:\n", - " node_id: 0\n", - " config:\n", - " c2_server_ip_address: 192.168.10.21\n", - " keep_alive_frequency: 10\n", - " masquerade_protocol: TCP\n", - " masquerade_port: DNS\n", + " node_name: web_server\n", + " c2_server_ip_address: 192.168.10.21\n", + " keep_alive_frequency: 10\n", + " masquerade_protocol: tcp\n", + " masquerade_port: dns\n", " 9:\n", " action: configure-c2-beacon\n", " options:\n", - " node_id: 0\n", - " config:\n", - " c2_server_ip_address: 192.168.10.22\n", - " keep_alive_frequency:\n", - " masquerade_protocol:\n", - " masquerade_port:\n", + " node_name: web_server\n", + " c2_server_ip_address: 192.168.10.22\n", "\n", " reward_function:\n", " reward_components:\n", - " - type: DUMMY\n", + " - type: dummy\n", "\"\"\"\n", "c2_agent_yaml = yaml.safe_load(custom_c2_agent)" ] @@ -287,13 +277,6 @@ "\n", "```yaml\n", " action_space:\n", - " options:\n", - " nodes: # Node List\n", - " - node_name: web_server\n", - " applications: \n", - " - application_name: C2Beacon\n", - " ...\n", - " ...\n", " action_map:\n", " 3:\n", " action: node-application-execute\n", @@ -352,13 +335,6 @@ "\n", "``` yaml\n", " action_space:\n", - " options:\n", - " nodes: # Node List\n", - " ...\n", - " - node_name: client_1\n", - " applications: \n", - " - application_name: C2Server\n", - " ...\n", " action_map:\n", " 4:\n", " action: c2-server-terminal-command\n", @@ -408,13 +384,6 @@ "\n", "``` yaml\n", " action_space:\n", - " options:\n", - " nodes: # Node List\n", - " ...\n", - " - node_name: client_1\n", - " applications: \n", - " - application_name: C2Server\n", - " ...\n", " action_map:\n", " 5:\n", " action: c2-server-ransomware-configure\n", @@ -459,13 +428,6 @@ "\n", "``` yaml\n", " action_space:\n", - " options:\n", - " nodes: # Node List\n", - " ...\n", - " - node_name: client_1\n", - " applications: \n", - " - application_name: C2Server\n", - " ...\n", " action_map:\n", " 6:\n", " action: c2-server-data-exfiltrate\n", @@ -524,13 +486,6 @@ "\n", "``` yaml\n", " action_space:\n", - " options:\n", - " nodes: # Node List\n", - " ...\n", - " - node_name: client_1\n", - " applications: \n", - " - application_name: C2Server\n", - " ...\n", " action_map:\n", " 7:\n", " action: c2-server-ransomware-launch\n", @@ -584,8 +539,8 @@ " type: custom\n", " options:\n", " components:\n", - " - type: NODES\n", - " label: NODES\n", + " - type: nodes\n", + " label: nodes\n", " options:\n", " hosts:\n", " - hostname: web_server\n", @@ -667,55 +622,29 @@ " 1:\n", " action: node-application-remove\n", " options:\n", - " node_id: 0\n", - " application_name: C2Beacon\n", + " node_name: web_server\n", + " application_name: c2-beacon\n", " 2:\n", " action: node-shutdown\n", " options:\n", - " node_id: 0\n", + " node_name: web_server\n", " 3:\n", " action: router-acl-add-rule\n", " options:\n", " target_router: router_1\n", " position: 1\n", - " permission: 2\n", - " source_ip_id: 2\n", - " dest_ip_id: 3\n", - " source_port_id: 2\n", - " dest_port_id: 2\n", - " protocol_id: 1\n", - " source_wildcard_id: 0\n", - " dest_wildcard_id: 0\n", + " permission: DENY\n", + " src_ip: 192.168.10.21\n", + " dst_ip: 192.168.1.12\n", + " src_port: HTTP\n", + " dst_port: HTTP\n", + " protocol_name: ALL\n", + " src_wildcard: 0.0.0.1\n", + " dst_wildcard: 0.0.0.1\n", "\n", - "\n", - " options:\n", - " nodes:\n", - " - node_name: web_server\n", - " applications:\n", - " - application_name: C2Beacon\n", - "\n", - " - node_name: database_server\n", - " folders:\n", - " - folder_name: database\n", - " files:\n", - " - file_name: database.db\n", - " services:\n", - " - service_name: DatabaseService\n", - " - node_name: router_1\n", - "\n", - " max_folders_per_node: 2\n", - " max_files_per_folder: 2\n", - " max_services_per_node: 2\n", - " max_nics_per_node: 8\n", - " max_acl_rules: 10\n", - " ip_list:\n", - " - 192.168.10.21\n", - " - 192.168.1.12\n", - " wildcard_list:\n", - " - 0.0.0.1\n", " reward_function:\n", " reward_components:\n", - " - type: DUMMY\n", + " - type: dummy\n", "\n", " agent_settings:\n", " flatten_obs: False\n", @@ -1112,7 +1041,7 @@ "outputs": [], "source": [ "# Attempting to install the C2 RansomwareScript\n", - "ransomware_install_command = {\"commands\":[[\"software_manager\", \"application\", \"install\", \"ransomware-script\"]],\n", + "ransomware_install_command = {\"commands\":[[\"software_manager\", \"application\", \"install\", \"RansomwareScript\"]],\n", " \"username\": \"admin\",\n", " \"password\": \"admin\"}\n", "\n", @@ -1200,7 +1129,7 @@ "outputs": [], "source": [ "# Attempting to install the C2 RansomwareScript\n", - "ransomware_install_command = {\"commands\":[\"software_manager\", \"application\", \"install\", \"ransomware-script\"],\n", + "ransomware_install_command = {\"commands\":[\"software_manager\", \"application\", \"install\", \"RansomwareScript\"],\n", " \"username\": \"admin\",\n", " \"password\": \"admin\"}\n", "\n", @@ -1325,7 +1254,7 @@ "metadata": {}, "outputs": [], "source": [ - "database_server: Server = blue_env.game.simulation.network.get_node_by_hostname(\"database-server\")\n", + "database_server: Server = blue_env.game.simulation.network.get_node_by_hostname(\"database_server\")\n", "database_server.software_manager.file_system.show(full=True)" ] }, @@ -1369,12 +1298,14 @@ "source": [ "As demonstrated earlier, red agents can use the ``configure-c2-beacon`` action to configure these settings mid episode through the configuration options:\n", "\n", - "``` YAML\n", - "...\n", - " action: configure_c2_beacon\n", - " options:\n", - " node_id: 0\n", - " config:\n", + "```YAML\n", + "\n", + " action_space:\n", + " action_map:\n", + " 8:\n", + " action: configure-c2-beacon\n", + " options:\n", + " node_name: web_server\n", " c2_server_ip_address: 192.168.10.21\n", " keep_alive_frequency: 10\n", " masquerade_protocol: tcp\n", @@ -1739,16 +1670,6 @@ "\n", "display_obs_diffs(tcp_c2_obs, udp_c2_obs, blue_config_env.game.step_counter)" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "\n", - "env.game.agents[\"CustomC2Agent\"].show_history()" - ] } ], "metadata": { From cf33dcdcf9e9948bcd9c22331e304fafff15088e Mon Sep 17 00:00:00 2001 From: Marek Wolan Date: Wed, 26 Feb 2025 18:12:20 +0000 Subject: [PATCH 91/95] remove outdated information from agents doc page --- docs/source/configuration/agents.rst | 6 ------ 1 file changed, 6 deletions(-) diff --git a/docs/source/configuration/agents.rst b/docs/source/configuration/agents.rst index ee84aede..c2674e31 100644 --- a/docs/source/configuration/agents.rst +++ b/docs/source/configuration/agents.rst @@ -20,12 +20,6 @@ Agents can be scripted (deterministic and stochastic), or controlled by a reinfo - ref: green_agent_example team: GREEN type: probabilistic-agent - observation_space: - type: UC2GreenObservation # TODO: what - action_space: - reward_function: - reward_components: - - type: dummy agent_settings: start_settings: From f1a36cafaac887573f8b7d9be6d552d0383a4951 Mon Sep 17 00:00:00 2001 From: Marek Wolan Date: Wed, 26 Feb 2025 18:13:45 +0000 Subject: [PATCH 92/95] remove outdated information from data manipulation bot doc page --- .../applications/data_manipulation_bot.rst | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/docs/source/simulation_components/system/applications/data_manipulation_bot.rst b/docs/source/simulation_components/system/applications/data_manipulation_bot.rst index 3ddb8bca..04c581bd 100644 --- a/docs/source/simulation_components/system/applications/data_manipulation_bot.rst +++ b/docs/source/simulation_components/system/applications/data_manipulation_bot.rst @@ -97,26 +97,6 @@ If not using the data manipulation bot manually, it needs to be used with a data team: RED type: red-database-corrupting-agent - observation_space: - type: uc2-red-observation #TODO what - options: - nodes: - - node_name: client_1 - observations: - - logon_status - - operating_status - applications: - - application_ref: data_manipulation_bot - observations: - operating_status - health_status - folders: {} - - action_space: - reward_function: - reward_components: - - type: dummy - agent_settings: start_settings: start_step: 25 From fd367d1f0eff88095612cfefcdf7a20f559b395d Mon Sep 17 00:00:00 2001 From: Marek Wolan Date: Wed, 26 Feb 2025 18:21:28 +0000 Subject: [PATCH 93/95] Fix typos and duplicate identifiers in docs --- docs/source/request_system.rst | 2 -- .../system/applications/database_client.rst | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/source/request_system.rst b/docs/source/request_system.rst index 30ced50a..93fc2a9f 100644 --- a/docs/source/request_system.rst +++ b/docs/source/request_system.rst @@ -4,8 +4,6 @@ .. _request_system: -.. _request_system: - Request System ************** diff --git a/docs/source/simulation_components/system/applications/database_client.rst b/docs/source/simulation_components/system/applications/database_client.rst index 472b504c..465827d9 100644 --- a/docs/source/simulation_components/system/applications/database_client.rst +++ b/docs/source/simulation_components/system/applications/database_client.rst @@ -59,7 +59,7 @@ Python # install DatabaseClient client.software_manager.install(DatabaseClient) - database_client: DatabaseClient = client.software_manager.software.get("database-sclient") + database_client: DatabaseClient = client.software_manager.software.get("database-client") # Configure the DatabaseClient database_client.configure(server_ip_address=IPv4Address("192.168.0.1")) # address of the DatabaseService From bab40603788f2f06fa6945527238217b79e54743 Mon Sep 17 00:00:00 2001 From: Marek Wolan Date: Wed, 26 Feb 2025 18:26:54 +0000 Subject: [PATCH 94/95] Fix agent config in terminal processing notebook --- src/primaite/notebooks/Terminal-Processing.ipynb | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/src/primaite/notebooks/Terminal-Processing.ipynb b/src/primaite/notebooks/Terminal-Processing.ipynb index 07d38791..755b0184 100644 --- a/src/primaite/notebooks/Terminal-Processing.ipynb +++ b/src/primaite/notebooks/Terminal-Processing.ipynb @@ -298,21 +298,7 @@ " - ref: CustomC2Agent\n", " team: RED\n", " type: proxy-agent\n", - " observation_space: null\n", " action_space:\n", - " options:\n", - " nodes:\n", - " - node_name: client_1\n", - " max_folders_per_node: 1\n", - " max_files_per_folder: 1\n", - " max_services_per_node: 2\n", - " max_nics_per_node: 8\n", - " max_acl_rules: 10\n", - " ip_list:\n", - " - 192.168.1.21\n", - " - 192.168.1.14\n", - " wildcard_list:\n", - " - 0.0.0.1\n", " action_map:\n", " 0:\n", " action: do-nothing\n", @@ -508,7 +494,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "venv", "language": "python", "name": "python3" }, From 2b04695c2e8e8d0ca2af074c6eb45f41ee78d63f Mon Sep 17 00:00:00 2001 From: Marek Wolan Date: Thu, 27 Feb 2025 10:07:17 +0000 Subject: [PATCH 95/95] Apply suggestions from code review --- src/primaite/VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/primaite/VERSION b/src/primaite/VERSION index 21a8fb4d..d9b058f1 100644 --- a/src/primaite/VERSION +++ b/src/primaite/VERSION @@ -1 +1 @@ -4.0.0a1-dev +4.0.0-dev