From 3f94c40434d11034cd1dc5dc9e07800c03bbcec4 Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Wed, 22 Jan 2025 10:49:42 +0000 Subject: [PATCH 01/37] Fix logger inititialisation in agents --- src/primaite/game/agent/interface.py | 7 +++ src/primaite/game/game.py | 1 + .../notebooks/Training-an-SB3-Agent.ipynb | 48 ++++++++++++++++--- 3 files changed, 49 insertions(+), 7 deletions(-) diff --git a/src/primaite/game/agent/interface.py b/src/primaite/game/agent/interface.py index aac898e1..6a6bc323 100644 --- a/src/primaite/game/agent/interface.py +++ b/src/primaite/game/agent/interface.py @@ -68,6 +68,10 @@ class AbstractAgent(BaseModel, ABC): ) reward_function: RewardFunction.ConfigSchema = Field(default_factory=lambda: RewardFunction.ConfigSchema()) + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.logger: AgentLog = AgentLog(agent_name=self.config.ref) + config: "AbstractAgent.ConfigSchema" = Field(default_factory=lambda: AbstractAgent.ConfigSchema()) logger: AgentLog = AgentLog(agent_name="Abstract_Agent") @@ -81,6 +85,7 @@ class AbstractAgent(BaseModel, ABC): def __init_subclass__(cls, identifier: Optional[str] = None, **kwargs: Any) -> None: super().__init_subclass__(**kwargs) + print("cls identifier:", identifier) if identifier is None: return if identifier in cls._registry: @@ -157,6 +162,8 @@ class AbstractAgent(BaseModel, ABC): def from_config(cls, config: Dict) -> AbstractAgent: """Grab the relevant agent class and construct an instance from a config dict.""" agent_type = config["type"] + print("agent_type:", agent_type) + print("cls._registry:", cls._registry) agent_class = cls._registry[agent_type] return agent_class(config=config) diff --git a/src/primaite/game/game.py b/src/primaite/game/game.py index f59117f4..c828a462 100644 --- a/src/primaite/game/game.py +++ b/src/primaite/game/game.py @@ -493,6 +493,7 @@ class PrimaiteGame: agents_cfg = cfg.get("agents", []) for agent_cfg in agents_cfg: + print("agent_cfg:", agent_cfg) new_agent = AbstractAgent.from_config(agent_cfg) game.agents[agent_cfg["ref"]] = new_agent if isinstance(new_agent, ProxyAgent): diff --git a/src/primaite/notebooks/Training-an-SB3-Agent.ipynb b/src/primaite/notebooks/Training-an-SB3-Agent.ipynb index 2b554475..0e3245ae 100644 --- a/src/primaite/notebooks/Training-an-SB3-Agent.ipynb +++ b/src/primaite/notebooks/Training-an-SB3-Agent.ipynb @@ -29,18 +29,28 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "cls identifier: AbstractScriptedAgent\n", + "cls identifier: ProxyAgent\n" + ] + } + ], "source": [ "from primaite.game.game import PrimaiteGame\n", "from primaite.session.environment import PrimaiteGymEnv\n", + "from primaite.game.agent.scripted_agents import probabilistic_agent\n", "import yaml" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -49,7 +59,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -69,9 +79,33 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "agent_cfg: {'ref': 'client_2_green_user', 'team': 'GREEN', 'type': 'ProbabilisticAgent', 'agent_settings': {'action_probabilities': {0: 0.3, 1: 0.6, 2: 0.1}}, 'action_space': {'action_map': {0: {'action': 'do_nothing', 'options': {}}, 1: {'action': 'node_application_execute', 'options': {'node_name': 'client_2', 'application_name': 'WebBrowser'}}, 2: {'action': 'node_application_execute', 'options': {'node_name': 'client_2', 'application_name': 'DatabaseClient'}}}}, 'reward_function': {'reward_components': [{'type': 'WEBPAGE_UNAVAILABLE_PENALTY', 'weight': 0.25, 'options': {'node_hostname': 'client_2'}}, {'type': 'GREEN_ADMIN_DATABASE_UNREACHABLE_PENALTY', 'weight': 0.05, 'options': {'node_hostname': 'client_2'}}]}}\n", + "agent_type: ProbabilisticAgent\n", + "cls._registry: {'AbstractScriptedAgent': , 'ProxyAgent': }\n" + ] + }, + { + "ename": "KeyError", + "evalue": "'ProbabilisticAgent'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[4], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m gym \u001b[38;5;241m=\u001b[39m \u001b[43mPrimaiteGymEnv\u001b[49m\u001b[43m(\u001b[49m\u001b[43menv_config\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcfg\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/session/environment.py:71\u001b[0m, in \u001b[0;36mPrimaiteGymEnv.__init__\u001b[0;34m(self, env_config)\u001b[0m\n\u001b[1;32m 69\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mio \u001b[38;5;241m=\u001b[39m PrimaiteIO\u001b[38;5;241m.\u001b[39mfrom_config(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mepisode_scheduler(\u001b[38;5;241m0\u001b[39m)\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mio_settings\u001b[39m\u001b[38;5;124m\"\u001b[39m, {}))\n\u001b[1;32m 70\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Handles IO for the environment. This produces sys logs, agent logs, etc.\"\"\"\u001b[39;00m\n\u001b[0;32m---> 71\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgame: PrimaiteGame \u001b[38;5;241m=\u001b[39m \u001b[43mPrimaiteGame\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfrom_config\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mepisode_scheduler\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 72\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Current game.\"\"\"\u001b[39;00m\n\u001b[1;32m 73\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_agent_name \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mnext\u001b[39m(\u001b[38;5;28miter\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgame\u001b[38;5;241m.\u001b[39mrl_agents))\n", + "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/game/game.py:497\u001b[0m, in \u001b[0;36mPrimaiteGame.from_config\u001b[0;34m(cls, cfg)\u001b[0m\n\u001b[1;32m 495\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m agent_cfg \u001b[38;5;129;01min\u001b[39;00m agents_cfg:\n\u001b[1;32m 496\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124magent_cfg:\u001b[39m\u001b[38;5;124m\"\u001b[39m, agent_cfg)\n\u001b[0;32m--> 497\u001b[0m new_agent \u001b[38;5;241m=\u001b[39m \u001b[43mAbstractAgent\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfrom_config\u001b[49m\u001b[43m(\u001b[49m\u001b[43magent_cfg\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 498\u001b[0m game\u001b[38;5;241m.\u001b[39magents[agent_cfg[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mref\u001b[39m\u001b[38;5;124m\"\u001b[39m]] \u001b[38;5;241m=\u001b[39m new_agent\n\u001b[1;32m 499\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(new_agent, ProxyAgent):\n", + "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/game/agent/interface.py:163\u001b[0m, in \u001b[0;36mAbstractAgent.from_config\u001b[0;34m(cls, config)\u001b[0m\n\u001b[1;32m 161\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124magent_type:\u001b[39m\u001b[38;5;124m\"\u001b[39m, agent_type)\n\u001b[1;32m 162\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcls._registry:\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28mcls\u001b[39m\u001b[38;5;241m.\u001b[39m_registry)\n\u001b[0;32m--> 163\u001b[0m agent_class \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mcls\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_registry\u001b[49m\u001b[43m[\u001b[49m\u001b[43magent_type\u001b[49m\u001b[43m]\u001b[49m\n\u001b[1;32m 164\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m agent_class(config\u001b[38;5;241m=\u001b[39mconfig)\n", + "\u001b[0;31mKeyError\u001b[0m: 'ProbabilisticAgent'" + ] + } + ], "source": [ "gym = PrimaiteGymEnv(env_config=cfg)" ] @@ -191,7 +225,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.11" + "version": "3.10.12" } }, "nbformat": 4, From 0aa691752aa268880f1a9f7531286e5a159136d0 Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Wed, 22 Jan 2025 12:20:45 +0000 Subject: [PATCH 02/37] #3075: Tidy up debug print statements. --- src/primaite/game/agent/interface.py | 3 -- src/primaite/game/game.py | 2 +- .../notebooks/Training-an-SB3-Agent.ipynb | 51 ++++--------------- 3 files changed, 10 insertions(+), 46 deletions(-) diff --git a/src/primaite/game/agent/interface.py b/src/primaite/game/agent/interface.py index 6a6bc323..5dab6aac 100644 --- a/src/primaite/game/agent/interface.py +++ b/src/primaite/game/agent/interface.py @@ -85,7 +85,6 @@ class AbstractAgent(BaseModel, ABC): def __init_subclass__(cls, identifier: Optional[str] = None, **kwargs: Any) -> None: super().__init_subclass__(**kwargs) - print("cls identifier:", identifier) if identifier is None: return if identifier in cls._registry: @@ -162,8 +161,6 @@ class AbstractAgent(BaseModel, ABC): def from_config(cls, config: Dict) -> AbstractAgent: """Grab the relevant agent class and construct an instance from a config dict.""" agent_type = config["type"] - print("agent_type:", agent_type) - print("cls._registry:", cls._registry) agent_class = cls._registry[agent_type] return agent_class(config=config) diff --git a/src/primaite/game/game.py b/src/primaite/game/game.py index c828a462..d9f6fa3c 100644 --- a/src/primaite/game/game.py +++ b/src/primaite/game/game.py @@ -493,7 +493,7 @@ class PrimaiteGame: agents_cfg = cfg.get("agents", []) for agent_cfg in agents_cfg: - print("agent_cfg:", agent_cfg) + new_agent = AbstractAgent.from_config(agent_cfg) game.agents[agent_cfg["ref"]] = new_agent if isinstance(new_agent, ProxyAgent): diff --git a/src/primaite/notebooks/Training-an-SB3-Agent.ipynb b/src/primaite/notebooks/Training-an-SB3-Agent.ipynb index 0e3245ae..d3492f92 100644 --- a/src/primaite/notebooks/Training-an-SB3-Agent.ipynb +++ b/src/primaite/notebooks/Training-an-SB3-Agent.ipynb @@ -29,18 +29,9 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "cls identifier: AbstractScriptedAgent\n", - "cls identifier: ProxyAgent\n" - ] - } - ], + "outputs": [], "source": [ "from primaite.game.game import PrimaiteGame\n", "from primaite.session.environment import PrimaiteGymEnv\n", @@ -50,7 +41,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -59,7 +50,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -79,33 +70,9 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "agent_cfg: {'ref': 'client_2_green_user', 'team': 'GREEN', 'type': 'ProbabilisticAgent', 'agent_settings': {'action_probabilities': {0: 0.3, 1: 0.6, 2: 0.1}}, 'action_space': {'action_map': {0: {'action': 'do_nothing', 'options': {}}, 1: {'action': 'node_application_execute', 'options': {'node_name': 'client_2', 'application_name': 'WebBrowser'}}, 2: {'action': 'node_application_execute', 'options': {'node_name': 'client_2', 'application_name': 'DatabaseClient'}}}}, 'reward_function': {'reward_components': [{'type': 'WEBPAGE_UNAVAILABLE_PENALTY', 'weight': 0.25, 'options': {'node_hostname': 'client_2'}}, {'type': 'GREEN_ADMIN_DATABASE_UNREACHABLE_PENALTY', 'weight': 0.05, 'options': {'node_hostname': 'client_2'}}]}}\n", - "agent_type: ProbabilisticAgent\n", - "cls._registry: {'AbstractScriptedAgent': , 'ProxyAgent': }\n" - ] - }, - { - "ename": "KeyError", - "evalue": "'ProbabilisticAgent'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[4], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m gym \u001b[38;5;241m=\u001b[39m \u001b[43mPrimaiteGymEnv\u001b[49m\u001b[43m(\u001b[49m\u001b[43menv_config\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcfg\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/session/environment.py:71\u001b[0m, in \u001b[0;36mPrimaiteGymEnv.__init__\u001b[0;34m(self, env_config)\u001b[0m\n\u001b[1;32m 69\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mio \u001b[38;5;241m=\u001b[39m PrimaiteIO\u001b[38;5;241m.\u001b[39mfrom_config(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mepisode_scheduler(\u001b[38;5;241m0\u001b[39m)\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mio_settings\u001b[39m\u001b[38;5;124m\"\u001b[39m, {}))\n\u001b[1;32m 70\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Handles IO for the environment. This produces sys logs, agent logs, etc.\"\"\"\u001b[39;00m\n\u001b[0;32m---> 71\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgame: PrimaiteGame \u001b[38;5;241m=\u001b[39m \u001b[43mPrimaiteGame\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfrom_config\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mepisode_scheduler\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 72\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Current game.\"\"\"\u001b[39;00m\n\u001b[1;32m 73\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_agent_name \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mnext\u001b[39m(\u001b[38;5;28miter\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgame\u001b[38;5;241m.\u001b[39mrl_agents))\n", - "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/game/game.py:497\u001b[0m, in \u001b[0;36mPrimaiteGame.from_config\u001b[0;34m(cls, cfg)\u001b[0m\n\u001b[1;32m 495\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m agent_cfg \u001b[38;5;129;01min\u001b[39;00m agents_cfg:\n\u001b[1;32m 496\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124magent_cfg:\u001b[39m\u001b[38;5;124m\"\u001b[39m, agent_cfg)\n\u001b[0;32m--> 497\u001b[0m new_agent \u001b[38;5;241m=\u001b[39m \u001b[43mAbstractAgent\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfrom_config\u001b[49m\u001b[43m(\u001b[49m\u001b[43magent_cfg\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 498\u001b[0m game\u001b[38;5;241m.\u001b[39magents[agent_cfg[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mref\u001b[39m\u001b[38;5;124m\"\u001b[39m]] \u001b[38;5;241m=\u001b[39m new_agent\n\u001b[1;32m 499\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(new_agent, ProxyAgent):\n", - "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/game/agent/interface.py:163\u001b[0m, in \u001b[0;36mAbstractAgent.from_config\u001b[0;34m(cls, config)\u001b[0m\n\u001b[1;32m 161\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124magent_type:\u001b[39m\u001b[38;5;124m\"\u001b[39m, agent_type)\n\u001b[1;32m 162\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcls._registry:\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28mcls\u001b[39m\u001b[38;5;241m.\u001b[39m_registry)\n\u001b[0;32m--> 163\u001b[0m agent_class \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mcls\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_registry\u001b[49m\u001b[43m[\u001b[49m\u001b[43magent_type\u001b[49m\u001b[43m]\u001b[49m\n\u001b[1;32m 164\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m agent_class(config\u001b[38;5;241m=\u001b[39mconfig)\n", - "\u001b[0;31mKeyError\u001b[0m: 'ProbabilisticAgent'" - ] - } - ], + "outputs": [], "source": [ "gym = PrimaiteGymEnv(env_config=cfg)" ] @@ -134,7 +101,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ @@ -166,7 +133,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -211,7 +178,7 @@ ], "metadata": { "kernelspec": { - "display_name": ".venv", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, From 8eeba691a9755e1bc237f0687bba2e82061a8fdb Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Thu, 23 Jan 2025 15:16:49 +0000 Subject: [PATCH 03/37] #3075: Remove unnecessary print statement. --- src/primaite/game/agent/actions/node.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/primaite/game/agent/actions/node.py b/src/primaite/game/agent/actions/node.py index fbab18f0..5e1b6725 100644 --- a/src/primaite/game/agent/actions/node.py +++ b/src/primaite/game/agent/actions/node.py @@ -36,7 +36,6 @@ class NodeAbstractAction(AbstractAction, identifier="node_abstract"): @classmethod def form_request(cls, config: ConfigSchema) -> RequestFormat: """Return the action formatted as a request which can be ingested by the PrimAITE simulation.""" - print(config) return ["network", "node", config.node_name, config.verb] From f4b73057d2094f9c248ad8f3ae757501378afff3 Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Mon, 27 Jan 2025 17:08:33 +0000 Subject: [PATCH 04/37] #3075: Update notebooks to reflect extension changes. --- src/primaite/notebooks/Action-masking.ipynb | 17 +++++----- ...ommand-and-Control-E2E-Demonstration.ipynb | 16 +++++++-- ...a-Manipulation-Customising-Red-Agent.ipynb | 33 +++++++++++++------ .../Data-Manipulation-E2E-Demonstration.ipynb | 15 +++++---- .../Getting-Information-Out-Of-PrimAITE.ipynb | 6 ++-- ...ege-Escalation-and-Data-Loss-Example.ipynb | 15 +++++++-- .../Training-an-RLLIB-MARL-System.ipynb | 20 +++++++++-- .../notebooks/Training-an-RLLib-Agent.ipynb | 23 ++++++++++--- .../notebooks/Using-Episode-Schedules.ipynb | 15 ++------- src/primaite/notebooks/multi-processing.ipynb | 18 +++++----- 10 files changed, 116 insertions(+), 62 deletions(-) diff --git a/src/primaite/notebooks/Action-masking.ipynb b/src/primaite/notebooks/Action-masking.ipynb index 7fde0a49..64015080 100644 --- a/src/primaite/notebooks/Action-masking.ipynb +++ b/src/primaite/notebooks/Action-masking.ipynb @@ -22,13 +22,14 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "from primaite.session.environment import PrimaiteGymEnv\n", "from primaite.config.load import data_manipulation_config_path\n", - "from prettytable import PrettyTable" + "from prettytable import PrettyTable\n", + "from primaite.game.agent.scripted_agents import probabilistic_agent\n" ] }, { @@ -103,7 +104,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -116,7 +117,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ @@ -156,7 +157,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ @@ -167,7 +168,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "metadata": {}, "outputs": [], "source": [ @@ -204,7 +205,7 @@ ], "metadata": { "kernelspec": { - "display_name": ".venv", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -218,7 +219,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.11" + "version": "3.10.12" } }, "nbformat": 4, diff --git a/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb b/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb index 66a684de..625510b8 100644 --- a/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb +++ b/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb @@ -16,6 +16,15 @@ "execution_count": null, "metadata": {}, "outputs": [], + "source": [ + "!primaite setup" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], "source": [ "# Imports\n", "import yaml\n", @@ -27,7 +36,8 @@ "from primaite.simulator.system.applications.red_applications.c2.abstract_c2 import C2Command\n", "from primaite.simulator.system.applications.red_applications.ransomware_script import RansomwareScript\n", "from primaite.simulator.network.hardware.nodes.host.computer import Computer\n", - "from primaite.simulator.network.hardware.nodes.host.server import Server" + "from primaite.simulator.network.hardware.nodes.host.server import Server\n", + "from primaite.game.agent.scripted_agents import probabilistic_agent" ] }, { @@ -43,7 +53,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -1763,7 +1773,7 @@ ], "metadata": { "kernelspec": { - "display_name": ".venv", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, diff --git a/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb b/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb index 756fc44f..de6d3fe5 100644 --- a/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb +++ b/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb @@ -26,7 +26,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -36,12 +36,13 @@ "from primaite.game.agent.interface import AgentHistoryItem\n", "from primaite.session.environment import PrimaiteGymEnv\n", "import yaml\n", - "from pprint import pprint" + "from pprint import pprint\n", + "from primaite.game.agent.scripted_agents import probabilistic_agent, data_manipulation_bot" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -68,7 +69,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -79,7 +80,8 @@ " if red_action == 'do_nothing':\n", " red_str = 'DO NOTHING'\n", " elif red_action == 'node_application_execute':\n", - " client = \"client 1\" if red_info.parameters['node_id'] == 0 else \"client 2\"\n", + " client = \"client 1\" if red_info.parameters['node_name'] == 0 else \"client 2\"\n", + "\n", " red_str = f\"ATTACK from {client}\"\n", " return red_str" ] @@ -243,7 +245,8 @@ "outputs": [], "source": [ "change = yaml.safe_load(\"\"\"\n", - "start_settings:\n", + " possible_start_nodes: [client_1, client_2]\n", + " target_application: DataManipulationBot\n", " start_step: 25\n", " frequency: 20\n", " variance: 0\n", @@ -253,7 +256,9 @@ " cfg = yaml.safe_load(f)\n", " for agent in cfg['agents']:\n", " if agent['ref'] == \"data_manipulation_attacker\":\n", + " print(f\"{agent['agent_settings']=}\")\n", " agent['agent_settings'] = change\n", + " print(f\"{agent['agent_settings']=}\")\n", "\n", "env = PrimaiteGymEnv(env_config = cfg)\n", "env.reset()\n", @@ -310,9 +315,17 @@ "outputs": [], "source": [ "change = yaml.safe_load(\"\"\"\n", - "# TODO:\n", + " action_space:\n", + " action_map:\n", + " 0:\n", + " action: do_nothing\n", + " options: {}\n", + " 1:\n", + " action: node_application_execute\n", + " options:\n", + " node_name: client_1\n", + " application_name: DataManipulationBot\n", "\"\"\")\n", - "#TODO 2869 fix\n", "\n", "with open(data_manipulation_config_path(), 'r') as f:\n", " cfg = yaml.safe_load(f)\n", @@ -438,7 +451,7 @@ ], "metadata": { "kernelspec": { - "display_name": ".venv", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -452,7 +465,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.11" + "version": "3.10.12" } }, "nbformat": 4, diff --git a/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb b/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb index dbc6f0c1..3818bb18 100644 --- a/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb +++ b/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb @@ -382,7 +382,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": { "tags": [] }, @@ -394,7 +394,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": { "tags": [] }, @@ -405,7 +405,8 @@ "from primaite.session.environment import PrimaiteGymEnv\n", "from primaite.game.agent.interface import AgentHistoryItem\n", "import yaml\n", - "from pprint import pprint\n" + "from pprint import pprint\n", + "from primaite.game.agent.scripted_agents import probabilistic_agent, data_manipulation_bot\n" ] }, { @@ -429,9 +430,9 @@ " cfg = yaml.safe_load(f)\n", " # set success probability to 1.0 to avoid rerunning cells.\n", " cfg['simulation']['network']['nodes'][8]['applications'][0]['options']['data_manipulation_p_of_success'] = 1.0\n", - " cfg['simulation']['network']['nodes'][9]['applications'][0]['options']['data_manipulation_p_of_success'] = 1.0\n", + " cfg['simulation']['network']['nodes'][9]['applications'][1]['options']['data_manipulation_p_of_success'] = 1.0\n", " cfg['simulation']['network']['nodes'][8]['applications'][0]['options']['port_scan_p_of_success'] = 1.0\n", - " cfg['simulation']['network']['nodes'][9]['applications'][0]['options']['port_scan_p_of_success'] = 1.0\n", + " cfg['simulation']['network']['nodes'][9]['applications'][1]['options']['port_scan_p_of_success'] = 1.0\n", " # don't flatten observations so that we can see what is going on\n", " cfg['agents'][3]['agent_settings']['flatten_obs'] = False\n", "\n", @@ -450,7 +451,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -461,7 +462,7 @@ " if red_action == 'do_nothing':\n", " red_str = 'DO NOTHING'\n", " elif red_action == 'node_application_execute':\n", - " client = \"client 1\" if red_info.parameters['node_id'] == 0 else \"client 2\"\n", + " client = \"client 1\" if red_info.parameters['node_name'] == 0 else \"client 2\"\n", " red_str = f\"ATTACK from {client}\"\n", " return red_str" ] diff --git a/src/primaite/notebooks/Getting-Information-Out-Of-PrimAITE.ipynb b/src/primaite/notebooks/Getting-Information-Out-Of-PrimAITE.ipynb index f8691d7d..d7d60d76 100644 --- a/src/primaite/notebooks/Getting-Information-Out-Of-PrimAITE.ipynb +++ b/src/primaite/notebooks/Getting-Information-Out-Of-PrimAITE.ipynb @@ -32,6 +32,8 @@ "from primaite.session.environment import PrimaiteGymEnv\n", "from primaite.simulator.network.hardware.nodes.host.computer import Computer\n", "from notebook.services.config import ConfigManager\n", + "from primaite.game.agent.scripted_agents import probabilistic_agent\n", + "\n", "\n", "cm = ConfigManager().update('notebook', {'limit_output': 50}) # limit output lines to 50 - for neatness\n", "\n", @@ -157,7 +159,7 @@ ], "metadata": { "kernelspec": { - "display_name": "venv", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -171,7 +173,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.8" + "version": "3.10.12" } }, "nbformat": 4, diff --git a/src/primaite/notebooks/Privilege-Escalation-and-Data-Loss-Example.ipynb b/src/primaite/notebooks/Privilege-Escalation-and-Data-Loss-Example.ipynb index fcda4dbd..35d3813a 100644 --- a/src/primaite/notebooks/Privilege-Escalation-and-Data-Loss-Example.ipynb +++ b/src/primaite/notebooks/Privilege-Escalation-and-Data-Loss-Example.ipynb @@ -54,6 +54,15 @@ { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!primaite setup" + ] + }, + { + "cell_type": "code", + "execution_count": 2, "metadata": { "tags": [] }, @@ -68,7 +77,7 @@ "from primaite.simulator.network.hardware.nodes.host.server import Server\n", "from primaite.simulator.system.applications.database_client import DatabaseClient\n", "from primaite.simulator.system.applications.web_browser import WebBrowser\n", - "from primaite.simulator.system.services.database.database_service import DatabaseService" + "from primaite.simulator.system.services.database.database_service import DatabaseService\n" ] }, { @@ -80,7 +89,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": { "tags": [] }, @@ -103,7 +112,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": { "tags": [] }, diff --git a/src/primaite/notebooks/Training-an-RLLIB-MARL-System.ipynb b/src/primaite/notebooks/Training-an-RLLIB-MARL-System.ipynb index 76cab86a..dadb399e 100644 --- a/src/primaite/notebooks/Training-an-RLLIB-MARL-System.ipynb +++ b/src/primaite/notebooks/Training-an-RLLIB-MARL-System.ipynb @@ -40,6 +40,7 @@ "import ray\n", "from ray.rllib.algorithms.ppo import PPOConfig\n", "from primaite.session.ray_envs import PrimaiteRayMARLEnv\n", + "from primaite.game.agent.scripted_agents import probabilistic_agent\n", "\n", "with open(PRIMAITE_PATHS.user_config_path / 'example_config/data_manipulation_marl.yaml', 'r') as f:\n", " cfg = yaml.safe_load(f)\n", @@ -56,7 +57,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -102,7 +103,20 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "*** SIGTERM received at time=1737996337 on cpu 7 ***\n", + "PC: @ 0x7f3649b0fe2e (unknown) epoll_wait\n", + " @ 0x7f3649a2c520 (unknown) (unknown)\n", + "[2025-01-27 16:45:37,381 E 117142 117142] logging.cc:440: *** SIGTERM received at time=1737996337 on cpu 7 ***\n", + "[2025-01-27 16:45:37,381 E 117142 117142] logging.cc:440: PC: @ 0x7f3649b0fe2e (unknown) epoll_wait\n", + "[2025-01-27 16:45:37,381 E 117142 117142] logging.cc:440: @ 0x7f3649a2c520 (unknown) (unknown)\n" + ] + } + ], "source": [ "eval = algo.evaluate()" ] @@ -110,7 +124,7 @@ ], "metadata": { "kernelspec": { - "display_name": "venv", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, diff --git a/src/primaite/notebooks/Training-an-RLLib-Agent.ipynb b/src/primaite/notebooks/Training-an-RLLib-Agent.ipynb index 7252b046..64a9e7ab 100644 --- a/src/primaite/notebooks/Training-an-RLLib-Agent.ipynb +++ b/src/primaite/notebooks/Training-an-RLLib-Agent.ipynb @@ -32,6 +32,8 @@ "from primaite.session.ray_envs import PrimaiteRayEnv\n", "import ray\n", "from ray.rllib.algorithms.ppo import PPOConfig\n", + "from primaite.game.agent.scripted_agents import probabilistic_agent\n", + "\n", "\n", "# If you get an error saying this config file doesn't exist, you may need to run `primaite setup` in your command line\n", "# to copy the files to your user data path.\n", @@ -50,7 +52,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -96,7 +98,20 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "*** SIGTERM received at time=1737996055 on cpu 1 ***\n", + "PC: @ 0x7f6e254a6e2e (unknown) epoll_wait\n", + " @ 0x7f6e253c3520 (unknown) (unknown)\n", + "[2025-01-27 16:40:55,343 E 114171 114171] logging.cc:440: *** SIGTERM received at time=1737996055 on cpu 1 ***\n", + "[2025-01-27 16:40:55,343 E 114171 114171] logging.cc:440: PC: @ 0x7f6e254a6e2e (unknown) epoll_wait\n", + "[2025-01-27 16:40:55,344 E 114171 114171] logging.cc:440: @ 0x7f6e253c3520 (unknown) (unknown)\n" + ] + } + ], "source": [ "eval = algo.evaluate()" ] @@ -104,7 +119,7 @@ ], "metadata": { "kernelspec": { - "display_name": ".venv", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -118,7 +133,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.11" + "version": "3.10.12" } }, "nbformat": 4, diff --git a/src/primaite/notebooks/Using-Episode-Schedules.ipynb b/src/primaite/notebooks/Using-Episode-Schedules.ipynb index d08ca67b..311fe4fb 100644 --- a/src/primaite/notebooks/Using-Episode-Schedules.ipynb +++ b/src/primaite/notebooks/Using-Episode-Schedules.ipynb @@ -48,6 +48,7 @@ "from primaite.session.environment import PrimaiteGymEnv\n", "from primaite import PRIMAITE_PATHS\n", "from prettytable import PrettyTable\n", + "from primaite.game.agent.scripted_agents import probabilistic_agent, data_manipulation_bot\n", "scenario_path = PRIMAITE_PATHS.user_config_path / \"example_config/scenario_with_placeholders\"" ] }, @@ -409,21 +410,9 @@ ], "metadata": { "kernelspec": { - "display_name": "venv", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.11" } }, "nbformat": 4, diff --git a/src/primaite/notebooks/multi-processing.ipynb b/src/primaite/notebooks/multi-processing.ipynb index ad386f34..798bf3ff 100644 --- a/src/primaite/notebooks/multi-processing.ipynb +++ b/src/primaite/notebooks/multi-processing.ipynb @@ -29,7 +29,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 30, "metadata": {}, "outputs": [], "source": [ @@ -37,13 +37,13 @@ "from stable_baselines3 import PPO\n", "from stable_baselines3.common.utils import set_random_seed\n", "from stable_baselines3.common.vec_env import SubprocVecEnv\n", - "\n", - "from primaite.session.environment import PrimaiteGymEnv\n" + "from primaite.session.environment import PrimaiteGymEnv\n", + "from primaite.game.agent.scripted_agents import probabilistic_agent\n" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 31, "metadata": {}, "outputs": [], "source": [ @@ -52,7 +52,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 32, "metadata": {}, "outputs": [], "source": [ @@ -69,7 +69,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 33, "metadata": {}, "outputs": [], "source": [ @@ -90,7 +90,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 36, "metadata": {}, "outputs": [], "source": [ @@ -138,7 +138,7 @@ ], "metadata": { "kernelspec": { - "display_name": ".venv", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -152,7 +152,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.11" + "version": "3.10.12" } }, "nbformat": 4, From 4fb4c5e0f9e26b2d0c797beec562b8f35344fe74 Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Tue, 28 Jan 2025 17:18:07 +0000 Subject: [PATCH 05/37] #3075: Fix error in config file. --- src/primaite/config/_package_data/data_manipulation.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/primaite/config/_package_data/data_manipulation.yaml b/src/primaite/config/_package_data/data_manipulation.yaml index b0d5d087..fc48c8e6 100644 --- a/src/primaite/config/_package_data/data_manipulation.yaml +++ b/src/primaite/config/_package_data/data_manipulation.yaml @@ -81,7 +81,7 @@ agents: action: node_application_execute options: node_name: client_1 - application_name: WebBrowser + application_name: DatabaseClient reward_function: reward_components: From 32de95917e636c8febd08d38aa41492e7e02c606 Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Mon, 3 Feb 2025 11:33:21 +0000 Subject: [PATCH 06/37] #3075: Add repeat parameter to config file. --- .../notebooks/Data-Manipulation-E2E-Demonstration.ipynb | 3 +-- .../applications/red_applications/data_manipulation_bot.py | 2 ++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb b/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb index 3818bb18..4a34619f 100644 --- a/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb +++ b/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb @@ -405,8 +405,7 @@ "from primaite.session.environment import PrimaiteGymEnv\n", "from primaite.game.agent.interface import AgentHistoryItem\n", "import yaml\n", - "from pprint import pprint\n", - "from primaite.game.agent.scripted_agents import probabilistic_agent, data_manipulation_bot\n" + "from pprint import pprint\n" ] }, { diff --git a/src/primaite/simulator/system/applications/red_applications/data_manipulation_bot.py b/src/primaite/simulator/system/applications/red_applications/data_manipulation_bot.py index 392cdfba..79ef1e67 100644 --- a/src/primaite/simulator/system/applications/red_applications/data_manipulation_bot.py +++ b/src/primaite/simulator/system/applications/red_applications/data_manipulation_bot.py @@ -52,6 +52,7 @@ class DataManipulationBot(Application, identifier="DataManipulationBot"): payload: str = "DELETE" port_scan_p_of_success: float = 0.1 data_manipulation_p_of_success: float = 0.1 + repeat: bool = True config: "DataManipulationBot.ConfigSchema" = Field(default_factory=lambda: DataManipulationBot.ConfigSchema()) @@ -76,6 +77,7 @@ class DataManipulationBot(Application, identifier="DataManipulationBot"): self.payload = self.config.payload self.port_scan_p_of_success = self.config.port_scan_p_of_success self.data_manipulation_p_of_success = self.config.data_manipulation_p_of_success + self.repeat = self.config.repeat def describe_state(self) -> Dict: """ From d24f9872ffac89b6cf28830c073c598ef214cb2f Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Mon, 3 Feb 2025 17:06:12 +0000 Subject: [PATCH 07/37] #3075: C2C-E2E-Demo internal YAML changes. --- ...ommand-and-Control-E2E-Demonstration.ipynb | 1966 +++++++++++++++-- 1 file changed, 1747 insertions(+), 219 deletions(-) diff --git a/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb b/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb index 625510b8..2c1a94c8 100644 --- a/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb +++ b/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb @@ -13,16 +13,31 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-02-03 16:04:19,828: Performing the PrimAITE first-time setup...\n", + "2025-02-03 16:04:19,829: Building the PrimAITE app directories...\n", + "2025-02-03 16:04:19,829: Building primaite_config.yaml...\n", + "2025-02-03 16:04:19,829: Rebuilding the demo notebooks...\n", + "/home/nick/primaite/4.0.0a1-dev/notebooks/example_notebooks/Command-and-Control-E2E-Demonstration.ipynb\n", + "2025-02-03 16:04:19,831: Reset example notebook: /home/nick/primaite/4.0.0a1-dev/notebooks/example_notebooks/Command-and-Control-E2E-Demonstration.ipynb\n", + "2025-02-03 16:04:19,836: Rebuilding the example notebooks...\n", + "2025-02-03 16:04:19,840: PrimAITE setup complete!\n" + ] + } + ], "source": [ "!primaite setup" ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -36,8 +51,7 @@ "from primaite.simulator.system.applications.red_applications.c2.abstract_c2 import C2Command\n", "from primaite.simulator.system.applications.red_applications.ransomware_script import RansomwareScript\n", "from primaite.simulator.network.hardware.nodes.host.computer import Computer\n", - "from primaite.simulator.network.hardware.nodes.host.server import Server\n", - "from primaite.game.agent.scripted_agents import probabilistic_agent" + "from primaite.simulator.network.hardware.nodes.host.server import Server" ] }, { @@ -53,7 +67,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -63,24 +77,24 @@ " type: ProxyAgent\n", "\n", " action_space:\n", - " options:\n", - " nodes:\n", - " - node_name: web_server\n", - " applications:\n", - " - application_name: C2Beacon\n", - " - node_name: client_1\n", - " applications:\n", - " - application_name: C2Server\n", - " max_folders_per_node: 1\n", - " max_files_per_folder: 1\n", - " max_services_per_node: 2\n", - " max_nics_per_node: 8\n", - " max_acl_rules: 10\n", - " ip_list:\n", - " - 192.168.1.21\n", - " - 192.168.1.14\n", - " wildcard_list:\n", - " - 0.0.0.1\n", + " # options:\n", + " # nodes:\n", + " # - node_name: web_server\n", + " # applications:\n", + " # - application_name: C2Beacon\n", + " # - node_name: client_1\n", + " # applications:\n", + " # - application_name: C2Server\n", + " # max_folders_per_node: 1\n", + " # max_files_per_folder: 1\n", + " # max_services_per_node: 2\n", + " # max_nics_per_node: 8\n", + " # max_acl_rules: 10\n", + " # ip_list:\n", + " # - 192.168.1.21\n", + " # - 192.168.1.14\n", + " # wildcard_list:\n", + " # - 0.0.0.1\n", " action_map:\n", " 0:\n", " action: do_nothing\n", @@ -88,30 +102,30 @@ " 1:\n", " action: node_application_install\n", " options:\n", - " node_id: 0\n", + " node_name: web_server\n", " application_name: C2Beacon\n", " 2:\n", " action: configure_c2_beacon\n", " options:\n", - " node_id: 0\n", - " config:\n", - " c2_server_ip_address: 192.168.10.21\n", - " keep_alive_frequency:\n", - " masquerade_protocol:\n", - " masquerade_port:\n", + " node_name: web_server\n", + " # config:\n", + " c2_server_ip_address: 192.168.10.21\n", + " # keep_alive_frequency: 10\n", + " # masquerade_protocol: TCP\n", + " # masquerade_port: DNS\n", " 3:\n", " action: node_application_execute\n", " options:\n", - " node_id: 0\n", - " application_id: 0\n", + " node_name: web_server\n", + " application_name: C2Beacon\n", " 4:\n", " action: c2_server_terminal_command\n", " options:\n", - " node_id: 1\n", + " node_name: client_1\n", " ip_address:\n", - " account:\n", - " username: admin\n", - " password: admin\n", + " # account:\n", + " username: admin\n", + " password: admin\n", " commands:\n", " -\n", " - software_manager\n", @@ -121,44 +135,44 @@ " 5:\n", " action: c2_server_ransomware_configure\n", " options:\n", - " node_id: 1\n", - " config:\n", - " server_ip_address: 192.168.1.14\n", - " payload: ENCRYPT\n", + " node_name: client_1\n", + " # config:\n", + " server_ip_address: 192.168.1.14\n", + " payload: ENCRYPT\n", " 6:\n", " action: c2_server_data_exfiltrate\n", " options:\n", - " node_id: 1\n", + " node_name: client_1\n", " target_file_name: \"database.db\"\n", " target_folder_name: \"database\"\n", " exfiltration_folder_name: \"spoils\"\n", " target_ip_address: 192.168.1.14\n", - " account:\n", - " username: admin\n", - " password: admin\n", + " # account:\n", + " username: admin\n", + " password: admin\n", "\n", " 7:\n", " action: c2_server_ransomware_launch\n", " options:\n", - " node_id: 1\n", + " node_name: client_1\n", " 8:\n", " action: configure_c2_beacon\n", " options:\n", - " node_id: 0\n", - " config:\n", - " c2_server_ip_address: 192.168.10.21\n", - " keep_alive_frequency: 10\n", - " masquerade_protocol: TCP\n", - " masquerade_port: DNS\n", + " node_name: web_server\n", + " # config:\n", + " c2_server_ip_address: 192.168.10.21\n", + " # keep_alive_frequency: 10\n", + " # masquerade_protocol: TCP\n", + " # masquerade_port: DNS\n", " 9:\n", " action: configure_c2_beacon\n", " options:\n", - " node_id: 0\n", - " config:\n", - " c2_server_ip_address: 192.168.10.22\n", - " keep_alive_frequency:\n", - " masquerade_protocol:\n", - " masquerade_port:\n", + " node_name: web_server\n", + " # config:\n", + " c2_server_ip_address: 192.168.10.22\n", + " # keep_alive_frequency: 10\n", + " # masquerade_protocol: TCP\n", + " # masquerade_port: DNS\n", "\n", " reward_function:\n", " reward_components:\n", @@ -169,9 +183,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-03 16:04:24,734: PrimaiteGymEnv RNG seed = None\n" + ] + } + ], "source": [ "with open(data_manipulation_config_path()) as f:\n", " cfg = yaml.safe_load(f)\n", @@ -197,9 +219,35 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+--------------------------------------------------------------------------------------+\n", + "| client_1 Software Manager |\n", + "+---------------------+-------------+-----------------+--------------+------+----------+\n", + "| Name | Type | Operating State | Health State | Port | Protocol |\n", + "+---------------------+-------------+-----------------+--------------+------+----------+\n", + "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", + "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", + "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", + "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", + "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", + "| NMAP | Application | RUNNING | GOOD | None | none |\n", + "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", + "| UserManager | Service | RUNNING | GOOD | None | none |\n", + "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", + "| FTPClient | Service | RUNNING | GOOD | 21 | tcp |\n", + "| DataManipulationBot | Application | RUNNING | GOOD | None | none |\n", + "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", + "| C2Server | Application | RUNNING | GOOD | None | tcp |\n", + "+---------------------+-------------+-----------------+--------------+------+----------+\n" + ] + } + ], "source": [ "client_1: Computer = env.game.simulation.network.get_node_by_hostname(\"client_1\")\n", "client_1.software_manager.install(C2Server)\n", @@ -249,9 +297,34 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+-------------------------------------------------------------------------------------+\n", + "| web_server Software Manager |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| Name | Type | Operating State | Health State | Port | Protocol |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", + "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", + "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", + "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", + "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", + "| NMAP | Application | RUNNING | GOOD | None | none |\n", + "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", + "| UserManager | Service | RUNNING | GOOD | None | none |\n", + "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", + "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", + "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", + "| C2Beacon | Application | INSTALLING | UNUSED | None | tcp |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n" + ] + } + ], "source": [ "env.step(1)\n", "web_server: Computer = env.game.simulation.network.get_node_by_hostname(\"web_server\")\n", @@ -291,9 +364,41 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+-------------------------------------------------------------------------------------+\n", + "| web_server Software Manager |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| Name | Type | Operating State | Health State | Port | Protocol |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", + "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", + "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", + "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", + "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", + "| NMAP | Application | RUNNING | GOOD | None | none |\n", + "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", + "| UserManager | Service | RUNNING | GOOD | None | none |\n", + "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", + "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", + "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", + "| C2Beacon | Application | RUNNING | GOOD | None | tcp |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", + "| C2Beacon Running Status |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| False | 192.168.10.21 | 0 | 5 | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n" + ] + } + ], "source": [ "env.step(2)\n", "c2_beacon: C2Beacon = web_server.software_manager.software[\"C2Beacon\"]\n", @@ -332,18 +437,54 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "(0,\n", + " 0.0,\n", + " False,\n", + " False,\n", + " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=2, action='node_application_execute', parameters={'node_name': 'web_server', 'application_name': 'C2Beacon'}, request=['network', 'node', 'web_server', 'application', 'C2Beacon', 'execute'], response=RequestResponse(status='success', data={}), reward=0.0, reward_info={})}})" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "env.step(3)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", + "| C2Beacon Running Status |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| True | 192.168.10.21 | 1 | 5 | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "+-----------------------------------------------------------------------------------------------------+\n", + "| C2Server Running Status |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n", + "| True | 192.168.1.12 | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n" + ] + } + ], "source": [ "c2_beacon.show()\n", "c2_server.show()" @@ -406,18 +547,59 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "(0,\n", + " 0.0,\n", + " False,\n", + " False,\n", + " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=3, action='c2_server_terminal_command', parameters={'node_name': 'client_1', 'ip_address': None, 'username': 'admin', 'password': 'admin', 'commands': [['software_manager', 'application', 'install', 'RansomwareScript']]}, request=['network', 'node', 'client_1', 'application', 'C2Server', 'terminal_command', {'commands': [['software_manager', 'application', 'install', 'RansomwareScript']], 'ip_address': None, 'username': 'admin', 'password': 'admin'}], response=RequestResponse(status='success', data={0: RequestResponse(status='success', data={})}), reward=0.0, reward_info={})}})" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "env.step(4)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+--------------------------------------------------------------------------------------+\n", + "| client_1 Software Manager |\n", + "+---------------------+-------------+-----------------+--------------+------+----------+\n", + "| Name | Type | Operating State | Health State | Port | Protocol |\n", + "+---------------------+-------------+-----------------+--------------+------+----------+\n", + "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", + "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", + "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", + "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", + "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", + "| NMAP | Application | RUNNING | GOOD | None | none |\n", + "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", + "| UserManager | Service | RUNNING | GOOD | None | none |\n", + "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", + "| FTPClient | Service | RUNNING | GOOD | 21 | tcp |\n", + "| DataManipulationBot | Application | RUNNING | GOOD | None | none |\n", + "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", + "| C2Server | Application | RUNNING | GOOD | None | tcp |\n", + "+---------------------+-------------+-----------------+--------------+------+----------+\n" + ] + } + ], "source": [ "client_1.software_manager.show()" ] @@ -456,18 +638,66 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "(0,\n", + " 0.0,\n", + " False,\n", + " False,\n", + " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=4, action='c2_server_ransomware_configure', parameters={'node_name': 'client_1', 'server_ip_address': '192.168.1.14', 'payload': 'ENCRYPT'}, request=['network', 'node', 'client_1', 'application', 'C2Server', 'ransomware_configure', {'server_ip_address': '192.168.1.14', 'server_password': None, 'payload': 'ENCRYPT'}], response=RequestResponse(status='success', data={}), reward=0.0, reward_info={})}})" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "env.step(5)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+-------------------------------------------------------------------------------------+\n", + "| web_server Software Manager |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| Name | Type | Operating State | Health State | Port | Protocol |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", + "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", + "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", + "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", + "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", + "| NMAP | Application | RUNNING | GOOD | None | none |\n", + "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", + "| UserManager | Service | RUNNING | GOOD | None | none |\n", + "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", + "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", + "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", + "| C2Beacon | Application | RUNNING | GOOD | None | tcp |\n", + "| RansomwareScript | Application | RUNNING | GOOD | None | none |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "+------------------------------------+\n", + "| RansomwareScript Running Status |\n", + "+--------------------------+---------+\n", + "| Target Server IP Address | Payload |\n", + "+--------------------------+---------+\n", + "| 192.168.1.14 | ENCRYPT |\n", + "+--------------------------+---------+\n" + ] + } + ], "source": [ "ransomware_script: RansomwareScript = web_server.software_manager.software[\"RansomwareScript\"]\n", "web_server.software_manager.show()\n", @@ -513,18 +743,48 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "(0,\n", + " 0.0,\n", + " False,\n", + " False,\n", + " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=5, action='c2_server_data_exfiltrate', parameters={'node_name': 'client_1', 'target_file_name': 'database.db', 'target_folder_name': 'database', 'exfiltration_folder_name': 'spoils', 'target_ip_address': '192.168.1.14', 'username': 'admin', 'password': 'admin'}, request=['network', 'node', 'client_1', 'application', 'C2Server', 'exfiltrate', {'target_file_name': 'database.db', 'target_folder_name': 'database', 'exfiltration_folder_name': 'spoils', 'target_ip_address': '192.168.1.14', 'username': 'admin', 'password': 'admin'}], response=RequestResponse(status='success', data={}), reward=0.0, reward_info={})}})" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "env.step(6)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+--------------------------------------------------------------------------------+\n", + "| client_1 File System |\n", + "+--------------------+---------+---------------+-----------------------+---------+\n", + "| File Path | Size | Health status | Visible health status | Deleted |\n", + "+--------------------+---------+---------------+-----------------------+---------+\n", + "| root | 0 B | GOOD | NONE | False |\n", + "| spoils/database.db | 4.77 MB | GOOD | NONE | False |\n", + "+--------------------+---------+---------------+-----------------------+---------+\n" + ] + } + ], "source": [ "client_1: Computer = env.game.simulation.network.get_node_by_hostname(\"client_1\")\n", "client_1.software_manager.file_system.show(full=True)" @@ -532,9 +792,25 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+---------------------------------------------------------------------------------+\n", + "| web_server File System |\n", + "+---------------------+---------+---------------+-----------------------+---------+\n", + "| File Path | Size | Health status | Visible health status | Deleted |\n", + "+---------------------+---------+---------------+-----------------------+---------+\n", + "| primaite/index.html | 15.0 KB | GOOD | NONE | False |\n", + "| root | 0 B | GOOD | NONE | False |\n", + "| spoils/database.db | 4.77 MB | GOOD | NONE | False |\n", + "+---------------------+---------+---------------+-----------------------+---------+\n" + ] + } + ], "source": [ "web_server: Computer = env.game.simulation.network.get_node_by_hostname(\"web_server\")\n", "web_server.software_manager.file_system.show(full=True)" @@ -571,18 +847,48 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "(0,\n", + " 0.0,\n", + " False,\n", + " False,\n", + " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=6, action='c2_server_ransomware_launch', parameters={'node_name': 'client_1'}, request=['network', 'node', 'client_1', 'application', 'C2Server', 'ransomware_launch'], response=RequestResponse(status='success', data={}), reward=0.0, reward_info={})}})" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "env.step(7)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----------------------------------------------------------------------------------+\n", + "| database_server File System |\n", + "+----------------------+---------+---------------+-----------------------+---------+\n", + "| File Path | Size | Health status | Visible health status | Deleted |\n", + "+----------------------+---------+---------------+-----------------------+---------+\n", + "| database/database.db | 4.77 MB | CORRUPT | NONE | False |\n", + "| root | 0 B | GOOD | NONE | False |\n", + "+----------------------+---------+---------------+-----------------------+---------+\n" + ] + } + ], "source": [ "database_server: Server = env.game.simulation.network.get_node_by_hostname(\"database_server\")\n", "database_server.software_manager.file_system.show(full=True)" @@ -601,7 +907,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "metadata": {}, "outputs": [], "source": [ @@ -697,52 +1003,52 @@ " 1:\n", " action: node_application_remove\n", " options:\n", - " node_id: 0\n", + " node_name: web_server\n", " application_name: C2Beacon\n", " 2:\n", " action: node_shutdown\n", " options:\n", - " node_id: 0\n", + " node_name: web_server\n", " 3:\n", " action: router_acl_add_rule\n", " options:\n", " target_router: router_1\n", " position: 1\n", - " permission: 2\n", - " source_ip_id: 2\n", - " dest_ip_id: 3\n", - " source_port_id: 2\n", - " dest_port_id: 2\n", - " protocol_id: 1\n", - " source_wildcard_id: 0\n", - " dest_wildcard_id: 0\n", + " permission: DENY\n", + " src_ip: 192.168.10.21\n", + " dst_ip: 192.168.1.12\n", + " src_port: HTTP\n", + " dst_port: HTTP\n", + " protocol_name: ALL\n", + " src_wildcard: NONE\n", + " dst_wildcard: NONE\n", "\n", "\n", - " options:\n", - " nodes:\n", - " - node_name: web_server\n", - " applications:\n", - " - application_name: C2Beacon\n", + " # options:\n", + " # nodes:\n", + " # - node_name: web_server\n", + " # applications:\n", + " # - application_name: C2Beacon\n", "\n", - " - node_name: database_server\n", - " folders:\n", - " - folder_name: database\n", - " files:\n", - " - file_name: database.db\n", - " services:\n", - " - service_name: DatabaseService\n", - " - node_name: router_1\n", + " # - node_name: database_server\n", + " # folders:\n", + " # - folder_name: database\n", + " # files:\n", + " # - file_name: database.db\n", + " # services:\n", + " # - service_name: DatabaseService\n", + " # - node_name: router_1\n", "\n", - " max_folders_per_node: 2\n", - " max_files_per_folder: 2\n", - " max_services_per_node: 2\n", - " max_nics_per_node: 8\n", - " max_acl_rules: 10\n", - " ip_list:\n", - " - 192.168.10.21\n", - " - 192.168.1.12\n", - " wildcard_list:\n", - " - 0.0.0.1\n", + " # max_folders_per_node: 2\n", + " # max_files_per_folder: 2\n", + " # max_services_per_node: 2\n", + " # max_nics_per_node: 8\n", + " # max_acl_rules: 10\n", + " # ip_list:\n", + " # - 192.168.10.21\n", + " # - 192.168.1.12\n", + " # wildcard_list:\n", + " # - 0.0.0.1\n", " reward_function:\n", " reward_components:\n", " - type: DUMMY\n", @@ -755,9 +1061,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-03 16:04:26,020: PrimaiteGymEnv RNG seed = None\n" + ] + } + ], "source": [ "with open(data_manipulation_config_path()) as f:\n", " cfg = yaml.safe_load(f)\n", @@ -771,7 +1085,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 21, "metadata": {}, "outputs": [], "source": [ @@ -814,9 +1128,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 22, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-03 16:04:26,440: Resetting environment, episode 0, avg. reward: 0.0\n", + "2025-02-03 16:04:26,445: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-03/16-04-20/agent_actions/episode_0.json\n" + ] + } + ], "source": [ "# Resetting the environment and capturing the default observation space.\n", "blue_env.reset()\n", @@ -825,9 +1148,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 23, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Setting up the C2 Suite via the simulation API.\n", "\n", @@ -848,7 +1182,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 24, "metadata": {}, "outputs": [], "source": [ @@ -858,9 +1192,26 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 25, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 2\n", + "root['NODES']['HOST0']['APPLICATIONS'][1]['operating_status']: 0 -> 1\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", + "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", + "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" + ] + } + ], "source": [ "display_obs_diffs(default_obs, c2_configuration_obs, blue_env.game.step_counter)" ] @@ -880,9 +1231,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 26, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "RequestResponse(status='success', data={0: RequestResponse(status='success', data={})})" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Installing RansomwareScript via C2 Terminal Commands\n", "ransomware_install_command = {\"commands\":[[\"software_manager\", \"application\", \"install\", \"RansomwareScript\"]],\n", @@ -893,9 +1255,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 27, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "RequestResponse(status='success', data={})" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Configuring the RansomwareScript\n", "ransomware_config = {\"server_ip_address\": \"192.168.1.14\", \"payload\": \"ENCRYPT\"}\n", @@ -904,7 +1277,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 28, "metadata": {}, "outputs": [], "source": [ @@ -914,9 +1287,28 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 29, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 7\n", + "root['NODES']['HOST0']['APPLICATIONS'][1]['operating_status']: 0 -> 1\n", + "root['NODES']['HOST0']['APPLICATIONS'][2]['operating_status']: 0 -> 3\n", + "root['NODES']['HOST0']['users']['local_login']: 0 -> 1\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", + "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", + "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" + ] + } + ], "source": [ "display_obs_diffs(default_obs, c2_ransomware_obs, env.game.step_counter)" ] @@ -932,7 +1324,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 30, "metadata": {}, "outputs": [], "source": [ @@ -948,16 +1340,27 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 31, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "RequestResponse(status='success', data={})" + ] + }, + "execution_count": 31, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "c2_server.send_command(given_command=C2Command.DATA_EXFILTRATION, command_options=exfil_options)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 32, "metadata": {}, "outputs": [], "source": [ @@ -966,9 +1369,22 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 33, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 7\n", + "root['NODES']['HOST0']['APPLICATIONS'][2]['operating_status']: 3 -> 1\n", + "root['NODES']['HOST1']['users']['remote_sessions']: 0 -> 1\n" + ] + } + ], "source": [ "display_obs_diffs(c2_ransomware_obs, c2_exfil_obs, env.game.step_counter)" ] @@ -984,9 +1400,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 34, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "RequestResponse(status='success', data={})" + ] + }, + "execution_count": 34, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Configuring the RansomwareScript\n", "ransomware_config = {\"server_ip_address\": \"192.168.1.14\", \"payload\": \"ENCRYPT\"}\n", @@ -995,9 +1422,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 35, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "RequestResponse(status='success', data={})" + ] + }, + "execution_count": 35, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Waiting for the ransomware to finish installing and then launching the RansomwareScript.\n", "blue_env.step(0)\n", @@ -1006,7 +1444,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 36, "metadata": {}, "outputs": [], "source": [ @@ -1016,9 +1454,30 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 37, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 6\n", + "root['NODES']['HOST0']['APPLICATIONS'][2]['operating_status']: 3 -> 1\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['NODES']['HOST1']['users']['remote_sessions']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" + ] + } + ], "source": [ "display_obs_diffs(c2_ransomware_obs, c2_final_obs, blue_env.game.step_counter)" ] @@ -1034,7 +1493,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 38, "metadata": {}, "outputs": [], "source": [ @@ -1066,16 +1525,200 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 39, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-03 16:04:27,571: Resetting environment, episode 1, avg. reward: 0.0\n", + "2025-02-03 16:04:27,574: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-03/16-04-20/agent_actions/episode_1.json\n" + ] + }, + { + "data": { + "text/plain": [ + "({'NODES': {'HOST0': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 1,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'HOST1': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 1,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'HOST2': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 0,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'HOST3': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 0,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'ROUTER0': {'ACL': {1: {'position': 0,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 2: {'position': 1,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 3: {'position': 2,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 4: {'position': 3,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 5: {'position': 4,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 6: {'position': 5,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 7: {'position': 6,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 8: {'position': 7,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 9: {'position': 8,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 10: {'position': 9,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0}},\n", + " 'PORTS': {1: {'operating_status': 1},\n", + " 2: {'operating_status': 1},\n", + " 3: {'operating_status': 2}},\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}}},\n", + " 'LINKS': {1: {'PROTOCOLS': {'ALL': 1}},\n", + " 2: {'PROTOCOLS': {'ALL': 1}},\n", + " 3: {'PROTOCOLS': {'ALL': 0}},\n", + " 4: {'PROTOCOLS': {'ALL': 1}},\n", + " 5: {'PROTOCOLS': {'ALL': 1}},\n", + " 6: {'PROTOCOLS': {'ALL': 1}},\n", + " 7: {'PROTOCOLS': {'ALL': 1}},\n", + " 8: {'PROTOCOLS': {'ALL': 1}},\n", + " 9: {'PROTOCOLS': {'ALL': 1}},\n", + " 10: {'PROTOCOLS': {'ALL': 0}}},\n", + " 'ICS': 0},\n", + " {})" + ] + }, + "execution_count": 39, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "blue_env.reset()" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 40, "metadata": {}, "outputs": [], "source": [ @@ -1094,7 +1737,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 41, "metadata": {}, "outputs": [], "source": [ @@ -1111,9 +1754,33 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 42, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+-------------------------------------------------------------------------------------+\n", + "| web_server Software Manager |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| Name | Type | Operating State | Health State | Port | Protocol |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", + "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", + "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", + "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", + "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", + "| NMAP | Application | RUNNING | GOOD | None | none |\n", + "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", + "| UserManager | Service | RUNNING | GOOD | None | none |\n", + "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", + "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", + "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n" + ] + } + ], "source": [ "blue_env.step(0)\n", "web_server.software_manager.show()" @@ -1121,9 +1788,26 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 43, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 3\n", + "root['NODES']['HOST0']['APPLICATIONS'][1]['operating_status']: 1 -> 0\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", + "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", + "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" + ] + } + ], "source": [ "display_obs_diffs(pre_blue_action_obs, post_blue_action_obs, blue_env.game.step_counter)" ] @@ -1137,9 +1821,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 44, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "RequestResponse(status='failure', data={'Reason': 'Command sent to the C2 Beacon but no response was ever received.'})" + ] + }, + "execution_count": 44, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Attempting to install the C2 RansomwareScript\n", "ransomware_install_command = {\"commands\":[[\"software_manager\", \"application\", \"install\", \"RansomwareScript\"]],\n", @@ -1161,16 +1856,200 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 45, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-03 16:04:28,041: Resetting environment, episode 2, avg. reward: 0.0\n", + "2025-02-03 16:04:28,045: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-03/16-04-20/agent_actions/episode_2.json\n" + ] + }, + { + "data": { + "text/plain": [ + "({'NODES': {'HOST0': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 1,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'HOST1': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 1,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'HOST2': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 0,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'HOST3': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 0,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'ROUTER0': {'ACL': {1: {'position': 0,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 2: {'position': 1,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 3: {'position': 2,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 4: {'position': 3,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 5: {'position': 4,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 6: {'position': 5,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 7: {'position': 6,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 8: {'position': 7,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 9: {'position': 8,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 10: {'position': 9,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0}},\n", + " 'PORTS': {1: {'operating_status': 1},\n", + " 2: {'operating_status': 1},\n", + " 3: {'operating_status': 2}},\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}}},\n", + " 'LINKS': {1: {'PROTOCOLS': {'ALL': 1}},\n", + " 2: {'PROTOCOLS': {'ALL': 1}},\n", + " 3: {'PROTOCOLS': {'ALL': 0}},\n", + " 4: {'PROTOCOLS': {'ALL': 1}},\n", + " 5: {'PROTOCOLS': {'ALL': 1}},\n", + " 6: {'PROTOCOLS': {'ALL': 1}},\n", + " 7: {'PROTOCOLS': {'ALL': 1}},\n", + " 8: {'PROTOCOLS': {'ALL': 1}},\n", + " 9: {'PROTOCOLS': {'ALL': 1}},\n", + " 10: {'PROTOCOLS': {'ALL': 0}}},\n", + " 'ICS': 0},\n", + " {})" + ] + }, + "execution_count": 45, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "blue_env.reset()" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 46, "metadata": {}, "outputs": [], "source": [ @@ -1189,7 +2068,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 47, "metadata": {}, "outputs": [], "source": [ @@ -1206,9 +2085,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 48, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "NodeOperatingState.SHUTTING_DOWN\n" + ] + } + ], "source": [ "web_server = blue_env.game.simulation.network.get_node_by_hostname(\"web_server\")\n", "print(web_server.operating_state)" @@ -1216,18 +2103,48 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 49, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 2\n", + "root['NODES']['HOST0']['operating_status']: 1 -> 4\n", + "root['NODES']['HOST0']['APPLICATIONS'][1]['operating_status']: 1 -> 0\n", + "root['NODES']['HOST0']['NICS'][1]['nic_status']: 1 -> 0\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", + "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", + "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" + ] + } + ], "source": [ "display_obs_diffs(pre_blue_action_obs, post_blue_action_obs, blue_env.game.step_counter)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 50, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "RequestResponse(status='failure', data={'Reason': 'Command sent to the C2 Beacon but no response was ever received.'})" + ] + }, + "execution_count": 50, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Attempting to install the C2 RansomwareScript\n", "ransomware_install_command = {\"commands\":[\"software_manager\", \"application\", \"install\", \"RansomwareScript\"],\n", @@ -1251,16 +2168,200 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 51, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-03 16:04:28,560: Resetting environment, episode 3, avg. reward: 0.0\n", + "2025-02-03 16:04:28,564: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-03/16-04-20/agent_actions/episode_3.json\n" + ] + }, + { + "data": { + "text/plain": [ + "({'NODES': {'HOST0': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 1,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'HOST1': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 1,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'HOST2': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 0,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'HOST3': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 0,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'ROUTER0': {'ACL': {1: {'position': 0,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 2: {'position': 1,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 3: {'position': 2,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 4: {'position': 3,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 5: {'position': 4,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 6: {'position': 5,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 7: {'position': 6,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 8: {'position': 7,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 9: {'position': 8,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 10: {'position': 9,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0}},\n", + " 'PORTS': {1: {'operating_status': 1},\n", + " 2: {'operating_status': 1},\n", + " 3: {'operating_status': 2}},\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}}},\n", + " 'LINKS': {1: {'PROTOCOLS': {'ALL': 1}},\n", + " 2: {'PROTOCOLS': {'ALL': 1}},\n", + " 3: {'PROTOCOLS': {'ALL': 0}},\n", + " 4: {'PROTOCOLS': {'ALL': 1}},\n", + " 5: {'PROTOCOLS': {'ALL': 1}},\n", + " 6: {'PROTOCOLS': {'ALL': 1}},\n", + " 7: {'PROTOCOLS': {'ALL': 1}},\n", + " 8: {'PROTOCOLS': {'ALL': 1}},\n", + " 9: {'PROTOCOLS': {'ALL': 1}},\n", + " 10: {'PROTOCOLS': {'ALL': 0}}},\n", + " 'ICS': 0},\n", + " {})" + ] + }, + "execution_count": 51, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "blue_env.reset()" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 52, "metadata": {}, "outputs": [], "source": [ @@ -1279,7 +2380,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 53, "metadata": {}, "outputs": [], "source": [ @@ -1296,11 +2397,44 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 54, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+---------------------------------------------------------------+\n", + "| router_1 Network Interfaces |\n", + "+------+-------------------+-----------------+-------+----------+\n", + "| Port | MAC Address | Address | Speed | Status |\n", + "+------+-------------------+-----------------+-------+----------+\n", + "| 1 | dd:6e:95:d4:3f:74 | 192.168.1.1/24 | 100.0 | Enabled |\n", + "| 2 | 8b:79:07:fc:69:2c | 192.168.10.1/24 | 100.0 | Enabled |\n", + "| 3 | 1f:fd:c4:ae:7a:00 | 127.0.0.1/8 | 100.0 | Disabled |\n", + "| 4 | 7b:e3:bf:4b:76:e8 | 127.0.0.1/8 | 100.0 | Disabled |\n", + "| 5 | 4f:37:b0:6b:5d:44 | 127.0.0.1/8 | 100.0 | Disabled |\n", + "+------+-------------------+-----------------+-------+----------+\n", + "+------------------------------------------------------------------------------------------------------------------------+\n", + "| router_1 Access Control List |\n", + "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n", + "| Index | Action | Protocol | Src IP | Src Wildcard | Src Port | Dst IP | Dst Wildcard | Dst Port | Matched |\n", + "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n", + "| 1 | DENY | ANY | 192.168.10.21 | ANY | 80 | 192.168.1.12 | ANY | 80 | 0 |\n", + "| 18 | PERMIT | ANY | ANY | ANY | 5432 | ANY | ANY | 5432 | 0 |\n", + "| 19 | PERMIT | ANY | ANY | ANY | 53 | ANY | ANY | 53 | 0 |\n", + "| 20 | PERMIT | ANY | ANY | ANY | 21 | ANY | ANY | 21 | 0 |\n", + "| 21 | PERMIT | ANY | ANY | ANY | 80 | ANY | ANY | 80 | 4 |\n", + "| 22 | PERMIT | ANY | ANY | ANY | 219 | ANY | ANY | 219 | 10 |\n", + "| 23 | PERMIT | icmp | ANY | ANY | ANY | ANY | ANY | ANY | 0 |\n", + "| 24 | DENY | ANY | ANY | ANY | ANY | ANY | ANY | ANY | 0 |\n", + "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n" + ] + } + ], "source": [ "router_1: Router = blue_env.game.simulation.network.get_node_by_hostname(\"router_1\")\n", + "router_1.show()\n", "router_1.acl.show()" ] }, @@ -1313,9 +2447,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 55, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "RequestResponse(status='failure', data={'Reason': 'Command sent to the C2 Beacon but no response was ever received.'})" + ] + }, + "execution_count": 55, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "blue_env.step(0)\n", "\n", @@ -1326,9 +2471,30 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 56, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+------------------------------------------------------------------------------------------------------------------------+\n", + "| router_1 Access Control List |\n", + "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n", + "| Index | Action | Protocol | Src IP | Src Wildcard | Src Port | Dst IP | Dst Wildcard | Dst Port | Matched |\n", + "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n", + "| 1 | DENY | ANY | 192.168.10.21 | ANY | 80 | 192.168.1.12 | ANY | 80 | 2 |\n", + "| 18 | PERMIT | ANY | ANY | ANY | 5432 | ANY | ANY | 5432 | 0 |\n", + "| 19 | PERMIT | ANY | ANY | ANY | 53 | ANY | ANY | 53 | 0 |\n", + "| 20 | PERMIT | ANY | ANY | ANY | 21 | ANY | ANY | 21 | 0 |\n", + "| 21 | PERMIT | ANY | ANY | ANY | 80 | ANY | ANY | 80 | 4 |\n", + "| 22 | PERMIT | ANY | ANY | ANY | 219 | ANY | ANY | 219 | 10 |\n", + "| 23 | PERMIT | icmp | ANY | ANY | ANY | ANY | ANY | ANY | 0 |\n", + "| 24 | DENY | ANY | ANY | ANY | ANY | ANY | ANY | ANY | 0 |\n", + "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n" + ] + } + ], "source": [ "router_1.acl.show()" ] @@ -1342,18 +2508,58 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 57, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+-------------------------------------------------------------------------------------+\n", + "| web_server Software Manager |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| Name | Type | Operating State | Health State | Port | Protocol |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", + "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", + "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", + "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", + "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", + "| NMAP | Application | RUNNING | GOOD | None | none |\n", + "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", + "| UserManager | Service | RUNNING | GOOD | None | none |\n", + "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", + "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", + "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", + "| C2Beacon | Application | RUNNING | GOOD | None | tcp |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n" + ] + } + ], "source": [ "web_server.software_manager.show()" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 58, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----------------------------------------------------------------------------------+\n", + "| database_server File System |\n", + "+----------------------+---------+---------------+-----------------------+---------+\n", + "| File Path | Size | Health status | Visible health status | Deleted |\n", + "+----------------------+---------+---------------+-----------------------+---------+\n", + "| database/database.db | 4.77 MB | GOOD | NONE | False |\n", + "| root | 0 B | GOOD | NONE | False |\n", + "+----------------------+---------+---------------+-----------------------+---------+\n" + ] + } + ], "source": [ "database_server: Server = blue_env.game.simulation.network.get_node_by_hostname(\"database_server\")\n", "database_server.software_manager.file_system.show(full=True)" @@ -1361,9 +2567,33 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 59, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 3\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", + "root['NODES']['ROUTER0']['ACL'][1]['permission']: 0 -> 2\n", + "root['NODES']['ROUTER0']['ACL'][1]['source_ip_id']: 0 -> 7\n", + "root['NODES']['ROUTER0']['ACL'][1]['source_wildcard_id']: 0 -> 1\n", + "root['NODES']['ROUTER0']['ACL'][1]['source_port_id']: 0 -> 2\n", + "root['NODES']['ROUTER0']['ACL'][1]['dest_ip_id']: 0 -> 3\n", + "root['NODES']['ROUTER0']['ACL'][1]['dest_wildcard_id']: 0 -> 1\n", + "root['NODES']['ROUTER0']['ACL'][1]['dest_port_id']: 0 -> 2\n", + "root['NODES']['ROUTER0']['ACL'][1]['protocol_id']: 0 -> 1\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", + "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", + "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" + ] + } + ], "source": [ "display_obs_diffs(pre_blue_action_obs, post_blue_action_obs, blue_env.game.step_counter)" ] @@ -1427,9 +2657,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 60, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-03 16:04:29,610: PrimaiteGymEnv RNG seed = None\n" + ] + } + ], "source": [ "with open(data_manipulation_config_path()) as f:\n", " cfg = yaml.safe_load(f)\n", @@ -1450,7 +2688,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 61, "metadata": {}, "outputs": [], "source": [ @@ -1478,9 +2716,30 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 62, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", + "| C2Beacon Running Status |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| False | None | 0 | 5 | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "+-----------------------------------------------------------------------------------------------------+\n", + "| C2Server Running Status |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n", + "| False | None | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n" + ] + } + ], "source": [ "env.step(2) # Agent Action Equivalent to c2_beacon.configure(c2_server_ip_address=\"192.168.10.21\")\n", "env.step(3) # Agent action Equivalent to c2_beacon.establish()\n", @@ -1497,9 +2756,30 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 63, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", + "| C2Beacon Running Status |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| False | None | 0 | 5 | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "+-----------------------------------------------------------------------------------------------------+\n", + "| C2Server Running Status |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n", + "| False | None | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n" + ] + } + ], "source": [ "env.step(9) # Equivalent of to c2_beacon.configure(c2_server_ip_address=\"192.168.10.22\")\n", "env.step(3)\n", @@ -1517,9 +2797,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 64, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+-----------------------------------------------------------------------------------------------------+\n", + "| C2Server Running Status |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n", + "| False | None | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n" + ] + } + ], "source": [ "for i in range(6):\n", " env.step(0)\n", @@ -1542,9 +2836,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 65, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-03 16:04:30,011: PrimaiteGymEnv RNG seed = None\n" + ] + } + ], "source": [ "with open(data_manipulation_config_path()) as f:\n", " cfg = yaml.safe_load(f)\n", @@ -1560,7 +2862,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 66, "metadata": {}, "outputs": [], "source": [ @@ -1584,9 +2886,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 67, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", + "| C2Beacon Running Status |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| True | 192.168.10.21 | 0 | 5 | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n" + ] + } + ], "source": [ "c2_beacon.configure(c2_server_ip_address=\"192.168.10.21\")\n", "c2_beacon.establish()\n", @@ -1604,9 +2920,72 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 68, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 4\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 5\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 6\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 7\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 8\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 9\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 10\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 11\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 12\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 13\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" + ] + } + ], "source": [ "for i in range(10):\n", " keep_alive_obs, _, _, _, _ = blue_config_env.step(0)\n", @@ -1622,9 +3001,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 69, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", + "| C2Beacon Running Status |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| True | 192.168.10.21 | 0 | 1 | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n" + ] + } + ], "source": [ "c2_beacon.configure(c2_server_ip_address=\"192.168.10.21\", keep_alive_frequency=1)\n", "c2_beacon.establish()\n", @@ -1640,9 +3033,40 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 70, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 14\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 15\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" + ] + } + ], "source": [ "# Comparing the OBS of the default frequency to a timestep frequency of 1\n", "for i in range(2):\n", @@ -1661,9 +3085,52 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 71, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 16\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 17\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 18\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 19\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 20\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 21\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 22\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" + ] + } + ], "source": [ "c2_beacon.configure(c2_server_ip_address=\"192.168.10.21\", keep_alive_frequency=7)\n", "\n", @@ -1700,9 +3167,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 72, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-03 16:04:30,864: Resetting environment, episode 0, avg. reward: 0.0\n", + "2025-02-03 16:04:30,867: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-03/16-04-20/agent_actions/episode_0.json\n" + ] + } + ], "source": [ "blue_config_env.reset()\n", "\n", @@ -1724,9 +3200,28 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 73, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 5\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" + ] + } + ], "source": [ "# Capturing default C2 Traffic\n", "for i in range(3):\n", @@ -1744,9 +3239,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 74, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", + "| C2Beacon Running Status |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| True | 192.168.10.21 | 0 | 5 | udp | 53 |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n" + ] + } + ], "source": [ "from primaite.utils.validation.ip_protocol import PROTOCOL_LOOKUP\n", "from primaite.utils.validation.port import PORT_LOOKUP\n", @@ -1759,9 +3268,28 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 75, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 10\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 1 -> 0\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 1 -> 0\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['udp'][53]['inbound']: 0 -> 1\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['udp'][53]['outbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 1 -> 0\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 1 -> 0\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['udp'][53]['inbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['udp'][53]['outbound']: 0 -> 1\n" + ] + } + ], "source": [ "# Capturing UDP C2 Traffic\n", "for i in range(5):\n", From 454bd61fb27e2e8b8702d86c0e83806deae7cd0c Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Tue, 4 Feb 2025 10:50:53 +0000 Subject: [PATCH 08/37] #3075: Update internal YAML fragments in C2C-E2E-Demo notebook. --- ...ommand-and-Control-E2E-Demonstration.ipynb | 1802 ++--------------- 1 file changed, 137 insertions(+), 1665 deletions(-) diff --git a/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb b/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb index 2c1a94c8..c2a87e45 100644 --- a/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb +++ b/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb @@ -13,24 +13,9 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-02-03 16:04:19,828: Performing the PrimAITE first-time setup...\n", - "2025-02-03 16:04:19,829: Building the PrimAITE app directories...\n", - "2025-02-03 16:04:19,829: Building primaite_config.yaml...\n", - "2025-02-03 16:04:19,829: Rebuilding the demo notebooks...\n", - "/home/nick/primaite/4.0.0a1-dev/notebooks/example_notebooks/Command-and-Control-E2E-Demonstration.ipynb\n", - "2025-02-03 16:04:19,831: Reset example notebook: /home/nick/primaite/4.0.0a1-dev/notebooks/example_notebooks/Command-and-Control-E2E-Demonstration.ipynb\n", - "2025-02-03 16:04:19,836: Rebuilding the example notebooks...\n", - "2025-02-03 16:04:19,840: PrimAITE setup complete!\n" - ] - } - ], + "outputs": [], "source": [ "!primaite setup" ] @@ -183,17 +168,9 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-03 16:04:24,734: PrimaiteGymEnv RNG seed = None\n" - ] - } - ], + "outputs": [], "source": [ "with open(data_manipulation_config_path()) as f:\n", " cfg = yaml.safe_load(f)\n", @@ -219,35 +196,9 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+--------------------------------------------------------------------------------------+\n", - "| client_1 Software Manager |\n", - "+---------------------+-------------+-----------------+--------------+------+----------+\n", - "| Name | Type | Operating State | Health State | Port | Protocol |\n", - "+---------------------+-------------+-----------------+--------------+------+----------+\n", - "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", - "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", - "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", - "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", - "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", - "| NMAP | Application | RUNNING | GOOD | None | none |\n", - "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", - "| UserManager | Service | RUNNING | GOOD | None | none |\n", - "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", - "| FTPClient | Service | RUNNING | GOOD | 21 | tcp |\n", - "| DataManipulationBot | Application | RUNNING | GOOD | None | none |\n", - "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", - "| C2Server | Application | RUNNING | GOOD | None | tcp |\n", - "+---------------------+-------------+-----------------+--------------+------+----------+\n" - ] - } - ], + "outputs": [], "source": [ "client_1: Computer = env.game.simulation.network.get_node_by_hostname(\"client_1\")\n", "client_1.software_manager.install(C2Server)\n", @@ -297,34 +248,9 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+-------------------------------------------------------------------------------------+\n", - "| web_server Software Manager |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| Name | Type | Operating State | Health State | Port | Protocol |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", - "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", - "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", - "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", - "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", - "| NMAP | Application | RUNNING | GOOD | None | none |\n", - "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", - "| UserManager | Service | RUNNING | GOOD | None | none |\n", - "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", - "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", - "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", - "| C2Beacon | Application | INSTALLING | UNUSED | None | tcp |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n" - ] - } - ], + "outputs": [], "source": [ "env.step(1)\n", "web_server: Computer = env.game.simulation.network.get_node_by_hostname(\"web_server\")\n", @@ -364,41 +290,9 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+-------------------------------------------------------------------------------------+\n", - "| web_server Software Manager |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| Name | Type | Operating State | Health State | Port | Protocol |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", - "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", - "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", - "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", - "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", - "| NMAP | Application | RUNNING | GOOD | None | none |\n", - "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", - "| UserManager | Service | RUNNING | GOOD | None | none |\n", - "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", - "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", - "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", - "| C2Beacon | Application | RUNNING | GOOD | None | tcp |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", - "| C2Beacon Running Status |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| False | 192.168.10.21 | 0 | 5 | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n" - ] - } - ], + "outputs": [], "source": [ "env.step(2)\n", "c2_beacon: C2Beacon = web_server.software_manager.software[\"C2Beacon\"]\n", @@ -437,54 +331,18 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(0,\n", - " 0.0,\n", - " False,\n", - " False,\n", - " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=2, action='node_application_execute', parameters={'node_name': 'web_server', 'application_name': 'C2Beacon'}, request=['network', 'node', 'web_server', 'application', 'C2Beacon', 'execute'], response=RequestResponse(status='success', data={}), reward=0.0, reward_info={})}})" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "env.step(3)" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", - "| C2Beacon Running Status |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| True | 192.168.10.21 | 1 | 5 | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "+-----------------------------------------------------------------------------------------------------+\n", - "| C2Server Running Status |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n", - "| True | 192.168.1.12 | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n" - ] - } - ], + "outputs": [], "source": [ "c2_beacon.show()\n", "c2_server.show()" @@ -547,59 +405,18 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(0,\n", - " 0.0,\n", - " False,\n", - " False,\n", - " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=3, action='c2_server_terminal_command', parameters={'node_name': 'client_1', 'ip_address': None, 'username': 'admin', 'password': 'admin', 'commands': [['software_manager', 'application', 'install', 'RansomwareScript']]}, request=['network', 'node', 'client_1', 'application', 'C2Server', 'terminal_command', {'commands': [['software_manager', 'application', 'install', 'RansomwareScript']], 'ip_address': None, 'username': 'admin', 'password': 'admin'}], response=RequestResponse(status='success', data={0: RequestResponse(status='success', data={})}), reward=0.0, reward_info={})}})" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "env.step(4)" ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+--------------------------------------------------------------------------------------+\n", - "| client_1 Software Manager |\n", - "+---------------------+-------------+-----------------+--------------+------+----------+\n", - "| Name | Type | Operating State | Health State | Port | Protocol |\n", - "+---------------------+-------------+-----------------+--------------+------+----------+\n", - "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", - "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", - "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", - "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", - "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", - "| NMAP | Application | RUNNING | GOOD | None | none |\n", - "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", - "| UserManager | Service | RUNNING | GOOD | None | none |\n", - "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", - "| FTPClient | Service | RUNNING | GOOD | 21 | tcp |\n", - "| DataManipulationBot | Application | RUNNING | GOOD | None | none |\n", - "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", - "| C2Server | Application | RUNNING | GOOD | None | tcp |\n", - "+---------------------+-------------+-----------------+--------------+------+----------+\n" - ] - } - ], + "outputs": [], "source": [ "client_1.software_manager.show()" ] @@ -638,66 +455,18 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(0,\n", - " 0.0,\n", - " False,\n", - " False,\n", - " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=4, action='c2_server_ransomware_configure', parameters={'node_name': 'client_1', 'server_ip_address': '192.168.1.14', 'payload': 'ENCRYPT'}, request=['network', 'node', 'client_1', 'application', 'C2Server', 'ransomware_configure', {'server_ip_address': '192.168.1.14', 'server_password': None, 'payload': 'ENCRYPT'}], response=RequestResponse(status='success', data={}), reward=0.0, reward_info={})}})" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "env.step(5)" ] }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+-------------------------------------------------------------------------------------+\n", - "| web_server Software Manager |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| Name | Type | Operating State | Health State | Port | Protocol |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", - "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", - "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", - "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", - "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", - "| NMAP | Application | RUNNING | GOOD | None | none |\n", - "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", - "| UserManager | Service | RUNNING | GOOD | None | none |\n", - "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", - "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", - "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", - "| C2Beacon | Application | RUNNING | GOOD | None | tcp |\n", - "| RansomwareScript | Application | RUNNING | GOOD | None | none |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "+------------------------------------+\n", - "| RansomwareScript Running Status |\n", - "+--------------------------+---------+\n", - "| Target Server IP Address | Payload |\n", - "+--------------------------+---------+\n", - "| 192.168.1.14 | ENCRYPT |\n", - "+--------------------------+---------+\n" - ] - } - ], + "outputs": [], "source": [ "ransomware_script: RansomwareScript = web_server.software_manager.software[\"RansomwareScript\"]\n", "web_server.software_manager.show()\n", @@ -743,48 +512,18 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(0,\n", - " 0.0,\n", - " False,\n", - " False,\n", - " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=5, action='c2_server_data_exfiltrate', parameters={'node_name': 'client_1', 'target_file_name': 'database.db', 'target_folder_name': 'database', 'exfiltration_folder_name': 'spoils', 'target_ip_address': '192.168.1.14', 'username': 'admin', 'password': 'admin'}, request=['network', 'node', 'client_1', 'application', 'C2Server', 'exfiltrate', {'target_file_name': 'database.db', 'target_folder_name': 'database', 'exfiltration_folder_name': 'spoils', 'target_ip_address': '192.168.1.14', 'username': 'admin', 'password': 'admin'}], response=RequestResponse(status='success', data={}), reward=0.0, reward_info={})}})" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "env.step(6)" ] }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+--------------------------------------------------------------------------------+\n", - "| client_1 File System |\n", - "+--------------------+---------+---------------+-----------------------+---------+\n", - "| File Path | Size | Health status | Visible health status | Deleted |\n", - "+--------------------+---------+---------------+-----------------------+---------+\n", - "| root | 0 B | GOOD | NONE | False |\n", - "| spoils/database.db | 4.77 MB | GOOD | NONE | False |\n", - "+--------------------+---------+---------------+-----------------------+---------+\n" - ] - } - ], + "outputs": [], "source": [ "client_1: Computer = env.game.simulation.network.get_node_by_hostname(\"client_1\")\n", "client_1.software_manager.file_system.show(full=True)" @@ -792,25 +531,9 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+---------------------------------------------------------------------------------+\n", - "| web_server File System |\n", - "+---------------------+---------+---------------+-----------------------+---------+\n", - "| File Path | Size | Health status | Visible health status | Deleted |\n", - "+---------------------+---------+---------------+-----------------------+---------+\n", - "| primaite/index.html | 15.0 KB | GOOD | NONE | False |\n", - "| root | 0 B | GOOD | NONE | False |\n", - "| spoils/database.db | 4.77 MB | GOOD | NONE | False |\n", - "+---------------------+---------+---------------+-----------------------+---------+\n" - ] - } - ], + "outputs": [], "source": [ "web_server: Computer = env.game.simulation.network.get_node_by_hostname(\"web_server\")\n", "web_server.software_manager.file_system.show(full=True)" @@ -847,48 +570,18 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(0,\n", - " 0.0,\n", - " False,\n", - " False,\n", - " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=6, action='c2_server_ransomware_launch', parameters={'node_name': 'client_1'}, request=['network', 'node', 'client_1', 'application', 'C2Server', 'ransomware_launch'], response=RequestResponse(status='success', data={}), reward=0.0, reward_info={})}})" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "env.step(7)" ] }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+----------------------------------------------------------------------------------+\n", - "| database_server File System |\n", - "+----------------------+---------+---------------+-----------------------+---------+\n", - "| File Path | Size | Health status | Visible health status | Deleted |\n", - "+----------------------+---------+---------------+-----------------------+---------+\n", - "| database/database.db | 4.77 MB | CORRUPT | NONE | False |\n", - "| root | 0 B | GOOD | NONE | False |\n", - "+----------------------+---------+---------------+-----------------------+---------+\n" - ] - } - ], + "outputs": [], "source": [ "database_server: Server = env.game.simulation.network.get_node_by_hostname(\"database_server\")\n", "database_server.software_manager.file_system.show(full=True)" @@ -907,7 +600,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1025,30 +718,30 @@ "\n", "\n", " # options:\n", - " # nodes:\n", - " # - node_name: web_server\n", - " # applications:\n", - " # - application_name: C2Beacon\n", + " # # nodes:\n", + " # node_name: web_server\n", + " # # applications:\n", + " # application_name: C2Beacon\n", "\n", - " # - node_name: database_server\n", - " # folders:\n", - " # - folder_name: database\n", - " # files:\n", - " # - file_name: database.db\n", - " # services:\n", - " # - service_name: DatabaseService\n", - " # - node_name: router_1\n", + " # node_name: database_server\n", + " # folders:\n", + " # - folder_name: database\n", + " # files:\n", + " # - file_name: database.db\n", + " # services:\n", + " # - service_name: DatabaseService\n", + " # node_name: router_1\n", "\n", - " # max_folders_per_node: 2\n", - " # max_files_per_folder: 2\n", - " # max_services_per_node: 2\n", - " # max_nics_per_node: 8\n", - " # max_acl_rules: 10\n", - " # ip_list:\n", - " # - 192.168.10.21\n", - " # - 192.168.1.12\n", - " # wildcard_list:\n", - " # - 0.0.0.1\n", + " # max_folders_per_node: 2\n", + " # max_files_per_folder: 2\n", + " # max_services_per_node: 2\n", + " # max_nics_per_node: 8\n", + " # max_acl_rules: 10\n", + " # ip_list:\n", + " # - 192.168.10.21\n", + " # - 192.168.1.12\n", + " # wildcard_list:\n", + " # - 0.0.0.1\n", " reward_function:\n", " reward_components:\n", " - type: DUMMY\n", @@ -1061,17 +754,9 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-03 16:04:26,020: PrimaiteGymEnv RNG seed = None\n" - ] - } - ], + "outputs": [], "source": [ "with open(data_manipulation_config_path()) as f:\n", " cfg = yaml.safe_load(f)\n", @@ -1128,18 +813,9 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-03 16:04:26,440: Resetting environment, episode 0, avg. reward: 0.0\n", - "2025-02-03 16:04:26,445: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-03/16-04-20/agent_actions/episode_0.json\n" - ] - } - ], + "outputs": [], "source": [ "# Resetting the environment and capturing the default observation space.\n", "blue_env.reset()\n", @@ -1148,20 +824,9 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 23, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Setting up the C2 Suite via the simulation API.\n", "\n", @@ -1192,26 +857,9 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 2\n", - "root['NODES']['HOST0']['APPLICATIONS'][1]['operating_status']: 0 -> 1\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", - "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", - "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" - ] - } - ], + "outputs": [], "source": [ "display_obs_diffs(default_obs, c2_configuration_obs, blue_env.game.step_counter)" ] @@ -1231,20 +879,9 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "RequestResponse(status='success', data={0: RequestResponse(status='success', data={})})" - ] - }, - "execution_count": 26, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Installing RansomwareScript via C2 Terminal Commands\n", "ransomware_install_command = {\"commands\":[[\"software_manager\", \"application\", \"install\", \"RansomwareScript\"]],\n", @@ -1255,20 +892,9 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "RequestResponse(status='success', data={})" - ] - }, - "execution_count": 27, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Configuring the RansomwareScript\n", "ransomware_config = {\"server_ip_address\": \"192.168.1.14\", \"payload\": \"ENCRYPT\"}\n", @@ -1287,28 +913,9 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 7\n", - "root['NODES']['HOST0']['APPLICATIONS'][1]['operating_status']: 0 -> 1\n", - "root['NODES']['HOST0']['APPLICATIONS'][2]['operating_status']: 0 -> 3\n", - "root['NODES']['HOST0']['users']['local_login']: 0 -> 1\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", - "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", - "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" - ] - } - ], + "outputs": [], "source": [ "display_obs_diffs(default_obs, c2_ransomware_obs, env.game.step_counter)" ] @@ -1340,20 +947,9 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "RequestResponse(status='success', data={})" - ] - }, - "execution_count": 31, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "c2_server.send_command(given_command=C2Command.DATA_EXFILTRATION, command_options=exfil_options)" ] @@ -1369,22 +965,9 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 7\n", - "root['NODES']['HOST0']['APPLICATIONS'][2]['operating_status']: 3 -> 1\n", - "root['NODES']['HOST1']['users']['remote_sessions']: 0 -> 1\n" - ] - } - ], + "outputs": [], "source": [ "display_obs_diffs(c2_ransomware_obs, c2_exfil_obs, env.game.step_counter)" ] @@ -1400,20 +983,9 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "RequestResponse(status='success', data={})" - ] - }, - "execution_count": 34, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Configuring the RansomwareScript\n", "ransomware_config = {\"server_ip_address\": \"192.168.1.14\", \"payload\": \"ENCRYPT\"}\n", @@ -1422,20 +994,9 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "RequestResponse(status='success', data={})" - ] - }, - "execution_count": 35, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Waiting for the ransomware to finish installing and then launching the RansomwareScript.\n", "blue_env.step(0)\n", @@ -1454,30 +1015,9 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 6\n", - "root['NODES']['HOST0']['APPLICATIONS'][2]['operating_status']: 3 -> 1\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['NODES']['HOST1']['users']['remote_sessions']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" - ] - } - ], + "outputs": [], "source": [ "display_obs_diffs(c2_ransomware_obs, c2_final_obs, blue_env.game.step_counter)" ] @@ -1525,193 +1065,9 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-03 16:04:27,571: Resetting environment, episode 1, avg. reward: 0.0\n", - "2025-02-03 16:04:27,574: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-03/16-04-20/agent_actions/episode_1.json\n" - ] - }, - { - "data": { - "text/plain": [ - "({'NODES': {'HOST0': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 1,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'HOST1': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 1,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'HOST2': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 0,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'HOST3': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 0,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'ROUTER0': {'ACL': {1: {'position': 0,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 2: {'position': 1,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 3: {'position': 2,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 4: {'position': 3,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 5: {'position': 4,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 6: {'position': 5,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 7: {'position': 6,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 8: {'position': 7,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 9: {'position': 8,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 10: {'position': 9,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0}},\n", - " 'PORTS': {1: {'operating_status': 1},\n", - " 2: {'operating_status': 1},\n", - " 3: {'operating_status': 2}},\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}}},\n", - " 'LINKS': {1: {'PROTOCOLS': {'ALL': 1}},\n", - " 2: {'PROTOCOLS': {'ALL': 1}},\n", - " 3: {'PROTOCOLS': {'ALL': 0}},\n", - " 4: {'PROTOCOLS': {'ALL': 1}},\n", - " 5: {'PROTOCOLS': {'ALL': 1}},\n", - " 6: {'PROTOCOLS': {'ALL': 1}},\n", - " 7: {'PROTOCOLS': {'ALL': 1}},\n", - " 8: {'PROTOCOLS': {'ALL': 1}},\n", - " 9: {'PROTOCOLS': {'ALL': 1}},\n", - " 10: {'PROTOCOLS': {'ALL': 0}}},\n", - " 'ICS': 0},\n", - " {})" - ] - }, - "execution_count": 39, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "blue_env.reset()" ] @@ -1754,33 +1110,9 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+-------------------------------------------------------------------------------------+\n", - "| web_server Software Manager |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| Name | Type | Operating State | Health State | Port | Protocol |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", - "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", - "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", - "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", - "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", - "| NMAP | Application | RUNNING | GOOD | None | none |\n", - "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", - "| UserManager | Service | RUNNING | GOOD | None | none |\n", - "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", - "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", - "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n" - ] - } - ], + "outputs": [], "source": [ "blue_env.step(0)\n", "web_server.software_manager.show()" @@ -1788,26 +1120,9 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 3\n", - "root['NODES']['HOST0']['APPLICATIONS'][1]['operating_status']: 1 -> 0\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", - "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", - "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" - ] - } - ], + "outputs": [], "source": [ "display_obs_diffs(pre_blue_action_obs, post_blue_action_obs, blue_env.game.step_counter)" ] @@ -1821,20 +1136,9 @@ }, { "cell_type": "code", - "execution_count": 44, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "RequestResponse(status='failure', data={'Reason': 'Command sent to the C2 Beacon but no response was ever received.'})" - ] - }, - "execution_count": 44, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Attempting to install the C2 RansomwareScript\n", "ransomware_install_command = {\"commands\":[[\"software_manager\", \"application\", \"install\", \"RansomwareScript\"]],\n", @@ -1856,193 +1160,9 @@ }, { "cell_type": "code", - "execution_count": 45, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-03 16:04:28,041: Resetting environment, episode 2, avg. reward: 0.0\n", - "2025-02-03 16:04:28,045: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-03/16-04-20/agent_actions/episode_2.json\n" - ] - }, - { - "data": { - "text/plain": [ - "({'NODES': {'HOST0': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 1,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'HOST1': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 1,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'HOST2': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 0,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'HOST3': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 0,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'ROUTER0': {'ACL': {1: {'position': 0,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 2: {'position': 1,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 3: {'position': 2,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 4: {'position': 3,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 5: {'position': 4,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 6: {'position': 5,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 7: {'position': 6,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 8: {'position': 7,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 9: {'position': 8,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 10: {'position': 9,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0}},\n", - " 'PORTS': {1: {'operating_status': 1},\n", - " 2: {'operating_status': 1},\n", - " 3: {'operating_status': 2}},\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}}},\n", - " 'LINKS': {1: {'PROTOCOLS': {'ALL': 1}},\n", - " 2: {'PROTOCOLS': {'ALL': 1}},\n", - " 3: {'PROTOCOLS': {'ALL': 0}},\n", - " 4: {'PROTOCOLS': {'ALL': 1}},\n", - " 5: {'PROTOCOLS': {'ALL': 1}},\n", - " 6: {'PROTOCOLS': {'ALL': 1}},\n", - " 7: {'PROTOCOLS': {'ALL': 1}},\n", - " 8: {'PROTOCOLS': {'ALL': 1}},\n", - " 9: {'PROTOCOLS': {'ALL': 1}},\n", - " 10: {'PROTOCOLS': {'ALL': 0}}},\n", - " 'ICS': 0},\n", - " {})" - ] - }, - "execution_count": 45, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "blue_env.reset()" ] @@ -2085,17 +1205,9 @@ }, { "cell_type": "code", - "execution_count": 48, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "NodeOperatingState.SHUTTING_DOWN\n" - ] - } - ], + "outputs": [], "source": [ "web_server = blue_env.game.simulation.network.get_node_by_hostname(\"web_server\")\n", "print(web_server.operating_state)" @@ -2103,48 +1215,18 @@ }, { "cell_type": "code", - "execution_count": 49, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 2\n", - "root['NODES']['HOST0']['operating_status']: 1 -> 4\n", - "root['NODES']['HOST0']['APPLICATIONS'][1]['operating_status']: 1 -> 0\n", - "root['NODES']['HOST0']['NICS'][1]['nic_status']: 1 -> 0\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", - "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", - "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" - ] - } - ], + "outputs": [], "source": [ "display_obs_diffs(pre_blue_action_obs, post_blue_action_obs, blue_env.game.step_counter)" ] }, { "cell_type": "code", - "execution_count": 50, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "RequestResponse(status='failure', data={'Reason': 'Command sent to the C2 Beacon but no response was ever received.'})" - ] - }, - "execution_count": 50, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Attempting to install the C2 RansomwareScript\n", "ransomware_install_command = {\"commands\":[\"software_manager\", \"application\", \"install\", \"RansomwareScript\"],\n", @@ -2168,193 +1250,9 @@ }, { "cell_type": "code", - "execution_count": 51, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-03 16:04:28,560: Resetting environment, episode 3, avg. reward: 0.0\n", - "2025-02-03 16:04:28,564: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-03/16-04-20/agent_actions/episode_3.json\n" - ] - }, - { - "data": { - "text/plain": [ - "({'NODES': {'HOST0': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 1,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'HOST1': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 1,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'HOST2': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 0,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'HOST3': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 0,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'ROUTER0': {'ACL': {1: {'position': 0,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 2: {'position': 1,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 3: {'position': 2,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 4: {'position': 3,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 5: {'position': 4,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 6: {'position': 5,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 7: {'position': 6,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 8: {'position': 7,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 9: {'position': 8,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 10: {'position': 9,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0}},\n", - " 'PORTS': {1: {'operating_status': 1},\n", - " 2: {'operating_status': 1},\n", - " 3: {'operating_status': 2}},\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}}},\n", - " 'LINKS': {1: {'PROTOCOLS': {'ALL': 1}},\n", - " 2: {'PROTOCOLS': {'ALL': 1}},\n", - " 3: {'PROTOCOLS': {'ALL': 0}},\n", - " 4: {'PROTOCOLS': {'ALL': 1}},\n", - " 5: {'PROTOCOLS': {'ALL': 1}},\n", - " 6: {'PROTOCOLS': {'ALL': 1}},\n", - " 7: {'PROTOCOLS': {'ALL': 1}},\n", - " 8: {'PROTOCOLS': {'ALL': 1}},\n", - " 9: {'PROTOCOLS': {'ALL': 1}},\n", - " 10: {'PROTOCOLS': {'ALL': 0}}},\n", - " 'ICS': 0},\n", - " {})" - ] - }, - "execution_count": 51, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "blue_env.reset()" ] @@ -2397,41 +1295,9 @@ }, { "cell_type": "code", - "execution_count": 54, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+---------------------------------------------------------------+\n", - "| router_1 Network Interfaces |\n", - "+------+-------------------+-----------------+-------+----------+\n", - "| Port | MAC Address | Address | Speed | Status |\n", - "+------+-------------------+-----------------+-------+----------+\n", - "| 1 | dd:6e:95:d4:3f:74 | 192.168.1.1/24 | 100.0 | Enabled |\n", - "| 2 | 8b:79:07:fc:69:2c | 192.168.10.1/24 | 100.0 | Enabled |\n", - "| 3 | 1f:fd:c4:ae:7a:00 | 127.0.0.1/8 | 100.0 | Disabled |\n", - "| 4 | 7b:e3:bf:4b:76:e8 | 127.0.0.1/8 | 100.0 | Disabled |\n", - "| 5 | 4f:37:b0:6b:5d:44 | 127.0.0.1/8 | 100.0 | Disabled |\n", - "+------+-------------------+-----------------+-------+----------+\n", - "+------------------------------------------------------------------------------------------------------------------------+\n", - "| router_1 Access Control List |\n", - "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n", - "| Index | Action | Protocol | Src IP | Src Wildcard | Src Port | Dst IP | Dst Wildcard | Dst Port | Matched |\n", - "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n", - "| 1 | DENY | ANY | 192.168.10.21 | ANY | 80 | 192.168.1.12 | ANY | 80 | 0 |\n", - "| 18 | PERMIT | ANY | ANY | ANY | 5432 | ANY | ANY | 5432 | 0 |\n", - "| 19 | PERMIT | ANY | ANY | ANY | 53 | ANY | ANY | 53 | 0 |\n", - "| 20 | PERMIT | ANY | ANY | ANY | 21 | ANY | ANY | 21 | 0 |\n", - "| 21 | PERMIT | ANY | ANY | ANY | 80 | ANY | ANY | 80 | 4 |\n", - "| 22 | PERMIT | ANY | ANY | ANY | 219 | ANY | ANY | 219 | 10 |\n", - "| 23 | PERMIT | icmp | ANY | ANY | ANY | ANY | ANY | ANY | 0 |\n", - "| 24 | DENY | ANY | ANY | ANY | ANY | ANY | ANY | ANY | 0 |\n", - "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n" - ] - } - ], + "outputs": [], "source": [ "router_1: Router = blue_env.game.simulation.network.get_node_by_hostname(\"router_1\")\n", "router_1.show()\n", @@ -2447,20 +1313,9 @@ }, { "cell_type": "code", - "execution_count": 55, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "RequestResponse(status='failure', data={'Reason': 'Command sent to the C2 Beacon but no response was ever received.'})" - ] - }, - "execution_count": 55, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "blue_env.step(0)\n", "\n", @@ -2471,30 +1326,9 @@ }, { "cell_type": "code", - "execution_count": 56, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+------------------------------------------------------------------------------------------------------------------------+\n", - "| router_1 Access Control List |\n", - "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n", - "| Index | Action | Protocol | Src IP | Src Wildcard | Src Port | Dst IP | Dst Wildcard | Dst Port | Matched |\n", - "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n", - "| 1 | DENY | ANY | 192.168.10.21 | ANY | 80 | 192.168.1.12 | ANY | 80 | 2 |\n", - "| 18 | PERMIT | ANY | ANY | ANY | 5432 | ANY | ANY | 5432 | 0 |\n", - "| 19 | PERMIT | ANY | ANY | ANY | 53 | ANY | ANY | 53 | 0 |\n", - "| 20 | PERMIT | ANY | ANY | ANY | 21 | ANY | ANY | 21 | 0 |\n", - "| 21 | PERMIT | ANY | ANY | ANY | 80 | ANY | ANY | 80 | 4 |\n", - "| 22 | PERMIT | ANY | ANY | ANY | 219 | ANY | ANY | 219 | 10 |\n", - "| 23 | PERMIT | icmp | ANY | ANY | ANY | ANY | ANY | ANY | 0 |\n", - "| 24 | DENY | ANY | ANY | ANY | ANY | ANY | ANY | ANY | 0 |\n", - "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n" - ] - } - ], + "outputs": [], "source": [ "router_1.acl.show()" ] @@ -2508,58 +1342,18 @@ }, { "cell_type": "code", - "execution_count": 57, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+-------------------------------------------------------------------------------------+\n", - "| web_server Software Manager |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| Name | Type | Operating State | Health State | Port | Protocol |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", - "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", - "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", - "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", - "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", - "| NMAP | Application | RUNNING | GOOD | None | none |\n", - "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", - "| UserManager | Service | RUNNING | GOOD | None | none |\n", - "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", - "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", - "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", - "| C2Beacon | Application | RUNNING | GOOD | None | tcp |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n" - ] - } - ], + "outputs": [], "source": [ "web_server.software_manager.show()" ] }, { "cell_type": "code", - "execution_count": 58, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+----------------------------------------------------------------------------------+\n", - "| database_server File System |\n", - "+----------------------+---------+---------------+-----------------------+---------+\n", - "| File Path | Size | Health status | Visible health status | Deleted |\n", - "+----------------------+---------+---------------+-----------------------+---------+\n", - "| database/database.db | 4.77 MB | GOOD | NONE | False |\n", - "| root | 0 B | GOOD | NONE | False |\n", - "+----------------------+---------+---------------+-----------------------+---------+\n" - ] - } - ], + "outputs": [], "source": [ "database_server: Server = blue_env.game.simulation.network.get_node_by_hostname(\"database_server\")\n", "database_server.software_manager.file_system.show(full=True)" @@ -2567,33 +1361,9 @@ }, { "cell_type": "code", - "execution_count": 59, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 3\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", - "root['NODES']['ROUTER0']['ACL'][1]['permission']: 0 -> 2\n", - "root['NODES']['ROUTER0']['ACL'][1]['source_ip_id']: 0 -> 7\n", - "root['NODES']['ROUTER0']['ACL'][1]['source_wildcard_id']: 0 -> 1\n", - "root['NODES']['ROUTER0']['ACL'][1]['source_port_id']: 0 -> 2\n", - "root['NODES']['ROUTER0']['ACL'][1]['dest_ip_id']: 0 -> 3\n", - "root['NODES']['ROUTER0']['ACL'][1]['dest_wildcard_id']: 0 -> 1\n", - "root['NODES']['ROUTER0']['ACL'][1]['dest_port_id']: 0 -> 2\n", - "root['NODES']['ROUTER0']['ACL'][1]['protocol_id']: 0 -> 1\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", - "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", - "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" - ] - } - ], + "outputs": [], "source": [ "display_obs_diffs(pre_blue_action_obs, post_blue_action_obs, blue_env.game.step_counter)" ] @@ -2657,17 +1427,9 @@ }, { "cell_type": "code", - "execution_count": 60, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-03 16:04:29,610: PrimaiteGymEnv RNG seed = None\n" - ] - } - ], + "outputs": [], "source": [ "with open(data_manipulation_config_path()) as f:\n", " cfg = yaml.safe_load(f)\n", @@ -2716,30 +1478,9 @@ }, { "cell_type": "code", - "execution_count": 62, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", - "| C2Beacon Running Status |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| False | None | 0 | 5 | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "+-----------------------------------------------------------------------------------------------------+\n", - "| C2Server Running Status |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n", - "| False | None | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n" - ] - } - ], + "outputs": [], "source": [ "env.step(2) # Agent Action Equivalent to c2_beacon.configure(c2_server_ip_address=\"192.168.10.21\")\n", "env.step(3) # Agent action Equivalent to c2_beacon.establish()\n", @@ -2756,30 +1497,9 @@ }, { "cell_type": "code", - "execution_count": 63, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", - "| C2Beacon Running Status |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| False | None | 0 | 5 | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "+-----------------------------------------------------------------------------------------------------+\n", - "| C2Server Running Status |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n", - "| False | None | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n" - ] - } - ], + "outputs": [], "source": [ "env.step(9) # Equivalent of to c2_beacon.configure(c2_server_ip_address=\"192.168.10.22\")\n", "env.step(3)\n", @@ -2797,23 +1517,9 @@ }, { "cell_type": "code", - "execution_count": 64, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+-----------------------------------------------------------------------------------------------------+\n", - "| C2Server Running Status |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n", - "| False | None | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n" - ] - } - ], + "outputs": [], "source": [ "for i in range(6):\n", " env.step(0)\n", @@ -2836,17 +1542,9 @@ }, { "cell_type": "code", - "execution_count": 65, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-03 16:04:30,011: PrimaiteGymEnv RNG seed = None\n" - ] - } - ], + "outputs": [], "source": [ "with open(data_manipulation_config_path()) as f:\n", " cfg = yaml.safe_load(f)\n", @@ -2886,23 +1584,9 @@ }, { "cell_type": "code", - "execution_count": 67, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", - "| C2Beacon Running Status |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| True | 192.168.10.21 | 0 | 5 | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n" - ] - } - ], + "outputs": [], "source": [ "c2_beacon.configure(c2_server_ip_address=\"192.168.10.21\")\n", "c2_beacon.establish()\n", @@ -2920,72 +1604,9 @@ }, { "cell_type": "code", - "execution_count": 68, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 4\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 5\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 6\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 7\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 8\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 9\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 10\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 11\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 12\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 13\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" - ] - } - ], + "outputs": [], "source": [ "for i in range(10):\n", " keep_alive_obs, _, _, _, _ = blue_config_env.step(0)\n", @@ -3001,23 +1622,9 @@ }, { "cell_type": "code", - "execution_count": 69, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", - "| C2Beacon Running Status |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| True | 192.168.10.21 | 0 | 1 | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n" - ] - } - ], + "outputs": [], "source": [ "c2_beacon.configure(c2_server_ip_address=\"192.168.10.21\", keep_alive_frequency=1)\n", "c2_beacon.establish()\n", @@ -3033,40 +1640,9 @@ }, { "cell_type": "code", - "execution_count": 70, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 14\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 15\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" - ] - } - ], + "outputs": [], "source": [ "# Comparing the OBS of the default frequency to a timestep frequency of 1\n", "for i in range(2):\n", @@ -3085,52 +1661,9 @@ }, { "cell_type": "code", - "execution_count": 71, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 16\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 17\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 18\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 19\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 20\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 21\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 22\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" - ] - } - ], + "outputs": [], "source": [ "c2_beacon.configure(c2_server_ip_address=\"192.168.10.21\", keep_alive_frequency=7)\n", "\n", @@ -3167,18 +1700,9 @@ }, { "cell_type": "code", - "execution_count": 72, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-03 16:04:30,864: Resetting environment, episode 0, avg. reward: 0.0\n", - "2025-02-03 16:04:30,867: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-03/16-04-20/agent_actions/episode_0.json\n" - ] - } - ], + "outputs": [], "source": [ "blue_config_env.reset()\n", "\n", @@ -3200,28 +1724,9 @@ }, { "cell_type": "code", - "execution_count": 73, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 5\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" - ] - } - ], + "outputs": [], "source": [ "# Capturing default C2 Traffic\n", "for i in range(3):\n", @@ -3239,23 +1744,9 @@ }, { "cell_type": "code", - "execution_count": 74, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", - "| C2Beacon Running Status |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| True | 192.168.10.21 | 0 | 5 | udp | 53 |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n" - ] - } - ], + "outputs": [], "source": [ "from primaite.utils.validation.ip_protocol import PROTOCOL_LOOKUP\n", "from primaite.utils.validation.port import PORT_LOOKUP\n", @@ -3268,28 +1759,9 @@ }, { "cell_type": "code", - "execution_count": 75, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 10\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 1 -> 0\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 1 -> 0\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['udp'][53]['inbound']: 0 -> 1\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['udp'][53]['outbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 1 -> 0\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 1 -> 0\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['udp'][53]['inbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['udp'][53]['outbound']: 0 -> 1\n" - ] - } - ], + "outputs": [], "source": [ "# Capturing UDP C2 Traffic\n", "for i in range(5):\n", From f7c6ee3df43bc1d46af90dbac0f321a1e911e524 Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Tue, 4 Feb 2025 15:45:19 +0000 Subject: [PATCH 09/37] #3075: C2C-E2E-Demo wildcard_list changes. --- ...ommand-and-Control-E2E-Demonstration.ipynb | 1822 +++++++++++++++-- 1 file changed, 1645 insertions(+), 177 deletions(-) diff --git a/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb b/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb index c2a87e45..b8d1423f 100644 --- a/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb +++ b/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb @@ -13,9 +13,24 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-02-04 15:37:30,830: Performing the PrimAITE first-time setup...\n", + "2025-02-04 15:37:30,830: Building the PrimAITE app directories...\n", + "2025-02-04 15:37:30,830: Building primaite_config.yaml...\n", + "2025-02-04 15:37:30,830: Rebuilding the demo notebooks...\n", + "/home/nick/primaite/4.0.0a1-dev/notebooks/example_notebooks/Command-and-Control-E2E-Demonstration.ipynb\n", + "2025-02-04 15:37:30,832: Reset example notebook: /home/nick/primaite/4.0.0a1-dev/notebooks/example_notebooks/Command-and-Control-E2E-Demonstration.ipynb\n", + "2025-02-04 15:37:30,838: Rebuilding the example notebooks...\n", + "2025-02-04 15:37:30,841: PrimAITE setup complete!\n" + ] + } + ], "source": [ "!primaite setup" ] @@ -62,24 +77,6 @@ " type: ProxyAgent\n", "\n", " action_space:\n", - " # options:\n", - " # nodes:\n", - " # - node_name: web_server\n", - " # applications:\n", - " # - application_name: C2Beacon\n", - " # - node_name: client_1\n", - " # applications:\n", - " # - application_name: C2Server\n", - " # max_folders_per_node: 1\n", - " # max_files_per_folder: 1\n", - " # max_services_per_node: 2\n", - " # max_nics_per_node: 8\n", - " # max_acl_rules: 10\n", - " # ip_list:\n", - " # - 192.168.1.21\n", - " # - 192.168.1.14\n", - " # wildcard_list:\n", - " # - 0.0.0.1\n", " action_map:\n", " 0:\n", " action: do_nothing\n", @@ -93,11 +90,7 @@ " action: configure_c2_beacon\n", " options:\n", " node_name: web_server\n", - " # config:\n", " c2_server_ip_address: 192.168.10.21\n", - " # keep_alive_frequency: 10\n", - " # masquerade_protocol: TCP\n", - " # masquerade_port: DNS\n", " 3:\n", " action: node_application_execute\n", " options:\n", @@ -108,7 +101,6 @@ " options:\n", " node_name: client_1\n", " ip_address:\n", - " # account:\n", " username: admin\n", " password: admin\n", " commands:\n", @@ -121,7 +113,6 @@ " action: c2_server_ransomware_configure\n", " options:\n", " node_name: client_1\n", - " # config:\n", " server_ip_address: 192.168.1.14\n", " payload: ENCRYPT\n", " 6:\n", @@ -132,7 +123,6 @@ " target_folder_name: \"database\"\n", " exfiltration_folder_name: \"spoils\"\n", " target_ip_address: 192.168.1.14\n", - " # account:\n", " username: admin\n", " password: admin\n", "\n", @@ -144,20 +134,12 @@ " action: configure_c2_beacon\n", " options:\n", " node_name: web_server\n", - " # config:\n", " c2_server_ip_address: 192.168.10.21\n", - " # keep_alive_frequency: 10\n", - " # masquerade_protocol: TCP\n", - " # masquerade_port: DNS\n", " 9:\n", " action: configure_c2_beacon\n", " options:\n", " node_name: web_server\n", - " # config:\n", " c2_server_ip_address: 192.168.10.22\n", - " # keep_alive_frequency: 10\n", - " # masquerade_protocol: TCP\n", - " # masquerade_port: DNS\n", "\n", " reward_function:\n", " reward_components:\n", @@ -168,9 +150,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-04 15:37:35,293: PrimaiteGymEnv RNG seed = None\n" + ] + } + ], "source": [ "with open(data_manipulation_config_path()) as f:\n", " cfg = yaml.safe_load(f)\n", @@ -196,9 +186,35 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+--------------------------------------------------------------------------------------+\n", + "| client_1 Software Manager |\n", + "+---------------------+-------------+-----------------+--------------+------+----------+\n", + "| Name | Type | Operating State | Health State | Port | Protocol |\n", + "+---------------------+-------------+-----------------+--------------+------+----------+\n", + "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", + "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", + "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", + "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", + "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", + "| NMAP | Application | RUNNING | GOOD | None | none |\n", + "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", + "| UserManager | Service | RUNNING | GOOD | None | none |\n", + "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", + "| FTPClient | Service | RUNNING | GOOD | 21 | tcp |\n", + "| DataManipulationBot | Application | RUNNING | GOOD | None | none |\n", + "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", + "| C2Server | Application | RUNNING | GOOD | None | tcp |\n", + "+---------------------+-------------+-----------------+--------------+------+----------+\n" + ] + } + ], "source": [ "client_1: Computer = env.game.simulation.network.get_node_by_hostname(\"client_1\")\n", "client_1.software_manager.install(C2Server)\n", @@ -248,9 +264,34 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+-------------------------------------------------------------------------------------+\n", + "| web_server Software Manager |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| Name | Type | Operating State | Health State | Port | Protocol |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", + "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", + "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", + "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", + "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", + "| NMAP | Application | RUNNING | GOOD | None | none |\n", + "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", + "| UserManager | Service | RUNNING | GOOD | None | none |\n", + "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", + "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", + "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", + "| C2Beacon | Application | INSTALLING | UNUSED | None | tcp |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n" + ] + } + ], "source": [ "env.step(1)\n", "web_server: Computer = env.game.simulation.network.get_node_by_hostname(\"web_server\")\n", @@ -290,9 +331,41 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+-------------------------------------------------------------------------------------+\n", + "| web_server Software Manager |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| Name | Type | Operating State | Health State | Port | Protocol |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", + "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", + "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", + "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", + "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", + "| NMAP | Application | RUNNING | GOOD | None | none |\n", + "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", + "| UserManager | Service | RUNNING | GOOD | None | none |\n", + "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", + "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", + "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", + "| C2Beacon | Application | RUNNING | GOOD | None | tcp |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", + "| C2Beacon Running Status |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| False | 192.168.10.21 | 0 | 5 | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n" + ] + } + ], "source": [ "env.step(2)\n", "c2_beacon: C2Beacon = web_server.software_manager.software[\"C2Beacon\"]\n", @@ -331,18 +404,54 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "(0,\n", + " 0.0,\n", + " False,\n", + " False,\n", + " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=2, action='node_application_execute', parameters={'node_name': 'web_server', 'application_name': 'C2Beacon'}, request=['network', 'node', 'web_server', 'application', 'C2Beacon', 'execute'], response=RequestResponse(status='success', data={}), reward=0.0, reward_info={})}})" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "env.step(3)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", + "| C2Beacon Running Status |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| True | 192.168.10.21 | 1 | 5 | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "+-----------------------------------------------------------------------------------------------------+\n", + "| C2Server Running Status |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n", + "| True | 192.168.1.12 | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n" + ] + } + ], "source": [ "c2_beacon.show()\n", "c2_server.show()" @@ -405,18 +514,59 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "(0,\n", + " 0.0,\n", + " False,\n", + " False,\n", + " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=3, action='c2_server_terminal_command', parameters={'node_name': 'client_1', 'ip_address': None, 'username': 'admin', 'password': 'admin', 'commands': [['software_manager', 'application', 'install', 'RansomwareScript']]}, request=['network', 'node', 'client_1', 'application', 'C2Server', 'terminal_command', {'commands': [['software_manager', 'application', 'install', 'RansomwareScript']], 'ip_address': None, 'username': 'admin', 'password': 'admin'}], response=RequestResponse(status='success', data={0: RequestResponse(status='success', data={})}), reward=0.0, reward_info={})}})" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "env.step(4)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+--------------------------------------------------------------------------------------+\n", + "| client_1 Software Manager |\n", + "+---------------------+-------------+-----------------+--------------+------+----------+\n", + "| Name | Type | Operating State | Health State | Port | Protocol |\n", + "+---------------------+-------------+-----------------+--------------+------+----------+\n", + "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", + "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", + "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", + "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", + "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", + "| NMAP | Application | RUNNING | GOOD | None | none |\n", + "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", + "| UserManager | Service | RUNNING | GOOD | None | none |\n", + "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", + "| FTPClient | Service | RUNNING | GOOD | 21 | tcp |\n", + "| DataManipulationBot | Application | RUNNING | GOOD | None | none |\n", + "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", + "| C2Server | Application | RUNNING | GOOD | None | tcp |\n", + "+---------------------+-------------+-----------------+--------------+------+----------+\n" + ] + } + ], "source": [ "client_1.software_manager.show()" ] @@ -455,18 +605,66 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "(0,\n", + " 0.0,\n", + " False,\n", + " False,\n", + " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=4, action='c2_server_ransomware_configure', parameters={'node_name': 'client_1', 'server_ip_address': '192.168.1.14', 'payload': 'ENCRYPT'}, request=['network', 'node', 'client_1', 'application', 'C2Server', 'ransomware_configure', {'server_ip_address': '192.168.1.14', 'server_password': None, 'payload': 'ENCRYPT'}], response=RequestResponse(status='success', data={}), reward=0.0, reward_info={})}})" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "env.step(5)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+-------------------------------------------------------------------------------------+\n", + "| web_server Software Manager |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| Name | Type | Operating State | Health State | Port | Protocol |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", + "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", + "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", + "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", + "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", + "| NMAP | Application | RUNNING | GOOD | None | none |\n", + "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", + "| UserManager | Service | RUNNING | GOOD | None | none |\n", + "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", + "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", + "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", + "| C2Beacon | Application | RUNNING | GOOD | None | tcp |\n", + "| RansomwareScript | Application | RUNNING | GOOD | None | none |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "+------------------------------------+\n", + "| RansomwareScript Running Status |\n", + "+--------------------------+---------+\n", + "| Target Server IP Address | Payload |\n", + "+--------------------------+---------+\n", + "| 192.168.1.14 | ENCRYPT |\n", + "+--------------------------+---------+\n" + ] + } + ], "source": [ "ransomware_script: RansomwareScript = web_server.software_manager.software[\"RansomwareScript\"]\n", "web_server.software_manager.show()\n", @@ -512,18 +710,48 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "(0,\n", + " 0.0,\n", + " False,\n", + " False,\n", + " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=5, action='c2_server_data_exfiltrate', parameters={'node_name': 'client_1', 'target_file_name': 'database.db', 'target_folder_name': 'database', 'exfiltration_folder_name': 'spoils', 'target_ip_address': '192.168.1.14', 'username': 'admin', 'password': 'admin'}, request=['network', 'node', 'client_1', 'application', 'C2Server', 'exfiltrate', {'target_file_name': 'database.db', 'target_folder_name': 'database', 'exfiltration_folder_name': 'spoils', 'target_ip_address': '192.168.1.14', 'username': 'admin', 'password': 'admin'}], response=RequestResponse(status='success', data={}), reward=0.0, reward_info={})}})" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "env.step(6)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+--------------------------------------------------------------------------------+\n", + "| client_1 File System |\n", + "+--------------------+---------+---------------+-----------------------+---------+\n", + "| File Path | Size | Health status | Visible health status | Deleted |\n", + "+--------------------+---------+---------------+-----------------------+---------+\n", + "| root | 0 B | GOOD | NONE | False |\n", + "| spoils/database.db | 4.77 MB | GOOD | NONE | False |\n", + "+--------------------+---------+---------------+-----------------------+---------+\n" + ] + } + ], "source": [ "client_1: Computer = env.game.simulation.network.get_node_by_hostname(\"client_1\")\n", "client_1.software_manager.file_system.show(full=True)" @@ -531,9 +759,25 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+---------------------------------------------------------------------------------+\n", + "| web_server File System |\n", + "+---------------------+---------+---------------+-----------------------+---------+\n", + "| File Path | Size | Health status | Visible health status | Deleted |\n", + "+---------------------+---------+---------------+-----------------------+---------+\n", + "| primaite/index.html | 15.0 KB | GOOD | NONE | False |\n", + "| root | 0 B | GOOD | NONE | False |\n", + "| spoils/database.db | 4.77 MB | GOOD | NONE | False |\n", + "+---------------------+---------+---------------+-----------------------+---------+\n" + ] + } + ], "source": [ "web_server: Computer = env.game.simulation.network.get_node_by_hostname(\"web_server\")\n", "web_server.software_manager.file_system.show(full=True)" @@ -570,18 +814,48 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "(0,\n", + " 0.0,\n", + " False,\n", + " False,\n", + " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=6, action='c2_server_ransomware_launch', parameters={'node_name': 'client_1'}, request=['network', 'node', 'client_1', 'application', 'C2Server', 'ransomware_launch'], response=RequestResponse(status='success', data={}), reward=0.0, reward_info={})}})" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "env.step(7)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----------------------------------------------------------------------------------+\n", + "| database_server File System |\n", + "+----------------------+---------+---------------+-----------------------+---------+\n", + "| File Path | Size | Health status | Visible health status | Deleted |\n", + "+----------------------+---------+---------------+-----------------------+---------+\n", + "| database/database.db | 4.77 MB | CORRUPT | NONE | False |\n", + "| root | 0 B | GOOD | NONE | False |\n", + "+----------------------+---------+---------------+-----------------------+---------+\n" + ] + } + ], "source": [ "database_server: Server = env.game.simulation.network.get_node_by_hostname(\"database_server\")\n", "database_server.software_manager.file_system.show(full=True)" @@ -600,7 +874,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "metadata": {}, "outputs": [], "source": [ @@ -713,35 +987,9 @@ " src_port: HTTP\n", " dst_port: HTTP\n", " protocol_name: ALL\n", - " src_wildcard: NONE\n", - " dst_wildcard: NONE\n", + " src_wildcard: 0.0.0.1\n", + " dst_wildcard: 0.0.0.1\n", "\n", - "\n", - " # options:\n", - " # # nodes:\n", - " # node_name: web_server\n", - " # # applications:\n", - " # application_name: C2Beacon\n", - "\n", - " # node_name: database_server\n", - " # folders:\n", - " # - folder_name: database\n", - " # files:\n", - " # - file_name: database.db\n", - " # services:\n", - " # - service_name: DatabaseService\n", - " # node_name: router_1\n", - "\n", - " # max_folders_per_node: 2\n", - " # max_files_per_folder: 2\n", - " # max_services_per_node: 2\n", - " # max_nics_per_node: 8\n", - " # max_acl_rules: 10\n", - " # ip_list:\n", - " # - 192.168.10.21\n", - " # - 192.168.1.12\n", - " # wildcard_list:\n", - " # - 0.0.0.1\n", " reward_function:\n", " reward_components:\n", " - type: DUMMY\n", @@ -754,9 +1002,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-04 15:37:36,689: PrimaiteGymEnv RNG seed = None\n" + ] + } + ], "source": [ "with open(data_manipulation_config_path()) as f:\n", " cfg = yaml.safe_load(f)\n", @@ -813,9 +1069,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 22, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-04 15:37:37,079: Resetting environment, episode 0, avg. reward: 0.0\n", + "2025-02-04 15:37:37,082: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-04/15-37-31/agent_actions/episode_0.json\n" + ] + } + ], "source": [ "# Resetting the environment and capturing the default observation space.\n", "blue_env.reset()\n", @@ -824,9 +1089,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 23, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Setting up the C2 Suite via the simulation API.\n", "\n", @@ -857,9 +1133,26 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 25, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 2\n", + "root['NODES']['HOST0']['APPLICATIONS'][1]['operating_status']: 0 -> 1\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", + "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", + "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" + ] + } + ], "source": [ "display_obs_diffs(default_obs, c2_configuration_obs, blue_env.game.step_counter)" ] @@ -879,9 +1172,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 26, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "RequestResponse(status='success', data={0: RequestResponse(status='success', data={})})" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Installing RansomwareScript via C2 Terminal Commands\n", "ransomware_install_command = {\"commands\":[[\"software_manager\", \"application\", \"install\", \"RansomwareScript\"]],\n", @@ -892,9 +1196,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 27, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "RequestResponse(status='success', data={})" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Configuring the RansomwareScript\n", "ransomware_config = {\"server_ip_address\": \"192.168.1.14\", \"payload\": \"ENCRYPT\"}\n", @@ -913,9 +1228,28 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 29, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 7\n", + "root['NODES']['HOST0']['APPLICATIONS'][1]['operating_status']: 0 -> 1\n", + "root['NODES']['HOST0']['APPLICATIONS'][2]['operating_status']: 0 -> 3\n", + "root['NODES']['HOST0']['users']['local_login']: 0 -> 1\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", + "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", + "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" + ] + } + ], "source": [ "display_obs_diffs(default_obs, c2_ransomware_obs, env.game.step_counter)" ] @@ -947,9 +1281,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 31, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "RequestResponse(status='success', data={})" + ] + }, + "execution_count": 31, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "c2_server.send_command(given_command=C2Command.DATA_EXFILTRATION, command_options=exfil_options)" ] @@ -965,9 +1310,22 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 33, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 7\n", + "root['NODES']['HOST0']['APPLICATIONS'][2]['operating_status']: 3 -> 1\n", + "root['NODES']['HOST1']['users']['remote_sessions']: 0 -> 1\n" + ] + } + ], "source": [ "display_obs_diffs(c2_ransomware_obs, c2_exfil_obs, env.game.step_counter)" ] @@ -983,9 +1341,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 34, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "RequestResponse(status='success', data={})" + ] + }, + "execution_count": 34, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Configuring the RansomwareScript\n", "ransomware_config = {\"server_ip_address\": \"192.168.1.14\", \"payload\": \"ENCRYPT\"}\n", @@ -994,9 +1363,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 35, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "RequestResponse(status='success', data={})" + ] + }, + "execution_count": 35, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Waiting for the ransomware to finish installing and then launching the RansomwareScript.\n", "blue_env.step(0)\n", @@ -1015,9 +1395,30 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 37, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 6\n", + "root['NODES']['HOST0']['APPLICATIONS'][2]['operating_status']: 3 -> 1\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['NODES']['HOST1']['users']['remote_sessions']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" + ] + } + ], "source": [ "display_obs_diffs(c2_ransomware_obs, c2_final_obs, blue_env.game.step_counter)" ] @@ -1065,9 +1466,193 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 39, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-04 15:37:38,317: Resetting environment, episode 1, avg. reward: 0.0\n", + "2025-02-04 15:37:38,321: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-04/15-37-31/agent_actions/episode_1.json\n" + ] + }, + { + "data": { + "text/plain": [ + "({'NODES': {'HOST0': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 1,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'HOST1': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 1,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'HOST2': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 0,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'HOST3': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 0,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'ROUTER0': {'ACL': {1: {'position': 0,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 2: {'position': 1,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 3: {'position': 2,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 4: {'position': 3,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 5: {'position': 4,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 6: {'position': 5,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 7: {'position': 6,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 8: {'position': 7,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 9: {'position': 8,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 10: {'position': 9,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0}},\n", + " 'PORTS': {1: {'operating_status': 1},\n", + " 2: {'operating_status': 1},\n", + " 3: {'operating_status': 2}},\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}}},\n", + " 'LINKS': {1: {'PROTOCOLS': {'ALL': 1}},\n", + " 2: {'PROTOCOLS': {'ALL': 1}},\n", + " 3: {'PROTOCOLS': {'ALL': 0}},\n", + " 4: {'PROTOCOLS': {'ALL': 1}},\n", + " 5: {'PROTOCOLS': {'ALL': 1}},\n", + " 6: {'PROTOCOLS': {'ALL': 1}},\n", + " 7: {'PROTOCOLS': {'ALL': 1}},\n", + " 8: {'PROTOCOLS': {'ALL': 1}},\n", + " 9: {'PROTOCOLS': {'ALL': 1}},\n", + " 10: {'PROTOCOLS': {'ALL': 0}}},\n", + " 'ICS': 0},\n", + " {})" + ] + }, + "execution_count": 39, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "blue_env.reset()" ] @@ -1110,9 +1695,33 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 42, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+-------------------------------------------------------------------------------------+\n", + "| web_server Software Manager |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| Name | Type | Operating State | Health State | Port | Protocol |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", + "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", + "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", + "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", + "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", + "| NMAP | Application | RUNNING | GOOD | None | none |\n", + "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", + "| UserManager | Service | RUNNING | GOOD | None | none |\n", + "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", + "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", + "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n" + ] + } + ], "source": [ "blue_env.step(0)\n", "web_server.software_manager.show()" @@ -1120,9 +1729,26 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 43, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 3\n", + "root['NODES']['HOST0']['APPLICATIONS'][1]['operating_status']: 1 -> 0\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", + "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", + "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" + ] + } + ], "source": [ "display_obs_diffs(pre_blue_action_obs, post_blue_action_obs, blue_env.game.step_counter)" ] @@ -1136,9 +1762,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 44, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "RequestResponse(status='failure', data={'Reason': 'Command sent to the C2 Beacon but no response was ever received.'})" + ] + }, + "execution_count": 44, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Attempting to install the C2 RansomwareScript\n", "ransomware_install_command = {\"commands\":[[\"software_manager\", \"application\", \"install\", \"RansomwareScript\"]],\n", @@ -1160,9 +1797,193 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 45, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-04 15:37:38,778: Resetting environment, episode 2, avg. reward: 0.0\n", + "2025-02-04 15:37:38,781: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-04/15-37-31/agent_actions/episode_2.json\n" + ] + }, + { + "data": { + "text/plain": [ + "({'NODES': {'HOST0': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 1,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'HOST1': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 1,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'HOST2': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 0,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'HOST3': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 0,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'ROUTER0': {'ACL': {1: {'position': 0,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 2: {'position': 1,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 3: {'position': 2,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 4: {'position': 3,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 5: {'position': 4,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 6: {'position': 5,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 7: {'position': 6,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 8: {'position': 7,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 9: {'position': 8,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 10: {'position': 9,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0}},\n", + " 'PORTS': {1: {'operating_status': 1},\n", + " 2: {'operating_status': 1},\n", + " 3: {'operating_status': 2}},\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}}},\n", + " 'LINKS': {1: {'PROTOCOLS': {'ALL': 1}},\n", + " 2: {'PROTOCOLS': {'ALL': 1}},\n", + " 3: {'PROTOCOLS': {'ALL': 0}},\n", + " 4: {'PROTOCOLS': {'ALL': 1}},\n", + " 5: {'PROTOCOLS': {'ALL': 1}},\n", + " 6: {'PROTOCOLS': {'ALL': 1}},\n", + " 7: {'PROTOCOLS': {'ALL': 1}},\n", + " 8: {'PROTOCOLS': {'ALL': 1}},\n", + " 9: {'PROTOCOLS': {'ALL': 1}},\n", + " 10: {'PROTOCOLS': {'ALL': 0}}},\n", + " 'ICS': 0},\n", + " {})" + ] + }, + "execution_count": 45, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "blue_env.reset()" ] @@ -1205,9 +2026,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 48, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "NodeOperatingState.SHUTTING_DOWN\n" + ] + } + ], "source": [ "web_server = blue_env.game.simulation.network.get_node_by_hostname(\"web_server\")\n", "print(web_server.operating_state)" @@ -1215,18 +2044,48 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 49, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 2\n", + "root['NODES']['HOST0']['operating_status']: 1 -> 4\n", + "root['NODES']['HOST0']['APPLICATIONS'][1]['operating_status']: 1 -> 0\n", + "root['NODES']['HOST0']['NICS'][1]['nic_status']: 1 -> 0\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", + "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", + "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" + ] + } + ], "source": [ "display_obs_diffs(pre_blue_action_obs, post_blue_action_obs, blue_env.game.step_counter)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 50, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "RequestResponse(status='failure', data={'Reason': 'Command sent to the C2 Beacon but no response was ever received.'})" + ] + }, + "execution_count": 50, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Attempting to install the C2 RansomwareScript\n", "ransomware_install_command = {\"commands\":[\"software_manager\", \"application\", \"install\", \"RansomwareScript\"],\n", @@ -1250,9 +2109,193 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 51, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-04 15:37:39,257: Resetting environment, episode 3, avg. reward: 0.0\n", + "2025-02-04 15:37:39,260: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-04/15-37-31/agent_actions/episode_3.json\n" + ] + }, + { + "data": { + "text/plain": [ + "({'NODES': {'HOST0': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 1,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'HOST1': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 1,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'HOST2': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 0,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'HOST3': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 0,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'ROUTER0': {'ACL': {1: {'position': 0,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 2: {'position': 1,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 3: {'position': 2,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 4: {'position': 3,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 5: {'position': 4,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 6: {'position': 5,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 7: {'position': 6,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 8: {'position': 7,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 9: {'position': 8,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 10: {'position': 9,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0}},\n", + " 'PORTS': {1: {'operating_status': 1},\n", + " 2: {'operating_status': 1},\n", + " 3: {'operating_status': 2}},\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}}},\n", + " 'LINKS': {1: {'PROTOCOLS': {'ALL': 1}},\n", + " 2: {'PROTOCOLS': {'ALL': 1}},\n", + " 3: {'PROTOCOLS': {'ALL': 0}},\n", + " 4: {'PROTOCOLS': {'ALL': 1}},\n", + " 5: {'PROTOCOLS': {'ALL': 1}},\n", + " 6: {'PROTOCOLS': {'ALL': 1}},\n", + " 7: {'PROTOCOLS': {'ALL': 1}},\n", + " 8: {'PROTOCOLS': {'ALL': 1}},\n", + " 9: {'PROTOCOLS': {'ALL': 1}},\n", + " 10: {'PROTOCOLS': {'ALL': 0}}},\n", + " 'ICS': 0},\n", + " {})" + ] + }, + "execution_count": 51, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "blue_env.reset()" ] @@ -1295,12 +2338,43 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 54, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+---------------------------------------------------------------+\n", + "| router_1 Network Interfaces |\n", + "+------+-------------------+-----------------+-------+----------+\n", + "| Port | MAC Address | Address | Speed | Status |\n", + "+------+-------------------+-----------------+-------+----------+\n", + "| 1 | dd:bc:17:75:a2:c4 | 192.168.1.1/24 | 100.0 | Enabled |\n", + "| 2 | 00:f2:f5:65:51:75 | 192.168.10.1/24 | 100.0 | Enabled |\n", + "| 3 | de:7d:ac:0e:58:e1 | 127.0.0.1/8 | 100.0 | Disabled |\n", + "| 4 | 8a:b4:55:12:c7:9e | 127.0.0.1/8 | 100.0 | Disabled |\n", + "| 5 | a5:ef:c6:53:97:c4 | 127.0.0.1/8 | 100.0 | Disabled |\n", + "+------+-------------------+-----------------+-------+----------+\n", + "+------------------------------------------------------------------------------------------------------------------------+\n", + "| router_1 Access Control List |\n", + "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n", + "| Index | Action | Protocol | Src IP | Src Wildcard | Src Port | Dst IP | Dst Wildcard | Dst Port | Matched |\n", + "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n", + "| 1 | DENY | ANY | 192.168.10.21 | 0.0.0.1 | 80 | 192.168.1.12 | 0.0.0.1 | 80 | 0 |\n", + "| 18 | PERMIT | ANY | ANY | ANY | 5432 | ANY | ANY | 5432 | 0 |\n", + "| 19 | PERMIT | ANY | ANY | ANY | 53 | ANY | ANY | 53 | 0 |\n", + "| 20 | PERMIT | ANY | ANY | ANY | 21 | ANY | ANY | 21 | 0 |\n", + "| 21 | PERMIT | ANY | ANY | ANY | 80 | ANY | ANY | 80 | 4 |\n", + "| 22 | PERMIT | ANY | ANY | ANY | 219 | ANY | ANY | 219 | 10 |\n", + "| 23 | PERMIT | icmp | ANY | ANY | ANY | ANY | ANY | ANY | 0 |\n", + "| 24 | DENY | ANY | ANY | ANY | ANY | ANY | ANY | ANY | 0 |\n", + "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n" + ] + } + ], "source": [ "router_1: Router = blue_env.game.simulation.network.get_node_by_hostname(\"router_1\")\n", - "router_1.show()\n", "router_1.acl.show()" ] }, @@ -1313,9 +2387,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 55, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "RequestResponse(status='failure', data={'Reason': 'Command sent to the C2 Beacon but no response was ever received.'})" + ] + }, + "execution_count": 55, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "blue_env.step(0)\n", "\n", @@ -1326,9 +2411,30 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 56, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+------------------------------------------------------------------------------------------------------------------------+\n", + "| router_1 Access Control List |\n", + "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n", + "| Index | Action | Protocol | Src IP | Src Wildcard | Src Port | Dst IP | Dst Wildcard | Dst Port | Matched |\n", + "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n", + "| 1 | DENY | ANY | 192.168.10.21 | 0.0.0.1 | 80 | 192.168.1.12 | 0.0.0.1 | 80 | 2 |\n", + "| 18 | PERMIT | ANY | ANY | ANY | 5432 | ANY | ANY | 5432 | 0 |\n", + "| 19 | PERMIT | ANY | ANY | ANY | 53 | ANY | ANY | 53 | 0 |\n", + "| 20 | PERMIT | ANY | ANY | ANY | 21 | ANY | ANY | 21 | 0 |\n", + "| 21 | PERMIT | ANY | ANY | ANY | 80 | ANY | ANY | 80 | 4 |\n", + "| 22 | PERMIT | ANY | ANY | ANY | 219 | ANY | ANY | 219 | 10 |\n", + "| 23 | PERMIT | icmp | ANY | ANY | ANY | ANY | ANY | ANY | 0 |\n", + "| 24 | DENY | ANY | ANY | ANY | ANY | ANY | ANY | ANY | 0 |\n", + "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n" + ] + } + ], "source": [ "router_1.acl.show()" ] @@ -1342,18 +2448,58 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 57, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+-------------------------------------------------------------------------------------+\n", + "| web_server Software Manager |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| Name | Type | Operating State | Health State | Port | Protocol |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", + "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", + "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", + "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", + "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", + "| NMAP | Application | RUNNING | GOOD | None | none |\n", + "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", + "| UserManager | Service | RUNNING | GOOD | None | none |\n", + "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", + "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", + "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", + "| C2Beacon | Application | RUNNING | GOOD | None | tcp |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n" + ] + } + ], "source": [ "web_server.software_manager.show()" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 58, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----------------------------------------------------------------------------------+\n", + "| database_server File System |\n", + "+----------------------+---------+---------------+-----------------------+---------+\n", + "| File Path | Size | Health status | Visible health status | Deleted |\n", + "+----------------------+---------+---------------+-----------------------+---------+\n", + "| database/database.db | 4.77 MB | GOOD | NONE | False |\n", + "| root | 0 B | GOOD | NONE | False |\n", + "+----------------------+---------+---------------+-----------------------+---------+\n" + ] + } + ], "source": [ "database_server: Server = blue_env.game.simulation.network.get_node_by_hostname(\"database_server\")\n", "database_server.software_manager.file_system.show(full=True)" @@ -1361,9 +2507,33 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 59, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 3\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", + "root['NODES']['ROUTER0']['ACL'][1]['permission']: 0 -> 2\n", + "root['NODES']['ROUTER0']['ACL'][1]['source_ip_id']: 0 -> 7\n", + "root['NODES']['ROUTER0']['ACL'][1]['source_wildcard_id']: 0 -> 2\n", + "root['NODES']['ROUTER0']['ACL'][1]['source_port_id']: 0 -> 2\n", + "root['NODES']['ROUTER0']['ACL'][1]['dest_ip_id']: 0 -> 3\n", + "root['NODES']['ROUTER0']['ACL'][1]['dest_wildcard_id']: 0 -> 2\n", + "root['NODES']['ROUTER0']['ACL'][1]['dest_port_id']: 0 -> 2\n", + "root['NODES']['ROUTER0']['ACL'][1]['protocol_id']: 0 -> 1\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", + "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", + "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" + ] + } + ], "source": [ "display_obs_diffs(pre_blue_action_obs, post_blue_action_obs, blue_env.game.step_counter)" ] @@ -1427,9 +2597,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 60, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-04 15:37:40,175: PrimaiteGymEnv RNG seed = None\n" + ] + } + ], "source": [ "with open(data_manipulation_config_path()) as f:\n", " cfg = yaml.safe_load(f)\n", @@ -1478,9 +2656,30 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 62, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", + "| C2Beacon Running Status |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| False | None | 0 | 5 | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "+-----------------------------------------------------------------------------------------------------+\n", + "| C2Server Running Status |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n", + "| False | None | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n" + ] + } + ], "source": [ "env.step(2) # Agent Action Equivalent to c2_beacon.configure(c2_server_ip_address=\"192.168.10.21\")\n", "env.step(3) # Agent action Equivalent to c2_beacon.establish()\n", @@ -1497,9 +2696,30 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 63, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", + "| C2Beacon Running Status |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| False | None | 0 | 5 | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "+-----------------------------------------------------------------------------------------------------+\n", + "| C2Server Running Status |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n", + "| False | None | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n" + ] + } + ], "source": [ "env.step(9) # Equivalent of to c2_beacon.configure(c2_server_ip_address=\"192.168.10.22\")\n", "env.step(3)\n", @@ -1517,9 +2737,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 64, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+-----------------------------------------------------------------------------------------------------+\n", + "| C2Server Running Status |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n", + "| False | None | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n" + ] + } + ], "source": [ "for i in range(6):\n", " env.step(0)\n", @@ -1542,9 +2776,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 65, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-04 15:37:40,588: PrimaiteGymEnv RNG seed = None\n" + ] + } + ], "source": [ "with open(data_manipulation_config_path()) as f:\n", " cfg = yaml.safe_load(f)\n", @@ -1584,9 +2826,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 67, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", + "| C2Beacon Running Status |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| True | 192.168.10.21 | 0 | 5 | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n" + ] + } + ], "source": [ "c2_beacon.configure(c2_server_ip_address=\"192.168.10.21\")\n", "c2_beacon.establish()\n", @@ -1604,9 +2860,72 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 68, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 4\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 5\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 6\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 7\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 8\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 9\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 10\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 11\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 12\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 13\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" + ] + } + ], "source": [ "for i in range(10):\n", " keep_alive_obs, _, _, _, _ = blue_config_env.step(0)\n", @@ -1622,9 +2941,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 69, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", + "| C2Beacon Running Status |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| True | 192.168.10.21 | 0 | 1 | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n" + ] + } + ], "source": [ "c2_beacon.configure(c2_server_ip_address=\"192.168.10.21\", keep_alive_frequency=1)\n", "c2_beacon.establish()\n", @@ -1640,9 +2973,40 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 70, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 14\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 15\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" + ] + } + ], "source": [ "# Comparing the OBS of the default frequency to a timestep frequency of 1\n", "for i in range(2):\n", @@ -1661,9 +3025,52 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 71, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 16\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 17\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 18\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 19\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 20\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 21\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 22\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" + ] + } + ], "source": [ "c2_beacon.configure(c2_server_ip_address=\"192.168.10.21\", keep_alive_frequency=7)\n", "\n", @@ -1700,9 +3107,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 72, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-04 15:37:41,322: Resetting environment, episode 0, avg. reward: 0.0\n", + "2025-02-04 15:37:41,325: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-04/15-37-31/agent_actions/episode_0.json\n" + ] + } + ], "source": [ "blue_config_env.reset()\n", "\n", @@ -1724,9 +3140,28 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 73, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 5\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" + ] + } + ], "source": [ "# Capturing default C2 Traffic\n", "for i in range(3):\n", @@ -1744,9 +3179,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 74, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", + "| C2Beacon Running Status |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| True | 192.168.10.21 | 0 | 5 | udp | 53 |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n" + ] + } + ], "source": [ "from primaite.utils.validation.ip_protocol import PROTOCOL_LOOKUP\n", "from primaite.utils.validation.port import PORT_LOOKUP\n", @@ -1759,9 +3208,28 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 75, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 10\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 1 -> 0\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 1 -> 0\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['udp'][53]['inbound']: 0 -> 1\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['udp'][53]['outbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 1 -> 0\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 1 -> 0\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['udp'][53]['inbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['udp'][53]['outbound']: 0 -> 1\n" + ] + } + ], "source": [ "# Capturing UDP C2 Traffic\n", "for i in range(5):\n", From 3a29593f600ba3fa1b9b74806333bf7934e4303f Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Wed, 5 Feb 2025 11:50:38 +0000 Subject: [PATCH 10/37] 3075: Fix bug in notebook helper function. --- ...a-Manipulation-Customising-Red-Agent.ipynb | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb b/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb index de6d3fe5..14938c02 100644 --- a/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb +++ b/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb @@ -6,7 +6,7 @@ "source": [ "# Customising UC2 Red Agents\n", "\n", - "© Crown-owned copyright 2024, Defence Science and Technology Laboratory UK\n", + "© Crown-owned copyright 2025, Defence Science and Technology Laboratory UK\n", "\n", "This notebook will go over some examples of how red agent behaviour can be varied by changing its configuration parameters.\n", "\n", @@ -26,7 +26,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -36,13 +36,12 @@ "from primaite.game.agent.interface import AgentHistoryItem\n", "from primaite.session.environment import PrimaiteGymEnv\n", "import yaml\n", - "from pprint import pprint\n", - "from primaite.game.agent.scripted_agents import probabilistic_agent, data_manipulation_bot" + "from pprint import pprint" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -69,7 +68,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -80,9 +79,7 @@ " if red_action == 'do_nothing':\n", " red_str = 'DO NOTHING'\n", " elif red_action == 'node_application_execute':\n", - " client = \"client 1\" if red_info.parameters['node_name'] == 0 else \"client 2\"\n", - "\n", - " red_str = f\"ATTACK from {client}\"\n", + " red_str = f\"ATTACK from {red_info.parameters['node_name']}\"\n", " return red_str" ] }, @@ -245,7 +242,7 @@ "outputs": [], "source": [ "change = yaml.safe_load(\"\"\"\n", - " possible_start_nodes: [client_1, client_2]\n", + " possible_start_nodes: [client_1]\n", " target_application: DataManipulationBot\n", " start_step: 25\n", " frequency: 20\n", @@ -315,6 +312,10 @@ "outputs": [], "source": [ "change = yaml.safe_load(\"\"\"\n", + " agent_settings:\n", + " possible_start_nodes: [client_1]\n", + " target_application: DataManipulationBot\n", + "\n", " action_space:\n", " action_map:\n", " 0:\n", From a4c19608a69251c0dabff77c69fb02fab544b3c7 Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Wed, 5 Feb 2025 11:51:22 +0000 Subject: [PATCH 11/37] 3075: Pre-commit tidy up. --- docs/source/how_to_guides/extensible_nodes.rst | 2 +- src/primaite/game/game.py | 1 - src/primaite/notebooks/Action-masking.ipynb | 15 +++++++-------- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/docs/source/how_to_guides/extensible_nodes.rst b/docs/source/how_to_guides/extensible_nodes.rst index 043d0f06..4819cbb2 100644 --- a/docs/source/how_to_guides/extensible_nodes.rst +++ b/docs/source/how_to_guides/extensible_nodes.rst @@ -52,4 +52,4 @@ class Router(NetworkNode, identifier="router"): Changes to YAML file. ===================== -While effort has been made to ensure that nodes defined within configuration YAML files for use with PrimAITE 3.X remain compatible with PrimAITE v4+, it is encouraged to review for minor changes needed. +While effort has been made to ensure that nodes defined within configuration YAML files for use with PrimAITE 3.X remain compatible with PrimAITE v4+, it is encouraged to review for minor changes needed. diff --git a/src/primaite/game/game.py b/src/primaite/game/game.py index f819fa05..e9941a12 100644 --- a/src/primaite/game/game.py +++ b/src/primaite/game/game.py @@ -404,7 +404,6 @@ class PrimaiteGame: agents_cfg = cfg.get("agents", []) for agent_cfg in agents_cfg: - new_agent = AbstractAgent.from_config(agent_cfg) game.agents[agent_cfg["ref"]] = new_agent if isinstance(new_agent, ProxyAgent): diff --git a/src/primaite/notebooks/Action-masking.ipynb b/src/primaite/notebooks/Action-masking.ipynb index 64015080..74504878 100644 --- a/src/primaite/notebooks/Action-masking.ipynb +++ b/src/primaite/notebooks/Action-masking.ipynb @@ -6,7 +6,7 @@ "source": [ "# Action Masking\n", "\n", - "© Crown-owned copyright 2024, Defence Science and Technology Laboratory UK\n", + "© Crown-owned copyright 2025, Defence Science and Technology Laboratory UK\n", "\n", "PrimAITE environments support action masking. The action mask shows which of the agent's actions are applicable with the current environment state. For example, a node can only be turned on if it is currently turned off." ] @@ -22,14 +22,13 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from primaite.session.environment import PrimaiteGymEnv\n", "from primaite.config.load import data_manipulation_config_path\n", - "from prettytable import PrettyTable\n", - "from primaite.game.agent.scripted_agents import probabilistic_agent\n" + "from prettytable import PrettyTable" ] }, { @@ -104,7 +103,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -117,7 +116,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -157,7 +156,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -168,7 +167,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ From b22ba65eb69644a25ad6451d3fadfe87e5c8112d Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Wed, 5 Feb 2025 16:40:59 +0000 Subject: [PATCH 12/37] #3075: Further notebook updates. --- ...ommand-and-Control-E2E-Demonstration.ipynb | 1792 ++--------------- .../Data-Manipulation-E2E-Demonstration.ipynb | 6 +- .../Getting-Information-Out-Of-PrimAITE.ipynb | 4 +- ...ege-Escalation-and-Data-Loss-Example.ipynb | 11 +- .../notebooks/Requests-and-Responses.ipynb | 22 +- .../notebooks/Terminal-Processing.ipynb | 43 +- .../Training-an-RLLIB-MARL-System.ipynb | 31 +- .../notebooks/Training-an-RLLib-Agent.ipynb | 31 +- .../notebooks/Training-an-SB3-Agent.ipynb | 2 +- .../notebooks/Using-Episode-Schedules.ipynb | 18 +- src/primaite/notebooks/multi-processing.ipynb | 15 +- .../simulator/network/hardware/base.py | 8 +- 12 files changed, 224 insertions(+), 1759 deletions(-) diff --git a/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb b/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb index b8d1423f..d3c414d7 100644 --- a/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb +++ b/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb @@ -13,31 +13,16 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-02-04 15:37:30,830: Performing the PrimAITE first-time setup...\n", - "2025-02-04 15:37:30,830: Building the PrimAITE app directories...\n", - "2025-02-04 15:37:30,830: Building primaite_config.yaml...\n", - "2025-02-04 15:37:30,830: Rebuilding the demo notebooks...\n", - "/home/nick/primaite/4.0.0a1-dev/notebooks/example_notebooks/Command-and-Control-E2E-Demonstration.ipynb\n", - "2025-02-04 15:37:30,832: Reset example notebook: /home/nick/primaite/4.0.0a1-dev/notebooks/example_notebooks/Command-and-Control-E2E-Demonstration.ipynb\n", - "2025-02-04 15:37:30,838: Rebuilding the example notebooks...\n", - "2025-02-04 15:37:30,841: PrimAITE setup complete!\n" - ] - } - ], + "outputs": [], "source": [ "!primaite setup" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -67,7 +52,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -150,17 +135,9 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-04 15:37:35,293: PrimaiteGymEnv RNG seed = None\n" - ] - } - ], + "outputs": [], "source": [ "with open(data_manipulation_config_path()) as f:\n", " cfg = yaml.safe_load(f)\n", @@ -186,35 +163,9 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+--------------------------------------------------------------------------------------+\n", - "| client_1 Software Manager |\n", - "+---------------------+-------------+-----------------+--------------+------+----------+\n", - "| Name | Type | Operating State | Health State | Port | Protocol |\n", - "+---------------------+-------------+-----------------+--------------+------+----------+\n", - "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", - "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", - "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", - "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", - "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", - "| NMAP | Application | RUNNING | GOOD | None | none |\n", - "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", - "| UserManager | Service | RUNNING | GOOD | None | none |\n", - "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", - "| FTPClient | Service | RUNNING | GOOD | 21 | tcp |\n", - "| DataManipulationBot | Application | RUNNING | GOOD | None | none |\n", - "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", - "| C2Server | Application | RUNNING | GOOD | None | tcp |\n", - "+---------------------+-------------+-----------------+--------------+------+----------+\n" - ] - } - ], + "outputs": [], "source": [ "client_1: Computer = env.game.simulation.network.get_node_by_hostname(\"client_1\")\n", "client_1.software_manager.install(C2Server)\n", @@ -264,34 +215,9 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+-------------------------------------------------------------------------------------+\n", - "| web_server Software Manager |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| Name | Type | Operating State | Health State | Port | Protocol |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", - "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", - "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", - "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", - "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", - "| NMAP | Application | RUNNING | GOOD | None | none |\n", - "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", - "| UserManager | Service | RUNNING | GOOD | None | none |\n", - "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", - "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", - "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", - "| C2Beacon | Application | INSTALLING | UNUSED | None | tcp |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n" - ] - } - ], + "outputs": [], "source": [ "env.step(1)\n", "web_server: Computer = env.game.simulation.network.get_node_by_hostname(\"web_server\")\n", @@ -331,41 +257,9 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+-------------------------------------------------------------------------------------+\n", - "| web_server Software Manager |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| Name | Type | Operating State | Health State | Port | Protocol |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", - "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", - "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", - "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", - "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", - "| NMAP | Application | RUNNING | GOOD | None | none |\n", - "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", - "| UserManager | Service | RUNNING | GOOD | None | none |\n", - "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", - "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", - "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", - "| C2Beacon | Application | RUNNING | GOOD | None | tcp |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", - "| C2Beacon Running Status |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| False | 192.168.10.21 | 0 | 5 | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n" - ] - } - ], + "outputs": [], "source": [ "env.step(2)\n", "c2_beacon: C2Beacon = web_server.software_manager.software[\"C2Beacon\"]\n", @@ -404,54 +298,18 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(0,\n", - " 0.0,\n", - " False,\n", - " False,\n", - " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=2, action='node_application_execute', parameters={'node_name': 'web_server', 'application_name': 'C2Beacon'}, request=['network', 'node', 'web_server', 'application', 'C2Beacon', 'execute'], response=RequestResponse(status='success', data={}), reward=0.0, reward_info={})}})" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "env.step(3)" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", - "| C2Beacon Running Status |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| True | 192.168.10.21 | 1 | 5 | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "+-----------------------------------------------------------------------------------------------------+\n", - "| C2Server Running Status |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n", - "| True | 192.168.1.12 | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n" - ] - } - ], + "outputs": [], "source": [ "c2_beacon.show()\n", "c2_server.show()" @@ -514,59 +372,18 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(0,\n", - " 0.0,\n", - " False,\n", - " False,\n", - " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=3, action='c2_server_terminal_command', parameters={'node_name': 'client_1', 'ip_address': None, 'username': 'admin', 'password': 'admin', 'commands': [['software_manager', 'application', 'install', 'RansomwareScript']]}, request=['network', 'node', 'client_1', 'application', 'C2Server', 'terminal_command', {'commands': [['software_manager', 'application', 'install', 'RansomwareScript']], 'ip_address': None, 'username': 'admin', 'password': 'admin'}], response=RequestResponse(status='success', data={0: RequestResponse(status='success', data={})}), reward=0.0, reward_info={})}})" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "env.step(4)" ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+--------------------------------------------------------------------------------------+\n", - "| client_1 Software Manager |\n", - "+---------------------+-------------+-----------------+--------------+------+----------+\n", - "| Name | Type | Operating State | Health State | Port | Protocol |\n", - "+---------------------+-------------+-----------------+--------------+------+----------+\n", - "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", - "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", - "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", - "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", - "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", - "| NMAP | Application | RUNNING | GOOD | None | none |\n", - "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", - "| UserManager | Service | RUNNING | GOOD | None | none |\n", - "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", - "| FTPClient | Service | RUNNING | GOOD | 21 | tcp |\n", - "| DataManipulationBot | Application | RUNNING | GOOD | None | none |\n", - "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", - "| C2Server | Application | RUNNING | GOOD | None | tcp |\n", - "+---------------------+-------------+-----------------+--------------+------+----------+\n" - ] - } - ], + "outputs": [], "source": [ "client_1.software_manager.show()" ] @@ -605,66 +422,18 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(0,\n", - " 0.0,\n", - " False,\n", - " False,\n", - " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=4, action='c2_server_ransomware_configure', parameters={'node_name': 'client_1', 'server_ip_address': '192.168.1.14', 'payload': 'ENCRYPT'}, request=['network', 'node', 'client_1', 'application', 'C2Server', 'ransomware_configure', {'server_ip_address': '192.168.1.14', 'server_password': None, 'payload': 'ENCRYPT'}], response=RequestResponse(status='success', data={}), reward=0.0, reward_info={})}})" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "env.step(5)" ] }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+-------------------------------------------------------------------------------------+\n", - "| web_server Software Manager |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| Name | Type | Operating State | Health State | Port | Protocol |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", - "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", - "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", - "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", - "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", - "| NMAP | Application | RUNNING | GOOD | None | none |\n", - "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", - "| UserManager | Service | RUNNING | GOOD | None | none |\n", - "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", - "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", - "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", - "| C2Beacon | Application | RUNNING | GOOD | None | tcp |\n", - "| RansomwareScript | Application | RUNNING | GOOD | None | none |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "+------------------------------------+\n", - "| RansomwareScript Running Status |\n", - "+--------------------------+---------+\n", - "| Target Server IP Address | Payload |\n", - "+--------------------------+---------+\n", - "| 192.168.1.14 | ENCRYPT |\n", - "+--------------------------+---------+\n" - ] - } - ], + "outputs": [], "source": [ "ransomware_script: RansomwareScript = web_server.software_manager.software[\"RansomwareScript\"]\n", "web_server.software_manager.show()\n", @@ -710,48 +479,18 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(0,\n", - " 0.0,\n", - " False,\n", - " False,\n", - " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=5, action='c2_server_data_exfiltrate', parameters={'node_name': 'client_1', 'target_file_name': 'database.db', 'target_folder_name': 'database', 'exfiltration_folder_name': 'spoils', 'target_ip_address': '192.168.1.14', 'username': 'admin', 'password': 'admin'}, request=['network', 'node', 'client_1', 'application', 'C2Server', 'exfiltrate', {'target_file_name': 'database.db', 'target_folder_name': 'database', 'exfiltration_folder_name': 'spoils', 'target_ip_address': '192.168.1.14', 'username': 'admin', 'password': 'admin'}], response=RequestResponse(status='success', data={}), reward=0.0, reward_info={})}})" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "env.step(6)" ] }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+--------------------------------------------------------------------------------+\n", - "| client_1 File System |\n", - "+--------------------+---------+---------------+-----------------------+---------+\n", - "| File Path | Size | Health status | Visible health status | Deleted |\n", - "+--------------------+---------+---------------+-----------------------+---------+\n", - "| root | 0 B | GOOD | NONE | False |\n", - "| spoils/database.db | 4.77 MB | GOOD | NONE | False |\n", - "+--------------------+---------+---------------+-----------------------+---------+\n" - ] - } - ], + "outputs": [], "source": [ "client_1: Computer = env.game.simulation.network.get_node_by_hostname(\"client_1\")\n", "client_1.software_manager.file_system.show(full=True)" @@ -759,25 +498,9 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+---------------------------------------------------------------------------------+\n", - "| web_server File System |\n", - "+---------------------+---------+---------------+-----------------------+---------+\n", - "| File Path | Size | Health status | Visible health status | Deleted |\n", - "+---------------------+---------+---------------+-----------------------+---------+\n", - "| primaite/index.html | 15.0 KB | GOOD | NONE | False |\n", - "| root | 0 B | GOOD | NONE | False |\n", - "| spoils/database.db | 4.77 MB | GOOD | NONE | False |\n", - "+---------------------+---------+---------------+-----------------------+---------+\n" - ] - } - ], + "outputs": [], "source": [ "web_server: Computer = env.game.simulation.network.get_node_by_hostname(\"web_server\")\n", "web_server.software_manager.file_system.show(full=True)" @@ -814,48 +537,18 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(0,\n", - " 0.0,\n", - " False,\n", - " False,\n", - " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=6, action='c2_server_ransomware_launch', parameters={'node_name': 'client_1'}, request=['network', 'node', 'client_1', 'application', 'C2Server', 'ransomware_launch'], response=RequestResponse(status='success', data={}), reward=0.0, reward_info={})}})" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "env.step(7)" ] }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+----------------------------------------------------------------------------------+\n", - "| database_server File System |\n", - "+----------------------+---------+---------------+-----------------------+---------+\n", - "| File Path | Size | Health status | Visible health status | Deleted |\n", - "+----------------------+---------+---------------+-----------------------+---------+\n", - "| database/database.db | 4.77 MB | CORRUPT | NONE | False |\n", - "| root | 0 B | GOOD | NONE | False |\n", - "+----------------------+---------+---------------+-----------------------+---------+\n" - ] - } - ], + "outputs": [], "source": [ "database_server: Server = env.game.simulation.network.get_node_by_hostname(\"database_server\")\n", "database_server.software_manager.file_system.show(full=True)" @@ -874,7 +567,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1002,17 +695,9 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-04 15:37:36,689: PrimaiteGymEnv RNG seed = None\n" - ] - } - ], + "outputs": [], "source": [ "with open(data_manipulation_config_path()) as f:\n", " cfg = yaml.safe_load(f)\n", @@ -1026,7 +711,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1069,18 +754,9 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-04 15:37:37,079: Resetting environment, episode 0, avg. reward: 0.0\n", - "2025-02-04 15:37:37,082: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-04/15-37-31/agent_actions/episode_0.json\n" - ] - } - ], + "outputs": [], "source": [ "# Resetting the environment and capturing the default observation space.\n", "blue_env.reset()\n", @@ -1089,20 +765,9 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 23, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Setting up the C2 Suite via the simulation API.\n", "\n", @@ -1123,7 +788,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1133,26 +798,9 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 2\n", - "root['NODES']['HOST0']['APPLICATIONS'][1]['operating_status']: 0 -> 1\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", - "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", - "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" - ] - } - ], + "outputs": [], "source": [ "display_obs_diffs(default_obs, c2_configuration_obs, blue_env.game.step_counter)" ] @@ -1172,20 +820,9 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "RequestResponse(status='success', data={0: RequestResponse(status='success', data={})})" - ] - }, - "execution_count": 26, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Installing RansomwareScript via C2 Terminal Commands\n", "ransomware_install_command = {\"commands\":[[\"software_manager\", \"application\", \"install\", \"RansomwareScript\"]],\n", @@ -1196,20 +833,9 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "RequestResponse(status='success', data={})" - ] - }, - "execution_count": 27, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Configuring the RansomwareScript\n", "ransomware_config = {\"server_ip_address\": \"192.168.1.14\", \"payload\": \"ENCRYPT\"}\n", @@ -1218,7 +844,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1228,28 +854,9 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 7\n", - "root['NODES']['HOST0']['APPLICATIONS'][1]['operating_status']: 0 -> 1\n", - "root['NODES']['HOST0']['APPLICATIONS'][2]['operating_status']: 0 -> 3\n", - "root['NODES']['HOST0']['users']['local_login']: 0 -> 1\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", - "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", - "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" - ] - } - ], + "outputs": [], "source": [ "display_obs_diffs(default_obs, c2_ransomware_obs, env.game.step_counter)" ] @@ -1265,7 +872,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1281,27 +888,16 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "RequestResponse(status='success', data={})" - ] - }, - "execution_count": 31, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "c2_server.send_command(given_command=C2Command.DATA_EXFILTRATION, command_options=exfil_options)" ] }, { "cell_type": "code", - "execution_count": 32, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1310,22 +906,9 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 7\n", - "root['NODES']['HOST0']['APPLICATIONS'][2]['operating_status']: 3 -> 1\n", - "root['NODES']['HOST1']['users']['remote_sessions']: 0 -> 1\n" - ] - } - ], + "outputs": [], "source": [ "display_obs_diffs(c2_ransomware_obs, c2_exfil_obs, env.game.step_counter)" ] @@ -1341,20 +924,9 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "RequestResponse(status='success', data={})" - ] - }, - "execution_count": 34, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Configuring the RansomwareScript\n", "ransomware_config = {\"server_ip_address\": \"192.168.1.14\", \"payload\": \"ENCRYPT\"}\n", @@ -1363,20 +935,9 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "RequestResponse(status='success', data={})" - ] - }, - "execution_count": 35, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Waiting for the ransomware to finish installing and then launching the RansomwareScript.\n", "blue_env.step(0)\n", @@ -1385,7 +946,7 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1395,30 +956,9 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 6\n", - "root['NODES']['HOST0']['APPLICATIONS'][2]['operating_status']: 3 -> 1\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['NODES']['HOST1']['users']['remote_sessions']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" - ] - } - ], + "outputs": [], "source": [ "display_obs_diffs(c2_ransomware_obs, c2_final_obs, blue_env.game.step_counter)" ] @@ -1434,7 +974,7 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1466,200 +1006,16 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-04 15:37:38,317: Resetting environment, episode 1, avg. reward: 0.0\n", - "2025-02-04 15:37:38,321: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-04/15-37-31/agent_actions/episode_1.json\n" - ] - }, - { - "data": { - "text/plain": [ - "({'NODES': {'HOST0': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 1,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'HOST1': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 1,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'HOST2': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 0,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'HOST3': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 0,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'ROUTER0': {'ACL': {1: {'position': 0,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 2: {'position': 1,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 3: {'position': 2,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 4: {'position': 3,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 5: {'position': 4,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 6: {'position': 5,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 7: {'position': 6,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 8: {'position': 7,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 9: {'position': 8,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 10: {'position': 9,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0}},\n", - " 'PORTS': {1: {'operating_status': 1},\n", - " 2: {'operating_status': 1},\n", - " 3: {'operating_status': 2}},\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}}},\n", - " 'LINKS': {1: {'PROTOCOLS': {'ALL': 1}},\n", - " 2: {'PROTOCOLS': {'ALL': 1}},\n", - " 3: {'PROTOCOLS': {'ALL': 0}},\n", - " 4: {'PROTOCOLS': {'ALL': 1}},\n", - " 5: {'PROTOCOLS': {'ALL': 1}},\n", - " 6: {'PROTOCOLS': {'ALL': 1}},\n", - " 7: {'PROTOCOLS': {'ALL': 1}},\n", - " 8: {'PROTOCOLS': {'ALL': 1}},\n", - " 9: {'PROTOCOLS': {'ALL': 1}},\n", - " 10: {'PROTOCOLS': {'ALL': 0}}},\n", - " 'ICS': 0},\n", - " {})" - ] - }, - "execution_count": 39, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "blue_env.reset()" ] }, { "cell_type": "code", - "execution_count": 40, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1678,7 +1034,7 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1695,33 +1051,9 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+-------------------------------------------------------------------------------------+\n", - "| web_server Software Manager |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| Name | Type | Operating State | Health State | Port | Protocol |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", - "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", - "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", - "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", - "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", - "| NMAP | Application | RUNNING | GOOD | None | none |\n", - "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", - "| UserManager | Service | RUNNING | GOOD | None | none |\n", - "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", - "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", - "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n" - ] - } - ], + "outputs": [], "source": [ "blue_env.step(0)\n", "web_server.software_manager.show()" @@ -1729,26 +1061,9 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 3\n", - "root['NODES']['HOST0']['APPLICATIONS'][1]['operating_status']: 1 -> 0\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", - "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", - "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" - ] - } - ], + "outputs": [], "source": [ "display_obs_diffs(pre_blue_action_obs, post_blue_action_obs, blue_env.game.step_counter)" ] @@ -1762,20 +1077,9 @@ }, { "cell_type": "code", - "execution_count": 44, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "RequestResponse(status='failure', data={'Reason': 'Command sent to the C2 Beacon but no response was ever received.'})" - ] - }, - "execution_count": 44, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Attempting to install the C2 RansomwareScript\n", "ransomware_install_command = {\"commands\":[[\"software_manager\", \"application\", \"install\", \"RansomwareScript\"]],\n", @@ -1797,200 +1101,16 @@ }, { "cell_type": "code", - "execution_count": 45, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-04 15:37:38,778: Resetting environment, episode 2, avg. reward: 0.0\n", - "2025-02-04 15:37:38,781: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-04/15-37-31/agent_actions/episode_2.json\n" - ] - }, - { - "data": { - "text/plain": [ - "({'NODES': {'HOST0': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 1,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'HOST1': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 1,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'HOST2': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 0,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'HOST3': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 0,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'ROUTER0': {'ACL': {1: {'position': 0,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 2: {'position': 1,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 3: {'position': 2,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 4: {'position': 3,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 5: {'position': 4,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 6: {'position': 5,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 7: {'position': 6,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 8: {'position': 7,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 9: {'position': 8,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 10: {'position': 9,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0}},\n", - " 'PORTS': {1: {'operating_status': 1},\n", - " 2: {'operating_status': 1},\n", - " 3: {'operating_status': 2}},\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}}},\n", - " 'LINKS': {1: {'PROTOCOLS': {'ALL': 1}},\n", - " 2: {'PROTOCOLS': {'ALL': 1}},\n", - " 3: {'PROTOCOLS': {'ALL': 0}},\n", - " 4: {'PROTOCOLS': {'ALL': 1}},\n", - " 5: {'PROTOCOLS': {'ALL': 1}},\n", - " 6: {'PROTOCOLS': {'ALL': 1}},\n", - " 7: {'PROTOCOLS': {'ALL': 1}},\n", - " 8: {'PROTOCOLS': {'ALL': 1}},\n", - " 9: {'PROTOCOLS': {'ALL': 1}},\n", - " 10: {'PROTOCOLS': {'ALL': 0}}},\n", - " 'ICS': 0},\n", - " {})" - ] - }, - "execution_count": 45, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "blue_env.reset()" ] }, { "cell_type": "code", - "execution_count": 46, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -2009,7 +1129,7 @@ }, { "cell_type": "code", - "execution_count": 47, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -2026,17 +1146,9 @@ }, { "cell_type": "code", - "execution_count": 48, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "NodeOperatingState.SHUTTING_DOWN\n" - ] - } - ], + "outputs": [], "source": [ "web_server = blue_env.game.simulation.network.get_node_by_hostname(\"web_server\")\n", "print(web_server.operating_state)" @@ -2044,48 +1156,18 @@ }, { "cell_type": "code", - "execution_count": 49, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 2\n", - "root['NODES']['HOST0']['operating_status']: 1 -> 4\n", - "root['NODES']['HOST0']['APPLICATIONS'][1]['operating_status']: 1 -> 0\n", - "root['NODES']['HOST0']['NICS'][1]['nic_status']: 1 -> 0\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", - "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", - "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" - ] - } - ], + "outputs": [], "source": [ "display_obs_diffs(pre_blue_action_obs, post_blue_action_obs, blue_env.game.step_counter)" ] }, { "cell_type": "code", - "execution_count": 50, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "RequestResponse(status='failure', data={'Reason': 'Command sent to the C2 Beacon but no response was ever received.'})" - ] - }, - "execution_count": 50, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Attempting to install the C2 RansomwareScript\n", "ransomware_install_command = {\"commands\":[\"software_manager\", \"application\", \"install\", \"RansomwareScript\"],\n", @@ -2109,200 +1191,16 @@ }, { "cell_type": "code", - "execution_count": 51, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-04 15:37:39,257: Resetting environment, episode 3, avg. reward: 0.0\n", - "2025-02-04 15:37:39,260: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-04/15-37-31/agent_actions/episode_3.json\n" - ] - }, - { - "data": { - "text/plain": [ - "({'NODES': {'HOST0': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 1,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'HOST1': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 1,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'HOST2': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 0,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'HOST3': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 0,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'ROUTER0': {'ACL': {1: {'position': 0,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 2: {'position': 1,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 3: {'position': 2,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 4: {'position': 3,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 5: {'position': 4,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 6: {'position': 5,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 7: {'position': 6,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 8: {'position': 7,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 9: {'position': 8,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 10: {'position': 9,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0}},\n", - " 'PORTS': {1: {'operating_status': 1},\n", - " 2: {'operating_status': 1},\n", - " 3: {'operating_status': 2}},\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}}},\n", - " 'LINKS': {1: {'PROTOCOLS': {'ALL': 1}},\n", - " 2: {'PROTOCOLS': {'ALL': 1}},\n", - " 3: {'PROTOCOLS': {'ALL': 0}},\n", - " 4: {'PROTOCOLS': {'ALL': 1}},\n", - " 5: {'PROTOCOLS': {'ALL': 1}},\n", - " 6: {'PROTOCOLS': {'ALL': 1}},\n", - " 7: {'PROTOCOLS': {'ALL': 1}},\n", - " 8: {'PROTOCOLS': {'ALL': 1}},\n", - " 9: {'PROTOCOLS': {'ALL': 1}},\n", - " 10: {'PROTOCOLS': {'ALL': 0}}},\n", - " 'ICS': 0},\n", - " {})" - ] - }, - "execution_count": 51, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "blue_env.reset()" ] }, { "cell_type": "code", - "execution_count": 52, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -2321,7 +1219,7 @@ }, { "cell_type": "code", - "execution_count": 53, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -2338,41 +1236,9 @@ }, { "cell_type": "code", - "execution_count": 54, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+---------------------------------------------------------------+\n", - "| router_1 Network Interfaces |\n", - "+------+-------------------+-----------------+-------+----------+\n", - "| Port | MAC Address | Address | Speed | Status |\n", - "+------+-------------------+-----------------+-------+----------+\n", - "| 1 | dd:bc:17:75:a2:c4 | 192.168.1.1/24 | 100.0 | Enabled |\n", - "| 2 | 00:f2:f5:65:51:75 | 192.168.10.1/24 | 100.0 | Enabled |\n", - "| 3 | de:7d:ac:0e:58:e1 | 127.0.0.1/8 | 100.0 | Disabled |\n", - "| 4 | 8a:b4:55:12:c7:9e | 127.0.0.1/8 | 100.0 | Disabled |\n", - "| 5 | a5:ef:c6:53:97:c4 | 127.0.0.1/8 | 100.0 | Disabled |\n", - "+------+-------------------+-----------------+-------+----------+\n", - "+------------------------------------------------------------------------------------------------------------------------+\n", - "| router_1 Access Control List |\n", - "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n", - "| Index | Action | Protocol | Src IP | Src Wildcard | Src Port | Dst IP | Dst Wildcard | Dst Port | Matched |\n", - "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n", - "| 1 | DENY | ANY | 192.168.10.21 | 0.0.0.1 | 80 | 192.168.1.12 | 0.0.0.1 | 80 | 0 |\n", - "| 18 | PERMIT | ANY | ANY | ANY | 5432 | ANY | ANY | 5432 | 0 |\n", - "| 19 | PERMIT | ANY | ANY | ANY | 53 | ANY | ANY | 53 | 0 |\n", - "| 20 | PERMIT | ANY | ANY | ANY | 21 | ANY | ANY | 21 | 0 |\n", - "| 21 | PERMIT | ANY | ANY | ANY | 80 | ANY | ANY | 80 | 4 |\n", - "| 22 | PERMIT | ANY | ANY | ANY | 219 | ANY | ANY | 219 | 10 |\n", - "| 23 | PERMIT | icmp | ANY | ANY | ANY | ANY | ANY | ANY | 0 |\n", - "| 24 | DENY | ANY | ANY | ANY | ANY | ANY | ANY | ANY | 0 |\n", - "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n" - ] - } - ], + "outputs": [], "source": [ "router_1: Router = blue_env.game.simulation.network.get_node_by_hostname(\"router_1\")\n", "router_1.acl.show()" @@ -2387,20 +1253,9 @@ }, { "cell_type": "code", - "execution_count": 55, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "RequestResponse(status='failure', data={'Reason': 'Command sent to the C2 Beacon but no response was ever received.'})" - ] - }, - "execution_count": 55, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "blue_env.step(0)\n", "\n", @@ -2411,30 +1266,9 @@ }, { "cell_type": "code", - "execution_count": 56, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+------------------------------------------------------------------------------------------------------------------------+\n", - "| router_1 Access Control List |\n", - "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n", - "| Index | Action | Protocol | Src IP | Src Wildcard | Src Port | Dst IP | Dst Wildcard | Dst Port | Matched |\n", - "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n", - "| 1 | DENY | ANY | 192.168.10.21 | 0.0.0.1 | 80 | 192.168.1.12 | 0.0.0.1 | 80 | 2 |\n", - "| 18 | PERMIT | ANY | ANY | ANY | 5432 | ANY | ANY | 5432 | 0 |\n", - "| 19 | PERMIT | ANY | ANY | ANY | 53 | ANY | ANY | 53 | 0 |\n", - "| 20 | PERMIT | ANY | ANY | ANY | 21 | ANY | ANY | 21 | 0 |\n", - "| 21 | PERMIT | ANY | ANY | ANY | 80 | ANY | ANY | 80 | 4 |\n", - "| 22 | PERMIT | ANY | ANY | ANY | 219 | ANY | ANY | 219 | 10 |\n", - "| 23 | PERMIT | icmp | ANY | ANY | ANY | ANY | ANY | ANY | 0 |\n", - "| 24 | DENY | ANY | ANY | ANY | ANY | ANY | ANY | ANY | 0 |\n", - "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n" - ] - } - ], + "outputs": [], "source": [ "router_1.acl.show()" ] @@ -2448,58 +1282,18 @@ }, { "cell_type": "code", - "execution_count": 57, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+-------------------------------------------------------------------------------------+\n", - "| web_server Software Manager |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| Name | Type | Operating State | Health State | Port | Protocol |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", - "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", - "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", - "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", - "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", - "| NMAP | Application | RUNNING | GOOD | None | none |\n", - "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", - "| UserManager | Service | RUNNING | GOOD | None | none |\n", - "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", - "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", - "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", - "| C2Beacon | Application | RUNNING | GOOD | None | tcp |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n" - ] - } - ], + "outputs": [], "source": [ "web_server.software_manager.show()" ] }, { "cell_type": "code", - "execution_count": 58, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+----------------------------------------------------------------------------------+\n", - "| database_server File System |\n", - "+----------------------+---------+---------------+-----------------------+---------+\n", - "| File Path | Size | Health status | Visible health status | Deleted |\n", - "+----------------------+---------+---------------+-----------------------+---------+\n", - "| database/database.db | 4.77 MB | GOOD | NONE | False |\n", - "| root | 0 B | GOOD | NONE | False |\n", - "+----------------------+---------+---------------+-----------------------+---------+\n" - ] - } - ], + "outputs": [], "source": [ "database_server: Server = blue_env.game.simulation.network.get_node_by_hostname(\"database_server\")\n", "database_server.software_manager.file_system.show(full=True)" @@ -2507,33 +1301,9 @@ }, { "cell_type": "code", - "execution_count": 59, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 3\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", - "root['NODES']['ROUTER0']['ACL'][1]['permission']: 0 -> 2\n", - "root['NODES']['ROUTER0']['ACL'][1]['source_ip_id']: 0 -> 7\n", - "root['NODES']['ROUTER0']['ACL'][1]['source_wildcard_id']: 0 -> 2\n", - "root['NODES']['ROUTER0']['ACL'][1]['source_port_id']: 0 -> 2\n", - "root['NODES']['ROUTER0']['ACL'][1]['dest_ip_id']: 0 -> 3\n", - "root['NODES']['ROUTER0']['ACL'][1]['dest_wildcard_id']: 0 -> 2\n", - "root['NODES']['ROUTER0']['ACL'][1]['dest_port_id']: 0 -> 2\n", - "root['NODES']['ROUTER0']['ACL'][1]['protocol_id']: 0 -> 1\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", - "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", - "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" - ] - } - ], + "outputs": [], "source": [ "display_obs_diffs(pre_blue_action_obs, post_blue_action_obs, blue_env.game.step_counter)" ] @@ -2597,17 +1367,9 @@ }, { "cell_type": "code", - "execution_count": 60, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-04 15:37:40,175: PrimaiteGymEnv RNG seed = None\n" - ] - } - ], + "outputs": [], "source": [ "with open(data_manipulation_config_path()) as f:\n", " cfg = yaml.safe_load(f)\n", @@ -2628,7 +1390,7 @@ }, { "cell_type": "code", - "execution_count": 61, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -2656,30 +1418,9 @@ }, { "cell_type": "code", - "execution_count": 62, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", - "| C2Beacon Running Status |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| False | None | 0 | 5 | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "+-----------------------------------------------------------------------------------------------------+\n", - "| C2Server Running Status |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n", - "| False | None | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n" - ] - } - ], + "outputs": [], "source": [ "env.step(2) # Agent Action Equivalent to c2_beacon.configure(c2_server_ip_address=\"192.168.10.21\")\n", "env.step(3) # Agent action Equivalent to c2_beacon.establish()\n", @@ -2696,30 +1437,9 @@ }, { "cell_type": "code", - "execution_count": 63, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", - "| C2Beacon Running Status |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| False | None | 0 | 5 | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "+-----------------------------------------------------------------------------------------------------+\n", - "| C2Server Running Status |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n", - "| False | None | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n" - ] - } - ], + "outputs": [], "source": [ "env.step(9) # Equivalent of to c2_beacon.configure(c2_server_ip_address=\"192.168.10.22\")\n", "env.step(3)\n", @@ -2737,23 +1457,9 @@ }, { "cell_type": "code", - "execution_count": 64, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+-----------------------------------------------------------------------------------------------------+\n", - "| C2Server Running Status |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n", - "| False | None | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n" - ] - } - ], + "outputs": [], "source": [ "for i in range(6):\n", " env.step(0)\n", @@ -2776,17 +1482,9 @@ }, { "cell_type": "code", - "execution_count": 65, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-04 15:37:40,588: PrimaiteGymEnv RNG seed = None\n" - ] - } - ], + "outputs": [], "source": [ "with open(data_manipulation_config_path()) as f:\n", " cfg = yaml.safe_load(f)\n", @@ -2802,7 +1500,7 @@ }, { "cell_type": "code", - "execution_count": 66, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -2826,23 +1524,9 @@ }, { "cell_type": "code", - "execution_count": 67, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", - "| C2Beacon Running Status |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| True | 192.168.10.21 | 0 | 5 | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n" - ] - } - ], + "outputs": [], "source": [ "c2_beacon.configure(c2_server_ip_address=\"192.168.10.21\")\n", "c2_beacon.establish()\n", @@ -2860,72 +1544,9 @@ }, { "cell_type": "code", - "execution_count": 68, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 4\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 5\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 6\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 7\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 8\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 9\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 10\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 11\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 12\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 13\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" - ] - } - ], + "outputs": [], "source": [ "for i in range(10):\n", " keep_alive_obs, _, _, _, _ = blue_config_env.step(0)\n", @@ -2941,23 +1562,9 @@ }, { "cell_type": "code", - "execution_count": 69, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", - "| C2Beacon Running Status |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| True | 192.168.10.21 | 0 | 1 | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n" - ] - } - ], + "outputs": [], "source": [ "c2_beacon.configure(c2_server_ip_address=\"192.168.10.21\", keep_alive_frequency=1)\n", "c2_beacon.establish()\n", @@ -2973,40 +1580,9 @@ }, { "cell_type": "code", - "execution_count": 70, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 14\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 15\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" - ] - } - ], + "outputs": [], "source": [ "# Comparing the OBS of the default frequency to a timestep frequency of 1\n", "for i in range(2):\n", @@ -3025,52 +1601,9 @@ }, { "cell_type": "code", - "execution_count": 71, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 16\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 17\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 18\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 19\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 20\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 21\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 22\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" - ] - } - ], + "outputs": [], "source": [ "c2_beacon.configure(c2_server_ip_address=\"192.168.10.21\", keep_alive_frequency=7)\n", "\n", @@ -3107,18 +1640,9 @@ }, { "cell_type": "code", - "execution_count": 72, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-04 15:37:41,322: Resetting environment, episode 0, avg. reward: 0.0\n", - "2025-02-04 15:37:41,325: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-04/15-37-31/agent_actions/episode_0.json\n" - ] - } - ], + "outputs": [], "source": [ "blue_config_env.reset()\n", "\n", @@ -3140,28 +1664,9 @@ }, { "cell_type": "code", - "execution_count": 73, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 5\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" - ] - } - ], + "outputs": [], "source": [ "# Capturing default C2 Traffic\n", "for i in range(3):\n", @@ -3179,23 +1684,9 @@ }, { "cell_type": "code", - "execution_count": 74, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", - "| C2Beacon Running Status |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| True | 192.168.10.21 | 0 | 5 | udp | 53 |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n" - ] - } - ], + "outputs": [], "source": [ "from primaite.utils.validation.ip_protocol import PROTOCOL_LOOKUP\n", "from primaite.utils.validation.port import PORT_LOOKUP\n", @@ -3208,28 +1699,9 @@ }, { "cell_type": "code", - "execution_count": 75, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 10\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 1 -> 0\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 1 -> 0\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['udp'][53]['inbound']: 0 -> 1\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['udp'][53]['outbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 1 -> 0\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 1 -> 0\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['udp'][53]['inbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['udp'][53]['outbound']: 0 -> 1\n" - ] - } - ], + "outputs": [], "source": [ "# Capturing UDP C2 Traffic\n", "for i in range(5):\n", diff --git a/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb b/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb index 4a34619f..41b75e43 100644 --- a/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb +++ b/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb @@ -382,7 +382,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": { "tags": [] }, @@ -394,7 +394,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": { "tags": [] }, @@ -450,7 +450,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ diff --git a/src/primaite/notebooks/Getting-Information-Out-Of-PrimAITE.ipynb b/src/primaite/notebooks/Getting-Information-Out-Of-PrimAITE.ipynb index d7d60d76..c7fdafad 100644 --- a/src/primaite/notebooks/Getting-Information-Out-Of-PrimAITE.ipynb +++ b/src/primaite/notebooks/Getting-Information-Out-Of-PrimAITE.ipynb @@ -6,7 +6,7 @@ "source": [ "# Getting information out of PrimAITE\n", "\n", - "© Crown-owned copyright 2024, Defence Science and Technology Laboratory UK\n" + "© Crown-owned copyright 2025, Defence Science and Technology Laboratory UK\n" ] }, { @@ -32,8 +32,6 @@ "from primaite.session.environment import PrimaiteGymEnv\n", "from primaite.simulator.network.hardware.nodes.host.computer import Computer\n", "from notebook.services.config import ConfigManager\n", - "from primaite.game.agent.scripted_agents import probabilistic_agent\n", - "\n", "\n", "cm = ConfigManager().update('notebook', {'limit_output': 50}) # limit output lines to 50 - for neatness\n", "\n", diff --git a/src/primaite/notebooks/Privilege-Escalation-and-Data-Loss-Example.ipynb b/src/primaite/notebooks/Privilege-Escalation-and-Data-Loss-Example.ipynb index 35d3813a..cbb898ea 100644 --- a/src/primaite/notebooks/Privilege-Escalation-and-Data-Loss-Example.ipynb +++ b/src/primaite/notebooks/Privilege-Escalation-and-Data-Loss-Example.ipynb @@ -6,7 +6,7 @@ "source": [ "# Simulating Privilege Escalation and Data Loss Using SSH and ACLs Manipulation\n", "\n", - "© Crown-owned copyright 2024, Defence Science and Technology Laboratory UK\n", + "© Crown-owned copyright 2025, Defence Science and Technology Laboratory UK\n", "\n", "## Overview\n", "\n", @@ -62,7 +62,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": { "tags": [] }, @@ -77,7 +77,8 @@ "from primaite.simulator.network.hardware.nodes.host.server import Server\n", "from primaite.simulator.system.applications.database_client import DatabaseClient\n", "from primaite.simulator.system.applications.web_browser import WebBrowser\n", - "from primaite.simulator.system.services.database.database_service import DatabaseService\n" + "from primaite.simulator.system.services.database.database_service import DatabaseService\n", + "from primaite.simulator.network.hardware.nodes.network import firewall\n" ] }, { @@ -89,7 +90,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": { "tags": [] }, @@ -112,7 +113,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": { "tags": [] }, diff --git a/src/primaite/notebooks/Requests-and-Responses.ipynb b/src/primaite/notebooks/Requests-and-Responses.ipynb index 83aed07c..01a6cffa 100644 --- a/src/primaite/notebooks/Requests-and-Responses.ipynb +++ b/src/primaite/notebooks/Requests-and-Responses.ipynb @@ -6,7 +6,7 @@ "source": [ "# Requests and Responses\n", "\n", - "© Crown-owned copyright 2024, Defence Science and Technology Laboratory UK\n", + "© Crown-owned copyright 2025, Defence Science and Technology Laboratory UK\n", "\n", "Agents interact with the PrimAITE simulation via the Request system.\n" ] @@ -36,7 +36,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -47,17 +47,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "sim = Simulation()\n", "sim.network.add_node(\n", - " HostNode(\n", - " hostname=\"client\",\n", - " ip_address='10.0.0.1',\n", - " subnet_mask='255.255.255.0',\n", - " operating_state=NodeOperatingState.ON)\n", + " HostNode.from_config(\n", + " config = {\n", + " 'type': \"hostnode\",\n", + " 'hostname': \"client\",\n", + " 'ip_address': '10.0.0.1',\n", + " 'subnet_mask': '255.255.255.0',\n", + " 'operating_state': NodeOperatingState.ON,\n", + " }\n", + " )\n", ")\n", "client = sim.network.get_node_by_hostname('client')\n" ] @@ -210,7 +214,7 @@ ], "metadata": { "kernelspec": { - "display_name": "venv", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, diff --git a/src/primaite/notebooks/Terminal-Processing.ipynb b/src/primaite/notebooks/Terminal-Processing.ipynb index 9aa4e96a..48318c13 100644 --- a/src/primaite/notebooks/Terminal-Processing.ipynb +++ b/src/primaite/notebooks/Terminal-Processing.ipynb @@ -29,7 +29,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -39,12 +39,43 @@ "from primaite.simulator.system.applications.red_applications.ransomware_script import RansomwareScript\n", "from primaite.simulator.system.services.terminal.terminal import RemoteTerminalConnection\n", "\n", + "# print(dir(Computer))\n", + "# node_a = Computer.from_config(\n", + "# config = {\n", + "# \"type\": \"computer\",\n", + "# \"hostname\": \"node_a\",\n", + "# \"ip_address\": \"192.168.0.10\",\n", + "# \"subnet_mask\": \"255.255.255.0\",\n", + "# \"startup_duration\": 0,\n", + "# }\n", + "# )\n", + "# print(f\"{node_a=}\")\n", + "\n", "def basic_network() -> Network:\n", " \"\"\"Utility function for creating a default network to demonstrate Terminal functionality\"\"\"\n", " network = Network()\n", - " node_a = Computer(hostname=\"node_a\", ip_address=\"192.168.0.10\", subnet_mask=\"255.255.255.0\", start_up_duration=0)\n", + " # node_a = Computer(hostname=\"node_a\", ip_address=\"192.168.0.10\", subnet_mask=\"255.255.255.0\", start_up_duration=0)\n", + " node_a = Computer.from_config(\n", + " config = {\n", + " \"type\": \"computer\",\n", + " \"hostname\": \"node_a\",\n", + " \"ip_address\": \"192.168.0.10\",\n", + " \"subnet_mask\": \"255.255.255.0\",\n", + " \"startup_duration\": 0,\n", + " }\n", + " )\n", + " print(f\"{node_a=}\")\n", " node_a.power_on()\n", - " node_b = Computer(hostname=\"node_b\", ip_address=\"192.168.0.11\", subnet_mask=\"255.255.255.0\", start_up_duration=0)\n", + " # node_b = Computer(hostname=\"node_b\", ip_address=\"192.168.0.11\", subnet_mask=\"255.255.255.0\", start_up_duration=0)\n", + " node_b = Computer.from_config(\n", + " config = {\n", + " \"type\": \"computer\",\n", + " \"hostname\": \"node_b\",\n", + " \"ip_address\": \"192.168.0.11\",\n", + " \"subnet_mask\": \"255.255.255.0\",\n", + " \"startup_duration\": 0,\n", + " }\n", + " )\n", " node_b.power_on()\n", " network.connect(node_a.network_interface[1], node_b.network_interface[1])\n", " return network" @@ -84,7 +115,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -211,7 +242,7 @@ ], "metadata": { "kernelspec": { - "display_name": ".venv", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -225,7 +256,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.11" + "version": "3.10.12" } }, "nbformat": 4, diff --git a/src/primaite/notebooks/Training-an-RLLIB-MARL-System.ipynb b/src/primaite/notebooks/Training-an-RLLIB-MARL-System.ipynb index dadb399e..87d9c377 100644 --- a/src/primaite/notebooks/Training-an-RLLIB-MARL-System.ipynb +++ b/src/primaite/notebooks/Training-an-RLLIB-MARL-System.ipynb @@ -6,7 +6,7 @@ "source": [ "# Train a Multi agent system using RLLIB\n", "\n", - "© Crown-owned copyright 2024, Defence Science and Technology Laboratory UK\n", + "© Crown-owned copyright 2025, Defence Science and Technology Laboratory UK\n", "\n", "This notebook will demonstrate how to use the `PrimaiteRayMARLEnv` to train a very basic system with two PPO agents." ] @@ -57,7 +57,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -103,20 +103,7 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "*** SIGTERM received at time=1737996337 on cpu 7 ***\n", - "PC: @ 0x7f3649b0fe2e (unknown) epoll_wait\n", - " @ 0x7f3649a2c520 (unknown) (unknown)\n", - "[2025-01-27 16:45:37,381 E 117142 117142] logging.cc:440: *** SIGTERM received at time=1737996337 on cpu 7 ***\n", - "[2025-01-27 16:45:37,381 E 117142 117142] logging.cc:440: PC: @ 0x7f3649b0fe2e (unknown) epoll_wait\n", - "[2025-01-27 16:45:37,381 E 117142 117142] logging.cc:440: @ 0x7f3649a2c520 (unknown) (unknown)\n" - ] - } - ], + "outputs": [], "source": [ "eval = algo.evaluate()" ] @@ -127,18 +114,6 @@ "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.12" } }, "nbformat": 4, diff --git a/src/primaite/notebooks/Training-an-RLLib-Agent.ipynb b/src/primaite/notebooks/Training-an-RLLib-Agent.ipynb index 64a9e7ab..79740bca 100644 --- a/src/primaite/notebooks/Training-an-RLLib-Agent.ipynb +++ b/src/primaite/notebooks/Training-an-RLLib-Agent.ipynb @@ -6,7 +6,7 @@ "source": [ "# Train a Single agent system using RLLib\n", "\n", - "© Crown-owned copyright 2024, Defence Science and Technology Laboratory UK\n", + "© Crown-owned copyright 2025, Defence Science and Technology Laboratory UK\n", "\n", "This notebook will demonstrate how to use PrimaiteRayEnv to train a basic PPO agent." ] @@ -52,7 +52,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -98,20 +98,7 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "*** SIGTERM received at time=1737996055 on cpu 1 ***\n", - "PC: @ 0x7f6e254a6e2e (unknown) epoll_wait\n", - " @ 0x7f6e253c3520 (unknown) (unknown)\n", - "[2025-01-27 16:40:55,343 E 114171 114171] logging.cc:440: *** SIGTERM received at time=1737996055 on cpu 1 ***\n", - "[2025-01-27 16:40:55,343 E 114171 114171] logging.cc:440: PC: @ 0x7f6e254a6e2e (unknown) epoll_wait\n", - "[2025-01-27 16:40:55,344 E 114171 114171] logging.cc:440: @ 0x7f6e253c3520 (unknown) (unknown)\n" - ] - } - ], + "outputs": [], "source": [ "eval = algo.evaluate()" ] @@ -122,18 +109,6 @@ "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.12" } }, "nbformat": 4, diff --git a/src/primaite/notebooks/Training-an-SB3-Agent.ipynb b/src/primaite/notebooks/Training-an-SB3-Agent.ipynb index d3492f92..10328989 100644 --- a/src/primaite/notebooks/Training-an-SB3-Agent.ipynb +++ b/src/primaite/notebooks/Training-an-SB3-Agent.ipynb @@ -6,7 +6,7 @@ "source": [ "# Training an SB3 Agent\n", "\n", - "© Crown-owned copyright 2024, Defence Science and Technology Laboratory UK\n", + "© Crown-owned copyright 2025, Defence Science and Technology Laboratory UK\n", "\n", "This notebook will demonstrate how to use primaite to create and train a PPO agent, using a pre-defined configuration file." ] diff --git a/src/primaite/notebooks/Using-Episode-Schedules.ipynb b/src/primaite/notebooks/Using-Episode-Schedules.ipynb index 311fe4fb..44305266 100644 --- a/src/primaite/notebooks/Using-Episode-Schedules.ipynb +++ b/src/primaite/notebooks/Using-Episode-Schedules.ipynb @@ -6,7 +6,7 @@ "source": [ "# Using Episode Schedules\n", "\n", - "© Crown-owned copyright 2024, Defence Science and Technology Laboratory UK\n", + "© Crown-owned copyright 2025, Defence Science and Technology Laboratory UK\n", "\n", "PrimAITE supports the ability to use different variations on a scenario at different episodes. This can be used to increase \n", "domain randomisation to prevent overfitting, or to set up curriculum learning to train agents to perform more complicated tasks.\n", @@ -40,7 +40,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -325,7 +325,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "metadata": {}, "outputs": [], "source": [ @@ -413,6 +413,18 @@ "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" } }, "nbformat": 4, diff --git a/src/primaite/notebooks/multi-processing.ipynb b/src/primaite/notebooks/multi-processing.ipynb index 798bf3ff..e56bf362 100644 --- a/src/primaite/notebooks/multi-processing.ipynb +++ b/src/primaite/notebooks/multi-processing.ipynb @@ -6,7 +6,7 @@ "source": [ "# Simple multi-processing demonstration\n", "\n", - "© Crown-owned copyright 2024, Defence Science and Technology Laboratory UK\n", + "© Crown-owned copyright 2025, Defence Science and Technology Laboratory UK\n", "\n", "This notebook uses SubprocVecEnv from SB3." ] @@ -29,7 +29,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -37,13 +37,12 @@ "from stable_baselines3 import PPO\n", "from stable_baselines3.common.utils import set_random_seed\n", "from stable_baselines3.common.vec_env import SubprocVecEnv\n", - "from primaite.session.environment import PrimaiteGymEnv\n", - "from primaite.game.agent.scripted_agents import probabilistic_agent\n" + "from primaite.session.environment import PrimaiteGymEnv\n" ] }, { "cell_type": "code", - "execution_count": 31, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -52,7 +51,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -69,7 +68,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -90,7 +89,7 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ diff --git a/src/primaite/simulator/network/hardware/base.py b/src/primaite/simulator/network/hardware/base.py index 6543d793..cc3d4150 100644 --- a/src/primaite/simulator/network/hardware/base.py +++ b/src/primaite/simulator/network/hardware/base.py @@ -1197,7 +1197,7 @@ class UserSessionManager(Service, identifier="UserSessionManager"): """Request should take the form [username, password, remote_ip_address].""" username, password, remote_ip_address = request response = RequestResponse.from_bool(self.remote_login(username, password, remote_ip_address)) - response.data = {"remote_hostname": self.parent.hostname, "username": username} + response.data = {"remote_hostname": self.parent.config.hostname, "username": username} return response rm.add_request("remote_login", RequestType(func=_remote_login)) @@ -1230,7 +1230,7 @@ class UserSessionManager(Service, identifier="UserSessionManager"): if markdown: table.set_style(MARKDOWN) table.align = "l" - table.title = f"{self.parent.hostname} User Sessions" + table.title = f"{self.parent.config.hostname} User Sessions" def _add_session_to_table(user_session: UserSession): """ @@ -1627,9 +1627,7 @@ class Node(SimComponent, ABC): dns_server=kwargs["config"].dns_server, ) super().__init__(**kwargs) - self.operating_state = ( - NodeOperatingState.ON if not (p := kwargs["config"].operating_state) else NodeOperatingState[p.upper()] - ) + self.operating_state = NodeOperatingState.ON if not (p := kwargs["config"].operating_state) else p self._install_system_software() self.session_manager.node = self self.session_manager.software_manager = self.software_manager From 7b2a9c1d2a6d64d870de17fa0a3c8afc15641919 Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Wed, 5 Feb 2025 16:46:25 +0000 Subject: [PATCH 13/37] #3075: Pre-commit changes. --- src/primaite/notebooks/Requests-and-Responses.ipynb | 4 ++-- src/primaite/notebooks/Terminal-Processing.ipynb | 4 ++-- src/primaite/notebooks/Training-an-SB3-Agent.ipynb | 10 +++++----- src/primaite/notebooks/Using-Episode-Schedules.ipynb | 4 ++-- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/primaite/notebooks/Requests-and-Responses.ipynb b/src/primaite/notebooks/Requests-and-Responses.ipynb index 01a6cffa..48f1cf3e 100644 --- a/src/primaite/notebooks/Requests-and-Responses.ipynb +++ b/src/primaite/notebooks/Requests-and-Responses.ipynb @@ -36,7 +36,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -47,7 +47,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ diff --git a/src/primaite/notebooks/Terminal-Processing.ipynb b/src/primaite/notebooks/Terminal-Processing.ipynb index 48318c13..dc870db5 100644 --- a/src/primaite/notebooks/Terminal-Processing.ipynb +++ b/src/primaite/notebooks/Terminal-Processing.ipynb @@ -29,7 +29,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -115,7 +115,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ diff --git a/src/primaite/notebooks/Training-an-SB3-Agent.ipynb b/src/primaite/notebooks/Training-an-SB3-Agent.ipynb index 10328989..1bec0183 100644 --- a/src/primaite/notebooks/Training-an-SB3-Agent.ipynb +++ b/src/primaite/notebooks/Training-an-SB3-Agent.ipynb @@ -29,7 +29,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -41,7 +41,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -50,7 +50,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -101,7 +101,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -133,7 +133,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ diff --git a/src/primaite/notebooks/Using-Episode-Schedules.ipynb b/src/primaite/notebooks/Using-Episode-Schedules.ipynb index 44305266..4bc49a70 100644 --- a/src/primaite/notebooks/Using-Episode-Schedules.ipynb +++ b/src/primaite/notebooks/Using-Episode-Schedules.ipynb @@ -40,7 +40,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -325,7 +325,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ From 1a8c3b9471e591659eb69853a4865b1a23708ad8 Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Thu, 6 Feb 2025 16:42:26 +0000 Subject: [PATCH 14/37] #3075: Fix notebooks after test changes. --- ...ommand-and-Control-E2E-Demonstration.ipynb | 82 +++++++++---------- ...a-Manipulation-Customising-Red-Agent.ipynb | 16 ++-- .../Data-Manipulation-E2E-Demonstration.ipynb | 4 +- ...ege-Escalation-and-Data-Loss-Example.ipynb | 30 +++---- .../notebooks/Requests-and-Responses.ipynb | 13 +-- .../notebooks/Terminal-Processing.ipynb | 26 ++---- .../simulator/network/hardware/base.py | 4 +- .../network/hardware/nodes/host/host_node.py | 2 +- .../configs/basic_switched_network.yaml | 6 +- 9 files changed, 86 insertions(+), 97 deletions(-) diff --git a/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb b/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb index 17b7b79e..474dadff 100644 --- a/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb +++ b/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb @@ -59,30 +59,30 @@ "custom_c2_agent = \"\"\"\n", " - ref: CustomC2Agent\n", " team: RED\n", - " type: ProxyAgent\n", + " type: proxy-agent\n", "\n", " action_space:\n", " action_map:\n", " 0:\n", - " action: do_nothing\n", + " action: do-nothing\n", " options: {}\n", " 1:\n", - " action: node_application_install\n", + " action: node-application-install\n", " options:\n", " node_name: web_server\n", - " application_name: C2Beacon\n", + " application_name: c2-beacon\n", " 2:\n", - " action: configure_c2_beacon\n", + " action: configure-c2-beacon\n", " options:\n", " node_name: web_server\n", " c2_server_ip_address: 192.168.10.21\n", " 3:\n", - " action: node_application_execute\n", + " action: node-application-execute\n", " options:\n", " node_name: web_server\n", - " application_name: C2Beacon\n", + " application_name: c2-beacon\n", " 4:\n", - " action: c2_server_terminal_command\n", + " action: c2-server-terminal-command\n", " options:\n", " node_name: client_1\n", " ip_address:\n", @@ -93,7 +93,7 @@ " - software_manager\n", " - application\n", " - install\n", - " - RansomwareScript\n", + " - ransomware-script\n", " 5:\n", " action: c2-server-ransomware-configure\n", " options:\n", @@ -101,7 +101,7 @@ " server_ip_address: 192.168.1.14\n", " payload: ENCRYPT\n", " 6:\n", - " action: c2_server_data_exfiltrate\n", + " action: c2-server-data-exfiltrate\n", " options:\n", " node_name: client_1\n", " target_file_name: \"database.db\"\n", @@ -112,23 +112,23 @@ " password: admin\n", "\n", " 7:\n", - " action: c2_server_ransomware_launch\n", + " action: c2-server-ransomware-launch\n", " options:\n", " node_name: client_1\n", " 8:\n", - " action: configure_c2_beacon\n", + " action: configure-c2-beacon\n", " options:\n", " node_name: web_server\n", " c2_server_ip_address: 192.168.10.21\n", " 9:\n", - " action: configure_c2_beacon\n", + " action: configure-c2-beacon\n", " options:\n", " node_name: web_server\n", " c2_server_ip_address: 192.168.10.22\n", "\n", " reward_function:\n", " reward_components:\n", - " - type: DUMMY\n", + " - type: dummy\n", "\"\"\"\n", "c2_agent_yaml = yaml.safe_load(custom_c2_agent)" ] @@ -169,7 +169,7 @@ "source": [ "client_1: Computer = env.game.simulation.network.get_node_by_hostname(\"client_1\")\n", "client_1.software_manager.install(C2Server)\n", - "c2_server: C2Server = client_1.software_manager.software[\"C2Server\"]\n", + "c2_server: C2Server = client_1.software_manager.software[\"c2-server\"]\n", "c2_server.run()\n", "client_1.software_manager.show()" ] @@ -262,7 +262,7 @@ "outputs": [], "source": [ "env.step(2)\n", - "c2_beacon: C2Beacon = web_server.software_manager.software[\"C2Beacon\"]\n", + "c2_beacon: C2Beacon = web_server.software_manager.software[\"c2-beacon\"]\n", "web_server.software_manager.show()\n", "c2_beacon.show()" ] @@ -435,7 +435,7 @@ "metadata": {}, "outputs": [], "source": [ - "ransomware_script: RansomwareScript = web_server.software_manager.software[\"RansomwareScript\"]\n", + "ransomware_script: RansomwareScript = web_server.software_manager.software[\"ransomware-script\"]\n", "web_server.software_manager.show()\n", "ransomware_script.show()" ] @@ -574,20 +574,20 @@ "custom_blue_agent_yaml = \"\"\"\n", " - ref: defender\n", " team: BLUE\n", - " type: ProxyAgent\n", + " type: proxy-agent\n", "\n", " observation_space:\n", - " type: CUSTOM\n", + " type: custom\n", " options:\n", " components:\n", - " - type: NODES\n", - " label: NODES\n", + " - type: nodes\n", + " label: nodes\n", " options:\n", " hosts:\n", " - hostname: web_server\n", " applications:\n", - " - application_name: C2Beacon\n", - " - application_name: RansomwareScript\n", + " - application_name: c2-beacon\n", + " - application_name: ransomware-script\n", " folders:\n", " - folder_name: exfiltration_folder\n", " files:\n", @@ -637,7 +637,7 @@ " - UDP\n", " num_rules: 10\n", "\n", - " - type: LINKS\n", + " - type: links\n", " label: LINKS\n", " options:\n", " link_references:\n", @@ -651,26 +651,26 @@ " - switch_2:eth-1<->client_1:eth-1\n", " - switch_2:eth-2<->client_2:eth-1\n", " - switch_2:eth-7<->security_suite:eth-2\n", - " - type: \"NONE\"\n", + " - type: \"none\"\n", " label: ICS\n", " options: {}\n", "\n", " action_space:\n", " action_map:\n", " 0:\n", - " action: do_nothing\n", + " action: do-nothing\n", " options: {}\n", " 1:\n", - " action: node_application_remove\n", + " action: node-application-remove\n", " options:\n", " node_name: web_server\n", - " application_name: C2Beacon\n", + " application_name: c2-beacon\n", " 2:\n", - " action: node_shutdown\n", + " action: node-shutdown\n", " options:\n", " node_name: web_server\n", " 3:\n", - " action: router_acl_add_rule\n", + " action: router-acl-add-rule\n", " options:\n", " target_router: router_1\n", " position: 1\n", @@ -685,7 +685,7 @@ "\n", " reward_function:\n", " reward_components:\n", - " - type: DUMMY\n", + " - type: dummy\n", "\n", " agent_settings:\n", " flatten_obs: False\n", @@ -776,12 +776,12 @@ "\n", "# Installing the C2 Server.\n", "client_1.software_manager.install(C2Server)\n", - "c2_server: C2Server = client_1.software_manager.software[\"C2Server\"]\n", + "c2_server: C2Server = client_1.software_manager.software[\"c2-server\"]\n", "c2_server.run()\n", "\n", "# Installing the C2 Beacon.\n", "web_server.software_manager.install(C2Beacon)\n", - "c2_beacon: C2Beacon = web_server.software_manager.software[\"C2Beacon\"]\n", + "c2_beacon: C2Beacon = web_server.software_manager.software[\"c2-beacon\"]\n", "c2_beacon.configure(c2_server_ip_address=\"192.168.10.21\")\n", "c2_beacon.establish()" ] @@ -825,7 +825,7 @@ "outputs": [], "source": [ "# Installing RansomwareScript via C2 Terminal Commands\n", - "ransomware_install_command = {\"commands\":[[\"software_manager\", \"application\", \"install\", \"RansomwareScript\"]],\n", + "ransomware_install_command = {\"commands\":[[\"software_manager\", \"application\", \"install\", \"ransomware-script\"]],\n", " \"username\": \"admin\",\n", " \"password\": \"admin\"}\n", "c2_server.send_command(C2Command.TERMINAL, command_options=ransomware_install_command)\n" @@ -984,11 +984,11 @@ " web_server: Server = given_env.game.simulation.network.get_node_by_hostname(\"web_server\")\n", "\n", " client_1.software_manager.install(C2Server)\n", - " c2_server: C2Server = client_1.software_manager.software[\"C2Server\"]\n", + " c2_server: C2Server = client_1.software_manager.software[\"c2-server\"]\n", " c2_server.run()\n", "\n", " web_server.software_manager.install(C2Beacon)\n", - " c2_beacon: C2Beacon = web_server.software_manager.software[\"C2Beacon\"]\n", + " c2_beacon: C2Beacon = web_server.software_manager.software[\"c2-beacon\"]\n", " c2_beacon.configure(c2_server_ip_address=\"192.168.10.21\")\n", " c2_beacon.establish()\n", "\n", @@ -1086,7 +1086,7 @@ " \"username\": \"admin\",\n", " \"password\": \"admin\"}\n", "\n", - "c2_server: C2Server = client_1.software_manager.software[\"C2Server\"]\n", + "c2_server: C2Server = client_1.software_manager.software[\"c2-server\"]\n", "c2_server.send_command(C2Command.TERMINAL, command_options=ransomware_install_command)" ] }, @@ -1174,7 +1174,7 @@ " \"username\": \"admin\",\n", " \"password\": \"admin\"}\n", "\n", - "c2_server: C2Server = client_1.software_manager.software[\"C2Server\"]\n", + "c2_server: C2Server = client_1.software_manager.software[\"c2-server\"]\n", "c2_server.send_command(C2Command.TERMINAL, command_options=ransomware_install_command)" ] }, @@ -1396,16 +1396,16 @@ "source": [ "web_server: Server = c2_config_env.game.simulation.network.get_node_by_hostname(\"web_server\")\n", "web_server.software_manager.install(C2Beacon)\n", - "c2_beacon: C2Beacon = web_server.software_manager.software[\"C2Beacon\"]\n", + "c2_beacon: C2Beacon = web_server.software_manager.software[\"c2-beacon\"]\n", "\n", "client_1: Computer = c2_config_env.game.simulation.network.get_node_by_hostname(\"client_1\")\n", "client_1.software_manager.install(C2Server)\n", - "c2_server_1: C2Server = client_1.software_manager.software[\"C2Server\"]\n", + "c2_server_1: C2Server = client_1.software_manager.software[\"c2-server\"]\n", "c2_server_1.run()\n", "\n", "client_2: Computer = c2_config_env.game.simulation.network.get_node_by_hostname(\"client_2\")\n", "client_2.software_manager.install(C2Server)\n", - "c2_server_2: C2Server = client_2.software_manager.software[\"C2Server\"]\n", + "c2_server_2: C2Server = client_2.software_manager.software[\"c2-server\"]\n", "c2_server_2.run()" ] }, diff --git a/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb b/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb index 14938c02..2e0a0d40 100644 --- a/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb +++ b/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb @@ -76,9 +76,9 @@ " # parse the info dict form step output and write out what the red agent is doing\n", " red_info : AgentHistoryItem = info['agent_actions']['data_manipulation_attacker']\n", " red_action = red_info.action\n", - " if red_action == 'do_nothing':\n", + " if red_action == 'do-nothing':\n", " red_str = 'DO NOTHING'\n", - " elif red_action == 'node_application_execute':\n", + " elif red_action == 'node-application-execute':\n", " red_str = f\"ATTACK from {red_info.parameters['node_name']}\"\n", " return red_str" ] @@ -369,18 +369,18 @@ "change = yaml.safe_load(\"\"\"\n", " applications:\n", " - ref: data_manipulation_bot\n", - " type: DataManipulationBot\n", + " type: data-manipulation-bot\n", " options:\n", " port_scan_p_of_success: 1.0\n", " data_manipulation_p_of_success: 1.0\n", " payload: \"DELETE\"\n", " server_ip: 192.168.1.14\n", " - ref: client_1_web_browser\n", - " type: WebBrowser\n", + " type: web-browser\n", " options:\n", " target_url: http://arcd.com/users/\n", " - ref: client_1_database_client\n", - " type: DatabaseClient\n", + " type: database-client\n", " options:\n", " db_server_ip: 192.168.1.14\n", "\"\"\")\n", @@ -414,18 +414,18 @@ "change = yaml.safe_load(\"\"\"\n", " applications:\n", " - ref: data_manipulation_bot\n", - " type: DataManipulationBot\n", + " type: data-manipulation-bot\n", " options:\n", " port_scan_p_of_success: 0.0\n", " data_manipulation_p_of_success: 0.0\n", " payload: \"DELETE\"\n", " server_ip: 192.168.1.14\n", " - ref: client_1_web_browser\n", - " type: WebBrowser\n", + " type: web-browser\n", " options:\n", " target_url: http://arcd.com/users/\n", " - ref: client_1_database_client\n", - " type: DatabaseClient\n", + " type: database-client\n", " options:\n", " db_server_ip: 192.168.1.14\n", "\"\"\")\n", diff --git a/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb b/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb index 41b75e43..2d3f99b2 100644 --- a/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb +++ b/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb @@ -458,9 +458,9 @@ " # parse the info dict form step output and write out what the red agent is doing\n", " red_info : AgentHistoryItem = info['agent_actions']['data_manipulation_attacker']\n", " red_action = red_info.action\n", - " if red_action == 'do_nothing':\n", + " if red_action == 'do-nothing':\n", " red_str = 'DO NOTHING'\n", - " elif red_action == 'node_application_execute':\n", + " elif red_action == 'node-application-execute':\n", " client = \"client 1\" if red_info.parameters['node_name'] == 0 else \"client 2\"\n", " red_str = f\"ATTACK from {client}\"\n", " return red_str" diff --git a/src/primaite/notebooks/Privilege-Escalation-and-Data-Loss-Example.ipynb b/src/primaite/notebooks/Privilege-Escalation-and-Data-Loss-Example.ipynb index cbb898ea..deb38eea 100644 --- a/src/primaite/notebooks/Privilege-Escalation-and-Data-Loss-Example.ipynb +++ b/src/primaite/notebooks/Privilege-Escalation-and-Data-Loss-Example.ipynb @@ -120,11 +120,11 @@ "outputs": [], "source": [ "some_tech_jnr_dev_pc: Computer = game.simulation.network.get_node_by_hostname(\"some_tech_jnr_dev_pc\")\n", - "some_tech_jnr_dev_db_client: DatabaseClient = some_tech_jnr_dev_pc.software_manager.software[\"DatabaseClient\"]\n", - "some_tech_jnr_dev_web_browser: WebBrowser = some_tech_jnr_dev_pc.software_manager.software[\"WebBrowser\"]\n", + "some_tech_jnr_dev_db_client: DatabaseClient = some_tech_jnr_dev_pc.software_manager.software[\"database-client\"]\n", + "some_tech_jnr_dev_web_browser: WebBrowser = some_tech_jnr_dev_pc.software_manager.software[\"web-browser\"]\n", "some_tech_rt: Router = game.simulation.network.get_node_by_hostname(\"some_tech_rt\")\n", "some_tech_db_srv: Server = game.simulation.network.get_node_by_hostname(\"some_tech_db_srv\")\n", - "some_tech_db_service: DatabaseService = some_tech_db_srv.software_manager.software[\"DatabaseService\"]\n", + "some_tech_db_service: DatabaseService = some_tech_db_srv.software_manager.software[\"database-service\"]\n", "some_tech_storage_srv: Server = game.simulation.network.get_node_by_hostname(\"some_tech_storage_srv\")\n", "some_tech_web_srv: Server = game.simulation.network.get_node_by_hostname(\"some_tech_web_srv\")" ] @@ -211,7 +211,7 @@ "source": [ "caos_action = [\n", " \"network\", \"node\", \"some_tech_jnr_dev_pc\", \n", - " \"service\", \"Terminal\", \"node_session_remote_login\", \"admin\", \"admin\", str(some_tech_storage_srv.network_interface[1].ip_address)\n", + " \"service\", \"terminal\", \"node-session-remote-login\", \"admin\", \"admin\", str(some_tech_storage_srv.network_interface[1].ip_address)\n", "]\n", "game.simulation.apply_request(caos_action)" ] @@ -233,7 +233,7 @@ }, "outputs": [], "source": [ - "caos_action = [\"network\", \"node\", \"some_tech_jnr_dev_pc\", \"application\", \"WebBrowser\", \"execute\"]\n", + "caos_action = [\"network\", \"node\", \"some_tech_jnr_dev_pc\", \"application\", \"web-browser\", \"execute\"]\n", "game.simulation.apply_request(caos_action)" ] }, @@ -256,7 +256,7 @@ "metadata": {}, "outputs": [], "source": [ - "game.get_sim_state()[\"network\"][\"nodes\"][\"some_tech_rt\"][\"services\"][\"UserSessionManager\"][\"active_remote_sessions\"]" + "game.get_sim_state()[\"network\"][\"nodes\"][\"some_tech_rt\"][\"services\"][\"user-session-manager\"][\"active_remote_sessions\"]" ] }, { @@ -269,7 +269,7 @@ "source": [ "caos_action = [\n", " \"network\", \"node\", \"some_tech_jnr_dev_pc\", \n", - " \"service\", \"Terminal\", \"node_session_remote_login\", \"admin\", \"admin\", str(some_tech_rt.network_interface[4].ip_address)\n", + " \"service\", \"terminal\", \"node-session-remote-login\", \"admin\", \"admin\", str(some_tech_rt.network_interface[4].ip_address)\n", "]\n", "game.simulation.apply_request(caos_action)" ] @@ -282,7 +282,7 @@ }, "outputs": [], "source": [ - "game.get_sim_state()[\"network\"][\"nodes\"][\"some_tech_rt\"][\"services\"][\"UserSessionManager\"][\"active_remote_sessions\"]" + "game.get_sim_state()[\"network\"][\"nodes\"][\"some_tech_rt\"][\"services\"][\"user-session-manager\"][\"active_remote_sessions\"]" ] }, { @@ -315,7 +315,7 @@ "source": [ "caos_action = [\n", " \"network\", \"node\", \"some_tech_jnr_dev_pc\", \n", - " \"service\", \"Terminal\", \"send_remote_command\", str(some_tech_rt.network_interface[4].ip_address),\n", + " \"service\", \"terminal\", \"send_remote_command\", str(some_tech_rt.network_interface[4].ip_address),\n", " {\n", " \"command\": [\n", " \"acl\", \"add_rule\", \"PERMIT\", \"TCP\",\n", @@ -368,7 +368,7 @@ "source": [ "caos_action = [\n", " \"network\", \"node\", \"some_tech_jnr_dev_pc\", \n", - " \"service\", \"Terminal\", \"remote_logoff\", str(some_tech_rt.network_interface[4].ip_address)\n", + " \"service\", \"terminal\", \"remote_logoff\", str(some_tech_rt.network_interface[4].ip_address)\n", "]\n", "game.simulation.apply_request(caos_action)" ] @@ -386,7 +386,7 @@ "metadata": {}, "outputs": [], "source": [ - "game.get_sim_state()[\"network\"][\"nodes\"][\"some_tech_rt\"][\"services\"][\"UserSessionManager\"][\"active_remote_sessions\"]" + "game.get_sim_state()[\"network\"][\"nodes\"][\"some_tech_rt\"][\"services\"][\"user-session-manager\"][\"active_remote_sessions\"]" ] }, { @@ -406,7 +406,7 @@ "source": [ "caos_action = [\n", " \"network\", \"node\", \"some_tech_jnr_dev_pc\", \n", - " \"service\", \"Terminal\", \"node_session_remote_login\", \"admin\", \"admin\", str(some_tech_storage_srv.network_interface[1].ip_address)\n", + " \"service\", \"terminal\", \"node-session-remote-login\", \"admin\", \"admin\", str(some_tech_storage_srv.network_interface[1].ip_address)\n", "]\n", "game.simulation.apply_request(caos_action)" ] @@ -421,7 +421,7 @@ "source": [ "caos_action = [\n", " \"network\", \"node\", \"some_tech_jnr_dev_pc\", \n", - " \"service\", \"Terminal\", \"send_remote_command\", str(some_tech_storage_srv.network_interface[1].ip_address),\n", + " \"service\", \"terminal\", \"send_remote_command\", str(some_tech_storage_srv.network_interface[1].ip_address),\n", " {\n", " \"command\": [\n", " \"file_system\", \"delete\", \"file\", db_backup_folder, \"database.db\"\n", @@ -476,7 +476,7 @@ }, "outputs": [], "source": [ - "caos_action = [\"network\", \"node\", \"some_tech_jnr_dev_pc\", \"application\", \"WebBrowser\", \"execute\"]\n", + "caos_action = [\"network\", \"node\", \"some_tech_jnr_dev_pc\", \"application\", \"web-browser\", \"execute\"]\n", "game.simulation.apply_request(caos_action)" ] }, @@ -535,7 +535,7 @@ }, "outputs": [], "source": [ - "caos_action = [\"network\", \"node\", \"some_tech_jnr_dev_pc\", \"application\", \"WebBrowser\", \"execute\"]\n", + "caos_action = [\"network\", \"node\", \"some_tech_jnr_dev_pc\", \"application\", \"web-browser\", \"execute\"]\n", "game.simulation.apply_request(caos_action)" ] }, diff --git a/src/primaite/notebooks/Requests-and-Responses.ipynb b/src/primaite/notebooks/Requests-and-Responses.ipynb index 48f1cf3e..c29d41dc 100644 --- a/src/primaite/notebooks/Requests-and-Responses.ipynb +++ b/src/primaite/notebooks/Requests-and-Responses.ipynb @@ -52,14 +52,15 @@ "outputs": [], "source": [ "sim = Simulation()\n", + "\n", "sim.network.add_node(\n", " HostNode.from_config(\n", " config = {\n", - " 'type': \"hostnode\",\n", + " 'type': \"host-node\",\n", " 'hostname': \"client\",\n", " 'ip_address': '10.0.0.1',\n", " 'subnet_mask': '255.255.255.0',\n", - " 'operating_state': NodeOperatingState.ON,\n", + " 'operating_state': \"ON\",\n", " }\n", " )\n", ")\n", @@ -98,7 +99,7 @@ "outputs": [], "source": [ "response = sim.apply_request(\n", - " request=[\"network\", \"node\", \"client\", \"service\", \"DNSClient\", \"stop\"],\n", + " request=[\"network\", \"node\", \"client\", \"service\", \"dns-client\", \"stop\"],\n", " context={}\n", " )\n", "print(response)" @@ -118,7 +119,7 @@ "metadata": {}, "outputs": [], "source": [ - "print(f\"DNS Client state: {client.software_manager.software.get('DNSClient').operating_state.name}\")" + "print(f\"DNS Client state: {client.software_manager.software.get('dns-client').operating_state.name}\")" ] }, { @@ -142,7 +143,7 @@ "outputs": [], "source": [ "response = sim.apply_request(\n", - " request=[\"network\", \"node\", \"client\", \"service\", \"NonExistentApplication\", \"stop\"],\n", + " request=[\"network\", \"node\", \"client\", \"service\", \"non-existent-application\", \"stop\"],\n", " context={}\n", " )\n", "print(response)" @@ -205,7 +206,7 @@ "outputs": [], "source": [ "response = sim.apply_request(\n", - " request=[\"network\", \"node\", \"client\", \"service\", \"DNSClient\", \"start\"],\n", + " request=[\"network\", \"node\", \"client\", \"service\", \"dns-client\", \"start\"],\n", " context={}\n", " )\n", "print(response)" diff --git a/src/primaite/notebooks/Terminal-Processing.ipynb b/src/primaite/notebooks/Terminal-Processing.ipynb index dc870db5..5077732b 100644 --- a/src/primaite/notebooks/Terminal-Processing.ipynb +++ b/src/primaite/notebooks/Terminal-Processing.ipynb @@ -6,7 +6,7 @@ "source": [ "# Terminal Processing\n", "\n", - "© Crown-owned copyright 2024, Defence Science and Technology Laboratory UK" + "© Crown-owned copyright 2025, Defence Science and Technology Laboratory UK" ] }, { @@ -39,41 +39,27 @@ "from primaite.simulator.system.applications.red_applications.ransomware_script import RansomwareScript\n", "from primaite.simulator.system.services.terminal.terminal import RemoteTerminalConnection\n", "\n", - "# print(dir(Computer))\n", - "# node_a = Computer.from_config(\n", - "# config = {\n", - "# \"type\": \"computer\",\n", - "# \"hostname\": \"node_a\",\n", - "# \"ip_address\": \"192.168.0.10\",\n", - "# \"subnet_mask\": \"255.255.255.0\",\n", - "# \"startup_duration\": 0,\n", - "# }\n", - "# )\n", - "# print(f\"{node_a=}\")\n", - "\n", "def basic_network() -> Network:\n", " \"\"\"Utility function for creating a default network to demonstrate Terminal functionality\"\"\"\n", " network = Network()\n", - " # node_a = Computer(hostname=\"node_a\", ip_address=\"192.168.0.10\", subnet_mask=\"255.255.255.0\", start_up_duration=0)\n", " node_a = Computer.from_config(\n", " config = {\n", " \"type\": \"computer\",\n", " \"hostname\": \"node_a\",\n", " \"ip_address\": \"192.168.0.10\",\n", " \"subnet_mask\": \"255.255.255.0\",\n", - " \"startup_duration\": 0,\n", + " # \"startup_duration\": 0,\n", " }\n", " )\n", " print(f\"{node_a=}\")\n", " node_a.power_on()\n", - " # node_b = Computer(hostname=\"node_b\", ip_address=\"192.168.0.11\", subnet_mask=\"255.255.255.0\", start_up_duration=0)\n", " node_b = Computer.from_config(\n", " config = {\n", " \"type\": \"computer\",\n", " \"hostname\": \"node_b\",\n", " \"ip_address\": \"192.168.0.11\",\n", " \"subnet_mask\": \"255.255.255.0\",\n", - " \"startup_duration\": 0,\n", + " # \"startup_duration\": 0,\n", " }\n", " )\n", " node_b.power_on()\n", @@ -98,9 +84,9 @@ "source": [ "network: Network = basic_network()\n", "computer_a: Computer = network.get_node_by_hostname(\"node_a\")\n", - "terminal_a: Terminal = computer_a.software_manager.software.get(\"Terminal\")\n", + "terminal_a: Terminal = computer_a.software_manager.software.get(\"terminal\")\n", "computer_b: Computer = network.get_node_by_hostname(\"node_b\")\n", - "terminal_b: Terminal = computer_b.software_manager.software.get(\"Terminal\")" + "terminal_b: Terminal = computer_b.software_manager.software.get(\"terminal\")" ] }, { @@ -152,7 +138,7 @@ "metadata": {}, "outputs": [], "source": [ - "term_a_term_b_remote_connection.execute([\"software_manager\", \"application\", \"install\", \"RansomwareScript\"])" + "term_a_term_b_remote_connection.execute([\"software_manager\", \"application\", \"install\", \"ransomware-script\"])" ] }, { diff --git a/src/primaite/simulator/network/hardware/base.py b/src/primaite/simulator/network/hardware/base.py index 4712af60..4c252050 100644 --- a/src/primaite/simulator/network/hardware/base.py +++ b/src/primaite/simulator/network/hardware/base.py @@ -1632,7 +1632,9 @@ class Node(SimComponent, ABC): dns_server=kwargs["config"].dns_server, ) super().__init__(**kwargs) - self.operating_state = NodeOperatingState.ON if not (p := kwargs["config"].operating_state) else p + self.operating_state = ( + NodeOperatingState.ON if not (p := kwargs["config"].operating_state) else NodeOperatingState[p.upper()] + ) self._install_system_software() self.session_manager.node = self self.session_manager.software_manager = self.software_manager diff --git a/src/primaite/simulator/network/hardware/nodes/host/host_node.py b/src/primaite/simulator/network/hardware/nodes/host/host_node.py index 76d9167c..d38ba097 100644 --- a/src/primaite/simulator/network/hardware/nodes/host/host_node.py +++ b/src/primaite/simulator/network/hardware/nodes/host/host_node.py @@ -313,7 +313,7 @@ class HostNode(Node, discriminator="host-node"): """ SYSTEM_SOFTWARE: ClassVar[Dict] = { - "HostARP": HostARP, + "host-arp": HostARP, "icmp": ICMP, "dns-client": DNSClient, "ntp-client": NTPClient, diff --git a/tests/assets/configs/basic_switched_network.yaml b/tests/assets/configs/basic_switched_network.yaml index c9ac5f8d..4c6d2557 100644 --- a/tests/assets/configs/basic_switched_network.yaml +++ b/tests/assets/configs/basic_switched_network.yaml @@ -34,10 +34,10 @@ agents: action_space: action_map: 0: - action: do-nothing + action: do_nothing options: {} 1: - action: node-application-execute + action: node_application_execute options: node_name: client_2 application_name: web-browser @@ -109,7 +109,7 @@ agents: action_space: action_map: 0: - action: do-nothing + action: do_nothing options: {} reward_function: From fc66e125d64d882c78109d008e7f84039cc19e32 Mon Sep 17 00:00:00 2001 From: Marek Wolan Date: Fri, 7 Feb 2025 10:08:18 +0000 Subject: [PATCH 15/37] Fix action schema for change password --- src/primaite/game/agent/actions/session.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/primaite/game/agent/actions/session.py b/src/primaite/game/agent/actions/session.py index 58a8a555..b7b56d17 100644 --- a/src/primaite/game/agent/actions/session.py +++ b/src/primaite/game/agent/actions/session.py @@ -63,8 +63,6 @@ class NodeSessionsRemoteLoginAction(NodeSessionAbstractAction, discriminator="no class NodeSessionsRemoteLogoutAction(NodeSessionAbstractAction, discriminator="node-session-remote-logoff"): """Action which performs a remote session logout.""" - config: "NodeSessionsRemoteLogoutAction.ConfigSchema" - class ConfigSchema(NodeSessionAbstractAction.ConfigSchema): """Configuration schema for NodeSessionsRemoteLogoutAction.""" @@ -78,14 +76,13 @@ class NodeSessionsRemoteLogoutAction(NodeSessionAbstractAction, discriminator="n return ["network", "node", config.node_name, "service", "terminal", config.verb, config.remote_ip] -class NodeAccountChangePasswordAction(NodeSessionAbstractAction, discriminator="node-account-change-password"): +class NodeAccountChangePasswordAction(AbstractAction, discriminator="node-account-change-password"): """Action which changes the password for a user.""" - config: "NodeAccountChangePasswordAction.ConfigSchema" - - class ConfigSchema(NodeSessionAbstractAction.ConfigSchema): + class ConfigSchema(AbstractAction.ConfigSchema): """Configuration schema for NodeAccountsChangePasswordAction.""" + node_name: str username: str current_password: str new_password: str From 56e81a020ca31c8a8770d7a737f8008166474b50 Mon Sep 17 00:00:00 2001 From: Charlie Crane Date: Tue, 11 Feb 2025 10:53:41 +0000 Subject: [PATCH 16/37] #3060 - Adding version to YAML files within PrimAITE --- .../config/_package_data/basic_lan_network_example.yaml | 2 ++ .../_package_data/client_server_p2p_network_example.yaml | 2 ++ src/primaite/config/_package_data/data_manipulation.yaml | 2 ++ .../config/_package_data/data_manipulation_marl.yaml | 2 ++ .../base_scenario.yaml | 2 ++ .../_package_data/multi_lan_internet_network_example.yaml | 2 ++ .../_package_data/scenario_with_placeholders/scenario.yaml | 2 ++ src/primaite/game/game.py | 5 +++++ tests/assets/configs/action_penalty.yaml | 2 ++ tests/assets/configs/bad_primaite_session.yaml | 2 ++ tests/assets/configs/basic_c2_setup.yaml | 2 ++ tests/assets/configs/basic_firewall.yaml | 1 + .../configs/basic_node_with_software_listening_ports.yaml | 2 ++ tests/assets/configs/basic_node_with_users.yaml | 2 ++ tests/assets/configs/basic_switched_network.yaml | 2 ++ tests/assets/configs/data_manipulation.yaml | 2 ++ tests/assets/configs/dmz_network.yaml | 2 ++ tests/assets/configs/eval_only_primaite_session.yaml | 2 ++ tests/assets/configs/extended_config.yaml | 2 ++ tests/assets/configs/firewall_actions_network.yaml | 2 ++ tests/assets/configs/fixing_duration_one_item.yaml | 2 ++ tests/assets/configs/install_and_configure_apps.yaml | 2 ++ tests/assets/configs/multi_agent_session.yaml | 2 ++ .../configs/nmap_network_service_recon_red_agent_config.yaml | 2 ++ tests/assets/configs/nmap_ping_scan_red_agent_config.yaml | 2 ++ tests/assets/configs/nmap_port_scan_red_agent_config.yaml | 2 ++ tests/assets/configs/no_nodes_links_agents_network.yaml | 2 ++ .../assets/configs/scenario_with_placeholders/scenario.yaml | 2 ++ tests/assets/configs/shared_rewards.yaml | 2 ++ tests/assets/configs/software_fixing_duration.yaml | 2 ++ tests/assets/configs/test_application_install.yaml | 2 ++ tests/assets/configs/test_primaite_session.yaml | 2 ++ tests/assets/configs/wireless_wan_network_config.yaml | 2 ++ .../wireless_wan_network_config_freq_max_override.yaml | 2 ++ ...ireless_wan_network_config_freq_max_override_blocked.yaml | 2 ++ 35 files changed, 72 insertions(+) diff --git a/src/primaite/config/_package_data/basic_lan_network_example.yaml b/src/primaite/config/_package_data/basic_lan_network_example.yaml index 9490ff00..854dfcf3 100644 --- a/src/primaite/config/_package_data/basic_lan_network_example.yaml +++ b/src/primaite/config/_package_data/basic_lan_network_example.yaml @@ -1,3 +1,5 @@ +version: 3.0 + game: ports: - ARP diff --git a/src/primaite/config/_package_data/client_server_p2p_network_example.yaml b/src/primaite/config/_package_data/client_server_p2p_network_example.yaml index 798dd318..702c864f 100644 --- a/src/primaite/config/_package_data/client_server_p2p_network_example.yaml +++ b/src/primaite/config/_package_data/client_server_p2p_network_example.yaml @@ -1,3 +1,5 @@ +version: 3.0 + game: ports: - ARP diff --git a/src/primaite/config/_package_data/data_manipulation.yaml b/src/primaite/config/_package_data/data_manipulation.yaml index 4705c135..72b3f671 100644 --- a/src/primaite/config/_package_data/data_manipulation.yaml +++ b/src/primaite/config/_package_data/data_manipulation.yaml @@ -1,3 +1,5 @@ +version: 3.0 + io_settings: save_agent_actions: true save_step_metadata: false diff --git a/src/primaite/config/_package_data/data_manipulation_marl.yaml b/src/primaite/config/_package_data/data_manipulation_marl.yaml index 0263edb0..178b3322 100644 --- a/src/primaite/config/_package_data/data_manipulation_marl.yaml +++ b/src/primaite/config/_package_data/data_manipulation_marl.yaml @@ -1,3 +1,5 @@ +version: 3.0 + io_settings: save_agent_actions: true save_step_metadata: false diff --git a/src/primaite/config/_package_data/mini_scenario_with_simulation_variation/base_scenario.yaml b/src/primaite/config/_package_data/mini_scenario_with_simulation_variation/base_scenario.yaml index 4c7734d8..9530305a 100644 --- a/src/primaite/config/_package_data/mini_scenario_with_simulation_variation/base_scenario.yaml +++ b/src/primaite/config/_package_data/mini_scenario_with_simulation_variation/base_scenario.yaml @@ -1,3 +1,5 @@ +version: 3.0 + game: max_episode_length: 128 ports: [] diff --git a/src/primaite/config/_package_data/multi_lan_internet_network_example.yaml b/src/primaite/config/_package_data/multi_lan_internet_network_example.yaml index cbd1c01e..4e8f085e 100644 --- a/src/primaite/config/_package_data/multi_lan_internet_network_example.yaml +++ b/src/primaite/config/_package_data/multi_lan_internet_network_example.yaml @@ -1,3 +1,5 @@ +version: 3.0 + game: ports: - ARP diff --git a/src/primaite/config/_package_data/scenario_with_placeholders/scenario.yaml b/src/primaite/config/_package_data/scenario_with_placeholders/scenario.yaml index 1a96e3a0..d34fb310 100644 --- a/src/primaite/config/_package_data/scenario_with_placeholders/scenario.yaml +++ b/src/primaite/config/_package_data/scenario_with_placeholders/scenario.yaml @@ -1,3 +1,5 @@ +version: 3.0 + io_settings: save_agent_actions: true save_step_metadata: false diff --git a/src/primaite/game/game.py b/src/primaite/game/game.py index 1427776e..639b37cf 100644 --- a/src/primaite/game/game.py +++ b/src/primaite/game/game.py @@ -247,6 +247,11 @@ class PrimaiteGame: game.options = PrimaiteGameOptions(**cfg["game"]) game.save_step_metadata = cfg.get("io_settings", {}).get("save_step_metadata") or False + # TODO: Future YAML config should specify the PrimAITE version they are written for. + # For now, we warn that if it is missing, pending a mechanism to handle variations. + if not cfg["version"]: + _LOGGER.warning("Version definition is missing from provided configuration. ") + # 1. create simulation sim = game.simulation net = sim.network diff --git a/tests/assets/configs/action_penalty.yaml b/tests/assets/configs/action_penalty.yaml index 9c117dfe..9561db4f 100644 --- a/tests/assets/configs/action_penalty.yaml +++ b/tests/assets/configs/action_penalty.yaml @@ -1,3 +1,5 @@ +version: 3.0 + io_settings: save_agent_actions: false save_step_metadata: false diff --git a/tests/assets/configs/bad_primaite_session.yaml b/tests/assets/configs/bad_primaite_session.yaml index 18b37466..1f59225b 100644 --- a/tests/assets/configs/bad_primaite_session.yaml +++ b/tests/assets/configs/bad_primaite_session.yaml @@ -1,3 +1,5 @@ +version: 3.0 + game: ports: - ARP diff --git a/tests/assets/configs/basic_c2_setup.yaml b/tests/assets/configs/basic_c2_setup.yaml index ac2b026e..3011457f 100644 --- a/tests/assets/configs/basic_c2_setup.yaml +++ b/tests/assets/configs/basic_c2_setup.yaml @@ -4,6 +4,8 @@ # | node_a |------| switch_1 |------| node_b | # -------------- -------------- -------------- # +version: 3.0 + io_settings: save_step_metadata: false save_pcap_logs: true diff --git a/tests/assets/configs/basic_firewall.yaml b/tests/assets/configs/basic_firewall.yaml index 8108ecbe..1aa1361a 100644 --- a/tests/assets/configs/basic_firewall.yaml +++ b/tests/assets/configs/basic_firewall.yaml @@ -4,6 +4,7 @@ # | client_1 |------| switch_1 |------| client_2 | # -------------- -------------- -------------- # +version: 3.0 io_settings: save_step_metadata: false diff --git a/tests/assets/configs/basic_node_with_software_listening_ports.yaml b/tests/assets/configs/basic_node_with_software_listening_ports.yaml index 7b04196b..b503d66d 100644 --- a/tests/assets/configs/basic_node_with_software_listening_ports.yaml +++ b/tests/assets/configs/basic_node_with_software_listening_ports.yaml @@ -1,3 +1,5 @@ +version: 3.0 + io_settings: save_step_metadata: false save_pcap_logs: true diff --git a/tests/assets/configs/basic_node_with_users.yaml b/tests/assets/configs/basic_node_with_users.yaml index 064519dd..3009b79e 100644 --- a/tests/assets/configs/basic_node_with_users.yaml +++ b/tests/assets/configs/basic_node_with_users.yaml @@ -1,3 +1,5 @@ +version: 3.0 + io_settings: save_step_metadata: false save_pcap_logs: true diff --git a/tests/assets/configs/basic_switched_network.yaml b/tests/assets/configs/basic_switched_network.yaml index c9ac5f8d..7e8a5715 100644 --- a/tests/assets/configs/basic_switched_network.yaml +++ b/tests/assets/configs/basic_switched_network.yaml @@ -4,6 +4,8 @@ # | client_1 |------| switch_1 |------| client_2 | # -------------- -------------- -------------- # +version: 3.0 + io_settings: save_step_metadata: false save_pcap_logs: true diff --git a/tests/assets/configs/data_manipulation.yaml b/tests/assets/configs/data_manipulation.yaml index e9464f0f..baf9df7d 100644 --- a/tests/assets/configs/data_manipulation.yaml +++ b/tests/assets/configs/data_manipulation.yaml @@ -1,3 +1,5 @@ +version: 3.0 + io_settings: save_agent_actions: true save_step_metadata: false diff --git a/tests/assets/configs/dmz_network.yaml b/tests/assets/configs/dmz_network.yaml index 2aa472d8..456920f1 100644 --- a/tests/assets/configs/dmz_network.yaml +++ b/tests/assets/configs/dmz_network.yaml @@ -30,6 +30,8 @@ # | external_computer |------| switch_3 |------| external_server | # ----------------------- -------------- --------------------- # +version: 3.0 + io_settings: save_step_metadata: false save_pcap_logs: true diff --git a/tests/assets/configs/eval_only_primaite_session.yaml b/tests/assets/configs/eval_only_primaite_session.yaml index 3403a7ff..c0a61b9d 100644 --- a/tests/assets/configs/eval_only_primaite_session.yaml +++ b/tests/assets/configs/eval_only_primaite_session.yaml @@ -1,3 +1,5 @@ +version: 3.0 + game: ports: - ARP diff --git a/tests/assets/configs/extended_config.yaml b/tests/assets/configs/extended_config.yaml index 8bf7d506..1b0772c1 100644 --- a/tests/assets/configs/extended_config.yaml +++ b/tests/assets/configs/extended_config.yaml @@ -1,3 +1,5 @@ +version: 3.0 + io_settings: save_agent_actions: true save_step_metadata: false diff --git a/tests/assets/configs/firewall_actions_network.yaml b/tests/assets/configs/firewall_actions_network.yaml index eb39aa1a..b624be52 100644 --- a/tests/assets/configs/firewall_actions_network.yaml +++ b/tests/assets/configs/firewall_actions_network.yaml @@ -30,6 +30,8 @@ # | external_computer |------| switch_3 |------| external_server | # ----------------------- -------------- --------------------- # +version: 3.0 + io_settings: save_step_metadata: false save_pcap_logs: true diff --git a/tests/assets/configs/fixing_duration_one_item.yaml b/tests/assets/configs/fixing_duration_one_item.yaml index ff3a6504..ad94a9da 100644 --- a/tests/assets/configs/fixing_duration_one_item.yaml +++ b/tests/assets/configs/fixing_duration_one_item.yaml @@ -4,6 +4,8 @@ # | client_1 |------| switch_1 |------| client_2 | # -------------- -------------- -------------- # +version: 3.0 + io_settings: save_step_metadata: false save_pcap_logs: true diff --git a/tests/assets/configs/install_and_configure_apps.yaml b/tests/assets/configs/install_and_configure_apps.yaml index 6b80519c..4f96496e 100644 --- a/tests/assets/configs/install_and_configure_apps.yaml +++ b/tests/assets/configs/install_and_configure_apps.yaml @@ -1,3 +1,5 @@ +version: 3.0 + io_settings: save_step_metadata: false save_pcap_logs: false diff --git a/tests/assets/configs/multi_agent_session.yaml b/tests/assets/configs/multi_agent_session.yaml index e99ea49b..263d5f66 100644 --- a/tests/assets/configs/multi_agent_session.yaml +++ b/tests/assets/configs/multi_agent_session.yaml @@ -1,3 +1,5 @@ +version: 3.0 + io_settings: save_agent_actions: false save_step_metadata: false diff --git a/tests/assets/configs/nmap_network_service_recon_red_agent_config.yaml b/tests/assets/configs/nmap_network_service_recon_red_agent_config.yaml index 9b49d466..6abb11ce 100644 --- a/tests/assets/configs/nmap_network_service_recon_red_agent_config.yaml +++ b/tests/assets/configs/nmap_network_service_recon_red_agent_config.yaml @@ -1,3 +1,5 @@ +version: 3.0 + io_settings: save_step_metadata: false save_pcap_logs: true diff --git a/tests/assets/configs/nmap_ping_scan_red_agent_config.yaml b/tests/assets/configs/nmap_ping_scan_red_agent_config.yaml index e3cb33e1..869f4fb8 100644 --- a/tests/assets/configs/nmap_ping_scan_red_agent_config.yaml +++ b/tests/assets/configs/nmap_ping_scan_red_agent_config.yaml @@ -1,3 +1,5 @@ +version: 3.0 + io_settings: save_step_metadata: false save_pcap_logs: true diff --git a/tests/assets/configs/nmap_port_scan_red_agent_config.yaml b/tests/assets/configs/nmap_port_scan_red_agent_config.yaml index 2692a3bc..1d5ab323 100644 --- a/tests/assets/configs/nmap_port_scan_red_agent_config.yaml +++ b/tests/assets/configs/nmap_port_scan_red_agent_config.yaml @@ -1,3 +1,5 @@ +version: 3.0 + io_settings: save_step_metadata: false save_pcap_logs: true diff --git a/tests/assets/configs/no_nodes_links_agents_network.yaml b/tests/assets/configs/no_nodes_links_agents_network.yaml index b20835bc..b79e6bc6 100644 --- a/tests/assets/configs/no_nodes_links_agents_network.yaml +++ b/tests/assets/configs/no_nodes_links_agents_network.yaml @@ -1,3 +1,5 @@ +version: 3.0 + io_settings: save_step_metadata: false save_pcap_logs: true diff --git a/tests/assets/configs/scenario_with_placeholders/scenario.yaml b/tests/assets/configs/scenario_with_placeholders/scenario.yaml index ab8c968c..76b51626 100644 --- a/tests/assets/configs/scenario_with_placeholders/scenario.yaml +++ b/tests/assets/configs/scenario_with_placeholders/scenario.yaml @@ -1,3 +1,5 @@ +version: 3.0 + io_settings: save_agent_actions: true save_step_metadata: false diff --git a/tests/assets/configs/shared_rewards.yaml b/tests/assets/configs/shared_rewards.yaml index 07256f01..68b3b310 100644 --- a/tests/assets/configs/shared_rewards.yaml +++ b/tests/assets/configs/shared_rewards.yaml @@ -1,3 +1,5 @@ +version: 3.0 + io_settings: save_agent_actions: false save_step_metadata: false diff --git a/tests/assets/configs/software_fixing_duration.yaml b/tests/assets/configs/software_fixing_duration.yaml index 63409f2b..2f5613e1 100644 --- a/tests/assets/configs/software_fixing_duration.yaml +++ b/tests/assets/configs/software_fixing_duration.yaml @@ -4,6 +4,8 @@ # | client_1 |------| switch_1 |------| client_2 | # -------------- -------------- -------------- # +version: 3.0 + io_settings: save_step_metadata: false save_pcap_logs: true diff --git a/tests/assets/configs/test_application_install.yaml b/tests/assets/configs/test_application_install.yaml index 0266dd00..d49d9bf2 100644 --- a/tests/assets/configs/test_application_install.yaml +++ b/tests/assets/configs/test_application_install.yaml @@ -1,3 +1,5 @@ +version: 3.0 + io_settings: save_agent_actions: true save_step_metadata: false diff --git a/tests/assets/configs/test_primaite_session.yaml b/tests/assets/configs/test_primaite_session.yaml index 65d16a47..c1206634 100644 --- a/tests/assets/configs/test_primaite_session.yaml +++ b/tests/assets/configs/test_primaite_session.yaml @@ -1,3 +1,5 @@ +version: 3.0 + io_settings: save_agent_actions: true save_step_metadata: true diff --git a/tests/assets/configs/wireless_wan_network_config.yaml b/tests/assets/configs/wireless_wan_network_config.yaml index 40a9126c..b722e50f 100644 --- a/tests/assets/configs/wireless_wan_network_config.yaml +++ b/tests/assets/configs/wireless_wan_network_config.yaml @@ -1,3 +1,5 @@ +version: 3.0 + game: max_episode_length: 256 ports: diff --git a/tests/assets/configs/wireless_wan_network_config_freq_max_override.yaml b/tests/assets/configs/wireless_wan_network_config_freq_max_override.yaml index 13ceee16..4c2f4f86 100644 --- a/tests/assets/configs/wireless_wan_network_config_freq_max_override.yaml +++ b/tests/assets/configs/wireless_wan_network_config_freq_max_override.yaml @@ -1,3 +1,5 @@ +version: 3.0 + game: max_episode_length: 256 ports: diff --git a/tests/assets/configs/wireless_wan_network_config_freq_max_override_blocked.yaml b/tests/assets/configs/wireless_wan_network_config_freq_max_override_blocked.yaml index 6c52fd95..a98b6ee2 100644 --- a/tests/assets/configs/wireless_wan_network_config_freq_max_override_blocked.yaml +++ b/tests/assets/configs/wireless_wan_network_config_freq_max_override_blocked.yaml @@ -1,3 +1,5 @@ +version: 3.0 + game: max_episode_length: 256 ports: From dadf93bc7fd70e459f525d27c429b6514932614b Mon Sep 17 00:00:00 2001 From: Charlie Crane Date: Tue, 11 Feb 2025 14:59:25 +0000 Subject: [PATCH 17/37] #3060 - Address some missed pre-commit errors --- docs/source/how_to_guides/extensible_nodes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/how_to_guides/extensible_nodes.rst b/docs/source/how_to_guides/extensible_nodes.rst index 043d0f06..4819cbb2 100644 --- a/docs/source/how_to_guides/extensible_nodes.rst +++ b/docs/source/how_to_guides/extensible_nodes.rst @@ -52,4 +52,4 @@ class Router(NetworkNode, identifier="router"): Changes to YAML file. ===================== -While effort has been made to ensure that nodes defined within configuration YAML files for use with PrimAITE 3.X remain compatible with PrimAITE v4+, it is encouraged to review for minor changes needed. +While effort has been made to ensure that nodes defined within configuration YAML files for use with PrimAITE 3.X remain compatible with PrimAITE v4+, it is encouraged to review for minor changes needed. From 70fc11f4f110e138c16c4a80030b408b3ecb7f15 Mon Sep 17 00:00:00 2001 From: Charlie Crane Date: Tue, 11 Feb 2025 16:20:11 +0000 Subject: [PATCH 18/37] #3060 - Correct change in game.py that caused test failures --- src/primaite/game/game.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/primaite/game/game.py b/src/primaite/game/game.py index 639b37cf..4741199c 100644 --- a/src/primaite/game/game.py +++ b/src/primaite/game/game.py @@ -247,9 +247,11 @@ class PrimaiteGame: game.options = PrimaiteGameOptions(**cfg["game"]) game.save_step_metadata = cfg.get("io_settings", {}).get("save_step_metadata") or False + config_version = cfg.get("version", {}) + # TODO: Future YAML config should specify the PrimAITE version they are written for. # For now, we warn that if it is missing, pending a mechanism to handle variations. - if not cfg["version"]: + if not isinstance(config_version, int): _LOGGER.warning("Version definition is missing from provided configuration. ") # 1. create simulation From 56699d2377a706c54648b54c5fb72456fbc1961d Mon Sep 17 00:00:00 2001 From: Charlie Crane Date: Wed, 12 Feb 2025 16:18:50 +0000 Subject: [PATCH 19/37] Resolve hardcoding of Agent Logger having agent_name as abstract_agent --- src/primaite/game/agent/interface.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/primaite/game/agent/interface.py b/src/primaite/game/agent/interface.py index a55cd3ff..82edab88 100644 --- a/src/primaite/game/agent/interface.py +++ b/src/primaite/game/agent/interface.py @@ -70,7 +70,7 @@ class AbstractAgent(BaseModel, ABC): config: ConfigSchema = Field(default_factory=lambda: AbstractAgent.ConfigSchema()) - logger: AgentLog = AgentLog(agent_name="Abstract_Agent") + logger: AgentLog = None history: List[AgentHistoryItem] = [] action_manager: ActionManager = Field(default_factory=lambda: ActionManager()) @@ -79,6 +79,12 @@ class AbstractAgent(BaseModel, ABC): _registry: ClassVar[Dict[str, Type[AbstractAgent]]] = {} + def __init__(self, **kwargs): + """init""" + + super().__init__(**kwargs) + self.logger: AgentLog = AgentLog(agent_name=kwargs["config"]["ref"]) + def __init_subclass__(cls, discriminator: Optional[str] = None, **kwargs: Any) -> None: super().__init_subclass__(**kwargs) if discriminator is None: From 7e138d1d61b9e1e2fad0bf5fd9e00ba4e4ea7701 Mon Sep 17 00:00:00 2001 From: Charlie Crane Date: Fri, 14 Feb 2025 11:38:15 +0000 Subject: [PATCH 20/37] #Bug and test fixes --- src/primaite/game/agent/interface.py | 2 +- src/primaite/simulator/network/container.py | 65 ++++++++----------- .../simulator/network/hardware/base.py | 4 +- src/primaite/simulator/system/software.py | 2 + .../extensions/nodes/giga_switch.py | 2 +- .../actions/test_terminal_actions.py | 2 - .../_primaite/_game/_agent/test_agent.py | 2 +- 7 files changed, 34 insertions(+), 45 deletions(-) diff --git a/src/primaite/game/agent/interface.py b/src/primaite/game/agent/interface.py index 82edab88..571c850d 100644 --- a/src/primaite/game/agent/interface.py +++ b/src/primaite/game/agent/interface.py @@ -80,7 +80,7 @@ class AbstractAgent(BaseModel, ABC): _registry: ClassVar[Dict[str, Type[AbstractAgent]]] = {} def __init__(self, **kwargs): - """init""" + """Initialise and setup agent logger""" super().__init__(**kwargs) self.logger: AgentLog = AgentLog(agent_name=kwargs["config"]["ref"]) diff --git a/src/primaite/simulator/network/container.py b/src/primaite/simulator/network/container.py index 2e494910..1fa8c680 100644 --- a/src/primaite/simulator/network/container.py +++ b/src/primaite/simulator/network/container.py @@ -163,15 +163,6 @@ class Network(SimComponent): :param links: Include link details in the output. Defaults to True. :param markdown: Use Markdown style in table output. Defaults to False. """ - nodes_type_map = { - "Router": self.router_nodes, - "Firewall": self.firewall_nodes, - "Switch": self.switch_nodes, - "Server": self.server_nodes, - "Computer": self.computer_nodes, - "Printer": self.printer_nodes, - "Wireless Router": self.wireless_router_nodes, - } if nodes: table = PrettyTable(["Node", "Type", "Operating State"]) @@ -189,19 +180,18 @@ class Network(SimComponent): table.set_style(MARKDOWN) table.align = "l" table.title = "IP Addresses" - for nodes in nodes_type_map.values(): - for node in nodes: - for i, port in node.network_interface.items(): - if hasattr(port, "ip_address"): - if port.ip_address != IPv4Address("127.0.0.1"): - port_str = port.port_name if port.port_name else port.port_num - table.add_row( - [ - node.config.hostname, - port_str, - port.ip_address, - port.subnet_mask, - node.config.default_gateway, + for node in self.nodes.values(): + for i, port in node.network_interface.items(): + if hasattr(port, "ip_address"): + if port.ip_address != IPv4Address("127.0.0.1"): + port_str = port.port_name if port.port_name else port.port_num + table.add_row( + [ + node.config.hostname, + port_str, + port.ip_address, + port.subnet_mask, + node.config.default_gateway, ] ) print(table) @@ -215,22 +205,21 @@ class Network(SimComponent): table.align = "l" table.title = "Links" links = list(self.links.values()) - for nodes in nodes_type_map.values(): - for node in nodes: - for link in links[::-1]: - if node in [link.endpoint_a.parent, link.endpoint_b.parent]: - table.add_row( - [ - link.endpoint_a.parent.config.hostname, - str(link.endpoint_a), - link.endpoint_b.parent.config.hostname, - str(link.endpoint_b), - link.is_up, - link.bandwidth, - link.current_load_percent, - ] - ) - links.remove(link) + for node in self.nodes.values(): + for link in links[::-1]: + if node in [link.endpoint_a.parent, link.endpoint_b.parent]: + table.add_row( + [ + link.endpoint_a.parent.config.hostname, + str(link.endpoint_a), + link.endpoint_b.parent.config.hostname, + str(link.endpoint_b), + link.is_up, + link.bandwidth, + link.current_load_percent, + ] + ) + links.remove(link) print(table) def clear_links(self): diff --git a/src/primaite/simulator/network/hardware/base.py b/src/primaite/simulator/network/hardware/base.py index 8653359a..4c252050 100644 --- a/src/primaite/simulator/network/hardware/base.py +++ b/src/primaite/simulator/network/hardware/base.py @@ -1197,7 +1197,7 @@ class UserSessionManager(Service, discriminator="user-session-manager"): """Request should take the form [username, password, remote_ip_address].""" username, password, remote_ip_address = request response = RequestResponse.from_bool(self.remote_login(username, password, remote_ip_address)) - response.data = {"remote_hostname": self.parent.hostname, "username": username} + response.data = {"remote_hostname": self.parent.config.hostname, "username": username} return response rm.add_request("remote_login", RequestType(func=_remote_login)) @@ -1230,7 +1230,7 @@ class UserSessionManager(Service, discriminator="user-session-manager"): if markdown: table.set_style(MARKDOWN) table.align = "l" - table.title = f"{self.parent.hostname} User Sessions" + table.title = f"{self.parent.config.hostname} User Sessions" def _add_session_to_table(user_session: UserSession): """ diff --git a/src/primaite/simulator/system/software.py b/src/primaite/simulator/system/software.py index 950f77c6..42468057 100644 --- a/src/primaite/simulator/system/software.py +++ b/src/primaite/simulator/system/software.py @@ -20,6 +20,7 @@ from primaite.utils.validation.port import Port if TYPE_CHECKING: from primaite.simulator.system.core.software_manager import SoftwareManager + from primaite.simulator.network.hardware.base import Node class SoftwareType(Enum): @@ -110,6 +111,7 @@ class Software(SimComponent, ABC): "The folder on the file system the Software uses." _fixing_countdown: Optional[int] = None "Current number of ticks left to patch the software." + # parent: Optional[Node] = None def __init__(self, **kwargs): super().__init__(**kwargs) diff --git a/tests/integration_tests/extensions/nodes/giga_switch.py b/tests/integration_tests/extensions/nodes/giga_switch.py index d9599618..5c202ed2 100644 --- a/tests/integration_tests/extensions/nodes/giga_switch.py +++ b/tests/integration_tests/extensions/nodes/giga_switch.py @@ -49,7 +49,7 @@ class GigaSwitch(NetworkNode, discriminator="gigaswitch"): if markdown: table.set_style(MARKDOWN) table.align = "l" - table.title = f"{self.hostname} Switch Ports" + table.title = f"{self.config.hostname} Switch Ports" for port_num, port in self.network_interface.items(): table.add_row([port_num, port.mac_address, port.speed, "Enabled" if port.enabled else "Disabled"]) print(table) diff --git a/tests/integration_tests/game_layer/actions/test_terminal_actions.py b/tests/integration_tests/game_layer/actions/test_terminal_actions.py index c39d8263..6d30644c 100644 --- a/tests/integration_tests/game_layer/actions/test_terminal_actions.py +++ b/tests/integration_tests/game_layer/actions/test_terminal_actions.py @@ -106,7 +106,6 @@ def test_remote_login_change_password(game_and_agent_fixture: Tuple[PrimaiteGame "username": "user123", "current_password": "password", "new_password": "different_password", - "remote_ip": str(server_1.network_interface[1].ip_address), }, ) agent.store_action(action) @@ -146,7 +145,6 @@ def test_change_password_logs_out_user(game_and_agent_fixture: Tuple[PrimaiteGam "username": "user123", "current_password": "password", "new_password": "different_password", - "remote_ip": str(server_1.network_interface[1].ip_address), }, ) agent.store_action(action) diff --git a/tests/unit_tests/_primaite/_game/_agent/test_agent.py b/tests/unit_tests/_primaite/_game/_agent/test_agent.py index a2693591..064537c2 100644 --- a/tests/unit_tests/_primaite/_game/_agent/test_agent.py +++ b/tests/unit_tests/_primaite/_game/_agent/test_agent.py @@ -4,7 +4,7 @@ from primaite.game.agent.scripted_agents.random_agent import RandomAgent def test_creating_empty_agent(): - agent = RandomAgent() + agent = RandomAgent(config={"ref" :"Empty Agent"}) assert len(agent.action_manager.action_map) == 0 assert isinstance(agent.observation_manager.obs, NullObservation) assert len(agent.reward_function.reward_components) == 0 From 7b80d15d8131bc5203c0a34b676746d572700486 Mon Sep 17 00:00:00 2001 From: Charlie Crane Date: Fri, 14 Feb 2025 11:39:25 +0000 Subject: [PATCH 21/37] # Pre-commit actions --- docs/source/how_to_guides/extensible_nodes.rst | 2 +- src/primaite/game/agent/interface.py | 3 +-- src/primaite/simulator/network/container.py | 5 ++--- src/primaite/simulator/system/software.py | 1 - tests/unit_tests/_primaite/_game/_agent/test_agent.py | 2 +- 5 files changed, 5 insertions(+), 8 deletions(-) diff --git a/docs/source/how_to_guides/extensible_nodes.rst b/docs/source/how_to_guides/extensible_nodes.rst index 043d0f06..4819cbb2 100644 --- a/docs/source/how_to_guides/extensible_nodes.rst +++ b/docs/source/how_to_guides/extensible_nodes.rst @@ -52,4 +52,4 @@ class Router(NetworkNode, identifier="router"): Changes to YAML file. ===================== -While effort has been made to ensure that nodes defined within configuration YAML files for use with PrimAITE 3.X remain compatible with PrimAITE v4+, it is encouraged to review for minor changes needed. +While effort has been made to ensure that nodes defined within configuration YAML files for use with PrimAITE 3.X remain compatible with PrimAITE v4+, it is encouraged to review for minor changes needed. diff --git a/src/primaite/game/agent/interface.py b/src/primaite/game/agent/interface.py index 571c850d..07363012 100644 --- a/src/primaite/game/agent/interface.py +++ b/src/primaite/game/agent/interface.py @@ -80,8 +80,7 @@ class AbstractAgent(BaseModel, ABC): _registry: ClassVar[Dict[str, Type[AbstractAgent]]] = {} def __init__(self, **kwargs): - """Initialise and setup agent logger""" - + """Initialise and setup agent logger.""" super().__init__(**kwargs) self.logger: AgentLog = AgentLog(agent_name=kwargs["config"]["ref"]) diff --git a/src/primaite/simulator/network/container.py b/src/primaite/simulator/network/container.py index 1fa8c680..b0426537 100644 --- a/src/primaite/simulator/network/container.py +++ b/src/primaite/simulator/network/container.py @@ -163,7 +163,6 @@ class Network(SimComponent): :param links: Include link details in the output. Defaults to True. :param markdown: Use Markdown style in table output. Defaults to False. """ - if nodes: table = PrettyTable(["Node", "Type", "Operating State"]) if markdown: @@ -192,8 +191,8 @@ class Network(SimComponent): port.ip_address, port.subnet_mask, node.config.default_gateway, - ] - ) + ] + ) print(table) if links: diff --git a/src/primaite/simulator/system/software.py b/src/primaite/simulator/system/software.py index 42468057..8fb64d73 100644 --- a/src/primaite/simulator/system/software.py +++ b/src/primaite/simulator/system/software.py @@ -20,7 +20,6 @@ from primaite.utils.validation.port import Port if TYPE_CHECKING: from primaite.simulator.system.core.software_manager import SoftwareManager - from primaite.simulator.network.hardware.base import Node class SoftwareType(Enum): diff --git a/tests/unit_tests/_primaite/_game/_agent/test_agent.py b/tests/unit_tests/_primaite/_game/_agent/test_agent.py index 064537c2..b555f1b2 100644 --- a/tests/unit_tests/_primaite/_game/_agent/test_agent.py +++ b/tests/unit_tests/_primaite/_game/_agent/test_agent.py @@ -4,7 +4,7 @@ from primaite.game.agent.scripted_agents.random_agent import RandomAgent def test_creating_empty_agent(): - agent = RandomAgent(config={"ref" :"Empty Agent"}) + agent = RandomAgent(config={"ref": "Empty Agent"}) assert len(agent.action_manager.action_map) == 0 assert isinstance(agent.observation_manager.obs, NullObservation) assert len(agent.reward_function.reward_components) == 0 From 2830a2aef5b3002028a580a3aa9aebe469a459f4 Mon Sep 17 00:00:00 2001 From: Charlie Crane Date: Fri, 14 Feb 2025 12:36:23 +0000 Subject: [PATCH 22/37] #3060 - Amendment to new version tag in configuration yaml. Now under the metadata tag --- .../config/_package_data/basic_lan_network_example.yaml | 3 ++- .../_package_data/client_server_p2p_network_example.yaml | 3 ++- src/primaite/config/_package_data/data_manipulation.yaml | 3 ++- src/primaite/config/_package_data/data_manipulation_marl.yaml | 3 ++- .../mini_scenario_with_simulation_variation/base_scenario.yaml | 3 ++- .../_package_data/multi_lan_internet_network_example.yaml | 3 ++- .../_package_data/scenario_with_placeholders/scenario.yaml | 3 ++- tests/assets/configs/action_penalty.yaml | 3 ++- tests/assets/configs/bad_primaite_session.yaml | 3 ++- tests/assets/configs/basic_c2_setup.yaml | 3 ++- tests/assets/configs/basic_firewall.yaml | 3 ++- .../configs/basic_node_with_software_listening_ports.yaml | 3 ++- tests/assets/configs/basic_node_with_users.yaml | 3 ++- tests/assets/configs/basic_switched_network.yaml | 3 ++- tests/assets/configs/data_manipulation.yaml | 3 ++- tests/assets/configs/dmz_network.yaml | 3 ++- tests/assets/configs/eval_only_primaite_session.yaml | 3 ++- tests/assets/configs/extended_config.yaml | 3 ++- tests/assets/configs/firewall_actions_network.yaml | 3 ++- tests/assets/configs/fixing_duration_one_item.yaml | 3 ++- tests/assets/configs/install_and_configure_apps.yaml | 3 ++- tests/assets/configs/multi_agent_session.yaml | 3 ++- .../configs/nmap_network_service_recon_red_agent_config.yaml | 3 ++- tests/assets/configs/nmap_ping_scan_red_agent_config.yaml | 3 ++- tests/assets/configs/nmap_port_scan_red_agent_config.yaml | 3 ++- tests/assets/configs/no_nodes_links_agents_network.yaml | 3 ++- tests/assets/configs/scenario_with_placeholders/scenario.yaml | 3 ++- tests/assets/configs/shared_rewards.yaml | 3 ++- tests/assets/configs/software_fixing_duration.yaml | 3 ++- tests/assets/configs/test_application_install.yaml | 3 ++- tests/assets/configs/test_primaite_session.yaml | 3 ++- tests/assets/configs/wireless_wan_network_config.yaml | 3 ++- .../configs/wireless_wan_network_config_freq_max_override.yaml | 3 ++- .../wireless_wan_network_config_freq_max_override_blocked.yaml | 3 ++- 34 files changed, 68 insertions(+), 34 deletions(-) diff --git a/src/primaite/config/_package_data/basic_lan_network_example.yaml b/src/primaite/config/_package_data/basic_lan_network_example.yaml index 854dfcf3..9996be84 100644 --- a/src/primaite/config/_package_data/basic_lan_network_example.yaml +++ b/src/primaite/config/_package_data/basic_lan_network_example.yaml @@ -1,4 +1,5 @@ -version: 3.0 +metadata: + version: 3.0 game: ports: diff --git a/src/primaite/config/_package_data/client_server_p2p_network_example.yaml b/src/primaite/config/_package_data/client_server_p2p_network_example.yaml index 702c864f..1a9fca98 100644 --- a/src/primaite/config/_package_data/client_server_p2p_network_example.yaml +++ b/src/primaite/config/_package_data/client_server_p2p_network_example.yaml @@ -1,4 +1,5 @@ -version: 3.0 +metadata: + version: 3.0 game: ports: diff --git a/src/primaite/config/_package_data/data_manipulation.yaml b/src/primaite/config/_package_data/data_manipulation.yaml index 72b3f671..81f9aabe 100644 --- a/src/primaite/config/_package_data/data_manipulation.yaml +++ b/src/primaite/config/_package_data/data_manipulation.yaml @@ -1,4 +1,5 @@ -version: 3.0 +metadata: + version: 3.0 io_settings: save_agent_actions: true diff --git a/src/primaite/config/_package_data/data_manipulation_marl.yaml b/src/primaite/config/_package_data/data_manipulation_marl.yaml index 178b3322..71507acb 100644 --- a/src/primaite/config/_package_data/data_manipulation_marl.yaml +++ b/src/primaite/config/_package_data/data_manipulation_marl.yaml @@ -1,4 +1,5 @@ -version: 3.0 +metadata: + version: 3.0 io_settings: save_agent_actions: true diff --git a/src/primaite/config/_package_data/mini_scenario_with_simulation_variation/base_scenario.yaml b/src/primaite/config/_package_data/mini_scenario_with_simulation_variation/base_scenario.yaml index 9530305a..2ea18867 100644 --- a/src/primaite/config/_package_data/mini_scenario_with_simulation_variation/base_scenario.yaml +++ b/src/primaite/config/_package_data/mini_scenario_with_simulation_variation/base_scenario.yaml @@ -1,4 +1,5 @@ -version: 3.0 +metadata: + version: 3.0 game: max_episode_length: 128 diff --git a/src/primaite/config/_package_data/multi_lan_internet_network_example.yaml b/src/primaite/config/_package_data/multi_lan_internet_network_example.yaml index 4e8f085e..deaef3bd 100644 --- a/src/primaite/config/_package_data/multi_lan_internet_network_example.yaml +++ b/src/primaite/config/_package_data/multi_lan_internet_network_example.yaml @@ -1,4 +1,5 @@ -version: 3.0 +metadata: + version: 3.0 game: ports: diff --git a/src/primaite/config/_package_data/scenario_with_placeholders/scenario.yaml b/src/primaite/config/_package_data/scenario_with_placeholders/scenario.yaml index d34fb310..4ec3d257 100644 --- a/src/primaite/config/_package_data/scenario_with_placeholders/scenario.yaml +++ b/src/primaite/config/_package_data/scenario_with_placeholders/scenario.yaml @@ -1,4 +1,5 @@ -version: 3.0 +metadata: + version: 3.0 io_settings: save_agent_actions: true diff --git a/tests/assets/configs/action_penalty.yaml b/tests/assets/configs/action_penalty.yaml index 9561db4f..6700172e 100644 --- a/tests/assets/configs/action_penalty.yaml +++ b/tests/assets/configs/action_penalty.yaml @@ -1,4 +1,5 @@ -version: 3.0 +metadata: + version: 3.0 io_settings: save_agent_actions: false diff --git a/tests/assets/configs/bad_primaite_session.yaml b/tests/assets/configs/bad_primaite_session.yaml index 1f59225b..b8551caf 100644 --- a/tests/assets/configs/bad_primaite_session.yaml +++ b/tests/assets/configs/bad_primaite_session.yaml @@ -1,4 +1,5 @@ -version: 3.0 +metadata: + version: 3.0 game: ports: diff --git a/tests/assets/configs/basic_c2_setup.yaml b/tests/assets/configs/basic_c2_setup.yaml index 3011457f..9b569b44 100644 --- a/tests/assets/configs/basic_c2_setup.yaml +++ b/tests/assets/configs/basic_c2_setup.yaml @@ -4,7 +4,8 @@ # | node_a |------| switch_1 |------| node_b | # -------------- -------------- -------------- # -version: 3.0 +metadata: + version: 3.0 io_settings: save_step_metadata: false diff --git a/tests/assets/configs/basic_firewall.yaml b/tests/assets/configs/basic_firewall.yaml index 1aa1361a..26038270 100644 --- a/tests/assets/configs/basic_firewall.yaml +++ b/tests/assets/configs/basic_firewall.yaml @@ -4,7 +4,8 @@ # | client_1 |------| switch_1 |------| client_2 | # -------------- -------------- -------------- # -version: 3.0 +metadata: + version: 3.0 io_settings: save_step_metadata: false diff --git a/tests/assets/configs/basic_node_with_software_listening_ports.yaml b/tests/assets/configs/basic_node_with_software_listening_ports.yaml index b503d66d..6372de54 100644 --- a/tests/assets/configs/basic_node_with_software_listening_ports.yaml +++ b/tests/assets/configs/basic_node_with_software_listening_ports.yaml @@ -1,4 +1,5 @@ -version: 3.0 +metadata: + version: 3.0 io_settings: save_step_metadata: false diff --git a/tests/assets/configs/basic_node_with_users.yaml b/tests/assets/configs/basic_node_with_users.yaml index 3009b79e..20331ff2 100644 --- a/tests/assets/configs/basic_node_with_users.yaml +++ b/tests/assets/configs/basic_node_with_users.yaml @@ -1,4 +1,5 @@ -version: 3.0 +metadata: + version: 3.0 io_settings: save_step_metadata: false diff --git a/tests/assets/configs/basic_switched_network.yaml b/tests/assets/configs/basic_switched_network.yaml index 7e8a5715..b57ed3e0 100644 --- a/tests/assets/configs/basic_switched_network.yaml +++ b/tests/assets/configs/basic_switched_network.yaml @@ -4,7 +4,8 @@ # | client_1 |------| switch_1 |------| client_2 | # -------------- -------------- -------------- # -version: 3.0 +metadata: + version: 3.0 io_settings: save_step_metadata: false diff --git a/tests/assets/configs/data_manipulation.yaml b/tests/assets/configs/data_manipulation.yaml index baf9df7d..59f97644 100644 --- a/tests/assets/configs/data_manipulation.yaml +++ b/tests/assets/configs/data_manipulation.yaml @@ -1,4 +1,5 @@ -version: 3.0 +metadata: + version: 3.0 io_settings: save_agent_actions: true diff --git a/tests/assets/configs/dmz_network.yaml b/tests/assets/configs/dmz_network.yaml index 456920f1..0accb3b2 100644 --- a/tests/assets/configs/dmz_network.yaml +++ b/tests/assets/configs/dmz_network.yaml @@ -30,7 +30,8 @@ # | external_computer |------| switch_3 |------| external_server | # ----------------------- -------------- --------------------- # -version: 3.0 +metadata: + version: 3.0 io_settings: save_step_metadata: false diff --git a/tests/assets/configs/eval_only_primaite_session.yaml b/tests/assets/configs/eval_only_primaite_session.yaml index c0a61b9d..6085b1e7 100644 --- a/tests/assets/configs/eval_only_primaite_session.yaml +++ b/tests/assets/configs/eval_only_primaite_session.yaml @@ -1,4 +1,5 @@ -version: 3.0 +metadata: + version: 3.0 game: ports: diff --git a/tests/assets/configs/extended_config.yaml b/tests/assets/configs/extended_config.yaml index 1b0772c1..a58a9d4a 100644 --- a/tests/assets/configs/extended_config.yaml +++ b/tests/assets/configs/extended_config.yaml @@ -1,4 +1,5 @@ -version: 3.0 +metadata: + version: 3.0 io_settings: save_agent_actions: true diff --git a/tests/assets/configs/firewall_actions_network.yaml b/tests/assets/configs/firewall_actions_network.yaml index b624be52..66470f5a 100644 --- a/tests/assets/configs/firewall_actions_network.yaml +++ b/tests/assets/configs/firewall_actions_network.yaml @@ -30,7 +30,8 @@ # | external_computer |------| switch_3 |------| external_server | # ----------------------- -------------- --------------------- # -version: 3.0 +metadata: + version: 3.0 io_settings: save_step_metadata: false diff --git a/tests/assets/configs/fixing_duration_one_item.yaml b/tests/assets/configs/fixing_duration_one_item.yaml index ad94a9da..02b69e5c 100644 --- a/tests/assets/configs/fixing_duration_one_item.yaml +++ b/tests/assets/configs/fixing_duration_one_item.yaml @@ -4,7 +4,8 @@ # | client_1 |------| switch_1 |------| client_2 | # -------------- -------------- -------------- # -version: 3.0 +metadata: + version: 3.0 io_settings: save_step_metadata: false diff --git a/tests/assets/configs/install_and_configure_apps.yaml b/tests/assets/configs/install_and_configure_apps.yaml index 4f96496e..35546902 100644 --- a/tests/assets/configs/install_and_configure_apps.yaml +++ b/tests/assets/configs/install_and_configure_apps.yaml @@ -1,4 +1,5 @@ -version: 3.0 +metadata: + version: 3.0 io_settings: save_step_metadata: false diff --git a/tests/assets/configs/multi_agent_session.yaml b/tests/assets/configs/multi_agent_session.yaml index 263d5f66..de0cdad9 100644 --- a/tests/assets/configs/multi_agent_session.yaml +++ b/tests/assets/configs/multi_agent_session.yaml @@ -1,4 +1,5 @@ -version: 3.0 +metadata: + version: 3.0 io_settings: save_agent_actions: false diff --git a/tests/assets/configs/nmap_network_service_recon_red_agent_config.yaml b/tests/assets/configs/nmap_network_service_recon_red_agent_config.yaml index 6abb11ce..f7b8431e 100644 --- a/tests/assets/configs/nmap_network_service_recon_red_agent_config.yaml +++ b/tests/assets/configs/nmap_network_service_recon_red_agent_config.yaml @@ -1,4 +1,5 @@ -version: 3.0 +metadata: + version: 3.0 io_settings: save_step_metadata: false diff --git a/tests/assets/configs/nmap_ping_scan_red_agent_config.yaml b/tests/assets/configs/nmap_ping_scan_red_agent_config.yaml index 869f4fb8..112d7266 100644 --- a/tests/assets/configs/nmap_ping_scan_red_agent_config.yaml +++ b/tests/assets/configs/nmap_ping_scan_red_agent_config.yaml @@ -1,4 +1,5 @@ -version: 3.0 +metadata: + version: 3.0 io_settings: save_step_metadata: false diff --git a/tests/assets/configs/nmap_port_scan_red_agent_config.yaml b/tests/assets/configs/nmap_port_scan_red_agent_config.yaml index 1d5ab323..acd5319a 100644 --- a/tests/assets/configs/nmap_port_scan_red_agent_config.yaml +++ b/tests/assets/configs/nmap_port_scan_red_agent_config.yaml @@ -1,4 +1,5 @@ -version: 3.0 +metadata: + version: 3.0 io_settings: save_step_metadata: false diff --git a/tests/assets/configs/no_nodes_links_agents_network.yaml b/tests/assets/configs/no_nodes_links_agents_network.yaml index b79e6bc6..ed279b51 100644 --- a/tests/assets/configs/no_nodes_links_agents_network.yaml +++ b/tests/assets/configs/no_nodes_links_agents_network.yaml @@ -1,4 +1,5 @@ -version: 3.0 +metadata: + version: 3.0 io_settings: save_step_metadata: false diff --git a/tests/assets/configs/scenario_with_placeholders/scenario.yaml b/tests/assets/configs/scenario_with_placeholders/scenario.yaml index 76b51626..57fe59ab 100644 --- a/tests/assets/configs/scenario_with_placeholders/scenario.yaml +++ b/tests/assets/configs/scenario_with_placeholders/scenario.yaml @@ -1,4 +1,5 @@ -version: 3.0 +metadata: + version: 3.0 io_settings: save_agent_actions: true diff --git a/tests/assets/configs/shared_rewards.yaml b/tests/assets/configs/shared_rewards.yaml index 68b3b310..5aeb99fa 100644 --- a/tests/assets/configs/shared_rewards.yaml +++ b/tests/assets/configs/shared_rewards.yaml @@ -1,4 +1,5 @@ -version: 3.0 +metadata: + version: 3.0 io_settings: save_agent_actions: false diff --git a/tests/assets/configs/software_fixing_duration.yaml b/tests/assets/configs/software_fixing_duration.yaml index 2f5613e1..66ba6f18 100644 --- a/tests/assets/configs/software_fixing_duration.yaml +++ b/tests/assets/configs/software_fixing_duration.yaml @@ -4,7 +4,8 @@ # | client_1 |------| switch_1 |------| client_2 | # -------------- -------------- -------------- # -version: 3.0 +metadata: + version: 3.0 io_settings: save_step_metadata: false diff --git a/tests/assets/configs/test_application_install.yaml b/tests/assets/configs/test_application_install.yaml index d49d9bf2..8a292f83 100644 --- a/tests/assets/configs/test_application_install.yaml +++ b/tests/assets/configs/test_application_install.yaml @@ -1,4 +1,5 @@ -version: 3.0 +metadata: + version: 3.0 io_settings: save_agent_actions: true diff --git a/tests/assets/configs/test_primaite_session.yaml b/tests/assets/configs/test_primaite_session.yaml index c1206634..ad43732f 100644 --- a/tests/assets/configs/test_primaite_session.yaml +++ b/tests/assets/configs/test_primaite_session.yaml @@ -1,4 +1,5 @@ -version: 3.0 +metadata: + version: 3.0 io_settings: save_agent_actions: true diff --git a/tests/assets/configs/wireless_wan_network_config.yaml b/tests/assets/configs/wireless_wan_network_config.yaml index b722e50f..45721d8a 100644 --- a/tests/assets/configs/wireless_wan_network_config.yaml +++ b/tests/assets/configs/wireless_wan_network_config.yaml @@ -1,4 +1,5 @@ -version: 3.0 +metadata: + version: 3.0 game: max_episode_length: 256 diff --git a/tests/assets/configs/wireless_wan_network_config_freq_max_override.yaml b/tests/assets/configs/wireless_wan_network_config_freq_max_override.yaml index 4c2f4f86..20e48a89 100644 --- a/tests/assets/configs/wireless_wan_network_config_freq_max_override.yaml +++ b/tests/assets/configs/wireless_wan_network_config_freq_max_override.yaml @@ -1,4 +1,5 @@ -version: 3.0 +metadata: + version: 3.0 game: max_episode_length: 256 diff --git a/tests/assets/configs/wireless_wan_network_config_freq_max_override_blocked.yaml b/tests/assets/configs/wireless_wan_network_config_freq_max_override_blocked.yaml index a98b6ee2..6342d1b1 100644 --- a/tests/assets/configs/wireless_wan_network_config_freq_max_override_blocked.yaml +++ b/tests/assets/configs/wireless_wan_network_config_freq_max_override_blocked.yaml @@ -1,4 +1,5 @@ -version: 3.0 +metadata: + version: 3.0 game: max_episode_length: 256 From bf30b70bd2055500dfaf88d78682734cdf124af2 Mon Sep 17 00:00:00 2001 From: Charlie Crane Date: Mon, 17 Feb 2025 10:09:39 +0000 Subject: [PATCH 23/37] #3060 - Removal of un-necessary check in game.py --- src/primaite/game/game.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/primaite/game/game.py b/src/primaite/game/game.py index 4741199c..1427776e 100644 --- a/src/primaite/game/game.py +++ b/src/primaite/game/game.py @@ -247,13 +247,6 @@ class PrimaiteGame: game.options = PrimaiteGameOptions(**cfg["game"]) game.save_step_metadata = cfg.get("io_settings", {}).get("save_step_metadata") or False - config_version = cfg.get("version", {}) - - # TODO: Future YAML config should specify the PrimAITE version they are written for. - # For now, we warn that if it is missing, pending a mechanism to handle variations. - if not isinstance(config_version, int): - _LOGGER.warning("Version definition is missing from provided configuration. ") - # 1. create simulation sim = game.simulation net = sim.network From 486c797d60422c6f0acfea7a2e9cf9094bf4b42c Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Mon, 17 Feb 2025 11:54:13 +0000 Subject: [PATCH 24/37] #3075: Change duplicate application_name in config file. --- src/primaite/config/_package_data/data_manipulation.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/primaite/config/_package_data/data_manipulation.yaml b/src/primaite/config/_package_data/data_manipulation.yaml index 4705c135..ceee607d 100644 --- a/src/primaite/config/_package_data/data_manipulation.yaml +++ b/src/primaite/config/_package_data/data_manipulation.yaml @@ -81,7 +81,7 @@ agents: action: node-application-execute options: node_name: client_1 - application_name: web-browser + application_name: database-client reward_function: reward_components: From 7e82de919c4287eb64d4e47b84808be2d2525f61 Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Mon, 17 Feb 2025 14:19:56 +0000 Subject: [PATCH 25/37] #3075: Markdown changes. --- ...ommand-and-Control-E2E-Demonstration.ipynb | 85 +- ...a-Manipulation-Customising-Red-Agent.ipynb | 48 +- .../Data-Manipulation-E2E-Demonstration.ipynb | 4531 ++++++++++++++++- 3 files changed, 4548 insertions(+), 116 deletions(-) diff --git a/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb b/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb index 474dadff..6dc8e077 100644 --- a/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb +++ b/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb @@ -6,7 +6,7 @@ "source": [ "# Command and Control Application Suite E2E Demonstration\n", "\n", - "© Crown-owned copyright 2024, Defence Science and Technology Laboratory UK\n", + "© Crown-owned copyright 2025, Defence Science and Technology Laboratory UK\n", "\n", "This notebook demonstrates the current implementation of the command and control (C2) server and beacon applications in primAITE." ] @@ -201,15 +201,15 @@ " nodes: # Node List\n", " - node_name: web_server\n", " applications: \n", - " - application_name: C2Beacon\n", + " - application_name: c2-beacon\n", " ...\n", " ...\n", " action_map:\n", " 1:\n", - " action: node_application_install \n", + " action: node-application-install \n", " options:\n", " node_id: 0 # Index 0 at the node list.\n", - " application_name: C2Beacon\n", + " application_name: c2-beacon\n", "```" ] }, @@ -244,7 +244,7 @@ " action_map:\n", " ...\n", " 2:\n", - " action: configure_c2_beacon\n", + " action: configure-c2-beacon\n", " options:\n", " node_id: 0 # Node Index\n", " config: # Further information about these config options can be found at the bottom of this notebook.\n", @@ -273,7 +273,7 @@ "source": [ "### **Command and Control** | C2 Beacon Actions | node_application_execute\n", "\n", - "The final action is ``node_application_execute`` which is used to establish a connection for the C2 application. This action can be called by the Red Agent via action ``3`` in it's action map. \n", + "The final action is ``node-application-execute`` which is used to establish a connection for the C2 application. This action can be called by the Red Agent via action ``3`` in it's action map. \n", "\n", "The yaml snippet below shows all the relevant agent options for this action:\n", "\n", @@ -283,13 +283,13 @@ " nodes: # Node List\n", " - node_name: web_server\n", " applications: \n", - " - application_name: C2Beacon\n", + " - application_name: c2-beacon\n", " ...\n", " ...\n", " action_map:\n", " ...\n", " 3:\n", - " action: node_application_execute\n", + " action: node-application-execute\n", " options:\n", " node_id: 0\n", " application_id: 0\n", @@ -331,34 +331,27 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### **Command and Control** | C2 Server Actions | C2_SERVER_TERMINAL_COMMAND\n", + "### **Command and Control** | C2 Server Actions | c2-server-terminal-command\n", "\n", - "The C2 Server's terminal action: ``C2_SERVER_TERMINAL_COMMAND`` is indexed at ``4`` in it's action map. \n", + "The C2 Server's terminal action: ``c2-server-terminal-command`` is indexed at ``4`` in it's action map. \n", "\n", "This action leverages the terminal service that is installed by default on all nodes to grant red agents a lot more configurability. If you're unfamiliar with terminals then it's recommended that you refer to the ``Terminal Processing`` notebook.\n", "\n", "It's worth noting that an additional benefit a red agent has when using the terminal service via the C2 Server is that you can execute multiple commands in one action. \n", "\n", - "In this notebook, the ``C2_SERVER_TERMINAL_COMMAND`` is used to install a RansomwareScript application on the ``web_server`` node.\n", + "In this notebook, the ``c2-server-terminal-command`` is used to install a RansomwareScript application on the ``web_server`` node.\n", "\n", "The yaml snippet below shows all the relevant agent options for this action:\n", "\n", "``` yaml\n", " action_space:\n", - " options:\n", - " nodes: # Node List\n", - " ...\n", - " - node_name: client_1\n", - " applications: \n", - " - application_name: C2Server\n", - " ...\n", " action_map:\n", + " ...\n", " 4:\n", - " action: C2_SERVER_TERMINAL_COMMAND\n", + " action: c2-server-terminal-command\n", " options:\n", - " node_id: 1\n", + " node_name: client_1\n", " ip_address:\n", - " account:\n", " username: admin\n", " password: admin\n", " commands:\n", @@ -366,7 +359,7 @@ " - software_manager\n", " - application\n", " - install\n", - " - RansomwareScript\n", + " - ransomware-script\n", "```" ] }, @@ -402,14 +395,8 @@ "\n", "``` yaml\n", " action_space:\n", - " options:\n", - " nodes: # Node List\n", - " ...\n", - " - node_name: client_1\n", - " applications: \n", - " - application_name: C2Server\n", - " ...\n", " action_map:\n", + " ...\n", " 5:\n", " action: c2-server-ransomware-configure\n", " options:\n", @@ -444,9 +431,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### **Command and Control** | C2 Server Actions | c2_server_data_exfiltrate\n", + "### **Command and Control** | C2 Server Actions | c2-server-data-exfiltrate\n", "\n", - "The second to last action available is the ``c2_server_data_exfiltrate`` which is indexed as action ``6`` in the action map.\n", + "The second to last action available is the ``c2-server-data-exfiltrate`` which is indexed as action ``6`` in the action map.\n", "\n", "This action can be used to exfiltrate a target file on a remote node to the C2 Beacon and the C2 Server's host file system via the ``FTP`` services.\n", "\n", @@ -454,25 +441,18 @@ "\n", "``` yaml\n", " action_space:\n", - " options:\n", - " nodes: # Node List\n", - " ...\n", - " - node_name: client_1\n", - " applications: \n", - " - application_name: C2Server\n", - " ...\n", " action_map:\n", + " ...\n", " 6:\n", - " action: c2_server_data_exfiltrate\n", + " action: c2-server-data-exfiltrate\n", " options:\n", " node_id: 1\n", " target_file_name: \"database.db\"\n", " target_folder_name: \"database\"\n", " exfiltration_folder_name: \"spoils\"\n", " target_ip_address: \"192.168.1.14\"\n", - " account:\n", - " username: \"admin\",\n", - " password: \"admin\"\n", + " username: \"admin\",\n", + " password: \"admin\"\n", "\n", "```" ] @@ -510,9 +490,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### **Command and Control** | C2 Server Actions | c2_server_ransomware_launch\n", + "### **Command and Control** | C2 Server Actions | c2-server-ransomware-launch\n", "\n", - "Finally, the last available action is for the c2_server_ransomware_launch to start the ransomware script installed on the same node as the C2 beacon.\n", + "Finally, the last available action is for the c2-server-ransomware-launch to start the ransomware script installed on the same node as the C2 beacon.\n", "\n", "This action is indexed as action ``7``.\n", "\n", @@ -520,16 +500,10 @@ "\n", "``` yaml\n", " action_space:\n", - " options:\n", - " nodes: # Node List\n", - " ...\n", - " - node_name: client_1\n", - " applications: \n", - " - application_name: C2Server\n", " ...\n", " action_map:\n", " 7:\n", - " action: c2_server_ransomware_launch\n", + " action: c2-server-ransomware-launch\n", " options:\n", " node_id: 1\n", "```\n" @@ -1337,18 +1311,15 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "As demonstrated earlier, red agents can use the ``configure_c2_beacon`` action to configure these settings mid episode through the configuration options:\n", + "As demonstrated earlier, red agents can use the ``configure-c2-beacon`` action to configure these settings mid episode through the configuration options:\n", "\n", "``` YAML\n", "...\n", - " action: configure_c2_beacon\n", + " action: configure-c2-beacon\n", " options:\n", - " node_id: 0\n", + " node_name: web_server\n", " config:\n", " c2_server_ip_address: 192.168.10.21\n", - " keep_alive_frequency: 10\n", - " masquerade_protocol: TCP\n", - " masquerade_port: DNS\n", "```" ] }, diff --git a/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb b/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb index 2e0a0d40..1eb814e3 100644 --- a/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb +++ b/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb @@ -146,42 +146,15 @@ "```yaml\n", " - ref: data_manipulation_attacker # name of agent\n", " team: RED # not used, just for human reference\n", - " type: RedDatabaseCorruptingAgent # type of agent - this lets primaite know which agent class to use\n", - "\n", - " # Since the agent does not need to react to what is happening in the environment, the observation space is empty.\n", - " observation_space:\n", - " type: UC2RedObservation\n", - " options:\n", - " nodes: {}\n", - "\n", - " action_space:\n", - " \n", - " # The agent has access to the DataManipulationBoth on clients 1 and 2.\n", - " options:\n", - " nodes:\n", - " - node_name: client_1 # The network should have a node called client_1\n", - " applications:\n", - " - application_name: DataManipulationBot # The node client_1 should have DataManipulationBot configured on it\n", - " - node_name: client_2 # The network should have a node called client_2\n", - " applications:\n", - " - application_name: DataManipulationBot # The node client_2 should have DataManipulationBot configured on it\n", - "\n", - " # not important\n", - " max_folders_per_node: 1\n", - " max_files_per_folder: 1\n", - " max_services_per_node: 1\n", - "\n", - " # red agent does not need a reward function\n", - " reward_function:\n", - " reward_components:\n", - " - type: DUMMY\n", + " type: red-database-corrupting-agent # type of agent - this lets primaite know which agent class to use\n", "\n", " # These actions are passed to the RedDatabaseCorruptingAgent init method, they dictate the schedule of attacks\n", " agent_settings:\n", - " start_settings:\n", - " start_step: 25 # first attack at step 25\n", - " frequency: 20 # attacks will happen every 20 steps (on average)\n", - " variance: 5 # the timing of attacks will vary by up to 5 steps earlier or later\n", + " possible_start_nodes: [client_1, client_2] # List of clients the attack can start from\n", + " target_application: data-manipulation-bot\n", + " start_step: 25 # first attack at step 25\n", + " frequency: 20 # attacks will happen every 20 steps (on average)\n", + " variance: 5 # the timing of attacks will vary by up to 5 steps earlier or later\n", "```" ] }, @@ -201,8 +174,7 @@ "simulation:\n", " network:\n", " nodes:\n", - " - ref: client_1\n", - " hostname: client_1\n", + " - hostname: client_1\n", " type: computer\n", " ip_address: 192.168.10.21\n", " subnet_mask: 255.255.255.0\n", @@ -210,15 +182,13 @@ " \n", " # \n", " applications:\n", - " - ref: data_manipulation_bot\n", - " type: DataManipulationBot\n", + " - type: data-manipulation-bot\n", " options:\n", " port_scan_p_of_success: 0.8 # Probability that port scan is successful\n", " data_manipulation_p_of_success: 0.8 # Probability that SQL attack is successful\n", " payload: \"DELETE\" # The SQL query which causes the attack (this has to be DELETE)\n", " server_ip: 192.168.1.14 # IP address of server hosting the database\n", - " - ref: client_1_database_client\n", - " type: DatabaseClient # Database client must be installed in order for DataManipulationBot to function\n", + " - type: database-client # Database client must be installed in order for DataManipulationBot to function\n", " options:\n", " db_server_ip: 192.168.1.14 # IP address of server hosting the database\n", "```" diff --git a/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb b/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb index 2d3f99b2..f3e3fb98 100644 --- a/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb +++ b/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb @@ -6,7 +6,7 @@ "source": [ "# Data Manipulation Scenario\n", "\n", - "© Crown-owned copyright 2024, Defence Science and Technology Laboratory UK" + "© Crown-owned copyright 2025, Defence Science and Technology Laboratory UK" ] }, { @@ -373,16 +373,29 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-02-17 11:15:30,959: Performing the PrimAITE first-time setup...\n", + "2025-02-17 11:15:30,960: Building the PrimAITE app directories...\n", + "2025-02-17 11:15:30,960: Building primaite_config.yaml...\n", + "2025-02-17 11:15:30,960: Rebuilding the demo notebooks...\n", + "2025-02-17 11:15:30,967: Rebuilding the example notebooks...\n", + "2025-02-17 11:15:30,970: PrimAITE setup complete!\n" + ] + } + ], "source": [ "!primaite setup" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": { "tags": [] }, @@ -394,7 +407,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": { "tags": [] }, @@ -420,9 +433,254 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-17 11:15:35,108: PrimaiteGymEnv RNG seed = None\n", + "2025-02-17 11:15:35,109: Resetting environment, episode 0, avg. reward: 0.0\n", + "2025-02-17 11:15:35,111: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-17/11-15-31/agent_actions/episode_0.json\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "env created successfully\n", + "{'ICS': 0,\n", + " 'LINKS': {1: {'PROTOCOLS': {'ALL': 0}},\n", + " 2: {'PROTOCOLS': {'ALL': 0}},\n", + " 3: {'PROTOCOLS': {'ALL': 0}},\n", + " 4: {'PROTOCOLS': {'ALL': 0}},\n", + " 5: {'PROTOCOLS': {'ALL': 0}},\n", + " 6: {'PROTOCOLS': {'ALL': 0}},\n", + " 7: {'PROTOCOLS': {'ALL': 0}},\n", + " 8: {'PROTOCOLS': {'ALL': 0}},\n", + " 9: {'PROTOCOLS': {'ALL': 0}},\n", + " 10: {'PROTOCOLS': {'ALL': 0}}},\n", + " 'NODES': {'HOST0': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0,\n", + " 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0,\n", + " 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0,\n", + " 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0,\n", + " 'outbound': 0}}},\n", + " 'nic_status': 0}},\n", + " 'SERVICES': {1: {'health_status': 0,\n", + " 'operating_status': 0}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'HOST1': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0,\n", + " 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0,\n", + " 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0,\n", + " 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0,\n", + " 'outbound': 0}}},\n", + " 'nic_status': 0}},\n", + " 'SERVICES': {1: {'health_status': 0,\n", + " 'operating_status': 1}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'HOST2': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0,\n", + " 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0,\n", + " 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0,\n", + " 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0,\n", + " 'outbound': 0}}},\n", + " 'nic_status': 0}},\n", + " 'SERVICES': {1: {'health_status': 0,\n", + " 'operating_status': 0}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'HOST3': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0,\n", + " 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0,\n", + " 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0,\n", + " 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0,\n", + " 'outbound': 0}}},\n", + " 'nic_status': 0}},\n", + " 'SERVICES': {1: {'health_status': 0,\n", + " 'operating_status': 0}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'HOST4': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0,\n", + " 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0,\n", + " 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0,\n", + " 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0,\n", + " 'outbound': 0}}},\n", + " 'nic_status': 1}},\n", + " 'SERVICES': {1: {'health_status': 0,\n", + " 'operating_status': 0}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'HOST5': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0,\n", + " 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0,\n", + " 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0,\n", + " 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0,\n", + " 'outbound': 0}}},\n", + " 'nic_status': 0}},\n", + " 'SERVICES': {1: {'health_status': 0,\n", + " 'operating_status': 0}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'HOST6': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0,\n", + " 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0,\n", + " 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0,\n", + " 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0,\n", + " 'outbound': 0}}},\n", + " 'nic_status': 0}},\n", + " 'SERVICES': {1: {'health_status': 0,\n", + " 'operating_status': 0}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'ROUTER0': {'ACL': {1: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 0,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 2: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 1,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 3: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 2,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 4: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 3,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 5: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 4,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 6: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 5,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 7: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 6,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 8: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 7,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 9: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 8,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 10: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 9,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0}},\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}}}}\n" + ] + } + ], "source": [ "# create the env\n", "with open(data_manipulation_config_path(), 'r') as f:\n", @@ -450,7 +708,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -468,9 +726,51 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "step: 1, Red action: DO NOTHING, Blue reward:0.65\n", + "step: 2, Red action: DO NOTHING, Blue reward:0.65\n", + "step: 3, Red action: DO NOTHING, Blue reward:0.65\n", + "step: 4, Red action: DO NOTHING, Blue reward:0.90\n", + "step: 5, Red action: DO NOTHING, Blue reward:0.90\n", + "step: 6, Red action: DO NOTHING, Blue reward:0.90\n", + "step: 7, Red action: DO NOTHING, Blue reward:0.90\n", + "step: 8, Red action: DO NOTHING, Blue reward:0.90\n", + "step: 9, Red action: DO NOTHING, Blue reward:0.90\n", + "step: 10, Red action: DO NOTHING, Blue reward:0.90\n", + "step: 11, Red action: DO NOTHING, Blue reward:0.90\n", + "step: 12, Red action: DO NOTHING, Blue reward:0.90\n", + "step: 13, Red action: DO NOTHING, Blue reward:0.90\n", + "step: 14, Red action: DO NOTHING, Blue reward:0.90\n", + "step: 15, Red action: DO NOTHING, Blue reward:0.90\n", + "step: 16, Red action: DO NOTHING, Blue reward:0.90\n", + "step: 17, Red action: DO NOTHING, Blue reward:0.95\n", + "step: 18, Red action: DO NOTHING, Blue reward:0.95\n", + "step: 19, Red action: DO NOTHING, Blue reward:0.95\n", + "step: 20, Red action: DO NOTHING, Blue reward:0.95\n", + "step: 21, Red action: DO NOTHING, Blue reward:0.95\n", + "step: 22, Red action: DO NOTHING, Blue reward:0.95\n", + "step: 23, Red action: DO NOTHING, Blue reward:0.95\n", + "step: 24, Red action: DO NOTHING, Blue reward:0.95\n", + "step: 25, Red action: DO NOTHING, Blue reward:0.95\n", + "step: 26, Red action: ATTACK from client 2, Blue reward:0.15\n", + "step: 27, Red action: DO NOTHING, Blue reward:-0.35\n", + "step: 28, Red action: DO NOTHING, Blue reward:-0.85\n", + "step: 29, Red action: DO NOTHING, Blue reward:-0.85\n", + "step: 30, Red action: DO NOTHING, Blue reward:-0.85\n", + "step: 31, Red action: DO NOTHING, Blue reward:-0.85\n", + "step: 32, Red action: DO NOTHING, Blue reward:-0.85\n", + "step: 33, Red action: DO NOTHING, Blue reward:-0.85\n", + "step: 34, Red action: DO NOTHING, Blue reward:-0.85\n", + "step: 35, Red action: DO NOTHING, Blue reward:-0.85\n" + ] + } + ], "source": [ "for step in range(35):\n", " obs, reward, terminated, truncated, info = env.step(0)\n", @@ -486,9 +786,198 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'HOST0': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 0}},\n", + " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'HOST1': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 0}},\n", + " 'SERVICES': {1: {'health_status': 0, 'operating_status': 1}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'HOST2': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 0}},\n", + " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'HOST3': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 0}},\n", + " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'HOST4': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 1}},\n", + " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'HOST5': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 0}},\n", + " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'HOST6': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 0}},\n", + " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'ROUTER0': {'ACL': {1: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 0,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 2: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 1,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 3: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 2,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 4: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 3,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 5: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 4,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 6: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 5,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 7: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 6,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 8: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 7,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 9: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 8,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 10: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 9,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0}},\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}}}\n" + ] + } + ], "source": [ "pprint(obs['NODES'])" ] @@ -502,9 +991,198 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'HOST0': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 0}},\n", + " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'HOST1': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 0}},\n", + " 'SERVICES': {1: {'health_status': 3, 'operating_status': 1}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'HOST2': {'FOLDERS': {1: {'FILES': {1: {'health_status': 2}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 0}},\n", + " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'HOST3': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 0}},\n", + " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'HOST4': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 1}},\n", + " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'HOST5': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 0}},\n", + " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'HOST6': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 0}},\n", + " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'ROUTER0': {'ACL': {1: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 0,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 2: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 1,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 3: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 2,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 4: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 3,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 5: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 4,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 6: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 5,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 7: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 6,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 8: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 7,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 9: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 8,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 10: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 9,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0}},\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}}}\n" + ] + } + ], "source": [ "obs, reward, terminated, truncated, info = env.step(9) # scan database file\n", "obs, reward, terminated, truncated, info = env.step(1) # scan webapp service\n", @@ -536,9 +1214,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "step: 38\n", + "Red action: do-nothing\n", + "Green action: do-nothing\n", + "Green action: node-application-execute\n", + "Blue reward:-0.8500000000000001\n" + ] + } + ], "source": [ "obs, reward, terminated, truncated, info = env.step(13) # patch the database\n", "print(f\"step: {env.game.step_counter}\")\n", @@ -561,9 +1251,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "step: 39\n", + "Red action: do-nothing\n", + "Green action: timestep=38 action='node-application-execute' parameters={'node_name': 'client_2', 'application_name': 'web-browser'} request=['network', 'node', 'client_2', 'application', 'web-browser', 'execute'] response=RequestResponse(status='failure', data={}) reward=-0.2 reward_info={'connection_attempt_status': 'n/a'}\n", + "Green action: timestep=38 action='node-application-execute' parameters={'node_name': 'client_1', 'application_name': 'web-browser'} request=['network', 'node', 'client_1', 'application', 'web-browser', 'execute'] response=RequestResponse(status='failure', data={}) reward=-0.25 reward_info={'connection_attempt_status': 'n/a'}\n", + "Blue reward:-0.05\n" + ] + } + ], "source": [ "obs, reward, terminated, truncated, info = env.step(0) # do nothing\n", "print(f\"step: {env.game.step_counter}\")\n", @@ -586,7 +1288,3796 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "step: 40, Red action: do-nothing, Blue reward:-0.05\n", + "step: 41, Red action: do-nothing, Blue reward:-0.05\n", + "step: 42, Red action: do-nothing, Blue reward:-0.05\n", + "step: 43, Red action: do-nothing, Blue reward:0.95\n", + "step: 44, Red action: do-nothing, Blue reward:0.95\n", + "step: 45, Red action: do-nothing, Blue reward:0.95\n", + "step: 46, Red action: do-nothing, Blue reward:0.95\n", + "step: 47, Red action: do-nothing, Blue reward:0.95\n", + "step: 48, Red action: do-nothing, Blue reward:0.95\n", + "step: 49, Red action: do-nothing, Blue reward:0.95\n", + "step: 50, Red action: do-nothing, Blue reward:0.95\n", + "step: 51, Red action: node-application-execute, Blue reward:0.95\n", + "step: 52, Red action: do-nothing, Blue reward:0.95\n", + "step: 53, Red action: do-nothing, Blue reward:0.85\n", + "step: 54, Red action: do-nothing, Blue reward:0.85\n", + "step: 55, Red action: do-nothing, Blue reward:0.85\n", + "step: 56, Red action: do-nothing, Blue reward:0.85\n", + "step: 57, Red action: do-nothing, Blue reward:0.85\n", + "step: 58, Red action: do-nothing, Blue reward:0.85\n", + "step: 59, Red action: do-nothing, Blue reward:0.85\n", + "step: 60, Red action: do-nothing, Blue reward:0.85\n", + "step: 61, Red action: do-nothing, Blue reward:0.85\n", + "step: 62, Red action: do-nothing, Blue reward:0.85\n", + "step: 63, Red action: do-nothing, Blue reward:0.85\n", + "step: 64, Red action: do-nothing, Blue reward:0.85\n", + "step: 65, Red action: do-nothing, Blue reward:0.85\n", + "step: 66, Red action: do-nothing, Blue reward:0.85\n", + "step: 67, Red action: do-nothing, Blue reward:0.85\n", + "step: 68, Red action: do-nothing, Blue reward:0.85\n", + "step: 69, Red action: do-nothing, Blue reward:0.85\n", + "step: 70, Red action: do-nothing, Blue reward:0.85\n", + "step: 71, Red action: do-nothing, Blue reward:0.85\n", + "step: 72, Red action: do-nothing, Blue reward:0.85\n", + "step: 73, Red action: do-nothing, Blue reward:0.85\n", + "step: 74, Red action: do-nothing, Blue reward:0.85\n", + "step: 75, Red action: node-application-execute, Blue reward:0.85\n", + "step: 76, Red action: do-nothing, Blue reward:0.85\n", + "step: 77, Red action: do-nothing, Blue reward:0.85\n", + "step: 78, Red action: do-nothing, Blue reward:0.85\n", + "step: 79, Red action: do-nothing, Blue reward:0.85\n", + "step: 80, Red action: do-nothing, Blue reward:0.85\n", + "step: 81, Red action: do-nothing, Blue reward:0.85\n", + "step: 82, Red action: do-nothing, Blue reward:0.85\n", + "step: 83, Red action: do-nothing, Blue reward:0.85\n", + "step: 84, Red action: do-nothing, Blue reward:0.85\n", + "step: 85, Red action: do-nothing, Blue reward:0.85\n", + "step: 86, Red action: do-nothing, Blue reward:0.85\n", + "step: 87, Red action: do-nothing, Blue reward:0.85\n", + "step: 88, Red action: do-nothing, Blue reward:0.85\n", + "step: 89, Red action: do-nothing, Blue reward:0.85\n", + "step: 90, Red action: do-nothing, Blue reward:0.85\n", + "step: 91, Red action: do-nothing, Blue reward:0.85\n", + "step: 92, Red action: do-nothing, Blue reward:0.85\n", + "step: 93, Red action: do-nothing, Blue reward:0.85\n", + "step: 94, Red action: do-nothing, Blue reward:0.85\n", + "step: 95, Red action: do-nothing, Blue reward:0.85\n", + "step: 96, Red action: do-nothing, Blue reward:0.85\n", + "step: 97, Red action: node-application-execute, Blue reward:0.85\n", + "step: 98, Red action: do-nothing, Blue reward:0.85\n", + "step: 99, Red action: do-nothing, Blue reward:0.85\n", + "step: 100, Red action: do-nothing, Blue reward:0.85\n", + "step: 101, Red action: do-nothing, Blue reward:0.85\n", + "step: 102, Red action: do-nothing, Blue reward:0.85\n", + "step: 103, Red action: do-nothing, Blue reward:0.85\n", + "step: 104, Red action: do-nothing, Blue reward:0.85\n", + "step: 105, Red action: do-nothing, Blue reward:0.85\n", + "step: 106, Red action: do-nothing, Blue reward:0.85\n", + "step: 107, Red action: do-nothing, Blue reward:0.85\n", + "step: 108, Red action: do-nothing, Blue reward:0.85\n", + "step: 109, Red action: do-nothing, Blue reward:0.85\n", + "step: 110, Red action: do-nothing, Blue reward:0.85\n", + "step: 111, Red action: do-nothing, Blue reward:0.85\n", + "step: 112, Red action: do-nothing, Blue reward:0.85\n", + "step: 113, Red action: node-application-execute, Blue reward:0.85\n", + "step: 114, Red action: do-nothing, Blue reward:0.85\n", + "step: 115, Red action: do-nothing, Blue reward:0.85\n", + "step: 116, Red action: do-nothing, Blue reward:0.85\n", + "step: 117, Red action: do-nothing, Blue reward:0.85\n", + "step: 118, Red action: do-nothing, Blue reward:0.85\n", + "step: 119, Red action: do-nothing, Blue reward:0.85\n", + "step: 120, Red action: do-nothing, Blue reward:0.85\n", + "step: 121, Red action: do-nothing, Blue reward:0.85\n", + "step: 122, Red action: do-nothing, Blue reward:0.85\n", + "step: 123, Red action: do-nothing, Blue reward:0.85\n", + "step: 124, Red action: do-nothing, Blue reward:0.85\n", + "step: 125, Red action: do-nothing, Blue reward:0.85\n", + "step: 126, Red action: do-nothing, Blue reward:0.85\n", + "step: 127, Red action: do-nothing, Blue reward:0.85\n", + "step: 128, Red action: do-nothing, Blue reward:0.85\n", + "step: 129, Red action: do-nothing, Blue reward:0.85\n", + "step: 130, Red action: do-nothing, Blue reward:0.85\n", + "step: 131, Red action: do-nothing, Blue reward:0.85\n", + "step: 132, Red action: node-application-execute, Blue reward:0.85\n", + "step: 133, Red action: do-nothing, Blue reward:0.85\n", + "step: 134, Red action: do-nothing, Blue reward:0.85\n", + "step: 135, Red action: do-nothing, Blue reward:0.85\n", + "step: 136, Red action: do-nothing, Blue reward:0.85\n", + "step: 137, Red action: do-nothing, Blue reward:0.85\n", + "step: 138, Red action: do-nothing, Blue reward:0.85\n", + "step: 139, Red action: do-nothing, Blue reward:0.85\n", + "step: 140, Red action: do-nothing, Blue reward:0.85\n", + "step: 141, Red action: do-nothing, Blue reward:0.85\n", + "step: 142, Red action: do-nothing, Blue reward:0.85\n", + "step: 143, Red action: do-nothing, Blue reward:0.85\n", + "step: 144, Red action: do-nothing, Blue reward:0.85\n", + "step: 145, Red action: do-nothing, Blue reward:0.85\n", + "step: 146, Red action: do-nothing, Blue reward:0.85\n", + "step: 147, Red action: do-nothing, Blue reward:0.85\n", + "step: 148, Red action: do-nothing, Blue reward:0.85\n", + "step: 149, Red action: do-nothing, Blue reward:0.85\n", + "step: 150, Red action: do-nothing, Blue reward:0.85\n", + "step: 151, Red action: do-nothing, Blue reward:0.85\n", + "step: 152, Red action: do-nothing, Blue reward:0.85\n", + "step: 153, Red action: do-nothing, Blue reward:0.85\n", + "step: 154, Red action: do-nothing, Blue reward:0.85\n", + "step: 155, Red action: do-nothing, Blue reward:0.85\n", + "step: 156, Red action: do-nothing, Blue reward:0.85\n", + "step: 157, Red action: node-application-execute, Blue reward:0.85\n", + "step: 158, Red action: do-nothing, Blue reward:0.85\n", + "step: 159, Red action: do-nothing, Blue reward:0.85\n", + "step: 160, Red action: do-nothing, Blue reward:0.85\n", + "step: 161, Red action: do-nothing, Blue reward:0.85\n", + "step: 162, Red action: do-nothing, Blue reward:0.85\n", + "step: 163, Red action: do-nothing, Blue reward:0.85\n", + "step: 164, Red action: do-nothing, Blue reward:0.85\n", + "step: 165, Red action: do-nothing, Blue reward:0.85\n", + "step: 166, Red action: do-nothing, Blue reward:0.85\n", + "step: 167, Red action: do-nothing, Blue reward:0.85\n", + "step: 168, Red action: do-nothing, Blue reward:0.85\n", + "step: 169, Red action: do-nothing, Blue reward:0.85\n", + "step: 170, Red action: do-nothing, Blue reward:0.85\n", + "step: 171, Red action: do-nothing, Blue reward:0.85\n", + "step: 172, Red action: do-nothing, Blue reward:0.85\n", + "step: 173, Red action: do-nothing, Blue reward:0.85\n", + "step: 174, Red action: do-nothing, Blue reward:0.85\n", + "step: 175, Red action: do-nothing, Blue reward:0.85\n", + "step: 176, Red action: do-nothing, Blue reward:0.85\n", + "step: 177, Red action: do-nothing, Blue reward:0.85\n", + "step: 178, Red action: node-application-execute, Blue reward:0.85\n", + "step: 179, Red action: do-nothing, Blue reward:0.85\n", + "step: 180, Red action: do-nothing, Blue reward:0.85\n", + "step: 181, Red action: do-nothing, Blue reward:0.85\n", + "step: 182, Red action: do-nothing, Blue reward:0.85\n", + "step: 183, Red action: do-nothing, Blue reward:0.85\n", + "step: 184, Red action: do-nothing, Blue reward:0.85\n", + "step: 185, Red action: do-nothing, Blue reward:0.85\n", + "step: 186, Red action: do-nothing, Blue reward:0.85\n", + "step: 187, Red action: do-nothing, Blue reward:0.85\n", + "step: 188, Red action: do-nothing, Blue reward:0.85\n", + "step: 189, Red action: do-nothing, Blue reward:0.85\n", + "step: 190, Red action: do-nothing, Blue reward:0.85\n", + "step: 191, Red action: do-nothing, Blue reward:0.85\n", + "step: 192, Red action: do-nothing, Blue reward:0.85\n", + "step: 193, Red action: do-nothing, Blue reward:0.85\n", + "step: 194, Red action: do-nothing, Blue reward:0.85\n", + "step: 195, Red action: do-nothing, Blue reward:0.85\n", + "step: 196, Red action: do-nothing, Blue reward:0.85\n", + "step: 197, Red action: do-nothing, Blue reward:0.85\n", + "step: 198, Red action: node-application-execute, Blue reward:0.85\n", + "step: 199, Red action: do-nothing, Blue reward:0.85\n", + "step: 200, Red action: do-nothing, Blue reward:0.85\n", + "step: 201, Red action: do-nothing, Blue reward:0.85\n", + "step: 202, Red action: do-nothing, Blue reward:0.85\n", + "step: 203, Red action: do-nothing, Blue reward:0.85\n", + "step: 204, Red action: do-nothing, Blue reward:0.85\n", + "step: 205, Red action: do-nothing, Blue reward:0.85\n", + "step: 206, Red action: do-nothing, Blue reward:0.85\n", + "step: 207, Red action: do-nothing, Blue reward:0.85\n", + "step: 208, Red action: do-nothing, Blue reward:0.85\n", + "step: 209, Red action: do-nothing, Blue reward:0.85\n", + "step: 210, Red action: do-nothing, Blue reward:0.85\n", + "step: 211, Red action: do-nothing, Blue reward:0.85\n", + "step: 212, Red action: do-nothing, Blue reward:0.85\n", + "step: 213, Red action: do-nothing, Blue reward:0.85\n", + "step: 214, Red action: do-nothing, Blue reward:0.85\n", + "step: 215, Red action: do-nothing, Blue reward:0.85\n", + "step: 216, Red action: do-nothing, Blue reward:0.85\n", + "step: 217, Red action: do-nothing, Blue reward:0.85\n", + "step: 218, Red action: node-application-execute, Blue reward:0.85\n", + "step: 219, Red action: do-nothing, Blue reward:0.85\n", + "step: 220, Red action: do-nothing, Blue reward:0.85\n", + "step: 221, Red action: do-nothing, Blue reward:0.85\n", + "step: 222, Red action: do-nothing, Blue reward:0.85\n", + "step: 223, Red action: do-nothing, Blue reward:0.85\n", + "step: 224, Red action: do-nothing, Blue reward:0.85\n", + "step: 225, Red action: do-nothing, Blue reward:0.85\n", + "step: 226, Red action: do-nothing, Blue reward:0.85\n", + "step: 227, Red action: do-nothing, Blue reward:0.85\n", + "step: 228, Red action: do-nothing, Blue reward:0.85\n", + "step: 229, Red action: do-nothing, Blue reward:0.85\n", + "step: 230, Red action: do-nothing, Blue reward:0.85\n", + "step: 231, Red action: do-nothing, Blue reward:0.85\n", + "step: 232, Red action: do-nothing, Blue reward:0.85\n", + "step: 233, Red action: do-nothing, Blue reward:0.85\n", + "step: 234, Red action: do-nothing, Blue reward:0.85\n", + "step: 235, Red action: do-nothing, Blue reward:0.85\n", + "step: 236, Red action: do-nothing, Blue reward:0.85\n", + "step: 237, Red action: do-nothing, Blue reward:0.85\n", + "step: 238, Red action: do-nothing, Blue reward:0.85\n", + "step: 239, Red action: do-nothing, Blue reward:0.85\n", + "step: 240, Red action: do-nothing, Blue reward:0.85\n", + "step: 241, Red action: node-application-execute, Blue reward:0.85\n", + "step: 242, Red action: do-nothing, Blue reward:0.85\n", + "step: 243, Red action: do-nothing, Blue reward:0.85\n", + "step: 244, Red action: do-nothing, Blue reward:0.85\n", + "step: 245, Red action: do-nothing, Blue reward:0.85\n", + "step: 246, Red action: do-nothing, Blue reward:0.85\n", + "step: 247, Red action: do-nothing, Blue reward:0.85\n", + "step: 248, Red action: do-nothing, Blue reward:0.85\n", + "step: 249, Red action: do-nothing, Blue reward:0.85\n", + "step: 250, Red action: do-nothing, Blue reward:0.85\n", + "step: 251, Red action: do-nothing, Blue reward:0.85\n", + "step: 252, Red action: do-nothing, Blue reward:0.85\n", + "step: 253, Red action: do-nothing, Blue reward:0.85\n", + "step: 254, Red action: do-nothing, Blue reward:0.85\n", + "step: 255, Red action: do-nothing, Blue reward:0.85\n", + "step: 256, Red action: do-nothing, Blue reward:0.85\n", + "step: 257, Red action: do-nothing, Blue reward:0.85\n", + "step: 258, Red action: do-nothing, Blue reward:0.85\n", + "step: 259, Red action: do-nothing, Blue reward:0.85\n", + "step: 260, Red action: do-nothing, Blue reward:0.85\n", + "step: 261, Red action: node-application-execute, Blue reward:0.85\n", + "step: 262, Red action: do-nothing, Blue reward:0.85\n", + "step: 263, Red action: do-nothing, Blue reward:0.85\n", + "step: 264, Red action: do-nothing, Blue reward:0.85\n", + "step: 265, Red action: do-nothing, Blue reward:0.85\n", + "step: 266, Red action: do-nothing, Blue reward:0.85\n", + "step: 267, Red action: do-nothing, Blue reward:0.85\n", + "step: 268, Red action: do-nothing, Blue reward:0.85\n", + "step: 269, Red action: do-nothing, Blue reward:0.85\n", + "step: 270, Red action: do-nothing, Blue reward:0.85\n", + "step: 271, Red action: do-nothing, Blue reward:0.85\n", + "step: 272, Red action: do-nothing, Blue reward:0.85\n", + "step: 273, Red action: do-nothing, Blue reward:0.85\n", + "step: 274, Red action: do-nothing, Blue reward:0.85\n", + "step: 275, Red action: do-nothing, Blue reward:0.85\n", + "step: 276, Red action: do-nothing, Blue reward:0.85\n", + "step: 277, Red action: do-nothing, Blue reward:0.85\n", + "step: 278, Red action: do-nothing, Blue reward:0.85\n", + "step: 279, Red action: do-nothing, Blue reward:0.85\n", + "step: 280, Red action: do-nothing, Blue reward:0.85\n", + "step: 281, Red action: node-application-execute, Blue reward:0.85\n", + "step: 282, Red action: do-nothing, Blue reward:0.85\n", + "step: 283, Red action: do-nothing, Blue reward:0.85\n", + "step: 284, Red action: do-nothing, Blue reward:0.85\n", + "step: 285, Red action: do-nothing, Blue reward:0.85\n", + "step: 286, Red action: do-nothing, Blue reward:0.85\n", + "step: 287, Red action: do-nothing, Blue reward:0.85\n", + "step: 288, Red action: do-nothing, Blue reward:0.85\n", + "step: 289, Red action: do-nothing, Blue reward:0.85\n", + "step: 290, Red action: do-nothing, Blue reward:0.85\n", + "step: 291, Red action: do-nothing, Blue reward:0.85\n", + "step: 292, Red action: do-nothing, Blue reward:0.85\n", + "step: 293, Red action: do-nothing, Blue reward:0.85\n", + "step: 294, Red action: do-nothing, Blue reward:0.85\n", + "step: 295, Red action: do-nothing, Blue reward:0.85\n", + "step: 296, Red action: do-nothing, Blue reward:0.85\n", + "step: 297, Red action: do-nothing, Blue reward:0.85\n", + "step: 298, Red action: do-nothing, Blue reward:0.85\n", + "step: 299, Red action: do-nothing, Blue reward:0.85\n", + "step: 300, Red action: do-nothing, Blue reward:0.85\n", + "step: 301, Red action: do-nothing, Blue reward:0.85\n", + "step: 302, Red action: do-nothing, Blue reward:0.85\n", + "step: 303, Red action: do-nothing, Blue reward:0.85\n", + "step: 304, Red action: node-application-execute, Blue reward:0.85\n", + "step: 305, Red action: do-nothing, Blue reward:0.85\n", + "step: 306, Red action: do-nothing, Blue reward:0.85\n", + "step: 307, Red action: do-nothing, Blue reward:0.85\n", + "step: 308, Red action: do-nothing, Blue reward:0.85\n", + "step: 309, Red action: do-nothing, Blue reward:0.85\n", + "step: 310, Red action: do-nothing, Blue reward:0.85\n", + "step: 311, Red action: do-nothing, Blue reward:0.85\n", + "step: 312, Red action: do-nothing, Blue reward:0.85\n", + "step: 313, Red action: do-nothing, Blue reward:0.85\n", + "step: 314, Red action: do-nothing, Blue reward:0.85\n", + "step: 315, Red action: do-nothing, Blue reward:0.85\n", + "step: 316, Red action: do-nothing, Blue reward:0.85\n", + "step: 317, Red action: do-nothing, Blue reward:0.85\n", + "step: 318, Red action: do-nothing, Blue reward:0.85\n", + "step: 319, Red action: do-nothing, Blue reward:0.85\n", + "step: 320, Red action: do-nothing, Blue reward:0.85\n", + "step: 321, Red action: do-nothing, Blue reward:0.85\n", + "step: 322, Red action: do-nothing, Blue reward:0.85\n", + "step: 323, Red action: do-nothing, Blue reward:0.85\n", + "step: 324, Red action: do-nothing, Blue reward:0.85\n", + "step: 325, Red action: node-application-execute, Blue reward:0.85\n", + "step: 326, Red action: do-nothing, Blue reward:0.85\n", + "step: 327, Red action: do-nothing, Blue reward:0.85\n", + "step: 328, Red action: do-nothing, Blue reward:0.85\n", + "step: 329, Red action: do-nothing, Blue reward:0.85\n", + "step: 330, Red action: do-nothing, Blue reward:0.85\n", + "step: 331, Red action: do-nothing, Blue reward:0.85\n", + "step: 332, Red action: do-nothing, Blue reward:0.85\n", + "step: 333, Red action: do-nothing, Blue reward:0.85\n", + "step: 334, Red action: do-nothing, Blue reward:0.85\n", + "step: 335, Red action: do-nothing, Blue reward:0.85\n", + "step: 336, Red action: do-nothing, Blue reward:0.85\n", + "step: 337, Red action: do-nothing, Blue reward:0.85\n", + "step: 338, Red action: do-nothing, Blue reward:0.85\n", + "step: 339, Red action: do-nothing, Blue reward:0.85\n", + "step: 340, Red action: do-nothing, Blue reward:0.85\n", + "step: 341, Red action: do-nothing, Blue reward:0.85\n", + "step: 342, Red action: do-nothing, Blue reward:0.85\n", + "step: 343, Red action: do-nothing, Blue reward:0.85\n", + "step: 344, Red action: do-nothing, Blue reward:0.85\n", + "step: 345, Red action: do-nothing, Blue reward:0.85\n", + "step: 346, Red action: do-nothing, Blue reward:0.85\n", + "step: 347, Red action: do-nothing, Blue reward:0.85\n", + "step: 348, Red action: do-nothing, Blue reward:0.85\n", + "step: 349, Red action: node-application-execute, Blue reward:0.85\n", + "step: 350, Red action: do-nothing, Blue reward:0.85\n", + "step: 351, Red action: do-nothing, Blue reward:0.85\n", + "step: 352, Red action: do-nothing, Blue reward:0.85\n", + "step: 353, Red action: do-nothing, Blue reward:0.85\n", + "step: 354, Red action: do-nothing, Blue reward:0.85\n", + "step: 355, Red action: do-nothing, Blue reward:0.85\n", + "step: 356, Red action: do-nothing, Blue reward:0.85\n", + "step: 357, Red action: do-nothing, Blue reward:0.85\n", + "step: 358, Red action: do-nothing, Blue reward:0.85\n", + "step: 359, Red action: do-nothing, Blue reward:0.85\n", + "step: 360, Red action: do-nothing, Blue reward:0.85\n", + "step: 361, Red action: do-nothing, Blue reward:0.85\n", + "step: 362, Red action: do-nothing, Blue reward:0.85\n", + "step: 363, Red action: do-nothing, Blue reward:0.85\n", + "step: 364, Red action: do-nothing, Blue reward:0.85\n", + "step: 365, Red action: do-nothing, Blue reward:0.85\n", + "step: 366, Red action: do-nothing, Blue reward:0.85\n", + "step: 367, Red action: do-nothing, Blue reward:0.85\n", + "step: 368, Red action: do-nothing, Blue reward:0.85\n", + "step: 369, Red action: node-application-execute, Blue reward:0.85\n", + "step: 370, Red action: do-nothing, Blue reward:0.85\n", + "step: 371, Red action: do-nothing, Blue reward:0.85\n", + "step: 372, Red action: do-nothing, Blue reward:0.85\n", + "step: 373, Red action: do-nothing, Blue reward:0.85\n", + "step: 374, Red action: do-nothing, Blue reward:0.85\n", + "step: 375, Red action: do-nothing, Blue reward:0.85\n", + "step: 376, Red action: do-nothing, Blue reward:0.85\n", + "step: 377, Red action: do-nothing, Blue reward:0.85\n", + "step: 378, Red action: do-nothing, Blue reward:0.85\n", + "step: 379, Red action: do-nothing, Blue reward:0.85\n", + "step: 380, Red action: do-nothing, Blue reward:0.85\n", + "step: 381, Red action: do-nothing, Blue reward:0.85\n", + "step: 382, Red action: do-nothing, Blue reward:0.85\n", + "step: 383, Red action: do-nothing, Blue reward:0.85\n", + "step: 384, Red action: do-nothing, Blue reward:0.85\n", + "step: 385, Red action: do-nothing, Blue reward:0.85\n", + "step: 386, Red action: do-nothing, Blue reward:0.85\n", + "step: 387, Red action: do-nothing, Blue reward:0.85\n", + "step: 388, Red action: do-nothing, Blue reward:0.85\n", + "step: 389, Red action: do-nothing, Blue reward:0.85\n", + "step: 390, Red action: do-nothing, Blue reward:0.85\n", + "step: 391, Red action: do-nothing, Blue reward:0.85\n", + "step: 392, Red action: do-nothing, Blue reward:0.85\n", + "step: 393, Red action: do-nothing, Blue reward:0.85\n", + "step: 394, Red action: node-application-execute, Blue reward:0.85\n", + "step: 395, Red action: do-nothing, Blue reward:0.85\n", + "step: 396, Red action: do-nothing, Blue reward:0.85\n", + "step: 397, Red action: do-nothing, Blue reward:0.85\n", + "step: 398, Red action: do-nothing, Blue reward:0.85\n", + "step: 399, Red action: do-nothing, Blue reward:0.85\n", + "step: 400, Red action: do-nothing, Blue reward:0.85\n", + "step: 401, Red action: do-nothing, Blue reward:0.85\n", + "step: 402, Red action: do-nothing, Blue reward:0.85\n", + "step: 403, Red action: do-nothing, Blue reward:0.85\n", + "step: 404, Red action: do-nothing, Blue reward:0.85\n", + "step: 405, Red action: do-nothing, Blue reward:0.85\n", + "step: 406, Red action: do-nothing, Blue reward:0.85\n", + "step: 407, Red action: do-nothing, Blue reward:0.85\n", + "step: 408, Red action: do-nothing, Blue reward:0.85\n", + "step: 409, Red action: do-nothing, Blue reward:0.85\n", + "step: 410, Red action: do-nothing, Blue reward:0.85\n", + "step: 411, Red action: do-nothing, Blue reward:0.85\n", + "step: 412, Red action: do-nothing, Blue reward:0.85\n", + "step: 413, Red action: do-nothing, Blue reward:0.85\n", + "step: 414, Red action: do-nothing, Blue reward:0.85\n", + "step: 415, Red action: do-nothing, Blue reward:0.85\n", + "step: 416, Red action: do-nothing, Blue reward:0.85\n", + "step: 417, Red action: do-nothing, Blue reward:0.85\n", + "step: 418, Red action: node-application-execute, Blue reward:0.85\n", + "step: 419, Red action: do-nothing, Blue reward:0.85\n", + "step: 420, Red action: do-nothing, Blue reward:0.85\n", + "step: 421, Red action: do-nothing, Blue reward:0.85\n", + "step: 422, Red action: do-nothing, Blue reward:0.85\n", + "step: 423, Red action: do-nothing, Blue reward:0.85\n", + "step: 424, Red action: do-nothing, Blue reward:0.85\n", + "step: 425, Red action: do-nothing, Blue reward:0.85\n", + "step: 426, Red action: do-nothing, Blue reward:0.85\n", + "step: 427, Red action: do-nothing, Blue reward:0.85\n", + "step: 428, Red action: do-nothing, Blue reward:0.85\n", + "step: 429, Red action: do-nothing, Blue reward:0.85\n", + "step: 430, Red action: do-nothing, Blue reward:0.85\n", + "step: 431, Red action: do-nothing, Blue reward:0.85\n", + "step: 432, Red action: do-nothing, Blue reward:0.85\n", + "step: 433, Red action: do-nothing, Blue reward:0.85\n", + "step: 434, Red action: do-nothing, Blue reward:0.85\n", + "step: 435, Red action: do-nothing, Blue reward:0.85\n", + "step: 436, Red action: do-nothing, Blue reward:0.85\n", + "step: 437, Red action: do-nothing, Blue reward:0.85\n", + "step: 438, Red action: do-nothing, Blue reward:0.85\n", + "step: 439, Red action: do-nothing, Blue reward:0.85\n", + "step: 440, Red action: do-nothing, Blue reward:0.85\n", + "step: 441, Red action: do-nothing, Blue reward:0.85\n", + "step: 442, Red action: do-nothing, Blue reward:0.85\n", + "step: 443, Red action: node-application-execute, Blue reward:0.85\n", + "step: 444, Red action: do-nothing, Blue reward:0.85\n", + "step: 445, Red action: do-nothing, Blue reward:0.85\n", + "step: 446, Red action: do-nothing, Blue reward:0.85\n", + "step: 447, Red action: do-nothing, Blue reward:0.85\n", + "step: 448, Red action: do-nothing, Blue reward:0.85\n", + "step: 449, Red action: do-nothing, Blue reward:0.85\n", + "step: 450, Red action: do-nothing, Blue reward:0.85\n", + "step: 451, Red action: do-nothing, Blue reward:0.85\n", + "step: 452, Red action: do-nothing, Blue reward:0.85\n", + "step: 453, Red action: do-nothing, Blue reward:0.85\n", + "step: 454, Red action: do-nothing, Blue reward:0.85\n", + "step: 455, Red action: do-nothing, Blue reward:0.85\n", + "step: 456, Red action: do-nothing, Blue reward:0.85\n", + "step: 457, Red action: do-nothing, Blue reward:0.85\n", + "step: 458, Red action: do-nothing, Blue reward:0.85\n", + "step: 459, Red action: do-nothing, Blue reward:0.85\n", + "step: 460, Red action: do-nothing, Blue reward:0.85\n", + "step: 461, Red action: node-application-execute, Blue reward:0.85\n", + "step: 462, Red action: do-nothing, Blue reward:0.85\n", + "step: 463, Red action: do-nothing, Blue reward:0.85\n", + "step: 464, Red action: do-nothing, Blue reward:0.85\n", + "step: 465, Red action: do-nothing, Blue reward:0.85\n", + "step: 466, Red action: do-nothing, Blue reward:0.85\n", + "step: 467, Red action: do-nothing, Blue reward:0.85\n", + "step: 468, Red action: do-nothing, Blue reward:0.85\n", + "step: 469, Red action: do-nothing, Blue reward:0.85\n", + "step: 470, Red action: do-nothing, Blue reward:0.85\n", + "step: 471, Red action: do-nothing, Blue reward:0.85\n", + "step: 472, Red action: do-nothing, Blue reward:0.85\n", + "step: 473, Red action: do-nothing, Blue reward:0.85\n", + "step: 474, Red action: do-nothing, Blue reward:0.85\n", + "step: 475, Red action: do-nothing, Blue reward:0.85\n", + "step: 476, Red action: do-nothing, Blue reward:0.85\n", + "step: 477, Red action: do-nothing, Blue reward:0.85\n", + "step: 478, Red action: do-nothing, Blue reward:0.85\n", + "step: 479, Red action: do-nothing, Blue reward:0.85\n", + "step: 480, Red action: node-application-execute, Blue reward:0.85\n", + "step: 481, Red action: do-nothing, Blue reward:0.85\n", + "step: 482, Red action: do-nothing, Blue reward:0.85\n", + "step: 483, Red action: do-nothing, Blue reward:0.85\n", + "step: 484, Red action: do-nothing, Blue reward:0.85\n", + "step: 485, Red action: do-nothing, Blue reward:0.85\n", + "step: 486, Red action: do-nothing, Blue reward:0.85\n", + "step: 487, Red action: do-nothing, Blue reward:0.85\n", + "step: 488, Red action: do-nothing, Blue reward:0.85\n", + "step: 489, Red action: do-nothing, Blue reward:0.85\n", + "step: 490, Red action: do-nothing, Blue reward:0.85\n", + "step: 491, Red action: do-nothing, Blue reward:0.85\n", + "step: 492, Red action: do-nothing, Blue reward:0.85\n", + "step: 493, Red action: do-nothing, Blue reward:0.85\n", + "step: 494, Red action: do-nothing, Blue reward:0.85\n", + "step: 495, Red action: do-nothing, Blue reward:0.85\n", + "step: 496, Red action: do-nothing, Blue reward:0.85\n", + "step: 497, Red action: do-nothing, Blue reward:0.85\n", + "step: 498, Red action: do-nothing, Blue reward:0.85\n", + "step: 499, Red action: do-nothing, Blue reward:0.85\n", + "step: 500, Red action: do-nothing, Blue reward:0.85\n", + "step: 501, Red action: do-nothing, Blue reward:0.85\n", + "step: 502, Red action: do-nothing, Blue reward:0.85\n", + "step: 503, Red action: do-nothing, Blue reward:0.85\n", + "step: 504, Red action: node-application-execute, Blue reward:0.85\n", + "step: 505, Red action: do-nothing, Blue reward:0.85\n", + "step: 506, Red action: do-nothing, Blue reward:0.85\n", + "step: 507, Red action: do-nothing, Blue reward:0.85\n", + "step: 508, Red action: do-nothing, Blue reward:0.85\n", + "step: 509, Red action: do-nothing, Blue reward:0.85\n", + "step: 510, Red action: do-nothing, Blue reward:0.85\n", + "step: 511, Red action: do-nothing, Blue reward:0.85\n", + "step: 512, Red action: do-nothing, Blue reward:0.85\n", + "step: 513, Red action: do-nothing, Blue reward:0.85\n", + "step: 514, Red action: do-nothing, Blue reward:0.85\n", + "step: 515, Red action: do-nothing, Blue reward:0.85\n", + "step: 516, Red action: do-nothing, Blue reward:0.85\n", + "step: 517, Red action: do-nothing, Blue reward:0.85\n", + "step: 518, Red action: do-nothing, Blue reward:0.85\n", + "step: 519, Red action: do-nothing, Blue reward:0.85\n", + "step: 520, Red action: do-nothing, Blue reward:0.85\n", + "step: 521, Red action: do-nothing, Blue reward:0.85\n", + "step: 522, Red action: do-nothing, Blue reward:0.85\n", + "step: 523, Red action: do-nothing, Blue reward:0.85\n", + "step: 524, Red action: do-nothing, Blue reward:0.85\n", + "step: 525, Red action: do-nothing, Blue reward:0.85\n", + "step: 526, Red action: do-nothing, Blue reward:0.85\n", + "step: 527, Red action: node-application-execute, Blue reward:0.85\n", + "step: 528, Red action: do-nothing, Blue reward:0.85\n", + "step: 529, Red action: do-nothing, Blue reward:0.85\n", + "step: 530, Red action: do-nothing, Blue reward:0.85\n", + "step: 531, Red action: do-nothing, Blue reward:0.85\n", + "step: 532, Red action: do-nothing, Blue reward:0.85\n", + "step: 533, Red action: do-nothing, Blue reward:0.85\n", + "step: 534, Red action: do-nothing, Blue reward:0.85\n", + "step: 535, Red action: do-nothing, Blue reward:0.85\n", + "step: 536, Red action: do-nothing, Blue reward:0.85\n", + "step: 537, Red action: do-nothing, Blue reward:0.85\n", + "step: 538, Red action: do-nothing, Blue reward:0.85\n", + "step: 539, Red action: do-nothing, Blue reward:0.85\n", + "step: 540, Red action: do-nothing, Blue reward:0.85\n", + "step: 541, Red action: do-nothing, Blue reward:0.85\n", + "step: 542, Red action: do-nothing, Blue reward:0.85\n", + "step: 543, Red action: do-nothing, Blue reward:0.85\n", + "step: 544, Red action: do-nothing, Blue reward:0.85\n", + "step: 545, Red action: do-nothing, Blue reward:0.85\n", + "step: 546, Red action: do-nothing, Blue reward:0.85\n", + "step: 547, Red action: do-nothing, Blue reward:0.85\n", + "step: 548, Red action: do-nothing, Blue reward:0.85\n", + "step: 549, Red action: node-application-execute, Blue reward:0.85\n", + "step: 550, Red action: do-nothing, Blue reward:0.85\n", + "step: 551, Red action: do-nothing, Blue reward:0.85\n", + "step: 552, Red action: do-nothing, Blue reward:0.85\n", + "step: 553, Red action: do-nothing, Blue reward:0.85\n", + "step: 554, Red action: do-nothing, Blue reward:0.85\n", + "step: 555, Red action: do-nothing, Blue reward:0.85\n", + "step: 556, Red action: do-nothing, Blue reward:0.85\n", + "step: 557, Red action: do-nothing, Blue reward:0.85\n", + "step: 558, Red action: do-nothing, Blue reward:0.85\n", + "step: 559, Red action: do-nothing, Blue reward:0.85\n", + "step: 560, Red action: do-nothing, Blue reward:0.85\n", + "step: 561, Red action: do-nothing, Blue reward:0.85\n", + "step: 562, Red action: do-nothing, Blue reward:0.85\n", + "step: 563, Red action: do-nothing, Blue reward:0.85\n", + "step: 564, Red action: do-nothing, Blue reward:0.85\n", + "step: 565, Red action: node-application-execute, Blue reward:0.85\n", + "step: 566, Red action: do-nothing, Blue reward:0.85\n", + "step: 567, Red action: do-nothing, Blue reward:0.85\n", + "step: 568, Red action: do-nothing, Blue reward:0.85\n", + "step: 569, Red action: do-nothing, Blue reward:0.85\n", + "step: 570, Red action: do-nothing, Blue reward:0.85\n", + "step: 571, Red action: do-nothing, Blue reward:0.85\n", + "step: 572, Red action: do-nothing, Blue reward:0.85\n", + "step: 573, Red action: do-nothing, Blue reward:0.85\n", + "step: 574, Red action: do-nothing, Blue reward:0.85\n", + "step: 575, Red action: do-nothing, Blue reward:0.85\n", + "step: 576, Red action: do-nothing, Blue reward:0.85\n", + "step: 577, Red action: do-nothing, Blue reward:0.85\n", + "step: 578, Red action: do-nothing, Blue reward:0.85\n", + "step: 579, Red action: do-nothing, Blue reward:0.85\n", + "step: 580, Red action: do-nothing, Blue reward:0.85\n", + "step: 581, Red action: do-nothing, Blue reward:0.85\n", + "step: 582, Red action: do-nothing, Blue reward:0.85\n", + "step: 583, Red action: do-nothing, Blue reward:0.85\n", + "step: 584, Red action: do-nothing, Blue reward:0.85\n", + "step: 585, Red action: do-nothing, Blue reward:0.85\n", + "step: 586, Red action: do-nothing, Blue reward:0.85\n", + "step: 587, Red action: do-nothing, Blue reward:0.85\n", + "step: 588, Red action: do-nothing, Blue reward:0.85\n", + "step: 589, Red action: node-application-execute, Blue reward:0.85\n", + "step: 590, Red action: do-nothing, Blue reward:0.85\n", + "step: 591, Red action: do-nothing, Blue reward:0.85\n", + "step: 592, Red action: do-nothing, Blue reward:0.85\n", + "step: 593, Red action: do-nothing, Blue reward:0.85\n", + "step: 594, Red action: do-nothing, Blue reward:0.85\n", + "step: 595, Red action: do-nothing, Blue reward:0.85\n", + "step: 596, Red action: do-nothing, Blue reward:0.85\n", + "step: 597, Red action: do-nothing, Blue reward:0.85\n", + "step: 598, Red action: do-nothing, Blue reward:0.85\n", + "step: 599, Red action: do-nothing, Blue reward:0.85\n", + "step: 600, Red action: do-nothing, Blue reward:0.85\n", + "step: 601, Red action: do-nothing, Blue reward:0.85\n", + "step: 602, Red action: do-nothing, Blue reward:0.85\n", + "step: 603, Red action: do-nothing, Blue reward:0.85\n", + "step: 604, Red action: do-nothing, Blue reward:0.85\n", + "step: 605, Red action: do-nothing, Blue reward:0.85\n", + "step: 606, Red action: do-nothing, Blue reward:0.85\n", + "step: 607, Red action: do-nothing, Blue reward:0.85\n", + "step: 608, Red action: do-nothing, Blue reward:0.85\n", + "step: 609, Red action: do-nothing, Blue reward:0.85\n", + "step: 610, Red action: node-application-execute, Blue reward:0.85\n", + "step: 611, Red action: do-nothing, Blue reward:0.85\n", + "step: 612, Red action: do-nothing, Blue reward:0.85\n", + "step: 613, Red action: do-nothing, Blue reward:0.85\n", + "step: 614, Red action: do-nothing, Blue reward:0.85\n", + "step: 615, Red action: do-nothing, Blue reward:0.85\n", + "step: 616, Red action: do-nothing, Blue reward:0.85\n", + "step: 617, Red action: do-nothing, Blue reward:0.85\n", + "step: 618, Red action: do-nothing, Blue reward:0.85\n", + "step: 619, Red action: do-nothing, Blue reward:0.85\n", + "step: 620, Red action: do-nothing, Blue reward:0.85\n", + "step: 621, Red action: do-nothing, Blue reward:0.85\n", + "step: 622, Red action: do-nothing, Blue reward:0.85\n", + "step: 623, Red action: do-nothing, Blue reward:0.85\n", + "step: 624, Red action: do-nothing, Blue reward:0.85\n", + "step: 625, Red action: do-nothing, Blue reward:0.85\n", + "step: 626, Red action: node-application-execute, Blue reward:0.85\n", + "step: 627, Red action: do-nothing, Blue reward:0.85\n", + "step: 628, Red action: do-nothing, Blue reward:0.85\n", + "step: 629, Red action: do-nothing, Blue reward:0.85\n", + "step: 630, Red action: do-nothing, Blue reward:0.85\n", + "step: 631, Red action: do-nothing, Blue reward:0.85\n", + "step: 632, Red action: do-nothing, Blue reward:0.85\n", + "step: 633, Red action: do-nothing, Blue reward:0.85\n", + "step: 634, Red action: do-nothing, Blue reward:0.85\n", + "step: 635, Red action: do-nothing, Blue reward:0.85\n", + "step: 636, Red action: do-nothing, Blue reward:0.85\n", + "step: 637, Red action: do-nothing, Blue reward:0.85\n", + "step: 638, Red action: do-nothing, Blue reward:0.85\n", + "step: 639, Red action: do-nothing, Blue reward:0.85\n", + "step: 640, Red action: do-nothing, Blue reward:0.85\n", + "step: 641, Red action: do-nothing, Blue reward:0.85\n", + "step: 642, Red action: do-nothing, Blue reward:0.85\n", + "step: 643, Red action: do-nothing, Blue reward:0.85\n", + "step: 644, Red action: node-application-execute, Blue reward:0.85\n", + "step: 645, Red action: do-nothing, Blue reward:0.85\n", + "step: 646, Red action: do-nothing, Blue reward:0.85\n", + "step: 647, Red action: do-nothing, Blue reward:0.85\n", + "step: 648, Red action: do-nothing, Blue reward:0.85\n", + "step: 649, Red action: do-nothing, Blue reward:0.85\n", + "step: 650, Red action: do-nothing, Blue reward:0.85\n", + "step: 651, Red action: do-nothing, Blue reward:0.85\n", + "step: 652, Red action: do-nothing, Blue reward:0.85\n", + "step: 653, Red action: do-nothing, Blue reward:0.85\n", + "step: 654, Red action: do-nothing, Blue reward:0.85\n", + "step: 655, Red action: do-nothing, Blue reward:0.85\n", + "step: 656, Red action: do-nothing, Blue reward:0.85\n", + "step: 657, Red action: do-nothing, Blue reward:0.85\n", + "step: 658, Red action: do-nothing, Blue reward:0.85\n", + "step: 659, Red action: do-nothing, Blue reward:0.85\n", + "step: 660, Red action: do-nothing, Blue reward:0.85\n", + "step: 661, Red action: do-nothing, Blue reward:0.85\n", + "step: 662, Red action: do-nothing, Blue reward:0.85\n", + "step: 663, Red action: do-nothing, Blue reward:0.85\n", + "step: 664, Red action: do-nothing, Blue reward:0.85\n", + "step: 665, Red action: do-nothing, Blue reward:0.85\n", + "step: 666, Red action: do-nothing, Blue reward:0.85\n", + "step: 667, Red action: node-application-execute, Blue reward:0.85\n", + "step: 668, Red action: do-nothing, Blue reward:0.85\n", + "step: 669, Red action: do-nothing, Blue reward:0.85\n", + "step: 670, Red action: do-nothing, Blue reward:0.85\n", + "step: 671, Red action: do-nothing, Blue reward:0.85\n", + "step: 672, Red action: do-nothing, Blue reward:0.85\n", + "step: 673, Red action: do-nothing, Blue reward:0.85\n", + "step: 674, Red action: do-nothing, Blue reward:0.85\n", + "step: 675, Red action: do-nothing, Blue reward:0.85\n", + "step: 676, Red action: do-nothing, Blue reward:0.85\n", + "step: 677, Red action: do-nothing, Blue reward:0.85\n", + "step: 678, Red action: do-nothing, Blue reward:0.85\n", + "step: 679, Red action: do-nothing, Blue reward:0.85\n", + "step: 680, Red action: do-nothing, Blue reward:0.85\n", + "step: 681, Red action: do-nothing, Blue reward:0.85\n", + "step: 682, Red action: do-nothing, Blue reward:0.85\n", + "step: 683, Red action: do-nothing, Blue reward:0.85\n", + "step: 684, Red action: do-nothing, Blue reward:0.85\n", + "step: 685, Red action: node-application-execute, Blue reward:0.85\n", + "step: 686, Red action: do-nothing, Blue reward:0.85\n", + "step: 687, Red action: do-nothing, Blue reward:0.85\n", + "step: 688, Red action: do-nothing, Blue reward:0.85\n", + "step: 689, Red action: do-nothing, Blue reward:0.85\n", + "step: 690, Red action: do-nothing, Blue reward:0.85\n", + "step: 691, Red action: do-nothing, Blue reward:0.85\n", + "step: 692, Red action: do-nothing, Blue reward:0.85\n", + "step: 693, Red action: do-nothing, Blue reward:0.85\n", + "step: 694, Red action: do-nothing, Blue reward:0.85\n", + "step: 695, Red action: do-nothing, Blue reward:0.85\n", + "step: 696, Red action: do-nothing, Blue reward:0.85\n", + "step: 697, Red action: do-nothing, Blue reward:0.85\n", + "step: 698, Red action: do-nothing, Blue reward:0.85\n", + "step: 699, Red action: do-nothing, Blue reward:0.85\n", + "step: 700, Red action: do-nothing, Blue reward:0.85\n", + "step: 701, Red action: node-application-execute, Blue reward:0.85\n", + "step: 702, Red action: do-nothing, Blue reward:0.85\n", + "step: 703, Red action: do-nothing, Blue reward:0.85\n", + "step: 704, Red action: do-nothing, Blue reward:0.85\n", + "step: 705, Red action: do-nothing, Blue reward:0.85\n", + "step: 706, Red action: do-nothing, Blue reward:0.85\n", + "step: 707, Red action: do-nothing, Blue reward:0.85\n", + "step: 708, Red action: do-nothing, Blue reward:0.85\n", + "step: 709, Red action: do-nothing, Blue reward:0.85\n", + "step: 710, Red action: do-nothing, Blue reward:0.85\n", + "step: 711, Red action: do-nothing, Blue reward:0.85\n", + "step: 712, Red action: do-nothing, Blue reward:0.85\n", + "step: 713, Red action: do-nothing, Blue reward:0.85\n", + "step: 714, Red action: do-nothing, Blue reward:0.85\n", + "step: 715, Red action: do-nothing, Blue reward:0.85\n", + "step: 716, Red action: do-nothing, Blue reward:0.85\n", + "step: 717, Red action: do-nothing, Blue reward:0.85\n", + "step: 718, Red action: node-application-execute, Blue reward:0.85\n", + "step: 719, Red action: do-nothing, Blue reward:0.85\n", + "step: 720, Red action: do-nothing, Blue reward:0.85\n", + "step: 721, Red action: do-nothing, Blue reward:0.85\n", + "step: 722, Red action: do-nothing, Blue reward:0.85\n", + "step: 723, Red action: do-nothing, Blue reward:0.85\n", + "step: 724, Red action: do-nothing, Blue reward:0.85\n", + "step: 725, Red action: do-nothing, Blue reward:0.85\n", + "step: 726, Red action: do-nothing, Blue reward:0.85\n", + "step: 727, Red action: do-nothing, Blue reward:0.85\n", + "step: 728, Red action: do-nothing, Blue reward:0.85\n", + "step: 729, Red action: do-nothing, Blue reward:0.85\n", + "step: 730, Red action: do-nothing, Blue reward:0.85\n", + "step: 731, Red action: do-nothing, Blue reward:0.85\n", + "step: 732, Red action: do-nothing, Blue reward:0.85\n", + "step: 733, Red action: do-nothing, Blue reward:0.85\n", + "step: 734, Red action: do-nothing, Blue reward:0.85\n", + "step: 735, Red action: do-nothing, Blue reward:0.85\n", + "step: 736, Red action: do-nothing, Blue reward:0.85\n", + "step: 737, Red action: do-nothing, Blue reward:0.85\n", + "step: 738, Red action: do-nothing, Blue reward:0.85\n", + "step: 739, Red action: do-nothing, Blue reward:0.85\n", + "step: 740, Red action: do-nothing, Blue reward:0.85\n", + "step: 741, Red action: do-nothing, Blue reward:0.85\n", + "step: 742, Red action: do-nothing, Blue reward:0.85\n", + "step: 743, Red action: node-application-execute, Blue reward:0.85\n", + "step: 744, Red action: do-nothing, Blue reward:0.85\n", + "step: 745, Red action: do-nothing, Blue reward:0.85\n", + "step: 746, Red action: do-nothing, Blue reward:0.85\n", + "step: 747, Red action: do-nothing, Blue reward:0.85\n", + "step: 748, Red action: do-nothing, Blue reward:0.85\n", + "step: 749, Red action: do-nothing, Blue reward:0.85\n", + "step: 750, Red action: do-nothing, Blue reward:0.85\n", + "step: 751, Red action: do-nothing, Blue reward:0.85\n", + "step: 752, Red action: do-nothing, Blue reward:0.85\n", + "step: 753, Red action: do-nothing, Blue reward:0.85\n", + "step: 754, Red action: do-nothing, Blue reward:0.85\n", + "step: 755, Red action: do-nothing, Blue reward:0.85\n", + "step: 756, Red action: do-nothing, Blue reward:0.85\n", + "step: 757, Red action: do-nothing, Blue reward:0.85\n", + "step: 758, Red action: do-nothing, Blue reward:0.85\n", + "step: 759, Red action: do-nothing, Blue reward:0.85\n", + "step: 760, Red action: node-application-execute, Blue reward:0.85\n", + "step: 761, Red action: do-nothing, Blue reward:0.85\n", + "step: 762, Red action: do-nothing, Blue reward:0.85\n", + "step: 763, Red action: do-nothing, Blue reward:0.85\n", + "step: 764, Red action: do-nothing, Blue reward:0.85\n", + "step: 765, Red action: do-nothing, Blue reward:0.85\n", + "step: 766, Red action: do-nothing, Blue reward:0.85\n", + "step: 767, Red action: do-nothing, Blue reward:0.85\n", + "step: 768, Red action: do-nothing, Blue reward:0.85\n", + "step: 769, Red action: do-nothing, Blue reward:0.85\n", + "step: 770, Red action: do-nothing, Blue reward:0.85\n", + "step: 771, Red action: do-nothing, Blue reward:0.85\n", + "step: 772, Red action: do-nothing, Blue reward:0.85\n", + "step: 773, Red action: do-nothing, Blue reward:0.85\n", + "step: 774, Red action: do-nothing, Blue reward:0.85\n", + "step: 775, Red action: do-nothing, Blue reward:0.85\n", + "step: 776, Red action: do-nothing, Blue reward:0.85\n", + "step: 777, Red action: do-nothing, Blue reward:0.85\n", + "step: 778, Red action: do-nothing, Blue reward:0.85\n", + "step: 779, Red action: do-nothing, Blue reward:0.85\n", + "step: 780, Red action: do-nothing, Blue reward:0.85\n", + "step: 781, Red action: do-nothing, Blue reward:0.85\n", + "step: 782, Red action: do-nothing, Blue reward:0.85\n", + "step: 783, Red action: node-application-execute, Blue reward:0.85\n", + "step: 784, Red action: do-nothing, Blue reward:0.85\n", + "step: 785, Red action: do-nothing, Blue reward:0.85\n", + "step: 786, Red action: do-nothing, Blue reward:0.85\n", + "step: 787, Red action: do-nothing, Blue reward:0.85\n", + "step: 788, Red action: do-nothing, Blue reward:0.85\n", + "step: 789, Red action: do-nothing, Blue reward:0.85\n", + "step: 790, Red action: do-nothing, Blue reward:0.85\n", + "step: 791, Red action: do-nothing, Blue reward:0.85\n", + "step: 792, Red action: do-nothing, Blue reward:0.85\n", + "step: 793, Red action: do-nothing, Blue reward:0.85\n", + "step: 794, Red action: do-nothing, Blue reward:0.85\n", + "step: 795, Red action: do-nothing, Blue reward:0.85\n", + "step: 796, Red action: do-nothing, Blue reward:0.85\n", + "step: 797, Red action: do-nothing, Blue reward:0.85\n", + "step: 798, Red action: do-nothing, Blue reward:0.85\n", + "step: 799, Red action: do-nothing, Blue reward:0.85\n", + "step: 800, Red action: do-nothing, Blue reward:0.85\n", + "step: 801, Red action: do-nothing, Blue reward:0.85\n", + "step: 802, Red action: do-nothing, Blue reward:0.85\n", + "step: 803, Red action: node-application-execute, Blue reward:0.85\n", + "step: 804, Red action: do-nothing, Blue reward:0.85\n", + "step: 805, Red action: do-nothing, Blue reward:0.85\n", + "step: 806, Red action: do-nothing, Blue reward:0.85\n", + "step: 807, Red action: do-nothing, Blue reward:0.85\n", + "step: 808, Red action: do-nothing, Blue reward:0.85\n", + "step: 809, Red action: do-nothing, Blue reward:0.85\n", + "step: 810, Red action: do-nothing, Blue reward:0.85\n", + "step: 811, Red action: do-nothing, Blue reward:0.85\n", + "step: 812, Red action: do-nothing, Blue reward:0.85\n", + "step: 813, Red action: do-nothing, Blue reward:0.85\n", + "step: 814, Red action: do-nothing, Blue reward:0.85\n", + "step: 815, Red action: do-nothing, Blue reward:0.85\n", + "step: 816, Red action: do-nothing, Blue reward:0.85\n", + "step: 817, Red action: do-nothing, Blue reward:0.85\n", + "step: 818, Red action: do-nothing, Blue reward:0.85\n", + "step: 819, Red action: do-nothing, Blue reward:0.85\n", + "step: 820, Red action: do-nothing, Blue reward:0.85\n", + "step: 821, Red action: do-nothing, Blue reward:0.85\n", + "step: 822, Red action: do-nothing, Blue reward:0.85\n", + "step: 823, Red action: do-nothing, Blue reward:0.85\n", + "step: 824, Red action: do-nothing, Blue reward:0.85\n", + "step: 825, Red action: do-nothing, Blue reward:0.85\n", + "step: 826, Red action: do-nothing, Blue reward:0.85\n", + "step: 827, Red action: node-application-execute, Blue reward:0.85\n", + "step: 828, Red action: do-nothing, Blue reward:0.85\n", + "step: 829, Red action: do-nothing, Blue reward:0.85\n", + "step: 830, Red action: do-nothing, Blue reward:0.85\n", + "step: 831, Red action: do-nothing, Blue reward:0.85\n", + "step: 832, Red action: do-nothing, Blue reward:0.85\n", + "step: 833, Red action: do-nothing, Blue reward:0.85\n", + "step: 834, Red action: do-nothing, Blue reward:0.85\n", + "step: 835, Red action: do-nothing, Blue reward:0.85\n", + "step: 836, Red action: do-nothing, Blue reward:0.85\n", + "step: 837, Red action: do-nothing, Blue reward:0.85\n", + "step: 838, Red action: do-nothing, Blue reward:0.85\n", + "step: 839, Red action: do-nothing, Blue reward:0.85\n", + "step: 840, Red action: do-nothing, Blue reward:0.85\n", + "step: 841, Red action: do-nothing, Blue reward:0.85\n", + "step: 842, Red action: do-nothing, Blue reward:0.85\n", + "step: 843, Red action: do-nothing, Blue reward:0.85\n", + "step: 844, Red action: do-nothing, Blue reward:0.85\n", + "step: 845, Red action: do-nothing, Blue reward:0.85\n", + "step: 846, Red action: do-nothing, Blue reward:0.85\n", + "step: 847, Red action: node-application-execute, Blue reward:0.85\n", + "step: 848, Red action: do-nothing, Blue reward:0.85\n", + "step: 849, Red action: do-nothing, Blue reward:0.85\n", + "step: 850, Red action: do-nothing, Blue reward:0.85\n", + "step: 851, Red action: do-nothing, Blue reward:0.85\n", + "step: 852, Red action: do-nothing, Blue reward:0.85\n", + "step: 853, Red action: do-nothing, Blue reward:0.85\n", + "step: 854, Red action: do-nothing, Blue reward:0.85\n", + "step: 855, Red action: do-nothing, Blue reward:0.85\n", + "step: 856, Red action: do-nothing, Blue reward:0.85\n", + "step: 857, Red action: do-nothing, Blue reward:0.85\n", + "step: 858, Red action: do-nothing, Blue reward:0.85\n", + "step: 859, Red action: do-nothing, Blue reward:0.85\n", + "step: 860, Red action: do-nothing, Blue reward:0.85\n", + "step: 861, Red action: do-nothing, Blue reward:0.85\n", + "step: 862, Red action: do-nothing, Blue reward:0.85\n", + "step: 863, Red action: do-nothing, Blue reward:0.85\n", + "step: 864, Red action: do-nothing, Blue reward:0.85\n", + "step: 865, Red action: do-nothing, Blue reward:0.85\n", + "step: 866, Red action: do-nothing, Blue reward:0.85\n", + "step: 867, Red action: do-nothing, Blue reward:0.85\n", + "step: 868, Red action: do-nothing, Blue reward:0.85\n", + "step: 869, Red action: do-nothing, Blue reward:0.85\n", + "step: 870, Red action: do-nothing, Blue reward:0.85\n", + "step: 871, Red action: node-application-execute, Blue reward:0.85\n", + "step: 872, Red action: do-nothing, Blue reward:0.85\n", + "step: 873, Red action: do-nothing, Blue reward:0.85\n", + "step: 874, Red action: do-nothing, Blue reward:0.85\n", + "step: 875, Red action: do-nothing, Blue reward:0.85\n", + "step: 876, Red action: do-nothing, Blue reward:0.85\n", + "step: 877, Red action: do-nothing, Blue reward:0.85\n", + "step: 878, Red action: do-nothing, Blue reward:0.85\n", + "step: 879, Red action: do-nothing, Blue reward:0.85\n", + "step: 880, Red action: do-nothing, Blue reward:0.85\n", + "step: 881, Red action: do-nothing, Blue reward:0.85\n", + "step: 882, Red action: do-nothing, Blue reward:0.85\n", + "step: 883, Red action: do-nothing, Blue reward:0.85\n", + "step: 884, Red action: do-nothing, Blue reward:0.85\n", + "step: 885, Red action: do-nothing, Blue reward:0.85\n", + "step: 886, Red action: do-nothing, Blue reward:0.85\n", + "step: 887, Red action: do-nothing, Blue reward:0.85\n", + "step: 888, Red action: do-nothing, Blue reward:0.85\n", + "step: 889, Red action: do-nothing, Blue reward:0.85\n", + "step: 890, Red action: do-nothing, Blue reward:0.85\n", + "step: 891, Red action: node-application-execute, Blue reward:0.85\n", + "step: 892, Red action: do-nothing, Blue reward:0.85\n", + "step: 893, Red action: do-nothing, Blue reward:0.85\n", + "step: 894, Red action: do-nothing, Blue reward:0.85\n", + "step: 895, Red action: do-nothing, Blue reward:0.85\n", + "step: 896, Red action: do-nothing, Blue reward:0.85\n", + "step: 897, Red action: do-nothing, Blue reward:0.85\n", + "step: 898, Red action: do-nothing, Blue reward:0.85\n", + "step: 899, Red action: do-nothing, Blue reward:0.85\n", + "step: 900, Red action: do-nothing, Blue reward:0.85\n", + "step: 901, Red action: do-nothing, Blue reward:0.85\n", + "step: 902, Red action: do-nothing, Blue reward:0.85\n", + "step: 903, Red action: do-nothing, Blue reward:0.85\n", + "step: 904, Red action: do-nothing, Blue reward:0.85\n", + "step: 905, Red action: do-nothing, Blue reward:0.85\n", + "step: 906, Red action: node-application-execute, Blue reward:0.85\n", + "step: 907, Red action: do-nothing, Blue reward:0.85\n", + "step: 908, Red action: do-nothing, Blue reward:0.85\n", + "step: 909, Red action: do-nothing, Blue reward:0.85\n", + "step: 910, Red action: do-nothing, Blue reward:0.85\n", + "step: 911, Red action: do-nothing, Blue reward:0.85\n", + "step: 912, Red action: do-nothing, Blue reward:0.85\n", + "step: 913, Red action: do-nothing, Blue reward:0.85\n", + "step: 914, Red action: do-nothing, Blue reward:0.85\n", + "step: 915, Red action: do-nothing, Blue reward:0.85\n", + "step: 916, Red action: do-nothing, Blue reward:0.85\n", + "step: 917, Red action: do-nothing, Blue reward:0.85\n", + "step: 918, Red action: do-nothing, Blue reward:0.85\n", + "step: 919, Red action: do-nothing, Blue reward:0.85\n", + "step: 920, Red action: do-nothing, Blue reward:0.85\n", + "step: 921, Red action: do-nothing, Blue reward:0.85\n", + "step: 922, Red action: do-nothing, Blue reward:0.85\n", + "step: 923, Red action: do-nothing, Blue reward:0.85\n", + "step: 924, Red action: do-nothing, Blue reward:0.85\n", + "step: 925, Red action: do-nothing, Blue reward:0.85\n", + "step: 926, Red action: do-nothing, Blue reward:0.85\n", + "step: 927, Red action: do-nothing, Blue reward:0.85\n", + "step: 928, Red action: do-nothing, Blue reward:0.85\n", + "step: 929, Red action: do-nothing, Blue reward:0.85\n", + "step: 930, Red action: node-application-execute, Blue reward:0.85\n", + "step: 931, Red action: do-nothing, Blue reward:0.85\n", + "step: 932, Red action: do-nothing, Blue reward:0.85\n", + "step: 933, Red action: do-nothing, Blue reward:0.85\n", + "step: 934, Red action: do-nothing, Blue reward:0.85\n", + "step: 935, Red action: do-nothing, Blue reward:0.85\n", + "step: 936, Red action: do-nothing, Blue reward:0.85\n", + "step: 937, Red action: do-nothing, Blue reward:0.85\n", + "step: 938, Red action: do-nothing, Blue reward:0.85\n", + "step: 939, Red action: do-nothing, Blue reward:0.85\n", + "step: 940, Red action: do-nothing, Blue reward:0.85\n", + "step: 941, Red action: do-nothing, Blue reward:0.85\n", + "step: 942, Red action: do-nothing, Blue reward:0.85\n", + "step: 943, Red action: do-nothing, Blue reward:0.85\n", + "step: 944, Red action: do-nothing, Blue reward:0.85\n", + "step: 945, Red action: do-nothing, Blue reward:0.85\n", + "step: 946, Red action: do-nothing, Blue reward:0.85\n", + "step: 947, Red action: node-application-execute, Blue reward:0.85\n", + "step: 948, Red action: do-nothing, Blue reward:0.85\n", + "step: 949, Red action: do-nothing, Blue reward:0.85\n", + "step: 950, Red action: do-nothing, Blue reward:0.85\n", + "step: 951, Red action: do-nothing, Blue reward:0.85\n", + "step: 952, Red action: do-nothing, Blue reward:0.85\n", + "step: 953, Red action: do-nothing, Blue reward:0.85\n", + "step: 954, Red action: do-nothing, Blue reward:0.85\n", + "step: 955, Red action: do-nothing, Blue reward:0.85\n", + "step: 956, Red action: do-nothing, Blue reward:0.85\n", + "step: 957, Red action: do-nothing, Blue reward:0.85\n", + "step: 958, Red action: do-nothing, Blue reward:0.85\n", + "step: 959, Red action: do-nothing, Blue reward:0.85\n", + "step: 960, Red action: do-nothing, Blue reward:0.85\n", + "step: 961, Red action: do-nothing, Blue reward:0.85\n", + "step: 962, Red action: do-nothing, Blue reward:0.85\n", + "step: 963, Red action: node-application-execute, Blue reward:0.85\n", + "step: 964, Red action: do-nothing, Blue reward:0.85\n", + "step: 965, Red action: do-nothing, Blue reward:0.85\n", + "step: 966, Red action: do-nothing, Blue reward:0.85\n", + "step: 967, Red action: do-nothing, Blue reward:0.85\n", + "step: 968, Red action: do-nothing, Blue reward:0.85\n", + "step: 969, Red action: do-nothing, Blue reward:0.85\n", + "step: 970, Red action: do-nothing, Blue reward:0.85\n", + "step: 971, Red action: do-nothing, Blue reward:0.85\n", + "step: 972, Red action: do-nothing, Blue reward:0.85\n", + "step: 973, Red action: do-nothing, Blue reward:0.85\n", + "step: 974, Red action: do-nothing, Blue reward:0.85\n", + "step: 975, Red action: do-nothing, Blue reward:0.85\n", + "step: 976, Red action: do-nothing, Blue reward:0.85\n", + "step: 977, Red action: do-nothing, Blue reward:0.85\n", + "step: 978, Red action: do-nothing, Blue reward:0.85\n", + "step: 979, Red action: do-nothing, Blue reward:0.85\n", + "step: 980, Red action: do-nothing, Blue reward:0.85\n", + "step: 981, Red action: do-nothing, Blue reward:0.85\n", + "step: 982, Red action: do-nothing, Blue reward:0.85\n", + "step: 983, Red action: do-nothing, Blue reward:0.85\n", + "step: 984, Red action: do-nothing, Blue reward:0.85\n", + "step: 985, Red action: do-nothing, Blue reward:0.85\n", + "step: 986, Red action: do-nothing, Blue reward:0.85\n", + "step: 987, Red action: do-nothing, Blue reward:0.85\n", + "step: 988, Red action: node-application-execute, Blue reward:0.85\n", + "step: 989, Red action: do-nothing, Blue reward:0.85\n", + "step: 990, Red action: do-nothing, Blue reward:0.85\n", + "step: 991, Red action: do-nothing, Blue reward:0.85\n", + "step: 992, Red action: do-nothing, Blue reward:0.85\n", + "step: 993, Red action: do-nothing, Blue reward:0.85\n", + "step: 994, Red action: do-nothing, Blue reward:0.85\n", + "step: 995, Red action: do-nothing, Blue reward:0.85\n", + "step: 996, Red action: do-nothing, Blue reward:0.85\n", + "step: 997, Red action: do-nothing, Blue reward:0.85\n", + "step: 998, Red action: do-nothing, Blue reward:0.85\n", + "step: 999, Red action: do-nothing, Blue reward:0.85\n", + "step: 1000, Red action: do-nothing, Blue reward:0.85\n", + "step: 1001, Red action: do-nothing, Blue reward:0.85\n", + "step: 1002, Red action: do-nothing, Blue reward:0.85\n", + "step: 1003, Red action: do-nothing, Blue reward:0.85\n", + "step: 1004, Red action: do-nothing, Blue reward:0.85\n", + "step: 1005, Red action: do-nothing, Blue reward:0.85\n", + "step: 1006, Red action: do-nothing, Blue reward:0.85\n", + "step: 1007, Red action: do-nothing, Blue reward:0.85\n", + "step: 1008, Red action: do-nothing, Blue reward:0.85\n", + "step: 1009, Red action: do-nothing, Blue reward:0.85\n", + "step: 1010, Red action: do-nothing, Blue reward:0.85\n", + "step: 1011, Red action: do-nothing, Blue reward:0.85\n", + "step: 1012, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1013, Red action: do-nothing, Blue reward:0.85\n", + "step: 1014, Red action: do-nothing, Blue reward:0.85\n", + "step: 1015, Red action: do-nothing, Blue reward:0.85\n", + "step: 1016, Red action: do-nothing, Blue reward:0.85\n", + "step: 1017, Red action: do-nothing, Blue reward:0.85\n", + "step: 1018, Red action: do-nothing, Blue reward:0.85\n", + "step: 1019, Red action: do-nothing, Blue reward:0.85\n", + "step: 1020, Red action: do-nothing, Blue reward:0.85\n", + "step: 1021, Red action: do-nothing, Blue reward:0.85\n", + "step: 1022, Red action: do-nothing, Blue reward:0.85\n", + "step: 1023, Red action: do-nothing, Blue reward:0.85\n", + "step: 1024, Red action: do-nothing, Blue reward:0.85\n", + "step: 1025, Red action: do-nothing, Blue reward:0.85\n", + "step: 1026, Red action: do-nothing, Blue reward:0.85\n", + "step: 1027, Red action: do-nothing, Blue reward:0.85\n", + "step: 1028, Red action: do-nothing, Blue reward:0.85\n", + "step: 1029, Red action: do-nothing, Blue reward:0.85\n", + "step: 1030, Red action: do-nothing, Blue reward:0.85\n", + "step: 1031, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1032, Red action: do-nothing, Blue reward:0.85\n", + "step: 1033, Red action: do-nothing, Blue reward:0.85\n", + "step: 1034, Red action: do-nothing, Blue reward:0.85\n", + "step: 1035, Red action: do-nothing, Blue reward:0.85\n", + "step: 1036, Red action: do-nothing, Blue reward:0.85\n", + "step: 1037, Red action: do-nothing, Blue reward:0.85\n", + "step: 1038, Red action: do-nothing, Blue reward:0.85\n", + "step: 1039, Red action: do-nothing, Blue reward:0.85\n", + "step: 1040, Red action: do-nothing, Blue reward:0.85\n", + "step: 1041, Red action: do-nothing, Blue reward:0.85\n", + "step: 1042, Red action: do-nothing, Blue reward:0.85\n", + "step: 1043, Red action: do-nothing, Blue reward:0.85\n", + "step: 1044, Red action: do-nothing, Blue reward:0.85\n", + "step: 1045, Red action: do-nothing, Blue reward:0.85\n", + "step: 1046, Red action: do-nothing, Blue reward:0.85\n", + "step: 1047, Red action: do-nothing, Blue reward:0.85\n", + "step: 1048, Red action: do-nothing, Blue reward:0.85\n", + "step: 1049, Red action: do-nothing, Blue reward:0.85\n", + "step: 1050, Red action: do-nothing, Blue reward:0.85\n", + "step: 1051, Red action: do-nothing, Blue reward:0.85\n", + "step: 1052, Red action: do-nothing, Blue reward:0.85\n", + "step: 1053, Red action: do-nothing, Blue reward:0.85\n", + "step: 1054, Red action: do-nothing, Blue reward:0.85\n", + "step: 1055, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1056, Red action: do-nothing, Blue reward:0.85\n", + "step: 1057, Red action: do-nothing, Blue reward:0.85\n", + "step: 1058, Red action: do-nothing, Blue reward:0.85\n", + "step: 1059, Red action: do-nothing, Blue reward:0.85\n", + "step: 1060, Red action: do-nothing, Blue reward:0.85\n", + "step: 1061, Red action: do-nothing, Blue reward:0.85\n", + "step: 1062, Red action: do-nothing, Blue reward:0.85\n", + "step: 1063, Red action: do-nothing, Blue reward:0.85\n", + "step: 1064, Red action: do-nothing, Blue reward:0.85\n", + "step: 1065, Red action: do-nothing, Blue reward:0.85\n", + "step: 1066, Red action: do-nothing, Blue reward:0.85\n", + "step: 1067, Red action: do-nothing, Blue reward:0.85\n", + "step: 1068, Red action: do-nothing, Blue reward:0.85\n", + "step: 1069, Red action: do-nothing, Blue reward:0.85\n", + "step: 1070, Red action: do-nothing, Blue reward:0.85\n", + "step: 1071, Red action: do-nothing, Blue reward:0.85\n", + "step: 1072, Red action: do-nothing, Blue reward:0.85\n", + "step: 1073, Red action: do-nothing, Blue reward:0.85\n", + "step: 1074, Red action: do-nothing, Blue reward:0.85\n", + "step: 1075, Red action: do-nothing, Blue reward:0.85\n", + "step: 1076, Red action: do-nothing, Blue reward:0.85\n", + "step: 1077, Red action: do-nothing, Blue reward:0.85\n", + "step: 1078, Red action: do-nothing, Blue reward:0.85\n", + "step: 1079, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1080, Red action: do-nothing, Blue reward:0.85\n", + "step: 1081, Red action: do-nothing, Blue reward:0.85\n", + "step: 1082, Red action: do-nothing, Blue reward:0.85\n", + "step: 1083, Red action: do-nothing, Blue reward:0.85\n", + "step: 1084, Red action: do-nothing, Blue reward:0.85\n", + "step: 1085, Red action: do-nothing, Blue reward:0.85\n", + "step: 1086, Red action: do-nothing, Blue reward:0.85\n", + "step: 1087, Red action: do-nothing, Blue reward:0.85\n", + "step: 1088, Red action: do-nothing, Blue reward:0.85\n", + "step: 1089, Red action: do-nothing, Blue reward:0.85\n", + "step: 1090, Red action: do-nothing, Blue reward:0.85\n", + "step: 1091, Red action: do-nothing, Blue reward:0.85\n", + "step: 1092, Red action: do-nothing, Blue reward:0.85\n", + "step: 1093, Red action: do-nothing, Blue reward:0.85\n", + "step: 1094, Red action: do-nothing, Blue reward:0.85\n", + "step: 1095, Red action: do-nothing, Blue reward:0.85\n", + "step: 1096, Red action: do-nothing, Blue reward:0.85\n", + "step: 1097, Red action: do-nothing, Blue reward:0.85\n", + "step: 1098, Red action: do-nothing, Blue reward:0.85\n", + "step: 1099, Red action: do-nothing, Blue reward:0.85\n", + "step: 1100, Red action: do-nothing, Blue reward:0.85\n", + "step: 1101, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1102, Red action: do-nothing, Blue reward:0.85\n", + "step: 1103, Red action: do-nothing, Blue reward:0.85\n", + "step: 1104, Red action: do-nothing, Blue reward:0.85\n", + "step: 1105, Red action: do-nothing, Blue reward:0.85\n", + "step: 1106, Red action: do-nothing, Blue reward:0.85\n", + "step: 1107, Red action: do-nothing, Blue reward:0.85\n", + "step: 1108, Red action: do-nothing, Blue reward:0.85\n", + "step: 1109, Red action: do-nothing, Blue reward:0.85\n", + "step: 1110, Red action: do-nothing, Blue reward:0.85\n", + "step: 1111, Red action: do-nothing, Blue reward:0.85\n", + "step: 1112, Red action: do-nothing, Blue reward:0.85\n", + "step: 1113, Red action: do-nothing, Blue reward:0.85\n", + "step: 1114, Red action: do-nothing, Blue reward:0.85\n", + "step: 1115, Red action: do-nothing, Blue reward:0.85\n", + "step: 1116, Red action: do-nothing, Blue reward:0.85\n", + "step: 1117, Red action: do-nothing, Blue reward:0.85\n", + "step: 1118, Red action: do-nothing, Blue reward:0.85\n", + "step: 1119, Red action: do-nothing, Blue reward:0.85\n", + "step: 1120, Red action: do-nothing, Blue reward:0.85\n", + "step: 1121, Red action: do-nothing, Blue reward:0.85\n", + "step: 1122, Red action: do-nothing, Blue reward:0.85\n", + "step: 1123, Red action: do-nothing, Blue reward:0.85\n", + "step: 1124, Red action: do-nothing, Blue reward:0.85\n", + "step: 1125, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1126, Red action: do-nothing, Blue reward:0.85\n", + "step: 1127, Red action: do-nothing, Blue reward:0.85\n", + "step: 1128, Red action: do-nothing, Blue reward:0.85\n", + "step: 1129, Red action: do-nothing, Blue reward:0.85\n", + "step: 1130, Red action: do-nothing, Blue reward:0.85\n", + "step: 1131, Red action: do-nothing, Blue reward:0.85\n", + "step: 1132, Red action: do-nothing, Blue reward:0.85\n", + "step: 1133, Red action: do-nothing, Blue reward:0.85\n", + "step: 1134, Red action: do-nothing, Blue reward:0.85\n", + "step: 1135, Red action: do-nothing, Blue reward:0.85\n", + "step: 1136, Red action: do-nothing, Blue reward:0.85\n", + "step: 1137, Red action: do-nothing, Blue reward:0.85\n", + "step: 1138, Red action: do-nothing, Blue reward:0.85\n", + "step: 1139, Red action: do-nothing, Blue reward:0.85\n", + "step: 1140, Red action: do-nothing, Blue reward:0.85\n", + "step: 1141, Red action: do-nothing, Blue reward:0.85\n", + "step: 1142, Red action: do-nothing, Blue reward:0.85\n", + "step: 1143, Red action: do-nothing, Blue reward:0.85\n", + "step: 1144, Red action: do-nothing, Blue reward:0.85\n", + "step: 1145, Red action: do-nothing, Blue reward:0.85\n", + "step: 1146, Red action: do-nothing, Blue reward:0.85\n", + "step: 1147, Red action: do-nothing, Blue reward:0.85\n", + "step: 1148, Red action: do-nothing, Blue reward:0.85\n", + "step: 1149, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1150, Red action: do-nothing, Blue reward:0.85\n", + "step: 1151, Red action: do-nothing, Blue reward:0.85\n", + "step: 1152, Red action: do-nothing, Blue reward:0.85\n", + "step: 1153, Red action: do-nothing, Blue reward:0.85\n", + "step: 1154, Red action: do-nothing, Blue reward:0.85\n", + "step: 1155, Red action: do-nothing, Blue reward:0.85\n", + "step: 1156, Red action: do-nothing, Blue reward:0.85\n", + "step: 1157, Red action: do-nothing, Blue reward:0.85\n", + "step: 1158, Red action: do-nothing, Blue reward:0.85\n", + "step: 1159, Red action: do-nothing, Blue reward:0.85\n", + "step: 1160, Red action: do-nothing, Blue reward:0.85\n", + "step: 1161, Red action: do-nothing, Blue reward:0.85\n", + "step: 1162, Red action: do-nothing, Blue reward:0.85\n", + "step: 1163, Red action: do-nothing, Blue reward:0.85\n", + "step: 1164, Red action: do-nothing, Blue reward:0.85\n", + "step: 1165, Red action: do-nothing, Blue reward:0.85\n", + "step: 1166, Red action: do-nothing, Blue reward:0.85\n", + "step: 1167, Red action: do-nothing, Blue reward:0.85\n", + "step: 1168, Red action: do-nothing, Blue reward:0.85\n", + "step: 1169, Red action: do-nothing, Blue reward:0.85\n", + "step: 1170, Red action: do-nothing, Blue reward:0.85\n", + "step: 1171, Red action: do-nothing, Blue reward:0.85\n", + "step: 1172, Red action: do-nothing, Blue reward:0.85\n", + "step: 1173, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1174, Red action: do-nothing, Blue reward:0.85\n", + "step: 1175, Red action: do-nothing, Blue reward:0.85\n", + "step: 1176, Red action: do-nothing, Blue reward:0.85\n", + "step: 1177, Red action: do-nothing, Blue reward:0.85\n", + "step: 1178, Red action: do-nothing, Blue reward:0.85\n", + "step: 1179, Red action: do-nothing, Blue reward:0.85\n", + "step: 1180, Red action: do-nothing, Blue reward:0.85\n", + "step: 1181, Red action: do-nothing, Blue reward:0.85\n", + "step: 1182, Red action: do-nothing, Blue reward:0.85\n", + "step: 1183, Red action: do-nothing, Blue reward:0.85\n", + "step: 1184, Red action: do-nothing, Blue reward:0.85\n", + "step: 1185, Red action: do-nothing, Blue reward:0.85\n", + "step: 1186, Red action: do-nothing, Blue reward:0.85\n", + "step: 1187, Red action: do-nothing, Blue reward:0.85\n", + "step: 1188, Red action: do-nothing, Blue reward:0.85\n", + "step: 1189, Red action: do-nothing, Blue reward:0.85\n", + "step: 1190, Red action: do-nothing, Blue reward:0.85\n", + "step: 1191, Red action: do-nothing, Blue reward:0.85\n", + "step: 1192, Red action: do-nothing, Blue reward:0.85\n", + "step: 1193, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1194, Red action: do-nothing, Blue reward:0.85\n", + "step: 1195, Red action: do-nothing, Blue reward:0.85\n", + "step: 1196, Red action: do-nothing, Blue reward:0.85\n", + "step: 1197, Red action: do-nothing, Blue reward:0.85\n", + "step: 1198, Red action: do-nothing, Blue reward:0.85\n", + "step: 1199, Red action: do-nothing, Blue reward:0.85\n", + "step: 1200, Red action: do-nothing, Blue reward:0.85\n", + "step: 1201, Red action: do-nothing, Blue reward:0.85\n", + "step: 1202, Red action: do-nothing, Blue reward:0.85\n", + "step: 1203, Red action: do-nothing, Blue reward:0.85\n", + "step: 1204, Red action: do-nothing, Blue reward:0.85\n", + "step: 1205, Red action: do-nothing, Blue reward:0.85\n", + "step: 1206, Red action: do-nothing, Blue reward:0.85\n", + "step: 1207, Red action: do-nothing, Blue reward:0.85\n", + "step: 1208, Red action: do-nothing, Blue reward:0.85\n", + "step: 1209, Red action: do-nothing, Blue reward:0.85\n", + "step: 1210, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1211, Red action: do-nothing, Blue reward:0.85\n", + "step: 1212, Red action: do-nothing, Blue reward:0.85\n", + "step: 1213, Red action: do-nothing, Blue reward:0.85\n", + "step: 1214, Red action: do-nothing, Blue reward:0.85\n", + "step: 1215, Red action: do-nothing, Blue reward:0.85\n", + "step: 1216, Red action: do-nothing, Blue reward:0.85\n", + "step: 1217, Red action: do-nothing, Blue reward:0.85\n", + "step: 1218, Red action: do-nothing, Blue reward:0.85\n", + "step: 1219, Red action: do-nothing, Blue reward:0.85\n", + "step: 1220, Red action: do-nothing, Blue reward:0.85\n", + "step: 1221, Red action: do-nothing, Blue reward:0.85\n", + "step: 1222, Red action: do-nothing, Blue reward:0.85\n", + "step: 1223, Red action: do-nothing, Blue reward:0.85\n", + "step: 1224, Red action: do-nothing, Blue reward:0.85\n", + "step: 1225, Red action: do-nothing, Blue reward:0.85\n", + "step: 1226, Red action: do-nothing, Blue reward:0.85\n", + "step: 1227, Red action: do-nothing, Blue reward:0.85\n", + "step: 1228, Red action: do-nothing, Blue reward:0.85\n", + "step: 1229, Red action: do-nothing, Blue reward:0.85\n", + "step: 1230, Red action: do-nothing, Blue reward:0.85\n", + "step: 1231, Red action: do-nothing, Blue reward:0.85\n", + "step: 1232, Red action: do-nothing, Blue reward:0.85\n", + "step: 1233, Red action: do-nothing, Blue reward:0.85\n", + "step: 1234, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1235, Red action: do-nothing, Blue reward:0.85\n", + "step: 1236, Red action: do-nothing, Blue reward:0.85\n", + "step: 1237, Red action: do-nothing, Blue reward:0.85\n", + "step: 1238, Red action: do-nothing, Blue reward:0.85\n", + "step: 1239, Red action: do-nothing, Blue reward:0.85\n", + "step: 1240, Red action: do-nothing, Blue reward:0.85\n", + "step: 1241, Red action: do-nothing, Blue reward:0.85\n", + "step: 1242, Red action: do-nothing, Blue reward:0.85\n", + "step: 1243, Red action: do-nothing, Blue reward:0.85\n", + "step: 1244, Red action: do-nothing, Blue reward:0.85\n", + "step: 1245, Red action: do-nothing, Blue reward:0.85\n", + "step: 1246, Red action: do-nothing, Blue reward:0.85\n", + "step: 1247, Red action: do-nothing, Blue reward:0.85\n", + "step: 1248, Red action: do-nothing, Blue reward:0.85\n", + "step: 1249, Red action: do-nothing, Blue reward:0.85\n", + "step: 1250, Red action: do-nothing, Blue reward:0.85\n", + "step: 1251, Red action: do-nothing, Blue reward:0.85\n", + "step: 1252, Red action: do-nothing, Blue reward:0.85\n", + "step: 1253, Red action: do-nothing, Blue reward:0.85\n", + "step: 1254, Red action: do-nothing, Blue reward:0.85\n", + "step: 1255, Red action: do-nothing, Blue reward:0.85\n", + "step: 1256, Red action: do-nothing, Blue reward:0.85\n", + "step: 1257, Red action: do-nothing, Blue reward:0.85\n", + "step: 1258, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1259, Red action: do-nothing, Blue reward:0.85\n", + "step: 1260, Red action: do-nothing, Blue reward:0.85\n", + "step: 1261, Red action: do-nothing, Blue reward:0.85\n", + "step: 1262, Red action: do-nothing, Blue reward:0.85\n", + "step: 1263, Red action: do-nothing, Blue reward:0.85\n", + "step: 1264, Red action: do-nothing, Blue reward:0.85\n", + "step: 1265, Red action: do-nothing, Blue reward:0.85\n", + "step: 1266, Red action: do-nothing, Blue reward:0.85\n", + "step: 1267, Red action: do-nothing, Blue reward:0.85\n", + "step: 1268, Red action: do-nothing, Blue reward:0.85\n", + "step: 1269, Red action: do-nothing, Blue reward:0.85\n", + "step: 1270, Red action: do-nothing, Blue reward:0.85\n", + "step: 1271, Red action: do-nothing, Blue reward:0.85\n", + "step: 1272, Red action: do-nothing, Blue reward:0.85\n", + "step: 1273, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1274, Red action: do-nothing, Blue reward:0.85\n", + "step: 1275, Red action: do-nothing, Blue reward:0.85\n", + "step: 1276, Red action: do-nothing, Blue reward:0.85\n", + "step: 1277, Red action: do-nothing, Blue reward:0.85\n", + "step: 1278, Red action: do-nothing, Blue reward:0.85\n", + "step: 1279, Red action: do-nothing, Blue reward:0.85\n", + "step: 1280, Red action: do-nothing, Blue reward:0.85\n", + "step: 1281, Red action: do-nothing, Blue reward:0.85\n", + "step: 1282, Red action: do-nothing, Blue reward:0.85\n", + "step: 1283, Red action: do-nothing, Blue reward:0.85\n", + "step: 1284, Red action: do-nothing, Blue reward:0.85\n", + "step: 1285, Red action: do-nothing, Blue reward:0.85\n", + "step: 1286, Red action: do-nothing, Blue reward:0.85\n", + "step: 1287, Red action: do-nothing, Blue reward:0.85\n", + "step: 1288, Red action: do-nothing, Blue reward:0.85\n", + "step: 1289, Red action: do-nothing, Blue reward:0.85\n", + "step: 1290, Red action: do-nothing, Blue reward:0.85\n", + "step: 1291, Red action: do-nothing, Blue reward:0.85\n", + "step: 1292, Red action: do-nothing, Blue reward:0.85\n", + "step: 1293, Red action: do-nothing, Blue reward:0.85\n", + "step: 1294, Red action: do-nothing, Blue reward:0.85\n", + "step: 1295, Red action: do-nothing, Blue reward:0.85\n", + "step: 1296, Red action: do-nothing, Blue reward:0.85\n", + "step: 1297, Red action: do-nothing, Blue reward:0.85\n", + "step: 1298, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1299, Red action: do-nothing, Blue reward:0.85\n", + "step: 1300, Red action: do-nothing, Blue reward:0.85\n", + "step: 1301, Red action: do-nothing, Blue reward:0.85\n", + "step: 1302, Red action: do-nothing, Blue reward:0.85\n", + "step: 1303, Red action: do-nothing, Blue reward:0.85\n", + "step: 1304, Red action: do-nothing, Blue reward:0.85\n", + "step: 1305, Red action: do-nothing, Blue reward:0.85\n", + "step: 1306, Red action: do-nothing, Blue reward:0.85\n", + "step: 1307, Red action: do-nothing, Blue reward:0.85\n", + "step: 1308, Red action: do-nothing, Blue reward:0.85\n", + "step: 1309, Red action: do-nothing, Blue reward:0.85\n", + "step: 1310, Red action: do-nothing, Blue reward:0.85\n", + "step: 1311, Red action: do-nothing, Blue reward:0.85\n", + "step: 1312, Red action: do-nothing, Blue reward:0.85\n", + "step: 1313, Red action: do-nothing, Blue reward:0.85\n", + "step: 1314, Red action: do-nothing, Blue reward:0.85\n", + "step: 1315, Red action: do-nothing, Blue reward:0.85\n", + "step: 1316, Red action: do-nothing, Blue reward:0.85\n", + "step: 1317, Red action: do-nothing, Blue reward:0.85\n", + "step: 1318, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1319, Red action: do-nothing, Blue reward:0.85\n", + "step: 1320, Red action: do-nothing, Blue reward:0.85\n", + "step: 1321, Red action: do-nothing, Blue reward:0.85\n", + "step: 1322, Red action: do-nothing, Blue reward:0.85\n", + "step: 1323, Red action: do-nothing, Blue reward:0.85\n", + "step: 1324, Red action: do-nothing, Blue reward:0.85\n", + "step: 1325, Red action: do-nothing, Blue reward:0.85\n", + "step: 1326, Red action: do-nothing, Blue reward:0.85\n", + "step: 1327, Red action: do-nothing, Blue reward:0.85\n", + "step: 1328, Red action: do-nothing, Blue reward:0.85\n", + "step: 1329, Red action: do-nothing, Blue reward:0.85\n", + "step: 1330, Red action: do-nothing, Blue reward:0.85\n", + "step: 1331, Red action: do-nothing, Blue reward:0.85\n", + "step: 1332, Red action: do-nothing, Blue reward:0.85\n", + "step: 1333, Red action: do-nothing, Blue reward:0.85\n", + "step: 1334, Red action: do-nothing, Blue reward:0.85\n", + "step: 1335, Red action: do-nothing, Blue reward:0.85\n", + "step: 1336, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1337, Red action: do-nothing, Blue reward:0.85\n", + "step: 1338, Red action: do-nothing, Blue reward:0.85\n", + "step: 1339, Red action: do-nothing, Blue reward:0.85\n", + "step: 1340, Red action: do-nothing, Blue reward:0.85\n", + "step: 1341, Red action: do-nothing, Blue reward:0.85\n", + "step: 1342, Red action: do-nothing, Blue reward:0.85\n", + "step: 1343, Red action: do-nothing, Blue reward:0.85\n", + "step: 1344, Red action: do-nothing, Blue reward:0.85\n", + "step: 1345, Red action: do-nothing, Blue reward:0.85\n", + "step: 1346, Red action: do-nothing, Blue reward:0.85\n", + "step: 1347, Red action: do-nothing, Blue reward:0.85\n", + "step: 1348, Red action: do-nothing, Blue reward:0.85\n", + "step: 1349, Red action: do-nothing, Blue reward:0.85\n", + "step: 1350, Red action: do-nothing, Blue reward:0.85\n", + "step: 1351, Red action: do-nothing, Blue reward:0.85\n", + "step: 1352, Red action: do-nothing, Blue reward:0.85\n", + "step: 1353, Red action: do-nothing, Blue reward:0.85\n", + "step: 1354, Red action: do-nothing, Blue reward:0.85\n", + "step: 1355, Red action: do-nothing, Blue reward:0.85\n", + "step: 1356, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1357, Red action: do-nothing, Blue reward:0.85\n", + "step: 1358, Red action: do-nothing, Blue reward:0.85\n", + "step: 1359, Red action: do-nothing, Blue reward:0.85\n", + "step: 1360, Red action: do-nothing, Blue reward:0.85\n", + "step: 1361, Red action: do-nothing, Blue reward:0.85\n", + "step: 1362, Red action: do-nothing, Blue reward:0.85\n", + "step: 1363, Red action: do-nothing, Blue reward:0.85\n", + "step: 1364, Red action: do-nothing, Blue reward:0.85\n", + "step: 1365, Red action: do-nothing, Blue reward:0.85\n", + "step: 1366, Red action: do-nothing, Blue reward:0.85\n", + "step: 1367, Red action: do-nothing, Blue reward:0.85\n", + "step: 1368, Red action: do-nothing, Blue reward:0.85\n", + "step: 1369, Red action: do-nothing, Blue reward:0.85\n", + "step: 1370, Red action: do-nothing, Blue reward:0.85\n", + "step: 1371, Red action: do-nothing, Blue reward:0.85\n", + "step: 1372, Red action: do-nothing, Blue reward:0.85\n", + "step: 1373, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1374, Red action: do-nothing, Blue reward:0.85\n", + "step: 1375, Red action: do-nothing, Blue reward:0.85\n", + "step: 1376, Red action: do-nothing, Blue reward:0.85\n", + "step: 1377, Red action: do-nothing, Blue reward:0.85\n", + "step: 1378, Red action: do-nothing, Blue reward:0.85\n", + "step: 1379, Red action: do-nothing, Blue reward:0.85\n", + "step: 1380, Red action: do-nothing, Blue reward:0.85\n", + "step: 1381, Red action: do-nothing, Blue reward:0.85\n", + "step: 1382, Red action: do-nothing, Blue reward:0.85\n", + "step: 1383, Red action: do-nothing, Blue reward:0.85\n", + "step: 1384, Red action: do-nothing, Blue reward:0.85\n", + "step: 1385, Red action: do-nothing, Blue reward:0.85\n", + "step: 1386, Red action: do-nothing, Blue reward:0.85\n", + "step: 1387, Red action: do-nothing, Blue reward:0.85\n", + "step: 1388, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1389, Red action: do-nothing, Blue reward:0.85\n", + "step: 1390, Red action: do-nothing, Blue reward:0.85\n", + "step: 1391, Red action: do-nothing, Blue reward:0.85\n", + "step: 1392, Red action: do-nothing, Blue reward:0.85\n", + "step: 1393, Red action: do-nothing, Blue reward:0.85\n", + "step: 1394, Red action: do-nothing, Blue reward:0.85\n", + "step: 1395, Red action: do-nothing, Blue reward:0.85\n", + "step: 1396, Red action: do-nothing, Blue reward:0.85\n", + "step: 1397, Red action: do-nothing, Blue reward:0.85\n", + "step: 1398, Red action: do-nothing, Blue reward:0.85\n", + "step: 1399, Red action: do-nothing, Blue reward:0.85\n", + "step: 1400, Red action: do-nothing, Blue reward:0.85\n", + "step: 1401, Red action: do-nothing, Blue reward:0.85\n", + "step: 1402, Red action: do-nothing, Blue reward:0.85\n", + "step: 1403, Red action: do-nothing, Blue reward:0.85\n", + "step: 1404, Red action: do-nothing, Blue reward:0.85\n", + "step: 1405, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1406, Red action: do-nothing, Blue reward:0.85\n", + "step: 1407, Red action: do-nothing, Blue reward:0.85\n", + "step: 1408, Red action: do-nothing, Blue reward:0.85\n", + "step: 1409, Red action: do-nothing, Blue reward:0.85\n", + "step: 1410, Red action: do-nothing, Blue reward:0.85\n", + "step: 1411, Red action: do-nothing, Blue reward:0.85\n", + "step: 1412, Red action: do-nothing, Blue reward:0.85\n", + "step: 1413, Red action: do-nothing, Blue reward:0.85\n", + "step: 1414, Red action: do-nothing, Blue reward:0.85\n", + "step: 1415, Red action: do-nothing, Blue reward:0.85\n", + "step: 1416, Red action: do-nothing, Blue reward:0.85\n", + "step: 1417, Red action: do-nothing, Blue reward:0.85\n", + "step: 1418, Red action: do-nothing, Blue reward:0.85\n", + "step: 1419, Red action: do-nothing, Blue reward:0.85\n", + "step: 1420, Red action: do-nothing, Blue reward:0.85\n", + "step: 1421, Red action: do-nothing, Blue reward:0.85\n", + "step: 1422, Red action: do-nothing, Blue reward:0.85\n", + "step: 1423, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1424, Red action: do-nothing, Blue reward:0.85\n", + "step: 1425, Red action: do-nothing, Blue reward:0.85\n", + "step: 1426, Red action: do-nothing, Blue reward:0.85\n", + "step: 1427, Red action: do-nothing, Blue reward:0.85\n", + "step: 1428, Red action: do-nothing, Blue reward:0.85\n", + "step: 1429, Red action: do-nothing, Blue reward:0.85\n", + "step: 1430, Red action: do-nothing, Blue reward:0.85\n", + "step: 1431, Red action: do-nothing, Blue reward:0.85\n", + "step: 1432, Red action: do-nothing, Blue reward:0.85\n", + "step: 1433, Red action: do-nothing, Blue reward:0.85\n", + "step: 1434, Red action: do-nothing, Blue reward:0.85\n", + "step: 1435, Red action: do-nothing, Blue reward:0.85\n", + "step: 1436, Red action: do-nothing, Blue reward:0.85\n", + "step: 1437, Red action: do-nothing, Blue reward:0.85\n", + "step: 1438, Red action: do-nothing, Blue reward:0.85\n", + "step: 1439, Red action: do-nothing, Blue reward:0.85\n", + "step: 1440, Red action: do-nothing, Blue reward:0.85\n", + "step: 1441, Red action: do-nothing, Blue reward:0.85\n", + "step: 1442, Red action: do-nothing, Blue reward:0.85\n", + "step: 1443, Red action: do-nothing, Blue reward:0.85\n", + "step: 1444, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1445, Red action: do-nothing, Blue reward:0.85\n", + "step: 1446, Red action: do-nothing, Blue reward:0.85\n", + "step: 1447, Red action: do-nothing, Blue reward:0.85\n", + "step: 1448, Red action: do-nothing, Blue reward:0.85\n", + "step: 1449, Red action: do-nothing, Blue reward:0.85\n", + "step: 1450, Red action: do-nothing, Blue reward:0.85\n", + "step: 1451, Red action: do-nothing, Blue reward:0.85\n", + "step: 1452, Red action: do-nothing, Blue reward:0.85\n", + "step: 1453, Red action: do-nothing, Blue reward:0.85\n", + "step: 1454, Red action: do-nothing, Blue reward:0.85\n", + "step: 1455, Red action: do-nothing, Blue reward:0.85\n", + "step: 1456, Red action: do-nothing, Blue reward:0.85\n", + "step: 1457, Red action: do-nothing, Blue reward:0.85\n", + "step: 1458, Red action: do-nothing, Blue reward:0.85\n", + "step: 1459, Red action: do-nothing, Blue reward:0.85\n", + "step: 1460, Red action: do-nothing, Blue reward:0.85\n", + "step: 1461, Red action: do-nothing, Blue reward:0.85\n", + "step: 1462, Red action: do-nothing, Blue reward:0.85\n", + "step: 1463, Red action: do-nothing, Blue reward:0.85\n", + "step: 1464, Red action: do-nothing, Blue reward:0.85\n", + "step: 1465, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1466, Red action: do-nothing, Blue reward:0.85\n", + "step: 1467, Red action: do-nothing, Blue reward:0.85\n", + "step: 1468, Red action: do-nothing, Blue reward:0.85\n", + "step: 1469, Red action: do-nothing, Blue reward:0.85\n", + "step: 1470, Red action: do-nothing, Blue reward:0.85\n", + "step: 1471, Red action: do-nothing, Blue reward:0.85\n", + "step: 1472, Red action: do-nothing, Blue reward:0.85\n", + "step: 1473, Red action: do-nothing, Blue reward:0.85\n", + "step: 1474, Red action: do-nothing, Blue reward:0.85\n", + "step: 1475, Red action: do-nothing, Blue reward:0.85\n", + "step: 1476, Red action: do-nothing, Blue reward:0.85\n", + "step: 1477, Red action: do-nothing, Blue reward:0.85\n", + "step: 1478, Red action: do-nothing, Blue reward:0.85\n", + "step: 1479, Red action: do-nothing, Blue reward:0.85\n", + "step: 1480, Red action: do-nothing, Blue reward:0.85\n", + "step: 1481, Red action: do-nothing, Blue reward:0.85\n", + "step: 1482, Red action: do-nothing, Blue reward:0.85\n", + "step: 1483, Red action: do-nothing, Blue reward:0.85\n", + "step: 1484, Red action: do-nothing, Blue reward:0.85\n", + "step: 1485, Red action: do-nothing, Blue reward:0.85\n", + "step: 1486, Red action: do-nothing, Blue reward:0.85\n", + "step: 1487, Red action: do-nothing, Blue reward:0.85\n", + "step: 1488, Red action: do-nothing, Blue reward:0.85\n", + "step: 1489, Red action: do-nothing, Blue reward:0.85\n", + "step: 1490, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1491, Red action: do-nothing, Blue reward:0.85\n", + "step: 1492, Red action: do-nothing, Blue reward:0.85\n", + "step: 1493, Red action: do-nothing, Blue reward:0.85\n", + "step: 1494, Red action: do-nothing, Blue reward:0.85\n", + "step: 1495, Red action: do-nothing, Blue reward:0.85\n", + "step: 1496, Red action: do-nothing, Blue reward:0.85\n", + "step: 1497, Red action: do-nothing, Blue reward:0.85\n", + "step: 1498, Red action: do-nothing, Blue reward:0.85\n", + "step: 1499, Red action: do-nothing, Blue reward:0.85\n", + "step: 1500, Red action: do-nothing, Blue reward:0.85\n", + "step: 1501, Red action: do-nothing, Blue reward:0.85\n", + "step: 1502, Red action: do-nothing, Blue reward:0.85\n", + "step: 1503, Red action: do-nothing, Blue reward:0.85\n", + "step: 1504, Red action: do-nothing, Blue reward:0.85\n", + "step: 1505, Red action: do-nothing, Blue reward:0.85\n", + "step: 1506, Red action: do-nothing, Blue reward:0.85\n", + "step: 1507, Red action: do-nothing, Blue reward:0.85\n", + "step: 1508, Red action: do-nothing, Blue reward:0.85\n", + "step: 1509, Red action: do-nothing, Blue reward:0.85\n", + "step: 1510, Red action: do-nothing, Blue reward:0.85\n", + "step: 1511, Red action: do-nothing, Blue reward:0.85\n", + "step: 1512, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1513, Red action: do-nothing, Blue reward:0.85\n", + "step: 1514, Red action: do-nothing, Blue reward:0.85\n", + "step: 1515, Red action: do-nothing, Blue reward:0.85\n", + "step: 1516, Red action: do-nothing, Blue reward:0.85\n", + "step: 1517, Red action: do-nothing, Blue reward:0.85\n", + "step: 1518, Red action: do-nothing, Blue reward:0.85\n", + "step: 1519, Red action: do-nothing, Blue reward:0.85\n", + "step: 1520, Red action: do-nothing, Blue reward:0.85\n", + "step: 1521, Red action: do-nothing, Blue reward:0.85\n", + "step: 1522, Red action: do-nothing, Blue reward:0.85\n", + "step: 1523, Red action: do-nothing, Blue reward:0.85\n", + "step: 1524, Red action: do-nothing, Blue reward:0.85\n", + "step: 1525, Red action: do-nothing, Blue reward:0.85\n", + "step: 1526, Red action: do-nothing, Blue reward:0.85\n", + "step: 1527, Red action: do-nothing, Blue reward:0.85\n", + "step: 1528, Red action: do-nothing, Blue reward:0.85\n", + "step: 1529, Red action: do-nothing, Blue reward:0.85\n", + "step: 1530, Red action: do-nothing, Blue reward:0.85\n", + "step: 1531, Red action: do-nothing, Blue reward:0.85\n", + "step: 1532, Red action: do-nothing, Blue reward:0.85\n", + "step: 1533, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1534, Red action: do-nothing, Blue reward:0.85\n", + "step: 1535, Red action: do-nothing, Blue reward:0.85\n", + "step: 1536, Red action: do-nothing, Blue reward:0.85\n", + "step: 1537, Red action: do-nothing, Blue reward:0.85\n", + "step: 1538, Red action: do-nothing, Blue reward:0.85\n", + "step: 1539, Red action: do-nothing, Blue reward:0.85\n", + "step: 1540, Red action: do-nothing, Blue reward:0.85\n", + "step: 1541, Red action: do-nothing, Blue reward:0.85\n", + "step: 1542, Red action: do-nothing, Blue reward:0.85\n", + "step: 1543, Red action: do-nothing, Blue reward:0.85\n", + "step: 1544, Red action: do-nothing, Blue reward:0.85\n", + "step: 1545, Red action: do-nothing, Blue reward:0.85\n", + "step: 1546, Red action: do-nothing, Blue reward:0.85\n", + "step: 1547, Red action: do-nothing, Blue reward:0.85\n", + "step: 1548, Red action: do-nothing, Blue reward:0.85\n", + "step: 1549, Red action: do-nothing, Blue reward:0.85\n", + "step: 1550, Red action: do-nothing, Blue reward:0.85\n", + "step: 1551, Red action: do-nothing, Blue reward:0.85\n", + "step: 1552, Red action: do-nothing, Blue reward:0.85\n", + "step: 1553, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1554, Red action: do-nothing, Blue reward:0.85\n", + "step: 1555, Red action: do-nothing, Blue reward:0.85\n", + "step: 1556, Red action: do-nothing, Blue reward:0.85\n", + "step: 1557, Red action: do-nothing, Blue reward:0.85\n", + "step: 1558, Red action: do-nothing, Blue reward:0.85\n", + "step: 1559, Red action: do-nothing, Blue reward:0.85\n", + "step: 1560, Red action: do-nothing, Blue reward:0.85\n", + "step: 1561, Red action: do-nothing, Blue reward:0.85\n", + "step: 1562, Red action: do-nothing, Blue reward:0.85\n", + "step: 1563, Red action: do-nothing, Blue reward:0.85\n", + "step: 1564, Red action: do-nothing, Blue reward:0.85\n", + "step: 1565, Red action: do-nothing, Blue reward:0.85\n", + "step: 1566, Red action: do-nothing, Blue reward:0.85\n", + "step: 1567, Red action: do-nothing, Blue reward:0.85\n", + "step: 1568, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1569, Red action: do-nothing, Blue reward:0.85\n", + "step: 1570, Red action: do-nothing, Blue reward:0.85\n", + "step: 1571, Red action: do-nothing, Blue reward:0.85\n", + "step: 1572, Red action: do-nothing, Blue reward:0.85\n", + "step: 1573, Red action: do-nothing, Blue reward:0.85\n", + "step: 1574, Red action: do-nothing, Blue reward:0.85\n", + "step: 1575, Red action: do-nothing, Blue reward:0.85\n", + "step: 1576, Red action: do-nothing, Blue reward:0.85\n", + "step: 1577, Red action: do-nothing, Blue reward:0.85\n", + "step: 1578, Red action: do-nothing, Blue reward:0.85\n", + "step: 1579, Red action: do-nothing, Blue reward:0.85\n", + "step: 1580, Red action: do-nothing, Blue reward:0.85\n", + "step: 1581, Red action: do-nothing, Blue reward:0.85\n", + "step: 1582, Red action: do-nothing, Blue reward:0.85\n", + "step: 1583, Red action: do-nothing, Blue reward:0.85\n", + "step: 1584, Red action: do-nothing, Blue reward:0.85\n", + "step: 1585, Red action: do-nothing, Blue reward:0.85\n", + "step: 1586, Red action: do-nothing, Blue reward:0.85\n", + "step: 1587, Red action: do-nothing, Blue reward:0.85\n", + "step: 1588, Red action: do-nothing, Blue reward:0.85\n", + "step: 1589, Red action: do-nothing, Blue reward:0.85\n", + "step: 1590, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1591, Red action: do-nothing, Blue reward:0.85\n", + "step: 1592, Red action: do-nothing, Blue reward:0.85\n", + "step: 1593, Red action: do-nothing, Blue reward:0.85\n", + "step: 1594, Red action: do-nothing, Blue reward:0.85\n", + "step: 1595, Red action: do-nothing, Blue reward:0.85\n", + "step: 1596, Red action: do-nothing, Blue reward:0.85\n", + "step: 1597, Red action: do-nothing, Blue reward:0.85\n", + "step: 1598, Red action: do-nothing, Blue reward:0.85\n", + "step: 1599, Red action: do-nothing, Blue reward:0.85\n", + "step: 1600, Red action: do-nothing, Blue reward:0.85\n", + "step: 1601, Red action: do-nothing, Blue reward:0.85\n", + "step: 1602, Red action: do-nothing, Blue reward:0.85\n", + "step: 1603, Red action: do-nothing, Blue reward:0.85\n", + "step: 1604, Red action: do-nothing, Blue reward:0.85\n", + "step: 1605, Red action: do-nothing, Blue reward:0.85\n", + "step: 1606, Red action: do-nothing, Blue reward:0.85\n", + "step: 1607, Red action: do-nothing, Blue reward:0.85\n", + "step: 1608, Red action: do-nothing, Blue reward:0.85\n", + "step: 1609, Red action: do-nothing, Blue reward:0.85\n", + "step: 1610, Red action: do-nothing, Blue reward:0.85\n", + "step: 1611, Red action: do-nothing, Blue reward:0.85\n", + "step: 1612, Red action: do-nothing, Blue reward:0.85\n", + "step: 1613, Red action: do-nothing, Blue reward:0.85\n", + "step: 1614, Red action: do-nothing, Blue reward:0.85\n", + "step: 1615, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1616, Red action: do-nothing, Blue reward:0.85\n", + "step: 1617, Red action: do-nothing, Blue reward:0.85\n", + "step: 1618, Red action: do-nothing, Blue reward:0.85\n", + "step: 1619, Red action: do-nothing, Blue reward:0.85\n", + "step: 1620, Red action: do-nothing, Blue reward:0.85\n", + "step: 1621, Red action: do-nothing, Blue reward:0.85\n", + "step: 1622, Red action: do-nothing, Blue reward:0.85\n", + "step: 1623, Red action: do-nothing, Blue reward:0.85\n", + "step: 1624, Red action: do-nothing, Blue reward:0.85\n", + "step: 1625, Red action: do-nothing, Blue reward:0.85\n", + "step: 1626, Red action: do-nothing, Blue reward:0.85\n", + "step: 1627, Red action: do-nothing, Blue reward:0.85\n", + "step: 1628, Red action: do-nothing, Blue reward:0.85\n", + "step: 1629, Red action: do-nothing, Blue reward:0.85\n", + "step: 1630, Red action: do-nothing, Blue reward:0.85\n", + "step: 1631, Red action: do-nothing, Blue reward:0.85\n", + "step: 1632, Red action: do-nothing, Blue reward:0.85\n", + "step: 1633, Red action: do-nothing, Blue reward:0.85\n", + "step: 1634, Red action: do-nothing, Blue reward:0.85\n", + "step: 1635, Red action: do-nothing, Blue reward:0.85\n", + "step: 1636, Red action: do-nothing, Blue reward:0.85\n", + "step: 1637, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1638, Red action: do-nothing, Blue reward:0.85\n", + "step: 1639, Red action: do-nothing, Blue reward:0.85\n", + "step: 1640, Red action: do-nothing, Blue reward:0.85\n", + "step: 1641, Red action: do-nothing, Blue reward:0.85\n", + "step: 1642, Red action: do-nothing, Blue reward:0.85\n", + "step: 1643, Red action: do-nothing, Blue reward:0.85\n", + "step: 1644, Red action: do-nothing, Blue reward:0.85\n", + "step: 1645, Red action: do-nothing, Blue reward:0.85\n", + "step: 1646, Red action: do-nothing, Blue reward:0.85\n", + "step: 1647, Red action: do-nothing, Blue reward:0.85\n", + "step: 1648, Red action: do-nothing, Blue reward:0.85\n", + "step: 1649, Red action: do-nothing, Blue reward:0.85\n", + "step: 1650, Red action: do-nothing, Blue reward:0.85\n", + "step: 1651, Red action: do-nothing, Blue reward:0.85\n", + "step: 1652, Red action: do-nothing, Blue reward:0.85\n", + "step: 1653, Red action: do-nothing, Blue reward:0.85\n", + "step: 1654, Red action: do-nothing, Blue reward:0.85\n", + "step: 1655, Red action: do-nothing, Blue reward:0.85\n", + "step: 1656, Red action: do-nothing, Blue reward:0.85\n", + "step: 1657, Red action: do-nothing, Blue reward:0.85\n", + "step: 1658, Red action: do-nothing, Blue reward:0.85\n", + "step: 1659, Red action: do-nothing, Blue reward:0.85\n", + "step: 1660, Red action: do-nothing, Blue reward:0.85\n", + "step: 1661, Red action: do-nothing, Blue reward:0.85\n", + "step: 1662, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1663, Red action: do-nothing, Blue reward:0.85\n", + "step: 1664, Red action: do-nothing, Blue reward:0.85\n", + "step: 1665, Red action: do-nothing, Blue reward:0.85\n", + "step: 1666, Red action: do-nothing, Blue reward:0.85\n", + "step: 1667, Red action: do-nothing, Blue reward:0.85\n", + "step: 1668, Red action: do-nothing, Blue reward:0.85\n", + "step: 1669, Red action: do-nothing, Blue reward:0.85\n", + "step: 1670, Red action: do-nothing, Blue reward:0.85\n", + "step: 1671, Red action: do-nothing, Blue reward:0.85\n", + "step: 1672, Red action: do-nothing, Blue reward:0.85\n", + "step: 1673, Red action: do-nothing, Blue reward:0.85\n", + "step: 1674, Red action: do-nothing, Blue reward:0.85\n", + "step: 1675, Red action: do-nothing, Blue reward:0.85\n", + "step: 1676, Red action: do-nothing, Blue reward:0.85\n", + "step: 1677, Red action: do-nothing, Blue reward:0.85\n", + "step: 1678, Red action: do-nothing, Blue reward:0.85\n", + "step: 1679, Red action: do-nothing, Blue reward:0.85\n", + "step: 1680, Red action: do-nothing, Blue reward:0.85\n", + "step: 1681, Red action: do-nothing, Blue reward:0.85\n", + "step: 1682, Red action: do-nothing, Blue reward:0.85\n", + "step: 1683, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1684, Red action: do-nothing, Blue reward:0.85\n", + "step: 1685, Red action: do-nothing, Blue reward:0.85\n", + "step: 1686, Red action: do-nothing, Blue reward:0.85\n", + "step: 1687, Red action: do-nothing, Blue reward:0.85\n", + "step: 1688, Red action: do-nothing, Blue reward:0.85\n", + "step: 1689, Red action: do-nothing, Blue reward:0.85\n", + "step: 1690, Red action: do-nothing, Blue reward:0.85\n", + "step: 1691, Red action: do-nothing, Blue reward:0.85\n", + "step: 1692, Red action: do-nothing, Blue reward:0.85\n", + "step: 1693, Red action: do-nothing, Blue reward:0.85\n", + "step: 1694, Red action: do-nothing, Blue reward:0.85\n", + "step: 1695, Red action: do-nothing, Blue reward:0.85\n", + "step: 1696, Red action: do-nothing, Blue reward:0.85\n", + "step: 1697, Red action: do-nothing, Blue reward:0.85\n", + "step: 1698, Red action: do-nothing, Blue reward:0.85\n", + "step: 1699, Red action: do-nothing, Blue reward:0.85\n", + "step: 1700, Red action: do-nothing, Blue reward:0.85\n", + "step: 1701, Red action: do-nothing, Blue reward:0.85\n", + "step: 1702, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1703, Red action: do-nothing, Blue reward:0.85\n", + "step: 1704, Red action: do-nothing, Blue reward:0.85\n", + "step: 1705, Red action: do-nothing, Blue reward:0.85\n", + "step: 1706, Red action: do-nothing, Blue reward:0.85\n", + "step: 1707, Red action: do-nothing, Blue reward:0.85\n", + "step: 1708, Red action: do-nothing, Blue reward:0.85\n", + "step: 1709, Red action: do-nothing, Blue reward:0.85\n", + "step: 1710, Red action: do-nothing, Blue reward:0.85\n", + "step: 1711, Red action: do-nothing, Blue reward:0.85\n", + "step: 1712, Red action: do-nothing, Blue reward:0.85\n", + "step: 1713, Red action: do-nothing, Blue reward:0.85\n", + "step: 1714, Red action: do-nothing, Blue reward:0.85\n", + "step: 1715, Red action: do-nothing, Blue reward:0.85\n", + "step: 1716, Red action: do-nothing, Blue reward:0.85\n", + "step: 1717, Red action: do-nothing, Blue reward:0.85\n", + "step: 1718, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1719, Red action: do-nothing, Blue reward:0.85\n", + "step: 1720, Red action: do-nothing, Blue reward:0.85\n", + "step: 1721, Red action: do-nothing, Blue reward:0.85\n", + "step: 1722, Red action: do-nothing, Blue reward:0.85\n", + "step: 1723, Red action: do-nothing, Blue reward:0.85\n", + "step: 1724, Red action: do-nothing, Blue reward:0.85\n", + "step: 1725, Red action: do-nothing, Blue reward:0.85\n", + "step: 1726, Red action: do-nothing, Blue reward:0.85\n", + "step: 1727, Red action: do-nothing, Blue reward:0.85\n", + "step: 1728, Red action: do-nothing, Blue reward:0.85\n", + "step: 1729, Red action: do-nothing, Blue reward:0.85\n", + "step: 1730, Red action: do-nothing, Blue reward:0.85\n", + "step: 1731, Red action: do-nothing, Blue reward:0.85\n", + "step: 1732, Red action: do-nothing, Blue reward:0.85\n", + "step: 1733, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1734, Red action: do-nothing, Blue reward:0.85\n", + "step: 1735, Red action: do-nothing, Blue reward:0.85\n", + "step: 1736, Red action: do-nothing, Blue reward:0.85\n", + "step: 1737, Red action: do-nothing, Blue reward:0.85\n", + "step: 1738, Red action: do-nothing, Blue reward:0.85\n", + "step: 1739, Red action: do-nothing, Blue reward:0.85\n", + "step: 1740, Red action: do-nothing, Blue reward:0.85\n", + "step: 1741, Red action: do-nothing, Blue reward:0.85\n", + "step: 1742, Red action: do-nothing, Blue reward:0.85\n", + "step: 1743, Red action: do-nothing, Blue reward:0.85\n", + "step: 1744, Red action: do-nothing, Blue reward:0.85\n", + "step: 1745, Red action: do-nothing, Blue reward:0.85\n", + "step: 1746, Red action: do-nothing, Blue reward:0.85\n", + "step: 1747, Red action: do-nothing, Blue reward:0.85\n", + "step: 1748, Red action: do-nothing, Blue reward:0.85\n", + "step: 1749, Red action: do-nothing, Blue reward:0.85\n", + "step: 1750, Red action: do-nothing, Blue reward:0.85\n", + "step: 1751, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1752, Red action: do-nothing, Blue reward:0.85\n", + "step: 1753, Red action: do-nothing, Blue reward:0.85\n", + "step: 1754, Red action: do-nothing, Blue reward:0.85\n", + "step: 1755, Red action: do-nothing, Blue reward:0.85\n", + "step: 1756, Red action: do-nothing, Blue reward:0.85\n", + "step: 1757, Red action: do-nothing, Blue reward:0.85\n", + "step: 1758, Red action: do-nothing, Blue reward:0.85\n", + "step: 1759, Red action: do-nothing, Blue reward:0.85\n", + "step: 1760, Red action: do-nothing, Blue reward:0.85\n", + "step: 1761, Red action: do-nothing, Blue reward:0.85\n", + "step: 1762, Red action: do-nothing, Blue reward:0.85\n", + "step: 1763, Red action: do-nothing, Blue reward:0.85\n", + "step: 1764, Red action: do-nothing, Blue reward:0.85\n", + "step: 1765, Red action: do-nothing, Blue reward:0.85\n", + "step: 1766, Red action: do-nothing, Blue reward:0.85\n", + "step: 1767, Red action: do-nothing, Blue reward:0.85\n", + "step: 1768, Red action: do-nothing, Blue reward:0.85\n", + "step: 1769, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1770, Red action: do-nothing, Blue reward:0.85\n", + "step: 1771, Red action: do-nothing, Blue reward:0.85\n", + "step: 1772, Red action: do-nothing, Blue reward:0.85\n", + "step: 1773, Red action: do-nothing, Blue reward:0.85\n", + "step: 1774, Red action: do-nothing, Blue reward:0.85\n", + "step: 1775, Red action: do-nothing, Blue reward:0.85\n", + "step: 1776, Red action: do-nothing, Blue reward:0.85\n", + "step: 1777, Red action: do-nothing, Blue reward:0.85\n", + "step: 1778, Red action: do-nothing, Blue reward:0.85\n", + "step: 1779, Red action: do-nothing, Blue reward:0.85\n", + "step: 1780, Red action: do-nothing, Blue reward:0.85\n", + "step: 1781, Red action: do-nothing, Blue reward:0.85\n", + "step: 1782, Red action: do-nothing, Blue reward:0.85\n", + "step: 1783, Red action: do-nothing, Blue reward:0.85\n", + "step: 1784, Red action: do-nothing, Blue reward:0.85\n", + "step: 1785, Red action: do-nothing, Blue reward:0.85\n", + "step: 1786, Red action: do-nothing, Blue reward:0.85\n", + "step: 1787, Red action: do-nothing, Blue reward:0.85\n", + "step: 1788, Red action: do-nothing, Blue reward:0.85\n", + "step: 1789, Red action: do-nothing, Blue reward:0.85\n", + "step: 1790, Red action: do-nothing, Blue reward:0.85\n", + "step: 1791, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1792, Red action: do-nothing, Blue reward:0.85\n", + "step: 1793, Red action: do-nothing, Blue reward:0.85\n", + "step: 1794, Red action: do-nothing, Blue reward:0.85\n", + "step: 1795, Red action: do-nothing, Blue reward:0.85\n", + "step: 1796, Red action: do-nothing, Blue reward:0.85\n", + "step: 1797, Red action: do-nothing, Blue reward:0.85\n", + "step: 1798, Red action: do-nothing, Blue reward:0.85\n", + "step: 1799, Red action: do-nothing, Blue reward:0.85\n", + "step: 1800, Red action: do-nothing, Blue reward:0.85\n", + "step: 1801, Red action: do-nothing, Blue reward:0.85\n", + "step: 1802, Red action: do-nothing, Blue reward:0.85\n", + "step: 1803, Red action: do-nothing, Blue reward:0.85\n", + "step: 1804, Red action: do-nothing, Blue reward:0.85\n", + "step: 1805, Red action: do-nothing, Blue reward:0.85\n", + "step: 1806, Red action: do-nothing, Blue reward:0.85\n", + "step: 1807, Red action: do-nothing, Blue reward:0.85\n", + "step: 1808, Red action: do-nothing, Blue reward:0.85\n", + "step: 1809, Red action: do-nothing, Blue reward:0.85\n", + "step: 1810, Red action: do-nothing, Blue reward:0.85\n", + "step: 1811, Red action: do-nothing, Blue reward:0.85\n", + "step: 1812, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1813, Red action: do-nothing, Blue reward:0.85\n", + "step: 1814, Red action: do-nothing, Blue reward:0.85\n", + "step: 1815, Red action: do-nothing, Blue reward:0.85\n", + "step: 1816, Red action: do-nothing, Blue reward:0.85\n", + "step: 1817, Red action: do-nothing, Blue reward:0.85\n", + "step: 1818, Red action: do-nothing, Blue reward:0.85\n", + "step: 1819, Red action: do-nothing, Blue reward:0.85\n", + "step: 1820, Red action: do-nothing, Blue reward:0.85\n", + "step: 1821, Red action: do-nothing, Blue reward:0.85\n", + "step: 1822, Red action: do-nothing, Blue reward:0.85\n", + "step: 1823, Red action: do-nothing, Blue reward:0.85\n", + "step: 1824, Red action: do-nothing, Blue reward:0.85\n", + "step: 1825, Red action: do-nothing, Blue reward:0.85\n", + "step: 1826, Red action: do-nothing, Blue reward:0.85\n", + "step: 1827, Red action: do-nothing, Blue reward:0.85\n", + "step: 1828, Red action: do-nothing, Blue reward:0.85\n", + "step: 1829, Red action: do-nothing, Blue reward:0.85\n", + "step: 1830, Red action: do-nothing, Blue reward:0.85\n", + "step: 1831, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1832, Red action: do-nothing, Blue reward:0.85\n", + "step: 1833, Red action: do-nothing, Blue reward:0.85\n", + "step: 1834, Red action: do-nothing, Blue reward:0.85\n", + "step: 1835, Red action: do-nothing, Blue reward:0.85\n", + "step: 1836, Red action: do-nothing, Blue reward:0.85\n", + "step: 1837, Red action: do-nothing, Blue reward:0.85\n", + "step: 1838, Red action: do-nothing, Blue reward:0.85\n", + "step: 1839, Red action: do-nothing, Blue reward:0.85\n", + "step: 1840, Red action: do-nothing, Blue reward:0.85\n", + "step: 1841, Red action: do-nothing, Blue reward:0.85\n", + "step: 1842, Red action: do-nothing, Blue reward:0.85\n", + "step: 1843, Red action: do-nothing, Blue reward:0.85\n", + "step: 1844, Red action: do-nothing, Blue reward:0.85\n", + "step: 1845, Red action: do-nothing, Blue reward:0.85\n", + "step: 1846, Red action: do-nothing, Blue reward:0.85\n", + "step: 1847, Red action: do-nothing, Blue reward:0.85\n", + "step: 1848, Red action: do-nothing, Blue reward:0.85\n", + "step: 1849, Red action: do-nothing, Blue reward:0.85\n", + "step: 1850, Red action: do-nothing, Blue reward:0.85\n", + "step: 1851, Red action: do-nothing, Blue reward:0.85\n", + "step: 1852, Red action: do-nothing, Blue reward:0.85\n", + "step: 1853, Red action: do-nothing, Blue reward:0.85\n", + "step: 1854, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1855, Red action: do-nothing, Blue reward:0.85\n", + "step: 1856, Red action: do-nothing, Blue reward:0.85\n", + "step: 1857, Red action: do-nothing, Blue reward:0.85\n", + "step: 1858, Red action: do-nothing, Blue reward:0.85\n", + "step: 1859, Red action: do-nothing, Blue reward:0.85\n", + "step: 1860, Red action: do-nothing, Blue reward:0.85\n", + "step: 1861, Red action: do-nothing, Blue reward:0.85\n", + "step: 1862, Red action: do-nothing, Blue reward:0.85\n", + "step: 1863, Red action: do-nothing, Blue reward:0.85\n", + "step: 1864, Red action: do-nothing, Blue reward:0.85\n", + "step: 1865, Red action: do-nothing, Blue reward:0.85\n", + "step: 1866, Red action: do-nothing, Blue reward:0.85\n", + "step: 1867, Red action: do-nothing, Blue reward:0.85\n", + "step: 1868, Red action: do-nothing, Blue reward:0.85\n", + "step: 1869, Red action: do-nothing, Blue reward:0.85\n", + "step: 1870, Red action: do-nothing, Blue reward:0.85\n", + "step: 1871, Red action: do-nothing, Blue reward:0.85\n", + "step: 1872, Red action: do-nothing, Blue reward:0.85\n", + "step: 1873, Red action: do-nothing, Blue reward:0.85\n", + "step: 1874, Red action: do-nothing, Blue reward:0.85\n", + "step: 1875, Red action: do-nothing, Blue reward:0.85\n", + "step: 1876, Red action: do-nothing, Blue reward:0.85\n", + "step: 1877, Red action: do-nothing, Blue reward:0.85\n", + "step: 1878, Red action: do-nothing, Blue reward:0.85\n", + "step: 1879, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1880, Red action: do-nothing, Blue reward:0.85\n", + "step: 1881, Red action: do-nothing, Blue reward:0.85\n", + "step: 1882, Red action: do-nothing, Blue reward:0.85\n", + "step: 1883, Red action: do-nothing, Blue reward:0.85\n", + "step: 1884, Red action: do-nothing, Blue reward:0.85\n", + "step: 1885, Red action: do-nothing, Blue reward:0.85\n", + "step: 1886, Red action: do-nothing, Blue reward:0.85\n", + "step: 1887, Red action: do-nothing, Blue reward:0.85\n", + "step: 1888, Red action: do-nothing, Blue reward:0.85\n", + "step: 1889, Red action: do-nothing, Blue reward:0.85\n", + "step: 1890, Red action: do-nothing, Blue reward:0.85\n", + "step: 1891, Red action: do-nothing, Blue reward:0.85\n", + "step: 1892, Red action: do-nothing, Blue reward:0.85\n", + "step: 1893, Red action: do-nothing, Blue reward:0.85\n", + "step: 1894, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1895, Red action: do-nothing, Blue reward:0.85\n", + "step: 1896, Red action: do-nothing, Blue reward:0.85\n", + "step: 1897, Red action: do-nothing, Blue reward:0.85\n", + "step: 1898, Red action: do-nothing, Blue reward:0.85\n", + "step: 1899, Red action: do-nothing, Blue reward:0.85\n", + "step: 1900, Red action: do-nothing, Blue reward:0.85\n", + "step: 1901, Red action: do-nothing, Blue reward:0.85\n", + "step: 1902, Red action: do-nothing, Blue reward:0.85\n", + "step: 1903, Red action: do-nothing, Blue reward:0.85\n", + "step: 1904, Red action: do-nothing, Blue reward:0.85\n", + "step: 1905, Red action: do-nothing, Blue reward:0.85\n", + "step: 1906, Red action: do-nothing, Blue reward:0.85\n", + "step: 1907, Red action: do-nothing, Blue reward:0.85\n", + "step: 1908, Red action: do-nothing, Blue reward:0.85\n", + "step: 1909, Red action: do-nothing, Blue reward:0.85\n", + "step: 1910, Red action: do-nothing, Blue reward:0.85\n", + "step: 1911, Red action: do-nothing, Blue reward:0.85\n", + "step: 1912, Red action: do-nothing, Blue reward:0.85\n", + "step: 1913, Red action: do-nothing, Blue reward:0.85\n", + "step: 1914, Red action: do-nothing, Blue reward:0.85\n", + "step: 1915, Red action: do-nothing, Blue reward:0.85\n", + "step: 1916, Red action: do-nothing, Blue reward:0.85\n", + "step: 1917, Red action: do-nothing, Blue reward:0.85\n", + "step: 1918, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1919, Red action: do-nothing, Blue reward:0.85\n", + "step: 1920, Red action: do-nothing, Blue reward:0.85\n", + "step: 1921, Red action: do-nothing, Blue reward:0.85\n", + "step: 1922, Red action: do-nothing, Blue reward:0.85\n", + "step: 1923, Red action: do-nothing, Blue reward:0.85\n", + "step: 1924, Red action: do-nothing, Blue reward:0.85\n", + "step: 1925, Red action: do-nothing, Blue reward:0.85\n", + "step: 1926, Red action: do-nothing, Blue reward:0.85\n", + "step: 1927, Red action: do-nothing, Blue reward:0.85\n", + "step: 1928, Red action: do-nothing, Blue reward:0.85\n", + "step: 1929, Red action: do-nothing, Blue reward:0.85\n", + "step: 1930, Red action: do-nothing, Blue reward:0.85\n", + "step: 1931, Red action: do-nothing, Blue reward:0.85\n", + "step: 1932, Red action: do-nothing, Blue reward:0.85\n", + "step: 1933, Red action: do-nothing, Blue reward:0.85\n", + "step: 1934, Red action: do-nothing, Blue reward:0.85\n", + "step: 1935, Red action: do-nothing, Blue reward:0.85\n", + "step: 1936, Red action: do-nothing, Blue reward:0.85\n", + "step: 1937, Red action: do-nothing, Blue reward:0.85\n", + "step: 1938, Red action: do-nothing, Blue reward:0.85\n", + "step: 1939, Red action: do-nothing, Blue reward:0.85\n", + "step: 1940, Red action: do-nothing, Blue reward:0.85\n", + "step: 1941, Red action: do-nothing, Blue reward:0.85\n", + "step: 1942, Red action: do-nothing, Blue reward:0.85\n", + "step: 1943, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1944, Red action: do-nothing, Blue reward:0.85\n", + "step: 1945, Red action: do-nothing, Blue reward:0.85\n", + "step: 1946, Red action: do-nothing, Blue reward:0.85\n", + "step: 1947, Red action: do-nothing, Blue reward:0.85\n", + "step: 1948, Red action: do-nothing, Blue reward:0.85\n", + "step: 1949, Red action: do-nothing, Blue reward:0.85\n", + "step: 1950, Red action: do-nothing, Blue reward:0.85\n", + "step: 1951, Red action: do-nothing, Blue reward:0.85\n", + "step: 1952, Red action: do-nothing, Blue reward:0.85\n", + "step: 1953, Red action: do-nothing, Blue reward:0.85\n", + "step: 1954, Red action: do-nothing, Blue reward:0.85\n", + "step: 1955, Red action: do-nothing, Blue reward:0.85\n", + "step: 1956, Red action: do-nothing, Blue reward:0.85\n", + "step: 1957, Red action: do-nothing, Blue reward:0.85\n", + "step: 1958, Red action: do-nothing, Blue reward:0.85\n", + "step: 1959, Red action: do-nothing, Blue reward:0.85\n", + "step: 1960, Red action: do-nothing, Blue reward:0.85\n", + "step: 1961, Red action: do-nothing, Blue reward:0.85\n", + "step: 1962, Red action: do-nothing, Blue reward:0.85\n", + "step: 1963, Red action: do-nothing, Blue reward:0.85\n", + "step: 1964, Red action: do-nothing, Blue reward:0.85\n", + "step: 1965, Red action: do-nothing, Blue reward:0.85\n", + "step: 1966, Red action: do-nothing, Blue reward:0.85\n", + "step: 1967, Red action: do-nothing, Blue reward:0.85\n", + "step: 1968, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1969, Red action: do-nothing, Blue reward:0.85\n", + "step: 1970, Red action: do-nothing, Blue reward:0.85\n", + "step: 1971, Red action: do-nothing, Blue reward:0.85\n", + "step: 1972, Red action: do-nothing, Blue reward:0.85\n", + "step: 1973, Red action: do-nothing, Blue reward:0.85\n", + "step: 1974, Red action: do-nothing, Blue reward:0.85\n", + "step: 1975, Red action: do-nothing, Blue reward:0.85\n", + "step: 1976, Red action: do-nothing, Blue reward:0.85\n", + "step: 1977, Red action: do-nothing, Blue reward:0.85\n", + "step: 1978, Red action: do-nothing, Blue reward:0.85\n", + "step: 1979, Red action: do-nothing, Blue reward:0.85\n", + "step: 1980, Red action: do-nothing, Blue reward:0.85\n", + "step: 1981, Red action: do-nothing, Blue reward:0.85\n", + "step: 1982, Red action: do-nothing, Blue reward:0.85\n", + "step: 1983, Red action: do-nothing, Blue reward:0.85\n", + "step: 1984, Red action: do-nothing, Blue reward:0.85\n", + "step: 1985, Red action: do-nothing, Blue reward:0.85\n", + "step: 1986, Red action: do-nothing, Blue reward:0.85\n", + "step: 1987, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1988, Red action: do-nothing, Blue reward:0.85\n", + "step: 1989, Red action: do-nothing, Blue reward:0.85\n", + "step: 1990, Red action: do-nothing, Blue reward:0.85\n", + "step: 1991, Red action: do-nothing, Blue reward:0.85\n", + "step: 1992, Red action: do-nothing, Blue reward:0.85\n", + "step: 1993, Red action: do-nothing, Blue reward:0.85\n", + "step: 1994, Red action: do-nothing, Blue reward:0.85\n", + "step: 1995, Red action: do-nothing, Blue reward:0.85\n", + "step: 1996, Red action: do-nothing, Blue reward:0.85\n", + "step: 1997, Red action: do-nothing, Blue reward:0.85\n", + "step: 1998, Red action: do-nothing, Blue reward:0.85\n", + "step: 1999, Red action: do-nothing, Blue reward:0.85\n", + "step: 2000, Red action: do-nothing, Blue reward:0.85\n", + "step: 2001, Red action: do-nothing, Blue reward:0.85\n", + "step: 2002, Red action: do-nothing, Blue reward:0.85\n", + "step: 2003, Red action: do-nothing, Blue reward:0.85\n", + "step: 2004, Red action: do-nothing, Blue reward:0.85\n", + "step: 2005, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2006, Red action: do-nothing, Blue reward:0.85\n", + "step: 2007, Red action: do-nothing, Blue reward:0.85\n", + "step: 2008, Red action: do-nothing, Blue reward:0.85\n", + "step: 2009, Red action: do-nothing, Blue reward:0.85\n", + "step: 2010, Red action: do-nothing, Blue reward:0.85\n", + "step: 2011, Red action: do-nothing, Blue reward:0.85\n", + "step: 2012, Red action: do-nothing, Blue reward:0.85\n", + "step: 2013, Red action: do-nothing, Blue reward:0.85\n", + "step: 2014, Red action: do-nothing, Blue reward:0.85\n", + "step: 2015, Red action: do-nothing, Blue reward:0.85\n", + "step: 2016, Red action: do-nothing, Blue reward:0.85\n", + "step: 2017, Red action: do-nothing, Blue reward:0.85\n", + "step: 2018, Red action: do-nothing, Blue reward:0.85\n", + "step: 2019, Red action: do-nothing, Blue reward:0.85\n", + "step: 2020, Red action: do-nothing, Blue reward:0.85\n", + "step: 2021, Red action: do-nothing, Blue reward:0.85\n", + "step: 2022, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2023, Red action: do-nothing, Blue reward:0.85\n", + "step: 2024, Red action: do-nothing, Blue reward:0.85\n", + "step: 2025, Red action: do-nothing, Blue reward:0.85\n", + "step: 2026, Red action: do-nothing, Blue reward:0.85\n", + "step: 2027, Red action: do-nothing, Blue reward:0.85\n", + "step: 2028, Red action: do-nothing, Blue reward:0.85\n", + "step: 2029, Red action: do-nothing, Blue reward:0.85\n", + "step: 2030, Red action: do-nothing, Blue reward:0.85\n", + "step: 2031, Red action: do-nothing, Blue reward:0.85\n", + "step: 2032, Red action: do-nothing, Blue reward:0.85\n", + "step: 2033, Red action: do-nothing, Blue reward:0.85\n", + "step: 2034, Red action: do-nothing, Blue reward:0.85\n", + "step: 2035, Red action: do-nothing, Blue reward:0.85\n", + "step: 2036, Red action: do-nothing, Blue reward:0.85\n", + "step: 2037, Red action: do-nothing, Blue reward:0.85\n", + "step: 2038, Red action: do-nothing, Blue reward:0.85\n", + "step: 2039, Red action: do-nothing, Blue reward:0.85\n", + "step: 2040, Red action: do-nothing, Blue reward:0.85\n", + "step: 2041, Red action: do-nothing, Blue reward:0.85\n", + "step: 2042, Red action: do-nothing, Blue reward:0.85\n", + "step: 2043, Red action: do-nothing, Blue reward:0.85\n", + "step: 2044, Red action: do-nothing, Blue reward:0.85\n", + "step: 2045, Red action: do-nothing, Blue reward:0.85\n", + "step: 2046, Red action: do-nothing, Blue reward:0.85\n", + "step: 2047, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2048, Red action: do-nothing, Blue reward:0.85\n", + "step: 2049, Red action: do-nothing, Blue reward:0.85\n", + "step: 2050, Red action: do-nothing, Blue reward:0.85\n", + "step: 2051, Red action: do-nothing, Blue reward:0.85\n", + "step: 2052, Red action: do-nothing, Blue reward:0.85\n", + "step: 2053, Red action: do-nothing, Blue reward:0.85\n", + "step: 2054, Red action: do-nothing, Blue reward:0.85\n", + "step: 2055, Red action: do-nothing, Blue reward:0.85\n", + "step: 2056, Red action: do-nothing, Blue reward:0.85\n", + "step: 2057, Red action: do-nothing, Blue reward:0.85\n", + "step: 2058, Red action: do-nothing, Blue reward:0.85\n", + "step: 2059, Red action: do-nothing, Blue reward:0.85\n", + "step: 2060, Red action: do-nothing, Blue reward:0.85\n", + "step: 2061, Red action: do-nothing, Blue reward:0.85\n", + "step: 2062, Red action: do-nothing, Blue reward:0.85\n", + "step: 2063, Red action: do-nothing, Blue reward:0.85\n", + "step: 2064, Red action: do-nothing, Blue reward:0.85\n", + "step: 2065, Red action: do-nothing, Blue reward:0.85\n", + "step: 2066, Red action: do-nothing, Blue reward:0.85\n", + "step: 2067, Red action: do-nothing, Blue reward:0.85\n", + "step: 2068, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2069, Red action: do-nothing, Blue reward:0.85\n", + "step: 2070, Red action: do-nothing, Blue reward:0.85\n", + "step: 2071, Red action: do-nothing, Blue reward:0.85\n", + "step: 2072, Red action: do-nothing, Blue reward:0.85\n", + "step: 2073, Red action: do-nothing, Blue reward:0.85\n", + "step: 2074, Red action: do-nothing, Blue reward:0.85\n", + "step: 2075, Red action: do-nothing, Blue reward:0.85\n", + "step: 2076, Red action: do-nothing, Blue reward:0.85\n", + "step: 2077, Red action: do-nothing, Blue reward:0.85\n", + "step: 2078, Red action: do-nothing, Blue reward:0.85\n", + "step: 2079, Red action: do-nothing, Blue reward:0.85\n", + "step: 2080, Red action: do-nothing, Blue reward:0.85\n", + "step: 2081, Red action: do-nothing, Blue reward:0.85\n", + "step: 2082, Red action: do-nothing, Blue reward:0.85\n", + "step: 2083, Red action: do-nothing, Blue reward:0.85\n", + "step: 2084, Red action: do-nothing, Blue reward:0.85\n", + "step: 2085, Red action: do-nothing, Blue reward:0.85\n", + "step: 2086, Red action: do-nothing, Blue reward:0.85\n", + "step: 2087, Red action: do-nothing, Blue reward:0.85\n", + "step: 2088, Red action: do-nothing, Blue reward:0.85\n", + "step: 2089, Red action: do-nothing, Blue reward:0.85\n", + "step: 2090, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2091, Red action: do-nothing, Blue reward:0.85\n", + "step: 2092, Red action: do-nothing, Blue reward:0.85\n", + "step: 2093, Red action: do-nothing, Blue reward:0.85\n", + "step: 2094, Red action: do-nothing, Blue reward:0.85\n", + "step: 2095, Red action: do-nothing, Blue reward:0.85\n", + "step: 2096, Red action: do-nothing, Blue reward:0.85\n", + "step: 2097, Red action: do-nothing, Blue reward:0.85\n", + "step: 2098, Red action: do-nothing, Blue reward:0.85\n", + "step: 2099, Red action: do-nothing, Blue reward:0.85\n", + "step: 2100, Red action: do-nothing, Blue reward:0.85\n", + "step: 2101, Red action: do-nothing, Blue reward:0.85\n", + "step: 2102, Red action: do-nothing, Blue reward:0.85\n", + "step: 2103, Red action: do-nothing, Blue reward:0.85\n", + "step: 2104, Red action: do-nothing, Blue reward:0.85\n", + "step: 2105, Red action: do-nothing, Blue reward:0.85\n", + "step: 2106, Red action: do-nothing, Blue reward:0.85\n", + "step: 2107, Red action: do-nothing, Blue reward:0.85\n", + "step: 2108, Red action: do-nothing, Blue reward:0.85\n", + "step: 2109, Red action: do-nothing, Blue reward:0.85\n", + "step: 2110, Red action: do-nothing, Blue reward:0.85\n", + "step: 2111, Red action: do-nothing, Blue reward:0.85\n", + "step: 2112, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2113, Red action: do-nothing, Blue reward:0.85\n", + "step: 2114, Red action: do-nothing, Blue reward:0.85\n", + "step: 2115, Red action: do-nothing, Blue reward:0.85\n", + "step: 2116, Red action: do-nothing, Blue reward:0.85\n", + "step: 2117, Red action: do-nothing, Blue reward:0.85\n", + "step: 2118, Red action: do-nothing, Blue reward:0.85\n", + "step: 2119, Red action: do-nothing, Blue reward:0.85\n", + "step: 2120, Red action: do-nothing, Blue reward:0.85\n", + "step: 2121, Red action: do-nothing, Blue reward:0.85\n", + "step: 2122, Red action: do-nothing, Blue reward:0.85\n", + "step: 2123, Red action: do-nothing, Blue reward:0.85\n", + "step: 2124, Red action: do-nothing, Blue reward:0.85\n", + "step: 2125, Red action: do-nothing, Blue reward:0.85\n", + "step: 2126, Red action: do-nothing, Blue reward:0.85\n", + "step: 2127, Red action: do-nothing, Blue reward:0.85\n", + "step: 2128, Red action: do-nothing, Blue reward:0.85\n", + "step: 2129, Red action: do-nothing, Blue reward:0.85\n", + "step: 2130, Red action: do-nothing, Blue reward:0.85\n", + "step: 2131, Red action: do-nothing, Blue reward:0.85\n", + "step: 2132, Red action: do-nothing, Blue reward:0.85\n", + "step: 2133, Red action: do-nothing, Blue reward:0.85\n", + "step: 2134, Red action: do-nothing, Blue reward:0.85\n", + "step: 2135, Red action: do-nothing, Blue reward:0.85\n", + "step: 2136, Red action: do-nothing, Blue reward:0.85\n", + "step: 2137, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2138, Red action: do-nothing, Blue reward:0.85\n", + "step: 2139, Red action: do-nothing, Blue reward:0.85\n", + "step: 2140, Red action: do-nothing, Blue reward:0.85\n", + "step: 2141, Red action: do-nothing, Blue reward:0.85\n", + "step: 2142, Red action: do-nothing, Blue reward:0.85\n", + "step: 2143, Red action: do-nothing, Blue reward:0.85\n", + "step: 2144, Red action: do-nothing, Blue reward:0.85\n", + "step: 2145, Red action: do-nothing, Blue reward:0.85\n", + "step: 2146, Red action: do-nothing, Blue reward:0.85\n", + "step: 2147, Red action: do-nothing, Blue reward:0.85\n", + "step: 2148, Red action: do-nothing, Blue reward:0.85\n", + "step: 2149, Red action: do-nothing, Blue reward:0.85\n", + "step: 2150, Red action: do-nothing, Blue reward:0.85\n", + "step: 2151, Red action: do-nothing, Blue reward:0.85\n", + "step: 2152, Red action: do-nothing, Blue reward:0.85\n", + "step: 2153, Red action: do-nothing, Blue reward:0.85\n", + "step: 2154, Red action: do-nothing, Blue reward:0.85\n", + "step: 2155, Red action: do-nothing, Blue reward:0.85\n", + "step: 2156, Red action: do-nothing, Blue reward:0.85\n", + "step: 2157, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2158, Red action: do-nothing, Blue reward:0.85\n", + "step: 2159, Red action: do-nothing, Blue reward:0.85\n", + "step: 2160, Red action: do-nothing, Blue reward:0.85\n", + "step: 2161, Red action: do-nothing, Blue reward:0.85\n", + "step: 2162, Red action: do-nothing, Blue reward:0.85\n", + "step: 2163, Red action: do-nothing, Blue reward:0.85\n", + "step: 2164, Red action: do-nothing, Blue reward:0.85\n", + "step: 2165, Red action: do-nothing, Blue reward:0.85\n", + "step: 2166, Red action: do-nothing, Blue reward:0.85\n", + "step: 2167, Red action: do-nothing, Blue reward:0.85\n", + "step: 2168, Red action: do-nothing, Blue reward:0.85\n", + "step: 2169, Red action: do-nothing, Blue reward:0.85\n", + "step: 2170, Red action: do-nothing, Blue reward:0.85\n", + "step: 2171, Red action: do-nothing, Blue reward:0.85\n", + "step: 2172, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2173, Red action: do-nothing, Blue reward:0.85\n", + "step: 2174, Red action: do-nothing, Blue reward:0.85\n", + "step: 2175, Red action: do-nothing, Blue reward:0.85\n", + "step: 2176, Red action: do-nothing, Blue reward:0.85\n", + "step: 2177, Red action: do-nothing, Blue reward:0.85\n", + "step: 2178, Red action: do-nothing, Blue reward:0.85\n", + "step: 2179, Red action: do-nothing, Blue reward:0.85\n", + "step: 2180, Red action: do-nothing, Blue reward:0.85\n", + "step: 2181, Red action: do-nothing, Blue reward:0.85\n", + "step: 2182, Red action: do-nothing, Blue reward:0.85\n", + "step: 2183, Red action: do-nothing, Blue reward:0.85\n", + "step: 2184, Red action: do-nothing, Blue reward:0.85\n", + "step: 2185, Red action: do-nothing, Blue reward:0.85\n", + "step: 2186, Red action: do-nothing, Blue reward:0.85\n", + "step: 2187, Red action: do-nothing, Blue reward:0.85\n", + "step: 2188, Red action: do-nothing, Blue reward:0.85\n", + "step: 2189, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2190, Red action: do-nothing, Blue reward:0.85\n", + "step: 2191, Red action: do-nothing, Blue reward:0.85\n", + "step: 2192, Red action: do-nothing, Blue reward:0.85\n", + "step: 2193, Red action: do-nothing, Blue reward:0.85\n", + "step: 2194, Red action: do-nothing, Blue reward:0.85\n", + "step: 2195, Red action: do-nothing, Blue reward:0.85\n", + "step: 2196, Red action: do-nothing, Blue reward:0.85\n", + "step: 2197, Red action: do-nothing, Blue reward:0.85\n", + "step: 2198, Red action: do-nothing, Blue reward:0.85\n", + "step: 2199, Red action: do-nothing, Blue reward:0.85\n", + "step: 2200, Red action: do-nothing, Blue reward:0.85\n", + "step: 2201, Red action: do-nothing, Blue reward:0.85\n", + "step: 2202, Red action: do-nothing, Blue reward:0.85\n", + "step: 2203, Red action: do-nothing, Blue reward:0.85\n", + "step: 2204, Red action: do-nothing, Blue reward:0.85\n", + "step: 2205, Red action: do-nothing, Blue reward:0.85\n", + "step: 2206, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2207, Red action: do-nothing, Blue reward:0.85\n", + "step: 2208, Red action: do-nothing, Blue reward:0.85\n", + "step: 2209, Red action: do-nothing, Blue reward:0.85\n", + "step: 2210, Red action: do-nothing, Blue reward:0.85\n", + "step: 2211, Red action: do-nothing, Blue reward:0.85\n", + "step: 2212, Red action: do-nothing, Blue reward:0.85\n", + "step: 2213, Red action: do-nothing, Blue reward:0.85\n", + "step: 2214, Red action: do-nothing, Blue reward:0.85\n", + "step: 2215, Red action: do-nothing, Blue reward:0.85\n", + "step: 2216, Red action: do-nothing, Blue reward:0.85\n", + "step: 2217, Red action: do-nothing, Blue reward:0.85\n", + "step: 2218, Red action: do-nothing, Blue reward:0.85\n", + "step: 2219, Red action: do-nothing, Blue reward:0.85\n", + "step: 2220, Red action: do-nothing, Blue reward:0.85\n", + "step: 2221, Red action: do-nothing, Blue reward:0.85\n", + "step: 2222, Red action: do-nothing, Blue reward:0.85\n", + "step: 2223, Red action: do-nothing, Blue reward:0.85\n", + "step: 2224, Red action: do-nothing, Blue reward:0.85\n", + "step: 2225, Red action: do-nothing, Blue reward:0.85\n", + "step: 2226, Red action: do-nothing, Blue reward:0.85\n", + "step: 2227, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2228, Red action: do-nothing, Blue reward:0.85\n", + "step: 2229, Red action: do-nothing, Blue reward:0.85\n", + "step: 2230, Red action: do-nothing, Blue reward:0.85\n", + "step: 2231, Red action: do-nothing, Blue reward:0.85\n", + "step: 2232, Red action: do-nothing, Blue reward:0.85\n", + "step: 2233, Red action: do-nothing, Blue reward:0.85\n", + "step: 2234, Red action: do-nothing, Blue reward:0.85\n", + "step: 2235, Red action: do-nothing, Blue reward:0.85\n", + "step: 2236, Red action: do-nothing, Blue reward:0.85\n", + "step: 2237, Red action: do-nothing, Blue reward:0.85\n", + "step: 2238, Red action: do-nothing, Blue reward:0.85\n", + "step: 2239, Red action: do-nothing, Blue reward:0.85\n", + "step: 2240, Red action: do-nothing, Blue reward:0.85\n", + "step: 2241, Red action: do-nothing, Blue reward:0.85\n", + "step: 2242, Red action: do-nothing, Blue reward:0.85\n", + "step: 2243, Red action: do-nothing, Blue reward:0.85\n", + "step: 2244, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2245, Red action: do-nothing, Blue reward:0.85\n", + "step: 2246, Red action: do-nothing, Blue reward:0.85\n", + "step: 2247, Red action: do-nothing, Blue reward:0.85\n", + "step: 2248, Red action: do-nothing, Blue reward:0.85\n", + "step: 2249, Red action: do-nothing, Blue reward:0.85\n", + "step: 2250, Red action: do-nothing, Blue reward:0.85\n", + "step: 2251, Red action: do-nothing, Blue reward:0.85\n", + "step: 2252, Red action: do-nothing, Blue reward:0.85\n", + "step: 2253, Red action: do-nothing, Blue reward:0.85\n", + "step: 2254, Red action: do-nothing, Blue reward:0.85\n", + "step: 2255, Red action: do-nothing, Blue reward:0.85\n", + "step: 2256, Red action: do-nothing, Blue reward:0.85\n", + "step: 2257, Red action: do-nothing, Blue reward:0.85\n", + "step: 2258, Red action: do-nothing, Blue reward:0.85\n", + "step: 2259, Red action: do-nothing, Blue reward:0.85\n", + "step: 2260, Red action: do-nothing, Blue reward:0.85\n", + "step: 2261, Red action: do-nothing, Blue reward:0.85\n", + "step: 2262, Red action: do-nothing, Blue reward:0.85\n", + "step: 2263, Red action: do-nothing, Blue reward:0.85\n", + "step: 2264, Red action: do-nothing, Blue reward:0.85\n", + "step: 2265, Red action: do-nothing, Blue reward:0.85\n", + "step: 2266, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2267, Red action: do-nothing, Blue reward:0.85\n", + "step: 2268, Red action: do-nothing, Blue reward:0.85\n", + "step: 2269, Red action: do-nothing, Blue reward:0.85\n", + "step: 2270, Red action: do-nothing, Blue reward:0.85\n", + "step: 2271, Red action: do-nothing, Blue reward:0.85\n", + "step: 2272, Red action: do-nothing, Blue reward:0.85\n", + "step: 2273, Red action: do-nothing, Blue reward:0.85\n", + "step: 2274, Red action: do-nothing, Blue reward:0.85\n", + "step: 2275, Red action: do-nothing, Blue reward:0.85\n", + "step: 2276, Red action: do-nothing, Blue reward:0.85\n", + "step: 2277, Red action: do-nothing, Blue reward:0.85\n", + "step: 2278, Red action: do-nothing, Blue reward:0.85\n", + "step: 2279, Red action: do-nothing, Blue reward:0.85\n", + "step: 2280, Red action: do-nothing, Blue reward:0.85\n", + "step: 2281, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2282, Red action: do-nothing, Blue reward:0.85\n", + "step: 2283, Red action: do-nothing, Blue reward:0.85\n", + "step: 2284, Red action: do-nothing, Blue reward:0.85\n", + "step: 2285, Red action: do-nothing, Blue reward:0.85\n", + "step: 2286, Red action: do-nothing, Blue reward:0.85\n", + "step: 2287, Red action: do-nothing, Blue reward:0.85\n", + "step: 2288, Red action: do-nothing, Blue reward:0.85\n", + "step: 2289, Red action: do-nothing, Blue reward:0.85\n", + "step: 2290, Red action: do-nothing, Blue reward:0.85\n", + "step: 2291, Red action: do-nothing, Blue reward:0.85\n", + "step: 2292, Red action: do-nothing, Blue reward:0.85\n", + "step: 2293, Red action: do-nothing, Blue reward:0.85\n", + "step: 2294, Red action: do-nothing, Blue reward:0.85\n", + "step: 2295, Red action: do-nothing, Blue reward:0.85\n", + "step: 2296, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2297, Red action: do-nothing, Blue reward:0.85\n", + "step: 2298, Red action: do-nothing, Blue reward:0.85\n", + "step: 2299, Red action: do-nothing, Blue reward:0.85\n", + "step: 2300, Red action: do-nothing, Blue reward:0.85\n", + "step: 2301, Red action: do-nothing, Blue reward:0.85\n", + "step: 2302, Red action: do-nothing, Blue reward:0.85\n", + "step: 2303, Red action: do-nothing, Blue reward:0.85\n", + "step: 2304, Red action: do-nothing, Blue reward:0.85\n", + "step: 2305, Red action: do-nothing, Blue reward:0.85\n", + "step: 2306, Red action: do-nothing, Blue reward:0.85\n", + "step: 2307, Red action: do-nothing, Blue reward:0.85\n", + "step: 2308, Red action: do-nothing, Blue reward:0.85\n", + "step: 2309, Red action: do-nothing, Blue reward:0.85\n", + "step: 2310, Red action: do-nothing, Blue reward:0.85\n", + "step: 2311, Red action: do-nothing, Blue reward:0.85\n", + "step: 2312, Red action: do-nothing, Blue reward:0.85\n", + "step: 2313, Red action: do-nothing, Blue reward:0.85\n", + "step: 2314, Red action: do-nothing, Blue reward:0.85\n", + "step: 2315, Red action: do-nothing, Blue reward:0.85\n", + "step: 2316, Red action: do-nothing, Blue reward:0.85\n", + "step: 2317, Red action: do-nothing, Blue reward:0.85\n", + "step: 2318, Red action: do-nothing, Blue reward:0.85\n", + "step: 2319, Red action: do-nothing, Blue reward:0.85\n", + "step: 2320, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2321, Red action: do-nothing, Blue reward:0.85\n", + "step: 2322, Red action: do-nothing, Blue reward:0.85\n", + "step: 2323, Red action: do-nothing, Blue reward:0.85\n", + "step: 2324, Red action: do-nothing, Blue reward:0.85\n", + "step: 2325, Red action: do-nothing, Blue reward:0.85\n", + "step: 2326, Red action: do-nothing, Blue reward:0.85\n", + "step: 2327, Red action: do-nothing, Blue reward:0.85\n", + "step: 2328, Red action: do-nothing, Blue reward:0.85\n", + "step: 2329, Red action: do-nothing, Blue reward:0.85\n", + "step: 2330, Red action: do-nothing, Blue reward:0.85\n", + "step: 2331, Red action: do-nothing, Blue reward:0.85\n", + "step: 2332, Red action: do-nothing, Blue reward:0.85\n", + "step: 2333, Red action: do-nothing, Blue reward:0.85\n", + "step: 2334, Red action: do-nothing, Blue reward:0.85\n", + "step: 2335, Red action: do-nothing, Blue reward:0.85\n", + "step: 2336, Red action: do-nothing, Blue reward:0.85\n", + "step: 2337, Red action: do-nothing, Blue reward:0.85\n", + "step: 2338, Red action: do-nothing, Blue reward:0.85\n", + "step: 2339, Red action: do-nothing, Blue reward:0.85\n", + "step: 2340, Red action: do-nothing, Blue reward:0.85\n", + "step: 2341, Red action: do-nothing, Blue reward:0.85\n", + "step: 2342, Red action: do-nothing, Blue reward:0.85\n", + "step: 2343, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2344, Red action: do-nothing, Blue reward:0.85\n", + "step: 2345, Red action: do-nothing, Blue reward:0.85\n", + "step: 2346, Red action: do-nothing, Blue reward:0.85\n", + "step: 2347, Red action: do-nothing, Blue reward:0.85\n", + "step: 2348, Red action: do-nothing, Blue reward:0.85\n", + "step: 2349, Red action: do-nothing, Blue reward:0.85\n", + "step: 2350, Red action: do-nothing, Blue reward:0.85\n", + "step: 2351, Red action: do-nothing, Blue reward:0.85\n", + "step: 2352, Red action: do-nothing, Blue reward:0.85\n", + "step: 2353, Red action: do-nothing, Blue reward:0.85\n", + "step: 2354, Red action: do-nothing, Blue reward:0.85\n", + "step: 2355, Red action: do-nothing, Blue reward:0.85\n", + "step: 2356, Red action: do-nothing, Blue reward:0.85\n", + "step: 2357, Red action: do-nothing, Blue reward:0.85\n", + "step: 2358, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2359, Red action: do-nothing, Blue reward:0.85\n", + "step: 2360, Red action: do-nothing, Blue reward:0.85\n", + "step: 2361, Red action: do-nothing, Blue reward:0.85\n", + "step: 2362, Red action: do-nothing, Blue reward:0.85\n", + "step: 2363, Red action: do-nothing, Blue reward:0.85\n", + "step: 2364, Red action: do-nothing, Blue reward:0.85\n", + "step: 2365, Red action: do-nothing, Blue reward:0.85\n", + "step: 2366, Red action: do-nothing, Blue reward:0.85\n", + "step: 2367, Red action: do-nothing, Blue reward:0.85\n", + "step: 2368, Red action: do-nothing, Blue reward:0.85\n", + "step: 2369, Red action: do-nothing, Blue reward:0.85\n", + "step: 2370, Red action: do-nothing, Blue reward:0.85\n", + "step: 2371, Red action: do-nothing, Blue reward:0.85\n", + "step: 2372, Red action: do-nothing, Blue reward:0.85\n", + "step: 2373, Red action: do-nothing, Blue reward:0.85\n", + "step: 2374, Red action: do-nothing, Blue reward:0.85\n", + "step: 2375, Red action: do-nothing, Blue reward:0.85\n", + "step: 2376, Red action: do-nothing, Blue reward:0.85\n", + "step: 2377, Red action: do-nothing, Blue reward:0.85\n", + "step: 2378, Red action: do-nothing, Blue reward:0.85\n", + "step: 2379, Red action: do-nothing, Blue reward:0.85\n", + "step: 2380, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2381, Red action: do-nothing, Blue reward:0.85\n", + "step: 2382, Red action: do-nothing, Blue reward:0.85\n", + "step: 2383, Red action: do-nothing, Blue reward:0.85\n", + "step: 2384, Red action: do-nothing, Blue reward:0.85\n", + "step: 2385, Red action: do-nothing, Blue reward:0.85\n", + "step: 2386, Red action: do-nothing, Blue reward:0.85\n", + "step: 2387, Red action: do-nothing, Blue reward:0.85\n", + "step: 2388, Red action: do-nothing, Blue reward:0.85\n", + "step: 2389, Red action: do-nothing, Blue reward:0.85\n", + "step: 2390, Red action: do-nothing, Blue reward:0.85\n", + "step: 2391, Red action: do-nothing, Blue reward:0.85\n", + "step: 2392, Red action: do-nothing, Blue reward:0.85\n", + "step: 2393, Red action: do-nothing, Blue reward:0.85\n", + "step: 2394, Red action: do-nothing, Blue reward:0.85\n", + "step: 2395, Red action: do-nothing, Blue reward:0.85\n", + "step: 2396, Red action: do-nothing, Blue reward:0.85\n", + "step: 2397, Red action: do-nothing, Blue reward:0.85\n", + "step: 2398, Red action: do-nothing, Blue reward:0.85\n", + "step: 2399, Red action: do-nothing, Blue reward:0.85\n", + "step: 2400, Red action: do-nothing, Blue reward:0.85\n", + "step: 2401, Red action: do-nothing, Blue reward:0.85\n", + "step: 2402, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2403, Red action: do-nothing, Blue reward:0.85\n", + "step: 2404, Red action: do-nothing, Blue reward:0.85\n", + "step: 2405, Red action: do-nothing, Blue reward:0.85\n", + "step: 2406, Red action: do-nothing, Blue reward:0.85\n", + "step: 2407, Red action: do-nothing, Blue reward:0.85\n", + "step: 2408, Red action: do-nothing, Blue reward:0.85\n", + "step: 2409, Red action: do-nothing, Blue reward:0.85\n", + "step: 2410, Red action: do-nothing, Blue reward:0.85\n", + "step: 2411, Red action: do-nothing, Blue reward:0.85\n", + "step: 2412, Red action: do-nothing, Blue reward:0.85\n", + "step: 2413, Red action: do-nothing, Blue reward:0.85\n", + "step: 2414, Red action: do-nothing, Blue reward:0.85\n", + "step: 2415, Red action: do-nothing, Blue reward:0.85\n", + "step: 2416, Red action: do-nothing, Blue reward:0.85\n", + "step: 2417, Red action: do-nothing, Blue reward:0.85\n", + "step: 2418, Red action: do-nothing, Blue reward:0.85\n", + "step: 2419, Red action: do-nothing, Blue reward:0.85\n", + "step: 2420, Red action: do-nothing, Blue reward:0.85\n", + "step: 2421, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2422, Red action: do-nothing, Blue reward:0.85\n", + "step: 2423, Red action: do-nothing, Blue reward:0.85\n", + "step: 2424, Red action: do-nothing, Blue reward:0.85\n", + "step: 2425, Red action: do-nothing, Blue reward:0.85\n", + "step: 2426, Red action: do-nothing, Blue reward:0.85\n", + "step: 2427, Red action: do-nothing, Blue reward:0.85\n", + "step: 2428, Red action: do-nothing, Blue reward:0.85\n", + "step: 2429, Red action: do-nothing, Blue reward:0.85\n", + "step: 2430, Red action: do-nothing, Blue reward:0.85\n", + "step: 2431, Red action: do-nothing, Blue reward:0.85\n", + "step: 2432, Red action: do-nothing, Blue reward:0.85\n", + "step: 2433, Red action: do-nothing, Blue reward:0.85\n", + "step: 2434, Red action: do-nothing, Blue reward:0.85\n", + "step: 2435, Red action: do-nothing, Blue reward:0.85\n", + "step: 2436, Red action: do-nothing, Blue reward:0.85\n", + "step: 2437, Red action: do-nothing, Blue reward:0.85\n", + "step: 2438, Red action: do-nothing, Blue reward:0.85\n", + "step: 2439, Red action: do-nothing, Blue reward:0.85\n", + "step: 2440, Red action: do-nothing, Blue reward:0.85\n", + "step: 2441, Red action: do-nothing, Blue reward:0.85\n", + "step: 2442, Red action: do-nothing, Blue reward:0.85\n", + "step: 2443, Red action: do-nothing, Blue reward:0.85\n", + "step: 2444, Red action: do-nothing, Blue reward:0.85\n", + "step: 2445, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2446, Red action: do-nothing, Blue reward:0.85\n", + "step: 2447, Red action: do-nothing, Blue reward:0.85\n", + "step: 2448, Red action: do-nothing, Blue reward:0.85\n", + "step: 2449, Red action: do-nothing, Blue reward:0.85\n", + "step: 2450, Red action: do-nothing, Blue reward:0.85\n", + "step: 2451, Red action: do-nothing, Blue reward:0.85\n", + "step: 2452, Red action: do-nothing, Blue reward:0.85\n", + "step: 2453, Red action: do-nothing, Blue reward:0.85\n", + "step: 2454, Red action: do-nothing, Blue reward:0.85\n", + "step: 2455, Red action: do-nothing, Blue reward:0.85\n", + "step: 2456, Red action: do-nothing, Blue reward:0.85\n", + "step: 2457, Red action: do-nothing, Blue reward:0.85\n", + "step: 2458, Red action: do-nothing, Blue reward:0.85\n", + "step: 2459, Red action: do-nothing, Blue reward:0.85\n", + "step: 2460, Red action: do-nothing, Blue reward:0.85\n", + "step: 2461, Red action: do-nothing, Blue reward:0.85\n", + "step: 2462, Red action: do-nothing, Blue reward:0.85\n", + "step: 2463, Red action: do-nothing, Blue reward:0.85\n", + "step: 2464, Red action: do-nothing, Blue reward:0.85\n", + "step: 2465, Red action: do-nothing, Blue reward:0.85\n", + "step: 2466, Red action: do-nothing, Blue reward:0.85\n", + "step: 2467, Red action: do-nothing, Blue reward:0.85\n", + "step: 2468, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2469, Red action: do-nothing, Blue reward:0.85\n", + "step: 2470, Red action: do-nothing, Blue reward:0.85\n", + "step: 2471, Red action: do-nothing, Blue reward:0.85\n", + "step: 2472, Red action: do-nothing, Blue reward:0.85\n", + "step: 2473, Red action: do-nothing, Blue reward:0.85\n", + "step: 2474, Red action: do-nothing, Blue reward:0.85\n", + "step: 2475, Red action: do-nothing, Blue reward:0.85\n", + "step: 2476, Red action: do-nothing, Blue reward:0.85\n", + "step: 2477, Red action: do-nothing, Blue reward:0.85\n", + "step: 2478, Red action: do-nothing, Blue reward:0.85\n", + "step: 2479, Red action: do-nothing, Blue reward:0.85\n", + "step: 2480, Red action: do-nothing, Blue reward:0.85\n", + "step: 2481, Red action: do-nothing, Blue reward:0.85\n", + "step: 2482, Red action: do-nothing, Blue reward:0.85\n", + "step: 2483, Red action: do-nothing, Blue reward:0.85\n", + "step: 2484, Red action: do-nothing, Blue reward:0.85\n", + "step: 2485, Red action: do-nothing, Blue reward:0.85\n", + "step: 2486, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2487, Red action: do-nothing, Blue reward:0.85\n", + "step: 2488, Red action: do-nothing, Blue reward:0.85\n", + "step: 2489, Red action: do-nothing, Blue reward:0.85\n", + "step: 2490, Red action: do-nothing, Blue reward:0.85\n", + "step: 2491, Red action: do-nothing, Blue reward:0.85\n", + "step: 2492, Red action: do-nothing, Blue reward:0.85\n", + "step: 2493, Red action: do-nothing, Blue reward:0.85\n", + "step: 2494, Red action: do-nothing, Blue reward:0.85\n", + "step: 2495, Red action: do-nothing, Blue reward:0.85\n", + "step: 2496, Red action: do-nothing, Blue reward:0.85\n", + "step: 2497, Red action: do-nothing, Blue reward:0.85\n", + "step: 2498, Red action: do-nothing, Blue reward:0.85\n", + "step: 2499, Red action: do-nothing, Blue reward:0.85\n", + "step: 2500, Red action: do-nothing, Blue reward:0.85\n", + "step: 2501, Red action: do-nothing, Blue reward:0.85\n", + "step: 2502, Red action: do-nothing, Blue reward:0.85\n", + "step: 2503, Red action: do-nothing, Blue reward:0.85\n", + "step: 2504, Red action: do-nothing, Blue reward:0.85\n", + "step: 2505, Red action: do-nothing, Blue reward:0.85\n", + "step: 2506, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2507, Red action: do-nothing, Blue reward:0.85\n", + "step: 2508, Red action: do-nothing, Blue reward:0.85\n", + "step: 2509, Red action: do-nothing, Blue reward:0.85\n", + "step: 2510, Red action: do-nothing, Blue reward:0.85\n", + "step: 2511, Red action: do-nothing, Blue reward:0.85\n", + "step: 2512, Red action: do-nothing, Blue reward:0.85\n", + "step: 2513, Red action: do-nothing, Blue reward:0.85\n", + "step: 2514, Red action: do-nothing, Blue reward:0.85\n", + "step: 2515, Red action: do-nothing, Blue reward:0.85\n", + "step: 2516, Red action: do-nothing, Blue reward:0.85\n", + "step: 2517, Red action: do-nothing, Blue reward:0.85\n", + "step: 2518, Red action: do-nothing, Blue reward:0.85\n", + "step: 2519, Red action: do-nothing, Blue reward:0.85\n", + "step: 2520, Red action: do-nothing, Blue reward:0.85\n", + "step: 2521, Red action: do-nothing, Blue reward:0.85\n", + "step: 2522, Red action: do-nothing, Blue reward:0.85\n", + "step: 2523, Red action: do-nothing, Blue reward:0.85\n", + "step: 2524, Red action: do-nothing, Blue reward:0.85\n", + "step: 2525, Red action: do-nothing, Blue reward:0.85\n", + "step: 2526, Red action: do-nothing, Blue reward:0.85\n", + "step: 2527, Red action: do-nothing, Blue reward:0.85\n", + "step: 2528, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2529, Red action: do-nothing, Blue reward:0.85\n", + "step: 2530, Red action: do-nothing, Blue reward:0.85\n", + "step: 2531, Red action: do-nothing, Blue reward:0.85\n", + "step: 2532, Red action: do-nothing, Blue reward:0.85\n", + "step: 2533, Red action: do-nothing, Blue reward:0.85\n", + "step: 2534, Red action: do-nothing, Blue reward:0.85\n", + "step: 2535, Red action: do-nothing, Blue reward:0.85\n", + "step: 2536, Red action: do-nothing, Blue reward:0.85\n", + "step: 2537, Red action: do-nothing, Blue reward:0.85\n", + "step: 2538, Red action: do-nothing, Blue reward:0.85\n", + "step: 2539, Red action: do-nothing, Blue reward:0.85\n", + "step: 2540, Red action: do-nothing, Blue reward:0.85\n", + "step: 2541, Red action: do-nothing, Blue reward:0.85\n", + "step: 2542, Red action: do-nothing, Blue reward:0.85\n", + "step: 2543, Red action: do-nothing, Blue reward:0.85\n", + "step: 2544, Red action: do-nothing, Blue reward:0.85\n", + "step: 2545, Red action: do-nothing, Blue reward:0.85\n", + "step: 2546, Red action: do-nothing, Blue reward:0.85\n", + "step: 2547, Red action: do-nothing, Blue reward:0.85\n", + "step: 2548, Red action: do-nothing, Blue reward:0.85\n", + "step: 2549, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2550, Red action: do-nothing, Blue reward:0.85\n", + "step: 2551, Red action: do-nothing, Blue reward:0.85\n", + "step: 2552, Red action: do-nothing, Blue reward:0.85\n", + "step: 2553, Red action: do-nothing, Blue reward:0.85\n", + "step: 2554, Red action: do-nothing, Blue reward:0.85\n", + "step: 2555, Red action: do-nothing, Blue reward:0.85\n", + "step: 2556, Red action: do-nothing, Blue reward:0.85\n", + "step: 2557, Red action: do-nothing, Blue reward:0.85\n", + "step: 2558, Red action: do-nothing, Blue reward:0.85\n", + "step: 2559, Red action: do-nothing, Blue reward:0.85\n", + "step: 2560, Red action: do-nothing, Blue reward:0.85\n", + "step: 2561, Red action: do-nothing, Blue reward:0.85\n", + "step: 2562, Red action: do-nothing, Blue reward:0.85\n", + "step: 2563, Red action: do-nothing, Blue reward:0.85\n", + "step: 2564, Red action: do-nothing, Blue reward:0.85\n", + "step: 2565, Red action: do-nothing, Blue reward:0.85\n", + "step: 2566, Red action: do-nothing, Blue reward:0.85\n", + "step: 2567, Red action: do-nothing, Blue reward:0.85\n", + "step: 2568, Red action: do-nothing, Blue reward:0.85\n", + "step: 2569, Red action: do-nothing, Blue reward:0.85\n", + "step: 2570, Red action: do-nothing, Blue reward:0.85\n", + "step: 2571, Red action: do-nothing, Blue reward:0.85\n", + "step: 2572, Red action: do-nothing, Blue reward:0.85\n", + "step: 2573, Red action: do-nothing, Blue reward:0.85\n", + "step: 2574, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2575, Red action: do-nothing, Blue reward:0.85\n", + "step: 2576, Red action: do-nothing, Blue reward:0.85\n", + "step: 2577, Red action: do-nothing, Blue reward:0.85\n", + "step: 2578, Red action: do-nothing, Blue reward:0.85\n", + "step: 2579, Red action: do-nothing, Blue reward:0.85\n", + "step: 2580, Red action: do-nothing, Blue reward:0.85\n", + "step: 2581, Red action: do-nothing, Blue reward:0.85\n", + "step: 2582, Red action: do-nothing, Blue reward:0.85\n", + "step: 2583, Red action: do-nothing, Blue reward:0.85\n", + "step: 2584, Red action: do-nothing, Blue reward:0.85\n", + "step: 2585, Red action: do-nothing, Blue reward:0.85\n", + "step: 2586, Red action: do-nothing, Blue reward:0.85\n", + "step: 2587, Red action: do-nothing, Blue reward:0.85\n", + "step: 2588, Red action: do-nothing, Blue reward:0.85\n", + "step: 2589, Red action: do-nothing, Blue reward:0.85\n", + "step: 2590, Red action: do-nothing, Blue reward:0.85\n", + "step: 2591, Red action: do-nothing, Blue reward:0.85\n", + "step: 2592, Red action: do-nothing, Blue reward:0.85\n", + "step: 2593, Red action: do-nothing, Blue reward:0.85\n", + "step: 2594, Red action: do-nothing, Blue reward:0.85\n", + "step: 2595, Red action: do-nothing, Blue reward:0.85\n", + "step: 2596, Red action: do-nothing, Blue reward:0.85\n", + "step: 2597, Red action: do-nothing, Blue reward:0.85\n", + "step: 2598, Red action: do-nothing, Blue reward:0.85\n", + "step: 2599, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2600, Red action: do-nothing, Blue reward:0.85\n", + "step: 2601, Red action: do-nothing, Blue reward:0.85\n", + "step: 2602, Red action: do-nothing, Blue reward:0.85\n", + "step: 2603, Red action: do-nothing, Blue reward:0.85\n", + "step: 2604, Red action: do-nothing, Blue reward:0.85\n", + "step: 2605, Red action: do-nothing, Blue reward:0.85\n", + "step: 2606, Red action: do-nothing, Blue reward:0.85\n", + "step: 2607, Red action: do-nothing, Blue reward:0.85\n", + "step: 2608, Red action: do-nothing, Blue reward:0.85\n", + "step: 2609, Red action: do-nothing, Blue reward:0.85\n", + "step: 2610, Red action: do-nothing, Blue reward:0.85\n", + "step: 2611, Red action: do-nothing, Blue reward:0.85\n", + "step: 2612, Red action: do-nothing, Blue reward:0.85\n", + "step: 2613, Red action: do-nothing, Blue reward:0.85\n", + "step: 2614, Red action: do-nothing, Blue reward:0.85\n", + "step: 2615, Red action: do-nothing, Blue reward:0.85\n", + "step: 2616, Red action: do-nothing, Blue reward:0.85\n", + "step: 2617, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2618, Red action: do-nothing, Blue reward:0.85\n", + "step: 2619, Red action: do-nothing, Blue reward:0.85\n", + "step: 2620, Red action: do-nothing, Blue reward:0.85\n", + "step: 2621, Red action: do-nothing, Blue reward:0.85\n", + "step: 2622, Red action: do-nothing, Blue reward:0.85\n", + "step: 2623, Red action: do-nothing, Blue reward:0.85\n", + "step: 2624, Red action: do-nothing, Blue reward:0.85\n", + "step: 2625, Red action: do-nothing, Blue reward:0.85\n", + "step: 2626, Red action: do-nothing, Blue reward:0.85\n", + "step: 2627, Red action: do-nothing, Blue reward:0.85\n", + "step: 2628, Red action: do-nothing, Blue reward:0.85\n", + "step: 2629, Red action: do-nothing, Blue reward:0.85\n", + "step: 2630, Red action: do-nothing, Blue reward:0.85\n", + "step: 2631, Red action: do-nothing, Blue reward:0.85\n", + "step: 2632, Red action: do-nothing, Blue reward:0.85\n", + "step: 2633, Red action: do-nothing, Blue reward:0.85\n", + "step: 2634, Red action: do-nothing, Blue reward:0.85\n", + "step: 2635, Red action: do-nothing, Blue reward:0.85\n", + "step: 2636, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2637, Red action: do-nothing, Blue reward:0.85\n", + "step: 2638, Red action: do-nothing, Blue reward:0.85\n", + "step: 2639, Red action: do-nothing, Blue reward:0.85\n", + "step: 2640, Red action: do-nothing, Blue reward:0.85\n", + "step: 2641, Red action: do-nothing, Blue reward:0.85\n", + "step: 2642, Red action: do-nothing, Blue reward:0.85\n", + "step: 2643, Red action: do-nothing, Blue reward:0.85\n", + "step: 2644, Red action: do-nothing, Blue reward:0.85\n", + "step: 2645, Red action: do-nothing, Blue reward:0.85\n", + "step: 2646, Red action: do-nothing, Blue reward:0.85\n", + "step: 2647, Red action: do-nothing, Blue reward:0.85\n", + "step: 2648, Red action: do-nothing, Blue reward:0.85\n", + "step: 2649, Red action: do-nothing, Blue reward:0.85\n", + "step: 2650, Red action: do-nothing, Blue reward:0.85\n", + "step: 2651, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2652, Red action: do-nothing, Blue reward:0.85\n", + "step: 2653, Red action: do-nothing, Blue reward:0.85\n", + "step: 2654, Red action: do-nothing, Blue reward:0.85\n", + "step: 2655, Red action: do-nothing, Blue reward:0.85\n", + "step: 2656, Red action: do-nothing, Blue reward:0.85\n", + "step: 2657, Red action: do-nothing, Blue reward:0.85\n", + "step: 2658, Red action: do-nothing, Blue reward:0.85\n", + "step: 2659, Red action: do-nothing, Blue reward:0.85\n", + "step: 2660, Red action: do-nothing, Blue reward:0.85\n", + "step: 2661, Red action: do-nothing, Blue reward:0.85\n", + "step: 2662, Red action: do-nothing, Blue reward:0.85\n", + "step: 2663, Red action: do-nothing, Blue reward:0.85\n", + "step: 2664, Red action: do-nothing, Blue reward:0.85\n", + "step: 2665, Red action: do-nothing, Blue reward:0.85\n", + "step: 2666, Red action: do-nothing, Blue reward:0.85\n", + "step: 2667, Red action: do-nothing, Blue reward:0.85\n", + "step: 2668, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2669, Red action: do-nothing, Blue reward:0.85\n", + "step: 2670, Red action: do-nothing, Blue reward:0.85\n", + "step: 2671, Red action: do-nothing, Blue reward:0.85\n", + "step: 2672, Red action: do-nothing, Blue reward:0.85\n", + "step: 2673, Red action: do-nothing, Blue reward:0.85\n", + "step: 2674, Red action: do-nothing, Blue reward:0.85\n", + "step: 2675, Red action: do-nothing, Blue reward:0.85\n", + "step: 2676, Red action: do-nothing, Blue reward:0.85\n", + "step: 2677, Red action: do-nothing, Blue reward:0.85\n", + "step: 2678, Red action: do-nothing, Blue reward:0.85\n", + "step: 2679, Red action: do-nothing, Blue reward:0.85\n", + "step: 2680, Red action: do-nothing, Blue reward:0.85\n", + "step: 2681, Red action: do-nothing, Blue reward:0.85\n", + "step: 2682, Red action: do-nothing, Blue reward:0.85\n", + "step: 2683, Red action: do-nothing, Blue reward:0.85\n", + "step: 2684, Red action: do-nothing, Blue reward:0.85\n", + "step: 2685, Red action: do-nothing, Blue reward:0.85\n", + "step: 2686, Red action: do-nothing, Blue reward:0.85\n", + "step: 2687, Red action: do-nothing, Blue reward:0.85\n", + "step: 2688, Red action: do-nothing, Blue reward:0.85\n", + "step: 2689, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2690, Red action: do-nothing, Blue reward:0.85\n", + "step: 2691, Red action: do-nothing, Blue reward:0.85\n", + "step: 2692, Red action: do-nothing, Blue reward:0.85\n", + "step: 2693, Red action: do-nothing, Blue reward:0.85\n", + "step: 2694, Red action: do-nothing, Blue reward:0.85\n", + "step: 2695, Red action: do-nothing, Blue reward:0.85\n", + "step: 2696, Red action: do-nothing, Blue reward:0.85\n", + "step: 2697, Red action: do-nothing, Blue reward:0.85\n", + "step: 2698, Red action: do-nothing, Blue reward:0.85\n", + "step: 2699, Red action: do-nothing, Blue reward:0.85\n", + "step: 2700, Red action: do-nothing, Blue reward:0.85\n", + "step: 2701, Red action: do-nothing, Blue reward:0.85\n", + "step: 2702, Red action: do-nothing, Blue reward:0.85\n", + "step: 2703, Red action: do-nothing, Blue reward:0.85\n", + "step: 2704, Red action: do-nothing, Blue reward:0.85\n", + "step: 2705, Red action: do-nothing, Blue reward:0.85\n", + "step: 2706, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2707, Red action: do-nothing, Blue reward:0.85\n", + "step: 2708, Red action: do-nothing, Blue reward:0.85\n", + "step: 2709, Red action: do-nothing, Blue reward:0.85\n", + "step: 2710, Red action: do-nothing, Blue reward:0.85\n", + "step: 2711, Red action: do-nothing, Blue reward:0.85\n", + "step: 2712, Red action: do-nothing, Blue reward:0.85\n", + "step: 2713, Red action: do-nothing, Blue reward:0.85\n", + "step: 2714, Red action: do-nothing, Blue reward:0.85\n", + "step: 2715, Red action: do-nothing, Blue reward:0.85\n", + "step: 2716, Red action: do-nothing, Blue reward:0.85\n", + "step: 2717, Red action: do-nothing, Blue reward:0.85\n", + "step: 2718, Red action: do-nothing, Blue reward:0.85\n", + "step: 2719, Red action: do-nothing, Blue reward:0.85\n", + "step: 2720, Red action: do-nothing, Blue reward:0.85\n", + "step: 2721, Red action: do-nothing, Blue reward:0.85\n", + "step: 2722, Red action: do-nothing, Blue reward:0.85\n", + "step: 2723, Red action: do-nothing, Blue reward:0.85\n", + "step: 2724, Red action: do-nothing, Blue reward:0.85\n", + "step: 2725, Red action: do-nothing, Blue reward:0.85\n", + "step: 2726, Red action: do-nothing, Blue reward:0.85\n", + "step: 2727, Red action: do-nothing, Blue reward:0.85\n", + "step: 2728, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2729, Red action: do-nothing, Blue reward:0.85\n", + "step: 2730, Red action: do-nothing, Blue reward:0.85\n", + "step: 2731, Red action: do-nothing, Blue reward:0.85\n", + "step: 2732, Red action: do-nothing, Blue reward:0.85\n", + "step: 2733, Red action: do-nothing, Blue reward:0.85\n", + "step: 2734, Red action: do-nothing, Blue reward:0.85\n", + "step: 2735, Red action: do-nothing, Blue reward:0.85\n", + "step: 2736, Red action: do-nothing, Blue reward:0.85\n", + "step: 2737, Red action: do-nothing, Blue reward:0.85\n", + "step: 2738, Red action: do-nothing, Blue reward:0.85\n", + "step: 2739, Red action: do-nothing, Blue reward:0.85\n", + "step: 2740, Red action: do-nothing, Blue reward:0.85\n", + "step: 2741, Red action: do-nothing, Blue reward:0.85\n", + "step: 2742, Red action: do-nothing, Blue reward:0.85\n", + "step: 2743, Red action: do-nothing, Blue reward:0.85\n", + "step: 2744, Red action: do-nothing, Blue reward:0.85\n", + "step: 2745, Red action: do-nothing, Blue reward:0.85\n", + "step: 2746, Red action: do-nothing, Blue reward:0.85\n", + "step: 2747, Red action: do-nothing, Blue reward:0.85\n", + "step: 2748, Red action: do-nothing, Blue reward:0.85\n", + "step: 2749, Red action: do-nothing, Blue reward:0.85\n", + "step: 2750, Red action: do-nothing, Blue reward:0.85\n", + "step: 2751, Red action: do-nothing, Blue reward:0.85\n", + "step: 2752, Red action: do-nothing, Blue reward:0.85\n", + "step: 2753, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2754, Red action: do-nothing, Blue reward:0.85\n", + "step: 2755, Red action: do-nothing, Blue reward:0.85\n", + "step: 2756, Red action: do-nothing, Blue reward:0.85\n", + "step: 2757, Red action: do-nothing, Blue reward:0.85\n", + "step: 2758, Red action: do-nothing, Blue reward:0.85\n", + "step: 2759, Red action: do-nothing, Blue reward:0.85\n", + "step: 2760, Red action: do-nothing, Blue reward:0.85\n", + "step: 2761, Red action: do-nothing, Blue reward:0.85\n", + "step: 2762, Red action: do-nothing, Blue reward:0.85\n", + "step: 2763, Red action: do-nothing, Blue reward:0.85\n", + "step: 2764, Red action: do-nothing, Blue reward:0.85\n", + "step: 2765, Red action: do-nothing, Blue reward:0.85\n", + "step: 2766, Red action: do-nothing, Blue reward:0.85\n", + "step: 2767, Red action: do-nothing, Blue reward:0.85\n", + "step: 2768, Red action: do-nothing, Blue reward:0.85\n", + "step: 2769, Red action: do-nothing, Blue reward:0.85\n", + "step: 2770, Red action: do-nothing, Blue reward:0.85\n", + "step: 2771, Red action: do-nothing, Blue reward:0.85\n", + "step: 2772, Red action: do-nothing, Blue reward:0.85\n", + "step: 2773, Red action: do-nothing, Blue reward:0.85\n", + "step: 2774, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2775, Red action: do-nothing, Blue reward:0.85\n", + "step: 2776, Red action: do-nothing, Blue reward:0.85\n", + "step: 2777, Red action: do-nothing, Blue reward:0.85\n", + "step: 2778, Red action: do-nothing, Blue reward:0.85\n", + "step: 2779, Red action: do-nothing, Blue reward:0.85\n", + "step: 2780, Red action: do-nothing, Blue reward:0.85\n", + "step: 2781, Red action: do-nothing, Blue reward:0.85\n", + "step: 2782, Red action: do-nothing, Blue reward:0.85\n", + "step: 2783, Red action: do-nothing, Blue reward:0.85\n", + "step: 2784, Red action: do-nothing, Blue reward:0.85\n", + "step: 2785, Red action: do-nothing, Blue reward:0.85\n", + "step: 2786, Red action: do-nothing, Blue reward:0.85\n", + "step: 2787, Red action: do-nothing, Blue reward:0.85\n", + "step: 2788, Red action: do-nothing, Blue reward:0.85\n", + "step: 2789, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2790, Red action: do-nothing, Blue reward:0.85\n", + "step: 2791, Red action: do-nothing, Blue reward:0.85\n", + "step: 2792, Red action: do-nothing, Blue reward:0.85\n", + "step: 2793, Red action: do-nothing, Blue reward:0.85\n", + "step: 2794, Red action: do-nothing, Blue reward:0.85\n", + "step: 2795, Red action: do-nothing, Blue reward:0.85\n", + "step: 2796, Red action: do-nothing, Blue reward:0.85\n", + "step: 2797, Red action: do-nothing, Blue reward:0.85\n", + "step: 2798, Red action: do-nothing, Blue reward:0.85\n", + "step: 2799, Red action: do-nothing, Blue reward:0.85\n", + "step: 2800, Red action: do-nothing, Blue reward:0.85\n", + "step: 2801, Red action: do-nothing, Blue reward:0.85\n", + "step: 2802, Red action: do-nothing, Blue reward:0.85\n", + "step: 2803, Red action: do-nothing, Blue reward:0.85\n", + "step: 2804, Red action: do-nothing, Blue reward:0.85\n", + "step: 2805, Red action: do-nothing, Blue reward:0.85\n", + "step: 2806, Red action: do-nothing, Blue reward:0.85\n", + "step: 2807, Red action: do-nothing, Blue reward:0.85\n", + "step: 2808, Red action: do-nothing, Blue reward:0.85\n", + "step: 2809, Red action: do-nothing, Blue reward:0.85\n", + "step: 2810, Red action: do-nothing, Blue reward:0.85\n", + "step: 2811, Red action: do-nothing, Blue reward:0.85\n", + "step: 2812, Red action: do-nothing, Blue reward:0.85\n", + "step: 2813, Red action: do-nothing, Blue reward:0.85\n", + "step: 2814, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2815, Red action: do-nothing, Blue reward:0.85\n", + "step: 2816, Red action: do-nothing, Blue reward:0.85\n", + "step: 2817, Red action: do-nothing, Blue reward:0.85\n", + "step: 2818, Red action: do-nothing, Blue reward:0.85\n", + "step: 2819, Red action: do-nothing, Blue reward:0.85\n", + "step: 2820, Red action: do-nothing, Blue reward:0.85\n", + "step: 2821, Red action: do-nothing, Blue reward:0.85\n", + "step: 2822, Red action: do-nothing, Blue reward:0.85\n", + "step: 2823, Red action: do-nothing, Blue reward:0.85\n", + "step: 2824, Red action: do-nothing, Blue reward:0.85\n", + "step: 2825, Red action: do-nothing, Blue reward:0.85\n", + "step: 2826, Red action: do-nothing, Blue reward:0.85\n", + "step: 2827, Red action: do-nothing, Blue reward:0.85\n", + "step: 2828, Red action: do-nothing, Blue reward:0.85\n", + "step: 2829, Red action: do-nothing, Blue reward:0.85\n", + "step: 2830, Red action: do-nothing, Blue reward:0.85\n", + "step: 2831, Red action: do-nothing, Blue reward:0.85\n", + "step: 2832, Red action: do-nothing, Blue reward:0.85\n", + "step: 2833, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2834, Red action: do-nothing, Blue reward:0.85\n", + "step: 2835, Red action: do-nothing, Blue reward:0.85\n", + "step: 2836, Red action: do-nothing, Blue reward:0.85\n", + "step: 2837, Red action: do-nothing, Blue reward:0.85\n", + "step: 2838, Red action: do-nothing, Blue reward:0.85\n", + "step: 2839, Red action: do-nothing, Blue reward:0.85\n", + "step: 2840, Red action: do-nothing, Blue reward:0.85\n", + "step: 2841, Red action: do-nothing, Blue reward:0.85\n", + "step: 2842, Red action: do-nothing, Blue reward:0.85\n", + "step: 2843, Red action: do-nothing, Blue reward:0.85\n", + "step: 2844, Red action: do-nothing, Blue reward:0.85\n", + "step: 2845, Red action: do-nothing, Blue reward:0.85\n", + "step: 2846, Red action: do-nothing, Blue reward:0.85\n", + "step: 2847, Red action: do-nothing, Blue reward:0.85\n", + "step: 2848, Red action: do-nothing, Blue reward:0.85\n", + "step: 2849, Red action: do-nothing, Blue reward:0.85\n", + "step: 2850, Red action: do-nothing, Blue reward:0.85\n", + "step: 2851, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2852, Red action: do-nothing, Blue reward:0.85\n", + "step: 2853, Red action: do-nothing, Blue reward:0.85\n", + "step: 2854, Red action: do-nothing, Blue reward:0.85\n", + "step: 2855, Red action: do-nothing, Blue reward:0.85\n", + "step: 2856, Red action: do-nothing, Blue reward:0.85\n", + "step: 2857, Red action: do-nothing, Blue reward:0.85\n", + "step: 2858, Red action: do-nothing, Blue reward:0.85\n", + "step: 2859, Red action: do-nothing, Blue reward:0.85\n", + "step: 2860, Red action: do-nothing, Blue reward:0.85\n", + "step: 2861, Red action: do-nothing, Blue reward:0.85\n", + "step: 2862, Red action: do-nothing, Blue reward:0.85\n", + "step: 2863, Red action: do-nothing, Blue reward:0.85\n", + "step: 2864, Red action: do-nothing, Blue reward:0.85\n", + "step: 2865, Red action: do-nothing, Blue reward:0.85\n", + "step: 2866, Red action: do-nothing, Blue reward:0.85\n", + "step: 2867, Red action: do-nothing, Blue reward:0.85\n", + "step: 2868, Red action: do-nothing, Blue reward:0.85\n", + "step: 2869, Red action: do-nothing, Blue reward:0.85\n", + "step: 2870, Red action: do-nothing, Blue reward:0.85\n", + "step: 2871, Red action: do-nothing, Blue reward:0.85\n", + "step: 2872, Red action: do-nothing, Blue reward:0.85\n", + "step: 2873, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2874, Red action: do-nothing, Blue reward:0.85\n", + "step: 2875, Red action: do-nothing, Blue reward:0.85\n", + "step: 2876, Red action: do-nothing, Blue reward:0.85\n", + "step: 2877, Red action: do-nothing, Blue reward:0.85\n", + "step: 2878, Red action: do-nothing, Blue reward:0.85\n", + "step: 2879, Red action: do-nothing, Blue reward:0.85\n", + "step: 2880, Red action: do-nothing, Blue reward:0.85\n", + "step: 2881, Red action: do-nothing, Blue reward:0.85\n", + "step: 2882, Red action: do-nothing, Blue reward:0.85\n", + "step: 2883, Red action: do-nothing, Blue reward:0.85\n", + "step: 2884, Red action: do-nothing, Blue reward:0.85\n", + "step: 2885, Red action: do-nothing, Blue reward:0.85\n", + "step: 2886, Red action: do-nothing, Blue reward:0.85\n", + "step: 2887, Red action: do-nothing, Blue reward:0.85\n", + "step: 2888, Red action: do-nothing, Blue reward:0.85\n", + "step: 2889, Red action: do-nothing, Blue reward:0.85\n", + "step: 2890, Red action: do-nothing, Blue reward:0.85\n", + "step: 2891, Red action: do-nothing, Blue reward:0.85\n", + "step: 2892, Red action: do-nothing, Blue reward:0.85\n", + "step: 2893, Red action: do-nothing, Blue reward:0.85\n", + "step: 2894, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2895, Red action: do-nothing, Blue reward:0.85\n", + "step: 2896, Red action: do-nothing, Blue reward:0.85\n", + "step: 2897, Red action: do-nothing, Blue reward:0.85\n", + "step: 2898, Red action: do-nothing, Blue reward:0.85\n", + "step: 2899, Red action: do-nothing, Blue reward:0.85\n", + "step: 2900, Red action: do-nothing, Blue reward:0.85\n", + "step: 2901, Red action: do-nothing, Blue reward:0.85\n", + "step: 2902, Red action: do-nothing, Blue reward:0.85\n", + "step: 2903, Red action: do-nothing, Blue reward:0.85\n", + "step: 2904, Red action: do-nothing, Blue reward:0.85\n", + "step: 2905, Red action: do-nothing, Blue reward:0.85\n", + "step: 2906, Red action: do-nothing, Blue reward:0.85\n", + "step: 2907, Red action: do-nothing, Blue reward:0.85\n", + "step: 2908, Red action: do-nothing, Blue reward:0.85\n", + "step: 2909, Red action: do-nothing, Blue reward:0.85\n", + "step: 2910, Red action: do-nothing, Blue reward:0.85\n", + "step: 2911, Red action: do-nothing, Blue reward:0.85\n", + "step: 2912, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2913, Red action: do-nothing, Blue reward:0.85\n", + "step: 2914, Red action: do-nothing, Blue reward:0.85\n", + "step: 2915, Red action: do-nothing, Blue reward:0.85\n", + "step: 2916, Red action: do-nothing, Blue reward:0.85\n", + "step: 2917, Red action: do-nothing, Blue reward:0.85\n", + "step: 2918, Red action: do-nothing, Blue reward:0.85\n", + "step: 2919, Red action: do-nothing, Blue reward:0.85\n", + "step: 2920, Red action: do-nothing, Blue reward:0.85\n", + "step: 2921, Red action: do-nothing, Blue reward:0.85\n", + "step: 2922, Red action: do-nothing, Blue reward:0.85\n", + "step: 2923, Red action: do-nothing, Blue reward:0.85\n", + "step: 2924, Red action: do-nothing, Blue reward:0.85\n", + "step: 2925, Red action: do-nothing, Blue reward:0.85\n", + "step: 2926, Red action: do-nothing, Blue reward:0.85\n", + "step: 2927, Red action: do-nothing, Blue reward:0.85\n", + "step: 2928, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2929, Red action: do-nothing, Blue reward:0.85\n", + "step: 2930, Red action: do-nothing, Blue reward:0.85\n", + "step: 2931, Red action: do-nothing, Blue reward:0.85\n", + "step: 2932, Red action: do-nothing, Blue reward:0.85\n", + "step: 2933, Red action: do-nothing, Blue reward:0.85\n", + "step: 2934, Red action: do-nothing, Blue reward:0.85\n", + "step: 2935, Red action: do-nothing, Blue reward:0.85\n", + "step: 2936, Red action: do-nothing, Blue reward:0.85\n", + "step: 2937, Red action: do-nothing, Blue reward:0.85\n", + "step: 2938, Red action: do-nothing, Blue reward:0.85\n", + "step: 2939, Red action: do-nothing, Blue reward:0.85\n", + "step: 2940, Red action: do-nothing, Blue reward:0.85\n", + "step: 2941, Red action: do-nothing, Blue reward:0.85\n", + "step: 2942, Red action: do-nothing, Blue reward:0.85\n", + "step: 2943, Red action: do-nothing, Blue reward:0.85\n", + "step: 2944, Red action: do-nothing, Blue reward:0.85\n", + "step: 2945, Red action: do-nothing, Blue reward:0.85\n", + "step: 2946, Red action: do-nothing, Blue reward:0.85\n", + "step: 2947, Red action: do-nothing, Blue reward:0.85\n", + "step: 2948, Red action: do-nothing, Blue reward:0.85\n", + "step: 2949, Red action: do-nothing, Blue reward:0.85\n", + "step: 2950, Red action: do-nothing, Blue reward:0.85\n", + "step: 2951, Red action: do-nothing, Blue reward:0.85\n", + "step: 2952, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2953, Red action: do-nothing, Blue reward:0.85\n", + "step: 2954, Red action: do-nothing, Blue reward:0.85\n", + "step: 2955, Red action: do-nothing, Blue reward:0.85\n", + "step: 2956, Red action: do-nothing, Blue reward:0.85\n", + "step: 2957, Red action: do-nothing, Blue reward:0.85\n", + "step: 2958, Red action: do-nothing, Blue reward:0.85\n", + "step: 2959, Red action: do-nothing, Blue reward:0.85\n", + "step: 2960, Red action: do-nothing, Blue reward:0.85\n", + "step: 2961, Red action: do-nothing, Blue reward:0.85\n", + "step: 2962, Red action: do-nothing, Blue reward:0.85\n", + "step: 2963, Red action: do-nothing, Blue reward:0.85\n", + "step: 2964, Red action: do-nothing, Blue reward:0.85\n", + "step: 2965, Red action: do-nothing, Blue reward:0.85\n", + "step: 2966, Red action: do-nothing, Blue reward:0.85\n", + "step: 2967, Red action: do-nothing, Blue reward:0.85\n", + "step: 2968, Red action: do-nothing, Blue reward:0.85\n", + "step: 2969, Red action: do-nothing, Blue reward:0.85\n", + "step: 2970, Red action: do-nothing, Blue reward:0.85\n", + "step: 2971, Red action: do-nothing, Blue reward:0.85\n", + "step: 2972, Red action: do-nothing, Blue reward:0.85\n", + "step: 2973, Red action: do-nothing, Blue reward:0.85\n", + "step: 2974, Red action: do-nothing, Blue reward:0.85\n", + "step: 2975, Red action: do-nothing, Blue reward:0.85\n", + "step: 2976, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2977, Red action: do-nothing, Blue reward:0.85\n", + "step: 2978, Red action: do-nothing, Blue reward:0.85\n", + "step: 2979, Red action: do-nothing, Blue reward:0.85\n", + "step: 2980, Red action: do-nothing, Blue reward:0.85\n", + "step: 2981, Red action: do-nothing, Blue reward:0.85\n", + "step: 2982, Red action: do-nothing, Blue reward:0.85\n", + "step: 2983, Red action: do-nothing, Blue reward:0.85\n", + "step: 2984, Red action: do-nothing, Blue reward:0.85\n", + "step: 2985, Red action: do-nothing, Blue reward:0.85\n", + "step: 2986, Red action: do-nothing, Blue reward:0.85\n", + "step: 2987, Red action: do-nothing, Blue reward:0.85\n", + "step: 2988, Red action: do-nothing, Blue reward:0.85\n", + "step: 2989, Red action: do-nothing, Blue reward:0.85\n", + "step: 2990, Red action: do-nothing, Blue reward:0.85\n", + "step: 2991, Red action: do-nothing, Blue reward:0.85\n", + "step: 2992, Red action: do-nothing, Blue reward:0.85\n", + "step: 2993, Red action: do-nothing, Blue reward:0.85\n", + "step: 2994, Red action: do-nothing, Blue reward:0.85\n", + "step: 2995, Red action: do-nothing, Blue reward:0.85\n", + "step: 2996, Red action: do-nothing, Blue reward:0.85\n", + "step: 2997, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2998, Red action: do-nothing, Blue reward:0.85\n", + "step: 2999, Red action: do-nothing, Blue reward:0.85\n", + "step: 3000, Red action: do-nothing, Blue reward:0.85\n", + "step: 3001, Red action: do-nothing, Blue reward:0.85\n", + "step: 3002, Red action: do-nothing, Blue reward:0.85\n", + "step: 3003, Red action: do-nothing, Blue reward:0.85\n", + "step: 3004, Red action: do-nothing, Blue reward:0.85\n", + "step: 3005, Red action: do-nothing, Blue reward:0.85\n", + "step: 3006, Red action: do-nothing, Blue reward:0.85\n", + "step: 3007, Red action: do-nothing, Blue reward:0.85\n", + "step: 3008, Red action: do-nothing, Blue reward:0.85\n", + "step: 3009, Red action: do-nothing, Blue reward:0.85\n", + "step: 3010, Red action: do-nothing, Blue reward:0.85\n", + "step: 3011, Red action: do-nothing, Blue reward:0.85\n", + "step: 3012, Red action: do-nothing, Blue reward:0.85\n", + "step: 3013, Red action: do-nothing, Blue reward:0.85\n", + "step: 3014, Red action: do-nothing, Blue reward:0.85\n", + "step: 3015, Red action: do-nothing, Blue reward:0.85\n", + "step: 3016, Red action: do-nothing, Blue reward:0.85\n", + "step: 3017, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3018, Red action: do-nothing, Blue reward:0.85\n", + "step: 3019, Red action: do-nothing, Blue reward:0.85\n", + "step: 3020, Red action: do-nothing, Blue reward:0.85\n", + "step: 3021, Red action: do-nothing, Blue reward:0.85\n", + "step: 3022, Red action: do-nothing, Blue reward:0.85\n", + "step: 3023, Red action: do-nothing, Blue reward:0.85\n", + "step: 3024, Red action: do-nothing, Blue reward:0.85\n", + "step: 3025, Red action: do-nothing, Blue reward:0.85\n", + "step: 3026, Red action: do-nothing, Blue reward:0.85\n", + "step: 3027, Red action: do-nothing, Blue reward:0.85\n", + "step: 3028, Red action: do-nothing, Blue reward:0.85\n", + "step: 3029, Red action: do-nothing, Blue reward:0.85\n", + "step: 3030, Red action: do-nothing, Blue reward:0.85\n", + "step: 3031, Red action: do-nothing, Blue reward:0.85\n", + "step: 3032, Red action: do-nothing, Blue reward:0.85\n", + "step: 3033, Red action: do-nothing, Blue reward:0.85\n", + "step: 3034, Red action: do-nothing, Blue reward:0.85\n", + "step: 3035, Red action: do-nothing, Blue reward:0.85\n", + "step: 3036, Red action: do-nothing, Blue reward:0.85\n", + "step: 3037, Red action: do-nothing, Blue reward:0.85\n", + "step: 3038, Red action: do-nothing, Blue reward:0.85\n", + "step: 3039, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3040, Red action: do-nothing, Blue reward:0.85\n", + "step: 3041, Red action: do-nothing, Blue reward:0.85\n", + "step: 3042, Red action: do-nothing, Blue reward:0.85\n", + "step: 3043, Red action: do-nothing, Blue reward:0.85\n", + "step: 3044, Red action: do-nothing, Blue reward:0.85\n", + "step: 3045, Red action: do-nothing, Blue reward:0.85\n", + "step: 3046, Red action: do-nothing, Blue reward:0.85\n", + "step: 3047, Red action: do-nothing, Blue reward:0.85\n", + "step: 3048, Red action: do-nothing, Blue reward:0.85\n", + "step: 3049, Red action: do-nothing, Blue reward:0.85\n", + "step: 3050, Red action: do-nothing, Blue reward:0.85\n", + "step: 3051, Red action: do-nothing, Blue reward:0.85\n", + "step: 3052, Red action: do-nothing, Blue reward:0.85\n", + "step: 3053, Red action: do-nothing, Blue reward:0.85\n", + "step: 3054, Red action: do-nothing, Blue reward:0.85\n", + "step: 3055, Red action: do-nothing, Blue reward:0.85\n", + "step: 3056, Red action: do-nothing, Blue reward:0.85\n", + "step: 3057, Red action: do-nothing, Blue reward:0.85\n", + "step: 3058, Red action: do-nothing, Blue reward:0.85\n", + "step: 3059, Red action: do-nothing, Blue reward:0.85\n", + "step: 3060, Red action: do-nothing, Blue reward:0.85\n", + "step: 3061, Red action: do-nothing, Blue reward:0.85\n", + "step: 3062, Red action: do-nothing, Blue reward:0.85\n", + "step: 3063, Red action: do-nothing, Blue reward:0.85\n", + "step: 3064, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3065, Red action: do-nothing, Blue reward:0.85\n", + "step: 3066, Red action: do-nothing, Blue reward:0.85\n", + "step: 3067, Red action: do-nothing, Blue reward:0.85\n", + "step: 3068, Red action: do-nothing, Blue reward:0.85\n", + "step: 3069, Red action: do-nothing, Blue reward:0.85\n", + "step: 3070, Red action: do-nothing, Blue reward:0.85\n", + "step: 3071, Red action: do-nothing, Blue reward:0.85\n", + "step: 3072, Red action: do-nothing, Blue reward:0.85\n", + "step: 3073, Red action: do-nothing, Blue reward:0.85\n", + "step: 3074, Red action: do-nothing, Blue reward:0.85\n", + "step: 3075, Red action: do-nothing, Blue reward:0.85\n", + "step: 3076, Red action: do-nothing, Blue reward:0.85\n", + "step: 3077, Red action: do-nothing, Blue reward:0.85\n", + "step: 3078, Red action: do-nothing, Blue reward:0.85\n", + "step: 3079, Red action: do-nothing, Blue reward:0.85\n", + "step: 3080, Red action: do-nothing, Blue reward:0.85\n", + "step: 3081, Red action: do-nothing, Blue reward:0.85\n", + "step: 3082, Red action: do-nothing, Blue reward:0.85\n", + "step: 3083, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3084, Red action: do-nothing, Blue reward:0.85\n", + "step: 3085, Red action: do-nothing, Blue reward:0.85\n", + "step: 3086, Red action: do-nothing, Blue reward:0.85\n", + "step: 3087, Red action: do-nothing, Blue reward:0.85\n", + "step: 3088, Red action: do-nothing, Blue reward:0.85\n", + "step: 3089, Red action: do-nothing, Blue reward:0.85\n", + "step: 3090, Red action: do-nothing, Blue reward:0.85\n", + "step: 3091, Red action: do-nothing, Blue reward:0.85\n", + "step: 3092, Red action: do-nothing, Blue reward:0.85\n", + "step: 3093, Red action: do-nothing, Blue reward:0.85\n", + "step: 3094, Red action: do-nothing, Blue reward:0.85\n", + "step: 3095, Red action: do-nothing, Blue reward:0.85\n", + "step: 3096, Red action: do-nothing, Blue reward:0.85\n", + "step: 3097, Red action: do-nothing, Blue reward:0.85\n", + "step: 3098, Red action: do-nothing, Blue reward:0.85\n", + "step: 3099, Red action: do-nothing, Blue reward:0.85\n", + "step: 3100, Red action: do-nothing, Blue reward:0.85\n", + "step: 3101, Red action: do-nothing, Blue reward:0.85\n", + "step: 3102, Red action: do-nothing, Blue reward:0.85\n", + "step: 3103, Red action: do-nothing, Blue reward:0.85\n", + "step: 3104, Red action: do-nothing, Blue reward:0.85\n", + "step: 3105, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3106, Red action: do-nothing, Blue reward:0.85\n", + "step: 3107, Red action: do-nothing, Blue reward:0.85\n", + "step: 3108, Red action: do-nothing, Blue reward:0.85\n", + "step: 3109, Red action: do-nothing, Blue reward:0.85\n", + "step: 3110, Red action: do-nothing, Blue reward:0.85\n", + "step: 3111, Red action: do-nothing, Blue reward:0.85\n", + "step: 3112, Red action: do-nothing, Blue reward:0.85\n", + "step: 3113, Red action: do-nothing, Blue reward:0.85\n", + "step: 3114, Red action: do-nothing, Blue reward:0.85\n", + "step: 3115, Red action: do-nothing, Blue reward:0.85\n", + "step: 3116, Red action: do-nothing, Blue reward:0.85\n", + "step: 3117, Red action: do-nothing, Blue reward:0.85\n", + "step: 3118, Red action: do-nothing, Blue reward:0.85\n", + "step: 3119, Red action: do-nothing, Blue reward:0.85\n", + "step: 3120, Red action: do-nothing, Blue reward:0.85\n", + "step: 3121, Red action: do-nothing, Blue reward:0.85\n", + "step: 3122, Red action: do-nothing, Blue reward:0.85\n", + "step: 3123, Red action: do-nothing, Blue reward:0.85\n", + "step: 3124, Red action: do-nothing, Blue reward:0.85\n", + "step: 3125, Red action: do-nothing, Blue reward:0.85\n", + "step: 3126, Red action: do-nothing, Blue reward:0.85\n", + "step: 3127, Red action: do-nothing, Blue reward:0.85\n", + "step: 3128, Red action: do-nothing, Blue reward:0.85\n", + "step: 3129, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3130, Red action: do-nothing, Blue reward:0.85\n", + "step: 3131, Red action: do-nothing, Blue reward:0.85\n", + "step: 3132, Red action: do-nothing, Blue reward:0.85\n", + "step: 3133, Red action: do-nothing, Blue reward:0.85\n", + "step: 3134, Red action: do-nothing, Blue reward:0.85\n", + "step: 3135, Red action: do-nothing, Blue reward:0.85\n", + "step: 3136, Red action: do-nothing, Blue reward:0.85\n", + "step: 3137, Red action: do-nothing, Blue reward:0.85\n", + "step: 3138, Red action: do-nothing, Blue reward:0.85\n", + "step: 3139, Red action: do-nothing, Blue reward:0.85\n", + "step: 3140, Red action: do-nothing, Blue reward:0.85\n", + "step: 3141, Red action: do-nothing, Blue reward:0.85\n", + "step: 3142, Red action: do-nothing, Blue reward:0.85\n", + "step: 3143, Red action: do-nothing, Blue reward:0.85\n", + "step: 3144, Red action: do-nothing, Blue reward:0.85\n", + "step: 3145, Red action: do-nothing, Blue reward:0.85\n", + "step: 3146, Red action: do-nothing, Blue reward:0.85\n", + "step: 3147, Red action: do-nothing, Blue reward:0.85\n", + "step: 3148, Red action: do-nothing, Blue reward:0.85\n", + "step: 3149, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3150, Red action: do-nothing, Blue reward:0.85\n", + "step: 3151, Red action: do-nothing, Blue reward:0.85\n", + "step: 3152, Red action: do-nothing, Blue reward:0.85\n", + "step: 3153, Red action: do-nothing, Blue reward:0.85\n", + "step: 3154, Red action: do-nothing, Blue reward:0.85\n", + "step: 3155, Red action: do-nothing, Blue reward:0.85\n", + "step: 3156, Red action: do-nothing, Blue reward:0.85\n", + "step: 3157, Red action: do-nothing, Blue reward:0.85\n", + "step: 3158, Red action: do-nothing, Blue reward:0.85\n", + "step: 3159, Red action: do-nothing, Blue reward:0.85\n", + "step: 3160, Red action: do-nothing, Blue reward:0.85\n", + "step: 3161, Red action: do-nothing, Blue reward:0.85\n", + "step: 3162, Red action: do-nothing, Blue reward:0.85\n", + "step: 3163, Red action: do-nothing, Blue reward:0.85\n", + "step: 3164, Red action: do-nothing, Blue reward:0.85\n", + "step: 3165, Red action: do-nothing, Blue reward:0.85\n", + "step: 3166, Red action: do-nothing, Blue reward:0.85\n", + "step: 3167, Red action: do-nothing, Blue reward:0.85\n", + "step: 3168, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3169, Red action: do-nothing, Blue reward:0.85\n", + "step: 3170, Red action: do-nothing, Blue reward:0.85\n", + "step: 3171, Red action: do-nothing, Blue reward:0.85\n", + "step: 3172, Red action: do-nothing, Blue reward:0.85\n", + "step: 3173, Red action: do-nothing, Blue reward:0.85\n", + "step: 3174, Red action: do-nothing, Blue reward:0.85\n", + "step: 3175, Red action: do-nothing, Blue reward:0.85\n", + "step: 3176, Red action: do-nothing, Blue reward:0.85\n", + "step: 3177, Red action: do-nothing, Blue reward:0.85\n", + "step: 3178, Red action: do-nothing, Blue reward:0.85\n", + "step: 3179, Red action: do-nothing, Blue reward:0.85\n", + "step: 3180, Red action: do-nothing, Blue reward:0.85\n", + "step: 3181, Red action: do-nothing, Blue reward:0.85\n", + "step: 3182, Red action: do-nothing, Blue reward:0.85\n", + "step: 3183, Red action: do-nothing, Blue reward:0.85\n", + "step: 3184, Red action: do-nothing, Blue reward:0.85\n", + "step: 3185, Red action: do-nothing, Blue reward:0.85\n", + "step: 3186, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3187, Red action: do-nothing, Blue reward:0.85\n", + "step: 3188, Red action: do-nothing, Blue reward:0.85\n", + "step: 3189, Red action: do-nothing, Blue reward:0.85\n", + "step: 3190, Red action: do-nothing, Blue reward:0.85\n", + "step: 3191, Red action: do-nothing, Blue reward:0.85\n", + "step: 3192, Red action: do-nothing, Blue reward:0.85\n", + "step: 3193, Red action: do-nothing, Blue reward:0.85\n", + "step: 3194, Red action: do-nothing, Blue reward:0.85\n", + "step: 3195, Red action: do-nothing, Blue reward:0.85\n", + "step: 3196, Red action: do-nothing, Blue reward:0.85\n", + "step: 3197, Red action: do-nothing, Blue reward:0.85\n", + "step: 3198, Red action: do-nothing, Blue reward:0.85\n", + "step: 3199, Red action: do-nothing, Blue reward:0.85\n", + "step: 3200, Red action: do-nothing, Blue reward:0.85\n", + "step: 3201, Red action: do-nothing, Blue reward:0.85\n", + "step: 3202, Red action: do-nothing, Blue reward:0.85\n", + "step: 3203, Red action: do-nothing, Blue reward:0.85\n", + "step: 3204, Red action: do-nothing, Blue reward:0.85\n", + "step: 3205, Red action: do-nothing, Blue reward:0.85\n", + "step: 3206, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3207, Red action: do-nothing, Blue reward:0.85\n", + "step: 3208, Red action: do-nothing, Blue reward:0.85\n", + "step: 3209, Red action: do-nothing, Blue reward:0.85\n", + "step: 3210, Red action: do-nothing, Blue reward:0.85\n", + "step: 3211, Red action: do-nothing, Blue reward:0.85\n", + "step: 3212, Red action: do-nothing, Blue reward:0.85\n", + "step: 3213, Red action: do-nothing, Blue reward:0.85\n", + "step: 3214, Red action: do-nothing, Blue reward:0.85\n", + "step: 3215, Red action: do-nothing, Blue reward:0.85\n", + "step: 3216, Red action: do-nothing, Blue reward:0.85\n", + "step: 3217, Red action: do-nothing, Blue reward:0.85\n", + "step: 3218, Red action: do-nothing, Blue reward:0.85\n", + "step: 3219, Red action: do-nothing, Blue reward:0.85\n", + "step: 3220, Red action: do-nothing, Blue reward:0.85\n", + "step: 3221, Red action: do-nothing, Blue reward:0.85\n", + "step: 3222, Red action: do-nothing, Blue reward:0.85\n", + "step: 3223, Red action: do-nothing, Blue reward:0.85\n", + "step: 3224, Red action: do-nothing, Blue reward:0.85\n", + "step: 3225, Red action: do-nothing, Blue reward:0.85\n", + "step: 3226, Red action: do-nothing, Blue reward:0.85\n", + "step: 3227, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3228, Red action: do-nothing, Blue reward:0.85\n", + "step: 3229, Red action: do-nothing, Blue reward:0.85\n", + "step: 3230, Red action: do-nothing, Blue reward:0.85\n", + "step: 3231, Red action: do-nothing, Blue reward:0.85\n", + "step: 3232, Red action: do-nothing, Blue reward:0.85\n", + "step: 3233, Red action: do-nothing, Blue reward:0.85\n", + "step: 3234, Red action: do-nothing, Blue reward:0.85\n", + "step: 3235, Red action: do-nothing, Blue reward:0.85\n", + "step: 3236, Red action: do-nothing, Blue reward:0.85\n", + "step: 3237, Red action: do-nothing, Blue reward:0.85\n", + "step: 3238, Red action: do-nothing, Blue reward:0.85\n", + "step: 3239, Red action: do-nothing, Blue reward:0.85\n", + "step: 3240, Red action: do-nothing, Blue reward:0.85\n", + "step: 3241, Red action: do-nothing, Blue reward:0.85\n", + "step: 3242, Red action: do-nothing, Blue reward:0.85\n", + "step: 3243, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3244, Red action: do-nothing, Blue reward:0.85\n", + "step: 3245, Red action: do-nothing, Blue reward:0.85\n", + "step: 3246, Red action: do-nothing, Blue reward:0.85\n", + "step: 3247, Red action: do-nothing, Blue reward:0.85\n", + "step: 3248, Red action: do-nothing, Blue reward:0.85\n", + "step: 3249, Red action: do-nothing, Blue reward:0.85\n", + "step: 3250, Red action: do-nothing, Blue reward:0.85\n", + "step: 3251, Red action: do-nothing, Blue reward:0.85\n", + "step: 3252, Red action: do-nothing, Blue reward:0.85\n", + "step: 3253, Red action: do-nothing, Blue reward:0.85\n", + "step: 3254, Red action: do-nothing, Blue reward:0.85\n", + "step: 3255, Red action: do-nothing, Blue reward:0.85\n", + "step: 3256, Red action: do-nothing, Blue reward:0.85\n", + "step: 3257, Red action: do-nothing, Blue reward:0.85\n", + "step: 3258, Red action: do-nothing, Blue reward:0.85\n", + "step: 3259, Red action: do-nothing, Blue reward:0.85\n", + "step: 3260, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3261, Red action: do-nothing, Blue reward:0.85\n", + "step: 3262, Red action: do-nothing, Blue reward:0.85\n", + "step: 3263, Red action: do-nothing, Blue reward:0.85\n", + "step: 3264, Red action: do-nothing, Blue reward:0.85\n", + "step: 3265, Red action: do-nothing, Blue reward:0.85\n", + "step: 3266, Red action: do-nothing, Blue reward:0.85\n", + "step: 3267, Red action: do-nothing, Blue reward:0.85\n", + "step: 3268, Red action: do-nothing, Blue reward:0.85\n", + "step: 3269, Red action: do-nothing, Blue reward:0.85\n", + "step: 3270, Red action: do-nothing, Blue reward:0.85\n", + "step: 3271, Red action: do-nothing, Blue reward:0.85\n", + "step: 3272, Red action: do-nothing, Blue reward:0.85\n", + "step: 3273, Red action: do-nothing, Blue reward:0.85\n", + "step: 3274, Red action: do-nothing, Blue reward:0.85\n", + "step: 3275, Red action: do-nothing, Blue reward:0.85\n", + "step: 3276, Red action: do-nothing, Blue reward:0.85\n", + "step: 3277, Red action: do-nothing, Blue reward:0.85\n", + "step: 3278, Red action: do-nothing, Blue reward:0.85\n", + "step: 3279, Red action: do-nothing, Blue reward:0.85\n", + "step: 3280, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3281, Red action: do-nothing, Blue reward:0.85\n", + "step: 3282, Red action: do-nothing, Blue reward:0.85\n", + "step: 3283, Red action: do-nothing, Blue reward:0.85\n", + "step: 3284, Red action: do-nothing, Blue reward:0.85\n", + "step: 3285, Red action: do-nothing, Blue reward:0.85\n", + "step: 3286, Red action: do-nothing, Blue reward:0.85\n", + "step: 3287, Red action: do-nothing, Blue reward:0.85\n", + "step: 3288, Red action: do-nothing, Blue reward:0.85\n", + "step: 3289, Red action: do-nothing, Blue reward:0.85\n", + "step: 3290, Red action: do-nothing, Blue reward:0.85\n", + "step: 3291, Red action: do-nothing, Blue reward:0.85\n", + "step: 3292, Red action: do-nothing, Blue reward:0.85\n", + "step: 3293, Red action: do-nothing, Blue reward:0.85\n", + "step: 3294, Red action: do-nothing, Blue reward:0.85\n", + "step: 3295, Red action: do-nothing, Blue reward:0.85\n", + "step: 3296, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3297, Red action: do-nothing, Blue reward:0.85\n", + "step: 3298, Red action: do-nothing, Blue reward:0.85\n", + "step: 3299, Red action: do-nothing, Blue reward:0.85\n", + "step: 3300, Red action: do-nothing, Blue reward:0.85\n", + "step: 3301, Red action: do-nothing, Blue reward:0.85\n", + "step: 3302, Red action: do-nothing, Blue reward:0.85\n", + "step: 3303, Red action: do-nothing, Blue reward:0.85\n", + "step: 3304, Red action: do-nothing, Blue reward:0.85\n", + "step: 3305, Red action: do-nothing, Blue reward:0.85\n", + "step: 3306, Red action: do-nothing, Blue reward:0.85\n", + "step: 3307, Red action: do-nothing, Blue reward:0.85\n", + "step: 3308, Red action: do-nothing, Blue reward:0.85\n", + "step: 3309, Red action: do-nothing, Blue reward:0.85\n", + "step: 3310, Red action: do-nothing, Blue reward:0.85\n", + "step: 3311, Red action: do-nothing, Blue reward:0.85\n", + "step: 3312, Red action: do-nothing, Blue reward:0.85\n", + "step: 3313, Red action: do-nothing, Blue reward:0.85\n", + "step: 3314, Red action: do-nothing, Blue reward:0.85\n", + "step: 3315, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3316, Red action: do-nothing, Blue reward:0.85\n", + "step: 3317, Red action: do-nothing, Blue reward:0.85\n", + "step: 3318, Red action: do-nothing, Blue reward:0.85\n", + "step: 3319, Red action: do-nothing, Blue reward:0.85\n", + "step: 3320, Red action: do-nothing, Blue reward:0.85\n", + "step: 3321, Red action: do-nothing, Blue reward:0.85\n", + "step: 3322, Red action: do-nothing, Blue reward:0.85\n", + "step: 3323, Red action: do-nothing, Blue reward:0.85\n", + "step: 3324, Red action: do-nothing, Blue reward:0.85\n", + "step: 3325, Red action: do-nothing, Blue reward:0.85\n", + "step: 3326, Red action: do-nothing, Blue reward:0.85\n", + "step: 3327, Red action: do-nothing, Blue reward:0.85\n", + "step: 3328, Red action: do-nothing, Blue reward:0.85\n", + "step: 3329, Red action: do-nothing, Blue reward:0.85\n", + "step: 3330, Red action: do-nothing, Blue reward:0.85\n", + "step: 3331, Red action: do-nothing, Blue reward:0.85\n", + "step: 3332, Red action: do-nothing, Blue reward:0.85\n", + "step: 3333, Red action: do-nothing, Blue reward:0.85\n", + "step: 3334, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3335, Red action: do-nothing, Blue reward:0.85\n", + "step: 3336, Red action: do-nothing, Blue reward:0.85\n", + "step: 3337, Red action: do-nothing, Blue reward:0.85\n", + "step: 3338, Red action: do-nothing, Blue reward:0.85\n", + "step: 3339, Red action: do-nothing, Blue reward:0.85\n", + "step: 3340, Red action: do-nothing, Blue reward:0.85\n", + "step: 3341, Red action: do-nothing, Blue reward:0.85\n", + "step: 3342, Red action: do-nothing, Blue reward:0.85\n", + "step: 3343, Red action: do-nothing, Blue reward:0.85\n", + "step: 3344, Red action: do-nothing, Blue reward:0.85\n", + "step: 3345, Red action: do-nothing, Blue reward:0.85\n", + "step: 3346, Red action: do-nothing, Blue reward:0.85\n", + "step: 3347, Red action: do-nothing, Blue reward:0.85\n", + "step: 3348, Red action: do-nothing, Blue reward:0.85\n", + "step: 3349, Red action: do-nothing, Blue reward:0.85\n", + "step: 3350, Red action: do-nothing, Blue reward:0.85\n", + "step: 3351, Red action: do-nothing, Blue reward:0.85\n", + "step: 3352, Red action: do-nothing, Blue reward:0.85\n", + "step: 3353, Red action: do-nothing, Blue reward:0.85\n", + "step: 3354, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3355, Red action: do-nothing, Blue reward:0.85\n", + "step: 3356, Red action: do-nothing, Blue reward:0.85\n", + "step: 3357, Red action: do-nothing, Blue reward:0.85\n", + "step: 3358, Red action: do-nothing, Blue reward:0.85\n", + "step: 3359, Red action: do-nothing, Blue reward:0.85\n", + "step: 3360, Red action: do-nothing, Blue reward:0.85\n", + "step: 3361, Red action: do-nothing, Blue reward:0.85\n", + "step: 3362, Red action: do-nothing, Blue reward:0.85\n", + "step: 3363, Red action: do-nothing, Blue reward:0.85\n", + "step: 3364, Red action: do-nothing, Blue reward:0.85\n", + "step: 3365, Red action: do-nothing, Blue reward:0.85\n", + "step: 3366, Red action: do-nothing, Blue reward:0.85\n", + "step: 3367, Red action: do-nothing, Blue reward:0.85\n", + "step: 3368, Red action: do-nothing, Blue reward:0.85\n", + "step: 3369, Red action: do-nothing, Blue reward:0.85\n", + "step: 3370, Red action: do-nothing, Blue reward:0.85\n", + "step: 3371, Red action: do-nothing, Blue reward:0.85\n", + "step: 3372, Red action: do-nothing, Blue reward:0.85\n", + "step: 3373, Red action: do-nothing, Blue reward:0.85\n", + "step: 3374, Red action: do-nothing, Blue reward:0.85\n", + "step: 3375, Red action: do-nothing, Blue reward:0.85\n", + "step: 3376, Red action: do-nothing, Blue reward:0.85\n", + "step: 3377, Red action: do-nothing, Blue reward:0.85\n", + "step: 3378, Red action: do-nothing, Blue reward:0.85\n", + "step: 3379, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3380, Red action: do-nothing, Blue reward:0.85\n", + "step: 3381, Red action: do-nothing, Blue reward:0.85\n", + "step: 3382, Red action: do-nothing, Blue reward:0.85\n", + "step: 3383, Red action: do-nothing, Blue reward:0.85\n", + "step: 3384, Red action: do-nothing, Blue reward:0.85\n", + "step: 3385, Red action: do-nothing, Blue reward:0.85\n", + "step: 3386, Red action: do-nothing, Blue reward:0.85\n", + "step: 3387, Red action: do-nothing, Blue reward:0.85\n", + "step: 3388, Red action: do-nothing, Blue reward:0.85\n", + "step: 3389, Red action: do-nothing, Blue reward:0.85\n", + "step: 3390, Red action: do-nothing, Blue reward:0.85\n", + "step: 3391, Red action: do-nothing, Blue reward:0.85\n", + "step: 3392, Red action: do-nothing, Blue reward:0.85\n", + "step: 3393, Red action: do-nothing, Blue reward:0.85\n", + "step: 3394, Red action: do-nothing, Blue reward:0.85\n", + "step: 3395, Red action: do-nothing, Blue reward:0.85\n", + "step: 3396, Red action: do-nothing, Blue reward:0.85\n", + "step: 3397, Red action: do-nothing, Blue reward:0.85\n", + "step: 3398, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3399, Red action: do-nothing, Blue reward:0.85\n", + "step: 3400, Red action: do-nothing, Blue reward:0.85\n", + "step: 3401, Red action: do-nothing, Blue reward:0.85\n", + "step: 3402, Red action: do-nothing, Blue reward:0.85\n", + "step: 3403, Red action: do-nothing, Blue reward:0.85\n", + "step: 3404, Red action: do-nothing, Blue reward:0.85\n", + "step: 3405, Red action: do-nothing, Blue reward:0.85\n", + "step: 3406, Red action: do-nothing, Blue reward:0.85\n", + "step: 3407, Red action: do-nothing, Blue reward:0.85\n", + "step: 3408, Red action: do-nothing, Blue reward:0.85\n", + "step: 3409, Red action: do-nothing, Blue reward:0.85\n", + "step: 3410, Red action: do-nothing, Blue reward:0.85\n", + "step: 3411, Red action: do-nothing, Blue reward:0.85\n", + "step: 3412, Red action: do-nothing, Blue reward:0.85\n", + "step: 3413, Red action: do-nothing, Blue reward:0.85\n", + "step: 3414, Red action: do-nothing, Blue reward:0.85\n", + "step: 3415, Red action: do-nothing, Blue reward:0.85\n", + "step: 3416, Red action: do-nothing, Blue reward:0.85\n", + "step: 3417, Red action: do-nothing, Blue reward:0.85\n", + "step: 3418, Red action: do-nothing, Blue reward:0.85\n", + "step: 3419, Red action: do-nothing, Blue reward:0.85\n", + "step: 3420, Red action: do-nothing, Blue reward:0.85\n", + "step: 3421, Red action: do-nothing, Blue reward:0.85\n", + "step: 3422, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3423, Red action: do-nothing, Blue reward:0.85\n", + "step: 3424, Red action: do-nothing, Blue reward:0.85\n", + "step: 3425, Red action: do-nothing, Blue reward:0.85\n", + "step: 3426, Red action: do-nothing, Blue reward:0.85\n", + "step: 3427, Red action: do-nothing, Blue reward:0.85\n", + "step: 3428, Red action: do-nothing, Blue reward:0.85\n", + "step: 3429, Red action: do-nothing, Blue reward:0.85\n", + "step: 3430, Red action: do-nothing, Blue reward:0.85\n", + "step: 3431, Red action: do-nothing, Blue reward:0.85\n", + "step: 3432, Red action: do-nothing, Blue reward:0.85\n", + "step: 3433, Red action: do-nothing, Blue reward:0.85\n", + "step: 3434, Red action: do-nothing, Blue reward:0.85\n", + "step: 3435, Red action: do-nothing, Blue reward:0.85\n", + "step: 3436, Red action: do-nothing, Blue reward:0.85\n", + "step: 3437, Red action: do-nothing, Blue reward:0.85\n", + "step: 3438, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3439, Red action: do-nothing, Blue reward:0.85\n", + "step: 3440, Red action: do-nothing, Blue reward:0.85\n", + "step: 3441, Red action: do-nothing, Blue reward:0.85\n", + "step: 3442, Red action: do-nothing, Blue reward:0.85\n", + "step: 3443, Red action: do-nothing, Blue reward:0.85\n", + "step: 3444, Red action: do-nothing, Blue reward:0.85\n", + "step: 3445, Red action: do-nothing, Blue reward:0.85\n", + "step: 3446, Red action: do-nothing, Blue reward:0.85\n", + "step: 3447, Red action: do-nothing, Blue reward:0.85\n", + "step: 3448, Red action: do-nothing, Blue reward:0.85\n", + "step: 3449, Red action: do-nothing, Blue reward:0.85\n", + "step: 3450, Red action: do-nothing, Blue reward:0.85\n", + "step: 3451, Red action: do-nothing, Blue reward:0.85\n", + "step: 3452, Red action: do-nothing, Blue reward:0.85\n", + "step: 3453, Red action: do-nothing, Blue reward:0.85\n", + "step: 3454, Red action: do-nothing, Blue reward:0.85\n", + "step: 3455, Red action: do-nothing, Blue reward:0.85\n", + "step: 3456, Red action: do-nothing, Blue reward:0.85\n", + "step: 3457, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3458, Red action: do-nothing, Blue reward:0.85\n", + "step: 3459, Red action: do-nothing, Blue reward:0.85\n", + "step: 3460, Red action: do-nothing, Blue reward:0.85\n", + "step: 3461, Red action: do-nothing, Blue reward:0.85\n", + "step: 3462, Red action: do-nothing, Blue reward:0.85\n", + "step: 3463, Red action: do-nothing, Blue reward:0.85\n", + "step: 3464, Red action: do-nothing, Blue reward:0.85\n", + "step: 3465, Red action: do-nothing, Blue reward:0.85\n", + "step: 3466, Red action: do-nothing, Blue reward:0.85\n", + "step: 3467, Red action: do-nothing, Blue reward:0.85\n", + "step: 3468, Red action: do-nothing, Blue reward:0.85\n", + "step: 3469, Red action: do-nothing, Blue reward:0.85\n", + "step: 3470, Red action: do-nothing, Blue reward:0.85\n", + "step: 3471, Red action: do-nothing, Blue reward:0.85\n", + "step: 3472, Red action: do-nothing, Blue reward:0.85\n", + "step: 3473, Red action: do-nothing, Blue reward:0.85\n", + "step: 3474, Red action: do-nothing, Blue reward:0.85\n", + "step: 3475, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3476, Red action: do-nothing, Blue reward:0.85\n", + "step: 3477, Red action: do-nothing, Blue reward:0.85\n", + "step: 3478, Red action: do-nothing, Blue reward:0.85\n", + "step: 3479, Red action: do-nothing, Blue reward:0.85\n", + "step: 3480, Red action: do-nothing, Blue reward:0.85\n", + "step: 3481, Red action: do-nothing, Blue reward:0.85\n", + "step: 3482, Red action: do-nothing, Blue reward:0.85\n", + "step: 3483, Red action: do-nothing, Blue reward:0.85\n", + "step: 3484, Red action: do-nothing, Blue reward:0.85\n", + "step: 3485, Red action: do-nothing, Blue reward:0.85\n", + "step: 3486, Red action: do-nothing, Blue reward:0.85\n", + "step: 3487, Red action: do-nothing, Blue reward:0.85\n", + "step: 3488, Red action: do-nothing, Blue reward:0.85\n", + "step: 3489, Red action: do-nothing, Blue reward:0.85\n", + "step: 3490, Red action: do-nothing, Blue reward:0.85\n", + "step: 3491, Red action: do-nothing, Blue reward:0.85\n", + "step: 3492, Red action: do-nothing, Blue reward:0.85\n", + "step: 3493, Red action: do-nothing, Blue reward:0.85\n", + "step: 3494, Red action: do-nothing, Blue reward:0.85\n", + "step: 3495, Red action: do-nothing, Blue reward:0.85\n", + "step: 3496, Red action: do-nothing, Blue reward:0.85\n", + "step: 3497, Red action: do-nothing, Blue reward:0.85\n", + "step: 3498, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3499, Red action: do-nothing, Blue reward:0.85\n", + "step: 3500, Red action: do-nothing, Blue reward:0.85\n", + "step: 3501, Red action: do-nothing, Blue reward:0.85\n", + "step: 3502, Red action: do-nothing, Blue reward:0.85\n", + "step: 3503, Red action: do-nothing, Blue reward:0.85\n", + "step: 3504, Red action: do-nothing, Blue reward:0.85\n", + "step: 3505, Red action: do-nothing, Blue reward:0.85\n", + "step: 3506, Red action: do-nothing, Blue reward:0.85\n", + "step: 3507, Red action: do-nothing, Blue reward:0.85\n", + "step: 3508, Red action: do-nothing, Blue reward:0.85\n", + "step: 3509, Red action: do-nothing, Blue reward:0.85\n", + "step: 3510, Red action: do-nothing, Blue reward:0.85\n", + "step: 3511, Red action: do-nothing, Blue reward:0.85\n", + "step: 3512, Red action: do-nothing, Blue reward:0.85\n", + "step: 3513, Red action: do-nothing, Blue reward:0.85\n", + "step: 3514, Red action: do-nothing, Blue reward:0.85\n", + "step: 3515, Red action: do-nothing, Blue reward:0.85\n", + "step: 3516, Red action: do-nothing, Blue reward:0.85\n", + "step: 3517, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3518, Red action: do-nothing, Blue reward:0.85\n", + "step: 3519, Red action: do-nothing, Blue reward:0.85\n", + "step: 3520, Red action: do-nothing, Blue reward:0.85\n", + "step: 3521, Red action: do-nothing, Blue reward:0.85\n", + "step: 3522, Red action: do-nothing, Blue reward:0.85\n", + "step: 3523, Red action: do-nothing, Blue reward:0.85\n", + "step: 3524, Red action: do-nothing, Blue reward:0.85\n", + "step: 3525, Red action: do-nothing, Blue reward:0.85\n", + "step: 3526, Red action: do-nothing, Blue reward:0.85\n", + "step: 3527, Red action: do-nothing, Blue reward:0.85\n", + "step: 3528, Red action: do-nothing, Blue reward:0.85\n", + "step: 3529, Red action: do-nothing, Blue reward:0.85\n", + "step: 3530, Red action: do-nothing, Blue reward:0.85\n", + "step: 3531, Red action: do-nothing, Blue reward:0.85\n", + "step: 3532, Red action: do-nothing, Blue reward:0.85\n", + "step: 3533, Red action: do-nothing, Blue reward:0.85\n", + "step: 3534, Red action: do-nothing, Blue reward:0.85\n", + "step: 3535, Red action: do-nothing, Blue reward:0.85\n", + "step: 3536, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3537, Red action: do-nothing, Blue reward:0.85\n", + "step: 3538, Red action: do-nothing, Blue reward:0.85\n", + "step: 3539, Red action: do-nothing, Blue reward:0.85\n", + "step: 3540, Red action: do-nothing, Blue reward:0.85\n", + "step: 3541, Red action: do-nothing, Blue reward:0.85\n", + "step: 3542, Red action: do-nothing, Blue reward:0.85\n", + "step: 3543, Red action: do-nothing, Blue reward:0.85\n", + "step: 3544, Red action: do-nothing, Blue reward:0.85\n", + "step: 3545, Red action: do-nothing, Blue reward:0.85\n", + "step: 3546, Red action: do-nothing, Blue reward:0.85\n", + "step: 3547, Red action: do-nothing, Blue reward:0.85\n", + "step: 3548, Red action: do-nothing, Blue reward:0.85\n", + "step: 3549, Red action: do-nothing, Blue reward:0.85\n", + "step: 3550, Red action: do-nothing, Blue reward:0.85\n", + "step: 3551, Red action: do-nothing, Blue reward:0.85\n", + "step: 3552, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3553, Red action: do-nothing, Blue reward:0.85\n", + "step: 3554, Red action: do-nothing, Blue reward:0.85\n", + "step: 3555, Red action: do-nothing, Blue reward:0.85\n", + "step: 3556, Red action: do-nothing, Blue reward:0.85\n", + "step: 3557, Red action: do-nothing, Blue reward:0.85\n", + "step: 3558, Red action: do-nothing, Blue reward:0.85\n", + "step: 3559, Red action: do-nothing, Blue reward:0.85\n", + "step: 3560, Red action: do-nothing, Blue reward:0.85\n", + "step: 3561, Red action: do-nothing, Blue reward:0.85\n", + "step: 3562, Red action: do-nothing, Blue reward:0.85\n", + "step: 3563, Red action: do-nothing, Blue reward:0.85\n", + "step: 3564, Red action: do-nothing, Blue reward:0.85\n", + "step: 3565, Red action: do-nothing, Blue reward:0.85\n", + "step: 3566, Red action: do-nothing, Blue reward:0.85\n", + "step: 3567, Red action: do-nothing, Blue reward:0.85\n", + "step: 3568, Red action: do-nothing, Blue reward:0.85\n", + "step: 3569, Red action: do-nothing, Blue reward:0.85\n", + "step: 3570, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3571, Red action: do-nothing, Blue reward:0.85\n", + "step: 3572, Red action: do-nothing, Blue reward:0.85\n", + "step: 3573, Red action: do-nothing, Blue reward:0.85\n", + "step: 3574, Red action: do-nothing, Blue reward:0.85\n", + "step: 3575, Red action: do-nothing, Blue reward:0.85\n", + "step: 3576, Red action: do-nothing, Blue reward:0.85\n", + "step: 3577, Red action: do-nothing, Blue reward:0.85\n", + "step: 3578, Red action: do-nothing, Blue reward:0.85\n", + "step: 3579, Red action: do-nothing, Blue reward:0.85\n", + "step: 3580, Red action: do-nothing, Blue reward:0.85\n", + "step: 3581, Red action: do-nothing, Blue reward:0.85\n", + "step: 3582, Red action: do-nothing, Blue reward:0.85\n", + "step: 3583, Red action: do-nothing, Blue reward:0.85\n", + "step: 3584, Red action: do-nothing, Blue reward:0.85\n", + "step: 3585, Red action: do-nothing, Blue reward:0.85\n", + "step: 3586, Red action: do-nothing, Blue reward:0.85\n", + "step: 3587, Red action: do-nothing, Blue reward:0.85\n", + "step: 3588, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3589, Red action: do-nothing, Blue reward:0.85\n", + "step: 3590, Red action: do-nothing, Blue reward:0.85\n", + "step: 3591, Red action: do-nothing, Blue reward:0.85\n", + "step: 3592, Red action: do-nothing, Blue reward:0.85\n", + "step: 3593, Red action: do-nothing, Blue reward:0.85\n", + "step: 3594, Red action: do-nothing, Blue reward:0.85\n", + "step: 3595, Red action: do-nothing, Blue reward:0.85\n", + "step: 3596, Red action: do-nothing, Blue reward:0.85\n", + "step: 3597, Red action: do-nothing, Blue reward:0.85\n", + "step: 3598, Red action: do-nothing, Blue reward:0.85\n", + "step: 3599, Red action: do-nothing, Blue reward:0.85\n", + "step: 3600, Red action: do-nothing, Blue reward:0.85\n", + "step: 3601, Red action: do-nothing, Blue reward:0.85\n", + "step: 3602, Red action: do-nothing, Blue reward:0.85\n", + "step: 3603, Red action: do-nothing, Blue reward:0.85\n", + "step: 3604, Red action: do-nothing, Blue reward:0.85\n", + "step: 3605, Red action: do-nothing, Blue reward:0.85\n", + "step: 3606, Red action: do-nothing, Blue reward:0.85\n", + "step: 3607, Red action: do-nothing, Blue reward:0.85\n", + "step: 3608, Red action: do-nothing, Blue reward:0.85\n", + "step: 3609, Red action: do-nothing, Blue reward:0.85\n", + "step: 3610, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3611, Red action: do-nothing, Blue reward:0.85\n", + "step: 3612, Red action: do-nothing, Blue reward:0.85\n", + "step: 3613, Red action: do-nothing, Blue reward:0.85\n", + "step: 3614, Red action: do-nothing, Blue reward:0.85\n", + "step: 3615, Red action: do-nothing, Blue reward:0.85\n", + "step: 3616, Red action: do-nothing, Blue reward:0.85\n", + "step: 3617, Red action: do-nothing, Blue reward:0.85\n", + "step: 3618, Red action: do-nothing, Blue reward:0.85\n", + "step: 3619, Red action: do-nothing, Blue reward:0.85\n", + "step: 3620, Red action: do-nothing, Blue reward:0.85\n", + "step: 3621, Red action: do-nothing, Blue reward:0.85\n", + "step: 3622, Red action: do-nothing, Blue reward:0.85\n", + "step: 3623, Red action: do-nothing, Blue reward:0.85\n", + "step: 3624, Red action: do-nothing, Blue reward:0.85\n", + "step: 3625, Red action: do-nothing, Blue reward:0.85\n", + "step: 3626, Red action: do-nothing, Blue reward:0.85\n", + "step: 3627, Red action: do-nothing, Blue reward:0.85\n", + "step: 3628, Red action: do-nothing, Blue reward:0.85\n", + "step: 3629, Red action: do-nothing, Blue reward:0.85\n", + "step: 3630, Red action: do-nothing, Blue reward:0.85\n", + "step: 3631, Red action: do-nothing, Blue reward:0.85\n", + "step: 3632, Red action: do-nothing, Blue reward:0.85\n", + "step: 3633, Red action: do-nothing, Blue reward:0.85\n", + "step: 3634, Red action: do-nothing, Blue reward:0.85\n", + "step: 3635, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3636, Red action: do-nothing, Blue reward:0.85\n", + "step: 3637, Red action: do-nothing, Blue reward:0.85\n", + "step: 3638, Red action: do-nothing, Blue reward:0.85\n", + "step: 3639, Red action: do-nothing, Blue reward:0.85\n", + "step: 3640, Red action: do-nothing, Blue reward:0.85\n", + "step: 3641, Red action: do-nothing, Blue reward:0.85\n", + "step: 3642, Red action: do-nothing, Blue reward:0.85\n", + "step: 3643, Red action: do-nothing, Blue reward:0.85\n", + "step: 3644, Red action: do-nothing, Blue reward:0.85\n", + "step: 3645, Red action: do-nothing, Blue reward:0.85\n", + "step: 3646, Red action: do-nothing, Blue reward:0.85\n", + "step: 3647, Red action: do-nothing, Blue reward:0.85\n", + "step: 3648, Red action: do-nothing, Blue reward:0.85\n", + "step: 3649, Red action: do-nothing, Blue reward:0.85\n", + "step: 3650, Red action: do-nothing, Blue reward:0.85\n", + "step: 3651, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3652, Red action: do-nothing, Blue reward:0.85\n", + "step: 3653, Red action: do-nothing, Blue reward:0.85\n", + "step: 3654, Red action: do-nothing, Blue reward:0.85\n", + "step: 3655, Red action: do-nothing, Blue reward:0.85\n", + "step: 3656, Red action: do-nothing, Blue reward:0.85\n", + "step: 3657, Red action: do-nothing, Blue reward:0.85\n", + "step: 3658, Red action: do-nothing, Blue reward:0.85\n", + "step: 3659, Red action: do-nothing, Blue reward:0.85\n", + "step: 3660, Red action: do-nothing, Blue reward:0.85\n", + "step: 3661, Red action: do-nothing, Blue reward:0.85\n", + "step: 3662, Red action: do-nothing, Blue reward:0.85\n", + "step: 3663, Red action: do-nothing, Blue reward:0.85\n", + "step: 3664, Red action: do-nothing, Blue reward:0.85\n", + "step: 3665, Red action: do-nothing, Blue reward:0.85\n", + "step: 3666, Red action: do-nothing, Blue reward:0.85\n", + "step: 3667, Red action: do-nothing, Blue reward:0.85\n", + "step: 3668, Red action: do-nothing, Blue reward:0.85\n", + "step: 3669, Red action: do-nothing, Blue reward:0.85\n", + "step: 3670, Red action: do-nothing, Blue reward:0.85\n", + "step: 3671, Red action: do-nothing, Blue reward:0.85\n", + "step: 3672, Red action: do-nothing, Blue reward:0.85\n", + "step: 3673, Red action: do-nothing, Blue reward:0.85\n", + "step: 3674, Red action: do-nothing, Blue reward:0.85\n", + "step: 3675, Red action: do-nothing, Blue reward:0.85\n", + "step: 3676, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3677, Red action: do-nothing, Blue reward:0.85\n", + "step: 3678, Red action: do-nothing, Blue reward:0.85\n", + "step: 3679, Red action: do-nothing, Blue reward:0.85\n", + "step: 3680, Red action: do-nothing, Blue reward:0.85\n", + "step: 3681, Red action: do-nothing, Blue reward:0.85\n", + "step: 3682, Red action: do-nothing, Blue reward:0.85\n", + "step: 3683, Red action: do-nothing, Blue reward:0.85\n", + "step: 3684, Red action: do-nothing, Blue reward:0.85\n", + "step: 3685, Red action: do-nothing, Blue reward:0.85\n", + "step: 3686, Red action: do-nothing, Blue reward:0.85\n", + "step: 3687, Red action: do-nothing, Blue reward:0.85\n", + "step: 3688, Red action: do-nothing, Blue reward:0.85\n", + "step: 3689, Red action: do-nothing, Blue reward:0.85\n", + "step: 3690, Red action: do-nothing, Blue reward:0.85\n", + "step: 3691, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3692, Red action: do-nothing, Blue reward:0.85\n", + "step: 3693, Red action: do-nothing, Blue reward:0.85\n", + "step: 3694, Red action: do-nothing, Blue reward:0.85\n", + "step: 3695, Red action: do-nothing, Blue reward:0.85\n", + "step: 3696, Red action: do-nothing, Blue reward:0.85\n", + "step: 3697, Red action: do-nothing, Blue reward:0.85\n", + "step: 3698, Red action: do-nothing, Blue reward:0.85\n", + "step: 3699, Red action: do-nothing, Blue reward:0.85\n", + "step: 3700, Red action: do-nothing, Blue reward:0.85\n", + "step: 3701, Red action: do-nothing, Blue reward:0.85\n", + "step: 3702, Red action: do-nothing, Blue reward:0.85\n", + "step: 3703, Red action: do-nothing, Blue reward:0.85\n", + "step: 3704, Red action: do-nothing, Blue reward:0.85\n", + "step: 3705, Red action: do-nothing, Blue reward:0.85\n", + "step: 3706, Red action: do-nothing, Blue reward:0.85\n", + "step: 3707, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3708, Red action: do-nothing, Blue reward:0.85\n", + "step: 3709, Red action: do-nothing, Blue reward:0.85\n", + "step: 3710, Red action: do-nothing, Blue reward:0.85\n", + "step: 3711, Red action: do-nothing, Blue reward:0.85\n", + "step: 3712, Red action: do-nothing, Blue reward:0.85\n", + "step: 3713, Red action: do-nothing, Blue reward:0.85\n", + "step: 3714, Red action: do-nothing, Blue reward:0.85\n", + "step: 3715, Red action: do-nothing, Blue reward:0.85\n", + "step: 3716, Red action: do-nothing, Blue reward:0.85\n", + "step: 3717, Red action: do-nothing, Blue reward:0.85\n", + "step: 3718, Red action: do-nothing, Blue reward:0.85\n", + "step: 3719, Red action: do-nothing, Blue reward:0.85\n", + "step: 3720, Red action: do-nothing, Blue reward:0.85\n", + "step: 3721, Red action: do-nothing, Blue reward:0.85\n", + "step: 3722, Red action: do-nothing, Blue reward:0.85\n", + "step: 3723, Red action: do-nothing, Blue reward:0.85\n", + "step: 3724, Red action: do-nothing, Blue reward:0.85\n", + "step: 3725, Red action: do-nothing, Blue reward:0.85\n", + "step: 3726, Red action: do-nothing, Blue reward:0.85\n", + "step: 3727, Red action: do-nothing, Blue reward:0.85\n", + "step: 3728, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3729, Red action: do-nothing, Blue reward:0.85\n", + "step: 3730, Red action: do-nothing, Blue reward:0.85\n", + "step: 3731, Red action: do-nothing, Blue reward:0.85\n", + "step: 3732, Red action: do-nothing, Blue reward:0.85\n", + "step: 3733, Red action: do-nothing, Blue reward:0.85\n", + "step: 3734, Red action: do-nothing, Blue reward:0.85\n", + "step: 3735, Red action: do-nothing, Blue reward:0.85\n", + "step: 3736, Red action: do-nothing, Blue reward:0.85\n", + "step: 3737, Red action: do-nothing, Blue reward:0.85\n", + "step: 3738, Red action: do-nothing, Blue reward:0.85\n", + "step: 3739, Red action: do-nothing, Blue reward:0.85\n", + "step: 3740, Red action: do-nothing, Blue reward:0.85\n", + "step: 3741, Red action: do-nothing, Blue reward:0.85\n", + "step: 3742, Red action: do-nothing, Blue reward:0.85\n", + "step: 3743, Red action: do-nothing, Blue reward:0.85\n", + "step: 3744, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3745, Red action: do-nothing, Blue reward:0.85\n", + "step: 3746, Red action: do-nothing, Blue reward:0.85\n", + "step: 3747, Red action: do-nothing, Blue reward:0.85\n", + "step: 3748, Red action: do-nothing, Blue reward:0.85\n", + "step: 3749, Red action: do-nothing, Blue reward:0.85\n", + "step: 3750, Red action: do-nothing, Blue reward:0.85\n", + "step: 3751, Red action: do-nothing, Blue reward:0.85\n", + "step: 3752, Red action: do-nothing, Blue reward:0.85\n", + "step: 3753, Red action: do-nothing, Blue reward:0.85\n", + "step: 3754, Red action: do-nothing, Blue reward:0.85\n", + "step: 3755, Red action: do-nothing, Blue reward:0.85\n", + "step: 3756, Red action: do-nothing, Blue reward:0.85\n", + "step: 3757, Red action: do-nothing, Blue reward:0.85\n", + "step: 3758, Red action: do-nothing, Blue reward:0.85\n", + "step: 3759, Red action: do-nothing, Blue reward:0.85\n", + "step: 3760, Red action: do-nothing, Blue reward:0.85\n", + "step: 3761, Red action: do-nothing, Blue reward:0.85\n", + "step: 3762, Red action: do-nothing, Blue reward:0.85\n", + "step: 3763, Red action: do-nothing, Blue reward:0.85\n", + "step: 3764, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3765, Red action: do-nothing, Blue reward:0.85\n", + "step: 3766, Red action: do-nothing, Blue reward:0.85\n", + "step: 3767, Red action: do-nothing, Blue reward:0.85\n", + "step: 3768, Red action: do-nothing, Blue reward:0.85\n", + "step: 3769, Red action: do-nothing, Blue reward:0.85\n", + "step: 3770, Red action: do-nothing, Blue reward:0.85\n", + "step: 3771, Red action: do-nothing, Blue reward:0.85\n", + "step: 3772, Red action: do-nothing, Blue reward:0.85\n", + "step: 3773, Red action: do-nothing, Blue reward:0.85\n", + "step: 3774, Red action: do-nothing, Blue reward:0.85\n", + "step: 3775, Red action: do-nothing, Blue reward:0.85\n", + "step: 3776, Red action: do-nothing, Blue reward:0.85\n", + "step: 3777, Red action: do-nothing, Blue reward:0.85\n", + "step: 3778, Red action: do-nothing, Blue reward:0.85\n", + "step: 3779, Red action: do-nothing, Blue reward:0.85\n", + "step: 3780, Red action: do-nothing, Blue reward:0.85\n", + "step: 3781, Red action: do-nothing, Blue reward:0.85\n", + "step: 3782, Red action: do-nothing, Blue reward:0.85\n", + "step: 3783, Red action: do-nothing, Blue reward:0.85\n", + "step: 3784, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3785, Red action: do-nothing, Blue reward:0.85\n", + "step: 3786, Red action: do-nothing, Blue reward:0.85\n", + "step: 3787, Red action: do-nothing, Blue reward:0.85\n", + "step: 3788, Red action: do-nothing, Blue reward:0.85\n", + "step: 3789, Red action: do-nothing, Blue reward:0.85\n", + "step: 3790, Red action: do-nothing, Blue reward:0.85\n", + "step: 3791, Red action: do-nothing, Blue reward:0.85\n", + "step: 3792, Red action: do-nothing, Blue reward:0.85\n", + "step: 3793, Red action: do-nothing, Blue reward:0.85\n", + "step: 3794, Red action: do-nothing, Blue reward:0.85\n", + "step: 3795, Red action: do-nothing, Blue reward:0.85\n", + "step: 3796, Red action: do-nothing, Blue reward:0.85\n", + "step: 3797, Red action: do-nothing, Blue reward:0.85\n", + "step: 3798, Red action: do-nothing, Blue reward:0.85\n", + "step: 3799, Red action: do-nothing, Blue reward:0.85\n", + "step: 3800, Red action: do-nothing, Blue reward:0.85\n", + "step: 3801, Red action: do-nothing, Blue reward:0.85\n", + "step: 3802, Red action: do-nothing, Blue reward:0.85\n" + ] + }, + { + "ename": "KeyboardInterrupt", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[11], line 11\u001b[0m\n\u001b[1;32m 8\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mstep: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00menv\u001b[38;5;241m.\u001b[39mgame\u001b[38;5;241m.\u001b[39mstep_counter\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m, Red action: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00minfo[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124magent_actions\u001b[39m\u001b[38;5;124m'\u001b[39m][\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mdata_manipulation_attacker\u001b[39m\u001b[38;5;124m'\u001b[39m]\u001b[38;5;241m.\u001b[39maction\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m, Blue reward:\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mreward\u001b[38;5;132;01m:\u001b[39;00m\u001b[38;5;124m.2f\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m )\n\u001b[1;32m 10\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28mabs\u001b[39m(reward \u001b[38;5;241m-\u001b[39m \u001b[38;5;241m0.8\u001b[39m) \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m1e-5\u001b[39m:\n\u001b[0;32m---> 11\u001b[0m obs, reward, terminated, truncated, info \u001b[38;5;241m=\u001b[39m \u001b[43menv\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstep\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m)\u001b[49m \u001b[38;5;66;03m# do nothing\u001b[39;00m\n\u001b[1;32m 12\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mstep: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00menv\u001b[38;5;241m.\u001b[39mgame\u001b[38;5;241m.\u001b[39mstep_counter\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m, Red action: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00minfo[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124magent_actions\u001b[39m\u001b[38;5;124m'\u001b[39m][\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mdata_manipulation_attacker\u001b[39m\u001b[38;5;124m'\u001b[39m]\u001b[38;5;241m.\u001b[39maction\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m, Blue reward:\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mreward\u001b[38;5;132;01m:\u001b[39;00m\u001b[38;5;124m.2f\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m )\n\u001b[1;32m 13\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m env\u001b[38;5;241m.\u001b[39mgame\u001b[38;5;241m.\u001b[39mstep_counter \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m10000\u001b[39m:\n", + "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/session/environment.py:111\u001b[0m, in \u001b[0;36mPrimaiteGymEnv.step\u001b[0;34m(self, action)\u001b[0m\n\u001b[1;32m 109\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgame\u001b[38;5;241m.\u001b[39mapply_agent_actions()\n\u001b[1;32m 110\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgame\u001b[38;5;241m.\u001b[39madvance_timestep()\n\u001b[0;32m--> 111\u001b[0m state \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgame\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_sim_state\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 112\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgame\u001b[38;5;241m.\u001b[39mupdate_agents(state)\n\u001b[1;32m 114\u001b[0m next_obs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_get_obs() \u001b[38;5;66;03m# this doesn't update observation, just gets the current observation\u001b[39;00m\n", + "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/game/game.py:151\u001b[0m, in \u001b[0;36mPrimaiteGame.get_sim_state\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 149\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mget_sim_state\u001b[39m(\u001b[38;5;28mself\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Dict:\n\u001b[1;32m 150\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"Get the current state of the simulation.\"\"\"\u001b[39;00m\n\u001b[0;32m--> 151\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msimulation\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdescribe_state\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/simulator/sim_container.py:57\u001b[0m, in \u001b[0;36mSimulation.describe_state\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 46\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 47\u001b[0m \u001b[38;5;124;03mProduce a dictionary describing the current state of this object.\u001b[39;00m\n\u001b[1;32m 48\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 52\u001b[0m \u001b[38;5;124;03m:rtype: Dict\u001b[39;00m\n\u001b[1;32m 53\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 54\u001b[0m state \u001b[38;5;241m=\u001b[39m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39mdescribe_state()\n\u001b[1;32m 55\u001b[0m state\u001b[38;5;241m.\u001b[39mupdate(\n\u001b[1;32m 56\u001b[0m {\n\u001b[0;32m---> 57\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnetwork\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mnetwork\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdescribe_state\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m,\n\u001b[1;32m 58\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdomain\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdomain\u001b[38;5;241m.\u001b[39mdescribe_state(),\n\u001b[1;32m 59\u001b[0m }\n\u001b[1;32m 60\u001b[0m )\n\u001b[1;32m 61\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m state\n", + "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/simulator/network/container.py:260\u001b[0m, in \u001b[0;36mNetwork.describe_state\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 252\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 253\u001b[0m \u001b[38;5;124;03mProduce a dictionary describing the current state of the Network.\u001b[39;00m\n\u001b[1;32m 254\u001b[0m \n\u001b[1;32m 255\u001b[0m \u001b[38;5;124;03m:return: A dictionary capturing the current state of the Network and its child objects.\u001b[39;00m\n\u001b[1;32m 256\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 257\u001b[0m state \u001b[38;5;241m=\u001b[39m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39mdescribe_state()\n\u001b[1;32m 258\u001b[0m state\u001b[38;5;241m.\u001b[39mupdate(\n\u001b[1;32m 259\u001b[0m {\n\u001b[0;32m--> 260\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnodes\u001b[39m\u001b[38;5;124m\"\u001b[39m: {node\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mhostname: node\u001b[38;5;241m.\u001b[39mdescribe_state() \u001b[38;5;28;01mfor\u001b[39;00m node \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnodes\u001b[38;5;241m.\u001b[39mvalues()},\n\u001b[1;32m 261\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mlinks\u001b[39m\u001b[38;5;124m\"\u001b[39m: {},\n\u001b[1;32m 262\u001b[0m }\n\u001b[1;32m 263\u001b[0m )\n\u001b[1;32m 264\u001b[0m \u001b[38;5;66;03m# Update the links one-by-one. The key is a 4-tuple of `hostname_a, port_a, hostname_b, port_b`\u001b[39;00m\n\u001b[1;32m 265\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m _, link \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlinks\u001b[38;5;241m.\u001b[39mitems():\n", + "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/simulator/network/container.py:260\u001b[0m, in \u001b[0;36m\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 252\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 253\u001b[0m \u001b[38;5;124;03mProduce a dictionary describing the current state of the Network.\u001b[39;00m\n\u001b[1;32m 254\u001b[0m \n\u001b[1;32m 255\u001b[0m \u001b[38;5;124;03m:return: A dictionary capturing the current state of the Network and its child objects.\u001b[39;00m\n\u001b[1;32m 256\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 257\u001b[0m state \u001b[38;5;241m=\u001b[39m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39mdescribe_state()\n\u001b[1;32m 258\u001b[0m state\u001b[38;5;241m.\u001b[39mupdate(\n\u001b[1;32m 259\u001b[0m {\n\u001b[0;32m--> 260\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnodes\u001b[39m\u001b[38;5;124m\"\u001b[39m: {node\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mhostname: \u001b[43mnode\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdescribe_state\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mfor\u001b[39;00m node \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnodes\u001b[38;5;241m.\u001b[39mvalues()},\n\u001b[1;32m 261\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mlinks\u001b[39m\u001b[38;5;124m\"\u001b[39m: {},\n\u001b[1;32m 262\u001b[0m }\n\u001b[1;32m 263\u001b[0m )\n\u001b[1;32m 264\u001b[0m \u001b[38;5;66;03m# Update the links one-by-one. The key is a 4-tuple of `hostname_a, port_a, hostname_b, port_b`\u001b[39;00m\n\u001b[1;32m 265\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m _, link \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlinks\u001b[38;5;241m.\u001b[39mitems():\n", + "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/simulator/network/hardware/base.py:1900\u001b[0m, in \u001b[0;36mNode.describe_state\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1881\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 1882\u001b[0m \u001b[38;5;124;03mProduce a dictionary describing the current state of this object.\u001b[39;00m\n\u001b[1;32m 1883\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1887\u001b[0m \u001b[38;5;124;03m:rtype: Dict\u001b[39;00m\n\u001b[1;32m 1888\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 1889\u001b[0m state \u001b[38;5;241m=\u001b[39m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39mdescribe_state()\n\u001b[1;32m 1890\u001b[0m state\u001b[38;5;241m.\u001b[39mupdate(\n\u001b[1;32m 1891\u001b[0m {\n\u001b[1;32m 1892\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhostname\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mhostname,\n\u001b[1;32m 1893\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moperating_state\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39moperating_state\u001b[38;5;241m.\u001b[39mvalue,\n\u001b[1;32m 1894\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mNICs\u001b[39m\u001b[38;5;124m\"\u001b[39m: {\n\u001b[1;32m 1895\u001b[0m eth_num: network_interface\u001b[38;5;241m.\u001b[39mdescribe_state()\n\u001b[1;32m 1896\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m eth_num, network_interface \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnetwork_interface\u001b[38;5;241m.\u001b[39mitems()\n\u001b[1;32m 1897\u001b[0m },\n\u001b[1;32m 1898\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfile_system\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfile_system\u001b[38;5;241m.\u001b[39mdescribe_state(),\n\u001b[1;32m 1899\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mapplications\u001b[39m\u001b[38;5;124m\"\u001b[39m: {app\u001b[38;5;241m.\u001b[39mname: app\u001b[38;5;241m.\u001b[39mdescribe_state() \u001b[38;5;28;01mfor\u001b[39;00m app \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mapplications\u001b[38;5;241m.\u001b[39mvalues()},\n\u001b[0;32m-> 1900\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mservices\u001b[39m\u001b[38;5;124m\"\u001b[39m: {svc\u001b[38;5;241m.\u001b[39mname: svc\u001b[38;5;241m.\u001b[39mdescribe_state() \u001b[38;5;28;01mfor\u001b[39;00m svc \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mservices\u001b[38;5;241m.\u001b[39mvalues()},\n\u001b[1;32m 1901\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mprocess\u001b[39m\u001b[38;5;124m\"\u001b[39m: {proc\u001b[38;5;241m.\u001b[39mname: proc\u001b[38;5;241m.\u001b[39mdescribe_state() \u001b[38;5;28;01mfor\u001b[39;00m proc \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprocesses\u001b[38;5;241m.\u001b[39mvalues()},\n\u001b[1;32m 1902\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mrevealed_to_red\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mrevealed_to_red,\n\u001b[1;32m 1903\u001b[0m }\n\u001b[1;32m 1904\u001b[0m )\n\u001b[1;32m 1905\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m state\n", + "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/simulator/network/hardware/base.py:1900\u001b[0m, in \u001b[0;36m\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 1881\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 1882\u001b[0m \u001b[38;5;124;03mProduce a dictionary describing the current state of this object.\u001b[39;00m\n\u001b[1;32m 1883\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1887\u001b[0m \u001b[38;5;124;03m:rtype: Dict\u001b[39;00m\n\u001b[1;32m 1888\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 1889\u001b[0m state \u001b[38;5;241m=\u001b[39m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39mdescribe_state()\n\u001b[1;32m 1890\u001b[0m state\u001b[38;5;241m.\u001b[39mupdate(\n\u001b[1;32m 1891\u001b[0m {\n\u001b[1;32m 1892\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhostname\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mhostname,\n\u001b[1;32m 1893\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moperating_state\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39moperating_state\u001b[38;5;241m.\u001b[39mvalue,\n\u001b[1;32m 1894\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mNICs\u001b[39m\u001b[38;5;124m\"\u001b[39m: {\n\u001b[1;32m 1895\u001b[0m eth_num: network_interface\u001b[38;5;241m.\u001b[39mdescribe_state()\n\u001b[1;32m 1896\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m eth_num, network_interface \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnetwork_interface\u001b[38;5;241m.\u001b[39mitems()\n\u001b[1;32m 1897\u001b[0m },\n\u001b[1;32m 1898\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfile_system\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfile_system\u001b[38;5;241m.\u001b[39mdescribe_state(),\n\u001b[1;32m 1899\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mapplications\u001b[39m\u001b[38;5;124m\"\u001b[39m: {app\u001b[38;5;241m.\u001b[39mname: app\u001b[38;5;241m.\u001b[39mdescribe_state() \u001b[38;5;28;01mfor\u001b[39;00m app \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mapplications\u001b[38;5;241m.\u001b[39mvalues()},\n\u001b[0;32m-> 1900\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mservices\u001b[39m\u001b[38;5;124m\"\u001b[39m: {svc\u001b[38;5;241m.\u001b[39mname: \u001b[43msvc\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdescribe_state\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mfor\u001b[39;00m svc \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mservices\u001b[38;5;241m.\u001b[39mvalues()},\n\u001b[1;32m 1901\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mprocess\u001b[39m\u001b[38;5;124m\"\u001b[39m: {proc\u001b[38;5;241m.\u001b[39mname: proc\u001b[38;5;241m.\u001b[39mdescribe_state() \u001b[38;5;28;01mfor\u001b[39;00m proc \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprocesses\u001b[38;5;241m.\u001b[39mvalues()},\n\u001b[1;32m 1902\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mrevealed_to_red\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mrevealed_to_red,\n\u001b[1;32m 1903\u001b[0m }\n\u001b[1;32m 1904\u001b[0m )\n\u001b[1;32m 1905\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m state\n", + "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/simulator/system/services/ntp/ntp_client.py:61\u001b[0m, in \u001b[0;36mNTPClient.describe_state\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 50\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdescribe_state\u001b[39m(\u001b[38;5;28mself\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Dict:\n\u001b[1;32m 51\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 52\u001b[0m \u001b[38;5;124;03m Describes the current state of the software.\u001b[39;00m\n\u001b[1;32m 53\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 59\u001b[0m \u001b[38;5;124;03m :rtype: Dict\u001b[39;00m\n\u001b[1;32m 60\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m---> 61\u001b[0m state \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241m.\u001b[39mdescribe_state()\n\u001b[1;32m 62\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m state\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + ] + } + ], "source": [ "env.step(13) # Patch the database\n", "print(f\"step: {env.game.step_counter}, Red action: {info['agent_actions']['data_manipulation_attacker'].action}, Blue reward:{reward:.2f}\" )\n", @@ -600,7 +5091,7 @@ "while abs(reward - 0.8) > 1e-5:\n", " obs, reward, terminated, truncated, info = env.step(0) # do nothing\n", " print(f\"step: {env.game.step_counter}, Red action: {info['agent_actions']['data_manipulation_attacker'].action}, Blue reward:{reward:.2f}\" )\n", - " if env.game.step_counter > 10000:\n", + " if env.game.step_counter > 2000:\n", " break # make sure there's no infinite loop if something went wrong" ] }, From 7dafec85176130dc3c9ce66bf4dbe8ec3b2faa7e Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Mon, 17 Feb 2025 15:01:02 +0000 Subject: [PATCH 26/37] #3075: Test fix --- .../game_layer/actions/test_terminal_actions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration_tests/game_layer/actions/test_terminal_actions.py b/tests/integration_tests/game_layer/actions/test_terminal_actions.py index c39d8263..af9c7b74 100644 --- a/tests/integration_tests/game_layer/actions/test_terminal_actions.py +++ b/tests/integration_tests/game_layer/actions/test_terminal_actions.py @@ -106,7 +106,7 @@ def test_remote_login_change_password(game_and_agent_fixture: Tuple[PrimaiteGame "username": "user123", "current_password": "password", "new_password": "different_password", - "remote_ip": str(server_1.network_interface[1].ip_address), + # "remote_ip": str(server_1.network_interface[1].ip_address), }, ) agent.store_action(action) @@ -146,7 +146,7 @@ def test_change_password_logs_out_user(game_and_agent_fixture: Tuple[PrimaiteGam "username": "user123", "current_password": "password", "new_password": "different_password", - "remote_ip": str(server_1.network_interface[1].ip_address), + # "remote_ip": str(server_1.network_interface[1].ip_address), }, ) agent.store_action(action) From 5f076ba225bf239a61377478949e8b474b7504d7 Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Mon, 17 Feb 2025 15:12:12 +0000 Subject: [PATCH 27/37] #3075: Fix pre-commit errors --- .../Data-Manipulation-E2E-Demonstration.ipynb | 4527 +---------------- 1 file changed, 18 insertions(+), 4509 deletions(-) diff --git a/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb b/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb index f3e3fb98..31e1f9d3 100644 --- a/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb +++ b/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb @@ -373,29 +373,16 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-02-17 11:15:30,959: Performing the PrimAITE first-time setup...\n", - "2025-02-17 11:15:30,960: Building the PrimAITE app directories...\n", - "2025-02-17 11:15:30,960: Building primaite_config.yaml...\n", - "2025-02-17 11:15:30,960: Rebuilding the demo notebooks...\n", - "2025-02-17 11:15:30,967: Rebuilding the example notebooks...\n", - "2025-02-17 11:15:30,970: PrimAITE setup complete!\n" - ] - } - ], + "outputs": [], "source": [ "!primaite setup" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": { "tags": [] }, @@ -407,7 +394,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": { "tags": [] }, @@ -433,254 +420,9 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-17 11:15:35,108: PrimaiteGymEnv RNG seed = None\n", - "2025-02-17 11:15:35,109: Resetting environment, episode 0, avg. reward: 0.0\n", - "2025-02-17 11:15:35,111: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-17/11-15-31/agent_actions/episode_0.json\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "env created successfully\n", - "{'ICS': 0,\n", - " 'LINKS': {1: {'PROTOCOLS': {'ALL': 0}},\n", - " 2: {'PROTOCOLS': {'ALL': 0}},\n", - " 3: {'PROTOCOLS': {'ALL': 0}},\n", - " 4: {'PROTOCOLS': {'ALL': 0}},\n", - " 5: {'PROTOCOLS': {'ALL': 0}},\n", - " 6: {'PROTOCOLS': {'ALL': 0}},\n", - " 7: {'PROTOCOLS': {'ALL': 0}},\n", - " 8: {'PROTOCOLS': {'ALL': 0}},\n", - " 9: {'PROTOCOLS': {'ALL': 0}},\n", - " 10: {'PROTOCOLS': {'ALL': 0}}},\n", - " 'NODES': {'HOST0': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0,\n", - " 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0,\n", - " 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0,\n", - " 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0,\n", - " 'outbound': 0}}},\n", - " 'nic_status': 0}},\n", - " 'SERVICES': {1: {'health_status': 0,\n", - " 'operating_status': 0}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'HOST1': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0,\n", - " 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0,\n", - " 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0,\n", - " 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0,\n", - " 'outbound': 0}}},\n", - " 'nic_status': 0}},\n", - " 'SERVICES': {1: {'health_status': 0,\n", - " 'operating_status': 1}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'HOST2': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0,\n", - " 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0,\n", - " 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0,\n", - " 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0,\n", - " 'outbound': 0}}},\n", - " 'nic_status': 0}},\n", - " 'SERVICES': {1: {'health_status': 0,\n", - " 'operating_status': 0}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'HOST3': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0,\n", - " 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0,\n", - " 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0,\n", - " 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0,\n", - " 'outbound': 0}}},\n", - " 'nic_status': 0}},\n", - " 'SERVICES': {1: {'health_status': 0,\n", - " 'operating_status': 0}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'HOST4': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0,\n", - " 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0,\n", - " 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0,\n", - " 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0,\n", - " 'outbound': 0}}},\n", - " 'nic_status': 1}},\n", - " 'SERVICES': {1: {'health_status': 0,\n", - " 'operating_status': 0}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'HOST5': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0,\n", - " 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0,\n", - " 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0,\n", - " 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0,\n", - " 'outbound': 0}}},\n", - " 'nic_status': 0}},\n", - " 'SERVICES': {1: {'health_status': 0,\n", - " 'operating_status': 0}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'HOST6': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0,\n", - " 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0,\n", - " 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0,\n", - " 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0,\n", - " 'outbound': 0}}},\n", - " 'nic_status': 0}},\n", - " 'SERVICES': {1: {'health_status': 0,\n", - " 'operating_status': 0}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'ROUTER0': {'ACL': {1: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 0,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 2: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 1,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 3: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 2,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 4: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 3,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 5: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 4,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 6: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 5,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 7: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 6,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 8: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 7,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 9: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 8,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 10: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 9,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0}},\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}}}}\n" - ] - } - ], + "outputs": [], "source": [ "# create the env\n", "with open(data_manipulation_config_path(), 'r') as f:\n", @@ -708,7 +450,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -726,51 +468,9 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "step: 1, Red action: DO NOTHING, Blue reward:0.65\n", - "step: 2, Red action: DO NOTHING, Blue reward:0.65\n", - "step: 3, Red action: DO NOTHING, Blue reward:0.65\n", - "step: 4, Red action: DO NOTHING, Blue reward:0.90\n", - "step: 5, Red action: DO NOTHING, Blue reward:0.90\n", - "step: 6, Red action: DO NOTHING, Blue reward:0.90\n", - "step: 7, Red action: DO NOTHING, Blue reward:0.90\n", - "step: 8, Red action: DO NOTHING, Blue reward:0.90\n", - "step: 9, Red action: DO NOTHING, Blue reward:0.90\n", - "step: 10, Red action: DO NOTHING, Blue reward:0.90\n", - "step: 11, Red action: DO NOTHING, Blue reward:0.90\n", - "step: 12, Red action: DO NOTHING, Blue reward:0.90\n", - "step: 13, Red action: DO NOTHING, Blue reward:0.90\n", - "step: 14, Red action: DO NOTHING, Blue reward:0.90\n", - "step: 15, Red action: DO NOTHING, Blue reward:0.90\n", - "step: 16, Red action: DO NOTHING, Blue reward:0.90\n", - "step: 17, Red action: DO NOTHING, Blue reward:0.95\n", - "step: 18, Red action: DO NOTHING, Blue reward:0.95\n", - "step: 19, Red action: DO NOTHING, Blue reward:0.95\n", - "step: 20, Red action: DO NOTHING, Blue reward:0.95\n", - "step: 21, Red action: DO NOTHING, Blue reward:0.95\n", - "step: 22, Red action: DO NOTHING, Blue reward:0.95\n", - "step: 23, Red action: DO NOTHING, Blue reward:0.95\n", - "step: 24, Red action: DO NOTHING, Blue reward:0.95\n", - "step: 25, Red action: DO NOTHING, Blue reward:0.95\n", - "step: 26, Red action: ATTACK from client 2, Blue reward:0.15\n", - "step: 27, Red action: DO NOTHING, Blue reward:-0.35\n", - "step: 28, Red action: DO NOTHING, Blue reward:-0.85\n", - "step: 29, Red action: DO NOTHING, Blue reward:-0.85\n", - "step: 30, Red action: DO NOTHING, Blue reward:-0.85\n", - "step: 31, Red action: DO NOTHING, Blue reward:-0.85\n", - "step: 32, Red action: DO NOTHING, Blue reward:-0.85\n", - "step: 33, Red action: DO NOTHING, Blue reward:-0.85\n", - "step: 34, Red action: DO NOTHING, Blue reward:-0.85\n", - "step: 35, Red action: DO NOTHING, Blue reward:-0.85\n" - ] - } - ], + "outputs": [], "source": [ "for step in range(35):\n", " obs, reward, terminated, truncated, info = env.step(0)\n", @@ -786,198 +486,9 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'HOST0': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 0}},\n", - " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'HOST1': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 0}},\n", - " 'SERVICES': {1: {'health_status': 0, 'operating_status': 1}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'HOST2': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 0}},\n", - " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'HOST3': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 0}},\n", - " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'HOST4': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 1}},\n", - " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'HOST5': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 0}},\n", - " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'HOST6': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 0}},\n", - " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'ROUTER0': {'ACL': {1: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 0,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 2: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 1,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 3: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 2,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 4: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 3,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 5: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 4,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 6: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 5,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 7: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 6,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 8: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 7,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 9: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 8,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 10: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 9,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0}},\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}}}\n" - ] - } - ], + "outputs": [], "source": [ "pprint(obs['NODES'])" ] @@ -991,198 +502,9 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'HOST0': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 0}},\n", - " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'HOST1': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 0}},\n", - " 'SERVICES': {1: {'health_status': 3, 'operating_status': 1}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'HOST2': {'FOLDERS': {1: {'FILES': {1: {'health_status': 2}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 0}},\n", - " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'HOST3': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 0}},\n", - " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'HOST4': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 1}},\n", - " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'HOST5': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 0}},\n", - " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'HOST6': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 0}},\n", - " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'ROUTER0': {'ACL': {1: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 0,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 2: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 1,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 3: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 2,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 4: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 3,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 5: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 4,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 6: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 5,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 7: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 6,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 8: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 7,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 9: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 8,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 10: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 9,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0}},\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}}}\n" - ] - } - ], + "outputs": [], "source": [ "obs, reward, terminated, truncated, info = env.step(9) # scan database file\n", "obs, reward, terminated, truncated, info = env.step(1) # scan webapp service\n", @@ -1214,21 +536,9 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "step: 38\n", - "Red action: do-nothing\n", - "Green action: do-nothing\n", - "Green action: node-application-execute\n", - "Blue reward:-0.8500000000000001\n" - ] - } - ], + "outputs": [], "source": [ "obs, reward, terminated, truncated, info = env.step(13) # patch the database\n", "print(f\"step: {env.game.step_counter}\")\n", @@ -1251,21 +561,9 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "step: 39\n", - "Red action: do-nothing\n", - "Green action: timestep=38 action='node-application-execute' parameters={'node_name': 'client_2', 'application_name': 'web-browser'} request=['network', 'node', 'client_2', 'application', 'web-browser', 'execute'] response=RequestResponse(status='failure', data={}) reward=-0.2 reward_info={'connection_attempt_status': 'n/a'}\n", - "Green action: timestep=38 action='node-application-execute' parameters={'node_name': 'client_1', 'application_name': 'web-browser'} request=['network', 'node', 'client_1', 'application', 'web-browser', 'execute'] response=RequestResponse(status='failure', data={}) reward=-0.25 reward_info={'connection_attempt_status': 'n/a'}\n", - "Blue reward:-0.05\n" - ] - } - ], + "outputs": [], "source": [ "obs, reward, terminated, truncated, info = env.step(0) # do nothing\n", "print(f\"step: {env.game.step_counter}\")\n", @@ -1288,3796 +586,7 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "step: 40, Red action: do-nothing, Blue reward:-0.05\n", - "step: 41, Red action: do-nothing, Blue reward:-0.05\n", - "step: 42, Red action: do-nothing, Blue reward:-0.05\n", - "step: 43, Red action: do-nothing, Blue reward:0.95\n", - "step: 44, Red action: do-nothing, Blue reward:0.95\n", - "step: 45, Red action: do-nothing, Blue reward:0.95\n", - "step: 46, Red action: do-nothing, Blue reward:0.95\n", - "step: 47, Red action: do-nothing, Blue reward:0.95\n", - "step: 48, Red action: do-nothing, Blue reward:0.95\n", - "step: 49, Red action: do-nothing, Blue reward:0.95\n", - "step: 50, Red action: do-nothing, Blue reward:0.95\n", - "step: 51, Red action: node-application-execute, Blue reward:0.95\n", - "step: 52, Red action: do-nothing, Blue reward:0.95\n", - "step: 53, Red action: do-nothing, Blue reward:0.85\n", - "step: 54, Red action: do-nothing, Blue reward:0.85\n", - "step: 55, Red action: do-nothing, Blue reward:0.85\n", - "step: 56, Red action: do-nothing, Blue reward:0.85\n", - "step: 57, Red action: do-nothing, Blue reward:0.85\n", - "step: 58, Red action: do-nothing, Blue reward:0.85\n", - "step: 59, Red action: do-nothing, Blue reward:0.85\n", - "step: 60, Red action: do-nothing, Blue reward:0.85\n", - "step: 61, Red action: do-nothing, Blue reward:0.85\n", - "step: 62, Red action: do-nothing, Blue reward:0.85\n", - "step: 63, Red action: do-nothing, Blue reward:0.85\n", - "step: 64, Red action: do-nothing, Blue reward:0.85\n", - "step: 65, Red action: do-nothing, Blue reward:0.85\n", - "step: 66, Red action: do-nothing, Blue reward:0.85\n", - "step: 67, Red action: do-nothing, Blue reward:0.85\n", - "step: 68, Red action: do-nothing, Blue reward:0.85\n", - "step: 69, Red action: do-nothing, Blue reward:0.85\n", - "step: 70, Red action: do-nothing, Blue reward:0.85\n", - "step: 71, Red action: do-nothing, Blue reward:0.85\n", - "step: 72, Red action: do-nothing, Blue reward:0.85\n", - "step: 73, Red action: do-nothing, Blue reward:0.85\n", - "step: 74, Red action: do-nothing, Blue reward:0.85\n", - "step: 75, Red action: node-application-execute, Blue reward:0.85\n", - "step: 76, Red action: do-nothing, Blue reward:0.85\n", - "step: 77, Red action: do-nothing, Blue reward:0.85\n", - "step: 78, Red action: do-nothing, Blue reward:0.85\n", - "step: 79, Red action: do-nothing, Blue reward:0.85\n", - "step: 80, Red action: do-nothing, Blue reward:0.85\n", - "step: 81, Red action: do-nothing, Blue reward:0.85\n", - "step: 82, Red action: do-nothing, Blue reward:0.85\n", - "step: 83, Red action: do-nothing, Blue reward:0.85\n", - "step: 84, Red action: do-nothing, Blue reward:0.85\n", - "step: 85, Red action: do-nothing, Blue reward:0.85\n", - "step: 86, Red action: do-nothing, Blue reward:0.85\n", - "step: 87, Red action: do-nothing, Blue reward:0.85\n", - "step: 88, Red action: do-nothing, Blue reward:0.85\n", - "step: 89, Red action: do-nothing, Blue reward:0.85\n", - "step: 90, Red action: do-nothing, Blue reward:0.85\n", - "step: 91, Red action: do-nothing, Blue reward:0.85\n", - "step: 92, Red action: do-nothing, Blue reward:0.85\n", - "step: 93, Red action: do-nothing, Blue reward:0.85\n", - "step: 94, Red action: do-nothing, Blue reward:0.85\n", - "step: 95, Red action: do-nothing, Blue reward:0.85\n", - "step: 96, Red action: do-nothing, Blue reward:0.85\n", - "step: 97, Red action: node-application-execute, Blue reward:0.85\n", - "step: 98, Red action: do-nothing, Blue reward:0.85\n", - "step: 99, Red action: do-nothing, Blue reward:0.85\n", - "step: 100, Red action: do-nothing, Blue reward:0.85\n", - "step: 101, Red action: do-nothing, Blue reward:0.85\n", - "step: 102, Red action: do-nothing, Blue reward:0.85\n", - "step: 103, Red action: do-nothing, Blue reward:0.85\n", - "step: 104, Red action: do-nothing, Blue reward:0.85\n", - "step: 105, Red action: do-nothing, Blue reward:0.85\n", - "step: 106, Red action: do-nothing, Blue reward:0.85\n", - "step: 107, Red action: do-nothing, Blue reward:0.85\n", - "step: 108, Red action: do-nothing, Blue reward:0.85\n", - "step: 109, Red action: do-nothing, Blue reward:0.85\n", - "step: 110, Red action: do-nothing, Blue reward:0.85\n", - "step: 111, Red action: do-nothing, Blue reward:0.85\n", - "step: 112, Red action: do-nothing, Blue reward:0.85\n", - "step: 113, Red action: node-application-execute, Blue reward:0.85\n", - "step: 114, Red action: do-nothing, Blue reward:0.85\n", - "step: 115, Red action: do-nothing, Blue reward:0.85\n", - "step: 116, Red action: do-nothing, Blue reward:0.85\n", - "step: 117, Red action: do-nothing, Blue reward:0.85\n", - "step: 118, Red action: do-nothing, Blue reward:0.85\n", - "step: 119, Red action: do-nothing, Blue reward:0.85\n", - "step: 120, Red action: do-nothing, Blue reward:0.85\n", - "step: 121, Red action: do-nothing, Blue reward:0.85\n", - "step: 122, Red action: do-nothing, Blue reward:0.85\n", - "step: 123, Red action: do-nothing, Blue reward:0.85\n", - "step: 124, Red action: do-nothing, Blue reward:0.85\n", - "step: 125, Red action: do-nothing, Blue reward:0.85\n", - "step: 126, Red action: do-nothing, Blue reward:0.85\n", - "step: 127, Red action: do-nothing, Blue reward:0.85\n", - "step: 128, Red action: do-nothing, Blue reward:0.85\n", - "step: 129, Red action: do-nothing, Blue reward:0.85\n", - "step: 130, Red action: do-nothing, Blue reward:0.85\n", - "step: 131, Red action: do-nothing, Blue reward:0.85\n", - "step: 132, Red action: node-application-execute, Blue reward:0.85\n", - "step: 133, Red action: do-nothing, Blue reward:0.85\n", - "step: 134, Red action: do-nothing, Blue reward:0.85\n", - "step: 135, Red action: do-nothing, Blue reward:0.85\n", - "step: 136, Red action: do-nothing, Blue reward:0.85\n", - "step: 137, Red action: do-nothing, Blue reward:0.85\n", - "step: 138, Red action: do-nothing, Blue reward:0.85\n", - "step: 139, Red action: do-nothing, Blue reward:0.85\n", - "step: 140, Red action: do-nothing, Blue reward:0.85\n", - "step: 141, Red action: do-nothing, Blue reward:0.85\n", - "step: 142, Red action: do-nothing, Blue reward:0.85\n", - "step: 143, Red action: do-nothing, Blue reward:0.85\n", - "step: 144, Red action: do-nothing, Blue reward:0.85\n", - "step: 145, Red action: do-nothing, Blue reward:0.85\n", - "step: 146, Red action: do-nothing, Blue reward:0.85\n", - "step: 147, Red action: do-nothing, Blue reward:0.85\n", - "step: 148, Red action: do-nothing, Blue reward:0.85\n", - "step: 149, Red action: do-nothing, Blue reward:0.85\n", - "step: 150, Red action: do-nothing, Blue reward:0.85\n", - "step: 151, Red action: do-nothing, Blue reward:0.85\n", - "step: 152, Red action: do-nothing, Blue reward:0.85\n", - "step: 153, Red action: do-nothing, Blue reward:0.85\n", - "step: 154, Red action: do-nothing, Blue reward:0.85\n", - "step: 155, Red action: do-nothing, Blue reward:0.85\n", - "step: 156, Red action: do-nothing, Blue reward:0.85\n", - "step: 157, Red action: node-application-execute, Blue reward:0.85\n", - "step: 158, Red action: do-nothing, Blue reward:0.85\n", - "step: 159, Red action: do-nothing, Blue reward:0.85\n", - "step: 160, Red action: do-nothing, Blue reward:0.85\n", - "step: 161, Red action: do-nothing, Blue reward:0.85\n", - "step: 162, Red action: do-nothing, Blue reward:0.85\n", - "step: 163, Red action: do-nothing, Blue reward:0.85\n", - "step: 164, Red action: do-nothing, Blue reward:0.85\n", - "step: 165, Red action: do-nothing, Blue reward:0.85\n", - "step: 166, Red action: do-nothing, Blue reward:0.85\n", - "step: 167, Red action: do-nothing, Blue reward:0.85\n", - "step: 168, Red action: do-nothing, Blue reward:0.85\n", - "step: 169, Red action: do-nothing, Blue reward:0.85\n", - "step: 170, Red action: do-nothing, Blue reward:0.85\n", - "step: 171, Red action: do-nothing, Blue reward:0.85\n", - "step: 172, Red action: do-nothing, Blue reward:0.85\n", - "step: 173, Red action: do-nothing, Blue reward:0.85\n", - "step: 174, Red action: do-nothing, Blue reward:0.85\n", - "step: 175, Red action: do-nothing, Blue reward:0.85\n", - "step: 176, Red action: do-nothing, Blue reward:0.85\n", - "step: 177, Red action: do-nothing, Blue reward:0.85\n", - "step: 178, Red action: node-application-execute, Blue reward:0.85\n", - "step: 179, Red action: do-nothing, Blue reward:0.85\n", - "step: 180, Red action: do-nothing, Blue reward:0.85\n", - "step: 181, Red action: do-nothing, Blue reward:0.85\n", - "step: 182, Red action: do-nothing, Blue reward:0.85\n", - "step: 183, Red action: do-nothing, Blue reward:0.85\n", - "step: 184, Red action: do-nothing, Blue reward:0.85\n", - "step: 185, Red action: do-nothing, Blue reward:0.85\n", - "step: 186, Red action: do-nothing, Blue reward:0.85\n", - "step: 187, Red action: do-nothing, Blue reward:0.85\n", - "step: 188, Red action: do-nothing, Blue reward:0.85\n", - "step: 189, Red action: do-nothing, Blue reward:0.85\n", - "step: 190, Red action: do-nothing, Blue reward:0.85\n", - "step: 191, Red action: do-nothing, Blue reward:0.85\n", - "step: 192, Red action: do-nothing, Blue reward:0.85\n", - "step: 193, Red action: do-nothing, Blue reward:0.85\n", - "step: 194, Red action: do-nothing, Blue reward:0.85\n", - "step: 195, Red action: do-nothing, Blue reward:0.85\n", - "step: 196, Red action: do-nothing, Blue reward:0.85\n", - "step: 197, Red action: do-nothing, Blue reward:0.85\n", - "step: 198, Red action: node-application-execute, Blue reward:0.85\n", - "step: 199, Red action: do-nothing, Blue reward:0.85\n", - "step: 200, Red action: do-nothing, Blue reward:0.85\n", - "step: 201, Red action: do-nothing, Blue reward:0.85\n", - "step: 202, Red action: do-nothing, Blue reward:0.85\n", - "step: 203, Red action: do-nothing, Blue reward:0.85\n", - "step: 204, Red action: do-nothing, Blue reward:0.85\n", - "step: 205, Red action: do-nothing, Blue reward:0.85\n", - "step: 206, Red action: do-nothing, Blue reward:0.85\n", - "step: 207, Red action: do-nothing, Blue reward:0.85\n", - "step: 208, Red action: do-nothing, Blue reward:0.85\n", - "step: 209, Red action: do-nothing, Blue reward:0.85\n", - "step: 210, Red action: do-nothing, Blue reward:0.85\n", - "step: 211, Red action: do-nothing, Blue reward:0.85\n", - "step: 212, Red action: do-nothing, Blue reward:0.85\n", - "step: 213, Red action: do-nothing, Blue reward:0.85\n", - "step: 214, Red action: do-nothing, Blue reward:0.85\n", - "step: 215, Red action: do-nothing, Blue reward:0.85\n", - "step: 216, Red action: do-nothing, Blue reward:0.85\n", - "step: 217, Red action: do-nothing, Blue reward:0.85\n", - "step: 218, Red action: node-application-execute, Blue reward:0.85\n", - "step: 219, Red action: do-nothing, Blue reward:0.85\n", - "step: 220, Red action: do-nothing, Blue reward:0.85\n", - "step: 221, Red action: do-nothing, Blue reward:0.85\n", - "step: 222, Red action: do-nothing, Blue reward:0.85\n", - "step: 223, Red action: do-nothing, Blue reward:0.85\n", - "step: 224, Red action: do-nothing, Blue reward:0.85\n", - "step: 225, Red action: do-nothing, Blue reward:0.85\n", - "step: 226, Red action: do-nothing, Blue reward:0.85\n", - "step: 227, Red action: do-nothing, Blue reward:0.85\n", - "step: 228, Red action: do-nothing, Blue reward:0.85\n", - "step: 229, Red action: do-nothing, Blue reward:0.85\n", - "step: 230, Red action: do-nothing, Blue reward:0.85\n", - "step: 231, Red action: do-nothing, Blue reward:0.85\n", - "step: 232, Red action: do-nothing, Blue reward:0.85\n", - "step: 233, Red action: do-nothing, Blue reward:0.85\n", - "step: 234, Red action: do-nothing, Blue reward:0.85\n", - "step: 235, Red action: do-nothing, Blue reward:0.85\n", - "step: 236, Red action: do-nothing, Blue reward:0.85\n", - "step: 237, Red action: do-nothing, Blue reward:0.85\n", - "step: 238, Red action: do-nothing, Blue reward:0.85\n", - "step: 239, Red action: do-nothing, Blue reward:0.85\n", - "step: 240, Red action: do-nothing, Blue reward:0.85\n", - "step: 241, Red action: node-application-execute, Blue reward:0.85\n", - "step: 242, Red action: do-nothing, Blue reward:0.85\n", - "step: 243, Red action: do-nothing, Blue reward:0.85\n", - "step: 244, Red action: do-nothing, Blue reward:0.85\n", - "step: 245, Red action: do-nothing, Blue reward:0.85\n", - "step: 246, Red action: do-nothing, Blue reward:0.85\n", - "step: 247, Red action: do-nothing, Blue reward:0.85\n", - "step: 248, Red action: do-nothing, Blue reward:0.85\n", - "step: 249, Red action: do-nothing, Blue reward:0.85\n", - "step: 250, Red action: do-nothing, Blue reward:0.85\n", - "step: 251, Red action: do-nothing, Blue reward:0.85\n", - "step: 252, Red action: do-nothing, Blue reward:0.85\n", - "step: 253, Red action: do-nothing, Blue reward:0.85\n", - "step: 254, Red action: do-nothing, Blue reward:0.85\n", - "step: 255, Red action: do-nothing, Blue reward:0.85\n", - "step: 256, Red action: do-nothing, Blue reward:0.85\n", - "step: 257, Red action: do-nothing, Blue reward:0.85\n", - "step: 258, Red action: do-nothing, Blue reward:0.85\n", - "step: 259, Red action: do-nothing, Blue reward:0.85\n", - "step: 260, Red action: do-nothing, Blue reward:0.85\n", - "step: 261, Red action: node-application-execute, Blue reward:0.85\n", - "step: 262, Red action: do-nothing, Blue reward:0.85\n", - "step: 263, Red action: do-nothing, Blue reward:0.85\n", - "step: 264, Red action: do-nothing, Blue reward:0.85\n", - "step: 265, Red action: do-nothing, Blue reward:0.85\n", - "step: 266, Red action: do-nothing, Blue reward:0.85\n", - "step: 267, Red action: do-nothing, Blue reward:0.85\n", - "step: 268, Red action: do-nothing, Blue reward:0.85\n", - "step: 269, Red action: do-nothing, Blue reward:0.85\n", - "step: 270, Red action: do-nothing, Blue reward:0.85\n", - "step: 271, Red action: do-nothing, Blue reward:0.85\n", - "step: 272, Red action: do-nothing, Blue reward:0.85\n", - "step: 273, Red action: do-nothing, Blue reward:0.85\n", - "step: 274, Red action: do-nothing, Blue reward:0.85\n", - "step: 275, Red action: do-nothing, Blue reward:0.85\n", - "step: 276, Red action: do-nothing, Blue reward:0.85\n", - "step: 277, Red action: do-nothing, Blue reward:0.85\n", - "step: 278, Red action: do-nothing, Blue reward:0.85\n", - "step: 279, Red action: do-nothing, Blue reward:0.85\n", - "step: 280, Red action: do-nothing, Blue reward:0.85\n", - "step: 281, Red action: node-application-execute, Blue reward:0.85\n", - "step: 282, Red action: do-nothing, Blue reward:0.85\n", - "step: 283, Red action: do-nothing, Blue reward:0.85\n", - "step: 284, Red action: do-nothing, Blue reward:0.85\n", - "step: 285, Red action: do-nothing, Blue reward:0.85\n", - "step: 286, Red action: do-nothing, Blue reward:0.85\n", - "step: 287, Red action: do-nothing, Blue reward:0.85\n", - "step: 288, Red action: do-nothing, Blue reward:0.85\n", - "step: 289, Red action: do-nothing, Blue reward:0.85\n", - "step: 290, Red action: do-nothing, Blue reward:0.85\n", - "step: 291, Red action: do-nothing, Blue reward:0.85\n", - "step: 292, Red action: do-nothing, Blue reward:0.85\n", - "step: 293, Red action: do-nothing, Blue reward:0.85\n", - "step: 294, Red action: do-nothing, Blue reward:0.85\n", - "step: 295, Red action: do-nothing, Blue reward:0.85\n", - "step: 296, Red action: do-nothing, Blue reward:0.85\n", - "step: 297, Red action: do-nothing, Blue reward:0.85\n", - "step: 298, Red action: do-nothing, Blue reward:0.85\n", - "step: 299, Red action: do-nothing, Blue reward:0.85\n", - "step: 300, Red action: do-nothing, Blue reward:0.85\n", - "step: 301, Red action: do-nothing, Blue reward:0.85\n", - "step: 302, Red action: do-nothing, Blue reward:0.85\n", - "step: 303, Red action: do-nothing, Blue reward:0.85\n", - "step: 304, Red action: node-application-execute, Blue reward:0.85\n", - "step: 305, Red action: do-nothing, Blue reward:0.85\n", - "step: 306, Red action: do-nothing, Blue reward:0.85\n", - "step: 307, Red action: do-nothing, Blue reward:0.85\n", - "step: 308, Red action: do-nothing, Blue reward:0.85\n", - "step: 309, Red action: do-nothing, Blue reward:0.85\n", - "step: 310, Red action: do-nothing, Blue reward:0.85\n", - "step: 311, Red action: do-nothing, Blue reward:0.85\n", - "step: 312, Red action: do-nothing, Blue reward:0.85\n", - "step: 313, Red action: do-nothing, Blue reward:0.85\n", - "step: 314, Red action: do-nothing, Blue reward:0.85\n", - "step: 315, Red action: do-nothing, Blue reward:0.85\n", - "step: 316, Red action: do-nothing, Blue reward:0.85\n", - "step: 317, Red action: do-nothing, Blue reward:0.85\n", - "step: 318, Red action: do-nothing, Blue reward:0.85\n", - "step: 319, Red action: do-nothing, Blue reward:0.85\n", - "step: 320, Red action: do-nothing, Blue reward:0.85\n", - "step: 321, Red action: do-nothing, Blue reward:0.85\n", - "step: 322, Red action: do-nothing, Blue reward:0.85\n", - "step: 323, Red action: do-nothing, Blue reward:0.85\n", - "step: 324, Red action: do-nothing, Blue reward:0.85\n", - "step: 325, Red action: node-application-execute, Blue reward:0.85\n", - "step: 326, Red action: do-nothing, Blue reward:0.85\n", - "step: 327, Red action: do-nothing, Blue reward:0.85\n", - "step: 328, Red action: do-nothing, Blue reward:0.85\n", - "step: 329, Red action: do-nothing, Blue reward:0.85\n", - "step: 330, Red action: do-nothing, Blue reward:0.85\n", - "step: 331, Red action: do-nothing, Blue reward:0.85\n", - "step: 332, Red action: do-nothing, Blue reward:0.85\n", - "step: 333, Red action: do-nothing, Blue reward:0.85\n", - "step: 334, Red action: do-nothing, Blue reward:0.85\n", - "step: 335, Red action: do-nothing, Blue reward:0.85\n", - "step: 336, Red action: do-nothing, Blue reward:0.85\n", - "step: 337, Red action: do-nothing, Blue reward:0.85\n", - "step: 338, Red action: do-nothing, Blue reward:0.85\n", - "step: 339, Red action: do-nothing, Blue reward:0.85\n", - "step: 340, Red action: do-nothing, Blue reward:0.85\n", - "step: 341, Red action: do-nothing, Blue reward:0.85\n", - "step: 342, Red action: do-nothing, Blue reward:0.85\n", - "step: 343, Red action: do-nothing, Blue reward:0.85\n", - "step: 344, Red action: do-nothing, Blue reward:0.85\n", - "step: 345, Red action: do-nothing, Blue reward:0.85\n", - "step: 346, Red action: do-nothing, Blue reward:0.85\n", - "step: 347, Red action: do-nothing, Blue reward:0.85\n", - "step: 348, Red action: do-nothing, Blue reward:0.85\n", - "step: 349, Red action: node-application-execute, Blue reward:0.85\n", - "step: 350, Red action: do-nothing, Blue reward:0.85\n", - "step: 351, Red action: do-nothing, Blue reward:0.85\n", - "step: 352, Red action: do-nothing, Blue reward:0.85\n", - "step: 353, Red action: do-nothing, Blue reward:0.85\n", - "step: 354, Red action: do-nothing, Blue reward:0.85\n", - "step: 355, Red action: do-nothing, Blue reward:0.85\n", - "step: 356, Red action: do-nothing, Blue reward:0.85\n", - "step: 357, Red action: do-nothing, Blue reward:0.85\n", - "step: 358, Red action: do-nothing, Blue reward:0.85\n", - "step: 359, Red action: do-nothing, Blue reward:0.85\n", - "step: 360, Red action: do-nothing, Blue reward:0.85\n", - "step: 361, Red action: do-nothing, Blue reward:0.85\n", - "step: 362, Red action: do-nothing, Blue reward:0.85\n", - "step: 363, Red action: do-nothing, Blue reward:0.85\n", - "step: 364, Red action: do-nothing, Blue reward:0.85\n", - "step: 365, Red action: do-nothing, Blue reward:0.85\n", - "step: 366, Red action: do-nothing, Blue reward:0.85\n", - "step: 367, Red action: do-nothing, Blue reward:0.85\n", - "step: 368, Red action: do-nothing, Blue reward:0.85\n", - "step: 369, Red action: node-application-execute, Blue reward:0.85\n", - "step: 370, Red action: do-nothing, Blue reward:0.85\n", - "step: 371, Red action: do-nothing, Blue reward:0.85\n", - "step: 372, Red action: do-nothing, Blue reward:0.85\n", - "step: 373, Red action: do-nothing, Blue reward:0.85\n", - "step: 374, Red action: do-nothing, Blue reward:0.85\n", - "step: 375, Red action: do-nothing, Blue reward:0.85\n", - "step: 376, Red action: do-nothing, Blue reward:0.85\n", - "step: 377, Red action: do-nothing, Blue reward:0.85\n", - "step: 378, Red action: do-nothing, Blue reward:0.85\n", - "step: 379, Red action: do-nothing, Blue reward:0.85\n", - "step: 380, Red action: do-nothing, Blue reward:0.85\n", - "step: 381, Red action: do-nothing, Blue reward:0.85\n", - "step: 382, Red action: do-nothing, Blue reward:0.85\n", - "step: 383, Red action: do-nothing, Blue reward:0.85\n", - "step: 384, Red action: do-nothing, Blue reward:0.85\n", - "step: 385, Red action: do-nothing, Blue reward:0.85\n", - "step: 386, Red action: do-nothing, Blue reward:0.85\n", - "step: 387, Red action: do-nothing, Blue reward:0.85\n", - "step: 388, Red action: do-nothing, Blue reward:0.85\n", - "step: 389, Red action: do-nothing, Blue reward:0.85\n", - "step: 390, Red action: do-nothing, Blue reward:0.85\n", - "step: 391, Red action: do-nothing, Blue reward:0.85\n", - "step: 392, Red action: do-nothing, Blue reward:0.85\n", - "step: 393, Red action: do-nothing, Blue reward:0.85\n", - "step: 394, Red action: node-application-execute, Blue reward:0.85\n", - "step: 395, Red action: do-nothing, Blue reward:0.85\n", - "step: 396, Red action: do-nothing, Blue reward:0.85\n", - "step: 397, Red action: do-nothing, Blue reward:0.85\n", - "step: 398, Red action: do-nothing, Blue reward:0.85\n", - "step: 399, Red action: do-nothing, Blue reward:0.85\n", - "step: 400, Red action: do-nothing, Blue reward:0.85\n", - "step: 401, Red action: do-nothing, Blue reward:0.85\n", - "step: 402, Red action: do-nothing, Blue reward:0.85\n", - "step: 403, Red action: do-nothing, Blue reward:0.85\n", - "step: 404, Red action: do-nothing, Blue reward:0.85\n", - "step: 405, Red action: do-nothing, Blue reward:0.85\n", - "step: 406, Red action: do-nothing, Blue reward:0.85\n", - "step: 407, Red action: do-nothing, Blue reward:0.85\n", - "step: 408, Red action: do-nothing, Blue reward:0.85\n", - "step: 409, Red action: do-nothing, Blue reward:0.85\n", - "step: 410, Red action: do-nothing, Blue reward:0.85\n", - "step: 411, Red action: do-nothing, Blue reward:0.85\n", - "step: 412, Red action: do-nothing, Blue reward:0.85\n", - "step: 413, Red action: do-nothing, Blue reward:0.85\n", - "step: 414, Red action: do-nothing, Blue reward:0.85\n", - "step: 415, Red action: do-nothing, Blue reward:0.85\n", - "step: 416, Red action: do-nothing, Blue reward:0.85\n", - "step: 417, Red action: do-nothing, Blue reward:0.85\n", - "step: 418, Red action: node-application-execute, Blue reward:0.85\n", - "step: 419, Red action: do-nothing, Blue reward:0.85\n", - "step: 420, Red action: do-nothing, Blue reward:0.85\n", - "step: 421, Red action: do-nothing, Blue reward:0.85\n", - "step: 422, Red action: do-nothing, Blue reward:0.85\n", - "step: 423, Red action: do-nothing, Blue reward:0.85\n", - "step: 424, Red action: do-nothing, Blue reward:0.85\n", - "step: 425, Red action: do-nothing, Blue reward:0.85\n", - "step: 426, Red action: do-nothing, Blue reward:0.85\n", - "step: 427, Red action: do-nothing, Blue reward:0.85\n", - "step: 428, Red action: do-nothing, Blue reward:0.85\n", - "step: 429, Red action: do-nothing, Blue reward:0.85\n", - "step: 430, Red action: do-nothing, Blue reward:0.85\n", - "step: 431, Red action: do-nothing, Blue reward:0.85\n", - "step: 432, Red action: do-nothing, Blue reward:0.85\n", - "step: 433, Red action: do-nothing, Blue reward:0.85\n", - "step: 434, Red action: do-nothing, Blue reward:0.85\n", - "step: 435, Red action: do-nothing, Blue reward:0.85\n", - "step: 436, Red action: do-nothing, Blue reward:0.85\n", - "step: 437, Red action: do-nothing, Blue reward:0.85\n", - "step: 438, Red action: do-nothing, Blue reward:0.85\n", - "step: 439, Red action: do-nothing, Blue reward:0.85\n", - "step: 440, Red action: do-nothing, Blue reward:0.85\n", - "step: 441, Red action: do-nothing, Blue reward:0.85\n", - "step: 442, Red action: do-nothing, Blue reward:0.85\n", - "step: 443, Red action: node-application-execute, Blue reward:0.85\n", - "step: 444, Red action: do-nothing, Blue reward:0.85\n", - "step: 445, Red action: do-nothing, Blue reward:0.85\n", - "step: 446, Red action: do-nothing, Blue reward:0.85\n", - "step: 447, Red action: do-nothing, Blue reward:0.85\n", - "step: 448, Red action: do-nothing, Blue reward:0.85\n", - "step: 449, Red action: do-nothing, Blue reward:0.85\n", - "step: 450, Red action: do-nothing, Blue reward:0.85\n", - "step: 451, Red action: do-nothing, Blue reward:0.85\n", - "step: 452, Red action: do-nothing, Blue reward:0.85\n", - "step: 453, Red action: do-nothing, Blue reward:0.85\n", - "step: 454, Red action: do-nothing, Blue reward:0.85\n", - "step: 455, Red action: do-nothing, Blue reward:0.85\n", - "step: 456, Red action: do-nothing, Blue reward:0.85\n", - "step: 457, Red action: do-nothing, Blue reward:0.85\n", - "step: 458, Red action: do-nothing, Blue reward:0.85\n", - "step: 459, Red action: do-nothing, Blue reward:0.85\n", - "step: 460, Red action: do-nothing, Blue reward:0.85\n", - "step: 461, Red action: node-application-execute, Blue reward:0.85\n", - "step: 462, Red action: do-nothing, Blue reward:0.85\n", - "step: 463, Red action: do-nothing, Blue reward:0.85\n", - "step: 464, Red action: do-nothing, Blue reward:0.85\n", - "step: 465, Red action: do-nothing, Blue reward:0.85\n", - "step: 466, Red action: do-nothing, Blue reward:0.85\n", - "step: 467, Red action: do-nothing, Blue reward:0.85\n", - "step: 468, Red action: do-nothing, Blue reward:0.85\n", - "step: 469, Red action: do-nothing, Blue reward:0.85\n", - "step: 470, Red action: do-nothing, Blue reward:0.85\n", - "step: 471, Red action: do-nothing, Blue reward:0.85\n", - "step: 472, Red action: do-nothing, Blue reward:0.85\n", - "step: 473, Red action: do-nothing, Blue reward:0.85\n", - "step: 474, Red action: do-nothing, Blue reward:0.85\n", - "step: 475, Red action: do-nothing, Blue reward:0.85\n", - "step: 476, Red action: do-nothing, Blue reward:0.85\n", - "step: 477, Red action: do-nothing, Blue reward:0.85\n", - "step: 478, Red action: do-nothing, Blue reward:0.85\n", - "step: 479, Red action: do-nothing, Blue reward:0.85\n", - "step: 480, Red action: node-application-execute, Blue reward:0.85\n", - "step: 481, Red action: do-nothing, Blue reward:0.85\n", - "step: 482, Red action: do-nothing, Blue reward:0.85\n", - "step: 483, Red action: do-nothing, Blue reward:0.85\n", - "step: 484, Red action: do-nothing, Blue reward:0.85\n", - "step: 485, Red action: do-nothing, Blue reward:0.85\n", - "step: 486, Red action: do-nothing, Blue reward:0.85\n", - "step: 487, Red action: do-nothing, Blue reward:0.85\n", - "step: 488, Red action: do-nothing, Blue reward:0.85\n", - "step: 489, Red action: do-nothing, Blue reward:0.85\n", - "step: 490, Red action: do-nothing, Blue reward:0.85\n", - "step: 491, Red action: do-nothing, Blue reward:0.85\n", - "step: 492, Red action: do-nothing, Blue reward:0.85\n", - "step: 493, Red action: do-nothing, Blue reward:0.85\n", - "step: 494, Red action: do-nothing, Blue reward:0.85\n", - "step: 495, Red action: do-nothing, Blue reward:0.85\n", - "step: 496, Red action: do-nothing, Blue reward:0.85\n", - "step: 497, Red action: do-nothing, Blue reward:0.85\n", - "step: 498, Red action: do-nothing, Blue reward:0.85\n", - "step: 499, Red action: do-nothing, Blue reward:0.85\n", - "step: 500, Red action: do-nothing, Blue reward:0.85\n", - "step: 501, Red action: do-nothing, Blue reward:0.85\n", - "step: 502, Red action: do-nothing, Blue reward:0.85\n", - "step: 503, Red action: do-nothing, Blue reward:0.85\n", - "step: 504, Red action: node-application-execute, Blue reward:0.85\n", - "step: 505, Red action: do-nothing, Blue reward:0.85\n", - "step: 506, Red action: do-nothing, Blue reward:0.85\n", - "step: 507, Red action: do-nothing, Blue reward:0.85\n", - "step: 508, Red action: do-nothing, Blue reward:0.85\n", - "step: 509, Red action: do-nothing, Blue reward:0.85\n", - "step: 510, Red action: do-nothing, Blue reward:0.85\n", - "step: 511, Red action: do-nothing, Blue reward:0.85\n", - "step: 512, Red action: do-nothing, Blue reward:0.85\n", - "step: 513, Red action: do-nothing, Blue reward:0.85\n", - "step: 514, Red action: do-nothing, Blue reward:0.85\n", - "step: 515, Red action: do-nothing, Blue reward:0.85\n", - "step: 516, Red action: do-nothing, Blue reward:0.85\n", - "step: 517, Red action: do-nothing, Blue reward:0.85\n", - "step: 518, Red action: do-nothing, Blue reward:0.85\n", - "step: 519, Red action: do-nothing, Blue reward:0.85\n", - "step: 520, Red action: do-nothing, Blue reward:0.85\n", - "step: 521, Red action: do-nothing, Blue reward:0.85\n", - "step: 522, Red action: do-nothing, Blue reward:0.85\n", - "step: 523, Red action: do-nothing, Blue reward:0.85\n", - "step: 524, Red action: do-nothing, Blue reward:0.85\n", - "step: 525, Red action: do-nothing, Blue reward:0.85\n", - "step: 526, Red action: do-nothing, Blue reward:0.85\n", - "step: 527, Red action: node-application-execute, Blue reward:0.85\n", - "step: 528, Red action: do-nothing, Blue reward:0.85\n", - "step: 529, Red action: do-nothing, Blue reward:0.85\n", - "step: 530, Red action: do-nothing, Blue reward:0.85\n", - "step: 531, Red action: do-nothing, Blue reward:0.85\n", - "step: 532, Red action: do-nothing, Blue reward:0.85\n", - "step: 533, Red action: do-nothing, Blue reward:0.85\n", - "step: 534, Red action: do-nothing, Blue reward:0.85\n", - "step: 535, Red action: do-nothing, Blue reward:0.85\n", - "step: 536, Red action: do-nothing, Blue reward:0.85\n", - "step: 537, Red action: do-nothing, Blue reward:0.85\n", - "step: 538, Red action: do-nothing, Blue reward:0.85\n", - "step: 539, Red action: do-nothing, Blue reward:0.85\n", - "step: 540, Red action: do-nothing, Blue reward:0.85\n", - "step: 541, Red action: do-nothing, Blue reward:0.85\n", - "step: 542, Red action: do-nothing, Blue reward:0.85\n", - "step: 543, Red action: do-nothing, Blue reward:0.85\n", - "step: 544, Red action: do-nothing, Blue reward:0.85\n", - "step: 545, Red action: do-nothing, Blue reward:0.85\n", - "step: 546, Red action: do-nothing, Blue reward:0.85\n", - "step: 547, Red action: do-nothing, Blue reward:0.85\n", - "step: 548, Red action: do-nothing, Blue reward:0.85\n", - "step: 549, Red action: node-application-execute, Blue reward:0.85\n", - "step: 550, Red action: do-nothing, Blue reward:0.85\n", - "step: 551, Red action: do-nothing, Blue reward:0.85\n", - "step: 552, Red action: do-nothing, Blue reward:0.85\n", - "step: 553, Red action: do-nothing, Blue reward:0.85\n", - "step: 554, Red action: do-nothing, Blue reward:0.85\n", - "step: 555, Red action: do-nothing, Blue reward:0.85\n", - "step: 556, Red action: do-nothing, Blue reward:0.85\n", - "step: 557, Red action: do-nothing, Blue reward:0.85\n", - "step: 558, Red action: do-nothing, Blue reward:0.85\n", - "step: 559, Red action: do-nothing, Blue reward:0.85\n", - "step: 560, Red action: do-nothing, Blue reward:0.85\n", - "step: 561, Red action: do-nothing, Blue reward:0.85\n", - "step: 562, Red action: do-nothing, Blue reward:0.85\n", - "step: 563, Red action: do-nothing, Blue reward:0.85\n", - "step: 564, Red action: do-nothing, Blue reward:0.85\n", - "step: 565, Red action: node-application-execute, Blue reward:0.85\n", - "step: 566, Red action: do-nothing, Blue reward:0.85\n", - "step: 567, Red action: do-nothing, Blue reward:0.85\n", - "step: 568, Red action: do-nothing, Blue reward:0.85\n", - "step: 569, Red action: do-nothing, Blue reward:0.85\n", - "step: 570, Red action: do-nothing, Blue reward:0.85\n", - "step: 571, Red action: do-nothing, Blue reward:0.85\n", - "step: 572, Red action: do-nothing, Blue reward:0.85\n", - "step: 573, Red action: do-nothing, Blue reward:0.85\n", - "step: 574, Red action: do-nothing, Blue reward:0.85\n", - "step: 575, Red action: do-nothing, Blue reward:0.85\n", - "step: 576, Red action: do-nothing, Blue reward:0.85\n", - "step: 577, Red action: do-nothing, Blue reward:0.85\n", - "step: 578, Red action: do-nothing, Blue reward:0.85\n", - "step: 579, Red action: do-nothing, Blue reward:0.85\n", - "step: 580, Red action: do-nothing, Blue reward:0.85\n", - "step: 581, Red action: do-nothing, Blue reward:0.85\n", - "step: 582, Red action: do-nothing, Blue reward:0.85\n", - "step: 583, Red action: do-nothing, Blue reward:0.85\n", - "step: 584, Red action: do-nothing, Blue reward:0.85\n", - "step: 585, Red action: do-nothing, Blue reward:0.85\n", - "step: 586, Red action: do-nothing, Blue reward:0.85\n", - "step: 587, Red action: do-nothing, Blue reward:0.85\n", - "step: 588, Red action: do-nothing, Blue reward:0.85\n", - "step: 589, Red action: node-application-execute, Blue reward:0.85\n", - "step: 590, Red action: do-nothing, Blue reward:0.85\n", - "step: 591, Red action: do-nothing, Blue reward:0.85\n", - "step: 592, Red action: do-nothing, Blue reward:0.85\n", - "step: 593, Red action: do-nothing, Blue reward:0.85\n", - "step: 594, Red action: do-nothing, Blue reward:0.85\n", - "step: 595, Red action: do-nothing, Blue reward:0.85\n", - "step: 596, Red action: do-nothing, Blue reward:0.85\n", - "step: 597, Red action: do-nothing, Blue reward:0.85\n", - "step: 598, Red action: do-nothing, Blue reward:0.85\n", - "step: 599, Red action: do-nothing, Blue reward:0.85\n", - "step: 600, Red action: do-nothing, Blue reward:0.85\n", - "step: 601, Red action: do-nothing, Blue reward:0.85\n", - "step: 602, Red action: do-nothing, Blue reward:0.85\n", - "step: 603, Red action: do-nothing, Blue reward:0.85\n", - "step: 604, Red action: do-nothing, Blue reward:0.85\n", - "step: 605, Red action: do-nothing, Blue reward:0.85\n", - "step: 606, Red action: do-nothing, Blue reward:0.85\n", - "step: 607, Red action: do-nothing, Blue reward:0.85\n", - "step: 608, Red action: do-nothing, Blue reward:0.85\n", - "step: 609, Red action: do-nothing, Blue reward:0.85\n", - "step: 610, Red action: node-application-execute, Blue reward:0.85\n", - "step: 611, Red action: do-nothing, Blue reward:0.85\n", - "step: 612, Red action: do-nothing, Blue reward:0.85\n", - "step: 613, Red action: do-nothing, Blue reward:0.85\n", - "step: 614, Red action: do-nothing, Blue reward:0.85\n", - "step: 615, Red action: do-nothing, Blue reward:0.85\n", - "step: 616, Red action: do-nothing, Blue reward:0.85\n", - "step: 617, Red action: do-nothing, Blue reward:0.85\n", - "step: 618, Red action: do-nothing, Blue reward:0.85\n", - "step: 619, Red action: do-nothing, Blue reward:0.85\n", - "step: 620, Red action: do-nothing, Blue reward:0.85\n", - "step: 621, Red action: do-nothing, Blue reward:0.85\n", - "step: 622, Red action: do-nothing, Blue reward:0.85\n", - "step: 623, Red action: do-nothing, Blue reward:0.85\n", - "step: 624, Red action: do-nothing, Blue reward:0.85\n", - "step: 625, Red action: do-nothing, Blue reward:0.85\n", - "step: 626, Red action: node-application-execute, Blue reward:0.85\n", - "step: 627, Red action: do-nothing, Blue reward:0.85\n", - "step: 628, Red action: do-nothing, Blue reward:0.85\n", - "step: 629, Red action: do-nothing, Blue reward:0.85\n", - "step: 630, Red action: do-nothing, Blue reward:0.85\n", - "step: 631, Red action: do-nothing, Blue reward:0.85\n", - "step: 632, Red action: do-nothing, Blue reward:0.85\n", - "step: 633, Red action: do-nothing, Blue reward:0.85\n", - "step: 634, Red action: do-nothing, Blue reward:0.85\n", - "step: 635, Red action: do-nothing, Blue reward:0.85\n", - "step: 636, Red action: do-nothing, Blue reward:0.85\n", - "step: 637, Red action: do-nothing, Blue reward:0.85\n", - "step: 638, Red action: do-nothing, Blue reward:0.85\n", - "step: 639, Red action: do-nothing, Blue reward:0.85\n", - "step: 640, Red action: do-nothing, Blue reward:0.85\n", - "step: 641, Red action: do-nothing, Blue reward:0.85\n", - "step: 642, Red action: do-nothing, Blue reward:0.85\n", - "step: 643, Red action: do-nothing, Blue reward:0.85\n", - "step: 644, Red action: node-application-execute, Blue reward:0.85\n", - "step: 645, Red action: do-nothing, Blue reward:0.85\n", - "step: 646, Red action: do-nothing, Blue reward:0.85\n", - "step: 647, Red action: do-nothing, Blue reward:0.85\n", - "step: 648, Red action: do-nothing, Blue reward:0.85\n", - "step: 649, Red action: do-nothing, Blue reward:0.85\n", - "step: 650, Red action: do-nothing, Blue reward:0.85\n", - "step: 651, Red action: do-nothing, Blue reward:0.85\n", - "step: 652, Red action: do-nothing, Blue reward:0.85\n", - "step: 653, Red action: do-nothing, Blue reward:0.85\n", - "step: 654, Red action: do-nothing, Blue reward:0.85\n", - "step: 655, Red action: do-nothing, Blue reward:0.85\n", - "step: 656, Red action: do-nothing, Blue reward:0.85\n", - "step: 657, Red action: do-nothing, Blue reward:0.85\n", - "step: 658, Red action: do-nothing, Blue reward:0.85\n", - "step: 659, Red action: do-nothing, Blue reward:0.85\n", - "step: 660, Red action: do-nothing, Blue reward:0.85\n", - "step: 661, Red action: do-nothing, Blue reward:0.85\n", - "step: 662, Red action: do-nothing, Blue reward:0.85\n", - "step: 663, Red action: do-nothing, Blue reward:0.85\n", - "step: 664, Red action: do-nothing, Blue reward:0.85\n", - "step: 665, Red action: do-nothing, Blue reward:0.85\n", - "step: 666, Red action: do-nothing, Blue reward:0.85\n", - "step: 667, Red action: node-application-execute, Blue reward:0.85\n", - "step: 668, Red action: do-nothing, Blue reward:0.85\n", - "step: 669, Red action: do-nothing, Blue reward:0.85\n", - "step: 670, Red action: do-nothing, Blue reward:0.85\n", - "step: 671, Red action: do-nothing, Blue reward:0.85\n", - "step: 672, Red action: do-nothing, Blue reward:0.85\n", - "step: 673, Red action: do-nothing, Blue reward:0.85\n", - "step: 674, Red action: do-nothing, Blue reward:0.85\n", - "step: 675, Red action: do-nothing, Blue reward:0.85\n", - "step: 676, Red action: do-nothing, Blue reward:0.85\n", - "step: 677, Red action: do-nothing, Blue reward:0.85\n", - "step: 678, Red action: do-nothing, Blue reward:0.85\n", - "step: 679, Red action: do-nothing, Blue reward:0.85\n", - "step: 680, Red action: do-nothing, Blue reward:0.85\n", - "step: 681, Red action: do-nothing, Blue reward:0.85\n", - "step: 682, Red action: do-nothing, Blue reward:0.85\n", - "step: 683, Red action: do-nothing, Blue reward:0.85\n", - "step: 684, Red action: do-nothing, Blue reward:0.85\n", - "step: 685, Red action: node-application-execute, Blue reward:0.85\n", - "step: 686, Red action: do-nothing, Blue reward:0.85\n", - "step: 687, Red action: do-nothing, Blue reward:0.85\n", - "step: 688, Red action: do-nothing, Blue reward:0.85\n", - "step: 689, Red action: do-nothing, Blue reward:0.85\n", - "step: 690, Red action: do-nothing, Blue reward:0.85\n", - "step: 691, Red action: do-nothing, Blue reward:0.85\n", - "step: 692, Red action: do-nothing, Blue reward:0.85\n", - "step: 693, Red action: do-nothing, Blue reward:0.85\n", - "step: 694, Red action: do-nothing, Blue reward:0.85\n", - "step: 695, Red action: do-nothing, Blue reward:0.85\n", - "step: 696, Red action: do-nothing, Blue reward:0.85\n", - "step: 697, Red action: do-nothing, Blue reward:0.85\n", - "step: 698, Red action: do-nothing, Blue reward:0.85\n", - "step: 699, Red action: do-nothing, Blue reward:0.85\n", - "step: 700, Red action: do-nothing, Blue reward:0.85\n", - "step: 701, Red action: node-application-execute, Blue reward:0.85\n", - "step: 702, Red action: do-nothing, Blue reward:0.85\n", - "step: 703, Red action: do-nothing, Blue reward:0.85\n", - "step: 704, Red action: do-nothing, Blue reward:0.85\n", - "step: 705, Red action: do-nothing, Blue reward:0.85\n", - "step: 706, Red action: do-nothing, Blue reward:0.85\n", - "step: 707, Red action: do-nothing, Blue reward:0.85\n", - "step: 708, Red action: do-nothing, Blue reward:0.85\n", - "step: 709, Red action: do-nothing, Blue reward:0.85\n", - "step: 710, Red action: do-nothing, Blue reward:0.85\n", - "step: 711, Red action: do-nothing, Blue reward:0.85\n", - "step: 712, Red action: do-nothing, Blue reward:0.85\n", - "step: 713, Red action: do-nothing, Blue reward:0.85\n", - "step: 714, Red action: do-nothing, Blue reward:0.85\n", - "step: 715, Red action: do-nothing, Blue reward:0.85\n", - "step: 716, Red action: do-nothing, Blue reward:0.85\n", - "step: 717, Red action: do-nothing, Blue reward:0.85\n", - "step: 718, Red action: node-application-execute, Blue reward:0.85\n", - "step: 719, Red action: do-nothing, Blue reward:0.85\n", - "step: 720, Red action: do-nothing, Blue reward:0.85\n", - "step: 721, Red action: do-nothing, Blue reward:0.85\n", - "step: 722, Red action: do-nothing, Blue reward:0.85\n", - "step: 723, Red action: do-nothing, Blue reward:0.85\n", - "step: 724, Red action: do-nothing, Blue reward:0.85\n", - "step: 725, Red action: do-nothing, Blue reward:0.85\n", - "step: 726, Red action: do-nothing, Blue reward:0.85\n", - "step: 727, Red action: do-nothing, Blue reward:0.85\n", - "step: 728, Red action: do-nothing, Blue reward:0.85\n", - "step: 729, Red action: do-nothing, Blue reward:0.85\n", - "step: 730, Red action: do-nothing, Blue reward:0.85\n", - "step: 731, Red action: do-nothing, Blue reward:0.85\n", - "step: 732, Red action: do-nothing, Blue reward:0.85\n", - "step: 733, Red action: do-nothing, Blue reward:0.85\n", - "step: 734, Red action: do-nothing, Blue reward:0.85\n", - "step: 735, Red action: do-nothing, Blue reward:0.85\n", - "step: 736, Red action: do-nothing, Blue reward:0.85\n", - "step: 737, Red action: do-nothing, Blue reward:0.85\n", - "step: 738, Red action: do-nothing, Blue reward:0.85\n", - "step: 739, Red action: do-nothing, Blue reward:0.85\n", - "step: 740, Red action: do-nothing, Blue reward:0.85\n", - "step: 741, Red action: do-nothing, Blue reward:0.85\n", - "step: 742, Red action: do-nothing, Blue reward:0.85\n", - "step: 743, Red action: node-application-execute, Blue reward:0.85\n", - "step: 744, Red action: do-nothing, Blue reward:0.85\n", - "step: 745, Red action: do-nothing, Blue reward:0.85\n", - "step: 746, Red action: do-nothing, Blue reward:0.85\n", - "step: 747, Red action: do-nothing, Blue reward:0.85\n", - "step: 748, Red action: do-nothing, Blue reward:0.85\n", - "step: 749, Red action: do-nothing, Blue reward:0.85\n", - "step: 750, Red action: do-nothing, Blue reward:0.85\n", - "step: 751, Red action: do-nothing, Blue reward:0.85\n", - "step: 752, Red action: do-nothing, Blue reward:0.85\n", - "step: 753, Red action: do-nothing, Blue reward:0.85\n", - "step: 754, Red action: do-nothing, Blue reward:0.85\n", - "step: 755, Red action: do-nothing, Blue reward:0.85\n", - "step: 756, Red action: do-nothing, Blue reward:0.85\n", - "step: 757, Red action: do-nothing, Blue reward:0.85\n", - "step: 758, Red action: do-nothing, Blue reward:0.85\n", - "step: 759, Red action: do-nothing, Blue reward:0.85\n", - "step: 760, Red action: node-application-execute, Blue reward:0.85\n", - "step: 761, Red action: do-nothing, Blue reward:0.85\n", - "step: 762, Red action: do-nothing, Blue reward:0.85\n", - "step: 763, Red action: do-nothing, Blue reward:0.85\n", - "step: 764, Red action: do-nothing, Blue reward:0.85\n", - "step: 765, Red action: do-nothing, Blue reward:0.85\n", - "step: 766, Red action: do-nothing, Blue reward:0.85\n", - "step: 767, Red action: do-nothing, Blue reward:0.85\n", - "step: 768, Red action: do-nothing, Blue reward:0.85\n", - "step: 769, Red action: do-nothing, Blue reward:0.85\n", - "step: 770, Red action: do-nothing, Blue reward:0.85\n", - "step: 771, Red action: do-nothing, Blue reward:0.85\n", - "step: 772, Red action: do-nothing, Blue reward:0.85\n", - "step: 773, Red action: do-nothing, Blue reward:0.85\n", - "step: 774, Red action: do-nothing, Blue reward:0.85\n", - "step: 775, Red action: do-nothing, Blue reward:0.85\n", - "step: 776, Red action: do-nothing, Blue reward:0.85\n", - "step: 777, Red action: do-nothing, Blue reward:0.85\n", - "step: 778, Red action: do-nothing, Blue reward:0.85\n", - "step: 779, Red action: do-nothing, Blue reward:0.85\n", - "step: 780, Red action: do-nothing, Blue reward:0.85\n", - "step: 781, Red action: do-nothing, Blue reward:0.85\n", - "step: 782, Red action: do-nothing, Blue reward:0.85\n", - "step: 783, Red action: node-application-execute, Blue reward:0.85\n", - "step: 784, Red action: do-nothing, Blue reward:0.85\n", - "step: 785, Red action: do-nothing, Blue reward:0.85\n", - "step: 786, Red action: do-nothing, Blue reward:0.85\n", - "step: 787, Red action: do-nothing, Blue reward:0.85\n", - "step: 788, Red action: do-nothing, Blue reward:0.85\n", - "step: 789, Red action: do-nothing, Blue reward:0.85\n", - "step: 790, Red action: do-nothing, Blue reward:0.85\n", - "step: 791, Red action: do-nothing, Blue reward:0.85\n", - "step: 792, Red action: do-nothing, Blue reward:0.85\n", - "step: 793, Red action: do-nothing, Blue reward:0.85\n", - "step: 794, Red action: do-nothing, Blue reward:0.85\n", - "step: 795, Red action: do-nothing, Blue reward:0.85\n", - "step: 796, Red action: do-nothing, Blue reward:0.85\n", - "step: 797, Red action: do-nothing, Blue reward:0.85\n", - "step: 798, Red action: do-nothing, Blue reward:0.85\n", - "step: 799, Red action: do-nothing, Blue reward:0.85\n", - "step: 800, Red action: do-nothing, Blue reward:0.85\n", - "step: 801, Red action: do-nothing, Blue reward:0.85\n", - "step: 802, Red action: do-nothing, Blue reward:0.85\n", - "step: 803, Red action: node-application-execute, Blue reward:0.85\n", - "step: 804, Red action: do-nothing, Blue reward:0.85\n", - "step: 805, Red action: do-nothing, Blue reward:0.85\n", - "step: 806, Red action: do-nothing, Blue reward:0.85\n", - "step: 807, Red action: do-nothing, Blue reward:0.85\n", - "step: 808, Red action: do-nothing, Blue reward:0.85\n", - "step: 809, Red action: do-nothing, Blue reward:0.85\n", - "step: 810, Red action: do-nothing, Blue reward:0.85\n", - "step: 811, Red action: do-nothing, Blue reward:0.85\n", - "step: 812, Red action: do-nothing, Blue reward:0.85\n", - "step: 813, Red action: do-nothing, Blue reward:0.85\n", - "step: 814, Red action: do-nothing, Blue reward:0.85\n", - "step: 815, Red action: do-nothing, Blue reward:0.85\n", - "step: 816, Red action: do-nothing, Blue reward:0.85\n", - "step: 817, Red action: do-nothing, Blue reward:0.85\n", - "step: 818, Red action: do-nothing, Blue reward:0.85\n", - "step: 819, Red action: do-nothing, Blue reward:0.85\n", - "step: 820, Red action: do-nothing, Blue reward:0.85\n", - "step: 821, Red action: do-nothing, Blue reward:0.85\n", - "step: 822, Red action: do-nothing, Blue reward:0.85\n", - "step: 823, Red action: do-nothing, Blue reward:0.85\n", - "step: 824, Red action: do-nothing, Blue reward:0.85\n", - "step: 825, Red action: do-nothing, Blue reward:0.85\n", - "step: 826, Red action: do-nothing, Blue reward:0.85\n", - "step: 827, Red action: node-application-execute, Blue reward:0.85\n", - "step: 828, Red action: do-nothing, Blue reward:0.85\n", - "step: 829, Red action: do-nothing, Blue reward:0.85\n", - "step: 830, Red action: do-nothing, Blue reward:0.85\n", - "step: 831, Red action: do-nothing, Blue reward:0.85\n", - "step: 832, Red action: do-nothing, Blue reward:0.85\n", - "step: 833, Red action: do-nothing, Blue reward:0.85\n", - "step: 834, Red action: do-nothing, Blue reward:0.85\n", - "step: 835, Red action: do-nothing, Blue reward:0.85\n", - "step: 836, Red action: do-nothing, Blue reward:0.85\n", - "step: 837, Red action: do-nothing, Blue reward:0.85\n", - "step: 838, Red action: do-nothing, Blue reward:0.85\n", - "step: 839, Red action: do-nothing, Blue reward:0.85\n", - "step: 840, Red action: do-nothing, Blue reward:0.85\n", - "step: 841, Red action: do-nothing, Blue reward:0.85\n", - "step: 842, Red action: do-nothing, Blue reward:0.85\n", - "step: 843, Red action: do-nothing, Blue reward:0.85\n", - "step: 844, Red action: do-nothing, Blue reward:0.85\n", - "step: 845, Red action: do-nothing, Blue reward:0.85\n", - "step: 846, Red action: do-nothing, Blue reward:0.85\n", - "step: 847, Red action: node-application-execute, Blue reward:0.85\n", - "step: 848, Red action: do-nothing, Blue reward:0.85\n", - "step: 849, Red action: do-nothing, Blue reward:0.85\n", - "step: 850, Red action: do-nothing, Blue reward:0.85\n", - "step: 851, Red action: do-nothing, Blue reward:0.85\n", - "step: 852, Red action: do-nothing, Blue reward:0.85\n", - "step: 853, Red action: do-nothing, Blue reward:0.85\n", - "step: 854, Red action: do-nothing, Blue reward:0.85\n", - "step: 855, Red action: do-nothing, Blue reward:0.85\n", - "step: 856, Red action: do-nothing, Blue reward:0.85\n", - "step: 857, Red action: do-nothing, Blue reward:0.85\n", - "step: 858, Red action: do-nothing, Blue reward:0.85\n", - "step: 859, Red action: do-nothing, Blue reward:0.85\n", - "step: 860, Red action: do-nothing, Blue reward:0.85\n", - "step: 861, Red action: do-nothing, Blue reward:0.85\n", - "step: 862, Red action: do-nothing, Blue reward:0.85\n", - "step: 863, Red action: do-nothing, Blue reward:0.85\n", - "step: 864, Red action: do-nothing, Blue reward:0.85\n", - "step: 865, Red action: do-nothing, Blue reward:0.85\n", - "step: 866, Red action: do-nothing, Blue reward:0.85\n", - "step: 867, Red action: do-nothing, Blue reward:0.85\n", - "step: 868, Red action: do-nothing, Blue reward:0.85\n", - "step: 869, Red action: do-nothing, Blue reward:0.85\n", - "step: 870, Red action: do-nothing, Blue reward:0.85\n", - "step: 871, Red action: node-application-execute, Blue reward:0.85\n", - "step: 872, Red action: do-nothing, Blue reward:0.85\n", - "step: 873, Red action: do-nothing, Blue reward:0.85\n", - "step: 874, Red action: do-nothing, Blue reward:0.85\n", - "step: 875, Red action: do-nothing, Blue reward:0.85\n", - "step: 876, Red action: do-nothing, Blue reward:0.85\n", - "step: 877, Red action: do-nothing, Blue reward:0.85\n", - "step: 878, Red action: do-nothing, Blue reward:0.85\n", - "step: 879, Red action: do-nothing, Blue reward:0.85\n", - "step: 880, Red action: do-nothing, Blue reward:0.85\n", - "step: 881, Red action: do-nothing, Blue reward:0.85\n", - "step: 882, Red action: do-nothing, Blue reward:0.85\n", - "step: 883, Red action: do-nothing, Blue reward:0.85\n", - "step: 884, Red action: do-nothing, Blue reward:0.85\n", - "step: 885, Red action: do-nothing, Blue reward:0.85\n", - "step: 886, Red action: do-nothing, Blue reward:0.85\n", - "step: 887, Red action: do-nothing, Blue reward:0.85\n", - "step: 888, Red action: do-nothing, Blue reward:0.85\n", - "step: 889, Red action: do-nothing, Blue reward:0.85\n", - "step: 890, Red action: do-nothing, Blue reward:0.85\n", - "step: 891, Red action: node-application-execute, Blue reward:0.85\n", - "step: 892, Red action: do-nothing, Blue reward:0.85\n", - "step: 893, Red action: do-nothing, Blue reward:0.85\n", - "step: 894, Red action: do-nothing, Blue reward:0.85\n", - "step: 895, Red action: do-nothing, Blue reward:0.85\n", - "step: 896, Red action: do-nothing, Blue reward:0.85\n", - "step: 897, Red action: do-nothing, Blue reward:0.85\n", - "step: 898, Red action: do-nothing, Blue reward:0.85\n", - "step: 899, Red action: do-nothing, Blue reward:0.85\n", - "step: 900, Red action: do-nothing, Blue reward:0.85\n", - "step: 901, Red action: do-nothing, Blue reward:0.85\n", - "step: 902, Red action: do-nothing, Blue reward:0.85\n", - "step: 903, Red action: do-nothing, Blue reward:0.85\n", - "step: 904, Red action: do-nothing, Blue reward:0.85\n", - "step: 905, Red action: do-nothing, Blue reward:0.85\n", - "step: 906, Red action: node-application-execute, Blue reward:0.85\n", - "step: 907, Red action: do-nothing, Blue reward:0.85\n", - "step: 908, Red action: do-nothing, Blue reward:0.85\n", - "step: 909, Red action: do-nothing, Blue reward:0.85\n", - "step: 910, Red action: do-nothing, Blue reward:0.85\n", - "step: 911, Red action: do-nothing, Blue reward:0.85\n", - "step: 912, Red action: do-nothing, Blue reward:0.85\n", - "step: 913, Red action: do-nothing, Blue reward:0.85\n", - "step: 914, Red action: do-nothing, Blue reward:0.85\n", - "step: 915, Red action: do-nothing, Blue reward:0.85\n", - "step: 916, Red action: do-nothing, Blue reward:0.85\n", - "step: 917, Red action: do-nothing, Blue reward:0.85\n", - "step: 918, Red action: do-nothing, Blue reward:0.85\n", - "step: 919, Red action: do-nothing, Blue reward:0.85\n", - "step: 920, Red action: do-nothing, Blue reward:0.85\n", - "step: 921, Red action: do-nothing, Blue reward:0.85\n", - "step: 922, Red action: do-nothing, Blue reward:0.85\n", - "step: 923, Red action: do-nothing, Blue reward:0.85\n", - "step: 924, Red action: do-nothing, Blue reward:0.85\n", - "step: 925, Red action: do-nothing, Blue reward:0.85\n", - "step: 926, Red action: do-nothing, Blue reward:0.85\n", - "step: 927, Red action: do-nothing, Blue reward:0.85\n", - "step: 928, Red action: do-nothing, Blue reward:0.85\n", - "step: 929, Red action: do-nothing, Blue reward:0.85\n", - "step: 930, Red action: node-application-execute, Blue reward:0.85\n", - "step: 931, Red action: do-nothing, Blue reward:0.85\n", - "step: 932, Red action: do-nothing, Blue reward:0.85\n", - "step: 933, Red action: do-nothing, Blue reward:0.85\n", - "step: 934, Red action: do-nothing, Blue reward:0.85\n", - "step: 935, Red action: do-nothing, Blue reward:0.85\n", - "step: 936, Red action: do-nothing, Blue reward:0.85\n", - "step: 937, Red action: do-nothing, Blue reward:0.85\n", - "step: 938, Red action: do-nothing, Blue reward:0.85\n", - "step: 939, Red action: do-nothing, Blue reward:0.85\n", - "step: 940, Red action: do-nothing, Blue reward:0.85\n", - "step: 941, Red action: do-nothing, Blue reward:0.85\n", - "step: 942, Red action: do-nothing, Blue reward:0.85\n", - "step: 943, Red action: do-nothing, Blue reward:0.85\n", - "step: 944, Red action: do-nothing, Blue reward:0.85\n", - "step: 945, Red action: do-nothing, Blue reward:0.85\n", - "step: 946, Red action: do-nothing, Blue reward:0.85\n", - "step: 947, Red action: node-application-execute, Blue reward:0.85\n", - "step: 948, Red action: do-nothing, Blue reward:0.85\n", - "step: 949, Red action: do-nothing, Blue reward:0.85\n", - "step: 950, Red action: do-nothing, Blue reward:0.85\n", - "step: 951, Red action: do-nothing, Blue reward:0.85\n", - "step: 952, Red action: do-nothing, Blue reward:0.85\n", - "step: 953, Red action: do-nothing, Blue reward:0.85\n", - "step: 954, Red action: do-nothing, Blue reward:0.85\n", - "step: 955, Red action: do-nothing, Blue reward:0.85\n", - "step: 956, Red action: do-nothing, Blue reward:0.85\n", - "step: 957, Red action: do-nothing, Blue reward:0.85\n", - "step: 958, Red action: do-nothing, Blue reward:0.85\n", - "step: 959, Red action: do-nothing, Blue reward:0.85\n", - "step: 960, Red action: do-nothing, Blue reward:0.85\n", - "step: 961, Red action: do-nothing, Blue reward:0.85\n", - "step: 962, Red action: do-nothing, Blue reward:0.85\n", - "step: 963, Red action: node-application-execute, Blue reward:0.85\n", - "step: 964, Red action: do-nothing, Blue reward:0.85\n", - "step: 965, Red action: do-nothing, Blue reward:0.85\n", - "step: 966, Red action: do-nothing, Blue reward:0.85\n", - "step: 967, Red action: do-nothing, Blue reward:0.85\n", - "step: 968, Red action: do-nothing, Blue reward:0.85\n", - "step: 969, Red action: do-nothing, Blue reward:0.85\n", - "step: 970, Red action: do-nothing, Blue reward:0.85\n", - "step: 971, Red action: do-nothing, Blue reward:0.85\n", - "step: 972, Red action: do-nothing, Blue reward:0.85\n", - "step: 973, Red action: do-nothing, Blue reward:0.85\n", - "step: 974, Red action: do-nothing, Blue reward:0.85\n", - "step: 975, Red action: do-nothing, Blue reward:0.85\n", - "step: 976, Red action: do-nothing, Blue reward:0.85\n", - "step: 977, Red action: do-nothing, Blue reward:0.85\n", - "step: 978, Red action: do-nothing, Blue reward:0.85\n", - "step: 979, Red action: do-nothing, Blue reward:0.85\n", - "step: 980, Red action: do-nothing, Blue reward:0.85\n", - "step: 981, Red action: do-nothing, Blue reward:0.85\n", - "step: 982, Red action: do-nothing, Blue reward:0.85\n", - "step: 983, Red action: do-nothing, Blue reward:0.85\n", - "step: 984, Red action: do-nothing, Blue reward:0.85\n", - "step: 985, Red action: do-nothing, Blue reward:0.85\n", - "step: 986, Red action: do-nothing, Blue reward:0.85\n", - "step: 987, Red action: do-nothing, Blue reward:0.85\n", - "step: 988, Red action: node-application-execute, Blue reward:0.85\n", - "step: 989, Red action: do-nothing, Blue reward:0.85\n", - "step: 990, Red action: do-nothing, Blue reward:0.85\n", - "step: 991, Red action: do-nothing, Blue reward:0.85\n", - "step: 992, Red action: do-nothing, Blue reward:0.85\n", - "step: 993, Red action: do-nothing, Blue reward:0.85\n", - "step: 994, Red action: do-nothing, Blue reward:0.85\n", - "step: 995, Red action: do-nothing, Blue reward:0.85\n", - "step: 996, Red action: do-nothing, Blue reward:0.85\n", - "step: 997, Red action: do-nothing, Blue reward:0.85\n", - "step: 998, Red action: do-nothing, Blue reward:0.85\n", - "step: 999, Red action: do-nothing, Blue reward:0.85\n", - "step: 1000, Red action: do-nothing, Blue reward:0.85\n", - "step: 1001, Red action: do-nothing, Blue reward:0.85\n", - "step: 1002, Red action: do-nothing, Blue reward:0.85\n", - "step: 1003, Red action: do-nothing, Blue reward:0.85\n", - "step: 1004, Red action: do-nothing, Blue reward:0.85\n", - "step: 1005, Red action: do-nothing, Blue reward:0.85\n", - "step: 1006, Red action: do-nothing, Blue reward:0.85\n", - "step: 1007, Red action: do-nothing, Blue reward:0.85\n", - "step: 1008, Red action: do-nothing, Blue reward:0.85\n", - "step: 1009, Red action: do-nothing, Blue reward:0.85\n", - "step: 1010, Red action: do-nothing, Blue reward:0.85\n", - "step: 1011, Red action: do-nothing, Blue reward:0.85\n", - "step: 1012, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1013, Red action: do-nothing, Blue reward:0.85\n", - "step: 1014, Red action: do-nothing, Blue reward:0.85\n", - "step: 1015, Red action: do-nothing, Blue reward:0.85\n", - "step: 1016, Red action: do-nothing, Blue reward:0.85\n", - "step: 1017, Red action: do-nothing, Blue reward:0.85\n", - "step: 1018, Red action: do-nothing, Blue reward:0.85\n", - "step: 1019, Red action: do-nothing, Blue reward:0.85\n", - "step: 1020, Red action: do-nothing, Blue reward:0.85\n", - "step: 1021, Red action: do-nothing, Blue reward:0.85\n", - "step: 1022, Red action: do-nothing, Blue reward:0.85\n", - "step: 1023, Red action: do-nothing, Blue reward:0.85\n", - "step: 1024, Red action: do-nothing, Blue reward:0.85\n", - "step: 1025, Red action: do-nothing, Blue reward:0.85\n", - "step: 1026, Red action: do-nothing, Blue reward:0.85\n", - "step: 1027, Red action: do-nothing, Blue reward:0.85\n", - "step: 1028, Red action: do-nothing, Blue reward:0.85\n", - "step: 1029, Red action: do-nothing, Blue reward:0.85\n", - "step: 1030, Red action: do-nothing, Blue reward:0.85\n", - "step: 1031, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1032, Red action: do-nothing, Blue reward:0.85\n", - "step: 1033, Red action: do-nothing, Blue reward:0.85\n", - "step: 1034, Red action: do-nothing, Blue reward:0.85\n", - "step: 1035, Red action: do-nothing, Blue reward:0.85\n", - "step: 1036, Red action: do-nothing, Blue reward:0.85\n", - "step: 1037, Red action: do-nothing, Blue reward:0.85\n", - "step: 1038, Red action: do-nothing, Blue reward:0.85\n", - "step: 1039, Red action: do-nothing, Blue reward:0.85\n", - "step: 1040, Red action: do-nothing, Blue reward:0.85\n", - "step: 1041, Red action: do-nothing, Blue reward:0.85\n", - "step: 1042, Red action: do-nothing, Blue reward:0.85\n", - "step: 1043, Red action: do-nothing, Blue reward:0.85\n", - "step: 1044, Red action: do-nothing, Blue reward:0.85\n", - "step: 1045, Red action: do-nothing, Blue reward:0.85\n", - "step: 1046, Red action: do-nothing, Blue reward:0.85\n", - "step: 1047, Red action: do-nothing, Blue reward:0.85\n", - "step: 1048, Red action: do-nothing, Blue reward:0.85\n", - "step: 1049, Red action: do-nothing, Blue reward:0.85\n", - "step: 1050, Red action: do-nothing, Blue reward:0.85\n", - "step: 1051, Red action: do-nothing, Blue reward:0.85\n", - "step: 1052, Red action: do-nothing, Blue reward:0.85\n", - "step: 1053, Red action: do-nothing, Blue reward:0.85\n", - "step: 1054, Red action: do-nothing, Blue reward:0.85\n", - "step: 1055, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1056, Red action: do-nothing, Blue reward:0.85\n", - "step: 1057, Red action: do-nothing, Blue reward:0.85\n", - "step: 1058, Red action: do-nothing, Blue reward:0.85\n", - "step: 1059, Red action: do-nothing, Blue reward:0.85\n", - "step: 1060, Red action: do-nothing, Blue reward:0.85\n", - "step: 1061, Red action: do-nothing, Blue reward:0.85\n", - "step: 1062, Red action: do-nothing, Blue reward:0.85\n", - "step: 1063, Red action: do-nothing, Blue reward:0.85\n", - "step: 1064, Red action: do-nothing, Blue reward:0.85\n", - "step: 1065, Red action: do-nothing, Blue reward:0.85\n", - "step: 1066, Red action: do-nothing, Blue reward:0.85\n", - "step: 1067, Red action: do-nothing, Blue reward:0.85\n", - "step: 1068, Red action: do-nothing, Blue reward:0.85\n", - "step: 1069, Red action: do-nothing, Blue reward:0.85\n", - "step: 1070, Red action: do-nothing, Blue reward:0.85\n", - "step: 1071, Red action: do-nothing, Blue reward:0.85\n", - "step: 1072, Red action: do-nothing, Blue reward:0.85\n", - "step: 1073, Red action: do-nothing, Blue reward:0.85\n", - "step: 1074, Red action: do-nothing, Blue reward:0.85\n", - "step: 1075, Red action: do-nothing, Blue reward:0.85\n", - "step: 1076, Red action: do-nothing, Blue reward:0.85\n", - "step: 1077, Red action: do-nothing, Blue reward:0.85\n", - "step: 1078, Red action: do-nothing, Blue reward:0.85\n", - "step: 1079, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1080, Red action: do-nothing, Blue reward:0.85\n", - "step: 1081, Red action: do-nothing, Blue reward:0.85\n", - "step: 1082, Red action: do-nothing, Blue reward:0.85\n", - "step: 1083, Red action: do-nothing, Blue reward:0.85\n", - "step: 1084, Red action: do-nothing, Blue reward:0.85\n", - "step: 1085, Red action: do-nothing, Blue reward:0.85\n", - "step: 1086, Red action: do-nothing, Blue reward:0.85\n", - "step: 1087, Red action: do-nothing, Blue reward:0.85\n", - "step: 1088, Red action: do-nothing, Blue reward:0.85\n", - "step: 1089, Red action: do-nothing, Blue reward:0.85\n", - "step: 1090, Red action: do-nothing, Blue reward:0.85\n", - "step: 1091, Red action: do-nothing, Blue reward:0.85\n", - "step: 1092, Red action: do-nothing, Blue reward:0.85\n", - "step: 1093, Red action: do-nothing, Blue reward:0.85\n", - "step: 1094, Red action: do-nothing, Blue reward:0.85\n", - "step: 1095, Red action: do-nothing, Blue reward:0.85\n", - "step: 1096, Red action: do-nothing, Blue reward:0.85\n", - "step: 1097, Red action: do-nothing, Blue reward:0.85\n", - "step: 1098, Red action: do-nothing, Blue reward:0.85\n", - "step: 1099, Red action: do-nothing, Blue reward:0.85\n", - "step: 1100, Red action: do-nothing, Blue reward:0.85\n", - "step: 1101, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1102, Red action: do-nothing, Blue reward:0.85\n", - "step: 1103, Red action: do-nothing, Blue reward:0.85\n", - "step: 1104, Red action: do-nothing, Blue reward:0.85\n", - "step: 1105, Red action: do-nothing, Blue reward:0.85\n", - "step: 1106, Red action: do-nothing, Blue reward:0.85\n", - "step: 1107, Red action: do-nothing, Blue reward:0.85\n", - "step: 1108, Red action: do-nothing, Blue reward:0.85\n", - "step: 1109, Red action: do-nothing, Blue reward:0.85\n", - "step: 1110, Red action: do-nothing, Blue reward:0.85\n", - "step: 1111, Red action: do-nothing, Blue reward:0.85\n", - "step: 1112, Red action: do-nothing, Blue reward:0.85\n", - "step: 1113, Red action: do-nothing, Blue reward:0.85\n", - "step: 1114, Red action: do-nothing, Blue reward:0.85\n", - "step: 1115, Red action: do-nothing, Blue reward:0.85\n", - "step: 1116, Red action: do-nothing, Blue reward:0.85\n", - "step: 1117, Red action: do-nothing, Blue reward:0.85\n", - "step: 1118, Red action: do-nothing, Blue reward:0.85\n", - "step: 1119, Red action: do-nothing, Blue reward:0.85\n", - "step: 1120, Red action: do-nothing, Blue reward:0.85\n", - "step: 1121, Red action: do-nothing, Blue reward:0.85\n", - "step: 1122, Red action: do-nothing, Blue reward:0.85\n", - "step: 1123, Red action: do-nothing, Blue reward:0.85\n", - "step: 1124, Red action: do-nothing, Blue reward:0.85\n", - "step: 1125, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1126, Red action: do-nothing, Blue reward:0.85\n", - "step: 1127, Red action: do-nothing, Blue reward:0.85\n", - "step: 1128, Red action: do-nothing, Blue reward:0.85\n", - "step: 1129, Red action: do-nothing, Blue reward:0.85\n", - "step: 1130, Red action: do-nothing, Blue reward:0.85\n", - "step: 1131, Red action: do-nothing, Blue reward:0.85\n", - "step: 1132, Red action: do-nothing, Blue reward:0.85\n", - "step: 1133, Red action: do-nothing, Blue reward:0.85\n", - "step: 1134, Red action: do-nothing, Blue reward:0.85\n", - "step: 1135, Red action: do-nothing, Blue reward:0.85\n", - "step: 1136, Red action: do-nothing, Blue reward:0.85\n", - "step: 1137, Red action: do-nothing, Blue reward:0.85\n", - "step: 1138, Red action: do-nothing, Blue reward:0.85\n", - "step: 1139, Red action: do-nothing, Blue reward:0.85\n", - "step: 1140, Red action: do-nothing, Blue reward:0.85\n", - "step: 1141, Red action: do-nothing, Blue reward:0.85\n", - "step: 1142, Red action: do-nothing, Blue reward:0.85\n", - "step: 1143, Red action: do-nothing, Blue reward:0.85\n", - "step: 1144, Red action: do-nothing, Blue reward:0.85\n", - "step: 1145, Red action: do-nothing, Blue reward:0.85\n", - "step: 1146, Red action: do-nothing, Blue reward:0.85\n", - "step: 1147, Red action: do-nothing, Blue reward:0.85\n", - "step: 1148, Red action: do-nothing, Blue reward:0.85\n", - "step: 1149, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1150, Red action: do-nothing, Blue reward:0.85\n", - "step: 1151, Red action: do-nothing, Blue reward:0.85\n", - "step: 1152, Red action: do-nothing, Blue reward:0.85\n", - "step: 1153, Red action: do-nothing, Blue reward:0.85\n", - "step: 1154, Red action: do-nothing, Blue reward:0.85\n", - "step: 1155, Red action: do-nothing, Blue reward:0.85\n", - "step: 1156, Red action: do-nothing, Blue reward:0.85\n", - "step: 1157, Red action: do-nothing, Blue reward:0.85\n", - "step: 1158, Red action: do-nothing, Blue reward:0.85\n", - "step: 1159, Red action: do-nothing, Blue reward:0.85\n", - "step: 1160, Red action: do-nothing, Blue reward:0.85\n", - "step: 1161, Red action: do-nothing, Blue reward:0.85\n", - "step: 1162, Red action: do-nothing, Blue reward:0.85\n", - "step: 1163, Red action: do-nothing, Blue reward:0.85\n", - "step: 1164, Red action: do-nothing, Blue reward:0.85\n", - "step: 1165, Red action: do-nothing, Blue reward:0.85\n", - "step: 1166, Red action: do-nothing, Blue reward:0.85\n", - "step: 1167, Red action: do-nothing, Blue reward:0.85\n", - "step: 1168, Red action: do-nothing, Blue reward:0.85\n", - "step: 1169, Red action: do-nothing, Blue reward:0.85\n", - "step: 1170, Red action: do-nothing, Blue reward:0.85\n", - "step: 1171, Red action: do-nothing, Blue reward:0.85\n", - "step: 1172, Red action: do-nothing, Blue reward:0.85\n", - "step: 1173, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1174, Red action: do-nothing, Blue reward:0.85\n", - "step: 1175, Red action: do-nothing, Blue reward:0.85\n", - "step: 1176, Red action: do-nothing, Blue reward:0.85\n", - "step: 1177, Red action: do-nothing, Blue reward:0.85\n", - "step: 1178, Red action: do-nothing, Blue reward:0.85\n", - "step: 1179, Red action: do-nothing, Blue reward:0.85\n", - "step: 1180, Red action: do-nothing, Blue reward:0.85\n", - "step: 1181, Red action: do-nothing, Blue reward:0.85\n", - "step: 1182, Red action: do-nothing, Blue reward:0.85\n", - "step: 1183, Red action: do-nothing, Blue reward:0.85\n", - "step: 1184, Red action: do-nothing, Blue reward:0.85\n", - "step: 1185, Red action: do-nothing, Blue reward:0.85\n", - "step: 1186, Red action: do-nothing, Blue reward:0.85\n", - "step: 1187, Red action: do-nothing, Blue reward:0.85\n", - "step: 1188, Red action: do-nothing, Blue reward:0.85\n", - "step: 1189, Red action: do-nothing, Blue reward:0.85\n", - "step: 1190, Red action: do-nothing, Blue reward:0.85\n", - "step: 1191, Red action: do-nothing, Blue reward:0.85\n", - "step: 1192, Red action: do-nothing, Blue reward:0.85\n", - "step: 1193, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1194, Red action: do-nothing, Blue reward:0.85\n", - "step: 1195, Red action: do-nothing, Blue reward:0.85\n", - "step: 1196, Red action: do-nothing, Blue reward:0.85\n", - "step: 1197, Red action: do-nothing, Blue reward:0.85\n", - "step: 1198, Red action: do-nothing, Blue reward:0.85\n", - "step: 1199, Red action: do-nothing, Blue reward:0.85\n", - "step: 1200, Red action: do-nothing, Blue reward:0.85\n", - "step: 1201, Red action: do-nothing, Blue reward:0.85\n", - "step: 1202, Red action: do-nothing, Blue reward:0.85\n", - "step: 1203, Red action: do-nothing, Blue reward:0.85\n", - "step: 1204, Red action: do-nothing, Blue reward:0.85\n", - "step: 1205, Red action: do-nothing, Blue reward:0.85\n", - "step: 1206, Red action: do-nothing, Blue reward:0.85\n", - "step: 1207, Red action: do-nothing, Blue reward:0.85\n", - "step: 1208, Red action: do-nothing, Blue reward:0.85\n", - "step: 1209, Red action: do-nothing, Blue reward:0.85\n", - "step: 1210, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1211, Red action: do-nothing, Blue reward:0.85\n", - "step: 1212, Red action: do-nothing, Blue reward:0.85\n", - "step: 1213, Red action: do-nothing, Blue reward:0.85\n", - "step: 1214, Red action: do-nothing, Blue reward:0.85\n", - "step: 1215, Red action: do-nothing, Blue reward:0.85\n", - "step: 1216, Red action: do-nothing, Blue reward:0.85\n", - "step: 1217, Red action: do-nothing, Blue reward:0.85\n", - "step: 1218, Red action: do-nothing, Blue reward:0.85\n", - "step: 1219, Red action: do-nothing, Blue reward:0.85\n", - "step: 1220, Red action: do-nothing, Blue reward:0.85\n", - "step: 1221, Red action: do-nothing, Blue reward:0.85\n", - "step: 1222, Red action: do-nothing, Blue reward:0.85\n", - "step: 1223, Red action: do-nothing, Blue reward:0.85\n", - "step: 1224, Red action: do-nothing, Blue reward:0.85\n", - "step: 1225, Red action: do-nothing, Blue reward:0.85\n", - "step: 1226, Red action: do-nothing, Blue reward:0.85\n", - "step: 1227, Red action: do-nothing, Blue reward:0.85\n", - "step: 1228, Red action: do-nothing, Blue reward:0.85\n", - "step: 1229, Red action: do-nothing, Blue reward:0.85\n", - "step: 1230, Red action: do-nothing, Blue reward:0.85\n", - "step: 1231, Red action: do-nothing, Blue reward:0.85\n", - "step: 1232, Red action: do-nothing, Blue reward:0.85\n", - "step: 1233, Red action: do-nothing, Blue reward:0.85\n", - "step: 1234, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1235, Red action: do-nothing, Blue reward:0.85\n", - "step: 1236, Red action: do-nothing, Blue reward:0.85\n", - "step: 1237, Red action: do-nothing, Blue reward:0.85\n", - "step: 1238, Red action: do-nothing, Blue reward:0.85\n", - "step: 1239, Red action: do-nothing, Blue reward:0.85\n", - "step: 1240, Red action: do-nothing, Blue reward:0.85\n", - "step: 1241, Red action: do-nothing, Blue reward:0.85\n", - "step: 1242, Red action: do-nothing, Blue reward:0.85\n", - "step: 1243, Red action: do-nothing, Blue reward:0.85\n", - "step: 1244, Red action: do-nothing, Blue reward:0.85\n", - "step: 1245, Red action: do-nothing, Blue reward:0.85\n", - "step: 1246, Red action: do-nothing, Blue reward:0.85\n", - "step: 1247, Red action: do-nothing, Blue reward:0.85\n", - "step: 1248, Red action: do-nothing, Blue reward:0.85\n", - "step: 1249, Red action: do-nothing, Blue reward:0.85\n", - "step: 1250, Red action: do-nothing, Blue reward:0.85\n", - "step: 1251, Red action: do-nothing, Blue reward:0.85\n", - "step: 1252, Red action: do-nothing, Blue reward:0.85\n", - "step: 1253, Red action: do-nothing, Blue reward:0.85\n", - "step: 1254, Red action: do-nothing, Blue reward:0.85\n", - "step: 1255, Red action: do-nothing, Blue reward:0.85\n", - "step: 1256, Red action: do-nothing, Blue reward:0.85\n", - "step: 1257, Red action: do-nothing, Blue reward:0.85\n", - "step: 1258, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1259, Red action: do-nothing, Blue reward:0.85\n", - "step: 1260, Red action: do-nothing, Blue reward:0.85\n", - "step: 1261, Red action: do-nothing, Blue reward:0.85\n", - "step: 1262, Red action: do-nothing, Blue reward:0.85\n", - "step: 1263, Red action: do-nothing, Blue reward:0.85\n", - "step: 1264, Red action: do-nothing, Blue reward:0.85\n", - "step: 1265, Red action: do-nothing, Blue reward:0.85\n", - "step: 1266, Red action: do-nothing, Blue reward:0.85\n", - "step: 1267, Red action: do-nothing, Blue reward:0.85\n", - "step: 1268, Red action: do-nothing, Blue reward:0.85\n", - "step: 1269, Red action: do-nothing, Blue reward:0.85\n", - "step: 1270, Red action: do-nothing, Blue reward:0.85\n", - "step: 1271, Red action: do-nothing, Blue reward:0.85\n", - "step: 1272, Red action: do-nothing, Blue reward:0.85\n", - "step: 1273, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1274, Red action: do-nothing, Blue reward:0.85\n", - "step: 1275, Red action: do-nothing, Blue reward:0.85\n", - "step: 1276, Red action: do-nothing, Blue reward:0.85\n", - "step: 1277, Red action: do-nothing, Blue reward:0.85\n", - "step: 1278, Red action: do-nothing, Blue reward:0.85\n", - "step: 1279, Red action: do-nothing, Blue reward:0.85\n", - "step: 1280, Red action: do-nothing, Blue reward:0.85\n", - "step: 1281, Red action: do-nothing, Blue reward:0.85\n", - "step: 1282, Red action: do-nothing, Blue reward:0.85\n", - "step: 1283, Red action: do-nothing, Blue reward:0.85\n", - "step: 1284, Red action: do-nothing, Blue reward:0.85\n", - "step: 1285, Red action: do-nothing, Blue reward:0.85\n", - "step: 1286, Red action: do-nothing, Blue reward:0.85\n", - "step: 1287, Red action: do-nothing, Blue reward:0.85\n", - "step: 1288, Red action: do-nothing, Blue reward:0.85\n", - "step: 1289, Red action: do-nothing, Blue reward:0.85\n", - "step: 1290, Red action: do-nothing, Blue reward:0.85\n", - "step: 1291, Red action: do-nothing, Blue reward:0.85\n", - "step: 1292, Red action: do-nothing, Blue reward:0.85\n", - "step: 1293, Red action: do-nothing, Blue reward:0.85\n", - "step: 1294, Red action: do-nothing, Blue reward:0.85\n", - "step: 1295, Red action: do-nothing, Blue reward:0.85\n", - "step: 1296, Red action: do-nothing, Blue reward:0.85\n", - "step: 1297, Red action: do-nothing, Blue reward:0.85\n", - "step: 1298, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1299, Red action: do-nothing, Blue reward:0.85\n", - "step: 1300, Red action: do-nothing, Blue reward:0.85\n", - "step: 1301, Red action: do-nothing, Blue reward:0.85\n", - "step: 1302, Red action: do-nothing, Blue reward:0.85\n", - "step: 1303, Red action: do-nothing, Blue reward:0.85\n", - "step: 1304, Red action: do-nothing, Blue reward:0.85\n", - "step: 1305, Red action: do-nothing, Blue reward:0.85\n", - "step: 1306, Red action: do-nothing, Blue reward:0.85\n", - "step: 1307, Red action: do-nothing, Blue reward:0.85\n", - "step: 1308, Red action: do-nothing, Blue reward:0.85\n", - "step: 1309, Red action: do-nothing, Blue reward:0.85\n", - "step: 1310, Red action: do-nothing, Blue reward:0.85\n", - "step: 1311, Red action: do-nothing, Blue reward:0.85\n", - "step: 1312, Red action: do-nothing, Blue reward:0.85\n", - "step: 1313, Red action: do-nothing, Blue reward:0.85\n", - "step: 1314, Red action: do-nothing, Blue reward:0.85\n", - "step: 1315, Red action: do-nothing, Blue reward:0.85\n", - "step: 1316, Red action: do-nothing, Blue reward:0.85\n", - "step: 1317, Red action: do-nothing, Blue reward:0.85\n", - "step: 1318, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1319, Red action: do-nothing, Blue reward:0.85\n", - "step: 1320, Red action: do-nothing, Blue reward:0.85\n", - "step: 1321, Red action: do-nothing, Blue reward:0.85\n", - "step: 1322, Red action: do-nothing, Blue reward:0.85\n", - "step: 1323, Red action: do-nothing, Blue reward:0.85\n", - "step: 1324, Red action: do-nothing, Blue reward:0.85\n", - "step: 1325, Red action: do-nothing, Blue reward:0.85\n", - "step: 1326, Red action: do-nothing, Blue reward:0.85\n", - "step: 1327, Red action: do-nothing, Blue reward:0.85\n", - "step: 1328, Red action: do-nothing, Blue reward:0.85\n", - "step: 1329, Red action: do-nothing, Blue reward:0.85\n", - "step: 1330, Red action: do-nothing, Blue reward:0.85\n", - "step: 1331, Red action: do-nothing, Blue reward:0.85\n", - "step: 1332, Red action: do-nothing, Blue reward:0.85\n", - "step: 1333, Red action: do-nothing, Blue reward:0.85\n", - "step: 1334, Red action: do-nothing, Blue reward:0.85\n", - "step: 1335, Red action: do-nothing, Blue reward:0.85\n", - "step: 1336, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1337, Red action: do-nothing, Blue reward:0.85\n", - "step: 1338, Red action: do-nothing, Blue reward:0.85\n", - "step: 1339, Red action: do-nothing, Blue reward:0.85\n", - "step: 1340, Red action: do-nothing, Blue reward:0.85\n", - "step: 1341, Red action: do-nothing, Blue reward:0.85\n", - "step: 1342, Red action: do-nothing, Blue reward:0.85\n", - "step: 1343, Red action: do-nothing, Blue reward:0.85\n", - "step: 1344, Red action: do-nothing, Blue reward:0.85\n", - "step: 1345, Red action: do-nothing, Blue reward:0.85\n", - "step: 1346, Red action: do-nothing, Blue reward:0.85\n", - "step: 1347, Red action: do-nothing, Blue reward:0.85\n", - "step: 1348, Red action: do-nothing, Blue reward:0.85\n", - "step: 1349, Red action: do-nothing, Blue reward:0.85\n", - "step: 1350, Red action: do-nothing, Blue reward:0.85\n", - "step: 1351, Red action: do-nothing, Blue reward:0.85\n", - "step: 1352, Red action: do-nothing, Blue reward:0.85\n", - "step: 1353, Red action: do-nothing, Blue reward:0.85\n", - "step: 1354, Red action: do-nothing, Blue reward:0.85\n", - "step: 1355, Red action: do-nothing, Blue reward:0.85\n", - "step: 1356, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1357, Red action: do-nothing, Blue reward:0.85\n", - "step: 1358, Red action: do-nothing, Blue reward:0.85\n", - "step: 1359, Red action: do-nothing, Blue reward:0.85\n", - "step: 1360, Red action: do-nothing, Blue reward:0.85\n", - "step: 1361, Red action: do-nothing, Blue reward:0.85\n", - "step: 1362, Red action: do-nothing, Blue reward:0.85\n", - "step: 1363, Red action: do-nothing, Blue reward:0.85\n", - "step: 1364, Red action: do-nothing, Blue reward:0.85\n", - "step: 1365, Red action: do-nothing, Blue reward:0.85\n", - "step: 1366, Red action: do-nothing, Blue reward:0.85\n", - "step: 1367, Red action: do-nothing, Blue reward:0.85\n", - "step: 1368, Red action: do-nothing, Blue reward:0.85\n", - "step: 1369, Red action: do-nothing, Blue reward:0.85\n", - "step: 1370, Red action: do-nothing, Blue reward:0.85\n", - "step: 1371, Red action: do-nothing, Blue reward:0.85\n", - "step: 1372, Red action: do-nothing, Blue reward:0.85\n", - "step: 1373, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1374, Red action: do-nothing, Blue reward:0.85\n", - "step: 1375, Red action: do-nothing, Blue reward:0.85\n", - "step: 1376, Red action: do-nothing, Blue reward:0.85\n", - "step: 1377, Red action: do-nothing, Blue reward:0.85\n", - "step: 1378, Red action: do-nothing, Blue reward:0.85\n", - "step: 1379, Red action: do-nothing, Blue reward:0.85\n", - "step: 1380, Red action: do-nothing, Blue reward:0.85\n", - "step: 1381, Red action: do-nothing, Blue reward:0.85\n", - "step: 1382, Red action: do-nothing, Blue reward:0.85\n", - "step: 1383, Red action: do-nothing, Blue reward:0.85\n", - "step: 1384, Red action: do-nothing, Blue reward:0.85\n", - "step: 1385, Red action: do-nothing, Blue reward:0.85\n", - "step: 1386, Red action: do-nothing, Blue reward:0.85\n", - "step: 1387, Red action: do-nothing, Blue reward:0.85\n", - "step: 1388, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1389, Red action: do-nothing, Blue reward:0.85\n", - "step: 1390, Red action: do-nothing, Blue reward:0.85\n", - "step: 1391, Red action: do-nothing, Blue reward:0.85\n", - "step: 1392, Red action: do-nothing, Blue reward:0.85\n", - "step: 1393, Red action: do-nothing, Blue reward:0.85\n", - "step: 1394, Red action: do-nothing, Blue reward:0.85\n", - "step: 1395, Red action: do-nothing, Blue reward:0.85\n", - "step: 1396, Red action: do-nothing, Blue reward:0.85\n", - "step: 1397, Red action: do-nothing, Blue reward:0.85\n", - "step: 1398, Red action: do-nothing, Blue reward:0.85\n", - "step: 1399, Red action: do-nothing, Blue reward:0.85\n", - "step: 1400, Red action: do-nothing, Blue reward:0.85\n", - "step: 1401, Red action: do-nothing, Blue reward:0.85\n", - "step: 1402, Red action: do-nothing, Blue reward:0.85\n", - "step: 1403, Red action: do-nothing, Blue reward:0.85\n", - "step: 1404, Red action: do-nothing, Blue reward:0.85\n", - "step: 1405, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1406, Red action: do-nothing, Blue reward:0.85\n", - "step: 1407, Red action: do-nothing, Blue reward:0.85\n", - "step: 1408, Red action: do-nothing, Blue reward:0.85\n", - "step: 1409, Red action: do-nothing, Blue reward:0.85\n", - "step: 1410, Red action: do-nothing, Blue reward:0.85\n", - "step: 1411, Red action: do-nothing, Blue reward:0.85\n", - "step: 1412, Red action: do-nothing, Blue reward:0.85\n", - "step: 1413, Red action: do-nothing, Blue reward:0.85\n", - "step: 1414, Red action: do-nothing, Blue reward:0.85\n", - "step: 1415, Red action: do-nothing, Blue reward:0.85\n", - "step: 1416, Red action: do-nothing, Blue reward:0.85\n", - "step: 1417, Red action: do-nothing, Blue reward:0.85\n", - "step: 1418, Red action: do-nothing, Blue reward:0.85\n", - "step: 1419, Red action: do-nothing, Blue reward:0.85\n", - "step: 1420, Red action: do-nothing, Blue reward:0.85\n", - "step: 1421, Red action: do-nothing, Blue reward:0.85\n", - "step: 1422, Red action: do-nothing, Blue reward:0.85\n", - "step: 1423, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1424, Red action: do-nothing, Blue reward:0.85\n", - "step: 1425, Red action: do-nothing, Blue reward:0.85\n", - "step: 1426, Red action: do-nothing, Blue reward:0.85\n", - "step: 1427, Red action: do-nothing, Blue reward:0.85\n", - "step: 1428, Red action: do-nothing, Blue reward:0.85\n", - "step: 1429, Red action: do-nothing, Blue reward:0.85\n", - "step: 1430, Red action: do-nothing, Blue reward:0.85\n", - "step: 1431, Red action: do-nothing, Blue reward:0.85\n", - "step: 1432, Red action: do-nothing, Blue reward:0.85\n", - "step: 1433, Red action: do-nothing, Blue reward:0.85\n", - "step: 1434, Red action: do-nothing, Blue reward:0.85\n", - "step: 1435, Red action: do-nothing, Blue reward:0.85\n", - "step: 1436, Red action: do-nothing, Blue reward:0.85\n", - "step: 1437, Red action: do-nothing, Blue reward:0.85\n", - "step: 1438, Red action: do-nothing, Blue reward:0.85\n", - "step: 1439, Red action: do-nothing, Blue reward:0.85\n", - "step: 1440, Red action: do-nothing, Blue reward:0.85\n", - "step: 1441, Red action: do-nothing, Blue reward:0.85\n", - "step: 1442, Red action: do-nothing, Blue reward:0.85\n", - "step: 1443, Red action: do-nothing, Blue reward:0.85\n", - "step: 1444, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1445, Red action: do-nothing, Blue reward:0.85\n", - "step: 1446, Red action: do-nothing, Blue reward:0.85\n", - "step: 1447, Red action: do-nothing, Blue reward:0.85\n", - "step: 1448, Red action: do-nothing, Blue reward:0.85\n", - "step: 1449, Red action: do-nothing, Blue reward:0.85\n", - "step: 1450, Red action: do-nothing, Blue reward:0.85\n", - "step: 1451, Red action: do-nothing, Blue reward:0.85\n", - "step: 1452, Red action: do-nothing, Blue reward:0.85\n", - "step: 1453, Red action: do-nothing, Blue reward:0.85\n", - "step: 1454, Red action: do-nothing, Blue reward:0.85\n", - "step: 1455, Red action: do-nothing, Blue reward:0.85\n", - "step: 1456, Red action: do-nothing, Blue reward:0.85\n", - "step: 1457, Red action: do-nothing, Blue reward:0.85\n", - "step: 1458, Red action: do-nothing, Blue reward:0.85\n", - "step: 1459, Red action: do-nothing, Blue reward:0.85\n", - "step: 1460, Red action: do-nothing, Blue reward:0.85\n", - "step: 1461, Red action: do-nothing, Blue reward:0.85\n", - "step: 1462, Red action: do-nothing, Blue reward:0.85\n", - "step: 1463, Red action: do-nothing, Blue reward:0.85\n", - "step: 1464, Red action: do-nothing, Blue reward:0.85\n", - "step: 1465, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1466, Red action: do-nothing, Blue reward:0.85\n", - "step: 1467, Red action: do-nothing, Blue reward:0.85\n", - "step: 1468, Red action: do-nothing, Blue reward:0.85\n", - "step: 1469, Red action: do-nothing, Blue reward:0.85\n", - "step: 1470, Red action: do-nothing, Blue reward:0.85\n", - "step: 1471, Red action: do-nothing, Blue reward:0.85\n", - "step: 1472, Red action: do-nothing, Blue reward:0.85\n", - "step: 1473, Red action: do-nothing, Blue reward:0.85\n", - "step: 1474, Red action: do-nothing, Blue reward:0.85\n", - "step: 1475, Red action: do-nothing, Blue reward:0.85\n", - "step: 1476, Red action: do-nothing, Blue reward:0.85\n", - "step: 1477, Red action: do-nothing, Blue reward:0.85\n", - "step: 1478, Red action: do-nothing, Blue reward:0.85\n", - "step: 1479, Red action: do-nothing, Blue reward:0.85\n", - "step: 1480, Red action: do-nothing, Blue reward:0.85\n", - "step: 1481, Red action: do-nothing, Blue reward:0.85\n", - "step: 1482, Red action: do-nothing, Blue reward:0.85\n", - "step: 1483, Red action: do-nothing, Blue reward:0.85\n", - "step: 1484, Red action: do-nothing, Blue reward:0.85\n", - "step: 1485, Red action: do-nothing, Blue reward:0.85\n", - "step: 1486, Red action: do-nothing, Blue reward:0.85\n", - "step: 1487, Red action: do-nothing, Blue reward:0.85\n", - "step: 1488, Red action: do-nothing, Blue reward:0.85\n", - "step: 1489, Red action: do-nothing, Blue reward:0.85\n", - "step: 1490, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1491, Red action: do-nothing, Blue reward:0.85\n", - "step: 1492, Red action: do-nothing, Blue reward:0.85\n", - "step: 1493, Red action: do-nothing, Blue reward:0.85\n", - "step: 1494, Red action: do-nothing, Blue reward:0.85\n", - "step: 1495, Red action: do-nothing, Blue reward:0.85\n", - "step: 1496, Red action: do-nothing, Blue reward:0.85\n", - "step: 1497, Red action: do-nothing, Blue reward:0.85\n", - "step: 1498, Red action: do-nothing, Blue reward:0.85\n", - "step: 1499, Red action: do-nothing, Blue reward:0.85\n", - "step: 1500, Red action: do-nothing, Blue reward:0.85\n", - "step: 1501, Red action: do-nothing, Blue reward:0.85\n", - "step: 1502, Red action: do-nothing, Blue reward:0.85\n", - "step: 1503, Red action: do-nothing, Blue reward:0.85\n", - "step: 1504, Red action: do-nothing, Blue reward:0.85\n", - "step: 1505, Red action: do-nothing, Blue reward:0.85\n", - "step: 1506, Red action: do-nothing, Blue reward:0.85\n", - "step: 1507, Red action: do-nothing, Blue reward:0.85\n", - "step: 1508, Red action: do-nothing, Blue reward:0.85\n", - "step: 1509, Red action: do-nothing, Blue reward:0.85\n", - "step: 1510, Red action: do-nothing, Blue reward:0.85\n", - "step: 1511, Red action: do-nothing, Blue reward:0.85\n", - "step: 1512, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1513, Red action: do-nothing, Blue reward:0.85\n", - "step: 1514, Red action: do-nothing, Blue reward:0.85\n", - "step: 1515, Red action: do-nothing, Blue reward:0.85\n", - "step: 1516, Red action: do-nothing, Blue reward:0.85\n", - "step: 1517, Red action: do-nothing, Blue reward:0.85\n", - "step: 1518, Red action: do-nothing, Blue reward:0.85\n", - "step: 1519, Red action: do-nothing, Blue reward:0.85\n", - "step: 1520, Red action: do-nothing, Blue reward:0.85\n", - "step: 1521, Red action: do-nothing, Blue reward:0.85\n", - "step: 1522, Red action: do-nothing, Blue reward:0.85\n", - "step: 1523, Red action: do-nothing, Blue reward:0.85\n", - "step: 1524, Red action: do-nothing, Blue reward:0.85\n", - "step: 1525, Red action: do-nothing, Blue reward:0.85\n", - "step: 1526, Red action: do-nothing, Blue reward:0.85\n", - "step: 1527, Red action: do-nothing, Blue reward:0.85\n", - "step: 1528, Red action: do-nothing, Blue reward:0.85\n", - "step: 1529, Red action: do-nothing, Blue reward:0.85\n", - "step: 1530, Red action: do-nothing, Blue reward:0.85\n", - "step: 1531, Red action: do-nothing, Blue reward:0.85\n", - "step: 1532, Red action: do-nothing, Blue reward:0.85\n", - "step: 1533, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1534, Red action: do-nothing, Blue reward:0.85\n", - "step: 1535, Red action: do-nothing, Blue reward:0.85\n", - "step: 1536, Red action: do-nothing, Blue reward:0.85\n", - "step: 1537, Red action: do-nothing, Blue reward:0.85\n", - "step: 1538, Red action: do-nothing, Blue reward:0.85\n", - "step: 1539, Red action: do-nothing, Blue reward:0.85\n", - "step: 1540, Red action: do-nothing, Blue reward:0.85\n", - "step: 1541, Red action: do-nothing, Blue reward:0.85\n", - "step: 1542, Red action: do-nothing, Blue reward:0.85\n", - "step: 1543, Red action: do-nothing, Blue reward:0.85\n", - "step: 1544, Red action: do-nothing, Blue reward:0.85\n", - "step: 1545, Red action: do-nothing, Blue reward:0.85\n", - "step: 1546, Red action: do-nothing, Blue reward:0.85\n", - "step: 1547, Red action: do-nothing, Blue reward:0.85\n", - "step: 1548, Red action: do-nothing, Blue reward:0.85\n", - "step: 1549, Red action: do-nothing, Blue reward:0.85\n", - "step: 1550, Red action: do-nothing, Blue reward:0.85\n", - "step: 1551, Red action: do-nothing, Blue reward:0.85\n", - "step: 1552, Red action: do-nothing, Blue reward:0.85\n", - "step: 1553, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1554, Red action: do-nothing, Blue reward:0.85\n", - "step: 1555, Red action: do-nothing, Blue reward:0.85\n", - "step: 1556, Red action: do-nothing, Blue reward:0.85\n", - "step: 1557, Red action: do-nothing, Blue reward:0.85\n", - "step: 1558, Red action: do-nothing, Blue reward:0.85\n", - "step: 1559, Red action: do-nothing, Blue reward:0.85\n", - "step: 1560, Red action: do-nothing, Blue reward:0.85\n", - "step: 1561, Red action: do-nothing, Blue reward:0.85\n", - "step: 1562, Red action: do-nothing, Blue reward:0.85\n", - "step: 1563, Red action: do-nothing, Blue reward:0.85\n", - "step: 1564, Red action: do-nothing, Blue reward:0.85\n", - "step: 1565, Red action: do-nothing, Blue reward:0.85\n", - "step: 1566, Red action: do-nothing, Blue reward:0.85\n", - "step: 1567, Red action: do-nothing, Blue reward:0.85\n", - "step: 1568, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1569, Red action: do-nothing, Blue reward:0.85\n", - "step: 1570, Red action: do-nothing, Blue reward:0.85\n", - "step: 1571, Red action: do-nothing, Blue reward:0.85\n", - "step: 1572, Red action: do-nothing, Blue reward:0.85\n", - "step: 1573, Red action: do-nothing, Blue reward:0.85\n", - "step: 1574, Red action: do-nothing, Blue reward:0.85\n", - "step: 1575, Red action: do-nothing, Blue reward:0.85\n", - "step: 1576, Red action: do-nothing, Blue reward:0.85\n", - "step: 1577, Red action: do-nothing, Blue reward:0.85\n", - "step: 1578, Red action: do-nothing, Blue reward:0.85\n", - "step: 1579, Red action: do-nothing, Blue reward:0.85\n", - "step: 1580, Red action: do-nothing, Blue reward:0.85\n", - "step: 1581, Red action: do-nothing, Blue reward:0.85\n", - "step: 1582, Red action: do-nothing, Blue reward:0.85\n", - "step: 1583, Red action: do-nothing, Blue reward:0.85\n", - "step: 1584, Red action: do-nothing, Blue reward:0.85\n", - "step: 1585, Red action: do-nothing, Blue reward:0.85\n", - "step: 1586, Red action: do-nothing, Blue reward:0.85\n", - "step: 1587, Red action: do-nothing, Blue reward:0.85\n", - "step: 1588, Red action: do-nothing, Blue reward:0.85\n", - "step: 1589, Red action: do-nothing, Blue reward:0.85\n", - "step: 1590, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1591, Red action: do-nothing, Blue reward:0.85\n", - "step: 1592, Red action: do-nothing, Blue reward:0.85\n", - "step: 1593, Red action: do-nothing, Blue reward:0.85\n", - "step: 1594, Red action: do-nothing, Blue reward:0.85\n", - "step: 1595, Red action: do-nothing, Blue reward:0.85\n", - "step: 1596, Red action: do-nothing, Blue reward:0.85\n", - "step: 1597, Red action: do-nothing, Blue reward:0.85\n", - "step: 1598, Red action: do-nothing, Blue reward:0.85\n", - "step: 1599, Red action: do-nothing, Blue reward:0.85\n", - "step: 1600, Red action: do-nothing, Blue reward:0.85\n", - "step: 1601, Red action: do-nothing, Blue reward:0.85\n", - "step: 1602, Red action: do-nothing, Blue reward:0.85\n", - "step: 1603, Red action: do-nothing, Blue reward:0.85\n", - "step: 1604, Red action: do-nothing, Blue reward:0.85\n", - "step: 1605, Red action: do-nothing, Blue reward:0.85\n", - "step: 1606, Red action: do-nothing, Blue reward:0.85\n", - "step: 1607, Red action: do-nothing, Blue reward:0.85\n", - "step: 1608, Red action: do-nothing, Blue reward:0.85\n", - "step: 1609, Red action: do-nothing, Blue reward:0.85\n", - "step: 1610, Red action: do-nothing, Blue reward:0.85\n", - "step: 1611, Red action: do-nothing, Blue reward:0.85\n", - "step: 1612, Red action: do-nothing, Blue reward:0.85\n", - "step: 1613, Red action: do-nothing, Blue reward:0.85\n", - "step: 1614, Red action: do-nothing, Blue reward:0.85\n", - "step: 1615, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1616, Red action: do-nothing, Blue reward:0.85\n", - "step: 1617, Red action: do-nothing, Blue reward:0.85\n", - "step: 1618, Red action: do-nothing, Blue reward:0.85\n", - "step: 1619, Red action: do-nothing, Blue reward:0.85\n", - "step: 1620, Red action: do-nothing, Blue reward:0.85\n", - "step: 1621, Red action: do-nothing, Blue reward:0.85\n", - "step: 1622, Red action: do-nothing, Blue reward:0.85\n", - "step: 1623, Red action: do-nothing, Blue reward:0.85\n", - "step: 1624, Red action: do-nothing, Blue reward:0.85\n", - "step: 1625, Red action: do-nothing, Blue reward:0.85\n", - "step: 1626, Red action: do-nothing, Blue reward:0.85\n", - "step: 1627, Red action: do-nothing, Blue reward:0.85\n", - "step: 1628, Red action: do-nothing, Blue reward:0.85\n", - "step: 1629, Red action: do-nothing, Blue reward:0.85\n", - "step: 1630, Red action: do-nothing, Blue reward:0.85\n", - "step: 1631, Red action: do-nothing, Blue reward:0.85\n", - "step: 1632, Red action: do-nothing, Blue reward:0.85\n", - "step: 1633, Red action: do-nothing, Blue reward:0.85\n", - "step: 1634, Red action: do-nothing, Blue reward:0.85\n", - "step: 1635, Red action: do-nothing, Blue reward:0.85\n", - "step: 1636, Red action: do-nothing, Blue reward:0.85\n", - "step: 1637, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1638, Red action: do-nothing, Blue reward:0.85\n", - "step: 1639, Red action: do-nothing, Blue reward:0.85\n", - "step: 1640, Red action: do-nothing, Blue reward:0.85\n", - "step: 1641, Red action: do-nothing, Blue reward:0.85\n", - "step: 1642, Red action: do-nothing, Blue reward:0.85\n", - "step: 1643, Red action: do-nothing, Blue reward:0.85\n", - "step: 1644, Red action: do-nothing, Blue reward:0.85\n", - "step: 1645, Red action: do-nothing, Blue reward:0.85\n", - "step: 1646, Red action: do-nothing, Blue reward:0.85\n", - "step: 1647, Red action: do-nothing, Blue reward:0.85\n", - "step: 1648, Red action: do-nothing, Blue reward:0.85\n", - "step: 1649, Red action: do-nothing, Blue reward:0.85\n", - "step: 1650, Red action: do-nothing, Blue reward:0.85\n", - "step: 1651, Red action: do-nothing, Blue reward:0.85\n", - "step: 1652, Red action: do-nothing, Blue reward:0.85\n", - "step: 1653, Red action: do-nothing, Blue reward:0.85\n", - "step: 1654, Red action: do-nothing, Blue reward:0.85\n", - "step: 1655, Red action: do-nothing, Blue reward:0.85\n", - "step: 1656, Red action: do-nothing, Blue reward:0.85\n", - "step: 1657, Red action: do-nothing, Blue reward:0.85\n", - "step: 1658, Red action: do-nothing, Blue reward:0.85\n", - "step: 1659, Red action: do-nothing, Blue reward:0.85\n", - "step: 1660, Red action: do-nothing, Blue reward:0.85\n", - "step: 1661, Red action: do-nothing, Blue reward:0.85\n", - "step: 1662, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1663, Red action: do-nothing, Blue reward:0.85\n", - "step: 1664, Red action: do-nothing, Blue reward:0.85\n", - "step: 1665, Red action: do-nothing, Blue reward:0.85\n", - "step: 1666, Red action: do-nothing, Blue reward:0.85\n", - "step: 1667, Red action: do-nothing, Blue reward:0.85\n", - "step: 1668, Red action: do-nothing, Blue reward:0.85\n", - "step: 1669, Red action: do-nothing, Blue reward:0.85\n", - "step: 1670, Red action: do-nothing, Blue reward:0.85\n", - "step: 1671, Red action: do-nothing, Blue reward:0.85\n", - "step: 1672, Red action: do-nothing, Blue reward:0.85\n", - "step: 1673, Red action: do-nothing, Blue reward:0.85\n", - "step: 1674, Red action: do-nothing, Blue reward:0.85\n", - "step: 1675, Red action: do-nothing, Blue reward:0.85\n", - "step: 1676, Red action: do-nothing, Blue reward:0.85\n", - "step: 1677, Red action: do-nothing, Blue reward:0.85\n", - "step: 1678, Red action: do-nothing, Blue reward:0.85\n", - "step: 1679, Red action: do-nothing, Blue reward:0.85\n", - "step: 1680, Red action: do-nothing, Blue reward:0.85\n", - "step: 1681, Red action: do-nothing, Blue reward:0.85\n", - "step: 1682, Red action: do-nothing, Blue reward:0.85\n", - "step: 1683, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1684, Red action: do-nothing, Blue reward:0.85\n", - "step: 1685, Red action: do-nothing, Blue reward:0.85\n", - "step: 1686, Red action: do-nothing, Blue reward:0.85\n", - "step: 1687, Red action: do-nothing, Blue reward:0.85\n", - "step: 1688, Red action: do-nothing, Blue reward:0.85\n", - "step: 1689, Red action: do-nothing, Blue reward:0.85\n", - "step: 1690, Red action: do-nothing, Blue reward:0.85\n", - "step: 1691, Red action: do-nothing, Blue reward:0.85\n", - "step: 1692, Red action: do-nothing, Blue reward:0.85\n", - "step: 1693, Red action: do-nothing, Blue reward:0.85\n", - "step: 1694, Red action: do-nothing, Blue reward:0.85\n", - "step: 1695, Red action: do-nothing, Blue reward:0.85\n", - "step: 1696, Red action: do-nothing, Blue reward:0.85\n", - "step: 1697, Red action: do-nothing, Blue reward:0.85\n", - "step: 1698, Red action: do-nothing, Blue reward:0.85\n", - "step: 1699, Red action: do-nothing, Blue reward:0.85\n", - "step: 1700, Red action: do-nothing, Blue reward:0.85\n", - "step: 1701, Red action: do-nothing, Blue reward:0.85\n", - "step: 1702, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1703, Red action: do-nothing, Blue reward:0.85\n", - "step: 1704, Red action: do-nothing, Blue reward:0.85\n", - "step: 1705, Red action: do-nothing, Blue reward:0.85\n", - "step: 1706, Red action: do-nothing, Blue reward:0.85\n", - "step: 1707, Red action: do-nothing, Blue reward:0.85\n", - "step: 1708, Red action: do-nothing, Blue reward:0.85\n", - "step: 1709, Red action: do-nothing, Blue reward:0.85\n", - "step: 1710, Red action: do-nothing, Blue reward:0.85\n", - "step: 1711, Red action: do-nothing, Blue reward:0.85\n", - "step: 1712, Red action: do-nothing, Blue reward:0.85\n", - "step: 1713, Red action: do-nothing, Blue reward:0.85\n", - "step: 1714, Red action: do-nothing, Blue reward:0.85\n", - "step: 1715, Red action: do-nothing, Blue reward:0.85\n", - "step: 1716, Red action: do-nothing, Blue reward:0.85\n", - "step: 1717, Red action: do-nothing, Blue reward:0.85\n", - "step: 1718, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1719, Red action: do-nothing, Blue reward:0.85\n", - "step: 1720, Red action: do-nothing, Blue reward:0.85\n", - "step: 1721, Red action: do-nothing, Blue reward:0.85\n", - "step: 1722, Red action: do-nothing, Blue reward:0.85\n", - "step: 1723, Red action: do-nothing, Blue reward:0.85\n", - "step: 1724, Red action: do-nothing, Blue reward:0.85\n", - "step: 1725, Red action: do-nothing, Blue reward:0.85\n", - "step: 1726, Red action: do-nothing, Blue reward:0.85\n", - "step: 1727, Red action: do-nothing, Blue reward:0.85\n", - "step: 1728, Red action: do-nothing, Blue reward:0.85\n", - "step: 1729, Red action: do-nothing, Blue reward:0.85\n", - "step: 1730, Red action: do-nothing, Blue reward:0.85\n", - "step: 1731, Red action: do-nothing, Blue reward:0.85\n", - "step: 1732, Red action: do-nothing, Blue reward:0.85\n", - "step: 1733, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1734, Red action: do-nothing, Blue reward:0.85\n", - "step: 1735, Red action: do-nothing, Blue reward:0.85\n", - "step: 1736, Red action: do-nothing, Blue reward:0.85\n", - "step: 1737, Red action: do-nothing, Blue reward:0.85\n", - "step: 1738, Red action: do-nothing, Blue reward:0.85\n", - "step: 1739, Red action: do-nothing, Blue reward:0.85\n", - "step: 1740, Red action: do-nothing, Blue reward:0.85\n", - "step: 1741, Red action: do-nothing, Blue reward:0.85\n", - "step: 1742, Red action: do-nothing, Blue reward:0.85\n", - "step: 1743, Red action: do-nothing, Blue reward:0.85\n", - "step: 1744, Red action: do-nothing, Blue reward:0.85\n", - "step: 1745, Red action: do-nothing, Blue reward:0.85\n", - "step: 1746, Red action: do-nothing, Blue reward:0.85\n", - "step: 1747, Red action: do-nothing, Blue reward:0.85\n", - "step: 1748, Red action: do-nothing, Blue reward:0.85\n", - "step: 1749, Red action: do-nothing, Blue reward:0.85\n", - "step: 1750, Red action: do-nothing, Blue reward:0.85\n", - "step: 1751, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1752, Red action: do-nothing, Blue reward:0.85\n", - "step: 1753, Red action: do-nothing, Blue reward:0.85\n", - "step: 1754, Red action: do-nothing, Blue reward:0.85\n", - "step: 1755, Red action: do-nothing, Blue reward:0.85\n", - "step: 1756, Red action: do-nothing, Blue reward:0.85\n", - "step: 1757, Red action: do-nothing, Blue reward:0.85\n", - "step: 1758, Red action: do-nothing, Blue reward:0.85\n", - "step: 1759, Red action: do-nothing, Blue reward:0.85\n", - "step: 1760, Red action: do-nothing, Blue reward:0.85\n", - "step: 1761, Red action: do-nothing, Blue reward:0.85\n", - "step: 1762, Red action: do-nothing, Blue reward:0.85\n", - "step: 1763, Red action: do-nothing, Blue reward:0.85\n", - "step: 1764, Red action: do-nothing, Blue reward:0.85\n", - "step: 1765, Red action: do-nothing, Blue reward:0.85\n", - "step: 1766, Red action: do-nothing, Blue reward:0.85\n", - "step: 1767, Red action: do-nothing, Blue reward:0.85\n", - "step: 1768, Red action: do-nothing, Blue reward:0.85\n", - "step: 1769, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1770, Red action: do-nothing, Blue reward:0.85\n", - "step: 1771, Red action: do-nothing, Blue reward:0.85\n", - "step: 1772, Red action: do-nothing, Blue reward:0.85\n", - "step: 1773, Red action: do-nothing, Blue reward:0.85\n", - "step: 1774, Red action: do-nothing, Blue reward:0.85\n", - "step: 1775, Red action: do-nothing, Blue reward:0.85\n", - "step: 1776, Red action: do-nothing, Blue reward:0.85\n", - "step: 1777, Red action: do-nothing, Blue reward:0.85\n", - "step: 1778, Red action: do-nothing, Blue reward:0.85\n", - "step: 1779, Red action: do-nothing, Blue reward:0.85\n", - "step: 1780, Red action: do-nothing, Blue reward:0.85\n", - "step: 1781, Red action: do-nothing, Blue reward:0.85\n", - "step: 1782, Red action: do-nothing, Blue reward:0.85\n", - "step: 1783, Red action: do-nothing, Blue reward:0.85\n", - "step: 1784, Red action: do-nothing, Blue reward:0.85\n", - "step: 1785, Red action: do-nothing, Blue reward:0.85\n", - "step: 1786, Red action: do-nothing, Blue reward:0.85\n", - "step: 1787, Red action: do-nothing, Blue reward:0.85\n", - "step: 1788, Red action: do-nothing, Blue reward:0.85\n", - "step: 1789, Red action: do-nothing, Blue reward:0.85\n", - "step: 1790, Red action: do-nothing, Blue reward:0.85\n", - "step: 1791, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1792, Red action: do-nothing, Blue reward:0.85\n", - "step: 1793, Red action: do-nothing, Blue reward:0.85\n", - "step: 1794, Red action: do-nothing, Blue reward:0.85\n", - "step: 1795, Red action: do-nothing, Blue reward:0.85\n", - "step: 1796, Red action: do-nothing, Blue reward:0.85\n", - "step: 1797, Red action: do-nothing, Blue reward:0.85\n", - "step: 1798, Red action: do-nothing, Blue reward:0.85\n", - "step: 1799, Red action: do-nothing, Blue reward:0.85\n", - "step: 1800, Red action: do-nothing, Blue reward:0.85\n", - "step: 1801, Red action: do-nothing, Blue reward:0.85\n", - "step: 1802, Red action: do-nothing, Blue reward:0.85\n", - "step: 1803, Red action: do-nothing, Blue reward:0.85\n", - "step: 1804, Red action: do-nothing, Blue reward:0.85\n", - "step: 1805, Red action: do-nothing, Blue reward:0.85\n", - "step: 1806, Red action: do-nothing, Blue reward:0.85\n", - "step: 1807, Red action: do-nothing, Blue reward:0.85\n", - "step: 1808, Red action: do-nothing, Blue reward:0.85\n", - "step: 1809, Red action: do-nothing, Blue reward:0.85\n", - "step: 1810, Red action: do-nothing, Blue reward:0.85\n", - "step: 1811, Red action: do-nothing, Blue reward:0.85\n", - "step: 1812, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1813, Red action: do-nothing, Blue reward:0.85\n", - "step: 1814, Red action: do-nothing, Blue reward:0.85\n", - "step: 1815, Red action: do-nothing, Blue reward:0.85\n", - "step: 1816, Red action: do-nothing, Blue reward:0.85\n", - "step: 1817, Red action: do-nothing, Blue reward:0.85\n", - "step: 1818, Red action: do-nothing, Blue reward:0.85\n", - "step: 1819, Red action: do-nothing, Blue reward:0.85\n", - "step: 1820, Red action: do-nothing, Blue reward:0.85\n", - "step: 1821, Red action: do-nothing, Blue reward:0.85\n", - "step: 1822, Red action: do-nothing, Blue reward:0.85\n", - "step: 1823, Red action: do-nothing, Blue reward:0.85\n", - "step: 1824, Red action: do-nothing, Blue reward:0.85\n", - "step: 1825, Red action: do-nothing, Blue reward:0.85\n", - "step: 1826, Red action: do-nothing, Blue reward:0.85\n", - "step: 1827, Red action: do-nothing, Blue reward:0.85\n", - "step: 1828, Red action: do-nothing, Blue reward:0.85\n", - "step: 1829, Red action: do-nothing, Blue reward:0.85\n", - "step: 1830, Red action: do-nothing, Blue reward:0.85\n", - "step: 1831, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1832, Red action: do-nothing, Blue reward:0.85\n", - "step: 1833, Red action: do-nothing, Blue reward:0.85\n", - "step: 1834, Red action: do-nothing, Blue reward:0.85\n", - "step: 1835, Red action: do-nothing, Blue reward:0.85\n", - "step: 1836, Red action: do-nothing, Blue reward:0.85\n", - "step: 1837, Red action: do-nothing, Blue reward:0.85\n", - "step: 1838, Red action: do-nothing, Blue reward:0.85\n", - "step: 1839, Red action: do-nothing, Blue reward:0.85\n", - "step: 1840, Red action: do-nothing, Blue reward:0.85\n", - "step: 1841, Red action: do-nothing, Blue reward:0.85\n", - "step: 1842, Red action: do-nothing, Blue reward:0.85\n", - "step: 1843, Red action: do-nothing, Blue reward:0.85\n", - "step: 1844, Red action: do-nothing, Blue reward:0.85\n", - "step: 1845, Red action: do-nothing, Blue reward:0.85\n", - "step: 1846, Red action: do-nothing, Blue reward:0.85\n", - "step: 1847, Red action: do-nothing, Blue reward:0.85\n", - "step: 1848, Red action: do-nothing, Blue reward:0.85\n", - "step: 1849, Red action: do-nothing, Blue reward:0.85\n", - "step: 1850, Red action: do-nothing, Blue reward:0.85\n", - "step: 1851, Red action: do-nothing, Blue reward:0.85\n", - "step: 1852, Red action: do-nothing, Blue reward:0.85\n", - "step: 1853, Red action: do-nothing, Blue reward:0.85\n", - "step: 1854, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1855, Red action: do-nothing, Blue reward:0.85\n", - "step: 1856, Red action: do-nothing, Blue reward:0.85\n", - "step: 1857, Red action: do-nothing, Blue reward:0.85\n", - "step: 1858, Red action: do-nothing, Blue reward:0.85\n", - "step: 1859, Red action: do-nothing, Blue reward:0.85\n", - "step: 1860, Red action: do-nothing, Blue reward:0.85\n", - "step: 1861, Red action: do-nothing, Blue reward:0.85\n", - "step: 1862, Red action: do-nothing, Blue reward:0.85\n", - "step: 1863, Red action: do-nothing, Blue reward:0.85\n", - "step: 1864, Red action: do-nothing, Blue reward:0.85\n", - "step: 1865, Red action: do-nothing, Blue reward:0.85\n", - "step: 1866, Red action: do-nothing, Blue reward:0.85\n", - "step: 1867, Red action: do-nothing, Blue reward:0.85\n", - "step: 1868, Red action: do-nothing, Blue reward:0.85\n", - "step: 1869, Red action: do-nothing, Blue reward:0.85\n", - "step: 1870, Red action: do-nothing, Blue reward:0.85\n", - "step: 1871, Red action: do-nothing, Blue reward:0.85\n", - "step: 1872, Red action: do-nothing, Blue reward:0.85\n", - "step: 1873, Red action: do-nothing, Blue reward:0.85\n", - "step: 1874, Red action: do-nothing, Blue reward:0.85\n", - "step: 1875, Red action: do-nothing, Blue reward:0.85\n", - "step: 1876, Red action: do-nothing, Blue reward:0.85\n", - "step: 1877, Red action: do-nothing, Blue reward:0.85\n", - "step: 1878, Red action: do-nothing, Blue reward:0.85\n", - "step: 1879, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1880, Red action: do-nothing, Blue reward:0.85\n", - "step: 1881, Red action: do-nothing, Blue reward:0.85\n", - "step: 1882, Red action: do-nothing, Blue reward:0.85\n", - "step: 1883, Red action: do-nothing, Blue reward:0.85\n", - "step: 1884, Red action: do-nothing, Blue reward:0.85\n", - "step: 1885, Red action: do-nothing, Blue reward:0.85\n", - "step: 1886, Red action: do-nothing, Blue reward:0.85\n", - "step: 1887, Red action: do-nothing, Blue reward:0.85\n", - "step: 1888, Red action: do-nothing, Blue reward:0.85\n", - "step: 1889, Red action: do-nothing, Blue reward:0.85\n", - "step: 1890, Red action: do-nothing, Blue reward:0.85\n", - "step: 1891, Red action: do-nothing, Blue reward:0.85\n", - "step: 1892, Red action: do-nothing, Blue reward:0.85\n", - "step: 1893, Red action: do-nothing, Blue reward:0.85\n", - "step: 1894, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1895, Red action: do-nothing, Blue reward:0.85\n", - "step: 1896, Red action: do-nothing, Blue reward:0.85\n", - "step: 1897, Red action: do-nothing, Blue reward:0.85\n", - "step: 1898, Red action: do-nothing, Blue reward:0.85\n", - "step: 1899, Red action: do-nothing, Blue reward:0.85\n", - "step: 1900, Red action: do-nothing, Blue reward:0.85\n", - "step: 1901, Red action: do-nothing, Blue reward:0.85\n", - "step: 1902, Red action: do-nothing, Blue reward:0.85\n", - "step: 1903, Red action: do-nothing, Blue reward:0.85\n", - "step: 1904, Red action: do-nothing, Blue reward:0.85\n", - "step: 1905, Red action: do-nothing, Blue reward:0.85\n", - "step: 1906, Red action: do-nothing, Blue reward:0.85\n", - "step: 1907, Red action: do-nothing, Blue reward:0.85\n", - "step: 1908, Red action: do-nothing, Blue reward:0.85\n", - "step: 1909, Red action: do-nothing, Blue reward:0.85\n", - "step: 1910, Red action: do-nothing, Blue reward:0.85\n", - "step: 1911, Red action: do-nothing, Blue reward:0.85\n", - "step: 1912, Red action: do-nothing, Blue reward:0.85\n", - "step: 1913, Red action: do-nothing, Blue reward:0.85\n", - "step: 1914, Red action: do-nothing, Blue reward:0.85\n", - "step: 1915, Red action: do-nothing, Blue reward:0.85\n", - "step: 1916, Red action: do-nothing, Blue reward:0.85\n", - "step: 1917, Red action: do-nothing, Blue reward:0.85\n", - "step: 1918, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1919, Red action: do-nothing, Blue reward:0.85\n", - "step: 1920, Red action: do-nothing, Blue reward:0.85\n", - "step: 1921, Red action: do-nothing, Blue reward:0.85\n", - "step: 1922, Red action: do-nothing, Blue reward:0.85\n", - "step: 1923, Red action: do-nothing, Blue reward:0.85\n", - "step: 1924, Red action: do-nothing, Blue reward:0.85\n", - "step: 1925, Red action: do-nothing, Blue reward:0.85\n", - "step: 1926, Red action: do-nothing, Blue reward:0.85\n", - "step: 1927, Red action: do-nothing, Blue reward:0.85\n", - "step: 1928, Red action: do-nothing, Blue reward:0.85\n", - "step: 1929, Red action: do-nothing, Blue reward:0.85\n", - "step: 1930, Red action: do-nothing, Blue reward:0.85\n", - "step: 1931, Red action: do-nothing, Blue reward:0.85\n", - "step: 1932, Red action: do-nothing, Blue reward:0.85\n", - "step: 1933, Red action: do-nothing, Blue reward:0.85\n", - "step: 1934, Red action: do-nothing, Blue reward:0.85\n", - "step: 1935, Red action: do-nothing, Blue reward:0.85\n", - "step: 1936, Red action: do-nothing, Blue reward:0.85\n", - "step: 1937, Red action: do-nothing, Blue reward:0.85\n", - "step: 1938, Red action: do-nothing, Blue reward:0.85\n", - "step: 1939, Red action: do-nothing, Blue reward:0.85\n", - "step: 1940, Red action: do-nothing, Blue reward:0.85\n", - "step: 1941, Red action: do-nothing, Blue reward:0.85\n", - "step: 1942, Red action: do-nothing, Blue reward:0.85\n", - "step: 1943, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1944, Red action: do-nothing, Blue reward:0.85\n", - "step: 1945, Red action: do-nothing, Blue reward:0.85\n", - "step: 1946, Red action: do-nothing, Blue reward:0.85\n", - "step: 1947, Red action: do-nothing, Blue reward:0.85\n", - "step: 1948, Red action: do-nothing, Blue reward:0.85\n", - "step: 1949, Red action: do-nothing, Blue reward:0.85\n", - "step: 1950, Red action: do-nothing, Blue reward:0.85\n", - "step: 1951, Red action: do-nothing, Blue reward:0.85\n", - "step: 1952, Red action: do-nothing, Blue reward:0.85\n", - "step: 1953, Red action: do-nothing, Blue reward:0.85\n", - "step: 1954, Red action: do-nothing, Blue reward:0.85\n", - "step: 1955, Red action: do-nothing, Blue reward:0.85\n", - "step: 1956, Red action: do-nothing, Blue reward:0.85\n", - "step: 1957, Red action: do-nothing, Blue reward:0.85\n", - "step: 1958, Red action: do-nothing, Blue reward:0.85\n", - "step: 1959, Red action: do-nothing, Blue reward:0.85\n", - "step: 1960, Red action: do-nothing, Blue reward:0.85\n", - "step: 1961, Red action: do-nothing, Blue reward:0.85\n", - "step: 1962, Red action: do-nothing, Blue reward:0.85\n", - "step: 1963, Red action: do-nothing, Blue reward:0.85\n", - "step: 1964, Red action: do-nothing, Blue reward:0.85\n", - "step: 1965, Red action: do-nothing, Blue reward:0.85\n", - "step: 1966, Red action: do-nothing, Blue reward:0.85\n", - "step: 1967, Red action: do-nothing, Blue reward:0.85\n", - "step: 1968, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1969, Red action: do-nothing, Blue reward:0.85\n", - "step: 1970, Red action: do-nothing, Blue reward:0.85\n", - "step: 1971, Red action: do-nothing, Blue reward:0.85\n", - "step: 1972, Red action: do-nothing, Blue reward:0.85\n", - "step: 1973, Red action: do-nothing, Blue reward:0.85\n", - "step: 1974, Red action: do-nothing, Blue reward:0.85\n", - "step: 1975, Red action: do-nothing, Blue reward:0.85\n", - "step: 1976, Red action: do-nothing, Blue reward:0.85\n", - "step: 1977, Red action: do-nothing, Blue reward:0.85\n", - "step: 1978, Red action: do-nothing, Blue reward:0.85\n", - "step: 1979, Red action: do-nothing, Blue reward:0.85\n", - "step: 1980, Red action: do-nothing, Blue reward:0.85\n", - "step: 1981, Red action: do-nothing, Blue reward:0.85\n", - "step: 1982, Red action: do-nothing, Blue reward:0.85\n", - "step: 1983, Red action: do-nothing, Blue reward:0.85\n", - "step: 1984, Red action: do-nothing, Blue reward:0.85\n", - "step: 1985, Red action: do-nothing, Blue reward:0.85\n", - "step: 1986, Red action: do-nothing, Blue reward:0.85\n", - "step: 1987, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1988, Red action: do-nothing, Blue reward:0.85\n", - "step: 1989, Red action: do-nothing, Blue reward:0.85\n", - "step: 1990, Red action: do-nothing, Blue reward:0.85\n", - "step: 1991, Red action: do-nothing, Blue reward:0.85\n", - "step: 1992, Red action: do-nothing, Blue reward:0.85\n", - "step: 1993, Red action: do-nothing, Blue reward:0.85\n", - "step: 1994, Red action: do-nothing, Blue reward:0.85\n", - "step: 1995, Red action: do-nothing, Blue reward:0.85\n", - "step: 1996, Red action: do-nothing, Blue reward:0.85\n", - "step: 1997, Red action: do-nothing, Blue reward:0.85\n", - "step: 1998, Red action: do-nothing, Blue reward:0.85\n", - "step: 1999, Red action: do-nothing, Blue reward:0.85\n", - "step: 2000, Red action: do-nothing, Blue reward:0.85\n", - "step: 2001, Red action: do-nothing, Blue reward:0.85\n", - "step: 2002, Red action: do-nothing, Blue reward:0.85\n", - "step: 2003, Red action: do-nothing, Blue reward:0.85\n", - "step: 2004, Red action: do-nothing, Blue reward:0.85\n", - "step: 2005, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2006, Red action: do-nothing, Blue reward:0.85\n", - "step: 2007, Red action: do-nothing, Blue reward:0.85\n", - "step: 2008, Red action: do-nothing, Blue reward:0.85\n", - "step: 2009, Red action: do-nothing, Blue reward:0.85\n", - "step: 2010, Red action: do-nothing, Blue reward:0.85\n", - "step: 2011, Red action: do-nothing, Blue reward:0.85\n", - "step: 2012, Red action: do-nothing, Blue reward:0.85\n", - "step: 2013, Red action: do-nothing, Blue reward:0.85\n", - "step: 2014, Red action: do-nothing, Blue reward:0.85\n", - "step: 2015, Red action: do-nothing, Blue reward:0.85\n", - "step: 2016, Red action: do-nothing, Blue reward:0.85\n", - "step: 2017, Red action: do-nothing, Blue reward:0.85\n", - "step: 2018, Red action: do-nothing, Blue reward:0.85\n", - "step: 2019, Red action: do-nothing, Blue reward:0.85\n", - "step: 2020, Red action: do-nothing, Blue reward:0.85\n", - "step: 2021, Red action: do-nothing, Blue reward:0.85\n", - "step: 2022, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2023, Red action: do-nothing, Blue reward:0.85\n", - "step: 2024, Red action: do-nothing, Blue reward:0.85\n", - "step: 2025, Red action: do-nothing, Blue reward:0.85\n", - "step: 2026, Red action: do-nothing, Blue reward:0.85\n", - "step: 2027, Red action: do-nothing, Blue reward:0.85\n", - "step: 2028, Red action: do-nothing, Blue reward:0.85\n", - "step: 2029, Red action: do-nothing, Blue reward:0.85\n", - "step: 2030, Red action: do-nothing, Blue reward:0.85\n", - "step: 2031, Red action: do-nothing, Blue reward:0.85\n", - "step: 2032, Red action: do-nothing, Blue reward:0.85\n", - "step: 2033, Red action: do-nothing, Blue reward:0.85\n", - "step: 2034, Red action: do-nothing, Blue reward:0.85\n", - "step: 2035, Red action: do-nothing, Blue reward:0.85\n", - "step: 2036, Red action: do-nothing, Blue reward:0.85\n", - "step: 2037, Red action: do-nothing, Blue reward:0.85\n", - "step: 2038, Red action: do-nothing, Blue reward:0.85\n", - "step: 2039, Red action: do-nothing, Blue reward:0.85\n", - "step: 2040, Red action: do-nothing, Blue reward:0.85\n", - "step: 2041, Red action: do-nothing, Blue reward:0.85\n", - "step: 2042, Red action: do-nothing, Blue reward:0.85\n", - "step: 2043, Red action: do-nothing, Blue reward:0.85\n", - "step: 2044, Red action: do-nothing, Blue reward:0.85\n", - "step: 2045, Red action: do-nothing, Blue reward:0.85\n", - "step: 2046, Red action: do-nothing, Blue reward:0.85\n", - "step: 2047, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2048, Red action: do-nothing, Blue reward:0.85\n", - "step: 2049, Red action: do-nothing, Blue reward:0.85\n", - "step: 2050, Red action: do-nothing, Blue reward:0.85\n", - "step: 2051, Red action: do-nothing, Blue reward:0.85\n", - "step: 2052, Red action: do-nothing, Blue reward:0.85\n", - "step: 2053, Red action: do-nothing, Blue reward:0.85\n", - "step: 2054, Red action: do-nothing, Blue reward:0.85\n", - "step: 2055, Red action: do-nothing, Blue reward:0.85\n", - "step: 2056, Red action: do-nothing, Blue reward:0.85\n", - "step: 2057, Red action: do-nothing, Blue reward:0.85\n", - "step: 2058, Red action: do-nothing, Blue reward:0.85\n", - "step: 2059, Red action: do-nothing, Blue reward:0.85\n", - "step: 2060, Red action: do-nothing, Blue reward:0.85\n", - "step: 2061, Red action: do-nothing, Blue reward:0.85\n", - "step: 2062, Red action: do-nothing, Blue reward:0.85\n", - "step: 2063, Red action: do-nothing, Blue reward:0.85\n", - "step: 2064, Red action: do-nothing, Blue reward:0.85\n", - "step: 2065, Red action: do-nothing, Blue reward:0.85\n", - "step: 2066, Red action: do-nothing, Blue reward:0.85\n", - "step: 2067, Red action: do-nothing, Blue reward:0.85\n", - "step: 2068, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2069, Red action: do-nothing, Blue reward:0.85\n", - "step: 2070, Red action: do-nothing, Blue reward:0.85\n", - "step: 2071, Red action: do-nothing, Blue reward:0.85\n", - "step: 2072, Red action: do-nothing, Blue reward:0.85\n", - "step: 2073, Red action: do-nothing, Blue reward:0.85\n", - "step: 2074, Red action: do-nothing, Blue reward:0.85\n", - "step: 2075, Red action: do-nothing, Blue reward:0.85\n", - "step: 2076, Red action: do-nothing, Blue reward:0.85\n", - "step: 2077, Red action: do-nothing, Blue reward:0.85\n", - "step: 2078, Red action: do-nothing, Blue reward:0.85\n", - "step: 2079, Red action: do-nothing, Blue reward:0.85\n", - "step: 2080, Red action: do-nothing, Blue reward:0.85\n", - "step: 2081, Red action: do-nothing, Blue reward:0.85\n", - "step: 2082, Red action: do-nothing, Blue reward:0.85\n", - "step: 2083, Red action: do-nothing, Blue reward:0.85\n", - "step: 2084, Red action: do-nothing, Blue reward:0.85\n", - "step: 2085, Red action: do-nothing, Blue reward:0.85\n", - "step: 2086, Red action: do-nothing, Blue reward:0.85\n", - "step: 2087, Red action: do-nothing, Blue reward:0.85\n", - "step: 2088, Red action: do-nothing, Blue reward:0.85\n", - "step: 2089, Red action: do-nothing, Blue reward:0.85\n", - "step: 2090, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2091, Red action: do-nothing, Blue reward:0.85\n", - "step: 2092, Red action: do-nothing, Blue reward:0.85\n", - "step: 2093, Red action: do-nothing, Blue reward:0.85\n", - "step: 2094, Red action: do-nothing, Blue reward:0.85\n", - "step: 2095, Red action: do-nothing, Blue reward:0.85\n", - "step: 2096, Red action: do-nothing, Blue reward:0.85\n", - "step: 2097, Red action: do-nothing, Blue reward:0.85\n", - "step: 2098, Red action: do-nothing, Blue reward:0.85\n", - "step: 2099, Red action: do-nothing, Blue reward:0.85\n", - "step: 2100, Red action: do-nothing, Blue reward:0.85\n", - "step: 2101, Red action: do-nothing, Blue reward:0.85\n", - "step: 2102, Red action: do-nothing, Blue reward:0.85\n", - "step: 2103, Red action: do-nothing, Blue reward:0.85\n", - "step: 2104, Red action: do-nothing, Blue reward:0.85\n", - "step: 2105, Red action: do-nothing, Blue reward:0.85\n", - "step: 2106, Red action: do-nothing, Blue reward:0.85\n", - "step: 2107, Red action: do-nothing, Blue reward:0.85\n", - "step: 2108, Red action: do-nothing, Blue reward:0.85\n", - "step: 2109, Red action: do-nothing, Blue reward:0.85\n", - "step: 2110, Red action: do-nothing, Blue reward:0.85\n", - "step: 2111, Red action: do-nothing, Blue reward:0.85\n", - "step: 2112, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2113, Red action: do-nothing, Blue reward:0.85\n", - "step: 2114, Red action: do-nothing, Blue reward:0.85\n", - "step: 2115, Red action: do-nothing, Blue reward:0.85\n", - "step: 2116, Red action: do-nothing, Blue reward:0.85\n", - "step: 2117, Red action: do-nothing, Blue reward:0.85\n", - "step: 2118, Red action: do-nothing, Blue reward:0.85\n", - "step: 2119, Red action: do-nothing, Blue reward:0.85\n", - "step: 2120, Red action: do-nothing, Blue reward:0.85\n", - "step: 2121, Red action: do-nothing, Blue reward:0.85\n", - "step: 2122, Red action: do-nothing, Blue reward:0.85\n", - "step: 2123, Red action: do-nothing, Blue reward:0.85\n", - "step: 2124, Red action: do-nothing, Blue reward:0.85\n", - "step: 2125, Red action: do-nothing, Blue reward:0.85\n", - "step: 2126, Red action: do-nothing, Blue reward:0.85\n", - "step: 2127, Red action: do-nothing, Blue reward:0.85\n", - "step: 2128, Red action: do-nothing, Blue reward:0.85\n", - "step: 2129, Red action: do-nothing, Blue reward:0.85\n", - "step: 2130, Red action: do-nothing, Blue reward:0.85\n", - "step: 2131, Red action: do-nothing, Blue reward:0.85\n", - "step: 2132, Red action: do-nothing, Blue reward:0.85\n", - "step: 2133, Red action: do-nothing, Blue reward:0.85\n", - "step: 2134, Red action: do-nothing, Blue reward:0.85\n", - "step: 2135, Red action: do-nothing, Blue reward:0.85\n", - "step: 2136, Red action: do-nothing, Blue reward:0.85\n", - "step: 2137, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2138, Red action: do-nothing, Blue reward:0.85\n", - "step: 2139, Red action: do-nothing, Blue reward:0.85\n", - "step: 2140, Red action: do-nothing, Blue reward:0.85\n", - "step: 2141, Red action: do-nothing, Blue reward:0.85\n", - "step: 2142, Red action: do-nothing, Blue reward:0.85\n", - "step: 2143, Red action: do-nothing, Blue reward:0.85\n", - "step: 2144, Red action: do-nothing, Blue reward:0.85\n", - "step: 2145, Red action: do-nothing, Blue reward:0.85\n", - "step: 2146, Red action: do-nothing, Blue reward:0.85\n", - "step: 2147, Red action: do-nothing, Blue reward:0.85\n", - "step: 2148, Red action: do-nothing, Blue reward:0.85\n", - "step: 2149, Red action: do-nothing, Blue reward:0.85\n", - "step: 2150, Red action: do-nothing, Blue reward:0.85\n", - "step: 2151, Red action: do-nothing, Blue reward:0.85\n", - "step: 2152, Red action: do-nothing, Blue reward:0.85\n", - "step: 2153, Red action: do-nothing, Blue reward:0.85\n", - "step: 2154, Red action: do-nothing, Blue reward:0.85\n", - "step: 2155, Red action: do-nothing, Blue reward:0.85\n", - "step: 2156, Red action: do-nothing, Blue reward:0.85\n", - "step: 2157, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2158, Red action: do-nothing, Blue reward:0.85\n", - "step: 2159, Red action: do-nothing, Blue reward:0.85\n", - "step: 2160, Red action: do-nothing, Blue reward:0.85\n", - "step: 2161, Red action: do-nothing, Blue reward:0.85\n", - "step: 2162, Red action: do-nothing, Blue reward:0.85\n", - "step: 2163, Red action: do-nothing, Blue reward:0.85\n", - "step: 2164, Red action: do-nothing, Blue reward:0.85\n", - "step: 2165, Red action: do-nothing, Blue reward:0.85\n", - "step: 2166, Red action: do-nothing, Blue reward:0.85\n", - "step: 2167, Red action: do-nothing, Blue reward:0.85\n", - "step: 2168, Red action: do-nothing, Blue reward:0.85\n", - "step: 2169, Red action: do-nothing, Blue reward:0.85\n", - "step: 2170, Red action: do-nothing, Blue reward:0.85\n", - "step: 2171, Red action: do-nothing, Blue reward:0.85\n", - "step: 2172, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2173, Red action: do-nothing, Blue reward:0.85\n", - "step: 2174, Red action: do-nothing, Blue reward:0.85\n", - "step: 2175, Red action: do-nothing, Blue reward:0.85\n", - "step: 2176, Red action: do-nothing, Blue reward:0.85\n", - "step: 2177, Red action: do-nothing, Blue reward:0.85\n", - "step: 2178, Red action: do-nothing, Blue reward:0.85\n", - "step: 2179, Red action: do-nothing, Blue reward:0.85\n", - "step: 2180, Red action: do-nothing, Blue reward:0.85\n", - "step: 2181, Red action: do-nothing, Blue reward:0.85\n", - "step: 2182, Red action: do-nothing, Blue reward:0.85\n", - "step: 2183, Red action: do-nothing, Blue reward:0.85\n", - "step: 2184, Red action: do-nothing, Blue reward:0.85\n", - "step: 2185, Red action: do-nothing, Blue reward:0.85\n", - "step: 2186, Red action: do-nothing, Blue reward:0.85\n", - "step: 2187, Red action: do-nothing, Blue reward:0.85\n", - "step: 2188, Red action: do-nothing, Blue reward:0.85\n", - "step: 2189, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2190, Red action: do-nothing, Blue reward:0.85\n", - "step: 2191, Red action: do-nothing, Blue reward:0.85\n", - "step: 2192, Red action: do-nothing, Blue reward:0.85\n", - "step: 2193, Red action: do-nothing, Blue reward:0.85\n", - "step: 2194, Red action: do-nothing, Blue reward:0.85\n", - "step: 2195, Red action: do-nothing, Blue reward:0.85\n", - "step: 2196, Red action: do-nothing, Blue reward:0.85\n", - "step: 2197, Red action: do-nothing, Blue reward:0.85\n", - "step: 2198, Red action: do-nothing, Blue reward:0.85\n", - "step: 2199, Red action: do-nothing, Blue reward:0.85\n", - "step: 2200, Red action: do-nothing, Blue reward:0.85\n", - "step: 2201, Red action: do-nothing, Blue reward:0.85\n", - "step: 2202, Red action: do-nothing, Blue reward:0.85\n", - "step: 2203, Red action: do-nothing, Blue reward:0.85\n", - "step: 2204, Red action: do-nothing, Blue reward:0.85\n", - "step: 2205, Red action: do-nothing, Blue reward:0.85\n", - "step: 2206, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2207, Red action: do-nothing, Blue reward:0.85\n", - "step: 2208, Red action: do-nothing, Blue reward:0.85\n", - "step: 2209, Red action: do-nothing, Blue reward:0.85\n", - "step: 2210, Red action: do-nothing, Blue reward:0.85\n", - "step: 2211, Red action: do-nothing, Blue reward:0.85\n", - "step: 2212, Red action: do-nothing, Blue reward:0.85\n", - "step: 2213, Red action: do-nothing, Blue reward:0.85\n", - "step: 2214, Red action: do-nothing, Blue reward:0.85\n", - "step: 2215, Red action: do-nothing, Blue reward:0.85\n", - "step: 2216, Red action: do-nothing, Blue reward:0.85\n", - "step: 2217, Red action: do-nothing, Blue reward:0.85\n", - "step: 2218, Red action: do-nothing, Blue reward:0.85\n", - "step: 2219, Red action: do-nothing, Blue reward:0.85\n", - "step: 2220, Red action: do-nothing, Blue reward:0.85\n", - "step: 2221, Red action: do-nothing, Blue reward:0.85\n", - "step: 2222, Red action: do-nothing, Blue reward:0.85\n", - "step: 2223, Red action: do-nothing, Blue reward:0.85\n", - "step: 2224, Red action: do-nothing, Blue reward:0.85\n", - "step: 2225, Red action: do-nothing, Blue reward:0.85\n", - "step: 2226, Red action: do-nothing, Blue reward:0.85\n", - "step: 2227, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2228, Red action: do-nothing, Blue reward:0.85\n", - "step: 2229, Red action: do-nothing, Blue reward:0.85\n", - "step: 2230, Red action: do-nothing, Blue reward:0.85\n", - "step: 2231, Red action: do-nothing, Blue reward:0.85\n", - "step: 2232, Red action: do-nothing, Blue reward:0.85\n", - "step: 2233, Red action: do-nothing, Blue reward:0.85\n", - "step: 2234, Red action: do-nothing, Blue reward:0.85\n", - "step: 2235, Red action: do-nothing, Blue reward:0.85\n", - "step: 2236, Red action: do-nothing, Blue reward:0.85\n", - "step: 2237, Red action: do-nothing, Blue reward:0.85\n", - "step: 2238, Red action: do-nothing, Blue reward:0.85\n", - "step: 2239, Red action: do-nothing, Blue reward:0.85\n", - "step: 2240, Red action: do-nothing, Blue reward:0.85\n", - "step: 2241, Red action: do-nothing, Blue reward:0.85\n", - "step: 2242, Red action: do-nothing, Blue reward:0.85\n", - "step: 2243, Red action: do-nothing, Blue reward:0.85\n", - "step: 2244, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2245, Red action: do-nothing, Blue reward:0.85\n", - "step: 2246, Red action: do-nothing, Blue reward:0.85\n", - "step: 2247, Red action: do-nothing, Blue reward:0.85\n", - "step: 2248, Red action: do-nothing, Blue reward:0.85\n", - "step: 2249, Red action: do-nothing, Blue reward:0.85\n", - "step: 2250, Red action: do-nothing, Blue reward:0.85\n", - "step: 2251, Red action: do-nothing, Blue reward:0.85\n", - "step: 2252, Red action: do-nothing, Blue reward:0.85\n", - "step: 2253, Red action: do-nothing, Blue reward:0.85\n", - "step: 2254, Red action: do-nothing, Blue reward:0.85\n", - "step: 2255, Red action: do-nothing, Blue reward:0.85\n", - "step: 2256, Red action: do-nothing, Blue reward:0.85\n", - "step: 2257, Red action: do-nothing, Blue reward:0.85\n", - "step: 2258, Red action: do-nothing, Blue reward:0.85\n", - "step: 2259, Red action: do-nothing, Blue reward:0.85\n", - "step: 2260, Red action: do-nothing, Blue reward:0.85\n", - "step: 2261, Red action: do-nothing, Blue reward:0.85\n", - "step: 2262, Red action: do-nothing, Blue reward:0.85\n", - "step: 2263, Red action: do-nothing, Blue reward:0.85\n", - "step: 2264, Red action: do-nothing, Blue reward:0.85\n", - "step: 2265, Red action: do-nothing, Blue reward:0.85\n", - "step: 2266, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2267, Red action: do-nothing, Blue reward:0.85\n", - "step: 2268, Red action: do-nothing, Blue reward:0.85\n", - "step: 2269, Red action: do-nothing, Blue reward:0.85\n", - "step: 2270, Red action: do-nothing, Blue reward:0.85\n", - "step: 2271, Red action: do-nothing, Blue reward:0.85\n", - "step: 2272, Red action: do-nothing, Blue reward:0.85\n", - "step: 2273, Red action: do-nothing, Blue reward:0.85\n", - "step: 2274, Red action: do-nothing, Blue reward:0.85\n", - "step: 2275, Red action: do-nothing, Blue reward:0.85\n", - "step: 2276, Red action: do-nothing, Blue reward:0.85\n", - "step: 2277, Red action: do-nothing, Blue reward:0.85\n", - "step: 2278, Red action: do-nothing, Blue reward:0.85\n", - "step: 2279, Red action: do-nothing, Blue reward:0.85\n", - "step: 2280, Red action: do-nothing, Blue reward:0.85\n", - "step: 2281, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2282, Red action: do-nothing, Blue reward:0.85\n", - "step: 2283, Red action: do-nothing, Blue reward:0.85\n", - "step: 2284, Red action: do-nothing, Blue reward:0.85\n", - "step: 2285, Red action: do-nothing, Blue reward:0.85\n", - "step: 2286, Red action: do-nothing, Blue reward:0.85\n", - "step: 2287, Red action: do-nothing, Blue reward:0.85\n", - "step: 2288, Red action: do-nothing, Blue reward:0.85\n", - "step: 2289, Red action: do-nothing, Blue reward:0.85\n", - "step: 2290, Red action: do-nothing, Blue reward:0.85\n", - "step: 2291, Red action: do-nothing, Blue reward:0.85\n", - "step: 2292, Red action: do-nothing, Blue reward:0.85\n", - "step: 2293, Red action: do-nothing, Blue reward:0.85\n", - "step: 2294, Red action: do-nothing, Blue reward:0.85\n", - "step: 2295, Red action: do-nothing, Blue reward:0.85\n", - "step: 2296, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2297, Red action: do-nothing, Blue reward:0.85\n", - "step: 2298, Red action: do-nothing, Blue reward:0.85\n", - "step: 2299, Red action: do-nothing, Blue reward:0.85\n", - "step: 2300, Red action: do-nothing, Blue reward:0.85\n", - "step: 2301, Red action: do-nothing, Blue reward:0.85\n", - "step: 2302, Red action: do-nothing, Blue reward:0.85\n", - "step: 2303, Red action: do-nothing, Blue reward:0.85\n", - "step: 2304, Red action: do-nothing, Blue reward:0.85\n", - "step: 2305, Red action: do-nothing, Blue reward:0.85\n", - "step: 2306, Red action: do-nothing, Blue reward:0.85\n", - "step: 2307, Red action: do-nothing, Blue reward:0.85\n", - "step: 2308, Red action: do-nothing, Blue reward:0.85\n", - "step: 2309, Red action: do-nothing, Blue reward:0.85\n", - "step: 2310, Red action: do-nothing, Blue reward:0.85\n", - "step: 2311, Red action: do-nothing, Blue reward:0.85\n", - "step: 2312, Red action: do-nothing, Blue reward:0.85\n", - "step: 2313, Red action: do-nothing, Blue reward:0.85\n", - "step: 2314, Red action: do-nothing, Blue reward:0.85\n", - "step: 2315, Red action: do-nothing, Blue reward:0.85\n", - "step: 2316, Red action: do-nothing, Blue reward:0.85\n", - "step: 2317, Red action: do-nothing, Blue reward:0.85\n", - "step: 2318, Red action: do-nothing, Blue reward:0.85\n", - "step: 2319, Red action: do-nothing, Blue reward:0.85\n", - "step: 2320, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2321, Red action: do-nothing, Blue reward:0.85\n", - "step: 2322, Red action: do-nothing, Blue reward:0.85\n", - "step: 2323, Red action: do-nothing, Blue reward:0.85\n", - "step: 2324, Red action: do-nothing, Blue reward:0.85\n", - "step: 2325, Red action: do-nothing, Blue reward:0.85\n", - "step: 2326, Red action: do-nothing, Blue reward:0.85\n", - "step: 2327, Red action: do-nothing, Blue reward:0.85\n", - "step: 2328, Red action: do-nothing, Blue reward:0.85\n", - "step: 2329, Red action: do-nothing, Blue reward:0.85\n", - "step: 2330, Red action: do-nothing, Blue reward:0.85\n", - "step: 2331, Red action: do-nothing, Blue reward:0.85\n", - "step: 2332, Red action: do-nothing, Blue reward:0.85\n", - "step: 2333, Red action: do-nothing, Blue reward:0.85\n", - "step: 2334, Red action: do-nothing, Blue reward:0.85\n", - "step: 2335, Red action: do-nothing, Blue reward:0.85\n", - "step: 2336, Red action: do-nothing, Blue reward:0.85\n", - "step: 2337, Red action: do-nothing, Blue reward:0.85\n", - "step: 2338, Red action: do-nothing, Blue reward:0.85\n", - "step: 2339, Red action: do-nothing, Blue reward:0.85\n", - "step: 2340, Red action: do-nothing, Blue reward:0.85\n", - "step: 2341, Red action: do-nothing, Blue reward:0.85\n", - "step: 2342, Red action: do-nothing, Blue reward:0.85\n", - "step: 2343, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2344, Red action: do-nothing, Blue reward:0.85\n", - "step: 2345, Red action: do-nothing, Blue reward:0.85\n", - "step: 2346, Red action: do-nothing, Blue reward:0.85\n", - "step: 2347, Red action: do-nothing, Blue reward:0.85\n", - "step: 2348, Red action: do-nothing, Blue reward:0.85\n", - "step: 2349, Red action: do-nothing, Blue reward:0.85\n", - "step: 2350, Red action: do-nothing, Blue reward:0.85\n", - "step: 2351, Red action: do-nothing, Blue reward:0.85\n", - "step: 2352, Red action: do-nothing, Blue reward:0.85\n", - "step: 2353, Red action: do-nothing, Blue reward:0.85\n", - "step: 2354, Red action: do-nothing, Blue reward:0.85\n", - "step: 2355, Red action: do-nothing, Blue reward:0.85\n", - "step: 2356, Red action: do-nothing, Blue reward:0.85\n", - "step: 2357, Red action: do-nothing, Blue reward:0.85\n", - "step: 2358, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2359, Red action: do-nothing, Blue reward:0.85\n", - "step: 2360, Red action: do-nothing, Blue reward:0.85\n", - "step: 2361, Red action: do-nothing, Blue reward:0.85\n", - "step: 2362, Red action: do-nothing, Blue reward:0.85\n", - "step: 2363, Red action: do-nothing, Blue reward:0.85\n", - "step: 2364, Red action: do-nothing, Blue reward:0.85\n", - "step: 2365, Red action: do-nothing, Blue reward:0.85\n", - "step: 2366, Red action: do-nothing, Blue reward:0.85\n", - "step: 2367, Red action: do-nothing, Blue reward:0.85\n", - "step: 2368, Red action: do-nothing, Blue reward:0.85\n", - "step: 2369, Red action: do-nothing, Blue reward:0.85\n", - "step: 2370, Red action: do-nothing, Blue reward:0.85\n", - "step: 2371, Red action: do-nothing, Blue reward:0.85\n", - "step: 2372, Red action: do-nothing, Blue reward:0.85\n", - "step: 2373, Red action: do-nothing, Blue reward:0.85\n", - "step: 2374, Red action: do-nothing, Blue reward:0.85\n", - "step: 2375, Red action: do-nothing, Blue reward:0.85\n", - "step: 2376, Red action: do-nothing, Blue reward:0.85\n", - "step: 2377, Red action: do-nothing, Blue reward:0.85\n", - "step: 2378, Red action: do-nothing, Blue reward:0.85\n", - "step: 2379, Red action: do-nothing, Blue reward:0.85\n", - "step: 2380, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2381, Red action: do-nothing, Blue reward:0.85\n", - "step: 2382, Red action: do-nothing, Blue reward:0.85\n", - "step: 2383, Red action: do-nothing, Blue reward:0.85\n", - "step: 2384, Red action: do-nothing, Blue reward:0.85\n", - "step: 2385, Red action: do-nothing, Blue reward:0.85\n", - "step: 2386, Red action: do-nothing, Blue reward:0.85\n", - "step: 2387, Red action: do-nothing, Blue reward:0.85\n", - "step: 2388, Red action: do-nothing, Blue reward:0.85\n", - "step: 2389, Red action: do-nothing, Blue reward:0.85\n", - "step: 2390, Red action: do-nothing, Blue reward:0.85\n", - "step: 2391, Red action: do-nothing, Blue reward:0.85\n", - "step: 2392, Red action: do-nothing, Blue reward:0.85\n", - "step: 2393, Red action: do-nothing, Blue reward:0.85\n", - "step: 2394, Red action: do-nothing, Blue reward:0.85\n", - "step: 2395, Red action: do-nothing, Blue reward:0.85\n", - "step: 2396, Red action: do-nothing, Blue reward:0.85\n", - "step: 2397, Red action: do-nothing, Blue reward:0.85\n", - "step: 2398, Red action: do-nothing, Blue reward:0.85\n", - "step: 2399, Red action: do-nothing, Blue reward:0.85\n", - "step: 2400, Red action: do-nothing, Blue reward:0.85\n", - "step: 2401, Red action: do-nothing, Blue reward:0.85\n", - "step: 2402, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2403, Red action: do-nothing, Blue reward:0.85\n", - "step: 2404, Red action: do-nothing, Blue reward:0.85\n", - "step: 2405, Red action: do-nothing, Blue reward:0.85\n", - "step: 2406, Red action: do-nothing, Blue reward:0.85\n", - "step: 2407, Red action: do-nothing, Blue reward:0.85\n", - "step: 2408, Red action: do-nothing, Blue reward:0.85\n", - "step: 2409, Red action: do-nothing, Blue reward:0.85\n", - "step: 2410, Red action: do-nothing, Blue reward:0.85\n", - "step: 2411, Red action: do-nothing, Blue reward:0.85\n", - "step: 2412, Red action: do-nothing, Blue reward:0.85\n", - "step: 2413, Red action: do-nothing, Blue reward:0.85\n", - "step: 2414, Red action: do-nothing, Blue reward:0.85\n", - "step: 2415, Red action: do-nothing, Blue reward:0.85\n", - "step: 2416, Red action: do-nothing, Blue reward:0.85\n", - "step: 2417, Red action: do-nothing, Blue reward:0.85\n", - "step: 2418, Red action: do-nothing, Blue reward:0.85\n", - "step: 2419, Red action: do-nothing, Blue reward:0.85\n", - "step: 2420, Red action: do-nothing, Blue reward:0.85\n", - "step: 2421, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2422, Red action: do-nothing, Blue reward:0.85\n", - "step: 2423, Red action: do-nothing, Blue reward:0.85\n", - "step: 2424, Red action: do-nothing, Blue reward:0.85\n", - "step: 2425, Red action: do-nothing, Blue reward:0.85\n", - "step: 2426, Red action: do-nothing, Blue reward:0.85\n", - "step: 2427, Red action: do-nothing, Blue reward:0.85\n", - "step: 2428, Red action: do-nothing, Blue reward:0.85\n", - "step: 2429, Red action: do-nothing, Blue reward:0.85\n", - "step: 2430, Red action: do-nothing, Blue reward:0.85\n", - "step: 2431, Red action: do-nothing, Blue reward:0.85\n", - "step: 2432, Red action: do-nothing, Blue reward:0.85\n", - "step: 2433, Red action: do-nothing, Blue reward:0.85\n", - "step: 2434, Red action: do-nothing, Blue reward:0.85\n", - "step: 2435, Red action: do-nothing, Blue reward:0.85\n", - "step: 2436, Red action: do-nothing, Blue reward:0.85\n", - "step: 2437, Red action: do-nothing, Blue reward:0.85\n", - "step: 2438, Red action: do-nothing, Blue reward:0.85\n", - "step: 2439, Red action: do-nothing, Blue reward:0.85\n", - "step: 2440, Red action: do-nothing, Blue reward:0.85\n", - "step: 2441, Red action: do-nothing, Blue reward:0.85\n", - "step: 2442, Red action: do-nothing, Blue reward:0.85\n", - "step: 2443, Red action: do-nothing, Blue reward:0.85\n", - "step: 2444, Red action: do-nothing, Blue reward:0.85\n", - "step: 2445, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2446, Red action: do-nothing, Blue reward:0.85\n", - "step: 2447, Red action: do-nothing, Blue reward:0.85\n", - "step: 2448, Red action: do-nothing, Blue reward:0.85\n", - "step: 2449, Red action: do-nothing, Blue reward:0.85\n", - "step: 2450, Red action: do-nothing, Blue reward:0.85\n", - "step: 2451, Red action: do-nothing, Blue reward:0.85\n", - "step: 2452, Red action: do-nothing, Blue reward:0.85\n", - "step: 2453, Red action: do-nothing, Blue reward:0.85\n", - "step: 2454, Red action: do-nothing, Blue reward:0.85\n", - "step: 2455, Red action: do-nothing, Blue reward:0.85\n", - "step: 2456, Red action: do-nothing, Blue reward:0.85\n", - "step: 2457, Red action: do-nothing, Blue reward:0.85\n", - "step: 2458, Red action: do-nothing, Blue reward:0.85\n", - "step: 2459, Red action: do-nothing, Blue reward:0.85\n", - "step: 2460, Red action: do-nothing, Blue reward:0.85\n", - "step: 2461, Red action: do-nothing, Blue reward:0.85\n", - "step: 2462, Red action: do-nothing, Blue reward:0.85\n", - "step: 2463, Red action: do-nothing, Blue reward:0.85\n", - "step: 2464, Red action: do-nothing, Blue reward:0.85\n", - "step: 2465, Red action: do-nothing, Blue reward:0.85\n", - "step: 2466, Red action: do-nothing, Blue reward:0.85\n", - "step: 2467, Red action: do-nothing, Blue reward:0.85\n", - "step: 2468, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2469, Red action: do-nothing, Blue reward:0.85\n", - "step: 2470, Red action: do-nothing, Blue reward:0.85\n", - "step: 2471, Red action: do-nothing, Blue reward:0.85\n", - "step: 2472, Red action: do-nothing, Blue reward:0.85\n", - "step: 2473, Red action: do-nothing, Blue reward:0.85\n", - "step: 2474, Red action: do-nothing, Blue reward:0.85\n", - "step: 2475, Red action: do-nothing, Blue reward:0.85\n", - "step: 2476, Red action: do-nothing, Blue reward:0.85\n", - "step: 2477, Red action: do-nothing, Blue reward:0.85\n", - "step: 2478, Red action: do-nothing, Blue reward:0.85\n", - "step: 2479, Red action: do-nothing, Blue reward:0.85\n", - "step: 2480, Red action: do-nothing, Blue reward:0.85\n", - "step: 2481, Red action: do-nothing, Blue reward:0.85\n", - "step: 2482, Red action: do-nothing, Blue reward:0.85\n", - "step: 2483, Red action: do-nothing, Blue reward:0.85\n", - "step: 2484, Red action: do-nothing, Blue reward:0.85\n", - "step: 2485, Red action: do-nothing, Blue reward:0.85\n", - "step: 2486, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2487, Red action: do-nothing, Blue reward:0.85\n", - "step: 2488, Red action: do-nothing, Blue reward:0.85\n", - "step: 2489, Red action: do-nothing, Blue reward:0.85\n", - "step: 2490, Red action: do-nothing, Blue reward:0.85\n", - "step: 2491, Red action: do-nothing, Blue reward:0.85\n", - "step: 2492, Red action: do-nothing, Blue reward:0.85\n", - "step: 2493, Red action: do-nothing, Blue reward:0.85\n", - "step: 2494, Red action: do-nothing, Blue reward:0.85\n", - "step: 2495, Red action: do-nothing, Blue reward:0.85\n", - "step: 2496, Red action: do-nothing, Blue reward:0.85\n", - "step: 2497, Red action: do-nothing, Blue reward:0.85\n", - "step: 2498, Red action: do-nothing, Blue reward:0.85\n", - "step: 2499, Red action: do-nothing, Blue reward:0.85\n", - "step: 2500, Red action: do-nothing, Blue reward:0.85\n", - "step: 2501, Red action: do-nothing, Blue reward:0.85\n", - "step: 2502, Red action: do-nothing, Blue reward:0.85\n", - "step: 2503, Red action: do-nothing, Blue reward:0.85\n", - "step: 2504, Red action: do-nothing, Blue reward:0.85\n", - "step: 2505, Red action: do-nothing, Blue reward:0.85\n", - "step: 2506, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2507, Red action: do-nothing, Blue reward:0.85\n", - "step: 2508, Red action: do-nothing, Blue reward:0.85\n", - "step: 2509, Red action: do-nothing, Blue reward:0.85\n", - "step: 2510, Red action: do-nothing, Blue reward:0.85\n", - "step: 2511, Red action: do-nothing, Blue reward:0.85\n", - "step: 2512, Red action: do-nothing, Blue reward:0.85\n", - "step: 2513, Red action: do-nothing, Blue reward:0.85\n", - "step: 2514, Red action: do-nothing, Blue reward:0.85\n", - "step: 2515, Red action: do-nothing, Blue reward:0.85\n", - "step: 2516, Red action: do-nothing, Blue reward:0.85\n", - "step: 2517, Red action: do-nothing, Blue reward:0.85\n", - "step: 2518, Red action: do-nothing, Blue reward:0.85\n", - "step: 2519, Red action: do-nothing, Blue reward:0.85\n", - "step: 2520, Red action: do-nothing, Blue reward:0.85\n", - "step: 2521, Red action: do-nothing, Blue reward:0.85\n", - "step: 2522, Red action: do-nothing, Blue reward:0.85\n", - "step: 2523, Red action: do-nothing, Blue reward:0.85\n", - "step: 2524, Red action: do-nothing, Blue reward:0.85\n", - "step: 2525, Red action: do-nothing, Blue reward:0.85\n", - "step: 2526, Red action: do-nothing, Blue reward:0.85\n", - "step: 2527, Red action: do-nothing, Blue reward:0.85\n", - "step: 2528, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2529, Red action: do-nothing, Blue reward:0.85\n", - "step: 2530, Red action: do-nothing, Blue reward:0.85\n", - "step: 2531, Red action: do-nothing, Blue reward:0.85\n", - "step: 2532, Red action: do-nothing, Blue reward:0.85\n", - "step: 2533, Red action: do-nothing, Blue reward:0.85\n", - "step: 2534, Red action: do-nothing, Blue reward:0.85\n", - "step: 2535, Red action: do-nothing, Blue reward:0.85\n", - "step: 2536, Red action: do-nothing, Blue reward:0.85\n", - "step: 2537, Red action: do-nothing, Blue reward:0.85\n", - "step: 2538, Red action: do-nothing, Blue reward:0.85\n", - "step: 2539, Red action: do-nothing, Blue reward:0.85\n", - "step: 2540, Red action: do-nothing, Blue reward:0.85\n", - "step: 2541, Red action: do-nothing, Blue reward:0.85\n", - "step: 2542, Red action: do-nothing, Blue reward:0.85\n", - "step: 2543, Red action: do-nothing, Blue reward:0.85\n", - "step: 2544, Red action: do-nothing, Blue reward:0.85\n", - "step: 2545, Red action: do-nothing, Blue reward:0.85\n", - "step: 2546, Red action: do-nothing, Blue reward:0.85\n", - "step: 2547, Red action: do-nothing, Blue reward:0.85\n", - "step: 2548, Red action: do-nothing, Blue reward:0.85\n", - "step: 2549, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2550, Red action: do-nothing, Blue reward:0.85\n", - "step: 2551, Red action: do-nothing, Blue reward:0.85\n", - "step: 2552, Red action: do-nothing, Blue reward:0.85\n", - "step: 2553, Red action: do-nothing, Blue reward:0.85\n", - "step: 2554, Red action: do-nothing, Blue reward:0.85\n", - "step: 2555, Red action: do-nothing, Blue reward:0.85\n", - "step: 2556, Red action: do-nothing, Blue reward:0.85\n", - "step: 2557, Red action: do-nothing, Blue reward:0.85\n", - "step: 2558, Red action: do-nothing, Blue reward:0.85\n", - "step: 2559, Red action: do-nothing, Blue reward:0.85\n", - "step: 2560, Red action: do-nothing, Blue reward:0.85\n", - "step: 2561, Red action: do-nothing, Blue reward:0.85\n", - "step: 2562, Red action: do-nothing, Blue reward:0.85\n", - "step: 2563, Red action: do-nothing, Blue reward:0.85\n", - "step: 2564, Red action: do-nothing, Blue reward:0.85\n", - "step: 2565, Red action: do-nothing, Blue reward:0.85\n", - "step: 2566, Red action: do-nothing, Blue reward:0.85\n", - "step: 2567, Red action: do-nothing, Blue reward:0.85\n", - "step: 2568, Red action: do-nothing, Blue reward:0.85\n", - "step: 2569, Red action: do-nothing, Blue reward:0.85\n", - "step: 2570, Red action: do-nothing, Blue reward:0.85\n", - "step: 2571, Red action: do-nothing, Blue reward:0.85\n", - "step: 2572, Red action: do-nothing, Blue reward:0.85\n", - "step: 2573, Red action: do-nothing, Blue reward:0.85\n", - "step: 2574, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2575, Red action: do-nothing, Blue reward:0.85\n", - "step: 2576, Red action: do-nothing, Blue reward:0.85\n", - "step: 2577, Red action: do-nothing, Blue reward:0.85\n", - "step: 2578, Red action: do-nothing, Blue reward:0.85\n", - "step: 2579, Red action: do-nothing, Blue reward:0.85\n", - "step: 2580, Red action: do-nothing, Blue reward:0.85\n", - "step: 2581, Red action: do-nothing, Blue reward:0.85\n", - "step: 2582, Red action: do-nothing, Blue reward:0.85\n", - "step: 2583, Red action: do-nothing, Blue reward:0.85\n", - "step: 2584, Red action: do-nothing, Blue reward:0.85\n", - "step: 2585, Red action: do-nothing, Blue reward:0.85\n", - "step: 2586, Red action: do-nothing, Blue reward:0.85\n", - "step: 2587, Red action: do-nothing, Blue reward:0.85\n", - "step: 2588, Red action: do-nothing, Blue reward:0.85\n", - "step: 2589, Red action: do-nothing, Blue reward:0.85\n", - "step: 2590, Red action: do-nothing, Blue reward:0.85\n", - "step: 2591, Red action: do-nothing, Blue reward:0.85\n", - "step: 2592, Red action: do-nothing, Blue reward:0.85\n", - "step: 2593, Red action: do-nothing, Blue reward:0.85\n", - "step: 2594, Red action: do-nothing, Blue reward:0.85\n", - "step: 2595, Red action: do-nothing, Blue reward:0.85\n", - "step: 2596, Red action: do-nothing, Blue reward:0.85\n", - "step: 2597, Red action: do-nothing, Blue reward:0.85\n", - "step: 2598, Red action: do-nothing, Blue reward:0.85\n", - "step: 2599, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2600, Red action: do-nothing, Blue reward:0.85\n", - "step: 2601, Red action: do-nothing, Blue reward:0.85\n", - "step: 2602, Red action: do-nothing, Blue reward:0.85\n", - "step: 2603, Red action: do-nothing, Blue reward:0.85\n", - "step: 2604, Red action: do-nothing, Blue reward:0.85\n", - "step: 2605, Red action: do-nothing, Blue reward:0.85\n", - "step: 2606, Red action: do-nothing, Blue reward:0.85\n", - "step: 2607, Red action: do-nothing, Blue reward:0.85\n", - "step: 2608, Red action: do-nothing, Blue reward:0.85\n", - "step: 2609, Red action: do-nothing, Blue reward:0.85\n", - "step: 2610, Red action: do-nothing, Blue reward:0.85\n", - "step: 2611, Red action: do-nothing, Blue reward:0.85\n", - "step: 2612, Red action: do-nothing, Blue reward:0.85\n", - "step: 2613, Red action: do-nothing, Blue reward:0.85\n", - "step: 2614, Red action: do-nothing, Blue reward:0.85\n", - "step: 2615, Red action: do-nothing, Blue reward:0.85\n", - "step: 2616, Red action: do-nothing, Blue reward:0.85\n", - "step: 2617, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2618, Red action: do-nothing, Blue reward:0.85\n", - "step: 2619, Red action: do-nothing, Blue reward:0.85\n", - "step: 2620, Red action: do-nothing, Blue reward:0.85\n", - "step: 2621, Red action: do-nothing, Blue reward:0.85\n", - "step: 2622, Red action: do-nothing, Blue reward:0.85\n", - "step: 2623, Red action: do-nothing, Blue reward:0.85\n", - "step: 2624, Red action: do-nothing, Blue reward:0.85\n", - "step: 2625, Red action: do-nothing, Blue reward:0.85\n", - "step: 2626, Red action: do-nothing, Blue reward:0.85\n", - "step: 2627, Red action: do-nothing, Blue reward:0.85\n", - "step: 2628, Red action: do-nothing, Blue reward:0.85\n", - "step: 2629, Red action: do-nothing, Blue reward:0.85\n", - "step: 2630, Red action: do-nothing, Blue reward:0.85\n", - "step: 2631, Red action: do-nothing, Blue reward:0.85\n", - "step: 2632, Red action: do-nothing, Blue reward:0.85\n", - "step: 2633, Red action: do-nothing, Blue reward:0.85\n", - "step: 2634, Red action: do-nothing, Blue reward:0.85\n", - "step: 2635, Red action: do-nothing, Blue reward:0.85\n", - "step: 2636, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2637, Red action: do-nothing, Blue reward:0.85\n", - "step: 2638, Red action: do-nothing, Blue reward:0.85\n", - "step: 2639, Red action: do-nothing, Blue reward:0.85\n", - "step: 2640, Red action: do-nothing, Blue reward:0.85\n", - "step: 2641, Red action: do-nothing, Blue reward:0.85\n", - "step: 2642, Red action: do-nothing, Blue reward:0.85\n", - "step: 2643, Red action: do-nothing, Blue reward:0.85\n", - "step: 2644, Red action: do-nothing, Blue reward:0.85\n", - "step: 2645, Red action: do-nothing, Blue reward:0.85\n", - "step: 2646, Red action: do-nothing, Blue reward:0.85\n", - "step: 2647, Red action: do-nothing, Blue reward:0.85\n", - "step: 2648, Red action: do-nothing, Blue reward:0.85\n", - "step: 2649, Red action: do-nothing, Blue reward:0.85\n", - "step: 2650, Red action: do-nothing, Blue reward:0.85\n", - "step: 2651, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2652, Red action: do-nothing, Blue reward:0.85\n", - "step: 2653, Red action: do-nothing, Blue reward:0.85\n", - "step: 2654, Red action: do-nothing, Blue reward:0.85\n", - "step: 2655, Red action: do-nothing, Blue reward:0.85\n", - "step: 2656, Red action: do-nothing, Blue reward:0.85\n", - "step: 2657, Red action: do-nothing, Blue reward:0.85\n", - "step: 2658, Red action: do-nothing, Blue reward:0.85\n", - "step: 2659, Red action: do-nothing, Blue reward:0.85\n", - "step: 2660, Red action: do-nothing, Blue reward:0.85\n", - "step: 2661, Red action: do-nothing, Blue reward:0.85\n", - "step: 2662, Red action: do-nothing, Blue reward:0.85\n", - "step: 2663, Red action: do-nothing, Blue reward:0.85\n", - "step: 2664, Red action: do-nothing, Blue reward:0.85\n", - "step: 2665, Red action: do-nothing, Blue reward:0.85\n", - "step: 2666, Red action: do-nothing, Blue reward:0.85\n", - "step: 2667, Red action: do-nothing, Blue reward:0.85\n", - "step: 2668, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2669, Red action: do-nothing, Blue reward:0.85\n", - "step: 2670, Red action: do-nothing, Blue reward:0.85\n", - "step: 2671, Red action: do-nothing, Blue reward:0.85\n", - "step: 2672, Red action: do-nothing, Blue reward:0.85\n", - "step: 2673, Red action: do-nothing, Blue reward:0.85\n", - "step: 2674, Red action: do-nothing, Blue reward:0.85\n", - "step: 2675, Red action: do-nothing, Blue reward:0.85\n", - "step: 2676, Red action: do-nothing, Blue reward:0.85\n", - "step: 2677, Red action: do-nothing, Blue reward:0.85\n", - "step: 2678, Red action: do-nothing, Blue reward:0.85\n", - "step: 2679, Red action: do-nothing, Blue reward:0.85\n", - "step: 2680, Red action: do-nothing, Blue reward:0.85\n", - "step: 2681, Red action: do-nothing, Blue reward:0.85\n", - "step: 2682, Red action: do-nothing, Blue reward:0.85\n", - "step: 2683, Red action: do-nothing, Blue reward:0.85\n", - "step: 2684, Red action: do-nothing, Blue reward:0.85\n", - "step: 2685, Red action: do-nothing, Blue reward:0.85\n", - "step: 2686, Red action: do-nothing, Blue reward:0.85\n", - "step: 2687, Red action: do-nothing, Blue reward:0.85\n", - "step: 2688, Red action: do-nothing, Blue reward:0.85\n", - "step: 2689, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2690, Red action: do-nothing, Blue reward:0.85\n", - "step: 2691, Red action: do-nothing, Blue reward:0.85\n", - "step: 2692, Red action: do-nothing, Blue reward:0.85\n", - "step: 2693, Red action: do-nothing, Blue reward:0.85\n", - "step: 2694, Red action: do-nothing, Blue reward:0.85\n", - "step: 2695, Red action: do-nothing, Blue reward:0.85\n", - "step: 2696, Red action: do-nothing, Blue reward:0.85\n", - "step: 2697, Red action: do-nothing, Blue reward:0.85\n", - "step: 2698, Red action: do-nothing, Blue reward:0.85\n", - "step: 2699, Red action: do-nothing, Blue reward:0.85\n", - "step: 2700, Red action: do-nothing, Blue reward:0.85\n", - "step: 2701, Red action: do-nothing, Blue reward:0.85\n", - "step: 2702, Red action: do-nothing, Blue reward:0.85\n", - "step: 2703, Red action: do-nothing, Blue reward:0.85\n", - "step: 2704, Red action: do-nothing, Blue reward:0.85\n", - "step: 2705, Red action: do-nothing, Blue reward:0.85\n", - "step: 2706, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2707, Red action: do-nothing, Blue reward:0.85\n", - "step: 2708, Red action: do-nothing, Blue reward:0.85\n", - "step: 2709, Red action: do-nothing, Blue reward:0.85\n", - "step: 2710, Red action: do-nothing, Blue reward:0.85\n", - "step: 2711, Red action: do-nothing, Blue reward:0.85\n", - "step: 2712, Red action: do-nothing, Blue reward:0.85\n", - "step: 2713, Red action: do-nothing, Blue reward:0.85\n", - "step: 2714, Red action: do-nothing, Blue reward:0.85\n", - "step: 2715, Red action: do-nothing, Blue reward:0.85\n", - "step: 2716, Red action: do-nothing, Blue reward:0.85\n", - "step: 2717, Red action: do-nothing, Blue reward:0.85\n", - "step: 2718, Red action: do-nothing, Blue reward:0.85\n", - "step: 2719, Red action: do-nothing, Blue reward:0.85\n", - "step: 2720, Red action: do-nothing, Blue reward:0.85\n", - "step: 2721, Red action: do-nothing, Blue reward:0.85\n", - "step: 2722, Red action: do-nothing, Blue reward:0.85\n", - "step: 2723, Red action: do-nothing, Blue reward:0.85\n", - "step: 2724, Red action: do-nothing, Blue reward:0.85\n", - "step: 2725, Red action: do-nothing, Blue reward:0.85\n", - "step: 2726, Red action: do-nothing, Blue reward:0.85\n", - "step: 2727, Red action: do-nothing, Blue reward:0.85\n", - "step: 2728, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2729, Red action: do-nothing, Blue reward:0.85\n", - "step: 2730, Red action: do-nothing, Blue reward:0.85\n", - "step: 2731, Red action: do-nothing, Blue reward:0.85\n", - "step: 2732, Red action: do-nothing, Blue reward:0.85\n", - "step: 2733, Red action: do-nothing, Blue reward:0.85\n", - "step: 2734, Red action: do-nothing, Blue reward:0.85\n", - "step: 2735, Red action: do-nothing, Blue reward:0.85\n", - "step: 2736, Red action: do-nothing, Blue reward:0.85\n", - "step: 2737, Red action: do-nothing, Blue reward:0.85\n", - "step: 2738, Red action: do-nothing, Blue reward:0.85\n", - "step: 2739, Red action: do-nothing, Blue reward:0.85\n", - "step: 2740, Red action: do-nothing, Blue reward:0.85\n", - "step: 2741, Red action: do-nothing, Blue reward:0.85\n", - "step: 2742, Red action: do-nothing, Blue reward:0.85\n", - "step: 2743, Red action: do-nothing, Blue reward:0.85\n", - "step: 2744, Red action: do-nothing, Blue reward:0.85\n", - "step: 2745, Red action: do-nothing, Blue reward:0.85\n", - "step: 2746, Red action: do-nothing, Blue reward:0.85\n", - "step: 2747, Red action: do-nothing, Blue reward:0.85\n", - "step: 2748, Red action: do-nothing, Blue reward:0.85\n", - "step: 2749, Red action: do-nothing, Blue reward:0.85\n", - "step: 2750, Red action: do-nothing, Blue reward:0.85\n", - "step: 2751, Red action: do-nothing, Blue reward:0.85\n", - "step: 2752, Red action: do-nothing, Blue reward:0.85\n", - "step: 2753, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2754, Red action: do-nothing, Blue reward:0.85\n", - "step: 2755, Red action: do-nothing, Blue reward:0.85\n", - "step: 2756, Red action: do-nothing, Blue reward:0.85\n", - "step: 2757, Red action: do-nothing, Blue reward:0.85\n", - "step: 2758, Red action: do-nothing, Blue reward:0.85\n", - "step: 2759, Red action: do-nothing, Blue reward:0.85\n", - "step: 2760, Red action: do-nothing, Blue reward:0.85\n", - "step: 2761, Red action: do-nothing, Blue reward:0.85\n", - "step: 2762, Red action: do-nothing, Blue reward:0.85\n", - "step: 2763, Red action: do-nothing, Blue reward:0.85\n", - "step: 2764, Red action: do-nothing, Blue reward:0.85\n", - "step: 2765, Red action: do-nothing, Blue reward:0.85\n", - "step: 2766, Red action: do-nothing, Blue reward:0.85\n", - "step: 2767, Red action: do-nothing, Blue reward:0.85\n", - "step: 2768, Red action: do-nothing, Blue reward:0.85\n", - "step: 2769, Red action: do-nothing, Blue reward:0.85\n", - "step: 2770, Red action: do-nothing, Blue reward:0.85\n", - "step: 2771, Red action: do-nothing, Blue reward:0.85\n", - "step: 2772, Red action: do-nothing, Blue reward:0.85\n", - "step: 2773, Red action: do-nothing, Blue reward:0.85\n", - "step: 2774, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2775, Red action: do-nothing, Blue reward:0.85\n", - "step: 2776, Red action: do-nothing, Blue reward:0.85\n", - "step: 2777, Red action: do-nothing, Blue reward:0.85\n", - "step: 2778, Red action: do-nothing, Blue reward:0.85\n", - "step: 2779, Red action: do-nothing, Blue reward:0.85\n", - "step: 2780, Red action: do-nothing, Blue reward:0.85\n", - "step: 2781, Red action: do-nothing, Blue reward:0.85\n", - "step: 2782, Red action: do-nothing, Blue reward:0.85\n", - "step: 2783, Red action: do-nothing, Blue reward:0.85\n", - "step: 2784, Red action: do-nothing, Blue reward:0.85\n", - "step: 2785, Red action: do-nothing, Blue reward:0.85\n", - "step: 2786, Red action: do-nothing, Blue reward:0.85\n", - "step: 2787, Red action: do-nothing, Blue reward:0.85\n", - "step: 2788, Red action: do-nothing, Blue reward:0.85\n", - "step: 2789, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2790, Red action: do-nothing, Blue reward:0.85\n", - "step: 2791, Red action: do-nothing, Blue reward:0.85\n", - "step: 2792, Red action: do-nothing, Blue reward:0.85\n", - "step: 2793, Red action: do-nothing, Blue reward:0.85\n", - "step: 2794, Red action: do-nothing, Blue reward:0.85\n", - "step: 2795, Red action: do-nothing, Blue reward:0.85\n", - "step: 2796, Red action: do-nothing, Blue reward:0.85\n", - "step: 2797, Red action: do-nothing, Blue reward:0.85\n", - "step: 2798, Red action: do-nothing, Blue reward:0.85\n", - "step: 2799, Red action: do-nothing, Blue reward:0.85\n", - "step: 2800, Red action: do-nothing, Blue reward:0.85\n", - "step: 2801, Red action: do-nothing, Blue reward:0.85\n", - "step: 2802, Red action: do-nothing, Blue reward:0.85\n", - "step: 2803, Red action: do-nothing, Blue reward:0.85\n", - "step: 2804, Red action: do-nothing, Blue reward:0.85\n", - "step: 2805, Red action: do-nothing, Blue reward:0.85\n", - "step: 2806, Red action: do-nothing, Blue reward:0.85\n", - "step: 2807, Red action: do-nothing, Blue reward:0.85\n", - "step: 2808, Red action: do-nothing, Blue reward:0.85\n", - "step: 2809, Red action: do-nothing, Blue reward:0.85\n", - "step: 2810, Red action: do-nothing, Blue reward:0.85\n", - "step: 2811, Red action: do-nothing, Blue reward:0.85\n", - "step: 2812, Red action: do-nothing, Blue reward:0.85\n", - "step: 2813, Red action: do-nothing, Blue reward:0.85\n", - "step: 2814, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2815, Red action: do-nothing, Blue reward:0.85\n", - "step: 2816, Red action: do-nothing, Blue reward:0.85\n", - "step: 2817, Red action: do-nothing, Blue reward:0.85\n", - "step: 2818, Red action: do-nothing, Blue reward:0.85\n", - "step: 2819, Red action: do-nothing, Blue reward:0.85\n", - "step: 2820, Red action: do-nothing, Blue reward:0.85\n", - "step: 2821, Red action: do-nothing, Blue reward:0.85\n", - "step: 2822, Red action: do-nothing, Blue reward:0.85\n", - "step: 2823, Red action: do-nothing, Blue reward:0.85\n", - "step: 2824, Red action: do-nothing, Blue reward:0.85\n", - "step: 2825, Red action: do-nothing, Blue reward:0.85\n", - "step: 2826, Red action: do-nothing, Blue reward:0.85\n", - "step: 2827, Red action: do-nothing, Blue reward:0.85\n", - "step: 2828, Red action: do-nothing, Blue reward:0.85\n", - "step: 2829, Red action: do-nothing, Blue reward:0.85\n", - "step: 2830, Red action: do-nothing, Blue reward:0.85\n", - "step: 2831, Red action: do-nothing, Blue reward:0.85\n", - "step: 2832, Red action: do-nothing, Blue reward:0.85\n", - "step: 2833, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2834, Red action: do-nothing, Blue reward:0.85\n", - "step: 2835, Red action: do-nothing, Blue reward:0.85\n", - "step: 2836, Red action: do-nothing, Blue reward:0.85\n", - "step: 2837, Red action: do-nothing, Blue reward:0.85\n", - "step: 2838, Red action: do-nothing, Blue reward:0.85\n", - "step: 2839, Red action: do-nothing, Blue reward:0.85\n", - "step: 2840, Red action: do-nothing, Blue reward:0.85\n", - "step: 2841, Red action: do-nothing, Blue reward:0.85\n", - "step: 2842, Red action: do-nothing, Blue reward:0.85\n", - "step: 2843, Red action: do-nothing, Blue reward:0.85\n", - "step: 2844, Red action: do-nothing, Blue reward:0.85\n", - "step: 2845, Red action: do-nothing, Blue reward:0.85\n", - "step: 2846, Red action: do-nothing, Blue reward:0.85\n", - "step: 2847, Red action: do-nothing, Blue reward:0.85\n", - "step: 2848, Red action: do-nothing, Blue reward:0.85\n", - "step: 2849, Red action: do-nothing, Blue reward:0.85\n", - "step: 2850, Red action: do-nothing, Blue reward:0.85\n", - "step: 2851, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2852, Red action: do-nothing, Blue reward:0.85\n", - "step: 2853, Red action: do-nothing, Blue reward:0.85\n", - "step: 2854, Red action: do-nothing, Blue reward:0.85\n", - "step: 2855, Red action: do-nothing, Blue reward:0.85\n", - "step: 2856, Red action: do-nothing, Blue reward:0.85\n", - "step: 2857, Red action: do-nothing, Blue reward:0.85\n", - "step: 2858, Red action: do-nothing, Blue reward:0.85\n", - "step: 2859, Red action: do-nothing, Blue reward:0.85\n", - "step: 2860, Red action: do-nothing, Blue reward:0.85\n", - "step: 2861, Red action: do-nothing, Blue reward:0.85\n", - "step: 2862, Red action: do-nothing, Blue reward:0.85\n", - "step: 2863, Red action: do-nothing, Blue reward:0.85\n", - "step: 2864, Red action: do-nothing, Blue reward:0.85\n", - "step: 2865, Red action: do-nothing, Blue reward:0.85\n", - "step: 2866, Red action: do-nothing, Blue reward:0.85\n", - "step: 2867, Red action: do-nothing, Blue reward:0.85\n", - "step: 2868, Red action: do-nothing, Blue reward:0.85\n", - "step: 2869, Red action: do-nothing, Blue reward:0.85\n", - "step: 2870, Red action: do-nothing, Blue reward:0.85\n", - "step: 2871, Red action: do-nothing, Blue reward:0.85\n", - "step: 2872, Red action: do-nothing, Blue reward:0.85\n", - "step: 2873, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2874, Red action: do-nothing, Blue reward:0.85\n", - "step: 2875, Red action: do-nothing, Blue reward:0.85\n", - "step: 2876, Red action: do-nothing, Blue reward:0.85\n", - "step: 2877, Red action: do-nothing, Blue reward:0.85\n", - "step: 2878, Red action: do-nothing, Blue reward:0.85\n", - "step: 2879, Red action: do-nothing, Blue reward:0.85\n", - "step: 2880, Red action: do-nothing, Blue reward:0.85\n", - "step: 2881, Red action: do-nothing, Blue reward:0.85\n", - "step: 2882, Red action: do-nothing, Blue reward:0.85\n", - "step: 2883, Red action: do-nothing, Blue reward:0.85\n", - "step: 2884, Red action: do-nothing, Blue reward:0.85\n", - "step: 2885, Red action: do-nothing, Blue reward:0.85\n", - "step: 2886, Red action: do-nothing, Blue reward:0.85\n", - "step: 2887, Red action: do-nothing, Blue reward:0.85\n", - "step: 2888, Red action: do-nothing, Blue reward:0.85\n", - "step: 2889, Red action: do-nothing, Blue reward:0.85\n", - "step: 2890, Red action: do-nothing, Blue reward:0.85\n", - "step: 2891, Red action: do-nothing, Blue reward:0.85\n", - "step: 2892, Red action: do-nothing, Blue reward:0.85\n", - "step: 2893, Red action: do-nothing, Blue reward:0.85\n", - "step: 2894, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2895, Red action: do-nothing, Blue reward:0.85\n", - "step: 2896, Red action: do-nothing, Blue reward:0.85\n", - "step: 2897, Red action: do-nothing, Blue reward:0.85\n", - "step: 2898, Red action: do-nothing, Blue reward:0.85\n", - "step: 2899, Red action: do-nothing, Blue reward:0.85\n", - "step: 2900, Red action: do-nothing, Blue reward:0.85\n", - "step: 2901, Red action: do-nothing, Blue reward:0.85\n", - "step: 2902, Red action: do-nothing, Blue reward:0.85\n", - "step: 2903, Red action: do-nothing, Blue reward:0.85\n", - "step: 2904, Red action: do-nothing, Blue reward:0.85\n", - "step: 2905, Red action: do-nothing, Blue reward:0.85\n", - "step: 2906, Red action: do-nothing, Blue reward:0.85\n", - "step: 2907, Red action: do-nothing, Blue reward:0.85\n", - "step: 2908, Red action: do-nothing, Blue reward:0.85\n", - "step: 2909, Red action: do-nothing, Blue reward:0.85\n", - "step: 2910, Red action: do-nothing, Blue reward:0.85\n", - "step: 2911, Red action: do-nothing, Blue reward:0.85\n", - "step: 2912, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2913, Red action: do-nothing, Blue reward:0.85\n", - "step: 2914, Red action: do-nothing, Blue reward:0.85\n", - "step: 2915, Red action: do-nothing, Blue reward:0.85\n", - "step: 2916, Red action: do-nothing, Blue reward:0.85\n", - "step: 2917, Red action: do-nothing, Blue reward:0.85\n", - "step: 2918, Red action: do-nothing, Blue reward:0.85\n", - "step: 2919, Red action: do-nothing, Blue reward:0.85\n", - "step: 2920, Red action: do-nothing, Blue reward:0.85\n", - "step: 2921, Red action: do-nothing, Blue reward:0.85\n", - "step: 2922, Red action: do-nothing, Blue reward:0.85\n", - "step: 2923, Red action: do-nothing, Blue reward:0.85\n", - "step: 2924, Red action: do-nothing, Blue reward:0.85\n", - "step: 2925, Red action: do-nothing, Blue reward:0.85\n", - "step: 2926, Red action: do-nothing, Blue reward:0.85\n", - "step: 2927, Red action: do-nothing, Blue reward:0.85\n", - "step: 2928, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2929, Red action: do-nothing, Blue reward:0.85\n", - "step: 2930, Red action: do-nothing, Blue reward:0.85\n", - "step: 2931, Red action: do-nothing, Blue reward:0.85\n", - "step: 2932, Red action: do-nothing, Blue reward:0.85\n", - "step: 2933, Red action: do-nothing, Blue reward:0.85\n", - "step: 2934, Red action: do-nothing, Blue reward:0.85\n", - "step: 2935, Red action: do-nothing, Blue reward:0.85\n", - "step: 2936, Red action: do-nothing, Blue reward:0.85\n", - "step: 2937, Red action: do-nothing, Blue reward:0.85\n", - "step: 2938, Red action: do-nothing, Blue reward:0.85\n", - "step: 2939, Red action: do-nothing, Blue reward:0.85\n", - "step: 2940, Red action: do-nothing, Blue reward:0.85\n", - "step: 2941, Red action: do-nothing, Blue reward:0.85\n", - "step: 2942, Red action: do-nothing, Blue reward:0.85\n", - "step: 2943, Red action: do-nothing, Blue reward:0.85\n", - "step: 2944, Red action: do-nothing, Blue reward:0.85\n", - "step: 2945, Red action: do-nothing, Blue reward:0.85\n", - "step: 2946, Red action: do-nothing, Blue reward:0.85\n", - "step: 2947, Red action: do-nothing, Blue reward:0.85\n", - "step: 2948, Red action: do-nothing, Blue reward:0.85\n", - "step: 2949, Red action: do-nothing, Blue reward:0.85\n", - "step: 2950, Red action: do-nothing, Blue reward:0.85\n", - "step: 2951, Red action: do-nothing, Blue reward:0.85\n", - "step: 2952, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2953, Red action: do-nothing, Blue reward:0.85\n", - "step: 2954, Red action: do-nothing, Blue reward:0.85\n", - "step: 2955, Red action: do-nothing, Blue reward:0.85\n", - "step: 2956, Red action: do-nothing, Blue reward:0.85\n", - "step: 2957, Red action: do-nothing, Blue reward:0.85\n", - "step: 2958, Red action: do-nothing, Blue reward:0.85\n", - "step: 2959, Red action: do-nothing, Blue reward:0.85\n", - "step: 2960, Red action: do-nothing, Blue reward:0.85\n", - "step: 2961, Red action: do-nothing, Blue reward:0.85\n", - "step: 2962, Red action: do-nothing, Blue reward:0.85\n", - "step: 2963, Red action: do-nothing, Blue reward:0.85\n", - "step: 2964, Red action: do-nothing, Blue reward:0.85\n", - "step: 2965, Red action: do-nothing, Blue reward:0.85\n", - "step: 2966, Red action: do-nothing, Blue reward:0.85\n", - "step: 2967, Red action: do-nothing, Blue reward:0.85\n", - "step: 2968, Red action: do-nothing, Blue reward:0.85\n", - "step: 2969, Red action: do-nothing, Blue reward:0.85\n", - "step: 2970, Red action: do-nothing, Blue reward:0.85\n", - "step: 2971, Red action: do-nothing, Blue reward:0.85\n", - "step: 2972, Red action: do-nothing, Blue reward:0.85\n", - "step: 2973, Red action: do-nothing, Blue reward:0.85\n", - "step: 2974, Red action: do-nothing, Blue reward:0.85\n", - "step: 2975, Red action: do-nothing, Blue reward:0.85\n", - "step: 2976, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2977, Red action: do-nothing, Blue reward:0.85\n", - "step: 2978, Red action: do-nothing, Blue reward:0.85\n", - "step: 2979, Red action: do-nothing, Blue reward:0.85\n", - "step: 2980, Red action: do-nothing, Blue reward:0.85\n", - "step: 2981, Red action: do-nothing, Blue reward:0.85\n", - "step: 2982, Red action: do-nothing, Blue reward:0.85\n", - "step: 2983, Red action: do-nothing, Blue reward:0.85\n", - "step: 2984, Red action: do-nothing, Blue reward:0.85\n", - "step: 2985, Red action: do-nothing, Blue reward:0.85\n", - "step: 2986, Red action: do-nothing, Blue reward:0.85\n", - "step: 2987, Red action: do-nothing, Blue reward:0.85\n", - "step: 2988, Red action: do-nothing, Blue reward:0.85\n", - "step: 2989, Red action: do-nothing, Blue reward:0.85\n", - "step: 2990, Red action: do-nothing, Blue reward:0.85\n", - "step: 2991, Red action: do-nothing, Blue reward:0.85\n", - "step: 2992, Red action: do-nothing, Blue reward:0.85\n", - "step: 2993, Red action: do-nothing, Blue reward:0.85\n", - "step: 2994, Red action: do-nothing, Blue reward:0.85\n", - "step: 2995, Red action: do-nothing, Blue reward:0.85\n", - "step: 2996, Red action: do-nothing, Blue reward:0.85\n", - "step: 2997, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2998, Red action: do-nothing, Blue reward:0.85\n", - "step: 2999, Red action: do-nothing, Blue reward:0.85\n", - "step: 3000, Red action: do-nothing, Blue reward:0.85\n", - "step: 3001, Red action: do-nothing, Blue reward:0.85\n", - "step: 3002, Red action: do-nothing, Blue reward:0.85\n", - "step: 3003, Red action: do-nothing, Blue reward:0.85\n", - "step: 3004, Red action: do-nothing, Blue reward:0.85\n", - "step: 3005, Red action: do-nothing, Blue reward:0.85\n", - "step: 3006, Red action: do-nothing, Blue reward:0.85\n", - "step: 3007, Red action: do-nothing, Blue reward:0.85\n", - "step: 3008, Red action: do-nothing, Blue reward:0.85\n", - "step: 3009, Red action: do-nothing, Blue reward:0.85\n", - "step: 3010, Red action: do-nothing, Blue reward:0.85\n", - "step: 3011, Red action: do-nothing, Blue reward:0.85\n", - "step: 3012, Red action: do-nothing, Blue reward:0.85\n", - "step: 3013, Red action: do-nothing, Blue reward:0.85\n", - "step: 3014, Red action: do-nothing, Blue reward:0.85\n", - "step: 3015, Red action: do-nothing, Blue reward:0.85\n", - "step: 3016, Red action: do-nothing, Blue reward:0.85\n", - "step: 3017, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3018, Red action: do-nothing, Blue reward:0.85\n", - "step: 3019, Red action: do-nothing, Blue reward:0.85\n", - "step: 3020, Red action: do-nothing, Blue reward:0.85\n", - "step: 3021, Red action: do-nothing, Blue reward:0.85\n", - "step: 3022, Red action: do-nothing, Blue reward:0.85\n", - "step: 3023, Red action: do-nothing, Blue reward:0.85\n", - "step: 3024, Red action: do-nothing, Blue reward:0.85\n", - "step: 3025, Red action: do-nothing, Blue reward:0.85\n", - "step: 3026, Red action: do-nothing, Blue reward:0.85\n", - "step: 3027, Red action: do-nothing, Blue reward:0.85\n", - "step: 3028, Red action: do-nothing, Blue reward:0.85\n", - "step: 3029, Red action: do-nothing, Blue reward:0.85\n", - "step: 3030, Red action: do-nothing, Blue reward:0.85\n", - "step: 3031, Red action: do-nothing, Blue reward:0.85\n", - "step: 3032, Red action: do-nothing, Blue reward:0.85\n", - "step: 3033, Red action: do-nothing, Blue reward:0.85\n", - "step: 3034, Red action: do-nothing, Blue reward:0.85\n", - "step: 3035, Red action: do-nothing, Blue reward:0.85\n", - "step: 3036, Red action: do-nothing, Blue reward:0.85\n", - "step: 3037, Red action: do-nothing, Blue reward:0.85\n", - "step: 3038, Red action: do-nothing, Blue reward:0.85\n", - "step: 3039, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3040, Red action: do-nothing, Blue reward:0.85\n", - "step: 3041, Red action: do-nothing, Blue reward:0.85\n", - "step: 3042, Red action: do-nothing, Blue reward:0.85\n", - "step: 3043, Red action: do-nothing, Blue reward:0.85\n", - "step: 3044, Red action: do-nothing, Blue reward:0.85\n", - "step: 3045, Red action: do-nothing, Blue reward:0.85\n", - "step: 3046, Red action: do-nothing, Blue reward:0.85\n", - "step: 3047, Red action: do-nothing, Blue reward:0.85\n", - "step: 3048, Red action: do-nothing, Blue reward:0.85\n", - "step: 3049, Red action: do-nothing, Blue reward:0.85\n", - "step: 3050, Red action: do-nothing, Blue reward:0.85\n", - "step: 3051, Red action: do-nothing, Blue reward:0.85\n", - "step: 3052, Red action: do-nothing, Blue reward:0.85\n", - "step: 3053, Red action: do-nothing, Blue reward:0.85\n", - "step: 3054, Red action: do-nothing, Blue reward:0.85\n", - "step: 3055, Red action: do-nothing, Blue reward:0.85\n", - "step: 3056, Red action: do-nothing, Blue reward:0.85\n", - "step: 3057, Red action: do-nothing, Blue reward:0.85\n", - "step: 3058, Red action: do-nothing, Blue reward:0.85\n", - "step: 3059, Red action: do-nothing, Blue reward:0.85\n", - "step: 3060, Red action: do-nothing, Blue reward:0.85\n", - "step: 3061, Red action: do-nothing, Blue reward:0.85\n", - "step: 3062, Red action: do-nothing, Blue reward:0.85\n", - "step: 3063, Red action: do-nothing, Blue reward:0.85\n", - "step: 3064, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3065, Red action: do-nothing, Blue reward:0.85\n", - "step: 3066, Red action: do-nothing, Blue reward:0.85\n", - "step: 3067, Red action: do-nothing, Blue reward:0.85\n", - "step: 3068, Red action: do-nothing, Blue reward:0.85\n", - "step: 3069, Red action: do-nothing, Blue reward:0.85\n", - "step: 3070, Red action: do-nothing, Blue reward:0.85\n", - "step: 3071, Red action: do-nothing, Blue reward:0.85\n", - "step: 3072, Red action: do-nothing, Blue reward:0.85\n", - "step: 3073, Red action: do-nothing, Blue reward:0.85\n", - "step: 3074, Red action: do-nothing, Blue reward:0.85\n", - "step: 3075, Red action: do-nothing, Blue reward:0.85\n", - "step: 3076, Red action: do-nothing, Blue reward:0.85\n", - "step: 3077, Red action: do-nothing, Blue reward:0.85\n", - "step: 3078, Red action: do-nothing, Blue reward:0.85\n", - "step: 3079, Red action: do-nothing, Blue reward:0.85\n", - "step: 3080, Red action: do-nothing, Blue reward:0.85\n", - "step: 3081, Red action: do-nothing, Blue reward:0.85\n", - "step: 3082, Red action: do-nothing, Blue reward:0.85\n", - "step: 3083, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3084, Red action: do-nothing, Blue reward:0.85\n", - "step: 3085, Red action: do-nothing, Blue reward:0.85\n", - "step: 3086, Red action: do-nothing, Blue reward:0.85\n", - "step: 3087, Red action: do-nothing, Blue reward:0.85\n", - "step: 3088, Red action: do-nothing, Blue reward:0.85\n", - "step: 3089, Red action: do-nothing, Blue reward:0.85\n", - "step: 3090, Red action: do-nothing, Blue reward:0.85\n", - "step: 3091, Red action: do-nothing, Blue reward:0.85\n", - "step: 3092, Red action: do-nothing, Blue reward:0.85\n", - "step: 3093, Red action: do-nothing, Blue reward:0.85\n", - "step: 3094, Red action: do-nothing, Blue reward:0.85\n", - "step: 3095, Red action: do-nothing, Blue reward:0.85\n", - "step: 3096, Red action: do-nothing, Blue reward:0.85\n", - "step: 3097, Red action: do-nothing, Blue reward:0.85\n", - "step: 3098, Red action: do-nothing, Blue reward:0.85\n", - "step: 3099, Red action: do-nothing, Blue reward:0.85\n", - "step: 3100, Red action: do-nothing, Blue reward:0.85\n", - "step: 3101, Red action: do-nothing, Blue reward:0.85\n", - "step: 3102, Red action: do-nothing, Blue reward:0.85\n", - "step: 3103, Red action: do-nothing, Blue reward:0.85\n", - "step: 3104, Red action: do-nothing, Blue reward:0.85\n", - "step: 3105, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3106, Red action: do-nothing, Blue reward:0.85\n", - "step: 3107, Red action: do-nothing, Blue reward:0.85\n", - "step: 3108, Red action: do-nothing, Blue reward:0.85\n", - "step: 3109, Red action: do-nothing, Blue reward:0.85\n", - "step: 3110, Red action: do-nothing, Blue reward:0.85\n", - "step: 3111, Red action: do-nothing, Blue reward:0.85\n", - "step: 3112, Red action: do-nothing, Blue reward:0.85\n", - "step: 3113, Red action: do-nothing, Blue reward:0.85\n", - "step: 3114, Red action: do-nothing, Blue reward:0.85\n", - "step: 3115, Red action: do-nothing, Blue reward:0.85\n", - "step: 3116, Red action: do-nothing, Blue reward:0.85\n", - "step: 3117, Red action: do-nothing, Blue reward:0.85\n", - "step: 3118, Red action: do-nothing, Blue reward:0.85\n", - "step: 3119, Red action: do-nothing, Blue reward:0.85\n", - "step: 3120, Red action: do-nothing, Blue reward:0.85\n", - "step: 3121, Red action: do-nothing, Blue reward:0.85\n", - "step: 3122, Red action: do-nothing, Blue reward:0.85\n", - "step: 3123, Red action: do-nothing, Blue reward:0.85\n", - "step: 3124, Red action: do-nothing, Blue reward:0.85\n", - "step: 3125, Red action: do-nothing, Blue reward:0.85\n", - "step: 3126, Red action: do-nothing, Blue reward:0.85\n", - "step: 3127, Red action: do-nothing, Blue reward:0.85\n", - "step: 3128, Red action: do-nothing, Blue reward:0.85\n", - "step: 3129, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3130, Red action: do-nothing, Blue reward:0.85\n", - "step: 3131, Red action: do-nothing, Blue reward:0.85\n", - "step: 3132, Red action: do-nothing, Blue reward:0.85\n", - "step: 3133, Red action: do-nothing, Blue reward:0.85\n", - "step: 3134, Red action: do-nothing, Blue reward:0.85\n", - "step: 3135, Red action: do-nothing, Blue reward:0.85\n", - "step: 3136, Red action: do-nothing, Blue reward:0.85\n", - "step: 3137, Red action: do-nothing, Blue reward:0.85\n", - "step: 3138, Red action: do-nothing, Blue reward:0.85\n", - "step: 3139, Red action: do-nothing, Blue reward:0.85\n", - "step: 3140, Red action: do-nothing, Blue reward:0.85\n", - "step: 3141, Red action: do-nothing, Blue reward:0.85\n", - "step: 3142, Red action: do-nothing, Blue reward:0.85\n", - "step: 3143, Red action: do-nothing, Blue reward:0.85\n", - "step: 3144, Red action: do-nothing, Blue reward:0.85\n", - "step: 3145, Red action: do-nothing, Blue reward:0.85\n", - "step: 3146, Red action: do-nothing, Blue reward:0.85\n", - "step: 3147, Red action: do-nothing, Blue reward:0.85\n", - "step: 3148, Red action: do-nothing, Blue reward:0.85\n", - "step: 3149, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3150, Red action: do-nothing, Blue reward:0.85\n", - "step: 3151, Red action: do-nothing, Blue reward:0.85\n", - "step: 3152, Red action: do-nothing, Blue reward:0.85\n", - "step: 3153, Red action: do-nothing, Blue reward:0.85\n", - "step: 3154, Red action: do-nothing, Blue reward:0.85\n", - "step: 3155, Red action: do-nothing, Blue reward:0.85\n", - "step: 3156, Red action: do-nothing, Blue reward:0.85\n", - "step: 3157, Red action: do-nothing, Blue reward:0.85\n", - "step: 3158, Red action: do-nothing, Blue reward:0.85\n", - "step: 3159, Red action: do-nothing, Blue reward:0.85\n", - "step: 3160, Red action: do-nothing, Blue reward:0.85\n", - "step: 3161, Red action: do-nothing, Blue reward:0.85\n", - "step: 3162, Red action: do-nothing, Blue reward:0.85\n", - "step: 3163, Red action: do-nothing, Blue reward:0.85\n", - "step: 3164, Red action: do-nothing, Blue reward:0.85\n", - "step: 3165, Red action: do-nothing, Blue reward:0.85\n", - "step: 3166, Red action: do-nothing, Blue reward:0.85\n", - "step: 3167, Red action: do-nothing, Blue reward:0.85\n", - "step: 3168, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3169, Red action: do-nothing, Blue reward:0.85\n", - "step: 3170, Red action: do-nothing, Blue reward:0.85\n", - "step: 3171, Red action: do-nothing, Blue reward:0.85\n", - "step: 3172, Red action: do-nothing, Blue reward:0.85\n", - "step: 3173, Red action: do-nothing, Blue reward:0.85\n", - "step: 3174, Red action: do-nothing, Blue reward:0.85\n", - "step: 3175, Red action: do-nothing, Blue reward:0.85\n", - "step: 3176, Red action: do-nothing, Blue reward:0.85\n", - "step: 3177, Red action: do-nothing, Blue reward:0.85\n", - "step: 3178, Red action: do-nothing, Blue reward:0.85\n", - "step: 3179, Red action: do-nothing, Blue reward:0.85\n", - "step: 3180, Red action: do-nothing, Blue reward:0.85\n", - "step: 3181, Red action: do-nothing, Blue reward:0.85\n", - "step: 3182, Red action: do-nothing, Blue reward:0.85\n", - "step: 3183, Red action: do-nothing, Blue reward:0.85\n", - "step: 3184, Red action: do-nothing, Blue reward:0.85\n", - "step: 3185, Red action: do-nothing, Blue reward:0.85\n", - "step: 3186, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3187, Red action: do-nothing, Blue reward:0.85\n", - "step: 3188, Red action: do-nothing, Blue reward:0.85\n", - "step: 3189, Red action: do-nothing, Blue reward:0.85\n", - "step: 3190, Red action: do-nothing, Blue reward:0.85\n", - "step: 3191, Red action: do-nothing, Blue reward:0.85\n", - "step: 3192, Red action: do-nothing, Blue reward:0.85\n", - "step: 3193, Red action: do-nothing, Blue reward:0.85\n", - "step: 3194, Red action: do-nothing, Blue reward:0.85\n", - "step: 3195, Red action: do-nothing, Blue reward:0.85\n", - "step: 3196, Red action: do-nothing, Blue reward:0.85\n", - "step: 3197, Red action: do-nothing, Blue reward:0.85\n", - "step: 3198, Red action: do-nothing, Blue reward:0.85\n", - "step: 3199, Red action: do-nothing, Blue reward:0.85\n", - "step: 3200, Red action: do-nothing, Blue reward:0.85\n", - "step: 3201, Red action: do-nothing, Blue reward:0.85\n", - "step: 3202, Red action: do-nothing, Blue reward:0.85\n", - "step: 3203, Red action: do-nothing, Blue reward:0.85\n", - "step: 3204, Red action: do-nothing, Blue reward:0.85\n", - "step: 3205, Red action: do-nothing, Blue reward:0.85\n", - "step: 3206, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3207, Red action: do-nothing, Blue reward:0.85\n", - "step: 3208, Red action: do-nothing, Blue reward:0.85\n", - "step: 3209, Red action: do-nothing, Blue reward:0.85\n", - "step: 3210, Red action: do-nothing, Blue reward:0.85\n", - "step: 3211, Red action: do-nothing, Blue reward:0.85\n", - "step: 3212, Red action: do-nothing, Blue reward:0.85\n", - "step: 3213, Red action: do-nothing, Blue reward:0.85\n", - "step: 3214, Red action: do-nothing, Blue reward:0.85\n", - "step: 3215, Red action: do-nothing, Blue reward:0.85\n", - "step: 3216, Red action: do-nothing, Blue reward:0.85\n", - "step: 3217, Red action: do-nothing, Blue reward:0.85\n", - "step: 3218, Red action: do-nothing, Blue reward:0.85\n", - "step: 3219, Red action: do-nothing, Blue reward:0.85\n", - "step: 3220, Red action: do-nothing, Blue reward:0.85\n", - "step: 3221, Red action: do-nothing, Blue reward:0.85\n", - "step: 3222, Red action: do-nothing, Blue reward:0.85\n", - "step: 3223, Red action: do-nothing, Blue reward:0.85\n", - "step: 3224, Red action: do-nothing, Blue reward:0.85\n", - "step: 3225, Red action: do-nothing, Blue reward:0.85\n", - "step: 3226, Red action: do-nothing, Blue reward:0.85\n", - "step: 3227, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3228, Red action: do-nothing, Blue reward:0.85\n", - "step: 3229, Red action: do-nothing, Blue reward:0.85\n", - "step: 3230, Red action: do-nothing, Blue reward:0.85\n", - "step: 3231, Red action: do-nothing, Blue reward:0.85\n", - "step: 3232, Red action: do-nothing, Blue reward:0.85\n", - "step: 3233, Red action: do-nothing, Blue reward:0.85\n", - "step: 3234, Red action: do-nothing, Blue reward:0.85\n", - "step: 3235, Red action: do-nothing, Blue reward:0.85\n", - "step: 3236, Red action: do-nothing, Blue reward:0.85\n", - "step: 3237, Red action: do-nothing, Blue reward:0.85\n", - "step: 3238, Red action: do-nothing, Blue reward:0.85\n", - "step: 3239, Red action: do-nothing, Blue reward:0.85\n", - "step: 3240, Red action: do-nothing, Blue reward:0.85\n", - "step: 3241, Red action: do-nothing, Blue reward:0.85\n", - "step: 3242, Red action: do-nothing, Blue reward:0.85\n", - "step: 3243, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3244, Red action: do-nothing, Blue reward:0.85\n", - "step: 3245, Red action: do-nothing, Blue reward:0.85\n", - "step: 3246, Red action: do-nothing, Blue reward:0.85\n", - "step: 3247, Red action: do-nothing, Blue reward:0.85\n", - "step: 3248, Red action: do-nothing, Blue reward:0.85\n", - "step: 3249, Red action: do-nothing, Blue reward:0.85\n", - "step: 3250, Red action: do-nothing, Blue reward:0.85\n", - "step: 3251, Red action: do-nothing, Blue reward:0.85\n", - "step: 3252, Red action: do-nothing, Blue reward:0.85\n", - "step: 3253, Red action: do-nothing, Blue reward:0.85\n", - "step: 3254, Red action: do-nothing, Blue reward:0.85\n", - "step: 3255, Red action: do-nothing, Blue reward:0.85\n", - "step: 3256, Red action: do-nothing, Blue reward:0.85\n", - "step: 3257, Red action: do-nothing, Blue reward:0.85\n", - "step: 3258, Red action: do-nothing, Blue reward:0.85\n", - "step: 3259, Red action: do-nothing, Blue reward:0.85\n", - "step: 3260, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3261, Red action: do-nothing, Blue reward:0.85\n", - "step: 3262, Red action: do-nothing, Blue reward:0.85\n", - "step: 3263, Red action: do-nothing, Blue reward:0.85\n", - "step: 3264, Red action: do-nothing, Blue reward:0.85\n", - "step: 3265, Red action: do-nothing, Blue reward:0.85\n", - "step: 3266, Red action: do-nothing, Blue reward:0.85\n", - "step: 3267, Red action: do-nothing, Blue reward:0.85\n", - "step: 3268, Red action: do-nothing, Blue reward:0.85\n", - "step: 3269, Red action: do-nothing, Blue reward:0.85\n", - "step: 3270, Red action: do-nothing, Blue reward:0.85\n", - "step: 3271, Red action: do-nothing, Blue reward:0.85\n", - "step: 3272, Red action: do-nothing, Blue reward:0.85\n", - "step: 3273, Red action: do-nothing, Blue reward:0.85\n", - "step: 3274, Red action: do-nothing, Blue reward:0.85\n", - "step: 3275, Red action: do-nothing, Blue reward:0.85\n", - "step: 3276, Red action: do-nothing, Blue reward:0.85\n", - "step: 3277, Red action: do-nothing, Blue reward:0.85\n", - "step: 3278, Red action: do-nothing, Blue reward:0.85\n", - "step: 3279, Red action: do-nothing, Blue reward:0.85\n", - "step: 3280, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3281, Red action: do-nothing, Blue reward:0.85\n", - "step: 3282, Red action: do-nothing, Blue reward:0.85\n", - "step: 3283, Red action: do-nothing, Blue reward:0.85\n", - "step: 3284, Red action: do-nothing, Blue reward:0.85\n", - "step: 3285, Red action: do-nothing, Blue reward:0.85\n", - "step: 3286, Red action: do-nothing, Blue reward:0.85\n", - "step: 3287, Red action: do-nothing, Blue reward:0.85\n", - "step: 3288, Red action: do-nothing, Blue reward:0.85\n", - "step: 3289, Red action: do-nothing, Blue reward:0.85\n", - "step: 3290, Red action: do-nothing, Blue reward:0.85\n", - "step: 3291, Red action: do-nothing, Blue reward:0.85\n", - "step: 3292, Red action: do-nothing, Blue reward:0.85\n", - "step: 3293, Red action: do-nothing, Blue reward:0.85\n", - "step: 3294, Red action: do-nothing, Blue reward:0.85\n", - "step: 3295, Red action: do-nothing, Blue reward:0.85\n", - "step: 3296, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3297, Red action: do-nothing, Blue reward:0.85\n", - "step: 3298, Red action: do-nothing, Blue reward:0.85\n", - "step: 3299, Red action: do-nothing, Blue reward:0.85\n", - "step: 3300, Red action: do-nothing, Blue reward:0.85\n", - "step: 3301, Red action: do-nothing, Blue reward:0.85\n", - "step: 3302, Red action: do-nothing, Blue reward:0.85\n", - "step: 3303, Red action: do-nothing, Blue reward:0.85\n", - "step: 3304, Red action: do-nothing, Blue reward:0.85\n", - "step: 3305, Red action: do-nothing, Blue reward:0.85\n", - "step: 3306, Red action: do-nothing, Blue reward:0.85\n", - "step: 3307, Red action: do-nothing, Blue reward:0.85\n", - "step: 3308, Red action: do-nothing, Blue reward:0.85\n", - "step: 3309, Red action: do-nothing, Blue reward:0.85\n", - "step: 3310, Red action: do-nothing, Blue reward:0.85\n", - "step: 3311, Red action: do-nothing, Blue reward:0.85\n", - "step: 3312, Red action: do-nothing, Blue reward:0.85\n", - "step: 3313, Red action: do-nothing, Blue reward:0.85\n", - "step: 3314, Red action: do-nothing, Blue reward:0.85\n", - "step: 3315, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3316, Red action: do-nothing, Blue reward:0.85\n", - "step: 3317, Red action: do-nothing, Blue reward:0.85\n", - "step: 3318, Red action: do-nothing, Blue reward:0.85\n", - "step: 3319, Red action: do-nothing, Blue reward:0.85\n", - "step: 3320, Red action: do-nothing, Blue reward:0.85\n", - "step: 3321, Red action: do-nothing, Blue reward:0.85\n", - "step: 3322, Red action: do-nothing, Blue reward:0.85\n", - "step: 3323, Red action: do-nothing, Blue reward:0.85\n", - "step: 3324, Red action: do-nothing, Blue reward:0.85\n", - "step: 3325, Red action: do-nothing, Blue reward:0.85\n", - "step: 3326, Red action: do-nothing, Blue reward:0.85\n", - "step: 3327, Red action: do-nothing, Blue reward:0.85\n", - "step: 3328, Red action: do-nothing, Blue reward:0.85\n", - "step: 3329, Red action: do-nothing, Blue reward:0.85\n", - "step: 3330, Red action: do-nothing, Blue reward:0.85\n", - "step: 3331, Red action: do-nothing, Blue reward:0.85\n", - "step: 3332, Red action: do-nothing, Blue reward:0.85\n", - "step: 3333, Red action: do-nothing, Blue reward:0.85\n", - "step: 3334, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3335, Red action: do-nothing, Blue reward:0.85\n", - "step: 3336, Red action: do-nothing, Blue reward:0.85\n", - "step: 3337, Red action: do-nothing, Blue reward:0.85\n", - "step: 3338, Red action: do-nothing, Blue reward:0.85\n", - "step: 3339, Red action: do-nothing, Blue reward:0.85\n", - "step: 3340, Red action: do-nothing, Blue reward:0.85\n", - "step: 3341, Red action: do-nothing, Blue reward:0.85\n", - "step: 3342, Red action: do-nothing, Blue reward:0.85\n", - "step: 3343, Red action: do-nothing, Blue reward:0.85\n", - "step: 3344, Red action: do-nothing, Blue reward:0.85\n", - "step: 3345, Red action: do-nothing, Blue reward:0.85\n", - "step: 3346, Red action: do-nothing, Blue reward:0.85\n", - "step: 3347, Red action: do-nothing, Blue reward:0.85\n", - "step: 3348, Red action: do-nothing, Blue reward:0.85\n", - "step: 3349, Red action: do-nothing, Blue reward:0.85\n", - "step: 3350, Red action: do-nothing, Blue reward:0.85\n", - "step: 3351, Red action: do-nothing, Blue reward:0.85\n", - "step: 3352, Red action: do-nothing, Blue reward:0.85\n", - "step: 3353, Red action: do-nothing, Blue reward:0.85\n", - "step: 3354, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3355, Red action: do-nothing, Blue reward:0.85\n", - "step: 3356, Red action: do-nothing, Blue reward:0.85\n", - "step: 3357, Red action: do-nothing, Blue reward:0.85\n", - "step: 3358, Red action: do-nothing, Blue reward:0.85\n", - "step: 3359, Red action: do-nothing, Blue reward:0.85\n", - "step: 3360, Red action: do-nothing, Blue reward:0.85\n", - "step: 3361, Red action: do-nothing, Blue reward:0.85\n", - "step: 3362, Red action: do-nothing, Blue reward:0.85\n", - "step: 3363, Red action: do-nothing, Blue reward:0.85\n", - "step: 3364, Red action: do-nothing, Blue reward:0.85\n", - "step: 3365, Red action: do-nothing, Blue reward:0.85\n", - "step: 3366, Red action: do-nothing, Blue reward:0.85\n", - "step: 3367, Red action: do-nothing, Blue reward:0.85\n", - "step: 3368, Red action: do-nothing, Blue reward:0.85\n", - "step: 3369, Red action: do-nothing, Blue reward:0.85\n", - "step: 3370, Red action: do-nothing, Blue reward:0.85\n", - "step: 3371, Red action: do-nothing, Blue reward:0.85\n", - "step: 3372, Red action: do-nothing, Blue reward:0.85\n", - "step: 3373, Red action: do-nothing, Blue reward:0.85\n", - "step: 3374, Red action: do-nothing, Blue reward:0.85\n", - "step: 3375, Red action: do-nothing, Blue reward:0.85\n", - "step: 3376, Red action: do-nothing, Blue reward:0.85\n", - "step: 3377, Red action: do-nothing, Blue reward:0.85\n", - "step: 3378, Red action: do-nothing, Blue reward:0.85\n", - "step: 3379, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3380, Red action: do-nothing, Blue reward:0.85\n", - "step: 3381, Red action: do-nothing, Blue reward:0.85\n", - "step: 3382, Red action: do-nothing, Blue reward:0.85\n", - "step: 3383, Red action: do-nothing, Blue reward:0.85\n", - "step: 3384, Red action: do-nothing, Blue reward:0.85\n", - "step: 3385, Red action: do-nothing, Blue reward:0.85\n", - "step: 3386, Red action: do-nothing, Blue reward:0.85\n", - "step: 3387, Red action: do-nothing, Blue reward:0.85\n", - "step: 3388, Red action: do-nothing, Blue reward:0.85\n", - "step: 3389, Red action: do-nothing, Blue reward:0.85\n", - "step: 3390, Red action: do-nothing, Blue reward:0.85\n", - "step: 3391, Red action: do-nothing, Blue reward:0.85\n", - "step: 3392, Red action: do-nothing, Blue reward:0.85\n", - "step: 3393, Red action: do-nothing, Blue reward:0.85\n", - "step: 3394, Red action: do-nothing, Blue reward:0.85\n", - "step: 3395, Red action: do-nothing, Blue reward:0.85\n", - "step: 3396, Red action: do-nothing, Blue reward:0.85\n", - "step: 3397, Red action: do-nothing, Blue reward:0.85\n", - "step: 3398, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3399, Red action: do-nothing, Blue reward:0.85\n", - "step: 3400, Red action: do-nothing, Blue reward:0.85\n", - "step: 3401, Red action: do-nothing, Blue reward:0.85\n", - "step: 3402, Red action: do-nothing, Blue reward:0.85\n", - "step: 3403, Red action: do-nothing, Blue reward:0.85\n", - "step: 3404, Red action: do-nothing, Blue reward:0.85\n", - "step: 3405, Red action: do-nothing, Blue reward:0.85\n", - "step: 3406, Red action: do-nothing, Blue reward:0.85\n", - "step: 3407, Red action: do-nothing, Blue reward:0.85\n", - "step: 3408, Red action: do-nothing, Blue reward:0.85\n", - "step: 3409, Red action: do-nothing, Blue reward:0.85\n", - "step: 3410, Red action: do-nothing, Blue reward:0.85\n", - "step: 3411, Red action: do-nothing, Blue reward:0.85\n", - "step: 3412, Red action: do-nothing, Blue reward:0.85\n", - "step: 3413, Red action: do-nothing, Blue reward:0.85\n", - "step: 3414, Red action: do-nothing, Blue reward:0.85\n", - "step: 3415, Red action: do-nothing, Blue reward:0.85\n", - "step: 3416, Red action: do-nothing, Blue reward:0.85\n", - "step: 3417, Red action: do-nothing, Blue reward:0.85\n", - "step: 3418, Red action: do-nothing, Blue reward:0.85\n", - "step: 3419, Red action: do-nothing, Blue reward:0.85\n", - "step: 3420, Red action: do-nothing, Blue reward:0.85\n", - "step: 3421, Red action: do-nothing, Blue reward:0.85\n", - "step: 3422, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3423, Red action: do-nothing, Blue reward:0.85\n", - "step: 3424, Red action: do-nothing, Blue reward:0.85\n", - "step: 3425, Red action: do-nothing, Blue reward:0.85\n", - "step: 3426, Red action: do-nothing, Blue reward:0.85\n", - "step: 3427, Red action: do-nothing, Blue reward:0.85\n", - "step: 3428, Red action: do-nothing, Blue reward:0.85\n", - "step: 3429, Red action: do-nothing, Blue reward:0.85\n", - "step: 3430, Red action: do-nothing, Blue reward:0.85\n", - "step: 3431, Red action: do-nothing, Blue reward:0.85\n", - "step: 3432, Red action: do-nothing, Blue reward:0.85\n", - "step: 3433, Red action: do-nothing, Blue reward:0.85\n", - "step: 3434, Red action: do-nothing, Blue reward:0.85\n", - "step: 3435, Red action: do-nothing, Blue reward:0.85\n", - "step: 3436, Red action: do-nothing, Blue reward:0.85\n", - "step: 3437, Red action: do-nothing, Blue reward:0.85\n", - "step: 3438, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3439, Red action: do-nothing, Blue reward:0.85\n", - "step: 3440, Red action: do-nothing, Blue reward:0.85\n", - "step: 3441, Red action: do-nothing, Blue reward:0.85\n", - "step: 3442, Red action: do-nothing, Blue reward:0.85\n", - "step: 3443, Red action: do-nothing, Blue reward:0.85\n", - "step: 3444, Red action: do-nothing, Blue reward:0.85\n", - "step: 3445, Red action: do-nothing, Blue reward:0.85\n", - "step: 3446, Red action: do-nothing, Blue reward:0.85\n", - "step: 3447, Red action: do-nothing, Blue reward:0.85\n", - "step: 3448, Red action: do-nothing, Blue reward:0.85\n", - "step: 3449, Red action: do-nothing, Blue reward:0.85\n", - "step: 3450, Red action: do-nothing, Blue reward:0.85\n", - "step: 3451, Red action: do-nothing, Blue reward:0.85\n", - "step: 3452, Red action: do-nothing, Blue reward:0.85\n", - "step: 3453, Red action: do-nothing, Blue reward:0.85\n", - "step: 3454, Red action: do-nothing, Blue reward:0.85\n", - "step: 3455, Red action: do-nothing, Blue reward:0.85\n", - "step: 3456, Red action: do-nothing, Blue reward:0.85\n", - "step: 3457, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3458, Red action: do-nothing, Blue reward:0.85\n", - "step: 3459, Red action: do-nothing, Blue reward:0.85\n", - "step: 3460, Red action: do-nothing, Blue reward:0.85\n", - "step: 3461, Red action: do-nothing, Blue reward:0.85\n", - "step: 3462, Red action: do-nothing, Blue reward:0.85\n", - "step: 3463, Red action: do-nothing, Blue reward:0.85\n", - "step: 3464, Red action: do-nothing, Blue reward:0.85\n", - "step: 3465, Red action: do-nothing, Blue reward:0.85\n", - "step: 3466, Red action: do-nothing, Blue reward:0.85\n", - "step: 3467, Red action: do-nothing, Blue reward:0.85\n", - "step: 3468, Red action: do-nothing, Blue reward:0.85\n", - "step: 3469, Red action: do-nothing, Blue reward:0.85\n", - "step: 3470, Red action: do-nothing, Blue reward:0.85\n", - "step: 3471, Red action: do-nothing, Blue reward:0.85\n", - "step: 3472, Red action: do-nothing, Blue reward:0.85\n", - "step: 3473, Red action: do-nothing, Blue reward:0.85\n", - "step: 3474, Red action: do-nothing, Blue reward:0.85\n", - "step: 3475, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3476, Red action: do-nothing, Blue reward:0.85\n", - "step: 3477, Red action: do-nothing, Blue reward:0.85\n", - "step: 3478, Red action: do-nothing, Blue reward:0.85\n", - "step: 3479, Red action: do-nothing, Blue reward:0.85\n", - "step: 3480, Red action: do-nothing, Blue reward:0.85\n", - "step: 3481, Red action: do-nothing, Blue reward:0.85\n", - "step: 3482, Red action: do-nothing, Blue reward:0.85\n", - "step: 3483, Red action: do-nothing, Blue reward:0.85\n", - "step: 3484, Red action: do-nothing, Blue reward:0.85\n", - "step: 3485, Red action: do-nothing, Blue reward:0.85\n", - "step: 3486, Red action: do-nothing, Blue reward:0.85\n", - "step: 3487, Red action: do-nothing, Blue reward:0.85\n", - "step: 3488, Red action: do-nothing, Blue reward:0.85\n", - "step: 3489, Red action: do-nothing, Blue reward:0.85\n", - "step: 3490, Red action: do-nothing, Blue reward:0.85\n", - "step: 3491, Red action: do-nothing, Blue reward:0.85\n", - "step: 3492, Red action: do-nothing, Blue reward:0.85\n", - "step: 3493, Red action: do-nothing, Blue reward:0.85\n", - "step: 3494, Red action: do-nothing, Blue reward:0.85\n", - "step: 3495, Red action: do-nothing, Blue reward:0.85\n", - "step: 3496, Red action: do-nothing, Blue reward:0.85\n", - "step: 3497, Red action: do-nothing, Blue reward:0.85\n", - "step: 3498, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3499, Red action: do-nothing, Blue reward:0.85\n", - "step: 3500, Red action: do-nothing, Blue reward:0.85\n", - "step: 3501, Red action: do-nothing, Blue reward:0.85\n", - "step: 3502, Red action: do-nothing, Blue reward:0.85\n", - "step: 3503, Red action: do-nothing, Blue reward:0.85\n", - "step: 3504, Red action: do-nothing, Blue reward:0.85\n", - "step: 3505, Red action: do-nothing, Blue reward:0.85\n", - "step: 3506, Red action: do-nothing, Blue reward:0.85\n", - "step: 3507, Red action: do-nothing, Blue reward:0.85\n", - "step: 3508, Red action: do-nothing, Blue reward:0.85\n", - "step: 3509, Red action: do-nothing, Blue reward:0.85\n", - "step: 3510, Red action: do-nothing, Blue reward:0.85\n", - "step: 3511, Red action: do-nothing, Blue reward:0.85\n", - "step: 3512, Red action: do-nothing, Blue reward:0.85\n", - "step: 3513, Red action: do-nothing, Blue reward:0.85\n", - "step: 3514, Red action: do-nothing, Blue reward:0.85\n", - "step: 3515, Red action: do-nothing, Blue reward:0.85\n", - "step: 3516, Red action: do-nothing, Blue reward:0.85\n", - "step: 3517, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3518, Red action: do-nothing, Blue reward:0.85\n", - "step: 3519, Red action: do-nothing, Blue reward:0.85\n", - "step: 3520, Red action: do-nothing, Blue reward:0.85\n", - "step: 3521, Red action: do-nothing, Blue reward:0.85\n", - "step: 3522, Red action: do-nothing, Blue reward:0.85\n", - "step: 3523, Red action: do-nothing, Blue reward:0.85\n", - "step: 3524, Red action: do-nothing, Blue reward:0.85\n", - "step: 3525, Red action: do-nothing, Blue reward:0.85\n", - "step: 3526, Red action: do-nothing, Blue reward:0.85\n", - "step: 3527, Red action: do-nothing, Blue reward:0.85\n", - "step: 3528, Red action: do-nothing, Blue reward:0.85\n", - "step: 3529, Red action: do-nothing, Blue reward:0.85\n", - "step: 3530, Red action: do-nothing, Blue reward:0.85\n", - "step: 3531, Red action: do-nothing, Blue reward:0.85\n", - "step: 3532, Red action: do-nothing, Blue reward:0.85\n", - "step: 3533, Red action: do-nothing, Blue reward:0.85\n", - "step: 3534, Red action: do-nothing, Blue reward:0.85\n", - "step: 3535, Red action: do-nothing, Blue reward:0.85\n", - "step: 3536, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3537, Red action: do-nothing, Blue reward:0.85\n", - "step: 3538, Red action: do-nothing, Blue reward:0.85\n", - "step: 3539, Red action: do-nothing, Blue reward:0.85\n", - "step: 3540, Red action: do-nothing, Blue reward:0.85\n", - "step: 3541, Red action: do-nothing, Blue reward:0.85\n", - "step: 3542, Red action: do-nothing, Blue reward:0.85\n", - "step: 3543, Red action: do-nothing, Blue reward:0.85\n", - "step: 3544, Red action: do-nothing, Blue reward:0.85\n", - "step: 3545, Red action: do-nothing, Blue reward:0.85\n", - "step: 3546, Red action: do-nothing, Blue reward:0.85\n", - "step: 3547, Red action: do-nothing, Blue reward:0.85\n", - "step: 3548, Red action: do-nothing, Blue reward:0.85\n", - "step: 3549, Red action: do-nothing, Blue reward:0.85\n", - "step: 3550, Red action: do-nothing, Blue reward:0.85\n", - "step: 3551, Red action: do-nothing, Blue reward:0.85\n", - "step: 3552, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3553, Red action: do-nothing, Blue reward:0.85\n", - "step: 3554, Red action: do-nothing, Blue reward:0.85\n", - "step: 3555, Red action: do-nothing, Blue reward:0.85\n", - "step: 3556, Red action: do-nothing, Blue reward:0.85\n", - "step: 3557, Red action: do-nothing, Blue reward:0.85\n", - "step: 3558, Red action: do-nothing, Blue reward:0.85\n", - "step: 3559, Red action: do-nothing, Blue reward:0.85\n", - "step: 3560, Red action: do-nothing, Blue reward:0.85\n", - "step: 3561, Red action: do-nothing, Blue reward:0.85\n", - "step: 3562, Red action: do-nothing, Blue reward:0.85\n", - "step: 3563, Red action: do-nothing, Blue reward:0.85\n", - "step: 3564, Red action: do-nothing, Blue reward:0.85\n", - "step: 3565, Red action: do-nothing, Blue reward:0.85\n", - "step: 3566, Red action: do-nothing, Blue reward:0.85\n", - "step: 3567, Red action: do-nothing, Blue reward:0.85\n", - "step: 3568, Red action: do-nothing, Blue reward:0.85\n", - "step: 3569, Red action: do-nothing, Blue reward:0.85\n", - "step: 3570, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3571, Red action: do-nothing, Blue reward:0.85\n", - "step: 3572, Red action: do-nothing, Blue reward:0.85\n", - "step: 3573, Red action: do-nothing, Blue reward:0.85\n", - "step: 3574, Red action: do-nothing, Blue reward:0.85\n", - "step: 3575, Red action: do-nothing, Blue reward:0.85\n", - "step: 3576, Red action: do-nothing, Blue reward:0.85\n", - "step: 3577, Red action: do-nothing, Blue reward:0.85\n", - "step: 3578, Red action: do-nothing, Blue reward:0.85\n", - "step: 3579, Red action: do-nothing, Blue reward:0.85\n", - "step: 3580, Red action: do-nothing, Blue reward:0.85\n", - "step: 3581, Red action: do-nothing, Blue reward:0.85\n", - "step: 3582, Red action: do-nothing, Blue reward:0.85\n", - "step: 3583, Red action: do-nothing, Blue reward:0.85\n", - "step: 3584, Red action: do-nothing, Blue reward:0.85\n", - "step: 3585, Red action: do-nothing, Blue reward:0.85\n", - "step: 3586, Red action: do-nothing, Blue reward:0.85\n", - "step: 3587, Red action: do-nothing, Blue reward:0.85\n", - "step: 3588, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3589, Red action: do-nothing, Blue reward:0.85\n", - "step: 3590, Red action: do-nothing, Blue reward:0.85\n", - "step: 3591, Red action: do-nothing, Blue reward:0.85\n", - "step: 3592, Red action: do-nothing, Blue reward:0.85\n", - "step: 3593, Red action: do-nothing, Blue reward:0.85\n", - "step: 3594, Red action: do-nothing, Blue reward:0.85\n", - "step: 3595, Red action: do-nothing, Blue reward:0.85\n", - "step: 3596, Red action: do-nothing, Blue reward:0.85\n", - "step: 3597, Red action: do-nothing, Blue reward:0.85\n", - "step: 3598, Red action: do-nothing, Blue reward:0.85\n", - "step: 3599, Red action: do-nothing, Blue reward:0.85\n", - "step: 3600, Red action: do-nothing, Blue reward:0.85\n", - "step: 3601, Red action: do-nothing, Blue reward:0.85\n", - "step: 3602, Red action: do-nothing, Blue reward:0.85\n", - "step: 3603, Red action: do-nothing, Blue reward:0.85\n", - "step: 3604, Red action: do-nothing, Blue reward:0.85\n", - "step: 3605, Red action: do-nothing, Blue reward:0.85\n", - "step: 3606, Red action: do-nothing, Blue reward:0.85\n", - "step: 3607, Red action: do-nothing, Blue reward:0.85\n", - "step: 3608, Red action: do-nothing, Blue reward:0.85\n", - "step: 3609, Red action: do-nothing, Blue reward:0.85\n", - "step: 3610, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3611, Red action: do-nothing, Blue reward:0.85\n", - "step: 3612, Red action: do-nothing, Blue reward:0.85\n", - "step: 3613, Red action: do-nothing, Blue reward:0.85\n", - "step: 3614, Red action: do-nothing, Blue reward:0.85\n", - "step: 3615, Red action: do-nothing, Blue reward:0.85\n", - "step: 3616, Red action: do-nothing, Blue reward:0.85\n", - "step: 3617, Red action: do-nothing, Blue reward:0.85\n", - "step: 3618, Red action: do-nothing, Blue reward:0.85\n", - "step: 3619, Red action: do-nothing, Blue reward:0.85\n", - "step: 3620, Red action: do-nothing, Blue reward:0.85\n", - "step: 3621, Red action: do-nothing, Blue reward:0.85\n", - "step: 3622, Red action: do-nothing, Blue reward:0.85\n", - "step: 3623, Red action: do-nothing, Blue reward:0.85\n", - "step: 3624, Red action: do-nothing, Blue reward:0.85\n", - "step: 3625, Red action: do-nothing, Blue reward:0.85\n", - "step: 3626, Red action: do-nothing, Blue reward:0.85\n", - "step: 3627, Red action: do-nothing, Blue reward:0.85\n", - "step: 3628, Red action: do-nothing, Blue reward:0.85\n", - "step: 3629, Red action: do-nothing, Blue reward:0.85\n", - "step: 3630, Red action: do-nothing, Blue reward:0.85\n", - "step: 3631, Red action: do-nothing, Blue reward:0.85\n", - "step: 3632, Red action: do-nothing, Blue reward:0.85\n", - "step: 3633, Red action: do-nothing, Blue reward:0.85\n", - "step: 3634, Red action: do-nothing, Blue reward:0.85\n", - "step: 3635, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3636, Red action: do-nothing, Blue reward:0.85\n", - "step: 3637, Red action: do-nothing, Blue reward:0.85\n", - "step: 3638, Red action: do-nothing, Blue reward:0.85\n", - "step: 3639, Red action: do-nothing, Blue reward:0.85\n", - "step: 3640, Red action: do-nothing, Blue reward:0.85\n", - "step: 3641, Red action: do-nothing, Blue reward:0.85\n", - "step: 3642, Red action: do-nothing, Blue reward:0.85\n", - "step: 3643, Red action: do-nothing, Blue reward:0.85\n", - "step: 3644, Red action: do-nothing, Blue reward:0.85\n", - "step: 3645, Red action: do-nothing, Blue reward:0.85\n", - "step: 3646, Red action: do-nothing, Blue reward:0.85\n", - "step: 3647, Red action: do-nothing, Blue reward:0.85\n", - "step: 3648, Red action: do-nothing, Blue reward:0.85\n", - "step: 3649, Red action: do-nothing, Blue reward:0.85\n", - "step: 3650, Red action: do-nothing, Blue reward:0.85\n", - "step: 3651, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3652, Red action: do-nothing, Blue reward:0.85\n", - "step: 3653, Red action: do-nothing, Blue reward:0.85\n", - "step: 3654, Red action: do-nothing, Blue reward:0.85\n", - "step: 3655, Red action: do-nothing, Blue reward:0.85\n", - "step: 3656, Red action: do-nothing, Blue reward:0.85\n", - "step: 3657, Red action: do-nothing, Blue reward:0.85\n", - "step: 3658, Red action: do-nothing, Blue reward:0.85\n", - "step: 3659, Red action: do-nothing, Blue reward:0.85\n", - "step: 3660, Red action: do-nothing, Blue reward:0.85\n", - "step: 3661, Red action: do-nothing, Blue reward:0.85\n", - "step: 3662, Red action: do-nothing, Blue reward:0.85\n", - "step: 3663, Red action: do-nothing, Blue reward:0.85\n", - "step: 3664, Red action: do-nothing, Blue reward:0.85\n", - "step: 3665, Red action: do-nothing, Blue reward:0.85\n", - "step: 3666, Red action: do-nothing, Blue reward:0.85\n", - "step: 3667, Red action: do-nothing, Blue reward:0.85\n", - "step: 3668, Red action: do-nothing, Blue reward:0.85\n", - "step: 3669, Red action: do-nothing, Blue reward:0.85\n", - "step: 3670, Red action: do-nothing, Blue reward:0.85\n", - "step: 3671, Red action: do-nothing, Blue reward:0.85\n", - "step: 3672, Red action: do-nothing, Blue reward:0.85\n", - "step: 3673, Red action: do-nothing, Blue reward:0.85\n", - "step: 3674, Red action: do-nothing, Blue reward:0.85\n", - "step: 3675, Red action: do-nothing, Blue reward:0.85\n", - "step: 3676, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3677, Red action: do-nothing, Blue reward:0.85\n", - "step: 3678, Red action: do-nothing, Blue reward:0.85\n", - "step: 3679, Red action: do-nothing, Blue reward:0.85\n", - "step: 3680, Red action: do-nothing, Blue reward:0.85\n", - "step: 3681, Red action: do-nothing, Blue reward:0.85\n", - "step: 3682, Red action: do-nothing, Blue reward:0.85\n", - "step: 3683, Red action: do-nothing, Blue reward:0.85\n", - "step: 3684, Red action: do-nothing, Blue reward:0.85\n", - "step: 3685, Red action: do-nothing, Blue reward:0.85\n", - "step: 3686, Red action: do-nothing, Blue reward:0.85\n", - "step: 3687, Red action: do-nothing, Blue reward:0.85\n", - "step: 3688, Red action: do-nothing, Blue reward:0.85\n", - "step: 3689, Red action: do-nothing, Blue reward:0.85\n", - "step: 3690, Red action: do-nothing, Blue reward:0.85\n", - "step: 3691, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3692, Red action: do-nothing, Blue reward:0.85\n", - "step: 3693, Red action: do-nothing, Blue reward:0.85\n", - "step: 3694, Red action: do-nothing, Blue reward:0.85\n", - "step: 3695, Red action: do-nothing, Blue reward:0.85\n", - "step: 3696, Red action: do-nothing, Blue reward:0.85\n", - "step: 3697, Red action: do-nothing, Blue reward:0.85\n", - "step: 3698, Red action: do-nothing, Blue reward:0.85\n", - "step: 3699, Red action: do-nothing, Blue reward:0.85\n", - "step: 3700, Red action: do-nothing, Blue reward:0.85\n", - "step: 3701, Red action: do-nothing, Blue reward:0.85\n", - "step: 3702, Red action: do-nothing, Blue reward:0.85\n", - "step: 3703, Red action: do-nothing, Blue reward:0.85\n", - "step: 3704, Red action: do-nothing, Blue reward:0.85\n", - "step: 3705, Red action: do-nothing, Blue reward:0.85\n", - "step: 3706, Red action: do-nothing, Blue reward:0.85\n", - "step: 3707, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3708, Red action: do-nothing, Blue reward:0.85\n", - "step: 3709, Red action: do-nothing, Blue reward:0.85\n", - "step: 3710, Red action: do-nothing, Blue reward:0.85\n", - "step: 3711, Red action: do-nothing, Blue reward:0.85\n", - "step: 3712, Red action: do-nothing, Blue reward:0.85\n", - "step: 3713, Red action: do-nothing, Blue reward:0.85\n", - "step: 3714, Red action: do-nothing, Blue reward:0.85\n", - "step: 3715, Red action: do-nothing, Blue reward:0.85\n", - "step: 3716, Red action: do-nothing, Blue reward:0.85\n", - "step: 3717, Red action: do-nothing, Blue reward:0.85\n", - "step: 3718, Red action: do-nothing, Blue reward:0.85\n", - "step: 3719, Red action: do-nothing, Blue reward:0.85\n", - "step: 3720, Red action: do-nothing, Blue reward:0.85\n", - "step: 3721, Red action: do-nothing, Blue reward:0.85\n", - "step: 3722, Red action: do-nothing, Blue reward:0.85\n", - "step: 3723, Red action: do-nothing, Blue reward:0.85\n", - "step: 3724, Red action: do-nothing, Blue reward:0.85\n", - "step: 3725, Red action: do-nothing, Blue reward:0.85\n", - "step: 3726, Red action: do-nothing, Blue reward:0.85\n", - "step: 3727, Red action: do-nothing, Blue reward:0.85\n", - "step: 3728, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3729, Red action: do-nothing, Blue reward:0.85\n", - "step: 3730, Red action: do-nothing, Blue reward:0.85\n", - "step: 3731, Red action: do-nothing, Blue reward:0.85\n", - "step: 3732, Red action: do-nothing, Blue reward:0.85\n", - "step: 3733, Red action: do-nothing, Blue reward:0.85\n", - "step: 3734, Red action: do-nothing, Blue reward:0.85\n", - "step: 3735, Red action: do-nothing, Blue reward:0.85\n", - "step: 3736, Red action: do-nothing, Blue reward:0.85\n", - "step: 3737, Red action: do-nothing, Blue reward:0.85\n", - "step: 3738, Red action: do-nothing, Blue reward:0.85\n", - "step: 3739, Red action: do-nothing, Blue reward:0.85\n", - "step: 3740, Red action: do-nothing, Blue reward:0.85\n", - "step: 3741, Red action: do-nothing, Blue reward:0.85\n", - "step: 3742, Red action: do-nothing, Blue reward:0.85\n", - "step: 3743, Red action: do-nothing, Blue reward:0.85\n", - "step: 3744, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3745, Red action: do-nothing, Blue reward:0.85\n", - "step: 3746, Red action: do-nothing, Blue reward:0.85\n", - "step: 3747, Red action: do-nothing, Blue reward:0.85\n", - "step: 3748, Red action: do-nothing, Blue reward:0.85\n", - "step: 3749, Red action: do-nothing, Blue reward:0.85\n", - "step: 3750, Red action: do-nothing, Blue reward:0.85\n", - "step: 3751, Red action: do-nothing, Blue reward:0.85\n", - "step: 3752, Red action: do-nothing, Blue reward:0.85\n", - "step: 3753, Red action: do-nothing, Blue reward:0.85\n", - "step: 3754, Red action: do-nothing, Blue reward:0.85\n", - "step: 3755, Red action: do-nothing, Blue reward:0.85\n", - "step: 3756, Red action: do-nothing, Blue reward:0.85\n", - "step: 3757, Red action: do-nothing, Blue reward:0.85\n", - "step: 3758, Red action: do-nothing, Blue reward:0.85\n", - "step: 3759, Red action: do-nothing, Blue reward:0.85\n", - "step: 3760, Red action: do-nothing, Blue reward:0.85\n", - "step: 3761, Red action: do-nothing, Blue reward:0.85\n", - "step: 3762, Red action: do-nothing, Blue reward:0.85\n", - "step: 3763, Red action: do-nothing, Blue reward:0.85\n", - "step: 3764, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3765, Red action: do-nothing, Blue reward:0.85\n", - "step: 3766, Red action: do-nothing, Blue reward:0.85\n", - "step: 3767, Red action: do-nothing, Blue reward:0.85\n", - "step: 3768, Red action: do-nothing, Blue reward:0.85\n", - "step: 3769, Red action: do-nothing, Blue reward:0.85\n", - "step: 3770, Red action: do-nothing, Blue reward:0.85\n", - "step: 3771, Red action: do-nothing, Blue reward:0.85\n", - "step: 3772, Red action: do-nothing, Blue reward:0.85\n", - "step: 3773, Red action: do-nothing, Blue reward:0.85\n", - "step: 3774, Red action: do-nothing, Blue reward:0.85\n", - "step: 3775, Red action: do-nothing, Blue reward:0.85\n", - "step: 3776, Red action: do-nothing, Blue reward:0.85\n", - "step: 3777, Red action: do-nothing, Blue reward:0.85\n", - "step: 3778, Red action: do-nothing, Blue reward:0.85\n", - "step: 3779, Red action: do-nothing, Blue reward:0.85\n", - "step: 3780, Red action: do-nothing, Blue reward:0.85\n", - "step: 3781, Red action: do-nothing, Blue reward:0.85\n", - "step: 3782, Red action: do-nothing, Blue reward:0.85\n", - "step: 3783, Red action: do-nothing, Blue reward:0.85\n", - "step: 3784, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3785, Red action: do-nothing, Blue reward:0.85\n", - "step: 3786, Red action: do-nothing, Blue reward:0.85\n", - "step: 3787, Red action: do-nothing, Blue reward:0.85\n", - "step: 3788, Red action: do-nothing, Blue reward:0.85\n", - "step: 3789, Red action: do-nothing, Blue reward:0.85\n", - "step: 3790, Red action: do-nothing, Blue reward:0.85\n", - "step: 3791, Red action: do-nothing, Blue reward:0.85\n", - "step: 3792, Red action: do-nothing, Blue reward:0.85\n", - "step: 3793, Red action: do-nothing, Blue reward:0.85\n", - "step: 3794, Red action: do-nothing, Blue reward:0.85\n", - "step: 3795, Red action: do-nothing, Blue reward:0.85\n", - "step: 3796, Red action: do-nothing, Blue reward:0.85\n", - "step: 3797, Red action: do-nothing, Blue reward:0.85\n", - "step: 3798, Red action: do-nothing, Blue reward:0.85\n", - "step: 3799, Red action: do-nothing, Blue reward:0.85\n", - "step: 3800, Red action: do-nothing, Blue reward:0.85\n", - "step: 3801, Red action: do-nothing, Blue reward:0.85\n", - "step: 3802, Red action: do-nothing, Blue reward:0.85\n" - ] - }, - { - "ename": "KeyboardInterrupt", - "evalue": "", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[11], line 11\u001b[0m\n\u001b[1;32m 8\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mstep: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00menv\u001b[38;5;241m.\u001b[39mgame\u001b[38;5;241m.\u001b[39mstep_counter\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m, Red action: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00minfo[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124magent_actions\u001b[39m\u001b[38;5;124m'\u001b[39m][\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mdata_manipulation_attacker\u001b[39m\u001b[38;5;124m'\u001b[39m]\u001b[38;5;241m.\u001b[39maction\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m, Blue reward:\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mreward\u001b[38;5;132;01m:\u001b[39;00m\u001b[38;5;124m.2f\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m )\n\u001b[1;32m 10\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28mabs\u001b[39m(reward \u001b[38;5;241m-\u001b[39m \u001b[38;5;241m0.8\u001b[39m) \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m1e-5\u001b[39m:\n\u001b[0;32m---> 11\u001b[0m obs, reward, terminated, truncated, info \u001b[38;5;241m=\u001b[39m \u001b[43menv\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstep\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m)\u001b[49m \u001b[38;5;66;03m# do nothing\u001b[39;00m\n\u001b[1;32m 12\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mstep: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00menv\u001b[38;5;241m.\u001b[39mgame\u001b[38;5;241m.\u001b[39mstep_counter\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m, Red action: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00minfo[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124magent_actions\u001b[39m\u001b[38;5;124m'\u001b[39m][\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mdata_manipulation_attacker\u001b[39m\u001b[38;5;124m'\u001b[39m]\u001b[38;5;241m.\u001b[39maction\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m, Blue reward:\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mreward\u001b[38;5;132;01m:\u001b[39;00m\u001b[38;5;124m.2f\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m )\n\u001b[1;32m 13\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m env\u001b[38;5;241m.\u001b[39mgame\u001b[38;5;241m.\u001b[39mstep_counter \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m10000\u001b[39m:\n", - "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/session/environment.py:111\u001b[0m, in \u001b[0;36mPrimaiteGymEnv.step\u001b[0;34m(self, action)\u001b[0m\n\u001b[1;32m 109\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgame\u001b[38;5;241m.\u001b[39mapply_agent_actions()\n\u001b[1;32m 110\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgame\u001b[38;5;241m.\u001b[39madvance_timestep()\n\u001b[0;32m--> 111\u001b[0m state \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgame\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_sim_state\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 112\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgame\u001b[38;5;241m.\u001b[39mupdate_agents(state)\n\u001b[1;32m 114\u001b[0m next_obs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_get_obs() \u001b[38;5;66;03m# this doesn't update observation, just gets the current observation\u001b[39;00m\n", - "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/game/game.py:151\u001b[0m, in \u001b[0;36mPrimaiteGame.get_sim_state\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 149\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mget_sim_state\u001b[39m(\u001b[38;5;28mself\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Dict:\n\u001b[1;32m 150\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"Get the current state of the simulation.\"\"\"\u001b[39;00m\n\u001b[0;32m--> 151\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msimulation\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdescribe_state\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/simulator/sim_container.py:57\u001b[0m, in \u001b[0;36mSimulation.describe_state\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 46\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 47\u001b[0m \u001b[38;5;124;03mProduce a dictionary describing the current state of this object.\u001b[39;00m\n\u001b[1;32m 48\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 52\u001b[0m \u001b[38;5;124;03m:rtype: Dict\u001b[39;00m\n\u001b[1;32m 53\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 54\u001b[0m state \u001b[38;5;241m=\u001b[39m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39mdescribe_state()\n\u001b[1;32m 55\u001b[0m state\u001b[38;5;241m.\u001b[39mupdate(\n\u001b[1;32m 56\u001b[0m {\n\u001b[0;32m---> 57\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnetwork\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mnetwork\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdescribe_state\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m,\n\u001b[1;32m 58\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdomain\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdomain\u001b[38;5;241m.\u001b[39mdescribe_state(),\n\u001b[1;32m 59\u001b[0m }\n\u001b[1;32m 60\u001b[0m )\n\u001b[1;32m 61\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m state\n", - "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/simulator/network/container.py:260\u001b[0m, in \u001b[0;36mNetwork.describe_state\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 252\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 253\u001b[0m \u001b[38;5;124;03mProduce a dictionary describing the current state of the Network.\u001b[39;00m\n\u001b[1;32m 254\u001b[0m \n\u001b[1;32m 255\u001b[0m \u001b[38;5;124;03m:return: A dictionary capturing the current state of the Network and its child objects.\u001b[39;00m\n\u001b[1;32m 256\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 257\u001b[0m state \u001b[38;5;241m=\u001b[39m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39mdescribe_state()\n\u001b[1;32m 258\u001b[0m state\u001b[38;5;241m.\u001b[39mupdate(\n\u001b[1;32m 259\u001b[0m {\n\u001b[0;32m--> 260\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnodes\u001b[39m\u001b[38;5;124m\"\u001b[39m: {node\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mhostname: node\u001b[38;5;241m.\u001b[39mdescribe_state() \u001b[38;5;28;01mfor\u001b[39;00m node \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnodes\u001b[38;5;241m.\u001b[39mvalues()},\n\u001b[1;32m 261\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mlinks\u001b[39m\u001b[38;5;124m\"\u001b[39m: {},\n\u001b[1;32m 262\u001b[0m }\n\u001b[1;32m 263\u001b[0m )\n\u001b[1;32m 264\u001b[0m \u001b[38;5;66;03m# Update the links one-by-one. The key is a 4-tuple of `hostname_a, port_a, hostname_b, port_b`\u001b[39;00m\n\u001b[1;32m 265\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m _, link \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlinks\u001b[38;5;241m.\u001b[39mitems():\n", - "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/simulator/network/container.py:260\u001b[0m, in \u001b[0;36m\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 252\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 253\u001b[0m \u001b[38;5;124;03mProduce a dictionary describing the current state of the Network.\u001b[39;00m\n\u001b[1;32m 254\u001b[0m \n\u001b[1;32m 255\u001b[0m \u001b[38;5;124;03m:return: A dictionary capturing the current state of the Network and its child objects.\u001b[39;00m\n\u001b[1;32m 256\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 257\u001b[0m state \u001b[38;5;241m=\u001b[39m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39mdescribe_state()\n\u001b[1;32m 258\u001b[0m state\u001b[38;5;241m.\u001b[39mupdate(\n\u001b[1;32m 259\u001b[0m {\n\u001b[0;32m--> 260\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnodes\u001b[39m\u001b[38;5;124m\"\u001b[39m: {node\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mhostname: \u001b[43mnode\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdescribe_state\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mfor\u001b[39;00m node \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnodes\u001b[38;5;241m.\u001b[39mvalues()},\n\u001b[1;32m 261\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mlinks\u001b[39m\u001b[38;5;124m\"\u001b[39m: {},\n\u001b[1;32m 262\u001b[0m }\n\u001b[1;32m 263\u001b[0m )\n\u001b[1;32m 264\u001b[0m \u001b[38;5;66;03m# Update the links one-by-one. The key is a 4-tuple of `hostname_a, port_a, hostname_b, port_b`\u001b[39;00m\n\u001b[1;32m 265\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m _, link \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlinks\u001b[38;5;241m.\u001b[39mitems():\n", - "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/simulator/network/hardware/base.py:1900\u001b[0m, in \u001b[0;36mNode.describe_state\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1881\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 1882\u001b[0m \u001b[38;5;124;03mProduce a dictionary describing the current state of this object.\u001b[39;00m\n\u001b[1;32m 1883\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1887\u001b[0m \u001b[38;5;124;03m:rtype: Dict\u001b[39;00m\n\u001b[1;32m 1888\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 1889\u001b[0m state \u001b[38;5;241m=\u001b[39m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39mdescribe_state()\n\u001b[1;32m 1890\u001b[0m state\u001b[38;5;241m.\u001b[39mupdate(\n\u001b[1;32m 1891\u001b[0m {\n\u001b[1;32m 1892\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhostname\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mhostname,\n\u001b[1;32m 1893\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moperating_state\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39moperating_state\u001b[38;5;241m.\u001b[39mvalue,\n\u001b[1;32m 1894\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mNICs\u001b[39m\u001b[38;5;124m\"\u001b[39m: {\n\u001b[1;32m 1895\u001b[0m eth_num: network_interface\u001b[38;5;241m.\u001b[39mdescribe_state()\n\u001b[1;32m 1896\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m eth_num, network_interface \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnetwork_interface\u001b[38;5;241m.\u001b[39mitems()\n\u001b[1;32m 1897\u001b[0m },\n\u001b[1;32m 1898\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfile_system\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfile_system\u001b[38;5;241m.\u001b[39mdescribe_state(),\n\u001b[1;32m 1899\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mapplications\u001b[39m\u001b[38;5;124m\"\u001b[39m: {app\u001b[38;5;241m.\u001b[39mname: app\u001b[38;5;241m.\u001b[39mdescribe_state() \u001b[38;5;28;01mfor\u001b[39;00m app \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mapplications\u001b[38;5;241m.\u001b[39mvalues()},\n\u001b[0;32m-> 1900\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mservices\u001b[39m\u001b[38;5;124m\"\u001b[39m: {svc\u001b[38;5;241m.\u001b[39mname: svc\u001b[38;5;241m.\u001b[39mdescribe_state() \u001b[38;5;28;01mfor\u001b[39;00m svc \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mservices\u001b[38;5;241m.\u001b[39mvalues()},\n\u001b[1;32m 1901\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mprocess\u001b[39m\u001b[38;5;124m\"\u001b[39m: {proc\u001b[38;5;241m.\u001b[39mname: proc\u001b[38;5;241m.\u001b[39mdescribe_state() \u001b[38;5;28;01mfor\u001b[39;00m proc \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprocesses\u001b[38;5;241m.\u001b[39mvalues()},\n\u001b[1;32m 1902\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mrevealed_to_red\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mrevealed_to_red,\n\u001b[1;32m 1903\u001b[0m }\n\u001b[1;32m 1904\u001b[0m )\n\u001b[1;32m 1905\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m state\n", - "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/simulator/network/hardware/base.py:1900\u001b[0m, in \u001b[0;36m\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 1881\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 1882\u001b[0m \u001b[38;5;124;03mProduce a dictionary describing the current state of this object.\u001b[39;00m\n\u001b[1;32m 1883\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1887\u001b[0m \u001b[38;5;124;03m:rtype: Dict\u001b[39;00m\n\u001b[1;32m 1888\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 1889\u001b[0m state \u001b[38;5;241m=\u001b[39m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39mdescribe_state()\n\u001b[1;32m 1890\u001b[0m state\u001b[38;5;241m.\u001b[39mupdate(\n\u001b[1;32m 1891\u001b[0m {\n\u001b[1;32m 1892\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhostname\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mhostname,\n\u001b[1;32m 1893\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moperating_state\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39moperating_state\u001b[38;5;241m.\u001b[39mvalue,\n\u001b[1;32m 1894\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mNICs\u001b[39m\u001b[38;5;124m\"\u001b[39m: {\n\u001b[1;32m 1895\u001b[0m eth_num: network_interface\u001b[38;5;241m.\u001b[39mdescribe_state()\n\u001b[1;32m 1896\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m eth_num, network_interface \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnetwork_interface\u001b[38;5;241m.\u001b[39mitems()\n\u001b[1;32m 1897\u001b[0m },\n\u001b[1;32m 1898\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfile_system\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfile_system\u001b[38;5;241m.\u001b[39mdescribe_state(),\n\u001b[1;32m 1899\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mapplications\u001b[39m\u001b[38;5;124m\"\u001b[39m: {app\u001b[38;5;241m.\u001b[39mname: app\u001b[38;5;241m.\u001b[39mdescribe_state() \u001b[38;5;28;01mfor\u001b[39;00m app \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mapplications\u001b[38;5;241m.\u001b[39mvalues()},\n\u001b[0;32m-> 1900\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mservices\u001b[39m\u001b[38;5;124m\"\u001b[39m: {svc\u001b[38;5;241m.\u001b[39mname: \u001b[43msvc\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdescribe_state\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mfor\u001b[39;00m svc \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mservices\u001b[38;5;241m.\u001b[39mvalues()},\n\u001b[1;32m 1901\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mprocess\u001b[39m\u001b[38;5;124m\"\u001b[39m: {proc\u001b[38;5;241m.\u001b[39mname: proc\u001b[38;5;241m.\u001b[39mdescribe_state() \u001b[38;5;28;01mfor\u001b[39;00m proc \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprocesses\u001b[38;5;241m.\u001b[39mvalues()},\n\u001b[1;32m 1902\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mrevealed_to_red\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mrevealed_to_red,\n\u001b[1;32m 1903\u001b[0m }\n\u001b[1;32m 1904\u001b[0m )\n\u001b[1;32m 1905\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m state\n", - "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/simulator/system/services/ntp/ntp_client.py:61\u001b[0m, in \u001b[0;36mNTPClient.describe_state\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 50\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdescribe_state\u001b[39m(\u001b[38;5;28mself\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Dict:\n\u001b[1;32m 51\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 52\u001b[0m \u001b[38;5;124;03m Describes the current state of the software.\u001b[39;00m\n\u001b[1;32m 53\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 59\u001b[0m \u001b[38;5;124;03m :rtype: Dict\u001b[39;00m\n\u001b[1;32m 60\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m---> 61\u001b[0m state \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241m.\u001b[39mdescribe_state()\n\u001b[1;32m 62\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m state\n", - "\u001b[0;31mKeyboardInterrupt\u001b[0m: " - ] - } - ], + "outputs": [], "source": [ "env.step(13) # Patch the database\n", "print(f\"step: {env.game.step_counter}, Red action: {info['agent_actions']['data_manipulation_attacker'].action}, Blue reward:{reward:.2f}\" )\n", From 3651d033d6f2ba5d527de70c00f796c2ff96bfeb Mon Sep 17 00:00:00 2001 From: Marek Wolan Date: Mon, 17 Feb 2025 17:34:22 +0000 Subject: [PATCH 28/37] fix instantiation of network nodes --- src/primaite/simulator/network/hardware/base.py | 4 +--- .../simulator/network/hardware/nodes/host/host_node.py | 4 ++-- .../simulator/network/hardware/nodes/network/router.py | 1 - .../simulator/network/hardware/nodes/network/switch.py | 5 ++--- src/primaite/simulator/system/services/arp/arp.py | 4 ++-- src/primaite/simulator/system/software.py | 2 +- 6 files changed, 8 insertions(+), 12 deletions(-) diff --git a/src/primaite/simulator/network/hardware/base.py b/src/primaite/simulator/network/hardware/base.py index 8653359a..b20fb351 100644 --- a/src/primaite/simulator/network/hardware/base.py +++ b/src/primaite/simulator/network/hardware/base.py @@ -639,10 +639,8 @@ class IPWiredNetworkInterface(WiredNetworkInterface, Layer3Interface, ABC): `default_gateway_hello` method is not defined, ignoring such errors to proceed without interruption. """ super().enable() - try: + if hasattr(self._connected_node, "default_gateway_hello"): self._connected_node.default_gateway_hello() - except AttributeError: - pass return True @abstractmethod diff --git a/src/primaite/simulator/network/hardware/nodes/host/host_node.py b/src/primaite/simulator/network/hardware/nodes/host/host_node.py index 76d9167c..2c1910c4 100644 --- a/src/primaite/simulator/network/hardware/nodes/host/host_node.py +++ b/src/primaite/simulator/network/hardware/nodes/host/host_node.py @@ -333,7 +333,7 @@ class HostNode(Node, discriminator="host-node"): class ConfigSchema(Node.ConfigSchema): """Configuration Schema for HostNode class.""" - type: Literal["host-node"] + type: Literal["host-node"] = "host-node" hostname: str = "HostNode" subnet_mask: IPV4Address = "255.255.255.0" ip_address: IPV4Address @@ -375,7 +375,7 @@ class HostNode(Node, discriminator="host-node"): This method is invoked to ensure the host node can communicate with its default gateway, primarily to confirm network connectivity and populate the ARP cache with the gateway's MAC address. """ - if self.operating_state == NodeOperatingState.ON and self.default_gateway: + if self.operating_state == NodeOperatingState.ON and self.config.default_gateway: self.software_manager.arp.get_default_gateway_mac_address() def receive_frame(self, frame: Frame, from_network_interface: NIC): diff --git a/src/primaite/simulator/network/hardware/nodes/network/router.py b/src/primaite/simulator/network/hardware/nodes/network/router.py index 3b35600b..f2a4652c 100644 --- a/src/primaite/simulator/network/hardware/nodes/network/router.py +++ b/src/primaite/simulator/network/hardware/nodes/network/router.py @@ -1351,7 +1351,6 @@ class Router(NetworkNode, discriminator="router"): :return: A dictionary representing the current state. """ state = super().describe_state() - state["num_ports"] = self.config.num_ports state["acl"] = self.acl.describe_state() return state diff --git a/src/primaite/simulator/network/hardware/nodes/network/switch.py b/src/primaite/simulator/network/hardware/nodes/network/switch.py index 1f2bc135..6e5814d0 100644 --- a/src/primaite/simulator/network/hardware/nodes/network/switch.py +++ b/src/primaite/simulator/network/hardware/nodes/network/switch.py @@ -103,8 +103,8 @@ class Switch(NetworkNode, discriminator="switch"): type: Literal["switch"] = "switch" hostname: str = "Switch" - num_ports: int = 24 - "The number of ports on the switch. Default is 24." + num_ports: int = 8 + "The number of ports on the switch." config: ConfigSchema = Field(default_factory=lambda: Switch.ConfigSchema()) @@ -139,7 +139,6 @@ class Switch(NetworkNode, discriminator="switch"): """ state = super().describe_state() state["ports"] = {port_num: port.describe_state() for port_num, port in self.network_interface.items()} - state["num_ports"] = self.config.num_ports # redundant? state["mac_address_table"] = {mac: port.port_num for mac, port in self.mac_address_table.items()} return state diff --git a/src/primaite/simulator/system/services/arp/arp.py b/src/primaite/simulator/system/services/arp/arp.py index b0630d5d..348cc03e 100644 --- a/src/primaite/simulator/system/services/arp/arp.py +++ b/src/primaite/simulator/system/services/arp/arp.py @@ -137,8 +137,8 @@ class ARP(Service, discriminator="arp"): break if use_default_gateway: - if self.software_manager.node.default_gateway: - target_ip_address = self.software_manager.node.default_gateway + if self.software_manager.node.config.default_gateway: + target_ip_address = self.software_manager.node.config.default_gateway else: return diff --git a/src/primaite/simulator/system/software.py b/src/primaite/simulator/system/software.py index 950f77c6..86b57818 100644 --- a/src/primaite/simulator/system/software.py +++ b/src/primaite/simulator/system/software.py @@ -82,7 +82,7 @@ class Software(SimComponent, ABC): """Configurable options for all software.""" model_config = ConfigDict(extra="forbid") - starting_health_state: SoftwareHealthState = SoftwareHealthState.UNUSED + starting_health_state: SoftwareHealthState = SoftwareHealthState.GOOD criticality: SoftwareCriticality = SoftwareCriticality.LOWEST fixing_duration: int = 2 From de88974332076c0f1841b0178674f4b73543701f Mon Sep 17 00:00:00 2001 From: Marek Wolan Date: Mon, 17 Feb 2025 18:14:07 +0000 Subject: [PATCH 29/37] Fix airspace hello --- src/primaite/simulator/network/airspace.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/primaite/simulator/network/airspace.py b/src/primaite/simulator/network/airspace.py index 7ede0bb0..434940f8 100644 --- a/src/primaite/simulator/network/airspace.py +++ b/src/primaite/simulator/network/airspace.py @@ -449,10 +449,8 @@ class IPWirelessNetworkInterface(WirelessNetworkInterface, Layer3Interface, ABC) `default_gateway_hello` method is not defined, ignoring such errors to proceed without interruption. """ super().enable() - try: + if hasattr(self._connected_node, "default_gateway_hello"): self._connected_node.default_gateway_hello() - except AttributeError: - pass @abstractmethod def receive_frame(self, frame: Frame) -> bool: From 46240e49a41510b4f2b708b6d489a6a97e27d40f Mon Sep 17 00:00:00 2001 From: Marek Wolan Date: Mon, 17 Feb 2025 18:46:09 +0000 Subject: [PATCH 30/37] update tests and make office lan creation work like previously --- src/primaite/simulator/network/creation.py | 15 +++++++++++++-- .../_network/_hardware/nodes/test_switch.py | 1 - .../_system/_applications/test_applications.py | 6 +++--- .../_simulator/_system/_services/test_services.py | 12 ++++++------ .../_primaite/_simulator/_system/test_software.py | 4 ++-- 5 files changed, 24 insertions(+), 14 deletions(-) diff --git a/src/primaite/simulator/network/creation.py b/src/primaite/simulator/network/creation.py index 255b7bf5..089ed00d 100644 --- a/src/primaite/simulator/network/creation.py +++ b/src/primaite/simulator/network/creation.py @@ -155,7 +155,12 @@ class OfficeLANAdder(NetworkNodeAdder, discriminator="office-lan"): # Create a core switch if more than one edge switch is needed if num_of_switches > 1: core_switch = Switch.from_config( - config={"type": "switch", "hostname": f"switch_core_{config.lan_name}", "start_up_duration": 0} + config={ + "type": "switch", + "hostname": f"switch_core_{config.lan_name}", + "start_up_duration": 0, + "num_ports": 24, + } ) core_switch.power_on() network.add_node(core_switch) @@ -183,7 +188,12 @@ class OfficeLANAdder(NetworkNodeAdder, discriminator="office-lan"): switch_port = 0 switch_n = 1 switch = Switch.from_config( - config={"type": "switch", "hostname": f"switch_edge_{switch_n}_{config.lan_name}", "start_up_duration": 0} + config={ + "type": "switch", + "hostname": f"switch_edge_{switch_n}_{config.lan_name}", + "start_up_duration": 0, + "num_ports": 24, + } ) switch.power_on() network.add_node(switch) @@ -207,6 +217,7 @@ class OfficeLANAdder(NetworkNodeAdder, discriminator="office-lan"): "type": "switch", "hostname": f"switch_edge_{switch_n}_{config.lan_name}", "start_up_duration": 0, + "num_ports": 24, } ) switch.power_on() diff --git a/tests/unit_tests/_primaite/_simulator/_network/_hardware/nodes/test_switch.py b/tests/unit_tests/_primaite/_simulator/_network/_hardware/nodes/test_switch.py index e45fe45d..94b1764d 100644 --- a/tests/unit_tests/_primaite/_simulator/_network/_hardware/nodes/test_switch.py +++ b/tests/unit_tests/_primaite/_simulator/_network/_hardware/nodes/test_switch.py @@ -17,4 +17,3 @@ def switch() -> Switch: def test_describe_state(switch): state = switch.describe_state() assert len(state.get("ports")) is 8 - assert state.get("num_ports") is 8 diff --git a/tests/unit_tests/_primaite/_simulator/_system/_applications/test_applications.py b/tests/unit_tests/_primaite/_simulator/_system/_applications/test_applications.py index dd29f18e..6cccad91 100644 --- a/tests/unit_tests/_primaite/_simulator/_system/_applications/test_applications.py +++ b/tests/unit_tests/_primaite/_simulator/_system/_applications/test_applications.py @@ -18,7 +18,7 @@ def test_scan(application): def test_run_application(application): assert application.operating_state == ApplicationOperatingState.CLOSED - assert application.health_state_actual == SoftwareHealthState.UNUSED + assert application.health_state_actual == SoftwareHealthState.GOOD application.run() assert application.operating_state == ApplicationOperatingState.RUNNING @@ -37,9 +37,9 @@ def test_close_application(application): def test_application_describe_states(application): assert application.operating_state == ApplicationOperatingState.CLOSED - assert application.health_state_actual == SoftwareHealthState.UNUSED + assert application.health_state_actual == SoftwareHealthState.GOOD - assert SoftwareHealthState.UNUSED.value == application.describe_state().get("health_state_actual") + assert SoftwareHealthState.GOOD.value == application.describe_state().get("health_state_actual") application.run() assert SoftwareHealthState.GOOD.value == application.describe_state().get("health_state_actual") diff --git a/tests/unit_tests/_primaite/_simulator/_system/_services/test_services.py b/tests/unit_tests/_primaite/_simulator/_system/_services/test_services.py index 5598e1a7..fe78aa65 100644 --- a/tests/unit_tests/_primaite/_simulator/_system/_services/test_services.py +++ b/tests/unit_tests/_primaite/_simulator/_system/_services/test_services.py @@ -22,7 +22,7 @@ def test_scan(service): def test_start_service(service): assert service.operating_state == ServiceOperatingState.STOPPED - assert service.health_state_actual == SoftwareHealthState.UNUSED + assert service.health_state_actual == SoftwareHealthState.GOOD service.start() assert service.operating_state == ServiceOperatingState.RUNNING @@ -43,7 +43,7 @@ def test_pause_and_resume_service(service): assert service.operating_state == ServiceOperatingState.STOPPED service.resume() assert service.operating_state == ServiceOperatingState.STOPPED - assert service.health_state_actual == SoftwareHealthState.UNUSED + assert service.health_state_actual == SoftwareHealthState.GOOD service.start() assert service.health_state_actual == SoftwareHealthState.GOOD @@ -58,11 +58,11 @@ def test_pause_and_resume_service(service): def test_restart(service): assert service.operating_state == ServiceOperatingState.STOPPED - assert service.health_state_actual == SoftwareHealthState.UNUSED + assert service.health_state_actual == SoftwareHealthState.GOOD service.restart() # Service is STOPPED. Restart will only work if the service was PAUSED or RUNNING assert service.operating_state == ServiceOperatingState.STOPPED - assert service.health_state_actual == SoftwareHealthState.UNUSED + assert service.health_state_actual == SoftwareHealthState.GOOD service.start() assert service.operating_state == ServiceOperatingState.RUNNING @@ -157,11 +157,11 @@ def test_service_fixing(service): def test_enable_disable(service): service.disable() assert service.operating_state == ServiceOperatingState.DISABLED - assert service.health_state_actual == SoftwareHealthState.UNUSED + assert service.health_state_actual == SoftwareHealthState.GOOD service.enable() assert service.operating_state == ServiceOperatingState.STOPPED - assert service.health_state_actual == SoftwareHealthState.UNUSED + assert service.health_state_actual == SoftwareHealthState.GOOD def test_overwhelm_service(service): diff --git a/tests/unit_tests/_primaite/_simulator/_system/test_software.py b/tests/unit_tests/_primaite/_simulator/_system/test_software.py index 8c39c41d..9ad0dbcb 100644 --- a/tests/unit_tests/_primaite/_simulator/_system/test_software.py +++ b/tests/unit_tests/_primaite/_simulator/_system/test_software.py @@ -39,6 +39,6 @@ def test_software_creation(software): def test_software_set_health_state(software): - assert software.health_state_actual == SoftwareHealthState.UNUSED - software.set_health_state(SoftwareHealthState.GOOD) assert software.health_state_actual == SoftwareHealthState.GOOD + software.set_health_state(SoftwareHealthState.COMPROMISED) + assert software.health_state_actual == SoftwareHealthState.COMPROMISED From 22b197a79aaf287895398083e75396a1c9d7c04f Mon Sep 17 00:00:00 2001 From: Charlie Crane Date: Tue, 18 Feb 2025 09:13:33 +0000 Subject: [PATCH 31/37] Removal of leftover comment from software.py --- src/primaite/simulator/system/software.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/primaite/simulator/system/software.py b/src/primaite/simulator/system/software.py index 8fb64d73..950f77c6 100644 --- a/src/primaite/simulator/system/software.py +++ b/src/primaite/simulator/system/software.py @@ -110,7 +110,6 @@ class Software(SimComponent, ABC): "The folder on the file system the Software uses." _fixing_countdown: Optional[int] = None "Current number of ticks left to patch the software." - # parent: Optional[Node] = None def __init__(self, **kwargs): super().__init__(**kwargs) From 8c7f8cd0ecb257df594825201dce6b598f35f5cf Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Tue, 18 Feb 2025 11:36:22 +0000 Subject: [PATCH 32/37] #3075: Fix final two demo notebooks --- .../create-simulation_demo.ipynb | 49 ++++++++++++++++--- .../network_simulator_demo.ipynb | 16 +----- src/primaite/simulator/network/networks.py | 2 +- 3 files changed, 45 insertions(+), 22 deletions(-) diff --git a/src/primaite/simulator/_package_data/create-simulation_demo.ipynb b/src/primaite/simulator/_package_data/create-simulation_demo.ipynb index 690e7856..46f03be6 100644 --- a/src/primaite/simulator/_package_data/create-simulation_demo.ipynb +++ b/src/primaite/simulator/_package_data/create-simulation_demo.ipynb @@ -6,7 +6,7 @@ "source": [ "# Build a simulation using the Python API\n", "\n", - "© Crown-owned copyright 2024, Defence Science and Technology Laboratory UK\n", + "© Crown-owned copyright 2025, Defence Science and Technology Laboratory UK\n", "\n", "Currently, this notebook manipulates the simulation by directly placing objects inside of the attributes of the network and domain. It should be refactored when proper methods exist for adding these objects." ] @@ -70,9 +70,23 @@ "metadata": {}, "outputs": [], "source": [ - "my_pc = Computer(hostname=\"Computer\", ip_address=\"192.168.1.10\", subnet_mask=\"255.255.255.0\")\n", + "my_pc = Computer.from_config(\n", + " config={\n", + " \"type\": \"computer\",\n", + " \"hostname\":\"pc_1\",\n", + " \"ip_address\":\"192.168.1.10\",\n", + " \"subnet_mask\":\"255.255.255.0\",\n", + " }\n", + " )\n", "net.add_node(my_pc)\n", - "my_server = Server(hostname=\"Server\", ip_address=\"192.168.1.11\", subnet_mask=\"255.255.255.0\")\n", + "my_server = Server.from_config(\n", + " config={\n", + " \"type\": \"server\",\n", + " \"hostname\":\"Server\",\n", + " \"ip_address\":\"192.168.1.11\",\n", + " \"subnet_mask\":\"255.255.255.0\"\n", + " }\n", + ")\n", "net.add_node(my_server)\n" ] }, @@ -99,7 +113,13 @@ "metadata": {}, "outputs": [], "source": [ - "my_switch = Switch(hostname=\"switch1\", num_ports=12)\n", + "my_switch = Switch.from_config(\n", + " config = {\n", + " \"type\":\"switch\",\n", + " \"hostname\":\"switch1\",\n", + " \"num_ports\":12\n", + " }\n", + ")\n", "net.add_node(my_switch)\n", "\n", "pc_nic = NIC(ip_address=\"130.1.1.1\", gateway=\"130.1.1.255\", subnet_mask=\"255.255.255.0\")\n", @@ -163,16 +183,30 @@ "metadata": {}, "outputs": [], "source": [ + "from pydantic import Field\n", + "\n", "from pathlib import Path\n", "from primaite.simulator.system.applications.application import Application, ApplicationOperatingState\n", "from primaite.simulator.system.software import SoftwareHealthState, SoftwareCriticality\n", "from primaite.simulator.file_system.file_system import FileSystem\n", "from primaite.utils.validation.ip_protocol import PROTOCOL_LOOKUP\n", "from primaite.utils.validation.port import PORT_LOOKUP\n", + "from primaite.simulator.system.core.sys_log import SysLog\n", "\n", "\n", "# no applications exist yet so we will create our own.\n", "class MSPaint(Application, discriminator=\"MSPaint\"):\n", + " class ConfigSchema(Application.ConfigSchema):\n", + " type: str = \"MSPaint\"\n", + "\n", + " config: ConfigSchema = Field(default_factory=lambda: MSPaint.ConfigSchema())\n", + "\n", + " def __init__(self, **kwargs):\n", + " kwargs[\"name\"] = \"MSPaint\"\n", + " kwargs[\"port\"] = PORT_LOOKUP[\"HTTP\"]\n", + " kwargs[\"protocol\"] = PROTOCOL_LOOKUP[\"NONE\"]\n", + " super().__init__(**kwargs)\n", + "\n", " def describe_state(self):\n", " return super().describe_state()" ] @@ -183,7 +217,8 @@ "metadata": {}, "outputs": [], "source": [ - "mspaint = MSPaint(name = \"mspaint\", health_state_actual=SoftwareHealthState.GOOD, health_state_visible=SoftwareHealthState.GOOD, criticality=SoftwareCriticality.MEDIUM, port=PORT_LOOKUP[\"HTTP\"], protocol = PROTOCOL_LOOKUP[\"NONE\"],operating_state=ApplicationOperatingState.RUNNING,execution_control_status='manual', file_system=FileSystem(sys_log=SysLog(hostname=\"Test\"), sim_root=Path(__name__).parent),)" + "my_pc.software_manager.install(MSPaint)\n", + "mspaint = my_pc.software_manager.software.get(\"MSPaint\")" ] }, { @@ -250,7 +285,7 @@ ], "metadata": { "kernelspec": { - "display_name": ".venv", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -264,7 +299,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.11" + "version": "3.10.12" } }, "nbformat": 4, diff --git a/src/primaite/simulator/_package_data/network_simulator_demo.ipynb b/src/primaite/simulator/_package_data/network_simulator_demo.ipynb index 58c07fe5..be930ac0 100644 --- a/src/primaite/simulator/_package_data/network_simulator_demo.ipynb +++ b/src/primaite/simulator/_package_data/network_simulator_demo.ipynb @@ -7,7 +7,7 @@ "source": [ "# PrimAITE Router Simulation Demo\n", "\n", - "© Crown-owned copyright 2024, Defence Science and Technology Laboratory UK\n", + "© Crown-owned copyright 2025, Defence Science and Technology Laboratory UK\n", "\n", "This demo uses a modified version of the ARCD Use Case 2 Network (seen below) to demonstrate the capabilities of the Network simulator in PrimAITE." ] @@ -650,21 +650,9 @@ ], "metadata": { "kernelspec": { - "display_name": ".venv", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.11" } }, "nbformat": 4, diff --git a/src/primaite/simulator/network/networks.py b/src/primaite/simulator/network/networks.py index 5e677679..5d558e80 100644 --- a/src/primaite/simulator/network/networks.py +++ b/src/primaite/simulator/network/networks.py @@ -295,7 +295,7 @@ def arcd_uc2_network() -> Network: # Security Suite security_suite_cfg = { "type": "server", - "hostname": "backup_server", + "hostname": "security_suite", "ip_address": "192.168.1.110", "subnet_mask": "255.255.255.0", "default_gateway": "192.168.1.1", From 72f80e55da0a17766975bf1d73aaff75786ec303 Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Tue, 18 Feb 2025 11:51:35 +0000 Subject: [PATCH 33/37] #3075: Move demo notebook to correct folder. --- .../_package_data => notebooks}/create-simulation_demo.ipynb | 0 .../_package_data => notebooks}/network_simulator_demo.ipynb | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename src/primaite/{simulator/_package_data => notebooks}/create-simulation_demo.ipynb (100%) rename src/primaite/{simulator/_package_data => notebooks}/network_simulator_demo.ipynb (100%) diff --git a/src/primaite/simulator/_package_data/create-simulation_demo.ipynb b/src/primaite/notebooks/create-simulation_demo.ipynb similarity index 100% rename from src/primaite/simulator/_package_data/create-simulation_demo.ipynb rename to src/primaite/notebooks/create-simulation_demo.ipynb diff --git a/src/primaite/simulator/_package_data/network_simulator_demo.ipynb b/src/primaite/notebooks/network_simulator_demo.ipynb similarity index 100% rename from src/primaite/simulator/_package_data/network_simulator_demo.ipynb rename to src/primaite/notebooks/network_simulator_demo.ipynb From 90d7af3ff9fb0bc5fd60515b3d30752d7da27493 Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Tue, 18 Feb 2025 12:42:33 +0000 Subject: [PATCH 34/37] #3075: Remove references to src/primaite/simulator/_package_data after notebook move. --- .azure/azure-ci-build-pipeline.yaml | 4 ---- MANIFEST.in | 1 - 2 files changed, 5 deletions(-) diff --git a/.azure/azure-ci-build-pipeline.yaml b/.azure/azure-ci-build-pipeline.yaml index 624c9ca4..20c5b239 100644 --- a/.azure/azure-ci-build-pipeline.yaml +++ b/.azure/azure-ci-build-pipeline.yaml @@ -113,8 +113,6 @@ stages: - script: | pytest --nbmake -n=auto src/primaite/notebooks --junit-xml=./notebook-tests/notebooks.xml notebooks_exit_code=$? - pytest --nbmake -n=auto src/primaite/simulator/_package_data --junit-xml=./notebook-tests/package-notebooks.xml - package_notebooks_exit_code=$? # Fail step if either of these do not have exit code 0 if [ $notebooks_exit_code -ne 0 ] || [ $package_notebooks_exit_code -ne 0 ]; then exit 1 @@ -126,8 +124,6 @@ stages: - script: | pytest --nbmake -n=auto src/primaite/notebooks --junit-xml=./notebook-tests/notebooks.xml set notebooks_exit_code=%ERRORLEVEL% - pytest --nbmake -n=auto src/primaite/simulator/_package_data --junit-xml=./notebook-tests/package-notebooks.xml - set package_notebooks_exit_code=%ERRORLEVEL% rem Fail step if either of these do not have exit code 0 if %notebooks_exit_code% NEQ 0 exit /b 1 if %package_notebooks_exit_code% NEQ 0 exit /b 1 diff --git a/MANIFEST.in b/MANIFEST.in index 2ac7b306..51ae4ddf 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,2 @@ include src/primaite/setup/_package_data/primaite_config.yaml include src/primaite/config/_package_data/*.yaml -include src/primaite/simulator/_package_data/*.ipynb From f5cfaeedab633498274bbc05e34679dea19fec7a Mon Sep 17 00:00:00 2001 From: Archer Bowen Date: Fri, 21 Feb 2025 14:57:48 +0000 Subject: [PATCH 35/37] #3075 Cleaned up yaml snippets that were rendering as a strings. It seems that adding '...' to yaml snippets seems to end up causing the snippets to render as strings rather than actual yaml. --- ...ommand-and-Control-E2E-Demonstration.ipynb | 66 ++++++++----------- 1 file changed, 28 insertions(+), 38 deletions(-) diff --git a/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb b/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb index 6dc8e077..f187c8d5 100644 --- a/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb +++ b/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb @@ -120,6 +120,9 @@ " options:\n", " node_name: web_server\n", " c2_server_ip_address: 192.168.10.21\n", + " keep_alive_frequency: 10\n", + " masquerade_protocol: tcp\n", + " masquerade_port: dns\n", " 9:\n", " action: configure-c2-beacon\n", " options:\n", @@ -235,23 +238,18 @@ "The yaml snippet below shows all the relevant agent options for this action:\n", "\n", "```yaml\n", + "\n", " action_space:\n", - " options:\n", - " nodes: # Node List\n", - " - node_name: web_server\n", - " ...\n", - " ...\n", " action_map:\n", - " ...\n", " 2:\n", " action: configure-c2-beacon\n", " options:\n", - " node_id: 0 # Node Index\n", - " config: # Further information about these config options can be found at the bottom of this notebook.\n", - " c2_server_ip_address: 192.168.10.21\n", - " keep_alive_frequency:\n", - " masquerade_protocol:\n", - " masquerade_port:\n", + " node_name: web_server\n", + " c2_server_ip_address: 192.168.10.21 # Further information about these config options can be found at the bottom of this notebook.\n", + " keep_alive_frequency:\n", + " masquerade_protocol:\n", + " masquerade_port:\n", + "\n", "```" ] }, @@ -279,20 +277,12 @@ "\n", "```yaml\n", " action_space:\n", - " options:\n", - " nodes: # Node List\n", - " - node_name: web_server\n", - " applications: \n", - " - application_name: c2-beacon\n", - " ...\n", - " ...\n", " action_map:\n", - " ...\n", " 3:\n", " action: node-application-execute\n", " options:\n", - " node_id: 0\n", - " application_id: 0\n", + " node_name: web_server\n", + " application_name: c2-beacon\n", "```" ] }, @@ -346,7 +336,6 @@ "``` yaml\n", " action_space:\n", " action_map:\n", - " ...\n", " 4:\n", " action: c2-server-terminal-command\n", " options:\n", @@ -396,14 +385,12 @@ "``` yaml\n", " action_space:\n", " action_map:\n", - " ...\n", " 5:\n", " action: c2-server-ransomware-configure\n", " options:\n", - " node_id: 1\n", - " config:\n", - " server_ip_address: 192.168.1.14\n", - " payload: ENCRYPT\n", + " node_name: client_1\n", + " server_ip_address: 192.168.1.14\n", + " payload: ENCRYPT\n", "```\n" ] }, @@ -442,7 +429,6 @@ "``` yaml\n", " action_space:\n", " action_map:\n", - " ...\n", " 6:\n", " action: c2-server-data-exfiltrate\n", " options:\n", @@ -451,7 +437,7 @@ " target_folder_name: \"database\"\n", " exfiltration_folder_name: \"spoils\"\n", " target_ip_address: \"192.168.1.14\"\n", - " username: \"admin\",\n", + " username: \"admin\"\n", " password: \"admin\"\n", "\n", "```" @@ -500,7 +486,6 @@ "\n", "``` yaml\n", " action_space:\n", - " ...\n", " action_map:\n", " 7:\n", " action: c2-server-ransomware-launch\n", @@ -1313,13 +1298,18 @@ "source": [ "As demonstrated earlier, red agents can use the ``configure-c2-beacon`` action to configure these settings mid episode through the configuration options:\n", "\n", - "``` YAML\n", - "...\n", - " action: configure-c2-beacon\n", - " options:\n", - " node_name: web_server\n", - " config:\n", + "```YAML\n", + "\n", + " action_space:\n", + " action_map:\n", + " 8:\n", + " action: configure-c2-beacon\n", + " options:\n", + " node_name: web_server\n", " c2_server_ip_address: 192.168.10.21\n", + " keep_alive_frequency: 10\n", + " masquerade_protocol: tcp\n", + " masquerade_port: dns\n", "```" ] }, @@ -1684,7 +1674,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": ".venv", "language": "python", "name": "python3" }, From 892cdb82b91d621b92328dc0b2014348a08aac94 Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Tue, 25 Feb 2025 14:25:27 +0000 Subject: [PATCH 36/37] #3075: Code review changes. --- .azure/azure-ci-build-pipeline.yaml | 7 +++---- .../Data-Manipulation-Customising-Red-Agent.ipynb | 4 ++-- tests/assets/configs/basic_switched_network.yaml | 4 ++-- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.azure/azure-ci-build-pipeline.yaml b/.azure/azure-ci-build-pipeline.yaml index 20c5b239..b6f24777 100644 --- a/.azure/azure-ci-build-pipeline.yaml +++ b/.azure/azure-ci-build-pipeline.yaml @@ -113,8 +113,8 @@ stages: - script: | pytest --nbmake -n=auto src/primaite/notebooks --junit-xml=./notebook-tests/notebooks.xml notebooks_exit_code=$? - # Fail step if either of these do not have exit code 0 - if [ $notebooks_exit_code -ne 0 ] || [ $package_notebooks_exit_code -ne 0 ]; then + # Fail step if exit code not equal to 0 + if [ $notebooks_exit_code -ne 0 ]; then exit 1 fi displayName: 'Run notebooks on Linux and macOS' @@ -124,9 +124,8 @@ stages: - script: | pytest --nbmake -n=auto src/primaite/notebooks --junit-xml=./notebook-tests/notebooks.xml set notebooks_exit_code=%ERRORLEVEL% - rem Fail step if either of these do not have exit code 0 + rem Fail step if exit code not equal to 0 if %notebooks_exit_code% NEQ 0 exit /b 1 - if %package_notebooks_exit_code% NEQ 0 exit /b 1 displayName: 'Run notebooks on Windows' condition: eq(variables['Agent.OS'], 'Windows_NT') diff --git a/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb b/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb index 1eb814e3..5982407b 100644 --- a/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb +++ b/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb @@ -289,10 +289,10 @@ " action_space:\n", " action_map:\n", " 0:\n", - " action: do_nothing\n", + " action: do-nothing\n", " options: {}\n", " 1:\n", - " action: node_application_execute\n", + " action: node-application-execute\n", " options:\n", " node_name: client_1\n", " application_name: DataManipulationBot\n", diff --git a/tests/assets/configs/basic_switched_network.yaml b/tests/assets/configs/basic_switched_network.yaml index 2af48ea2..7339c34c 100644 --- a/tests/assets/configs/basic_switched_network.yaml +++ b/tests/assets/configs/basic_switched_network.yaml @@ -37,10 +37,10 @@ agents: action_space: action_map: 0: - action: do_nothing + action: do-nothing options: {} 1: - action: node_application_execute + action: node-application-execute options: node_name: client_2 application_name: web-browser From 1605b0989962a9c00c44cd63dc470a17b432e35e Mon Sep 17 00:00:00 2001 From: Marek Wolan Date: Wed, 26 Feb 2025 10:49:01 +0000 Subject: [PATCH 37/37] Apply suggestions from code review --- tests/assets/configs/basic_switched_network.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/assets/configs/basic_switched_network.yaml b/tests/assets/configs/basic_switched_network.yaml index 7339c34c..b57ed3e0 100644 --- a/tests/assets/configs/basic_switched_network.yaml +++ b/tests/assets/configs/basic_switched_network.yaml @@ -112,7 +112,7 @@ agents: action_space: action_map: 0: - action: do_nothing + action: do-nothing options: {} reward_function: