From 3f94c40434d11034cd1dc5dc9e07800c03bbcec4 Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Wed, 22 Jan 2025 10:49:42 +0000 Subject: [PATCH 01/24] Fix logger inititialisation in agents --- src/primaite/game/agent/interface.py | 7 +++ src/primaite/game/game.py | 1 + .../notebooks/Training-an-SB3-Agent.ipynb | 48 ++++++++++++++++--- 3 files changed, 49 insertions(+), 7 deletions(-) diff --git a/src/primaite/game/agent/interface.py b/src/primaite/game/agent/interface.py index aac898e1..6a6bc323 100644 --- a/src/primaite/game/agent/interface.py +++ b/src/primaite/game/agent/interface.py @@ -68,6 +68,10 @@ class AbstractAgent(BaseModel, ABC): ) reward_function: RewardFunction.ConfigSchema = Field(default_factory=lambda: RewardFunction.ConfigSchema()) + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.logger: AgentLog = AgentLog(agent_name=self.config.ref) + config: "AbstractAgent.ConfigSchema" = Field(default_factory=lambda: AbstractAgent.ConfigSchema()) logger: AgentLog = AgentLog(agent_name="Abstract_Agent") @@ -81,6 +85,7 @@ class AbstractAgent(BaseModel, ABC): def __init_subclass__(cls, identifier: Optional[str] = None, **kwargs: Any) -> None: super().__init_subclass__(**kwargs) + print("cls identifier:", identifier) if identifier is None: return if identifier in cls._registry: @@ -157,6 +162,8 @@ class AbstractAgent(BaseModel, ABC): def from_config(cls, config: Dict) -> AbstractAgent: """Grab the relevant agent class and construct an instance from a config dict.""" agent_type = config["type"] + print("agent_type:", agent_type) + print("cls._registry:", cls._registry) agent_class = cls._registry[agent_type] return agent_class(config=config) diff --git a/src/primaite/game/game.py b/src/primaite/game/game.py index f59117f4..c828a462 100644 --- a/src/primaite/game/game.py +++ b/src/primaite/game/game.py @@ -493,6 +493,7 @@ class PrimaiteGame: agents_cfg = cfg.get("agents", []) for agent_cfg in agents_cfg: + print("agent_cfg:", agent_cfg) new_agent = AbstractAgent.from_config(agent_cfg) game.agents[agent_cfg["ref"]] = new_agent if isinstance(new_agent, ProxyAgent): diff --git a/src/primaite/notebooks/Training-an-SB3-Agent.ipynb b/src/primaite/notebooks/Training-an-SB3-Agent.ipynb index 2b554475..0e3245ae 100644 --- a/src/primaite/notebooks/Training-an-SB3-Agent.ipynb +++ b/src/primaite/notebooks/Training-an-SB3-Agent.ipynb @@ -29,18 +29,28 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "cls identifier: AbstractScriptedAgent\n", + "cls identifier: ProxyAgent\n" + ] + } + ], "source": [ "from primaite.game.game import PrimaiteGame\n", "from primaite.session.environment import PrimaiteGymEnv\n", + "from primaite.game.agent.scripted_agents import probabilistic_agent\n", "import yaml" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -49,7 +59,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -69,9 +79,33 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "agent_cfg: {'ref': 'client_2_green_user', 'team': 'GREEN', 'type': 'ProbabilisticAgent', 'agent_settings': {'action_probabilities': {0: 0.3, 1: 0.6, 2: 0.1}}, 'action_space': {'action_map': {0: {'action': 'do_nothing', 'options': {}}, 1: {'action': 'node_application_execute', 'options': {'node_name': 'client_2', 'application_name': 'WebBrowser'}}, 2: {'action': 'node_application_execute', 'options': {'node_name': 'client_2', 'application_name': 'DatabaseClient'}}}}, 'reward_function': {'reward_components': [{'type': 'WEBPAGE_UNAVAILABLE_PENALTY', 'weight': 0.25, 'options': {'node_hostname': 'client_2'}}, {'type': 'GREEN_ADMIN_DATABASE_UNREACHABLE_PENALTY', 'weight': 0.05, 'options': {'node_hostname': 'client_2'}}]}}\n", + "agent_type: ProbabilisticAgent\n", + "cls._registry: {'AbstractScriptedAgent': , 'ProxyAgent': }\n" + ] + }, + { + "ename": "KeyError", + "evalue": "'ProbabilisticAgent'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[4], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m gym \u001b[38;5;241m=\u001b[39m \u001b[43mPrimaiteGymEnv\u001b[49m\u001b[43m(\u001b[49m\u001b[43menv_config\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcfg\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/session/environment.py:71\u001b[0m, in \u001b[0;36mPrimaiteGymEnv.__init__\u001b[0;34m(self, env_config)\u001b[0m\n\u001b[1;32m 69\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mio \u001b[38;5;241m=\u001b[39m PrimaiteIO\u001b[38;5;241m.\u001b[39mfrom_config(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mepisode_scheduler(\u001b[38;5;241m0\u001b[39m)\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mio_settings\u001b[39m\u001b[38;5;124m\"\u001b[39m, {}))\n\u001b[1;32m 70\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Handles IO for the environment. This produces sys logs, agent logs, etc.\"\"\"\u001b[39;00m\n\u001b[0;32m---> 71\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgame: PrimaiteGame \u001b[38;5;241m=\u001b[39m \u001b[43mPrimaiteGame\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfrom_config\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mepisode_scheduler\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 72\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Current game.\"\"\"\u001b[39;00m\n\u001b[1;32m 73\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_agent_name \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mnext\u001b[39m(\u001b[38;5;28miter\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgame\u001b[38;5;241m.\u001b[39mrl_agents))\n", + "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/game/game.py:497\u001b[0m, in \u001b[0;36mPrimaiteGame.from_config\u001b[0;34m(cls, cfg)\u001b[0m\n\u001b[1;32m 495\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m agent_cfg \u001b[38;5;129;01min\u001b[39;00m agents_cfg:\n\u001b[1;32m 496\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124magent_cfg:\u001b[39m\u001b[38;5;124m\"\u001b[39m, agent_cfg)\n\u001b[0;32m--> 497\u001b[0m new_agent \u001b[38;5;241m=\u001b[39m \u001b[43mAbstractAgent\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfrom_config\u001b[49m\u001b[43m(\u001b[49m\u001b[43magent_cfg\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 498\u001b[0m game\u001b[38;5;241m.\u001b[39magents[agent_cfg[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mref\u001b[39m\u001b[38;5;124m\"\u001b[39m]] \u001b[38;5;241m=\u001b[39m new_agent\n\u001b[1;32m 499\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(new_agent, ProxyAgent):\n", + "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/game/agent/interface.py:163\u001b[0m, in \u001b[0;36mAbstractAgent.from_config\u001b[0;34m(cls, config)\u001b[0m\n\u001b[1;32m 161\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124magent_type:\u001b[39m\u001b[38;5;124m\"\u001b[39m, agent_type)\n\u001b[1;32m 162\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcls._registry:\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28mcls\u001b[39m\u001b[38;5;241m.\u001b[39m_registry)\n\u001b[0;32m--> 163\u001b[0m agent_class \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mcls\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_registry\u001b[49m\u001b[43m[\u001b[49m\u001b[43magent_type\u001b[49m\u001b[43m]\u001b[49m\n\u001b[1;32m 164\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m agent_class(config\u001b[38;5;241m=\u001b[39mconfig)\n", + "\u001b[0;31mKeyError\u001b[0m: 'ProbabilisticAgent'" + ] + } + ], "source": [ "gym = PrimaiteGymEnv(env_config=cfg)" ] @@ -191,7 +225,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.11" + "version": "3.10.12" } }, "nbformat": 4, From 0aa691752aa268880f1a9f7531286e5a159136d0 Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Wed, 22 Jan 2025 12:20:45 +0000 Subject: [PATCH 02/24] #3075: Tidy up debug print statements. --- src/primaite/game/agent/interface.py | 3 -- src/primaite/game/game.py | 2 +- .../notebooks/Training-an-SB3-Agent.ipynb | 51 ++++--------------- 3 files changed, 10 insertions(+), 46 deletions(-) diff --git a/src/primaite/game/agent/interface.py b/src/primaite/game/agent/interface.py index 6a6bc323..5dab6aac 100644 --- a/src/primaite/game/agent/interface.py +++ b/src/primaite/game/agent/interface.py @@ -85,7 +85,6 @@ class AbstractAgent(BaseModel, ABC): def __init_subclass__(cls, identifier: Optional[str] = None, **kwargs: Any) -> None: super().__init_subclass__(**kwargs) - print("cls identifier:", identifier) if identifier is None: return if identifier in cls._registry: @@ -162,8 +161,6 @@ class AbstractAgent(BaseModel, ABC): def from_config(cls, config: Dict) -> AbstractAgent: """Grab the relevant agent class and construct an instance from a config dict.""" agent_type = config["type"] - print("agent_type:", agent_type) - print("cls._registry:", cls._registry) agent_class = cls._registry[agent_type] return agent_class(config=config) diff --git a/src/primaite/game/game.py b/src/primaite/game/game.py index c828a462..d9f6fa3c 100644 --- a/src/primaite/game/game.py +++ b/src/primaite/game/game.py @@ -493,7 +493,7 @@ class PrimaiteGame: agents_cfg = cfg.get("agents", []) for agent_cfg in agents_cfg: - print("agent_cfg:", agent_cfg) + new_agent = AbstractAgent.from_config(agent_cfg) game.agents[agent_cfg["ref"]] = new_agent if isinstance(new_agent, ProxyAgent): diff --git a/src/primaite/notebooks/Training-an-SB3-Agent.ipynb b/src/primaite/notebooks/Training-an-SB3-Agent.ipynb index 0e3245ae..d3492f92 100644 --- a/src/primaite/notebooks/Training-an-SB3-Agent.ipynb +++ b/src/primaite/notebooks/Training-an-SB3-Agent.ipynb @@ -29,18 +29,9 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "cls identifier: AbstractScriptedAgent\n", - "cls identifier: ProxyAgent\n" - ] - } - ], + "outputs": [], "source": [ "from primaite.game.game import PrimaiteGame\n", "from primaite.session.environment import PrimaiteGymEnv\n", @@ -50,7 +41,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -59,7 +50,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -79,33 +70,9 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "agent_cfg: {'ref': 'client_2_green_user', 'team': 'GREEN', 'type': 'ProbabilisticAgent', 'agent_settings': {'action_probabilities': {0: 0.3, 1: 0.6, 2: 0.1}}, 'action_space': {'action_map': {0: {'action': 'do_nothing', 'options': {}}, 1: {'action': 'node_application_execute', 'options': {'node_name': 'client_2', 'application_name': 'WebBrowser'}}, 2: {'action': 'node_application_execute', 'options': {'node_name': 'client_2', 'application_name': 'DatabaseClient'}}}}, 'reward_function': {'reward_components': [{'type': 'WEBPAGE_UNAVAILABLE_PENALTY', 'weight': 0.25, 'options': {'node_hostname': 'client_2'}}, {'type': 'GREEN_ADMIN_DATABASE_UNREACHABLE_PENALTY', 'weight': 0.05, 'options': {'node_hostname': 'client_2'}}]}}\n", - "agent_type: ProbabilisticAgent\n", - "cls._registry: {'AbstractScriptedAgent': , 'ProxyAgent': }\n" - ] - }, - { - "ename": "KeyError", - "evalue": "'ProbabilisticAgent'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[4], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m gym \u001b[38;5;241m=\u001b[39m \u001b[43mPrimaiteGymEnv\u001b[49m\u001b[43m(\u001b[49m\u001b[43menv_config\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcfg\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/session/environment.py:71\u001b[0m, in \u001b[0;36mPrimaiteGymEnv.__init__\u001b[0;34m(self, env_config)\u001b[0m\n\u001b[1;32m 69\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mio \u001b[38;5;241m=\u001b[39m PrimaiteIO\u001b[38;5;241m.\u001b[39mfrom_config(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mepisode_scheduler(\u001b[38;5;241m0\u001b[39m)\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mio_settings\u001b[39m\u001b[38;5;124m\"\u001b[39m, {}))\n\u001b[1;32m 70\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Handles IO for the environment. This produces sys logs, agent logs, etc.\"\"\"\u001b[39;00m\n\u001b[0;32m---> 71\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgame: PrimaiteGame \u001b[38;5;241m=\u001b[39m \u001b[43mPrimaiteGame\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfrom_config\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mepisode_scheduler\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 72\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Current game.\"\"\"\u001b[39;00m\n\u001b[1;32m 73\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_agent_name \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mnext\u001b[39m(\u001b[38;5;28miter\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgame\u001b[38;5;241m.\u001b[39mrl_agents))\n", - "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/game/game.py:497\u001b[0m, in \u001b[0;36mPrimaiteGame.from_config\u001b[0;34m(cls, cfg)\u001b[0m\n\u001b[1;32m 495\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m agent_cfg \u001b[38;5;129;01min\u001b[39;00m agents_cfg:\n\u001b[1;32m 496\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124magent_cfg:\u001b[39m\u001b[38;5;124m\"\u001b[39m, agent_cfg)\n\u001b[0;32m--> 497\u001b[0m new_agent \u001b[38;5;241m=\u001b[39m \u001b[43mAbstractAgent\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfrom_config\u001b[49m\u001b[43m(\u001b[49m\u001b[43magent_cfg\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 498\u001b[0m game\u001b[38;5;241m.\u001b[39magents[agent_cfg[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mref\u001b[39m\u001b[38;5;124m\"\u001b[39m]] \u001b[38;5;241m=\u001b[39m new_agent\n\u001b[1;32m 499\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(new_agent, ProxyAgent):\n", - "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/game/agent/interface.py:163\u001b[0m, in \u001b[0;36mAbstractAgent.from_config\u001b[0;34m(cls, config)\u001b[0m\n\u001b[1;32m 161\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124magent_type:\u001b[39m\u001b[38;5;124m\"\u001b[39m, agent_type)\n\u001b[1;32m 162\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcls._registry:\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28mcls\u001b[39m\u001b[38;5;241m.\u001b[39m_registry)\n\u001b[0;32m--> 163\u001b[0m agent_class \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mcls\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_registry\u001b[49m\u001b[43m[\u001b[49m\u001b[43magent_type\u001b[49m\u001b[43m]\u001b[49m\n\u001b[1;32m 164\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m agent_class(config\u001b[38;5;241m=\u001b[39mconfig)\n", - "\u001b[0;31mKeyError\u001b[0m: 'ProbabilisticAgent'" - ] - } - ], + "outputs": [], "source": [ "gym = PrimaiteGymEnv(env_config=cfg)" ] @@ -134,7 +101,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ @@ -166,7 +133,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -211,7 +178,7 @@ ], "metadata": { "kernelspec": { - "display_name": ".venv", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, From 8eeba691a9755e1bc237f0687bba2e82061a8fdb Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Thu, 23 Jan 2025 15:16:49 +0000 Subject: [PATCH 03/24] #3075: Remove unnecessary print statement. --- src/primaite/game/agent/actions/node.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/primaite/game/agent/actions/node.py b/src/primaite/game/agent/actions/node.py index fbab18f0..5e1b6725 100644 --- a/src/primaite/game/agent/actions/node.py +++ b/src/primaite/game/agent/actions/node.py @@ -36,7 +36,6 @@ class NodeAbstractAction(AbstractAction, identifier="node_abstract"): @classmethod def form_request(cls, config: ConfigSchema) -> RequestFormat: """Return the action formatted as a request which can be ingested by the PrimAITE simulation.""" - print(config) return ["network", "node", config.node_name, config.verb] From f4b73057d2094f9c248ad8f3ae757501378afff3 Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Mon, 27 Jan 2025 17:08:33 +0000 Subject: [PATCH 04/24] #3075: Update notebooks to reflect extension changes. --- src/primaite/notebooks/Action-masking.ipynb | 17 +++++----- ...ommand-and-Control-E2E-Demonstration.ipynb | 16 +++++++-- ...a-Manipulation-Customising-Red-Agent.ipynb | 33 +++++++++++++------ .../Data-Manipulation-E2E-Demonstration.ipynb | 15 +++++---- .../Getting-Information-Out-Of-PrimAITE.ipynb | 6 ++-- ...ege-Escalation-and-Data-Loss-Example.ipynb | 15 +++++++-- .../Training-an-RLLIB-MARL-System.ipynb | 20 +++++++++-- .../notebooks/Training-an-RLLib-Agent.ipynb | 23 ++++++++++--- .../notebooks/Using-Episode-Schedules.ipynb | 15 ++------- src/primaite/notebooks/multi-processing.ipynb | 18 +++++----- 10 files changed, 116 insertions(+), 62 deletions(-) diff --git a/src/primaite/notebooks/Action-masking.ipynb b/src/primaite/notebooks/Action-masking.ipynb index 7fde0a49..64015080 100644 --- a/src/primaite/notebooks/Action-masking.ipynb +++ b/src/primaite/notebooks/Action-masking.ipynb @@ -22,13 +22,14 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "from primaite.session.environment import PrimaiteGymEnv\n", "from primaite.config.load import data_manipulation_config_path\n", - "from prettytable import PrettyTable" + "from prettytable import PrettyTable\n", + "from primaite.game.agent.scripted_agents import probabilistic_agent\n" ] }, { @@ -103,7 +104,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -116,7 +117,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ @@ -156,7 +157,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ @@ -167,7 +168,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "metadata": {}, "outputs": [], "source": [ @@ -204,7 +205,7 @@ ], "metadata": { "kernelspec": { - "display_name": ".venv", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -218,7 +219,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.11" + "version": "3.10.12" } }, "nbformat": 4, diff --git a/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb b/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb index 66a684de..625510b8 100644 --- a/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb +++ b/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb @@ -16,6 +16,15 @@ "execution_count": null, "metadata": {}, "outputs": [], + "source": [ + "!primaite setup" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], "source": [ "# Imports\n", "import yaml\n", @@ -27,7 +36,8 @@ "from primaite.simulator.system.applications.red_applications.c2.abstract_c2 import C2Command\n", "from primaite.simulator.system.applications.red_applications.ransomware_script import RansomwareScript\n", "from primaite.simulator.network.hardware.nodes.host.computer import Computer\n", - "from primaite.simulator.network.hardware.nodes.host.server import Server" + "from primaite.simulator.network.hardware.nodes.host.server import Server\n", + "from primaite.game.agent.scripted_agents import probabilistic_agent" ] }, { @@ -43,7 +53,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -1763,7 +1773,7 @@ ], "metadata": { "kernelspec": { - "display_name": ".venv", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, diff --git a/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb b/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb index 756fc44f..de6d3fe5 100644 --- a/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb +++ b/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb @@ -26,7 +26,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -36,12 +36,13 @@ "from primaite.game.agent.interface import AgentHistoryItem\n", "from primaite.session.environment import PrimaiteGymEnv\n", "import yaml\n", - "from pprint import pprint" + "from pprint import pprint\n", + "from primaite.game.agent.scripted_agents import probabilistic_agent, data_manipulation_bot" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -68,7 +69,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -79,7 +80,8 @@ " if red_action == 'do_nothing':\n", " red_str = 'DO NOTHING'\n", " elif red_action == 'node_application_execute':\n", - " client = \"client 1\" if red_info.parameters['node_id'] == 0 else \"client 2\"\n", + " client = \"client 1\" if red_info.parameters['node_name'] == 0 else \"client 2\"\n", + "\n", " red_str = f\"ATTACK from {client}\"\n", " return red_str" ] @@ -243,7 +245,8 @@ "outputs": [], "source": [ "change = yaml.safe_load(\"\"\"\n", - "start_settings:\n", + " possible_start_nodes: [client_1, client_2]\n", + " target_application: DataManipulationBot\n", " start_step: 25\n", " frequency: 20\n", " variance: 0\n", @@ -253,7 +256,9 @@ " cfg = yaml.safe_load(f)\n", " for agent in cfg['agents']:\n", " if agent['ref'] == \"data_manipulation_attacker\":\n", + " print(f\"{agent['agent_settings']=}\")\n", " agent['agent_settings'] = change\n", + " print(f\"{agent['agent_settings']=}\")\n", "\n", "env = PrimaiteGymEnv(env_config = cfg)\n", "env.reset()\n", @@ -310,9 +315,17 @@ "outputs": [], "source": [ "change = yaml.safe_load(\"\"\"\n", - "# TODO:\n", + " action_space:\n", + " action_map:\n", + " 0:\n", + " action: do_nothing\n", + " options: {}\n", + " 1:\n", + " action: node_application_execute\n", + " options:\n", + " node_name: client_1\n", + " application_name: DataManipulationBot\n", "\"\"\")\n", - "#TODO 2869 fix\n", "\n", "with open(data_manipulation_config_path(), 'r') as f:\n", " cfg = yaml.safe_load(f)\n", @@ -438,7 +451,7 @@ ], "metadata": { "kernelspec": { - "display_name": ".venv", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -452,7 +465,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.11" + "version": "3.10.12" } }, "nbformat": 4, diff --git a/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb b/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb index dbc6f0c1..3818bb18 100644 --- a/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb +++ b/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb @@ -382,7 +382,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": { "tags": [] }, @@ -394,7 +394,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": { "tags": [] }, @@ -405,7 +405,8 @@ "from primaite.session.environment import PrimaiteGymEnv\n", "from primaite.game.agent.interface import AgentHistoryItem\n", "import yaml\n", - "from pprint import pprint\n" + "from pprint import pprint\n", + "from primaite.game.agent.scripted_agents import probabilistic_agent, data_manipulation_bot\n" ] }, { @@ -429,9 +430,9 @@ " cfg = yaml.safe_load(f)\n", " # set success probability to 1.0 to avoid rerunning cells.\n", " cfg['simulation']['network']['nodes'][8]['applications'][0]['options']['data_manipulation_p_of_success'] = 1.0\n", - " cfg['simulation']['network']['nodes'][9]['applications'][0]['options']['data_manipulation_p_of_success'] = 1.0\n", + " cfg['simulation']['network']['nodes'][9]['applications'][1]['options']['data_manipulation_p_of_success'] = 1.0\n", " cfg['simulation']['network']['nodes'][8]['applications'][0]['options']['port_scan_p_of_success'] = 1.0\n", - " cfg['simulation']['network']['nodes'][9]['applications'][0]['options']['port_scan_p_of_success'] = 1.0\n", + " cfg['simulation']['network']['nodes'][9]['applications'][1]['options']['port_scan_p_of_success'] = 1.0\n", " # don't flatten observations so that we can see what is going on\n", " cfg['agents'][3]['agent_settings']['flatten_obs'] = False\n", "\n", @@ -450,7 +451,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -461,7 +462,7 @@ " if red_action == 'do_nothing':\n", " red_str = 'DO NOTHING'\n", " elif red_action == 'node_application_execute':\n", - " client = \"client 1\" if red_info.parameters['node_id'] == 0 else \"client 2\"\n", + " client = \"client 1\" if red_info.parameters['node_name'] == 0 else \"client 2\"\n", " red_str = f\"ATTACK from {client}\"\n", " return red_str" ] diff --git a/src/primaite/notebooks/Getting-Information-Out-Of-PrimAITE.ipynb b/src/primaite/notebooks/Getting-Information-Out-Of-PrimAITE.ipynb index f8691d7d..d7d60d76 100644 --- a/src/primaite/notebooks/Getting-Information-Out-Of-PrimAITE.ipynb +++ b/src/primaite/notebooks/Getting-Information-Out-Of-PrimAITE.ipynb @@ -32,6 +32,8 @@ "from primaite.session.environment import PrimaiteGymEnv\n", "from primaite.simulator.network.hardware.nodes.host.computer import Computer\n", "from notebook.services.config import ConfigManager\n", + "from primaite.game.agent.scripted_agents import probabilistic_agent\n", + "\n", "\n", "cm = ConfigManager().update('notebook', {'limit_output': 50}) # limit output lines to 50 - for neatness\n", "\n", @@ -157,7 +159,7 @@ ], "metadata": { "kernelspec": { - "display_name": "venv", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -171,7 +173,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.8" + "version": "3.10.12" } }, "nbformat": 4, diff --git a/src/primaite/notebooks/Privilege-Escalation-and-Data-Loss-Example.ipynb b/src/primaite/notebooks/Privilege-Escalation-and-Data-Loss-Example.ipynb index fcda4dbd..35d3813a 100644 --- a/src/primaite/notebooks/Privilege-Escalation-and-Data-Loss-Example.ipynb +++ b/src/primaite/notebooks/Privilege-Escalation-and-Data-Loss-Example.ipynb @@ -54,6 +54,15 @@ { "cell_type": "code", "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!primaite setup" + ] + }, + { + "cell_type": "code", + "execution_count": 2, "metadata": { "tags": [] }, @@ -68,7 +77,7 @@ "from primaite.simulator.network.hardware.nodes.host.server import Server\n", "from primaite.simulator.system.applications.database_client import DatabaseClient\n", "from primaite.simulator.system.applications.web_browser import WebBrowser\n", - "from primaite.simulator.system.services.database.database_service import DatabaseService" + "from primaite.simulator.system.services.database.database_service import DatabaseService\n" ] }, { @@ -80,7 +89,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": { "tags": [] }, @@ -103,7 +112,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": { "tags": [] }, diff --git a/src/primaite/notebooks/Training-an-RLLIB-MARL-System.ipynb b/src/primaite/notebooks/Training-an-RLLIB-MARL-System.ipynb index 76cab86a..dadb399e 100644 --- a/src/primaite/notebooks/Training-an-RLLIB-MARL-System.ipynb +++ b/src/primaite/notebooks/Training-an-RLLIB-MARL-System.ipynb @@ -40,6 +40,7 @@ "import ray\n", "from ray.rllib.algorithms.ppo import PPOConfig\n", "from primaite.session.ray_envs import PrimaiteRayMARLEnv\n", + "from primaite.game.agent.scripted_agents import probabilistic_agent\n", "\n", "with open(PRIMAITE_PATHS.user_config_path / 'example_config/data_manipulation_marl.yaml', 'r') as f:\n", " cfg = yaml.safe_load(f)\n", @@ -56,7 +57,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -102,7 +103,20 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "*** SIGTERM received at time=1737996337 on cpu 7 ***\n", + "PC: @ 0x7f3649b0fe2e (unknown) epoll_wait\n", + " @ 0x7f3649a2c520 (unknown) (unknown)\n", + "[2025-01-27 16:45:37,381 E 117142 117142] logging.cc:440: *** SIGTERM received at time=1737996337 on cpu 7 ***\n", + "[2025-01-27 16:45:37,381 E 117142 117142] logging.cc:440: PC: @ 0x7f3649b0fe2e (unknown) epoll_wait\n", + "[2025-01-27 16:45:37,381 E 117142 117142] logging.cc:440: @ 0x7f3649a2c520 (unknown) (unknown)\n" + ] + } + ], "source": [ "eval = algo.evaluate()" ] @@ -110,7 +124,7 @@ ], "metadata": { "kernelspec": { - "display_name": "venv", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, diff --git a/src/primaite/notebooks/Training-an-RLLib-Agent.ipynb b/src/primaite/notebooks/Training-an-RLLib-Agent.ipynb index 7252b046..64a9e7ab 100644 --- a/src/primaite/notebooks/Training-an-RLLib-Agent.ipynb +++ b/src/primaite/notebooks/Training-an-RLLib-Agent.ipynb @@ -32,6 +32,8 @@ "from primaite.session.ray_envs import PrimaiteRayEnv\n", "import ray\n", "from ray.rllib.algorithms.ppo import PPOConfig\n", + "from primaite.game.agent.scripted_agents import probabilistic_agent\n", + "\n", "\n", "# If you get an error saying this config file doesn't exist, you may need to run `primaite setup` in your command line\n", "# to copy the files to your user data path.\n", @@ -50,7 +52,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -96,7 +98,20 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "*** SIGTERM received at time=1737996055 on cpu 1 ***\n", + "PC: @ 0x7f6e254a6e2e (unknown) epoll_wait\n", + " @ 0x7f6e253c3520 (unknown) (unknown)\n", + "[2025-01-27 16:40:55,343 E 114171 114171] logging.cc:440: *** SIGTERM received at time=1737996055 on cpu 1 ***\n", + "[2025-01-27 16:40:55,343 E 114171 114171] logging.cc:440: PC: @ 0x7f6e254a6e2e (unknown) epoll_wait\n", + "[2025-01-27 16:40:55,344 E 114171 114171] logging.cc:440: @ 0x7f6e253c3520 (unknown) (unknown)\n" + ] + } + ], "source": [ "eval = algo.evaluate()" ] @@ -104,7 +119,7 @@ ], "metadata": { "kernelspec": { - "display_name": ".venv", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -118,7 +133,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.11" + "version": "3.10.12" } }, "nbformat": 4, diff --git a/src/primaite/notebooks/Using-Episode-Schedules.ipynb b/src/primaite/notebooks/Using-Episode-Schedules.ipynb index d08ca67b..311fe4fb 100644 --- a/src/primaite/notebooks/Using-Episode-Schedules.ipynb +++ b/src/primaite/notebooks/Using-Episode-Schedules.ipynb @@ -48,6 +48,7 @@ "from primaite.session.environment import PrimaiteGymEnv\n", "from primaite import PRIMAITE_PATHS\n", "from prettytable import PrettyTable\n", + "from primaite.game.agent.scripted_agents import probabilistic_agent, data_manipulation_bot\n", "scenario_path = PRIMAITE_PATHS.user_config_path / \"example_config/scenario_with_placeholders\"" ] }, @@ -409,21 +410,9 @@ ], "metadata": { "kernelspec": { - "display_name": "venv", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.11" } }, "nbformat": 4, diff --git a/src/primaite/notebooks/multi-processing.ipynb b/src/primaite/notebooks/multi-processing.ipynb index ad386f34..798bf3ff 100644 --- a/src/primaite/notebooks/multi-processing.ipynb +++ b/src/primaite/notebooks/multi-processing.ipynb @@ -29,7 +29,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 30, "metadata": {}, "outputs": [], "source": [ @@ -37,13 +37,13 @@ "from stable_baselines3 import PPO\n", "from stable_baselines3.common.utils import set_random_seed\n", "from stable_baselines3.common.vec_env import SubprocVecEnv\n", - "\n", - "from primaite.session.environment import PrimaiteGymEnv\n" + "from primaite.session.environment import PrimaiteGymEnv\n", + "from primaite.game.agent.scripted_agents import probabilistic_agent\n" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 31, "metadata": {}, "outputs": [], "source": [ @@ -52,7 +52,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 32, "metadata": {}, "outputs": [], "source": [ @@ -69,7 +69,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 33, "metadata": {}, "outputs": [], "source": [ @@ -90,7 +90,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 36, "metadata": {}, "outputs": [], "source": [ @@ -138,7 +138,7 @@ ], "metadata": { "kernelspec": { - "display_name": ".venv", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -152,7 +152,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.11" + "version": "3.10.12" } }, "nbformat": 4, From 4fb4c5e0f9e26b2d0c797beec562b8f35344fe74 Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Tue, 28 Jan 2025 17:18:07 +0000 Subject: [PATCH 05/24] #3075: Fix error in config file. --- src/primaite/config/_package_data/data_manipulation.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/primaite/config/_package_data/data_manipulation.yaml b/src/primaite/config/_package_data/data_manipulation.yaml index b0d5d087..fc48c8e6 100644 --- a/src/primaite/config/_package_data/data_manipulation.yaml +++ b/src/primaite/config/_package_data/data_manipulation.yaml @@ -81,7 +81,7 @@ agents: action: node_application_execute options: node_name: client_1 - application_name: WebBrowser + application_name: DatabaseClient reward_function: reward_components: From 32de95917e636c8febd08d38aa41492e7e02c606 Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Mon, 3 Feb 2025 11:33:21 +0000 Subject: [PATCH 06/24] #3075: Add repeat parameter to config file. --- .../notebooks/Data-Manipulation-E2E-Demonstration.ipynb | 3 +-- .../applications/red_applications/data_manipulation_bot.py | 2 ++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb b/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb index 3818bb18..4a34619f 100644 --- a/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb +++ b/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb @@ -405,8 +405,7 @@ "from primaite.session.environment import PrimaiteGymEnv\n", "from primaite.game.agent.interface import AgentHistoryItem\n", "import yaml\n", - "from pprint import pprint\n", - "from primaite.game.agent.scripted_agents import probabilistic_agent, data_manipulation_bot\n" + "from pprint import pprint\n" ] }, { diff --git a/src/primaite/simulator/system/applications/red_applications/data_manipulation_bot.py b/src/primaite/simulator/system/applications/red_applications/data_manipulation_bot.py index 392cdfba..79ef1e67 100644 --- a/src/primaite/simulator/system/applications/red_applications/data_manipulation_bot.py +++ b/src/primaite/simulator/system/applications/red_applications/data_manipulation_bot.py @@ -52,6 +52,7 @@ class DataManipulationBot(Application, identifier="DataManipulationBot"): payload: str = "DELETE" port_scan_p_of_success: float = 0.1 data_manipulation_p_of_success: float = 0.1 + repeat: bool = True config: "DataManipulationBot.ConfigSchema" = Field(default_factory=lambda: DataManipulationBot.ConfigSchema()) @@ -76,6 +77,7 @@ class DataManipulationBot(Application, identifier="DataManipulationBot"): self.payload = self.config.payload self.port_scan_p_of_success = self.config.port_scan_p_of_success self.data_manipulation_p_of_success = self.config.data_manipulation_p_of_success + self.repeat = self.config.repeat def describe_state(self) -> Dict: """ From d24f9872ffac89b6cf28830c073c598ef214cb2f Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Mon, 3 Feb 2025 17:06:12 +0000 Subject: [PATCH 07/24] #3075: C2C-E2E-Demo internal YAML changes. --- ...ommand-and-Control-E2E-Demonstration.ipynb | 1966 +++++++++++++++-- 1 file changed, 1747 insertions(+), 219 deletions(-) diff --git a/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb b/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb index 625510b8..2c1a94c8 100644 --- a/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb +++ b/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb @@ -13,16 +13,31 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-02-03 16:04:19,828: Performing the PrimAITE first-time setup...\n", + "2025-02-03 16:04:19,829: Building the PrimAITE app directories...\n", + "2025-02-03 16:04:19,829: Building primaite_config.yaml...\n", + "2025-02-03 16:04:19,829: Rebuilding the demo notebooks...\n", + "/home/nick/primaite/4.0.0a1-dev/notebooks/example_notebooks/Command-and-Control-E2E-Demonstration.ipynb\n", + "2025-02-03 16:04:19,831: Reset example notebook: /home/nick/primaite/4.0.0a1-dev/notebooks/example_notebooks/Command-and-Control-E2E-Demonstration.ipynb\n", + "2025-02-03 16:04:19,836: Rebuilding the example notebooks...\n", + "2025-02-03 16:04:19,840: PrimAITE setup complete!\n" + ] + } + ], "source": [ "!primaite setup" ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -36,8 +51,7 @@ "from primaite.simulator.system.applications.red_applications.c2.abstract_c2 import C2Command\n", "from primaite.simulator.system.applications.red_applications.ransomware_script import RansomwareScript\n", "from primaite.simulator.network.hardware.nodes.host.computer import Computer\n", - "from primaite.simulator.network.hardware.nodes.host.server import Server\n", - "from primaite.game.agent.scripted_agents import probabilistic_agent" + "from primaite.simulator.network.hardware.nodes.host.server import Server" ] }, { @@ -53,7 +67,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -63,24 +77,24 @@ " type: ProxyAgent\n", "\n", " action_space:\n", - " options:\n", - " nodes:\n", - " - node_name: web_server\n", - " applications:\n", - " - application_name: C2Beacon\n", - " - node_name: client_1\n", - " applications:\n", - " - application_name: C2Server\n", - " max_folders_per_node: 1\n", - " max_files_per_folder: 1\n", - " max_services_per_node: 2\n", - " max_nics_per_node: 8\n", - " max_acl_rules: 10\n", - " ip_list:\n", - " - 192.168.1.21\n", - " - 192.168.1.14\n", - " wildcard_list:\n", - " - 0.0.0.1\n", + " # options:\n", + " # nodes:\n", + " # - node_name: web_server\n", + " # applications:\n", + " # - application_name: C2Beacon\n", + " # - node_name: client_1\n", + " # applications:\n", + " # - application_name: C2Server\n", + " # max_folders_per_node: 1\n", + " # max_files_per_folder: 1\n", + " # max_services_per_node: 2\n", + " # max_nics_per_node: 8\n", + " # max_acl_rules: 10\n", + " # ip_list:\n", + " # - 192.168.1.21\n", + " # - 192.168.1.14\n", + " # wildcard_list:\n", + " # - 0.0.0.1\n", " action_map:\n", " 0:\n", " action: do_nothing\n", @@ -88,30 +102,30 @@ " 1:\n", " action: node_application_install\n", " options:\n", - " node_id: 0\n", + " node_name: web_server\n", " application_name: C2Beacon\n", " 2:\n", " action: configure_c2_beacon\n", " options:\n", - " node_id: 0\n", - " config:\n", - " c2_server_ip_address: 192.168.10.21\n", - " keep_alive_frequency:\n", - " masquerade_protocol:\n", - " masquerade_port:\n", + " node_name: web_server\n", + " # config:\n", + " c2_server_ip_address: 192.168.10.21\n", + " # keep_alive_frequency: 10\n", + " # masquerade_protocol: TCP\n", + " # masquerade_port: DNS\n", " 3:\n", " action: node_application_execute\n", " options:\n", - " node_id: 0\n", - " application_id: 0\n", + " node_name: web_server\n", + " application_name: C2Beacon\n", " 4:\n", " action: c2_server_terminal_command\n", " options:\n", - " node_id: 1\n", + " node_name: client_1\n", " ip_address:\n", - " account:\n", - " username: admin\n", - " password: admin\n", + " # account:\n", + " username: admin\n", + " password: admin\n", " commands:\n", " -\n", " - software_manager\n", @@ -121,44 +135,44 @@ " 5:\n", " action: c2_server_ransomware_configure\n", " options:\n", - " node_id: 1\n", - " config:\n", - " server_ip_address: 192.168.1.14\n", - " payload: ENCRYPT\n", + " node_name: client_1\n", + " # config:\n", + " server_ip_address: 192.168.1.14\n", + " payload: ENCRYPT\n", " 6:\n", " action: c2_server_data_exfiltrate\n", " options:\n", - " node_id: 1\n", + " node_name: client_1\n", " target_file_name: \"database.db\"\n", " target_folder_name: \"database\"\n", " exfiltration_folder_name: \"spoils\"\n", " target_ip_address: 192.168.1.14\n", - " account:\n", - " username: admin\n", - " password: admin\n", + " # account:\n", + " username: admin\n", + " password: admin\n", "\n", " 7:\n", " action: c2_server_ransomware_launch\n", " options:\n", - " node_id: 1\n", + " node_name: client_1\n", " 8:\n", " action: configure_c2_beacon\n", " options:\n", - " node_id: 0\n", - " config:\n", - " c2_server_ip_address: 192.168.10.21\n", - " keep_alive_frequency: 10\n", - " masquerade_protocol: TCP\n", - " masquerade_port: DNS\n", + " node_name: web_server\n", + " # config:\n", + " c2_server_ip_address: 192.168.10.21\n", + " # keep_alive_frequency: 10\n", + " # masquerade_protocol: TCP\n", + " # masquerade_port: DNS\n", " 9:\n", " action: configure_c2_beacon\n", " options:\n", - " node_id: 0\n", - " config:\n", - " c2_server_ip_address: 192.168.10.22\n", - " keep_alive_frequency:\n", - " masquerade_protocol:\n", - " masquerade_port:\n", + " node_name: web_server\n", + " # config:\n", + " c2_server_ip_address: 192.168.10.22\n", + " # keep_alive_frequency: 10\n", + " # masquerade_protocol: TCP\n", + " # masquerade_port: DNS\n", "\n", " reward_function:\n", " reward_components:\n", @@ -169,9 +183,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-03 16:04:24,734: PrimaiteGymEnv RNG seed = None\n" + ] + } + ], "source": [ "with open(data_manipulation_config_path()) as f:\n", " cfg = yaml.safe_load(f)\n", @@ -197,9 +219,35 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+--------------------------------------------------------------------------------------+\n", + "| client_1 Software Manager |\n", + "+---------------------+-------------+-----------------+--------------+------+----------+\n", + "| Name | Type | Operating State | Health State | Port | Protocol |\n", + "+---------------------+-------------+-----------------+--------------+------+----------+\n", + "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", + "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", + "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", + "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", + "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", + "| NMAP | Application | RUNNING | GOOD | None | none |\n", + "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", + "| UserManager | Service | RUNNING | GOOD | None | none |\n", + "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", + "| FTPClient | Service | RUNNING | GOOD | 21 | tcp |\n", + "| DataManipulationBot | Application | RUNNING | GOOD | None | none |\n", + "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", + "| C2Server | Application | RUNNING | GOOD | None | tcp |\n", + "+---------------------+-------------+-----------------+--------------+------+----------+\n" + ] + } + ], "source": [ "client_1: Computer = env.game.simulation.network.get_node_by_hostname(\"client_1\")\n", "client_1.software_manager.install(C2Server)\n", @@ -249,9 +297,34 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+-------------------------------------------------------------------------------------+\n", + "| web_server Software Manager |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| Name | Type | Operating State | Health State | Port | Protocol |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", + "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", + "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", + "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", + "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", + "| NMAP | Application | RUNNING | GOOD | None | none |\n", + "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", + "| UserManager | Service | RUNNING | GOOD | None | none |\n", + "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", + "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", + "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", + "| C2Beacon | Application | INSTALLING | UNUSED | None | tcp |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n" + ] + } + ], "source": [ "env.step(1)\n", "web_server: Computer = env.game.simulation.network.get_node_by_hostname(\"web_server\")\n", @@ -291,9 +364,41 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+-------------------------------------------------------------------------------------+\n", + "| web_server Software Manager |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| Name | Type | Operating State | Health State | Port | Protocol |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", + "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", + "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", + "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", + "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", + "| NMAP | Application | RUNNING | GOOD | None | none |\n", + "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", + "| UserManager | Service | RUNNING | GOOD | None | none |\n", + "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", + "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", + "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", + "| C2Beacon | Application | RUNNING | GOOD | None | tcp |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", + "| C2Beacon Running Status |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| False | 192.168.10.21 | 0 | 5 | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n" + ] + } + ], "source": [ "env.step(2)\n", "c2_beacon: C2Beacon = web_server.software_manager.software[\"C2Beacon\"]\n", @@ -332,18 +437,54 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "(0,\n", + " 0.0,\n", + " False,\n", + " False,\n", + " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=2, action='node_application_execute', parameters={'node_name': 'web_server', 'application_name': 'C2Beacon'}, request=['network', 'node', 'web_server', 'application', 'C2Beacon', 'execute'], response=RequestResponse(status='success', data={}), reward=0.0, reward_info={})}})" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "env.step(3)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", + "| C2Beacon Running Status |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| True | 192.168.10.21 | 1 | 5 | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "+-----------------------------------------------------------------------------------------------------+\n", + "| C2Server Running Status |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n", + "| True | 192.168.1.12 | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n" + ] + } + ], "source": [ "c2_beacon.show()\n", "c2_server.show()" @@ -406,18 +547,59 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "(0,\n", + " 0.0,\n", + " False,\n", + " False,\n", + " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=3, action='c2_server_terminal_command', parameters={'node_name': 'client_1', 'ip_address': None, 'username': 'admin', 'password': 'admin', 'commands': [['software_manager', 'application', 'install', 'RansomwareScript']]}, request=['network', 'node', 'client_1', 'application', 'C2Server', 'terminal_command', {'commands': [['software_manager', 'application', 'install', 'RansomwareScript']], 'ip_address': None, 'username': 'admin', 'password': 'admin'}], response=RequestResponse(status='success', data={0: RequestResponse(status='success', data={})}), reward=0.0, reward_info={})}})" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "env.step(4)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+--------------------------------------------------------------------------------------+\n", + "| client_1 Software Manager |\n", + "+---------------------+-------------+-----------------+--------------+------+----------+\n", + "| Name | Type | Operating State | Health State | Port | Protocol |\n", + "+---------------------+-------------+-----------------+--------------+------+----------+\n", + "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", + "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", + "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", + "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", + "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", + "| NMAP | Application | RUNNING | GOOD | None | none |\n", + "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", + "| UserManager | Service | RUNNING | GOOD | None | none |\n", + "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", + "| FTPClient | Service | RUNNING | GOOD | 21 | tcp |\n", + "| DataManipulationBot | Application | RUNNING | GOOD | None | none |\n", + "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", + "| C2Server | Application | RUNNING | GOOD | None | tcp |\n", + "+---------------------+-------------+-----------------+--------------+------+----------+\n" + ] + } + ], "source": [ "client_1.software_manager.show()" ] @@ -456,18 +638,66 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "(0,\n", + " 0.0,\n", + " False,\n", + " False,\n", + " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=4, action='c2_server_ransomware_configure', parameters={'node_name': 'client_1', 'server_ip_address': '192.168.1.14', 'payload': 'ENCRYPT'}, request=['network', 'node', 'client_1', 'application', 'C2Server', 'ransomware_configure', {'server_ip_address': '192.168.1.14', 'server_password': None, 'payload': 'ENCRYPT'}], response=RequestResponse(status='success', data={}), reward=0.0, reward_info={})}})" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "env.step(5)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+-------------------------------------------------------------------------------------+\n", + "| web_server Software Manager |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| Name | Type | Operating State | Health State | Port | Protocol |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", + "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", + "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", + "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", + "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", + "| NMAP | Application | RUNNING | GOOD | None | none |\n", + "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", + "| UserManager | Service | RUNNING | GOOD | None | none |\n", + "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", + "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", + "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", + "| C2Beacon | Application | RUNNING | GOOD | None | tcp |\n", + "| RansomwareScript | Application | RUNNING | GOOD | None | none |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "+------------------------------------+\n", + "| RansomwareScript Running Status |\n", + "+--------------------------+---------+\n", + "| Target Server IP Address | Payload |\n", + "+--------------------------+---------+\n", + "| 192.168.1.14 | ENCRYPT |\n", + "+--------------------------+---------+\n" + ] + } + ], "source": [ "ransomware_script: RansomwareScript = web_server.software_manager.software[\"RansomwareScript\"]\n", "web_server.software_manager.show()\n", @@ -513,18 +743,48 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "(0,\n", + " 0.0,\n", + " False,\n", + " False,\n", + " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=5, action='c2_server_data_exfiltrate', parameters={'node_name': 'client_1', 'target_file_name': 'database.db', 'target_folder_name': 'database', 'exfiltration_folder_name': 'spoils', 'target_ip_address': '192.168.1.14', 'username': 'admin', 'password': 'admin'}, request=['network', 'node', 'client_1', 'application', 'C2Server', 'exfiltrate', {'target_file_name': 'database.db', 'target_folder_name': 'database', 'exfiltration_folder_name': 'spoils', 'target_ip_address': '192.168.1.14', 'username': 'admin', 'password': 'admin'}], response=RequestResponse(status='success', data={}), reward=0.0, reward_info={})}})" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "env.step(6)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+--------------------------------------------------------------------------------+\n", + "| client_1 File System |\n", + "+--------------------+---------+---------------+-----------------------+---------+\n", + "| File Path | Size | Health status | Visible health status | Deleted |\n", + "+--------------------+---------+---------------+-----------------------+---------+\n", + "| root | 0 B | GOOD | NONE | False |\n", + "| spoils/database.db | 4.77 MB | GOOD | NONE | False |\n", + "+--------------------+---------+---------------+-----------------------+---------+\n" + ] + } + ], "source": [ "client_1: Computer = env.game.simulation.network.get_node_by_hostname(\"client_1\")\n", "client_1.software_manager.file_system.show(full=True)" @@ -532,9 +792,25 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+---------------------------------------------------------------------------------+\n", + "| web_server File System |\n", + "+---------------------+---------+---------------+-----------------------+---------+\n", + "| File Path | Size | Health status | Visible health status | Deleted |\n", + "+---------------------+---------+---------------+-----------------------+---------+\n", + "| primaite/index.html | 15.0 KB | GOOD | NONE | False |\n", + "| root | 0 B | GOOD | NONE | False |\n", + "| spoils/database.db | 4.77 MB | GOOD | NONE | False |\n", + "+---------------------+---------+---------------+-----------------------+---------+\n" + ] + } + ], "source": [ "web_server: Computer = env.game.simulation.network.get_node_by_hostname(\"web_server\")\n", "web_server.software_manager.file_system.show(full=True)" @@ -571,18 +847,48 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "(0,\n", + " 0.0,\n", + " False,\n", + " False,\n", + " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=6, action='c2_server_ransomware_launch', parameters={'node_name': 'client_1'}, request=['network', 'node', 'client_1', 'application', 'C2Server', 'ransomware_launch'], response=RequestResponse(status='success', data={}), reward=0.0, reward_info={})}})" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "env.step(7)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----------------------------------------------------------------------------------+\n", + "| database_server File System |\n", + "+----------------------+---------+---------------+-----------------------+---------+\n", + "| File Path | Size | Health status | Visible health status | Deleted |\n", + "+----------------------+---------+---------------+-----------------------+---------+\n", + "| database/database.db | 4.77 MB | CORRUPT | NONE | False |\n", + "| root | 0 B | GOOD | NONE | False |\n", + "+----------------------+---------+---------------+-----------------------+---------+\n" + ] + } + ], "source": [ "database_server: Server = env.game.simulation.network.get_node_by_hostname(\"database_server\")\n", "database_server.software_manager.file_system.show(full=True)" @@ -601,7 +907,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "metadata": {}, "outputs": [], "source": [ @@ -697,52 +1003,52 @@ " 1:\n", " action: node_application_remove\n", " options:\n", - " node_id: 0\n", + " node_name: web_server\n", " application_name: C2Beacon\n", " 2:\n", " action: node_shutdown\n", " options:\n", - " node_id: 0\n", + " node_name: web_server\n", " 3:\n", " action: router_acl_add_rule\n", " options:\n", " target_router: router_1\n", " position: 1\n", - " permission: 2\n", - " source_ip_id: 2\n", - " dest_ip_id: 3\n", - " source_port_id: 2\n", - " dest_port_id: 2\n", - " protocol_id: 1\n", - " source_wildcard_id: 0\n", - " dest_wildcard_id: 0\n", + " permission: DENY\n", + " src_ip: 192.168.10.21\n", + " dst_ip: 192.168.1.12\n", + " src_port: HTTP\n", + " dst_port: HTTP\n", + " protocol_name: ALL\n", + " src_wildcard: NONE\n", + " dst_wildcard: NONE\n", "\n", "\n", - " options:\n", - " nodes:\n", - " - node_name: web_server\n", - " applications:\n", - " - application_name: C2Beacon\n", + " # options:\n", + " # nodes:\n", + " # - node_name: web_server\n", + " # applications:\n", + " # - application_name: C2Beacon\n", "\n", - " - node_name: database_server\n", - " folders:\n", - " - folder_name: database\n", - " files:\n", - " - file_name: database.db\n", - " services:\n", - " - service_name: DatabaseService\n", - " - node_name: router_1\n", + " # - node_name: database_server\n", + " # folders:\n", + " # - folder_name: database\n", + " # files:\n", + " # - file_name: database.db\n", + " # services:\n", + " # - service_name: DatabaseService\n", + " # - node_name: router_1\n", "\n", - " max_folders_per_node: 2\n", - " max_files_per_folder: 2\n", - " max_services_per_node: 2\n", - " max_nics_per_node: 8\n", - " max_acl_rules: 10\n", - " ip_list:\n", - " - 192.168.10.21\n", - " - 192.168.1.12\n", - " wildcard_list:\n", - " - 0.0.0.1\n", + " # max_folders_per_node: 2\n", + " # max_files_per_folder: 2\n", + " # max_services_per_node: 2\n", + " # max_nics_per_node: 8\n", + " # max_acl_rules: 10\n", + " # ip_list:\n", + " # - 192.168.10.21\n", + " # - 192.168.1.12\n", + " # wildcard_list:\n", + " # - 0.0.0.1\n", " reward_function:\n", " reward_components:\n", " - type: DUMMY\n", @@ -755,9 +1061,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-03 16:04:26,020: PrimaiteGymEnv RNG seed = None\n" + ] + } + ], "source": [ "with open(data_manipulation_config_path()) as f:\n", " cfg = yaml.safe_load(f)\n", @@ -771,7 +1085,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 21, "metadata": {}, "outputs": [], "source": [ @@ -814,9 +1128,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 22, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-03 16:04:26,440: Resetting environment, episode 0, avg. reward: 0.0\n", + "2025-02-03 16:04:26,445: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-03/16-04-20/agent_actions/episode_0.json\n" + ] + } + ], "source": [ "# Resetting the environment and capturing the default observation space.\n", "blue_env.reset()\n", @@ -825,9 +1148,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 23, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Setting up the C2 Suite via the simulation API.\n", "\n", @@ -848,7 +1182,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 24, "metadata": {}, "outputs": [], "source": [ @@ -858,9 +1192,26 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 25, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 2\n", + "root['NODES']['HOST0']['APPLICATIONS'][1]['operating_status']: 0 -> 1\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", + "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", + "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" + ] + } + ], "source": [ "display_obs_diffs(default_obs, c2_configuration_obs, blue_env.game.step_counter)" ] @@ -880,9 +1231,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 26, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "RequestResponse(status='success', data={0: RequestResponse(status='success', data={})})" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Installing RansomwareScript via C2 Terminal Commands\n", "ransomware_install_command = {\"commands\":[[\"software_manager\", \"application\", \"install\", \"RansomwareScript\"]],\n", @@ -893,9 +1255,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 27, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "RequestResponse(status='success', data={})" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Configuring the RansomwareScript\n", "ransomware_config = {\"server_ip_address\": \"192.168.1.14\", \"payload\": \"ENCRYPT\"}\n", @@ -904,7 +1277,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 28, "metadata": {}, "outputs": [], "source": [ @@ -914,9 +1287,28 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 29, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 7\n", + "root['NODES']['HOST0']['APPLICATIONS'][1]['operating_status']: 0 -> 1\n", + "root['NODES']['HOST0']['APPLICATIONS'][2]['operating_status']: 0 -> 3\n", + "root['NODES']['HOST0']['users']['local_login']: 0 -> 1\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", + "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", + "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" + ] + } + ], "source": [ "display_obs_diffs(default_obs, c2_ransomware_obs, env.game.step_counter)" ] @@ -932,7 +1324,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 30, "metadata": {}, "outputs": [], "source": [ @@ -948,16 +1340,27 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 31, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "RequestResponse(status='success', data={})" + ] + }, + "execution_count": 31, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "c2_server.send_command(given_command=C2Command.DATA_EXFILTRATION, command_options=exfil_options)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 32, "metadata": {}, "outputs": [], "source": [ @@ -966,9 +1369,22 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 33, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 7\n", + "root['NODES']['HOST0']['APPLICATIONS'][2]['operating_status']: 3 -> 1\n", + "root['NODES']['HOST1']['users']['remote_sessions']: 0 -> 1\n" + ] + } + ], "source": [ "display_obs_diffs(c2_ransomware_obs, c2_exfil_obs, env.game.step_counter)" ] @@ -984,9 +1400,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 34, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "RequestResponse(status='success', data={})" + ] + }, + "execution_count": 34, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Configuring the RansomwareScript\n", "ransomware_config = {\"server_ip_address\": \"192.168.1.14\", \"payload\": \"ENCRYPT\"}\n", @@ -995,9 +1422,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 35, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "RequestResponse(status='success', data={})" + ] + }, + "execution_count": 35, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Waiting for the ransomware to finish installing and then launching the RansomwareScript.\n", "blue_env.step(0)\n", @@ -1006,7 +1444,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 36, "metadata": {}, "outputs": [], "source": [ @@ -1016,9 +1454,30 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 37, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 6\n", + "root['NODES']['HOST0']['APPLICATIONS'][2]['operating_status']: 3 -> 1\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['NODES']['HOST1']['users']['remote_sessions']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" + ] + } + ], "source": [ "display_obs_diffs(c2_ransomware_obs, c2_final_obs, blue_env.game.step_counter)" ] @@ -1034,7 +1493,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 38, "metadata": {}, "outputs": [], "source": [ @@ -1066,16 +1525,200 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 39, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-03 16:04:27,571: Resetting environment, episode 1, avg. reward: 0.0\n", + "2025-02-03 16:04:27,574: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-03/16-04-20/agent_actions/episode_1.json\n" + ] + }, + { + "data": { + "text/plain": [ + "({'NODES': {'HOST0': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 1,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'HOST1': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 1,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'HOST2': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 0,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'HOST3': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 0,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'ROUTER0': {'ACL': {1: {'position': 0,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 2: {'position': 1,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 3: {'position': 2,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 4: {'position': 3,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 5: {'position': 4,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 6: {'position': 5,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 7: {'position': 6,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 8: {'position': 7,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 9: {'position': 8,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 10: {'position': 9,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0}},\n", + " 'PORTS': {1: {'operating_status': 1},\n", + " 2: {'operating_status': 1},\n", + " 3: {'operating_status': 2}},\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}}},\n", + " 'LINKS': {1: {'PROTOCOLS': {'ALL': 1}},\n", + " 2: {'PROTOCOLS': {'ALL': 1}},\n", + " 3: {'PROTOCOLS': {'ALL': 0}},\n", + " 4: {'PROTOCOLS': {'ALL': 1}},\n", + " 5: {'PROTOCOLS': {'ALL': 1}},\n", + " 6: {'PROTOCOLS': {'ALL': 1}},\n", + " 7: {'PROTOCOLS': {'ALL': 1}},\n", + " 8: {'PROTOCOLS': {'ALL': 1}},\n", + " 9: {'PROTOCOLS': {'ALL': 1}},\n", + " 10: {'PROTOCOLS': {'ALL': 0}}},\n", + " 'ICS': 0},\n", + " {})" + ] + }, + "execution_count": 39, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "blue_env.reset()" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 40, "metadata": {}, "outputs": [], "source": [ @@ -1094,7 +1737,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 41, "metadata": {}, "outputs": [], "source": [ @@ -1111,9 +1754,33 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 42, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+-------------------------------------------------------------------------------------+\n", + "| web_server Software Manager |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| Name | Type | Operating State | Health State | Port | Protocol |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", + "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", + "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", + "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", + "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", + "| NMAP | Application | RUNNING | GOOD | None | none |\n", + "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", + "| UserManager | Service | RUNNING | GOOD | None | none |\n", + "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", + "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", + "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n" + ] + } + ], "source": [ "blue_env.step(0)\n", "web_server.software_manager.show()" @@ -1121,9 +1788,26 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 43, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 3\n", + "root['NODES']['HOST0']['APPLICATIONS'][1]['operating_status']: 1 -> 0\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", + "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", + "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" + ] + } + ], "source": [ "display_obs_diffs(pre_blue_action_obs, post_blue_action_obs, blue_env.game.step_counter)" ] @@ -1137,9 +1821,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 44, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "RequestResponse(status='failure', data={'Reason': 'Command sent to the C2 Beacon but no response was ever received.'})" + ] + }, + "execution_count": 44, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Attempting to install the C2 RansomwareScript\n", "ransomware_install_command = {\"commands\":[[\"software_manager\", \"application\", \"install\", \"RansomwareScript\"]],\n", @@ -1161,16 +1856,200 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 45, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-03 16:04:28,041: Resetting environment, episode 2, avg. reward: 0.0\n", + "2025-02-03 16:04:28,045: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-03/16-04-20/agent_actions/episode_2.json\n" + ] + }, + { + "data": { + "text/plain": [ + "({'NODES': {'HOST0': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 1,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'HOST1': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 1,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'HOST2': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 0,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'HOST3': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 0,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'ROUTER0': {'ACL': {1: {'position': 0,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 2: {'position': 1,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 3: {'position': 2,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 4: {'position': 3,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 5: {'position': 4,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 6: {'position': 5,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 7: {'position': 6,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 8: {'position': 7,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 9: {'position': 8,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 10: {'position': 9,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0}},\n", + " 'PORTS': {1: {'operating_status': 1},\n", + " 2: {'operating_status': 1},\n", + " 3: {'operating_status': 2}},\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}}},\n", + " 'LINKS': {1: {'PROTOCOLS': {'ALL': 1}},\n", + " 2: {'PROTOCOLS': {'ALL': 1}},\n", + " 3: {'PROTOCOLS': {'ALL': 0}},\n", + " 4: {'PROTOCOLS': {'ALL': 1}},\n", + " 5: {'PROTOCOLS': {'ALL': 1}},\n", + " 6: {'PROTOCOLS': {'ALL': 1}},\n", + " 7: {'PROTOCOLS': {'ALL': 1}},\n", + " 8: {'PROTOCOLS': {'ALL': 1}},\n", + " 9: {'PROTOCOLS': {'ALL': 1}},\n", + " 10: {'PROTOCOLS': {'ALL': 0}}},\n", + " 'ICS': 0},\n", + " {})" + ] + }, + "execution_count": 45, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "blue_env.reset()" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 46, "metadata": {}, "outputs": [], "source": [ @@ -1189,7 +2068,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 47, "metadata": {}, "outputs": [], "source": [ @@ -1206,9 +2085,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 48, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "NodeOperatingState.SHUTTING_DOWN\n" + ] + } + ], "source": [ "web_server = blue_env.game.simulation.network.get_node_by_hostname(\"web_server\")\n", "print(web_server.operating_state)" @@ -1216,18 +2103,48 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 49, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 2\n", + "root['NODES']['HOST0']['operating_status']: 1 -> 4\n", + "root['NODES']['HOST0']['APPLICATIONS'][1]['operating_status']: 1 -> 0\n", + "root['NODES']['HOST0']['NICS'][1]['nic_status']: 1 -> 0\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", + "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", + "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" + ] + } + ], "source": [ "display_obs_diffs(pre_blue_action_obs, post_blue_action_obs, blue_env.game.step_counter)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 50, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "RequestResponse(status='failure', data={'Reason': 'Command sent to the C2 Beacon but no response was ever received.'})" + ] + }, + "execution_count": 50, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Attempting to install the C2 RansomwareScript\n", "ransomware_install_command = {\"commands\":[\"software_manager\", \"application\", \"install\", \"RansomwareScript\"],\n", @@ -1251,16 +2168,200 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 51, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-03 16:04:28,560: Resetting environment, episode 3, avg. reward: 0.0\n", + "2025-02-03 16:04:28,564: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-03/16-04-20/agent_actions/episode_3.json\n" + ] + }, + { + "data": { + "text/plain": [ + "({'NODES': {'HOST0': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 1,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'HOST1': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 1,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'HOST2': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 0,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'HOST3': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 0,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'ROUTER0': {'ACL': {1: {'position': 0,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 2: {'position': 1,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 3: {'position': 2,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 4: {'position': 3,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 5: {'position': 4,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 6: {'position': 5,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 7: {'position': 6,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 8: {'position': 7,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 9: {'position': 8,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 10: {'position': 9,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0}},\n", + " 'PORTS': {1: {'operating_status': 1},\n", + " 2: {'operating_status': 1},\n", + " 3: {'operating_status': 2}},\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}}},\n", + " 'LINKS': {1: {'PROTOCOLS': {'ALL': 1}},\n", + " 2: {'PROTOCOLS': {'ALL': 1}},\n", + " 3: {'PROTOCOLS': {'ALL': 0}},\n", + " 4: {'PROTOCOLS': {'ALL': 1}},\n", + " 5: {'PROTOCOLS': {'ALL': 1}},\n", + " 6: {'PROTOCOLS': {'ALL': 1}},\n", + " 7: {'PROTOCOLS': {'ALL': 1}},\n", + " 8: {'PROTOCOLS': {'ALL': 1}},\n", + " 9: {'PROTOCOLS': {'ALL': 1}},\n", + " 10: {'PROTOCOLS': {'ALL': 0}}},\n", + " 'ICS': 0},\n", + " {})" + ] + }, + "execution_count": 51, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "blue_env.reset()" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 52, "metadata": {}, "outputs": [], "source": [ @@ -1279,7 +2380,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 53, "metadata": {}, "outputs": [], "source": [ @@ -1296,11 +2397,44 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 54, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+---------------------------------------------------------------+\n", + "| router_1 Network Interfaces |\n", + "+------+-------------------+-----------------+-------+----------+\n", + "| Port | MAC Address | Address | Speed | Status |\n", + "+------+-------------------+-----------------+-------+----------+\n", + "| 1 | dd:6e:95:d4:3f:74 | 192.168.1.1/24 | 100.0 | Enabled |\n", + "| 2 | 8b:79:07:fc:69:2c | 192.168.10.1/24 | 100.0 | Enabled |\n", + "| 3 | 1f:fd:c4:ae:7a:00 | 127.0.0.1/8 | 100.0 | Disabled |\n", + "| 4 | 7b:e3:bf:4b:76:e8 | 127.0.0.1/8 | 100.0 | Disabled |\n", + "| 5 | 4f:37:b0:6b:5d:44 | 127.0.0.1/8 | 100.0 | Disabled |\n", + "+------+-------------------+-----------------+-------+----------+\n", + "+------------------------------------------------------------------------------------------------------------------------+\n", + "| router_1 Access Control List |\n", + "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n", + "| Index | Action | Protocol | Src IP | Src Wildcard | Src Port | Dst IP | Dst Wildcard | Dst Port | Matched |\n", + "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n", + "| 1 | DENY | ANY | 192.168.10.21 | ANY | 80 | 192.168.1.12 | ANY | 80 | 0 |\n", + "| 18 | PERMIT | ANY | ANY | ANY | 5432 | ANY | ANY | 5432 | 0 |\n", + "| 19 | PERMIT | ANY | ANY | ANY | 53 | ANY | ANY | 53 | 0 |\n", + "| 20 | PERMIT | ANY | ANY | ANY | 21 | ANY | ANY | 21 | 0 |\n", + "| 21 | PERMIT | ANY | ANY | ANY | 80 | ANY | ANY | 80 | 4 |\n", + "| 22 | PERMIT | ANY | ANY | ANY | 219 | ANY | ANY | 219 | 10 |\n", + "| 23 | PERMIT | icmp | ANY | ANY | ANY | ANY | ANY | ANY | 0 |\n", + "| 24 | DENY | ANY | ANY | ANY | ANY | ANY | ANY | ANY | 0 |\n", + "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n" + ] + } + ], "source": [ "router_1: Router = blue_env.game.simulation.network.get_node_by_hostname(\"router_1\")\n", + "router_1.show()\n", "router_1.acl.show()" ] }, @@ -1313,9 +2447,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 55, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "RequestResponse(status='failure', data={'Reason': 'Command sent to the C2 Beacon but no response was ever received.'})" + ] + }, + "execution_count": 55, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "blue_env.step(0)\n", "\n", @@ -1326,9 +2471,30 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 56, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+------------------------------------------------------------------------------------------------------------------------+\n", + "| router_1 Access Control List |\n", + "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n", + "| Index | Action | Protocol | Src IP | Src Wildcard | Src Port | Dst IP | Dst Wildcard | Dst Port | Matched |\n", + "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n", + "| 1 | DENY | ANY | 192.168.10.21 | ANY | 80 | 192.168.1.12 | ANY | 80 | 2 |\n", + "| 18 | PERMIT | ANY | ANY | ANY | 5432 | ANY | ANY | 5432 | 0 |\n", + "| 19 | PERMIT | ANY | ANY | ANY | 53 | ANY | ANY | 53 | 0 |\n", + "| 20 | PERMIT | ANY | ANY | ANY | 21 | ANY | ANY | 21 | 0 |\n", + "| 21 | PERMIT | ANY | ANY | ANY | 80 | ANY | ANY | 80 | 4 |\n", + "| 22 | PERMIT | ANY | ANY | ANY | 219 | ANY | ANY | 219 | 10 |\n", + "| 23 | PERMIT | icmp | ANY | ANY | ANY | ANY | ANY | ANY | 0 |\n", + "| 24 | DENY | ANY | ANY | ANY | ANY | ANY | ANY | ANY | 0 |\n", + "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n" + ] + } + ], "source": [ "router_1.acl.show()" ] @@ -1342,18 +2508,58 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 57, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+-------------------------------------------------------------------------------------+\n", + "| web_server Software Manager |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| Name | Type | Operating State | Health State | Port | Protocol |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", + "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", + "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", + "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", + "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", + "| NMAP | Application | RUNNING | GOOD | None | none |\n", + "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", + "| UserManager | Service | RUNNING | GOOD | None | none |\n", + "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", + "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", + "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", + "| C2Beacon | Application | RUNNING | GOOD | None | tcp |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n" + ] + } + ], "source": [ "web_server.software_manager.show()" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 58, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----------------------------------------------------------------------------------+\n", + "| database_server File System |\n", + "+----------------------+---------+---------------+-----------------------+---------+\n", + "| File Path | Size | Health status | Visible health status | Deleted |\n", + "+----------------------+---------+---------------+-----------------------+---------+\n", + "| database/database.db | 4.77 MB | GOOD | NONE | False |\n", + "| root | 0 B | GOOD | NONE | False |\n", + "+----------------------+---------+---------------+-----------------------+---------+\n" + ] + } + ], "source": [ "database_server: Server = blue_env.game.simulation.network.get_node_by_hostname(\"database_server\")\n", "database_server.software_manager.file_system.show(full=True)" @@ -1361,9 +2567,33 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 59, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 3\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", + "root['NODES']['ROUTER0']['ACL'][1]['permission']: 0 -> 2\n", + "root['NODES']['ROUTER0']['ACL'][1]['source_ip_id']: 0 -> 7\n", + "root['NODES']['ROUTER0']['ACL'][1]['source_wildcard_id']: 0 -> 1\n", + "root['NODES']['ROUTER0']['ACL'][1]['source_port_id']: 0 -> 2\n", + "root['NODES']['ROUTER0']['ACL'][1]['dest_ip_id']: 0 -> 3\n", + "root['NODES']['ROUTER0']['ACL'][1]['dest_wildcard_id']: 0 -> 1\n", + "root['NODES']['ROUTER0']['ACL'][1]['dest_port_id']: 0 -> 2\n", + "root['NODES']['ROUTER0']['ACL'][1]['protocol_id']: 0 -> 1\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", + "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", + "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" + ] + } + ], "source": [ "display_obs_diffs(pre_blue_action_obs, post_blue_action_obs, blue_env.game.step_counter)" ] @@ -1427,9 +2657,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 60, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-03 16:04:29,610: PrimaiteGymEnv RNG seed = None\n" + ] + } + ], "source": [ "with open(data_manipulation_config_path()) as f:\n", " cfg = yaml.safe_load(f)\n", @@ -1450,7 +2688,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 61, "metadata": {}, "outputs": [], "source": [ @@ -1478,9 +2716,30 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 62, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", + "| C2Beacon Running Status |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| False | None | 0 | 5 | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "+-----------------------------------------------------------------------------------------------------+\n", + "| C2Server Running Status |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n", + "| False | None | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n" + ] + } + ], "source": [ "env.step(2) # Agent Action Equivalent to c2_beacon.configure(c2_server_ip_address=\"192.168.10.21\")\n", "env.step(3) # Agent action Equivalent to c2_beacon.establish()\n", @@ -1497,9 +2756,30 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 63, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", + "| C2Beacon Running Status |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| False | None | 0 | 5 | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "+-----------------------------------------------------------------------------------------------------+\n", + "| C2Server Running Status |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n", + "| False | None | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n" + ] + } + ], "source": [ "env.step(9) # Equivalent of to c2_beacon.configure(c2_server_ip_address=\"192.168.10.22\")\n", "env.step(3)\n", @@ -1517,9 +2797,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 64, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+-----------------------------------------------------------------------------------------------------+\n", + "| C2Server Running Status |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n", + "| False | None | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n" + ] + } + ], "source": [ "for i in range(6):\n", " env.step(0)\n", @@ -1542,9 +2836,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 65, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-03 16:04:30,011: PrimaiteGymEnv RNG seed = None\n" + ] + } + ], "source": [ "with open(data_manipulation_config_path()) as f:\n", " cfg = yaml.safe_load(f)\n", @@ -1560,7 +2862,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 66, "metadata": {}, "outputs": [], "source": [ @@ -1584,9 +2886,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 67, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", + "| C2Beacon Running Status |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| True | 192.168.10.21 | 0 | 5 | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n" + ] + } + ], "source": [ "c2_beacon.configure(c2_server_ip_address=\"192.168.10.21\")\n", "c2_beacon.establish()\n", @@ -1604,9 +2920,72 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 68, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 4\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 5\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 6\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 7\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 8\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 9\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 10\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 11\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 12\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 13\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" + ] + } + ], "source": [ "for i in range(10):\n", " keep_alive_obs, _, _, _, _ = blue_config_env.step(0)\n", @@ -1622,9 +3001,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 69, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", + "| C2Beacon Running Status |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| True | 192.168.10.21 | 0 | 1 | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n" + ] + } + ], "source": [ "c2_beacon.configure(c2_server_ip_address=\"192.168.10.21\", keep_alive_frequency=1)\n", "c2_beacon.establish()\n", @@ -1640,9 +3033,40 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 70, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 14\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 15\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" + ] + } + ], "source": [ "# Comparing the OBS of the default frequency to a timestep frequency of 1\n", "for i in range(2):\n", @@ -1661,9 +3085,52 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 71, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 16\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 17\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 18\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 19\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 20\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 21\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 22\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" + ] + } + ], "source": [ "c2_beacon.configure(c2_server_ip_address=\"192.168.10.21\", keep_alive_frequency=7)\n", "\n", @@ -1700,9 +3167,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 72, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-03 16:04:30,864: Resetting environment, episode 0, avg. reward: 0.0\n", + "2025-02-03 16:04:30,867: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-03/16-04-20/agent_actions/episode_0.json\n" + ] + } + ], "source": [ "blue_config_env.reset()\n", "\n", @@ -1724,9 +3200,28 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 73, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 5\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" + ] + } + ], "source": [ "# Capturing default C2 Traffic\n", "for i in range(3):\n", @@ -1744,9 +3239,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 74, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", + "| C2Beacon Running Status |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| True | 192.168.10.21 | 0 | 5 | udp | 53 |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n" + ] + } + ], "source": [ "from primaite.utils.validation.ip_protocol import PROTOCOL_LOOKUP\n", "from primaite.utils.validation.port import PORT_LOOKUP\n", @@ -1759,9 +3268,28 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 75, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 10\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 1 -> 0\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 1 -> 0\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['udp'][53]['inbound']: 0 -> 1\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['udp'][53]['outbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 1 -> 0\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 1 -> 0\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['udp'][53]['inbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['udp'][53]['outbound']: 0 -> 1\n" + ] + } + ], "source": [ "# Capturing UDP C2 Traffic\n", "for i in range(5):\n", From 454bd61fb27e2e8b8702d86c0e83806deae7cd0c Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Tue, 4 Feb 2025 10:50:53 +0000 Subject: [PATCH 08/24] #3075: Update internal YAML fragments in C2C-E2E-Demo notebook. --- ...ommand-and-Control-E2E-Demonstration.ipynb | 1802 ++--------------- 1 file changed, 137 insertions(+), 1665 deletions(-) diff --git a/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb b/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb index 2c1a94c8..c2a87e45 100644 --- a/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb +++ b/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb @@ -13,24 +13,9 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-02-03 16:04:19,828: Performing the PrimAITE first-time setup...\n", - "2025-02-03 16:04:19,829: Building the PrimAITE app directories...\n", - "2025-02-03 16:04:19,829: Building primaite_config.yaml...\n", - "2025-02-03 16:04:19,829: Rebuilding the demo notebooks...\n", - "/home/nick/primaite/4.0.0a1-dev/notebooks/example_notebooks/Command-and-Control-E2E-Demonstration.ipynb\n", - "2025-02-03 16:04:19,831: Reset example notebook: /home/nick/primaite/4.0.0a1-dev/notebooks/example_notebooks/Command-and-Control-E2E-Demonstration.ipynb\n", - "2025-02-03 16:04:19,836: Rebuilding the example notebooks...\n", - "2025-02-03 16:04:19,840: PrimAITE setup complete!\n" - ] - } - ], + "outputs": [], "source": [ "!primaite setup" ] @@ -183,17 +168,9 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-03 16:04:24,734: PrimaiteGymEnv RNG seed = None\n" - ] - } - ], + "outputs": [], "source": [ "with open(data_manipulation_config_path()) as f:\n", " cfg = yaml.safe_load(f)\n", @@ -219,35 +196,9 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+--------------------------------------------------------------------------------------+\n", - "| client_1 Software Manager |\n", - "+---------------------+-------------+-----------------+--------------+------+----------+\n", - "| Name | Type | Operating State | Health State | Port | Protocol |\n", - "+---------------------+-------------+-----------------+--------------+------+----------+\n", - "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", - "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", - "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", - "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", - "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", - "| NMAP | Application | RUNNING | GOOD | None | none |\n", - "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", - "| UserManager | Service | RUNNING | GOOD | None | none |\n", - "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", - "| FTPClient | Service | RUNNING | GOOD | 21 | tcp |\n", - "| DataManipulationBot | Application | RUNNING | GOOD | None | none |\n", - "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", - "| C2Server | Application | RUNNING | GOOD | None | tcp |\n", - "+---------------------+-------------+-----------------+--------------+------+----------+\n" - ] - } - ], + "outputs": [], "source": [ "client_1: Computer = env.game.simulation.network.get_node_by_hostname(\"client_1\")\n", "client_1.software_manager.install(C2Server)\n", @@ -297,34 +248,9 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+-------------------------------------------------------------------------------------+\n", - "| web_server Software Manager |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| Name | Type | Operating State | Health State | Port | Protocol |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", - "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", - "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", - "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", - "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", - "| NMAP | Application | RUNNING | GOOD | None | none |\n", - "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", - "| UserManager | Service | RUNNING | GOOD | None | none |\n", - "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", - "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", - "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", - "| C2Beacon | Application | INSTALLING | UNUSED | None | tcp |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n" - ] - } - ], + "outputs": [], "source": [ "env.step(1)\n", "web_server: Computer = env.game.simulation.network.get_node_by_hostname(\"web_server\")\n", @@ -364,41 +290,9 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+-------------------------------------------------------------------------------------+\n", - "| web_server Software Manager |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| Name | Type | Operating State | Health State | Port | Protocol |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", - "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", - "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", - "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", - "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", - "| NMAP | Application | RUNNING | GOOD | None | none |\n", - "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", - "| UserManager | Service | RUNNING | GOOD | None | none |\n", - "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", - "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", - "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", - "| C2Beacon | Application | RUNNING | GOOD | None | tcp |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", - "| C2Beacon Running Status |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| False | 192.168.10.21 | 0 | 5 | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n" - ] - } - ], + "outputs": [], "source": [ "env.step(2)\n", "c2_beacon: C2Beacon = web_server.software_manager.software[\"C2Beacon\"]\n", @@ -437,54 +331,18 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(0,\n", - " 0.0,\n", - " False,\n", - " False,\n", - " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=2, action='node_application_execute', parameters={'node_name': 'web_server', 'application_name': 'C2Beacon'}, request=['network', 'node', 'web_server', 'application', 'C2Beacon', 'execute'], response=RequestResponse(status='success', data={}), reward=0.0, reward_info={})}})" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "env.step(3)" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", - "| C2Beacon Running Status |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| True | 192.168.10.21 | 1 | 5 | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "+-----------------------------------------------------------------------------------------------------+\n", - "| C2Server Running Status |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n", - "| True | 192.168.1.12 | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n" - ] - } - ], + "outputs": [], "source": [ "c2_beacon.show()\n", "c2_server.show()" @@ -547,59 +405,18 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(0,\n", - " 0.0,\n", - " False,\n", - " False,\n", - " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=3, action='c2_server_terminal_command', parameters={'node_name': 'client_1', 'ip_address': None, 'username': 'admin', 'password': 'admin', 'commands': [['software_manager', 'application', 'install', 'RansomwareScript']]}, request=['network', 'node', 'client_1', 'application', 'C2Server', 'terminal_command', {'commands': [['software_manager', 'application', 'install', 'RansomwareScript']], 'ip_address': None, 'username': 'admin', 'password': 'admin'}], response=RequestResponse(status='success', data={0: RequestResponse(status='success', data={})}), reward=0.0, reward_info={})}})" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "env.step(4)" ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+--------------------------------------------------------------------------------------+\n", - "| client_1 Software Manager |\n", - "+---------------------+-------------+-----------------+--------------+------+----------+\n", - "| Name | Type | Operating State | Health State | Port | Protocol |\n", - "+---------------------+-------------+-----------------+--------------+------+----------+\n", - "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", - "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", - "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", - "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", - "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", - "| NMAP | Application | RUNNING | GOOD | None | none |\n", - "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", - "| UserManager | Service | RUNNING | GOOD | None | none |\n", - "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", - "| FTPClient | Service | RUNNING | GOOD | 21 | tcp |\n", - "| DataManipulationBot | Application | RUNNING | GOOD | None | none |\n", - "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", - "| C2Server | Application | RUNNING | GOOD | None | tcp |\n", - "+---------------------+-------------+-----------------+--------------+------+----------+\n" - ] - } - ], + "outputs": [], "source": [ "client_1.software_manager.show()" ] @@ -638,66 +455,18 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(0,\n", - " 0.0,\n", - " False,\n", - " False,\n", - " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=4, action='c2_server_ransomware_configure', parameters={'node_name': 'client_1', 'server_ip_address': '192.168.1.14', 'payload': 'ENCRYPT'}, request=['network', 'node', 'client_1', 'application', 'C2Server', 'ransomware_configure', {'server_ip_address': '192.168.1.14', 'server_password': None, 'payload': 'ENCRYPT'}], response=RequestResponse(status='success', data={}), reward=0.0, reward_info={})}})" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "env.step(5)" ] }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+-------------------------------------------------------------------------------------+\n", - "| web_server Software Manager |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| Name | Type | Operating State | Health State | Port | Protocol |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", - "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", - "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", - "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", - "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", - "| NMAP | Application | RUNNING | GOOD | None | none |\n", - "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", - "| UserManager | Service | RUNNING | GOOD | None | none |\n", - "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", - "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", - "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", - "| C2Beacon | Application | RUNNING | GOOD | None | tcp |\n", - "| RansomwareScript | Application | RUNNING | GOOD | None | none |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "+------------------------------------+\n", - "| RansomwareScript Running Status |\n", - "+--------------------------+---------+\n", - "| Target Server IP Address | Payload |\n", - "+--------------------------+---------+\n", - "| 192.168.1.14 | ENCRYPT |\n", - "+--------------------------+---------+\n" - ] - } - ], + "outputs": [], "source": [ "ransomware_script: RansomwareScript = web_server.software_manager.software[\"RansomwareScript\"]\n", "web_server.software_manager.show()\n", @@ -743,48 +512,18 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(0,\n", - " 0.0,\n", - " False,\n", - " False,\n", - " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=5, action='c2_server_data_exfiltrate', parameters={'node_name': 'client_1', 'target_file_name': 'database.db', 'target_folder_name': 'database', 'exfiltration_folder_name': 'spoils', 'target_ip_address': '192.168.1.14', 'username': 'admin', 'password': 'admin'}, request=['network', 'node', 'client_1', 'application', 'C2Server', 'exfiltrate', {'target_file_name': 'database.db', 'target_folder_name': 'database', 'exfiltration_folder_name': 'spoils', 'target_ip_address': '192.168.1.14', 'username': 'admin', 'password': 'admin'}], response=RequestResponse(status='success', data={}), reward=0.0, reward_info={})}})" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "env.step(6)" ] }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+--------------------------------------------------------------------------------+\n", - "| client_1 File System |\n", - "+--------------------+---------+---------------+-----------------------+---------+\n", - "| File Path | Size | Health status | Visible health status | Deleted |\n", - "+--------------------+---------+---------------+-----------------------+---------+\n", - "| root | 0 B | GOOD | NONE | False |\n", - "| spoils/database.db | 4.77 MB | GOOD | NONE | False |\n", - "+--------------------+---------+---------------+-----------------------+---------+\n" - ] - } - ], + "outputs": [], "source": [ "client_1: Computer = env.game.simulation.network.get_node_by_hostname(\"client_1\")\n", "client_1.software_manager.file_system.show(full=True)" @@ -792,25 +531,9 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+---------------------------------------------------------------------------------+\n", - "| web_server File System |\n", - "+---------------------+---------+---------------+-----------------------+---------+\n", - "| File Path | Size | Health status | Visible health status | Deleted |\n", - "+---------------------+---------+---------------+-----------------------+---------+\n", - "| primaite/index.html | 15.0 KB | GOOD | NONE | False |\n", - "| root | 0 B | GOOD | NONE | False |\n", - "| spoils/database.db | 4.77 MB | GOOD | NONE | False |\n", - "+---------------------+---------+---------------+-----------------------+---------+\n" - ] - } - ], + "outputs": [], "source": [ "web_server: Computer = env.game.simulation.network.get_node_by_hostname(\"web_server\")\n", "web_server.software_manager.file_system.show(full=True)" @@ -847,48 +570,18 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(0,\n", - " 0.0,\n", - " False,\n", - " False,\n", - " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=6, action='c2_server_ransomware_launch', parameters={'node_name': 'client_1'}, request=['network', 'node', 'client_1', 'application', 'C2Server', 'ransomware_launch'], response=RequestResponse(status='success', data={}), reward=0.0, reward_info={})}})" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "env.step(7)" ] }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+----------------------------------------------------------------------------------+\n", - "| database_server File System |\n", - "+----------------------+---------+---------------+-----------------------+---------+\n", - "| File Path | Size | Health status | Visible health status | Deleted |\n", - "+----------------------+---------+---------------+-----------------------+---------+\n", - "| database/database.db | 4.77 MB | CORRUPT | NONE | False |\n", - "| root | 0 B | GOOD | NONE | False |\n", - "+----------------------+---------+---------------+-----------------------+---------+\n" - ] - } - ], + "outputs": [], "source": [ "database_server: Server = env.game.simulation.network.get_node_by_hostname(\"database_server\")\n", "database_server.software_manager.file_system.show(full=True)" @@ -907,7 +600,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1025,30 +718,30 @@ "\n", "\n", " # options:\n", - " # nodes:\n", - " # - node_name: web_server\n", - " # applications:\n", - " # - application_name: C2Beacon\n", + " # # nodes:\n", + " # node_name: web_server\n", + " # # applications:\n", + " # application_name: C2Beacon\n", "\n", - " # - node_name: database_server\n", - " # folders:\n", - " # - folder_name: database\n", - " # files:\n", - " # - file_name: database.db\n", - " # services:\n", - " # - service_name: DatabaseService\n", - " # - node_name: router_1\n", + " # node_name: database_server\n", + " # folders:\n", + " # - folder_name: database\n", + " # files:\n", + " # - file_name: database.db\n", + " # services:\n", + " # - service_name: DatabaseService\n", + " # node_name: router_1\n", "\n", - " # max_folders_per_node: 2\n", - " # max_files_per_folder: 2\n", - " # max_services_per_node: 2\n", - " # max_nics_per_node: 8\n", - " # max_acl_rules: 10\n", - " # ip_list:\n", - " # - 192.168.10.21\n", - " # - 192.168.1.12\n", - " # wildcard_list:\n", - " # - 0.0.0.1\n", + " # max_folders_per_node: 2\n", + " # max_files_per_folder: 2\n", + " # max_services_per_node: 2\n", + " # max_nics_per_node: 8\n", + " # max_acl_rules: 10\n", + " # ip_list:\n", + " # - 192.168.10.21\n", + " # - 192.168.1.12\n", + " # wildcard_list:\n", + " # - 0.0.0.1\n", " reward_function:\n", " reward_components:\n", " - type: DUMMY\n", @@ -1061,17 +754,9 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-03 16:04:26,020: PrimaiteGymEnv RNG seed = None\n" - ] - } - ], + "outputs": [], "source": [ "with open(data_manipulation_config_path()) as f:\n", " cfg = yaml.safe_load(f)\n", @@ -1128,18 +813,9 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-03 16:04:26,440: Resetting environment, episode 0, avg. reward: 0.0\n", - "2025-02-03 16:04:26,445: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-03/16-04-20/agent_actions/episode_0.json\n" - ] - } - ], + "outputs": [], "source": [ "# Resetting the environment and capturing the default observation space.\n", "blue_env.reset()\n", @@ -1148,20 +824,9 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 23, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Setting up the C2 Suite via the simulation API.\n", "\n", @@ -1192,26 +857,9 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 2\n", - "root['NODES']['HOST0']['APPLICATIONS'][1]['operating_status']: 0 -> 1\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", - "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", - "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" - ] - } - ], + "outputs": [], "source": [ "display_obs_diffs(default_obs, c2_configuration_obs, blue_env.game.step_counter)" ] @@ -1231,20 +879,9 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "RequestResponse(status='success', data={0: RequestResponse(status='success', data={})})" - ] - }, - "execution_count": 26, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Installing RansomwareScript via C2 Terminal Commands\n", "ransomware_install_command = {\"commands\":[[\"software_manager\", \"application\", \"install\", \"RansomwareScript\"]],\n", @@ -1255,20 +892,9 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "RequestResponse(status='success', data={})" - ] - }, - "execution_count": 27, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Configuring the RansomwareScript\n", "ransomware_config = {\"server_ip_address\": \"192.168.1.14\", \"payload\": \"ENCRYPT\"}\n", @@ -1287,28 +913,9 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 7\n", - "root['NODES']['HOST0']['APPLICATIONS'][1]['operating_status']: 0 -> 1\n", - "root['NODES']['HOST0']['APPLICATIONS'][2]['operating_status']: 0 -> 3\n", - "root['NODES']['HOST0']['users']['local_login']: 0 -> 1\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", - "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", - "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" - ] - } - ], + "outputs": [], "source": [ "display_obs_diffs(default_obs, c2_ransomware_obs, env.game.step_counter)" ] @@ -1340,20 +947,9 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "RequestResponse(status='success', data={})" - ] - }, - "execution_count": 31, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "c2_server.send_command(given_command=C2Command.DATA_EXFILTRATION, command_options=exfil_options)" ] @@ -1369,22 +965,9 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 7\n", - "root['NODES']['HOST0']['APPLICATIONS'][2]['operating_status']: 3 -> 1\n", - "root['NODES']['HOST1']['users']['remote_sessions']: 0 -> 1\n" - ] - } - ], + "outputs": [], "source": [ "display_obs_diffs(c2_ransomware_obs, c2_exfil_obs, env.game.step_counter)" ] @@ -1400,20 +983,9 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "RequestResponse(status='success', data={})" - ] - }, - "execution_count": 34, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Configuring the RansomwareScript\n", "ransomware_config = {\"server_ip_address\": \"192.168.1.14\", \"payload\": \"ENCRYPT\"}\n", @@ -1422,20 +994,9 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "RequestResponse(status='success', data={})" - ] - }, - "execution_count": 35, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Waiting for the ransomware to finish installing and then launching the RansomwareScript.\n", "blue_env.step(0)\n", @@ -1454,30 +1015,9 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 6\n", - "root['NODES']['HOST0']['APPLICATIONS'][2]['operating_status']: 3 -> 1\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['NODES']['HOST1']['users']['remote_sessions']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" - ] - } - ], + "outputs": [], "source": [ "display_obs_diffs(c2_ransomware_obs, c2_final_obs, blue_env.game.step_counter)" ] @@ -1525,193 +1065,9 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-03 16:04:27,571: Resetting environment, episode 1, avg. reward: 0.0\n", - "2025-02-03 16:04:27,574: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-03/16-04-20/agent_actions/episode_1.json\n" - ] - }, - { - "data": { - "text/plain": [ - "({'NODES': {'HOST0': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 1,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'HOST1': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 1,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'HOST2': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 0,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'HOST3': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 0,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'ROUTER0': {'ACL': {1: {'position': 0,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 2: {'position': 1,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 3: {'position': 2,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 4: {'position': 3,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 5: {'position': 4,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 6: {'position': 5,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 7: {'position': 6,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 8: {'position': 7,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 9: {'position': 8,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 10: {'position': 9,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0}},\n", - " 'PORTS': {1: {'operating_status': 1},\n", - " 2: {'operating_status': 1},\n", - " 3: {'operating_status': 2}},\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}}},\n", - " 'LINKS': {1: {'PROTOCOLS': {'ALL': 1}},\n", - " 2: {'PROTOCOLS': {'ALL': 1}},\n", - " 3: {'PROTOCOLS': {'ALL': 0}},\n", - " 4: {'PROTOCOLS': {'ALL': 1}},\n", - " 5: {'PROTOCOLS': {'ALL': 1}},\n", - " 6: {'PROTOCOLS': {'ALL': 1}},\n", - " 7: {'PROTOCOLS': {'ALL': 1}},\n", - " 8: {'PROTOCOLS': {'ALL': 1}},\n", - " 9: {'PROTOCOLS': {'ALL': 1}},\n", - " 10: {'PROTOCOLS': {'ALL': 0}}},\n", - " 'ICS': 0},\n", - " {})" - ] - }, - "execution_count": 39, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "blue_env.reset()" ] @@ -1754,33 +1110,9 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+-------------------------------------------------------------------------------------+\n", - "| web_server Software Manager |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| Name | Type | Operating State | Health State | Port | Protocol |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", - "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", - "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", - "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", - "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", - "| NMAP | Application | RUNNING | GOOD | None | none |\n", - "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", - "| UserManager | Service | RUNNING | GOOD | None | none |\n", - "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", - "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", - "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n" - ] - } - ], + "outputs": [], "source": [ "blue_env.step(0)\n", "web_server.software_manager.show()" @@ -1788,26 +1120,9 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 3\n", - "root['NODES']['HOST0']['APPLICATIONS'][1]['operating_status']: 1 -> 0\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", - "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", - "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" - ] - } - ], + "outputs": [], "source": [ "display_obs_diffs(pre_blue_action_obs, post_blue_action_obs, blue_env.game.step_counter)" ] @@ -1821,20 +1136,9 @@ }, { "cell_type": "code", - "execution_count": 44, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "RequestResponse(status='failure', data={'Reason': 'Command sent to the C2 Beacon but no response was ever received.'})" - ] - }, - "execution_count": 44, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Attempting to install the C2 RansomwareScript\n", "ransomware_install_command = {\"commands\":[[\"software_manager\", \"application\", \"install\", \"RansomwareScript\"]],\n", @@ -1856,193 +1160,9 @@ }, { "cell_type": "code", - "execution_count": 45, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-03 16:04:28,041: Resetting environment, episode 2, avg. reward: 0.0\n", - "2025-02-03 16:04:28,045: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-03/16-04-20/agent_actions/episode_2.json\n" - ] - }, - { - "data": { - "text/plain": [ - "({'NODES': {'HOST0': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 1,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'HOST1': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 1,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'HOST2': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 0,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'HOST3': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 0,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'ROUTER0': {'ACL': {1: {'position': 0,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 2: {'position': 1,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 3: {'position': 2,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 4: {'position': 3,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 5: {'position': 4,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 6: {'position': 5,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 7: {'position': 6,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 8: {'position': 7,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 9: {'position': 8,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 10: {'position': 9,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0}},\n", - " 'PORTS': {1: {'operating_status': 1},\n", - " 2: {'operating_status': 1},\n", - " 3: {'operating_status': 2}},\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}}},\n", - " 'LINKS': {1: {'PROTOCOLS': {'ALL': 1}},\n", - " 2: {'PROTOCOLS': {'ALL': 1}},\n", - " 3: {'PROTOCOLS': {'ALL': 0}},\n", - " 4: {'PROTOCOLS': {'ALL': 1}},\n", - " 5: {'PROTOCOLS': {'ALL': 1}},\n", - " 6: {'PROTOCOLS': {'ALL': 1}},\n", - " 7: {'PROTOCOLS': {'ALL': 1}},\n", - " 8: {'PROTOCOLS': {'ALL': 1}},\n", - " 9: {'PROTOCOLS': {'ALL': 1}},\n", - " 10: {'PROTOCOLS': {'ALL': 0}}},\n", - " 'ICS': 0},\n", - " {})" - ] - }, - "execution_count": 45, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "blue_env.reset()" ] @@ -2085,17 +1205,9 @@ }, { "cell_type": "code", - "execution_count": 48, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "NodeOperatingState.SHUTTING_DOWN\n" - ] - } - ], + "outputs": [], "source": [ "web_server = blue_env.game.simulation.network.get_node_by_hostname(\"web_server\")\n", "print(web_server.operating_state)" @@ -2103,48 +1215,18 @@ }, { "cell_type": "code", - "execution_count": 49, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 2\n", - "root['NODES']['HOST0']['operating_status']: 1 -> 4\n", - "root['NODES']['HOST0']['APPLICATIONS'][1]['operating_status']: 1 -> 0\n", - "root['NODES']['HOST0']['NICS'][1]['nic_status']: 1 -> 0\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", - "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", - "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" - ] - } - ], + "outputs": [], "source": [ "display_obs_diffs(pre_blue_action_obs, post_blue_action_obs, blue_env.game.step_counter)" ] }, { "cell_type": "code", - "execution_count": 50, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "RequestResponse(status='failure', data={'Reason': 'Command sent to the C2 Beacon but no response was ever received.'})" - ] - }, - "execution_count": 50, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Attempting to install the C2 RansomwareScript\n", "ransomware_install_command = {\"commands\":[\"software_manager\", \"application\", \"install\", \"RansomwareScript\"],\n", @@ -2168,193 +1250,9 @@ }, { "cell_type": "code", - "execution_count": 51, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-03 16:04:28,560: Resetting environment, episode 3, avg. reward: 0.0\n", - "2025-02-03 16:04:28,564: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-03/16-04-20/agent_actions/episode_3.json\n" - ] - }, - { - "data": { - "text/plain": [ - "({'NODES': {'HOST0': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 1,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'HOST1': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 1,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'HOST2': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 0,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'HOST3': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 0,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'ROUTER0': {'ACL': {1: {'position': 0,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 2: {'position': 1,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 3: {'position': 2,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 4: {'position': 3,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 5: {'position': 4,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 6: {'position': 5,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 7: {'position': 6,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 8: {'position': 7,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 9: {'position': 8,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 10: {'position': 9,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0}},\n", - " 'PORTS': {1: {'operating_status': 1},\n", - " 2: {'operating_status': 1},\n", - " 3: {'operating_status': 2}},\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}}},\n", - " 'LINKS': {1: {'PROTOCOLS': {'ALL': 1}},\n", - " 2: {'PROTOCOLS': {'ALL': 1}},\n", - " 3: {'PROTOCOLS': {'ALL': 0}},\n", - " 4: {'PROTOCOLS': {'ALL': 1}},\n", - " 5: {'PROTOCOLS': {'ALL': 1}},\n", - " 6: {'PROTOCOLS': {'ALL': 1}},\n", - " 7: {'PROTOCOLS': {'ALL': 1}},\n", - " 8: {'PROTOCOLS': {'ALL': 1}},\n", - " 9: {'PROTOCOLS': {'ALL': 1}},\n", - " 10: {'PROTOCOLS': {'ALL': 0}}},\n", - " 'ICS': 0},\n", - " {})" - ] - }, - "execution_count": 51, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "blue_env.reset()" ] @@ -2397,41 +1295,9 @@ }, { "cell_type": "code", - "execution_count": 54, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+---------------------------------------------------------------+\n", - "| router_1 Network Interfaces |\n", - "+------+-------------------+-----------------+-------+----------+\n", - "| Port | MAC Address | Address | Speed | Status |\n", - "+------+-------------------+-----------------+-------+----------+\n", - "| 1 | dd:6e:95:d4:3f:74 | 192.168.1.1/24 | 100.0 | Enabled |\n", - "| 2 | 8b:79:07:fc:69:2c | 192.168.10.1/24 | 100.0 | Enabled |\n", - "| 3 | 1f:fd:c4:ae:7a:00 | 127.0.0.1/8 | 100.0 | Disabled |\n", - "| 4 | 7b:e3:bf:4b:76:e8 | 127.0.0.1/8 | 100.0 | Disabled |\n", - "| 5 | 4f:37:b0:6b:5d:44 | 127.0.0.1/8 | 100.0 | Disabled |\n", - "+------+-------------------+-----------------+-------+----------+\n", - "+------------------------------------------------------------------------------------------------------------------------+\n", - "| router_1 Access Control List |\n", - "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n", - "| Index | Action | Protocol | Src IP | Src Wildcard | Src Port | Dst IP | Dst Wildcard | Dst Port | Matched |\n", - "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n", - "| 1 | DENY | ANY | 192.168.10.21 | ANY | 80 | 192.168.1.12 | ANY | 80 | 0 |\n", - "| 18 | PERMIT | ANY | ANY | ANY | 5432 | ANY | ANY | 5432 | 0 |\n", - "| 19 | PERMIT | ANY | ANY | ANY | 53 | ANY | ANY | 53 | 0 |\n", - "| 20 | PERMIT | ANY | ANY | ANY | 21 | ANY | ANY | 21 | 0 |\n", - "| 21 | PERMIT | ANY | ANY | ANY | 80 | ANY | ANY | 80 | 4 |\n", - "| 22 | PERMIT | ANY | ANY | ANY | 219 | ANY | ANY | 219 | 10 |\n", - "| 23 | PERMIT | icmp | ANY | ANY | ANY | ANY | ANY | ANY | 0 |\n", - "| 24 | DENY | ANY | ANY | ANY | ANY | ANY | ANY | ANY | 0 |\n", - "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n" - ] - } - ], + "outputs": [], "source": [ "router_1: Router = blue_env.game.simulation.network.get_node_by_hostname(\"router_1\")\n", "router_1.show()\n", @@ -2447,20 +1313,9 @@ }, { "cell_type": "code", - "execution_count": 55, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "RequestResponse(status='failure', data={'Reason': 'Command sent to the C2 Beacon but no response was ever received.'})" - ] - }, - "execution_count": 55, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "blue_env.step(0)\n", "\n", @@ -2471,30 +1326,9 @@ }, { "cell_type": "code", - "execution_count": 56, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+------------------------------------------------------------------------------------------------------------------------+\n", - "| router_1 Access Control List |\n", - "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n", - "| Index | Action | Protocol | Src IP | Src Wildcard | Src Port | Dst IP | Dst Wildcard | Dst Port | Matched |\n", - "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n", - "| 1 | DENY | ANY | 192.168.10.21 | ANY | 80 | 192.168.1.12 | ANY | 80 | 2 |\n", - "| 18 | PERMIT | ANY | ANY | ANY | 5432 | ANY | ANY | 5432 | 0 |\n", - "| 19 | PERMIT | ANY | ANY | ANY | 53 | ANY | ANY | 53 | 0 |\n", - "| 20 | PERMIT | ANY | ANY | ANY | 21 | ANY | ANY | 21 | 0 |\n", - "| 21 | PERMIT | ANY | ANY | ANY | 80 | ANY | ANY | 80 | 4 |\n", - "| 22 | PERMIT | ANY | ANY | ANY | 219 | ANY | ANY | 219 | 10 |\n", - "| 23 | PERMIT | icmp | ANY | ANY | ANY | ANY | ANY | ANY | 0 |\n", - "| 24 | DENY | ANY | ANY | ANY | ANY | ANY | ANY | ANY | 0 |\n", - "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n" - ] - } - ], + "outputs": [], "source": [ "router_1.acl.show()" ] @@ -2508,58 +1342,18 @@ }, { "cell_type": "code", - "execution_count": 57, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+-------------------------------------------------------------------------------------+\n", - "| web_server Software Manager |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| Name | Type | Operating State | Health State | Port | Protocol |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", - "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", - "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", - "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", - "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", - "| NMAP | Application | RUNNING | GOOD | None | none |\n", - "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", - "| UserManager | Service | RUNNING | GOOD | None | none |\n", - "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", - "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", - "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", - "| C2Beacon | Application | RUNNING | GOOD | None | tcp |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n" - ] - } - ], + "outputs": [], "source": [ "web_server.software_manager.show()" ] }, { "cell_type": "code", - "execution_count": 58, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+----------------------------------------------------------------------------------+\n", - "| database_server File System |\n", - "+----------------------+---------+---------------+-----------------------+---------+\n", - "| File Path | Size | Health status | Visible health status | Deleted |\n", - "+----------------------+---------+---------------+-----------------------+---------+\n", - "| database/database.db | 4.77 MB | GOOD | NONE | False |\n", - "| root | 0 B | GOOD | NONE | False |\n", - "+----------------------+---------+---------------+-----------------------+---------+\n" - ] - } - ], + "outputs": [], "source": [ "database_server: Server = blue_env.game.simulation.network.get_node_by_hostname(\"database_server\")\n", "database_server.software_manager.file_system.show(full=True)" @@ -2567,33 +1361,9 @@ }, { "cell_type": "code", - "execution_count": 59, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 3\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", - "root['NODES']['ROUTER0']['ACL'][1]['permission']: 0 -> 2\n", - "root['NODES']['ROUTER0']['ACL'][1]['source_ip_id']: 0 -> 7\n", - "root['NODES']['ROUTER0']['ACL'][1]['source_wildcard_id']: 0 -> 1\n", - "root['NODES']['ROUTER0']['ACL'][1]['source_port_id']: 0 -> 2\n", - "root['NODES']['ROUTER0']['ACL'][1]['dest_ip_id']: 0 -> 3\n", - "root['NODES']['ROUTER0']['ACL'][1]['dest_wildcard_id']: 0 -> 1\n", - "root['NODES']['ROUTER0']['ACL'][1]['dest_port_id']: 0 -> 2\n", - "root['NODES']['ROUTER0']['ACL'][1]['protocol_id']: 0 -> 1\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", - "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", - "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" - ] - } - ], + "outputs": [], "source": [ "display_obs_diffs(pre_blue_action_obs, post_blue_action_obs, blue_env.game.step_counter)" ] @@ -2657,17 +1427,9 @@ }, { "cell_type": "code", - "execution_count": 60, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-03 16:04:29,610: PrimaiteGymEnv RNG seed = None\n" - ] - } - ], + "outputs": [], "source": [ "with open(data_manipulation_config_path()) as f:\n", " cfg = yaml.safe_load(f)\n", @@ -2716,30 +1478,9 @@ }, { "cell_type": "code", - "execution_count": 62, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", - "| C2Beacon Running Status |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| False | None | 0 | 5 | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "+-----------------------------------------------------------------------------------------------------+\n", - "| C2Server Running Status |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n", - "| False | None | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n" - ] - } - ], + "outputs": [], "source": [ "env.step(2) # Agent Action Equivalent to c2_beacon.configure(c2_server_ip_address=\"192.168.10.21\")\n", "env.step(3) # Agent action Equivalent to c2_beacon.establish()\n", @@ -2756,30 +1497,9 @@ }, { "cell_type": "code", - "execution_count": 63, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", - "| C2Beacon Running Status |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| False | None | 0 | 5 | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "+-----------------------------------------------------------------------------------------------------+\n", - "| C2Server Running Status |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n", - "| False | None | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n" - ] - } - ], + "outputs": [], "source": [ "env.step(9) # Equivalent of to c2_beacon.configure(c2_server_ip_address=\"192.168.10.22\")\n", "env.step(3)\n", @@ -2797,23 +1517,9 @@ }, { "cell_type": "code", - "execution_count": 64, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+-----------------------------------------------------------------------------------------------------+\n", - "| C2Server Running Status |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n", - "| False | None | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n" - ] - } - ], + "outputs": [], "source": [ "for i in range(6):\n", " env.step(0)\n", @@ -2836,17 +1542,9 @@ }, { "cell_type": "code", - "execution_count": 65, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-03 16:04:30,011: PrimaiteGymEnv RNG seed = None\n" - ] - } - ], + "outputs": [], "source": [ "with open(data_manipulation_config_path()) as f:\n", " cfg = yaml.safe_load(f)\n", @@ -2886,23 +1584,9 @@ }, { "cell_type": "code", - "execution_count": 67, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", - "| C2Beacon Running Status |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| True | 192.168.10.21 | 0 | 5 | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n" - ] - } - ], + "outputs": [], "source": [ "c2_beacon.configure(c2_server_ip_address=\"192.168.10.21\")\n", "c2_beacon.establish()\n", @@ -2920,72 +1604,9 @@ }, { "cell_type": "code", - "execution_count": 68, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 4\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 5\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 6\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 7\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 8\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 9\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 10\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 11\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 12\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 13\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" - ] - } - ], + "outputs": [], "source": [ "for i in range(10):\n", " keep_alive_obs, _, _, _, _ = blue_config_env.step(0)\n", @@ -3001,23 +1622,9 @@ }, { "cell_type": "code", - "execution_count": 69, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", - "| C2Beacon Running Status |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| True | 192.168.10.21 | 0 | 1 | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n" - ] - } - ], + "outputs": [], "source": [ "c2_beacon.configure(c2_server_ip_address=\"192.168.10.21\", keep_alive_frequency=1)\n", "c2_beacon.establish()\n", @@ -3033,40 +1640,9 @@ }, { "cell_type": "code", - "execution_count": 70, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 14\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 15\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" - ] - } - ], + "outputs": [], "source": [ "# Comparing the OBS of the default frequency to a timestep frequency of 1\n", "for i in range(2):\n", @@ -3085,52 +1661,9 @@ }, { "cell_type": "code", - "execution_count": 71, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 16\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 17\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 18\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 19\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 20\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 21\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 22\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" - ] - } - ], + "outputs": [], "source": [ "c2_beacon.configure(c2_server_ip_address=\"192.168.10.21\", keep_alive_frequency=7)\n", "\n", @@ -3167,18 +1700,9 @@ }, { "cell_type": "code", - "execution_count": 72, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-03 16:04:30,864: Resetting environment, episode 0, avg. reward: 0.0\n", - "2025-02-03 16:04:30,867: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-03/16-04-20/agent_actions/episode_0.json\n" - ] - } - ], + "outputs": [], "source": [ "blue_config_env.reset()\n", "\n", @@ -3200,28 +1724,9 @@ }, { "cell_type": "code", - "execution_count": 73, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 5\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" - ] - } - ], + "outputs": [], "source": [ "# Capturing default C2 Traffic\n", "for i in range(3):\n", @@ -3239,23 +1744,9 @@ }, { "cell_type": "code", - "execution_count": 74, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", - "| C2Beacon Running Status |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| True | 192.168.10.21 | 0 | 5 | udp | 53 |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n" - ] - } - ], + "outputs": [], "source": [ "from primaite.utils.validation.ip_protocol import PROTOCOL_LOOKUP\n", "from primaite.utils.validation.port import PORT_LOOKUP\n", @@ -3268,28 +1759,9 @@ }, { "cell_type": "code", - "execution_count": 75, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 10\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 1 -> 0\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 1 -> 0\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['udp'][53]['inbound']: 0 -> 1\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['udp'][53]['outbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 1 -> 0\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 1 -> 0\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['udp'][53]['inbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['udp'][53]['outbound']: 0 -> 1\n" - ] - } - ], + "outputs": [], "source": [ "# Capturing UDP C2 Traffic\n", "for i in range(5):\n", From f7c6ee3df43bc1d46af90dbac0f321a1e911e524 Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Tue, 4 Feb 2025 15:45:19 +0000 Subject: [PATCH 09/24] #3075: C2C-E2E-Demo wildcard_list changes. --- ...ommand-and-Control-E2E-Demonstration.ipynb | 1822 +++++++++++++++-- 1 file changed, 1645 insertions(+), 177 deletions(-) diff --git a/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb b/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb index c2a87e45..b8d1423f 100644 --- a/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb +++ b/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb @@ -13,9 +13,24 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-02-04 15:37:30,830: Performing the PrimAITE first-time setup...\n", + "2025-02-04 15:37:30,830: Building the PrimAITE app directories...\n", + "2025-02-04 15:37:30,830: Building primaite_config.yaml...\n", + "2025-02-04 15:37:30,830: Rebuilding the demo notebooks...\n", + "/home/nick/primaite/4.0.0a1-dev/notebooks/example_notebooks/Command-and-Control-E2E-Demonstration.ipynb\n", + "2025-02-04 15:37:30,832: Reset example notebook: /home/nick/primaite/4.0.0a1-dev/notebooks/example_notebooks/Command-and-Control-E2E-Demonstration.ipynb\n", + "2025-02-04 15:37:30,838: Rebuilding the example notebooks...\n", + "2025-02-04 15:37:30,841: PrimAITE setup complete!\n" + ] + } + ], "source": [ "!primaite setup" ] @@ -62,24 +77,6 @@ " type: ProxyAgent\n", "\n", " action_space:\n", - " # options:\n", - " # nodes:\n", - " # - node_name: web_server\n", - " # applications:\n", - " # - application_name: C2Beacon\n", - " # - node_name: client_1\n", - " # applications:\n", - " # - application_name: C2Server\n", - " # max_folders_per_node: 1\n", - " # max_files_per_folder: 1\n", - " # max_services_per_node: 2\n", - " # max_nics_per_node: 8\n", - " # max_acl_rules: 10\n", - " # ip_list:\n", - " # - 192.168.1.21\n", - " # - 192.168.1.14\n", - " # wildcard_list:\n", - " # - 0.0.0.1\n", " action_map:\n", " 0:\n", " action: do_nothing\n", @@ -93,11 +90,7 @@ " action: configure_c2_beacon\n", " options:\n", " node_name: web_server\n", - " # config:\n", " c2_server_ip_address: 192.168.10.21\n", - " # keep_alive_frequency: 10\n", - " # masquerade_protocol: TCP\n", - " # masquerade_port: DNS\n", " 3:\n", " action: node_application_execute\n", " options:\n", @@ -108,7 +101,6 @@ " options:\n", " node_name: client_1\n", " ip_address:\n", - " # account:\n", " username: admin\n", " password: admin\n", " commands:\n", @@ -121,7 +113,6 @@ " action: c2_server_ransomware_configure\n", " options:\n", " node_name: client_1\n", - " # config:\n", " server_ip_address: 192.168.1.14\n", " payload: ENCRYPT\n", " 6:\n", @@ -132,7 +123,6 @@ " target_folder_name: \"database\"\n", " exfiltration_folder_name: \"spoils\"\n", " target_ip_address: 192.168.1.14\n", - " # account:\n", " username: admin\n", " password: admin\n", "\n", @@ -144,20 +134,12 @@ " action: configure_c2_beacon\n", " options:\n", " node_name: web_server\n", - " # config:\n", " c2_server_ip_address: 192.168.10.21\n", - " # keep_alive_frequency: 10\n", - " # masquerade_protocol: TCP\n", - " # masquerade_port: DNS\n", " 9:\n", " action: configure_c2_beacon\n", " options:\n", " node_name: web_server\n", - " # config:\n", " c2_server_ip_address: 192.168.10.22\n", - " # keep_alive_frequency: 10\n", - " # masquerade_protocol: TCP\n", - " # masquerade_port: DNS\n", "\n", " reward_function:\n", " reward_components:\n", @@ -168,9 +150,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-04 15:37:35,293: PrimaiteGymEnv RNG seed = None\n" + ] + } + ], "source": [ "with open(data_manipulation_config_path()) as f:\n", " cfg = yaml.safe_load(f)\n", @@ -196,9 +186,35 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+--------------------------------------------------------------------------------------+\n", + "| client_1 Software Manager |\n", + "+---------------------+-------------+-----------------+--------------+------+----------+\n", + "| Name | Type | Operating State | Health State | Port | Protocol |\n", + "+---------------------+-------------+-----------------+--------------+------+----------+\n", + "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", + "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", + "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", + "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", + "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", + "| NMAP | Application | RUNNING | GOOD | None | none |\n", + "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", + "| UserManager | Service | RUNNING | GOOD | None | none |\n", + "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", + "| FTPClient | Service | RUNNING | GOOD | 21 | tcp |\n", + "| DataManipulationBot | Application | RUNNING | GOOD | None | none |\n", + "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", + "| C2Server | Application | RUNNING | GOOD | None | tcp |\n", + "+---------------------+-------------+-----------------+--------------+------+----------+\n" + ] + } + ], "source": [ "client_1: Computer = env.game.simulation.network.get_node_by_hostname(\"client_1\")\n", "client_1.software_manager.install(C2Server)\n", @@ -248,9 +264,34 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+-------------------------------------------------------------------------------------+\n", + "| web_server Software Manager |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| Name | Type | Operating State | Health State | Port | Protocol |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", + "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", + "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", + "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", + "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", + "| NMAP | Application | RUNNING | GOOD | None | none |\n", + "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", + "| UserManager | Service | RUNNING | GOOD | None | none |\n", + "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", + "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", + "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", + "| C2Beacon | Application | INSTALLING | UNUSED | None | tcp |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n" + ] + } + ], "source": [ "env.step(1)\n", "web_server: Computer = env.game.simulation.network.get_node_by_hostname(\"web_server\")\n", @@ -290,9 +331,41 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+-------------------------------------------------------------------------------------+\n", + "| web_server Software Manager |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| Name | Type | Operating State | Health State | Port | Protocol |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", + "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", + "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", + "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", + "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", + "| NMAP | Application | RUNNING | GOOD | None | none |\n", + "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", + "| UserManager | Service | RUNNING | GOOD | None | none |\n", + "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", + "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", + "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", + "| C2Beacon | Application | RUNNING | GOOD | None | tcp |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", + "| C2Beacon Running Status |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| False | 192.168.10.21 | 0 | 5 | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n" + ] + } + ], "source": [ "env.step(2)\n", "c2_beacon: C2Beacon = web_server.software_manager.software[\"C2Beacon\"]\n", @@ -331,18 +404,54 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "(0,\n", + " 0.0,\n", + " False,\n", + " False,\n", + " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=2, action='node_application_execute', parameters={'node_name': 'web_server', 'application_name': 'C2Beacon'}, request=['network', 'node', 'web_server', 'application', 'C2Beacon', 'execute'], response=RequestResponse(status='success', data={}), reward=0.0, reward_info={})}})" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "env.step(3)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", + "| C2Beacon Running Status |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| True | 192.168.10.21 | 1 | 5 | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "+-----------------------------------------------------------------------------------------------------+\n", + "| C2Server Running Status |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n", + "| True | 192.168.1.12 | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n" + ] + } + ], "source": [ "c2_beacon.show()\n", "c2_server.show()" @@ -405,18 +514,59 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "(0,\n", + " 0.0,\n", + " False,\n", + " False,\n", + " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=3, action='c2_server_terminal_command', parameters={'node_name': 'client_1', 'ip_address': None, 'username': 'admin', 'password': 'admin', 'commands': [['software_manager', 'application', 'install', 'RansomwareScript']]}, request=['network', 'node', 'client_1', 'application', 'C2Server', 'terminal_command', {'commands': [['software_manager', 'application', 'install', 'RansomwareScript']], 'ip_address': None, 'username': 'admin', 'password': 'admin'}], response=RequestResponse(status='success', data={0: RequestResponse(status='success', data={})}), reward=0.0, reward_info={})}})" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "env.step(4)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+--------------------------------------------------------------------------------------+\n", + "| client_1 Software Manager |\n", + "+---------------------+-------------+-----------------+--------------+------+----------+\n", + "| Name | Type | Operating State | Health State | Port | Protocol |\n", + "+---------------------+-------------+-----------------+--------------+------+----------+\n", + "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", + "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", + "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", + "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", + "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", + "| NMAP | Application | RUNNING | GOOD | None | none |\n", + "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", + "| UserManager | Service | RUNNING | GOOD | None | none |\n", + "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", + "| FTPClient | Service | RUNNING | GOOD | 21 | tcp |\n", + "| DataManipulationBot | Application | RUNNING | GOOD | None | none |\n", + "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", + "| C2Server | Application | RUNNING | GOOD | None | tcp |\n", + "+---------------------+-------------+-----------------+--------------+------+----------+\n" + ] + } + ], "source": [ "client_1.software_manager.show()" ] @@ -455,18 +605,66 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "(0,\n", + " 0.0,\n", + " False,\n", + " False,\n", + " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=4, action='c2_server_ransomware_configure', parameters={'node_name': 'client_1', 'server_ip_address': '192.168.1.14', 'payload': 'ENCRYPT'}, request=['network', 'node', 'client_1', 'application', 'C2Server', 'ransomware_configure', {'server_ip_address': '192.168.1.14', 'server_password': None, 'payload': 'ENCRYPT'}], response=RequestResponse(status='success', data={}), reward=0.0, reward_info={})}})" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "env.step(5)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+-------------------------------------------------------------------------------------+\n", + "| web_server Software Manager |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| Name | Type | Operating State | Health State | Port | Protocol |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", + "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", + "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", + "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", + "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", + "| NMAP | Application | RUNNING | GOOD | None | none |\n", + "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", + "| UserManager | Service | RUNNING | GOOD | None | none |\n", + "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", + "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", + "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", + "| C2Beacon | Application | RUNNING | GOOD | None | tcp |\n", + "| RansomwareScript | Application | RUNNING | GOOD | None | none |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "+------------------------------------+\n", + "| RansomwareScript Running Status |\n", + "+--------------------------+---------+\n", + "| Target Server IP Address | Payload |\n", + "+--------------------------+---------+\n", + "| 192.168.1.14 | ENCRYPT |\n", + "+--------------------------+---------+\n" + ] + } + ], "source": [ "ransomware_script: RansomwareScript = web_server.software_manager.software[\"RansomwareScript\"]\n", "web_server.software_manager.show()\n", @@ -512,18 +710,48 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "(0,\n", + " 0.0,\n", + " False,\n", + " False,\n", + " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=5, action='c2_server_data_exfiltrate', parameters={'node_name': 'client_1', 'target_file_name': 'database.db', 'target_folder_name': 'database', 'exfiltration_folder_name': 'spoils', 'target_ip_address': '192.168.1.14', 'username': 'admin', 'password': 'admin'}, request=['network', 'node', 'client_1', 'application', 'C2Server', 'exfiltrate', {'target_file_name': 'database.db', 'target_folder_name': 'database', 'exfiltration_folder_name': 'spoils', 'target_ip_address': '192.168.1.14', 'username': 'admin', 'password': 'admin'}], response=RequestResponse(status='success', data={}), reward=0.0, reward_info={})}})" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "env.step(6)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+--------------------------------------------------------------------------------+\n", + "| client_1 File System |\n", + "+--------------------+---------+---------------+-----------------------+---------+\n", + "| File Path | Size | Health status | Visible health status | Deleted |\n", + "+--------------------+---------+---------------+-----------------------+---------+\n", + "| root | 0 B | GOOD | NONE | False |\n", + "| spoils/database.db | 4.77 MB | GOOD | NONE | False |\n", + "+--------------------+---------+---------------+-----------------------+---------+\n" + ] + } + ], "source": [ "client_1: Computer = env.game.simulation.network.get_node_by_hostname(\"client_1\")\n", "client_1.software_manager.file_system.show(full=True)" @@ -531,9 +759,25 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+---------------------------------------------------------------------------------+\n", + "| web_server File System |\n", + "+---------------------+---------+---------------+-----------------------+---------+\n", + "| File Path | Size | Health status | Visible health status | Deleted |\n", + "+---------------------+---------+---------------+-----------------------+---------+\n", + "| primaite/index.html | 15.0 KB | GOOD | NONE | False |\n", + "| root | 0 B | GOOD | NONE | False |\n", + "| spoils/database.db | 4.77 MB | GOOD | NONE | False |\n", + "+---------------------+---------+---------------+-----------------------+---------+\n" + ] + } + ], "source": [ "web_server: Computer = env.game.simulation.network.get_node_by_hostname(\"web_server\")\n", "web_server.software_manager.file_system.show(full=True)" @@ -570,18 +814,48 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "(0,\n", + " 0.0,\n", + " False,\n", + " False,\n", + " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=6, action='c2_server_ransomware_launch', parameters={'node_name': 'client_1'}, request=['network', 'node', 'client_1', 'application', 'C2Server', 'ransomware_launch'], response=RequestResponse(status='success', data={}), reward=0.0, reward_info={})}})" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "env.step(7)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----------------------------------------------------------------------------------+\n", + "| database_server File System |\n", + "+----------------------+---------+---------------+-----------------------+---------+\n", + "| File Path | Size | Health status | Visible health status | Deleted |\n", + "+----------------------+---------+---------------+-----------------------+---------+\n", + "| database/database.db | 4.77 MB | CORRUPT | NONE | False |\n", + "| root | 0 B | GOOD | NONE | False |\n", + "+----------------------+---------+---------------+-----------------------+---------+\n" + ] + } + ], "source": [ "database_server: Server = env.game.simulation.network.get_node_by_hostname(\"database_server\")\n", "database_server.software_manager.file_system.show(full=True)" @@ -600,7 +874,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "metadata": {}, "outputs": [], "source": [ @@ -713,35 +987,9 @@ " src_port: HTTP\n", " dst_port: HTTP\n", " protocol_name: ALL\n", - " src_wildcard: NONE\n", - " dst_wildcard: NONE\n", + " src_wildcard: 0.0.0.1\n", + " dst_wildcard: 0.0.0.1\n", "\n", - "\n", - " # options:\n", - " # # nodes:\n", - " # node_name: web_server\n", - " # # applications:\n", - " # application_name: C2Beacon\n", - "\n", - " # node_name: database_server\n", - " # folders:\n", - " # - folder_name: database\n", - " # files:\n", - " # - file_name: database.db\n", - " # services:\n", - " # - service_name: DatabaseService\n", - " # node_name: router_1\n", - "\n", - " # max_folders_per_node: 2\n", - " # max_files_per_folder: 2\n", - " # max_services_per_node: 2\n", - " # max_nics_per_node: 8\n", - " # max_acl_rules: 10\n", - " # ip_list:\n", - " # - 192.168.10.21\n", - " # - 192.168.1.12\n", - " # wildcard_list:\n", - " # - 0.0.0.1\n", " reward_function:\n", " reward_components:\n", " - type: DUMMY\n", @@ -754,9 +1002,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-04 15:37:36,689: PrimaiteGymEnv RNG seed = None\n" + ] + } + ], "source": [ "with open(data_manipulation_config_path()) as f:\n", " cfg = yaml.safe_load(f)\n", @@ -813,9 +1069,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 22, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-04 15:37:37,079: Resetting environment, episode 0, avg. reward: 0.0\n", + "2025-02-04 15:37:37,082: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-04/15-37-31/agent_actions/episode_0.json\n" + ] + } + ], "source": [ "# Resetting the environment and capturing the default observation space.\n", "blue_env.reset()\n", @@ -824,9 +1089,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 23, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Setting up the C2 Suite via the simulation API.\n", "\n", @@ -857,9 +1133,26 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 25, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 2\n", + "root['NODES']['HOST0']['APPLICATIONS'][1]['operating_status']: 0 -> 1\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", + "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", + "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" + ] + } + ], "source": [ "display_obs_diffs(default_obs, c2_configuration_obs, blue_env.game.step_counter)" ] @@ -879,9 +1172,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 26, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "RequestResponse(status='success', data={0: RequestResponse(status='success', data={})})" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Installing RansomwareScript via C2 Terminal Commands\n", "ransomware_install_command = {\"commands\":[[\"software_manager\", \"application\", \"install\", \"RansomwareScript\"]],\n", @@ -892,9 +1196,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 27, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "RequestResponse(status='success', data={})" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Configuring the RansomwareScript\n", "ransomware_config = {\"server_ip_address\": \"192.168.1.14\", \"payload\": \"ENCRYPT\"}\n", @@ -913,9 +1228,28 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 29, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 7\n", + "root['NODES']['HOST0']['APPLICATIONS'][1]['operating_status']: 0 -> 1\n", + "root['NODES']['HOST0']['APPLICATIONS'][2]['operating_status']: 0 -> 3\n", + "root['NODES']['HOST0']['users']['local_login']: 0 -> 1\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", + "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", + "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" + ] + } + ], "source": [ "display_obs_diffs(default_obs, c2_ransomware_obs, env.game.step_counter)" ] @@ -947,9 +1281,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 31, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "RequestResponse(status='success', data={})" + ] + }, + "execution_count": 31, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "c2_server.send_command(given_command=C2Command.DATA_EXFILTRATION, command_options=exfil_options)" ] @@ -965,9 +1310,22 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 33, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 7\n", + "root['NODES']['HOST0']['APPLICATIONS'][2]['operating_status']: 3 -> 1\n", + "root['NODES']['HOST1']['users']['remote_sessions']: 0 -> 1\n" + ] + } + ], "source": [ "display_obs_diffs(c2_ransomware_obs, c2_exfil_obs, env.game.step_counter)" ] @@ -983,9 +1341,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 34, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "RequestResponse(status='success', data={})" + ] + }, + "execution_count": 34, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Configuring the RansomwareScript\n", "ransomware_config = {\"server_ip_address\": \"192.168.1.14\", \"payload\": \"ENCRYPT\"}\n", @@ -994,9 +1363,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 35, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "RequestResponse(status='success', data={})" + ] + }, + "execution_count": 35, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Waiting for the ransomware to finish installing and then launching the RansomwareScript.\n", "blue_env.step(0)\n", @@ -1015,9 +1395,30 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 37, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 6\n", + "root['NODES']['HOST0']['APPLICATIONS'][2]['operating_status']: 3 -> 1\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['NODES']['HOST1']['users']['remote_sessions']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" + ] + } + ], "source": [ "display_obs_diffs(c2_ransomware_obs, c2_final_obs, blue_env.game.step_counter)" ] @@ -1065,9 +1466,193 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 39, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-04 15:37:38,317: Resetting environment, episode 1, avg. reward: 0.0\n", + "2025-02-04 15:37:38,321: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-04/15-37-31/agent_actions/episode_1.json\n" + ] + }, + { + "data": { + "text/plain": [ + "({'NODES': {'HOST0': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 1,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'HOST1': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 1,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'HOST2': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 0,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'HOST3': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 0,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'ROUTER0': {'ACL': {1: {'position': 0,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 2: {'position': 1,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 3: {'position': 2,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 4: {'position': 3,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 5: {'position': 4,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 6: {'position': 5,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 7: {'position': 6,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 8: {'position': 7,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 9: {'position': 8,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 10: {'position': 9,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0}},\n", + " 'PORTS': {1: {'operating_status': 1},\n", + " 2: {'operating_status': 1},\n", + " 3: {'operating_status': 2}},\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}}},\n", + " 'LINKS': {1: {'PROTOCOLS': {'ALL': 1}},\n", + " 2: {'PROTOCOLS': {'ALL': 1}},\n", + " 3: {'PROTOCOLS': {'ALL': 0}},\n", + " 4: {'PROTOCOLS': {'ALL': 1}},\n", + " 5: {'PROTOCOLS': {'ALL': 1}},\n", + " 6: {'PROTOCOLS': {'ALL': 1}},\n", + " 7: {'PROTOCOLS': {'ALL': 1}},\n", + " 8: {'PROTOCOLS': {'ALL': 1}},\n", + " 9: {'PROTOCOLS': {'ALL': 1}},\n", + " 10: {'PROTOCOLS': {'ALL': 0}}},\n", + " 'ICS': 0},\n", + " {})" + ] + }, + "execution_count": 39, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "blue_env.reset()" ] @@ -1110,9 +1695,33 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 42, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+-------------------------------------------------------------------------------------+\n", + "| web_server Software Manager |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| Name | Type | Operating State | Health State | Port | Protocol |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", + "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", + "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", + "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", + "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", + "| NMAP | Application | RUNNING | GOOD | None | none |\n", + "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", + "| UserManager | Service | RUNNING | GOOD | None | none |\n", + "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", + "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", + "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n" + ] + } + ], "source": [ "blue_env.step(0)\n", "web_server.software_manager.show()" @@ -1120,9 +1729,26 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 43, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 3\n", + "root['NODES']['HOST0']['APPLICATIONS'][1]['operating_status']: 1 -> 0\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", + "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", + "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" + ] + } + ], "source": [ "display_obs_diffs(pre_blue_action_obs, post_blue_action_obs, blue_env.game.step_counter)" ] @@ -1136,9 +1762,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 44, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "RequestResponse(status='failure', data={'Reason': 'Command sent to the C2 Beacon but no response was ever received.'})" + ] + }, + "execution_count": 44, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Attempting to install the C2 RansomwareScript\n", "ransomware_install_command = {\"commands\":[[\"software_manager\", \"application\", \"install\", \"RansomwareScript\"]],\n", @@ -1160,9 +1797,193 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 45, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-04 15:37:38,778: Resetting environment, episode 2, avg. reward: 0.0\n", + "2025-02-04 15:37:38,781: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-04/15-37-31/agent_actions/episode_2.json\n" + ] + }, + { + "data": { + "text/plain": [ + "({'NODES': {'HOST0': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 1,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'HOST1': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 1,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'HOST2': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 0,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'HOST3': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 0,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'ROUTER0': {'ACL': {1: {'position': 0,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 2: {'position': 1,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 3: {'position': 2,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 4: {'position': 3,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 5: {'position': 4,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 6: {'position': 5,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 7: {'position': 6,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 8: {'position': 7,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 9: {'position': 8,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 10: {'position': 9,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0}},\n", + " 'PORTS': {1: {'operating_status': 1},\n", + " 2: {'operating_status': 1},\n", + " 3: {'operating_status': 2}},\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}}},\n", + " 'LINKS': {1: {'PROTOCOLS': {'ALL': 1}},\n", + " 2: {'PROTOCOLS': {'ALL': 1}},\n", + " 3: {'PROTOCOLS': {'ALL': 0}},\n", + " 4: {'PROTOCOLS': {'ALL': 1}},\n", + " 5: {'PROTOCOLS': {'ALL': 1}},\n", + " 6: {'PROTOCOLS': {'ALL': 1}},\n", + " 7: {'PROTOCOLS': {'ALL': 1}},\n", + " 8: {'PROTOCOLS': {'ALL': 1}},\n", + " 9: {'PROTOCOLS': {'ALL': 1}},\n", + " 10: {'PROTOCOLS': {'ALL': 0}}},\n", + " 'ICS': 0},\n", + " {})" + ] + }, + "execution_count": 45, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "blue_env.reset()" ] @@ -1205,9 +2026,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 48, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "NodeOperatingState.SHUTTING_DOWN\n" + ] + } + ], "source": [ "web_server = blue_env.game.simulation.network.get_node_by_hostname(\"web_server\")\n", "print(web_server.operating_state)" @@ -1215,18 +2044,48 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 49, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 2\n", + "root['NODES']['HOST0']['operating_status']: 1 -> 4\n", + "root['NODES']['HOST0']['APPLICATIONS'][1]['operating_status']: 1 -> 0\n", + "root['NODES']['HOST0']['NICS'][1]['nic_status']: 1 -> 0\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", + "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", + "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" + ] + } + ], "source": [ "display_obs_diffs(pre_blue_action_obs, post_blue_action_obs, blue_env.game.step_counter)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 50, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "RequestResponse(status='failure', data={'Reason': 'Command sent to the C2 Beacon but no response was ever received.'})" + ] + }, + "execution_count": 50, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Attempting to install the C2 RansomwareScript\n", "ransomware_install_command = {\"commands\":[\"software_manager\", \"application\", \"install\", \"RansomwareScript\"],\n", @@ -1250,9 +2109,193 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 51, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-04 15:37:39,257: Resetting environment, episode 3, avg. reward: 0.0\n", + "2025-02-04 15:37:39,260: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-04/15-37-31/agent_actions/episode_3.json\n" + ] + }, + { + "data": { + "text/plain": [ + "({'NODES': {'HOST0': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 1,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'HOST1': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 1,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'HOST2': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 0,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'HOST3': {'APPLICATIONS': {1: {'operating_status': 0,\n", + " 'health_status': 0,\n", + " 'num_executions': 0},\n", + " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", + " 'FOLDERS': {1: {'health_status': 0,\n", + " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", + " 'NICS': {1: {'nic_status': 1,\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", + " 53: {'inbound': 0, 'outbound': 0},\n", + " 21: {'inbound': 0, 'outbound': 0}}}}},\n", + " 'num_file_creations': 0,\n", + " 'num_file_deletions': 0,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0},\n", + " 'operating_status': 1},\n", + " 'ROUTER0': {'ACL': {1: {'position': 0,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 2: {'position': 1,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 3: {'position': 2,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 4: {'position': 3,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 5: {'position': 4,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 6: {'position': 5,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 7: {'position': 6,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 8: {'position': 7,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 9: {'position': 8,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0},\n", + " 10: {'position': 9,\n", + " 'permission': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_wildcard_id': 0,\n", + " 'source_port_id': 0,\n", + " 'dest_ip_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'protocol_id': 0}},\n", + " 'PORTS': {1: {'operating_status': 1},\n", + " 2: {'operating_status': 1},\n", + " 3: {'operating_status': 2}},\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}}},\n", + " 'LINKS': {1: {'PROTOCOLS': {'ALL': 1}},\n", + " 2: {'PROTOCOLS': {'ALL': 1}},\n", + " 3: {'PROTOCOLS': {'ALL': 0}},\n", + " 4: {'PROTOCOLS': {'ALL': 1}},\n", + " 5: {'PROTOCOLS': {'ALL': 1}},\n", + " 6: {'PROTOCOLS': {'ALL': 1}},\n", + " 7: {'PROTOCOLS': {'ALL': 1}},\n", + " 8: {'PROTOCOLS': {'ALL': 1}},\n", + " 9: {'PROTOCOLS': {'ALL': 1}},\n", + " 10: {'PROTOCOLS': {'ALL': 0}}},\n", + " 'ICS': 0},\n", + " {})" + ] + }, + "execution_count": 51, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "blue_env.reset()" ] @@ -1295,12 +2338,43 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 54, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+---------------------------------------------------------------+\n", + "| router_1 Network Interfaces |\n", + "+------+-------------------+-----------------+-------+----------+\n", + "| Port | MAC Address | Address | Speed | Status |\n", + "+------+-------------------+-----------------+-------+----------+\n", + "| 1 | dd:bc:17:75:a2:c4 | 192.168.1.1/24 | 100.0 | Enabled |\n", + "| 2 | 00:f2:f5:65:51:75 | 192.168.10.1/24 | 100.0 | Enabled |\n", + "| 3 | de:7d:ac:0e:58:e1 | 127.0.0.1/8 | 100.0 | Disabled |\n", + "| 4 | 8a:b4:55:12:c7:9e | 127.0.0.1/8 | 100.0 | Disabled |\n", + "| 5 | a5:ef:c6:53:97:c4 | 127.0.0.1/8 | 100.0 | Disabled |\n", + "+------+-------------------+-----------------+-------+----------+\n", + "+------------------------------------------------------------------------------------------------------------------------+\n", + "| router_1 Access Control List |\n", + "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n", + "| Index | Action | Protocol | Src IP | Src Wildcard | Src Port | Dst IP | Dst Wildcard | Dst Port | Matched |\n", + "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n", + "| 1 | DENY | ANY | 192.168.10.21 | 0.0.0.1 | 80 | 192.168.1.12 | 0.0.0.1 | 80 | 0 |\n", + "| 18 | PERMIT | ANY | ANY | ANY | 5432 | ANY | ANY | 5432 | 0 |\n", + "| 19 | PERMIT | ANY | ANY | ANY | 53 | ANY | ANY | 53 | 0 |\n", + "| 20 | PERMIT | ANY | ANY | ANY | 21 | ANY | ANY | 21 | 0 |\n", + "| 21 | PERMIT | ANY | ANY | ANY | 80 | ANY | ANY | 80 | 4 |\n", + "| 22 | PERMIT | ANY | ANY | ANY | 219 | ANY | ANY | 219 | 10 |\n", + "| 23 | PERMIT | icmp | ANY | ANY | ANY | ANY | ANY | ANY | 0 |\n", + "| 24 | DENY | ANY | ANY | ANY | ANY | ANY | ANY | ANY | 0 |\n", + "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n" + ] + } + ], "source": [ "router_1: Router = blue_env.game.simulation.network.get_node_by_hostname(\"router_1\")\n", - "router_1.show()\n", "router_1.acl.show()" ] }, @@ -1313,9 +2387,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 55, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "RequestResponse(status='failure', data={'Reason': 'Command sent to the C2 Beacon but no response was ever received.'})" + ] + }, + "execution_count": 55, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "blue_env.step(0)\n", "\n", @@ -1326,9 +2411,30 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 56, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+------------------------------------------------------------------------------------------------------------------------+\n", + "| router_1 Access Control List |\n", + "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n", + "| Index | Action | Protocol | Src IP | Src Wildcard | Src Port | Dst IP | Dst Wildcard | Dst Port | Matched |\n", + "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n", + "| 1 | DENY | ANY | 192.168.10.21 | 0.0.0.1 | 80 | 192.168.1.12 | 0.0.0.1 | 80 | 2 |\n", + "| 18 | PERMIT | ANY | ANY | ANY | 5432 | ANY | ANY | 5432 | 0 |\n", + "| 19 | PERMIT | ANY | ANY | ANY | 53 | ANY | ANY | 53 | 0 |\n", + "| 20 | PERMIT | ANY | ANY | ANY | 21 | ANY | ANY | 21 | 0 |\n", + "| 21 | PERMIT | ANY | ANY | ANY | 80 | ANY | ANY | 80 | 4 |\n", + "| 22 | PERMIT | ANY | ANY | ANY | 219 | ANY | ANY | 219 | 10 |\n", + "| 23 | PERMIT | icmp | ANY | ANY | ANY | ANY | ANY | ANY | 0 |\n", + "| 24 | DENY | ANY | ANY | ANY | ANY | ANY | ANY | ANY | 0 |\n", + "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n" + ] + } + ], "source": [ "router_1.acl.show()" ] @@ -1342,18 +2448,58 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 57, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+-------------------------------------------------------------------------------------+\n", + "| web_server Software Manager |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| Name | Type | Operating State | Health State | Port | Protocol |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n", + "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", + "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", + "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", + "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", + "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", + "| NMAP | Application | RUNNING | GOOD | None | none |\n", + "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", + "| UserManager | Service | RUNNING | GOOD | None | none |\n", + "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", + "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", + "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", + "| C2Beacon | Application | RUNNING | GOOD | None | tcp |\n", + "+--------------------+-------------+-----------------+--------------+------+----------+\n" + ] + } + ], "source": [ "web_server.software_manager.show()" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 58, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----------------------------------------------------------------------------------+\n", + "| database_server File System |\n", + "+----------------------+---------+---------------+-----------------------+---------+\n", + "| File Path | Size | Health status | Visible health status | Deleted |\n", + "+----------------------+---------+---------------+-----------------------+---------+\n", + "| database/database.db | 4.77 MB | GOOD | NONE | False |\n", + "| root | 0 B | GOOD | NONE | False |\n", + "+----------------------+---------+---------------+-----------------------+---------+\n" + ] + } + ], "source": [ "database_server: Server = blue_env.game.simulation.network.get_node_by_hostname(\"database_server\")\n", "database_server.software_manager.file_system.show(full=True)" @@ -1361,9 +2507,33 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 59, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 3\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", + "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", + "root['NODES']['ROUTER0']['ACL'][1]['permission']: 0 -> 2\n", + "root['NODES']['ROUTER0']['ACL'][1]['source_ip_id']: 0 -> 7\n", + "root['NODES']['ROUTER0']['ACL'][1]['source_wildcard_id']: 0 -> 2\n", + "root['NODES']['ROUTER0']['ACL'][1]['source_port_id']: 0 -> 2\n", + "root['NODES']['ROUTER0']['ACL'][1]['dest_ip_id']: 0 -> 3\n", + "root['NODES']['ROUTER0']['ACL'][1]['dest_wildcard_id']: 0 -> 2\n", + "root['NODES']['ROUTER0']['ACL'][1]['dest_port_id']: 0 -> 2\n", + "root['NODES']['ROUTER0']['ACL'][1]['protocol_id']: 0 -> 1\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", + "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", + "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" + ] + } + ], "source": [ "display_obs_diffs(pre_blue_action_obs, post_blue_action_obs, blue_env.game.step_counter)" ] @@ -1427,9 +2597,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 60, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-04 15:37:40,175: PrimaiteGymEnv RNG seed = None\n" + ] + } + ], "source": [ "with open(data_manipulation_config_path()) as f:\n", " cfg = yaml.safe_load(f)\n", @@ -1478,9 +2656,30 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 62, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", + "| C2Beacon Running Status |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| False | None | 0 | 5 | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "+-----------------------------------------------------------------------------------------------------+\n", + "| C2Server Running Status |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n", + "| False | None | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n" + ] + } + ], "source": [ "env.step(2) # Agent Action Equivalent to c2_beacon.configure(c2_server_ip_address=\"192.168.10.21\")\n", "env.step(3) # Agent action Equivalent to c2_beacon.establish()\n", @@ -1497,9 +2696,30 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 63, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", + "| C2Beacon Running Status |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| False | None | 0 | 5 | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "+-----------------------------------------------------------------------------------------------------+\n", + "| C2Server Running Status |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n", + "| False | None | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n" + ] + } + ], "source": [ "env.step(9) # Equivalent of to c2_beacon.configure(c2_server_ip_address=\"192.168.10.22\")\n", "env.step(3)\n", @@ -1517,9 +2737,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 64, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+-----------------------------------------------------------------------------------------------------+\n", + "| C2Server Running Status |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n", + "| False | None | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------------+-------------------------+\n" + ] + } + ], "source": [ "for i in range(6):\n", " env.step(0)\n", @@ -1542,9 +2776,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 65, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-04 15:37:40,588: PrimaiteGymEnv RNG seed = None\n" + ] + } + ], "source": [ "with open(data_manipulation_config_path()) as f:\n", " cfg = yaml.safe_load(f)\n", @@ -1584,9 +2826,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 67, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", + "| C2Beacon Running Status |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| True | 192.168.10.21 | 0 | 5 | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n" + ] + } + ], "source": [ "c2_beacon.configure(c2_server_ip_address=\"192.168.10.21\")\n", "c2_beacon.establish()\n", @@ -1604,9 +2860,72 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 68, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 4\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 5\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 6\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 7\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 8\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 9\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 10\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 11\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 12\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 13\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" + ] + } + ], "source": [ "for i in range(10):\n", " keep_alive_obs, _, _, _, _ = blue_config_env.step(0)\n", @@ -1622,9 +2941,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 69, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", + "| C2Beacon Running Status |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| True | 192.168.10.21 | 0 | 1 | tcp | 80 |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n" + ] + } + ], "source": [ "c2_beacon.configure(c2_server_ip_address=\"192.168.10.21\", keep_alive_frequency=1)\n", "c2_beacon.establish()\n", @@ -1640,9 +2973,40 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 70, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 14\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 15\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" + ] + } + ], "source": [ "# Comparing the OBS of the default frequency to a timestep frequency of 1\n", "for i in range(2):\n", @@ -1661,9 +3025,52 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 71, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 16\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 17\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 18\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 19\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 20\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 21\n", + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 22\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" + ] + } + ], "source": [ "c2_beacon.configure(c2_server_ip_address=\"192.168.10.21\", keep_alive_frequency=7)\n", "\n", @@ -1700,9 +3107,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 72, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-04 15:37:41,322: Resetting environment, episode 0, avg. reward: 0.0\n", + "2025-02-04 15:37:41,325: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-04/15-37-31/agent_actions/episode_0.json\n" + ] + } + ], "source": [ "blue_config_env.reset()\n", "\n", @@ -1724,9 +3140,28 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 73, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 5\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", + "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", + "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" + ] + } + ], "source": [ "# Capturing default C2 Traffic\n", "for i in range(3):\n", @@ -1744,9 +3179,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 74, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", + "| C2Beacon Running Status |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", + "| True | 192.168.10.21 | 0 | 5 | udp | 53 |\n", + "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n" + ] + } + ], "source": [ "from primaite.utils.validation.ip_protocol import PROTOCOL_LOOKUP\n", "from primaite.utils.validation.port import PORT_LOOKUP\n", @@ -1759,9 +3208,28 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 75, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation space differences\n", + "-----------------------------\n", + "Step 10\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 1 -> 0\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 1 -> 0\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['udp'][53]['inbound']: 0 -> 1\n", + "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['udp'][53]['outbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 1 -> 0\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 1 -> 0\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['udp'][53]['inbound']: 0 -> 1\n", + "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['udp'][53]['outbound']: 0 -> 1\n" + ] + } + ], "source": [ "# Capturing UDP C2 Traffic\n", "for i in range(5):\n", From 3a29593f600ba3fa1b9b74806333bf7934e4303f Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Wed, 5 Feb 2025 11:50:38 +0000 Subject: [PATCH 10/24] 3075: Fix bug in notebook helper function. --- ...a-Manipulation-Customising-Red-Agent.ipynb | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb b/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb index de6d3fe5..14938c02 100644 --- a/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb +++ b/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb @@ -6,7 +6,7 @@ "source": [ "# Customising UC2 Red Agents\n", "\n", - "© Crown-owned copyright 2024, Defence Science and Technology Laboratory UK\n", + "© Crown-owned copyright 2025, Defence Science and Technology Laboratory UK\n", "\n", "This notebook will go over some examples of how red agent behaviour can be varied by changing its configuration parameters.\n", "\n", @@ -26,7 +26,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -36,13 +36,12 @@ "from primaite.game.agent.interface import AgentHistoryItem\n", "from primaite.session.environment import PrimaiteGymEnv\n", "import yaml\n", - "from pprint import pprint\n", - "from primaite.game.agent.scripted_agents import probabilistic_agent, data_manipulation_bot" + "from pprint import pprint" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -69,7 +68,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -80,9 +79,7 @@ " if red_action == 'do_nothing':\n", " red_str = 'DO NOTHING'\n", " elif red_action == 'node_application_execute':\n", - " client = \"client 1\" if red_info.parameters['node_name'] == 0 else \"client 2\"\n", - "\n", - " red_str = f\"ATTACK from {client}\"\n", + " red_str = f\"ATTACK from {red_info.parameters['node_name']}\"\n", " return red_str" ] }, @@ -245,7 +242,7 @@ "outputs": [], "source": [ "change = yaml.safe_load(\"\"\"\n", - " possible_start_nodes: [client_1, client_2]\n", + " possible_start_nodes: [client_1]\n", " target_application: DataManipulationBot\n", " start_step: 25\n", " frequency: 20\n", @@ -315,6 +312,10 @@ "outputs": [], "source": [ "change = yaml.safe_load(\"\"\"\n", + " agent_settings:\n", + " possible_start_nodes: [client_1]\n", + " target_application: DataManipulationBot\n", + "\n", " action_space:\n", " action_map:\n", " 0:\n", From a4c19608a69251c0dabff77c69fb02fab544b3c7 Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Wed, 5 Feb 2025 11:51:22 +0000 Subject: [PATCH 11/24] 3075: Pre-commit tidy up. --- docs/source/how_to_guides/extensible_nodes.rst | 2 +- src/primaite/game/game.py | 1 - src/primaite/notebooks/Action-masking.ipynb | 15 +++++++-------- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/docs/source/how_to_guides/extensible_nodes.rst b/docs/source/how_to_guides/extensible_nodes.rst index 043d0f06..4819cbb2 100644 --- a/docs/source/how_to_guides/extensible_nodes.rst +++ b/docs/source/how_to_guides/extensible_nodes.rst @@ -52,4 +52,4 @@ class Router(NetworkNode, identifier="router"): Changes to YAML file. ===================== -While effort has been made to ensure that nodes defined within configuration YAML files for use with PrimAITE 3.X remain compatible with PrimAITE v4+, it is encouraged to review for minor changes needed. +While effort has been made to ensure that nodes defined within configuration YAML files for use with PrimAITE 3.X remain compatible with PrimAITE v4+, it is encouraged to review for minor changes needed. diff --git a/src/primaite/game/game.py b/src/primaite/game/game.py index f819fa05..e9941a12 100644 --- a/src/primaite/game/game.py +++ b/src/primaite/game/game.py @@ -404,7 +404,6 @@ class PrimaiteGame: agents_cfg = cfg.get("agents", []) for agent_cfg in agents_cfg: - new_agent = AbstractAgent.from_config(agent_cfg) game.agents[agent_cfg["ref"]] = new_agent if isinstance(new_agent, ProxyAgent): diff --git a/src/primaite/notebooks/Action-masking.ipynb b/src/primaite/notebooks/Action-masking.ipynb index 64015080..74504878 100644 --- a/src/primaite/notebooks/Action-masking.ipynb +++ b/src/primaite/notebooks/Action-masking.ipynb @@ -6,7 +6,7 @@ "source": [ "# Action Masking\n", "\n", - "© Crown-owned copyright 2024, Defence Science and Technology Laboratory UK\n", + "© Crown-owned copyright 2025, Defence Science and Technology Laboratory UK\n", "\n", "PrimAITE environments support action masking. The action mask shows which of the agent's actions are applicable with the current environment state. For example, a node can only be turned on if it is currently turned off." ] @@ -22,14 +22,13 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from primaite.session.environment import PrimaiteGymEnv\n", "from primaite.config.load import data_manipulation_config_path\n", - "from prettytable import PrettyTable\n", - "from primaite.game.agent.scripted_agents import probabilistic_agent\n" + "from prettytable import PrettyTable" ] }, { @@ -104,7 +103,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -117,7 +116,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -157,7 +156,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -168,7 +167,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ From b22ba65eb69644a25ad6451d3fadfe87e5c8112d Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Wed, 5 Feb 2025 16:40:59 +0000 Subject: [PATCH 12/24] #3075: Further notebook updates. --- ...ommand-and-Control-E2E-Demonstration.ipynb | 1792 ++--------------- .../Data-Manipulation-E2E-Demonstration.ipynb | 6 +- .../Getting-Information-Out-Of-PrimAITE.ipynb | 4 +- ...ege-Escalation-and-Data-Loss-Example.ipynb | 11 +- .../notebooks/Requests-and-Responses.ipynb | 22 +- .../notebooks/Terminal-Processing.ipynb | 43 +- .../Training-an-RLLIB-MARL-System.ipynb | 31 +- .../notebooks/Training-an-RLLib-Agent.ipynb | 31 +- .../notebooks/Training-an-SB3-Agent.ipynb | 2 +- .../notebooks/Using-Episode-Schedules.ipynb | 18 +- src/primaite/notebooks/multi-processing.ipynb | 15 +- .../simulator/network/hardware/base.py | 8 +- 12 files changed, 224 insertions(+), 1759 deletions(-) diff --git a/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb b/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb index b8d1423f..d3c414d7 100644 --- a/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb +++ b/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb @@ -13,31 +13,16 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-02-04 15:37:30,830: Performing the PrimAITE first-time setup...\n", - "2025-02-04 15:37:30,830: Building the PrimAITE app directories...\n", - "2025-02-04 15:37:30,830: Building primaite_config.yaml...\n", - "2025-02-04 15:37:30,830: Rebuilding the demo notebooks...\n", - "/home/nick/primaite/4.0.0a1-dev/notebooks/example_notebooks/Command-and-Control-E2E-Demonstration.ipynb\n", - "2025-02-04 15:37:30,832: Reset example notebook: /home/nick/primaite/4.0.0a1-dev/notebooks/example_notebooks/Command-and-Control-E2E-Demonstration.ipynb\n", - "2025-02-04 15:37:30,838: Rebuilding the example notebooks...\n", - "2025-02-04 15:37:30,841: PrimAITE setup complete!\n" - ] - } - ], + "outputs": [], "source": [ "!primaite setup" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -67,7 +52,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -150,17 +135,9 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-04 15:37:35,293: PrimaiteGymEnv RNG seed = None\n" - ] - } - ], + "outputs": [], "source": [ "with open(data_manipulation_config_path()) as f:\n", " cfg = yaml.safe_load(f)\n", @@ -186,35 +163,9 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+--------------------------------------------------------------------------------------+\n", - "| client_1 Software Manager |\n", - "+---------------------+-------------+-----------------+--------------+------+----------+\n", - "| Name | Type | Operating State | Health State | Port | Protocol |\n", - "+---------------------+-------------+-----------------+--------------+------+----------+\n", - "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", - "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", - "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", - "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", - "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", - "| NMAP | Application | RUNNING | GOOD | None | none |\n", - "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", - "| UserManager | Service | RUNNING | GOOD | None | none |\n", - "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", - "| FTPClient | Service | RUNNING | GOOD | 21 | tcp |\n", - "| DataManipulationBot | Application | RUNNING | GOOD | None | none |\n", - "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", - "| C2Server | Application | RUNNING | GOOD | None | tcp |\n", - "+---------------------+-------------+-----------------+--------------+------+----------+\n" - ] - } - ], + "outputs": [], "source": [ "client_1: Computer = env.game.simulation.network.get_node_by_hostname(\"client_1\")\n", "client_1.software_manager.install(C2Server)\n", @@ -264,34 +215,9 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+-------------------------------------------------------------------------------------+\n", - "| web_server Software Manager |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| Name | Type | Operating State | Health State | Port | Protocol |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", - "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", - "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", - "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", - "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", - "| NMAP | Application | RUNNING | GOOD | None | none |\n", - "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", - "| UserManager | Service | RUNNING | GOOD | None | none |\n", - "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", - "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", - "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", - "| C2Beacon | Application | INSTALLING | UNUSED | None | tcp |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n" - ] - } - ], + "outputs": [], "source": [ "env.step(1)\n", "web_server: Computer = env.game.simulation.network.get_node_by_hostname(\"web_server\")\n", @@ -331,41 +257,9 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+-------------------------------------------------------------------------------------+\n", - "| web_server Software Manager |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| Name | Type | Operating State | Health State | Port | Protocol |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", - "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", - "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", - "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", - "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", - "| NMAP | Application | RUNNING | GOOD | None | none |\n", - "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", - "| UserManager | Service | RUNNING | GOOD | None | none |\n", - "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", - "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", - "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", - "| C2Beacon | Application | RUNNING | GOOD | None | tcp |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", - "| C2Beacon Running Status |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| False | 192.168.10.21 | 0 | 5 | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n" - ] - } - ], + "outputs": [], "source": [ "env.step(2)\n", "c2_beacon: C2Beacon = web_server.software_manager.software[\"C2Beacon\"]\n", @@ -404,54 +298,18 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(0,\n", - " 0.0,\n", - " False,\n", - " False,\n", - " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=2, action='node_application_execute', parameters={'node_name': 'web_server', 'application_name': 'C2Beacon'}, request=['network', 'node', 'web_server', 'application', 'C2Beacon', 'execute'], response=RequestResponse(status='success', data={}), reward=0.0, reward_info={})}})" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "env.step(3)" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", - "| C2Beacon Running Status |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| True | 192.168.10.21 | 1 | 5 | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "+-----------------------------------------------------------------------------------------------------+\n", - "| C2Server Running Status |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n", - "| True | 192.168.1.12 | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n" - ] - } - ], + "outputs": [], "source": [ "c2_beacon.show()\n", "c2_server.show()" @@ -514,59 +372,18 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(0,\n", - " 0.0,\n", - " False,\n", - " False,\n", - " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=3, action='c2_server_terminal_command', parameters={'node_name': 'client_1', 'ip_address': None, 'username': 'admin', 'password': 'admin', 'commands': [['software_manager', 'application', 'install', 'RansomwareScript']]}, request=['network', 'node', 'client_1', 'application', 'C2Server', 'terminal_command', {'commands': [['software_manager', 'application', 'install', 'RansomwareScript']], 'ip_address': None, 'username': 'admin', 'password': 'admin'}], response=RequestResponse(status='success', data={0: RequestResponse(status='success', data={})}), reward=0.0, reward_info={})}})" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "env.step(4)" ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+--------------------------------------------------------------------------------------+\n", - "| client_1 Software Manager |\n", - "+---------------------+-------------+-----------------+--------------+------+----------+\n", - "| Name | Type | Operating State | Health State | Port | Protocol |\n", - "+---------------------+-------------+-----------------+--------------+------+----------+\n", - "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", - "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", - "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", - "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", - "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", - "| NMAP | Application | RUNNING | GOOD | None | none |\n", - "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", - "| UserManager | Service | RUNNING | GOOD | None | none |\n", - "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", - "| FTPClient | Service | RUNNING | GOOD | 21 | tcp |\n", - "| DataManipulationBot | Application | RUNNING | GOOD | None | none |\n", - "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", - "| C2Server | Application | RUNNING | GOOD | None | tcp |\n", - "+---------------------+-------------+-----------------+--------------+------+----------+\n" - ] - } - ], + "outputs": [], "source": [ "client_1.software_manager.show()" ] @@ -605,66 +422,18 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(0,\n", - " 0.0,\n", - " False,\n", - " False,\n", - " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=4, action='c2_server_ransomware_configure', parameters={'node_name': 'client_1', 'server_ip_address': '192.168.1.14', 'payload': 'ENCRYPT'}, request=['network', 'node', 'client_1', 'application', 'C2Server', 'ransomware_configure', {'server_ip_address': '192.168.1.14', 'server_password': None, 'payload': 'ENCRYPT'}], response=RequestResponse(status='success', data={}), reward=0.0, reward_info={})}})" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "env.step(5)" ] }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+-------------------------------------------------------------------------------------+\n", - "| web_server Software Manager |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| Name | Type | Operating State | Health State | Port | Protocol |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", - "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", - "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", - "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", - "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", - "| NMAP | Application | RUNNING | GOOD | None | none |\n", - "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", - "| UserManager | Service | RUNNING | GOOD | None | none |\n", - "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", - "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", - "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", - "| C2Beacon | Application | RUNNING | GOOD | None | tcp |\n", - "| RansomwareScript | Application | RUNNING | GOOD | None | none |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "+------------------------------------+\n", - "| RansomwareScript Running Status |\n", - "+--------------------------+---------+\n", - "| Target Server IP Address | Payload |\n", - "+--------------------------+---------+\n", - "| 192.168.1.14 | ENCRYPT |\n", - "+--------------------------+---------+\n" - ] - } - ], + "outputs": [], "source": [ "ransomware_script: RansomwareScript = web_server.software_manager.software[\"RansomwareScript\"]\n", "web_server.software_manager.show()\n", @@ -710,48 +479,18 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(0,\n", - " 0.0,\n", - " False,\n", - " False,\n", - " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=5, action='c2_server_data_exfiltrate', parameters={'node_name': 'client_1', 'target_file_name': 'database.db', 'target_folder_name': 'database', 'exfiltration_folder_name': 'spoils', 'target_ip_address': '192.168.1.14', 'username': 'admin', 'password': 'admin'}, request=['network', 'node', 'client_1', 'application', 'C2Server', 'exfiltrate', {'target_file_name': 'database.db', 'target_folder_name': 'database', 'exfiltration_folder_name': 'spoils', 'target_ip_address': '192.168.1.14', 'username': 'admin', 'password': 'admin'}], response=RequestResponse(status='success', data={}), reward=0.0, reward_info={})}})" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "env.step(6)" ] }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+--------------------------------------------------------------------------------+\n", - "| client_1 File System |\n", - "+--------------------+---------+---------------+-----------------------+---------+\n", - "| File Path | Size | Health status | Visible health status | Deleted |\n", - "+--------------------+---------+---------------+-----------------------+---------+\n", - "| root | 0 B | GOOD | NONE | False |\n", - "| spoils/database.db | 4.77 MB | GOOD | NONE | False |\n", - "+--------------------+---------+---------------+-----------------------+---------+\n" - ] - } - ], + "outputs": [], "source": [ "client_1: Computer = env.game.simulation.network.get_node_by_hostname(\"client_1\")\n", "client_1.software_manager.file_system.show(full=True)" @@ -759,25 +498,9 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+---------------------------------------------------------------------------------+\n", - "| web_server File System |\n", - "+---------------------+---------+---------------+-----------------------+---------+\n", - "| File Path | Size | Health status | Visible health status | Deleted |\n", - "+---------------------+---------+---------------+-----------------------+---------+\n", - "| primaite/index.html | 15.0 KB | GOOD | NONE | False |\n", - "| root | 0 B | GOOD | NONE | False |\n", - "| spoils/database.db | 4.77 MB | GOOD | NONE | False |\n", - "+---------------------+---------+---------------+-----------------------+---------+\n" - ] - } - ], + "outputs": [], "source": [ "web_server: Computer = env.game.simulation.network.get_node_by_hostname(\"web_server\")\n", "web_server.software_manager.file_system.show(full=True)" @@ -814,48 +537,18 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(0,\n", - " 0.0,\n", - " False,\n", - " False,\n", - " {'agent_actions': {'CustomC2Agent': AgentHistoryItem(timestep=6, action='c2_server_ransomware_launch', parameters={'node_name': 'client_1'}, request=['network', 'node', 'client_1', 'application', 'C2Server', 'ransomware_launch'], response=RequestResponse(status='success', data={}), reward=0.0, reward_info={})}})" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "env.step(7)" ] }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+----------------------------------------------------------------------------------+\n", - "| database_server File System |\n", - "+----------------------+---------+---------------+-----------------------+---------+\n", - "| File Path | Size | Health status | Visible health status | Deleted |\n", - "+----------------------+---------+---------------+-----------------------+---------+\n", - "| database/database.db | 4.77 MB | CORRUPT | NONE | False |\n", - "| root | 0 B | GOOD | NONE | False |\n", - "+----------------------+---------+---------------+-----------------------+---------+\n" - ] - } - ], + "outputs": [], "source": [ "database_server: Server = env.game.simulation.network.get_node_by_hostname(\"database_server\")\n", "database_server.software_manager.file_system.show(full=True)" @@ -874,7 +567,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1002,17 +695,9 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-04 15:37:36,689: PrimaiteGymEnv RNG seed = None\n" - ] - } - ], + "outputs": [], "source": [ "with open(data_manipulation_config_path()) as f:\n", " cfg = yaml.safe_load(f)\n", @@ -1026,7 +711,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1069,18 +754,9 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-04 15:37:37,079: Resetting environment, episode 0, avg. reward: 0.0\n", - "2025-02-04 15:37:37,082: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-04/15-37-31/agent_actions/episode_0.json\n" - ] - } - ], + "outputs": [], "source": [ "# Resetting the environment and capturing the default observation space.\n", "blue_env.reset()\n", @@ -1089,20 +765,9 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 23, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Setting up the C2 Suite via the simulation API.\n", "\n", @@ -1123,7 +788,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1133,26 +798,9 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 2\n", - "root['NODES']['HOST0']['APPLICATIONS'][1]['operating_status']: 0 -> 1\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", - "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", - "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" - ] - } - ], + "outputs": [], "source": [ "display_obs_diffs(default_obs, c2_configuration_obs, blue_env.game.step_counter)" ] @@ -1172,20 +820,9 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "RequestResponse(status='success', data={0: RequestResponse(status='success', data={})})" - ] - }, - "execution_count": 26, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Installing RansomwareScript via C2 Terminal Commands\n", "ransomware_install_command = {\"commands\":[[\"software_manager\", \"application\", \"install\", \"RansomwareScript\"]],\n", @@ -1196,20 +833,9 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "RequestResponse(status='success', data={})" - ] - }, - "execution_count": 27, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Configuring the RansomwareScript\n", "ransomware_config = {\"server_ip_address\": \"192.168.1.14\", \"payload\": \"ENCRYPT\"}\n", @@ -1218,7 +844,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1228,28 +854,9 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 7\n", - "root['NODES']['HOST0']['APPLICATIONS'][1]['operating_status']: 0 -> 1\n", - "root['NODES']['HOST0']['APPLICATIONS'][2]['operating_status']: 0 -> 3\n", - "root['NODES']['HOST0']['users']['local_login']: 0 -> 1\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", - "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", - "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" - ] - } - ], + "outputs": [], "source": [ "display_obs_diffs(default_obs, c2_ransomware_obs, env.game.step_counter)" ] @@ -1265,7 +872,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1281,27 +888,16 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "RequestResponse(status='success', data={})" - ] - }, - "execution_count": 31, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "c2_server.send_command(given_command=C2Command.DATA_EXFILTRATION, command_options=exfil_options)" ] }, { "cell_type": "code", - "execution_count": 32, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1310,22 +906,9 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 7\n", - "root['NODES']['HOST0']['APPLICATIONS'][2]['operating_status']: 3 -> 1\n", - "root['NODES']['HOST1']['users']['remote_sessions']: 0 -> 1\n" - ] - } - ], + "outputs": [], "source": [ "display_obs_diffs(c2_ransomware_obs, c2_exfil_obs, env.game.step_counter)" ] @@ -1341,20 +924,9 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "RequestResponse(status='success', data={})" - ] - }, - "execution_count": 34, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Configuring the RansomwareScript\n", "ransomware_config = {\"server_ip_address\": \"192.168.1.14\", \"payload\": \"ENCRYPT\"}\n", @@ -1363,20 +935,9 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "RequestResponse(status='success', data={})" - ] - }, - "execution_count": 35, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Waiting for the ransomware to finish installing and then launching the RansomwareScript.\n", "blue_env.step(0)\n", @@ -1385,7 +946,7 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1395,30 +956,9 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 6\n", - "root['NODES']['HOST0']['APPLICATIONS'][2]['operating_status']: 3 -> 1\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['NODES']['HOST1']['users']['remote_sessions']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" - ] - } - ], + "outputs": [], "source": [ "display_obs_diffs(c2_ransomware_obs, c2_final_obs, blue_env.game.step_counter)" ] @@ -1434,7 +974,7 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1466,200 +1006,16 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-04 15:37:38,317: Resetting environment, episode 1, avg. reward: 0.0\n", - "2025-02-04 15:37:38,321: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-04/15-37-31/agent_actions/episode_1.json\n" - ] - }, - { - "data": { - "text/plain": [ - "({'NODES': {'HOST0': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 1,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'HOST1': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 1,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'HOST2': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 0,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'HOST3': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 0,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'ROUTER0': {'ACL': {1: {'position': 0,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 2: {'position': 1,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 3: {'position': 2,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 4: {'position': 3,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 5: {'position': 4,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 6: {'position': 5,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 7: {'position': 6,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 8: {'position': 7,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 9: {'position': 8,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 10: {'position': 9,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0}},\n", - " 'PORTS': {1: {'operating_status': 1},\n", - " 2: {'operating_status': 1},\n", - " 3: {'operating_status': 2}},\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}}},\n", - " 'LINKS': {1: {'PROTOCOLS': {'ALL': 1}},\n", - " 2: {'PROTOCOLS': {'ALL': 1}},\n", - " 3: {'PROTOCOLS': {'ALL': 0}},\n", - " 4: {'PROTOCOLS': {'ALL': 1}},\n", - " 5: {'PROTOCOLS': {'ALL': 1}},\n", - " 6: {'PROTOCOLS': {'ALL': 1}},\n", - " 7: {'PROTOCOLS': {'ALL': 1}},\n", - " 8: {'PROTOCOLS': {'ALL': 1}},\n", - " 9: {'PROTOCOLS': {'ALL': 1}},\n", - " 10: {'PROTOCOLS': {'ALL': 0}}},\n", - " 'ICS': 0},\n", - " {})" - ] - }, - "execution_count": 39, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "blue_env.reset()" ] }, { "cell_type": "code", - "execution_count": 40, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1678,7 +1034,7 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1695,33 +1051,9 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+-------------------------------------------------------------------------------------+\n", - "| web_server Software Manager |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| Name | Type | Operating State | Health State | Port | Protocol |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", - "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", - "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", - "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", - "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", - "| NMAP | Application | RUNNING | GOOD | None | none |\n", - "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", - "| UserManager | Service | RUNNING | GOOD | None | none |\n", - "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", - "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", - "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n" - ] - } - ], + "outputs": [], "source": [ "blue_env.step(0)\n", "web_server.software_manager.show()" @@ -1729,26 +1061,9 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 3\n", - "root['NODES']['HOST0']['APPLICATIONS'][1]['operating_status']: 1 -> 0\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", - "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", - "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" - ] - } - ], + "outputs": [], "source": [ "display_obs_diffs(pre_blue_action_obs, post_blue_action_obs, blue_env.game.step_counter)" ] @@ -1762,20 +1077,9 @@ }, { "cell_type": "code", - "execution_count": 44, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "RequestResponse(status='failure', data={'Reason': 'Command sent to the C2 Beacon but no response was ever received.'})" - ] - }, - "execution_count": 44, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Attempting to install the C2 RansomwareScript\n", "ransomware_install_command = {\"commands\":[[\"software_manager\", \"application\", \"install\", \"RansomwareScript\"]],\n", @@ -1797,200 +1101,16 @@ }, { "cell_type": "code", - "execution_count": 45, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-04 15:37:38,778: Resetting environment, episode 2, avg. reward: 0.0\n", - "2025-02-04 15:37:38,781: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-04/15-37-31/agent_actions/episode_2.json\n" - ] - }, - { - "data": { - "text/plain": [ - "({'NODES': {'HOST0': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 1,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'HOST1': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 1,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'HOST2': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 0,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'HOST3': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 0,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'ROUTER0': {'ACL': {1: {'position': 0,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 2: {'position': 1,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 3: {'position': 2,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 4: {'position': 3,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 5: {'position': 4,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 6: {'position': 5,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 7: {'position': 6,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 8: {'position': 7,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 9: {'position': 8,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 10: {'position': 9,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0}},\n", - " 'PORTS': {1: {'operating_status': 1},\n", - " 2: {'operating_status': 1},\n", - " 3: {'operating_status': 2}},\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}}},\n", - " 'LINKS': {1: {'PROTOCOLS': {'ALL': 1}},\n", - " 2: {'PROTOCOLS': {'ALL': 1}},\n", - " 3: {'PROTOCOLS': {'ALL': 0}},\n", - " 4: {'PROTOCOLS': {'ALL': 1}},\n", - " 5: {'PROTOCOLS': {'ALL': 1}},\n", - " 6: {'PROTOCOLS': {'ALL': 1}},\n", - " 7: {'PROTOCOLS': {'ALL': 1}},\n", - " 8: {'PROTOCOLS': {'ALL': 1}},\n", - " 9: {'PROTOCOLS': {'ALL': 1}},\n", - " 10: {'PROTOCOLS': {'ALL': 0}}},\n", - " 'ICS': 0},\n", - " {})" - ] - }, - "execution_count": 45, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "blue_env.reset()" ] }, { "cell_type": "code", - "execution_count": 46, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -2009,7 +1129,7 @@ }, { "cell_type": "code", - "execution_count": 47, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -2026,17 +1146,9 @@ }, { "cell_type": "code", - "execution_count": 48, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "NodeOperatingState.SHUTTING_DOWN\n" - ] - } - ], + "outputs": [], "source": [ "web_server = blue_env.game.simulation.network.get_node_by_hostname(\"web_server\")\n", "print(web_server.operating_state)" @@ -2044,48 +1156,18 @@ }, { "cell_type": "code", - "execution_count": 49, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 2\n", - "root['NODES']['HOST0']['operating_status']: 1 -> 4\n", - "root['NODES']['HOST0']['APPLICATIONS'][1]['operating_status']: 1 -> 0\n", - "root['NODES']['HOST0']['NICS'][1]['nic_status']: 1 -> 0\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", - "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", - "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" - ] - } - ], + "outputs": [], "source": [ "display_obs_diffs(pre_blue_action_obs, post_blue_action_obs, blue_env.game.step_counter)" ] }, { "cell_type": "code", - "execution_count": 50, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "RequestResponse(status='failure', data={'Reason': 'Command sent to the C2 Beacon but no response was ever received.'})" - ] - }, - "execution_count": 50, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Attempting to install the C2 RansomwareScript\n", "ransomware_install_command = {\"commands\":[\"software_manager\", \"application\", \"install\", \"RansomwareScript\"],\n", @@ -2109,200 +1191,16 @@ }, { "cell_type": "code", - "execution_count": 51, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-04 15:37:39,257: Resetting environment, episode 3, avg. reward: 0.0\n", - "2025-02-04 15:37:39,260: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-04/15-37-31/agent_actions/episode_3.json\n" - ] - }, - { - "data": { - "text/plain": [ - "({'NODES': {'HOST0': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 1,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'HOST1': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 1,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'HOST2': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 0,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'HOST3': {'APPLICATIONS': {1: {'operating_status': 0,\n", - " 'health_status': 0,\n", - " 'num_executions': 0},\n", - " 2: {'operating_status': 0, 'health_status': 0, 'num_executions': 0}},\n", - " 'FOLDERS': {1: {'health_status': 0,\n", - " 'FILES': {1: {'health_status': 0, 'num_access': 0}}}},\n", - " 'NICS': {1: {'nic_status': 1,\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {80: {'inbound': 0, 'outbound': 0},\n", - " 53: {'inbound': 0, 'outbound': 0},\n", - " 21: {'inbound': 0, 'outbound': 0}}}}},\n", - " 'num_file_creations': 0,\n", - " 'num_file_deletions': 0,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0},\n", - " 'operating_status': 1},\n", - " 'ROUTER0': {'ACL': {1: {'position': 0,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 2: {'position': 1,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 3: {'position': 2,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 4: {'position': 3,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 5: {'position': 4,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 6: {'position': 5,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 7: {'position': 6,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 8: {'position': 7,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 9: {'position': 8,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0},\n", - " 10: {'position': 9,\n", - " 'permission': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_wildcard_id': 0,\n", - " 'source_port_id': 0,\n", - " 'dest_ip_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'protocol_id': 0}},\n", - " 'PORTS': {1: {'operating_status': 1},\n", - " 2: {'operating_status': 1},\n", - " 3: {'operating_status': 2}},\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}}},\n", - " 'LINKS': {1: {'PROTOCOLS': {'ALL': 1}},\n", - " 2: {'PROTOCOLS': {'ALL': 1}},\n", - " 3: {'PROTOCOLS': {'ALL': 0}},\n", - " 4: {'PROTOCOLS': {'ALL': 1}},\n", - " 5: {'PROTOCOLS': {'ALL': 1}},\n", - " 6: {'PROTOCOLS': {'ALL': 1}},\n", - " 7: {'PROTOCOLS': {'ALL': 1}},\n", - " 8: {'PROTOCOLS': {'ALL': 1}},\n", - " 9: {'PROTOCOLS': {'ALL': 1}},\n", - " 10: {'PROTOCOLS': {'ALL': 0}}},\n", - " 'ICS': 0},\n", - " {})" - ] - }, - "execution_count": 51, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "blue_env.reset()" ] }, { "cell_type": "code", - "execution_count": 52, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -2321,7 +1219,7 @@ }, { "cell_type": "code", - "execution_count": 53, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -2338,41 +1236,9 @@ }, { "cell_type": "code", - "execution_count": 54, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+---------------------------------------------------------------+\n", - "| router_1 Network Interfaces |\n", - "+------+-------------------+-----------------+-------+----------+\n", - "| Port | MAC Address | Address | Speed | Status |\n", - "+------+-------------------+-----------------+-------+----------+\n", - "| 1 | dd:bc:17:75:a2:c4 | 192.168.1.1/24 | 100.0 | Enabled |\n", - "| 2 | 00:f2:f5:65:51:75 | 192.168.10.1/24 | 100.0 | Enabled |\n", - "| 3 | de:7d:ac:0e:58:e1 | 127.0.0.1/8 | 100.0 | Disabled |\n", - "| 4 | 8a:b4:55:12:c7:9e | 127.0.0.1/8 | 100.0 | Disabled |\n", - "| 5 | a5:ef:c6:53:97:c4 | 127.0.0.1/8 | 100.0 | Disabled |\n", - "+------+-------------------+-----------------+-------+----------+\n", - "+------------------------------------------------------------------------------------------------------------------------+\n", - "| router_1 Access Control List |\n", - "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n", - "| Index | Action | Protocol | Src IP | Src Wildcard | Src Port | Dst IP | Dst Wildcard | Dst Port | Matched |\n", - "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n", - "| 1 | DENY | ANY | 192.168.10.21 | 0.0.0.1 | 80 | 192.168.1.12 | 0.0.0.1 | 80 | 0 |\n", - "| 18 | PERMIT | ANY | ANY | ANY | 5432 | ANY | ANY | 5432 | 0 |\n", - "| 19 | PERMIT | ANY | ANY | ANY | 53 | ANY | ANY | 53 | 0 |\n", - "| 20 | PERMIT | ANY | ANY | ANY | 21 | ANY | ANY | 21 | 0 |\n", - "| 21 | PERMIT | ANY | ANY | ANY | 80 | ANY | ANY | 80 | 4 |\n", - "| 22 | PERMIT | ANY | ANY | ANY | 219 | ANY | ANY | 219 | 10 |\n", - "| 23 | PERMIT | icmp | ANY | ANY | ANY | ANY | ANY | ANY | 0 |\n", - "| 24 | DENY | ANY | ANY | ANY | ANY | ANY | ANY | ANY | 0 |\n", - "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n" - ] - } - ], + "outputs": [], "source": [ "router_1: Router = blue_env.game.simulation.network.get_node_by_hostname(\"router_1\")\n", "router_1.acl.show()" @@ -2387,20 +1253,9 @@ }, { "cell_type": "code", - "execution_count": 55, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "RequestResponse(status='failure', data={'Reason': 'Command sent to the C2 Beacon but no response was ever received.'})" - ] - }, - "execution_count": 55, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "blue_env.step(0)\n", "\n", @@ -2411,30 +1266,9 @@ }, { "cell_type": "code", - "execution_count": 56, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+------------------------------------------------------------------------------------------------------------------------+\n", - "| router_1 Access Control List |\n", - "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n", - "| Index | Action | Protocol | Src IP | Src Wildcard | Src Port | Dst IP | Dst Wildcard | Dst Port | Matched |\n", - "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n", - "| 1 | DENY | ANY | 192.168.10.21 | 0.0.0.1 | 80 | 192.168.1.12 | 0.0.0.1 | 80 | 2 |\n", - "| 18 | PERMIT | ANY | ANY | ANY | 5432 | ANY | ANY | 5432 | 0 |\n", - "| 19 | PERMIT | ANY | ANY | ANY | 53 | ANY | ANY | 53 | 0 |\n", - "| 20 | PERMIT | ANY | ANY | ANY | 21 | ANY | ANY | 21 | 0 |\n", - "| 21 | PERMIT | ANY | ANY | ANY | 80 | ANY | ANY | 80 | 4 |\n", - "| 22 | PERMIT | ANY | ANY | ANY | 219 | ANY | ANY | 219 | 10 |\n", - "| 23 | PERMIT | icmp | ANY | ANY | ANY | ANY | ANY | ANY | 0 |\n", - "| 24 | DENY | ANY | ANY | ANY | ANY | ANY | ANY | ANY | 0 |\n", - "+-------+--------+----------+---------------+--------------+----------+--------------+--------------+----------+---------+\n" - ] - } - ], + "outputs": [], "source": [ "router_1.acl.show()" ] @@ -2448,58 +1282,18 @@ }, { "cell_type": "code", - "execution_count": 57, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+-------------------------------------------------------------------------------------+\n", - "| web_server Software Manager |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| Name | Type | Operating State | Health State | Port | Protocol |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n", - "| ARP | Service | RUNNING | GOOD | 219 | udp |\n", - "| ICMP | Service | RUNNING | GOOD | None | icmp |\n", - "| DNSClient | Service | RUNNING | GOOD | 53 | tcp |\n", - "| NTPClient | Service | RUNNING | GOOD | 123 | udp |\n", - "| WebBrowser | Application | RUNNING | GOOD | 80 | tcp |\n", - "| NMAP | Application | RUNNING | GOOD | None | none |\n", - "| UserSessionManager | Service | RUNNING | GOOD | None | none |\n", - "| UserManager | Service | RUNNING | GOOD | None | none |\n", - "| Terminal | Service | RUNNING | GOOD | 22 | tcp |\n", - "| WebServer | Service | RUNNING | GOOD | 80 | tcp |\n", - "| DatabaseClient | Application | RUNNING | GOOD | 5432 | tcp |\n", - "| C2Beacon | Application | RUNNING | GOOD | None | tcp |\n", - "+--------------------+-------------+-----------------+--------------+------+----------+\n" - ] - } - ], + "outputs": [], "source": [ "web_server.software_manager.show()" ] }, { "cell_type": "code", - "execution_count": 58, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+----------------------------------------------------------------------------------+\n", - "| database_server File System |\n", - "+----------------------+---------+---------------+-----------------------+---------+\n", - "| File Path | Size | Health status | Visible health status | Deleted |\n", - "+----------------------+---------+---------------+-----------------------+---------+\n", - "| database/database.db | 4.77 MB | GOOD | NONE | False |\n", - "| root | 0 B | GOOD | NONE | False |\n", - "+----------------------+---------+---------------+-----------------------+---------+\n" - ] - } - ], + "outputs": [], "source": [ "database_server: Server = blue_env.game.simulation.network.get_node_by_hostname(\"database_server\")\n", "database_server.software_manager.file_system.show(full=True)" @@ -2507,33 +1301,9 @@ }, { "cell_type": "code", - "execution_count": 59, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 3\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['inbound']: 1 -> 0\n", - "root['NODES']['HOST1']['NICS'][1]['TRAFFIC']['tcp'][21]['outbound']: 4 -> 0\n", - "root['NODES']['ROUTER0']['ACL'][1]['permission']: 0 -> 2\n", - "root['NODES']['ROUTER0']['ACL'][1]['source_ip_id']: 0 -> 7\n", - "root['NODES']['ROUTER0']['ACL'][1]['source_wildcard_id']: 0 -> 2\n", - "root['NODES']['ROUTER0']['ACL'][1]['source_port_id']: 0 -> 2\n", - "root['NODES']['ROUTER0']['ACL'][1]['dest_ip_id']: 0 -> 3\n", - "root['NODES']['ROUTER0']['ACL'][1]['dest_wildcard_id']: 0 -> 2\n", - "root['NODES']['ROUTER0']['ACL'][1]['dest_port_id']: 0 -> 2\n", - "root['NODES']['ROUTER0']['ACL'][1]['protocol_id']: 0 -> 1\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 1 -> 0\n", - "root['LINKS'][5]['PROTOCOLS']['ALL']: 4 -> 0\n", - "root['LINKS'][6]['PROTOCOLS']['ALL']: 4 -> 0\n" - ] - } - ], + "outputs": [], "source": [ "display_obs_diffs(pre_blue_action_obs, post_blue_action_obs, blue_env.game.step_counter)" ] @@ -2597,17 +1367,9 @@ }, { "cell_type": "code", - "execution_count": 60, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-04 15:37:40,175: PrimaiteGymEnv RNG seed = None\n" - ] - } - ], + "outputs": [], "source": [ "with open(data_manipulation_config_path()) as f:\n", " cfg = yaml.safe_load(f)\n", @@ -2628,7 +1390,7 @@ }, { "cell_type": "code", - "execution_count": 61, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -2656,30 +1418,9 @@ }, { "cell_type": "code", - "execution_count": 62, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", - "| C2Beacon Running Status |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| False | None | 0 | 5 | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "+-----------------------------------------------------------------------------------------------------+\n", - "| C2Server Running Status |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n", - "| False | None | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n" - ] - } - ], + "outputs": [], "source": [ "env.step(2) # Agent Action Equivalent to c2_beacon.configure(c2_server_ip_address=\"192.168.10.21\")\n", "env.step(3) # Agent action Equivalent to c2_beacon.establish()\n", @@ -2696,30 +1437,9 @@ }, { "cell_type": "code", - "execution_count": 63, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", - "| C2Beacon Running Status |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| False | None | 0 | 5 | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "+-----------------------------------------------------------------------------------------------------+\n", - "| C2Server Running Status |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n", - "| False | None | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n" - ] - } - ], + "outputs": [], "source": [ "env.step(9) # Equivalent of to c2_beacon.configure(c2_server_ip_address=\"192.168.10.22\")\n", "env.step(3)\n", @@ -2737,23 +1457,9 @@ }, { "cell_type": "code", - "execution_count": 64, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+-----------------------------------------------------------------------------------------------------+\n", - "| C2Server Running Status |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n", - "| False | None | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------------+-------------------------+\n" - ] - } - ], + "outputs": [], "source": [ "for i in range(6):\n", " env.step(0)\n", @@ -2776,17 +1482,9 @@ }, { "cell_type": "code", - "execution_count": 65, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-04 15:37:40,588: PrimaiteGymEnv RNG seed = None\n" - ] - } - ], + "outputs": [], "source": [ "with open(data_manipulation_config_path()) as f:\n", " cfg = yaml.safe_load(f)\n", @@ -2802,7 +1500,7 @@ }, { "cell_type": "code", - "execution_count": 66, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -2826,23 +1524,9 @@ }, { "cell_type": "code", - "execution_count": 67, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", - "| C2Beacon Running Status |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| True | 192.168.10.21 | 0 | 5 | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n" - ] - } - ], + "outputs": [], "source": [ "c2_beacon.configure(c2_server_ip_address=\"192.168.10.21\")\n", "c2_beacon.establish()\n", @@ -2860,72 +1544,9 @@ }, { "cell_type": "code", - "execution_count": 68, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 4\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 5\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 6\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 7\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 8\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 9\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 10\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 11\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 12\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 13\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" - ] - } - ], + "outputs": [], "source": [ "for i in range(10):\n", " keep_alive_obs, _, _, _, _ = blue_config_env.step(0)\n", @@ -2941,23 +1562,9 @@ }, { "cell_type": "code", - "execution_count": 69, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", - "| C2Beacon Running Status |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| True | 192.168.10.21 | 0 | 1 | tcp | 80 |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n" - ] - } - ], + "outputs": [], "source": [ "c2_beacon.configure(c2_server_ip_address=\"192.168.10.21\", keep_alive_frequency=1)\n", "c2_beacon.establish()\n", @@ -2973,40 +1580,9 @@ }, { "cell_type": "code", - "execution_count": 70, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 14\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 15\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" - ] - } - ], + "outputs": [], "source": [ "# Comparing the OBS of the default frequency to a timestep frequency of 1\n", "for i in range(2):\n", @@ -3025,52 +1601,9 @@ }, { "cell_type": "code", - "execution_count": 71, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 16\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 17\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 18\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 19\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 20\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 21\n", - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 22\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" - ] - } - ], + "outputs": [], "source": [ "c2_beacon.configure(c2_server_ip_address=\"192.168.10.21\", keep_alive_frequency=7)\n", "\n", @@ -3107,18 +1640,9 @@ }, { "cell_type": "code", - "execution_count": 72, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-04 15:37:41,322: Resetting environment, episode 0, avg. reward: 0.0\n", - "2025-02-04 15:37:41,325: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-04/15-37-31/agent_actions/episode_0.json\n" - ] - } - ], + "outputs": [], "source": [ "blue_config_env.reset()\n", "\n", @@ -3140,28 +1664,9 @@ }, { "cell_type": "code", - "execution_count": 73, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 5\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 0 -> 1\n", - "root['LINKS'][1]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][2]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][4]['PROTOCOLS']['ALL']: 0 -> 1\n", - "root['LINKS'][8]['PROTOCOLS']['ALL']: 0 -> 1\n" - ] - } - ], + "outputs": [], "source": [ "# Capturing default C2 Traffic\n", "for i in range(3):\n", @@ -3179,23 +1684,9 @@ }, { "cell_type": "code", - "execution_count": 74, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+----------------------------------------------------------------------------------------------------------------------------------------------------+\n", - "| C2Beacon Running Status |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| C2 Connection Active | C2 Remote Connection | Keep Alive Inactivity | Keep Alive Frequency | Current Masquerade Protocol | Current Masquerade Port |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n", - "| True | 192.168.10.21 | 0 | 5 | udp | 53 |\n", - "+----------------------+----------------------+-----------------------+----------------------+-----------------------------+-------------------------+\n" - ] - } - ], + "outputs": [], "source": [ "from primaite.utils.validation.ip_protocol import PROTOCOL_LOOKUP\n", "from primaite.utils.validation.port import PORT_LOOKUP\n", @@ -3208,28 +1699,9 @@ }, { "cell_type": "code", - "execution_count": 75, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Observation space differences\n", - "-----------------------------\n", - "Step 10\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 1 -> 0\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 1 -> 0\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['udp'][53]['inbound']: 0 -> 1\n", - "root['NODES']['HOST0']['NICS'][1]['TRAFFIC']['udp'][53]['outbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['inbound']: 1 -> 0\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['tcp'][80]['outbound']: 1 -> 0\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['udp'][53]['inbound']: 0 -> 1\n", - "root['NODES']['HOST2']['NICS'][1]['TRAFFIC']['udp'][53]['outbound']: 0 -> 1\n" - ] - } - ], + "outputs": [], "source": [ "# Capturing UDP C2 Traffic\n", "for i in range(5):\n", diff --git a/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb b/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb index 4a34619f..41b75e43 100644 --- a/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb +++ b/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb @@ -382,7 +382,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": { "tags": [] }, @@ -394,7 +394,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": { "tags": [] }, @@ -450,7 +450,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ diff --git a/src/primaite/notebooks/Getting-Information-Out-Of-PrimAITE.ipynb b/src/primaite/notebooks/Getting-Information-Out-Of-PrimAITE.ipynb index d7d60d76..c7fdafad 100644 --- a/src/primaite/notebooks/Getting-Information-Out-Of-PrimAITE.ipynb +++ b/src/primaite/notebooks/Getting-Information-Out-Of-PrimAITE.ipynb @@ -6,7 +6,7 @@ "source": [ "# Getting information out of PrimAITE\n", "\n", - "© Crown-owned copyright 2024, Defence Science and Technology Laboratory UK\n" + "© Crown-owned copyright 2025, Defence Science and Technology Laboratory UK\n" ] }, { @@ -32,8 +32,6 @@ "from primaite.session.environment import PrimaiteGymEnv\n", "from primaite.simulator.network.hardware.nodes.host.computer import Computer\n", "from notebook.services.config import ConfigManager\n", - "from primaite.game.agent.scripted_agents import probabilistic_agent\n", - "\n", "\n", "cm = ConfigManager().update('notebook', {'limit_output': 50}) # limit output lines to 50 - for neatness\n", "\n", diff --git a/src/primaite/notebooks/Privilege-Escalation-and-Data-Loss-Example.ipynb b/src/primaite/notebooks/Privilege-Escalation-and-Data-Loss-Example.ipynb index 35d3813a..cbb898ea 100644 --- a/src/primaite/notebooks/Privilege-Escalation-and-Data-Loss-Example.ipynb +++ b/src/primaite/notebooks/Privilege-Escalation-and-Data-Loss-Example.ipynb @@ -6,7 +6,7 @@ "source": [ "# Simulating Privilege Escalation and Data Loss Using SSH and ACLs Manipulation\n", "\n", - "© Crown-owned copyright 2024, Defence Science and Technology Laboratory UK\n", + "© Crown-owned copyright 2025, Defence Science and Technology Laboratory UK\n", "\n", "## Overview\n", "\n", @@ -62,7 +62,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": { "tags": [] }, @@ -77,7 +77,8 @@ "from primaite.simulator.network.hardware.nodes.host.server import Server\n", "from primaite.simulator.system.applications.database_client import DatabaseClient\n", "from primaite.simulator.system.applications.web_browser import WebBrowser\n", - "from primaite.simulator.system.services.database.database_service import DatabaseService\n" + "from primaite.simulator.system.services.database.database_service import DatabaseService\n", + "from primaite.simulator.network.hardware.nodes.network import firewall\n" ] }, { @@ -89,7 +90,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": { "tags": [] }, @@ -112,7 +113,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": { "tags": [] }, diff --git a/src/primaite/notebooks/Requests-and-Responses.ipynb b/src/primaite/notebooks/Requests-and-Responses.ipynb index 83aed07c..01a6cffa 100644 --- a/src/primaite/notebooks/Requests-and-Responses.ipynb +++ b/src/primaite/notebooks/Requests-and-Responses.ipynb @@ -6,7 +6,7 @@ "source": [ "# Requests and Responses\n", "\n", - "© Crown-owned copyright 2024, Defence Science and Technology Laboratory UK\n", + "© Crown-owned copyright 2025, Defence Science and Technology Laboratory UK\n", "\n", "Agents interact with the PrimAITE simulation via the Request system.\n" ] @@ -36,7 +36,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -47,17 +47,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "sim = Simulation()\n", "sim.network.add_node(\n", - " HostNode(\n", - " hostname=\"client\",\n", - " ip_address='10.0.0.1',\n", - " subnet_mask='255.255.255.0',\n", - " operating_state=NodeOperatingState.ON)\n", + " HostNode.from_config(\n", + " config = {\n", + " 'type': \"hostnode\",\n", + " 'hostname': \"client\",\n", + " 'ip_address': '10.0.0.1',\n", + " 'subnet_mask': '255.255.255.0',\n", + " 'operating_state': NodeOperatingState.ON,\n", + " }\n", + " )\n", ")\n", "client = sim.network.get_node_by_hostname('client')\n" ] @@ -210,7 +214,7 @@ ], "metadata": { "kernelspec": { - "display_name": "venv", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, diff --git a/src/primaite/notebooks/Terminal-Processing.ipynb b/src/primaite/notebooks/Terminal-Processing.ipynb index 9aa4e96a..48318c13 100644 --- a/src/primaite/notebooks/Terminal-Processing.ipynb +++ b/src/primaite/notebooks/Terminal-Processing.ipynb @@ -29,7 +29,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -39,12 +39,43 @@ "from primaite.simulator.system.applications.red_applications.ransomware_script import RansomwareScript\n", "from primaite.simulator.system.services.terminal.terminal import RemoteTerminalConnection\n", "\n", + "# print(dir(Computer))\n", + "# node_a = Computer.from_config(\n", + "# config = {\n", + "# \"type\": \"computer\",\n", + "# \"hostname\": \"node_a\",\n", + "# \"ip_address\": \"192.168.0.10\",\n", + "# \"subnet_mask\": \"255.255.255.0\",\n", + "# \"startup_duration\": 0,\n", + "# }\n", + "# )\n", + "# print(f\"{node_a=}\")\n", + "\n", "def basic_network() -> Network:\n", " \"\"\"Utility function for creating a default network to demonstrate Terminal functionality\"\"\"\n", " network = Network()\n", - " node_a = Computer(hostname=\"node_a\", ip_address=\"192.168.0.10\", subnet_mask=\"255.255.255.0\", start_up_duration=0)\n", + " # node_a = Computer(hostname=\"node_a\", ip_address=\"192.168.0.10\", subnet_mask=\"255.255.255.0\", start_up_duration=0)\n", + " node_a = Computer.from_config(\n", + " config = {\n", + " \"type\": \"computer\",\n", + " \"hostname\": \"node_a\",\n", + " \"ip_address\": \"192.168.0.10\",\n", + " \"subnet_mask\": \"255.255.255.0\",\n", + " \"startup_duration\": 0,\n", + " }\n", + " )\n", + " print(f\"{node_a=}\")\n", " node_a.power_on()\n", - " node_b = Computer(hostname=\"node_b\", ip_address=\"192.168.0.11\", subnet_mask=\"255.255.255.0\", start_up_duration=0)\n", + " # node_b = Computer(hostname=\"node_b\", ip_address=\"192.168.0.11\", subnet_mask=\"255.255.255.0\", start_up_duration=0)\n", + " node_b = Computer.from_config(\n", + " config = {\n", + " \"type\": \"computer\",\n", + " \"hostname\": \"node_b\",\n", + " \"ip_address\": \"192.168.0.11\",\n", + " \"subnet_mask\": \"255.255.255.0\",\n", + " \"startup_duration\": 0,\n", + " }\n", + " )\n", " node_b.power_on()\n", " network.connect(node_a.network_interface[1], node_b.network_interface[1])\n", " return network" @@ -84,7 +115,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -211,7 +242,7 @@ ], "metadata": { "kernelspec": { - "display_name": ".venv", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -225,7 +256,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.11" + "version": "3.10.12" } }, "nbformat": 4, diff --git a/src/primaite/notebooks/Training-an-RLLIB-MARL-System.ipynb b/src/primaite/notebooks/Training-an-RLLIB-MARL-System.ipynb index dadb399e..87d9c377 100644 --- a/src/primaite/notebooks/Training-an-RLLIB-MARL-System.ipynb +++ b/src/primaite/notebooks/Training-an-RLLIB-MARL-System.ipynb @@ -6,7 +6,7 @@ "source": [ "# Train a Multi agent system using RLLIB\n", "\n", - "© Crown-owned copyright 2024, Defence Science and Technology Laboratory UK\n", + "© Crown-owned copyright 2025, Defence Science and Technology Laboratory UK\n", "\n", "This notebook will demonstrate how to use the `PrimaiteRayMARLEnv` to train a very basic system with two PPO agents." ] @@ -57,7 +57,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -103,20 +103,7 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "*** SIGTERM received at time=1737996337 on cpu 7 ***\n", - "PC: @ 0x7f3649b0fe2e (unknown) epoll_wait\n", - " @ 0x7f3649a2c520 (unknown) (unknown)\n", - "[2025-01-27 16:45:37,381 E 117142 117142] logging.cc:440: *** SIGTERM received at time=1737996337 on cpu 7 ***\n", - "[2025-01-27 16:45:37,381 E 117142 117142] logging.cc:440: PC: @ 0x7f3649b0fe2e (unknown) epoll_wait\n", - "[2025-01-27 16:45:37,381 E 117142 117142] logging.cc:440: @ 0x7f3649a2c520 (unknown) (unknown)\n" - ] - } - ], + "outputs": [], "source": [ "eval = algo.evaluate()" ] @@ -127,18 +114,6 @@ "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.12" } }, "nbformat": 4, diff --git a/src/primaite/notebooks/Training-an-RLLib-Agent.ipynb b/src/primaite/notebooks/Training-an-RLLib-Agent.ipynb index 64a9e7ab..79740bca 100644 --- a/src/primaite/notebooks/Training-an-RLLib-Agent.ipynb +++ b/src/primaite/notebooks/Training-an-RLLib-Agent.ipynb @@ -6,7 +6,7 @@ "source": [ "# Train a Single agent system using RLLib\n", "\n", - "© Crown-owned copyright 2024, Defence Science and Technology Laboratory UK\n", + "© Crown-owned copyright 2025, Defence Science and Technology Laboratory UK\n", "\n", "This notebook will demonstrate how to use PrimaiteRayEnv to train a basic PPO agent." ] @@ -52,7 +52,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -98,20 +98,7 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "*** SIGTERM received at time=1737996055 on cpu 1 ***\n", - "PC: @ 0x7f6e254a6e2e (unknown) epoll_wait\n", - " @ 0x7f6e253c3520 (unknown) (unknown)\n", - "[2025-01-27 16:40:55,343 E 114171 114171] logging.cc:440: *** SIGTERM received at time=1737996055 on cpu 1 ***\n", - "[2025-01-27 16:40:55,343 E 114171 114171] logging.cc:440: PC: @ 0x7f6e254a6e2e (unknown) epoll_wait\n", - "[2025-01-27 16:40:55,344 E 114171 114171] logging.cc:440: @ 0x7f6e253c3520 (unknown) (unknown)\n" - ] - } - ], + "outputs": [], "source": [ "eval = algo.evaluate()" ] @@ -122,18 +109,6 @@ "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.12" } }, "nbformat": 4, diff --git a/src/primaite/notebooks/Training-an-SB3-Agent.ipynb b/src/primaite/notebooks/Training-an-SB3-Agent.ipynb index d3492f92..10328989 100644 --- a/src/primaite/notebooks/Training-an-SB3-Agent.ipynb +++ b/src/primaite/notebooks/Training-an-SB3-Agent.ipynb @@ -6,7 +6,7 @@ "source": [ "# Training an SB3 Agent\n", "\n", - "© Crown-owned copyright 2024, Defence Science and Technology Laboratory UK\n", + "© Crown-owned copyright 2025, Defence Science and Technology Laboratory UK\n", "\n", "This notebook will demonstrate how to use primaite to create and train a PPO agent, using a pre-defined configuration file." ] diff --git a/src/primaite/notebooks/Using-Episode-Schedules.ipynb b/src/primaite/notebooks/Using-Episode-Schedules.ipynb index 311fe4fb..44305266 100644 --- a/src/primaite/notebooks/Using-Episode-Schedules.ipynb +++ b/src/primaite/notebooks/Using-Episode-Schedules.ipynb @@ -6,7 +6,7 @@ "source": [ "# Using Episode Schedules\n", "\n", - "© Crown-owned copyright 2024, Defence Science and Technology Laboratory UK\n", + "© Crown-owned copyright 2025, Defence Science and Technology Laboratory UK\n", "\n", "PrimAITE supports the ability to use different variations on a scenario at different episodes. This can be used to increase \n", "domain randomisation to prevent overfitting, or to set up curriculum learning to train agents to perform more complicated tasks.\n", @@ -40,7 +40,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -325,7 +325,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "metadata": {}, "outputs": [], "source": [ @@ -413,6 +413,18 @@ "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" } }, "nbformat": 4, diff --git a/src/primaite/notebooks/multi-processing.ipynb b/src/primaite/notebooks/multi-processing.ipynb index 798bf3ff..e56bf362 100644 --- a/src/primaite/notebooks/multi-processing.ipynb +++ b/src/primaite/notebooks/multi-processing.ipynb @@ -6,7 +6,7 @@ "source": [ "# Simple multi-processing demonstration\n", "\n", - "© Crown-owned copyright 2024, Defence Science and Technology Laboratory UK\n", + "© Crown-owned copyright 2025, Defence Science and Technology Laboratory UK\n", "\n", "This notebook uses SubprocVecEnv from SB3." ] @@ -29,7 +29,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -37,13 +37,12 @@ "from stable_baselines3 import PPO\n", "from stable_baselines3.common.utils import set_random_seed\n", "from stable_baselines3.common.vec_env import SubprocVecEnv\n", - "from primaite.session.environment import PrimaiteGymEnv\n", - "from primaite.game.agent.scripted_agents import probabilistic_agent\n" + "from primaite.session.environment import PrimaiteGymEnv\n" ] }, { "cell_type": "code", - "execution_count": 31, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -52,7 +51,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -69,7 +68,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -90,7 +89,7 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ diff --git a/src/primaite/simulator/network/hardware/base.py b/src/primaite/simulator/network/hardware/base.py index 6543d793..cc3d4150 100644 --- a/src/primaite/simulator/network/hardware/base.py +++ b/src/primaite/simulator/network/hardware/base.py @@ -1197,7 +1197,7 @@ class UserSessionManager(Service, identifier="UserSessionManager"): """Request should take the form [username, password, remote_ip_address].""" username, password, remote_ip_address = request response = RequestResponse.from_bool(self.remote_login(username, password, remote_ip_address)) - response.data = {"remote_hostname": self.parent.hostname, "username": username} + response.data = {"remote_hostname": self.parent.config.hostname, "username": username} return response rm.add_request("remote_login", RequestType(func=_remote_login)) @@ -1230,7 +1230,7 @@ class UserSessionManager(Service, identifier="UserSessionManager"): if markdown: table.set_style(MARKDOWN) table.align = "l" - table.title = f"{self.parent.hostname} User Sessions" + table.title = f"{self.parent.config.hostname} User Sessions" def _add_session_to_table(user_session: UserSession): """ @@ -1627,9 +1627,7 @@ class Node(SimComponent, ABC): dns_server=kwargs["config"].dns_server, ) super().__init__(**kwargs) - self.operating_state = ( - NodeOperatingState.ON if not (p := kwargs["config"].operating_state) else NodeOperatingState[p.upper()] - ) + self.operating_state = NodeOperatingState.ON if not (p := kwargs["config"].operating_state) else p self._install_system_software() self.session_manager.node = self self.session_manager.software_manager = self.software_manager From 7b2a9c1d2a6d64d870de17fa0a3c8afc15641919 Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Wed, 5 Feb 2025 16:46:25 +0000 Subject: [PATCH 13/24] #3075: Pre-commit changes. --- src/primaite/notebooks/Requests-and-Responses.ipynb | 4 ++-- src/primaite/notebooks/Terminal-Processing.ipynb | 4 ++-- src/primaite/notebooks/Training-an-SB3-Agent.ipynb | 10 +++++----- src/primaite/notebooks/Using-Episode-Schedules.ipynb | 4 ++-- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/primaite/notebooks/Requests-and-Responses.ipynb b/src/primaite/notebooks/Requests-and-Responses.ipynb index 01a6cffa..48f1cf3e 100644 --- a/src/primaite/notebooks/Requests-and-Responses.ipynb +++ b/src/primaite/notebooks/Requests-and-Responses.ipynb @@ -36,7 +36,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -47,7 +47,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ diff --git a/src/primaite/notebooks/Terminal-Processing.ipynb b/src/primaite/notebooks/Terminal-Processing.ipynb index 48318c13..dc870db5 100644 --- a/src/primaite/notebooks/Terminal-Processing.ipynb +++ b/src/primaite/notebooks/Terminal-Processing.ipynb @@ -29,7 +29,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -115,7 +115,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ diff --git a/src/primaite/notebooks/Training-an-SB3-Agent.ipynb b/src/primaite/notebooks/Training-an-SB3-Agent.ipynb index 10328989..1bec0183 100644 --- a/src/primaite/notebooks/Training-an-SB3-Agent.ipynb +++ b/src/primaite/notebooks/Training-an-SB3-Agent.ipynb @@ -29,7 +29,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -41,7 +41,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -50,7 +50,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -101,7 +101,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -133,7 +133,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ diff --git a/src/primaite/notebooks/Using-Episode-Schedules.ipynb b/src/primaite/notebooks/Using-Episode-Schedules.ipynb index 44305266..4bc49a70 100644 --- a/src/primaite/notebooks/Using-Episode-Schedules.ipynb +++ b/src/primaite/notebooks/Using-Episode-Schedules.ipynb @@ -40,7 +40,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -325,7 +325,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ From 1a8c3b9471e591659eb69853a4865b1a23708ad8 Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Thu, 6 Feb 2025 16:42:26 +0000 Subject: [PATCH 14/24] #3075: Fix notebooks after test changes. --- ...ommand-and-Control-E2E-Demonstration.ipynb | 82 +++++++++---------- ...a-Manipulation-Customising-Red-Agent.ipynb | 16 ++-- .../Data-Manipulation-E2E-Demonstration.ipynb | 4 +- ...ege-Escalation-and-Data-Loss-Example.ipynb | 30 +++---- .../notebooks/Requests-and-Responses.ipynb | 13 +-- .../notebooks/Terminal-Processing.ipynb | 26 ++---- .../simulator/network/hardware/base.py | 4 +- .../network/hardware/nodes/host/host_node.py | 2 +- .../configs/basic_switched_network.yaml | 6 +- 9 files changed, 86 insertions(+), 97 deletions(-) diff --git a/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb b/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb index 17b7b79e..474dadff 100644 --- a/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb +++ b/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb @@ -59,30 +59,30 @@ "custom_c2_agent = \"\"\"\n", " - ref: CustomC2Agent\n", " team: RED\n", - " type: ProxyAgent\n", + " type: proxy-agent\n", "\n", " action_space:\n", " action_map:\n", " 0:\n", - " action: do_nothing\n", + " action: do-nothing\n", " options: {}\n", " 1:\n", - " action: node_application_install\n", + " action: node-application-install\n", " options:\n", " node_name: web_server\n", - " application_name: C2Beacon\n", + " application_name: c2-beacon\n", " 2:\n", - " action: configure_c2_beacon\n", + " action: configure-c2-beacon\n", " options:\n", " node_name: web_server\n", " c2_server_ip_address: 192.168.10.21\n", " 3:\n", - " action: node_application_execute\n", + " action: node-application-execute\n", " options:\n", " node_name: web_server\n", - " application_name: C2Beacon\n", + " application_name: c2-beacon\n", " 4:\n", - " action: c2_server_terminal_command\n", + " action: c2-server-terminal-command\n", " options:\n", " node_name: client_1\n", " ip_address:\n", @@ -93,7 +93,7 @@ " - software_manager\n", " - application\n", " - install\n", - " - RansomwareScript\n", + " - ransomware-script\n", " 5:\n", " action: c2-server-ransomware-configure\n", " options:\n", @@ -101,7 +101,7 @@ " server_ip_address: 192.168.1.14\n", " payload: ENCRYPT\n", " 6:\n", - " action: c2_server_data_exfiltrate\n", + " action: c2-server-data-exfiltrate\n", " options:\n", " node_name: client_1\n", " target_file_name: \"database.db\"\n", @@ -112,23 +112,23 @@ " password: admin\n", "\n", " 7:\n", - " action: c2_server_ransomware_launch\n", + " action: c2-server-ransomware-launch\n", " options:\n", " node_name: client_1\n", " 8:\n", - " action: configure_c2_beacon\n", + " action: configure-c2-beacon\n", " options:\n", " node_name: web_server\n", " c2_server_ip_address: 192.168.10.21\n", " 9:\n", - " action: configure_c2_beacon\n", + " action: configure-c2-beacon\n", " options:\n", " node_name: web_server\n", " c2_server_ip_address: 192.168.10.22\n", "\n", " reward_function:\n", " reward_components:\n", - " - type: DUMMY\n", + " - type: dummy\n", "\"\"\"\n", "c2_agent_yaml = yaml.safe_load(custom_c2_agent)" ] @@ -169,7 +169,7 @@ "source": [ "client_1: Computer = env.game.simulation.network.get_node_by_hostname(\"client_1\")\n", "client_1.software_manager.install(C2Server)\n", - "c2_server: C2Server = client_1.software_manager.software[\"C2Server\"]\n", + "c2_server: C2Server = client_1.software_manager.software[\"c2-server\"]\n", "c2_server.run()\n", "client_1.software_manager.show()" ] @@ -262,7 +262,7 @@ "outputs": [], "source": [ "env.step(2)\n", - "c2_beacon: C2Beacon = web_server.software_manager.software[\"C2Beacon\"]\n", + "c2_beacon: C2Beacon = web_server.software_manager.software[\"c2-beacon\"]\n", "web_server.software_manager.show()\n", "c2_beacon.show()" ] @@ -435,7 +435,7 @@ "metadata": {}, "outputs": [], "source": [ - "ransomware_script: RansomwareScript = web_server.software_manager.software[\"RansomwareScript\"]\n", + "ransomware_script: RansomwareScript = web_server.software_manager.software[\"ransomware-script\"]\n", "web_server.software_manager.show()\n", "ransomware_script.show()" ] @@ -574,20 +574,20 @@ "custom_blue_agent_yaml = \"\"\"\n", " - ref: defender\n", " team: BLUE\n", - " type: ProxyAgent\n", + " type: proxy-agent\n", "\n", " observation_space:\n", - " type: CUSTOM\n", + " type: custom\n", " options:\n", " components:\n", - " - type: NODES\n", - " label: NODES\n", + " - type: nodes\n", + " label: nodes\n", " options:\n", " hosts:\n", " - hostname: web_server\n", " applications:\n", - " - application_name: C2Beacon\n", - " - application_name: RansomwareScript\n", + " - application_name: c2-beacon\n", + " - application_name: ransomware-script\n", " folders:\n", " - folder_name: exfiltration_folder\n", " files:\n", @@ -637,7 +637,7 @@ " - UDP\n", " num_rules: 10\n", "\n", - " - type: LINKS\n", + " - type: links\n", " label: LINKS\n", " options:\n", " link_references:\n", @@ -651,26 +651,26 @@ " - switch_2:eth-1<->client_1:eth-1\n", " - switch_2:eth-2<->client_2:eth-1\n", " - switch_2:eth-7<->security_suite:eth-2\n", - " - type: \"NONE\"\n", + " - type: \"none\"\n", " label: ICS\n", " options: {}\n", "\n", " action_space:\n", " action_map:\n", " 0:\n", - " action: do_nothing\n", + " action: do-nothing\n", " options: {}\n", " 1:\n", - " action: node_application_remove\n", + " action: node-application-remove\n", " options:\n", " node_name: web_server\n", - " application_name: C2Beacon\n", + " application_name: c2-beacon\n", " 2:\n", - " action: node_shutdown\n", + " action: node-shutdown\n", " options:\n", " node_name: web_server\n", " 3:\n", - " action: router_acl_add_rule\n", + " action: router-acl-add-rule\n", " options:\n", " target_router: router_1\n", " position: 1\n", @@ -685,7 +685,7 @@ "\n", " reward_function:\n", " reward_components:\n", - " - type: DUMMY\n", + " - type: dummy\n", "\n", " agent_settings:\n", " flatten_obs: False\n", @@ -776,12 +776,12 @@ "\n", "# Installing the C2 Server.\n", "client_1.software_manager.install(C2Server)\n", - "c2_server: C2Server = client_1.software_manager.software[\"C2Server\"]\n", + "c2_server: C2Server = client_1.software_manager.software[\"c2-server\"]\n", "c2_server.run()\n", "\n", "# Installing the C2 Beacon.\n", "web_server.software_manager.install(C2Beacon)\n", - "c2_beacon: C2Beacon = web_server.software_manager.software[\"C2Beacon\"]\n", + "c2_beacon: C2Beacon = web_server.software_manager.software[\"c2-beacon\"]\n", "c2_beacon.configure(c2_server_ip_address=\"192.168.10.21\")\n", "c2_beacon.establish()" ] @@ -825,7 +825,7 @@ "outputs": [], "source": [ "# Installing RansomwareScript via C2 Terminal Commands\n", - "ransomware_install_command = {\"commands\":[[\"software_manager\", \"application\", \"install\", \"RansomwareScript\"]],\n", + "ransomware_install_command = {\"commands\":[[\"software_manager\", \"application\", \"install\", \"ransomware-script\"]],\n", " \"username\": \"admin\",\n", " \"password\": \"admin\"}\n", "c2_server.send_command(C2Command.TERMINAL, command_options=ransomware_install_command)\n" @@ -984,11 +984,11 @@ " web_server: Server = given_env.game.simulation.network.get_node_by_hostname(\"web_server\")\n", "\n", " client_1.software_manager.install(C2Server)\n", - " c2_server: C2Server = client_1.software_manager.software[\"C2Server\"]\n", + " c2_server: C2Server = client_1.software_manager.software[\"c2-server\"]\n", " c2_server.run()\n", "\n", " web_server.software_manager.install(C2Beacon)\n", - " c2_beacon: C2Beacon = web_server.software_manager.software[\"C2Beacon\"]\n", + " c2_beacon: C2Beacon = web_server.software_manager.software[\"c2-beacon\"]\n", " c2_beacon.configure(c2_server_ip_address=\"192.168.10.21\")\n", " c2_beacon.establish()\n", "\n", @@ -1086,7 +1086,7 @@ " \"username\": \"admin\",\n", " \"password\": \"admin\"}\n", "\n", - "c2_server: C2Server = client_1.software_manager.software[\"C2Server\"]\n", + "c2_server: C2Server = client_1.software_manager.software[\"c2-server\"]\n", "c2_server.send_command(C2Command.TERMINAL, command_options=ransomware_install_command)" ] }, @@ -1174,7 +1174,7 @@ " \"username\": \"admin\",\n", " \"password\": \"admin\"}\n", "\n", - "c2_server: C2Server = client_1.software_manager.software[\"C2Server\"]\n", + "c2_server: C2Server = client_1.software_manager.software[\"c2-server\"]\n", "c2_server.send_command(C2Command.TERMINAL, command_options=ransomware_install_command)" ] }, @@ -1396,16 +1396,16 @@ "source": [ "web_server: Server = c2_config_env.game.simulation.network.get_node_by_hostname(\"web_server\")\n", "web_server.software_manager.install(C2Beacon)\n", - "c2_beacon: C2Beacon = web_server.software_manager.software[\"C2Beacon\"]\n", + "c2_beacon: C2Beacon = web_server.software_manager.software[\"c2-beacon\"]\n", "\n", "client_1: Computer = c2_config_env.game.simulation.network.get_node_by_hostname(\"client_1\")\n", "client_1.software_manager.install(C2Server)\n", - "c2_server_1: C2Server = client_1.software_manager.software[\"C2Server\"]\n", + "c2_server_1: C2Server = client_1.software_manager.software[\"c2-server\"]\n", "c2_server_1.run()\n", "\n", "client_2: Computer = c2_config_env.game.simulation.network.get_node_by_hostname(\"client_2\")\n", "client_2.software_manager.install(C2Server)\n", - "c2_server_2: C2Server = client_2.software_manager.software[\"C2Server\"]\n", + "c2_server_2: C2Server = client_2.software_manager.software[\"c2-server\"]\n", "c2_server_2.run()" ] }, diff --git a/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb b/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb index 14938c02..2e0a0d40 100644 --- a/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb +++ b/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb @@ -76,9 +76,9 @@ " # parse the info dict form step output and write out what the red agent is doing\n", " red_info : AgentHistoryItem = info['agent_actions']['data_manipulation_attacker']\n", " red_action = red_info.action\n", - " if red_action == 'do_nothing':\n", + " if red_action == 'do-nothing':\n", " red_str = 'DO NOTHING'\n", - " elif red_action == 'node_application_execute':\n", + " elif red_action == 'node-application-execute':\n", " red_str = f\"ATTACK from {red_info.parameters['node_name']}\"\n", " return red_str" ] @@ -369,18 +369,18 @@ "change = yaml.safe_load(\"\"\"\n", " applications:\n", " - ref: data_manipulation_bot\n", - " type: DataManipulationBot\n", + " type: data-manipulation-bot\n", " options:\n", " port_scan_p_of_success: 1.0\n", " data_manipulation_p_of_success: 1.0\n", " payload: \"DELETE\"\n", " server_ip: 192.168.1.14\n", " - ref: client_1_web_browser\n", - " type: WebBrowser\n", + " type: web-browser\n", " options:\n", " target_url: http://arcd.com/users/\n", " - ref: client_1_database_client\n", - " type: DatabaseClient\n", + " type: database-client\n", " options:\n", " db_server_ip: 192.168.1.14\n", "\"\"\")\n", @@ -414,18 +414,18 @@ "change = yaml.safe_load(\"\"\"\n", " applications:\n", " - ref: data_manipulation_bot\n", - " type: DataManipulationBot\n", + " type: data-manipulation-bot\n", " options:\n", " port_scan_p_of_success: 0.0\n", " data_manipulation_p_of_success: 0.0\n", " payload: \"DELETE\"\n", " server_ip: 192.168.1.14\n", " - ref: client_1_web_browser\n", - " type: WebBrowser\n", + " type: web-browser\n", " options:\n", " target_url: http://arcd.com/users/\n", " - ref: client_1_database_client\n", - " type: DatabaseClient\n", + " type: database-client\n", " options:\n", " db_server_ip: 192.168.1.14\n", "\"\"\")\n", diff --git a/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb b/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb index 41b75e43..2d3f99b2 100644 --- a/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb +++ b/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb @@ -458,9 +458,9 @@ " # parse the info dict form step output and write out what the red agent is doing\n", " red_info : AgentHistoryItem = info['agent_actions']['data_manipulation_attacker']\n", " red_action = red_info.action\n", - " if red_action == 'do_nothing':\n", + " if red_action == 'do-nothing':\n", " red_str = 'DO NOTHING'\n", - " elif red_action == 'node_application_execute':\n", + " elif red_action == 'node-application-execute':\n", " client = \"client 1\" if red_info.parameters['node_name'] == 0 else \"client 2\"\n", " red_str = f\"ATTACK from {client}\"\n", " return red_str" diff --git a/src/primaite/notebooks/Privilege-Escalation-and-Data-Loss-Example.ipynb b/src/primaite/notebooks/Privilege-Escalation-and-Data-Loss-Example.ipynb index cbb898ea..deb38eea 100644 --- a/src/primaite/notebooks/Privilege-Escalation-and-Data-Loss-Example.ipynb +++ b/src/primaite/notebooks/Privilege-Escalation-and-Data-Loss-Example.ipynb @@ -120,11 +120,11 @@ "outputs": [], "source": [ "some_tech_jnr_dev_pc: Computer = game.simulation.network.get_node_by_hostname(\"some_tech_jnr_dev_pc\")\n", - "some_tech_jnr_dev_db_client: DatabaseClient = some_tech_jnr_dev_pc.software_manager.software[\"DatabaseClient\"]\n", - "some_tech_jnr_dev_web_browser: WebBrowser = some_tech_jnr_dev_pc.software_manager.software[\"WebBrowser\"]\n", + "some_tech_jnr_dev_db_client: DatabaseClient = some_tech_jnr_dev_pc.software_manager.software[\"database-client\"]\n", + "some_tech_jnr_dev_web_browser: WebBrowser = some_tech_jnr_dev_pc.software_manager.software[\"web-browser\"]\n", "some_tech_rt: Router = game.simulation.network.get_node_by_hostname(\"some_tech_rt\")\n", "some_tech_db_srv: Server = game.simulation.network.get_node_by_hostname(\"some_tech_db_srv\")\n", - "some_tech_db_service: DatabaseService = some_tech_db_srv.software_manager.software[\"DatabaseService\"]\n", + "some_tech_db_service: DatabaseService = some_tech_db_srv.software_manager.software[\"database-service\"]\n", "some_tech_storage_srv: Server = game.simulation.network.get_node_by_hostname(\"some_tech_storage_srv\")\n", "some_tech_web_srv: Server = game.simulation.network.get_node_by_hostname(\"some_tech_web_srv\")" ] @@ -211,7 +211,7 @@ "source": [ "caos_action = [\n", " \"network\", \"node\", \"some_tech_jnr_dev_pc\", \n", - " \"service\", \"Terminal\", \"node_session_remote_login\", \"admin\", \"admin\", str(some_tech_storage_srv.network_interface[1].ip_address)\n", + " \"service\", \"terminal\", \"node-session-remote-login\", \"admin\", \"admin\", str(some_tech_storage_srv.network_interface[1].ip_address)\n", "]\n", "game.simulation.apply_request(caos_action)" ] @@ -233,7 +233,7 @@ }, "outputs": [], "source": [ - "caos_action = [\"network\", \"node\", \"some_tech_jnr_dev_pc\", \"application\", \"WebBrowser\", \"execute\"]\n", + "caos_action = [\"network\", \"node\", \"some_tech_jnr_dev_pc\", \"application\", \"web-browser\", \"execute\"]\n", "game.simulation.apply_request(caos_action)" ] }, @@ -256,7 +256,7 @@ "metadata": {}, "outputs": [], "source": [ - "game.get_sim_state()[\"network\"][\"nodes\"][\"some_tech_rt\"][\"services\"][\"UserSessionManager\"][\"active_remote_sessions\"]" + "game.get_sim_state()[\"network\"][\"nodes\"][\"some_tech_rt\"][\"services\"][\"user-session-manager\"][\"active_remote_sessions\"]" ] }, { @@ -269,7 +269,7 @@ "source": [ "caos_action = [\n", " \"network\", \"node\", \"some_tech_jnr_dev_pc\", \n", - " \"service\", \"Terminal\", \"node_session_remote_login\", \"admin\", \"admin\", str(some_tech_rt.network_interface[4].ip_address)\n", + " \"service\", \"terminal\", \"node-session-remote-login\", \"admin\", \"admin\", str(some_tech_rt.network_interface[4].ip_address)\n", "]\n", "game.simulation.apply_request(caos_action)" ] @@ -282,7 +282,7 @@ }, "outputs": [], "source": [ - "game.get_sim_state()[\"network\"][\"nodes\"][\"some_tech_rt\"][\"services\"][\"UserSessionManager\"][\"active_remote_sessions\"]" + "game.get_sim_state()[\"network\"][\"nodes\"][\"some_tech_rt\"][\"services\"][\"user-session-manager\"][\"active_remote_sessions\"]" ] }, { @@ -315,7 +315,7 @@ "source": [ "caos_action = [\n", " \"network\", \"node\", \"some_tech_jnr_dev_pc\", \n", - " \"service\", \"Terminal\", \"send_remote_command\", str(some_tech_rt.network_interface[4].ip_address),\n", + " \"service\", \"terminal\", \"send_remote_command\", str(some_tech_rt.network_interface[4].ip_address),\n", " {\n", " \"command\": [\n", " \"acl\", \"add_rule\", \"PERMIT\", \"TCP\",\n", @@ -368,7 +368,7 @@ "source": [ "caos_action = [\n", " \"network\", \"node\", \"some_tech_jnr_dev_pc\", \n", - " \"service\", \"Terminal\", \"remote_logoff\", str(some_tech_rt.network_interface[4].ip_address)\n", + " \"service\", \"terminal\", \"remote_logoff\", str(some_tech_rt.network_interface[4].ip_address)\n", "]\n", "game.simulation.apply_request(caos_action)" ] @@ -386,7 +386,7 @@ "metadata": {}, "outputs": [], "source": [ - "game.get_sim_state()[\"network\"][\"nodes\"][\"some_tech_rt\"][\"services\"][\"UserSessionManager\"][\"active_remote_sessions\"]" + "game.get_sim_state()[\"network\"][\"nodes\"][\"some_tech_rt\"][\"services\"][\"user-session-manager\"][\"active_remote_sessions\"]" ] }, { @@ -406,7 +406,7 @@ "source": [ "caos_action = [\n", " \"network\", \"node\", \"some_tech_jnr_dev_pc\", \n", - " \"service\", \"Terminal\", \"node_session_remote_login\", \"admin\", \"admin\", str(some_tech_storage_srv.network_interface[1].ip_address)\n", + " \"service\", \"terminal\", \"node-session-remote-login\", \"admin\", \"admin\", str(some_tech_storage_srv.network_interface[1].ip_address)\n", "]\n", "game.simulation.apply_request(caos_action)" ] @@ -421,7 +421,7 @@ "source": [ "caos_action = [\n", " \"network\", \"node\", \"some_tech_jnr_dev_pc\", \n", - " \"service\", \"Terminal\", \"send_remote_command\", str(some_tech_storage_srv.network_interface[1].ip_address),\n", + " \"service\", \"terminal\", \"send_remote_command\", str(some_tech_storage_srv.network_interface[1].ip_address),\n", " {\n", " \"command\": [\n", " \"file_system\", \"delete\", \"file\", db_backup_folder, \"database.db\"\n", @@ -476,7 +476,7 @@ }, "outputs": [], "source": [ - "caos_action = [\"network\", \"node\", \"some_tech_jnr_dev_pc\", \"application\", \"WebBrowser\", \"execute\"]\n", + "caos_action = [\"network\", \"node\", \"some_tech_jnr_dev_pc\", \"application\", \"web-browser\", \"execute\"]\n", "game.simulation.apply_request(caos_action)" ] }, @@ -535,7 +535,7 @@ }, "outputs": [], "source": [ - "caos_action = [\"network\", \"node\", \"some_tech_jnr_dev_pc\", \"application\", \"WebBrowser\", \"execute\"]\n", + "caos_action = [\"network\", \"node\", \"some_tech_jnr_dev_pc\", \"application\", \"web-browser\", \"execute\"]\n", "game.simulation.apply_request(caos_action)" ] }, diff --git a/src/primaite/notebooks/Requests-and-Responses.ipynb b/src/primaite/notebooks/Requests-and-Responses.ipynb index 48f1cf3e..c29d41dc 100644 --- a/src/primaite/notebooks/Requests-and-Responses.ipynb +++ b/src/primaite/notebooks/Requests-and-Responses.ipynb @@ -52,14 +52,15 @@ "outputs": [], "source": [ "sim = Simulation()\n", + "\n", "sim.network.add_node(\n", " HostNode.from_config(\n", " config = {\n", - " 'type': \"hostnode\",\n", + " 'type': \"host-node\",\n", " 'hostname': \"client\",\n", " 'ip_address': '10.0.0.1',\n", " 'subnet_mask': '255.255.255.0',\n", - " 'operating_state': NodeOperatingState.ON,\n", + " 'operating_state': \"ON\",\n", " }\n", " )\n", ")\n", @@ -98,7 +99,7 @@ "outputs": [], "source": [ "response = sim.apply_request(\n", - " request=[\"network\", \"node\", \"client\", \"service\", \"DNSClient\", \"stop\"],\n", + " request=[\"network\", \"node\", \"client\", \"service\", \"dns-client\", \"stop\"],\n", " context={}\n", " )\n", "print(response)" @@ -118,7 +119,7 @@ "metadata": {}, "outputs": [], "source": [ - "print(f\"DNS Client state: {client.software_manager.software.get('DNSClient').operating_state.name}\")" + "print(f\"DNS Client state: {client.software_manager.software.get('dns-client').operating_state.name}\")" ] }, { @@ -142,7 +143,7 @@ "outputs": [], "source": [ "response = sim.apply_request(\n", - " request=[\"network\", \"node\", \"client\", \"service\", \"NonExistentApplication\", \"stop\"],\n", + " request=[\"network\", \"node\", \"client\", \"service\", \"non-existent-application\", \"stop\"],\n", " context={}\n", " )\n", "print(response)" @@ -205,7 +206,7 @@ "outputs": [], "source": [ "response = sim.apply_request(\n", - " request=[\"network\", \"node\", \"client\", \"service\", \"DNSClient\", \"start\"],\n", + " request=[\"network\", \"node\", \"client\", \"service\", \"dns-client\", \"start\"],\n", " context={}\n", " )\n", "print(response)" diff --git a/src/primaite/notebooks/Terminal-Processing.ipynb b/src/primaite/notebooks/Terminal-Processing.ipynb index dc870db5..5077732b 100644 --- a/src/primaite/notebooks/Terminal-Processing.ipynb +++ b/src/primaite/notebooks/Terminal-Processing.ipynb @@ -6,7 +6,7 @@ "source": [ "# Terminal Processing\n", "\n", - "© Crown-owned copyright 2024, Defence Science and Technology Laboratory UK" + "© Crown-owned copyright 2025, Defence Science and Technology Laboratory UK" ] }, { @@ -39,41 +39,27 @@ "from primaite.simulator.system.applications.red_applications.ransomware_script import RansomwareScript\n", "from primaite.simulator.system.services.terminal.terminal import RemoteTerminalConnection\n", "\n", - "# print(dir(Computer))\n", - "# node_a = Computer.from_config(\n", - "# config = {\n", - "# \"type\": \"computer\",\n", - "# \"hostname\": \"node_a\",\n", - "# \"ip_address\": \"192.168.0.10\",\n", - "# \"subnet_mask\": \"255.255.255.0\",\n", - "# \"startup_duration\": 0,\n", - "# }\n", - "# )\n", - "# print(f\"{node_a=}\")\n", - "\n", "def basic_network() -> Network:\n", " \"\"\"Utility function for creating a default network to demonstrate Terminal functionality\"\"\"\n", " network = Network()\n", - " # node_a = Computer(hostname=\"node_a\", ip_address=\"192.168.0.10\", subnet_mask=\"255.255.255.0\", start_up_duration=0)\n", " node_a = Computer.from_config(\n", " config = {\n", " \"type\": \"computer\",\n", " \"hostname\": \"node_a\",\n", " \"ip_address\": \"192.168.0.10\",\n", " \"subnet_mask\": \"255.255.255.0\",\n", - " \"startup_duration\": 0,\n", + " # \"startup_duration\": 0,\n", " }\n", " )\n", " print(f\"{node_a=}\")\n", " node_a.power_on()\n", - " # node_b = Computer(hostname=\"node_b\", ip_address=\"192.168.0.11\", subnet_mask=\"255.255.255.0\", start_up_duration=0)\n", " node_b = Computer.from_config(\n", " config = {\n", " \"type\": \"computer\",\n", " \"hostname\": \"node_b\",\n", " \"ip_address\": \"192.168.0.11\",\n", " \"subnet_mask\": \"255.255.255.0\",\n", - " \"startup_duration\": 0,\n", + " # \"startup_duration\": 0,\n", " }\n", " )\n", " node_b.power_on()\n", @@ -98,9 +84,9 @@ "source": [ "network: Network = basic_network()\n", "computer_a: Computer = network.get_node_by_hostname(\"node_a\")\n", - "terminal_a: Terminal = computer_a.software_manager.software.get(\"Terminal\")\n", + "terminal_a: Terminal = computer_a.software_manager.software.get(\"terminal\")\n", "computer_b: Computer = network.get_node_by_hostname(\"node_b\")\n", - "terminal_b: Terminal = computer_b.software_manager.software.get(\"Terminal\")" + "terminal_b: Terminal = computer_b.software_manager.software.get(\"terminal\")" ] }, { @@ -152,7 +138,7 @@ "metadata": {}, "outputs": [], "source": [ - "term_a_term_b_remote_connection.execute([\"software_manager\", \"application\", \"install\", \"RansomwareScript\"])" + "term_a_term_b_remote_connection.execute([\"software_manager\", \"application\", \"install\", \"ransomware-script\"])" ] }, { diff --git a/src/primaite/simulator/network/hardware/base.py b/src/primaite/simulator/network/hardware/base.py index 4712af60..4c252050 100644 --- a/src/primaite/simulator/network/hardware/base.py +++ b/src/primaite/simulator/network/hardware/base.py @@ -1632,7 +1632,9 @@ class Node(SimComponent, ABC): dns_server=kwargs["config"].dns_server, ) super().__init__(**kwargs) - self.operating_state = NodeOperatingState.ON if not (p := kwargs["config"].operating_state) else p + self.operating_state = ( + NodeOperatingState.ON if not (p := kwargs["config"].operating_state) else NodeOperatingState[p.upper()] + ) self._install_system_software() self.session_manager.node = self self.session_manager.software_manager = self.software_manager diff --git a/src/primaite/simulator/network/hardware/nodes/host/host_node.py b/src/primaite/simulator/network/hardware/nodes/host/host_node.py index 76d9167c..d38ba097 100644 --- a/src/primaite/simulator/network/hardware/nodes/host/host_node.py +++ b/src/primaite/simulator/network/hardware/nodes/host/host_node.py @@ -313,7 +313,7 @@ class HostNode(Node, discriminator="host-node"): """ SYSTEM_SOFTWARE: ClassVar[Dict] = { - "HostARP": HostARP, + "host-arp": HostARP, "icmp": ICMP, "dns-client": DNSClient, "ntp-client": NTPClient, diff --git a/tests/assets/configs/basic_switched_network.yaml b/tests/assets/configs/basic_switched_network.yaml index c9ac5f8d..4c6d2557 100644 --- a/tests/assets/configs/basic_switched_network.yaml +++ b/tests/assets/configs/basic_switched_network.yaml @@ -34,10 +34,10 @@ agents: action_space: action_map: 0: - action: do-nothing + action: do_nothing options: {} 1: - action: node-application-execute + action: node_application_execute options: node_name: client_2 application_name: web-browser @@ -109,7 +109,7 @@ agents: action_space: action_map: 0: - action: do-nothing + action: do_nothing options: {} reward_function: From 486c797d60422c6f0acfea7a2e9cf9094bf4b42c Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Mon, 17 Feb 2025 11:54:13 +0000 Subject: [PATCH 15/24] #3075: Change duplicate application_name in config file. --- src/primaite/config/_package_data/data_manipulation.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/primaite/config/_package_data/data_manipulation.yaml b/src/primaite/config/_package_data/data_manipulation.yaml index 4705c135..ceee607d 100644 --- a/src/primaite/config/_package_data/data_manipulation.yaml +++ b/src/primaite/config/_package_data/data_manipulation.yaml @@ -81,7 +81,7 @@ agents: action: node-application-execute options: node_name: client_1 - application_name: web-browser + application_name: database-client reward_function: reward_components: From 7e82de919c4287eb64d4e47b84808be2d2525f61 Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Mon, 17 Feb 2025 14:19:56 +0000 Subject: [PATCH 16/24] #3075: Markdown changes. --- ...ommand-and-Control-E2E-Demonstration.ipynb | 85 +- ...a-Manipulation-Customising-Red-Agent.ipynb | 48 +- .../Data-Manipulation-E2E-Demonstration.ipynb | 4531 ++++++++++++++++- 3 files changed, 4548 insertions(+), 116 deletions(-) diff --git a/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb b/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb index 474dadff..6dc8e077 100644 --- a/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb +++ b/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb @@ -6,7 +6,7 @@ "source": [ "# Command and Control Application Suite E2E Demonstration\n", "\n", - "© Crown-owned copyright 2024, Defence Science and Technology Laboratory UK\n", + "© Crown-owned copyright 2025, Defence Science and Technology Laboratory UK\n", "\n", "This notebook demonstrates the current implementation of the command and control (C2) server and beacon applications in primAITE." ] @@ -201,15 +201,15 @@ " nodes: # Node List\n", " - node_name: web_server\n", " applications: \n", - " - application_name: C2Beacon\n", + " - application_name: c2-beacon\n", " ...\n", " ...\n", " action_map:\n", " 1:\n", - " action: node_application_install \n", + " action: node-application-install \n", " options:\n", " node_id: 0 # Index 0 at the node list.\n", - " application_name: C2Beacon\n", + " application_name: c2-beacon\n", "```" ] }, @@ -244,7 +244,7 @@ " action_map:\n", " ...\n", " 2:\n", - " action: configure_c2_beacon\n", + " action: configure-c2-beacon\n", " options:\n", " node_id: 0 # Node Index\n", " config: # Further information about these config options can be found at the bottom of this notebook.\n", @@ -273,7 +273,7 @@ "source": [ "### **Command and Control** | C2 Beacon Actions | node_application_execute\n", "\n", - "The final action is ``node_application_execute`` which is used to establish a connection for the C2 application. This action can be called by the Red Agent via action ``3`` in it's action map. \n", + "The final action is ``node-application-execute`` which is used to establish a connection for the C2 application. This action can be called by the Red Agent via action ``3`` in it's action map. \n", "\n", "The yaml snippet below shows all the relevant agent options for this action:\n", "\n", @@ -283,13 +283,13 @@ " nodes: # Node List\n", " - node_name: web_server\n", " applications: \n", - " - application_name: C2Beacon\n", + " - application_name: c2-beacon\n", " ...\n", " ...\n", " action_map:\n", " ...\n", " 3:\n", - " action: node_application_execute\n", + " action: node-application-execute\n", " options:\n", " node_id: 0\n", " application_id: 0\n", @@ -331,34 +331,27 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### **Command and Control** | C2 Server Actions | C2_SERVER_TERMINAL_COMMAND\n", + "### **Command and Control** | C2 Server Actions | c2-server-terminal-command\n", "\n", - "The C2 Server's terminal action: ``C2_SERVER_TERMINAL_COMMAND`` is indexed at ``4`` in it's action map. \n", + "The C2 Server's terminal action: ``c2-server-terminal-command`` is indexed at ``4`` in it's action map. \n", "\n", "This action leverages the terminal service that is installed by default on all nodes to grant red agents a lot more configurability. If you're unfamiliar with terminals then it's recommended that you refer to the ``Terminal Processing`` notebook.\n", "\n", "It's worth noting that an additional benefit a red agent has when using the terminal service via the C2 Server is that you can execute multiple commands in one action. \n", "\n", - "In this notebook, the ``C2_SERVER_TERMINAL_COMMAND`` is used to install a RansomwareScript application on the ``web_server`` node.\n", + "In this notebook, the ``c2-server-terminal-command`` is used to install a RansomwareScript application on the ``web_server`` node.\n", "\n", "The yaml snippet below shows all the relevant agent options for this action:\n", "\n", "``` yaml\n", " action_space:\n", - " options:\n", - " nodes: # Node List\n", - " ...\n", - " - node_name: client_1\n", - " applications: \n", - " - application_name: C2Server\n", - " ...\n", " action_map:\n", + " ...\n", " 4:\n", - " action: C2_SERVER_TERMINAL_COMMAND\n", + " action: c2-server-terminal-command\n", " options:\n", - " node_id: 1\n", + " node_name: client_1\n", " ip_address:\n", - " account:\n", " username: admin\n", " password: admin\n", " commands:\n", @@ -366,7 +359,7 @@ " - software_manager\n", " - application\n", " - install\n", - " - RansomwareScript\n", + " - ransomware-script\n", "```" ] }, @@ -402,14 +395,8 @@ "\n", "``` yaml\n", " action_space:\n", - " options:\n", - " nodes: # Node List\n", - " ...\n", - " - node_name: client_1\n", - " applications: \n", - " - application_name: C2Server\n", - " ...\n", " action_map:\n", + " ...\n", " 5:\n", " action: c2-server-ransomware-configure\n", " options:\n", @@ -444,9 +431,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### **Command and Control** | C2 Server Actions | c2_server_data_exfiltrate\n", + "### **Command and Control** | C2 Server Actions | c2-server-data-exfiltrate\n", "\n", - "The second to last action available is the ``c2_server_data_exfiltrate`` which is indexed as action ``6`` in the action map.\n", + "The second to last action available is the ``c2-server-data-exfiltrate`` which is indexed as action ``6`` in the action map.\n", "\n", "This action can be used to exfiltrate a target file on a remote node to the C2 Beacon and the C2 Server's host file system via the ``FTP`` services.\n", "\n", @@ -454,25 +441,18 @@ "\n", "``` yaml\n", " action_space:\n", - " options:\n", - " nodes: # Node List\n", - " ...\n", - " - node_name: client_1\n", - " applications: \n", - " - application_name: C2Server\n", - " ...\n", " action_map:\n", + " ...\n", " 6:\n", - " action: c2_server_data_exfiltrate\n", + " action: c2-server-data-exfiltrate\n", " options:\n", " node_id: 1\n", " target_file_name: \"database.db\"\n", " target_folder_name: \"database\"\n", " exfiltration_folder_name: \"spoils\"\n", " target_ip_address: \"192.168.1.14\"\n", - " account:\n", - " username: \"admin\",\n", - " password: \"admin\"\n", + " username: \"admin\",\n", + " password: \"admin\"\n", "\n", "```" ] @@ -510,9 +490,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### **Command and Control** | C2 Server Actions | c2_server_ransomware_launch\n", + "### **Command and Control** | C2 Server Actions | c2-server-ransomware-launch\n", "\n", - "Finally, the last available action is for the c2_server_ransomware_launch to start the ransomware script installed on the same node as the C2 beacon.\n", + "Finally, the last available action is for the c2-server-ransomware-launch to start the ransomware script installed on the same node as the C2 beacon.\n", "\n", "This action is indexed as action ``7``.\n", "\n", @@ -520,16 +500,10 @@ "\n", "``` yaml\n", " action_space:\n", - " options:\n", - " nodes: # Node List\n", - " ...\n", - " - node_name: client_1\n", - " applications: \n", - " - application_name: C2Server\n", " ...\n", " action_map:\n", " 7:\n", - " action: c2_server_ransomware_launch\n", + " action: c2-server-ransomware-launch\n", " options:\n", " node_id: 1\n", "```\n" @@ -1337,18 +1311,15 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "As demonstrated earlier, red agents can use the ``configure_c2_beacon`` action to configure these settings mid episode through the configuration options:\n", + "As demonstrated earlier, red agents can use the ``configure-c2-beacon`` action to configure these settings mid episode through the configuration options:\n", "\n", "``` YAML\n", "...\n", - " action: configure_c2_beacon\n", + " action: configure-c2-beacon\n", " options:\n", - " node_id: 0\n", + " node_name: web_server\n", " config:\n", " c2_server_ip_address: 192.168.10.21\n", - " keep_alive_frequency: 10\n", - " masquerade_protocol: TCP\n", - " masquerade_port: DNS\n", "```" ] }, diff --git a/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb b/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb index 2e0a0d40..1eb814e3 100644 --- a/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb +++ b/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb @@ -146,42 +146,15 @@ "```yaml\n", " - ref: data_manipulation_attacker # name of agent\n", " team: RED # not used, just for human reference\n", - " type: RedDatabaseCorruptingAgent # type of agent - this lets primaite know which agent class to use\n", - "\n", - " # Since the agent does not need to react to what is happening in the environment, the observation space is empty.\n", - " observation_space:\n", - " type: UC2RedObservation\n", - " options:\n", - " nodes: {}\n", - "\n", - " action_space:\n", - " \n", - " # The agent has access to the DataManipulationBoth on clients 1 and 2.\n", - " options:\n", - " nodes:\n", - " - node_name: client_1 # The network should have a node called client_1\n", - " applications:\n", - " - application_name: DataManipulationBot # The node client_1 should have DataManipulationBot configured on it\n", - " - node_name: client_2 # The network should have a node called client_2\n", - " applications:\n", - " - application_name: DataManipulationBot # The node client_2 should have DataManipulationBot configured on it\n", - "\n", - " # not important\n", - " max_folders_per_node: 1\n", - " max_files_per_folder: 1\n", - " max_services_per_node: 1\n", - "\n", - " # red agent does not need a reward function\n", - " reward_function:\n", - " reward_components:\n", - " - type: DUMMY\n", + " type: red-database-corrupting-agent # type of agent - this lets primaite know which agent class to use\n", "\n", " # These actions are passed to the RedDatabaseCorruptingAgent init method, they dictate the schedule of attacks\n", " agent_settings:\n", - " start_settings:\n", - " start_step: 25 # first attack at step 25\n", - " frequency: 20 # attacks will happen every 20 steps (on average)\n", - " variance: 5 # the timing of attacks will vary by up to 5 steps earlier or later\n", + " possible_start_nodes: [client_1, client_2] # List of clients the attack can start from\n", + " target_application: data-manipulation-bot\n", + " start_step: 25 # first attack at step 25\n", + " frequency: 20 # attacks will happen every 20 steps (on average)\n", + " variance: 5 # the timing of attacks will vary by up to 5 steps earlier or later\n", "```" ] }, @@ -201,8 +174,7 @@ "simulation:\n", " network:\n", " nodes:\n", - " - ref: client_1\n", - " hostname: client_1\n", + " - hostname: client_1\n", " type: computer\n", " ip_address: 192.168.10.21\n", " subnet_mask: 255.255.255.0\n", @@ -210,15 +182,13 @@ " \n", " # \n", " applications:\n", - " - ref: data_manipulation_bot\n", - " type: DataManipulationBot\n", + " - type: data-manipulation-bot\n", " options:\n", " port_scan_p_of_success: 0.8 # Probability that port scan is successful\n", " data_manipulation_p_of_success: 0.8 # Probability that SQL attack is successful\n", " payload: \"DELETE\" # The SQL query which causes the attack (this has to be DELETE)\n", " server_ip: 192.168.1.14 # IP address of server hosting the database\n", - " - ref: client_1_database_client\n", - " type: DatabaseClient # Database client must be installed in order for DataManipulationBot to function\n", + " - type: database-client # Database client must be installed in order for DataManipulationBot to function\n", " options:\n", " db_server_ip: 192.168.1.14 # IP address of server hosting the database\n", "```" diff --git a/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb b/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb index 2d3f99b2..f3e3fb98 100644 --- a/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb +++ b/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb @@ -6,7 +6,7 @@ "source": [ "# Data Manipulation Scenario\n", "\n", - "© Crown-owned copyright 2024, Defence Science and Technology Laboratory UK" + "© Crown-owned copyright 2025, Defence Science and Technology Laboratory UK" ] }, { @@ -373,16 +373,29 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2025-02-17 11:15:30,959: Performing the PrimAITE first-time setup...\n", + "2025-02-17 11:15:30,960: Building the PrimAITE app directories...\n", + "2025-02-17 11:15:30,960: Building primaite_config.yaml...\n", + "2025-02-17 11:15:30,960: Rebuilding the demo notebooks...\n", + "2025-02-17 11:15:30,967: Rebuilding the example notebooks...\n", + "2025-02-17 11:15:30,970: PrimAITE setup complete!\n" + ] + } + ], "source": [ "!primaite setup" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": { "tags": [] }, @@ -394,7 +407,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": { "tags": [] }, @@ -420,9 +433,254 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-02-17 11:15:35,108: PrimaiteGymEnv RNG seed = None\n", + "2025-02-17 11:15:35,109: Resetting environment, episode 0, avg. reward: 0.0\n", + "2025-02-17 11:15:35,111: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-17/11-15-31/agent_actions/episode_0.json\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "env created successfully\n", + "{'ICS': 0,\n", + " 'LINKS': {1: {'PROTOCOLS': {'ALL': 0}},\n", + " 2: {'PROTOCOLS': {'ALL': 0}},\n", + " 3: {'PROTOCOLS': {'ALL': 0}},\n", + " 4: {'PROTOCOLS': {'ALL': 0}},\n", + " 5: {'PROTOCOLS': {'ALL': 0}},\n", + " 6: {'PROTOCOLS': {'ALL': 0}},\n", + " 7: {'PROTOCOLS': {'ALL': 0}},\n", + " 8: {'PROTOCOLS': {'ALL': 0}},\n", + " 9: {'PROTOCOLS': {'ALL': 0}},\n", + " 10: {'PROTOCOLS': {'ALL': 0}}},\n", + " 'NODES': {'HOST0': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0,\n", + " 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0,\n", + " 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0,\n", + " 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0,\n", + " 'outbound': 0}}},\n", + " 'nic_status': 0}},\n", + " 'SERVICES': {1: {'health_status': 0,\n", + " 'operating_status': 0}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'HOST1': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0,\n", + " 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0,\n", + " 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0,\n", + " 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0,\n", + " 'outbound': 0}}},\n", + " 'nic_status': 0}},\n", + " 'SERVICES': {1: {'health_status': 0,\n", + " 'operating_status': 1}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'HOST2': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0,\n", + " 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0,\n", + " 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0,\n", + " 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0,\n", + " 'outbound': 0}}},\n", + " 'nic_status': 0}},\n", + " 'SERVICES': {1: {'health_status': 0,\n", + " 'operating_status': 0}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'HOST3': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0,\n", + " 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0,\n", + " 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0,\n", + " 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0,\n", + " 'outbound': 0}}},\n", + " 'nic_status': 0}},\n", + " 'SERVICES': {1: {'health_status': 0,\n", + " 'operating_status': 0}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'HOST4': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0,\n", + " 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0,\n", + " 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0,\n", + " 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0,\n", + " 'outbound': 0}}},\n", + " 'nic_status': 1}},\n", + " 'SERVICES': {1: {'health_status': 0,\n", + " 'operating_status': 0}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'HOST5': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0,\n", + " 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0,\n", + " 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0,\n", + " 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0,\n", + " 'outbound': 0}}},\n", + " 'nic_status': 0}},\n", + " 'SERVICES': {1: {'health_status': 0,\n", + " 'operating_status': 0}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'HOST6': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0,\n", + " 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0,\n", + " 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0,\n", + " 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0,\n", + " 'outbound': 0}}},\n", + " 'nic_status': 0}},\n", + " 'SERVICES': {1: {'health_status': 0,\n", + " 'operating_status': 0}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'ROUTER0': {'ACL': {1: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 0,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 2: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 1,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 3: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 2,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 4: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 3,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 5: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 4,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 6: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 5,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 7: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 6,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 8: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 7,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 9: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 8,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 10: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 9,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0}},\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}}}}\n" + ] + } + ], "source": [ "# create the env\n", "with open(data_manipulation_config_path(), 'r') as f:\n", @@ -450,7 +708,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -468,9 +726,51 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "step: 1, Red action: DO NOTHING, Blue reward:0.65\n", + "step: 2, Red action: DO NOTHING, Blue reward:0.65\n", + "step: 3, Red action: DO NOTHING, Blue reward:0.65\n", + "step: 4, Red action: DO NOTHING, Blue reward:0.90\n", + "step: 5, Red action: DO NOTHING, Blue reward:0.90\n", + "step: 6, Red action: DO NOTHING, Blue reward:0.90\n", + "step: 7, Red action: DO NOTHING, Blue reward:0.90\n", + "step: 8, Red action: DO NOTHING, Blue reward:0.90\n", + "step: 9, Red action: DO NOTHING, Blue reward:0.90\n", + "step: 10, Red action: DO NOTHING, Blue reward:0.90\n", + "step: 11, Red action: DO NOTHING, Blue reward:0.90\n", + "step: 12, Red action: DO NOTHING, Blue reward:0.90\n", + "step: 13, Red action: DO NOTHING, Blue reward:0.90\n", + "step: 14, Red action: DO NOTHING, Blue reward:0.90\n", + "step: 15, Red action: DO NOTHING, Blue reward:0.90\n", + "step: 16, Red action: DO NOTHING, Blue reward:0.90\n", + "step: 17, Red action: DO NOTHING, Blue reward:0.95\n", + "step: 18, Red action: DO NOTHING, Blue reward:0.95\n", + "step: 19, Red action: DO NOTHING, Blue reward:0.95\n", + "step: 20, Red action: DO NOTHING, Blue reward:0.95\n", + "step: 21, Red action: DO NOTHING, Blue reward:0.95\n", + "step: 22, Red action: DO NOTHING, Blue reward:0.95\n", + "step: 23, Red action: DO NOTHING, Blue reward:0.95\n", + "step: 24, Red action: DO NOTHING, Blue reward:0.95\n", + "step: 25, Red action: DO NOTHING, Blue reward:0.95\n", + "step: 26, Red action: ATTACK from client 2, Blue reward:0.15\n", + "step: 27, Red action: DO NOTHING, Blue reward:-0.35\n", + "step: 28, Red action: DO NOTHING, Blue reward:-0.85\n", + "step: 29, Red action: DO NOTHING, Blue reward:-0.85\n", + "step: 30, Red action: DO NOTHING, Blue reward:-0.85\n", + "step: 31, Red action: DO NOTHING, Blue reward:-0.85\n", + "step: 32, Red action: DO NOTHING, Blue reward:-0.85\n", + "step: 33, Red action: DO NOTHING, Blue reward:-0.85\n", + "step: 34, Red action: DO NOTHING, Blue reward:-0.85\n", + "step: 35, Red action: DO NOTHING, Blue reward:-0.85\n" + ] + } + ], "source": [ "for step in range(35):\n", " obs, reward, terminated, truncated, info = env.step(0)\n", @@ -486,9 +786,198 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'HOST0': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 0}},\n", + " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'HOST1': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 0}},\n", + " 'SERVICES': {1: {'health_status': 0, 'operating_status': 1}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'HOST2': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 0}},\n", + " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'HOST3': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 0}},\n", + " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'HOST4': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 1}},\n", + " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'HOST5': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 0}},\n", + " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'HOST6': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 0}},\n", + " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'ROUTER0': {'ACL': {1: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 0,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 2: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 1,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 3: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 2,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 4: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 3,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 5: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 4,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 6: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 5,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 7: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 6,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 8: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 7,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 9: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 8,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 10: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 9,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0}},\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}}}\n" + ] + } + ], "source": [ "pprint(obs['NODES'])" ] @@ -502,9 +991,198 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'HOST0': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 0}},\n", + " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'HOST1': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 0}},\n", + " 'SERVICES': {1: {'health_status': 3, 'operating_status': 1}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'HOST2': {'FOLDERS': {1: {'FILES': {1: {'health_status': 2}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 0}},\n", + " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'HOST3': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 0}},\n", + " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'HOST4': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 1}},\n", + " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'HOST5': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 0}},\n", + " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'HOST6': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", + " 'health_status': 0}},\n", + " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 1},\n", + " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", + " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", + " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", + " 'nic_status': 0}},\n", + " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", + " 'operating_status': 1,\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", + " 'ROUTER0': {'ACL': {1: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 0,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 2: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 1,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 3: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 2,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 4: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 3,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 5: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 4,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 6: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 5,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 7: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 6,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 8: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 7,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 9: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 8,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0},\n", + " 10: {'dest_ip_id': 0,\n", + " 'dest_port_id': 0,\n", + " 'dest_wildcard_id': 0,\n", + " 'permission': 0,\n", + " 'position': 9,\n", + " 'protocol_id': 0,\n", + " 'source_ip_id': 0,\n", + " 'source_port_id': 0,\n", + " 'source_wildcard_id': 0}},\n", + " 'users': {'local_login': 0, 'remote_sessions': 0}}}\n" + ] + } + ], "source": [ "obs, reward, terminated, truncated, info = env.step(9) # scan database file\n", "obs, reward, terminated, truncated, info = env.step(1) # scan webapp service\n", @@ -536,9 +1214,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "step: 38\n", + "Red action: do-nothing\n", + "Green action: do-nothing\n", + "Green action: node-application-execute\n", + "Blue reward:-0.8500000000000001\n" + ] + } + ], "source": [ "obs, reward, terminated, truncated, info = env.step(13) # patch the database\n", "print(f\"step: {env.game.step_counter}\")\n", @@ -561,9 +1251,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "step: 39\n", + "Red action: do-nothing\n", + "Green action: timestep=38 action='node-application-execute' parameters={'node_name': 'client_2', 'application_name': 'web-browser'} request=['network', 'node', 'client_2', 'application', 'web-browser', 'execute'] response=RequestResponse(status='failure', data={}) reward=-0.2 reward_info={'connection_attempt_status': 'n/a'}\n", + "Green action: timestep=38 action='node-application-execute' parameters={'node_name': 'client_1', 'application_name': 'web-browser'} request=['network', 'node', 'client_1', 'application', 'web-browser', 'execute'] response=RequestResponse(status='failure', data={}) reward=-0.25 reward_info={'connection_attempt_status': 'n/a'}\n", + "Blue reward:-0.05\n" + ] + } + ], "source": [ "obs, reward, terminated, truncated, info = env.step(0) # do nothing\n", "print(f\"step: {env.game.step_counter}\")\n", @@ -586,7 +1288,3796 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "step: 40, Red action: do-nothing, Blue reward:-0.05\n", + "step: 41, Red action: do-nothing, Blue reward:-0.05\n", + "step: 42, Red action: do-nothing, Blue reward:-0.05\n", + "step: 43, Red action: do-nothing, Blue reward:0.95\n", + "step: 44, Red action: do-nothing, Blue reward:0.95\n", + "step: 45, Red action: do-nothing, Blue reward:0.95\n", + "step: 46, Red action: do-nothing, Blue reward:0.95\n", + "step: 47, Red action: do-nothing, Blue reward:0.95\n", + "step: 48, Red action: do-nothing, Blue reward:0.95\n", + "step: 49, Red action: do-nothing, Blue reward:0.95\n", + "step: 50, Red action: do-nothing, Blue reward:0.95\n", + "step: 51, Red action: node-application-execute, Blue reward:0.95\n", + "step: 52, Red action: do-nothing, Blue reward:0.95\n", + "step: 53, Red action: do-nothing, Blue reward:0.85\n", + "step: 54, Red action: do-nothing, Blue reward:0.85\n", + "step: 55, Red action: do-nothing, Blue reward:0.85\n", + "step: 56, Red action: do-nothing, Blue reward:0.85\n", + "step: 57, Red action: do-nothing, Blue reward:0.85\n", + "step: 58, Red action: do-nothing, Blue reward:0.85\n", + "step: 59, Red action: do-nothing, Blue reward:0.85\n", + "step: 60, Red action: do-nothing, Blue reward:0.85\n", + "step: 61, Red action: do-nothing, Blue reward:0.85\n", + "step: 62, Red action: do-nothing, Blue reward:0.85\n", + "step: 63, Red action: do-nothing, Blue reward:0.85\n", + "step: 64, Red action: do-nothing, Blue reward:0.85\n", + "step: 65, Red action: do-nothing, Blue reward:0.85\n", + "step: 66, Red action: do-nothing, Blue reward:0.85\n", + "step: 67, Red action: do-nothing, Blue reward:0.85\n", + "step: 68, Red action: do-nothing, Blue reward:0.85\n", + "step: 69, Red action: do-nothing, Blue reward:0.85\n", + "step: 70, Red action: do-nothing, Blue reward:0.85\n", + "step: 71, Red action: do-nothing, Blue reward:0.85\n", + "step: 72, Red action: do-nothing, Blue reward:0.85\n", + "step: 73, Red action: do-nothing, Blue reward:0.85\n", + "step: 74, Red action: do-nothing, Blue reward:0.85\n", + "step: 75, Red action: node-application-execute, Blue reward:0.85\n", + "step: 76, Red action: do-nothing, Blue reward:0.85\n", + "step: 77, Red action: do-nothing, Blue reward:0.85\n", + "step: 78, Red action: do-nothing, Blue reward:0.85\n", + "step: 79, Red action: do-nothing, Blue reward:0.85\n", + "step: 80, Red action: do-nothing, Blue reward:0.85\n", + "step: 81, Red action: do-nothing, Blue reward:0.85\n", + "step: 82, Red action: do-nothing, Blue reward:0.85\n", + "step: 83, Red action: do-nothing, Blue reward:0.85\n", + "step: 84, Red action: do-nothing, Blue reward:0.85\n", + "step: 85, Red action: do-nothing, Blue reward:0.85\n", + "step: 86, Red action: do-nothing, Blue reward:0.85\n", + "step: 87, Red action: do-nothing, Blue reward:0.85\n", + "step: 88, Red action: do-nothing, Blue reward:0.85\n", + "step: 89, Red action: do-nothing, Blue reward:0.85\n", + "step: 90, Red action: do-nothing, Blue reward:0.85\n", + "step: 91, Red action: do-nothing, Blue reward:0.85\n", + "step: 92, Red action: do-nothing, Blue reward:0.85\n", + "step: 93, Red action: do-nothing, Blue reward:0.85\n", + "step: 94, Red action: do-nothing, Blue reward:0.85\n", + "step: 95, Red action: do-nothing, Blue reward:0.85\n", + "step: 96, Red action: do-nothing, Blue reward:0.85\n", + "step: 97, Red action: node-application-execute, Blue reward:0.85\n", + "step: 98, Red action: do-nothing, Blue reward:0.85\n", + "step: 99, Red action: do-nothing, Blue reward:0.85\n", + "step: 100, Red action: do-nothing, Blue reward:0.85\n", + "step: 101, Red action: do-nothing, Blue reward:0.85\n", + "step: 102, Red action: do-nothing, Blue reward:0.85\n", + "step: 103, Red action: do-nothing, Blue reward:0.85\n", + "step: 104, Red action: do-nothing, Blue reward:0.85\n", + "step: 105, Red action: do-nothing, Blue reward:0.85\n", + "step: 106, Red action: do-nothing, Blue reward:0.85\n", + "step: 107, Red action: do-nothing, Blue reward:0.85\n", + "step: 108, Red action: do-nothing, Blue reward:0.85\n", + "step: 109, Red action: do-nothing, Blue reward:0.85\n", + "step: 110, Red action: do-nothing, Blue reward:0.85\n", + "step: 111, Red action: do-nothing, Blue reward:0.85\n", + "step: 112, Red action: do-nothing, Blue reward:0.85\n", + "step: 113, Red action: node-application-execute, Blue reward:0.85\n", + "step: 114, Red action: do-nothing, Blue reward:0.85\n", + "step: 115, Red action: do-nothing, Blue reward:0.85\n", + "step: 116, Red action: do-nothing, Blue reward:0.85\n", + "step: 117, Red action: do-nothing, Blue reward:0.85\n", + "step: 118, Red action: do-nothing, Blue reward:0.85\n", + "step: 119, Red action: do-nothing, Blue reward:0.85\n", + "step: 120, Red action: do-nothing, Blue reward:0.85\n", + "step: 121, Red action: do-nothing, Blue reward:0.85\n", + "step: 122, Red action: do-nothing, Blue reward:0.85\n", + "step: 123, Red action: do-nothing, Blue reward:0.85\n", + "step: 124, Red action: do-nothing, Blue reward:0.85\n", + "step: 125, Red action: do-nothing, Blue reward:0.85\n", + "step: 126, Red action: do-nothing, Blue reward:0.85\n", + "step: 127, Red action: do-nothing, Blue reward:0.85\n", + "step: 128, Red action: do-nothing, Blue reward:0.85\n", + "step: 129, Red action: do-nothing, Blue reward:0.85\n", + "step: 130, Red action: do-nothing, Blue reward:0.85\n", + "step: 131, Red action: do-nothing, Blue reward:0.85\n", + "step: 132, Red action: node-application-execute, Blue reward:0.85\n", + "step: 133, Red action: do-nothing, Blue reward:0.85\n", + "step: 134, Red action: do-nothing, Blue reward:0.85\n", + "step: 135, Red action: do-nothing, Blue reward:0.85\n", + "step: 136, Red action: do-nothing, Blue reward:0.85\n", + "step: 137, Red action: do-nothing, Blue reward:0.85\n", + "step: 138, Red action: do-nothing, Blue reward:0.85\n", + "step: 139, Red action: do-nothing, Blue reward:0.85\n", + "step: 140, Red action: do-nothing, Blue reward:0.85\n", + "step: 141, Red action: do-nothing, Blue reward:0.85\n", + "step: 142, Red action: do-nothing, Blue reward:0.85\n", + "step: 143, Red action: do-nothing, Blue reward:0.85\n", + "step: 144, Red action: do-nothing, Blue reward:0.85\n", + "step: 145, Red action: do-nothing, Blue reward:0.85\n", + "step: 146, Red action: do-nothing, Blue reward:0.85\n", + "step: 147, Red action: do-nothing, Blue reward:0.85\n", + "step: 148, Red action: do-nothing, Blue reward:0.85\n", + "step: 149, Red action: do-nothing, Blue reward:0.85\n", + "step: 150, Red action: do-nothing, Blue reward:0.85\n", + "step: 151, Red action: do-nothing, Blue reward:0.85\n", + "step: 152, Red action: do-nothing, Blue reward:0.85\n", + "step: 153, Red action: do-nothing, Blue reward:0.85\n", + "step: 154, Red action: do-nothing, Blue reward:0.85\n", + "step: 155, Red action: do-nothing, Blue reward:0.85\n", + "step: 156, Red action: do-nothing, Blue reward:0.85\n", + "step: 157, Red action: node-application-execute, Blue reward:0.85\n", + "step: 158, Red action: do-nothing, Blue reward:0.85\n", + "step: 159, Red action: do-nothing, Blue reward:0.85\n", + "step: 160, Red action: do-nothing, Blue reward:0.85\n", + "step: 161, Red action: do-nothing, Blue reward:0.85\n", + "step: 162, Red action: do-nothing, Blue reward:0.85\n", + "step: 163, Red action: do-nothing, Blue reward:0.85\n", + "step: 164, Red action: do-nothing, Blue reward:0.85\n", + "step: 165, Red action: do-nothing, Blue reward:0.85\n", + "step: 166, Red action: do-nothing, Blue reward:0.85\n", + "step: 167, Red action: do-nothing, Blue reward:0.85\n", + "step: 168, Red action: do-nothing, Blue reward:0.85\n", + "step: 169, Red action: do-nothing, Blue reward:0.85\n", + "step: 170, Red action: do-nothing, Blue reward:0.85\n", + "step: 171, Red action: do-nothing, Blue reward:0.85\n", + "step: 172, Red action: do-nothing, Blue reward:0.85\n", + "step: 173, Red action: do-nothing, Blue reward:0.85\n", + "step: 174, Red action: do-nothing, Blue reward:0.85\n", + "step: 175, Red action: do-nothing, Blue reward:0.85\n", + "step: 176, Red action: do-nothing, Blue reward:0.85\n", + "step: 177, Red action: do-nothing, Blue reward:0.85\n", + "step: 178, Red action: node-application-execute, Blue reward:0.85\n", + "step: 179, Red action: do-nothing, Blue reward:0.85\n", + "step: 180, Red action: do-nothing, Blue reward:0.85\n", + "step: 181, Red action: do-nothing, Blue reward:0.85\n", + "step: 182, Red action: do-nothing, Blue reward:0.85\n", + "step: 183, Red action: do-nothing, Blue reward:0.85\n", + "step: 184, Red action: do-nothing, Blue reward:0.85\n", + "step: 185, Red action: do-nothing, Blue reward:0.85\n", + "step: 186, Red action: do-nothing, Blue reward:0.85\n", + "step: 187, Red action: do-nothing, Blue reward:0.85\n", + "step: 188, Red action: do-nothing, Blue reward:0.85\n", + "step: 189, Red action: do-nothing, Blue reward:0.85\n", + "step: 190, Red action: do-nothing, Blue reward:0.85\n", + "step: 191, Red action: do-nothing, Blue reward:0.85\n", + "step: 192, Red action: do-nothing, Blue reward:0.85\n", + "step: 193, Red action: do-nothing, Blue reward:0.85\n", + "step: 194, Red action: do-nothing, Blue reward:0.85\n", + "step: 195, Red action: do-nothing, Blue reward:0.85\n", + "step: 196, Red action: do-nothing, Blue reward:0.85\n", + "step: 197, Red action: do-nothing, Blue reward:0.85\n", + "step: 198, Red action: node-application-execute, Blue reward:0.85\n", + "step: 199, Red action: do-nothing, Blue reward:0.85\n", + "step: 200, Red action: do-nothing, Blue reward:0.85\n", + "step: 201, Red action: do-nothing, Blue reward:0.85\n", + "step: 202, Red action: do-nothing, Blue reward:0.85\n", + "step: 203, Red action: do-nothing, Blue reward:0.85\n", + "step: 204, Red action: do-nothing, Blue reward:0.85\n", + "step: 205, Red action: do-nothing, Blue reward:0.85\n", + "step: 206, Red action: do-nothing, Blue reward:0.85\n", + "step: 207, Red action: do-nothing, Blue reward:0.85\n", + "step: 208, Red action: do-nothing, Blue reward:0.85\n", + "step: 209, Red action: do-nothing, Blue reward:0.85\n", + "step: 210, Red action: do-nothing, Blue reward:0.85\n", + "step: 211, Red action: do-nothing, Blue reward:0.85\n", + "step: 212, Red action: do-nothing, Blue reward:0.85\n", + "step: 213, Red action: do-nothing, Blue reward:0.85\n", + "step: 214, Red action: do-nothing, Blue reward:0.85\n", + "step: 215, Red action: do-nothing, Blue reward:0.85\n", + "step: 216, Red action: do-nothing, Blue reward:0.85\n", + "step: 217, Red action: do-nothing, Blue reward:0.85\n", + "step: 218, Red action: node-application-execute, Blue reward:0.85\n", + "step: 219, Red action: do-nothing, Blue reward:0.85\n", + "step: 220, Red action: do-nothing, Blue reward:0.85\n", + "step: 221, Red action: do-nothing, Blue reward:0.85\n", + "step: 222, Red action: do-nothing, Blue reward:0.85\n", + "step: 223, Red action: do-nothing, Blue reward:0.85\n", + "step: 224, Red action: do-nothing, Blue reward:0.85\n", + "step: 225, Red action: do-nothing, Blue reward:0.85\n", + "step: 226, Red action: do-nothing, Blue reward:0.85\n", + "step: 227, Red action: do-nothing, Blue reward:0.85\n", + "step: 228, Red action: do-nothing, Blue reward:0.85\n", + "step: 229, Red action: do-nothing, Blue reward:0.85\n", + "step: 230, Red action: do-nothing, Blue reward:0.85\n", + "step: 231, Red action: do-nothing, Blue reward:0.85\n", + "step: 232, Red action: do-nothing, Blue reward:0.85\n", + "step: 233, Red action: do-nothing, Blue reward:0.85\n", + "step: 234, Red action: do-nothing, Blue reward:0.85\n", + "step: 235, Red action: do-nothing, Blue reward:0.85\n", + "step: 236, Red action: do-nothing, Blue reward:0.85\n", + "step: 237, Red action: do-nothing, Blue reward:0.85\n", + "step: 238, Red action: do-nothing, Blue reward:0.85\n", + "step: 239, Red action: do-nothing, Blue reward:0.85\n", + "step: 240, Red action: do-nothing, Blue reward:0.85\n", + "step: 241, Red action: node-application-execute, Blue reward:0.85\n", + "step: 242, Red action: do-nothing, Blue reward:0.85\n", + "step: 243, Red action: do-nothing, Blue reward:0.85\n", + "step: 244, Red action: do-nothing, Blue reward:0.85\n", + "step: 245, Red action: do-nothing, Blue reward:0.85\n", + "step: 246, Red action: do-nothing, Blue reward:0.85\n", + "step: 247, Red action: do-nothing, Blue reward:0.85\n", + "step: 248, Red action: do-nothing, Blue reward:0.85\n", + "step: 249, Red action: do-nothing, Blue reward:0.85\n", + "step: 250, Red action: do-nothing, Blue reward:0.85\n", + "step: 251, Red action: do-nothing, Blue reward:0.85\n", + "step: 252, Red action: do-nothing, Blue reward:0.85\n", + "step: 253, Red action: do-nothing, Blue reward:0.85\n", + "step: 254, Red action: do-nothing, Blue reward:0.85\n", + "step: 255, Red action: do-nothing, Blue reward:0.85\n", + "step: 256, Red action: do-nothing, Blue reward:0.85\n", + "step: 257, Red action: do-nothing, Blue reward:0.85\n", + "step: 258, Red action: do-nothing, Blue reward:0.85\n", + "step: 259, Red action: do-nothing, Blue reward:0.85\n", + "step: 260, Red action: do-nothing, Blue reward:0.85\n", + "step: 261, Red action: node-application-execute, Blue reward:0.85\n", + "step: 262, Red action: do-nothing, Blue reward:0.85\n", + "step: 263, Red action: do-nothing, Blue reward:0.85\n", + "step: 264, Red action: do-nothing, Blue reward:0.85\n", + "step: 265, Red action: do-nothing, Blue reward:0.85\n", + "step: 266, Red action: do-nothing, Blue reward:0.85\n", + "step: 267, Red action: do-nothing, Blue reward:0.85\n", + "step: 268, Red action: do-nothing, Blue reward:0.85\n", + "step: 269, Red action: do-nothing, Blue reward:0.85\n", + "step: 270, Red action: do-nothing, Blue reward:0.85\n", + "step: 271, Red action: do-nothing, Blue reward:0.85\n", + "step: 272, Red action: do-nothing, Blue reward:0.85\n", + "step: 273, Red action: do-nothing, Blue reward:0.85\n", + "step: 274, Red action: do-nothing, Blue reward:0.85\n", + "step: 275, Red action: do-nothing, Blue reward:0.85\n", + "step: 276, Red action: do-nothing, Blue reward:0.85\n", + "step: 277, Red action: do-nothing, Blue reward:0.85\n", + "step: 278, Red action: do-nothing, Blue reward:0.85\n", + "step: 279, Red action: do-nothing, Blue reward:0.85\n", + "step: 280, Red action: do-nothing, Blue reward:0.85\n", + "step: 281, Red action: node-application-execute, Blue reward:0.85\n", + "step: 282, Red action: do-nothing, Blue reward:0.85\n", + "step: 283, Red action: do-nothing, Blue reward:0.85\n", + "step: 284, Red action: do-nothing, Blue reward:0.85\n", + "step: 285, Red action: do-nothing, Blue reward:0.85\n", + "step: 286, Red action: do-nothing, Blue reward:0.85\n", + "step: 287, Red action: do-nothing, Blue reward:0.85\n", + "step: 288, Red action: do-nothing, Blue reward:0.85\n", + "step: 289, Red action: do-nothing, Blue reward:0.85\n", + "step: 290, Red action: do-nothing, Blue reward:0.85\n", + "step: 291, Red action: do-nothing, Blue reward:0.85\n", + "step: 292, Red action: do-nothing, Blue reward:0.85\n", + "step: 293, Red action: do-nothing, Blue reward:0.85\n", + "step: 294, Red action: do-nothing, Blue reward:0.85\n", + "step: 295, Red action: do-nothing, Blue reward:0.85\n", + "step: 296, Red action: do-nothing, Blue reward:0.85\n", + "step: 297, Red action: do-nothing, Blue reward:0.85\n", + "step: 298, Red action: do-nothing, Blue reward:0.85\n", + "step: 299, Red action: do-nothing, Blue reward:0.85\n", + "step: 300, Red action: do-nothing, Blue reward:0.85\n", + "step: 301, Red action: do-nothing, Blue reward:0.85\n", + "step: 302, Red action: do-nothing, Blue reward:0.85\n", + "step: 303, Red action: do-nothing, Blue reward:0.85\n", + "step: 304, Red action: node-application-execute, Blue reward:0.85\n", + "step: 305, Red action: do-nothing, Blue reward:0.85\n", + "step: 306, Red action: do-nothing, Blue reward:0.85\n", + "step: 307, Red action: do-nothing, Blue reward:0.85\n", + "step: 308, Red action: do-nothing, Blue reward:0.85\n", + "step: 309, Red action: do-nothing, Blue reward:0.85\n", + "step: 310, Red action: do-nothing, Blue reward:0.85\n", + "step: 311, Red action: do-nothing, Blue reward:0.85\n", + "step: 312, Red action: do-nothing, Blue reward:0.85\n", + "step: 313, Red action: do-nothing, Blue reward:0.85\n", + "step: 314, Red action: do-nothing, Blue reward:0.85\n", + "step: 315, Red action: do-nothing, Blue reward:0.85\n", + "step: 316, Red action: do-nothing, Blue reward:0.85\n", + "step: 317, Red action: do-nothing, Blue reward:0.85\n", + "step: 318, Red action: do-nothing, Blue reward:0.85\n", + "step: 319, Red action: do-nothing, Blue reward:0.85\n", + "step: 320, Red action: do-nothing, Blue reward:0.85\n", + "step: 321, Red action: do-nothing, Blue reward:0.85\n", + "step: 322, Red action: do-nothing, Blue reward:0.85\n", + "step: 323, Red action: do-nothing, Blue reward:0.85\n", + "step: 324, Red action: do-nothing, Blue reward:0.85\n", + "step: 325, Red action: node-application-execute, Blue reward:0.85\n", + "step: 326, Red action: do-nothing, Blue reward:0.85\n", + "step: 327, Red action: do-nothing, Blue reward:0.85\n", + "step: 328, Red action: do-nothing, Blue reward:0.85\n", + "step: 329, Red action: do-nothing, Blue reward:0.85\n", + "step: 330, Red action: do-nothing, Blue reward:0.85\n", + "step: 331, Red action: do-nothing, Blue reward:0.85\n", + "step: 332, Red action: do-nothing, Blue reward:0.85\n", + "step: 333, Red action: do-nothing, Blue reward:0.85\n", + "step: 334, Red action: do-nothing, Blue reward:0.85\n", + "step: 335, Red action: do-nothing, Blue reward:0.85\n", + "step: 336, Red action: do-nothing, Blue reward:0.85\n", + "step: 337, Red action: do-nothing, Blue reward:0.85\n", + "step: 338, Red action: do-nothing, Blue reward:0.85\n", + "step: 339, Red action: do-nothing, Blue reward:0.85\n", + "step: 340, Red action: do-nothing, Blue reward:0.85\n", + "step: 341, Red action: do-nothing, Blue reward:0.85\n", + "step: 342, Red action: do-nothing, Blue reward:0.85\n", + "step: 343, Red action: do-nothing, Blue reward:0.85\n", + "step: 344, Red action: do-nothing, Blue reward:0.85\n", + "step: 345, Red action: do-nothing, Blue reward:0.85\n", + "step: 346, Red action: do-nothing, Blue reward:0.85\n", + "step: 347, Red action: do-nothing, Blue reward:0.85\n", + "step: 348, Red action: do-nothing, Blue reward:0.85\n", + "step: 349, Red action: node-application-execute, Blue reward:0.85\n", + "step: 350, Red action: do-nothing, Blue reward:0.85\n", + "step: 351, Red action: do-nothing, Blue reward:0.85\n", + "step: 352, Red action: do-nothing, Blue reward:0.85\n", + "step: 353, Red action: do-nothing, Blue reward:0.85\n", + "step: 354, Red action: do-nothing, Blue reward:0.85\n", + "step: 355, Red action: do-nothing, Blue reward:0.85\n", + "step: 356, Red action: do-nothing, Blue reward:0.85\n", + "step: 357, Red action: do-nothing, Blue reward:0.85\n", + "step: 358, Red action: do-nothing, Blue reward:0.85\n", + "step: 359, Red action: do-nothing, Blue reward:0.85\n", + "step: 360, Red action: do-nothing, Blue reward:0.85\n", + "step: 361, Red action: do-nothing, Blue reward:0.85\n", + "step: 362, Red action: do-nothing, Blue reward:0.85\n", + "step: 363, Red action: do-nothing, Blue reward:0.85\n", + "step: 364, Red action: do-nothing, Blue reward:0.85\n", + "step: 365, Red action: do-nothing, Blue reward:0.85\n", + "step: 366, Red action: do-nothing, Blue reward:0.85\n", + "step: 367, Red action: do-nothing, Blue reward:0.85\n", + "step: 368, Red action: do-nothing, Blue reward:0.85\n", + "step: 369, Red action: node-application-execute, Blue reward:0.85\n", + "step: 370, Red action: do-nothing, Blue reward:0.85\n", + "step: 371, Red action: do-nothing, Blue reward:0.85\n", + "step: 372, Red action: do-nothing, Blue reward:0.85\n", + "step: 373, Red action: do-nothing, Blue reward:0.85\n", + "step: 374, Red action: do-nothing, Blue reward:0.85\n", + "step: 375, Red action: do-nothing, Blue reward:0.85\n", + "step: 376, Red action: do-nothing, Blue reward:0.85\n", + "step: 377, Red action: do-nothing, Blue reward:0.85\n", + "step: 378, Red action: do-nothing, Blue reward:0.85\n", + "step: 379, Red action: do-nothing, Blue reward:0.85\n", + "step: 380, Red action: do-nothing, Blue reward:0.85\n", + "step: 381, Red action: do-nothing, Blue reward:0.85\n", + "step: 382, Red action: do-nothing, Blue reward:0.85\n", + "step: 383, Red action: do-nothing, Blue reward:0.85\n", + "step: 384, Red action: do-nothing, Blue reward:0.85\n", + "step: 385, Red action: do-nothing, Blue reward:0.85\n", + "step: 386, Red action: do-nothing, Blue reward:0.85\n", + "step: 387, Red action: do-nothing, Blue reward:0.85\n", + "step: 388, Red action: do-nothing, Blue reward:0.85\n", + "step: 389, Red action: do-nothing, Blue reward:0.85\n", + "step: 390, Red action: do-nothing, Blue reward:0.85\n", + "step: 391, Red action: do-nothing, Blue reward:0.85\n", + "step: 392, Red action: do-nothing, Blue reward:0.85\n", + "step: 393, Red action: do-nothing, Blue reward:0.85\n", + "step: 394, Red action: node-application-execute, Blue reward:0.85\n", + "step: 395, Red action: do-nothing, Blue reward:0.85\n", + "step: 396, Red action: do-nothing, Blue reward:0.85\n", + "step: 397, Red action: do-nothing, Blue reward:0.85\n", + "step: 398, Red action: do-nothing, Blue reward:0.85\n", + "step: 399, Red action: do-nothing, Blue reward:0.85\n", + "step: 400, Red action: do-nothing, Blue reward:0.85\n", + "step: 401, Red action: do-nothing, Blue reward:0.85\n", + "step: 402, Red action: do-nothing, Blue reward:0.85\n", + "step: 403, Red action: do-nothing, Blue reward:0.85\n", + "step: 404, Red action: do-nothing, Blue reward:0.85\n", + "step: 405, Red action: do-nothing, Blue reward:0.85\n", + "step: 406, Red action: do-nothing, Blue reward:0.85\n", + "step: 407, Red action: do-nothing, Blue reward:0.85\n", + "step: 408, Red action: do-nothing, Blue reward:0.85\n", + "step: 409, Red action: do-nothing, Blue reward:0.85\n", + "step: 410, Red action: do-nothing, Blue reward:0.85\n", + "step: 411, Red action: do-nothing, Blue reward:0.85\n", + "step: 412, Red action: do-nothing, Blue reward:0.85\n", + "step: 413, Red action: do-nothing, Blue reward:0.85\n", + "step: 414, Red action: do-nothing, Blue reward:0.85\n", + "step: 415, Red action: do-nothing, Blue reward:0.85\n", + "step: 416, Red action: do-nothing, Blue reward:0.85\n", + "step: 417, Red action: do-nothing, Blue reward:0.85\n", + "step: 418, Red action: node-application-execute, Blue reward:0.85\n", + "step: 419, Red action: do-nothing, Blue reward:0.85\n", + "step: 420, Red action: do-nothing, Blue reward:0.85\n", + "step: 421, Red action: do-nothing, Blue reward:0.85\n", + "step: 422, Red action: do-nothing, Blue reward:0.85\n", + "step: 423, Red action: do-nothing, Blue reward:0.85\n", + "step: 424, Red action: do-nothing, Blue reward:0.85\n", + "step: 425, Red action: do-nothing, Blue reward:0.85\n", + "step: 426, Red action: do-nothing, Blue reward:0.85\n", + "step: 427, Red action: do-nothing, Blue reward:0.85\n", + "step: 428, Red action: do-nothing, Blue reward:0.85\n", + "step: 429, Red action: do-nothing, Blue reward:0.85\n", + "step: 430, Red action: do-nothing, Blue reward:0.85\n", + "step: 431, Red action: do-nothing, Blue reward:0.85\n", + "step: 432, Red action: do-nothing, Blue reward:0.85\n", + "step: 433, Red action: do-nothing, Blue reward:0.85\n", + "step: 434, Red action: do-nothing, Blue reward:0.85\n", + "step: 435, Red action: do-nothing, Blue reward:0.85\n", + "step: 436, Red action: do-nothing, Blue reward:0.85\n", + "step: 437, Red action: do-nothing, Blue reward:0.85\n", + "step: 438, Red action: do-nothing, Blue reward:0.85\n", + "step: 439, Red action: do-nothing, Blue reward:0.85\n", + "step: 440, Red action: do-nothing, Blue reward:0.85\n", + "step: 441, Red action: do-nothing, Blue reward:0.85\n", + "step: 442, Red action: do-nothing, Blue reward:0.85\n", + "step: 443, Red action: node-application-execute, Blue reward:0.85\n", + "step: 444, Red action: do-nothing, Blue reward:0.85\n", + "step: 445, Red action: do-nothing, Blue reward:0.85\n", + "step: 446, Red action: do-nothing, Blue reward:0.85\n", + "step: 447, Red action: do-nothing, Blue reward:0.85\n", + "step: 448, Red action: do-nothing, Blue reward:0.85\n", + "step: 449, Red action: do-nothing, Blue reward:0.85\n", + "step: 450, Red action: do-nothing, Blue reward:0.85\n", + "step: 451, Red action: do-nothing, Blue reward:0.85\n", + "step: 452, Red action: do-nothing, Blue reward:0.85\n", + "step: 453, Red action: do-nothing, Blue reward:0.85\n", + "step: 454, Red action: do-nothing, Blue reward:0.85\n", + "step: 455, Red action: do-nothing, Blue reward:0.85\n", + "step: 456, Red action: do-nothing, Blue reward:0.85\n", + "step: 457, Red action: do-nothing, Blue reward:0.85\n", + "step: 458, Red action: do-nothing, Blue reward:0.85\n", + "step: 459, Red action: do-nothing, Blue reward:0.85\n", + "step: 460, Red action: do-nothing, Blue reward:0.85\n", + "step: 461, Red action: node-application-execute, Blue reward:0.85\n", + "step: 462, Red action: do-nothing, Blue reward:0.85\n", + "step: 463, Red action: do-nothing, Blue reward:0.85\n", + "step: 464, Red action: do-nothing, Blue reward:0.85\n", + "step: 465, Red action: do-nothing, Blue reward:0.85\n", + "step: 466, Red action: do-nothing, Blue reward:0.85\n", + "step: 467, Red action: do-nothing, Blue reward:0.85\n", + "step: 468, Red action: do-nothing, Blue reward:0.85\n", + "step: 469, Red action: do-nothing, Blue reward:0.85\n", + "step: 470, Red action: do-nothing, Blue reward:0.85\n", + "step: 471, Red action: do-nothing, Blue reward:0.85\n", + "step: 472, Red action: do-nothing, Blue reward:0.85\n", + "step: 473, Red action: do-nothing, Blue reward:0.85\n", + "step: 474, Red action: do-nothing, Blue reward:0.85\n", + "step: 475, Red action: do-nothing, Blue reward:0.85\n", + "step: 476, Red action: do-nothing, Blue reward:0.85\n", + "step: 477, Red action: do-nothing, Blue reward:0.85\n", + "step: 478, Red action: do-nothing, Blue reward:0.85\n", + "step: 479, Red action: do-nothing, Blue reward:0.85\n", + "step: 480, Red action: node-application-execute, Blue reward:0.85\n", + "step: 481, Red action: do-nothing, Blue reward:0.85\n", + "step: 482, Red action: do-nothing, Blue reward:0.85\n", + "step: 483, Red action: do-nothing, Blue reward:0.85\n", + "step: 484, Red action: do-nothing, Blue reward:0.85\n", + "step: 485, Red action: do-nothing, Blue reward:0.85\n", + "step: 486, Red action: do-nothing, Blue reward:0.85\n", + "step: 487, Red action: do-nothing, Blue reward:0.85\n", + "step: 488, Red action: do-nothing, Blue reward:0.85\n", + "step: 489, Red action: do-nothing, Blue reward:0.85\n", + "step: 490, Red action: do-nothing, Blue reward:0.85\n", + "step: 491, Red action: do-nothing, Blue reward:0.85\n", + "step: 492, Red action: do-nothing, Blue reward:0.85\n", + "step: 493, Red action: do-nothing, Blue reward:0.85\n", + "step: 494, Red action: do-nothing, Blue reward:0.85\n", + "step: 495, Red action: do-nothing, Blue reward:0.85\n", + "step: 496, Red action: do-nothing, Blue reward:0.85\n", + "step: 497, Red action: do-nothing, Blue reward:0.85\n", + "step: 498, Red action: do-nothing, Blue reward:0.85\n", + "step: 499, Red action: do-nothing, Blue reward:0.85\n", + "step: 500, Red action: do-nothing, Blue reward:0.85\n", + "step: 501, Red action: do-nothing, Blue reward:0.85\n", + "step: 502, Red action: do-nothing, Blue reward:0.85\n", + "step: 503, Red action: do-nothing, Blue reward:0.85\n", + "step: 504, Red action: node-application-execute, Blue reward:0.85\n", + "step: 505, Red action: do-nothing, Blue reward:0.85\n", + "step: 506, Red action: do-nothing, Blue reward:0.85\n", + "step: 507, Red action: do-nothing, Blue reward:0.85\n", + "step: 508, Red action: do-nothing, Blue reward:0.85\n", + "step: 509, Red action: do-nothing, Blue reward:0.85\n", + "step: 510, Red action: do-nothing, Blue reward:0.85\n", + "step: 511, Red action: do-nothing, Blue reward:0.85\n", + "step: 512, Red action: do-nothing, Blue reward:0.85\n", + "step: 513, Red action: do-nothing, Blue reward:0.85\n", + "step: 514, Red action: do-nothing, Blue reward:0.85\n", + "step: 515, Red action: do-nothing, Blue reward:0.85\n", + "step: 516, Red action: do-nothing, Blue reward:0.85\n", + "step: 517, Red action: do-nothing, Blue reward:0.85\n", + "step: 518, Red action: do-nothing, Blue reward:0.85\n", + "step: 519, Red action: do-nothing, Blue reward:0.85\n", + "step: 520, Red action: do-nothing, Blue reward:0.85\n", + "step: 521, Red action: do-nothing, Blue reward:0.85\n", + "step: 522, Red action: do-nothing, Blue reward:0.85\n", + "step: 523, Red action: do-nothing, Blue reward:0.85\n", + "step: 524, Red action: do-nothing, Blue reward:0.85\n", + "step: 525, Red action: do-nothing, Blue reward:0.85\n", + "step: 526, Red action: do-nothing, Blue reward:0.85\n", + "step: 527, Red action: node-application-execute, Blue reward:0.85\n", + "step: 528, Red action: do-nothing, Blue reward:0.85\n", + "step: 529, Red action: do-nothing, Blue reward:0.85\n", + "step: 530, Red action: do-nothing, Blue reward:0.85\n", + "step: 531, Red action: do-nothing, Blue reward:0.85\n", + "step: 532, Red action: do-nothing, Blue reward:0.85\n", + "step: 533, Red action: do-nothing, Blue reward:0.85\n", + "step: 534, Red action: do-nothing, Blue reward:0.85\n", + "step: 535, Red action: do-nothing, Blue reward:0.85\n", + "step: 536, Red action: do-nothing, Blue reward:0.85\n", + "step: 537, Red action: do-nothing, Blue reward:0.85\n", + "step: 538, Red action: do-nothing, Blue reward:0.85\n", + "step: 539, Red action: do-nothing, Blue reward:0.85\n", + "step: 540, Red action: do-nothing, Blue reward:0.85\n", + "step: 541, Red action: do-nothing, Blue reward:0.85\n", + "step: 542, Red action: do-nothing, Blue reward:0.85\n", + "step: 543, Red action: do-nothing, Blue reward:0.85\n", + "step: 544, Red action: do-nothing, Blue reward:0.85\n", + "step: 545, Red action: do-nothing, Blue reward:0.85\n", + "step: 546, Red action: do-nothing, Blue reward:0.85\n", + "step: 547, Red action: do-nothing, Blue reward:0.85\n", + "step: 548, Red action: do-nothing, Blue reward:0.85\n", + "step: 549, Red action: node-application-execute, Blue reward:0.85\n", + "step: 550, Red action: do-nothing, Blue reward:0.85\n", + "step: 551, Red action: do-nothing, Blue reward:0.85\n", + "step: 552, Red action: do-nothing, Blue reward:0.85\n", + "step: 553, Red action: do-nothing, Blue reward:0.85\n", + "step: 554, Red action: do-nothing, Blue reward:0.85\n", + "step: 555, Red action: do-nothing, Blue reward:0.85\n", + "step: 556, Red action: do-nothing, Blue reward:0.85\n", + "step: 557, Red action: do-nothing, Blue reward:0.85\n", + "step: 558, Red action: do-nothing, Blue reward:0.85\n", + "step: 559, Red action: do-nothing, Blue reward:0.85\n", + "step: 560, Red action: do-nothing, Blue reward:0.85\n", + "step: 561, Red action: do-nothing, Blue reward:0.85\n", + "step: 562, Red action: do-nothing, Blue reward:0.85\n", + "step: 563, Red action: do-nothing, Blue reward:0.85\n", + "step: 564, Red action: do-nothing, Blue reward:0.85\n", + "step: 565, Red action: node-application-execute, Blue reward:0.85\n", + "step: 566, Red action: do-nothing, Blue reward:0.85\n", + "step: 567, Red action: do-nothing, Blue reward:0.85\n", + "step: 568, Red action: do-nothing, Blue reward:0.85\n", + "step: 569, Red action: do-nothing, Blue reward:0.85\n", + "step: 570, Red action: do-nothing, Blue reward:0.85\n", + "step: 571, Red action: do-nothing, Blue reward:0.85\n", + "step: 572, Red action: do-nothing, Blue reward:0.85\n", + "step: 573, Red action: do-nothing, Blue reward:0.85\n", + "step: 574, Red action: do-nothing, Blue reward:0.85\n", + "step: 575, Red action: do-nothing, Blue reward:0.85\n", + "step: 576, Red action: do-nothing, Blue reward:0.85\n", + "step: 577, Red action: do-nothing, Blue reward:0.85\n", + "step: 578, Red action: do-nothing, Blue reward:0.85\n", + "step: 579, Red action: do-nothing, Blue reward:0.85\n", + "step: 580, Red action: do-nothing, Blue reward:0.85\n", + "step: 581, Red action: do-nothing, Blue reward:0.85\n", + "step: 582, Red action: do-nothing, Blue reward:0.85\n", + "step: 583, Red action: do-nothing, Blue reward:0.85\n", + "step: 584, Red action: do-nothing, Blue reward:0.85\n", + "step: 585, Red action: do-nothing, Blue reward:0.85\n", + "step: 586, Red action: do-nothing, Blue reward:0.85\n", + "step: 587, Red action: do-nothing, Blue reward:0.85\n", + "step: 588, Red action: do-nothing, Blue reward:0.85\n", + "step: 589, Red action: node-application-execute, Blue reward:0.85\n", + "step: 590, Red action: do-nothing, Blue reward:0.85\n", + "step: 591, Red action: do-nothing, Blue reward:0.85\n", + "step: 592, Red action: do-nothing, Blue reward:0.85\n", + "step: 593, Red action: do-nothing, Blue reward:0.85\n", + "step: 594, Red action: do-nothing, Blue reward:0.85\n", + "step: 595, Red action: do-nothing, Blue reward:0.85\n", + "step: 596, Red action: do-nothing, Blue reward:0.85\n", + "step: 597, Red action: do-nothing, Blue reward:0.85\n", + "step: 598, Red action: do-nothing, Blue reward:0.85\n", + "step: 599, Red action: do-nothing, Blue reward:0.85\n", + "step: 600, Red action: do-nothing, Blue reward:0.85\n", + "step: 601, Red action: do-nothing, Blue reward:0.85\n", + "step: 602, Red action: do-nothing, Blue reward:0.85\n", + "step: 603, Red action: do-nothing, Blue reward:0.85\n", + "step: 604, Red action: do-nothing, Blue reward:0.85\n", + "step: 605, Red action: do-nothing, Blue reward:0.85\n", + "step: 606, Red action: do-nothing, Blue reward:0.85\n", + "step: 607, Red action: do-nothing, Blue reward:0.85\n", + "step: 608, Red action: do-nothing, Blue reward:0.85\n", + "step: 609, Red action: do-nothing, Blue reward:0.85\n", + "step: 610, Red action: node-application-execute, Blue reward:0.85\n", + "step: 611, Red action: do-nothing, Blue reward:0.85\n", + "step: 612, Red action: do-nothing, Blue reward:0.85\n", + "step: 613, Red action: do-nothing, Blue reward:0.85\n", + "step: 614, Red action: do-nothing, Blue reward:0.85\n", + "step: 615, Red action: do-nothing, Blue reward:0.85\n", + "step: 616, Red action: do-nothing, Blue reward:0.85\n", + "step: 617, Red action: do-nothing, Blue reward:0.85\n", + "step: 618, Red action: do-nothing, Blue reward:0.85\n", + "step: 619, Red action: do-nothing, Blue reward:0.85\n", + "step: 620, Red action: do-nothing, Blue reward:0.85\n", + "step: 621, Red action: do-nothing, Blue reward:0.85\n", + "step: 622, Red action: do-nothing, Blue reward:0.85\n", + "step: 623, Red action: do-nothing, Blue reward:0.85\n", + "step: 624, Red action: do-nothing, Blue reward:0.85\n", + "step: 625, Red action: do-nothing, Blue reward:0.85\n", + "step: 626, Red action: node-application-execute, Blue reward:0.85\n", + "step: 627, Red action: do-nothing, Blue reward:0.85\n", + "step: 628, Red action: do-nothing, Blue reward:0.85\n", + "step: 629, Red action: do-nothing, Blue reward:0.85\n", + "step: 630, Red action: do-nothing, Blue reward:0.85\n", + "step: 631, Red action: do-nothing, Blue reward:0.85\n", + "step: 632, Red action: do-nothing, Blue reward:0.85\n", + "step: 633, Red action: do-nothing, Blue reward:0.85\n", + "step: 634, Red action: do-nothing, Blue reward:0.85\n", + "step: 635, Red action: do-nothing, Blue reward:0.85\n", + "step: 636, Red action: do-nothing, Blue reward:0.85\n", + "step: 637, Red action: do-nothing, Blue reward:0.85\n", + "step: 638, Red action: do-nothing, Blue reward:0.85\n", + "step: 639, Red action: do-nothing, Blue reward:0.85\n", + "step: 640, Red action: do-nothing, Blue reward:0.85\n", + "step: 641, Red action: do-nothing, Blue reward:0.85\n", + "step: 642, Red action: do-nothing, Blue reward:0.85\n", + "step: 643, Red action: do-nothing, Blue reward:0.85\n", + "step: 644, Red action: node-application-execute, Blue reward:0.85\n", + "step: 645, Red action: do-nothing, Blue reward:0.85\n", + "step: 646, Red action: do-nothing, Blue reward:0.85\n", + "step: 647, Red action: do-nothing, Blue reward:0.85\n", + "step: 648, Red action: do-nothing, Blue reward:0.85\n", + "step: 649, Red action: do-nothing, Blue reward:0.85\n", + "step: 650, Red action: do-nothing, Blue reward:0.85\n", + "step: 651, Red action: do-nothing, Blue reward:0.85\n", + "step: 652, Red action: do-nothing, Blue reward:0.85\n", + "step: 653, Red action: do-nothing, Blue reward:0.85\n", + "step: 654, Red action: do-nothing, Blue reward:0.85\n", + "step: 655, Red action: do-nothing, Blue reward:0.85\n", + "step: 656, Red action: do-nothing, Blue reward:0.85\n", + "step: 657, Red action: do-nothing, Blue reward:0.85\n", + "step: 658, Red action: do-nothing, Blue reward:0.85\n", + "step: 659, Red action: do-nothing, Blue reward:0.85\n", + "step: 660, Red action: do-nothing, Blue reward:0.85\n", + "step: 661, Red action: do-nothing, Blue reward:0.85\n", + "step: 662, Red action: do-nothing, Blue reward:0.85\n", + "step: 663, Red action: do-nothing, Blue reward:0.85\n", + "step: 664, Red action: do-nothing, Blue reward:0.85\n", + "step: 665, Red action: do-nothing, Blue reward:0.85\n", + "step: 666, Red action: do-nothing, Blue reward:0.85\n", + "step: 667, Red action: node-application-execute, Blue reward:0.85\n", + "step: 668, Red action: do-nothing, Blue reward:0.85\n", + "step: 669, Red action: do-nothing, Blue reward:0.85\n", + "step: 670, Red action: do-nothing, Blue reward:0.85\n", + "step: 671, Red action: do-nothing, Blue reward:0.85\n", + "step: 672, Red action: do-nothing, Blue reward:0.85\n", + "step: 673, Red action: do-nothing, Blue reward:0.85\n", + "step: 674, Red action: do-nothing, Blue reward:0.85\n", + "step: 675, Red action: do-nothing, Blue reward:0.85\n", + "step: 676, Red action: do-nothing, Blue reward:0.85\n", + "step: 677, Red action: do-nothing, Blue reward:0.85\n", + "step: 678, Red action: do-nothing, Blue reward:0.85\n", + "step: 679, Red action: do-nothing, Blue reward:0.85\n", + "step: 680, Red action: do-nothing, Blue reward:0.85\n", + "step: 681, Red action: do-nothing, Blue reward:0.85\n", + "step: 682, Red action: do-nothing, Blue reward:0.85\n", + "step: 683, Red action: do-nothing, Blue reward:0.85\n", + "step: 684, Red action: do-nothing, Blue reward:0.85\n", + "step: 685, Red action: node-application-execute, Blue reward:0.85\n", + "step: 686, Red action: do-nothing, Blue reward:0.85\n", + "step: 687, Red action: do-nothing, Blue reward:0.85\n", + "step: 688, Red action: do-nothing, Blue reward:0.85\n", + "step: 689, Red action: do-nothing, Blue reward:0.85\n", + "step: 690, Red action: do-nothing, Blue reward:0.85\n", + "step: 691, Red action: do-nothing, Blue reward:0.85\n", + "step: 692, Red action: do-nothing, Blue reward:0.85\n", + "step: 693, Red action: do-nothing, Blue reward:0.85\n", + "step: 694, Red action: do-nothing, Blue reward:0.85\n", + "step: 695, Red action: do-nothing, Blue reward:0.85\n", + "step: 696, Red action: do-nothing, Blue reward:0.85\n", + "step: 697, Red action: do-nothing, Blue reward:0.85\n", + "step: 698, Red action: do-nothing, Blue reward:0.85\n", + "step: 699, Red action: do-nothing, Blue reward:0.85\n", + "step: 700, Red action: do-nothing, Blue reward:0.85\n", + "step: 701, Red action: node-application-execute, Blue reward:0.85\n", + "step: 702, Red action: do-nothing, Blue reward:0.85\n", + "step: 703, Red action: do-nothing, Blue reward:0.85\n", + "step: 704, Red action: do-nothing, Blue reward:0.85\n", + "step: 705, Red action: do-nothing, Blue reward:0.85\n", + "step: 706, Red action: do-nothing, Blue reward:0.85\n", + "step: 707, Red action: do-nothing, Blue reward:0.85\n", + "step: 708, Red action: do-nothing, Blue reward:0.85\n", + "step: 709, Red action: do-nothing, Blue reward:0.85\n", + "step: 710, Red action: do-nothing, Blue reward:0.85\n", + "step: 711, Red action: do-nothing, Blue reward:0.85\n", + "step: 712, Red action: do-nothing, Blue reward:0.85\n", + "step: 713, Red action: do-nothing, Blue reward:0.85\n", + "step: 714, Red action: do-nothing, Blue reward:0.85\n", + "step: 715, Red action: do-nothing, Blue reward:0.85\n", + "step: 716, Red action: do-nothing, Blue reward:0.85\n", + "step: 717, Red action: do-nothing, Blue reward:0.85\n", + "step: 718, Red action: node-application-execute, Blue reward:0.85\n", + "step: 719, Red action: do-nothing, Blue reward:0.85\n", + "step: 720, Red action: do-nothing, Blue reward:0.85\n", + "step: 721, Red action: do-nothing, Blue reward:0.85\n", + "step: 722, Red action: do-nothing, Blue reward:0.85\n", + "step: 723, Red action: do-nothing, Blue reward:0.85\n", + "step: 724, Red action: do-nothing, Blue reward:0.85\n", + "step: 725, Red action: do-nothing, Blue reward:0.85\n", + "step: 726, Red action: do-nothing, Blue reward:0.85\n", + "step: 727, Red action: do-nothing, Blue reward:0.85\n", + "step: 728, Red action: do-nothing, Blue reward:0.85\n", + "step: 729, Red action: do-nothing, Blue reward:0.85\n", + "step: 730, Red action: do-nothing, Blue reward:0.85\n", + "step: 731, Red action: do-nothing, Blue reward:0.85\n", + "step: 732, Red action: do-nothing, Blue reward:0.85\n", + "step: 733, Red action: do-nothing, Blue reward:0.85\n", + "step: 734, Red action: do-nothing, Blue reward:0.85\n", + "step: 735, Red action: do-nothing, Blue reward:0.85\n", + "step: 736, Red action: do-nothing, Blue reward:0.85\n", + "step: 737, Red action: do-nothing, Blue reward:0.85\n", + "step: 738, Red action: do-nothing, Blue reward:0.85\n", + "step: 739, Red action: do-nothing, Blue reward:0.85\n", + "step: 740, Red action: do-nothing, Blue reward:0.85\n", + "step: 741, Red action: do-nothing, Blue reward:0.85\n", + "step: 742, Red action: do-nothing, Blue reward:0.85\n", + "step: 743, Red action: node-application-execute, Blue reward:0.85\n", + "step: 744, Red action: do-nothing, Blue reward:0.85\n", + "step: 745, Red action: do-nothing, Blue reward:0.85\n", + "step: 746, Red action: do-nothing, Blue reward:0.85\n", + "step: 747, Red action: do-nothing, Blue reward:0.85\n", + "step: 748, Red action: do-nothing, Blue reward:0.85\n", + "step: 749, Red action: do-nothing, Blue reward:0.85\n", + "step: 750, Red action: do-nothing, Blue reward:0.85\n", + "step: 751, Red action: do-nothing, Blue reward:0.85\n", + "step: 752, Red action: do-nothing, Blue reward:0.85\n", + "step: 753, Red action: do-nothing, Blue reward:0.85\n", + "step: 754, Red action: do-nothing, Blue reward:0.85\n", + "step: 755, Red action: do-nothing, Blue reward:0.85\n", + "step: 756, Red action: do-nothing, Blue reward:0.85\n", + "step: 757, Red action: do-nothing, Blue reward:0.85\n", + "step: 758, Red action: do-nothing, Blue reward:0.85\n", + "step: 759, Red action: do-nothing, Blue reward:0.85\n", + "step: 760, Red action: node-application-execute, Blue reward:0.85\n", + "step: 761, Red action: do-nothing, Blue reward:0.85\n", + "step: 762, Red action: do-nothing, Blue reward:0.85\n", + "step: 763, Red action: do-nothing, Blue reward:0.85\n", + "step: 764, Red action: do-nothing, Blue reward:0.85\n", + "step: 765, Red action: do-nothing, Blue reward:0.85\n", + "step: 766, Red action: do-nothing, Blue reward:0.85\n", + "step: 767, Red action: do-nothing, Blue reward:0.85\n", + "step: 768, Red action: do-nothing, Blue reward:0.85\n", + "step: 769, Red action: do-nothing, Blue reward:0.85\n", + "step: 770, Red action: do-nothing, Blue reward:0.85\n", + "step: 771, Red action: do-nothing, Blue reward:0.85\n", + "step: 772, Red action: do-nothing, Blue reward:0.85\n", + "step: 773, Red action: do-nothing, Blue reward:0.85\n", + "step: 774, Red action: do-nothing, Blue reward:0.85\n", + "step: 775, Red action: do-nothing, Blue reward:0.85\n", + "step: 776, Red action: do-nothing, Blue reward:0.85\n", + "step: 777, Red action: do-nothing, Blue reward:0.85\n", + "step: 778, Red action: do-nothing, Blue reward:0.85\n", + "step: 779, Red action: do-nothing, Blue reward:0.85\n", + "step: 780, Red action: do-nothing, Blue reward:0.85\n", + "step: 781, Red action: do-nothing, Blue reward:0.85\n", + "step: 782, Red action: do-nothing, Blue reward:0.85\n", + "step: 783, Red action: node-application-execute, Blue reward:0.85\n", + "step: 784, Red action: do-nothing, Blue reward:0.85\n", + "step: 785, Red action: do-nothing, Blue reward:0.85\n", + "step: 786, Red action: do-nothing, Blue reward:0.85\n", + "step: 787, Red action: do-nothing, Blue reward:0.85\n", + "step: 788, Red action: do-nothing, Blue reward:0.85\n", + "step: 789, Red action: do-nothing, Blue reward:0.85\n", + "step: 790, Red action: do-nothing, Blue reward:0.85\n", + "step: 791, Red action: do-nothing, Blue reward:0.85\n", + "step: 792, Red action: do-nothing, Blue reward:0.85\n", + "step: 793, Red action: do-nothing, Blue reward:0.85\n", + "step: 794, Red action: do-nothing, Blue reward:0.85\n", + "step: 795, Red action: do-nothing, Blue reward:0.85\n", + "step: 796, Red action: do-nothing, Blue reward:0.85\n", + "step: 797, Red action: do-nothing, Blue reward:0.85\n", + "step: 798, Red action: do-nothing, Blue reward:0.85\n", + "step: 799, Red action: do-nothing, Blue reward:0.85\n", + "step: 800, Red action: do-nothing, Blue reward:0.85\n", + "step: 801, Red action: do-nothing, Blue reward:0.85\n", + "step: 802, Red action: do-nothing, Blue reward:0.85\n", + "step: 803, Red action: node-application-execute, Blue reward:0.85\n", + "step: 804, Red action: do-nothing, Blue reward:0.85\n", + "step: 805, Red action: do-nothing, Blue reward:0.85\n", + "step: 806, Red action: do-nothing, Blue reward:0.85\n", + "step: 807, Red action: do-nothing, Blue reward:0.85\n", + "step: 808, Red action: do-nothing, Blue reward:0.85\n", + "step: 809, Red action: do-nothing, Blue reward:0.85\n", + "step: 810, Red action: do-nothing, Blue reward:0.85\n", + "step: 811, Red action: do-nothing, Blue reward:0.85\n", + "step: 812, Red action: do-nothing, Blue reward:0.85\n", + "step: 813, Red action: do-nothing, Blue reward:0.85\n", + "step: 814, Red action: do-nothing, Blue reward:0.85\n", + "step: 815, Red action: do-nothing, Blue reward:0.85\n", + "step: 816, Red action: do-nothing, Blue reward:0.85\n", + "step: 817, Red action: do-nothing, Blue reward:0.85\n", + "step: 818, Red action: do-nothing, Blue reward:0.85\n", + "step: 819, Red action: do-nothing, Blue reward:0.85\n", + "step: 820, Red action: do-nothing, Blue reward:0.85\n", + "step: 821, Red action: do-nothing, Blue reward:0.85\n", + "step: 822, Red action: do-nothing, Blue reward:0.85\n", + "step: 823, Red action: do-nothing, Blue reward:0.85\n", + "step: 824, Red action: do-nothing, Blue reward:0.85\n", + "step: 825, Red action: do-nothing, Blue reward:0.85\n", + "step: 826, Red action: do-nothing, Blue reward:0.85\n", + "step: 827, Red action: node-application-execute, Blue reward:0.85\n", + "step: 828, Red action: do-nothing, Blue reward:0.85\n", + "step: 829, Red action: do-nothing, Blue reward:0.85\n", + "step: 830, Red action: do-nothing, Blue reward:0.85\n", + "step: 831, Red action: do-nothing, Blue reward:0.85\n", + "step: 832, Red action: do-nothing, Blue reward:0.85\n", + "step: 833, Red action: do-nothing, Blue reward:0.85\n", + "step: 834, Red action: do-nothing, Blue reward:0.85\n", + "step: 835, Red action: do-nothing, Blue reward:0.85\n", + "step: 836, Red action: do-nothing, Blue reward:0.85\n", + "step: 837, Red action: do-nothing, Blue reward:0.85\n", + "step: 838, Red action: do-nothing, Blue reward:0.85\n", + "step: 839, Red action: do-nothing, Blue reward:0.85\n", + "step: 840, Red action: do-nothing, Blue reward:0.85\n", + "step: 841, Red action: do-nothing, Blue reward:0.85\n", + "step: 842, Red action: do-nothing, Blue reward:0.85\n", + "step: 843, Red action: do-nothing, Blue reward:0.85\n", + "step: 844, Red action: do-nothing, Blue reward:0.85\n", + "step: 845, Red action: do-nothing, Blue reward:0.85\n", + "step: 846, Red action: do-nothing, Blue reward:0.85\n", + "step: 847, Red action: node-application-execute, Blue reward:0.85\n", + "step: 848, Red action: do-nothing, Blue reward:0.85\n", + "step: 849, Red action: do-nothing, Blue reward:0.85\n", + "step: 850, Red action: do-nothing, Blue reward:0.85\n", + "step: 851, Red action: do-nothing, Blue reward:0.85\n", + "step: 852, Red action: do-nothing, Blue reward:0.85\n", + "step: 853, Red action: do-nothing, Blue reward:0.85\n", + "step: 854, Red action: do-nothing, Blue reward:0.85\n", + "step: 855, Red action: do-nothing, Blue reward:0.85\n", + "step: 856, Red action: do-nothing, Blue reward:0.85\n", + "step: 857, Red action: do-nothing, Blue reward:0.85\n", + "step: 858, Red action: do-nothing, Blue reward:0.85\n", + "step: 859, Red action: do-nothing, Blue reward:0.85\n", + "step: 860, Red action: do-nothing, Blue reward:0.85\n", + "step: 861, Red action: do-nothing, Blue reward:0.85\n", + "step: 862, Red action: do-nothing, Blue reward:0.85\n", + "step: 863, Red action: do-nothing, Blue reward:0.85\n", + "step: 864, Red action: do-nothing, Blue reward:0.85\n", + "step: 865, Red action: do-nothing, Blue reward:0.85\n", + "step: 866, Red action: do-nothing, Blue reward:0.85\n", + "step: 867, Red action: do-nothing, Blue reward:0.85\n", + "step: 868, Red action: do-nothing, Blue reward:0.85\n", + "step: 869, Red action: do-nothing, Blue reward:0.85\n", + "step: 870, Red action: do-nothing, Blue reward:0.85\n", + "step: 871, Red action: node-application-execute, Blue reward:0.85\n", + "step: 872, Red action: do-nothing, Blue reward:0.85\n", + "step: 873, Red action: do-nothing, Blue reward:0.85\n", + "step: 874, Red action: do-nothing, Blue reward:0.85\n", + "step: 875, Red action: do-nothing, Blue reward:0.85\n", + "step: 876, Red action: do-nothing, Blue reward:0.85\n", + "step: 877, Red action: do-nothing, Blue reward:0.85\n", + "step: 878, Red action: do-nothing, Blue reward:0.85\n", + "step: 879, Red action: do-nothing, Blue reward:0.85\n", + "step: 880, Red action: do-nothing, Blue reward:0.85\n", + "step: 881, Red action: do-nothing, Blue reward:0.85\n", + "step: 882, Red action: do-nothing, Blue reward:0.85\n", + "step: 883, Red action: do-nothing, Blue reward:0.85\n", + "step: 884, Red action: do-nothing, Blue reward:0.85\n", + "step: 885, Red action: do-nothing, Blue reward:0.85\n", + "step: 886, Red action: do-nothing, Blue reward:0.85\n", + "step: 887, Red action: do-nothing, Blue reward:0.85\n", + "step: 888, Red action: do-nothing, Blue reward:0.85\n", + "step: 889, Red action: do-nothing, Blue reward:0.85\n", + "step: 890, Red action: do-nothing, Blue reward:0.85\n", + "step: 891, Red action: node-application-execute, Blue reward:0.85\n", + "step: 892, Red action: do-nothing, Blue reward:0.85\n", + "step: 893, Red action: do-nothing, Blue reward:0.85\n", + "step: 894, Red action: do-nothing, Blue reward:0.85\n", + "step: 895, Red action: do-nothing, Blue reward:0.85\n", + "step: 896, Red action: do-nothing, Blue reward:0.85\n", + "step: 897, Red action: do-nothing, Blue reward:0.85\n", + "step: 898, Red action: do-nothing, Blue reward:0.85\n", + "step: 899, Red action: do-nothing, Blue reward:0.85\n", + "step: 900, Red action: do-nothing, Blue reward:0.85\n", + "step: 901, Red action: do-nothing, Blue reward:0.85\n", + "step: 902, Red action: do-nothing, Blue reward:0.85\n", + "step: 903, Red action: do-nothing, Blue reward:0.85\n", + "step: 904, Red action: do-nothing, Blue reward:0.85\n", + "step: 905, Red action: do-nothing, Blue reward:0.85\n", + "step: 906, Red action: node-application-execute, Blue reward:0.85\n", + "step: 907, Red action: do-nothing, Blue reward:0.85\n", + "step: 908, Red action: do-nothing, Blue reward:0.85\n", + "step: 909, Red action: do-nothing, Blue reward:0.85\n", + "step: 910, Red action: do-nothing, Blue reward:0.85\n", + "step: 911, Red action: do-nothing, Blue reward:0.85\n", + "step: 912, Red action: do-nothing, Blue reward:0.85\n", + "step: 913, Red action: do-nothing, Blue reward:0.85\n", + "step: 914, Red action: do-nothing, Blue reward:0.85\n", + "step: 915, Red action: do-nothing, Blue reward:0.85\n", + "step: 916, Red action: do-nothing, Blue reward:0.85\n", + "step: 917, Red action: do-nothing, Blue reward:0.85\n", + "step: 918, Red action: do-nothing, Blue reward:0.85\n", + "step: 919, Red action: do-nothing, Blue reward:0.85\n", + "step: 920, Red action: do-nothing, Blue reward:0.85\n", + "step: 921, Red action: do-nothing, Blue reward:0.85\n", + "step: 922, Red action: do-nothing, Blue reward:0.85\n", + "step: 923, Red action: do-nothing, Blue reward:0.85\n", + "step: 924, Red action: do-nothing, Blue reward:0.85\n", + "step: 925, Red action: do-nothing, Blue reward:0.85\n", + "step: 926, Red action: do-nothing, Blue reward:0.85\n", + "step: 927, Red action: do-nothing, Blue reward:0.85\n", + "step: 928, Red action: do-nothing, Blue reward:0.85\n", + "step: 929, Red action: do-nothing, Blue reward:0.85\n", + "step: 930, Red action: node-application-execute, Blue reward:0.85\n", + "step: 931, Red action: do-nothing, Blue reward:0.85\n", + "step: 932, Red action: do-nothing, Blue reward:0.85\n", + "step: 933, Red action: do-nothing, Blue reward:0.85\n", + "step: 934, Red action: do-nothing, Blue reward:0.85\n", + "step: 935, Red action: do-nothing, Blue reward:0.85\n", + "step: 936, Red action: do-nothing, Blue reward:0.85\n", + "step: 937, Red action: do-nothing, Blue reward:0.85\n", + "step: 938, Red action: do-nothing, Blue reward:0.85\n", + "step: 939, Red action: do-nothing, Blue reward:0.85\n", + "step: 940, Red action: do-nothing, Blue reward:0.85\n", + "step: 941, Red action: do-nothing, Blue reward:0.85\n", + "step: 942, Red action: do-nothing, Blue reward:0.85\n", + "step: 943, Red action: do-nothing, Blue reward:0.85\n", + "step: 944, Red action: do-nothing, Blue reward:0.85\n", + "step: 945, Red action: do-nothing, Blue reward:0.85\n", + "step: 946, Red action: do-nothing, Blue reward:0.85\n", + "step: 947, Red action: node-application-execute, Blue reward:0.85\n", + "step: 948, Red action: do-nothing, Blue reward:0.85\n", + "step: 949, Red action: do-nothing, Blue reward:0.85\n", + "step: 950, Red action: do-nothing, Blue reward:0.85\n", + "step: 951, Red action: do-nothing, Blue reward:0.85\n", + "step: 952, Red action: do-nothing, Blue reward:0.85\n", + "step: 953, Red action: do-nothing, Blue reward:0.85\n", + "step: 954, Red action: do-nothing, Blue reward:0.85\n", + "step: 955, Red action: do-nothing, Blue reward:0.85\n", + "step: 956, Red action: do-nothing, Blue reward:0.85\n", + "step: 957, Red action: do-nothing, Blue reward:0.85\n", + "step: 958, Red action: do-nothing, Blue reward:0.85\n", + "step: 959, Red action: do-nothing, Blue reward:0.85\n", + "step: 960, Red action: do-nothing, Blue reward:0.85\n", + "step: 961, Red action: do-nothing, Blue reward:0.85\n", + "step: 962, Red action: do-nothing, Blue reward:0.85\n", + "step: 963, Red action: node-application-execute, Blue reward:0.85\n", + "step: 964, Red action: do-nothing, Blue reward:0.85\n", + "step: 965, Red action: do-nothing, Blue reward:0.85\n", + "step: 966, Red action: do-nothing, Blue reward:0.85\n", + "step: 967, Red action: do-nothing, Blue reward:0.85\n", + "step: 968, Red action: do-nothing, Blue reward:0.85\n", + "step: 969, Red action: do-nothing, Blue reward:0.85\n", + "step: 970, Red action: do-nothing, Blue reward:0.85\n", + "step: 971, Red action: do-nothing, Blue reward:0.85\n", + "step: 972, Red action: do-nothing, Blue reward:0.85\n", + "step: 973, Red action: do-nothing, Blue reward:0.85\n", + "step: 974, Red action: do-nothing, Blue reward:0.85\n", + "step: 975, Red action: do-nothing, Blue reward:0.85\n", + "step: 976, Red action: do-nothing, Blue reward:0.85\n", + "step: 977, Red action: do-nothing, Blue reward:0.85\n", + "step: 978, Red action: do-nothing, Blue reward:0.85\n", + "step: 979, Red action: do-nothing, Blue reward:0.85\n", + "step: 980, Red action: do-nothing, Blue reward:0.85\n", + "step: 981, Red action: do-nothing, Blue reward:0.85\n", + "step: 982, Red action: do-nothing, Blue reward:0.85\n", + "step: 983, Red action: do-nothing, Blue reward:0.85\n", + "step: 984, Red action: do-nothing, Blue reward:0.85\n", + "step: 985, Red action: do-nothing, Blue reward:0.85\n", + "step: 986, Red action: do-nothing, Blue reward:0.85\n", + "step: 987, Red action: do-nothing, Blue reward:0.85\n", + "step: 988, Red action: node-application-execute, Blue reward:0.85\n", + "step: 989, Red action: do-nothing, Blue reward:0.85\n", + "step: 990, Red action: do-nothing, Blue reward:0.85\n", + "step: 991, Red action: do-nothing, Blue reward:0.85\n", + "step: 992, Red action: do-nothing, Blue reward:0.85\n", + "step: 993, Red action: do-nothing, Blue reward:0.85\n", + "step: 994, Red action: do-nothing, Blue reward:0.85\n", + "step: 995, Red action: do-nothing, Blue reward:0.85\n", + "step: 996, Red action: do-nothing, Blue reward:0.85\n", + "step: 997, Red action: do-nothing, Blue reward:0.85\n", + "step: 998, Red action: do-nothing, Blue reward:0.85\n", + "step: 999, Red action: do-nothing, Blue reward:0.85\n", + "step: 1000, Red action: do-nothing, Blue reward:0.85\n", + "step: 1001, Red action: do-nothing, Blue reward:0.85\n", + "step: 1002, Red action: do-nothing, Blue reward:0.85\n", + "step: 1003, Red action: do-nothing, Blue reward:0.85\n", + "step: 1004, Red action: do-nothing, Blue reward:0.85\n", + "step: 1005, Red action: do-nothing, Blue reward:0.85\n", + "step: 1006, Red action: do-nothing, Blue reward:0.85\n", + "step: 1007, Red action: do-nothing, Blue reward:0.85\n", + "step: 1008, Red action: do-nothing, Blue reward:0.85\n", + "step: 1009, Red action: do-nothing, Blue reward:0.85\n", + "step: 1010, Red action: do-nothing, Blue reward:0.85\n", + "step: 1011, Red action: do-nothing, Blue reward:0.85\n", + "step: 1012, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1013, Red action: do-nothing, Blue reward:0.85\n", + "step: 1014, Red action: do-nothing, Blue reward:0.85\n", + "step: 1015, Red action: do-nothing, Blue reward:0.85\n", + "step: 1016, Red action: do-nothing, Blue reward:0.85\n", + "step: 1017, Red action: do-nothing, Blue reward:0.85\n", + "step: 1018, Red action: do-nothing, Blue reward:0.85\n", + "step: 1019, Red action: do-nothing, Blue reward:0.85\n", + "step: 1020, Red action: do-nothing, Blue reward:0.85\n", + "step: 1021, Red action: do-nothing, Blue reward:0.85\n", + "step: 1022, Red action: do-nothing, Blue reward:0.85\n", + "step: 1023, Red action: do-nothing, Blue reward:0.85\n", + "step: 1024, Red action: do-nothing, Blue reward:0.85\n", + "step: 1025, Red action: do-nothing, Blue reward:0.85\n", + "step: 1026, Red action: do-nothing, Blue reward:0.85\n", + "step: 1027, Red action: do-nothing, Blue reward:0.85\n", + "step: 1028, Red action: do-nothing, Blue reward:0.85\n", + "step: 1029, Red action: do-nothing, Blue reward:0.85\n", + "step: 1030, Red action: do-nothing, Blue reward:0.85\n", + "step: 1031, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1032, Red action: do-nothing, Blue reward:0.85\n", + "step: 1033, Red action: do-nothing, Blue reward:0.85\n", + "step: 1034, Red action: do-nothing, Blue reward:0.85\n", + "step: 1035, Red action: do-nothing, Blue reward:0.85\n", + "step: 1036, Red action: do-nothing, Blue reward:0.85\n", + "step: 1037, Red action: do-nothing, Blue reward:0.85\n", + "step: 1038, Red action: do-nothing, Blue reward:0.85\n", + "step: 1039, Red action: do-nothing, Blue reward:0.85\n", + "step: 1040, Red action: do-nothing, Blue reward:0.85\n", + "step: 1041, Red action: do-nothing, Blue reward:0.85\n", + "step: 1042, Red action: do-nothing, Blue reward:0.85\n", + "step: 1043, Red action: do-nothing, Blue reward:0.85\n", + "step: 1044, Red action: do-nothing, Blue reward:0.85\n", + "step: 1045, Red action: do-nothing, Blue reward:0.85\n", + "step: 1046, Red action: do-nothing, Blue reward:0.85\n", + "step: 1047, Red action: do-nothing, Blue reward:0.85\n", + "step: 1048, Red action: do-nothing, Blue reward:0.85\n", + "step: 1049, Red action: do-nothing, Blue reward:0.85\n", + "step: 1050, Red action: do-nothing, Blue reward:0.85\n", + "step: 1051, Red action: do-nothing, Blue reward:0.85\n", + "step: 1052, Red action: do-nothing, Blue reward:0.85\n", + "step: 1053, Red action: do-nothing, Blue reward:0.85\n", + "step: 1054, Red action: do-nothing, Blue reward:0.85\n", + "step: 1055, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1056, Red action: do-nothing, Blue reward:0.85\n", + "step: 1057, Red action: do-nothing, Blue reward:0.85\n", + "step: 1058, Red action: do-nothing, Blue reward:0.85\n", + "step: 1059, Red action: do-nothing, Blue reward:0.85\n", + "step: 1060, Red action: do-nothing, Blue reward:0.85\n", + "step: 1061, Red action: do-nothing, Blue reward:0.85\n", + "step: 1062, Red action: do-nothing, Blue reward:0.85\n", + "step: 1063, Red action: do-nothing, Blue reward:0.85\n", + "step: 1064, Red action: do-nothing, Blue reward:0.85\n", + "step: 1065, Red action: do-nothing, Blue reward:0.85\n", + "step: 1066, Red action: do-nothing, Blue reward:0.85\n", + "step: 1067, Red action: do-nothing, Blue reward:0.85\n", + "step: 1068, Red action: do-nothing, Blue reward:0.85\n", + "step: 1069, Red action: do-nothing, Blue reward:0.85\n", + "step: 1070, Red action: do-nothing, Blue reward:0.85\n", + "step: 1071, Red action: do-nothing, Blue reward:0.85\n", + "step: 1072, Red action: do-nothing, Blue reward:0.85\n", + "step: 1073, Red action: do-nothing, Blue reward:0.85\n", + "step: 1074, Red action: do-nothing, Blue reward:0.85\n", + "step: 1075, Red action: do-nothing, Blue reward:0.85\n", + "step: 1076, Red action: do-nothing, Blue reward:0.85\n", + "step: 1077, Red action: do-nothing, Blue reward:0.85\n", + "step: 1078, Red action: do-nothing, Blue reward:0.85\n", + "step: 1079, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1080, Red action: do-nothing, Blue reward:0.85\n", + "step: 1081, Red action: do-nothing, Blue reward:0.85\n", + "step: 1082, Red action: do-nothing, Blue reward:0.85\n", + "step: 1083, Red action: do-nothing, Blue reward:0.85\n", + "step: 1084, Red action: do-nothing, Blue reward:0.85\n", + "step: 1085, Red action: do-nothing, Blue reward:0.85\n", + "step: 1086, Red action: do-nothing, Blue reward:0.85\n", + "step: 1087, Red action: do-nothing, Blue reward:0.85\n", + "step: 1088, Red action: do-nothing, Blue reward:0.85\n", + "step: 1089, Red action: do-nothing, Blue reward:0.85\n", + "step: 1090, Red action: do-nothing, Blue reward:0.85\n", + "step: 1091, Red action: do-nothing, Blue reward:0.85\n", + "step: 1092, Red action: do-nothing, Blue reward:0.85\n", + "step: 1093, Red action: do-nothing, Blue reward:0.85\n", + "step: 1094, Red action: do-nothing, Blue reward:0.85\n", + "step: 1095, Red action: do-nothing, Blue reward:0.85\n", + "step: 1096, Red action: do-nothing, Blue reward:0.85\n", + "step: 1097, Red action: do-nothing, Blue reward:0.85\n", + "step: 1098, Red action: do-nothing, Blue reward:0.85\n", + "step: 1099, Red action: do-nothing, Blue reward:0.85\n", + "step: 1100, Red action: do-nothing, Blue reward:0.85\n", + "step: 1101, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1102, Red action: do-nothing, Blue reward:0.85\n", + "step: 1103, Red action: do-nothing, Blue reward:0.85\n", + "step: 1104, Red action: do-nothing, Blue reward:0.85\n", + "step: 1105, Red action: do-nothing, Blue reward:0.85\n", + "step: 1106, Red action: do-nothing, Blue reward:0.85\n", + "step: 1107, Red action: do-nothing, Blue reward:0.85\n", + "step: 1108, Red action: do-nothing, Blue reward:0.85\n", + "step: 1109, Red action: do-nothing, Blue reward:0.85\n", + "step: 1110, Red action: do-nothing, Blue reward:0.85\n", + "step: 1111, Red action: do-nothing, Blue reward:0.85\n", + "step: 1112, Red action: do-nothing, Blue reward:0.85\n", + "step: 1113, Red action: do-nothing, Blue reward:0.85\n", + "step: 1114, Red action: do-nothing, Blue reward:0.85\n", + "step: 1115, Red action: do-nothing, Blue reward:0.85\n", + "step: 1116, Red action: do-nothing, Blue reward:0.85\n", + "step: 1117, Red action: do-nothing, Blue reward:0.85\n", + "step: 1118, Red action: do-nothing, Blue reward:0.85\n", + "step: 1119, Red action: do-nothing, Blue reward:0.85\n", + "step: 1120, Red action: do-nothing, Blue reward:0.85\n", + "step: 1121, Red action: do-nothing, Blue reward:0.85\n", + "step: 1122, Red action: do-nothing, Blue reward:0.85\n", + "step: 1123, Red action: do-nothing, Blue reward:0.85\n", + "step: 1124, Red action: do-nothing, Blue reward:0.85\n", + "step: 1125, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1126, Red action: do-nothing, Blue reward:0.85\n", + "step: 1127, Red action: do-nothing, Blue reward:0.85\n", + "step: 1128, Red action: do-nothing, Blue reward:0.85\n", + "step: 1129, Red action: do-nothing, Blue reward:0.85\n", + "step: 1130, Red action: do-nothing, Blue reward:0.85\n", + "step: 1131, Red action: do-nothing, Blue reward:0.85\n", + "step: 1132, Red action: do-nothing, Blue reward:0.85\n", + "step: 1133, Red action: do-nothing, Blue reward:0.85\n", + "step: 1134, Red action: do-nothing, Blue reward:0.85\n", + "step: 1135, Red action: do-nothing, Blue reward:0.85\n", + "step: 1136, Red action: do-nothing, Blue reward:0.85\n", + "step: 1137, Red action: do-nothing, Blue reward:0.85\n", + "step: 1138, Red action: do-nothing, Blue reward:0.85\n", + "step: 1139, Red action: do-nothing, Blue reward:0.85\n", + "step: 1140, Red action: do-nothing, Blue reward:0.85\n", + "step: 1141, Red action: do-nothing, Blue reward:0.85\n", + "step: 1142, Red action: do-nothing, Blue reward:0.85\n", + "step: 1143, Red action: do-nothing, Blue reward:0.85\n", + "step: 1144, Red action: do-nothing, Blue reward:0.85\n", + "step: 1145, Red action: do-nothing, Blue reward:0.85\n", + "step: 1146, Red action: do-nothing, Blue reward:0.85\n", + "step: 1147, Red action: do-nothing, Blue reward:0.85\n", + "step: 1148, Red action: do-nothing, Blue reward:0.85\n", + "step: 1149, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1150, Red action: do-nothing, Blue reward:0.85\n", + "step: 1151, Red action: do-nothing, Blue reward:0.85\n", + "step: 1152, Red action: do-nothing, Blue reward:0.85\n", + "step: 1153, Red action: do-nothing, Blue reward:0.85\n", + "step: 1154, Red action: do-nothing, Blue reward:0.85\n", + "step: 1155, Red action: do-nothing, Blue reward:0.85\n", + "step: 1156, Red action: do-nothing, Blue reward:0.85\n", + "step: 1157, Red action: do-nothing, Blue reward:0.85\n", + "step: 1158, Red action: do-nothing, Blue reward:0.85\n", + "step: 1159, Red action: do-nothing, Blue reward:0.85\n", + "step: 1160, Red action: do-nothing, Blue reward:0.85\n", + "step: 1161, Red action: do-nothing, Blue reward:0.85\n", + "step: 1162, Red action: do-nothing, Blue reward:0.85\n", + "step: 1163, Red action: do-nothing, Blue reward:0.85\n", + "step: 1164, Red action: do-nothing, Blue reward:0.85\n", + "step: 1165, Red action: do-nothing, Blue reward:0.85\n", + "step: 1166, Red action: do-nothing, Blue reward:0.85\n", + "step: 1167, Red action: do-nothing, Blue reward:0.85\n", + "step: 1168, Red action: do-nothing, Blue reward:0.85\n", + "step: 1169, Red action: do-nothing, Blue reward:0.85\n", + "step: 1170, Red action: do-nothing, Blue reward:0.85\n", + "step: 1171, Red action: do-nothing, Blue reward:0.85\n", + "step: 1172, Red action: do-nothing, Blue reward:0.85\n", + "step: 1173, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1174, Red action: do-nothing, Blue reward:0.85\n", + "step: 1175, Red action: do-nothing, Blue reward:0.85\n", + "step: 1176, Red action: do-nothing, Blue reward:0.85\n", + "step: 1177, Red action: do-nothing, Blue reward:0.85\n", + "step: 1178, Red action: do-nothing, Blue reward:0.85\n", + "step: 1179, Red action: do-nothing, Blue reward:0.85\n", + "step: 1180, Red action: do-nothing, Blue reward:0.85\n", + "step: 1181, Red action: do-nothing, Blue reward:0.85\n", + "step: 1182, Red action: do-nothing, Blue reward:0.85\n", + "step: 1183, Red action: do-nothing, Blue reward:0.85\n", + "step: 1184, Red action: do-nothing, Blue reward:0.85\n", + "step: 1185, Red action: do-nothing, Blue reward:0.85\n", + "step: 1186, Red action: do-nothing, Blue reward:0.85\n", + "step: 1187, Red action: do-nothing, Blue reward:0.85\n", + "step: 1188, Red action: do-nothing, Blue reward:0.85\n", + "step: 1189, Red action: do-nothing, Blue reward:0.85\n", + "step: 1190, Red action: do-nothing, Blue reward:0.85\n", + "step: 1191, Red action: do-nothing, Blue reward:0.85\n", + "step: 1192, Red action: do-nothing, Blue reward:0.85\n", + "step: 1193, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1194, Red action: do-nothing, Blue reward:0.85\n", + "step: 1195, Red action: do-nothing, Blue reward:0.85\n", + "step: 1196, Red action: do-nothing, Blue reward:0.85\n", + "step: 1197, Red action: do-nothing, Blue reward:0.85\n", + "step: 1198, Red action: do-nothing, Blue reward:0.85\n", + "step: 1199, Red action: do-nothing, Blue reward:0.85\n", + "step: 1200, Red action: do-nothing, Blue reward:0.85\n", + "step: 1201, Red action: do-nothing, Blue reward:0.85\n", + "step: 1202, Red action: do-nothing, Blue reward:0.85\n", + "step: 1203, Red action: do-nothing, Blue reward:0.85\n", + "step: 1204, Red action: do-nothing, Blue reward:0.85\n", + "step: 1205, Red action: do-nothing, Blue reward:0.85\n", + "step: 1206, Red action: do-nothing, Blue reward:0.85\n", + "step: 1207, Red action: do-nothing, Blue reward:0.85\n", + "step: 1208, Red action: do-nothing, Blue reward:0.85\n", + "step: 1209, Red action: do-nothing, Blue reward:0.85\n", + "step: 1210, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1211, Red action: do-nothing, Blue reward:0.85\n", + "step: 1212, Red action: do-nothing, Blue reward:0.85\n", + "step: 1213, Red action: do-nothing, Blue reward:0.85\n", + "step: 1214, Red action: do-nothing, Blue reward:0.85\n", + "step: 1215, Red action: do-nothing, Blue reward:0.85\n", + "step: 1216, Red action: do-nothing, Blue reward:0.85\n", + "step: 1217, Red action: do-nothing, Blue reward:0.85\n", + "step: 1218, Red action: do-nothing, Blue reward:0.85\n", + "step: 1219, Red action: do-nothing, Blue reward:0.85\n", + "step: 1220, Red action: do-nothing, Blue reward:0.85\n", + "step: 1221, Red action: do-nothing, Blue reward:0.85\n", + "step: 1222, Red action: do-nothing, Blue reward:0.85\n", + "step: 1223, Red action: do-nothing, Blue reward:0.85\n", + "step: 1224, Red action: do-nothing, Blue reward:0.85\n", + "step: 1225, Red action: do-nothing, Blue reward:0.85\n", + "step: 1226, Red action: do-nothing, Blue reward:0.85\n", + "step: 1227, Red action: do-nothing, Blue reward:0.85\n", + "step: 1228, Red action: do-nothing, Blue reward:0.85\n", + "step: 1229, Red action: do-nothing, Blue reward:0.85\n", + "step: 1230, Red action: do-nothing, Blue reward:0.85\n", + "step: 1231, Red action: do-nothing, Blue reward:0.85\n", + "step: 1232, Red action: do-nothing, Blue reward:0.85\n", + "step: 1233, Red action: do-nothing, Blue reward:0.85\n", + "step: 1234, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1235, Red action: do-nothing, Blue reward:0.85\n", + "step: 1236, Red action: do-nothing, Blue reward:0.85\n", + "step: 1237, Red action: do-nothing, Blue reward:0.85\n", + "step: 1238, Red action: do-nothing, Blue reward:0.85\n", + "step: 1239, Red action: do-nothing, Blue reward:0.85\n", + "step: 1240, Red action: do-nothing, Blue reward:0.85\n", + "step: 1241, Red action: do-nothing, Blue reward:0.85\n", + "step: 1242, Red action: do-nothing, Blue reward:0.85\n", + "step: 1243, Red action: do-nothing, Blue reward:0.85\n", + "step: 1244, Red action: do-nothing, Blue reward:0.85\n", + "step: 1245, Red action: do-nothing, Blue reward:0.85\n", + "step: 1246, Red action: do-nothing, Blue reward:0.85\n", + "step: 1247, Red action: do-nothing, Blue reward:0.85\n", + "step: 1248, Red action: do-nothing, Blue reward:0.85\n", + "step: 1249, Red action: do-nothing, Blue reward:0.85\n", + "step: 1250, Red action: do-nothing, Blue reward:0.85\n", + "step: 1251, Red action: do-nothing, Blue reward:0.85\n", + "step: 1252, Red action: do-nothing, Blue reward:0.85\n", + "step: 1253, Red action: do-nothing, Blue reward:0.85\n", + "step: 1254, Red action: do-nothing, Blue reward:0.85\n", + "step: 1255, Red action: do-nothing, Blue reward:0.85\n", + "step: 1256, Red action: do-nothing, Blue reward:0.85\n", + "step: 1257, Red action: do-nothing, Blue reward:0.85\n", + "step: 1258, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1259, Red action: do-nothing, Blue reward:0.85\n", + "step: 1260, Red action: do-nothing, Blue reward:0.85\n", + "step: 1261, Red action: do-nothing, Blue reward:0.85\n", + "step: 1262, Red action: do-nothing, Blue reward:0.85\n", + "step: 1263, Red action: do-nothing, Blue reward:0.85\n", + "step: 1264, Red action: do-nothing, Blue reward:0.85\n", + "step: 1265, Red action: do-nothing, Blue reward:0.85\n", + "step: 1266, Red action: do-nothing, Blue reward:0.85\n", + "step: 1267, Red action: do-nothing, Blue reward:0.85\n", + "step: 1268, Red action: do-nothing, Blue reward:0.85\n", + "step: 1269, Red action: do-nothing, Blue reward:0.85\n", + "step: 1270, Red action: do-nothing, Blue reward:0.85\n", + "step: 1271, Red action: do-nothing, Blue reward:0.85\n", + "step: 1272, Red action: do-nothing, Blue reward:0.85\n", + "step: 1273, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1274, Red action: do-nothing, Blue reward:0.85\n", + "step: 1275, Red action: do-nothing, Blue reward:0.85\n", + "step: 1276, Red action: do-nothing, Blue reward:0.85\n", + "step: 1277, Red action: do-nothing, Blue reward:0.85\n", + "step: 1278, Red action: do-nothing, Blue reward:0.85\n", + "step: 1279, Red action: do-nothing, Blue reward:0.85\n", + "step: 1280, Red action: do-nothing, Blue reward:0.85\n", + "step: 1281, Red action: do-nothing, Blue reward:0.85\n", + "step: 1282, Red action: do-nothing, Blue reward:0.85\n", + "step: 1283, Red action: do-nothing, Blue reward:0.85\n", + "step: 1284, Red action: do-nothing, Blue reward:0.85\n", + "step: 1285, Red action: do-nothing, Blue reward:0.85\n", + "step: 1286, Red action: do-nothing, Blue reward:0.85\n", + "step: 1287, Red action: do-nothing, Blue reward:0.85\n", + "step: 1288, Red action: do-nothing, Blue reward:0.85\n", + "step: 1289, Red action: do-nothing, Blue reward:0.85\n", + "step: 1290, Red action: do-nothing, Blue reward:0.85\n", + "step: 1291, Red action: do-nothing, Blue reward:0.85\n", + "step: 1292, Red action: do-nothing, Blue reward:0.85\n", + "step: 1293, Red action: do-nothing, Blue reward:0.85\n", + "step: 1294, Red action: do-nothing, Blue reward:0.85\n", + "step: 1295, Red action: do-nothing, Blue reward:0.85\n", + "step: 1296, Red action: do-nothing, Blue reward:0.85\n", + "step: 1297, Red action: do-nothing, Blue reward:0.85\n", + "step: 1298, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1299, Red action: do-nothing, Blue reward:0.85\n", + "step: 1300, Red action: do-nothing, Blue reward:0.85\n", + "step: 1301, Red action: do-nothing, Blue reward:0.85\n", + "step: 1302, Red action: do-nothing, Blue reward:0.85\n", + "step: 1303, Red action: do-nothing, Blue reward:0.85\n", + "step: 1304, Red action: do-nothing, Blue reward:0.85\n", + "step: 1305, Red action: do-nothing, Blue reward:0.85\n", + "step: 1306, Red action: do-nothing, Blue reward:0.85\n", + "step: 1307, Red action: do-nothing, Blue reward:0.85\n", + "step: 1308, Red action: do-nothing, Blue reward:0.85\n", + "step: 1309, Red action: do-nothing, Blue reward:0.85\n", + "step: 1310, Red action: do-nothing, Blue reward:0.85\n", + "step: 1311, Red action: do-nothing, Blue reward:0.85\n", + "step: 1312, Red action: do-nothing, Blue reward:0.85\n", + "step: 1313, Red action: do-nothing, Blue reward:0.85\n", + "step: 1314, Red action: do-nothing, Blue reward:0.85\n", + "step: 1315, Red action: do-nothing, Blue reward:0.85\n", + "step: 1316, Red action: do-nothing, Blue reward:0.85\n", + "step: 1317, Red action: do-nothing, Blue reward:0.85\n", + "step: 1318, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1319, Red action: do-nothing, Blue reward:0.85\n", + "step: 1320, Red action: do-nothing, Blue reward:0.85\n", + "step: 1321, Red action: do-nothing, Blue reward:0.85\n", + "step: 1322, Red action: do-nothing, Blue reward:0.85\n", + "step: 1323, Red action: do-nothing, Blue reward:0.85\n", + "step: 1324, Red action: do-nothing, Blue reward:0.85\n", + "step: 1325, Red action: do-nothing, Blue reward:0.85\n", + "step: 1326, Red action: do-nothing, Blue reward:0.85\n", + "step: 1327, Red action: do-nothing, Blue reward:0.85\n", + "step: 1328, Red action: do-nothing, Blue reward:0.85\n", + "step: 1329, Red action: do-nothing, Blue reward:0.85\n", + "step: 1330, Red action: do-nothing, Blue reward:0.85\n", + "step: 1331, Red action: do-nothing, Blue reward:0.85\n", + "step: 1332, Red action: do-nothing, Blue reward:0.85\n", + "step: 1333, Red action: do-nothing, Blue reward:0.85\n", + "step: 1334, Red action: do-nothing, Blue reward:0.85\n", + "step: 1335, Red action: do-nothing, Blue reward:0.85\n", + "step: 1336, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1337, Red action: do-nothing, Blue reward:0.85\n", + "step: 1338, Red action: do-nothing, Blue reward:0.85\n", + "step: 1339, Red action: do-nothing, Blue reward:0.85\n", + "step: 1340, Red action: do-nothing, Blue reward:0.85\n", + "step: 1341, Red action: do-nothing, Blue reward:0.85\n", + "step: 1342, Red action: do-nothing, Blue reward:0.85\n", + "step: 1343, Red action: do-nothing, Blue reward:0.85\n", + "step: 1344, Red action: do-nothing, Blue reward:0.85\n", + "step: 1345, Red action: do-nothing, Blue reward:0.85\n", + "step: 1346, Red action: do-nothing, Blue reward:0.85\n", + "step: 1347, Red action: do-nothing, Blue reward:0.85\n", + "step: 1348, Red action: do-nothing, Blue reward:0.85\n", + "step: 1349, Red action: do-nothing, Blue reward:0.85\n", + "step: 1350, Red action: do-nothing, Blue reward:0.85\n", + "step: 1351, Red action: do-nothing, Blue reward:0.85\n", + "step: 1352, Red action: do-nothing, Blue reward:0.85\n", + "step: 1353, Red action: do-nothing, Blue reward:0.85\n", + "step: 1354, Red action: do-nothing, Blue reward:0.85\n", + "step: 1355, Red action: do-nothing, Blue reward:0.85\n", + "step: 1356, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1357, Red action: do-nothing, Blue reward:0.85\n", + "step: 1358, Red action: do-nothing, Blue reward:0.85\n", + "step: 1359, Red action: do-nothing, Blue reward:0.85\n", + "step: 1360, Red action: do-nothing, Blue reward:0.85\n", + "step: 1361, Red action: do-nothing, Blue reward:0.85\n", + "step: 1362, Red action: do-nothing, Blue reward:0.85\n", + "step: 1363, Red action: do-nothing, Blue reward:0.85\n", + "step: 1364, Red action: do-nothing, Blue reward:0.85\n", + "step: 1365, Red action: do-nothing, Blue reward:0.85\n", + "step: 1366, Red action: do-nothing, Blue reward:0.85\n", + "step: 1367, Red action: do-nothing, Blue reward:0.85\n", + "step: 1368, Red action: do-nothing, Blue reward:0.85\n", + "step: 1369, Red action: do-nothing, Blue reward:0.85\n", + "step: 1370, Red action: do-nothing, Blue reward:0.85\n", + "step: 1371, Red action: do-nothing, Blue reward:0.85\n", + "step: 1372, Red action: do-nothing, Blue reward:0.85\n", + "step: 1373, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1374, Red action: do-nothing, Blue reward:0.85\n", + "step: 1375, Red action: do-nothing, Blue reward:0.85\n", + "step: 1376, Red action: do-nothing, Blue reward:0.85\n", + "step: 1377, Red action: do-nothing, Blue reward:0.85\n", + "step: 1378, Red action: do-nothing, Blue reward:0.85\n", + "step: 1379, Red action: do-nothing, Blue reward:0.85\n", + "step: 1380, Red action: do-nothing, Blue reward:0.85\n", + "step: 1381, Red action: do-nothing, Blue reward:0.85\n", + "step: 1382, Red action: do-nothing, Blue reward:0.85\n", + "step: 1383, Red action: do-nothing, Blue reward:0.85\n", + "step: 1384, Red action: do-nothing, Blue reward:0.85\n", + "step: 1385, Red action: do-nothing, Blue reward:0.85\n", + "step: 1386, Red action: do-nothing, Blue reward:0.85\n", + "step: 1387, Red action: do-nothing, Blue reward:0.85\n", + "step: 1388, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1389, Red action: do-nothing, Blue reward:0.85\n", + "step: 1390, Red action: do-nothing, Blue reward:0.85\n", + "step: 1391, Red action: do-nothing, Blue reward:0.85\n", + "step: 1392, Red action: do-nothing, Blue reward:0.85\n", + "step: 1393, Red action: do-nothing, Blue reward:0.85\n", + "step: 1394, Red action: do-nothing, Blue reward:0.85\n", + "step: 1395, Red action: do-nothing, Blue reward:0.85\n", + "step: 1396, Red action: do-nothing, Blue reward:0.85\n", + "step: 1397, Red action: do-nothing, Blue reward:0.85\n", + "step: 1398, Red action: do-nothing, Blue reward:0.85\n", + "step: 1399, Red action: do-nothing, Blue reward:0.85\n", + "step: 1400, Red action: do-nothing, Blue reward:0.85\n", + "step: 1401, Red action: do-nothing, Blue reward:0.85\n", + "step: 1402, Red action: do-nothing, Blue reward:0.85\n", + "step: 1403, Red action: do-nothing, Blue reward:0.85\n", + "step: 1404, Red action: do-nothing, Blue reward:0.85\n", + "step: 1405, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1406, Red action: do-nothing, Blue reward:0.85\n", + "step: 1407, Red action: do-nothing, Blue reward:0.85\n", + "step: 1408, Red action: do-nothing, Blue reward:0.85\n", + "step: 1409, Red action: do-nothing, Blue reward:0.85\n", + "step: 1410, Red action: do-nothing, Blue reward:0.85\n", + "step: 1411, Red action: do-nothing, Blue reward:0.85\n", + "step: 1412, Red action: do-nothing, Blue reward:0.85\n", + "step: 1413, Red action: do-nothing, Blue reward:0.85\n", + "step: 1414, Red action: do-nothing, Blue reward:0.85\n", + "step: 1415, Red action: do-nothing, Blue reward:0.85\n", + "step: 1416, Red action: do-nothing, Blue reward:0.85\n", + "step: 1417, Red action: do-nothing, Blue reward:0.85\n", + "step: 1418, Red action: do-nothing, Blue reward:0.85\n", + "step: 1419, Red action: do-nothing, Blue reward:0.85\n", + "step: 1420, Red action: do-nothing, Blue reward:0.85\n", + "step: 1421, Red action: do-nothing, Blue reward:0.85\n", + "step: 1422, Red action: do-nothing, Blue reward:0.85\n", + "step: 1423, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1424, Red action: do-nothing, Blue reward:0.85\n", + "step: 1425, Red action: do-nothing, Blue reward:0.85\n", + "step: 1426, Red action: do-nothing, Blue reward:0.85\n", + "step: 1427, Red action: do-nothing, Blue reward:0.85\n", + "step: 1428, Red action: do-nothing, Blue reward:0.85\n", + "step: 1429, Red action: do-nothing, Blue reward:0.85\n", + "step: 1430, Red action: do-nothing, Blue reward:0.85\n", + "step: 1431, Red action: do-nothing, Blue reward:0.85\n", + "step: 1432, Red action: do-nothing, Blue reward:0.85\n", + "step: 1433, Red action: do-nothing, Blue reward:0.85\n", + "step: 1434, Red action: do-nothing, Blue reward:0.85\n", + "step: 1435, Red action: do-nothing, Blue reward:0.85\n", + "step: 1436, Red action: do-nothing, Blue reward:0.85\n", + "step: 1437, Red action: do-nothing, Blue reward:0.85\n", + "step: 1438, Red action: do-nothing, Blue reward:0.85\n", + "step: 1439, Red action: do-nothing, Blue reward:0.85\n", + "step: 1440, Red action: do-nothing, Blue reward:0.85\n", + "step: 1441, Red action: do-nothing, Blue reward:0.85\n", + "step: 1442, Red action: do-nothing, Blue reward:0.85\n", + "step: 1443, Red action: do-nothing, Blue reward:0.85\n", + "step: 1444, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1445, Red action: do-nothing, Blue reward:0.85\n", + "step: 1446, Red action: do-nothing, Blue reward:0.85\n", + "step: 1447, Red action: do-nothing, Blue reward:0.85\n", + "step: 1448, Red action: do-nothing, Blue reward:0.85\n", + "step: 1449, Red action: do-nothing, Blue reward:0.85\n", + "step: 1450, Red action: do-nothing, Blue reward:0.85\n", + "step: 1451, Red action: do-nothing, Blue reward:0.85\n", + "step: 1452, Red action: do-nothing, Blue reward:0.85\n", + "step: 1453, Red action: do-nothing, Blue reward:0.85\n", + "step: 1454, Red action: do-nothing, Blue reward:0.85\n", + "step: 1455, Red action: do-nothing, Blue reward:0.85\n", + "step: 1456, Red action: do-nothing, Blue reward:0.85\n", + "step: 1457, Red action: do-nothing, Blue reward:0.85\n", + "step: 1458, Red action: do-nothing, Blue reward:0.85\n", + "step: 1459, Red action: do-nothing, Blue reward:0.85\n", + "step: 1460, Red action: do-nothing, Blue reward:0.85\n", + "step: 1461, Red action: do-nothing, Blue reward:0.85\n", + "step: 1462, Red action: do-nothing, Blue reward:0.85\n", + "step: 1463, Red action: do-nothing, Blue reward:0.85\n", + "step: 1464, Red action: do-nothing, Blue reward:0.85\n", + "step: 1465, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1466, Red action: do-nothing, Blue reward:0.85\n", + "step: 1467, Red action: do-nothing, Blue reward:0.85\n", + "step: 1468, Red action: do-nothing, Blue reward:0.85\n", + "step: 1469, Red action: do-nothing, Blue reward:0.85\n", + "step: 1470, Red action: do-nothing, Blue reward:0.85\n", + "step: 1471, Red action: do-nothing, Blue reward:0.85\n", + "step: 1472, Red action: do-nothing, Blue reward:0.85\n", + "step: 1473, Red action: do-nothing, Blue reward:0.85\n", + "step: 1474, Red action: do-nothing, Blue reward:0.85\n", + "step: 1475, Red action: do-nothing, Blue reward:0.85\n", + "step: 1476, Red action: do-nothing, Blue reward:0.85\n", + "step: 1477, Red action: do-nothing, Blue reward:0.85\n", + "step: 1478, Red action: do-nothing, Blue reward:0.85\n", + "step: 1479, Red action: do-nothing, Blue reward:0.85\n", + "step: 1480, Red action: do-nothing, Blue reward:0.85\n", + "step: 1481, Red action: do-nothing, Blue reward:0.85\n", + "step: 1482, Red action: do-nothing, Blue reward:0.85\n", + "step: 1483, Red action: do-nothing, Blue reward:0.85\n", + "step: 1484, Red action: do-nothing, Blue reward:0.85\n", + "step: 1485, Red action: do-nothing, Blue reward:0.85\n", + "step: 1486, Red action: do-nothing, Blue reward:0.85\n", + "step: 1487, Red action: do-nothing, Blue reward:0.85\n", + "step: 1488, Red action: do-nothing, Blue reward:0.85\n", + "step: 1489, Red action: do-nothing, Blue reward:0.85\n", + "step: 1490, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1491, Red action: do-nothing, Blue reward:0.85\n", + "step: 1492, Red action: do-nothing, Blue reward:0.85\n", + "step: 1493, Red action: do-nothing, Blue reward:0.85\n", + "step: 1494, Red action: do-nothing, Blue reward:0.85\n", + "step: 1495, Red action: do-nothing, Blue reward:0.85\n", + "step: 1496, Red action: do-nothing, Blue reward:0.85\n", + "step: 1497, Red action: do-nothing, Blue reward:0.85\n", + "step: 1498, Red action: do-nothing, Blue reward:0.85\n", + "step: 1499, Red action: do-nothing, Blue reward:0.85\n", + "step: 1500, Red action: do-nothing, Blue reward:0.85\n", + "step: 1501, Red action: do-nothing, Blue reward:0.85\n", + "step: 1502, Red action: do-nothing, Blue reward:0.85\n", + "step: 1503, Red action: do-nothing, Blue reward:0.85\n", + "step: 1504, Red action: do-nothing, Blue reward:0.85\n", + "step: 1505, Red action: do-nothing, Blue reward:0.85\n", + "step: 1506, Red action: do-nothing, Blue reward:0.85\n", + "step: 1507, Red action: do-nothing, Blue reward:0.85\n", + "step: 1508, Red action: do-nothing, Blue reward:0.85\n", + "step: 1509, Red action: do-nothing, Blue reward:0.85\n", + "step: 1510, Red action: do-nothing, Blue reward:0.85\n", + "step: 1511, Red action: do-nothing, Blue reward:0.85\n", + "step: 1512, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1513, Red action: do-nothing, Blue reward:0.85\n", + "step: 1514, Red action: do-nothing, Blue reward:0.85\n", + "step: 1515, Red action: do-nothing, Blue reward:0.85\n", + "step: 1516, Red action: do-nothing, Blue reward:0.85\n", + "step: 1517, Red action: do-nothing, Blue reward:0.85\n", + "step: 1518, Red action: do-nothing, Blue reward:0.85\n", + "step: 1519, Red action: do-nothing, Blue reward:0.85\n", + "step: 1520, Red action: do-nothing, Blue reward:0.85\n", + "step: 1521, Red action: do-nothing, Blue reward:0.85\n", + "step: 1522, Red action: do-nothing, Blue reward:0.85\n", + "step: 1523, Red action: do-nothing, Blue reward:0.85\n", + "step: 1524, Red action: do-nothing, Blue reward:0.85\n", + "step: 1525, Red action: do-nothing, Blue reward:0.85\n", + "step: 1526, Red action: do-nothing, Blue reward:0.85\n", + "step: 1527, Red action: do-nothing, Blue reward:0.85\n", + "step: 1528, Red action: do-nothing, Blue reward:0.85\n", + "step: 1529, Red action: do-nothing, Blue reward:0.85\n", + "step: 1530, Red action: do-nothing, Blue reward:0.85\n", + "step: 1531, Red action: do-nothing, Blue reward:0.85\n", + "step: 1532, Red action: do-nothing, Blue reward:0.85\n", + "step: 1533, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1534, Red action: do-nothing, Blue reward:0.85\n", + "step: 1535, Red action: do-nothing, Blue reward:0.85\n", + "step: 1536, Red action: do-nothing, Blue reward:0.85\n", + "step: 1537, Red action: do-nothing, Blue reward:0.85\n", + "step: 1538, Red action: do-nothing, Blue reward:0.85\n", + "step: 1539, Red action: do-nothing, Blue reward:0.85\n", + "step: 1540, Red action: do-nothing, Blue reward:0.85\n", + "step: 1541, Red action: do-nothing, Blue reward:0.85\n", + "step: 1542, Red action: do-nothing, Blue reward:0.85\n", + "step: 1543, Red action: do-nothing, Blue reward:0.85\n", + "step: 1544, Red action: do-nothing, Blue reward:0.85\n", + "step: 1545, Red action: do-nothing, Blue reward:0.85\n", + "step: 1546, Red action: do-nothing, Blue reward:0.85\n", + "step: 1547, Red action: do-nothing, Blue reward:0.85\n", + "step: 1548, Red action: do-nothing, Blue reward:0.85\n", + "step: 1549, Red action: do-nothing, Blue reward:0.85\n", + "step: 1550, Red action: do-nothing, Blue reward:0.85\n", + "step: 1551, Red action: do-nothing, Blue reward:0.85\n", + "step: 1552, Red action: do-nothing, Blue reward:0.85\n", + "step: 1553, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1554, Red action: do-nothing, Blue reward:0.85\n", + "step: 1555, Red action: do-nothing, Blue reward:0.85\n", + "step: 1556, Red action: do-nothing, Blue reward:0.85\n", + "step: 1557, Red action: do-nothing, Blue reward:0.85\n", + "step: 1558, Red action: do-nothing, Blue reward:0.85\n", + "step: 1559, Red action: do-nothing, Blue reward:0.85\n", + "step: 1560, Red action: do-nothing, Blue reward:0.85\n", + "step: 1561, Red action: do-nothing, Blue reward:0.85\n", + "step: 1562, Red action: do-nothing, Blue reward:0.85\n", + "step: 1563, Red action: do-nothing, Blue reward:0.85\n", + "step: 1564, Red action: do-nothing, Blue reward:0.85\n", + "step: 1565, Red action: do-nothing, Blue reward:0.85\n", + "step: 1566, Red action: do-nothing, Blue reward:0.85\n", + "step: 1567, Red action: do-nothing, Blue reward:0.85\n", + "step: 1568, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1569, Red action: do-nothing, Blue reward:0.85\n", + "step: 1570, Red action: do-nothing, Blue reward:0.85\n", + "step: 1571, Red action: do-nothing, Blue reward:0.85\n", + "step: 1572, Red action: do-nothing, Blue reward:0.85\n", + "step: 1573, Red action: do-nothing, Blue reward:0.85\n", + "step: 1574, Red action: do-nothing, Blue reward:0.85\n", + "step: 1575, Red action: do-nothing, Blue reward:0.85\n", + "step: 1576, Red action: do-nothing, Blue reward:0.85\n", + "step: 1577, Red action: do-nothing, Blue reward:0.85\n", + "step: 1578, Red action: do-nothing, Blue reward:0.85\n", + "step: 1579, Red action: do-nothing, Blue reward:0.85\n", + "step: 1580, Red action: do-nothing, Blue reward:0.85\n", + "step: 1581, Red action: do-nothing, Blue reward:0.85\n", + "step: 1582, Red action: do-nothing, Blue reward:0.85\n", + "step: 1583, Red action: do-nothing, Blue reward:0.85\n", + "step: 1584, Red action: do-nothing, Blue reward:0.85\n", + "step: 1585, Red action: do-nothing, Blue reward:0.85\n", + "step: 1586, Red action: do-nothing, Blue reward:0.85\n", + "step: 1587, Red action: do-nothing, Blue reward:0.85\n", + "step: 1588, Red action: do-nothing, Blue reward:0.85\n", + "step: 1589, Red action: do-nothing, Blue reward:0.85\n", + "step: 1590, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1591, Red action: do-nothing, Blue reward:0.85\n", + "step: 1592, Red action: do-nothing, Blue reward:0.85\n", + "step: 1593, Red action: do-nothing, Blue reward:0.85\n", + "step: 1594, Red action: do-nothing, Blue reward:0.85\n", + "step: 1595, Red action: do-nothing, Blue reward:0.85\n", + "step: 1596, Red action: do-nothing, Blue reward:0.85\n", + "step: 1597, Red action: do-nothing, Blue reward:0.85\n", + "step: 1598, Red action: do-nothing, Blue reward:0.85\n", + "step: 1599, Red action: do-nothing, Blue reward:0.85\n", + "step: 1600, Red action: do-nothing, Blue reward:0.85\n", + "step: 1601, Red action: do-nothing, Blue reward:0.85\n", + "step: 1602, Red action: do-nothing, Blue reward:0.85\n", + "step: 1603, Red action: do-nothing, Blue reward:0.85\n", + "step: 1604, Red action: do-nothing, Blue reward:0.85\n", + "step: 1605, Red action: do-nothing, Blue reward:0.85\n", + "step: 1606, Red action: do-nothing, Blue reward:0.85\n", + "step: 1607, Red action: do-nothing, Blue reward:0.85\n", + "step: 1608, Red action: do-nothing, Blue reward:0.85\n", + "step: 1609, Red action: do-nothing, Blue reward:0.85\n", + "step: 1610, Red action: do-nothing, Blue reward:0.85\n", + "step: 1611, Red action: do-nothing, Blue reward:0.85\n", + "step: 1612, Red action: do-nothing, Blue reward:0.85\n", + "step: 1613, Red action: do-nothing, Blue reward:0.85\n", + "step: 1614, Red action: do-nothing, Blue reward:0.85\n", + "step: 1615, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1616, Red action: do-nothing, Blue reward:0.85\n", + "step: 1617, Red action: do-nothing, Blue reward:0.85\n", + "step: 1618, Red action: do-nothing, Blue reward:0.85\n", + "step: 1619, Red action: do-nothing, Blue reward:0.85\n", + "step: 1620, Red action: do-nothing, Blue reward:0.85\n", + "step: 1621, Red action: do-nothing, Blue reward:0.85\n", + "step: 1622, Red action: do-nothing, Blue reward:0.85\n", + "step: 1623, Red action: do-nothing, Blue reward:0.85\n", + "step: 1624, Red action: do-nothing, Blue reward:0.85\n", + "step: 1625, Red action: do-nothing, Blue reward:0.85\n", + "step: 1626, Red action: do-nothing, Blue reward:0.85\n", + "step: 1627, Red action: do-nothing, Blue reward:0.85\n", + "step: 1628, Red action: do-nothing, Blue reward:0.85\n", + "step: 1629, Red action: do-nothing, Blue reward:0.85\n", + "step: 1630, Red action: do-nothing, Blue reward:0.85\n", + "step: 1631, Red action: do-nothing, Blue reward:0.85\n", + "step: 1632, Red action: do-nothing, Blue reward:0.85\n", + "step: 1633, Red action: do-nothing, Blue reward:0.85\n", + "step: 1634, Red action: do-nothing, Blue reward:0.85\n", + "step: 1635, Red action: do-nothing, Blue reward:0.85\n", + "step: 1636, Red action: do-nothing, Blue reward:0.85\n", + "step: 1637, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1638, Red action: do-nothing, Blue reward:0.85\n", + "step: 1639, Red action: do-nothing, Blue reward:0.85\n", + "step: 1640, Red action: do-nothing, Blue reward:0.85\n", + "step: 1641, Red action: do-nothing, Blue reward:0.85\n", + "step: 1642, Red action: do-nothing, Blue reward:0.85\n", + "step: 1643, Red action: do-nothing, Blue reward:0.85\n", + "step: 1644, Red action: do-nothing, Blue reward:0.85\n", + "step: 1645, Red action: do-nothing, Blue reward:0.85\n", + "step: 1646, Red action: do-nothing, Blue reward:0.85\n", + "step: 1647, Red action: do-nothing, Blue reward:0.85\n", + "step: 1648, Red action: do-nothing, Blue reward:0.85\n", + "step: 1649, Red action: do-nothing, Blue reward:0.85\n", + "step: 1650, Red action: do-nothing, Blue reward:0.85\n", + "step: 1651, Red action: do-nothing, Blue reward:0.85\n", + "step: 1652, Red action: do-nothing, Blue reward:0.85\n", + "step: 1653, Red action: do-nothing, Blue reward:0.85\n", + "step: 1654, Red action: do-nothing, Blue reward:0.85\n", + "step: 1655, Red action: do-nothing, Blue reward:0.85\n", + "step: 1656, Red action: do-nothing, Blue reward:0.85\n", + "step: 1657, Red action: do-nothing, Blue reward:0.85\n", + "step: 1658, Red action: do-nothing, Blue reward:0.85\n", + "step: 1659, Red action: do-nothing, Blue reward:0.85\n", + "step: 1660, Red action: do-nothing, Blue reward:0.85\n", + "step: 1661, Red action: do-nothing, Blue reward:0.85\n", + "step: 1662, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1663, Red action: do-nothing, Blue reward:0.85\n", + "step: 1664, Red action: do-nothing, Blue reward:0.85\n", + "step: 1665, Red action: do-nothing, Blue reward:0.85\n", + "step: 1666, Red action: do-nothing, Blue reward:0.85\n", + "step: 1667, Red action: do-nothing, Blue reward:0.85\n", + "step: 1668, Red action: do-nothing, Blue reward:0.85\n", + "step: 1669, Red action: do-nothing, Blue reward:0.85\n", + "step: 1670, Red action: do-nothing, Blue reward:0.85\n", + "step: 1671, Red action: do-nothing, Blue reward:0.85\n", + "step: 1672, Red action: do-nothing, Blue reward:0.85\n", + "step: 1673, Red action: do-nothing, Blue reward:0.85\n", + "step: 1674, Red action: do-nothing, Blue reward:0.85\n", + "step: 1675, Red action: do-nothing, Blue reward:0.85\n", + "step: 1676, Red action: do-nothing, Blue reward:0.85\n", + "step: 1677, Red action: do-nothing, Blue reward:0.85\n", + "step: 1678, Red action: do-nothing, Blue reward:0.85\n", + "step: 1679, Red action: do-nothing, Blue reward:0.85\n", + "step: 1680, Red action: do-nothing, Blue reward:0.85\n", + "step: 1681, Red action: do-nothing, Blue reward:0.85\n", + "step: 1682, Red action: do-nothing, Blue reward:0.85\n", + "step: 1683, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1684, Red action: do-nothing, Blue reward:0.85\n", + "step: 1685, Red action: do-nothing, Blue reward:0.85\n", + "step: 1686, Red action: do-nothing, Blue reward:0.85\n", + "step: 1687, Red action: do-nothing, Blue reward:0.85\n", + "step: 1688, Red action: do-nothing, Blue reward:0.85\n", + "step: 1689, Red action: do-nothing, Blue reward:0.85\n", + "step: 1690, Red action: do-nothing, Blue reward:0.85\n", + "step: 1691, Red action: do-nothing, Blue reward:0.85\n", + "step: 1692, Red action: do-nothing, Blue reward:0.85\n", + "step: 1693, Red action: do-nothing, Blue reward:0.85\n", + "step: 1694, Red action: do-nothing, Blue reward:0.85\n", + "step: 1695, Red action: do-nothing, Blue reward:0.85\n", + "step: 1696, Red action: do-nothing, Blue reward:0.85\n", + "step: 1697, Red action: do-nothing, Blue reward:0.85\n", + "step: 1698, Red action: do-nothing, Blue reward:0.85\n", + "step: 1699, Red action: do-nothing, Blue reward:0.85\n", + "step: 1700, Red action: do-nothing, Blue reward:0.85\n", + "step: 1701, Red action: do-nothing, Blue reward:0.85\n", + "step: 1702, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1703, Red action: do-nothing, Blue reward:0.85\n", + "step: 1704, Red action: do-nothing, Blue reward:0.85\n", + "step: 1705, Red action: do-nothing, Blue reward:0.85\n", + "step: 1706, Red action: do-nothing, Blue reward:0.85\n", + "step: 1707, Red action: do-nothing, Blue reward:0.85\n", + "step: 1708, Red action: do-nothing, Blue reward:0.85\n", + "step: 1709, Red action: do-nothing, Blue reward:0.85\n", + "step: 1710, Red action: do-nothing, Blue reward:0.85\n", + "step: 1711, Red action: do-nothing, Blue reward:0.85\n", + "step: 1712, Red action: do-nothing, Blue reward:0.85\n", + "step: 1713, Red action: do-nothing, Blue reward:0.85\n", + "step: 1714, Red action: do-nothing, Blue reward:0.85\n", + "step: 1715, Red action: do-nothing, Blue reward:0.85\n", + "step: 1716, Red action: do-nothing, Blue reward:0.85\n", + "step: 1717, Red action: do-nothing, Blue reward:0.85\n", + "step: 1718, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1719, Red action: do-nothing, Blue reward:0.85\n", + "step: 1720, Red action: do-nothing, Blue reward:0.85\n", + "step: 1721, Red action: do-nothing, Blue reward:0.85\n", + "step: 1722, Red action: do-nothing, Blue reward:0.85\n", + "step: 1723, Red action: do-nothing, Blue reward:0.85\n", + "step: 1724, Red action: do-nothing, Blue reward:0.85\n", + "step: 1725, Red action: do-nothing, Blue reward:0.85\n", + "step: 1726, Red action: do-nothing, Blue reward:0.85\n", + "step: 1727, Red action: do-nothing, Blue reward:0.85\n", + "step: 1728, Red action: do-nothing, Blue reward:0.85\n", + "step: 1729, Red action: do-nothing, Blue reward:0.85\n", + "step: 1730, Red action: do-nothing, Blue reward:0.85\n", + "step: 1731, Red action: do-nothing, Blue reward:0.85\n", + "step: 1732, Red action: do-nothing, Blue reward:0.85\n", + "step: 1733, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1734, Red action: do-nothing, Blue reward:0.85\n", + "step: 1735, Red action: do-nothing, Blue reward:0.85\n", + "step: 1736, Red action: do-nothing, Blue reward:0.85\n", + "step: 1737, Red action: do-nothing, Blue reward:0.85\n", + "step: 1738, Red action: do-nothing, Blue reward:0.85\n", + "step: 1739, Red action: do-nothing, Blue reward:0.85\n", + "step: 1740, Red action: do-nothing, Blue reward:0.85\n", + "step: 1741, Red action: do-nothing, Blue reward:0.85\n", + "step: 1742, Red action: do-nothing, Blue reward:0.85\n", + "step: 1743, Red action: do-nothing, Blue reward:0.85\n", + "step: 1744, Red action: do-nothing, Blue reward:0.85\n", + "step: 1745, Red action: do-nothing, Blue reward:0.85\n", + "step: 1746, Red action: do-nothing, Blue reward:0.85\n", + "step: 1747, Red action: do-nothing, Blue reward:0.85\n", + "step: 1748, Red action: do-nothing, Blue reward:0.85\n", + "step: 1749, Red action: do-nothing, Blue reward:0.85\n", + "step: 1750, Red action: do-nothing, Blue reward:0.85\n", + "step: 1751, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1752, Red action: do-nothing, Blue reward:0.85\n", + "step: 1753, Red action: do-nothing, Blue reward:0.85\n", + "step: 1754, Red action: do-nothing, Blue reward:0.85\n", + "step: 1755, Red action: do-nothing, Blue reward:0.85\n", + "step: 1756, Red action: do-nothing, Blue reward:0.85\n", + "step: 1757, Red action: do-nothing, Blue reward:0.85\n", + "step: 1758, Red action: do-nothing, Blue reward:0.85\n", + "step: 1759, Red action: do-nothing, Blue reward:0.85\n", + "step: 1760, Red action: do-nothing, Blue reward:0.85\n", + "step: 1761, Red action: do-nothing, Blue reward:0.85\n", + "step: 1762, Red action: do-nothing, Blue reward:0.85\n", + "step: 1763, Red action: do-nothing, Blue reward:0.85\n", + "step: 1764, Red action: do-nothing, Blue reward:0.85\n", + "step: 1765, Red action: do-nothing, Blue reward:0.85\n", + "step: 1766, Red action: do-nothing, Blue reward:0.85\n", + "step: 1767, Red action: do-nothing, Blue reward:0.85\n", + "step: 1768, Red action: do-nothing, Blue reward:0.85\n", + "step: 1769, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1770, Red action: do-nothing, Blue reward:0.85\n", + "step: 1771, Red action: do-nothing, Blue reward:0.85\n", + "step: 1772, Red action: do-nothing, Blue reward:0.85\n", + "step: 1773, Red action: do-nothing, Blue reward:0.85\n", + "step: 1774, Red action: do-nothing, Blue reward:0.85\n", + "step: 1775, Red action: do-nothing, Blue reward:0.85\n", + "step: 1776, Red action: do-nothing, Blue reward:0.85\n", + "step: 1777, Red action: do-nothing, Blue reward:0.85\n", + "step: 1778, Red action: do-nothing, Blue reward:0.85\n", + "step: 1779, Red action: do-nothing, Blue reward:0.85\n", + "step: 1780, Red action: do-nothing, Blue reward:0.85\n", + "step: 1781, Red action: do-nothing, Blue reward:0.85\n", + "step: 1782, Red action: do-nothing, Blue reward:0.85\n", + "step: 1783, Red action: do-nothing, Blue reward:0.85\n", + "step: 1784, Red action: do-nothing, Blue reward:0.85\n", + "step: 1785, Red action: do-nothing, Blue reward:0.85\n", + "step: 1786, Red action: do-nothing, Blue reward:0.85\n", + "step: 1787, Red action: do-nothing, Blue reward:0.85\n", + "step: 1788, Red action: do-nothing, Blue reward:0.85\n", + "step: 1789, Red action: do-nothing, Blue reward:0.85\n", + "step: 1790, Red action: do-nothing, Blue reward:0.85\n", + "step: 1791, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1792, Red action: do-nothing, Blue reward:0.85\n", + "step: 1793, Red action: do-nothing, Blue reward:0.85\n", + "step: 1794, Red action: do-nothing, Blue reward:0.85\n", + "step: 1795, Red action: do-nothing, Blue reward:0.85\n", + "step: 1796, Red action: do-nothing, Blue reward:0.85\n", + "step: 1797, Red action: do-nothing, Blue reward:0.85\n", + "step: 1798, Red action: do-nothing, Blue reward:0.85\n", + "step: 1799, Red action: do-nothing, Blue reward:0.85\n", + "step: 1800, Red action: do-nothing, Blue reward:0.85\n", + "step: 1801, Red action: do-nothing, Blue reward:0.85\n", + "step: 1802, Red action: do-nothing, Blue reward:0.85\n", + "step: 1803, Red action: do-nothing, Blue reward:0.85\n", + "step: 1804, Red action: do-nothing, Blue reward:0.85\n", + "step: 1805, Red action: do-nothing, Blue reward:0.85\n", + "step: 1806, Red action: do-nothing, Blue reward:0.85\n", + "step: 1807, Red action: do-nothing, Blue reward:0.85\n", + "step: 1808, Red action: do-nothing, Blue reward:0.85\n", + "step: 1809, Red action: do-nothing, Blue reward:0.85\n", + "step: 1810, Red action: do-nothing, Blue reward:0.85\n", + "step: 1811, Red action: do-nothing, Blue reward:0.85\n", + "step: 1812, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1813, Red action: do-nothing, Blue reward:0.85\n", + "step: 1814, Red action: do-nothing, Blue reward:0.85\n", + "step: 1815, Red action: do-nothing, Blue reward:0.85\n", + "step: 1816, Red action: do-nothing, Blue reward:0.85\n", + "step: 1817, Red action: do-nothing, Blue reward:0.85\n", + "step: 1818, Red action: do-nothing, Blue reward:0.85\n", + "step: 1819, Red action: do-nothing, Blue reward:0.85\n", + "step: 1820, Red action: do-nothing, Blue reward:0.85\n", + "step: 1821, Red action: do-nothing, Blue reward:0.85\n", + "step: 1822, Red action: do-nothing, Blue reward:0.85\n", + "step: 1823, Red action: do-nothing, Blue reward:0.85\n", + "step: 1824, Red action: do-nothing, Blue reward:0.85\n", + "step: 1825, Red action: do-nothing, Blue reward:0.85\n", + "step: 1826, Red action: do-nothing, Blue reward:0.85\n", + "step: 1827, Red action: do-nothing, Blue reward:0.85\n", + "step: 1828, Red action: do-nothing, Blue reward:0.85\n", + "step: 1829, Red action: do-nothing, Blue reward:0.85\n", + "step: 1830, Red action: do-nothing, Blue reward:0.85\n", + "step: 1831, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1832, Red action: do-nothing, Blue reward:0.85\n", + "step: 1833, Red action: do-nothing, Blue reward:0.85\n", + "step: 1834, Red action: do-nothing, Blue reward:0.85\n", + "step: 1835, Red action: do-nothing, Blue reward:0.85\n", + "step: 1836, Red action: do-nothing, Blue reward:0.85\n", + "step: 1837, Red action: do-nothing, Blue reward:0.85\n", + "step: 1838, Red action: do-nothing, Blue reward:0.85\n", + "step: 1839, Red action: do-nothing, Blue reward:0.85\n", + "step: 1840, Red action: do-nothing, Blue reward:0.85\n", + "step: 1841, Red action: do-nothing, Blue reward:0.85\n", + "step: 1842, Red action: do-nothing, Blue reward:0.85\n", + "step: 1843, Red action: do-nothing, Blue reward:0.85\n", + "step: 1844, Red action: do-nothing, Blue reward:0.85\n", + "step: 1845, Red action: do-nothing, Blue reward:0.85\n", + "step: 1846, Red action: do-nothing, Blue reward:0.85\n", + "step: 1847, Red action: do-nothing, Blue reward:0.85\n", + "step: 1848, Red action: do-nothing, Blue reward:0.85\n", + "step: 1849, Red action: do-nothing, Blue reward:0.85\n", + "step: 1850, Red action: do-nothing, Blue reward:0.85\n", + "step: 1851, Red action: do-nothing, Blue reward:0.85\n", + "step: 1852, Red action: do-nothing, Blue reward:0.85\n", + "step: 1853, Red action: do-nothing, Blue reward:0.85\n", + "step: 1854, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1855, Red action: do-nothing, Blue reward:0.85\n", + "step: 1856, Red action: do-nothing, Blue reward:0.85\n", + "step: 1857, Red action: do-nothing, Blue reward:0.85\n", + "step: 1858, Red action: do-nothing, Blue reward:0.85\n", + "step: 1859, Red action: do-nothing, Blue reward:0.85\n", + "step: 1860, Red action: do-nothing, Blue reward:0.85\n", + "step: 1861, Red action: do-nothing, Blue reward:0.85\n", + "step: 1862, Red action: do-nothing, Blue reward:0.85\n", + "step: 1863, Red action: do-nothing, Blue reward:0.85\n", + "step: 1864, Red action: do-nothing, Blue reward:0.85\n", + "step: 1865, Red action: do-nothing, Blue reward:0.85\n", + "step: 1866, Red action: do-nothing, Blue reward:0.85\n", + "step: 1867, Red action: do-nothing, Blue reward:0.85\n", + "step: 1868, Red action: do-nothing, Blue reward:0.85\n", + "step: 1869, Red action: do-nothing, Blue reward:0.85\n", + "step: 1870, Red action: do-nothing, Blue reward:0.85\n", + "step: 1871, Red action: do-nothing, Blue reward:0.85\n", + "step: 1872, Red action: do-nothing, Blue reward:0.85\n", + "step: 1873, Red action: do-nothing, Blue reward:0.85\n", + "step: 1874, Red action: do-nothing, Blue reward:0.85\n", + "step: 1875, Red action: do-nothing, Blue reward:0.85\n", + "step: 1876, Red action: do-nothing, Blue reward:0.85\n", + "step: 1877, Red action: do-nothing, Blue reward:0.85\n", + "step: 1878, Red action: do-nothing, Blue reward:0.85\n", + "step: 1879, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1880, Red action: do-nothing, Blue reward:0.85\n", + "step: 1881, Red action: do-nothing, Blue reward:0.85\n", + "step: 1882, Red action: do-nothing, Blue reward:0.85\n", + "step: 1883, Red action: do-nothing, Blue reward:0.85\n", + "step: 1884, Red action: do-nothing, Blue reward:0.85\n", + "step: 1885, Red action: do-nothing, Blue reward:0.85\n", + "step: 1886, Red action: do-nothing, Blue reward:0.85\n", + "step: 1887, Red action: do-nothing, Blue reward:0.85\n", + "step: 1888, Red action: do-nothing, Blue reward:0.85\n", + "step: 1889, Red action: do-nothing, Blue reward:0.85\n", + "step: 1890, Red action: do-nothing, Blue reward:0.85\n", + "step: 1891, Red action: do-nothing, Blue reward:0.85\n", + "step: 1892, Red action: do-nothing, Blue reward:0.85\n", + "step: 1893, Red action: do-nothing, Blue reward:0.85\n", + "step: 1894, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1895, Red action: do-nothing, Blue reward:0.85\n", + "step: 1896, Red action: do-nothing, Blue reward:0.85\n", + "step: 1897, Red action: do-nothing, Blue reward:0.85\n", + "step: 1898, Red action: do-nothing, Blue reward:0.85\n", + "step: 1899, Red action: do-nothing, Blue reward:0.85\n", + "step: 1900, Red action: do-nothing, Blue reward:0.85\n", + "step: 1901, Red action: do-nothing, Blue reward:0.85\n", + "step: 1902, Red action: do-nothing, Blue reward:0.85\n", + "step: 1903, Red action: do-nothing, Blue reward:0.85\n", + "step: 1904, Red action: do-nothing, Blue reward:0.85\n", + "step: 1905, Red action: do-nothing, Blue reward:0.85\n", + "step: 1906, Red action: do-nothing, Blue reward:0.85\n", + "step: 1907, Red action: do-nothing, Blue reward:0.85\n", + "step: 1908, Red action: do-nothing, Blue reward:0.85\n", + "step: 1909, Red action: do-nothing, Blue reward:0.85\n", + "step: 1910, Red action: do-nothing, Blue reward:0.85\n", + "step: 1911, Red action: do-nothing, Blue reward:0.85\n", + "step: 1912, Red action: do-nothing, Blue reward:0.85\n", + "step: 1913, Red action: do-nothing, Blue reward:0.85\n", + "step: 1914, Red action: do-nothing, Blue reward:0.85\n", + "step: 1915, Red action: do-nothing, Blue reward:0.85\n", + "step: 1916, Red action: do-nothing, Blue reward:0.85\n", + "step: 1917, Red action: do-nothing, Blue reward:0.85\n", + "step: 1918, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1919, Red action: do-nothing, Blue reward:0.85\n", + "step: 1920, Red action: do-nothing, Blue reward:0.85\n", + "step: 1921, Red action: do-nothing, Blue reward:0.85\n", + "step: 1922, Red action: do-nothing, Blue reward:0.85\n", + "step: 1923, Red action: do-nothing, Blue reward:0.85\n", + "step: 1924, Red action: do-nothing, Blue reward:0.85\n", + "step: 1925, Red action: do-nothing, Blue reward:0.85\n", + "step: 1926, Red action: do-nothing, Blue reward:0.85\n", + "step: 1927, Red action: do-nothing, Blue reward:0.85\n", + "step: 1928, Red action: do-nothing, Blue reward:0.85\n", + "step: 1929, Red action: do-nothing, Blue reward:0.85\n", + "step: 1930, Red action: do-nothing, Blue reward:0.85\n", + "step: 1931, Red action: do-nothing, Blue reward:0.85\n", + "step: 1932, Red action: do-nothing, Blue reward:0.85\n", + "step: 1933, Red action: do-nothing, Blue reward:0.85\n", + "step: 1934, Red action: do-nothing, Blue reward:0.85\n", + "step: 1935, Red action: do-nothing, Blue reward:0.85\n", + "step: 1936, Red action: do-nothing, Blue reward:0.85\n", + "step: 1937, Red action: do-nothing, Blue reward:0.85\n", + "step: 1938, Red action: do-nothing, Blue reward:0.85\n", + "step: 1939, Red action: do-nothing, Blue reward:0.85\n", + "step: 1940, Red action: do-nothing, Blue reward:0.85\n", + "step: 1941, Red action: do-nothing, Blue reward:0.85\n", + "step: 1942, Red action: do-nothing, Blue reward:0.85\n", + "step: 1943, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1944, Red action: do-nothing, Blue reward:0.85\n", + "step: 1945, Red action: do-nothing, Blue reward:0.85\n", + "step: 1946, Red action: do-nothing, Blue reward:0.85\n", + "step: 1947, Red action: do-nothing, Blue reward:0.85\n", + "step: 1948, Red action: do-nothing, Blue reward:0.85\n", + "step: 1949, Red action: do-nothing, Blue reward:0.85\n", + "step: 1950, Red action: do-nothing, Blue reward:0.85\n", + "step: 1951, Red action: do-nothing, Blue reward:0.85\n", + "step: 1952, Red action: do-nothing, Blue reward:0.85\n", + "step: 1953, Red action: do-nothing, Blue reward:0.85\n", + "step: 1954, Red action: do-nothing, Blue reward:0.85\n", + "step: 1955, Red action: do-nothing, Blue reward:0.85\n", + "step: 1956, Red action: do-nothing, Blue reward:0.85\n", + "step: 1957, Red action: do-nothing, Blue reward:0.85\n", + "step: 1958, Red action: do-nothing, Blue reward:0.85\n", + "step: 1959, Red action: do-nothing, Blue reward:0.85\n", + "step: 1960, Red action: do-nothing, Blue reward:0.85\n", + "step: 1961, Red action: do-nothing, Blue reward:0.85\n", + "step: 1962, Red action: do-nothing, Blue reward:0.85\n", + "step: 1963, Red action: do-nothing, Blue reward:0.85\n", + "step: 1964, Red action: do-nothing, Blue reward:0.85\n", + "step: 1965, Red action: do-nothing, Blue reward:0.85\n", + "step: 1966, Red action: do-nothing, Blue reward:0.85\n", + "step: 1967, Red action: do-nothing, Blue reward:0.85\n", + "step: 1968, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1969, Red action: do-nothing, Blue reward:0.85\n", + "step: 1970, Red action: do-nothing, Blue reward:0.85\n", + "step: 1971, Red action: do-nothing, Blue reward:0.85\n", + "step: 1972, Red action: do-nothing, Blue reward:0.85\n", + "step: 1973, Red action: do-nothing, Blue reward:0.85\n", + "step: 1974, Red action: do-nothing, Blue reward:0.85\n", + "step: 1975, Red action: do-nothing, Blue reward:0.85\n", + "step: 1976, Red action: do-nothing, Blue reward:0.85\n", + "step: 1977, Red action: do-nothing, Blue reward:0.85\n", + "step: 1978, Red action: do-nothing, Blue reward:0.85\n", + "step: 1979, Red action: do-nothing, Blue reward:0.85\n", + "step: 1980, Red action: do-nothing, Blue reward:0.85\n", + "step: 1981, Red action: do-nothing, Blue reward:0.85\n", + "step: 1982, Red action: do-nothing, Blue reward:0.85\n", + "step: 1983, Red action: do-nothing, Blue reward:0.85\n", + "step: 1984, Red action: do-nothing, Blue reward:0.85\n", + "step: 1985, Red action: do-nothing, Blue reward:0.85\n", + "step: 1986, Red action: do-nothing, Blue reward:0.85\n", + "step: 1987, Red action: node-application-execute, Blue reward:0.85\n", + "step: 1988, Red action: do-nothing, Blue reward:0.85\n", + "step: 1989, Red action: do-nothing, Blue reward:0.85\n", + "step: 1990, Red action: do-nothing, Blue reward:0.85\n", + "step: 1991, Red action: do-nothing, Blue reward:0.85\n", + "step: 1992, Red action: do-nothing, Blue reward:0.85\n", + "step: 1993, Red action: do-nothing, Blue reward:0.85\n", + "step: 1994, Red action: do-nothing, Blue reward:0.85\n", + "step: 1995, Red action: do-nothing, Blue reward:0.85\n", + "step: 1996, Red action: do-nothing, Blue reward:0.85\n", + "step: 1997, Red action: do-nothing, Blue reward:0.85\n", + "step: 1998, Red action: do-nothing, Blue reward:0.85\n", + "step: 1999, Red action: do-nothing, Blue reward:0.85\n", + "step: 2000, Red action: do-nothing, Blue reward:0.85\n", + "step: 2001, Red action: do-nothing, Blue reward:0.85\n", + "step: 2002, Red action: do-nothing, Blue reward:0.85\n", + "step: 2003, Red action: do-nothing, Blue reward:0.85\n", + "step: 2004, Red action: do-nothing, Blue reward:0.85\n", + "step: 2005, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2006, Red action: do-nothing, Blue reward:0.85\n", + "step: 2007, Red action: do-nothing, Blue reward:0.85\n", + "step: 2008, Red action: do-nothing, Blue reward:0.85\n", + "step: 2009, Red action: do-nothing, Blue reward:0.85\n", + "step: 2010, Red action: do-nothing, Blue reward:0.85\n", + "step: 2011, Red action: do-nothing, Blue reward:0.85\n", + "step: 2012, Red action: do-nothing, Blue reward:0.85\n", + "step: 2013, Red action: do-nothing, Blue reward:0.85\n", + "step: 2014, Red action: do-nothing, Blue reward:0.85\n", + "step: 2015, Red action: do-nothing, Blue reward:0.85\n", + "step: 2016, Red action: do-nothing, Blue reward:0.85\n", + "step: 2017, Red action: do-nothing, Blue reward:0.85\n", + "step: 2018, Red action: do-nothing, Blue reward:0.85\n", + "step: 2019, Red action: do-nothing, Blue reward:0.85\n", + "step: 2020, Red action: do-nothing, Blue reward:0.85\n", + "step: 2021, Red action: do-nothing, Blue reward:0.85\n", + "step: 2022, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2023, Red action: do-nothing, Blue reward:0.85\n", + "step: 2024, Red action: do-nothing, Blue reward:0.85\n", + "step: 2025, Red action: do-nothing, Blue reward:0.85\n", + "step: 2026, Red action: do-nothing, Blue reward:0.85\n", + "step: 2027, Red action: do-nothing, Blue reward:0.85\n", + "step: 2028, Red action: do-nothing, Blue reward:0.85\n", + "step: 2029, Red action: do-nothing, Blue reward:0.85\n", + "step: 2030, Red action: do-nothing, Blue reward:0.85\n", + "step: 2031, Red action: do-nothing, Blue reward:0.85\n", + "step: 2032, Red action: do-nothing, Blue reward:0.85\n", + "step: 2033, Red action: do-nothing, Blue reward:0.85\n", + "step: 2034, Red action: do-nothing, Blue reward:0.85\n", + "step: 2035, Red action: do-nothing, Blue reward:0.85\n", + "step: 2036, Red action: do-nothing, Blue reward:0.85\n", + "step: 2037, Red action: do-nothing, Blue reward:0.85\n", + "step: 2038, Red action: do-nothing, Blue reward:0.85\n", + "step: 2039, Red action: do-nothing, Blue reward:0.85\n", + "step: 2040, Red action: do-nothing, Blue reward:0.85\n", + "step: 2041, Red action: do-nothing, Blue reward:0.85\n", + "step: 2042, Red action: do-nothing, Blue reward:0.85\n", + "step: 2043, Red action: do-nothing, Blue reward:0.85\n", + "step: 2044, Red action: do-nothing, Blue reward:0.85\n", + "step: 2045, Red action: do-nothing, Blue reward:0.85\n", + "step: 2046, Red action: do-nothing, Blue reward:0.85\n", + "step: 2047, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2048, Red action: do-nothing, Blue reward:0.85\n", + "step: 2049, Red action: do-nothing, Blue reward:0.85\n", + "step: 2050, Red action: do-nothing, Blue reward:0.85\n", + "step: 2051, Red action: do-nothing, Blue reward:0.85\n", + "step: 2052, Red action: do-nothing, Blue reward:0.85\n", + "step: 2053, Red action: do-nothing, Blue reward:0.85\n", + "step: 2054, Red action: do-nothing, Blue reward:0.85\n", + "step: 2055, Red action: do-nothing, Blue reward:0.85\n", + "step: 2056, Red action: do-nothing, Blue reward:0.85\n", + "step: 2057, Red action: do-nothing, Blue reward:0.85\n", + "step: 2058, Red action: do-nothing, Blue reward:0.85\n", + "step: 2059, Red action: do-nothing, Blue reward:0.85\n", + "step: 2060, Red action: do-nothing, Blue reward:0.85\n", + "step: 2061, Red action: do-nothing, Blue reward:0.85\n", + "step: 2062, Red action: do-nothing, Blue reward:0.85\n", + "step: 2063, Red action: do-nothing, Blue reward:0.85\n", + "step: 2064, Red action: do-nothing, Blue reward:0.85\n", + "step: 2065, Red action: do-nothing, Blue reward:0.85\n", + "step: 2066, Red action: do-nothing, Blue reward:0.85\n", + "step: 2067, Red action: do-nothing, Blue reward:0.85\n", + "step: 2068, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2069, Red action: do-nothing, Blue reward:0.85\n", + "step: 2070, Red action: do-nothing, Blue reward:0.85\n", + "step: 2071, Red action: do-nothing, Blue reward:0.85\n", + "step: 2072, Red action: do-nothing, Blue reward:0.85\n", + "step: 2073, Red action: do-nothing, Blue reward:0.85\n", + "step: 2074, Red action: do-nothing, Blue reward:0.85\n", + "step: 2075, Red action: do-nothing, Blue reward:0.85\n", + "step: 2076, Red action: do-nothing, Blue reward:0.85\n", + "step: 2077, Red action: do-nothing, Blue reward:0.85\n", + "step: 2078, Red action: do-nothing, Blue reward:0.85\n", + "step: 2079, Red action: do-nothing, Blue reward:0.85\n", + "step: 2080, Red action: do-nothing, Blue reward:0.85\n", + "step: 2081, Red action: do-nothing, Blue reward:0.85\n", + "step: 2082, Red action: do-nothing, Blue reward:0.85\n", + "step: 2083, Red action: do-nothing, Blue reward:0.85\n", + "step: 2084, Red action: do-nothing, Blue reward:0.85\n", + "step: 2085, Red action: do-nothing, Blue reward:0.85\n", + "step: 2086, Red action: do-nothing, Blue reward:0.85\n", + "step: 2087, Red action: do-nothing, Blue reward:0.85\n", + "step: 2088, Red action: do-nothing, Blue reward:0.85\n", + "step: 2089, Red action: do-nothing, Blue reward:0.85\n", + "step: 2090, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2091, Red action: do-nothing, Blue reward:0.85\n", + "step: 2092, Red action: do-nothing, Blue reward:0.85\n", + "step: 2093, Red action: do-nothing, Blue reward:0.85\n", + "step: 2094, Red action: do-nothing, Blue reward:0.85\n", + "step: 2095, Red action: do-nothing, Blue reward:0.85\n", + "step: 2096, Red action: do-nothing, Blue reward:0.85\n", + "step: 2097, Red action: do-nothing, Blue reward:0.85\n", + "step: 2098, Red action: do-nothing, Blue reward:0.85\n", + "step: 2099, Red action: do-nothing, Blue reward:0.85\n", + "step: 2100, Red action: do-nothing, Blue reward:0.85\n", + "step: 2101, Red action: do-nothing, Blue reward:0.85\n", + "step: 2102, Red action: do-nothing, Blue reward:0.85\n", + "step: 2103, Red action: do-nothing, Blue reward:0.85\n", + "step: 2104, Red action: do-nothing, Blue reward:0.85\n", + "step: 2105, Red action: do-nothing, Blue reward:0.85\n", + "step: 2106, Red action: do-nothing, Blue reward:0.85\n", + "step: 2107, Red action: do-nothing, Blue reward:0.85\n", + "step: 2108, Red action: do-nothing, Blue reward:0.85\n", + "step: 2109, Red action: do-nothing, Blue reward:0.85\n", + "step: 2110, Red action: do-nothing, Blue reward:0.85\n", + "step: 2111, Red action: do-nothing, Blue reward:0.85\n", + "step: 2112, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2113, Red action: do-nothing, Blue reward:0.85\n", + "step: 2114, Red action: do-nothing, Blue reward:0.85\n", + "step: 2115, Red action: do-nothing, Blue reward:0.85\n", + "step: 2116, Red action: do-nothing, Blue reward:0.85\n", + "step: 2117, Red action: do-nothing, Blue reward:0.85\n", + "step: 2118, Red action: do-nothing, Blue reward:0.85\n", + "step: 2119, Red action: do-nothing, Blue reward:0.85\n", + "step: 2120, Red action: do-nothing, Blue reward:0.85\n", + "step: 2121, Red action: do-nothing, Blue reward:0.85\n", + "step: 2122, Red action: do-nothing, Blue reward:0.85\n", + "step: 2123, Red action: do-nothing, Blue reward:0.85\n", + "step: 2124, Red action: do-nothing, Blue reward:0.85\n", + "step: 2125, Red action: do-nothing, Blue reward:0.85\n", + "step: 2126, Red action: do-nothing, Blue reward:0.85\n", + "step: 2127, Red action: do-nothing, Blue reward:0.85\n", + "step: 2128, Red action: do-nothing, Blue reward:0.85\n", + "step: 2129, Red action: do-nothing, Blue reward:0.85\n", + "step: 2130, Red action: do-nothing, Blue reward:0.85\n", + "step: 2131, Red action: do-nothing, Blue reward:0.85\n", + "step: 2132, Red action: do-nothing, Blue reward:0.85\n", + "step: 2133, Red action: do-nothing, Blue reward:0.85\n", + "step: 2134, Red action: do-nothing, Blue reward:0.85\n", + "step: 2135, Red action: do-nothing, Blue reward:0.85\n", + "step: 2136, Red action: do-nothing, Blue reward:0.85\n", + "step: 2137, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2138, Red action: do-nothing, Blue reward:0.85\n", + "step: 2139, Red action: do-nothing, Blue reward:0.85\n", + "step: 2140, Red action: do-nothing, Blue reward:0.85\n", + "step: 2141, Red action: do-nothing, Blue reward:0.85\n", + "step: 2142, Red action: do-nothing, Blue reward:0.85\n", + "step: 2143, Red action: do-nothing, Blue reward:0.85\n", + "step: 2144, Red action: do-nothing, Blue reward:0.85\n", + "step: 2145, Red action: do-nothing, Blue reward:0.85\n", + "step: 2146, Red action: do-nothing, Blue reward:0.85\n", + "step: 2147, Red action: do-nothing, Blue reward:0.85\n", + "step: 2148, Red action: do-nothing, Blue reward:0.85\n", + "step: 2149, Red action: do-nothing, Blue reward:0.85\n", + "step: 2150, Red action: do-nothing, Blue reward:0.85\n", + "step: 2151, Red action: do-nothing, Blue reward:0.85\n", + "step: 2152, Red action: do-nothing, Blue reward:0.85\n", + "step: 2153, Red action: do-nothing, Blue reward:0.85\n", + "step: 2154, Red action: do-nothing, Blue reward:0.85\n", + "step: 2155, Red action: do-nothing, Blue reward:0.85\n", + "step: 2156, Red action: do-nothing, Blue reward:0.85\n", + "step: 2157, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2158, Red action: do-nothing, Blue reward:0.85\n", + "step: 2159, Red action: do-nothing, Blue reward:0.85\n", + "step: 2160, Red action: do-nothing, Blue reward:0.85\n", + "step: 2161, Red action: do-nothing, Blue reward:0.85\n", + "step: 2162, Red action: do-nothing, Blue reward:0.85\n", + "step: 2163, Red action: do-nothing, Blue reward:0.85\n", + "step: 2164, Red action: do-nothing, Blue reward:0.85\n", + "step: 2165, Red action: do-nothing, Blue reward:0.85\n", + "step: 2166, Red action: do-nothing, Blue reward:0.85\n", + "step: 2167, Red action: do-nothing, Blue reward:0.85\n", + "step: 2168, Red action: do-nothing, Blue reward:0.85\n", + "step: 2169, Red action: do-nothing, Blue reward:0.85\n", + "step: 2170, Red action: do-nothing, Blue reward:0.85\n", + "step: 2171, Red action: do-nothing, Blue reward:0.85\n", + "step: 2172, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2173, Red action: do-nothing, Blue reward:0.85\n", + "step: 2174, Red action: do-nothing, Blue reward:0.85\n", + "step: 2175, Red action: do-nothing, Blue reward:0.85\n", + "step: 2176, Red action: do-nothing, Blue reward:0.85\n", + "step: 2177, Red action: do-nothing, Blue reward:0.85\n", + "step: 2178, Red action: do-nothing, Blue reward:0.85\n", + "step: 2179, Red action: do-nothing, Blue reward:0.85\n", + "step: 2180, Red action: do-nothing, Blue reward:0.85\n", + "step: 2181, Red action: do-nothing, Blue reward:0.85\n", + "step: 2182, Red action: do-nothing, Blue reward:0.85\n", + "step: 2183, Red action: do-nothing, Blue reward:0.85\n", + "step: 2184, Red action: do-nothing, Blue reward:0.85\n", + "step: 2185, Red action: do-nothing, Blue reward:0.85\n", + "step: 2186, Red action: do-nothing, Blue reward:0.85\n", + "step: 2187, Red action: do-nothing, Blue reward:0.85\n", + "step: 2188, Red action: do-nothing, Blue reward:0.85\n", + "step: 2189, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2190, Red action: do-nothing, Blue reward:0.85\n", + "step: 2191, Red action: do-nothing, Blue reward:0.85\n", + "step: 2192, Red action: do-nothing, Blue reward:0.85\n", + "step: 2193, Red action: do-nothing, Blue reward:0.85\n", + "step: 2194, Red action: do-nothing, Blue reward:0.85\n", + "step: 2195, Red action: do-nothing, Blue reward:0.85\n", + "step: 2196, Red action: do-nothing, Blue reward:0.85\n", + "step: 2197, Red action: do-nothing, Blue reward:0.85\n", + "step: 2198, Red action: do-nothing, Blue reward:0.85\n", + "step: 2199, Red action: do-nothing, Blue reward:0.85\n", + "step: 2200, Red action: do-nothing, Blue reward:0.85\n", + "step: 2201, Red action: do-nothing, Blue reward:0.85\n", + "step: 2202, Red action: do-nothing, Blue reward:0.85\n", + "step: 2203, Red action: do-nothing, Blue reward:0.85\n", + "step: 2204, Red action: do-nothing, Blue reward:0.85\n", + "step: 2205, Red action: do-nothing, Blue reward:0.85\n", + "step: 2206, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2207, Red action: do-nothing, Blue reward:0.85\n", + "step: 2208, Red action: do-nothing, Blue reward:0.85\n", + "step: 2209, Red action: do-nothing, Blue reward:0.85\n", + "step: 2210, Red action: do-nothing, Blue reward:0.85\n", + "step: 2211, Red action: do-nothing, Blue reward:0.85\n", + "step: 2212, Red action: do-nothing, Blue reward:0.85\n", + "step: 2213, Red action: do-nothing, Blue reward:0.85\n", + "step: 2214, Red action: do-nothing, Blue reward:0.85\n", + "step: 2215, Red action: do-nothing, Blue reward:0.85\n", + "step: 2216, Red action: do-nothing, Blue reward:0.85\n", + "step: 2217, Red action: do-nothing, Blue reward:0.85\n", + "step: 2218, Red action: do-nothing, Blue reward:0.85\n", + "step: 2219, Red action: do-nothing, Blue reward:0.85\n", + "step: 2220, Red action: do-nothing, Blue reward:0.85\n", + "step: 2221, Red action: do-nothing, Blue reward:0.85\n", + "step: 2222, Red action: do-nothing, Blue reward:0.85\n", + "step: 2223, Red action: do-nothing, Blue reward:0.85\n", + "step: 2224, Red action: do-nothing, Blue reward:0.85\n", + "step: 2225, Red action: do-nothing, Blue reward:0.85\n", + "step: 2226, Red action: do-nothing, Blue reward:0.85\n", + "step: 2227, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2228, Red action: do-nothing, Blue reward:0.85\n", + "step: 2229, Red action: do-nothing, Blue reward:0.85\n", + "step: 2230, Red action: do-nothing, Blue reward:0.85\n", + "step: 2231, Red action: do-nothing, Blue reward:0.85\n", + "step: 2232, Red action: do-nothing, Blue reward:0.85\n", + "step: 2233, Red action: do-nothing, Blue reward:0.85\n", + "step: 2234, Red action: do-nothing, Blue reward:0.85\n", + "step: 2235, Red action: do-nothing, Blue reward:0.85\n", + "step: 2236, Red action: do-nothing, Blue reward:0.85\n", + "step: 2237, Red action: do-nothing, Blue reward:0.85\n", + "step: 2238, Red action: do-nothing, Blue reward:0.85\n", + "step: 2239, Red action: do-nothing, Blue reward:0.85\n", + "step: 2240, Red action: do-nothing, Blue reward:0.85\n", + "step: 2241, Red action: do-nothing, Blue reward:0.85\n", + "step: 2242, Red action: do-nothing, Blue reward:0.85\n", + "step: 2243, Red action: do-nothing, Blue reward:0.85\n", + "step: 2244, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2245, Red action: do-nothing, Blue reward:0.85\n", + "step: 2246, Red action: do-nothing, Blue reward:0.85\n", + "step: 2247, Red action: do-nothing, Blue reward:0.85\n", + "step: 2248, Red action: do-nothing, Blue reward:0.85\n", + "step: 2249, Red action: do-nothing, Blue reward:0.85\n", + "step: 2250, Red action: do-nothing, Blue reward:0.85\n", + "step: 2251, Red action: do-nothing, Blue reward:0.85\n", + "step: 2252, Red action: do-nothing, Blue reward:0.85\n", + "step: 2253, Red action: do-nothing, Blue reward:0.85\n", + "step: 2254, Red action: do-nothing, Blue reward:0.85\n", + "step: 2255, Red action: do-nothing, Blue reward:0.85\n", + "step: 2256, Red action: do-nothing, Blue reward:0.85\n", + "step: 2257, Red action: do-nothing, Blue reward:0.85\n", + "step: 2258, Red action: do-nothing, Blue reward:0.85\n", + "step: 2259, Red action: do-nothing, Blue reward:0.85\n", + "step: 2260, Red action: do-nothing, Blue reward:0.85\n", + "step: 2261, Red action: do-nothing, Blue reward:0.85\n", + "step: 2262, Red action: do-nothing, Blue reward:0.85\n", + "step: 2263, Red action: do-nothing, Blue reward:0.85\n", + "step: 2264, Red action: do-nothing, Blue reward:0.85\n", + "step: 2265, Red action: do-nothing, Blue reward:0.85\n", + "step: 2266, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2267, Red action: do-nothing, Blue reward:0.85\n", + "step: 2268, Red action: do-nothing, Blue reward:0.85\n", + "step: 2269, Red action: do-nothing, Blue reward:0.85\n", + "step: 2270, Red action: do-nothing, Blue reward:0.85\n", + "step: 2271, Red action: do-nothing, Blue reward:0.85\n", + "step: 2272, Red action: do-nothing, Blue reward:0.85\n", + "step: 2273, Red action: do-nothing, Blue reward:0.85\n", + "step: 2274, Red action: do-nothing, Blue reward:0.85\n", + "step: 2275, Red action: do-nothing, Blue reward:0.85\n", + "step: 2276, Red action: do-nothing, Blue reward:0.85\n", + "step: 2277, Red action: do-nothing, Blue reward:0.85\n", + "step: 2278, Red action: do-nothing, Blue reward:0.85\n", + "step: 2279, Red action: do-nothing, Blue reward:0.85\n", + "step: 2280, Red action: do-nothing, Blue reward:0.85\n", + "step: 2281, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2282, Red action: do-nothing, Blue reward:0.85\n", + "step: 2283, Red action: do-nothing, Blue reward:0.85\n", + "step: 2284, Red action: do-nothing, Blue reward:0.85\n", + "step: 2285, Red action: do-nothing, Blue reward:0.85\n", + "step: 2286, Red action: do-nothing, Blue reward:0.85\n", + "step: 2287, Red action: do-nothing, Blue reward:0.85\n", + "step: 2288, Red action: do-nothing, Blue reward:0.85\n", + "step: 2289, Red action: do-nothing, Blue reward:0.85\n", + "step: 2290, Red action: do-nothing, Blue reward:0.85\n", + "step: 2291, Red action: do-nothing, Blue reward:0.85\n", + "step: 2292, Red action: do-nothing, Blue reward:0.85\n", + "step: 2293, Red action: do-nothing, Blue reward:0.85\n", + "step: 2294, Red action: do-nothing, Blue reward:0.85\n", + "step: 2295, Red action: do-nothing, Blue reward:0.85\n", + "step: 2296, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2297, Red action: do-nothing, Blue reward:0.85\n", + "step: 2298, Red action: do-nothing, Blue reward:0.85\n", + "step: 2299, Red action: do-nothing, Blue reward:0.85\n", + "step: 2300, Red action: do-nothing, Blue reward:0.85\n", + "step: 2301, Red action: do-nothing, Blue reward:0.85\n", + "step: 2302, Red action: do-nothing, Blue reward:0.85\n", + "step: 2303, Red action: do-nothing, Blue reward:0.85\n", + "step: 2304, Red action: do-nothing, Blue reward:0.85\n", + "step: 2305, Red action: do-nothing, Blue reward:0.85\n", + "step: 2306, Red action: do-nothing, Blue reward:0.85\n", + "step: 2307, Red action: do-nothing, Blue reward:0.85\n", + "step: 2308, Red action: do-nothing, Blue reward:0.85\n", + "step: 2309, Red action: do-nothing, Blue reward:0.85\n", + "step: 2310, Red action: do-nothing, Blue reward:0.85\n", + "step: 2311, Red action: do-nothing, Blue reward:0.85\n", + "step: 2312, Red action: do-nothing, Blue reward:0.85\n", + "step: 2313, Red action: do-nothing, Blue reward:0.85\n", + "step: 2314, Red action: do-nothing, Blue reward:0.85\n", + "step: 2315, Red action: do-nothing, Blue reward:0.85\n", + "step: 2316, Red action: do-nothing, Blue reward:0.85\n", + "step: 2317, Red action: do-nothing, Blue reward:0.85\n", + "step: 2318, Red action: do-nothing, Blue reward:0.85\n", + "step: 2319, Red action: do-nothing, Blue reward:0.85\n", + "step: 2320, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2321, Red action: do-nothing, Blue reward:0.85\n", + "step: 2322, Red action: do-nothing, Blue reward:0.85\n", + "step: 2323, Red action: do-nothing, Blue reward:0.85\n", + "step: 2324, Red action: do-nothing, Blue reward:0.85\n", + "step: 2325, Red action: do-nothing, Blue reward:0.85\n", + "step: 2326, Red action: do-nothing, Blue reward:0.85\n", + "step: 2327, Red action: do-nothing, Blue reward:0.85\n", + "step: 2328, Red action: do-nothing, Blue reward:0.85\n", + "step: 2329, Red action: do-nothing, Blue reward:0.85\n", + "step: 2330, Red action: do-nothing, Blue reward:0.85\n", + "step: 2331, Red action: do-nothing, Blue reward:0.85\n", + "step: 2332, Red action: do-nothing, Blue reward:0.85\n", + "step: 2333, Red action: do-nothing, Blue reward:0.85\n", + "step: 2334, Red action: do-nothing, Blue reward:0.85\n", + "step: 2335, Red action: do-nothing, Blue reward:0.85\n", + "step: 2336, Red action: do-nothing, Blue reward:0.85\n", + "step: 2337, Red action: do-nothing, Blue reward:0.85\n", + "step: 2338, Red action: do-nothing, Blue reward:0.85\n", + "step: 2339, Red action: do-nothing, Blue reward:0.85\n", + "step: 2340, Red action: do-nothing, Blue reward:0.85\n", + "step: 2341, Red action: do-nothing, Blue reward:0.85\n", + "step: 2342, Red action: do-nothing, Blue reward:0.85\n", + "step: 2343, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2344, Red action: do-nothing, Blue reward:0.85\n", + "step: 2345, Red action: do-nothing, Blue reward:0.85\n", + "step: 2346, Red action: do-nothing, Blue reward:0.85\n", + "step: 2347, Red action: do-nothing, Blue reward:0.85\n", + "step: 2348, Red action: do-nothing, Blue reward:0.85\n", + "step: 2349, Red action: do-nothing, Blue reward:0.85\n", + "step: 2350, Red action: do-nothing, Blue reward:0.85\n", + "step: 2351, Red action: do-nothing, Blue reward:0.85\n", + "step: 2352, Red action: do-nothing, Blue reward:0.85\n", + "step: 2353, Red action: do-nothing, Blue reward:0.85\n", + "step: 2354, Red action: do-nothing, Blue reward:0.85\n", + "step: 2355, Red action: do-nothing, Blue reward:0.85\n", + "step: 2356, Red action: do-nothing, Blue reward:0.85\n", + "step: 2357, Red action: do-nothing, Blue reward:0.85\n", + "step: 2358, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2359, Red action: do-nothing, Blue reward:0.85\n", + "step: 2360, Red action: do-nothing, Blue reward:0.85\n", + "step: 2361, Red action: do-nothing, Blue reward:0.85\n", + "step: 2362, Red action: do-nothing, Blue reward:0.85\n", + "step: 2363, Red action: do-nothing, Blue reward:0.85\n", + "step: 2364, Red action: do-nothing, Blue reward:0.85\n", + "step: 2365, Red action: do-nothing, Blue reward:0.85\n", + "step: 2366, Red action: do-nothing, Blue reward:0.85\n", + "step: 2367, Red action: do-nothing, Blue reward:0.85\n", + "step: 2368, Red action: do-nothing, Blue reward:0.85\n", + "step: 2369, Red action: do-nothing, Blue reward:0.85\n", + "step: 2370, Red action: do-nothing, Blue reward:0.85\n", + "step: 2371, Red action: do-nothing, Blue reward:0.85\n", + "step: 2372, Red action: do-nothing, Blue reward:0.85\n", + "step: 2373, Red action: do-nothing, Blue reward:0.85\n", + "step: 2374, Red action: do-nothing, Blue reward:0.85\n", + "step: 2375, Red action: do-nothing, Blue reward:0.85\n", + "step: 2376, Red action: do-nothing, Blue reward:0.85\n", + "step: 2377, Red action: do-nothing, Blue reward:0.85\n", + "step: 2378, Red action: do-nothing, Blue reward:0.85\n", + "step: 2379, Red action: do-nothing, Blue reward:0.85\n", + "step: 2380, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2381, Red action: do-nothing, Blue reward:0.85\n", + "step: 2382, Red action: do-nothing, Blue reward:0.85\n", + "step: 2383, Red action: do-nothing, Blue reward:0.85\n", + "step: 2384, Red action: do-nothing, Blue reward:0.85\n", + "step: 2385, Red action: do-nothing, Blue reward:0.85\n", + "step: 2386, Red action: do-nothing, Blue reward:0.85\n", + "step: 2387, Red action: do-nothing, Blue reward:0.85\n", + "step: 2388, Red action: do-nothing, Blue reward:0.85\n", + "step: 2389, Red action: do-nothing, Blue reward:0.85\n", + "step: 2390, Red action: do-nothing, Blue reward:0.85\n", + "step: 2391, Red action: do-nothing, Blue reward:0.85\n", + "step: 2392, Red action: do-nothing, Blue reward:0.85\n", + "step: 2393, Red action: do-nothing, Blue reward:0.85\n", + "step: 2394, Red action: do-nothing, Blue reward:0.85\n", + "step: 2395, Red action: do-nothing, Blue reward:0.85\n", + "step: 2396, Red action: do-nothing, Blue reward:0.85\n", + "step: 2397, Red action: do-nothing, Blue reward:0.85\n", + "step: 2398, Red action: do-nothing, Blue reward:0.85\n", + "step: 2399, Red action: do-nothing, Blue reward:0.85\n", + "step: 2400, Red action: do-nothing, Blue reward:0.85\n", + "step: 2401, Red action: do-nothing, Blue reward:0.85\n", + "step: 2402, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2403, Red action: do-nothing, Blue reward:0.85\n", + "step: 2404, Red action: do-nothing, Blue reward:0.85\n", + "step: 2405, Red action: do-nothing, Blue reward:0.85\n", + "step: 2406, Red action: do-nothing, Blue reward:0.85\n", + "step: 2407, Red action: do-nothing, Blue reward:0.85\n", + "step: 2408, Red action: do-nothing, Blue reward:0.85\n", + "step: 2409, Red action: do-nothing, Blue reward:0.85\n", + "step: 2410, Red action: do-nothing, Blue reward:0.85\n", + "step: 2411, Red action: do-nothing, Blue reward:0.85\n", + "step: 2412, Red action: do-nothing, Blue reward:0.85\n", + "step: 2413, Red action: do-nothing, Blue reward:0.85\n", + "step: 2414, Red action: do-nothing, Blue reward:0.85\n", + "step: 2415, Red action: do-nothing, Blue reward:0.85\n", + "step: 2416, Red action: do-nothing, Blue reward:0.85\n", + "step: 2417, Red action: do-nothing, Blue reward:0.85\n", + "step: 2418, Red action: do-nothing, Blue reward:0.85\n", + "step: 2419, Red action: do-nothing, Blue reward:0.85\n", + "step: 2420, Red action: do-nothing, Blue reward:0.85\n", + "step: 2421, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2422, Red action: do-nothing, Blue reward:0.85\n", + "step: 2423, Red action: do-nothing, Blue reward:0.85\n", + "step: 2424, Red action: do-nothing, Blue reward:0.85\n", + "step: 2425, Red action: do-nothing, Blue reward:0.85\n", + "step: 2426, Red action: do-nothing, Blue reward:0.85\n", + "step: 2427, Red action: do-nothing, Blue reward:0.85\n", + "step: 2428, Red action: do-nothing, Blue reward:0.85\n", + "step: 2429, Red action: do-nothing, Blue reward:0.85\n", + "step: 2430, Red action: do-nothing, Blue reward:0.85\n", + "step: 2431, Red action: do-nothing, Blue reward:0.85\n", + "step: 2432, Red action: do-nothing, Blue reward:0.85\n", + "step: 2433, Red action: do-nothing, Blue reward:0.85\n", + "step: 2434, Red action: do-nothing, Blue reward:0.85\n", + "step: 2435, Red action: do-nothing, Blue reward:0.85\n", + "step: 2436, Red action: do-nothing, Blue reward:0.85\n", + "step: 2437, Red action: do-nothing, Blue reward:0.85\n", + "step: 2438, Red action: do-nothing, Blue reward:0.85\n", + "step: 2439, Red action: do-nothing, Blue reward:0.85\n", + "step: 2440, Red action: do-nothing, Blue reward:0.85\n", + "step: 2441, Red action: do-nothing, Blue reward:0.85\n", + "step: 2442, Red action: do-nothing, Blue reward:0.85\n", + "step: 2443, Red action: do-nothing, Blue reward:0.85\n", + "step: 2444, Red action: do-nothing, Blue reward:0.85\n", + "step: 2445, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2446, Red action: do-nothing, Blue reward:0.85\n", + "step: 2447, Red action: do-nothing, Blue reward:0.85\n", + "step: 2448, Red action: do-nothing, Blue reward:0.85\n", + "step: 2449, Red action: do-nothing, Blue reward:0.85\n", + "step: 2450, Red action: do-nothing, Blue reward:0.85\n", + "step: 2451, Red action: do-nothing, Blue reward:0.85\n", + "step: 2452, Red action: do-nothing, Blue reward:0.85\n", + "step: 2453, Red action: do-nothing, Blue reward:0.85\n", + "step: 2454, Red action: do-nothing, Blue reward:0.85\n", + "step: 2455, Red action: do-nothing, Blue reward:0.85\n", + "step: 2456, Red action: do-nothing, Blue reward:0.85\n", + "step: 2457, Red action: do-nothing, Blue reward:0.85\n", + "step: 2458, Red action: do-nothing, Blue reward:0.85\n", + "step: 2459, Red action: do-nothing, Blue reward:0.85\n", + "step: 2460, Red action: do-nothing, Blue reward:0.85\n", + "step: 2461, Red action: do-nothing, Blue reward:0.85\n", + "step: 2462, Red action: do-nothing, Blue reward:0.85\n", + "step: 2463, Red action: do-nothing, Blue reward:0.85\n", + "step: 2464, Red action: do-nothing, Blue reward:0.85\n", + "step: 2465, Red action: do-nothing, Blue reward:0.85\n", + "step: 2466, Red action: do-nothing, Blue reward:0.85\n", + "step: 2467, Red action: do-nothing, Blue reward:0.85\n", + "step: 2468, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2469, Red action: do-nothing, Blue reward:0.85\n", + "step: 2470, Red action: do-nothing, Blue reward:0.85\n", + "step: 2471, Red action: do-nothing, Blue reward:0.85\n", + "step: 2472, Red action: do-nothing, Blue reward:0.85\n", + "step: 2473, Red action: do-nothing, Blue reward:0.85\n", + "step: 2474, Red action: do-nothing, Blue reward:0.85\n", + "step: 2475, Red action: do-nothing, Blue reward:0.85\n", + "step: 2476, Red action: do-nothing, Blue reward:0.85\n", + "step: 2477, Red action: do-nothing, Blue reward:0.85\n", + "step: 2478, Red action: do-nothing, Blue reward:0.85\n", + "step: 2479, Red action: do-nothing, Blue reward:0.85\n", + "step: 2480, Red action: do-nothing, Blue reward:0.85\n", + "step: 2481, Red action: do-nothing, Blue reward:0.85\n", + "step: 2482, Red action: do-nothing, Blue reward:0.85\n", + "step: 2483, Red action: do-nothing, Blue reward:0.85\n", + "step: 2484, Red action: do-nothing, Blue reward:0.85\n", + "step: 2485, Red action: do-nothing, Blue reward:0.85\n", + "step: 2486, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2487, Red action: do-nothing, Blue reward:0.85\n", + "step: 2488, Red action: do-nothing, Blue reward:0.85\n", + "step: 2489, Red action: do-nothing, Blue reward:0.85\n", + "step: 2490, Red action: do-nothing, Blue reward:0.85\n", + "step: 2491, Red action: do-nothing, Blue reward:0.85\n", + "step: 2492, Red action: do-nothing, Blue reward:0.85\n", + "step: 2493, Red action: do-nothing, Blue reward:0.85\n", + "step: 2494, Red action: do-nothing, Blue reward:0.85\n", + "step: 2495, Red action: do-nothing, Blue reward:0.85\n", + "step: 2496, Red action: do-nothing, Blue reward:0.85\n", + "step: 2497, Red action: do-nothing, Blue reward:0.85\n", + "step: 2498, Red action: do-nothing, Blue reward:0.85\n", + "step: 2499, Red action: do-nothing, Blue reward:0.85\n", + "step: 2500, Red action: do-nothing, Blue reward:0.85\n", + "step: 2501, Red action: do-nothing, Blue reward:0.85\n", + "step: 2502, Red action: do-nothing, Blue reward:0.85\n", + "step: 2503, Red action: do-nothing, Blue reward:0.85\n", + "step: 2504, Red action: do-nothing, Blue reward:0.85\n", + "step: 2505, Red action: do-nothing, Blue reward:0.85\n", + "step: 2506, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2507, Red action: do-nothing, Blue reward:0.85\n", + "step: 2508, Red action: do-nothing, Blue reward:0.85\n", + "step: 2509, Red action: do-nothing, Blue reward:0.85\n", + "step: 2510, Red action: do-nothing, Blue reward:0.85\n", + "step: 2511, Red action: do-nothing, Blue reward:0.85\n", + "step: 2512, Red action: do-nothing, Blue reward:0.85\n", + "step: 2513, Red action: do-nothing, Blue reward:0.85\n", + "step: 2514, Red action: do-nothing, Blue reward:0.85\n", + "step: 2515, Red action: do-nothing, Blue reward:0.85\n", + "step: 2516, Red action: do-nothing, Blue reward:0.85\n", + "step: 2517, Red action: do-nothing, Blue reward:0.85\n", + "step: 2518, Red action: do-nothing, Blue reward:0.85\n", + "step: 2519, Red action: do-nothing, Blue reward:0.85\n", + "step: 2520, Red action: do-nothing, Blue reward:0.85\n", + "step: 2521, Red action: do-nothing, Blue reward:0.85\n", + "step: 2522, Red action: do-nothing, Blue reward:0.85\n", + "step: 2523, Red action: do-nothing, Blue reward:0.85\n", + "step: 2524, Red action: do-nothing, Blue reward:0.85\n", + "step: 2525, Red action: do-nothing, Blue reward:0.85\n", + "step: 2526, Red action: do-nothing, Blue reward:0.85\n", + "step: 2527, Red action: do-nothing, Blue reward:0.85\n", + "step: 2528, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2529, Red action: do-nothing, Blue reward:0.85\n", + "step: 2530, Red action: do-nothing, Blue reward:0.85\n", + "step: 2531, Red action: do-nothing, Blue reward:0.85\n", + "step: 2532, Red action: do-nothing, Blue reward:0.85\n", + "step: 2533, Red action: do-nothing, Blue reward:0.85\n", + "step: 2534, Red action: do-nothing, Blue reward:0.85\n", + "step: 2535, Red action: do-nothing, Blue reward:0.85\n", + "step: 2536, Red action: do-nothing, Blue reward:0.85\n", + "step: 2537, Red action: do-nothing, Blue reward:0.85\n", + "step: 2538, Red action: do-nothing, Blue reward:0.85\n", + "step: 2539, Red action: do-nothing, Blue reward:0.85\n", + "step: 2540, Red action: do-nothing, Blue reward:0.85\n", + "step: 2541, Red action: do-nothing, Blue reward:0.85\n", + "step: 2542, Red action: do-nothing, Blue reward:0.85\n", + "step: 2543, Red action: do-nothing, Blue reward:0.85\n", + "step: 2544, Red action: do-nothing, Blue reward:0.85\n", + "step: 2545, Red action: do-nothing, Blue reward:0.85\n", + "step: 2546, Red action: do-nothing, Blue reward:0.85\n", + "step: 2547, Red action: do-nothing, Blue reward:0.85\n", + "step: 2548, Red action: do-nothing, Blue reward:0.85\n", + "step: 2549, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2550, Red action: do-nothing, Blue reward:0.85\n", + "step: 2551, Red action: do-nothing, Blue reward:0.85\n", + "step: 2552, Red action: do-nothing, Blue reward:0.85\n", + "step: 2553, Red action: do-nothing, Blue reward:0.85\n", + "step: 2554, Red action: do-nothing, Blue reward:0.85\n", + "step: 2555, Red action: do-nothing, Blue reward:0.85\n", + "step: 2556, Red action: do-nothing, Blue reward:0.85\n", + "step: 2557, Red action: do-nothing, Blue reward:0.85\n", + "step: 2558, Red action: do-nothing, Blue reward:0.85\n", + "step: 2559, Red action: do-nothing, Blue reward:0.85\n", + "step: 2560, Red action: do-nothing, Blue reward:0.85\n", + "step: 2561, Red action: do-nothing, Blue reward:0.85\n", + "step: 2562, Red action: do-nothing, Blue reward:0.85\n", + "step: 2563, Red action: do-nothing, Blue reward:0.85\n", + "step: 2564, Red action: do-nothing, Blue reward:0.85\n", + "step: 2565, Red action: do-nothing, Blue reward:0.85\n", + "step: 2566, Red action: do-nothing, Blue reward:0.85\n", + "step: 2567, Red action: do-nothing, Blue reward:0.85\n", + "step: 2568, Red action: do-nothing, Blue reward:0.85\n", + "step: 2569, Red action: do-nothing, Blue reward:0.85\n", + "step: 2570, Red action: do-nothing, Blue reward:0.85\n", + "step: 2571, Red action: do-nothing, Blue reward:0.85\n", + "step: 2572, Red action: do-nothing, Blue reward:0.85\n", + "step: 2573, Red action: do-nothing, Blue reward:0.85\n", + "step: 2574, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2575, Red action: do-nothing, Blue reward:0.85\n", + "step: 2576, Red action: do-nothing, Blue reward:0.85\n", + "step: 2577, Red action: do-nothing, Blue reward:0.85\n", + "step: 2578, Red action: do-nothing, Blue reward:0.85\n", + "step: 2579, Red action: do-nothing, Blue reward:0.85\n", + "step: 2580, Red action: do-nothing, Blue reward:0.85\n", + "step: 2581, Red action: do-nothing, Blue reward:0.85\n", + "step: 2582, Red action: do-nothing, Blue reward:0.85\n", + "step: 2583, Red action: do-nothing, Blue reward:0.85\n", + "step: 2584, Red action: do-nothing, Blue reward:0.85\n", + "step: 2585, Red action: do-nothing, Blue reward:0.85\n", + "step: 2586, Red action: do-nothing, Blue reward:0.85\n", + "step: 2587, Red action: do-nothing, Blue reward:0.85\n", + "step: 2588, Red action: do-nothing, Blue reward:0.85\n", + "step: 2589, Red action: do-nothing, Blue reward:0.85\n", + "step: 2590, Red action: do-nothing, Blue reward:0.85\n", + "step: 2591, Red action: do-nothing, Blue reward:0.85\n", + "step: 2592, Red action: do-nothing, Blue reward:0.85\n", + "step: 2593, Red action: do-nothing, Blue reward:0.85\n", + "step: 2594, Red action: do-nothing, Blue reward:0.85\n", + "step: 2595, Red action: do-nothing, Blue reward:0.85\n", + "step: 2596, Red action: do-nothing, Blue reward:0.85\n", + "step: 2597, Red action: do-nothing, Blue reward:0.85\n", + "step: 2598, Red action: do-nothing, Blue reward:0.85\n", + "step: 2599, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2600, Red action: do-nothing, Blue reward:0.85\n", + "step: 2601, Red action: do-nothing, Blue reward:0.85\n", + "step: 2602, Red action: do-nothing, Blue reward:0.85\n", + "step: 2603, Red action: do-nothing, Blue reward:0.85\n", + "step: 2604, Red action: do-nothing, Blue reward:0.85\n", + "step: 2605, Red action: do-nothing, Blue reward:0.85\n", + "step: 2606, Red action: do-nothing, Blue reward:0.85\n", + "step: 2607, Red action: do-nothing, Blue reward:0.85\n", + "step: 2608, Red action: do-nothing, Blue reward:0.85\n", + "step: 2609, Red action: do-nothing, Blue reward:0.85\n", + "step: 2610, Red action: do-nothing, Blue reward:0.85\n", + "step: 2611, Red action: do-nothing, Blue reward:0.85\n", + "step: 2612, Red action: do-nothing, Blue reward:0.85\n", + "step: 2613, Red action: do-nothing, Blue reward:0.85\n", + "step: 2614, Red action: do-nothing, Blue reward:0.85\n", + "step: 2615, Red action: do-nothing, Blue reward:0.85\n", + "step: 2616, Red action: do-nothing, Blue reward:0.85\n", + "step: 2617, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2618, Red action: do-nothing, Blue reward:0.85\n", + "step: 2619, Red action: do-nothing, Blue reward:0.85\n", + "step: 2620, Red action: do-nothing, Blue reward:0.85\n", + "step: 2621, Red action: do-nothing, Blue reward:0.85\n", + "step: 2622, Red action: do-nothing, Blue reward:0.85\n", + "step: 2623, Red action: do-nothing, Blue reward:0.85\n", + "step: 2624, Red action: do-nothing, Blue reward:0.85\n", + "step: 2625, Red action: do-nothing, Blue reward:0.85\n", + "step: 2626, Red action: do-nothing, Blue reward:0.85\n", + "step: 2627, Red action: do-nothing, Blue reward:0.85\n", + "step: 2628, Red action: do-nothing, Blue reward:0.85\n", + "step: 2629, Red action: do-nothing, Blue reward:0.85\n", + "step: 2630, Red action: do-nothing, Blue reward:0.85\n", + "step: 2631, Red action: do-nothing, Blue reward:0.85\n", + "step: 2632, Red action: do-nothing, Blue reward:0.85\n", + "step: 2633, Red action: do-nothing, Blue reward:0.85\n", + "step: 2634, Red action: do-nothing, Blue reward:0.85\n", + "step: 2635, Red action: do-nothing, Blue reward:0.85\n", + "step: 2636, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2637, Red action: do-nothing, Blue reward:0.85\n", + "step: 2638, Red action: do-nothing, Blue reward:0.85\n", + "step: 2639, Red action: do-nothing, Blue reward:0.85\n", + "step: 2640, Red action: do-nothing, Blue reward:0.85\n", + "step: 2641, Red action: do-nothing, Blue reward:0.85\n", + "step: 2642, Red action: do-nothing, Blue reward:0.85\n", + "step: 2643, Red action: do-nothing, Blue reward:0.85\n", + "step: 2644, Red action: do-nothing, Blue reward:0.85\n", + "step: 2645, Red action: do-nothing, Blue reward:0.85\n", + "step: 2646, Red action: do-nothing, Blue reward:0.85\n", + "step: 2647, Red action: do-nothing, Blue reward:0.85\n", + "step: 2648, Red action: do-nothing, Blue reward:0.85\n", + "step: 2649, Red action: do-nothing, Blue reward:0.85\n", + "step: 2650, Red action: do-nothing, Blue reward:0.85\n", + "step: 2651, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2652, Red action: do-nothing, Blue reward:0.85\n", + "step: 2653, Red action: do-nothing, Blue reward:0.85\n", + "step: 2654, Red action: do-nothing, Blue reward:0.85\n", + "step: 2655, Red action: do-nothing, Blue reward:0.85\n", + "step: 2656, Red action: do-nothing, Blue reward:0.85\n", + "step: 2657, Red action: do-nothing, Blue reward:0.85\n", + "step: 2658, Red action: do-nothing, Blue reward:0.85\n", + "step: 2659, Red action: do-nothing, Blue reward:0.85\n", + "step: 2660, Red action: do-nothing, Blue reward:0.85\n", + "step: 2661, Red action: do-nothing, Blue reward:0.85\n", + "step: 2662, Red action: do-nothing, Blue reward:0.85\n", + "step: 2663, Red action: do-nothing, Blue reward:0.85\n", + "step: 2664, Red action: do-nothing, Blue reward:0.85\n", + "step: 2665, Red action: do-nothing, Blue reward:0.85\n", + "step: 2666, Red action: do-nothing, Blue reward:0.85\n", + "step: 2667, Red action: do-nothing, Blue reward:0.85\n", + "step: 2668, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2669, Red action: do-nothing, Blue reward:0.85\n", + "step: 2670, Red action: do-nothing, Blue reward:0.85\n", + "step: 2671, Red action: do-nothing, Blue reward:0.85\n", + "step: 2672, Red action: do-nothing, Blue reward:0.85\n", + "step: 2673, Red action: do-nothing, Blue reward:0.85\n", + "step: 2674, Red action: do-nothing, Blue reward:0.85\n", + "step: 2675, Red action: do-nothing, Blue reward:0.85\n", + "step: 2676, Red action: do-nothing, Blue reward:0.85\n", + "step: 2677, Red action: do-nothing, Blue reward:0.85\n", + "step: 2678, Red action: do-nothing, Blue reward:0.85\n", + "step: 2679, Red action: do-nothing, Blue reward:0.85\n", + "step: 2680, Red action: do-nothing, Blue reward:0.85\n", + "step: 2681, Red action: do-nothing, Blue reward:0.85\n", + "step: 2682, Red action: do-nothing, Blue reward:0.85\n", + "step: 2683, Red action: do-nothing, Blue reward:0.85\n", + "step: 2684, Red action: do-nothing, Blue reward:0.85\n", + "step: 2685, Red action: do-nothing, Blue reward:0.85\n", + "step: 2686, Red action: do-nothing, Blue reward:0.85\n", + "step: 2687, Red action: do-nothing, Blue reward:0.85\n", + "step: 2688, Red action: do-nothing, Blue reward:0.85\n", + "step: 2689, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2690, Red action: do-nothing, Blue reward:0.85\n", + "step: 2691, Red action: do-nothing, Blue reward:0.85\n", + "step: 2692, Red action: do-nothing, Blue reward:0.85\n", + "step: 2693, Red action: do-nothing, Blue reward:0.85\n", + "step: 2694, Red action: do-nothing, Blue reward:0.85\n", + "step: 2695, Red action: do-nothing, Blue reward:0.85\n", + "step: 2696, Red action: do-nothing, Blue reward:0.85\n", + "step: 2697, Red action: do-nothing, Blue reward:0.85\n", + "step: 2698, Red action: do-nothing, Blue reward:0.85\n", + "step: 2699, Red action: do-nothing, Blue reward:0.85\n", + "step: 2700, Red action: do-nothing, Blue reward:0.85\n", + "step: 2701, Red action: do-nothing, Blue reward:0.85\n", + "step: 2702, Red action: do-nothing, Blue reward:0.85\n", + "step: 2703, Red action: do-nothing, Blue reward:0.85\n", + "step: 2704, Red action: do-nothing, Blue reward:0.85\n", + "step: 2705, Red action: do-nothing, Blue reward:0.85\n", + "step: 2706, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2707, Red action: do-nothing, Blue reward:0.85\n", + "step: 2708, Red action: do-nothing, Blue reward:0.85\n", + "step: 2709, Red action: do-nothing, Blue reward:0.85\n", + "step: 2710, Red action: do-nothing, Blue reward:0.85\n", + "step: 2711, Red action: do-nothing, Blue reward:0.85\n", + "step: 2712, Red action: do-nothing, Blue reward:0.85\n", + "step: 2713, Red action: do-nothing, Blue reward:0.85\n", + "step: 2714, Red action: do-nothing, Blue reward:0.85\n", + "step: 2715, Red action: do-nothing, Blue reward:0.85\n", + "step: 2716, Red action: do-nothing, Blue reward:0.85\n", + "step: 2717, Red action: do-nothing, Blue reward:0.85\n", + "step: 2718, Red action: do-nothing, Blue reward:0.85\n", + "step: 2719, Red action: do-nothing, Blue reward:0.85\n", + "step: 2720, Red action: do-nothing, Blue reward:0.85\n", + "step: 2721, Red action: do-nothing, Blue reward:0.85\n", + "step: 2722, Red action: do-nothing, Blue reward:0.85\n", + "step: 2723, Red action: do-nothing, Blue reward:0.85\n", + "step: 2724, Red action: do-nothing, Blue reward:0.85\n", + "step: 2725, Red action: do-nothing, Blue reward:0.85\n", + "step: 2726, Red action: do-nothing, Blue reward:0.85\n", + "step: 2727, Red action: do-nothing, Blue reward:0.85\n", + "step: 2728, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2729, Red action: do-nothing, Blue reward:0.85\n", + "step: 2730, Red action: do-nothing, Blue reward:0.85\n", + "step: 2731, Red action: do-nothing, Blue reward:0.85\n", + "step: 2732, Red action: do-nothing, Blue reward:0.85\n", + "step: 2733, Red action: do-nothing, Blue reward:0.85\n", + "step: 2734, Red action: do-nothing, Blue reward:0.85\n", + "step: 2735, Red action: do-nothing, Blue reward:0.85\n", + "step: 2736, Red action: do-nothing, Blue reward:0.85\n", + "step: 2737, Red action: do-nothing, Blue reward:0.85\n", + "step: 2738, Red action: do-nothing, Blue reward:0.85\n", + "step: 2739, Red action: do-nothing, Blue reward:0.85\n", + "step: 2740, Red action: do-nothing, Blue reward:0.85\n", + "step: 2741, Red action: do-nothing, Blue reward:0.85\n", + "step: 2742, Red action: do-nothing, Blue reward:0.85\n", + "step: 2743, Red action: do-nothing, Blue reward:0.85\n", + "step: 2744, Red action: do-nothing, Blue reward:0.85\n", + "step: 2745, Red action: do-nothing, Blue reward:0.85\n", + "step: 2746, Red action: do-nothing, Blue reward:0.85\n", + "step: 2747, Red action: do-nothing, Blue reward:0.85\n", + "step: 2748, Red action: do-nothing, Blue reward:0.85\n", + "step: 2749, Red action: do-nothing, Blue reward:0.85\n", + "step: 2750, Red action: do-nothing, Blue reward:0.85\n", + "step: 2751, Red action: do-nothing, Blue reward:0.85\n", + "step: 2752, Red action: do-nothing, Blue reward:0.85\n", + "step: 2753, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2754, Red action: do-nothing, Blue reward:0.85\n", + "step: 2755, Red action: do-nothing, Blue reward:0.85\n", + "step: 2756, Red action: do-nothing, Blue reward:0.85\n", + "step: 2757, Red action: do-nothing, Blue reward:0.85\n", + "step: 2758, Red action: do-nothing, Blue reward:0.85\n", + "step: 2759, Red action: do-nothing, Blue reward:0.85\n", + "step: 2760, Red action: do-nothing, Blue reward:0.85\n", + "step: 2761, Red action: do-nothing, Blue reward:0.85\n", + "step: 2762, Red action: do-nothing, Blue reward:0.85\n", + "step: 2763, Red action: do-nothing, Blue reward:0.85\n", + "step: 2764, Red action: do-nothing, Blue reward:0.85\n", + "step: 2765, Red action: do-nothing, Blue reward:0.85\n", + "step: 2766, Red action: do-nothing, Blue reward:0.85\n", + "step: 2767, Red action: do-nothing, Blue reward:0.85\n", + "step: 2768, Red action: do-nothing, Blue reward:0.85\n", + "step: 2769, Red action: do-nothing, Blue reward:0.85\n", + "step: 2770, Red action: do-nothing, Blue reward:0.85\n", + "step: 2771, Red action: do-nothing, Blue reward:0.85\n", + "step: 2772, Red action: do-nothing, Blue reward:0.85\n", + "step: 2773, Red action: do-nothing, Blue reward:0.85\n", + "step: 2774, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2775, Red action: do-nothing, Blue reward:0.85\n", + "step: 2776, Red action: do-nothing, Blue reward:0.85\n", + "step: 2777, Red action: do-nothing, Blue reward:0.85\n", + "step: 2778, Red action: do-nothing, Blue reward:0.85\n", + "step: 2779, Red action: do-nothing, Blue reward:0.85\n", + "step: 2780, Red action: do-nothing, Blue reward:0.85\n", + "step: 2781, Red action: do-nothing, Blue reward:0.85\n", + "step: 2782, Red action: do-nothing, Blue reward:0.85\n", + "step: 2783, Red action: do-nothing, Blue reward:0.85\n", + "step: 2784, Red action: do-nothing, Blue reward:0.85\n", + "step: 2785, Red action: do-nothing, Blue reward:0.85\n", + "step: 2786, Red action: do-nothing, Blue reward:0.85\n", + "step: 2787, Red action: do-nothing, Blue reward:0.85\n", + "step: 2788, Red action: do-nothing, Blue reward:0.85\n", + "step: 2789, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2790, Red action: do-nothing, Blue reward:0.85\n", + "step: 2791, Red action: do-nothing, Blue reward:0.85\n", + "step: 2792, Red action: do-nothing, Blue reward:0.85\n", + "step: 2793, Red action: do-nothing, Blue reward:0.85\n", + "step: 2794, Red action: do-nothing, Blue reward:0.85\n", + "step: 2795, Red action: do-nothing, Blue reward:0.85\n", + "step: 2796, Red action: do-nothing, Blue reward:0.85\n", + "step: 2797, Red action: do-nothing, Blue reward:0.85\n", + "step: 2798, Red action: do-nothing, Blue reward:0.85\n", + "step: 2799, Red action: do-nothing, Blue reward:0.85\n", + "step: 2800, Red action: do-nothing, Blue reward:0.85\n", + "step: 2801, Red action: do-nothing, Blue reward:0.85\n", + "step: 2802, Red action: do-nothing, Blue reward:0.85\n", + "step: 2803, Red action: do-nothing, Blue reward:0.85\n", + "step: 2804, Red action: do-nothing, Blue reward:0.85\n", + "step: 2805, Red action: do-nothing, Blue reward:0.85\n", + "step: 2806, Red action: do-nothing, Blue reward:0.85\n", + "step: 2807, Red action: do-nothing, Blue reward:0.85\n", + "step: 2808, Red action: do-nothing, Blue reward:0.85\n", + "step: 2809, Red action: do-nothing, Blue reward:0.85\n", + "step: 2810, Red action: do-nothing, Blue reward:0.85\n", + "step: 2811, Red action: do-nothing, Blue reward:0.85\n", + "step: 2812, Red action: do-nothing, Blue reward:0.85\n", + "step: 2813, Red action: do-nothing, Blue reward:0.85\n", + "step: 2814, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2815, Red action: do-nothing, Blue reward:0.85\n", + "step: 2816, Red action: do-nothing, Blue reward:0.85\n", + "step: 2817, Red action: do-nothing, Blue reward:0.85\n", + "step: 2818, Red action: do-nothing, Blue reward:0.85\n", + "step: 2819, Red action: do-nothing, Blue reward:0.85\n", + "step: 2820, Red action: do-nothing, Blue reward:0.85\n", + "step: 2821, Red action: do-nothing, Blue reward:0.85\n", + "step: 2822, Red action: do-nothing, Blue reward:0.85\n", + "step: 2823, Red action: do-nothing, Blue reward:0.85\n", + "step: 2824, Red action: do-nothing, Blue reward:0.85\n", + "step: 2825, Red action: do-nothing, Blue reward:0.85\n", + "step: 2826, Red action: do-nothing, Blue reward:0.85\n", + "step: 2827, Red action: do-nothing, Blue reward:0.85\n", + "step: 2828, Red action: do-nothing, Blue reward:0.85\n", + "step: 2829, Red action: do-nothing, Blue reward:0.85\n", + "step: 2830, Red action: do-nothing, Blue reward:0.85\n", + "step: 2831, Red action: do-nothing, Blue reward:0.85\n", + "step: 2832, Red action: do-nothing, Blue reward:0.85\n", + "step: 2833, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2834, Red action: do-nothing, Blue reward:0.85\n", + "step: 2835, Red action: do-nothing, Blue reward:0.85\n", + "step: 2836, Red action: do-nothing, Blue reward:0.85\n", + "step: 2837, Red action: do-nothing, Blue reward:0.85\n", + "step: 2838, Red action: do-nothing, Blue reward:0.85\n", + "step: 2839, Red action: do-nothing, Blue reward:0.85\n", + "step: 2840, Red action: do-nothing, Blue reward:0.85\n", + "step: 2841, Red action: do-nothing, Blue reward:0.85\n", + "step: 2842, Red action: do-nothing, Blue reward:0.85\n", + "step: 2843, Red action: do-nothing, Blue reward:0.85\n", + "step: 2844, Red action: do-nothing, Blue reward:0.85\n", + "step: 2845, Red action: do-nothing, Blue reward:0.85\n", + "step: 2846, Red action: do-nothing, Blue reward:0.85\n", + "step: 2847, Red action: do-nothing, Blue reward:0.85\n", + "step: 2848, Red action: do-nothing, Blue reward:0.85\n", + "step: 2849, Red action: do-nothing, Blue reward:0.85\n", + "step: 2850, Red action: do-nothing, Blue reward:0.85\n", + "step: 2851, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2852, Red action: do-nothing, Blue reward:0.85\n", + "step: 2853, Red action: do-nothing, Blue reward:0.85\n", + "step: 2854, Red action: do-nothing, Blue reward:0.85\n", + "step: 2855, Red action: do-nothing, Blue reward:0.85\n", + "step: 2856, Red action: do-nothing, Blue reward:0.85\n", + "step: 2857, Red action: do-nothing, Blue reward:0.85\n", + "step: 2858, Red action: do-nothing, Blue reward:0.85\n", + "step: 2859, Red action: do-nothing, Blue reward:0.85\n", + "step: 2860, Red action: do-nothing, Blue reward:0.85\n", + "step: 2861, Red action: do-nothing, Blue reward:0.85\n", + "step: 2862, Red action: do-nothing, Blue reward:0.85\n", + "step: 2863, Red action: do-nothing, Blue reward:0.85\n", + "step: 2864, Red action: do-nothing, Blue reward:0.85\n", + "step: 2865, Red action: do-nothing, Blue reward:0.85\n", + "step: 2866, Red action: do-nothing, Blue reward:0.85\n", + "step: 2867, Red action: do-nothing, Blue reward:0.85\n", + "step: 2868, Red action: do-nothing, Blue reward:0.85\n", + "step: 2869, Red action: do-nothing, Blue reward:0.85\n", + "step: 2870, Red action: do-nothing, Blue reward:0.85\n", + "step: 2871, Red action: do-nothing, Blue reward:0.85\n", + "step: 2872, Red action: do-nothing, Blue reward:0.85\n", + "step: 2873, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2874, Red action: do-nothing, Blue reward:0.85\n", + "step: 2875, Red action: do-nothing, Blue reward:0.85\n", + "step: 2876, Red action: do-nothing, Blue reward:0.85\n", + "step: 2877, Red action: do-nothing, Blue reward:0.85\n", + "step: 2878, Red action: do-nothing, Blue reward:0.85\n", + "step: 2879, Red action: do-nothing, Blue reward:0.85\n", + "step: 2880, Red action: do-nothing, Blue reward:0.85\n", + "step: 2881, Red action: do-nothing, Blue reward:0.85\n", + "step: 2882, Red action: do-nothing, Blue reward:0.85\n", + "step: 2883, Red action: do-nothing, Blue reward:0.85\n", + "step: 2884, Red action: do-nothing, Blue reward:0.85\n", + "step: 2885, Red action: do-nothing, Blue reward:0.85\n", + "step: 2886, Red action: do-nothing, Blue reward:0.85\n", + "step: 2887, Red action: do-nothing, Blue reward:0.85\n", + "step: 2888, Red action: do-nothing, Blue reward:0.85\n", + "step: 2889, Red action: do-nothing, Blue reward:0.85\n", + "step: 2890, Red action: do-nothing, Blue reward:0.85\n", + "step: 2891, Red action: do-nothing, Blue reward:0.85\n", + "step: 2892, Red action: do-nothing, Blue reward:0.85\n", + "step: 2893, Red action: do-nothing, Blue reward:0.85\n", + "step: 2894, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2895, Red action: do-nothing, Blue reward:0.85\n", + "step: 2896, Red action: do-nothing, Blue reward:0.85\n", + "step: 2897, Red action: do-nothing, Blue reward:0.85\n", + "step: 2898, Red action: do-nothing, Blue reward:0.85\n", + "step: 2899, Red action: do-nothing, Blue reward:0.85\n", + "step: 2900, Red action: do-nothing, Blue reward:0.85\n", + "step: 2901, Red action: do-nothing, Blue reward:0.85\n", + "step: 2902, Red action: do-nothing, Blue reward:0.85\n", + "step: 2903, Red action: do-nothing, Blue reward:0.85\n", + "step: 2904, Red action: do-nothing, Blue reward:0.85\n", + "step: 2905, Red action: do-nothing, Blue reward:0.85\n", + "step: 2906, Red action: do-nothing, Blue reward:0.85\n", + "step: 2907, Red action: do-nothing, Blue reward:0.85\n", + "step: 2908, Red action: do-nothing, Blue reward:0.85\n", + "step: 2909, Red action: do-nothing, Blue reward:0.85\n", + "step: 2910, Red action: do-nothing, Blue reward:0.85\n", + "step: 2911, Red action: do-nothing, Blue reward:0.85\n", + "step: 2912, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2913, Red action: do-nothing, Blue reward:0.85\n", + "step: 2914, Red action: do-nothing, Blue reward:0.85\n", + "step: 2915, Red action: do-nothing, Blue reward:0.85\n", + "step: 2916, Red action: do-nothing, Blue reward:0.85\n", + "step: 2917, Red action: do-nothing, Blue reward:0.85\n", + "step: 2918, Red action: do-nothing, Blue reward:0.85\n", + "step: 2919, Red action: do-nothing, Blue reward:0.85\n", + "step: 2920, Red action: do-nothing, Blue reward:0.85\n", + "step: 2921, Red action: do-nothing, Blue reward:0.85\n", + "step: 2922, Red action: do-nothing, Blue reward:0.85\n", + "step: 2923, Red action: do-nothing, Blue reward:0.85\n", + "step: 2924, Red action: do-nothing, Blue reward:0.85\n", + "step: 2925, Red action: do-nothing, Blue reward:0.85\n", + "step: 2926, Red action: do-nothing, Blue reward:0.85\n", + "step: 2927, Red action: do-nothing, Blue reward:0.85\n", + "step: 2928, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2929, Red action: do-nothing, Blue reward:0.85\n", + "step: 2930, Red action: do-nothing, Blue reward:0.85\n", + "step: 2931, Red action: do-nothing, Blue reward:0.85\n", + "step: 2932, Red action: do-nothing, Blue reward:0.85\n", + "step: 2933, Red action: do-nothing, Blue reward:0.85\n", + "step: 2934, Red action: do-nothing, Blue reward:0.85\n", + "step: 2935, Red action: do-nothing, Blue reward:0.85\n", + "step: 2936, Red action: do-nothing, Blue reward:0.85\n", + "step: 2937, Red action: do-nothing, Blue reward:0.85\n", + "step: 2938, Red action: do-nothing, Blue reward:0.85\n", + "step: 2939, Red action: do-nothing, Blue reward:0.85\n", + "step: 2940, Red action: do-nothing, Blue reward:0.85\n", + "step: 2941, Red action: do-nothing, Blue reward:0.85\n", + "step: 2942, Red action: do-nothing, Blue reward:0.85\n", + "step: 2943, Red action: do-nothing, Blue reward:0.85\n", + "step: 2944, Red action: do-nothing, Blue reward:0.85\n", + "step: 2945, Red action: do-nothing, Blue reward:0.85\n", + "step: 2946, Red action: do-nothing, Blue reward:0.85\n", + "step: 2947, Red action: do-nothing, Blue reward:0.85\n", + "step: 2948, Red action: do-nothing, Blue reward:0.85\n", + "step: 2949, Red action: do-nothing, Blue reward:0.85\n", + "step: 2950, Red action: do-nothing, Blue reward:0.85\n", + "step: 2951, Red action: do-nothing, Blue reward:0.85\n", + "step: 2952, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2953, Red action: do-nothing, Blue reward:0.85\n", + "step: 2954, Red action: do-nothing, Blue reward:0.85\n", + "step: 2955, Red action: do-nothing, Blue reward:0.85\n", + "step: 2956, Red action: do-nothing, Blue reward:0.85\n", + "step: 2957, Red action: do-nothing, Blue reward:0.85\n", + "step: 2958, Red action: do-nothing, Blue reward:0.85\n", + "step: 2959, Red action: do-nothing, Blue reward:0.85\n", + "step: 2960, Red action: do-nothing, Blue reward:0.85\n", + "step: 2961, Red action: do-nothing, Blue reward:0.85\n", + "step: 2962, Red action: do-nothing, Blue reward:0.85\n", + "step: 2963, Red action: do-nothing, Blue reward:0.85\n", + "step: 2964, Red action: do-nothing, Blue reward:0.85\n", + "step: 2965, Red action: do-nothing, Blue reward:0.85\n", + "step: 2966, Red action: do-nothing, Blue reward:0.85\n", + "step: 2967, Red action: do-nothing, Blue reward:0.85\n", + "step: 2968, Red action: do-nothing, Blue reward:0.85\n", + "step: 2969, Red action: do-nothing, Blue reward:0.85\n", + "step: 2970, Red action: do-nothing, Blue reward:0.85\n", + "step: 2971, Red action: do-nothing, Blue reward:0.85\n", + "step: 2972, Red action: do-nothing, Blue reward:0.85\n", + "step: 2973, Red action: do-nothing, Blue reward:0.85\n", + "step: 2974, Red action: do-nothing, Blue reward:0.85\n", + "step: 2975, Red action: do-nothing, Blue reward:0.85\n", + "step: 2976, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2977, Red action: do-nothing, Blue reward:0.85\n", + "step: 2978, Red action: do-nothing, Blue reward:0.85\n", + "step: 2979, Red action: do-nothing, Blue reward:0.85\n", + "step: 2980, Red action: do-nothing, Blue reward:0.85\n", + "step: 2981, Red action: do-nothing, Blue reward:0.85\n", + "step: 2982, Red action: do-nothing, Blue reward:0.85\n", + "step: 2983, Red action: do-nothing, Blue reward:0.85\n", + "step: 2984, Red action: do-nothing, Blue reward:0.85\n", + "step: 2985, Red action: do-nothing, Blue reward:0.85\n", + "step: 2986, Red action: do-nothing, Blue reward:0.85\n", + "step: 2987, Red action: do-nothing, Blue reward:0.85\n", + "step: 2988, Red action: do-nothing, Blue reward:0.85\n", + "step: 2989, Red action: do-nothing, Blue reward:0.85\n", + "step: 2990, Red action: do-nothing, Blue reward:0.85\n", + "step: 2991, Red action: do-nothing, Blue reward:0.85\n", + "step: 2992, Red action: do-nothing, Blue reward:0.85\n", + "step: 2993, Red action: do-nothing, Blue reward:0.85\n", + "step: 2994, Red action: do-nothing, Blue reward:0.85\n", + "step: 2995, Red action: do-nothing, Blue reward:0.85\n", + "step: 2996, Red action: do-nothing, Blue reward:0.85\n", + "step: 2997, Red action: node-application-execute, Blue reward:0.85\n", + "step: 2998, Red action: do-nothing, Blue reward:0.85\n", + "step: 2999, Red action: do-nothing, Blue reward:0.85\n", + "step: 3000, Red action: do-nothing, Blue reward:0.85\n", + "step: 3001, Red action: do-nothing, Blue reward:0.85\n", + "step: 3002, Red action: do-nothing, Blue reward:0.85\n", + "step: 3003, Red action: do-nothing, Blue reward:0.85\n", + "step: 3004, Red action: do-nothing, Blue reward:0.85\n", + "step: 3005, Red action: do-nothing, Blue reward:0.85\n", + "step: 3006, Red action: do-nothing, Blue reward:0.85\n", + "step: 3007, Red action: do-nothing, Blue reward:0.85\n", + "step: 3008, Red action: do-nothing, Blue reward:0.85\n", + "step: 3009, Red action: do-nothing, Blue reward:0.85\n", + "step: 3010, Red action: do-nothing, Blue reward:0.85\n", + "step: 3011, Red action: do-nothing, Blue reward:0.85\n", + "step: 3012, Red action: do-nothing, Blue reward:0.85\n", + "step: 3013, Red action: do-nothing, Blue reward:0.85\n", + "step: 3014, Red action: do-nothing, Blue reward:0.85\n", + "step: 3015, Red action: do-nothing, Blue reward:0.85\n", + "step: 3016, Red action: do-nothing, Blue reward:0.85\n", + "step: 3017, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3018, Red action: do-nothing, Blue reward:0.85\n", + "step: 3019, Red action: do-nothing, Blue reward:0.85\n", + "step: 3020, Red action: do-nothing, Blue reward:0.85\n", + "step: 3021, Red action: do-nothing, Blue reward:0.85\n", + "step: 3022, Red action: do-nothing, Blue reward:0.85\n", + "step: 3023, Red action: do-nothing, Blue reward:0.85\n", + "step: 3024, Red action: do-nothing, Blue reward:0.85\n", + "step: 3025, Red action: do-nothing, Blue reward:0.85\n", + "step: 3026, Red action: do-nothing, Blue reward:0.85\n", + "step: 3027, Red action: do-nothing, Blue reward:0.85\n", + "step: 3028, Red action: do-nothing, Blue reward:0.85\n", + "step: 3029, Red action: do-nothing, Blue reward:0.85\n", + "step: 3030, Red action: do-nothing, Blue reward:0.85\n", + "step: 3031, Red action: do-nothing, Blue reward:0.85\n", + "step: 3032, Red action: do-nothing, Blue reward:0.85\n", + "step: 3033, Red action: do-nothing, Blue reward:0.85\n", + "step: 3034, Red action: do-nothing, Blue reward:0.85\n", + "step: 3035, Red action: do-nothing, Blue reward:0.85\n", + "step: 3036, Red action: do-nothing, Blue reward:0.85\n", + "step: 3037, Red action: do-nothing, Blue reward:0.85\n", + "step: 3038, Red action: do-nothing, Blue reward:0.85\n", + "step: 3039, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3040, Red action: do-nothing, Blue reward:0.85\n", + "step: 3041, Red action: do-nothing, Blue reward:0.85\n", + "step: 3042, Red action: do-nothing, Blue reward:0.85\n", + "step: 3043, Red action: do-nothing, Blue reward:0.85\n", + "step: 3044, Red action: do-nothing, Blue reward:0.85\n", + "step: 3045, Red action: do-nothing, Blue reward:0.85\n", + "step: 3046, Red action: do-nothing, Blue reward:0.85\n", + "step: 3047, Red action: do-nothing, Blue reward:0.85\n", + "step: 3048, Red action: do-nothing, Blue reward:0.85\n", + "step: 3049, Red action: do-nothing, Blue reward:0.85\n", + "step: 3050, Red action: do-nothing, Blue reward:0.85\n", + "step: 3051, Red action: do-nothing, Blue reward:0.85\n", + "step: 3052, Red action: do-nothing, Blue reward:0.85\n", + "step: 3053, Red action: do-nothing, Blue reward:0.85\n", + "step: 3054, Red action: do-nothing, Blue reward:0.85\n", + "step: 3055, Red action: do-nothing, Blue reward:0.85\n", + "step: 3056, Red action: do-nothing, Blue reward:0.85\n", + "step: 3057, Red action: do-nothing, Blue reward:0.85\n", + "step: 3058, Red action: do-nothing, Blue reward:0.85\n", + "step: 3059, Red action: do-nothing, Blue reward:0.85\n", + "step: 3060, Red action: do-nothing, Blue reward:0.85\n", + "step: 3061, Red action: do-nothing, Blue reward:0.85\n", + "step: 3062, Red action: do-nothing, Blue reward:0.85\n", + "step: 3063, Red action: do-nothing, Blue reward:0.85\n", + "step: 3064, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3065, Red action: do-nothing, Blue reward:0.85\n", + "step: 3066, Red action: do-nothing, Blue reward:0.85\n", + "step: 3067, Red action: do-nothing, Blue reward:0.85\n", + "step: 3068, Red action: do-nothing, Blue reward:0.85\n", + "step: 3069, Red action: do-nothing, Blue reward:0.85\n", + "step: 3070, Red action: do-nothing, Blue reward:0.85\n", + "step: 3071, Red action: do-nothing, Blue reward:0.85\n", + "step: 3072, Red action: do-nothing, Blue reward:0.85\n", + "step: 3073, Red action: do-nothing, Blue reward:0.85\n", + "step: 3074, Red action: do-nothing, Blue reward:0.85\n", + "step: 3075, Red action: do-nothing, Blue reward:0.85\n", + "step: 3076, Red action: do-nothing, Blue reward:0.85\n", + "step: 3077, Red action: do-nothing, Blue reward:0.85\n", + "step: 3078, Red action: do-nothing, Blue reward:0.85\n", + "step: 3079, Red action: do-nothing, Blue reward:0.85\n", + "step: 3080, Red action: do-nothing, Blue reward:0.85\n", + "step: 3081, Red action: do-nothing, Blue reward:0.85\n", + "step: 3082, Red action: do-nothing, Blue reward:0.85\n", + "step: 3083, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3084, Red action: do-nothing, Blue reward:0.85\n", + "step: 3085, Red action: do-nothing, Blue reward:0.85\n", + "step: 3086, Red action: do-nothing, Blue reward:0.85\n", + "step: 3087, Red action: do-nothing, Blue reward:0.85\n", + "step: 3088, Red action: do-nothing, Blue reward:0.85\n", + "step: 3089, Red action: do-nothing, Blue reward:0.85\n", + "step: 3090, Red action: do-nothing, Blue reward:0.85\n", + "step: 3091, Red action: do-nothing, Blue reward:0.85\n", + "step: 3092, Red action: do-nothing, Blue reward:0.85\n", + "step: 3093, Red action: do-nothing, Blue reward:0.85\n", + "step: 3094, Red action: do-nothing, Blue reward:0.85\n", + "step: 3095, Red action: do-nothing, Blue reward:0.85\n", + "step: 3096, Red action: do-nothing, Blue reward:0.85\n", + "step: 3097, Red action: do-nothing, Blue reward:0.85\n", + "step: 3098, Red action: do-nothing, Blue reward:0.85\n", + "step: 3099, Red action: do-nothing, Blue reward:0.85\n", + "step: 3100, Red action: do-nothing, Blue reward:0.85\n", + "step: 3101, Red action: do-nothing, Blue reward:0.85\n", + "step: 3102, Red action: do-nothing, Blue reward:0.85\n", + "step: 3103, Red action: do-nothing, Blue reward:0.85\n", + "step: 3104, Red action: do-nothing, Blue reward:0.85\n", + "step: 3105, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3106, Red action: do-nothing, Blue reward:0.85\n", + "step: 3107, Red action: do-nothing, Blue reward:0.85\n", + "step: 3108, Red action: do-nothing, Blue reward:0.85\n", + "step: 3109, Red action: do-nothing, Blue reward:0.85\n", + "step: 3110, Red action: do-nothing, Blue reward:0.85\n", + "step: 3111, Red action: do-nothing, Blue reward:0.85\n", + "step: 3112, Red action: do-nothing, Blue reward:0.85\n", + "step: 3113, Red action: do-nothing, Blue reward:0.85\n", + "step: 3114, Red action: do-nothing, Blue reward:0.85\n", + "step: 3115, Red action: do-nothing, Blue reward:0.85\n", + "step: 3116, Red action: do-nothing, Blue reward:0.85\n", + "step: 3117, Red action: do-nothing, Blue reward:0.85\n", + "step: 3118, Red action: do-nothing, Blue reward:0.85\n", + "step: 3119, Red action: do-nothing, Blue reward:0.85\n", + "step: 3120, Red action: do-nothing, Blue reward:0.85\n", + "step: 3121, Red action: do-nothing, Blue reward:0.85\n", + "step: 3122, Red action: do-nothing, Blue reward:0.85\n", + "step: 3123, Red action: do-nothing, Blue reward:0.85\n", + "step: 3124, Red action: do-nothing, Blue reward:0.85\n", + "step: 3125, Red action: do-nothing, Blue reward:0.85\n", + "step: 3126, Red action: do-nothing, Blue reward:0.85\n", + "step: 3127, Red action: do-nothing, Blue reward:0.85\n", + "step: 3128, Red action: do-nothing, Blue reward:0.85\n", + "step: 3129, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3130, Red action: do-nothing, Blue reward:0.85\n", + "step: 3131, Red action: do-nothing, Blue reward:0.85\n", + "step: 3132, Red action: do-nothing, Blue reward:0.85\n", + "step: 3133, Red action: do-nothing, Blue reward:0.85\n", + "step: 3134, Red action: do-nothing, Blue reward:0.85\n", + "step: 3135, Red action: do-nothing, Blue reward:0.85\n", + "step: 3136, Red action: do-nothing, Blue reward:0.85\n", + "step: 3137, Red action: do-nothing, Blue reward:0.85\n", + "step: 3138, Red action: do-nothing, Blue reward:0.85\n", + "step: 3139, Red action: do-nothing, Blue reward:0.85\n", + "step: 3140, Red action: do-nothing, Blue reward:0.85\n", + "step: 3141, Red action: do-nothing, Blue reward:0.85\n", + "step: 3142, Red action: do-nothing, Blue reward:0.85\n", + "step: 3143, Red action: do-nothing, Blue reward:0.85\n", + "step: 3144, Red action: do-nothing, Blue reward:0.85\n", + "step: 3145, Red action: do-nothing, Blue reward:0.85\n", + "step: 3146, Red action: do-nothing, Blue reward:0.85\n", + "step: 3147, Red action: do-nothing, Blue reward:0.85\n", + "step: 3148, Red action: do-nothing, Blue reward:0.85\n", + "step: 3149, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3150, Red action: do-nothing, Blue reward:0.85\n", + "step: 3151, Red action: do-nothing, Blue reward:0.85\n", + "step: 3152, Red action: do-nothing, Blue reward:0.85\n", + "step: 3153, Red action: do-nothing, Blue reward:0.85\n", + "step: 3154, Red action: do-nothing, Blue reward:0.85\n", + "step: 3155, Red action: do-nothing, Blue reward:0.85\n", + "step: 3156, Red action: do-nothing, Blue reward:0.85\n", + "step: 3157, Red action: do-nothing, Blue reward:0.85\n", + "step: 3158, Red action: do-nothing, Blue reward:0.85\n", + "step: 3159, Red action: do-nothing, Blue reward:0.85\n", + "step: 3160, Red action: do-nothing, Blue reward:0.85\n", + "step: 3161, Red action: do-nothing, Blue reward:0.85\n", + "step: 3162, Red action: do-nothing, Blue reward:0.85\n", + "step: 3163, Red action: do-nothing, Blue reward:0.85\n", + "step: 3164, Red action: do-nothing, Blue reward:0.85\n", + "step: 3165, Red action: do-nothing, Blue reward:0.85\n", + "step: 3166, Red action: do-nothing, Blue reward:0.85\n", + "step: 3167, Red action: do-nothing, Blue reward:0.85\n", + "step: 3168, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3169, Red action: do-nothing, Blue reward:0.85\n", + "step: 3170, Red action: do-nothing, Blue reward:0.85\n", + "step: 3171, Red action: do-nothing, Blue reward:0.85\n", + "step: 3172, Red action: do-nothing, Blue reward:0.85\n", + "step: 3173, Red action: do-nothing, Blue reward:0.85\n", + "step: 3174, Red action: do-nothing, Blue reward:0.85\n", + "step: 3175, Red action: do-nothing, Blue reward:0.85\n", + "step: 3176, Red action: do-nothing, Blue reward:0.85\n", + "step: 3177, Red action: do-nothing, Blue reward:0.85\n", + "step: 3178, Red action: do-nothing, Blue reward:0.85\n", + "step: 3179, Red action: do-nothing, Blue reward:0.85\n", + "step: 3180, Red action: do-nothing, Blue reward:0.85\n", + "step: 3181, Red action: do-nothing, Blue reward:0.85\n", + "step: 3182, Red action: do-nothing, Blue reward:0.85\n", + "step: 3183, Red action: do-nothing, Blue reward:0.85\n", + "step: 3184, Red action: do-nothing, Blue reward:0.85\n", + "step: 3185, Red action: do-nothing, Blue reward:0.85\n", + "step: 3186, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3187, Red action: do-nothing, Blue reward:0.85\n", + "step: 3188, Red action: do-nothing, Blue reward:0.85\n", + "step: 3189, Red action: do-nothing, Blue reward:0.85\n", + "step: 3190, Red action: do-nothing, Blue reward:0.85\n", + "step: 3191, Red action: do-nothing, Blue reward:0.85\n", + "step: 3192, Red action: do-nothing, Blue reward:0.85\n", + "step: 3193, Red action: do-nothing, Blue reward:0.85\n", + "step: 3194, Red action: do-nothing, Blue reward:0.85\n", + "step: 3195, Red action: do-nothing, Blue reward:0.85\n", + "step: 3196, Red action: do-nothing, Blue reward:0.85\n", + "step: 3197, Red action: do-nothing, Blue reward:0.85\n", + "step: 3198, Red action: do-nothing, Blue reward:0.85\n", + "step: 3199, Red action: do-nothing, Blue reward:0.85\n", + "step: 3200, Red action: do-nothing, Blue reward:0.85\n", + "step: 3201, Red action: do-nothing, Blue reward:0.85\n", + "step: 3202, Red action: do-nothing, Blue reward:0.85\n", + "step: 3203, Red action: do-nothing, Blue reward:0.85\n", + "step: 3204, Red action: do-nothing, Blue reward:0.85\n", + "step: 3205, Red action: do-nothing, Blue reward:0.85\n", + "step: 3206, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3207, Red action: do-nothing, Blue reward:0.85\n", + "step: 3208, Red action: do-nothing, Blue reward:0.85\n", + "step: 3209, Red action: do-nothing, Blue reward:0.85\n", + "step: 3210, Red action: do-nothing, Blue reward:0.85\n", + "step: 3211, Red action: do-nothing, Blue reward:0.85\n", + "step: 3212, Red action: do-nothing, Blue reward:0.85\n", + "step: 3213, Red action: do-nothing, Blue reward:0.85\n", + "step: 3214, Red action: do-nothing, Blue reward:0.85\n", + "step: 3215, Red action: do-nothing, Blue reward:0.85\n", + "step: 3216, Red action: do-nothing, Blue reward:0.85\n", + "step: 3217, Red action: do-nothing, Blue reward:0.85\n", + "step: 3218, Red action: do-nothing, Blue reward:0.85\n", + "step: 3219, Red action: do-nothing, Blue reward:0.85\n", + "step: 3220, Red action: do-nothing, Blue reward:0.85\n", + "step: 3221, Red action: do-nothing, Blue reward:0.85\n", + "step: 3222, Red action: do-nothing, Blue reward:0.85\n", + "step: 3223, Red action: do-nothing, Blue reward:0.85\n", + "step: 3224, Red action: do-nothing, Blue reward:0.85\n", + "step: 3225, Red action: do-nothing, Blue reward:0.85\n", + "step: 3226, Red action: do-nothing, Blue reward:0.85\n", + "step: 3227, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3228, Red action: do-nothing, Blue reward:0.85\n", + "step: 3229, Red action: do-nothing, Blue reward:0.85\n", + "step: 3230, Red action: do-nothing, Blue reward:0.85\n", + "step: 3231, Red action: do-nothing, Blue reward:0.85\n", + "step: 3232, Red action: do-nothing, Blue reward:0.85\n", + "step: 3233, Red action: do-nothing, Blue reward:0.85\n", + "step: 3234, Red action: do-nothing, Blue reward:0.85\n", + "step: 3235, Red action: do-nothing, Blue reward:0.85\n", + "step: 3236, Red action: do-nothing, Blue reward:0.85\n", + "step: 3237, Red action: do-nothing, Blue reward:0.85\n", + "step: 3238, Red action: do-nothing, Blue reward:0.85\n", + "step: 3239, Red action: do-nothing, Blue reward:0.85\n", + "step: 3240, Red action: do-nothing, Blue reward:0.85\n", + "step: 3241, Red action: do-nothing, Blue reward:0.85\n", + "step: 3242, Red action: do-nothing, Blue reward:0.85\n", + "step: 3243, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3244, Red action: do-nothing, Blue reward:0.85\n", + "step: 3245, Red action: do-nothing, Blue reward:0.85\n", + "step: 3246, Red action: do-nothing, Blue reward:0.85\n", + "step: 3247, Red action: do-nothing, Blue reward:0.85\n", + "step: 3248, Red action: do-nothing, Blue reward:0.85\n", + "step: 3249, Red action: do-nothing, Blue reward:0.85\n", + "step: 3250, Red action: do-nothing, Blue reward:0.85\n", + "step: 3251, Red action: do-nothing, Blue reward:0.85\n", + "step: 3252, Red action: do-nothing, Blue reward:0.85\n", + "step: 3253, Red action: do-nothing, Blue reward:0.85\n", + "step: 3254, Red action: do-nothing, Blue reward:0.85\n", + "step: 3255, Red action: do-nothing, Blue reward:0.85\n", + "step: 3256, Red action: do-nothing, Blue reward:0.85\n", + "step: 3257, Red action: do-nothing, Blue reward:0.85\n", + "step: 3258, Red action: do-nothing, Blue reward:0.85\n", + "step: 3259, Red action: do-nothing, Blue reward:0.85\n", + "step: 3260, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3261, Red action: do-nothing, Blue reward:0.85\n", + "step: 3262, Red action: do-nothing, Blue reward:0.85\n", + "step: 3263, Red action: do-nothing, Blue reward:0.85\n", + "step: 3264, Red action: do-nothing, Blue reward:0.85\n", + "step: 3265, Red action: do-nothing, Blue reward:0.85\n", + "step: 3266, Red action: do-nothing, Blue reward:0.85\n", + "step: 3267, Red action: do-nothing, Blue reward:0.85\n", + "step: 3268, Red action: do-nothing, Blue reward:0.85\n", + "step: 3269, Red action: do-nothing, Blue reward:0.85\n", + "step: 3270, Red action: do-nothing, Blue reward:0.85\n", + "step: 3271, Red action: do-nothing, Blue reward:0.85\n", + "step: 3272, Red action: do-nothing, Blue reward:0.85\n", + "step: 3273, Red action: do-nothing, Blue reward:0.85\n", + "step: 3274, Red action: do-nothing, Blue reward:0.85\n", + "step: 3275, Red action: do-nothing, Blue reward:0.85\n", + "step: 3276, Red action: do-nothing, Blue reward:0.85\n", + "step: 3277, Red action: do-nothing, Blue reward:0.85\n", + "step: 3278, Red action: do-nothing, Blue reward:0.85\n", + "step: 3279, Red action: do-nothing, Blue reward:0.85\n", + "step: 3280, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3281, Red action: do-nothing, Blue reward:0.85\n", + "step: 3282, Red action: do-nothing, Blue reward:0.85\n", + "step: 3283, Red action: do-nothing, Blue reward:0.85\n", + "step: 3284, Red action: do-nothing, Blue reward:0.85\n", + "step: 3285, Red action: do-nothing, Blue reward:0.85\n", + "step: 3286, Red action: do-nothing, Blue reward:0.85\n", + "step: 3287, Red action: do-nothing, Blue reward:0.85\n", + "step: 3288, Red action: do-nothing, Blue reward:0.85\n", + "step: 3289, Red action: do-nothing, Blue reward:0.85\n", + "step: 3290, Red action: do-nothing, Blue reward:0.85\n", + "step: 3291, Red action: do-nothing, Blue reward:0.85\n", + "step: 3292, Red action: do-nothing, Blue reward:0.85\n", + "step: 3293, Red action: do-nothing, Blue reward:0.85\n", + "step: 3294, Red action: do-nothing, Blue reward:0.85\n", + "step: 3295, Red action: do-nothing, Blue reward:0.85\n", + "step: 3296, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3297, Red action: do-nothing, Blue reward:0.85\n", + "step: 3298, Red action: do-nothing, Blue reward:0.85\n", + "step: 3299, Red action: do-nothing, Blue reward:0.85\n", + "step: 3300, Red action: do-nothing, Blue reward:0.85\n", + "step: 3301, Red action: do-nothing, Blue reward:0.85\n", + "step: 3302, Red action: do-nothing, Blue reward:0.85\n", + "step: 3303, Red action: do-nothing, Blue reward:0.85\n", + "step: 3304, Red action: do-nothing, Blue reward:0.85\n", + "step: 3305, Red action: do-nothing, Blue reward:0.85\n", + "step: 3306, Red action: do-nothing, Blue reward:0.85\n", + "step: 3307, Red action: do-nothing, Blue reward:0.85\n", + "step: 3308, Red action: do-nothing, Blue reward:0.85\n", + "step: 3309, Red action: do-nothing, Blue reward:0.85\n", + "step: 3310, Red action: do-nothing, Blue reward:0.85\n", + "step: 3311, Red action: do-nothing, Blue reward:0.85\n", + "step: 3312, Red action: do-nothing, Blue reward:0.85\n", + "step: 3313, Red action: do-nothing, Blue reward:0.85\n", + "step: 3314, Red action: do-nothing, Blue reward:0.85\n", + "step: 3315, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3316, Red action: do-nothing, Blue reward:0.85\n", + "step: 3317, Red action: do-nothing, Blue reward:0.85\n", + "step: 3318, Red action: do-nothing, Blue reward:0.85\n", + "step: 3319, Red action: do-nothing, Blue reward:0.85\n", + "step: 3320, Red action: do-nothing, Blue reward:0.85\n", + "step: 3321, Red action: do-nothing, Blue reward:0.85\n", + "step: 3322, Red action: do-nothing, Blue reward:0.85\n", + "step: 3323, Red action: do-nothing, Blue reward:0.85\n", + "step: 3324, Red action: do-nothing, Blue reward:0.85\n", + "step: 3325, Red action: do-nothing, Blue reward:0.85\n", + "step: 3326, Red action: do-nothing, Blue reward:0.85\n", + "step: 3327, Red action: do-nothing, Blue reward:0.85\n", + "step: 3328, Red action: do-nothing, Blue reward:0.85\n", + "step: 3329, Red action: do-nothing, Blue reward:0.85\n", + "step: 3330, Red action: do-nothing, Blue reward:0.85\n", + "step: 3331, Red action: do-nothing, Blue reward:0.85\n", + "step: 3332, Red action: do-nothing, Blue reward:0.85\n", + "step: 3333, Red action: do-nothing, Blue reward:0.85\n", + "step: 3334, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3335, Red action: do-nothing, Blue reward:0.85\n", + "step: 3336, Red action: do-nothing, Blue reward:0.85\n", + "step: 3337, Red action: do-nothing, Blue reward:0.85\n", + "step: 3338, Red action: do-nothing, Blue reward:0.85\n", + "step: 3339, Red action: do-nothing, Blue reward:0.85\n", + "step: 3340, Red action: do-nothing, Blue reward:0.85\n", + "step: 3341, Red action: do-nothing, Blue reward:0.85\n", + "step: 3342, Red action: do-nothing, Blue reward:0.85\n", + "step: 3343, Red action: do-nothing, Blue reward:0.85\n", + "step: 3344, Red action: do-nothing, Blue reward:0.85\n", + "step: 3345, Red action: do-nothing, Blue reward:0.85\n", + "step: 3346, Red action: do-nothing, Blue reward:0.85\n", + "step: 3347, Red action: do-nothing, Blue reward:0.85\n", + "step: 3348, Red action: do-nothing, Blue reward:0.85\n", + "step: 3349, Red action: do-nothing, Blue reward:0.85\n", + "step: 3350, Red action: do-nothing, Blue reward:0.85\n", + "step: 3351, Red action: do-nothing, Blue reward:0.85\n", + "step: 3352, Red action: do-nothing, Blue reward:0.85\n", + "step: 3353, Red action: do-nothing, Blue reward:0.85\n", + "step: 3354, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3355, Red action: do-nothing, Blue reward:0.85\n", + "step: 3356, Red action: do-nothing, Blue reward:0.85\n", + "step: 3357, Red action: do-nothing, Blue reward:0.85\n", + "step: 3358, Red action: do-nothing, Blue reward:0.85\n", + "step: 3359, Red action: do-nothing, Blue reward:0.85\n", + "step: 3360, Red action: do-nothing, Blue reward:0.85\n", + "step: 3361, Red action: do-nothing, Blue reward:0.85\n", + "step: 3362, Red action: do-nothing, Blue reward:0.85\n", + "step: 3363, Red action: do-nothing, Blue reward:0.85\n", + "step: 3364, Red action: do-nothing, Blue reward:0.85\n", + "step: 3365, Red action: do-nothing, Blue reward:0.85\n", + "step: 3366, Red action: do-nothing, Blue reward:0.85\n", + "step: 3367, Red action: do-nothing, Blue reward:0.85\n", + "step: 3368, Red action: do-nothing, Blue reward:0.85\n", + "step: 3369, Red action: do-nothing, Blue reward:0.85\n", + "step: 3370, Red action: do-nothing, Blue reward:0.85\n", + "step: 3371, Red action: do-nothing, Blue reward:0.85\n", + "step: 3372, Red action: do-nothing, Blue reward:0.85\n", + "step: 3373, Red action: do-nothing, Blue reward:0.85\n", + "step: 3374, Red action: do-nothing, Blue reward:0.85\n", + "step: 3375, Red action: do-nothing, Blue reward:0.85\n", + "step: 3376, Red action: do-nothing, Blue reward:0.85\n", + "step: 3377, Red action: do-nothing, Blue reward:0.85\n", + "step: 3378, Red action: do-nothing, Blue reward:0.85\n", + "step: 3379, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3380, Red action: do-nothing, Blue reward:0.85\n", + "step: 3381, Red action: do-nothing, Blue reward:0.85\n", + "step: 3382, Red action: do-nothing, Blue reward:0.85\n", + "step: 3383, Red action: do-nothing, Blue reward:0.85\n", + "step: 3384, Red action: do-nothing, Blue reward:0.85\n", + "step: 3385, Red action: do-nothing, Blue reward:0.85\n", + "step: 3386, Red action: do-nothing, Blue reward:0.85\n", + "step: 3387, Red action: do-nothing, Blue reward:0.85\n", + "step: 3388, Red action: do-nothing, Blue reward:0.85\n", + "step: 3389, Red action: do-nothing, Blue reward:0.85\n", + "step: 3390, Red action: do-nothing, Blue reward:0.85\n", + "step: 3391, Red action: do-nothing, Blue reward:0.85\n", + "step: 3392, Red action: do-nothing, Blue reward:0.85\n", + "step: 3393, Red action: do-nothing, Blue reward:0.85\n", + "step: 3394, Red action: do-nothing, Blue reward:0.85\n", + "step: 3395, Red action: do-nothing, Blue reward:0.85\n", + "step: 3396, Red action: do-nothing, Blue reward:0.85\n", + "step: 3397, Red action: do-nothing, Blue reward:0.85\n", + "step: 3398, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3399, Red action: do-nothing, Blue reward:0.85\n", + "step: 3400, Red action: do-nothing, Blue reward:0.85\n", + "step: 3401, Red action: do-nothing, Blue reward:0.85\n", + "step: 3402, Red action: do-nothing, Blue reward:0.85\n", + "step: 3403, Red action: do-nothing, Blue reward:0.85\n", + "step: 3404, Red action: do-nothing, Blue reward:0.85\n", + "step: 3405, Red action: do-nothing, Blue reward:0.85\n", + "step: 3406, Red action: do-nothing, Blue reward:0.85\n", + "step: 3407, Red action: do-nothing, Blue reward:0.85\n", + "step: 3408, Red action: do-nothing, Blue reward:0.85\n", + "step: 3409, Red action: do-nothing, Blue reward:0.85\n", + "step: 3410, Red action: do-nothing, Blue reward:0.85\n", + "step: 3411, Red action: do-nothing, Blue reward:0.85\n", + "step: 3412, Red action: do-nothing, Blue reward:0.85\n", + "step: 3413, Red action: do-nothing, Blue reward:0.85\n", + "step: 3414, Red action: do-nothing, Blue reward:0.85\n", + "step: 3415, Red action: do-nothing, Blue reward:0.85\n", + "step: 3416, Red action: do-nothing, Blue reward:0.85\n", + "step: 3417, Red action: do-nothing, Blue reward:0.85\n", + "step: 3418, Red action: do-nothing, Blue reward:0.85\n", + "step: 3419, Red action: do-nothing, Blue reward:0.85\n", + "step: 3420, Red action: do-nothing, Blue reward:0.85\n", + "step: 3421, Red action: do-nothing, Blue reward:0.85\n", + "step: 3422, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3423, Red action: do-nothing, Blue reward:0.85\n", + "step: 3424, Red action: do-nothing, Blue reward:0.85\n", + "step: 3425, Red action: do-nothing, Blue reward:0.85\n", + "step: 3426, Red action: do-nothing, Blue reward:0.85\n", + "step: 3427, Red action: do-nothing, Blue reward:0.85\n", + "step: 3428, Red action: do-nothing, Blue reward:0.85\n", + "step: 3429, Red action: do-nothing, Blue reward:0.85\n", + "step: 3430, Red action: do-nothing, Blue reward:0.85\n", + "step: 3431, Red action: do-nothing, Blue reward:0.85\n", + "step: 3432, Red action: do-nothing, Blue reward:0.85\n", + "step: 3433, Red action: do-nothing, Blue reward:0.85\n", + "step: 3434, Red action: do-nothing, Blue reward:0.85\n", + "step: 3435, Red action: do-nothing, Blue reward:0.85\n", + "step: 3436, Red action: do-nothing, Blue reward:0.85\n", + "step: 3437, Red action: do-nothing, Blue reward:0.85\n", + "step: 3438, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3439, Red action: do-nothing, Blue reward:0.85\n", + "step: 3440, Red action: do-nothing, Blue reward:0.85\n", + "step: 3441, Red action: do-nothing, Blue reward:0.85\n", + "step: 3442, Red action: do-nothing, Blue reward:0.85\n", + "step: 3443, Red action: do-nothing, Blue reward:0.85\n", + "step: 3444, Red action: do-nothing, Blue reward:0.85\n", + "step: 3445, Red action: do-nothing, Blue reward:0.85\n", + "step: 3446, Red action: do-nothing, Blue reward:0.85\n", + "step: 3447, Red action: do-nothing, Blue reward:0.85\n", + "step: 3448, Red action: do-nothing, Blue reward:0.85\n", + "step: 3449, Red action: do-nothing, Blue reward:0.85\n", + "step: 3450, Red action: do-nothing, Blue reward:0.85\n", + "step: 3451, Red action: do-nothing, Blue reward:0.85\n", + "step: 3452, Red action: do-nothing, Blue reward:0.85\n", + "step: 3453, Red action: do-nothing, Blue reward:0.85\n", + "step: 3454, Red action: do-nothing, Blue reward:0.85\n", + "step: 3455, Red action: do-nothing, Blue reward:0.85\n", + "step: 3456, Red action: do-nothing, Blue reward:0.85\n", + "step: 3457, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3458, Red action: do-nothing, Blue reward:0.85\n", + "step: 3459, Red action: do-nothing, Blue reward:0.85\n", + "step: 3460, Red action: do-nothing, Blue reward:0.85\n", + "step: 3461, Red action: do-nothing, Blue reward:0.85\n", + "step: 3462, Red action: do-nothing, Blue reward:0.85\n", + "step: 3463, Red action: do-nothing, Blue reward:0.85\n", + "step: 3464, Red action: do-nothing, Blue reward:0.85\n", + "step: 3465, Red action: do-nothing, Blue reward:0.85\n", + "step: 3466, Red action: do-nothing, Blue reward:0.85\n", + "step: 3467, Red action: do-nothing, Blue reward:0.85\n", + "step: 3468, Red action: do-nothing, Blue reward:0.85\n", + "step: 3469, Red action: do-nothing, Blue reward:0.85\n", + "step: 3470, Red action: do-nothing, Blue reward:0.85\n", + "step: 3471, Red action: do-nothing, Blue reward:0.85\n", + "step: 3472, Red action: do-nothing, Blue reward:0.85\n", + "step: 3473, Red action: do-nothing, Blue reward:0.85\n", + "step: 3474, Red action: do-nothing, Blue reward:0.85\n", + "step: 3475, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3476, Red action: do-nothing, Blue reward:0.85\n", + "step: 3477, Red action: do-nothing, Blue reward:0.85\n", + "step: 3478, Red action: do-nothing, Blue reward:0.85\n", + "step: 3479, Red action: do-nothing, Blue reward:0.85\n", + "step: 3480, Red action: do-nothing, Blue reward:0.85\n", + "step: 3481, Red action: do-nothing, Blue reward:0.85\n", + "step: 3482, Red action: do-nothing, Blue reward:0.85\n", + "step: 3483, Red action: do-nothing, Blue reward:0.85\n", + "step: 3484, Red action: do-nothing, Blue reward:0.85\n", + "step: 3485, Red action: do-nothing, Blue reward:0.85\n", + "step: 3486, Red action: do-nothing, Blue reward:0.85\n", + "step: 3487, Red action: do-nothing, Blue reward:0.85\n", + "step: 3488, Red action: do-nothing, Blue reward:0.85\n", + "step: 3489, Red action: do-nothing, Blue reward:0.85\n", + "step: 3490, Red action: do-nothing, Blue reward:0.85\n", + "step: 3491, Red action: do-nothing, Blue reward:0.85\n", + "step: 3492, Red action: do-nothing, Blue reward:0.85\n", + "step: 3493, Red action: do-nothing, Blue reward:0.85\n", + "step: 3494, Red action: do-nothing, Blue reward:0.85\n", + "step: 3495, Red action: do-nothing, Blue reward:0.85\n", + "step: 3496, Red action: do-nothing, Blue reward:0.85\n", + "step: 3497, Red action: do-nothing, Blue reward:0.85\n", + "step: 3498, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3499, Red action: do-nothing, Blue reward:0.85\n", + "step: 3500, Red action: do-nothing, Blue reward:0.85\n", + "step: 3501, Red action: do-nothing, Blue reward:0.85\n", + "step: 3502, Red action: do-nothing, Blue reward:0.85\n", + "step: 3503, Red action: do-nothing, Blue reward:0.85\n", + "step: 3504, Red action: do-nothing, Blue reward:0.85\n", + "step: 3505, Red action: do-nothing, Blue reward:0.85\n", + "step: 3506, Red action: do-nothing, Blue reward:0.85\n", + "step: 3507, Red action: do-nothing, Blue reward:0.85\n", + "step: 3508, Red action: do-nothing, Blue reward:0.85\n", + "step: 3509, Red action: do-nothing, Blue reward:0.85\n", + "step: 3510, Red action: do-nothing, Blue reward:0.85\n", + "step: 3511, Red action: do-nothing, Blue reward:0.85\n", + "step: 3512, Red action: do-nothing, Blue reward:0.85\n", + "step: 3513, Red action: do-nothing, Blue reward:0.85\n", + "step: 3514, Red action: do-nothing, Blue reward:0.85\n", + "step: 3515, Red action: do-nothing, Blue reward:0.85\n", + "step: 3516, Red action: do-nothing, Blue reward:0.85\n", + "step: 3517, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3518, Red action: do-nothing, Blue reward:0.85\n", + "step: 3519, Red action: do-nothing, Blue reward:0.85\n", + "step: 3520, Red action: do-nothing, Blue reward:0.85\n", + "step: 3521, Red action: do-nothing, Blue reward:0.85\n", + "step: 3522, Red action: do-nothing, Blue reward:0.85\n", + "step: 3523, Red action: do-nothing, Blue reward:0.85\n", + "step: 3524, Red action: do-nothing, Blue reward:0.85\n", + "step: 3525, Red action: do-nothing, Blue reward:0.85\n", + "step: 3526, Red action: do-nothing, Blue reward:0.85\n", + "step: 3527, Red action: do-nothing, Blue reward:0.85\n", + "step: 3528, Red action: do-nothing, Blue reward:0.85\n", + "step: 3529, Red action: do-nothing, Blue reward:0.85\n", + "step: 3530, Red action: do-nothing, Blue reward:0.85\n", + "step: 3531, Red action: do-nothing, Blue reward:0.85\n", + "step: 3532, Red action: do-nothing, Blue reward:0.85\n", + "step: 3533, Red action: do-nothing, Blue reward:0.85\n", + "step: 3534, Red action: do-nothing, Blue reward:0.85\n", + "step: 3535, Red action: do-nothing, Blue reward:0.85\n", + "step: 3536, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3537, Red action: do-nothing, Blue reward:0.85\n", + "step: 3538, Red action: do-nothing, Blue reward:0.85\n", + "step: 3539, Red action: do-nothing, Blue reward:0.85\n", + "step: 3540, Red action: do-nothing, Blue reward:0.85\n", + "step: 3541, Red action: do-nothing, Blue reward:0.85\n", + "step: 3542, Red action: do-nothing, Blue reward:0.85\n", + "step: 3543, Red action: do-nothing, Blue reward:0.85\n", + "step: 3544, Red action: do-nothing, Blue reward:0.85\n", + "step: 3545, Red action: do-nothing, Blue reward:0.85\n", + "step: 3546, Red action: do-nothing, Blue reward:0.85\n", + "step: 3547, Red action: do-nothing, Blue reward:0.85\n", + "step: 3548, Red action: do-nothing, Blue reward:0.85\n", + "step: 3549, Red action: do-nothing, Blue reward:0.85\n", + "step: 3550, Red action: do-nothing, Blue reward:0.85\n", + "step: 3551, Red action: do-nothing, Blue reward:0.85\n", + "step: 3552, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3553, Red action: do-nothing, Blue reward:0.85\n", + "step: 3554, Red action: do-nothing, Blue reward:0.85\n", + "step: 3555, Red action: do-nothing, Blue reward:0.85\n", + "step: 3556, Red action: do-nothing, Blue reward:0.85\n", + "step: 3557, Red action: do-nothing, Blue reward:0.85\n", + "step: 3558, Red action: do-nothing, Blue reward:0.85\n", + "step: 3559, Red action: do-nothing, Blue reward:0.85\n", + "step: 3560, Red action: do-nothing, Blue reward:0.85\n", + "step: 3561, Red action: do-nothing, Blue reward:0.85\n", + "step: 3562, Red action: do-nothing, Blue reward:0.85\n", + "step: 3563, Red action: do-nothing, Blue reward:0.85\n", + "step: 3564, Red action: do-nothing, Blue reward:0.85\n", + "step: 3565, Red action: do-nothing, Blue reward:0.85\n", + "step: 3566, Red action: do-nothing, Blue reward:0.85\n", + "step: 3567, Red action: do-nothing, Blue reward:0.85\n", + "step: 3568, Red action: do-nothing, Blue reward:0.85\n", + "step: 3569, Red action: do-nothing, Blue reward:0.85\n", + "step: 3570, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3571, Red action: do-nothing, Blue reward:0.85\n", + "step: 3572, Red action: do-nothing, Blue reward:0.85\n", + "step: 3573, Red action: do-nothing, Blue reward:0.85\n", + "step: 3574, Red action: do-nothing, Blue reward:0.85\n", + "step: 3575, Red action: do-nothing, Blue reward:0.85\n", + "step: 3576, Red action: do-nothing, Blue reward:0.85\n", + "step: 3577, Red action: do-nothing, Blue reward:0.85\n", + "step: 3578, Red action: do-nothing, Blue reward:0.85\n", + "step: 3579, Red action: do-nothing, Blue reward:0.85\n", + "step: 3580, Red action: do-nothing, Blue reward:0.85\n", + "step: 3581, Red action: do-nothing, Blue reward:0.85\n", + "step: 3582, Red action: do-nothing, Blue reward:0.85\n", + "step: 3583, Red action: do-nothing, Blue reward:0.85\n", + "step: 3584, Red action: do-nothing, Blue reward:0.85\n", + "step: 3585, Red action: do-nothing, Blue reward:0.85\n", + "step: 3586, Red action: do-nothing, Blue reward:0.85\n", + "step: 3587, Red action: do-nothing, Blue reward:0.85\n", + "step: 3588, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3589, Red action: do-nothing, Blue reward:0.85\n", + "step: 3590, Red action: do-nothing, Blue reward:0.85\n", + "step: 3591, Red action: do-nothing, Blue reward:0.85\n", + "step: 3592, Red action: do-nothing, Blue reward:0.85\n", + "step: 3593, Red action: do-nothing, Blue reward:0.85\n", + "step: 3594, Red action: do-nothing, Blue reward:0.85\n", + "step: 3595, Red action: do-nothing, Blue reward:0.85\n", + "step: 3596, Red action: do-nothing, Blue reward:0.85\n", + "step: 3597, Red action: do-nothing, Blue reward:0.85\n", + "step: 3598, Red action: do-nothing, Blue reward:0.85\n", + "step: 3599, Red action: do-nothing, Blue reward:0.85\n", + "step: 3600, Red action: do-nothing, Blue reward:0.85\n", + "step: 3601, Red action: do-nothing, Blue reward:0.85\n", + "step: 3602, Red action: do-nothing, Blue reward:0.85\n", + "step: 3603, Red action: do-nothing, Blue reward:0.85\n", + "step: 3604, Red action: do-nothing, Blue reward:0.85\n", + "step: 3605, Red action: do-nothing, Blue reward:0.85\n", + "step: 3606, Red action: do-nothing, Blue reward:0.85\n", + "step: 3607, Red action: do-nothing, Blue reward:0.85\n", + "step: 3608, Red action: do-nothing, Blue reward:0.85\n", + "step: 3609, Red action: do-nothing, Blue reward:0.85\n", + "step: 3610, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3611, Red action: do-nothing, Blue reward:0.85\n", + "step: 3612, Red action: do-nothing, Blue reward:0.85\n", + "step: 3613, Red action: do-nothing, Blue reward:0.85\n", + "step: 3614, Red action: do-nothing, Blue reward:0.85\n", + "step: 3615, Red action: do-nothing, Blue reward:0.85\n", + "step: 3616, Red action: do-nothing, Blue reward:0.85\n", + "step: 3617, Red action: do-nothing, Blue reward:0.85\n", + "step: 3618, Red action: do-nothing, Blue reward:0.85\n", + "step: 3619, Red action: do-nothing, Blue reward:0.85\n", + "step: 3620, Red action: do-nothing, Blue reward:0.85\n", + "step: 3621, Red action: do-nothing, Blue reward:0.85\n", + "step: 3622, Red action: do-nothing, Blue reward:0.85\n", + "step: 3623, Red action: do-nothing, Blue reward:0.85\n", + "step: 3624, Red action: do-nothing, Blue reward:0.85\n", + "step: 3625, Red action: do-nothing, Blue reward:0.85\n", + "step: 3626, Red action: do-nothing, Blue reward:0.85\n", + "step: 3627, Red action: do-nothing, Blue reward:0.85\n", + "step: 3628, Red action: do-nothing, Blue reward:0.85\n", + "step: 3629, Red action: do-nothing, Blue reward:0.85\n", + "step: 3630, Red action: do-nothing, Blue reward:0.85\n", + "step: 3631, Red action: do-nothing, Blue reward:0.85\n", + "step: 3632, Red action: do-nothing, Blue reward:0.85\n", + "step: 3633, Red action: do-nothing, Blue reward:0.85\n", + "step: 3634, Red action: do-nothing, Blue reward:0.85\n", + "step: 3635, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3636, Red action: do-nothing, Blue reward:0.85\n", + "step: 3637, Red action: do-nothing, Blue reward:0.85\n", + "step: 3638, Red action: do-nothing, Blue reward:0.85\n", + "step: 3639, Red action: do-nothing, Blue reward:0.85\n", + "step: 3640, Red action: do-nothing, Blue reward:0.85\n", + "step: 3641, Red action: do-nothing, Blue reward:0.85\n", + "step: 3642, Red action: do-nothing, Blue reward:0.85\n", + "step: 3643, Red action: do-nothing, Blue reward:0.85\n", + "step: 3644, Red action: do-nothing, Blue reward:0.85\n", + "step: 3645, Red action: do-nothing, Blue reward:0.85\n", + "step: 3646, Red action: do-nothing, Blue reward:0.85\n", + "step: 3647, Red action: do-nothing, Blue reward:0.85\n", + "step: 3648, Red action: do-nothing, Blue reward:0.85\n", + "step: 3649, Red action: do-nothing, Blue reward:0.85\n", + "step: 3650, Red action: do-nothing, Blue reward:0.85\n", + "step: 3651, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3652, Red action: do-nothing, Blue reward:0.85\n", + "step: 3653, Red action: do-nothing, Blue reward:0.85\n", + "step: 3654, Red action: do-nothing, Blue reward:0.85\n", + "step: 3655, Red action: do-nothing, Blue reward:0.85\n", + "step: 3656, Red action: do-nothing, Blue reward:0.85\n", + "step: 3657, Red action: do-nothing, Blue reward:0.85\n", + "step: 3658, Red action: do-nothing, Blue reward:0.85\n", + "step: 3659, Red action: do-nothing, Blue reward:0.85\n", + "step: 3660, Red action: do-nothing, Blue reward:0.85\n", + "step: 3661, Red action: do-nothing, Blue reward:0.85\n", + "step: 3662, Red action: do-nothing, Blue reward:0.85\n", + "step: 3663, Red action: do-nothing, Blue reward:0.85\n", + "step: 3664, Red action: do-nothing, Blue reward:0.85\n", + "step: 3665, Red action: do-nothing, Blue reward:0.85\n", + "step: 3666, Red action: do-nothing, Blue reward:0.85\n", + "step: 3667, Red action: do-nothing, Blue reward:0.85\n", + "step: 3668, Red action: do-nothing, Blue reward:0.85\n", + "step: 3669, Red action: do-nothing, Blue reward:0.85\n", + "step: 3670, Red action: do-nothing, Blue reward:0.85\n", + "step: 3671, Red action: do-nothing, Blue reward:0.85\n", + "step: 3672, Red action: do-nothing, Blue reward:0.85\n", + "step: 3673, Red action: do-nothing, Blue reward:0.85\n", + "step: 3674, Red action: do-nothing, Blue reward:0.85\n", + "step: 3675, Red action: do-nothing, Blue reward:0.85\n", + "step: 3676, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3677, Red action: do-nothing, Blue reward:0.85\n", + "step: 3678, Red action: do-nothing, Blue reward:0.85\n", + "step: 3679, Red action: do-nothing, Blue reward:0.85\n", + "step: 3680, Red action: do-nothing, Blue reward:0.85\n", + "step: 3681, Red action: do-nothing, Blue reward:0.85\n", + "step: 3682, Red action: do-nothing, Blue reward:0.85\n", + "step: 3683, Red action: do-nothing, Blue reward:0.85\n", + "step: 3684, Red action: do-nothing, Blue reward:0.85\n", + "step: 3685, Red action: do-nothing, Blue reward:0.85\n", + "step: 3686, Red action: do-nothing, Blue reward:0.85\n", + "step: 3687, Red action: do-nothing, Blue reward:0.85\n", + "step: 3688, Red action: do-nothing, Blue reward:0.85\n", + "step: 3689, Red action: do-nothing, Blue reward:0.85\n", + "step: 3690, Red action: do-nothing, Blue reward:0.85\n", + "step: 3691, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3692, Red action: do-nothing, Blue reward:0.85\n", + "step: 3693, Red action: do-nothing, Blue reward:0.85\n", + "step: 3694, Red action: do-nothing, Blue reward:0.85\n", + "step: 3695, Red action: do-nothing, Blue reward:0.85\n", + "step: 3696, Red action: do-nothing, Blue reward:0.85\n", + "step: 3697, Red action: do-nothing, Blue reward:0.85\n", + "step: 3698, Red action: do-nothing, Blue reward:0.85\n", + "step: 3699, Red action: do-nothing, Blue reward:0.85\n", + "step: 3700, Red action: do-nothing, Blue reward:0.85\n", + "step: 3701, Red action: do-nothing, Blue reward:0.85\n", + "step: 3702, Red action: do-nothing, Blue reward:0.85\n", + "step: 3703, Red action: do-nothing, Blue reward:0.85\n", + "step: 3704, Red action: do-nothing, Blue reward:0.85\n", + "step: 3705, Red action: do-nothing, Blue reward:0.85\n", + "step: 3706, Red action: do-nothing, Blue reward:0.85\n", + "step: 3707, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3708, Red action: do-nothing, Blue reward:0.85\n", + "step: 3709, Red action: do-nothing, Blue reward:0.85\n", + "step: 3710, Red action: do-nothing, Blue reward:0.85\n", + "step: 3711, Red action: do-nothing, Blue reward:0.85\n", + "step: 3712, Red action: do-nothing, Blue reward:0.85\n", + "step: 3713, Red action: do-nothing, Blue reward:0.85\n", + "step: 3714, Red action: do-nothing, Blue reward:0.85\n", + "step: 3715, Red action: do-nothing, Blue reward:0.85\n", + "step: 3716, Red action: do-nothing, Blue reward:0.85\n", + "step: 3717, Red action: do-nothing, Blue reward:0.85\n", + "step: 3718, Red action: do-nothing, Blue reward:0.85\n", + "step: 3719, Red action: do-nothing, Blue reward:0.85\n", + "step: 3720, Red action: do-nothing, Blue reward:0.85\n", + "step: 3721, Red action: do-nothing, Blue reward:0.85\n", + "step: 3722, Red action: do-nothing, Blue reward:0.85\n", + "step: 3723, Red action: do-nothing, Blue reward:0.85\n", + "step: 3724, Red action: do-nothing, Blue reward:0.85\n", + "step: 3725, Red action: do-nothing, Blue reward:0.85\n", + "step: 3726, Red action: do-nothing, Blue reward:0.85\n", + "step: 3727, Red action: do-nothing, Blue reward:0.85\n", + "step: 3728, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3729, Red action: do-nothing, Blue reward:0.85\n", + "step: 3730, Red action: do-nothing, Blue reward:0.85\n", + "step: 3731, Red action: do-nothing, Blue reward:0.85\n", + "step: 3732, Red action: do-nothing, Blue reward:0.85\n", + "step: 3733, Red action: do-nothing, Blue reward:0.85\n", + "step: 3734, Red action: do-nothing, Blue reward:0.85\n", + "step: 3735, Red action: do-nothing, Blue reward:0.85\n", + "step: 3736, Red action: do-nothing, Blue reward:0.85\n", + "step: 3737, Red action: do-nothing, Blue reward:0.85\n", + "step: 3738, Red action: do-nothing, Blue reward:0.85\n", + "step: 3739, Red action: do-nothing, Blue reward:0.85\n", + "step: 3740, Red action: do-nothing, Blue reward:0.85\n", + "step: 3741, Red action: do-nothing, Blue reward:0.85\n", + "step: 3742, Red action: do-nothing, Blue reward:0.85\n", + "step: 3743, Red action: do-nothing, Blue reward:0.85\n", + "step: 3744, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3745, Red action: do-nothing, Blue reward:0.85\n", + "step: 3746, Red action: do-nothing, Blue reward:0.85\n", + "step: 3747, Red action: do-nothing, Blue reward:0.85\n", + "step: 3748, Red action: do-nothing, Blue reward:0.85\n", + "step: 3749, Red action: do-nothing, Blue reward:0.85\n", + "step: 3750, Red action: do-nothing, Blue reward:0.85\n", + "step: 3751, Red action: do-nothing, Blue reward:0.85\n", + "step: 3752, Red action: do-nothing, Blue reward:0.85\n", + "step: 3753, Red action: do-nothing, Blue reward:0.85\n", + "step: 3754, Red action: do-nothing, Blue reward:0.85\n", + "step: 3755, Red action: do-nothing, Blue reward:0.85\n", + "step: 3756, Red action: do-nothing, Blue reward:0.85\n", + "step: 3757, Red action: do-nothing, Blue reward:0.85\n", + "step: 3758, Red action: do-nothing, Blue reward:0.85\n", + "step: 3759, Red action: do-nothing, Blue reward:0.85\n", + "step: 3760, Red action: do-nothing, Blue reward:0.85\n", + "step: 3761, Red action: do-nothing, Blue reward:0.85\n", + "step: 3762, Red action: do-nothing, Blue reward:0.85\n", + "step: 3763, Red action: do-nothing, Blue reward:0.85\n", + "step: 3764, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3765, Red action: do-nothing, Blue reward:0.85\n", + "step: 3766, Red action: do-nothing, Blue reward:0.85\n", + "step: 3767, Red action: do-nothing, Blue reward:0.85\n", + "step: 3768, Red action: do-nothing, Blue reward:0.85\n", + "step: 3769, Red action: do-nothing, Blue reward:0.85\n", + "step: 3770, Red action: do-nothing, Blue reward:0.85\n", + "step: 3771, Red action: do-nothing, Blue reward:0.85\n", + "step: 3772, Red action: do-nothing, Blue reward:0.85\n", + "step: 3773, Red action: do-nothing, Blue reward:0.85\n", + "step: 3774, Red action: do-nothing, Blue reward:0.85\n", + "step: 3775, Red action: do-nothing, Blue reward:0.85\n", + "step: 3776, Red action: do-nothing, Blue reward:0.85\n", + "step: 3777, Red action: do-nothing, Blue reward:0.85\n", + "step: 3778, Red action: do-nothing, Blue reward:0.85\n", + "step: 3779, Red action: do-nothing, Blue reward:0.85\n", + "step: 3780, Red action: do-nothing, Blue reward:0.85\n", + "step: 3781, Red action: do-nothing, Blue reward:0.85\n", + "step: 3782, Red action: do-nothing, Blue reward:0.85\n", + "step: 3783, Red action: do-nothing, Blue reward:0.85\n", + "step: 3784, Red action: node-application-execute, Blue reward:0.85\n", + "step: 3785, Red action: do-nothing, Blue reward:0.85\n", + "step: 3786, Red action: do-nothing, Blue reward:0.85\n", + "step: 3787, Red action: do-nothing, Blue reward:0.85\n", + "step: 3788, Red action: do-nothing, Blue reward:0.85\n", + "step: 3789, Red action: do-nothing, Blue reward:0.85\n", + "step: 3790, Red action: do-nothing, Blue reward:0.85\n", + "step: 3791, Red action: do-nothing, Blue reward:0.85\n", + "step: 3792, Red action: do-nothing, Blue reward:0.85\n", + "step: 3793, Red action: do-nothing, Blue reward:0.85\n", + "step: 3794, Red action: do-nothing, Blue reward:0.85\n", + "step: 3795, Red action: do-nothing, Blue reward:0.85\n", + "step: 3796, Red action: do-nothing, Blue reward:0.85\n", + "step: 3797, Red action: do-nothing, Blue reward:0.85\n", + "step: 3798, Red action: do-nothing, Blue reward:0.85\n", + "step: 3799, Red action: do-nothing, Blue reward:0.85\n", + "step: 3800, Red action: do-nothing, Blue reward:0.85\n", + "step: 3801, Red action: do-nothing, Blue reward:0.85\n", + "step: 3802, Red action: do-nothing, Blue reward:0.85\n" + ] + }, + { + "ename": "KeyboardInterrupt", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[11], line 11\u001b[0m\n\u001b[1;32m 8\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mstep: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00menv\u001b[38;5;241m.\u001b[39mgame\u001b[38;5;241m.\u001b[39mstep_counter\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m, Red action: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00minfo[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124magent_actions\u001b[39m\u001b[38;5;124m'\u001b[39m][\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mdata_manipulation_attacker\u001b[39m\u001b[38;5;124m'\u001b[39m]\u001b[38;5;241m.\u001b[39maction\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m, Blue reward:\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mreward\u001b[38;5;132;01m:\u001b[39;00m\u001b[38;5;124m.2f\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m )\n\u001b[1;32m 10\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28mabs\u001b[39m(reward \u001b[38;5;241m-\u001b[39m \u001b[38;5;241m0.8\u001b[39m) \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m1e-5\u001b[39m:\n\u001b[0;32m---> 11\u001b[0m obs, reward, terminated, truncated, info \u001b[38;5;241m=\u001b[39m \u001b[43menv\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstep\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m)\u001b[49m \u001b[38;5;66;03m# do nothing\u001b[39;00m\n\u001b[1;32m 12\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mstep: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00menv\u001b[38;5;241m.\u001b[39mgame\u001b[38;5;241m.\u001b[39mstep_counter\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m, Red action: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00minfo[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124magent_actions\u001b[39m\u001b[38;5;124m'\u001b[39m][\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mdata_manipulation_attacker\u001b[39m\u001b[38;5;124m'\u001b[39m]\u001b[38;5;241m.\u001b[39maction\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m, Blue reward:\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mreward\u001b[38;5;132;01m:\u001b[39;00m\u001b[38;5;124m.2f\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m )\n\u001b[1;32m 13\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m env\u001b[38;5;241m.\u001b[39mgame\u001b[38;5;241m.\u001b[39mstep_counter \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m10000\u001b[39m:\n", + "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/session/environment.py:111\u001b[0m, in \u001b[0;36mPrimaiteGymEnv.step\u001b[0;34m(self, action)\u001b[0m\n\u001b[1;32m 109\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgame\u001b[38;5;241m.\u001b[39mapply_agent_actions()\n\u001b[1;32m 110\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgame\u001b[38;5;241m.\u001b[39madvance_timestep()\n\u001b[0;32m--> 111\u001b[0m state \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgame\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_sim_state\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 112\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgame\u001b[38;5;241m.\u001b[39mupdate_agents(state)\n\u001b[1;32m 114\u001b[0m next_obs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_get_obs() \u001b[38;5;66;03m# this doesn't update observation, just gets the current observation\u001b[39;00m\n", + "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/game/game.py:151\u001b[0m, in \u001b[0;36mPrimaiteGame.get_sim_state\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 149\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mget_sim_state\u001b[39m(\u001b[38;5;28mself\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Dict:\n\u001b[1;32m 150\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"Get the current state of the simulation.\"\"\"\u001b[39;00m\n\u001b[0;32m--> 151\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msimulation\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdescribe_state\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/simulator/sim_container.py:57\u001b[0m, in \u001b[0;36mSimulation.describe_state\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 46\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 47\u001b[0m \u001b[38;5;124;03mProduce a dictionary describing the current state of this object.\u001b[39;00m\n\u001b[1;32m 48\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 52\u001b[0m \u001b[38;5;124;03m:rtype: Dict\u001b[39;00m\n\u001b[1;32m 53\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 54\u001b[0m state \u001b[38;5;241m=\u001b[39m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39mdescribe_state()\n\u001b[1;32m 55\u001b[0m state\u001b[38;5;241m.\u001b[39mupdate(\n\u001b[1;32m 56\u001b[0m {\n\u001b[0;32m---> 57\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnetwork\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mnetwork\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdescribe_state\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m,\n\u001b[1;32m 58\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdomain\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdomain\u001b[38;5;241m.\u001b[39mdescribe_state(),\n\u001b[1;32m 59\u001b[0m }\n\u001b[1;32m 60\u001b[0m )\n\u001b[1;32m 61\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m state\n", + "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/simulator/network/container.py:260\u001b[0m, in \u001b[0;36mNetwork.describe_state\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 252\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 253\u001b[0m \u001b[38;5;124;03mProduce a dictionary describing the current state of the Network.\u001b[39;00m\n\u001b[1;32m 254\u001b[0m \n\u001b[1;32m 255\u001b[0m \u001b[38;5;124;03m:return: A dictionary capturing the current state of the Network and its child objects.\u001b[39;00m\n\u001b[1;32m 256\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 257\u001b[0m state \u001b[38;5;241m=\u001b[39m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39mdescribe_state()\n\u001b[1;32m 258\u001b[0m state\u001b[38;5;241m.\u001b[39mupdate(\n\u001b[1;32m 259\u001b[0m {\n\u001b[0;32m--> 260\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnodes\u001b[39m\u001b[38;5;124m\"\u001b[39m: {node\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mhostname: node\u001b[38;5;241m.\u001b[39mdescribe_state() \u001b[38;5;28;01mfor\u001b[39;00m node \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnodes\u001b[38;5;241m.\u001b[39mvalues()},\n\u001b[1;32m 261\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mlinks\u001b[39m\u001b[38;5;124m\"\u001b[39m: {},\n\u001b[1;32m 262\u001b[0m }\n\u001b[1;32m 263\u001b[0m )\n\u001b[1;32m 264\u001b[0m \u001b[38;5;66;03m# Update the links one-by-one. The key is a 4-tuple of `hostname_a, port_a, hostname_b, port_b`\u001b[39;00m\n\u001b[1;32m 265\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m _, link \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlinks\u001b[38;5;241m.\u001b[39mitems():\n", + "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/simulator/network/container.py:260\u001b[0m, in \u001b[0;36m\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 252\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 253\u001b[0m \u001b[38;5;124;03mProduce a dictionary describing the current state of the Network.\u001b[39;00m\n\u001b[1;32m 254\u001b[0m \n\u001b[1;32m 255\u001b[0m \u001b[38;5;124;03m:return: A dictionary capturing the current state of the Network and its child objects.\u001b[39;00m\n\u001b[1;32m 256\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 257\u001b[0m state \u001b[38;5;241m=\u001b[39m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39mdescribe_state()\n\u001b[1;32m 258\u001b[0m state\u001b[38;5;241m.\u001b[39mupdate(\n\u001b[1;32m 259\u001b[0m {\n\u001b[0;32m--> 260\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnodes\u001b[39m\u001b[38;5;124m\"\u001b[39m: {node\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mhostname: \u001b[43mnode\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdescribe_state\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mfor\u001b[39;00m node \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnodes\u001b[38;5;241m.\u001b[39mvalues()},\n\u001b[1;32m 261\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mlinks\u001b[39m\u001b[38;5;124m\"\u001b[39m: {},\n\u001b[1;32m 262\u001b[0m }\n\u001b[1;32m 263\u001b[0m )\n\u001b[1;32m 264\u001b[0m \u001b[38;5;66;03m# Update the links one-by-one. The key is a 4-tuple of `hostname_a, port_a, hostname_b, port_b`\u001b[39;00m\n\u001b[1;32m 265\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m _, link \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlinks\u001b[38;5;241m.\u001b[39mitems():\n", + "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/simulator/network/hardware/base.py:1900\u001b[0m, in \u001b[0;36mNode.describe_state\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1881\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 1882\u001b[0m \u001b[38;5;124;03mProduce a dictionary describing the current state of this object.\u001b[39;00m\n\u001b[1;32m 1883\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1887\u001b[0m \u001b[38;5;124;03m:rtype: Dict\u001b[39;00m\n\u001b[1;32m 1888\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 1889\u001b[0m state \u001b[38;5;241m=\u001b[39m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39mdescribe_state()\n\u001b[1;32m 1890\u001b[0m state\u001b[38;5;241m.\u001b[39mupdate(\n\u001b[1;32m 1891\u001b[0m {\n\u001b[1;32m 1892\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhostname\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mhostname,\n\u001b[1;32m 1893\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moperating_state\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39moperating_state\u001b[38;5;241m.\u001b[39mvalue,\n\u001b[1;32m 1894\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mNICs\u001b[39m\u001b[38;5;124m\"\u001b[39m: {\n\u001b[1;32m 1895\u001b[0m eth_num: network_interface\u001b[38;5;241m.\u001b[39mdescribe_state()\n\u001b[1;32m 1896\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m eth_num, network_interface \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnetwork_interface\u001b[38;5;241m.\u001b[39mitems()\n\u001b[1;32m 1897\u001b[0m },\n\u001b[1;32m 1898\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfile_system\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfile_system\u001b[38;5;241m.\u001b[39mdescribe_state(),\n\u001b[1;32m 1899\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mapplications\u001b[39m\u001b[38;5;124m\"\u001b[39m: {app\u001b[38;5;241m.\u001b[39mname: app\u001b[38;5;241m.\u001b[39mdescribe_state() \u001b[38;5;28;01mfor\u001b[39;00m app \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mapplications\u001b[38;5;241m.\u001b[39mvalues()},\n\u001b[0;32m-> 1900\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mservices\u001b[39m\u001b[38;5;124m\"\u001b[39m: {svc\u001b[38;5;241m.\u001b[39mname: svc\u001b[38;5;241m.\u001b[39mdescribe_state() \u001b[38;5;28;01mfor\u001b[39;00m svc \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mservices\u001b[38;5;241m.\u001b[39mvalues()},\n\u001b[1;32m 1901\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mprocess\u001b[39m\u001b[38;5;124m\"\u001b[39m: {proc\u001b[38;5;241m.\u001b[39mname: proc\u001b[38;5;241m.\u001b[39mdescribe_state() \u001b[38;5;28;01mfor\u001b[39;00m proc \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprocesses\u001b[38;5;241m.\u001b[39mvalues()},\n\u001b[1;32m 1902\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mrevealed_to_red\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mrevealed_to_red,\n\u001b[1;32m 1903\u001b[0m }\n\u001b[1;32m 1904\u001b[0m )\n\u001b[1;32m 1905\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m state\n", + "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/simulator/network/hardware/base.py:1900\u001b[0m, in \u001b[0;36m\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 1881\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 1882\u001b[0m \u001b[38;5;124;03mProduce a dictionary describing the current state of this object.\u001b[39;00m\n\u001b[1;32m 1883\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1887\u001b[0m \u001b[38;5;124;03m:rtype: Dict\u001b[39;00m\n\u001b[1;32m 1888\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 1889\u001b[0m state \u001b[38;5;241m=\u001b[39m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39mdescribe_state()\n\u001b[1;32m 1890\u001b[0m state\u001b[38;5;241m.\u001b[39mupdate(\n\u001b[1;32m 1891\u001b[0m {\n\u001b[1;32m 1892\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhostname\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mhostname,\n\u001b[1;32m 1893\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moperating_state\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39moperating_state\u001b[38;5;241m.\u001b[39mvalue,\n\u001b[1;32m 1894\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mNICs\u001b[39m\u001b[38;5;124m\"\u001b[39m: {\n\u001b[1;32m 1895\u001b[0m eth_num: network_interface\u001b[38;5;241m.\u001b[39mdescribe_state()\n\u001b[1;32m 1896\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m eth_num, network_interface \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnetwork_interface\u001b[38;5;241m.\u001b[39mitems()\n\u001b[1;32m 1897\u001b[0m },\n\u001b[1;32m 1898\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfile_system\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfile_system\u001b[38;5;241m.\u001b[39mdescribe_state(),\n\u001b[1;32m 1899\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mapplications\u001b[39m\u001b[38;5;124m\"\u001b[39m: {app\u001b[38;5;241m.\u001b[39mname: app\u001b[38;5;241m.\u001b[39mdescribe_state() \u001b[38;5;28;01mfor\u001b[39;00m app \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mapplications\u001b[38;5;241m.\u001b[39mvalues()},\n\u001b[0;32m-> 1900\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mservices\u001b[39m\u001b[38;5;124m\"\u001b[39m: {svc\u001b[38;5;241m.\u001b[39mname: \u001b[43msvc\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdescribe_state\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mfor\u001b[39;00m svc \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mservices\u001b[38;5;241m.\u001b[39mvalues()},\n\u001b[1;32m 1901\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mprocess\u001b[39m\u001b[38;5;124m\"\u001b[39m: {proc\u001b[38;5;241m.\u001b[39mname: proc\u001b[38;5;241m.\u001b[39mdescribe_state() \u001b[38;5;28;01mfor\u001b[39;00m proc \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprocesses\u001b[38;5;241m.\u001b[39mvalues()},\n\u001b[1;32m 1902\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mrevealed_to_red\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mrevealed_to_red,\n\u001b[1;32m 1903\u001b[0m }\n\u001b[1;32m 1904\u001b[0m )\n\u001b[1;32m 1905\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m state\n", + "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/simulator/system/services/ntp/ntp_client.py:61\u001b[0m, in \u001b[0;36mNTPClient.describe_state\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 50\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdescribe_state\u001b[39m(\u001b[38;5;28mself\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Dict:\n\u001b[1;32m 51\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 52\u001b[0m \u001b[38;5;124;03m Describes the current state of the software.\u001b[39;00m\n\u001b[1;32m 53\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 59\u001b[0m \u001b[38;5;124;03m :rtype: Dict\u001b[39;00m\n\u001b[1;32m 60\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m---> 61\u001b[0m state \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241m.\u001b[39mdescribe_state()\n\u001b[1;32m 62\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m state\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + ] + } + ], "source": [ "env.step(13) # Patch the database\n", "print(f\"step: {env.game.step_counter}, Red action: {info['agent_actions']['data_manipulation_attacker'].action}, Blue reward:{reward:.2f}\" )\n", @@ -600,7 +5091,7 @@ "while abs(reward - 0.8) > 1e-5:\n", " obs, reward, terminated, truncated, info = env.step(0) # do nothing\n", " print(f\"step: {env.game.step_counter}, Red action: {info['agent_actions']['data_manipulation_attacker'].action}, Blue reward:{reward:.2f}\" )\n", - " if env.game.step_counter > 10000:\n", + " if env.game.step_counter > 2000:\n", " break # make sure there's no infinite loop if something went wrong" ] }, From 7dafec85176130dc3c9ce66bf4dbe8ec3b2faa7e Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Mon, 17 Feb 2025 15:01:02 +0000 Subject: [PATCH 17/24] #3075: Test fix --- .../game_layer/actions/test_terminal_actions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration_tests/game_layer/actions/test_terminal_actions.py b/tests/integration_tests/game_layer/actions/test_terminal_actions.py index c39d8263..af9c7b74 100644 --- a/tests/integration_tests/game_layer/actions/test_terminal_actions.py +++ b/tests/integration_tests/game_layer/actions/test_terminal_actions.py @@ -106,7 +106,7 @@ def test_remote_login_change_password(game_and_agent_fixture: Tuple[PrimaiteGame "username": "user123", "current_password": "password", "new_password": "different_password", - "remote_ip": str(server_1.network_interface[1].ip_address), + # "remote_ip": str(server_1.network_interface[1].ip_address), }, ) agent.store_action(action) @@ -146,7 +146,7 @@ def test_change_password_logs_out_user(game_and_agent_fixture: Tuple[PrimaiteGam "username": "user123", "current_password": "password", "new_password": "different_password", - "remote_ip": str(server_1.network_interface[1].ip_address), + # "remote_ip": str(server_1.network_interface[1].ip_address), }, ) agent.store_action(action) From 5f076ba225bf239a61377478949e8b474b7504d7 Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Mon, 17 Feb 2025 15:12:12 +0000 Subject: [PATCH 18/24] #3075: Fix pre-commit errors --- .../Data-Manipulation-E2E-Demonstration.ipynb | 4527 +---------------- 1 file changed, 18 insertions(+), 4509 deletions(-) diff --git a/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb b/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb index f3e3fb98..31e1f9d3 100644 --- a/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb +++ b/src/primaite/notebooks/Data-Manipulation-E2E-Demonstration.ipynb @@ -373,29 +373,16 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-02-17 11:15:30,959: Performing the PrimAITE first-time setup...\n", - "2025-02-17 11:15:30,960: Building the PrimAITE app directories...\n", - "2025-02-17 11:15:30,960: Building primaite_config.yaml...\n", - "2025-02-17 11:15:30,960: Rebuilding the demo notebooks...\n", - "2025-02-17 11:15:30,967: Rebuilding the example notebooks...\n", - "2025-02-17 11:15:30,970: PrimAITE setup complete!\n" - ] - } - ], + "outputs": [], "source": [ "!primaite setup" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": { "tags": [] }, @@ -407,7 +394,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": { "tags": [] }, @@ -433,254 +420,9 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-02-17 11:15:35,108: PrimaiteGymEnv RNG seed = None\n", - "2025-02-17 11:15:35,109: Resetting environment, episode 0, avg. reward: 0.0\n", - "2025-02-17 11:15:35,111: Saving agent action log to /home/nick/primaite/4.0.0a1-dev/sessions/2025-02-17/11-15-31/agent_actions/episode_0.json\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "env created successfully\n", - "{'ICS': 0,\n", - " 'LINKS': {1: {'PROTOCOLS': {'ALL': 0}},\n", - " 2: {'PROTOCOLS': {'ALL': 0}},\n", - " 3: {'PROTOCOLS': {'ALL': 0}},\n", - " 4: {'PROTOCOLS': {'ALL': 0}},\n", - " 5: {'PROTOCOLS': {'ALL': 0}},\n", - " 6: {'PROTOCOLS': {'ALL': 0}},\n", - " 7: {'PROTOCOLS': {'ALL': 0}},\n", - " 8: {'PROTOCOLS': {'ALL': 0}},\n", - " 9: {'PROTOCOLS': {'ALL': 0}},\n", - " 10: {'PROTOCOLS': {'ALL': 0}}},\n", - " 'NODES': {'HOST0': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0,\n", - " 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0,\n", - " 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0,\n", - " 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0,\n", - " 'outbound': 0}}},\n", - " 'nic_status': 0}},\n", - " 'SERVICES': {1: {'health_status': 0,\n", - " 'operating_status': 0}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'HOST1': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0,\n", - " 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0,\n", - " 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0,\n", - " 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0,\n", - " 'outbound': 0}}},\n", - " 'nic_status': 0}},\n", - " 'SERVICES': {1: {'health_status': 0,\n", - " 'operating_status': 1}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'HOST2': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0,\n", - " 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0,\n", - " 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0,\n", - " 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0,\n", - " 'outbound': 0}}},\n", - " 'nic_status': 0}},\n", - " 'SERVICES': {1: {'health_status': 0,\n", - " 'operating_status': 0}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'HOST3': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0,\n", - " 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0,\n", - " 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0,\n", - " 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0,\n", - " 'outbound': 0}}},\n", - " 'nic_status': 0}},\n", - " 'SERVICES': {1: {'health_status': 0,\n", - " 'operating_status': 0}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'HOST4': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0,\n", - " 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0,\n", - " 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0,\n", - " 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0,\n", - " 'outbound': 0}}},\n", - " 'nic_status': 1}},\n", - " 'SERVICES': {1: {'health_status': 0,\n", - " 'operating_status': 0}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'HOST5': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0,\n", - " 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0,\n", - " 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0,\n", - " 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0,\n", - " 'outbound': 0}}},\n", - " 'nic_status': 0}},\n", - " 'SERVICES': {1: {'health_status': 0,\n", - " 'operating_status': 0}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'HOST6': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0,\n", - " 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0,\n", - " 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0,\n", - " 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0,\n", - " 'outbound': 0}}},\n", - " 'nic_status': 0}},\n", - " 'SERVICES': {1: {'health_status': 0,\n", - " 'operating_status': 0}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'ROUTER0': {'ACL': {1: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 0,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 2: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 1,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 3: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 2,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 4: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 3,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 5: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 4,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 6: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 5,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 7: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 6,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 8: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 7,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 9: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 8,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 10: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 9,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0}},\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}}}}\n" - ] - } - ], + "outputs": [], "source": [ "# create the env\n", "with open(data_manipulation_config_path(), 'r') as f:\n", @@ -708,7 +450,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -726,51 +468,9 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "step: 1, Red action: DO NOTHING, Blue reward:0.65\n", - "step: 2, Red action: DO NOTHING, Blue reward:0.65\n", - "step: 3, Red action: DO NOTHING, Blue reward:0.65\n", - "step: 4, Red action: DO NOTHING, Blue reward:0.90\n", - "step: 5, Red action: DO NOTHING, Blue reward:0.90\n", - "step: 6, Red action: DO NOTHING, Blue reward:0.90\n", - "step: 7, Red action: DO NOTHING, Blue reward:0.90\n", - "step: 8, Red action: DO NOTHING, Blue reward:0.90\n", - "step: 9, Red action: DO NOTHING, Blue reward:0.90\n", - "step: 10, Red action: DO NOTHING, Blue reward:0.90\n", - "step: 11, Red action: DO NOTHING, Blue reward:0.90\n", - "step: 12, Red action: DO NOTHING, Blue reward:0.90\n", - "step: 13, Red action: DO NOTHING, Blue reward:0.90\n", - "step: 14, Red action: DO NOTHING, Blue reward:0.90\n", - "step: 15, Red action: DO NOTHING, Blue reward:0.90\n", - "step: 16, Red action: DO NOTHING, Blue reward:0.90\n", - "step: 17, Red action: DO NOTHING, Blue reward:0.95\n", - "step: 18, Red action: DO NOTHING, Blue reward:0.95\n", - "step: 19, Red action: DO NOTHING, Blue reward:0.95\n", - "step: 20, Red action: DO NOTHING, Blue reward:0.95\n", - "step: 21, Red action: DO NOTHING, Blue reward:0.95\n", - "step: 22, Red action: DO NOTHING, Blue reward:0.95\n", - "step: 23, Red action: DO NOTHING, Blue reward:0.95\n", - "step: 24, Red action: DO NOTHING, Blue reward:0.95\n", - "step: 25, Red action: DO NOTHING, Blue reward:0.95\n", - "step: 26, Red action: ATTACK from client 2, Blue reward:0.15\n", - "step: 27, Red action: DO NOTHING, Blue reward:-0.35\n", - "step: 28, Red action: DO NOTHING, Blue reward:-0.85\n", - "step: 29, Red action: DO NOTHING, Blue reward:-0.85\n", - "step: 30, Red action: DO NOTHING, Blue reward:-0.85\n", - "step: 31, Red action: DO NOTHING, Blue reward:-0.85\n", - "step: 32, Red action: DO NOTHING, Blue reward:-0.85\n", - "step: 33, Red action: DO NOTHING, Blue reward:-0.85\n", - "step: 34, Red action: DO NOTHING, Blue reward:-0.85\n", - "step: 35, Red action: DO NOTHING, Blue reward:-0.85\n" - ] - } - ], + "outputs": [], "source": [ "for step in range(35):\n", " obs, reward, terminated, truncated, info = env.step(0)\n", @@ -786,198 +486,9 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'HOST0': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 0}},\n", - " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'HOST1': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 0}},\n", - " 'SERVICES': {1: {'health_status': 0, 'operating_status': 1}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'HOST2': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 0}},\n", - " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'HOST3': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 0}},\n", - " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'HOST4': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 1}},\n", - " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'HOST5': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 0}},\n", - " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'HOST6': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 0}},\n", - " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'ROUTER0': {'ACL': {1: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 0,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 2: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 1,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 3: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 2,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 4: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 3,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 5: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 4,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 6: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 5,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 7: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 6,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 8: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 7,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 9: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 8,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 10: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 9,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0}},\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}}}\n" - ] - } - ], + "outputs": [], "source": [ "pprint(obs['NODES'])" ] @@ -991,198 +502,9 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'HOST0': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 0}},\n", - " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'HOST1': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 0}},\n", - " 'SERVICES': {1: {'health_status': 3, 'operating_status': 1}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'HOST2': {'FOLDERS': {1: {'FILES': {1: {'health_status': 2}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 0}},\n", - " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'HOST3': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 0}},\n", - " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'HOST4': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 1}},\n", - " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'HOST5': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 0}},\n", - " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'HOST6': {'FOLDERS': {1: {'FILES': {1: {'health_status': 0}},\n", - " 'health_status': 0}},\n", - " 'NICS': {1: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 1},\n", - " 2: {'NMNE': {'inbound': 0, 'outbound': 0},\n", - " 'TRAFFIC': {'icmp': {'inbound': 0, 'outbound': 0},\n", - " 'tcp': {53: {'inbound': 0, 'outbound': 0}}},\n", - " 'nic_status': 0}},\n", - " 'SERVICES': {1: {'health_status': 0, 'operating_status': 0}},\n", - " 'operating_status': 1,\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}},\n", - " 'ROUTER0': {'ACL': {1: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 0,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 2: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 1,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 3: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 2,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 4: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 3,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 5: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 4,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 6: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 5,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 7: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 6,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 8: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 7,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 9: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 8,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0},\n", - " 10: {'dest_ip_id': 0,\n", - " 'dest_port_id': 0,\n", - " 'dest_wildcard_id': 0,\n", - " 'permission': 0,\n", - " 'position': 9,\n", - " 'protocol_id': 0,\n", - " 'source_ip_id': 0,\n", - " 'source_port_id': 0,\n", - " 'source_wildcard_id': 0}},\n", - " 'users': {'local_login': 0, 'remote_sessions': 0}}}\n" - ] - } - ], + "outputs": [], "source": [ "obs, reward, terminated, truncated, info = env.step(9) # scan database file\n", "obs, reward, terminated, truncated, info = env.step(1) # scan webapp service\n", @@ -1214,21 +536,9 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "step: 38\n", - "Red action: do-nothing\n", - "Green action: do-nothing\n", - "Green action: node-application-execute\n", - "Blue reward:-0.8500000000000001\n" - ] - } - ], + "outputs": [], "source": [ "obs, reward, terminated, truncated, info = env.step(13) # patch the database\n", "print(f\"step: {env.game.step_counter}\")\n", @@ -1251,21 +561,9 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "step: 39\n", - "Red action: do-nothing\n", - "Green action: timestep=38 action='node-application-execute' parameters={'node_name': 'client_2', 'application_name': 'web-browser'} request=['network', 'node', 'client_2', 'application', 'web-browser', 'execute'] response=RequestResponse(status='failure', data={}) reward=-0.2 reward_info={'connection_attempt_status': 'n/a'}\n", - "Green action: timestep=38 action='node-application-execute' parameters={'node_name': 'client_1', 'application_name': 'web-browser'} request=['network', 'node', 'client_1', 'application', 'web-browser', 'execute'] response=RequestResponse(status='failure', data={}) reward=-0.25 reward_info={'connection_attempt_status': 'n/a'}\n", - "Blue reward:-0.05\n" - ] - } - ], + "outputs": [], "source": [ "obs, reward, terminated, truncated, info = env.step(0) # do nothing\n", "print(f\"step: {env.game.step_counter}\")\n", @@ -1288,3796 +586,7 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "step: 40, Red action: do-nothing, Blue reward:-0.05\n", - "step: 41, Red action: do-nothing, Blue reward:-0.05\n", - "step: 42, Red action: do-nothing, Blue reward:-0.05\n", - "step: 43, Red action: do-nothing, Blue reward:0.95\n", - "step: 44, Red action: do-nothing, Blue reward:0.95\n", - "step: 45, Red action: do-nothing, Blue reward:0.95\n", - "step: 46, Red action: do-nothing, Blue reward:0.95\n", - "step: 47, Red action: do-nothing, Blue reward:0.95\n", - "step: 48, Red action: do-nothing, Blue reward:0.95\n", - "step: 49, Red action: do-nothing, Blue reward:0.95\n", - "step: 50, Red action: do-nothing, Blue reward:0.95\n", - "step: 51, Red action: node-application-execute, Blue reward:0.95\n", - "step: 52, Red action: do-nothing, Blue reward:0.95\n", - "step: 53, Red action: do-nothing, Blue reward:0.85\n", - "step: 54, Red action: do-nothing, Blue reward:0.85\n", - "step: 55, Red action: do-nothing, Blue reward:0.85\n", - "step: 56, Red action: do-nothing, Blue reward:0.85\n", - "step: 57, Red action: do-nothing, Blue reward:0.85\n", - "step: 58, Red action: do-nothing, Blue reward:0.85\n", - "step: 59, Red action: do-nothing, Blue reward:0.85\n", - "step: 60, Red action: do-nothing, Blue reward:0.85\n", - "step: 61, Red action: do-nothing, Blue reward:0.85\n", - "step: 62, Red action: do-nothing, Blue reward:0.85\n", - "step: 63, Red action: do-nothing, Blue reward:0.85\n", - "step: 64, Red action: do-nothing, Blue reward:0.85\n", - "step: 65, Red action: do-nothing, Blue reward:0.85\n", - "step: 66, Red action: do-nothing, Blue reward:0.85\n", - "step: 67, Red action: do-nothing, Blue reward:0.85\n", - "step: 68, Red action: do-nothing, Blue reward:0.85\n", - "step: 69, Red action: do-nothing, Blue reward:0.85\n", - "step: 70, Red action: do-nothing, Blue reward:0.85\n", - "step: 71, Red action: do-nothing, Blue reward:0.85\n", - "step: 72, Red action: do-nothing, Blue reward:0.85\n", - "step: 73, Red action: do-nothing, Blue reward:0.85\n", - "step: 74, Red action: do-nothing, Blue reward:0.85\n", - "step: 75, Red action: node-application-execute, Blue reward:0.85\n", - "step: 76, Red action: do-nothing, Blue reward:0.85\n", - "step: 77, Red action: do-nothing, Blue reward:0.85\n", - "step: 78, Red action: do-nothing, Blue reward:0.85\n", - "step: 79, Red action: do-nothing, Blue reward:0.85\n", - "step: 80, Red action: do-nothing, Blue reward:0.85\n", - "step: 81, Red action: do-nothing, Blue reward:0.85\n", - "step: 82, Red action: do-nothing, Blue reward:0.85\n", - "step: 83, Red action: do-nothing, Blue reward:0.85\n", - "step: 84, Red action: do-nothing, Blue reward:0.85\n", - "step: 85, Red action: do-nothing, Blue reward:0.85\n", - "step: 86, Red action: do-nothing, Blue reward:0.85\n", - "step: 87, Red action: do-nothing, Blue reward:0.85\n", - "step: 88, Red action: do-nothing, Blue reward:0.85\n", - "step: 89, Red action: do-nothing, Blue reward:0.85\n", - "step: 90, Red action: do-nothing, Blue reward:0.85\n", - "step: 91, Red action: do-nothing, Blue reward:0.85\n", - "step: 92, Red action: do-nothing, Blue reward:0.85\n", - "step: 93, Red action: do-nothing, Blue reward:0.85\n", - "step: 94, Red action: do-nothing, Blue reward:0.85\n", - "step: 95, Red action: do-nothing, Blue reward:0.85\n", - "step: 96, Red action: do-nothing, Blue reward:0.85\n", - "step: 97, Red action: node-application-execute, Blue reward:0.85\n", - "step: 98, Red action: do-nothing, Blue reward:0.85\n", - "step: 99, Red action: do-nothing, Blue reward:0.85\n", - "step: 100, Red action: do-nothing, Blue reward:0.85\n", - "step: 101, Red action: do-nothing, Blue reward:0.85\n", - "step: 102, Red action: do-nothing, Blue reward:0.85\n", - "step: 103, Red action: do-nothing, Blue reward:0.85\n", - "step: 104, Red action: do-nothing, Blue reward:0.85\n", - "step: 105, Red action: do-nothing, Blue reward:0.85\n", - "step: 106, Red action: do-nothing, Blue reward:0.85\n", - "step: 107, Red action: do-nothing, Blue reward:0.85\n", - "step: 108, Red action: do-nothing, Blue reward:0.85\n", - "step: 109, Red action: do-nothing, Blue reward:0.85\n", - "step: 110, Red action: do-nothing, Blue reward:0.85\n", - "step: 111, Red action: do-nothing, Blue reward:0.85\n", - "step: 112, Red action: do-nothing, Blue reward:0.85\n", - "step: 113, Red action: node-application-execute, Blue reward:0.85\n", - "step: 114, Red action: do-nothing, Blue reward:0.85\n", - "step: 115, Red action: do-nothing, Blue reward:0.85\n", - "step: 116, Red action: do-nothing, Blue reward:0.85\n", - "step: 117, Red action: do-nothing, Blue reward:0.85\n", - "step: 118, Red action: do-nothing, Blue reward:0.85\n", - "step: 119, Red action: do-nothing, Blue reward:0.85\n", - "step: 120, Red action: do-nothing, Blue reward:0.85\n", - "step: 121, Red action: do-nothing, Blue reward:0.85\n", - "step: 122, Red action: do-nothing, Blue reward:0.85\n", - "step: 123, Red action: do-nothing, Blue reward:0.85\n", - "step: 124, Red action: do-nothing, Blue reward:0.85\n", - "step: 125, Red action: do-nothing, Blue reward:0.85\n", - "step: 126, Red action: do-nothing, Blue reward:0.85\n", - "step: 127, Red action: do-nothing, Blue reward:0.85\n", - "step: 128, Red action: do-nothing, Blue reward:0.85\n", - "step: 129, Red action: do-nothing, Blue reward:0.85\n", - "step: 130, Red action: do-nothing, Blue reward:0.85\n", - "step: 131, Red action: do-nothing, Blue reward:0.85\n", - "step: 132, Red action: node-application-execute, Blue reward:0.85\n", - "step: 133, Red action: do-nothing, Blue reward:0.85\n", - "step: 134, Red action: do-nothing, Blue reward:0.85\n", - "step: 135, Red action: do-nothing, Blue reward:0.85\n", - "step: 136, Red action: do-nothing, Blue reward:0.85\n", - "step: 137, Red action: do-nothing, Blue reward:0.85\n", - "step: 138, Red action: do-nothing, Blue reward:0.85\n", - "step: 139, Red action: do-nothing, Blue reward:0.85\n", - "step: 140, Red action: do-nothing, Blue reward:0.85\n", - "step: 141, Red action: do-nothing, Blue reward:0.85\n", - "step: 142, Red action: do-nothing, Blue reward:0.85\n", - "step: 143, Red action: do-nothing, Blue reward:0.85\n", - "step: 144, Red action: do-nothing, Blue reward:0.85\n", - "step: 145, Red action: do-nothing, Blue reward:0.85\n", - "step: 146, Red action: do-nothing, Blue reward:0.85\n", - "step: 147, Red action: do-nothing, Blue reward:0.85\n", - "step: 148, Red action: do-nothing, Blue reward:0.85\n", - "step: 149, Red action: do-nothing, Blue reward:0.85\n", - "step: 150, Red action: do-nothing, Blue reward:0.85\n", - "step: 151, Red action: do-nothing, Blue reward:0.85\n", - "step: 152, Red action: do-nothing, Blue reward:0.85\n", - "step: 153, Red action: do-nothing, Blue reward:0.85\n", - "step: 154, Red action: do-nothing, Blue reward:0.85\n", - "step: 155, Red action: do-nothing, Blue reward:0.85\n", - "step: 156, Red action: do-nothing, Blue reward:0.85\n", - "step: 157, Red action: node-application-execute, Blue reward:0.85\n", - "step: 158, Red action: do-nothing, Blue reward:0.85\n", - "step: 159, Red action: do-nothing, Blue reward:0.85\n", - "step: 160, Red action: do-nothing, Blue reward:0.85\n", - "step: 161, Red action: do-nothing, Blue reward:0.85\n", - "step: 162, Red action: do-nothing, Blue reward:0.85\n", - "step: 163, Red action: do-nothing, Blue reward:0.85\n", - "step: 164, Red action: do-nothing, Blue reward:0.85\n", - "step: 165, Red action: do-nothing, Blue reward:0.85\n", - "step: 166, Red action: do-nothing, Blue reward:0.85\n", - "step: 167, Red action: do-nothing, Blue reward:0.85\n", - "step: 168, Red action: do-nothing, Blue reward:0.85\n", - "step: 169, Red action: do-nothing, Blue reward:0.85\n", - "step: 170, Red action: do-nothing, Blue reward:0.85\n", - "step: 171, Red action: do-nothing, Blue reward:0.85\n", - "step: 172, Red action: do-nothing, Blue reward:0.85\n", - "step: 173, Red action: do-nothing, Blue reward:0.85\n", - "step: 174, Red action: do-nothing, Blue reward:0.85\n", - "step: 175, Red action: do-nothing, Blue reward:0.85\n", - "step: 176, Red action: do-nothing, Blue reward:0.85\n", - "step: 177, Red action: do-nothing, Blue reward:0.85\n", - "step: 178, Red action: node-application-execute, Blue reward:0.85\n", - "step: 179, Red action: do-nothing, Blue reward:0.85\n", - "step: 180, Red action: do-nothing, Blue reward:0.85\n", - "step: 181, Red action: do-nothing, Blue reward:0.85\n", - "step: 182, Red action: do-nothing, Blue reward:0.85\n", - "step: 183, Red action: do-nothing, Blue reward:0.85\n", - "step: 184, Red action: do-nothing, Blue reward:0.85\n", - "step: 185, Red action: do-nothing, Blue reward:0.85\n", - "step: 186, Red action: do-nothing, Blue reward:0.85\n", - "step: 187, Red action: do-nothing, Blue reward:0.85\n", - "step: 188, Red action: do-nothing, Blue reward:0.85\n", - "step: 189, Red action: do-nothing, Blue reward:0.85\n", - "step: 190, Red action: do-nothing, Blue reward:0.85\n", - "step: 191, Red action: do-nothing, Blue reward:0.85\n", - "step: 192, Red action: do-nothing, Blue reward:0.85\n", - "step: 193, Red action: do-nothing, Blue reward:0.85\n", - "step: 194, Red action: do-nothing, Blue reward:0.85\n", - "step: 195, Red action: do-nothing, Blue reward:0.85\n", - "step: 196, Red action: do-nothing, Blue reward:0.85\n", - "step: 197, Red action: do-nothing, Blue reward:0.85\n", - "step: 198, Red action: node-application-execute, Blue reward:0.85\n", - "step: 199, Red action: do-nothing, Blue reward:0.85\n", - "step: 200, Red action: do-nothing, Blue reward:0.85\n", - "step: 201, Red action: do-nothing, Blue reward:0.85\n", - "step: 202, Red action: do-nothing, Blue reward:0.85\n", - "step: 203, Red action: do-nothing, Blue reward:0.85\n", - "step: 204, Red action: do-nothing, Blue reward:0.85\n", - "step: 205, Red action: do-nothing, Blue reward:0.85\n", - "step: 206, Red action: do-nothing, Blue reward:0.85\n", - "step: 207, Red action: do-nothing, Blue reward:0.85\n", - "step: 208, Red action: do-nothing, Blue reward:0.85\n", - "step: 209, Red action: do-nothing, Blue reward:0.85\n", - "step: 210, Red action: do-nothing, Blue reward:0.85\n", - "step: 211, Red action: do-nothing, Blue reward:0.85\n", - "step: 212, Red action: do-nothing, Blue reward:0.85\n", - "step: 213, Red action: do-nothing, Blue reward:0.85\n", - "step: 214, Red action: do-nothing, Blue reward:0.85\n", - "step: 215, Red action: do-nothing, Blue reward:0.85\n", - "step: 216, Red action: do-nothing, Blue reward:0.85\n", - "step: 217, Red action: do-nothing, Blue reward:0.85\n", - "step: 218, Red action: node-application-execute, Blue reward:0.85\n", - "step: 219, Red action: do-nothing, Blue reward:0.85\n", - "step: 220, Red action: do-nothing, Blue reward:0.85\n", - "step: 221, Red action: do-nothing, Blue reward:0.85\n", - "step: 222, Red action: do-nothing, Blue reward:0.85\n", - "step: 223, Red action: do-nothing, Blue reward:0.85\n", - "step: 224, Red action: do-nothing, Blue reward:0.85\n", - "step: 225, Red action: do-nothing, Blue reward:0.85\n", - "step: 226, Red action: do-nothing, Blue reward:0.85\n", - "step: 227, Red action: do-nothing, Blue reward:0.85\n", - "step: 228, Red action: do-nothing, Blue reward:0.85\n", - "step: 229, Red action: do-nothing, Blue reward:0.85\n", - "step: 230, Red action: do-nothing, Blue reward:0.85\n", - "step: 231, Red action: do-nothing, Blue reward:0.85\n", - "step: 232, Red action: do-nothing, Blue reward:0.85\n", - "step: 233, Red action: do-nothing, Blue reward:0.85\n", - "step: 234, Red action: do-nothing, Blue reward:0.85\n", - "step: 235, Red action: do-nothing, Blue reward:0.85\n", - "step: 236, Red action: do-nothing, Blue reward:0.85\n", - "step: 237, Red action: do-nothing, Blue reward:0.85\n", - "step: 238, Red action: do-nothing, Blue reward:0.85\n", - "step: 239, Red action: do-nothing, Blue reward:0.85\n", - "step: 240, Red action: do-nothing, Blue reward:0.85\n", - "step: 241, Red action: node-application-execute, Blue reward:0.85\n", - "step: 242, Red action: do-nothing, Blue reward:0.85\n", - "step: 243, Red action: do-nothing, Blue reward:0.85\n", - "step: 244, Red action: do-nothing, Blue reward:0.85\n", - "step: 245, Red action: do-nothing, Blue reward:0.85\n", - "step: 246, Red action: do-nothing, Blue reward:0.85\n", - "step: 247, Red action: do-nothing, Blue reward:0.85\n", - "step: 248, Red action: do-nothing, Blue reward:0.85\n", - "step: 249, Red action: do-nothing, Blue reward:0.85\n", - "step: 250, Red action: do-nothing, Blue reward:0.85\n", - "step: 251, Red action: do-nothing, Blue reward:0.85\n", - "step: 252, Red action: do-nothing, Blue reward:0.85\n", - "step: 253, Red action: do-nothing, Blue reward:0.85\n", - "step: 254, Red action: do-nothing, Blue reward:0.85\n", - "step: 255, Red action: do-nothing, Blue reward:0.85\n", - "step: 256, Red action: do-nothing, Blue reward:0.85\n", - "step: 257, Red action: do-nothing, Blue reward:0.85\n", - "step: 258, Red action: do-nothing, Blue reward:0.85\n", - "step: 259, Red action: do-nothing, Blue reward:0.85\n", - "step: 260, Red action: do-nothing, Blue reward:0.85\n", - "step: 261, Red action: node-application-execute, Blue reward:0.85\n", - "step: 262, Red action: do-nothing, Blue reward:0.85\n", - "step: 263, Red action: do-nothing, Blue reward:0.85\n", - "step: 264, Red action: do-nothing, Blue reward:0.85\n", - "step: 265, Red action: do-nothing, Blue reward:0.85\n", - "step: 266, Red action: do-nothing, Blue reward:0.85\n", - "step: 267, Red action: do-nothing, Blue reward:0.85\n", - "step: 268, Red action: do-nothing, Blue reward:0.85\n", - "step: 269, Red action: do-nothing, Blue reward:0.85\n", - "step: 270, Red action: do-nothing, Blue reward:0.85\n", - "step: 271, Red action: do-nothing, Blue reward:0.85\n", - "step: 272, Red action: do-nothing, Blue reward:0.85\n", - "step: 273, Red action: do-nothing, Blue reward:0.85\n", - "step: 274, Red action: do-nothing, Blue reward:0.85\n", - "step: 275, Red action: do-nothing, Blue reward:0.85\n", - "step: 276, Red action: do-nothing, Blue reward:0.85\n", - "step: 277, Red action: do-nothing, Blue reward:0.85\n", - "step: 278, Red action: do-nothing, Blue reward:0.85\n", - "step: 279, Red action: do-nothing, Blue reward:0.85\n", - "step: 280, Red action: do-nothing, Blue reward:0.85\n", - "step: 281, Red action: node-application-execute, Blue reward:0.85\n", - "step: 282, Red action: do-nothing, Blue reward:0.85\n", - "step: 283, Red action: do-nothing, Blue reward:0.85\n", - "step: 284, Red action: do-nothing, Blue reward:0.85\n", - "step: 285, Red action: do-nothing, Blue reward:0.85\n", - "step: 286, Red action: do-nothing, Blue reward:0.85\n", - "step: 287, Red action: do-nothing, Blue reward:0.85\n", - "step: 288, Red action: do-nothing, Blue reward:0.85\n", - "step: 289, Red action: do-nothing, Blue reward:0.85\n", - "step: 290, Red action: do-nothing, Blue reward:0.85\n", - "step: 291, Red action: do-nothing, Blue reward:0.85\n", - "step: 292, Red action: do-nothing, Blue reward:0.85\n", - "step: 293, Red action: do-nothing, Blue reward:0.85\n", - "step: 294, Red action: do-nothing, Blue reward:0.85\n", - "step: 295, Red action: do-nothing, Blue reward:0.85\n", - "step: 296, Red action: do-nothing, Blue reward:0.85\n", - "step: 297, Red action: do-nothing, Blue reward:0.85\n", - "step: 298, Red action: do-nothing, Blue reward:0.85\n", - "step: 299, Red action: do-nothing, Blue reward:0.85\n", - "step: 300, Red action: do-nothing, Blue reward:0.85\n", - "step: 301, Red action: do-nothing, Blue reward:0.85\n", - "step: 302, Red action: do-nothing, Blue reward:0.85\n", - "step: 303, Red action: do-nothing, Blue reward:0.85\n", - "step: 304, Red action: node-application-execute, Blue reward:0.85\n", - "step: 305, Red action: do-nothing, Blue reward:0.85\n", - "step: 306, Red action: do-nothing, Blue reward:0.85\n", - "step: 307, Red action: do-nothing, Blue reward:0.85\n", - "step: 308, Red action: do-nothing, Blue reward:0.85\n", - "step: 309, Red action: do-nothing, Blue reward:0.85\n", - "step: 310, Red action: do-nothing, Blue reward:0.85\n", - "step: 311, Red action: do-nothing, Blue reward:0.85\n", - "step: 312, Red action: do-nothing, Blue reward:0.85\n", - "step: 313, Red action: do-nothing, Blue reward:0.85\n", - "step: 314, Red action: do-nothing, Blue reward:0.85\n", - "step: 315, Red action: do-nothing, Blue reward:0.85\n", - "step: 316, Red action: do-nothing, Blue reward:0.85\n", - "step: 317, Red action: do-nothing, Blue reward:0.85\n", - "step: 318, Red action: do-nothing, Blue reward:0.85\n", - "step: 319, Red action: do-nothing, Blue reward:0.85\n", - "step: 320, Red action: do-nothing, Blue reward:0.85\n", - "step: 321, Red action: do-nothing, Blue reward:0.85\n", - "step: 322, Red action: do-nothing, Blue reward:0.85\n", - "step: 323, Red action: do-nothing, Blue reward:0.85\n", - "step: 324, Red action: do-nothing, Blue reward:0.85\n", - "step: 325, Red action: node-application-execute, Blue reward:0.85\n", - "step: 326, Red action: do-nothing, Blue reward:0.85\n", - "step: 327, Red action: do-nothing, Blue reward:0.85\n", - "step: 328, Red action: do-nothing, Blue reward:0.85\n", - "step: 329, Red action: do-nothing, Blue reward:0.85\n", - "step: 330, Red action: do-nothing, Blue reward:0.85\n", - "step: 331, Red action: do-nothing, Blue reward:0.85\n", - "step: 332, Red action: do-nothing, Blue reward:0.85\n", - "step: 333, Red action: do-nothing, Blue reward:0.85\n", - "step: 334, Red action: do-nothing, Blue reward:0.85\n", - "step: 335, Red action: do-nothing, Blue reward:0.85\n", - "step: 336, Red action: do-nothing, Blue reward:0.85\n", - "step: 337, Red action: do-nothing, Blue reward:0.85\n", - "step: 338, Red action: do-nothing, Blue reward:0.85\n", - "step: 339, Red action: do-nothing, Blue reward:0.85\n", - "step: 340, Red action: do-nothing, Blue reward:0.85\n", - "step: 341, Red action: do-nothing, Blue reward:0.85\n", - "step: 342, Red action: do-nothing, Blue reward:0.85\n", - "step: 343, Red action: do-nothing, Blue reward:0.85\n", - "step: 344, Red action: do-nothing, Blue reward:0.85\n", - "step: 345, Red action: do-nothing, Blue reward:0.85\n", - "step: 346, Red action: do-nothing, Blue reward:0.85\n", - "step: 347, Red action: do-nothing, Blue reward:0.85\n", - "step: 348, Red action: do-nothing, Blue reward:0.85\n", - "step: 349, Red action: node-application-execute, Blue reward:0.85\n", - "step: 350, Red action: do-nothing, Blue reward:0.85\n", - "step: 351, Red action: do-nothing, Blue reward:0.85\n", - "step: 352, Red action: do-nothing, Blue reward:0.85\n", - "step: 353, Red action: do-nothing, Blue reward:0.85\n", - "step: 354, Red action: do-nothing, Blue reward:0.85\n", - "step: 355, Red action: do-nothing, Blue reward:0.85\n", - "step: 356, Red action: do-nothing, Blue reward:0.85\n", - "step: 357, Red action: do-nothing, Blue reward:0.85\n", - "step: 358, Red action: do-nothing, Blue reward:0.85\n", - "step: 359, Red action: do-nothing, Blue reward:0.85\n", - "step: 360, Red action: do-nothing, Blue reward:0.85\n", - "step: 361, Red action: do-nothing, Blue reward:0.85\n", - "step: 362, Red action: do-nothing, Blue reward:0.85\n", - "step: 363, Red action: do-nothing, Blue reward:0.85\n", - "step: 364, Red action: do-nothing, Blue reward:0.85\n", - "step: 365, Red action: do-nothing, Blue reward:0.85\n", - "step: 366, Red action: do-nothing, Blue reward:0.85\n", - "step: 367, Red action: do-nothing, Blue reward:0.85\n", - "step: 368, Red action: do-nothing, Blue reward:0.85\n", - "step: 369, Red action: node-application-execute, Blue reward:0.85\n", - "step: 370, Red action: do-nothing, Blue reward:0.85\n", - "step: 371, Red action: do-nothing, Blue reward:0.85\n", - "step: 372, Red action: do-nothing, Blue reward:0.85\n", - "step: 373, Red action: do-nothing, Blue reward:0.85\n", - "step: 374, Red action: do-nothing, Blue reward:0.85\n", - "step: 375, Red action: do-nothing, Blue reward:0.85\n", - "step: 376, Red action: do-nothing, Blue reward:0.85\n", - "step: 377, Red action: do-nothing, Blue reward:0.85\n", - "step: 378, Red action: do-nothing, Blue reward:0.85\n", - "step: 379, Red action: do-nothing, Blue reward:0.85\n", - "step: 380, Red action: do-nothing, Blue reward:0.85\n", - "step: 381, Red action: do-nothing, Blue reward:0.85\n", - "step: 382, Red action: do-nothing, Blue reward:0.85\n", - "step: 383, Red action: do-nothing, Blue reward:0.85\n", - "step: 384, Red action: do-nothing, Blue reward:0.85\n", - "step: 385, Red action: do-nothing, Blue reward:0.85\n", - "step: 386, Red action: do-nothing, Blue reward:0.85\n", - "step: 387, Red action: do-nothing, Blue reward:0.85\n", - "step: 388, Red action: do-nothing, Blue reward:0.85\n", - "step: 389, Red action: do-nothing, Blue reward:0.85\n", - "step: 390, Red action: do-nothing, Blue reward:0.85\n", - "step: 391, Red action: do-nothing, Blue reward:0.85\n", - "step: 392, Red action: do-nothing, Blue reward:0.85\n", - "step: 393, Red action: do-nothing, Blue reward:0.85\n", - "step: 394, Red action: node-application-execute, Blue reward:0.85\n", - "step: 395, Red action: do-nothing, Blue reward:0.85\n", - "step: 396, Red action: do-nothing, Blue reward:0.85\n", - "step: 397, Red action: do-nothing, Blue reward:0.85\n", - "step: 398, Red action: do-nothing, Blue reward:0.85\n", - "step: 399, Red action: do-nothing, Blue reward:0.85\n", - "step: 400, Red action: do-nothing, Blue reward:0.85\n", - "step: 401, Red action: do-nothing, Blue reward:0.85\n", - "step: 402, Red action: do-nothing, Blue reward:0.85\n", - "step: 403, Red action: do-nothing, Blue reward:0.85\n", - "step: 404, Red action: do-nothing, Blue reward:0.85\n", - "step: 405, Red action: do-nothing, Blue reward:0.85\n", - "step: 406, Red action: do-nothing, Blue reward:0.85\n", - "step: 407, Red action: do-nothing, Blue reward:0.85\n", - "step: 408, Red action: do-nothing, Blue reward:0.85\n", - "step: 409, Red action: do-nothing, Blue reward:0.85\n", - "step: 410, Red action: do-nothing, Blue reward:0.85\n", - "step: 411, Red action: do-nothing, Blue reward:0.85\n", - "step: 412, Red action: do-nothing, Blue reward:0.85\n", - "step: 413, Red action: do-nothing, Blue reward:0.85\n", - "step: 414, Red action: do-nothing, Blue reward:0.85\n", - "step: 415, Red action: do-nothing, Blue reward:0.85\n", - "step: 416, Red action: do-nothing, Blue reward:0.85\n", - "step: 417, Red action: do-nothing, Blue reward:0.85\n", - "step: 418, Red action: node-application-execute, Blue reward:0.85\n", - "step: 419, Red action: do-nothing, Blue reward:0.85\n", - "step: 420, Red action: do-nothing, Blue reward:0.85\n", - "step: 421, Red action: do-nothing, Blue reward:0.85\n", - "step: 422, Red action: do-nothing, Blue reward:0.85\n", - "step: 423, Red action: do-nothing, Blue reward:0.85\n", - "step: 424, Red action: do-nothing, Blue reward:0.85\n", - "step: 425, Red action: do-nothing, Blue reward:0.85\n", - "step: 426, Red action: do-nothing, Blue reward:0.85\n", - "step: 427, Red action: do-nothing, Blue reward:0.85\n", - "step: 428, Red action: do-nothing, Blue reward:0.85\n", - "step: 429, Red action: do-nothing, Blue reward:0.85\n", - "step: 430, Red action: do-nothing, Blue reward:0.85\n", - "step: 431, Red action: do-nothing, Blue reward:0.85\n", - "step: 432, Red action: do-nothing, Blue reward:0.85\n", - "step: 433, Red action: do-nothing, Blue reward:0.85\n", - "step: 434, Red action: do-nothing, Blue reward:0.85\n", - "step: 435, Red action: do-nothing, Blue reward:0.85\n", - "step: 436, Red action: do-nothing, Blue reward:0.85\n", - "step: 437, Red action: do-nothing, Blue reward:0.85\n", - "step: 438, Red action: do-nothing, Blue reward:0.85\n", - "step: 439, Red action: do-nothing, Blue reward:0.85\n", - "step: 440, Red action: do-nothing, Blue reward:0.85\n", - "step: 441, Red action: do-nothing, Blue reward:0.85\n", - "step: 442, Red action: do-nothing, Blue reward:0.85\n", - "step: 443, Red action: node-application-execute, Blue reward:0.85\n", - "step: 444, Red action: do-nothing, Blue reward:0.85\n", - "step: 445, Red action: do-nothing, Blue reward:0.85\n", - "step: 446, Red action: do-nothing, Blue reward:0.85\n", - "step: 447, Red action: do-nothing, Blue reward:0.85\n", - "step: 448, Red action: do-nothing, Blue reward:0.85\n", - "step: 449, Red action: do-nothing, Blue reward:0.85\n", - "step: 450, Red action: do-nothing, Blue reward:0.85\n", - "step: 451, Red action: do-nothing, Blue reward:0.85\n", - "step: 452, Red action: do-nothing, Blue reward:0.85\n", - "step: 453, Red action: do-nothing, Blue reward:0.85\n", - "step: 454, Red action: do-nothing, Blue reward:0.85\n", - "step: 455, Red action: do-nothing, Blue reward:0.85\n", - "step: 456, Red action: do-nothing, Blue reward:0.85\n", - "step: 457, Red action: do-nothing, Blue reward:0.85\n", - "step: 458, Red action: do-nothing, Blue reward:0.85\n", - "step: 459, Red action: do-nothing, Blue reward:0.85\n", - "step: 460, Red action: do-nothing, Blue reward:0.85\n", - "step: 461, Red action: node-application-execute, Blue reward:0.85\n", - "step: 462, Red action: do-nothing, Blue reward:0.85\n", - "step: 463, Red action: do-nothing, Blue reward:0.85\n", - "step: 464, Red action: do-nothing, Blue reward:0.85\n", - "step: 465, Red action: do-nothing, Blue reward:0.85\n", - "step: 466, Red action: do-nothing, Blue reward:0.85\n", - "step: 467, Red action: do-nothing, Blue reward:0.85\n", - "step: 468, Red action: do-nothing, Blue reward:0.85\n", - "step: 469, Red action: do-nothing, Blue reward:0.85\n", - "step: 470, Red action: do-nothing, Blue reward:0.85\n", - "step: 471, Red action: do-nothing, Blue reward:0.85\n", - "step: 472, Red action: do-nothing, Blue reward:0.85\n", - "step: 473, Red action: do-nothing, Blue reward:0.85\n", - "step: 474, Red action: do-nothing, Blue reward:0.85\n", - "step: 475, Red action: do-nothing, Blue reward:0.85\n", - "step: 476, Red action: do-nothing, Blue reward:0.85\n", - "step: 477, Red action: do-nothing, Blue reward:0.85\n", - "step: 478, Red action: do-nothing, Blue reward:0.85\n", - "step: 479, Red action: do-nothing, Blue reward:0.85\n", - "step: 480, Red action: node-application-execute, Blue reward:0.85\n", - "step: 481, Red action: do-nothing, Blue reward:0.85\n", - "step: 482, Red action: do-nothing, Blue reward:0.85\n", - "step: 483, Red action: do-nothing, Blue reward:0.85\n", - "step: 484, Red action: do-nothing, Blue reward:0.85\n", - "step: 485, Red action: do-nothing, Blue reward:0.85\n", - "step: 486, Red action: do-nothing, Blue reward:0.85\n", - "step: 487, Red action: do-nothing, Blue reward:0.85\n", - "step: 488, Red action: do-nothing, Blue reward:0.85\n", - "step: 489, Red action: do-nothing, Blue reward:0.85\n", - "step: 490, Red action: do-nothing, Blue reward:0.85\n", - "step: 491, Red action: do-nothing, Blue reward:0.85\n", - "step: 492, Red action: do-nothing, Blue reward:0.85\n", - "step: 493, Red action: do-nothing, Blue reward:0.85\n", - "step: 494, Red action: do-nothing, Blue reward:0.85\n", - "step: 495, Red action: do-nothing, Blue reward:0.85\n", - "step: 496, Red action: do-nothing, Blue reward:0.85\n", - "step: 497, Red action: do-nothing, Blue reward:0.85\n", - "step: 498, Red action: do-nothing, Blue reward:0.85\n", - "step: 499, Red action: do-nothing, Blue reward:0.85\n", - "step: 500, Red action: do-nothing, Blue reward:0.85\n", - "step: 501, Red action: do-nothing, Blue reward:0.85\n", - "step: 502, Red action: do-nothing, Blue reward:0.85\n", - "step: 503, Red action: do-nothing, Blue reward:0.85\n", - "step: 504, Red action: node-application-execute, Blue reward:0.85\n", - "step: 505, Red action: do-nothing, Blue reward:0.85\n", - "step: 506, Red action: do-nothing, Blue reward:0.85\n", - "step: 507, Red action: do-nothing, Blue reward:0.85\n", - "step: 508, Red action: do-nothing, Blue reward:0.85\n", - "step: 509, Red action: do-nothing, Blue reward:0.85\n", - "step: 510, Red action: do-nothing, Blue reward:0.85\n", - "step: 511, Red action: do-nothing, Blue reward:0.85\n", - "step: 512, Red action: do-nothing, Blue reward:0.85\n", - "step: 513, Red action: do-nothing, Blue reward:0.85\n", - "step: 514, Red action: do-nothing, Blue reward:0.85\n", - "step: 515, Red action: do-nothing, Blue reward:0.85\n", - "step: 516, Red action: do-nothing, Blue reward:0.85\n", - "step: 517, Red action: do-nothing, Blue reward:0.85\n", - "step: 518, Red action: do-nothing, Blue reward:0.85\n", - "step: 519, Red action: do-nothing, Blue reward:0.85\n", - "step: 520, Red action: do-nothing, Blue reward:0.85\n", - "step: 521, Red action: do-nothing, Blue reward:0.85\n", - "step: 522, Red action: do-nothing, Blue reward:0.85\n", - "step: 523, Red action: do-nothing, Blue reward:0.85\n", - "step: 524, Red action: do-nothing, Blue reward:0.85\n", - "step: 525, Red action: do-nothing, Blue reward:0.85\n", - "step: 526, Red action: do-nothing, Blue reward:0.85\n", - "step: 527, Red action: node-application-execute, Blue reward:0.85\n", - "step: 528, Red action: do-nothing, Blue reward:0.85\n", - "step: 529, Red action: do-nothing, Blue reward:0.85\n", - "step: 530, Red action: do-nothing, Blue reward:0.85\n", - "step: 531, Red action: do-nothing, Blue reward:0.85\n", - "step: 532, Red action: do-nothing, Blue reward:0.85\n", - "step: 533, Red action: do-nothing, Blue reward:0.85\n", - "step: 534, Red action: do-nothing, Blue reward:0.85\n", - "step: 535, Red action: do-nothing, Blue reward:0.85\n", - "step: 536, Red action: do-nothing, Blue reward:0.85\n", - "step: 537, Red action: do-nothing, Blue reward:0.85\n", - "step: 538, Red action: do-nothing, Blue reward:0.85\n", - "step: 539, Red action: do-nothing, Blue reward:0.85\n", - "step: 540, Red action: do-nothing, Blue reward:0.85\n", - "step: 541, Red action: do-nothing, Blue reward:0.85\n", - "step: 542, Red action: do-nothing, Blue reward:0.85\n", - "step: 543, Red action: do-nothing, Blue reward:0.85\n", - "step: 544, Red action: do-nothing, Blue reward:0.85\n", - "step: 545, Red action: do-nothing, Blue reward:0.85\n", - "step: 546, Red action: do-nothing, Blue reward:0.85\n", - "step: 547, Red action: do-nothing, Blue reward:0.85\n", - "step: 548, Red action: do-nothing, Blue reward:0.85\n", - "step: 549, Red action: node-application-execute, Blue reward:0.85\n", - "step: 550, Red action: do-nothing, Blue reward:0.85\n", - "step: 551, Red action: do-nothing, Blue reward:0.85\n", - "step: 552, Red action: do-nothing, Blue reward:0.85\n", - "step: 553, Red action: do-nothing, Blue reward:0.85\n", - "step: 554, Red action: do-nothing, Blue reward:0.85\n", - "step: 555, Red action: do-nothing, Blue reward:0.85\n", - "step: 556, Red action: do-nothing, Blue reward:0.85\n", - "step: 557, Red action: do-nothing, Blue reward:0.85\n", - "step: 558, Red action: do-nothing, Blue reward:0.85\n", - "step: 559, Red action: do-nothing, Blue reward:0.85\n", - "step: 560, Red action: do-nothing, Blue reward:0.85\n", - "step: 561, Red action: do-nothing, Blue reward:0.85\n", - "step: 562, Red action: do-nothing, Blue reward:0.85\n", - "step: 563, Red action: do-nothing, Blue reward:0.85\n", - "step: 564, Red action: do-nothing, Blue reward:0.85\n", - "step: 565, Red action: node-application-execute, Blue reward:0.85\n", - "step: 566, Red action: do-nothing, Blue reward:0.85\n", - "step: 567, Red action: do-nothing, Blue reward:0.85\n", - "step: 568, Red action: do-nothing, Blue reward:0.85\n", - "step: 569, Red action: do-nothing, Blue reward:0.85\n", - "step: 570, Red action: do-nothing, Blue reward:0.85\n", - "step: 571, Red action: do-nothing, Blue reward:0.85\n", - "step: 572, Red action: do-nothing, Blue reward:0.85\n", - "step: 573, Red action: do-nothing, Blue reward:0.85\n", - "step: 574, Red action: do-nothing, Blue reward:0.85\n", - "step: 575, Red action: do-nothing, Blue reward:0.85\n", - "step: 576, Red action: do-nothing, Blue reward:0.85\n", - "step: 577, Red action: do-nothing, Blue reward:0.85\n", - "step: 578, Red action: do-nothing, Blue reward:0.85\n", - "step: 579, Red action: do-nothing, Blue reward:0.85\n", - "step: 580, Red action: do-nothing, Blue reward:0.85\n", - "step: 581, Red action: do-nothing, Blue reward:0.85\n", - "step: 582, Red action: do-nothing, Blue reward:0.85\n", - "step: 583, Red action: do-nothing, Blue reward:0.85\n", - "step: 584, Red action: do-nothing, Blue reward:0.85\n", - "step: 585, Red action: do-nothing, Blue reward:0.85\n", - "step: 586, Red action: do-nothing, Blue reward:0.85\n", - "step: 587, Red action: do-nothing, Blue reward:0.85\n", - "step: 588, Red action: do-nothing, Blue reward:0.85\n", - "step: 589, Red action: node-application-execute, Blue reward:0.85\n", - "step: 590, Red action: do-nothing, Blue reward:0.85\n", - "step: 591, Red action: do-nothing, Blue reward:0.85\n", - "step: 592, Red action: do-nothing, Blue reward:0.85\n", - "step: 593, Red action: do-nothing, Blue reward:0.85\n", - "step: 594, Red action: do-nothing, Blue reward:0.85\n", - "step: 595, Red action: do-nothing, Blue reward:0.85\n", - "step: 596, Red action: do-nothing, Blue reward:0.85\n", - "step: 597, Red action: do-nothing, Blue reward:0.85\n", - "step: 598, Red action: do-nothing, Blue reward:0.85\n", - "step: 599, Red action: do-nothing, Blue reward:0.85\n", - "step: 600, Red action: do-nothing, Blue reward:0.85\n", - "step: 601, Red action: do-nothing, Blue reward:0.85\n", - "step: 602, Red action: do-nothing, Blue reward:0.85\n", - "step: 603, Red action: do-nothing, Blue reward:0.85\n", - "step: 604, Red action: do-nothing, Blue reward:0.85\n", - "step: 605, Red action: do-nothing, Blue reward:0.85\n", - "step: 606, Red action: do-nothing, Blue reward:0.85\n", - "step: 607, Red action: do-nothing, Blue reward:0.85\n", - "step: 608, Red action: do-nothing, Blue reward:0.85\n", - "step: 609, Red action: do-nothing, Blue reward:0.85\n", - "step: 610, Red action: node-application-execute, Blue reward:0.85\n", - "step: 611, Red action: do-nothing, Blue reward:0.85\n", - "step: 612, Red action: do-nothing, Blue reward:0.85\n", - "step: 613, Red action: do-nothing, Blue reward:0.85\n", - "step: 614, Red action: do-nothing, Blue reward:0.85\n", - "step: 615, Red action: do-nothing, Blue reward:0.85\n", - "step: 616, Red action: do-nothing, Blue reward:0.85\n", - "step: 617, Red action: do-nothing, Blue reward:0.85\n", - "step: 618, Red action: do-nothing, Blue reward:0.85\n", - "step: 619, Red action: do-nothing, Blue reward:0.85\n", - "step: 620, Red action: do-nothing, Blue reward:0.85\n", - "step: 621, Red action: do-nothing, Blue reward:0.85\n", - "step: 622, Red action: do-nothing, Blue reward:0.85\n", - "step: 623, Red action: do-nothing, Blue reward:0.85\n", - "step: 624, Red action: do-nothing, Blue reward:0.85\n", - "step: 625, Red action: do-nothing, Blue reward:0.85\n", - "step: 626, Red action: node-application-execute, Blue reward:0.85\n", - "step: 627, Red action: do-nothing, Blue reward:0.85\n", - "step: 628, Red action: do-nothing, Blue reward:0.85\n", - "step: 629, Red action: do-nothing, Blue reward:0.85\n", - "step: 630, Red action: do-nothing, Blue reward:0.85\n", - "step: 631, Red action: do-nothing, Blue reward:0.85\n", - "step: 632, Red action: do-nothing, Blue reward:0.85\n", - "step: 633, Red action: do-nothing, Blue reward:0.85\n", - "step: 634, Red action: do-nothing, Blue reward:0.85\n", - "step: 635, Red action: do-nothing, Blue reward:0.85\n", - "step: 636, Red action: do-nothing, Blue reward:0.85\n", - "step: 637, Red action: do-nothing, Blue reward:0.85\n", - "step: 638, Red action: do-nothing, Blue reward:0.85\n", - "step: 639, Red action: do-nothing, Blue reward:0.85\n", - "step: 640, Red action: do-nothing, Blue reward:0.85\n", - "step: 641, Red action: do-nothing, Blue reward:0.85\n", - "step: 642, Red action: do-nothing, Blue reward:0.85\n", - "step: 643, Red action: do-nothing, Blue reward:0.85\n", - "step: 644, Red action: node-application-execute, Blue reward:0.85\n", - "step: 645, Red action: do-nothing, Blue reward:0.85\n", - "step: 646, Red action: do-nothing, Blue reward:0.85\n", - "step: 647, Red action: do-nothing, Blue reward:0.85\n", - "step: 648, Red action: do-nothing, Blue reward:0.85\n", - "step: 649, Red action: do-nothing, Blue reward:0.85\n", - "step: 650, Red action: do-nothing, Blue reward:0.85\n", - "step: 651, Red action: do-nothing, Blue reward:0.85\n", - "step: 652, Red action: do-nothing, Blue reward:0.85\n", - "step: 653, Red action: do-nothing, Blue reward:0.85\n", - "step: 654, Red action: do-nothing, Blue reward:0.85\n", - "step: 655, Red action: do-nothing, Blue reward:0.85\n", - "step: 656, Red action: do-nothing, Blue reward:0.85\n", - "step: 657, Red action: do-nothing, Blue reward:0.85\n", - "step: 658, Red action: do-nothing, Blue reward:0.85\n", - "step: 659, Red action: do-nothing, Blue reward:0.85\n", - "step: 660, Red action: do-nothing, Blue reward:0.85\n", - "step: 661, Red action: do-nothing, Blue reward:0.85\n", - "step: 662, Red action: do-nothing, Blue reward:0.85\n", - "step: 663, Red action: do-nothing, Blue reward:0.85\n", - "step: 664, Red action: do-nothing, Blue reward:0.85\n", - "step: 665, Red action: do-nothing, Blue reward:0.85\n", - "step: 666, Red action: do-nothing, Blue reward:0.85\n", - "step: 667, Red action: node-application-execute, Blue reward:0.85\n", - "step: 668, Red action: do-nothing, Blue reward:0.85\n", - "step: 669, Red action: do-nothing, Blue reward:0.85\n", - "step: 670, Red action: do-nothing, Blue reward:0.85\n", - "step: 671, Red action: do-nothing, Blue reward:0.85\n", - "step: 672, Red action: do-nothing, Blue reward:0.85\n", - "step: 673, Red action: do-nothing, Blue reward:0.85\n", - "step: 674, Red action: do-nothing, Blue reward:0.85\n", - "step: 675, Red action: do-nothing, Blue reward:0.85\n", - "step: 676, Red action: do-nothing, Blue reward:0.85\n", - "step: 677, Red action: do-nothing, Blue reward:0.85\n", - "step: 678, Red action: do-nothing, Blue reward:0.85\n", - "step: 679, Red action: do-nothing, Blue reward:0.85\n", - "step: 680, Red action: do-nothing, Blue reward:0.85\n", - "step: 681, Red action: do-nothing, Blue reward:0.85\n", - "step: 682, Red action: do-nothing, Blue reward:0.85\n", - "step: 683, Red action: do-nothing, Blue reward:0.85\n", - "step: 684, Red action: do-nothing, Blue reward:0.85\n", - "step: 685, Red action: node-application-execute, Blue reward:0.85\n", - "step: 686, Red action: do-nothing, Blue reward:0.85\n", - "step: 687, Red action: do-nothing, Blue reward:0.85\n", - "step: 688, Red action: do-nothing, Blue reward:0.85\n", - "step: 689, Red action: do-nothing, Blue reward:0.85\n", - "step: 690, Red action: do-nothing, Blue reward:0.85\n", - "step: 691, Red action: do-nothing, Blue reward:0.85\n", - "step: 692, Red action: do-nothing, Blue reward:0.85\n", - "step: 693, Red action: do-nothing, Blue reward:0.85\n", - "step: 694, Red action: do-nothing, Blue reward:0.85\n", - "step: 695, Red action: do-nothing, Blue reward:0.85\n", - "step: 696, Red action: do-nothing, Blue reward:0.85\n", - "step: 697, Red action: do-nothing, Blue reward:0.85\n", - "step: 698, Red action: do-nothing, Blue reward:0.85\n", - "step: 699, Red action: do-nothing, Blue reward:0.85\n", - "step: 700, Red action: do-nothing, Blue reward:0.85\n", - "step: 701, Red action: node-application-execute, Blue reward:0.85\n", - "step: 702, Red action: do-nothing, Blue reward:0.85\n", - "step: 703, Red action: do-nothing, Blue reward:0.85\n", - "step: 704, Red action: do-nothing, Blue reward:0.85\n", - "step: 705, Red action: do-nothing, Blue reward:0.85\n", - "step: 706, Red action: do-nothing, Blue reward:0.85\n", - "step: 707, Red action: do-nothing, Blue reward:0.85\n", - "step: 708, Red action: do-nothing, Blue reward:0.85\n", - "step: 709, Red action: do-nothing, Blue reward:0.85\n", - "step: 710, Red action: do-nothing, Blue reward:0.85\n", - "step: 711, Red action: do-nothing, Blue reward:0.85\n", - "step: 712, Red action: do-nothing, Blue reward:0.85\n", - "step: 713, Red action: do-nothing, Blue reward:0.85\n", - "step: 714, Red action: do-nothing, Blue reward:0.85\n", - "step: 715, Red action: do-nothing, Blue reward:0.85\n", - "step: 716, Red action: do-nothing, Blue reward:0.85\n", - "step: 717, Red action: do-nothing, Blue reward:0.85\n", - "step: 718, Red action: node-application-execute, Blue reward:0.85\n", - "step: 719, Red action: do-nothing, Blue reward:0.85\n", - "step: 720, Red action: do-nothing, Blue reward:0.85\n", - "step: 721, Red action: do-nothing, Blue reward:0.85\n", - "step: 722, Red action: do-nothing, Blue reward:0.85\n", - "step: 723, Red action: do-nothing, Blue reward:0.85\n", - "step: 724, Red action: do-nothing, Blue reward:0.85\n", - "step: 725, Red action: do-nothing, Blue reward:0.85\n", - "step: 726, Red action: do-nothing, Blue reward:0.85\n", - "step: 727, Red action: do-nothing, Blue reward:0.85\n", - "step: 728, Red action: do-nothing, Blue reward:0.85\n", - "step: 729, Red action: do-nothing, Blue reward:0.85\n", - "step: 730, Red action: do-nothing, Blue reward:0.85\n", - "step: 731, Red action: do-nothing, Blue reward:0.85\n", - "step: 732, Red action: do-nothing, Blue reward:0.85\n", - "step: 733, Red action: do-nothing, Blue reward:0.85\n", - "step: 734, Red action: do-nothing, Blue reward:0.85\n", - "step: 735, Red action: do-nothing, Blue reward:0.85\n", - "step: 736, Red action: do-nothing, Blue reward:0.85\n", - "step: 737, Red action: do-nothing, Blue reward:0.85\n", - "step: 738, Red action: do-nothing, Blue reward:0.85\n", - "step: 739, Red action: do-nothing, Blue reward:0.85\n", - "step: 740, Red action: do-nothing, Blue reward:0.85\n", - "step: 741, Red action: do-nothing, Blue reward:0.85\n", - "step: 742, Red action: do-nothing, Blue reward:0.85\n", - "step: 743, Red action: node-application-execute, Blue reward:0.85\n", - "step: 744, Red action: do-nothing, Blue reward:0.85\n", - "step: 745, Red action: do-nothing, Blue reward:0.85\n", - "step: 746, Red action: do-nothing, Blue reward:0.85\n", - "step: 747, Red action: do-nothing, Blue reward:0.85\n", - "step: 748, Red action: do-nothing, Blue reward:0.85\n", - "step: 749, Red action: do-nothing, Blue reward:0.85\n", - "step: 750, Red action: do-nothing, Blue reward:0.85\n", - "step: 751, Red action: do-nothing, Blue reward:0.85\n", - "step: 752, Red action: do-nothing, Blue reward:0.85\n", - "step: 753, Red action: do-nothing, Blue reward:0.85\n", - "step: 754, Red action: do-nothing, Blue reward:0.85\n", - "step: 755, Red action: do-nothing, Blue reward:0.85\n", - "step: 756, Red action: do-nothing, Blue reward:0.85\n", - "step: 757, Red action: do-nothing, Blue reward:0.85\n", - "step: 758, Red action: do-nothing, Blue reward:0.85\n", - "step: 759, Red action: do-nothing, Blue reward:0.85\n", - "step: 760, Red action: node-application-execute, Blue reward:0.85\n", - "step: 761, Red action: do-nothing, Blue reward:0.85\n", - "step: 762, Red action: do-nothing, Blue reward:0.85\n", - "step: 763, Red action: do-nothing, Blue reward:0.85\n", - "step: 764, Red action: do-nothing, Blue reward:0.85\n", - "step: 765, Red action: do-nothing, Blue reward:0.85\n", - "step: 766, Red action: do-nothing, Blue reward:0.85\n", - "step: 767, Red action: do-nothing, Blue reward:0.85\n", - "step: 768, Red action: do-nothing, Blue reward:0.85\n", - "step: 769, Red action: do-nothing, Blue reward:0.85\n", - "step: 770, Red action: do-nothing, Blue reward:0.85\n", - "step: 771, Red action: do-nothing, Blue reward:0.85\n", - "step: 772, Red action: do-nothing, Blue reward:0.85\n", - "step: 773, Red action: do-nothing, Blue reward:0.85\n", - "step: 774, Red action: do-nothing, Blue reward:0.85\n", - "step: 775, Red action: do-nothing, Blue reward:0.85\n", - "step: 776, Red action: do-nothing, Blue reward:0.85\n", - "step: 777, Red action: do-nothing, Blue reward:0.85\n", - "step: 778, Red action: do-nothing, Blue reward:0.85\n", - "step: 779, Red action: do-nothing, Blue reward:0.85\n", - "step: 780, Red action: do-nothing, Blue reward:0.85\n", - "step: 781, Red action: do-nothing, Blue reward:0.85\n", - "step: 782, Red action: do-nothing, Blue reward:0.85\n", - "step: 783, Red action: node-application-execute, Blue reward:0.85\n", - "step: 784, Red action: do-nothing, Blue reward:0.85\n", - "step: 785, Red action: do-nothing, Blue reward:0.85\n", - "step: 786, Red action: do-nothing, Blue reward:0.85\n", - "step: 787, Red action: do-nothing, Blue reward:0.85\n", - "step: 788, Red action: do-nothing, Blue reward:0.85\n", - "step: 789, Red action: do-nothing, Blue reward:0.85\n", - "step: 790, Red action: do-nothing, Blue reward:0.85\n", - "step: 791, Red action: do-nothing, Blue reward:0.85\n", - "step: 792, Red action: do-nothing, Blue reward:0.85\n", - "step: 793, Red action: do-nothing, Blue reward:0.85\n", - "step: 794, Red action: do-nothing, Blue reward:0.85\n", - "step: 795, Red action: do-nothing, Blue reward:0.85\n", - "step: 796, Red action: do-nothing, Blue reward:0.85\n", - "step: 797, Red action: do-nothing, Blue reward:0.85\n", - "step: 798, Red action: do-nothing, Blue reward:0.85\n", - "step: 799, Red action: do-nothing, Blue reward:0.85\n", - "step: 800, Red action: do-nothing, Blue reward:0.85\n", - "step: 801, Red action: do-nothing, Blue reward:0.85\n", - "step: 802, Red action: do-nothing, Blue reward:0.85\n", - "step: 803, Red action: node-application-execute, Blue reward:0.85\n", - "step: 804, Red action: do-nothing, Blue reward:0.85\n", - "step: 805, Red action: do-nothing, Blue reward:0.85\n", - "step: 806, Red action: do-nothing, Blue reward:0.85\n", - "step: 807, Red action: do-nothing, Blue reward:0.85\n", - "step: 808, Red action: do-nothing, Blue reward:0.85\n", - "step: 809, Red action: do-nothing, Blue reward:0.85\n", - "step: 810, Red action: do-nothing, Blue reward:0.85\n", - "step: 811, Red action: do-nothing, Blue reward:0.85\n", - "step: 812, Red action: do-nothing, Blue reward:0.85\n", - "step: 813, Red action: do-nothing, Blue reward:0.85\n", - "step: 814, Red action: do-nothing, Blue reward:0.85\n", - "step: 815, Red action: do-nothing, Blue reward:0.85\n", - "step: 816, Red action: do-nothing, Blue reward:0.85\n", - "step: 817, Red action: do-nothing, Blue reward:0.85\n", - "step: 818, Red action: do-nothing, Blue reward:0.85\n", - "step: 819, Red action: do-nothing, Blue reward:0.85\n", - "step: 820, Red action: do-nothing, Blue reward:0.85\n", - "step: 821, Red action: do-nothing, Blue reward:0.85\n", - "step: 822, Red action: do-nothing, Blue reward:0.85\n", - "step: 823, Red action: do-nothing, Blue reward:0.85\n", - "step: 824, Red action: do-nothing, Blue reward:0.85\n", - "step: 825, Red action: do-nothing, Blue reward:0.85\n", - "step: 826, Red action: do-nothing, Blue reward:0.85\n", - "step: 827, Red action: node-application-execute, Blue reward:0.85\n", - "step: 828, Red action: do-nothing, Blue reward:0.85\n", - "step: 829, Red action: do-nothing, Blue reward:0.85\n", - "step: 830, Red action: do-nothing, Blue reward:0.85\n", - "step: 831, Red action: do-nothing, Blue reward:0.85\n", - "step: 832, Red action: do-nothing, Blue reward:0.85\n", - "step: 833, Red action: do-nothing, Blue reward:0.85\n", - "step: 834, Red action: do-nothing, Blue reward:0.85\n", - "step: 835, Red action: do-nothing, Blue reward:0.85\n", - "step: 836, Red action: do-nothing, Blue reward:0.85\n", - "step: 837, Red action: do-nothing, Blue reward:0.85\n", - "step: 838, Red action: do-nothing, Blue reward:0.85\n", - "step: 839, Red action: do-nothing, Blue reward:0.85\n", - "step: 840, Red action: do-nothing, Blue reward:0.85\n", - "step: 841, Red action: do-nothing, Blue reward:0.85\n", - "step: 842, Red action: do-nothing, Blue reward:0.85\n", - "step: 843, Red action: do-nothing, Blue reward:0.85\n", - "step: 844, Red action: do-nothing, Blue reward:0.85\n", - "step: 845, Red action: do-nothing, Blue reward:0.85\n", - "step: 846, Red action: do-nothing, Blue reward:0.85\n", - "step: 847, Red action: node-application-execute, Blue reward:0.85\n", - "step: 848, Red action: do-nothing, Blue reward:0.85\n", - "step: 849, Red action: do-nothing, Blue reward:0.85\n", - "step: 850, Red action: do-nothing, Blue reward:0.85\n", - "step: 851, Red action: do-nothing, Blue reward:0.85\n", - "step: 852, Red action: do-nothing, Blue reward:0.85\n", - "step: 853, Red action: do-nothing, Blue reward:0.85\n", - "step: 854, Red action: do-nothing, Blue reward:0.85\n", - "step: 855, Red action: do-nothing, Blue reward:0.85\n", - "step: 856, Red action: do-nothing, Blue reward:0.85\n", - "step: 857, Red action: do-nothing, Blue reward:0.85\n", - "step: 858, Red action: do-nothing, Blue reward:0.85\n", - "step: 859, Red action: do-nothing, Blue reward:0.85\n", - "step: 860, Red action: do-nothing, Blue reward:0.85\n", - "step: 861, Red action: do-nothing, Blue reward:0.85\n", - "step: 862, Red action: do-nothing, Blue reward:0.85\n", - "step: 863, Red action: do-nothing, Blue reward:0.85\n", - "step: 864, Red action: do-nothing, Blue reward:0.85\n", - "step: 865, Red action: do-nothing, Blue reward:0.85\n", - "step: 866, Red action: do-nothing, Blue reward:0.85\n", - "step: 867, Red action: do-nothing, Blue reward:0.85\n", - "step: 868, Red action: do-nothing, Blue reward:0.85\n", - "step: 869, Red action: do-nothing, Blue reward:0.85\n", - "step: 870, Red action: do-nothing, Blue reward:0.85\n", - "step: 871, Red action: node-application-execute, Blue reward:0.85\n", - "step: 872, Red action: do-nothing, Blue reward:0.85\n", - "step: 873, Red action: do-nothing, Blue reward:0.85\n", - "step: 874, Red action: do-nothing, Blue reward:0.85\n", - "step: 875, Red action: do-nothing, Blue reward:0.85\n", - "step: 876, Red action: do-nothing, Blue reward:0.85\n", - "step: 877, Red action: do-nothing, Blue reward:0.85\n", - "step: 878, Red action: do-nothing, Blue reward:0.85\n", - "step: 879, Red action: do-nothing, Blue reward:0.85\n", - "step: 880, Red action: do-nothing, Blue reward:0.85\n", - "step: 881, Red action: do-nothing, Blue reward:0.85\n", - "step: 882, Red action: do-nothing, Blue reward:0.85\n", - "step: 883, Red action: do-nothing, Blue reward:0.85\n", - "step: 884, Red action: do-nothing, Blue reward:0.85\n", - "step: 885, Red action: do-nothing, Blue reward:0.85\n", - "step: 886, Red action: do-nothing, Blue reward:0.85\n", - "step: 887, Red action: do-nothing, Blue reward:0.85\n", - "step: 888, Red action: do-nothing, Blue reward:0.85\n", - "step: 889, Red action: do-nothing, Blue reward:0.85\n", - "step: 890, Red action: do-nothing, Blue reward:0.85\n", - "step: 891, Red action: node-application-execute, Blue reward:0.85\n", - "step: 892, Red action: do-nothing, Blue reward:0.85\n", - "step: 893, Red action: do-nothing, Blue reward:0.85\n", - "step: 894, Red action: do-nothing, Blue reward:0.85\n", - "step: 895, Red action: do-nothing, Blue reward:0.85\n", - "step: 896, Red action: do-nothing, Blue reward:0.85\n", - "step: 897, Red action: do-nothing, Blue reward:0.85\n", - "step: 898, Red action: do-nothing, Blue reward:0.85\n", - "step: 899, Red action: do-nothing, Blue reward:0.85\n", - "step: 900, Red action: do-nothing, Blue reward:0.85\n", - "step: 901, Red action: do-nothing, Blue reward:0.85\n", - "step: 902, Red action: do-nothing, Blue reward:0.85\n", - "step: 903, Red action: do-nothing, Blue reward:0.85\n", - "step: 904, Red action: do-nothing, Blue reward:0.85\n", - "step: 905, Red action: do-nothing, Blue reward:0.85\n", - "step: 906, Red action: node-application-execute, Blue reward:0.85\n", - "step: 907, Red action: do-nothing, Blue reward:0.85\n", - "step: 908, Red action: do-nothing, Blue reward:0.85\n", - "step: 909, Red action: do-nothing, Blue reward:0.85\n", - "step: 910, Red action: do-nothing, Blue reward:0.85\n", - "step: 911, Red action: do-nothing, Blue reward:0.85\n", - "step: 912, Red action: do-nothing, Blue reward:0.85\n", - "step: 913, Red action: do-nothing, Blue reward:0.85\n", - "step: 914, Red action: do-nothing, Blue reward:0.85\n", - "step: 915, Red action: do-nothing, Blue reward:0.85\n", - "step: 916, Red action: do-nothing, Blue reward:0.85\n", - "step: 917, Red action: do-nothing, Blue reward:0.85\n", - "step: 918, Red action: do-nothing, Blue reward:0.85\n", - "step: 919, Red action: do-nothing, Blue reward:0.85\n", - "step: 920, Red action: do-nothing, Blue reward:0.85\n", - "step: 921, Red action: do-nothing, Blue reward:0.85\n", - "step: 922, Red action: do-nothing, Blue reward:0.85\n", - "step: 923, Red action: do-nothing, Blue reward:0.85\n", - "step: 924, Red action: do-nothing, Blue reward:0.85\n", - "step: 925, Red action: do-nothing, Blue reward:0.85\n", - "step: 926, Red action: do-nothing, Blue reward:0.85\n", - "step: 927, Red action: do-nothing, Blue reward:0.85\n", - "step: 928, Red action: do-nothing, Blue reward:0.85\n", - "step: 929, Red action: do-nothing, Blue reward:0.85\n", - "step: 930, Red action: node-application-execute, Blue reward:0.85\n", - "step: 931, Red action: do-nothing, Blue reward:0.85\n", - "step: 932, Red action: do-nothing, Blue reward:0.85\n", - "step: 933, Red action: do-nothing, Blue reward:0.85\n", - "step: 934, Red action: do-nothing, Blue reward:0.85\n", - "step: 935, Red action: do-nothing, Blue reward:0.85\n", - "step: 936, Red action: do-nothing, Blue reward:0.85\n", - "step: 937, Red action: do-nothing, Blue reward:0.85\n", - "step: 938, Red action: do-nothing, Blue reward:0.85\n", - "step: 939, Red action: do-nothing, Blue reward:0.85\n", - "step: 940, Red action: do-nothing, Blue reward:0.85\n", - "step: 941, Red action: do-nothing, Blue reward:0.85\n", - "step: 942, Red action: do-nothing, Blue reward:0.85\n", - "step: 943, Red action: do-nothing, Blue reward:0.85\n", - "step: 944, Red action: do-nothing, Blue reward:0.85\n", - "step: 945, Red action: do-nothing, Blue reward:0.85\n", - "step: 946, Red action: do-nothing, Blue reward:0.85\n", - "step: 947, Red action: node-application-execute, Blue reward:0.85\n", - "step: 948, Red action: do-nothing, Blue reward:0.85\n", - "step: 949, Red action: do-nothing, Blue reward:0.85\n", - "step: 950, Red action: do-nothing, Blue reward:0.85\n", - "step: 951, Red action: do-nothing, Blue reward:0.85\n", - "step: 952, Red action: do-nothing, Blue reward:0.85\n", - "step: 953, Red action: do-nothing, Blue reward:0.85\n", - "step: 954, Red action: do-nothing, Blue reward:0.85\n", - "step: 955, Red action: do-nothing, Blue reward:0.85\n", - "step: 956, Red action: do-nothing, Blue reward:0.85\n", - "step: 957, Red action: do-nothing, Blue reward:0.85\n", - "step: 958, Red action: do-nothing, Blue reward:0.85\n", - "step: 959, Red action: do-nothing, Blue reward:0.85\n", - "step: 960, Red action: do-nothing, Blue reward:0.85\n", - "step: 961, Red action: do-nothing, Blue reward:0.85\n", - "step: 962, Red action: do-nothing, Blue reward:0.85\n", - "step: 963, Red action: node-application-execute, Blue reward:0.85\n", - "step: 964, Red action: do-nothing, Blue reward:0.85\n", - "step: 965, Red action: do-nothing, Blue reward:0.85\n", - "step: 966, Red action: do-nothing, Blue reward:0.85\n", - "step: 967, Red action: do-nothing, Blue reward:0.85\n", - "step: 968, Red action: do-nothing, Blue reward:0.85\n", - "step: 969, Red action: do-nothing, Blue reward:0.85\n", - "step: 970, Red action: do-nothing, Blue reward:0.85\n", - "step: 971, Red action: do-nothing, Blue reward:0.85\n", - "step: 972, Red action: do-nothing, Blue reward:0.85\n", - "step: 973, Red action: do-nothing, Blue reward:0.85\n", - "step: 974, Red action: do-nothing, Blue reward:0.85\n", - "step: 975, Red action: do-nothing, Blue reward:0.85\n", - "step: 976, Red action: do-nothing, Blue reward:0.85\n", - "step: 977, Red action: do-nothing, Blue reward:0.85\n", - "step: 978, Red action: do-nothing, Blue reward:0.85\n", - "step: 979, Red action: do-nothing, Blue reward:0.85\n", - "step: 980, Red action: do-nothing, Blue reward:0.85\n", - "step: 981, Red action: do-nothing, Blue reward:0.85\n", - "step: 982, Red action: do-nothing, Blue reward:0.85\n", - "step: 983, Red action: do-nothing, Blue reward:0.85\n", - "step: 984, Red action: do-nothing, Blue reward:0.85\n", - "step: 985, Red action: do-nothing, Blue reward:0.85\n", - "step: 986, Red action: do-nothing, Blue reward:0.85\n", - "step: 987, Red action: do-nothing, Blue reward:0.85\n", - "step: 988, Red action: node-application-execute, Blue reward:0.85\n", - "step: 989, Red action: do-nothing, Blue reward:0.85\n", - "step: 990, Red action: do-nothing, Blue reward:0.85\n", - "step: 991, Red action: do-nothing, Blue reward:0.85\n", - "step: 992, Red action: do-nothing, Blue reward:0.85\n", - "step: 993, Red action: do-nothing, Blue reward:0.85\n", - "step: 994, Red action: do-nothing, Blue reward:0.85\n", - "step: 995, Red action: do-nothing, Blue reward:0.85\n", - "step: 996, Red action: do-nothing, Blue reward:0.85\n", - "step: 997, Red action: do-nothing, Blue reward:0.85\n", - "step: 998, Red action: do-nothing, Blue reward:0.85\n", - "step: 999, Red action: do-nothing, Blue reward:0.85\n", - "step: 1000, Red action: do-nothing, Blue reward:0.85\n", - "step: 1001, Red action: do-nothing, Blue reward:0.85\n", - "step: 1002, Red action: do-nothing, Blue reward:0.85\n", - "step: 1003, Red action: do-nothing, Blue reward:0.85\n", - "step: 1004, Red action: do-nothing, Blue reward:0.85\n", - "step: 1005, Red action: do-nothing, Blue reward:0.85\n", - "step: 1006, Red action: do-nothing, Blue reward:0.85\n", - "step: 1007, Red action: do-nothing, Blue reward:0.85\n", - "step: 1008, Red action: do-nothing, Blue reward:0.85\n", - "step: 1009, Red action: do-nothing, Blue reward:0.85\n", - "step: 1010, Red action: do-nothing, Blue reward:0.85\n", - "step: 1011, Red action: do-nothing, Blue reward:0.85\n", - "step: 1012, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1013, Red action: do-nothing, Blue reward:0.85\n", - "step: 1014, Red action: do-nothing, Blue reward:0.85\n", - "step: 1015, Red action: do-nothing, Blue reward:0.85\n", - "step: 1016, Red action: do-nothing, Blue reward:0.85\n", - "step: 1017, Red action: do-nothing, Blue reward:0.85\n", - "step: 1018, Red action: do-nothing, Blue reward:0.85\n", - "step: 1019, Red action: do-nothing, Blue reward:0.85\n", - "step: 1020, Red action: do-nothing, Blue reward:0.85\n", - "step: 1021, Red action: do-nothing, Blue reward:0.85\n", - "step: 1022, Red action: do-nothing, Blue reward:0.85\n", - "step: 1023, Red action: do-nothing, Blue reward:0.85\n", - "step: 1024, Red action: do-nothing, Blue reward:0.85\n", - "step: 1025, Red action: do-nothing, Blue reward:0.85\n", - "step: 1026, Red action: do-nothing, Blue reward:0.85\n", - "step: 1027, Red action: do-nothing, Blue reward:0.85\n", - "step: 1028, Red action: do-nothing, Blue reward:0.85\n", - "step: 1029, Red action: do-nothing, Blue reward:0.85\n", - "step: 1030, Red action: do-nothing, Blue reward:0.85\n", - "step: 1031, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1032, Red action: do-nothing, Blue reward:0.85\n", - "step: 1033, Red action: do-nothing, Blue reward:0.85\n", - "step: 1034, Red action: do-nothing, Blue reward:0.85\n", - "step: 1035, Red action: do-nothing, Blue reward:0.85\n", - "step: 1036, Red action: do-nothing, Blue reward:0.85\n", - "step: 1037, Red action: do-nothing, Blue reward:0.85\n", - "step: 1038, Red action: do-nothing, Blue reward:0.85\n", - "step: 1039, Red action: do-nothing, Blue reward:0.85\n", - "step: 1040, Red action: do-nothing, Blue reward:0.85\n", - "step: 1041, Red action: do-nothing, Blue reward:0.85\n", - "step: 1042, Red action: do-nothing, Blue reward:0.85\n", - "step: 1043, Red action: do-nothing, Blue reward:0.85\n", - "step: 1044, Red action: do-nothing, Blue reward:0.85\n", - "step: 1045, Red action: do-nothing, Blue reward:0.85\n", - "step: 1046, Red action: do-nothing, Blue reward:0.85\n", - "step: 1047, Red action: do-nothing, Blue reward:0.85\n", - "step: 1048, Red action: do-nothing, Blue reward:0.85\n", - "step: 1049, Red action: do-nothing, Blue reward:0.85\n", - "step: 1050, Red action: do-nothing, Blue reward:0.85\n", - "step: 1051, Red action: do-nothing, Blue reward:0.85\n", - "step: 1052, Red action: do-nothing, Blue reward:0.85\n", - "step: 1053, Red action: do-nothing, Blue reward:0.85\n", - "step: 1054, Red action: do-nothing, Blue reward:0.85\n", - "step: 1055, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1056, Red action: do-nothing, Blue reward:0.85\n", - "step: 1057, Red action: do-nothing, Blue reward:0.85\n", - "step: 1058, Red action: do-nothing, Blue reward:0.85\n", - "step: 1059, Red action: do-nothing, Blue reward:0.85\n", - "step: 1060, Red action: do-nothing, Blue reward:0.85\n", - "step: 1061, Red action: do-nothing, Blue reward:0.85\n", - "step: 1062, Red action: do-nothing, Blue reward:0.85\n", - "step: 1063, Red action: do-nothing, Blue reward:0.85\n", - "step: 1064, Red action: do-nothing, Blue reward:0.85\n", - "step: 1065, Red action: do-nothing, Blue reward:0.85\n", - "step: 1066, Red action: do-nothing, Blue reward:0.85\n", - "step: 1067, Red action: do-nothing, Blue reward:0.85\n", - "step: 1068, Red action: do-nothing, Blue reward:0.85\n", - "step: 1069, Red action: do-nothing, Blue reward:0.85\n", - "step: 1070, Red action: do-nothing, Blue reward:0.85\n", - "step: 1071, Red action: do-nothing, Blue reward:0.85\n", - "step: 1072, Red action: do-nothing, Blue reward:0.85\n", - "step: 1073, Red action: do-nothing, Blue reward:0.85\n", - "step: 1074, Red action: do-nothing, Blue reward:0.85\n", - "step: 1075, Red action: do-nothing, Blue reward:0.85\n", - "step: 1076, Red action: do-nothing, Blue reward:0.85\n", - "step: 1077, Red action: do-nothing, Blue reward:0.85\n", - "step: 1078, Red action: do-nothing, Blue reward:0.85\n", - "step: 1079, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1080, Red action: do-nothing, Blue reward:0.85\n", - "step: 1081, Red action: do-nothing, Blue reward:0.85\n", - "step: 1082, Red action: do-nothing, Blue reward:0.85\n", - "step: 1083, Red action: do-nothing, Blue reward:0.85\n", - "step: 1084, Red action: do-nothing, Blue reward:0.85\n", - "step: 1085, Red action: do-nothing, Blue reward:0.85\n", - "step: 1086, Red action: do-nothing, Blue reward:0.85\n", - "step: 1087, Red action: do-nothing, Blue reward:0.85\n", - "step: 1088, Red action: do-nothing, Blue reward:0.85\n", - "step: 1089, Red action: do-nothing, Blue reward:0.85\n", - "step: 1090, Red action: do-nothing, Blue reward:0.85\n", - "step: 1091, Red action: do-nothing, Blue reward:0.85\n", - "step: 1092, Red action: do-nothing, Blue reward:0.85\n", - "step: 1093, Red action: do-nothing, Blue reward:0.85\n", - "step: 1094, Red action: do-nothing, Blue reward:0.85\n", - "step: 1095, Red action: do-nothing, Blue reward:0.85\n", - "step: 1096, Red action: do-nothing, Blue reward:0.85\n", - "step: 1097, Red action: do-nothing, Blue reward:0.85\n", - "step: 1098, Red action: do-nothing, Blue reward:0.85\n", - "step: 1099, Red action: do-nothing, Blue reward:0.85\n", - "step: 1100, Red action: do-nothing, Blue reward:0.85\n", - "step: 1101, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1102, Red action: do-nothing, Blue reward:0.85\n", - "step: 1103, Red action: do-nothing, Blue reward:0.85\n", - "step: 1104, Red action: do-nothing, Blue reward:0.85\n", - "step: 1105, Red action: do-nothing, Blue reward:0.85\n", - "step: 1106, Red action: do-nothing, Blue reward:0.85\n", - "step: 1107, Red action: do-nothing, Blue reward:0.85\n", - "step: 1108, Red action: do-nothing, Blue reward:0.85\n", - "step: 1109, Red action: do-nothing, Blue reward:0.85\n", - "step: 1110, Red action: do-nothing, Blue reward:0.85\n", - "step: 1111, Red action: do-nothing, Blue reward:0.85\n", - "step: 1112, Red action: do-nothing, Blue reward:0.85\n", - "step: 1113, Red action: do-nothing, Blue reward:0.85\n", - "step: 1114, Red action: do-nothing, Blue reward:0.85\n", - "step: 1115, Red action: do-nothing, Blue reward:0.85\n", - "step: 1116, Red action: do-nothing, Blue reward:0.85\n", - "step: 1117, Red action: do-nothing, Blue reward:0.85\n", - "step: 1118, Red action: do-nothing, Blue reward:0.85\n", - "step: 1119, Red action: do-nothing, Blue reward:0.85\n", - "step: 1120, Red action: do-nothing, Blue reward:0.85\n", - "step: 1121, Red action: do-nothing, Blue reward:0.85\n", - "step: 1122, Red action: do-nothing, Blue reward:0.85\n", - "step: 1123, Red action: do-nothing, Blue reward:0.85\n", - "step: 1124, Red action: do-nothing, Blue reward:0.85\n", - "step: 1125, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1126, Red action: do-nothing, Blue reward:0.85\n", - "step: 1127, Red action: do-nothing, Blue reward:0.85\n", - "step: 1128, Red action: do-nothing, Blue reward:0.85\n", - "step: 1129, Red action: do-nothing, Blue reward:0.85\n", - "step: 1130, Red action: do-nothing, Blue reward:0.85\n", - "step: 1131, Red action: do-nothing, Blue reward:0.85\n", - "step: 1132, Red action: do-nothing, Blue reward:0.85\n", - "step: 1133, Red action: do-nothing, Blue reward:0.85\n", - "step: 1134, Red action: do-nothing, Blue reward:0.85\n", - "step: 1135, Red action: do-nothing, Blue reward:0.85\n", - "step: 1136, Red action: do-nothing, Blue reward:0.85\n", - "step: 1137, Red action: do-nothing, Blue reward:0.85\n", - "step: 1138, Red action: do-nothing, Blue reward:0.85\n", - "step: 1139, Red action: do-nothing, Blue reward:0.85\n", - "step: 1140, Red action: do-nothing, Blue reward:0.85\n", - "step: 1141, Red action: do-nothing, Blue reward:0.85\n", - "step: 1142, Red action: do-nothing, Blue reward:0.85\n", - "step: 1143, Red action: do-nothing, Blue reward:0.85\n", - "step: 1144, Red action: do-nothing, Blue reward:0.85\n", - "step: 1145, Red action: do-nothing, Blue reward:0.85\n", - "step: 1146, Red action: do-nothing, Blue reward:0.85\n", - "step: 1147, Red action: do-nothing, Blue reward:0.85\n", - "step: 1148, Red action: do-nothing, Blue reward:0.85\n", - "step: 1149, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1150, Red action: do-nothing, Blue reward:0.85\n", - "step: 1151, Red action: do-nothing, Blue reward:0.85\n", - "step: 1152, Red action: do-nothing, Blue reward:0.85\n", - "step: 1153, Red action: do-nothing, Blue reward:0.85\n", - "step: 1154, Red action: do-nothing, Blue reward:0.85\n", - "step: 1155, Red action: do-nothing, Blue reward:0.85\n", - "step: 1156, Red action: do-nothing, Blue reward:0.85\n", - "step: 1157, Red action: do-nothing, Blue reward:0.85\n", - "step: 1158, Red action: do-nothing, Blue reward:0.85\n", - "step: 1159, Red action: do-nothing, Blue reward:0.85\n", - "step: 1160, Red action: do-nothing, Blue reward:0.85\n", - "step: 1161, Red action: do-nothing, Blue reward:0.85\n", - "step: 1162, Red action: do-nothing, Blue reward:0.85\n", - "step: 1163, Red action: do-nothing, Blue reward:0.85\n", - "step: 1164, Red action: do-nothing, Blue reward:0.85\n", - "step: 1165, Red action: do-nothing, Blue reward:0.85\n", - "step: 1166, Red action: do-nothing, Blue reward:0.85\n", - "step: 1167, Red action: do-nothing, Blue reward:0.85\n", - "step: 1168, Red action: do-nothing, Blue reward:0.85\n", - "step: 1169, Red action: do-nothing, Blue reward:0.85\n", - "step: 1170, Red action: do-nothing, Blue reward:0.85\n", - "step: 1171, Red action: do-nothing, Blue reward:0.85\n", - "step: 1172, Red action: do-nothing, Blue reward:0.85\n", - "step: 1173, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1174, Red action: do-nothing, Blue reward:0.85\n", - "step: 1175, Red action: do-nothing, Blue reward:0.85\n", - "step: 1176, Red action: do-nothing, Blue reward:0.85\n", - "step: 1177, Red action: do-nothing, Blue reward:0.85\n", - "step: 1178, Red action: do-nothing, Blue reward:0.85\n", - "step: 1179, Red action: do-nothing, Blue reward:0.85\n", - "step: 1180, Red action: do-nothing, Blue reward:0.85\n", - "step: 1181, Red action: do-nothing, Blue reward:0.85\n", - "step: 1182, Red action: do-nothing, Blue reward:0.85\n", - "step: 1183, Red action: do-nothing, Blue reward:0.85\n", - "step: 1184, Red action: do-nothing, Blue reward:0.85\n", - "step: 1185, Red action: do-nothing, Blue reward:0.85\n", - "step: 1186, Red action: do-nothing, Blue reward:0.85\n", - "step: 1187, Red action: do-nothing, Blue reward:0.85\n", - "step: 1188, Red action: do-nothing, Blue reward:0.85\n", - "step: 1189, Red action: do-nothing, Blue reward:0.85\n", - "step: 1190, Red action: do-nothing, Blue reward:0.85\n", - "step: 1191, Red action: do-nothing, Blue reward:0.85\n", - "step: 1192, Red action: do-nothing, Blue reward:0.85\n", - "step: 1193, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1194, Red action: do-nothing, Blue reward:0.85\n", - "step: 1195, Red action: do-nothing, Blue reward:0.85\n", - "step: 1196, Red action: do-nothing, Blue reward:0.85\n", - "step: 1197, Red action: do-nothing, Blue reward:0.85\n", - "step: 1198, Red action: do-nothing, Blue reward:0.85\n", - "step: 1199, Red action: do-nothing, Blue reward:0.85\n", - "step: 1200, Red action: do-nothing, Blue reward:0.85\n", - "step: 1201, Red action: do-nothing, Blue reward:0.85\n", - "step: 1202, Red action: do-nothing, Blue reward:0.85\n", - "step: 1203, Red action: do-nothing, Blue reward:0.85\n", - "step: 1204, Red action: do-nothing, Blue reward:0.85\n", - "step: 1205, Red action: do-nothing, Blue reward:0.85\n", - "step: 1206, Red action: do-nothing, Blue reward:0.85\n", - "step: 1207, Red action: do-nothing, Blue reward:0.85\n", - "step: 1208, Red action: do-nothing, Blue reward:0.85\n", - "step: 1209, Red action: do-nothing, Blue reward:0.85\n", - "step: 1210, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1211, Red action: do-nothing, Blue reward:0.85\n", - "step: 1212, Red action: do-nothing, Blue reward:0.85\n", - "step: 1213, Red action: do-nothing, Blue reward:0.85\n", - "step: 1214, Red action: do-nothing, Blue reward:0.85\n", - "step: 1215, Red action: do-nothing, Blue reward:0.85\n", - "step: 1216, Red action: do-nothing, Blue reward:0.85\n", - "step: 1217, Red action: do-nothing, Blue reward:0.85\n", - "step: 1218, Red action: do-nothing, Blue reward:0.85\n", - "step: 1219, Red action: do-nothing, Blue reward:0.85\n", - "step: 1220, Red action: do-nothing, Blue reward:0.85\n", - "step: 1221, Red action: do-nothing, Blue reward:0.85\n", - "step: 1222, Red action: do-nothing, Blue reward:0.85\n", - "step: 1223, Red action: do-nothing, Blue reward:0.85\n", - "step: 1224, Red action: do-nothing, Blue reward:0.85\n", - "step: 1225, Red action: do-nothing, Blue reward:0.85\n", - "step: 1226, Red action: do-nothing, Blue reward:0.85\n", - "step: 1227, Red action: do-nothing, Blue reward:0.85\n", - "step: 1228, Red action: do-nothing, Blue reward:0.85\n", - "step: 1229, Red action: do-nothing, Blue reward:0.85\n", - "step: 1230, Red action: do-nothing, Blue reward:0.85\n", - "step: 1231, Red action: do-nothing, Blue reward:0.85\n", - "step: 1232, Red action: do-nothing, Blue reward:0.85\n", - "step: 1233, Red action: do-nothing, Blue reward:0.85\n", - "step: 1234, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1235, Red action: do-nothing, Blue reward:0.85\n", - "step: 1236, Red action: do-nothing, Blue reward:0.85\n", - "step: 1237, Red action: do-nothing, Blue reward:0.85\n", - "step: 1238, Red action: do-nothing, Blue reward:0.85\n", - "step: 1239, Red action: do-nothing, Blue reward:0.85\n", - "step: 1240, Red action: do-nothing, Blue reward:0.85\n", - "step: 1241, Red action: do-nothing, Blue reward:0.85\n", - "step: 1242, Red action: do-nothing, Blue reward:0.85\n", - "step: 1243, Red action: do-nothing, Blue reward:0.85\n", - "step: 1244, Red action: do-nothing, Blue reward:0.85\n", - "step: 1245, Red action: do-nothing, Blue reward:0.85\n", - "step: 1246, Red action: do-nothing, Blue reward:0.85\n", - "step: 1247, Red action: do-nothing, Blue reward:0.85\n", - "step: 1248, Red action: do-nothing, Blue reward:0.85\n", - "step: 1249, Red action: do-nothing, Blue reward:0.85\n", - "step: 1250, Red action: do-nothing, Blue reward:0.85\n", - "step: 1251, Red action: do-nothing, Blue reward:0.85\n", - "step: 1252, Red action: do-nothing, Blue reward:0.85\n", - "step: 1253, Red action: do-nothing, Blue reward:0.85\n", - "step: 1254, Red action: do-nothing, Blue reward:0.85\n", - "step: 1255, Red action: do-nothing, Blue reward:0.85\n", - "step: 1256, Red action: do-nothing, Blue reward:0.85\n", - "step: 1257, Red action: do-nothing, Blue reward:0.85\n", - "step: 1258, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1259, Red action: do-nothing, Blue reward:0.85\n", - "step: 1260, Red action: do-nothing, Blue reward:0.85\n", - "step: 1261, Red action: do-nothing, Blue reward:0.85\n", - "step: 1262, Red action: do-nothing, Blue reward:0.85\n", - "step: 1263, Red action: do-nothing, Blue reward:0.85\n", - "step: 1264, Red action: do-nothing, Blue reward:0.85\n", - "step: 1265, Red action: do-nothing, Blue reward:0.85\n", - "step: 1266, Red action: do-nothing, Blue reward:0.85\n", - "step: 1267, Red action: do-nothing, Blue reward:0.85\n", - "step: 1268, Red action: do-nothing, Blue reward:0.85\n", - "step: 1269, Red action: do-nothing, Blue reward:0.85\n", - "step: 1270, Red action: do-nothing, Blue reward:0.85\n", - "step: 1271, Red action: do-nothing, Blue reward:0.85\n", - "step: 1272, Red action: do-nothing, Blue reward:0.85\n", - "step: 1273, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1274, Red action: do-nothing, Blue reward:0.85\n", - "step: 1275, Red action: do-nothing, Blue reward:0.85\n", - "step: 1276, Red action: do-nothing, Blue reward:0.85\n", - "step: 1277, Red action: do-nothing, Blue reward:0.85\n", - "step: 1278, Red action: do-nothing, Blue reward:0.85\n", - "step: 1279, Red action: do-nothing, Blue reward:0.85\n", - "step: 1280, Red action: do-nothing, Blue reward:0.85\n", - "step: 1281, Red action: do-nothing, Blue reward:0.85\n", - "step: 1282, Red action: do-nothing, Blue reward:0.85\n", - "step: 1283, Red action: do-nothing, Blue reward:0.85\n", - "step: 1284, Red action: do-nothing, Blue reward:0.85\n", - "step: 1285, Red action: do-nothing, Blue reward:0.85\n", - "step: 1286, Red action: do-nothing, Blue reward:0.85\n", - "step: 1287, Red action: do-nothing, Blue reward:0.85\n", - "step: 1288, Red action: do-nothing, Blue reward:0.85\n", - "step: 1289, Red action: do-nothing, Blue reward:0.85\n", - "step: 1290, Red action: do-nothing, Blue reward:0.85\n", - "step: 1291, Red action: do-nothing, Blue reward:0.85\n", - "step: 1292, Red action: do-nothing, Blue reward:0.85\n", - "step: 1293, Red action: do-nothing, Blue reward:0.85\n", - "step: 1294, Red action: do-nothing, Blue reward:0.85\n", - "step: 1295, Red action: do-nothing, Blue reward:0.85\n", - "step: 1296, Red action: do-nothing, Blue reward:0.85\n", - "step: 1297, Red action: do-nothing, Blue reward:0.85\n", - "step: 1298, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1299, Red action: do-nothing, Blue reward:0.85\n", - "step: 1300, Red action: do-nothing, Blue reward:0.85\n", - "step: 1301, Red action: do-nothing, Blue reward:0.85\n", - "step: 1302, Red action: do-nothing, Blue reward:0.85\n", - "step: 1303, Red action: do-nothing, Blue reward:0.85\n", - "step: 1304, Red action: do-nothing, Blue reward:0.85\n", - "step: 1305, Red action: do-nothing, Blue reward:0.85\n", - "step: 1306, Red action: do-nothing, Blue reward:0.85\n", - "step: 1307, Red action: do-nothing, Blue reward:0.85\n", - "step: 1308, Red action: do-nothing, Blue reward:0.85\n", - "step: 1309, Red action: do-nothing, Blue reward:0.85\n", - "step: 1310, Red action: do-nothing, Blue reward:0.85\n", - "step: 1311, Red action: do-nothing, Blue reward:0.85\n", - "step: 1312, Red action: do-nothing, Blue reward:0.85\n", - "step: 1313, Red action: do-nothing, Blue reward:0.85\n", - "step: 1314, Red action: do-nothing, Blue reward:0.85\n", - "step: 1315, Red action: do-nothing, Blue reward:0.85\n", - "step: 1316, Red action: do-nothing, Blue reward:0.85\n", - "step: 1317, Red action: do-nothing, Blue reward:0.85\n", - "step: 1318, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1319, Red action: do-nothing, Blue reward:0.85\n", - "step: 1320, Red action: do-nothing, Blue reward:0.85\n", - "step: 1321, Red action: do-nothing, Blue reward:0.85\n", - "step: 1322, Red action: do-nothing, Blue reward:0.85\n", - "step: 1323, Red action: do-nothing, Blue reward:0.85\n", - "step: 1324, Red action: do-nothing, Blue reward:0.85\n", - "step: 1325, Red action: do-nothing, Blue reward:0.85\n", - "step: 1326, Red action: do-nothing, Blue reward:0.85\n", - "step: 1327, Red action: do-nothing, Blue reward:0.85\n", - "step: 1328, Red action: do-nothing, Blue reward:0.85\n", - "step: 1329, Red action: do-nothing, Blue reward:0.85\n", - "step: 1330, Red action: do-nothing, Blue reward:0.85\n", - "step: 1331, Red action: do-nothing, Blue reward:0.85\n", - "step: 1332, Red action: do-nothing, Blue reward:0.85\n", - "step: 1333, Red action: do-nothing, Blue reward:0.85\n", - "step: 1334, Red action: do-nothing, Blue reward:0.85\n", - "step: 1335, Red action: do-nothing, Blue reward:0.85\n", - "step: 1336, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1337, Red action: do-nothing, Blue reward:0.85\n", - "step: 1338, Red action: do-nothing, Blue reward:0.85\n", - "step: 1339, Red action: do-nothing, Blue reward:0.85\n", - "step: 1340, Red action: do-nothing, Blue reward:0.85\n", - "step: 1341, Red action: do-nothing, Blue reward:0.85\n", - "step: 1342, Red action: do-nothing, Blue reward:0.85\n", - "step: 1343, Red action: do-nothing, Blue reward:0.85\n", - "step: 1344, Red action: do-nothing, Blue reward:0.85\n", - "step: 1345, Red action: do-nothing, Blue reward:0.85\n", - "step: 1346, Red action: do-nothing, Blue reward:0.85\n", - "step: 1347, Red action: do-nothing, Blue reward:0.85\n", - "step: 1348, Red action: do-nothing, Blue reward:0.85\n", - "step: 1349, Red action: do-nothing, Blue reward:0.85\n", - "step: 1350, Red action: do-nothing, Blue reward:0.85\n", - "step: 1351, Red action: do-nothing, Blue reward:0.85\n", - "step: 1352, Red action: do-nothing, Blue reward:0.85\n", - "step: 1353, Red action: do-nothing, Blue reward:0.85\n", - "step: 1354, Red action: do-nothing, Blue reward:0.85\n", - "step: 1355, Red action: do-nothing, Blue reward:0.85\n", - "step: 1356, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1357, Red action: do-nothing, Blue reward:0.85\n", - "step: 1358, Red action: do-nothing, Blue reward:0.85\n", - "step: 1359, Red action: do-nothing, Blue reward:0.85\n", - "step: 1360, Red action: do-nothing, Blue reward:0.85\n", - "step: 1361, Red action: do-nothing, Blue reward:0.85\n", - "step: 1362, Red action: do-nothing, Blue reward:0.85\n", - "step: 1363, Red action: do-nothing, Blue reward:0.85\n", - "step: 1364, Red action: do-nothing, Blue reward:0.85\n", - "step: 1365, Red action: do-nothing, Blue reward:0.85\n", - "step: 1366, Red action: do-nothing, Blue reward:0.85\n", - "step: 1367, Red action: do-nothing, Blue reward:0.85\n", - "step: 1368, Red action: do-nothing, Blue reward:0.85\n", - "step: 1369, Red action: do-nothing, Blue reward:0.85\n", - "step: 1370, Red action: do-nothing, Blue reward:0.85\n", - "step: 1371, Red action: do-nothing, Blue reward:0.85\n", - "step: 1372, Red action: do-nothing, Blue reward:0.85\n", - "step: 1373, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1374, Red action: do-nothing, Blue reward:0.85\n", - "step: 1375, Red action: do-nothing, Blue reward:0.85\n", - "step: 1376, Red action: do-nothing, Blue reward:0.85\n", - "step: 1377, Red action: do-nothing, Blue reward:0.85\n", - "step: 1378, Red action: do-nothing, Blue reward:0.85\n", - "step: 1379, Red action: do-nothing, Blue reward:0.85\n", - "step: 1380, Red action: do-nothing, Blue reward:0.85\n", - "step: 1381, Red action: do-nothing, Blue reward:0.85\n", - "step: 1382, Red action: do-nothing, Blue reward:0.85\n", - "step: 1383, Red action: do-nothing, Blue reward:0.85\n", - "step: 1384, Red action: do-nothing, Blue reward:0.85\n", - "step: 1385, Red action: do-nothing, Blue reward:0.85\n", - "step: 1386, Red action: do-nothing, Blue reward:0.85\n", - "step: 1387, Red action: do-nothing, Blue reward:0.85\n", - "step: 1388, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1389, Red action: do-nothing, Blue reward:0.85\n", - "step: 1390, Red action: do-nothing, Blue reward:0.85\n", - "step: 1391, Red action: do-nothing, Blue reward:0.85\n", - "step: 1392, Red action: do-nothing, Blue reward:0.85\n", - "step: 1393, Red action: do-nothing, Blue reward:0.85\n", - "step: 1394, Red action: do-nothing, Blue reward:0.85\n", - "step: 1395, Red action: do-nothing, Blue reward:0.85\n", - "step: 1396, Red action: do-nothing, Blue reward:0.85\n", - "step: 1397, Red action: do-nothing, Blue reward:0.85\n", - "step: 1398, Red action: do-nothing, Blue reward:0.85\n", - "step: 1399, Red action: do-nothing, Blue reward:0.85\n", - "step: 1400, Red action: do-nothing, Blue reward:0.85\n", - "step: 1401, Red action: do-nothing, Blue reward:0.85\n", - "step: 1402, Red action: do-nothing, Blue reward:0.85\n", - "step: 1403, Red action: do-nothing, Blue reward:0.85\n", - "step: 1404, Red action: do-nothing, Blue reward:0.85\n", - "step: 1405, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1406, Red action: do-nothing, Blue reward:0.85\n", - "step: 1407, Red action: do-nothing, Blue reward:0.85\n", - "step: 1408, Red action: do-nothing, Blue reward:0.85\n", - "step: 1409, Red action: do-nothing, Blue reward:0.85\n", - "step: 1410, Red action: do-nothing, Blue reward:0.85\n", - "step: 1411, Red action: do-nothing, Blue reward:0.85\n", - "step: 1412, Red action: do-nothing, Blue reward:0.85\n", - "step: 1413, Red action: do-nothing, Blue reward:0.85\n", - "step: 1414, Red action: do-nothing, Blue reward:0.85\n", - "step: 1415, Red action: do-nothing, Blue reward:0.85\n", - "step: 1416, Red action: do-nothing, Blue reward:0.85\n", - "step: 1417, Red action: do-nothing, Blue reward:0.85\n", - "step: 1418, Red action: do-nothing, Blue reward:0.85\n", - "step: 1419, Red action: do-nothing, Blue reward:0.85\n", - "step: 1420, Red action: do-nothing, Blue reward:0.85\n", - "step: 1421, Red action: do-nothing, Blue reward:0.85\n", - "step: 1422, Red action: do-nothing, Blue reward:0.85\n", - "step: 1423, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1424, Red action: do-nothing, Blue reward:0.85\n", - "step: 1425, Red action: do-nothing, Blue reward:0.85\n", - "step: 1426, Red action: do-nothing, Blue reward:0.85\n", - "step: 1427, Red action: do-nothing, Blue reward:0.85\n", - "step: 1428, Red action: do-nothing, Blue reward:0.85\n", - "step: 1429, Red action: do-nothing, Blue reward:0.85\n", - "step: 1430, Red action: do-nothing, Blue reward:0.85\n", - "step: 1431, Red action: do-nothing, Blue reward:0.85\n", - "step: 1432, Red action: do-nothing, Blue reward:0.85\n", - "step: 1433, Red action: do-nothing, Blue reward:0.85\n", - "step: 1434, Red action: do-nothing, Blue reward:0.85\n", - "step: 1435, Red action: do-nothing, Blue reward:0.85\n", - "step: 1436, Red action: do-nothing, Blue reward:0.85\n", - "step: 1437, Red action: do-nothing, Blue reward:0.85\n", - "step: 1438, Red action: do-nothing, Blue reward:0.85\n", - "step: 1439, Red action: do-nothing, Blue reward:0.85\n", - "step: 1440, Red action: do-nothing, Blue reward:0.85\n", - "step: 1441, Red action: do-nothing, Blue reward:0.85\n", - "step: 1442, Red action: do-nothing, Blue reward:0.85\n", - "step: 1443, Red action: do-nothing, Blue reward:0.85\n", - "step: 1444, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1445, Red action: do-nothing, Blue reward:0.85\n", - "step: 1446, Red action: do-nothing, Blue reward:0.85\n", - "step: 1447, Red action: do-nothing, Blue reward:0.85\n", - "step: 1448, Red action: do-nothing, Blue reward:0.85\n", - "step: 1449, Red action: do-nothing, Blue reward:0.85\n", - "step: 1450, Red action: do-nothing, Blue reward:0.85\n", - "step: 1451, Red action: do-nothing, Blue reward:0.85\n", - "step: 1452, Red action: do-nothing, Blue reward:0.85\n", - "step: 1453, Red action: do-nothing, Blue reward:0.85\n", - "step: 1454, Red action: do-nothing, Blue reward:0.85\n", - "step: 1455, Red action: do-nothing, Blue reward:0.85\n", - "step: 1456, Red action: do-nothing, Blue reward:0.85\n", - "step: 1457, Red action: do-nothing, Blue reward:0.85\n", - "step: 1458, Red action: do-nothing, Blue reward:0.85\n", - "step: 1459, Red action: do-nothing, Blue reward:0.85\n", - "step: 1460, Red action: do-nothing, Blue reward:0.85\n", - "step: 1461, Red action: do-nothing, Blue reward:0.85\n", - "step: 1462, Red action: do-nothing, Blue reward:0.85\n", - "step: 1463, Red action: do-nothing, Blue reward:0.85\n", - "step: 1464, Red action: do-nothing, Blue reward:0.85\n", - "step: 1465, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1466, Red action: do-nothing, Blue reward:0.85\n", - "step: 1467, Red action: do-nothing, Blue reward:0.85\n", - "step: 1468, Red action: do-nothing, Blue reward:0.85\n", - "step: 1469, Red action: do-nothing, Blue reward:0.85\n", - "step: 1470, Red action: do-nothing, Blue reward:0.85\n", - "step: 1471, Red action: do-nothing, Blue reward:0.85\n", - "step: 1472, Red action: do-nothing, Blue reward:0.85\n", - "step: 1473, Red action: do-nothing, Blue reward:0.85\n", - "step: 1474, Red action: do-nothing, Blue reward:0.85\n", - "step: 1475, Red action: do-nothing, Blue reward:0.85\n", - "step: 1476, Red action: do-nothing, Blue reward:0.85\n", - "step: 1477, Red action: do-nothing, Blue reward:0.85\n", - "step: 1478, Red action: do-nothing, Blue reward:0.85\n", - "step: 1479, Red action: do-nothing, Blue reward:0.85\n", - "step: 1480, Red action: do-nothing, Blue reward:0.85\n", - "step: 1481, Red action: do-nothing, Blue reward:0.85\n", - "step: 1482, Red action: do-nothing, Blue reward:0.85\n", - "step: 1483, Red action: do-nothing, Blue reward:0.85\n", - "step: 1484, Red action: do-nothing, Blue reward:0.85\n", - "step: 1485, Red action: do-nothing, Blue reward:0.85\n", - "step: 1486, Red action: do-nothing, Blue reward:0.85\n", - "step: 1487, Red action: do-nothing, Blue reward:0.85\n", - "step: 1488, Red action: do-nothing, Blue reward:0.85\n", - "step: 1489, Red action: do-nothing, Blue reward:0.85\n", - "step: 1490, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1491, Red action: do-nothing, Blue reward:0.85\n", - "step: 1492, Red action: do-nothing, Blue reward:0.85\n", - "step: 1493, Red action: do-nothing, Blue reward:0.85\n", - "step: 1494, Red action: do-nothing, Blue reward:0.85\n", - "step: 1495, Red action: do-nothing, Blue reward:0.85\n", - "step: 1496, Red action: do-nothing, Blue reward:0.85\n", - "step: 1497, Red action: do-nothing, Blue reward:0.85\n", - "step: 1498, Red action: do-nothing, Blue reward:0.85\n", - "step: 1499, Red action: do-nothing, Blue reward:0.85\n", - "step: 1500, Red action: do-nothing, Blue reward:0.85\n", - "step: 1501, Red action: do-nothing, Blue reward:0.85\n", - "step: 1502, Red action: do-nothing, Blue reward:0.85\n", - "step: 1503, Red action: do-nothing, Blue reward:0.85\n", - "step: 1504, Red action: do-nothing, Blue reward:0.85\n", - "step: 1505, Red action: do-nothing, Blue reward:0.85\n", - "step: 1506, Red action: do-nothing, Blue reward:0.85\n", - "step: 1507, Red action: do-nothing, Blue reward:0.85\n", - "step: 1508, Red action: do-nothing, Blue reward:0.85\n", - "step: 1509, Red action: do-nothing, Blue reward:0.85\n", - "step: 1510, Red action: do-nothing, Blue reward:0.85\n", - "step: 1511, Red action: do-nothing, Blue reward:0.85\n", - "step: 1512, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1513, Red action: do-nothing, Blue reward:0.85\n", - "step: 1514, Red action: do-nothing, Blue reward:0.85\n", - "step: 1515, Red action: do-nothing, Blue reward:0.85\n", - "step: 1516, Red action: do-nothing, Blue reward:0.85\n", - "step: 1517, Red action: do-nothing, Blue reward:0.85\n", - "step: 1518, Red action: do-nothing, Blue reward:0.85\n", - "step: 1519, Red action: do-nothing, Blue reward:0.85\n", - "step: 1520, Red action: do-nothing, Blue reward:0.85\n", - "step: 1521, Red action: do-nothing, Blue reward:0.85\n", - "step: 1522, Red action: do-nothing, Blue reward:0.85\n", - "step: 1523, Red action: do-nothing, Blue reward:0.85\n", - "step: 1524, Red action: do-nothing, Blue reward:0.85\n", - "step: 1525, Red action: do-nothing, Blue reward:0.85\n", - "step: 1526, Red action: do-nothing, Blue reward:0.85\n", - "step: 1527, Red action: do-nothing, Blue reward:0.85\n", - "step: 1528, Red action: do-nothing, Blue reward:0.85\n", - "step: 1529, Red action: do-nothing, Blue reward:0.85\n", - "step: 1530, Red action: do-nothing, Blue reward:0.85\n", - "step: 1531, Red action: do-nothing, Blue reward:0.85\n", - "step: 1532, Red action: do-nothing, Blue reward:0.85\n", - "step: 1533, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1534, Red action: do-nothing, Blue reward:0.85\n", - "step: 1535, Red action: do-nothing, Blue reward:0.85\n", - "step: 1536, Red action: do-nothing, Blue reward:0.85\n", - "step: 1537, Red action: do-nothing, Blue reward:0.85\n", - "step: 1538, Red action: do-nothing, Blue reward:0.85\n", - "step: 1539, Red action: do-nothing, Blue reward:0.85\n", - "step: 1540, Red action: do-nothing, Blue reward:0.85\n", - "step: 1541, Red action: do-nothing, Blue reward:0.85\n", - "step: 1542, Red action: do-nothing, Blue reward:0.85\n", - "step: 1543, Red action: do-nothing, Blue reward:0.85\n", - "step: 1544, Red action: do-nothing, Blue reward:0.85\n", - "step: 1545, Red action: do-nothing, Blue reward:0.85\n", - "step: 1546, Red action: do-nothing, Blue reward:0.85\n", - "step: 1547, Red action: do-nothing, Blue reward:0.85\n", - "step: 1548, Red action: do-nothing, Blue reward:0.85\n", - "step: 1549, Red action: do-nothing, Blue reward:0.85\n", - "step: 1550, Red action: do-nothing, Blue reward:0.85\n", - "step: 1551, Red action: do-nothing, Blue reward:0.85\n", - "step: 1552, Red action: do-nothing, Blue reward:0.85\n", - "step: 1553, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1554, Red action: do-nothing, Blue reward:0.85\n", - "step: 1555, Red action: do-nothing, Blue reward:0.85\n", - "step: 1556, Red action: do-nothing, Blue reward:0.85\n", - "step: 1557, Red action: do-nothing, Blue reward:0.85\n", - "step: 1558, Red action: do-nothing, Blue reward:0.85\n", - "step: 1559, Red action: do-nothing, Blue reward:0.85\n", - "step: 1560, Red action: do-nothing, Blue reward:0.85\n", - "step: 1561, Red action: do-nothing, Blue reward:0.85\n", - "step: 1562, Red action: do-nothing, Blue reward:0.85\n", - "step: 1563, Red action: do-nothing, Blue reward:0.85\n", - "step: 1564, Red action: do-nothing, Blue reward:0.85\n", - "step: 1565, Red action: do-nothing, Blue reward:0.85\n", - "step: 1566, Red action: do-nothing, Blue reward:0.85\n", - "step: 1567, Red action: do-nothing, Blue reward:0.85\n", - "step: 1568, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1569, Red action: do-nothing, Blue reward:0.85\n", - "step: 1570, Red action: do-nothing, Blue reward:0.85\n", - "step: 1571, Red action: do-nothing, Blue reward:0.85\n", - "step: 1572, Red action: do-nothing, Blue reward:0.85\n", - "step: 1573, Red action: do-nothing, Blue reward:0.85\n", - "step: 1574, Red action: do-nothing, Blue reward:0.85\n", - "step: 1575, Red action: do-nothing, Blue reward:0.85\n", - "step: 1576, Red action: do-nothing, Blue reward:0.85\n", - "step: 1577, Red action: do-nothing, Blue reward:0.85\n", - "step: 1578, Red action: do-nothing, Blue reward:0.85\n", - "step: 1579, Red action: do-nothing, Blue reward:0.85\n", - "step: 1580, Red action: do-nothing, Blue reward:0.85\n", - "step: 1581, Red action: do-nothing, Blue reward:0.85\n", - "step: 1582, Red action: do-nothing, Blue reward:0.85\n", - "step: 1583, Red action: do-nothing, Blue reward:0.85\n", - "step: 1584, Red action: do-nothing, Blue reward:0.85\n", - "step: 1585, Red action: do-nothing, Blue reward:0.85\n", - "step: 1586, Red action: do-nothing, Blue reward:0.85\n", - "step: 1587, Red action: do-nothing, Blue reward:0.85\n", - "step: 1588, Red action: do-nothing, Blue reward:0.85\n", - "step: 1589, Red action: do-nothing, Blue reward:0.85\n", - "step: 1590, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1591, Red action: do-nothing, Blue reward:0.85\n", - "step: 1592, Red action: do-nothing, Blue reward:0.85\n", - "step: 1593, Red action: do-nothing, Blue reward:0.85\n", - "step: 1594, Red action: do-nothing, Blue reward:0.85\n", - "step: 1595, Red action: do-nothing, Blue reward:0.85\n", - "step: 1596, Red action: do-nothing, Blue reward:0.85\n", - "step: 1597, Red action: do-nothing, Blue reward:0.85\n", - "step: 1598, Red action: do-nothing, Blue reward:0.85\n", - "step: 1599, Red action: do-nothing, Blue reward:0.85\n", - "step: 1600, Red action: do-nothing, Blue reward:0.85\n", - "step: 1601, Red action: do-nothing, Blue reward:0.85\n", - "step: 1602, Red action: do-nothing, Blue reward:0.85\n", - "step: 1603, Red action: do-nothing, Blue reward:0.85\n", - "step: 1604, Red action: do-nothing, Blue reward:0.85\n", - "step: 1605, Red action: do-nothing, Blue reward:0.85\n", - "step: 1606, Red action: do-nothing, Blue reward:0.85\n", - "step: 1607, Red action: do-nothing, Blue reward:0.85\n", - "step: 1608, Red action: do-nothing, Blue reward:0.85\n", - "step: 1609, Red action: do-nothing, Blue reward:0.85\n", - "step: 1610, Red action: do-nothing, Blue reward:0.85\n", - "step: 1611, Red action: do-nothing, Blue reward:0.85\n", - "step: 1612, Red action: do-nothing, Blue reward:0.85\n", - "step: 1613, Red action: do-nothing, Blue reward:0.85\n", - "step: 1614, Red action: do-nothing, Blue reward:0.85\n", - "step: 1615, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1616, Red action: do-nothing, Blue reward:0.85\n", - "step: 1617, Red action: do-nothing, Blue reward:0.85\n", - "step: 1618, Red action: do-nothing, Blue reward:0.85\n", - "step: 1619, Red action: do-nothing, Blue reward:0.85\n", - "step: 1620, Red action: do-nothing, Blue reward:0.85\n", - "step: 1621, Red action: do-nothing, Blue reward:0.85\n", - "step: 1622, Red action: do-nothing, Blue reward:0.85\n", - "step: 1623, Red action: do-nothing, Blue reward:0.85\n", - "step: 1624, Red action: do-nothing, Blue reward:0.85\n", - "step: 1625, Red action: do-nothing, Blue reward:0.85\n", - "step: 1626, Red action: do-nothing, Blue reward:0.85\n", - "step: 1627, Red action: do-nothing, Blue reward:0.85\n", - "step: 1628, Red action: do-nothing, Blue reward:0.85\n", - "step: 1629, Red action: do-nothing, Blue reward:0.85\n", - "step: 1630, Red action: do-nothing, Blue reward:0.85\n", - "step: 1631, Red action: do-nothing, Blue reward:0.85\n", - "step: 1632, Red action: do-nothing, Blue reward:0.85\n", - "step: 1633, Red action: do-nothing, Blue reward:0.85\n", - "step: 1634, Red action: do-nothing, Blue reward:0.85\n", - "step: 1635, Red action: do-nothing, Blue reward:0.85\n", - "step: 1636, Red action: do-nothing, Blue reward:0.85\n", - "step: 1637, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1638, Red action: do-nothing, Blue reward:0.85\n", - "step: 1639, Red action: do-nothing, Blue reward:0.85\n", - "step: 1640, Red action: do-nothing, Blue reward:0.85\n", - "step: 1641, Red action: do-nothing, Blue reward:0.85\n", - "step: 1642, Red action: do-nothing, Blue reward:0.85\n", - "step: 1643, Red action: do-nothing, Blue reward:0.85\n", - "step: 1644, Red action: do-nothing, Blue reward:0.85\n", - "step: 1645, Red action: do-nothing, Blue reward:0.85\n", - "step: 1646, Red action: do-nothing, Blue reward:0.85\n", - "step: 1647, Red action: do-nothing, Blue reward:0.85\n", - "step: 1648, Red action: do-nothing, Blue reward:0.85\n", - "step: 1649, Red action: do-nothing, Blue reward:0.85\n", - "step: 1650, Red action: do-nothing, Blue reward:0.85\n", - "step: 1651, Red action: do-nothing, Blue reward:0.85\n", - "step: 1652, Red action: do-nothing, Blue reward:0.85\n", - "step: 1653, Red action: do-nothing, Blue reward:0.85\n", - "step: 1654, Red action: do-nothing, Blue reward:0.85\n", - "step: 1655, Red action: do-nothing, Blue reward:0.85\n", - "step: 1656, Red action: do-nothing, Blue reward:0.85\n", - "step: 1657, Red action: do-nothing, Blue reward:0.85\n", - "step: 1658, Red action: do-nothing, Blue reward:0.85\n", - "step: 1659, Red action: do-nothing, Blue reward:0.85\n", - "step: 1660, Red action: do-nothing, Blue reward:0.85\n", - "step: 1661, Red action: do-nothing, Blue reward:0.85\n", - "step: 1662, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1663, Red action: do-nothing, Blue reward:0.85\n", - "step: 1664, Red action: do-nothing, Blue reward:0.85\n", - "step: 1665, Red action: do-nothing, Blue reward:0.85\n", - "step: 1666, Red action: do-nothing, Blue reward:0.85\n", - "step: 1667, Red action: do-nothing, Blue reward:0.85\n", - "step: 1668, Red action: do-nothing, Blue reward:0.85\n", - "step: 1669, Red action: do-nothing, Blue reward:0.85\n", - "step: 1670, Red action: do-nothing, Blue reward:0.85\n", - "step: 1671, Red action: do-nothing, Blue reward:0.85\n", - "step: 1672, Red action: do-nothing, Blue reward:0.85\n", - "step: 1673, Red action: do-nothing, Blue reward:0.85\n", - "step: 1674, Red action: do-nothing, Blue reward:0.85\n", - "step: 1675, Red action: do-nothing, Blue reward:0.85\n", - "step: 1676, Red action: do-nothing, Blue reward:0.85\n", - "step: 1677, Red action: do-nothing, Blue reward:0.85\n", - "step: 1678, Red action: do-nothing, Blue reward:0.85\n", - "step: 1679, Red action: do-nothing, Blue reward:0.85\n", - "step: 1680, Red action: do-nothing, Blue reward:0.85\n", - "step: 1681, Red action: do-nothing, Blue reward:0.85\n", - "step: 1682, Red action: do-nothing, Blue reward:0.85\n", - "step: 1683, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1684, Red action: do-nothing, Blue reward:0.85\n", - "step: 1685, Red action: do-nothing, Blue reward:0.85\n", - "step: 1686, Red action: do-nothing, Blue reward:0.85\n", - "step: 1687, Red action: do-nothing, Blue reward:0.85\n", - "step: 1688, Red action: do-nothing, Blue reward:0.85\n", - "step: 1689, Red action: do-nothing, Blue reward:0.85\n", - "step: 1690, Red action: do-nothing, Blue reward:0.85\n", - "step: 1691, Red action: do-nothing, Blue reward:0.85\n", - "step: 1692, Red action: do-nothing, Blue reward:0.85\n", - "step: 1693, Red action: do-nothing, Blue reward:0.85\n", - "step: 1694, Red action: do-nothing, Blue reward:0.85\n", - "step: 1695, Red action: do-nothing, Blue reward:0.85\n", - "step: 1696, Red action: do-nothing, Blue reward:0.85\n", - "step: 1697, Red action: do-nothing, Blue reward:0.85\n", - "step: 1698, Red action: do-nothing, Blue reward:0.85\n", - "step: 1699, Red action: do-nothing, Blue reward:0.85\n", - "step: 1700, Red action: do-nothing, Blue reward:0.85\n", - "step: 1701, Red action: do-nothing, Blue reward:0.85\n", - "step: 1702, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1703, Red action: do-nothing, Blue reward:0.85\n", - "step: 1704, Red action: do-nothing, Blue reward:0.85\n", - "step: 1705, Red action: do-nothing, Blue reward:0.85\n", - "step: 1706, Red action: do-nothing, Blue reward:0.85\n", - "step: 1707, Red action: do-nothing, Blue reward:0.85\n", - "step: 1708, Red action: do-nothing, Blue reward:0.85\n", - "step: 1709, Red action: do-nothing, Blue reward:0.85\n", - "step: 1710, Red action: do-nothing, Blue reward:0.85\n", - "step: 1711, Red action: do-nothing, Blue reward:0.85\n", - "step: 1712, Red action: do-nothing, Blue reward:0.85\n", - "step: 1713, Red action: do-nothing, Blue reward:0.85\n", - "step: 1714, Red action: do-nothing, Blue reward:0.85\n", - "step: 1715, Red action: do-nothing, Blue reward:0.85\n", - "step: 1716, Red action: do-nothing, Blue reward:0.85\n", - "step: 1717, Red action: do-nothing, Blue reward:0.85\n", - "step: 1718, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1719, Red action: do-nothing, Blue reward:0.85\n", - "step: 1720, Red action: do-nothing, Blue reward:0.85\n", - "step: 1721, Red action: do-nothing, Blue reward:0.85\n", - "step: 1722, Red action: do-nothing, Blue reward:0.85\n", - "step: 1723, Red action: do-nothing, Blue reward:0.85\n", - "step: 1724, Red action: do-nothing, Blue reward:0.85\n", - "step: 1725, Red action: do-nothing, Blue reward:0.85\n", - "step: 1726, Red action: do-nothing, Blue reward:0.85\n", - "step: 1727, Red action: do-nothing, Blue reward:0.85\n", - "step: 1728, Red action: do-nothing, Blue reward:0.85\n", - "step: 1729, Red action: do-nothing, Blue reward:0.85\n", - "step: 1730, Red action: do-nothing, Blue reward:0.85\n", - "step: 1731, Red action: do-nothing, Blue reward:0.85\n", - "step: 1732, Red action: do-nothing, Blue reward:0.85\n", - "step: 1733, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1734, Red action: do-nothing, Blue reward:0.85\n", - "step: 1735, Red action: do-nothing, Blue reward:0.85\n", - "step: 1736, Red action: do-nothing, Blue reward:0.85\n", - "step: 1737, Red action: do-nothing, Blue reward:0.85\n", - "step: 1738, Red action: do-nothing, Blue reward:0.85\n", - "step: 1739, Red action: do-nothing, Blue reward:0.85\n", - "step: 1740, Red action: do-nothing, Blue reward:0.85\n", - "step: 1741, Red action: do-nothing, Blue reward:0.85\n", - "step: 1742, Red action: do-nothing, Blue reward:0.85\n", - "step: 1743, Red action: do-nothing, Blue reward:0.85\n", - "step: 1744, Red action: do-nothing, Blue reward:0.85\n", - "step: 1745, Red action: do-nothing, Blue reward:0.85\n", - "step: 1746, Red action: do-nothing, Blue reward:0.85\n", - "step: 1747, Red action: do-nothing, Blue reward:0.85\n", - "step: 1748, Red action: do-nothing, Blue reward:0.85\n", - "step: 1749, Red action: do-nothing, Blue reward:0.85\n", - "step: 1750, Red action: do-nothing, Blue reward:0.85\n", - "step: 1751, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1752, Red action: do-nothing, Blue reward:0.85\n", - "step: 1753, Red action: do-nothing, Blue reward:0.85\n", - "step: 1754, Red action: do-nothing, Blue reward:0.85\n", - "step: 1755, Red action: do-nothing, Blue reward:0.85\n", - "step: 1756, Red action: do-nothing, Blue reward:0.85\n", - "step: 1757, Red action: do-nothing, Blue reward:0.85\n", - "step: 1758, Red action: do-nothing, Blue reward:0.85\n", - "step: 1759, Red action: do-nothing, Blue reward:0.85\n", - "step: 1760, Red action: do-nothing, Blue reward:0.85\n", - "step: 1761, Red action: do-nothing, Blue reward:0.85\n", - "step: 1762, Red action: do-nothing, Blue reward:0.85\n", - "step: 1763, Red action: do-nothing, Blue reward:0.85\n", - "step: 1764, Red action: do-nothing, Blue reward:0.85\n", - "step: 1765, Red action: do-nothing, Blue reward:0.85\n", - "step: 1766, Red action: do-nothing, Blue reward:0.85\n", - "step: 1767, Red action: do-nothing, Blue reward:0.85\n", - "step: 1768, Red action: do-nothing, Blue reward:0.85\n", - "step: 1769, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1770, Red action: do-nothing, Blue reward:0.85\n", - "step: 1771, Red action: do-nothing, Blue reward:0.85\n", - "step: 1772, Red action: do-nothing, Blue reward:0.85\n", - "step: 1773, Red action: do-nothing, Blue reward:0.85\n", - "step: 1774, Red action: do-nothing, Blue reward:0.85\n", - "step: 1775, Red action: do-nothing, Blue reward:0.85\n", - "step: 1776, Red action: do-nothing, Blue reward:0.85\n", - "step: 1777, Red action: do-nothing, Blue reward:0.85\n", - "step: 1778, Red action: do-nothing, Blue reward:0.85\n", - "step: 1779, Red action: do-nothing, Blue reward:0.85\n", - "step: 1780, Red action: do-nothing, Blue reward:0.85\n", - "step: 1781, Red action: do-nothing, Blue reward:0.85\n", - "step: 1782, Red action: do-nothing, Blue reward:0.85\n", - "step: 1783, Red action: do-nothing, Blue reward:0.85\n", - "step: 1784, Red action: do-nothing, Blue reward:0.85\n", - "step: 1785, Red action: do-nothing, Blue reward:0.85\n", - "step: 1786, Red action: do-nothing, Blue reward:0.85\n", - "step: 1787, Red action: do-nothing, Blue reward:0.85\n", - "step: 1788, Red action: do-nothing, Blue reward:0.85\n", - "step: 1789, Red action: do-nothing, Blue reward:0.85\n", - "step: 1790, Red action: do-nothing, Blue reward:0.85\n", - "step: 1791, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1792, Red action: do-nothing, Blue reward:0.85\n", - "step: 1793, Red action: do-nothing, Blue reward:0.85\n", - "step: 1794, Red action: do-nothing, Blue reward:0.85\n", - "step: 1795, Red action: do-nothing, Blue reward:0.85\n", - "step: 1796, Red action: do-nothing, Blue reward:0.85\n", - "step: 1797, Red action: do-nothing, Blue reward:0.85\n", - "step: 1798, Red action: do-nothing, Blue reward:0.85\n", - "step: 1799, Red action: do-nothing, Blue reward:0.85\n", - "step: 1800, Red action: do-nothing, Blue reward:0.85\n", - "step: 1801, Red action: do-nothing, Blue reward:0.85\n", - "step: 1802, Red action: do-nothing, Blue reward:0.85\n", - "step: 1803, Red action: do-nothing, Blue reward:0.85\n", - "step: 1804, Red action: do-nothing, Blue reward:0.85\n", - "step: 1805, Red action: do-nothing, Blue reward:0.85\n", - "step: 1806, Red action: do-nothing, Blue reward:0.85\n", - "step: 1807, Red action: do-nothing, Blue reward:0.85\n", - "step: 1808, Red action: do-nothing, Blue reward:0.85\n", - "step: 1809, Red action: do-nothing, Blue reward:0.85\n", - "step: 1810, Red action: do-nothing, Blue reward:0.85\n", - "step: 1811, Red action: do-nothing, Blue reward:0.85\n", - "step: 1812, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1813, Red action: do-nothing, Blue reward:0.85\n", - "step: 1814, Red action: do-nothing, Blue reward:0.85\n", - "step: 1815, Red action: do-nothing, Blue reward:0.85\n", - "step: 1816, Red action: do-nothing, Blue reward:0.85\n", - "step: 1817, Red action: do-nothing, Blue reward:0.85\n", - "step: 1818, Red action: do-nothing, Blue reward:0.85\n", - "step: 1819, Red action: do-nothing, Blue reward:0.85\n", - "step: 1820, Red action: do-nothing, Blue reward:0.85\n", - "step: 1821, Red action: do-nothing, Blue reward:0.85\n", - "step: 1822, Red action: do-nothing, Blue reward:0.85\n", - "step: 1823, Red action: do-nothing, Blue reward:0.85\n", - "step: 1824, Red action: do-nothing, Blue reward:0.85\n", - "step: 1825, Red action: do-nothing, Blue reward:0.85\n", - "step: 1826, Red action: do-nothing, Blue reward:0.85\n", - "step: 1827, Red action: do-nothing, Blue reward:0.85\n", - "step: 1828, Red action: do-nothing, Blue reward:0.85\n", - "step: 1829, Red action: do-nothing, Blue reward:0.85\n", - "step: 1830, Red action: do-nothing, Blue reward:0.85\n", - "step: 1831, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1832, Red action: do-nothing, Blue reward:0.85\n", - "step: 1833, Red action: do-nothing, Blue reward:0.85\n", - "step: 1834, Red action: do-nothing, Blue reward:0.85\n", - "step: 1835, Red action: do-nothing, Blue reward:0.85\n", - "step: 1836, Red action: do-nothing, Blue reward:0.85\n", - "step: 1837, Red action: do-nothing, Blue reward:0.85\n", - "step: 1838, Red action: do-nothing, Blue reward:0.85\n", - "step: 1839, Red action: do-nothing, Blue reward:0.85\n", - "step: 1840, Red action: do-nothing, Blue reward:0.85\n", - "step: 1841, Red action: do-nothing, Blue reward:0.85\n", - "step: 1842, Red action: do-nothing, Blue reward:0.85\n", - "step: 1843, Red action: do-nothing, Blue reward:0.85\n", - "step: 1844, Red action: do-nothing, Blue reward:0.85\n", - "step: 1845, Red action: do-nothing, Blue reward:0.85\n", - "step: 1846, Red action: do-nothing, Blue reward:0.85\n", - "step: 1847, Red action: do-nothing, Blue reward:0.85\n", - "step: 1848, Red action: do-nothing, Blue reward:0.85\n", - "step: 1849, Red action: do-nothing, Blue reward:0.85\n", - "step: 1850, Red action: do-nothing, Blue reward:0.85\n", - "step: 1851, Red action: do-nothing, Blue reward:0.85\n", - "step: 1852, Red action: do-nothing, Blue reward:0.85\n", - "step: 1853, Red action: do-nothing, Blue reward:0.85\n", - "step: 1854, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1855, Red action: do-nothing, Blue reward:0.85\n", - "step: 1856, Red action: do-nothing, Blue reward:0.85\n", - "step: 1857, Red action: do-nothing, Blue reward:0.85\n", - "step: 1858, Red action: do-nothing, Blue reward:0.85\n", - "step: 1859, Red action: do-nothing, Blue reward:0.85\n", - "step: 1860, Red action: do-nothing, Blue reward:0.85\n", - "step: 1861, Red action: do-nothing, Blue reward:0.85\n", - "step: 1862, Red action: do-nothing, Blue reward:0.85\n", - "step: 1863, Red action: do-nothing, Blue reward:0.85\n", - "step: 1864, Red action: do-nothing, Blue reward:0.85\n", - "step: 1865, Red action: do-nothing, Blue reward:0.85\n", - "step: 1866, Red action: do-nothing, Blue reward:0.85\n", - "step: 1867, Red action: do-nothing, Blue reward:0.85\n", - "step: 1868, Red action: do-nothing, Blue reward:0.85\n", - "step: 1869, Red action: do-nothing, Blue reward:0.85\n", - "step: 1870, Red action: do-nothing, Blue reward:0.85\n", - "step: 1871, Red action: do-nothing, Blue reward:0.85\n", - "step: 1872, Red action: do-nothing, Blue reward:0.85\n", - "step: 1873, Red action: do-nothing, Blue reward:0.85\n", - "step: 1874, Red action: do-nothing, Blue reward:0.85\n", - "step: 1875, Red action: do-nothing, Blue reward:0.85\n", - "step: 1876, Red action: do-nothing, Blue reward:0.85\n", - "step: 1877, Red action: do-nothing, Blue reward:0.85\n", - "step: 1878, Red action: do-nothing, Blue reward:0.85\n", - "step: 1879, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1880, Red action: do-nothing, Blue reward:0.85\n", - "step: 1881, Red action: do-nothing, Blue reward:0.85\n", - "step: 1882, Red action: do-nothing, Blue reward:0.85\n", - "step: 1883, Red action: do-nothing, Blue reward:0.85\n", - "step: 1884, Red action: do-nothing, Blue reward:0.85\n", - "step: 1885, Red action: do-nothing, Blue reward:0.85\n", - "step: 1886, Red action: do-nothing, Blue reward:0.85\n", - "step: 1887, Red action: do-nothing, Blue reward:0.85\n", - "step: 1888, Red action: do-nothing, Blue reward:0.85\n", - "step: 1889, Red action: do-nothing, Blue reward:0.85\n", - "step: 1890, Red action: do-nothing, Blue reward:0.85\n", - "step: 1891, Red action: do-nothing, Blue reward:0.85\n", - "step: 1892, Red action: do-nothing, Blue reward:0.85\n", - "step: 1893, Red action: do-nothing, Blue reward:0.85\n", - "step: 1894, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1895, Red action: do-nothing, Blue reward:0.85\n", - "step: 1896, Red action: do-nothing, Blue reward:0.85\n", - "step: 1897, Red action: do-nothing, Blue reward:0.85\n", - "step: 1898, Red action: do-nothing, Blue reward:0.85\n", - "step: 1899, Red action: do-nothing, Blue reward:0.85\n", - "step: 1900, Red action: do-nothing, Blue reward:0.85\n", - "step: 1901, Red action: do-nothing, Blue reward:0.85\n", - "step: 1902, Red action: do-nothing, Blue reward:0.85\n", - "step: 1903, Red action: do-nothing, Blue reward:0.85\n", - "step: 1904, Red action: do-nothing, Blue reward:0.85\n", - "step: 1905, Red action: do-nothing, Blue reward:0.85\n", - "step: 1906, Red action: do-nothing, Blue reward:0.85\n", - "step: 1907, Red action: do-nothing, Blue reward:0.85\n", - "step: 1908, Red action: do-nothing, Blue reward:0.85\n", - "step: 1909, Red action: do-nothing, Blue reward:0.85\n", - "step: 1910, Red action: do-nothing, Blue reward:0.85\n", - "step: 1911, Red action: do-nothing, Blue reward:0.85\n", - "step: 1912, Red action: do-nothing, Blue reward:0.85\n", - "step: 1913, Red action: do-nothing, Blue reward:0.85\n", - "step: 1914, Red action: do-nothing, Blue reward:0.85\n", - "step: 1915, Red action: do-nothing, Blue reward:0.85\n", - "step: 1916, Red action: do-nothing, Blue reward:0.85\n", - "step: 1917, Red action: do-nothing, Blue reward:0.85\n", - "step: 1918, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1919, Red action: do-nothing, Blue reward:0.85\n", - "step: 1920, Red action: do-nothing, Blue reward:0.85\n", - "step: 1921, Red action: do-nothing, Blue reward:0.85\n", - "step: 1922, Red action: do-nothing, Blue reward:0.85\n", - "step: 1923, Red action: do-nothing, Blue reward:0.85\n", - "step: 1924, Red action: do-nothing, Blue reward:0.85\n", - "step: 1925, Red action: do-nothing, Blue reward:0.85\n", - "step: 1926, Red action: do-nothing, Blue reward:0.85\n", - "step: 1927, Red action: do-nothing, Blue reward:0.85\n", - "step: 1928, Red action: do-nothing, Blue reward:0.85\n", - "step: 1929, Red action: do-nothing, Blue reward:0.85\n", - "step: 1930, Red action: do-nothing, Blue reward:0.85\n", - "step: 1931, Red action: do-nothing, Blue reward:0.85\n", - "step: 1932, Red action: do-nothing, Blue reward:0.85\n", - "step: 1933, Red action: do-nothing, Blue reward:0.85\n", - "step: 1934, Red action: do-nothing, Blue reward:0.85\n", - "step: 1935, Red action: do-nothing, Blue reward:0.85\n", - "step: 1936, Red action: do-nothing, Blue reward:0.85\n", - "step: 1937, Red action: do-nothing, Blue reward:0.85\n", - "step: 1938, Red action: do-nothing, Blue reward:0.85\n", - "step: 1939, Red action: do-nothing, Blue reward:0.85\n", - "step: 1940, Red action: do-nothing, Blue reward:0.85\n", - "step: 1941, Red action: do-nothing, Blue reward:0.85\n", - "step: 1942, Red action: do-nothing, Blue reward:0.85\n", - "step: 1943, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1944, Red action: do-nothing, Blue reward:0.85\n", - "step: 1945, Red action: do-nothing, Blue reward:0.85\n", - "step: 1946, Red action: do-nothing, Blue reward:0.85\n", - "step: 1947, Red action: do-nothing, Blue reward:0.85\n", - "step: 1948, Red action: do-nothing, Blue reward:0.85\n", - "step: 1949, Red action: do-nothing, Blue reward:0.85\n", - "step: 1950, Red action: do-nothing, Blue reward:0.85\n", - "step: 1951, Red action: do-nothing, Blue reward:0.85\n", - "step: 1952, Red action: do-nothing, Blue reward:0.85\n", - "step: 1953, Red action: do-nothing, Blue reward:0.85\n", - "step: 1954, Red action: do-nothing, Blue reward:0.85\n", - "step: 1955, Red action: do-nothing, Blue reward:0.85\n", - "step: 1956, Red action: do-nothing, Blue reward:0.85\n", - "step: 1957, Red action: do-nothing, Blue reward:0.85\n", - "step: 1958, Red action: do-nothing, Blue reward:0.85\n", - "step: 1959, Red action: do-nothing, Blue reward:0.85\n", - "step: 1960, Red action: do-nothing, Blue reward:0.85\n", - "step: 1961, Red action: do-nothing, Blue reward:0.85\n", - "step: 1962, Red action: do-nothing, Blue reward:0.85\n", - "step: 1963, Red action: do-nothing, Blue reward:0.85\n", - "step: 1964, Red action: do-nothing, Blue reward:0.85\n", - "step: 1965, Red action: do-nothing, Blue reward:0.85\n", - "step: 1966, Red action: do-nothing, Blue reward:0.85\n", - "step: 1967, Red action: do-nothing, Blue reward:0.85\n", - "step: 1968, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1969, Red action: do-nothing, Blue reward:0.85\n", - "step: 1970, Red action: do-nothing, Blue reward:0.85\n", - "step: 1971, Red action: do-nothing, Blue reward:0.85\n", - "step: 1972, Red action: do-nothing, Blue reward:0.85\n", - "step: 1973, Red action: do-nothing, Blue reward:0.85\n", - "step: 1974, Red action: do-nothing, Blue reward:0.85\n", - "step: 1975, Red action: do-nothing, Blue reward:0.85\n", - "step: 1976, Red action: do-nothing, Blue reward:0.85\n", - "step: 1977, Red action: do-nothing, Blue reward:0.85\n", - "step: 1978, Red action: do-nothing, Blue reward:0.85\n", - "step: 1979, Red action: do-nothing, Blue reward:0.85\n", - "step: 1980, Red action: do-nothing, Blue reward:0.85\n", - "step: 1981, Red action: do-nothing, Blue reward:0.85\n", - "step: 1982, Red action: do-nothing, Blue reward:0.85\n", - "step: 1983, Red action: do-nothing, Blue reward:0.85\n", - "step: 1984, Red action: do-nothing, Blue reward:0.85\n", - "step: 1985, Red action: do-nothing, Blue reward:0.85\n", - "step: 1986, Red action: do-nothing, Blue reward:0.85\n", - "step: 1987, Red action: node-application-execute, Blue reward:0.85\n", - "step: 1988, Red action: do-nothing, Blue reward:0.85\n", - "step: 1989, Red action: do-nothing, Blue reward:0.85\n", - "step: 1990, Red action: do-nothing, Blue reward:0.85\n", - "step: 1991, Red action: do-nothing, Blue reward:0.85\n", - "step: 1992, Red action: do-nothing, Blue reward:0.85\n", - "step: 1993, Red action: do-nothing, Blue reward:0.85\n", - "step: 1994, Red action: do-nothing, Blue reward:0.85\n", - "step: 1995, Red action: do-nothing, Blue reward:0.85\n", - "step: 1996, Red action: do-nothing, Blue reward:0.85\n", - "step: 1997, Red action: do-nothing, Blue reward:0.85\n", - "step: 1998, Red action: do-nothing, Blue reward:0.85\n", - "step: 1999, Red action: do-nothing, Blue reward:0.85\n", - "step: 2000, Red action: do-nothing, Blue reward:0.85\n", - "step: 2001, Red action: do-nothing, Blue reward:0.85\n", - "step: 2002, Red action: do-nothing, Blue reward:0.85\n", - "step: 2003, Red action: do-nothing, Blue reward:0.85\n", - "step: 2004, Red action: do-nothing, Blue reward:0.85\n", - "step: 2005, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2006, Red action: do-nothing, Blue reward:0.85\n", - "step: 2007, Red action: do-nothing, Blue reward:0.85\n", - "step: 2008, Red action: do-nothing, Blue reward:0.85\n", - "step: 2009, Red action: do-nothing, Blue reward:0.85\n", - "step: 2010, Red action: do-nothing, Blue reward:0.85\n", - "step: 2011, Red action: do-nothing, Blue reward:0.85\n", - "step: 2012, Red action: do-nothing, Blue reward:0.85\n", - "step: 2013, Red action: do-nothing, Blue reward:0.85\n", - "step: 2014, Red action: do-nothing, Blue reward:0.85\n", - "step: 2015, Red action: do-nothing, Blue reward:0.85\n", - "step: 2016, Red action: do-nothing, Blue reward:0.85\n", - "step: 2017, Red action: do-nothing, Blue reward:0.85\n", - "step: 2018, Red action: do-nothing, Blue reward:0.85\n", - "step: 2019, Red action: do-nothing, Blue reward:0.85\n", - "step: 2020, Red action: do-nothing, Blue reward:0.85\n", - "step: 2021, Red action: do-nothing, Blue reward:0.85\n", - "step: 2022, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2023, Red action: do-nothing, Blue reward:0.85\n", - "step: 2024, Red action: do-nothing, Blue reward:0.85\n", - "step: 2025, Red action: do-nothing, Blue reward:0.85\n", - "step: 2026, Red action: do-nothing, Blue reward:0.85\n", - "step: 2027, Red action: do-nothing, Blue reward:0.85\n", - "step: 2028, Red action: do-nothing, Blue reward:0.85\n", - "step: 2029, Red action: do-nothing, Blue reward:0.85\n", - "step: 2030, Red action: do-nothing, Blue reward:0.85\n", - "step: 2031, Red action: do-nothing, Blue reward:0.85\n", - "step: 2032, Red action: do-nothing, Blue reward:0.85\n", - "step: 2033, Red action: do-nothing, Blue reward:0.85\n", - "step: 2034, Red action: do-nothing, Blue reward:0.85\n", - "step: 2035, Red action: do-nothing, Blue reward:0.85\n", - "step: 2036, Red action: do-nothing, Blue reward:0.85\n", - "step: 2037, Red action: do-nothing, Blue reward:0.85\n", - "step: 2038, Red action: do-nothing, Blue reward:0.85\n", - "step: 2039, Red action: do-nothing, Blue reward:0.85\n", - "step: 2040, Red action: do-nothing, Blue reward:0.85\n", - "step: 2041, Red action: do-nothing, Blue reward:0.85\n", - "step: 2042, Red action: do-nothing, Blue reward:0.85\n", - "step: 2043, Red action: do-nothing, Blue reward:0.85\n", - "step: 2044, Red action: do-nothing, Blue reward:0.85\n", - "step: 2045, Red action: do-nothing, Blue reward:0.85\n", - "step: 2046, Red action: do-nothing, Blue reward:0.85\n", - "step: 2047, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2048, Red action: do-nothing, Blue reward:0.85\n", - "step: 2049, Red action: do-nothing, Blue reward:0.85\n", - "step: 2050, Red action: do-nothing, Blue reward:0.85\n", - "step: 2051, Red action: do-nothing, Blue reward:0.85\n", - "step: 2052, Red action: do-nothing, Blue reward:0.85\n", - "step: 2053, Red action: do-nothing, Blue reward:0.85\n", - "step: 2054, Red action: do-nothing, Blue reward:0.85\n", - "step: 2055, Red action: do-nothing, Blue reward:0.85\n", - "step: 2056, Red action: do-nothing, Blue reward:0.85\n", - "step: 2057, Red action: do-nothing, Blue reward:0.85\n", - "step: 2058, Red action: do-nothing, Blue reward:0.85\n", - "step: 2059, Red action: do-nothing, Blue reward:0.85\n", - "step: 2060, Red action: do-nothing, Blue reward:0.85\n", - "step: 2061, Red action: do-nothing, Blue reward:0.85\n", - "step: 2062, Red action: do-nothing, Blue reward:0.85\n", - "step: 2063, Red action: do-nothing, Blue reward:0.85\n", - "step: 2064, Red action: do-nothing, Blue reward:0.85\n", - "step: 2065, Red action: do-nothing, Blue reward:0.85\n", - "step: 2066, Red action: do-nothing, Blue reward:0.85\n", - "step: 2067, Red action: do-nothing, Blue reward:0.85\n", - "step: 2068, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2069, Red action: do-nothing, Blue reward:0.85\n", - "step: 2070, Red action: do-nothing, Blue reward:0.85\n", - "step: 2071, Red action: do-nothing, Blue reward:0.85\n", - "step: 2072, Red action: do-nothing, Blue reward:0.85\n", - "step: 2073, Red action: do-nothing, Blue reward:0.85\n", - "step: 2074, Red action: do-nothing, Blue reward:0.85\n", - "step: 2075, Red action: do-nothing, Blue reward:0.85\n", - "step: 2076, Red action: do-nothing, Blue reward:0.85\n", - "step: 2077, Red action: do-nothing, Blue reward:0.85\n", - "step: 2078, Red action: do-nothing, Blue reward:0.85\n", - "step: 2079, Red action: do-nothing, Blue reward:0.85\n", - "step: 2080, Red action: do-nothing, Blue reward:0.85\n", - "step: 2081, Red action: do-nothing, Blue reward:0.85\n", - "step: 2082, Red action: do-nothing, Blue reward:0.85\n", - "step: 2083, Red action: do-nothing, Blue reward:0.85\n", - "step: 2084, Red action: do-nothing, Blue reward:0.85\n", - "step: 2085, Red action: do-nothing, Blue reward:0.85\n", - "step: 2086, Red action: do-nothing, Blue reward:0.85\n", - "step: 2087, Red action: do-nothing, Blue reward:0.85\n", - "step: 2088, Red action: do-nothing, Blue reward:0.85\n", - "step: 2089, Red action: do-nothing, Blue reward:0.85\n", - "step: 2090, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2091, Red action: do-nothing, Blue reward:0.85\n", - "step: 2092, Red action: do-nothing, Blue reward:0.85\n", - "step: 2093, Red action: do-nothing, Blue reward:0.85\n", - "step: 2094, Red action: do-nothing, Blue reward:0.85\n", - "step: 2095, Red action: do-nothing, Blue reward:0.85\n", - "step: 2096, Red action: do-nothing, Blue reward:0.85\n", - "step: 2097, Red action: do-nothing, Blue reward:0.85\n", - "step: 2098, Red action: do-nothing, Blue reward:0.85\n", - "step: 2099, Red action: do-nothing, Blue reward:0.85\n", - "step: 2100, Red action: do-nothing, Blue reward:0.85\n", - "step: 2101, Red action: do-nothing, Blue reward:0.85\n", - "step: 2102, Red action: do-nothing, Blue reward:0.85\n", - "step: 2103, Red action: do-nothing, Blue reward:0.85\n", - "step: 2104, Red action: do-nothing, Blue reward:0.85\n", - "step: 2105, Red action: do-nothing, Blue reward:0.85\n", - "step: 2106, Red action: do-nothing, Blue reward:0.85\n", - "step: 2107, Red action: do-nothing, Blue reward:0.85\n", - "step: 2108, Red action: do-nothing, Blue reward:0.85\n", - "step: 2109, Red action: do-nothing, Blue reward:0.85\n", - "step: 2110, Red action: do-nothing, Blue reward:0.85\n", - "step: 2111, Red action: do-nothing, Blue reward:0.85\n", - "step: 2112, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2113, Red action: do-nothing, Blue reward:0.85\n", - "step: 2114, Red action: do-nothing, Blue reward:0.85\n", - "step: 2115, Red action: do-nothing, Blue reward:0.85\n", - "step: 2116, Red action: do-nothing, Blue reward:0.85\n", - "step: 2117, Red action: do-nothing, Blue reward:0.85\n", - "step: 2118, Red action: do-nothing, Blue reward:0.85\n", - "step: 2119, Red action: do-nothing, Blue reward:0.85\n", - "step: 2120, Red action: do-nothing, Blue reward:0.85\n", - "step: 2121, Red action: do-nothing, Blue reward:0.85\n", - "step: 2122, Red action: do-nothing, Blue reward:0.85\n", - "step: 2123, Red action: do-nothing, Blue reward:0.85\n", - "step: 2124, Red action: do-nothing, Blue reward:0.85\n", - "step: 2125, Red action: do-nothing, Blue reward:0.85\n", - "step: 2126, Red action: do-nothing, Blue reward:0.85\n", - "step: 2127, Red action: do-nothing, Blue reward:0.85\n", - "step: 2128, Red action: do-nothing, Blue reward:0.85\n", - "step: 2129, Red action: do-nothing, Blue reward:0.85\n", - "step: 2130, Red action: do-nothing, Blue reward:0.85\n", - "step: 2131, Red action: do-nothing, Blue reward:0.85\n", - "step: 2132, Red action: do-nothing, Blue reward:0.85\n", - "step: 2133, Red action: do-nothing, Blue reward:0.85\n", - "step: 2134, Red action: do-nothing, Blue reward:0.85\n", - "step: 2135, Red action: do-nothing, Blue reward:0.85\n", - "step: 2136, Red action: do-nothing, Blue reward:0.85\n", - "step: 2137, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2138, Red action: do-nothing, Blue reward:0.85\n", - "step: 2139, Red action: do-nothing, Blue reward:0.85\n", - "step: 2140, Red action: do-nothing, Blue reward:0.85\n", - "step: 2141, Red action: do-nothing, Blue reward:0.85\n", - "step: 2142, Red action: do-nothing, Blue reward:0.85\n", - "step: 2143, Red action: do-nothing, Blue reward:0.85\n", - "step: 2144, Red action: do-nothing, Blue reward:0.85\n", - "step: 2145, Red action: do-nothing, Blue reward:0.85\n", - "step: 2146, Red action: do-nothing, Blue reward:0.85\n", - "step: 2147, Red action: do-nothing, Blue reward:0.85\n", - "step: 2148, Red action: do-nothing, Blue reward:0.85\n", - "step: 2149, Red action: do-nothing, Blue reward:0.85\n", - "step: 2150, Red action: do-nothing, Blue reward:0.85\n", - "step: 2151, Red action: do-nothing, Blue reward:0.85\n", - "step: 2152, Red action: do-nothing, Blue reward:0.85\n", - "step: 2153, Red action: do-nothing, Blue reward:0.85\n", - "step: 2154, Red action: do-nothing, Blue reward:0.85\n", - "step: 2155, Red action: do-nothing, Blue reward:0.85\n", - "step: 2156, Red action: do-nothing, Blue reward:0.85\n", - "step: 2157, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2158, Red action: do-nothing, Blue reward:0.85\n", - "step: 2159, Red action: do-nothing, Blue reward:0.85\n", - "step: 2160, Red action: do-nothing, Blue reward:0.85\n", - "step: 2161, Red action: do-nothing, Blue reward:0.85\n", - "step: 2162, Red action: do-nothing, Blue reward:0.85\n", - "step: 2163, Red action: do-nothing, Blue reward:0.85\n", - "step: 2164, Red action: do-nothing, Blue reward:0.85\n", - "step: 2165, Red action: do-nothing, Blue reward:0.85\n", - "step: 2166, Red action: do-nothing, Blue reward:0.85\n", - "step: 2167, Red action: do-nothing, Blue reward:0.85\n", - "step: 2168, Red action: do-nothing, Blue reward:0.85\n", - "step: 2169, Red action: do-nothing, Blue reward:0.85\n", - "step: 2170, Red action: do-nothing, Blue reward:0.85\n", - "step: 2171, Red action: do-nothing, Blue reward:0.85\n", - "step: 2172, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2173, Red action: do-nothing, Blue reward:0.85\n", - "step: 2174, Red action: do-nothing, Blue reward:0.85\n", - "step: 2175, Red action: do-nothing, Blue reward:0.85\n", - "step: 2176, Red action: do-nothing, Blue reward:0.85\n", - "step: 2177, Red action: do-nothing, Blue reward:0.85\n", - "step: 2178, Red action: do-nothing, Blue reward:0.85\n", - "step: 2179, Red action: do-nothing, Blue reward:0.85\n", - "step: 2180, Red action: do-nothing, Blue reward:0.85\n", - "step: 2181, Red action: do-nothing, Blue reward:0.85\n", - "step: 2182, Red action: do-nothing, Blue reward:0.85\n", - "step: 2183, Red action: do-nothing, Blue reward:0.85\n", - "step: 2184, Red action: do-nothing, Blue reward:0.85\n", - "step: 2185, Red action: do-nothing, Blue reward:0.85\n", - "step: 2186, Red action: do-nothing, Blue reward:0.85\n", - "step: 2187, Red action: do-nothing, Blue reward:0.85\n", - "step: 2188, Red action: do-nothing, Blue reward:0.85\n", - "step: 2189, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2190, Red action: do-nothing, Blue reward:0.85\n", - "step: 2191, Red action: do-nothing, Blue reward:0.85\n", - "step: 2192, Red action: do-nothing, Blue reward:0.85\n", - "step: 2193, Red action: do-nothing, Blue reward:0.85\n", - "step: 2194, Red action: do-nothing, Blue reward:0.85\n", - "step: 2195, Red action: do-nothing, Blue reward:0.85\n", - "step: 2196, Red action: do-nothing, Blue reward:0.85\n", - "step: 2197, Red action: do-nothing, Blue reward:0.85\n", - "step: 2198, Red action: do-nothing, Blue reward:0.85\n", - "step: 2199, Red action: do-nothing, Blue reward:0.85\n", - "step: 2200, Red action: do-nothing, Blue reward:0.85\n", - "step: 2201, Red action: do-nothing, Blue reward:0.85\n", - "step: 2202, Red action: do-nothing, Blue reward:0.85\n", - "step: 2203, Red action: do-nothing, Blue reward:0.85\n", - "step: 2204, Red action: do-nothing, Blue reward:0.85\n", - "step: 2205, Red action: do-nothing, Blue reward:0.85\n", - "step: 2206, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2207, Red action: do-nothing, Blue reward:0.85\n", - "step: 2208, Red action: do-nothing, Blue reward:0.85\n", - "step: 2209, Red action: do-nothing, Blue reward:0.85\n", - "step: 2210, Red action: do-nothing, Blue reward:0.85\n", - "step: 2211, Red action: do-nothing, Blue reward:0.85\n", - "step: 2212, Red action: do-nothing, Blue reward:0.85\n", - "step: 2213, Red action: do-nothing, Blue reward:0.85\n", - "step: 2214, Red action: do-nothing, Blue reward:0.85\n", - "step: 2215, Red action: do-nothing, Blue reward:0.85\n", - "step: 2216, Red action: do-nothing, Blue reward:0.85\n", - "step: 2217, Red action: do-nothing, Blue reward:0.85\n", - "step: 2218, Red action: do-nothing, Blue reward:0.85\n", - "step: 2219, Red action: do-nothing, Blue reward:0.85\n", - "step: 2220, Red action: do-nothing, Blue reward:0.85\n", - "step: 2221, Red action: do-nothing, Blue reward:0.85\n", - "step: 2222, Red action: do-nothing, Blue reward:0.85\n", - "step: 2223, Red action: do-nothing, Blue reward:0.85\n", - "step: 2224, Red action: do-nothing, Blue reward:0.85\n", - "step: 2225, Red action: do-nothing, Blue reward:0.85\n", - "step: 2226, Red action: do-nothing, Blue reward:0.85\n", - "step: 2227, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2228, Red action: do-nothing, Blue reward:0.85\n", - "step: 2229, Red action: do-nothing, Blue reward:0.85\n", - "step: 2230, Red action: do-nothing, Blue reward:0.85\n", - "step: 2231, Red action: do-nothing, Blue reward:0.85\n", - "step: 2232, Red action: do-nothing, Blue reward:0.85\n", - "step: 2233, Red action: do-nothing, Blue reward:0.85\n", - "step: 2234, Red action: do-nothing, Blue reward:0.85\n", - "step: 2235, Red action: do-nothing, Blue reward:0.85\n", - "step: 2236, Red action: do-nothing, Blue reward:0.85\n", - "step: 2237, Red action: do-nothing, Blue reward:0.85\n", - "step: 2238, Red action: do-nothing, Blue reward:0.85\n", - "step: 2239, Red action: do-nothing, Blue reward:0.85\n", - "step: 2240, Red action: do-nothing, Blue reward:0.85\n", - "step: 2241, Red action: do-nothing, Blue reward:0.85\n", - "step: 2242, Red action: do-nothing, Blue reward:0.85\n", - "step: 2243, Red action: do-nothing, Blue reward:0.85\n", - "step: 2244, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2245, Red action: do-nothing, Blue reward:0.85\n", - "step: 2246, Red action: do-nothing, Blue reward:0.85\n", - "step: 2247, Red action: do-nothing, Blue reward:0.85\n", - "step: 2248, Red action: do-nothing, Blue reward:0.85\n", - "step: 2249, Red action: do-nothing, Blue reward:0.85\n", - "step: 2250, Red action: do-nothing, Blue reward:0.85\n", - "step: 2251, Red action: do-nothing, Blue reward:0.85\n", - "step: 2252, Red action: do-nothing, Blue reward:0.85\n", - "step: 2253, Red action: do-nothing, Blue reward:0.85\n", - "step: 2254, Red action: do-nothing, Blue reward:0.85\n", - "step: 2255, Red action: do-nothing, Blue reward:0.85\n", - "step: 2256, Red action: do-nothing, Blue reward:0.85\n", - "step: 2257, Red action: do-nothing, Blue reward:0.85\n", - "step: 2258, Red action: do-nothing, Blue reward:0.85\n", - "step: 2259, Red action: do-nothing, Blue reward:0.85\n", - "step: 2260, Red action: do-nothing, Blue reward:0.85\n", - "step: 2261, Red action: do-nothing, Blue reward:0.85\n", - "step: 2262, Red action: do-nothing, Blue reward:0.85\n", - "step: 2263, Red action: do-nothing, Blue reward:0.85\n", - "step: 2264, Red action: do-nothing, Blue reward:0.85\n", - "step: 2265, Red action: do-nothing, Blue reward:0.85\n", - "step: 2266, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2267, Red action: do-nothing, Blue reward:0.85\n", - "step: 2268, Red action: do-nothing, Blue reward:0.85\n", - "step: 2269, Red action: do-nothing, Blue reward:0.85\n", - "step: 2270, Red action: do-nothing, Blue reward:0.85\n", - "step: 2271, Red action: do-nothing, Blue reward:0.85\n", - "step: 2272, Red action: do-nothing, Blue reward:0.85\n", - "step: 2273, Red action: do-nothing, Blue reward:0.85\n", - "step: 2274, Red action: do-nothing, Blue reward:0.85\n", - "step: 2275, Red action: do-nothing, Blue reward:0.85\n", - "step: 2276, Red action: do-nothing, Blue reward:0.85\n", - "step: 2277, Red action: do-nothing, Blue reward:0.85\n", - "step: 2278, Red action: do-nothing, Blue reward:0.85\n", - "step: 2279, Red action: do-nothing, Blue reward:0.85\n", - "step: 2280, Red action: do-nothing, Blue reward:0.85\n", - "step: 2281, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2282, Red action: do-nothing, Blue reward:0.85\n", - "step: 2283, Red action: do-nothing, Blue reward:0.85\n", - "step: 2284, Red action: do-nothing, Blue reward:0.85\n", - "step: 2285, Red action: do-nothing, Blue reward:0.85\n", - "step: 2286, Red action: do-nothing, Blue reward:0.85\n", - "step: 2287, Red action: do-nothing, Blue reward:0.85\n", - "step: 2288, Red action: do-nothing, Blue reward:0.85\n", - "step: 2289, Red action: do-nothing, Blue reward:0.85\n", - "step: 2290, Red action: do-nothing, Blue reward:0.85\n", - "step: 2291, Red action: do-nothing, Blue reward:0.85\n", - "step: 2292, Red action: do-nothing, Blue reward:0.85\n", - "step: 2293, Red action: do-nothing, Blue reward:0.85\n", - "step: 2294, Red action: do-nothing, Blue reward:0.85\n", - "step: 2295, Red action: do-nothing, Blue reward:0.85\n", - "step: 2296, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2297, Red action: do-nothing, Blue reward:0.85\n", - "step: 2298, Red action: do-nothing, Blue reward:0.85\n", - "step: 2299, Red action: do-nothing, Blue reward:0.85\n", - "step: 2300, Red action: do-nothing, Blue reward:0.85\n", - "step: 2301, Red action: do-nothing, Blue reward:0.85\n", - "step: 2302, Red action: do-nothing, Blue reward:0.85\n", - "step: 2303, Red action: do-nothing, Blue reward:0.85\n", - "step: 2304, Red action: do-nothing, Blue reward:0.85\n", - "step: 2305, Red action: do-nothing, Blue reward:0.85\n", - "step: 2306, Red action: do-nothing, Blue reward:0.85\n", - "step: 2307, Red action: do-nothing, Blue reward:0.85\n", - "step: 2308, Red action: do-nothing, Blue reward:0.85\n", - "step: 2309, Red action: do-nothing, Blue reward:0.85\n", - "step: 2310, Red action: do-nothing, Blue reward:0.85\n", - "step: 2311, Red action: do-nothing, Blue reward:0.85\n", - "step: 2312, Red action: do-nothing, Blue reward:0.85\n", - "step: 2313, Red action: do-nothing, Blue reward:0.85\n", - "step: 2314, Red action: do-nothing, Blue reward:0.85\n", - "step: 2315, Red action: do-nothing, Blue reward:0.85\n", - "step: 2316, Red action: do-nothing, Blue reward:0.85\n", - "step: 2317, Red action: do-nothing, Blue reward:0.85\n", - "step: 2318, Red action: do-nothing, Blue reward:0.85\n", - "step: 2319, Red action: do-nothing, Blue reward:0.85\n", - "step: 2320, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2321, Red action: do-nothing, Blue reward:0.85\n", - "step: 2322, Red action: do-nothing, Blue reward:0.85\n", - "step: 2323, Red action: do-nothing, Blue reward:0.85\n", - "step: 2324, Red action: do-nothing, Blue reward:0.85\n", - "step: 2325, Red action: do-nothing, Blue reward:0.85\n", - "step: 2326, Red action: do-nothing, Blue reward:0.85\n", - "step: 2327, Red action: do-nothing, Blue reward:0.85\n", - "step: 2328, Red action: do-nothing, Blue reward:0.85\n", - "step: 2329, Red action: do-nothing, Blue reward:0.85\n", - "step: 2330, Red action: do-nothing, Blue reward:0.85\n", - "step: 2331, Red action: do-nothing, Blue reward:0.85\n", - "step: 2332, Red action: do-nothing, Blue reward:0.85\n", - "step: 2333, Red action: do-nothing, Blue reward:0.85\n", - "step: 2334, Red action: do-nothing, Blue reward:0.85\n", - "step: 2335, Red action: do-nothing, Blue reward:0.85\n", - "step: 2336, Red action: do-nothing, Blue reward:0.85\n", - "step: 2337, Red action: do-nothing, Blue reward:0.85\n", - "step: 2338, Red action: do-nothing, Blue reward:0.85\n", - "step: 2339, Red action: do-nothing, Blue reward:0.85\n", - "step: 2340, Red action: do-nothing, Blue reward:0.85\n", - "step: 2341, Red action: do-nothing, Blue reward:0.85\n", - "step: 2342, Red action: do-nothing, Blue reward:0.85\n", - "step: 2343, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2344, Red action: do-nothing, Blue reward:0.85\n", - "step: 2345, Red action: do-nothing, Blue reward:0.85\n", - "step: 2346, Red action: do-nothing, Blue reward:0.85\n", - "step: 2347, Red action: do-nothing, Blue reward:0.85\n", - "step: 2348, Red action: do-nothing, Blue reward:0.85\n", - "step: 2349, Red action: do-nothing, Blue reward:0.85\n", - "step: 2350, Red action: do-nothing, Blue reward:0.85\n", - "step: 2351, Red action: do-nothing, Blue reward:0.85\n", - "step: 2352, Red action: do-nothing, Blue reward:0.85\n", - "step: 2353, Red action: do-nothing, Blue reward:0.85\n", - "step: 2354, Red action: do-nothing, Blue reward:0.85\n", - "step: 2355, Red action: do-nothing, Blue reward:0.85\n", - "step: 2356, Red action: do-nothing, Blue reward:0.85\n", - "step: 2357, Red action: do-nothing, Blue reward:0.85\n", - "step: 2358, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2359, Red action: do-nothing, Blue reward:0.85\n", - "step: 2360, Red action: do-nothing, Blue reward:0.85\n", - "step: 2361, Red action: do-nothing, Blue reward:0.85\n", - "step: 2362, Red action: do-nothing, Blue reward:0.85\n", - "step: 2363, Red action: do-nothing, Blue reward:0.85\n", - "step: 2364, Red action: do-nothing, Blue reward:0.85\n", - "step: 2365, Red action: do-nothing, Blue reward:0.85\n", - "step: 2366, Red action: do-nothing, Blue reward:0.85\n", - "step: 2367, Red action: do-nothing, Blue reward:0.85\n", - "step: 2368, Red action: do-nothing, Blue reward:0.85\n", - "step: 2369, Red action: do-nothing, Blue reward:0.85\n", - "step: 2370, Red action: do-nothing, Blue reward:0.85\n", - "step: 2371, Red action: do-nothing, Blue reward:0.85\n", - "step: 2372, Red action: do-nothing, Blue reward:0.85\n", - "step: 2373, Red action: do-nothing, Blue reward:0.85\n", - "step: 2374, Red action: do-nothing, Blue reward:0.85\n", - "step: 2375, Red action: do-nothing, Blue reward:0.85\n", - "step: 2376, Red action: do-nothing, Blue reward:0.85\n", - "step: 2377, Red action: do-nothing, Blue reward:0.85\n", - "step: 2378, Red action: do-nothing, Blue reward:0.85\n", - "step: 2379, Red action: do-nothing, Blue reward:0.85\n", - "step: 2380, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2381, Red action: do-nothing, Blue reward:0.85\n", - "step: 2382, Red action: do-nothing, Blue reward:0.85\n", - "step: 2383, Red action: do-nothing, Blue reward:0.85\n", - "step: 2384, Red action: do-nothing, Blue reward:0.85\n", - "step: 2385, Red action: do-nothing, Blue reward:0.85\n", - "step: 2386, Red action: do-nothing, Blue reward:0.85\n", - "step: 2387, Red action: do-nothing, Blue reward:0.85\n", - "step: 2388, Red action: do-nothing, Blue reward:0.85\n", - "step: 2389, Red action: do-nothing, Blue reward:0.85\n", - "step: 2390, Red action: do-nothing, Blue reward:0.85\n", - "step: 2391, Red action: do-nothing, Blue reward:0.85\n", - "step: 2392, Red action: do-nothing, Blue reward:0.85\n", - "step: 2393, Red action: do-nothing, Blue reward:0.85\n", - "step: 2394, Red action: do-nothing, Blue reward:0.85\n", - "step: 2395, Red action: do-nothing, Blue reward:0.85\n", - "step: 2396, Red action: do-nothing, Blue reward:0.85\n", - "step: 2397, Red action: do-nothing, Blue reward:0.85\n", - "step: 2398, Red action: do-nothing, Blue reward:0.85\n", - "step: 2399, Red action: do-nothing, Blue reward:0.85\n", - "step: 2400, Red action: do-nothing, Blue reward:0.85\n", - "step: 2401, Red action: do-nothing, Blue reward:0.85\n", - "step: 2402, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2403, Red action: do-nothing, Blue reward:0.85\n", - "step: 2404, Red action: do-nothing, Blue reward:0.85\n", - "step: 2405, Red action: do-nothing, Blue reward:0.85\n", - "step: 2406, Red action: do-nothing, Blue reward:0.85\n", - "step: 2407, Red action: do-nothing, Blue reward:0.85\n", - "step: 2408, Red action: do-nothing, Blue reward:0.85\n", - "step: 2409, Red action: do-nothing, Blue reward:0.85\n", - "step: 2410, Red action: do-nothing, Blue reward:0.85\n", - "step: 2411, Red action: do-nothing, Blue reward:0.85\n", - "step: 2412, Red action: do-nothing, Blue reward:0.85\n", - "step: 2413, Red action: do-nothing, Blue reward:0.85\n", - "step: 2414, Red action: do-nothing, Blue reward:0.85\n", - "step: 2415, Red action: do-nothing, Blue reward:0.85\n", - "step: 2416, Red action: do-nothing, Blue reward:0.85\n", - "step: 2417, Red action: do-nothing, Blue reward:0.85\n", - "step: 2418, Red action: do-nothing, Blue reward:0.85\n", - "step: 2419, Red action: do-nothing, Blue reward:0.85\n", - "step: 2420, Red action: do-nothing, Blue reward:0.85\n", - "step: 2421, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2422, Red action: do-nothing, Blue reward:0.85\n", - "step: 2423, Red action: do-nothing, Blue reward:0.85\n", - "step: 2424, Red action: do-nothing, Blue reward:0.85\n", - "step: 2425, Red action: do-nothing, Blue reward:0.85\n", - "step: 2426, Red action: do-nothing, Blue reward:0.85\n", - "step: 2427, Red action: do-nothing, Blue reward:0.85\n", - "step: 2428, Red action: do-nothing, Blue reward:0.85\n", - "step: 2429, Red action: do-nothing, Blue reward:0.85\n", - "step: 2430, Red action: do-nothing, Blue reward:0.85\n", - "step: 2431, Red action: do-nothing, Blue reward:0.85\n", - "step: 2432, Red action: do-nothing, Blue reward:0.85\n", - "step: 2433, Red action: do-nothing, Blue reward:0.85\n", - "step: 2434, Red action: do-nothing, Blue reward:0.85\n", - "step: 2435, Red action: do-nothing, Blue reward:0.85\n", - "step: 2436, Red action: do-nothing, Blue reward:0.85\n", - "step: 2437, Red action: do-nothing, Blue reward:0.85\n", - "step: 2438, Red action: do-nothing, Blue reward:0.85\n", - "step: 2439, Red action: do-nothing, Blue reward:0.85\n", - "step: 2440, Red action: do-nothing, Blue reward:0.85\n", - "step: 2441, Red action: do-nothing, Blue reward:0.85\n", - "step: 2442, Red action: do-nothing, Blue reward:0.85\n", - "step: 2443, Red action: do-nothing, Blue reward:0.85\n", - "step: 2444, Red action: do-nothing, Blue reward:0.85\n", - "step: 2445, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2446, Red action: do-nothing, Blue reward:0.85\n", - "step: 2447, Red action: do-nothing, Blue reward:0.85\n", - "step: 2448, Red action: do-nothing, Blue reward:0.85\n", - "step: 2449, Red action: do-nothing, Blue reward:0.85\n", - "step: 2450, Red action: do-nothing, Blue reward:0.85\n", - "step: 2451, Red action: do-nothing, Blue reward:0.85\n", - "step: 2452, Red action: do-nothing, Blue reward:0.85\n", - "step: 2453, Red action: do-nothing, Blue reward:0.85\n", - "step: 2454, Red action: do-nothing, Blue reward:0.85\n", - "step: 2455, Red action: do-nothing, Blue reward:0.85\n", - "step: 2456, Red action: do-nothing, Blue reward:0.85\n", - "step: 2457, Red action: do-nothing, Blue reward:0.85\n", - "step: 2458, Red action: do-nothing, Blue reward:0.85\n", - "step: 2459, Red action: do-nothing, Blue reward:0.85\n", - "step: 2460, Red action: do-nothing, Blue reward:0.85\n", - "step: 2461, Red action: do-nothing, Blue reward:0.85\n", - "step: 2462, Red action: do-nothing, Blue reward:0.85\n", - "step: 2463, Red action: do-nothing, Blue reward:0.85\n", - "step: 2464, Red action: do-nothing, Blue reward:0.85\n", - "step: 2465, Red action: do-nothing, Blue reward:0.85\n", - "step: 2466, Red action: do-nothing, Blue reward:0.85\n", - "step: 2467, Red action: do-nothing, Blue reward:0.85\n", - "step: 2468, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2469, Red action: do-nothing, Blue reward:0.85\n", - "step: 2470, Red action: do-nothing, Blue reward:0.85\n", - "step: 2471, Red action: do-nothing, Blue reward:0.85\n", - "step: 2472, Red action: do-nothing, Blue reward:0.85\n", - "step: 2473, Red action: do-nothing, Blue reward:0.85\n", - "step: 2474, Red action: do-nothing, Blue reward:0.85\n", - "step: 2475, Red action: do-nothing, Blue reward:0.85\n", - "step: 2476, Red action: do-nothing, Blue reward:0.85\n", - "step: 2477, Red action: do-nothing, Blue reward:0.85\n", - "step: 2478, Red action: do-nothing, Blue reward:0.85\n", - "step: 2479, Red action: do-nothing, Blue reward:0.85\n", - "step: 2480, Red action: do-nothing, Blue reward:0.85\n", - "step: 2481, Red action: do-nothing, Blue reward:0.85\n", - "step: 2482, Red action: do-nothing, Blue reward:0.85\n", - "step: 2483, Red action: do-nothing, Blue reward:0.85\n", - "step: 2484, Red action: do-nothing, Blue reward:0.85\n", - "step: 2485, Red action: do-nothing, Blue reward:0.85\n", - "step: 2486, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2487, Red action: do-nothing, Blue reward:0.85\n", - "step: 2488, Red action: do-nothing, Blue reward:0.85\n", - "step: 2489, Red action: do-nothing, Blue reward:0.85\n", - "step: 2490, Red action: do-nothing, Blue reward:0.85\n", - "step: 2491, Red action: do-nothing, Blue reward:0.85\n", - "step: 2492, Red action: do-nothing, Blue reward:0.85\n", - "step: 2493, Red action: do-nothing, Blue reward:0.85\n", - "step: 2494, Red action: do-nothing, Blue reward:0.85\n", - "step: 2495, Red action: do-nothing, Blue reward:0.85\n", - "step: 2496, Red action: do-nothing, Blue reward:0.85\n", - "step: 2497, Red action: do-nothing, Blue reward:0.85\n", - "step: 2498, Red action: do-nothing, Blue reward:0.85\n", - "step: 2499, Red action: do-nothing, Blue reward:0.85\n", - "step: 2500, Red action: do-nothing, Blue reward:0.85\n", - "step: 2501, Red action: do-nothing, Blue reward:0.85\n", - "step: 2502, Red action: do-nothing, Blue reward:0.85\n", - "step: 2503, Red action: do-nothing, Blue reward:0.85\n", - "step: 2504, Red action: do-nothing, Blue reward:0.85\n", - "step: 2505, Red action: do-nothing, Blue reward:0.85\n", - "step: 2506, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2507, Red action: do-nothing, Blue reward:0.85\n", - "step: 2508, Red action: do-nothing, Blue reward:0.85\n", - "step: 2509, Red action: do-nothing, Blue reward:0.85\n", - "step: 2510, Red action: do-nothing, Blue reward:0.85\n", - "step: 2511, Red action: do-nothing, Blue reward:0.85\n", - "step: 2512, Red action: do-nothing, Blue reward:0.85\n", - "step: 2513, Red action: do-nothing, Blue reward:0.85\n", - "step: 2514, Red action: do-nothing, Blue reward:0.85\n", - "step: 2515, Red action: do-nothing, Blue reward:0.85\n", - "step: 2516, Red action: do-nothing, Blue reward:0.85\n", - "step: 2517, Red action: do-nothing, Blue reward:0.85\n", - "step: 2518, Red action: do-nothing, Blue reward:0.85\n", - "step: 2519, Red action: do-nothing, Blue reward:0.85\n", - "step: 2520, Red action: do-nothing, Blue reward:0.85\n", - "step: 2521, Red action: do-nothing, Blue reward:0.85\n", - "step: 2522, Red action: do-nothing, Blue reward:0.85\n", - "step: 2523, Red action: do-nothing, Blue reward:0.85\n", - "step: 2524, Red action: do-nothing, Blue reward:0.85\n", - "step: 2525, Red action: do-nothing, Blue reward:0.85\n", - "step: 2526, Red action: do-nothing, Blue reward:0.85\n", - "step: 2527, Red action: do-nothing, Blue reward:0.85\n", - "step: 2528, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2529, Red action: do-nothing, Blue reward:0.85\n", - "step: 2530, Red action: do-nothing, Blue reward:0.85\n", - "step: 2531, Red action: do-nothing, Blue reward:0.85\n", - "step: 2532, Red action: do-nothing, Blue reward:0.85\n", - "step: 2533, Red action: do-nothing, Blue reward:0.85\n", - "step: 2534, Red action: do-nothing, Blue reward:0.85\n", - "step: 2535, Red action: do-nothing, Blue reward:0.85\n", - "step: 2536, Red action: do-nothing, Blue reward:0.85\n", - "step: 2537, Red action: do-nothing, Blue reward:0.85\n", - "step: 2538, Red action: do-nothing, Blue reward:0.85\n", - "step: 2539, Red action: do-nothing, Blue reward:0.85\n", - "step: 2540, Red action: do-nothing, Blue reward:0.85\n", - "step: 2541, Red action: do-nothing, Blue reward:0.85\n", - "step: 2542, Red action: do-nothing, Blue reward:0.85\n", - "step: 2543, Red action: do-nothing, Blue reward:0.85\n", - "step: 2544, Red action: do-nothing, Blue reward:0.85\n", - "step: 2545, Red action: do-nothing, Blue reward:0.85\n", - "step: 2546, Red action: do-nothing, Blue reward:0.85\n", - "step: 2547, Red action: do-nothing, Blue reward:0.85\n", - "step: 2548, Red action: do-nothing, Blue reward:0.85\n", - "step: 2549, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2550, Red action: do-nothing, Blue reward:0.85\n", - "step: 2551, Red action: do-nothing, Blue reward:0.85\n", - "step: 2552, Red action: do-nothing, Blue reward:0.85\n", - "step: 2553, Red action: do-nothing, Blue reward:0.85\n", - "step: 2554, Red action: do-nothing, Blue reward:0.85\n", - "step: 2555, Red action: do-nothing, Blue reward:0.85\n", - "step: 2556, Red action: do-nothing, Blue reward:0.85\n", - "step: 2557, Red action: do-nothing, Blue reward:0.85\n", - "step: 2558, Red action: do-nothing, Blue reward:0.85\n", - "step: 2559, Red action: do-nothing, Blue reward:0.85\n", - "step: 2560, Red action: do-nothing, Blue reward:0.85\n", - "step: 2561, Red action: do-nothing, Blue reward:0.85\n", - "step: 2562, Red action: do-nothing, Blue reward:0.85\n", - "step: 2563, Red action: do-nothing, Blue reward:0.85\n", - "step: 2564, Red action: do-nothing, Blue reward:0.85\n", - "step: 2565, Red action: do-nothing, Blue reward:0.85\n", - "step: 2566, Red action: do-nothing, Blue reward:0.85\n", - "step: 2567, Red action: do-nothing, Blue reward:0.85\n", - "step: 2568, Red action: do-nothing, Blue reward:0.85\n", - "step: 2569, Red action: do-nothing, Blue reward:0.85\n", - "step: 2570, Red action: do-nothing, Blue reward:0.85\n", - "step: 2571, Red action: do-nothing, Blue reward:0.85\n", - "step: 2572, Red action: do-nothing, Blue reward:0.85\n", - "step: 2573, Red action: do-nothing, Blue reward:0.85\n", - "step: 2574, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2575, Red action: do-nothing, Blue reward:0.85\n", - "step: 2576, Red action: do-nothing, Blue reward:0.85\n", - "step: 2577, Red action: do-nothing, Blue reward:0.85\n", - "step: 2578, Red action: do-nothing, Blue reward:0.85\n", - "step: 2579, Red action: do-nothing, Blue reward:0.85\n", - "step: 2580, Red action: do-nothing, Blue reward:0.85\n", - "step: 2581, Red action: do-nothing, Blue reward:0.85\n", - "step: 2582, Red action: do-nothing, Blue reward:0.85\n", - "step: 2583, Red action: do-nothing, Blue reward:0.85\n", - "step: 2584, Red action: do-nothing, Blue reward:0.85\n", - "step: 2585, Red action: do-nothing, Blue reward:0.85\n", - "step: 2586, Red action: do-nothing, Blue reward:0.85\n", - "step: 2587, Red action: do-nothing, Blue reward:0.85\n", - "step: 2588, Red action: do-nothing, Blue reward:0.85\n", - "step: 2589, Red action: do-nothing, Blue reward:0.85\n", - "step: 2590, Red action: do-nothing, Blue reward:0.85\n", - "step: 2591, Red action: do-nothing, Blue reward:0.85\n", - "step: 2592, Red action: do-nothing, Blue reward:0.85\n", - "step: 2593, Red action: do-nothing, Blue reward:0.85\n", - "step: 2594, Red action: do-nothing, Blue reward:0.85\n", - "step: 2595, Red action: do-nothing, Blue reward:0.85\n", - "step: 2596, Red action: do-nothing, Blue reward:0.85\n", - "step: 2597, Red action: do-nothing, Blue reward:0.85\n", - "step: 2598, Red action: do-nothing, Blue reward:0.85\n", - "step: 2599, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2600, Red action: do-nothing, Blue reward:0.85\n", - "step: 2601, Red action: do-nothing, Blue reward:0.85\n", - "step: 2602, Red action: do-nothing, Blue reward:0.85\n", - "step: 2603, Red action: do-nothing, Blue reward:0.85\n", - "step: 2604, Red action: do-nothing, Blue reward:0.85\n", - "step: 2605, Red action: do-nothing, Blue reward:0.85\n", - "step: 2606, Red action: do-nothing, Blue reward:0.85\n", - "step: 2607, Red action: do-nothing, Blue reward:0.85\n", - "step: 2608, Red action: do-nothing, Blue reward:0.85\n", - "step: 2609, Red action: do-nothing, Blue reward:0.85\n", - "step: 2610, Red action: do-nothing, Blue reward:0.85\n", - "step: 2611, Red action: do-nothing, Blue reward:0.85\n", - "step: 2612, Red action: do-nothing, Blue reward:0.85\n", - "step: 2613, Red action: do-nothing, Blue reward:0.85\n", - "step: 2614, Red action: do-nothing, Blue reward:0.85\n", - "step: 2615, Red action: do-nothing, Blue reward:0.85\n", - "step: 2616, Red action: do-nothing, Blue reward:0.85\n", - "step: 2617, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2618, Red action: do-nothing, Blue reward:0.85\n", - "step: 2619, Red action: do-nothing, Blue reward:0.85\n", - "step: 2620, Red action: do-nothing, Blue reward:0.85\n", - "step: 2621, Red action: do-nothing, Blue reward:0.85\n", - "step: 2622, Red action: do-nothing, Blue reward:0.85\n", - "step: 2623, Red action: do-nothing, Blue reward:0.85\n", - "step: 2624, Red action: do-nothing, Blue reward:0.85\n", - "step: 2625, Red action: do-nothing, Blue reward:0.85\n", - "step: 2626, Red action: do-nothing, Blue reward:0.85\n", - "step: 2627, Red action: do-nothing, Blue reward:0.85\n", - "step: 2628, Red action: do-nothing, Blue reward:0.85\n", - "step: 2629, Red action: do-nothing, Blue reward:0.85\n", - "step: 2630, Red action: do-nothing, Blue reward:0.85\n", - "step: 2631, Red action: do-nothing, Blue reward:0.85\n", - "step: 2632, Red action: do-nothing, Blue reward:0.85\n", - "step: 2633, Red action: do-nothing, Blue reward:0.85\n", - "step: 2634, Red action: do-nothing, Blue reward:0.85\n", - "step: 2635, Red action: do-nothing, Blue reward:0.85\n", - "step: 2636, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2637, Red action: do-nothing, Blue reward:0.85\n", - "step: 2638, Red action: do-nothing, Blue reward:0.85\n", - "step: 2639, Red action: do-nothing, Blue reward:0.85\n", - "step: 2640, Red action: do-nothing, Blue reward:0.85\n", - "step: 2641, Red action: do-nothing, Blue reward:0.85\n", - "step: 2642, Red action: do-nothing, Blue reward:0.85\n", - "step: 2643, Red action: do-nothing, Blue reward:0.85\n", - "step: 2644, Red action: do-nothing, Blue reward:0.85\n", - "step: 2645, Red action: do-nothing, Blue reward:0.85\n", - "step: 2646, Red action: do-nothing, Blue reward:0.85\n", - "step: 2647, Red action: do-nothing, Blue reward:0.85\n", - "step: 2648, Red action: do-nothing, Blue reward:0.85\n", - "step: 2649, Red action: do-nothing, Blue reward:0.85\n", - "step: 2650, Red action: do-nothing, Blue reward:0.85\n", - "step: 2651, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2652, Red action: do-nothing, Blue reward:0.85\n", - "step: 2653, Red action: do-nothing, Blue reward:0.85\n", - "step: 2654, Red action: do-nothing, Blue reward:0.85\n", - "step: 2655, Red action: do-nothing, Blue reward:0.85\n", - "step: 2656, Red action: do-nothing, Blue reward:0.85\n", - "step: 2657, Red action: do-nothing, Blue reward:0.85\n", - "step: 2658, Red action: do-nothing, Blue reward:0.85\n", - "step: 2659, Red action: do-nothing, Blue reward:0.85\n", - "step: 2660, Red action: do-nothing, Blue reward:0.85\n", - "step: 2661, Red action: do-nothing, Blue reward:0.85\n", - "step: 2662, Red action: do-nothing, Blue reward:0.85\n", - "step: 2663, Red action: do-nothing, Blue reward:0.85\n", - "step: 2664, Red action: do-nothing, Blue reward:0.85\n", - "step: 2665, Red action: do-nothing, Blue reward:0.85\n", - "step: 2666, Red action: do-nothing, Blue reward:0.85\n", - "step: 2667, Red action: do-nothing, Blue reward:0.85\n", - "step: 2668, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2669, Red action: do-nothing, Blue reward:0.85\n", - "step: 2670, Red action: do-nothing, Blue reward:0.85\n", - "step: 2671, Red action: do-nothing, Blue reward:0.85\n", - "step: 2672, Red action: do-nothing, Blue reward:0.85\n", - "step: 2673, Red action: do-nothing, Blue reward:0.85\n", - "step: 2674, Red action: do-nothing, Blue reward:0.85\n", - "step: 2675, Red action: do-nothing, Blue reward:0.85\n", - "step: 2676, Red action: do-nothing, Blue reward:0.85\n", - "step: 2677, Red action: do-nothing, Blue reward:0.85\n", - "step: 2678, Red action: do-nothing, Blue reward:0.85\n", - "step: 2679, Red action: do-nothing, Blue reward:0.85\n", - "step: 2680, Red action: do-nothing, Blue reward:0.85\n", - "step: 2681, Red action: do-nothing, Blue reward:0.85\n", - "step: 2682, Red action: do-nothing, Blue reward:0.85\n", - "step: 2683, Red action: do-nothing, Blue reward:0.85\n", - "step: 2684, Red action: do-nothing, Blue reward:0.85\n", - "step: 2685, Red action: do-nothing, Blue reward:0.85\n", - "step: 2686, Red action: do-nothing, Blue reward:0.85\n", - "step: 2687, Red action: do-nothing, Blue reward:0.85\n", - "step: 2688, Red action: do-nothing, Blue reward:0.85\n", - "step: 2689, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2690, Red action: do-nothing, Blue reward:0.85\n", - "step: 2691, Red action: do-nothing, Blue reward:0.85\n", - "step: 2692, Red action: do-nothing, Blue reward:0.85\n", - "step: 2693, Red action: do-nothing, Blue reward:0.85\n", - "step: 2694, Red action: do-nothing, Blue reward:0.85\n", - "step: 2695, Red action: do-nothing, Blue reward:0.85\n", - "step: 2696, Red action: do-nothing, Blue reward:0.85\n", - "step: 2697, Red action: do-nothing, Blue reward:0.85\n", - "step: 2698, Red action: do-nothing, Blue reward:0.85\n", - "step: 2699, Red action: do-nothing, Blue reward:0.85\n", - "step: 2700, Red action: do-nothing, Blue reward:0.85\n", - "step: 2701, Red action: do-nothing, Blue reward:0.85\n", - "step: 2702, Red action: do-nothing, Blue reward:0.85\n", - "step: 2703, Red action: do-nothing, Blue reward:0.85\n", - "step: 2704, Red action: do-nothing, Blue reward:0.85\n", - "step: 2705, Red action: do-nothing, Blue reward:0.85\n", - "step: 2706, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2707, Red action: do-nothing, Blue reward:0.85\n", - "step: 2708, Red action: do-nothing, Blue reward:0.85\n", - "step: 2709, Red action: do-nothing, Blue reward:0.85\n", - "step: 2710, Red action: do-nothing, Blue reward:0.85\n", - "step: 2711, Red action: do-nothing, Blue reward:0.85\n", - "step: 2712, Red action: do-nothing, Blue reward:0.85\n", - "step: 2713, Red action: do-nothing, Blue reward:0.85\n", - "step: 2714, Red action: do-nothing, Blue reward:0.85\n", - "step: 2715, Red action: do-nothing, Blue reward:0.85\n", - "step: 2716, Red action: do-nothing, Blue reward:0.85\n", - "step: 2717, Red action: do-nothing, Blue reward:0.85\n", - "step: 2718, Red action: do-nothing, Blue reward:0.85\n", - "step: 2719, Red action: do-nothing, Blue reward:0.85\n", - "step: 2720, Red action: do-nothing, Blue reward:0.85\n", - "step: 2721, Red action: do-nothing, Blue reward:0.85\n", - "step: 2722, Red action: do-nothing, Blue reward:0.85\n", - "step: 2723, Red action: do-nothing, Blue reward:0.85\n", - "step: 2724, Red action: do-nothing, Blue reward:0.85\n", - "step: 2725, Red action: do-nothing, Blue reward:0.85\n", - "step: 2726, Red action: do-nothing, Blue reward:0.85\n", - "step: 2727, Red action: do-nothing, Blue reward:0.85\n", - "step: 2728, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2729, Red action: do-nothing, Blue reward:0.85\n", - "step: 2730, Red action: do-nothing, Blue reward:0.85\n", - "step: 2731, Red action: do-nothing, Blue reward:0.85\n", - "step: 2732, Red action: do-nothing, Blue reward:0.85\n", - "step: 2733, Red action: do-nothing, Blue reward:0.85\n", - "step: 2734, Red action: do-nothing, Blue reward:0.85\n", - "step: 2735, Red action: do-nothing, Blue reward:0.85\n", - "step: 2736, Red action: do-nothing, Blue reward:0.85\n", - "step: 2737, Red action: do-nothing, Blue reward:0.85\n", - "step: 2738, Red action: do-nothing, Blue reward:0.85\n", - "step: 2739, Red action: do-nothing, Blue reward:0.85\n", - "step: 2740, Red action: do-nothing, Blue reward:0.85\n", - "step: 2741, Red action: do-nothing, Blue reward:0.85\n", - "step: 2742, Red action: do-nothing, Blue reward:0.85\n", - "step: 2743, Red action: do-nothing, Blue reward:0.85\n", - "step: 2744, Red action: do-nothing, Blue reward:0.85\n", - "step: 2745, Red action: do-nothing, Blue reward:0.85\n", - "step: 2746, Red action: do-nothing, Blue reward:0.85\n", - "step: 2747, Red action: do-nothing, Blue reward:0.85\n", - "step: 2748, Red action: do-nothing, Blue reward:0.85\n", - "step: 2749, Red action: do-nothing, Blue reward:0.85\n", - "step: 2750, Red action: do-nothing, Blue reward:0.85\n", - "step: 2751, Red action: do-nothing, Blue reward:0.85\n", - "step: 2752, Red action: do-nothing, Blue reward:0.85\n", - "step: 2753, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2754, Red action: do-nothing, Blue reward:0.85\n", - "step: 2755, Red action: do-nothing, Blue reward:0.85\n", - "step: 2756, Red action: do-nothing, Blue reward:0.85\n", - "step: 2757, Red action: do-nothing, Blue reward:0.85\n", - "step: 2758, Red action: do-nothing, Blue reward:0.85\n", - "step: 2759, Red action: do-nothing, Blue reward:0.85\n", - "step: 2760, Red action: do-nothing, Blue reward:0.85\n", - "step: 2761, Red action: do-nothing, Blue reward:0.85\n", - "step: 2762, Red action: do-nothing, Blue reward:0.85\n", - "step: 2763, Red action: do-nothing, Blue reward:0.85\n", - "step: 2764, Red action: do-nothing, Blue reward:0.85\n", - "step: 2765, Red action: do-nothing, Blue reward:0.85\n", - "step: 2766, Red action: do-nothing, Blue reward:0.85\n", - "step: 2767, Red action: do-nothing, Blue reward:0.85\n", - "step: 2768, Red action: do-nothing, Blue reward:0.85\n", - "step: 2769, Red action: do-nothing, Blue reward:0.85\n", - "step: 2770, Red action: do-nothing, Blue reward:0.85\n", - "step: 2771, Red action: do-nothing, Blue reward:0.85\n", - "step: 2772, Red action: do-nothing, Blue reward:0.85\n", - "step: 2773, Red action: do-nothing, Blue reward:0.85\n", - "step: 2774, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2775, Red action: do-nothing, Blue reward:0.85\n", - "step: 2776, Red action: do-nothing, Blue reward:0.85\n", - "step: 2777, Red action: do-nothing, Blue reward:0.85\n", - "step: 2778, Red action: do-nothing, Blue reward:0.85\n", - "step: 2779, Red action: do-nothing, Blue reward:0.85\n", - "step: 2780, Red action: do-nothing, Blue reward:0.85\n", - "step: 2781, Red action: do-nothing, Blue reward:0.85\n", - "step: 2782, Red action: do-nothing, Blue reward:0.85\n", - "step: 2783, Red action: do-nothing, Blue reward:0.85\n", - "step: 2784, Red action: do-nothing, Blue reward:0.85\n", - "step: 2785, Red action: do-nothing, Blue reward:0.85\n", - "step: 2786, Red action: do-nothing, Blue reward:0.85\n", - "step: 2787, Red action: do-nothing, Blue reward:0.85\n", - "step: 2788, Red action: do-nothing, Blue reward:0.85\n", - "step: 2789, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2790, Red action: do-nothing, Blue reward:0.85\n", - "step: 2791, Red action: do-nothing, Blue reward:0.85\n", - "step: 2792, Red action: do-nothing, Blue reward:0.85\n", - "step: 2793, Red action: do-nothing, Blue reward:0.85\n", - "step: 2794, Red action: do-nothing, Blue reward:0.85\n", - "step: 2795, Red action: do-nothing, Blue reward:0.85\n", - "step: 2796, Red action: do-nothing, Blue reward:0.85\n", - "step: 2797, Red action: do-nothing, Blue reward:0.85\n", - "step: 2798, Red action: do-nothing, Blue reward:0.85\n", - "step: 2799, Red action: do-nothing, Blue reward:0.85\n", - "step: 2800, Red action: do-nothing, Blue reward:0.85\n", - "step: 2801, Red action: do-nothing, Blue reward:0.85\n", - "step: 2802, Red action: do-nothing, Blue reward:0.85\n", - "step: 2803, Red action: do-nothing, Blue reward:0.85\n", - "step: 2804, Red action: do-nothing, Blue reward:0.85\n", - "step: 2805, Red action: do-nothing, Blue reward:0.85\n", - "step: 2806, Red action: do-nothing, Blue reward:0.85\n", - "step: 2807, Red action: do-nothing, Blue reward:0.85\n", - "step: 2808, Red action: do-nothing, Blue reward:0.85\n", - "step: 2809, Red action: do-nothing, Blue reward:0.85\n", - "step: 2810, Red action: do-nothing, Blue reward:0.85\n", - "step: 2811, Red action: do-nothing, Blue reward:0.85\n", - "step: 2812, Red action: do-nothing, Blue reward:0.85\n", - "step: 2813, Red action: do-nothing, Blue reward:0.85\n", - "step: 2814, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2815, Red action: do-nothing, Blue reward:0.85\n", - "step: 2816, Red action: do-nothing, Blue reward:0.85\n", - "step: 2817, Red action: do-nothing, Blue reward:0.85\n", - "step: 2818, Red action: do-nothing, Blue reward:0.85\n", - "step: 2819, Red action: do-nothing, Blue reward:0.85\n", - "step: 2820, Red action: do-nothing, Blue reward:0.85\n", - "step: 2821, Red action: do-nothing, Blue reward:0.85\n", - "step: 2822, Red action: do-nothing, Blue reward:0.85\n", - "step: 2823, Red action: do-nothing, Blue reward:0.85\n", - "step: 2824, Red action: do-nothing, Blue reward:0.85\n", - "step: 2825, Red action: do-nothing, Blue reward:0.85\n", - "step: 2826, Red action: do-nothing, Blue reward:0.85\n", - "step: 2827, Red action: do-nothing, Blue reward:0.85\n", - "step: 2828, Red action: do-nothing, Blue reward:0.85\n", - "step: 2829, Red action: do-nothing, Blue reward:0.85\n", - "step: 2830, Red action: do-nothing, Blue reward:0.85\n", - "step: 2831, Red action: do-nothing, Blue reward:0.85\n", - "step: 2832, Red action: do-nothing, Blue reward:0.85\n", - "step: 2833, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2834, Red action: do-nothing, Blue reward:0.85\n", - "step: 2835, Red action: do-nothing, Blue reward:0.85\n", - "step: 2836, Red action: do-nothing, Blue reward:0.85\n", - "step: 2837, Red action: do-nothing, Blue reward:0.85\n", - "step: 2838, Red action: do-nothing, Blue reward:0.85\n", - "step: 2839, Red action: do-nothing, Blue reward:0.85\n", - "step: 2840, Red action: do-nothing, Blue reward:0.85\n", - "step: 2841, Red action: do-nothing, Blue reward:0.85\n", - "step: 2842, Red action: do-nothing, Blue reward:0.85\n", - "step: 2843, Red action: do-nothing, Blue reward:0.85\n", - "step: 2844, Red action: do-nothing, Blue reward:0.85\n", - "step: 2845, Red action: do-nothing, Blue reward:0.85\n", - "step: 2846, Red action: do-nothing, Blue reward:0.85\n", - "step: 2847, Red action: do-nothing, Blue reward:0.85\n", - "step: 2848, Red action: do-nothing, Blue reward:0.85\n", - "step: 2849, Red action: do-nothing, Blue reward:0.85\n", - "step: 2850, Red action: do-nothing, Blue reward:0.85\n", - "step: 2851, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2852, Red action: do-nothing, Blue reward:0.85\n", - "step: 2853, Red action: do-nothing, Blue reward:0.85\n", - "step: 2854, Red action: do-nothing, Blue reward:0.85\n", - "step: 2855, Red action: do-nothing, Blue reward:0.85\n", - "step: 2856, Red action: do-nothing, Blue reward:0.85\n", - "step: 2857, Red action: do-nothing, Blue reward:0.85\n", - "step: 2858, Red action: do-nothing, Blue reward:0.85\n", - "step: 2859, Red action: do-nothing, Blue reward:0.85\n", - "step: 2860, Red action: do-nothing, Blue reward:0.85\n", - "step: 2861, Red action: do-nothing, Blue reward:0.85\n", - "step: 2862, Red action: do-nothing, Blue reward:0.85\n", - "step: 2863, Red action: do-nothing, Blue reward:0.85\n", - "step: 2864, Red action: do-nothing, Blue reward:0.85\n", - "step: 2865, Red action: do-nothing, Blue reward:0.85\n", - "step: 2866, Red action: do-nothing, Blue reward:0.85\n", - "step: 2867, Red action: do-nothing, Blue reward:0.85\n", - "step: 2868, Red action: do-nothing, Blue reward:0.85\n", - "step: 2869, Red action: do-nothing, Blue reward:0.85\n", - "step: 2870, Red action: do-nothing, Blue reward:0.85\n", - "step: 2871, Red action: do-nothing, Blue reward:0.85\n", - "step: 2872, Red action: do-nothing, Blue reward:0.85\n", - "step: 2873, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2874, Red action: do-nothing, Blue reward:0.85\n", - "step: 2875, Red action: do-nothing, Blue reward:0.85\n", - "step: 2876, Red action: do-nothing, Blue reward:0.85\n", - "step: 2877, Red action: do-nothing, Blue reward:0.85\n", - "step: 2878, Red action: do-nothing, Blue reward:0.85\n", - "step: 2879, Red action: do-nothing, Blue reward:0.85\n", - "step: 2880, Red action: do-nothing, Blue reward:0.85\n", - "step: 2881, Red action: do-nothing, Blue reward:0.85\n", - "step: 2882, Red action: do-nothing, Blue reward:0.85\n", - "step: 2883, Red action: do-nothing, Blue reward:0.85\n", - "step: 2884, Red action: do-nothing, Blue reward:0.85\n", - "step: 2885, Red action: do-nothing, Blue reward:0.85\n", - "step: 2886, Red action: do-nothing, Blue reward:0.85\n", - "step: 2887, Red action: do-nothing, Blue reward:0.85\n", - "step: 2888, Red action: do-nothing, Blue reward:0.85\n", - "step: 2889, Red action: do-nothing, Blue reward:0.85\n", - "step: 2890, Red action: do-nothing, Blue reward:0.85\n", - "step: 2891, Red action: do-nothing, Blue reward:0.85\n", - "step: 2892, Red action: do-nothing, Blue reward:0.85\n", - "step: 2893, Red action: do-nothing, Blue reward:0.85\n", - "step: 2894, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2895, Red action: do-nothing, Blue reward:0.85\n", - "step: 2896, Red action: do-nothing, Blue reward:0.85\n", - "step: 2897, Red action: do-nothing, Blue reward:0.85\n", - "step: 2898, Red action: do-nothing, Blue reward:0.85\n", - "step: 2899, Red action: do-nothing, Blue reward:0.85\n", - "step: 2900, Red action: do-nothing, Blue reward:0.85\n", - "step: 2901, Red action: do-nothing, Blue reward:0.85\n", - "step: 2902, Red action: do-nothing, Blue reward:0.85\n", - "step: 2903, Red action: do-nothing, Blue reward:0.85\n", - "step: 2904, Red action: do-nothing, Blue reward:0.85\n", - "step: 2905, Red action: do-nothing, Blue reward:0.85\n", - "step: 2906, Red action: do-nothing, Blue reward:0.85\n", - "step: 2907, Red action: do-nothing, Blue reward:0.85\n", - "step: 2908, Red action: do-nothing, Blue reward:0.85\n", - "step: 2909, Red action: do-nothing, Blue reward:0.85\n", - "step: 2910, Red action: do-nothing, Blue reward:0.85\n", - "step: 2911, Red action: do-nothing, Blue reward:0.85\n", - "step: 2912, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2913, Red action: do-nothing, Blue reward:0.85\n", - "step: 2914, Red action: do-nothing, Blue reward:0.85\n", - "step: 2915, Red action: do-nothing, Blue reward:0.85\n", - "step: 2916, Red action: do-nothing, Blue reward:0.85\n", - "step: 2917, Red action: do-nothing, Blue reward:0.85\n", - "step: 2918, Red action: do-nothing, Blue reward:0.85\n", - "step: 2919, Red action: do-nothing, Blue reward:0.85\n", - "step: 2920, Red action: do-nothing, Blue reward:0.85\n", - "step: 2921, Red action: do-nothing, Blue reward:0.85\n", - "step: 2922, Red action: do-nothing, Blue reward:0.85\n", - "step: 2923, Red action: do-nothing, Blue reward:0.85\n", - "step: 2924, Red action: do-nothing, Blue reward:0.85\n", - "step: 2925, Red action: do-nothing, Blue reward:0.85\n", - "step: 2926, Red action: do-nothing, Blue reward:0.85\n", - "step: 2927, Red action: do-nothing, Blue reward:0.85\n", - "step: 2928, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2929, Red action: do-nothing, Blue reward:0.85\n", - "step: 2930, Red action: do-nothing, Blue reward:0.85\n", - "step: 2931, Red action: do-nothing, Blue reward:0.85\n", - "step: 2932, Red action: do-nothing, Blue reward:0.85\n", - "step: 2933, Red action: do-nothing, Blue reward:0.85\n", - "step: 2934, Red action: do-nothing, Blue reward:0.85\n", - "step: 2935, Red action: do-nothing, Blue reward:0.85\n", - "step: 2936, Red action: do-nothing, Blue reward:0.85\n", - "step: 2937, Red action: do-nothing, Blue reward:0.85\n", - "step: 2938, Red action: do-nothing, Blue reward:0.85\n", - "step: 2939, Red action: do-nothing, Blue reward:0.85\n", - "step: 2940, Red action: do-nothing, Blue reward:0.85\n", - "step: 2941, Red action: do-nothing, Blue reward:0.85\n", - "step: 2942, Red action: do-nothing, Blue reward:0.85\n", - "step: 2943, Red action: do-nothing, Blue reward:0.85\n", - "step: 2944, Red action: do-nothing, Blue reward:0.85\n", - "step: 2945, Red action: do-nothing, Blue reward:0.85\n", - "step: 2946, Red action: do-nothing, Blue reward:0.85\n", - "step: 2947, Red action: do-nothing, Blue reward:0.85\n", - "step: 2948, Red action: do-nothing, Blue reward:0.85\n", - "step: 2949, Red action: do-nothing, Blue reward:0.85\n", - "step: 2950, Red action: do-nothing, Blue reward:0.85\n", - "step: 2951, Red action: do-nothing, Blue reward:0.85\n", - "step: 2952, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2953, Red action: do-nothing, Blue reward:0.85\n", - "step: 2954, Red action: do-nothing, Blue reward:0.85\n", - "step: 2955, Red action: do-nothing, Blue reward:0.85\n", - "step: 2956, Red action: do-nothing, Blue reward:0.85\n", - "step: 2957, Red action: do-nothing, Blue reward:0.85\n", - "step: 2958, Red action: do-nothing, Blue reward:0.85\n", - "step: 2959, Red action: do-nothing, Blue reward:0.85\n", - "step: 2960, Red action: do-nothing, Blue reward:0.85\n", - "step: 2961, Red action: do-nothing, Blue reward:0.85\n", - "step: 2962, Red action: do-nothing, Blue reward:0.85\n", - "step: 2963, Red action: do-nothing, Blue reward:0.85\n", - "step: 2964, Red action: do-nothing, Blue reward:0.85\n", - "step: 2965, Red action: do-nothing, Blue reward:0.85\n", - "step: 2966, Red action: do-nothing, Blue reward:0.85\n", - "step: 2967, Red action: do-nothing, Blue reward:0.85\n", - "step: 2968, Red action: do-nothing, Blue reward:0.85\n", - "step: 2969, Red action: do-nothing, Blue reward:0.85\n", - "step: 2970, Red action: do-nothing, Blue reward:0.85\n", - "step: 2971, Red action: do-nothing, Blue reward:0.85\n", - "step: 2972, Red action: do-nothing, Blue reward:0.85\n", - "step: 2973, Red action: do-nothing, Blue reward:0.85\n", - "step: 2974, Red action: do-nothing, Blue reward:0.85\n", - "step: 2975, Red action: do-nothing, Blue reward:0.85\n", - "step: 2976, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2977, Red action: do-nothing, Blue reward:0.85\n", - "step: 2978, Red action: do-nothing, Blue reward:0.85\n", - "step: 2979, Red action: do-nothing, Blue reward:0.85\n", - "step: 2980, Red action: do-nothing, Blue reward:0.85\n", - "step: 2981, Red action: do-nothing, Blue reward:0.85\n", - "step: 2982, Red action: do-nothing, Blue reward:0.85\n", - "step: 2983, Red action: do-nothing, Blue reward:0.85\n", - "step: 2984, Red action: do-nothing, Blue reward:0.85\n", - "step: 2985, Red action: do-nothing, Blue reward:0.85\n", - "step: 2986, Red action: do-nothing, Blue reward:0.85\n", - "step: 2987, Red action: do-nothing, Blue reward:0.85\n", - "step: 2988, Red action: do-nothing, Blue reward:0.85\n", - "step: 2989, Red action: do-nothing, Blue reward:0.85\n", - "step: 2990, Red action: do-nothing, Blue reward:0.85\n", - "step: 2991, Red action: do-nothing, Blue reward:0.85\n", - "step: 2992, Red action: do-nothing, Blue reward:0.85\n", - "step: 2993, Red action: do-nothing, Blue reward:0.85\n", - "step: 2994, Red action: do-nothing, Blue reward:0.85\n", - "step: 2995, Red action: do-nothing, Blue reward:0.85\n", - "step: 2996, Red action: do-nothing, Blue reward:0.85\n", - "step: 2997, Red action: node-application-execute, Blue reward:0.85\n", - "step: 2998, Red action: do-nothing, Blue reward:0.85\n", - "step: 2999, Red action: do-nothing, Blue reward:0.85\n", - "step: 3000, Red action: do-nothing, Blue reward:0.85\n", - "step: 3001, Red action: do-nothing, Blue reward:0.85\n", - "step: 3002, Red action: do-nothing, Blue reward:0.85\n", - "step: 3003, Red action: do-nothing, Blue reward:0.85\n", - "step: 3004, Red action: do-nothing, Blue reward:0.85\n", - "step: 3005, Red action: do-nothing, Blue reward:0.85\n", - "step: 3006, Red action: do-nothing, Blue reward:0.85\n", - "step: 3007, Red action: do-nothing, Blue reward:0.85\n", - "step: 3008, Red action: do-nothing, Blue reward:0.85\n", - "step: 3009, Red action: do-nothing, Blue reward:0.85\n", - "step: 3010, Red action: do-nothing, Blue reward:0.85\n", - "step: 3011, Red action: do-nothing, Blue reward:0.85\n", - "step: 3012, Red action: do-nothing, Blue reward:0.85\n", - "step: 3013, Red action: do-nothing, Blue reward:0.85\n", - "step: 3014, Red action: do-nothing, Blue reward:0.85\n", - "step: 3015, Red action: do-nothing, Blue reward:0.85\n", - "step: 3016, Red action: do-nothing, Blue reward:0.85\n", - "step: 3017, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3018, Red action: do-nothing, Blue reward:0.85\n", - "step: 3019, Red action: do-nothing, Blue reward:0.85\n", - "step: 3020, Red action: do-nothing, Blue reward:0.85\n", - "step: 3021, Red action: do-nothing, Blue reward:0.85\n", - "step: 3022, Red action: do-nothing, Blue reward:0.85\n", - "step: 3023, Red action: do-nothing, Blue reward:0.85\n", - "step: 3024, Red action: do-nothing, Blue reward:0.85\n", - "step: 3025, Red action: do-nothing, Blue reward:0.85\n", - "step: 3026, Red action: do-nothing, Blue reward:0.85\n", - "step: 3027, Red action: do-nothing, Blue reward:0.85\n", - "step: 3028, Red action: do-nothing, Blue reward:0.85\n", - "step: 3029, Red action: do-nothing, Blue reward:0.85\n", - "step: 3030, Red action: do-nothing, Blue reward:0.85\n", - "step: 3031, Red action: do-nothing, Blue reward:0.85\n", - "step: 3032, Red action: do-nothing, Blue reward:0.85\n", - "step: 3033, Red action: do-nothing, Blue reward:0.85\n", - "step: 3034, Red action: do-nothing, Blue reward:0.85\n", - "step: 3035, Red action: do-nothing, Blue reward:0.85\n", - "step: 3036, Red action: do-nothing, Blue reward:0.85\n", - "step: 3037, Red action: do-nothing, Blue reward:0.85\n", - "step: 3038, Red action: do-nothing, Blue reward:0.85\n", - "step: 3039, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3040, Red action: do-nothing, Blue reward:0.85\n", - "step: 3041, Red action: do-nothing, Blue reward:0.85\n", - "step: 3042, Red action: do-nothing, Blue reward:0.85\n", - "step: 3043, Red action: do-nothing, Blue reward:0.85\n", - "step: 3044, Red action: do-nothing, Blue reward:0.85\n", - "step: 3045, Red action: do-nothing, Blue reward:0.85\n", - "step: 3046, Red action: do-nothing, Blue reward:0.85\n", - "step: 3047, Red action: do-nothing, Blue reward:0.85\n", - "step: 3048, Red action: do-nothing, Blue reward:0.85\n", - "step: 3049, Red action: do-nothing, Blue reward:0.85\n", - "step: 3050, Red action: do-nothing, Blue reward:0.85\n", - "step: 3051, Red action: do-nothing, Blue reward:0.85\n", - "step: 3052, Red action: do-nothing, Blue reward:0.85\n", - "step: 3053, Red action: do-nothing, Blue reward:0.85\n", - "step: 3054, Red action: do-nothing, Blue reward:0.85\n", - "step: 3055, Red action: do-nothing, Blue reward:0.85\n", - "step: 3056, Red action: do-nothing, Blue reward:0.85\n", - "step: 3057, Red action: do-nothing, Blue reward:0.85\n", - "step: 3058, Red action: do-nothing, Blue reward:0.85\n", - "step: 3059, Red action: do-nothing, Blue reward:0.85\n", - "step: 3060, Red action: do-nothing, Blue reward:0.85\n", - "step: 3061, Red action: do-nothing, Blue reward:0.85\n", - "step: 3062, Red action: do-nothing, Blue reward:0.85\n", - "step: 3063, Red action: do-nothing, Blue reward:0.85\n", - "step: 3064, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3065, Red action: do-nothing, Blue reward:0.85\n", - "step: 3066, Red action: do-nothing, Blue reward:0.85\n", - "step: 3067, Red action: do-nothing, Blue reward:0.85\n", - "step: 3068, Red action: do-nothing, Blue reward:0.85\n", - "step: 3069, Red action: do-nothing, Blue reward:0.85\n", - "step: 3070, Red action: do-nothing, Blue reward:0.85\n", - "step: 3071, Red action: do-nothing, Blue reward:0.85\n", - "step: 3072, Red action: do-nothing, Blue reward:0.85\n", - "step: 3073, Red action: do-nothing, Blue reward:0.85\n", - "step: 3074, Red action: do-nothing, Blue reward:0.85\n", - "step: 3075, Red action: do-nothing, Blue reward:0.85\n", - "step: 3076, Red action: do-nothing, Blue reward:0.85\n", - "step: 3077, Red action: do-nothing, Blue reward:0.85\n", - "step: 3078, Red action: do-nothing, Blue reward:0.85\n", - "step: 3079, Red action: do-nothing, Blue reward:0.85\n", - "step: 3080, Red action: do-nothing, Blue reward:0.85\n", - "step: 3081, Red action: do-nothing, Blue reward:0.85\n", - "step: 3082, Red action: do-nothing, Blue reward:0.85\n", - "step: 3083, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3084, Red action: do-nothing, Blue reward:0.85\n", - "step: 3085, Red action: do-nothing, Blue reward:0.85\n", - "step: 3086, Red action: do-nothing, Blue reward:0.85\n", - "step: 3087, Red action: do-nothing, Blue reward:0.85\n", - "step: 3088, Red action: do-nothing, Blue reward:0.85\n", - "step: 3089, Red action: do-nothing, Blue reward:0.85\n", - "step: 3090, Red action: do-nothing, Blue reward:0.85\n", - "step: 3091, Red action: do-nothing, Blue reward:0.85\n", - "step: 3092, Red action: do-nothing, Blue reward:0.85\n", - "step: 3093, Red action: do-nothing, Blue reward:0.85\n", - "step: 3094, Red action: do-nothing, Blue reward:0.85\n", - "step: 3095, Red action: do-nothing, Blue reward:0.85\n", - "step: 3096, Red action: do-nothing, Blue reward:0.85\n", - "step: 3097, Red action: do-nothing, Blue reward:0.85\n", - "step: 3098, Red action: do-nothing, Blue reward:0.85\n", - "step: 3099, Red action: do-nothing, Blue reward:0.85\n", - "step: 3100, Red action: do-nothing, Blue reward:0.85\n", - "step: 3101, Red action: do-nothing, Blue reward:0.85\n", - "step: 3102, Red action: do-nothing, Blue reward:0.85\n", - "step: 3103, Red action: do-nothing, Blue reward:0.85\n", - "step: 3104, Red action: do-nothing, Blue reward:0.85\n", - "step: 3105, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3106, Red action: do-nothing, Blue reward:0.85\n", - "step: 3107, Red action: do-nothing, Blue reward:0.85\n", - "step: 3108, Red action: do-nothing, Blue reward:0.85\n", - "step: 3109, Red action: do-nothing, Blue reward:0.85\n", - "step: 3110, Red action: do-nothing, Blue reward:0.85\n", - "step: 3111, Red action: do-nothing, Blue reward:0.85\n", - "step: 3112, Red action: do-nothing, Blue reward:0.85\n", - "step: 3113, Red action: do-nothing, Blue reward:0.85\n", - "step: 3114, Red action: do-nothing, Blue reward:0.85\n", - "step: 3115, Red action: do-nothing, Blue reward:0.85\n", - "step: 3116, Red action: do-nothing, Blue reward:0.85\n", - "step: 3117, Red action: do-nothing, Blue reward:0.85\n", - "step: 3118, Red action: do-nothing, Blue reward:0.85\n", - "step: 3119, Red action: do-nothing, Blue reward:0.85\n", - "step: 3120, Red action: do-nothing, Blue reward:0.85\n", - "step: 3121, Red action: do-nothing, Blue reward:0.85\n", - "step: 3122, Red action: do-nothing, Blue reward:0.85\n", - "step: 3123, Red action: do-nothing, Blue reward:0.85\n", - "step: 3124, Red action: do-nothing, Blue reward:0.85\n", - "step: 3125, Red action: do-nothing, Blue reward:0.85\n", - "step: 3126, Red action: do-nothing, Blue reward:0.85\n", - "step: 3127, Red action: do-nothing, Blue reward:0.85\n", - "step: 3128, Red action: do-nothing, Blue reward:0.85\n", - "step: 3129, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3130, Red action: do-nothing, Blue reward:0.85\n", - "step: 3131, Red action: do-nothing, Blue reward:0.85\n", - "step: 3132, Red action: do-nothing, Blue reward:0.85\n", - "step: 3133, Red action: do-nothing, Blue reward:0.85\n", - "step: 3134, Red action: do-nothing, Blue reward:0.85\n", - "step: 3135, Red action: do-nothing, Blue reward:0.85\n", - "step: 3136, Red action: do-nothing, Blue reward:0.85\n", - "step: 3137, Red action: do-nothing, Blue reward:0.85\n", - "step: 3138, Red action: do-nothing, Blue reward:0.85\n", - "step: 3139, Red action: do-nothing, Blue reward:0.85\n", - "step: 3140, Red action: do-nothing, Blue reward:0.85\n", - "step: 3141, Red action: do-nothing, Blue reward:0.85\n", - "step: 3142, Red action: do-nothing, Blue reward:0.85\n", - "step: 3143, Red action: do-nothing, Blue reward:0.85\n", - "step: 3144, Red action: do-nothing, Blue reward:0.85\n", - "step: 3145, Red action: do-nothing, Blue reward:0.85\n", - "step: 3146, Red action: do-nothing, Blue reward:0.85\n", - "step: 3147, Red action: do-nothing, Blue reward:0.85\n", - "step: 3148, Red action: do-nothing, Blue reward:0.85\n", - "step: 3149, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3150, Red action: do-nothing, Blue reward:0.85\n", - "step: 3151, Red action: do-nothing, Blue reward:0.85\n", - "step: 3152, Red action: do-nothing, Blue reward:0.85\n", - "step: 3153, Red action: do-nothing, Blue reward:0.85\n", - "step: 3154, Red action: do-nothing, Blue reward:0.85\n", - "step: 3155, Red action: do-nothing, Blue reward:0.85\n", - "step: 3156, Red action: do-nothing, Blue reward:0.85\n", - "step: 3157, Red action: do-nothing, Blue reward:0.85\n", - "step: 3158, Red action: do-nothing, Blue reward:0.85\n", - "step: 3159, Red action: do-nothing, Blue reward:0.85\n", - "step: 3160, Red action: do-nothing, Blue reward:0.85\n", - "step: 3161, Red action: do-nothing, Blue reward:0.85\n", - "step: 3162, Red action: do-nothing, Blue reward:0.85\n", - "step: 3163, Red action: do-nothing, Blue reward:0.85\n", - "step: 3164, Red action: do-nothing, Blue reward:0.85\n", - "step: 3165, Red action: do-nothing, Blue reward:0.85\n", - "step: 3166, Red action: do-nothing, Blue reward:0.85\n", - "step: 3167, Red action: do-nothing, Blue reward:0.85\n", - "step: 3168, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3169, Red action: do-nothing, Blue reward:0.85\n", - "step: 3170, Red action: do-nothing, Blue reward:0.85\n", - "step: 3171, Red action: do-nothing, Blue reward:0.85\n", - "step: 3172, Red action: do-nothing, Blue reward:0.85\n", - "step: 3173, Red action: do-nothing, Blue reward:0.85\n", - "step: 3174, Red action: do-nothing, Blue reward:0.85\n", - "step: 3175, Red action: do-nothing, Blue reward:0.85\n", - "step: 3176, Red action: do-nothing, Blue reward:0.85\n", - "step: 3177, Red action: do-nothing, Blue reward:0.85\n", - "step: 3178, Red action: do-nothing, Blue reward:0.85\n", - "step: 3179, Red action: do-nothing, Blue reward:0.85\n", - "step: 3180, Red action: do-nothing, Blue reward:0.85\n", - "step: 3181, Red action: do-nothing, Blue reward:0.85\n", - "step: 3182, Red action: do-nothing, Blue reward:0.85\n", - "step: 3183, Red action: do-nothing, Blue reward:0.85\n", - "step: 3184, Red action: do-nothing, Blue reward:0.85\n", - "step: 3185, Red action: do-nothing, Blue reward:0.85\n", - "step: 3186, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3187, Red action: do-nothing, Blue reward:0.85\n", - "step: 3188, Red action: do-nothing, Blue reward:0.85\n", - "step: 3189, Red action: do-nothing, Blue reward:0.85\n", - "step: 3190, Red action: do-nothing, Blue reward:0.85\n", - "step: 3191, Red action: do-nothing, Blue reward:0.85\n", - "step: 3192, Red action: do-nothing, Blue reward:0.85\n", - "step: 3193, Red action: do-nothing, Blue reward:0.85\n", - "step: 3194, Red action: do-nothing, Blue reward:0.85\n", - "step: 3195, Red action: do-nothing, Blue reward:0.85\n", - "step: 3196, Red action: do-nothing, Blue reward:0.85\n", - "step: 3197, Red action: do-nothing, Blue reward:0.85\n", - "step: 3198, Red action: do-nothing, Blue reward:0.85\n", - "step: 3199, Red action: do-nothing, Blue reward:0.85\n", - "step: 3200, Red action: do-nothing, Blue reward:0.85\n", - "step: 3201, Red action: do-nothing, Blue reward:0.85\n", - "step: 3202, Red action: do-nothing, Blue reward:0.85\n", - "step: 3203, Red action: do-nothing, Blue reward:0.85\n", - "step: 3204, Red action: do-nothing, Blue reward:0.85\n", - "step: 3205, Red action: do-nothing, Blue reward:0.85\n", - "step: 3206, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3207, Red action: do-nothing, Blue reward:0.85\n", - "step: 3208, Red action: do-nothing, Blue reward:0.85\n", - "step: 3209, Red action: do-nothing, Blue reward:0.85\n", - "step: 3210, Red action: do-nothing, Blue reward:0.85\n", - "step: 3211, Red action: do-nothing, Blue reward:0.85\n", - "step: 3212, Red action: do-nothing, Blue reward:0.85\n", - "step: 3213, Red action: do-nothing, Blue reward:0.85\n", - "step: 3214, Red action: do-nothing, Blue reward:0.85\n", - "step: 3215, Red action: do-nothing, Blue reward:0.85\n", - "step: 3216, Red action: do-nothing, Blue reward:0.85\n", - "step: 3217, Red action: do-nothing, Blue reward:0.85\n", - "step: 3218, Red action: do-nothing, Blue reward:0.85\n", - "step: 3219, Red action: do-nothing, Blue reward:0.85\n", - "step: 3220, Red action: do-nothing, Blue reward:0.85\n", - "step: 3221, Red action: do-nothing, Blue reward:0.85\n", - "step: 3222, Red action: do-nothing, Blue reward:0.85\n", - "step: 3223, Red action: do-nothing, Blue reward:0.85\n", - "step: 3224, Red action: do-nothing, Blue reward:0.85\n", - "step: 3225, Red action: do-nothing, Blue reward:0.85\n", - "step: 3226, Red action: do-nothing, Blue reward:0.85\n", - "step: 3227, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3228, Red action: do-nothing, Blue reward:0.85\n", - "step: 3229, Red action: do-nothing, Blue reward:0.85\n", - "step: 3230, Red action: do-nothing, Blue reward:0.85\n", - "step: 3231, Red action: do-nothing, Blue reward:0.85\n", - "step: 3232, Red action: do-nothing, Blue reward:0.85\n", - "step: 3233, Red action: do-nothing, Blue reward:0.85\n", - "step: 3234, Red action: do-nothing, Blue reward:0.85\n", - "step: 3235, Red action: do-nothing, Blue reward:0.85\n", - "step: 3236, Red action: do-nothing, Blue reward:0.85\n", - "step: 3237, Red action: do-nothing, Blue reward:0.85\n", - "step: 3238, Red action: do-nothing, Blue reward:0.85\n", - "step: 3239, Red action: do-nothing, Blue reward:0.85\n", - "step: 3240, Red action: do-nothing, Blue reward:0.85\n", - "step: 3241, Red action: do-nothing, Blue reward:0.85\n", - "step: 3242, Red action: do-nothing, Blue reward:0.85\n", - "step: 3243, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3244, Red action: do-nothing, Blue reward:0.85\n", - "step: 3245, Red action: do-nothing, Blue reward:0.85\n", - "step: 3246, Red action: do-nothing, Blue reward:0.85\n", - "step: 3247, Red action: do-nothing, Blue reward:0.85\n", - "step: 3248, Red action: do-nothing, Blue reward:0.85\n", - "step: 3249, Red action: do-nothing, Blue reward:0.85\n", - "step: 3250, Red action: do-nothing, Blue reward:0.85\n", - "step: 3251, Red action: do-nothing, Blue reward:0.85\n", - "step: 3252, Red action: do-nothing, Blue reward:0.85\n", - "step: 3253, Red action: do-nothing, Blue reward:0.85\n", - "step: 3254, Red action: do-nothing, Blue reward:0.85\n", - "step: 3255, Red action: do-nothing, Blue reward:0.85\n", - "step: 3256, Red action: do-nothing, Blue reward:0.85\n", - "step: 3257, Red action: do-nothing, Blue reward:0.85\n", - "step: 3258, Red action: do-nothing, Blue reward:0.85\n", - "step: 3259, Red action: do-nothing, Blue reward:0.85\n", - "step: 3260, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3261, Red action: do-nothing, Blue reward:0.85\n", - "step: 3262, Red action: do-nothing, Blue reward:0.85\n", - "step: 3263, Red action: do-nothing, Blue reward:0.85\n", - "step: 3264, Red action: do-nothing, Blue reward:0.85\n", - "step: 3265, Red action: do-nothing, Blue reward:0.85\n", - "step: 3266, Red action: do-nothing, Blue reward:0.85\n", - "step: 3267, Red action: do-nothing, Blue reward:0.85\n", - "step: 3268, Red action: do-nothing, Blue reward:0.85\n", - "step: 3269, Red action: do-nothing, Blue reward:0.85\n", - "step: 3270, Red action: do-nothing, Blue reward:0.85\n", - "step: 3271, Red action: do-nothing, Blue reward:0.85\n", - "step: 3272, Red action: do-nothing, Blue reward:0.85\n", - "step: 3273, Red action: do-nothing, Blue reward:0.85\n", - "step: 3274, Red action: do-nothing, Blue reward:0.85\n", - "step: 3275, Red action: do-nothing, Blue reward:0.85\n", - "step: 3276, Red action: do-nothing, Blue reward:0.85\n", - "step: 3277, Red action: do-nothing, Blue reward:0.85\n", - "step: 3278, Red action: do-nothing, Blue reward:0.85\n", - "step: 3279, Red action: do-nothing, Blue reward:0.85\n", - "step: 3280, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3281, Red action: do-nothing, Blue reward:0.85\n", - "step: 3282, Red action: do-nothing, Blue reward:0.85\n", - "step: 3283, Red action: do-nothing, Blue reward:0.85\n", - "step: 3284, Red action: do-nothing, Blue reward:0.85\n", - "step: 3285, Red action: do-nothing, Blue reward:0.85\n", - "step: 3286, Red action: do-nothing, Blue reward:0.85\n", - "step: 3287, Red action: do-nothing, Blue reward:0.85\n", - "step: 3288, Red action: do-nothing, Blue reward:0.85\n", - "step: 3289, Red action: do-nothing, Blue reward:0.85\n", - "step: 3290, Red action: do-nothing, Blue reward:0.85\n", - "step: 3291, Red action: do-nothing, Blue reward:0.85\n", - "step: 3292, Red action: do-nothing, Blue reward:0.85\n", - "step: 3293, Red action: do-nothing, Blue reward:0.85\n", - "step: 3294, Red action: do-nothing, Blue reward:0.85\n", - "step: 3295, Red action: do-nothing, Blue reward:0.85\n", - "step: 3296, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3297, Red action: do-nothing, Blue reward:0.85\n", - "step: 3298, Red action: do-nothing, Blue reward:0.85\n", - "step: 3299, Red action: do-nothing, Blue reward:0.85\n", - "step: 3300, Red action: do-nothing, Blue reward:0.85\n", - "step: 3301, Red action: do-nothing, Blue reward:0.85\n", - "step: 3302, Red action: do-nothing, Blue reward:0.85\n", - "step: 3303, Red action: do-nothing, Blue reward:0.85\n", - "step: 3304, Red action: do-nothing, Blue reward:0.85\n", - "step: 3305, Red action: do-nothing, Blue reward:0.85\n", - "step: 3306, Red action: do-nothing, Blue reward:0.85\n", - "step: 3307, Red action: do-nothing, Blue reward:0.85\n", - "step: 3308, Red action: do-nothing, Blue reward:0.85\n", - "step: 3309, Red action: do-nothing, Blue reward:0.85\n", - "step: 3310, Red action: do-nothing, Blue reward:0.85\n", - "step: 3311, Red action: do-nothing, Blue reward:0.85\n", - "step: 3312, Red action: do-nothing, Blue reward:0.85\n", - "step: 3313, Red action: do-nothing, Blue reward:0.85\n", - "step: 3314, Red action: do-nothing, Blue reward:0.85\n", - "step: 3315, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3316, Red action: do-nothing, Blue reward:0.85\n", - "step: 3317, Red action: do-nothing, Blue reward:0.85\n", - "step: 3318, Red action: do-nothing, Blue reward:0.85\n", - "step: 3319, Red action: do-nothing, Blue reward:0.85\n", - "step: 3320, Red action: do-nothing, Blue reward:0.85\n", - "step: 3321, Red action: do-nothing, Blue reward:0.85\n", - "step: 3322, Red action: do-nothing, Blue reward:0.85\n", - "step: 3323, Red action: do-nothing, Blue reward:0.85\n", - "step: 3324, Red action: do-nothing, Blue reward:0.85\n", - "step: 3325, Red action: do-nothing, Blue reward:0.85\n", - "step: 3326, Red action: do-nothing, Blue reward:0.85\n", - "step: 3327, Red action: do-nothing, Blue reward:0.85\n", - "step: 3328, Red action: do-nothing, Blue reward:0.85\n", - "step: 3329, Red action: do-nothing, Blue reward:0.85\n", - "step: 3330, Red action: do-nothing, Blue reward:0.85\n", - "step: 3331, Red action: do-nothing, Blue reward:0.85\n", - "step: 3332, Red action: do-nothing, Blue reward:0.85\n", - "step: 3333, Red action: do-nothing, Blue reward:0.85\n", - "step: 3334, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3335, Red action: do-nothing, Blue reward:0.85\n", - "step: 3336, Red action: do-nothing, Blue reward:0.85\n", - "step: 3337, Red action: do-nothing, Blue reward:0.85\n", - "step: 3338, Red action: do-nothing, Blue reward:0.85\n", - "step: 3339, Red action: do-nothing, Blue reward:0.85\n", - "step: 3340, Red action: do-nothing, Blue reward:0.85\n", - "step: 3341, Red action: do-nothing, Blue reward:0.85\n", - "step: 3342, Red action: do-nothing, Blue reward:0.85\n", - "step: 3343, Red action: do-nothing, Blue reward:0.85\n", - "step: 3344, Red action: do-nothing, Blue reward:0.85\n", - "step: 3345, Red action: do-nothing, Blue reward:0.85\n", - "step: 3346, Red action: do-nothing, Blue reward:0.85\n", - "step: 3347, Red action: do-nothing, Blue reward:0.85\n", - "step: 3348, Red action: do-nothing, Blue reward:0.85\n", - "step: 3349, Red action: do-nothing, Blue reward:0.85\n", - "step: 3350, Red action: do-nothing, Blue reward:0.85\n", - "step: 3351, Red action: do-nothing, Blue reward:0.85\n", - "step: 3352, Red action: do-nothing, Blue reward:0.85\n", - "step: 3353, Red action: do-nothing, Blue reward:0.85\n", - "step: 3354, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3355, Red action: do-nothing, Blue reward:0.85\n", - "step: 3356, Red action: do-nothing, Blue reward:0.85\n", - "step: 3357, Red action: do-nothing, Blue reward:0.85\n", - "step: 3358, Red action: do-nothing, Blue reward:0.85\n", - "step: 3359, Red action: do-nothing, Blue reward:0.85\n", - "step: 3360, Red action: do-nothing, Blue reward:0.85\n", - "step: 3361, Red action: do-nothing, Blue reward:0.85\n", - "step: 3362, Red action: do-nothing, Blue reward:0.85\n", - "step: 3363, Red action: do-nothing, Blue reward:0.85\n", - "step: 3364, Red action: do-nothing, Blue reward:0.85\n", - "step: 3365, Red action: do-nothing, Blue reward:0.85\n", - "step: 3366, Red action: do-nothing, Blue reward:0.85\n", - "step: 3367, Red action: do-nothing, Blue reward:0.85\n", - "step: 3368, Red action: do-nothing, Blue reward:0.85\n", - "step: 3369, Red action: do-nothing, Blue reward:0.85\n", - "step: 3370, Red action: do-nothing, Blue reward:0.85\n", - "step: 3371, Red action: do-nothing, Blue reward:0.85\n", - "step: 3372, Red action: do-nothing, Blue reward:0.85\n", - "step: 3373, Red action: do-nothing, Blue reward:0.85\n", - "step: 3374, Red action: do-nothing, Blue reward:0.85\n", - "step: 3375, Red action: do-nothing, Blue reward:0.85\n", - "step: 3376, Red action: do-nothing, Blue reward:0.85\n", - "step: 3377, Red action: do-nothing, Blue reward:0.85\n", - "step: 3378, Red action: do-nothing, Blue reward:0.85\n", - "step: 3379, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3380, Red action: do-nothing, Blue reward:0.85\n", - "step: 3381, Red action: do-nothing, Blue reward:0.85\n", - "step: 3382, Red action: do-nothing, Blue reward:0.85\n", - "step: 3383, Red action: do-nothing, Blue reward:0.85\n", - "step: 3384, Red action: do-nothing, Blue reward:0.85\n", - "step: 3385, Red action: do-nothing, Blue reward:0.85\n", - "step: 3386, Red action: do-nothing, Blue reward:0.85\n", - "step: 3387, Red action: do-nothing, Blue reward:0.85\n", - "step: 3388, Red action: do-nothing, Blue reward:0.85\n", - "step: 3389, Red action: do-nothing, Blue reward:0.85\n", - "step: 3390, Red action: do-nothing, Blue reward:0.85\n", - "step: 3391, Red action: do-nothing, Blue reward:0.85\n", - "step: 3392, Red action: do-nothing, Blue reward:0.85\n", - "step: 3393, Red action: do-nothing, Blue reward:0.85\n", - "step: 3394, Red action: do-nothing, Blue reward:0.85\n", - "step: 3395, Red action: do-nothing, Blue reward:0.85\n", - "step: 3396, Red action: do-nothing, Blue reward:0.85\n", - "step: 3397, Red action: do-nothing, Blue reward:0.85\n", - "step: 3398, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3399, Red action: do-nothing, Blue reward:0.85\n", - "step: 3400, Red action: do-nothing, Blue reward:0.85\n", - "step: 3401, Red action: do-nothing, Blue reward:0.85\n", - "step: 3402, Red action: do-nothing, Blue reward:0.85\n", - "step: 3403, Red action: do-nothing, Blue reward:0.85\n", - "step: 3404, Red action: do-nothing, Blue reward:0.85\n", - "step: 3405, Red action: do-nothing, Blue reward:0.85\n", - "step: 3406, Red action: do-nothing, Blue reward:0.85\n", - "step: 3407, Red action: do-nothing, Blue reward:0.85\n", - "step: 3408, Red action: do-nothing, Blue reward:0.85\n", - "step: 3409, Red action: do-nothing, Blue reward:0.85\n", - "step: 3410, Red action: do-nothing, Blue reward:0.85\n", - "step: 3411, Red action: do-nothing, Blue reward:0.85\n", - "step: 3412, Red action: do-nothing, Blue reward:0.85\n", - "step: 3413, Red action: do-nothing, Blue reward:0.85\n", - "step: 3414, Red action: do-nothing, Blue reward:0.85\n", - "step: 3415, Red action: do-nothing, Blue reward:0.85\n", - "step: 3416, Red action: do-nothing, Blue reward:0.85\n", - "step: 3417, Red action: do-nothing, Blue reward:0.85\n", - "step: 3418, Red action: do-nothing, Blue reward:0.85\n", - "step: 3419, Red action: do-nothing, Blue reward:0.85\n", - "step: 3420, Red action: do-nothing, Blue reward:0.85\n", - "step: 3421, Red action: do-nothing, Blue reward:0.85\n", - "step: 3422, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3423, Red action: do-nothing, Blue reward:0.85\n", - "step: 3424, Red action: do-nothing, Blue reward:0.85\n", - "step: 3425, Red action: do-nothing, Blue reward:0.85\n", - "step: 3426, Red action: do-nothing, Blue reward:0.85\n", - "step: 3427, Red action: do-nothing, Blue reward:0.85\n", - "step: 3428, Red action: do-nothing, Blue reward:0.85\n", - "step: 3429, Red action: do-nothing, Blue reward:0.85\n", - "step: 3430, Red action: do-nothing, Blue reward:0.85\n", - "step: 3431, Red action: do-nothing, Blue reward:0.85\n", - "step: 3432, Red action: do-nothing, Blue reward:0.85\n", - "step: 3433, Red action: do-nothing, Blue reward:0.85\n", - "step: 3434, Red action: do-nothing, Blue reward:0.85\n", - "step: 3435, Red action: do-nothing, Blue reward:0.85\n", - "step: 3436, Red action: do-nothing, Blue reward:0.85\n", - "step: 3437, Red action: do-nothing, Blue reward:0.85\n", - "step: 3438, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3439, Red action: do-nothing, Blue reward:0.85\n", - "step: 3440, Red action: do-nothing, Blue reward:0.85\n", - "step: 3441, Red action: do-nothing, Blue reward:0.85\n", - "step: 3442, Red action: do-nothing, Blue reward:0.85\n", - "step: 3443, Red action: do-nothing, Blue reward:0.85\n", - "step: 3444, Red action: do-nothing, Blue reward:0.85\n", - "step: 3445, Red action: do-nothing, Blue reward:0.85\n", - "step: 3446, Red action: do-nothing, Blue reward:0.85\n", - "step: 3447, Red action: do-nothing, Blue reward:0.85\n", - "step: 3448, Red action: do-nothing, Blue reward:0.85\n", - "step: 3449, Red action: do-nothing, Blue reward:0.85\n", - "step: 3450, Red action: do-nothing, Blue reward:0.85\n", - "step: 3451, Red action: do-nothing, Blue reward:0.85\n", - "step: 3452, Red action: do-nothing, Blue reward:0.85\n", - "step: 3453, Red action: do-nothing, Blue reward:0.85\n", - "step: 3454, Red action: do-nothing, Blue reward:0.85\n", - "step: 3455, Red action: do-nothing, Blue reward:0.85\n", - "step: 3456, Red action: do-nothing, Blue reward:0.85\n", - "step: 3457, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3458, Red action: do-nothing, Blue reward:0.85\n", - "step: 3459, Red action: do-nothing, Blue reward:0.85\n", - "step: 3460, Red action: do-nothing, Blue reward:0.85\n", - "step: 3461, Red action: do-nothing, Blue reward:0.85\n", - "step: 3462, Red action: do-nothing, Blue reward:0.85\n", - "step: 3463, Red action: do-nothing, Blue reward:0.85\n", - "step: 3464, Red action: do-nothing, Blue reward:0.85\n", - "step: 3465, Red action: do-nothing, Blue reward:0.85\n", - "step: 3466, Red action: do-nothing, Blue reward:0.85\n", - "step: 3467, Red action: do-nothing, Blue reward:0.85\n", - "step: 3468, Red action: do-nothing, Blue reward:0.85\n", - "step: 3469, Red action: do-nothing, Blue reward:0.85\n", - "step: 3470, Red action: do-nothing, Blue reward:0.85\n", - "step: 3471, Red action: do-nothing, Blue reward:0.85\n", - "step: 3472, Red action: do-nothing, Blue reward:0.85\n", - "step: 3473, Red action: do-nothing, Blue reward:0.85\n", - "step: 3474, Red action: do-nothing, Blue reward:0.85\n", - "step: 3475, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3476, Red action: do-nothing, Blue reward:0.85\n", - "step: 3477, Red action: do-nothing, Blue reward:0.85\n", - "step: 3478, Red action: do-nothing, Blue reward:0.85\n", - "step: 3479, Red action: do-nothing, Blue reward:0.85\n", - "step: 3480, Red action: do-nothing, Blue reward:0.85\n", - "step: 3481, Red action: do-nothing, Blue reward:0.85\n", - "step: 3482, Red action: do-nothing, Blue reward:0.85\n", - "step: 3483, Red action: do-nothing, Blue reward:0.85\n", - "step: 3484, Red action: do-nothing, Blue reward:0.85\n", - "step: 3485, Red action: do-nothing, Blue reward:0.85\n", - "step: 3486, Red action: do-nothing, Blue reward:0.85\n", - "step: 3487, Red action: do-nothing, Blue reward:0.85\n", - "step: 3488, Red action: do-nothing, Blue reward:0.85\n", - "step: 3489, Red action: do-nothing, Blue reward:0.85\n", - "step: 3490, Red action: do-nothing, Blue reward:0.85\n", - "step: 3491, Red action: do-nothing, Blue reward:0.85\n", - "step: 3492, Red action: do-nothing, Blue reward:0.85\n", - "step: 3493, Red action: do-nothing, Blue reward:0.85\n", - "step: 3494, Red action: do-nothing, Blue reward:0.85\n", - "step: 3495, Red action: do-nothing, Blue reward:0.85\n", - "step: 3496, Red action: do-nothing, Blue reward:0.85\n", - "step: 3497, Red action: do-nothing, Blue reward:0.85\n", - "step: 3498, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3499, Red action: do-nothing, Blue reward:0.85\n", - "step: 3500, Red action: do-nothing, Blue reward:0.85\n", - "step: 3501, Red action: do-nothing, Blue reward:0.85\n", - "step: 3502, Red action: do-nothing, Blue reward:0.85\n", - "step: 3503, Red action: do-nothing, Blue reward:0.85\n", - "step: 3504, Red action: do-nothing, Blue reward:0.85\n", - "step: 3505, Red action: do-nothing, Blue reward:0.85\n", - "step: 3506, Red action: do-nothing, Blue reward:0.85\n", - "step: 3507, Red action: do-nothing, Blue reward:0.85\n", - "step: 3508, Red action: do-nothing, Blue reward:0.85\n", - "step: 3509, Red action: do-nothing, Blue reward:0.85\n", - "step: 3510, Red action: do-nothing, Blue reward:0.85\n", - "step: 3511, Red action: do-nothing, Blue reward:0.85\n", - "step: 3512, Red action: do-nothing, Blue reward:0.85\n", - "step: 3513, Red action: do-nothing, Blue reward:0.85\n", - "step: 3514, Red action: do-nothing, Blue reward:0.85\n", - "step: 3515, Red action: do-nothing, Blue reward:0.85\n", - "step: 3516, Red action: do-nothing, Blue reward:0.85\n", - "step: 3517, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3518, Red action: do-nothing, Blue reward:0.85\n", - "step: 3519, Red action: do-nothing, Blue reward:0.85\n", - "step: 3520, Red action: do-nothing, Blue reward:0.85\n", - "step: 3521, Red action: do-nothing, Blue reward:0.85\n", - "step: 3522, Red action: do-nothing, Blue reward:0.85\n", - "step: 3523, Red action: do-nothing, Blue reward:0.85\n", - "step: 3524, Red action: do-nothing, Blue reward:0.85\n", - "step: 3525, Red action: do-nothing, Blue reward:0.85\n", - "step: 3526, Red action: do-nothing, Blue reward:0.85\n", - "step: 3527, Red action: do-nothing, Blue reward:0.85\n", - "step: 3528, Red action: do-nothing, Blue reward:0.85\n", - "step: 3529, Red action: do-nothing, Blue reward:0.85\n", - "step: 3530, Red action: do-nothing, Blue reward:0.85\n", - "step: 3531, Red action: do-nothing, Blue reward:0.85\n", - "step: 3532, Red action: do-nothing, Blue reward:0.85\n", - "step: 3533, Red action: do-nothing, Blue reward:0.85\n", - "step: 3534, Red action: do-nothing, Blue reward:0.85\n", - "step: 3535, Red action: do-nothing, Blue reward:0.85\n", - "step: 3536, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3537, Red action: do-nothing, Blue reward:0.85\n", - "step: 3538, Red action: do-nothing, Blue reward:0.85\n", - "step: 3539, Red action: do-nothing, Blue reward:0.85\n", - "step: 3540, Red action: do-nothing, Blue reward:0.85\n", - "step: 3541, Red action: do-nothing, Blue reward:0.85\n", - "step: 3542, Red action: do-nothing, Blue reward:0.85\n", - "step: 3543, Red action: do-nothing, Blue reward:0.85\n", - "step: 3544, Red action: do-nothing, Blue reward:0.85\n", - "step: 3545, Red action: do-nothing, Blue reward:0.85\n", - "step: 3546, Red action: do-nothing, Blue reward:0.85\n", - "step: 3547, Red action: do-nothing, Blue reward:0.85\n", - "step: 3548, Red action: do-nothing, Blue reward:0.85\n", - "step: 3549, Red action: do-nothing, Blue reward:0.85\n", - "step: 3550, Red action: do-nothing, Blue reward:0.85\n", - "step: 3551, Red action: do-nothing, Blue reward:0.85\n", - "step: 3552, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3553, Red action: do-nothing, Blue reward:0.85\n", - "step: 3554, Red action: do-nothing, Blue reward:0.85\n", - "step: 3555, Red action: do-nothing, Blue reward:0.85\n", - "step: 3556, Red action: do-nothing, Blue reward:0.85\n", - "step: 3557, Red action: do-nothing, Blue reward:0.85\n", - "step: 3558, Red action: do-nothing, Blue reward:0.85\n", - "step: 3559, Red action: do-nothing, Blue reward:0.85\n", - "step: 3560, Red action: do-nothing, Blue reward:0.85\n", - "step: 3561, Red action: do-nothing, Blue reward:0.85\n", - "step: 3562, Red action: do-nothing, Blue reward:0.85\n", - "step: 3563, Red action: do-nothing, Blue reward:0.85\n", - "step: 3564, Red action: do-nothing, Blue reward:0.85\n", - "step: 3565, Red action: do-nothing, Blue reward:0.85\n", - "step: 3566, Red action: do-nothing, Blue reward:0.85\n", - "step: 3567, Red action: do-nothing, Blue reward:0.85\n", - "step: 3568, Red action: do-nothing, Blue reward:0.85\n", - "step: 3569, Red action: do-nothing, Blue reward:0.85\n", - "step: 3570, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3571, Red action: do-nothing, Blue reward:0.85\n", - "step: 3572, Red action: do-nothing, Blue reward:0.85\n", - "step: 3573, Red action: do-nothing, Blue reward:0.85\n", - "step: 3574, Red action: do-nothing, Blue reward:0.85\n", - "step: 3575, Red action: do-nothing, Blue reward:0.85\n", - "step: 3576, Red action: do-nothing, Blue reward:0.85\n", - "step: 3577, Red action: do-nothing, Blue reward:0.85\n", - "step: 3578, Red action: do-nothing, Blue reward:0.85\n", - "step: 3579, Red action: do-nothing, Blue reward:0.85\n", - "step: 3580, Red action: do-nothing, Blue reward:0.85\n", - "step: 3581, Red action: do-nothing, Blue reward:0.85\n", - "step: 3582, Red action: do-nothing, Blue reward:0.85\n", - "step: 3583, Red action: do-nothing, Blue reward:0.85\n", - "step: 3584, Red action: do-nothing, Blue reward:0.85\n", - "step: 3585, Red action: do-nothing, Blue reward:0.85\n", - "step: 3586, Red action: do-nothing, Blue reward:0.85\n", - "step: 3587, Red action: do-nothing, Blue reward:0.85\n", - "step: 3588, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3589, Red action: do-nothing, Blue reward:0.85\n", - "step: 3590, Red action: do-nothing, Blue reward:0.85\n", - "step: 3591, Red action: do-nothing, Blue reward:0.85\n", - "step: 3592, Red action: do-nothing, Blue reward:0.85\n", - "step: 3593, Red action: do-nothing, Blue reward:0.85\n", - "step: 3594, Red action: do-nothing, Blue reward:0.85\n", - "step: 3595, Red action: do-nothing, Blue reward:0.85\n", - "step: 3596, Red action: do-nothing, Blue reward:0.85\n", - "step: 3597, Red action: do-nothing, Blue reward:0.85\n", - "step: 3598, Red action: do-nothing, Blue reward:0.85\n", - "step: 3599, Red action: do-nothing, Blue reward:0.85\n", - "step: 3600, Red action: do-nothing, Blue reward:0.85\n", - "step: 3601, Red action: do-nothing, Blue reward:0.85\n", - "step: 3602, Red action: do-nothing, Blue reward:0.85\n", - "step: 3603, Red action: do-nothing, Blue reward:0.85\n", - "step: 3604, Red action: do-nothing, Blue reward:0.85\n", - "step: 3605, Red action: do-nothing, Blue reward:0.85\n", - "step: 3606, Red action: do-nothing, Blue reward:0.85\n", - "step: 3607, Red action: do-nothing, Blue reward:0.85\n", - "step: 3608, Red action: do-nothing, Blue reward:0.85\n", - "step: 3609, Red action: do-nothing, Blue reward:0.85\n", - "step: 3610, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3611, Red action: do-nothing, Blue reward:0.85\n", - "step: 3612, Red action: do-nothing, Blue reward:0.85\n", - "step: 3613, Red action: do-nothing, Blue reward:0.85\n", - "step: 3614, Red action: do-nothing, Blue reward:0.85\n", - "step: 3615, Red action: do-nothing, Blue reward:0.85\n", - "step: 3616, Red action: do-nothing, Blue reward:0.85\n", - "step: 3617, Red action: do-nothing, Blue reward:0.85\n", - "step: 3618, Red action: do-nothing, Blue reward:0.85\n", - "step: 3619, Red action: do-nothing, Blue reward:0.85\n", - "step: 3620, Red action: do-nothing, Blue reward:0.85\n", - "step: 3621, Red action: do-nothing, Blue reward:0.85\n", - "step: 3622, Red action: do-nothing, Blue reward:0.85\n", - "step: 3623, Red action: do-nothing, Blue reward:0.85\n", - "step: 3624, Red action: do-nothing, Blue reward:0.85\n", - "step: 3625, Red action: do-nothing, Blue reward:0.85\n", - "step: 3626, Red action: do-nothing, Blue reward:0.85\n", - "step: 3627, Red action: do-nothing, Blue reward:0.85\n", - "step: 3628, Red action: do-nothing, Blue reward:0.85\n", - "step: 3629, Red action: do-nothing, Blue reward:0.85\n", - "step: 3630, Red action: do-nothing, Blue reward:0.85\n", - "step: 3631, Red action: do-nothing, Blue reward:0.85\n", - "step: 3632, Red action: do-nothing, Blue reward:0.85\n", - "step: 3633, Red action: do-nothing, Blue reward:0.85\n", - "step: 3634, Red action: do-nothing, Blue reward:0.85\n", - "step: 3635, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3636, Red action: do-nothing, Blue reward:0.85\n", - "step: 3637, Red action: do-nothing, Blue reward:0.85\n", - "step: 3638, Red action: do-nothing, Blue reward:0.85\n", - "step: 3639, Red action: do-nothing, Blue reward:0.85\n", - "step: 3640, Red action: do-nothing, Blue reward:0.85\n", - "step: 3641, Red action: do-nothing, Blue reward:0.85\n", - "step: 3642, Red action: do-nothing, Blue reward:0.85\n", - "step: 3643, Red action: do-nothing, Blue reward:0.85\n", - "step: 3644, Red action: do-nothing, Blue reward:0.85\n", - "step: 3645, Red action: do-nothing, Blue reward:0.85\n", - "step: 3646, Red action: do-nothing, Blue reward:0.85\n", - "step: 3647, Red action: do-nothing, Blue reward:0.85\n", - "step: 3648, Red action: do-nothing, Blue reward:0.85\n", - "step: 3649, Red action: do-nothing, Blue reward:0.85\n", - "step: 3650, Red action: do-nothing, Blue reward:0.85\n", - "step: 3651, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3652, Red action: do-nothing, Blue reward:0.85\n", - "step: 3653, Red action: do-nothing, Blue reward:0.85\n", - "step: 3654, Red action: do-nothing, Blue reward:0.85\n", - "step: 3655, Red action: do-nothing, Blue reward:0.85\n", - "step: 3656, Red action: do-nothing, Blue reward:0.85\n", - "step: 3657, Red action: do-nothing, Blue reward:0.85\n", - "step: 3658, Red action: do-nothing, Blue reward:0.85\n", - "step: 3659, Red action: do-nothing, Blue reward:0.85\n", - "step: 3660, Red action: do-nothing, Blue reward:0.85\n", - "step: 3661, Red action: do-nothing, Blue reward:0.85\n", - "step: 3662, Red action: do-nothing, Blue reward:0.85\n", - "step: 3663, Red action: do-nothing, Blue reward:0.85\n", - "step: 3664, Red action: do-nothing, Blue reward:0.85\n", - "step: 3665, Red action: do-nothing, Blue reward:0.85\n", - "step: 3666, Red action: do-nothing, Blue reward:0.85\n", - "step: 3667, Red action: do-nothing, Blue reward:0.85\n", - "step: 3668, Red action: do-nothing, Blue reward:0.85\n", - "step: 3669, Red action: do-nothing, Blue reward:0.85\n", - "step: 3670, Red action: do-nothing, Blue reward:0.85\n", - "step: 3671, Red action: do-nothing, Blue reward:0.85\n", - "step: 3672, Red action: do-nothing, Blue reward:0.85\n", - "step: 3673, Red action: do-nothing, Blue reward:0.85\n", - "step: 3674, Red action: do-nothing, Blue reward:0.85\n", - "step: 3675, Red action: do-nothing, Blue reward:0.85\n", - "step: 3676, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3677, Red action: do-nothing, Blue reward:0.85\n", - "step: 3678, Red action: do-nothing, Blue reward:0.85\n", - "step: 3679, Red action: do-nothing, Blue reward:0.85\n", - "step: 3680, Red action: do-nothing, Blue reward:0.85\n", - "step: 3681, Red action: do-nothing, Blue reward:0.85\n", - "step: 3682, Red action: do-nothing, Blue reward:0.85\n", - "step: 3683, Red action: do-nothing, Blue reward:0.85\n", - "step: 3684, Red action: do-nothing, Blue reward:0.85\n", - "step: 3685, Red action: do-nothing, Blue reward:0.85\n", - "step: 3686, Red action: do-nothing, Blue reward:0.85\n", - "step: 3687, Red action: do-nothing, Blue reward:0.85\n", - "step: 3688, Red action: do-nothing, Blue reward:0.85\n", - "step: 3689, Red action: do-nothing, Blue reward:0.85\n", - "step: 3690, Red action: do-nothing, Blue reward:0.85\n", - "step: 3691, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3692, Red action: do-nothing, Blue reward:0.85\n", - "step: 3693, Red action: do-nothing, Blue reward:0.85\n", - "step: 3694, Red action: do-nothing, Blue reward:0.85\n", - "step: 3695, Red action: do-nothing, Blue reward:0.85\n", - "step: 3696, Red action: do-nothing, Blue reward:0.85\n", - "step: 3697, Red action: do-nothing, Blue reward:0.85\n", - "step: 3698, Red action: do-nothing, Blue reward:0.85\n", - "step: 3699, Red action: do-nothing, Blue reward:0.85\n", - "step: 3700, Red action: do-nothing, Blue reward:0.85\n", - "step: 3701, Red action: do-nothing, Blue reward:0.85\n", - "step: 3702, Red action: do-nothing, Blue reward:0.85\n", - "step: 3703, Red action: do-nothing, Blue reward:0.85\n", - "step: 3704, Red action: do-nothing, Blue reward:0.85\n", - "step: 3705, Red action: do-nothing, Blue reward:0.85\n", - "step: 3706, Red action: do-nothing, Blue reward:0.85\n", - "step: 3707, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3708, Red action: do-nothing, Blue reward:0.85\n", - "step: 3709, Red action: do-nothing, Blue reward:0.85\n", - "step: 3710, Red action: do-nothing, Blue reward:0.85\n", - "step: 3711, Red action: do-nothing, Blue reward:0.85\n", - "step: 3712, Red action: do-nothing, Blue reward:0.85\n", - "step: 3713, Red action: do-nothing, Blue reward:0.85\n", - "step: 3714, Red action: do-nothing, Blue reward:0.85\n", - "step: 3715, Red action: do-nothing, Blue reward:0.85\n", - "step: 3716, Red action: do-nothing, Blue reward:0.85\n", - "step: 3717, Red action: do-nothing, Blue reward:0.85\n", - "step: 3718, Red action: do-nothing, Blue reward:0.85\n", - "step: 3719, Red action: do-nothing, Blue reward:0.85\n", - "step: 3720, Red action: do-nothing, Blue reward:0.85\n", - "step: 3721, Red action: do-nothing, Blue reward:0.85\n", - "step: 3722, Red action: do-nothing, Blue reward:0.85\n", - "step: 3723, Red action: do-nothing, Blue reward:0.85\n", - "step: 3724, Red action: do-nothing, Blue reward:0.85\n", - "step: 3725, Red action: do-nothing, Blue reward:0.85\n", - "step: 3726, Red action: do-nothing, Blue reward:0.85\n", - "step: 3727, Red action: do-nothing, Blue reward:0.85\n", - "step: 3728, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3729, Red action: do-nothing, Blue reward:0.85\n", - "step: 3730, Red action: do-nothing, Blue reward:0.85\n", - "step: 3731, Red action: do-nothing, Blue reward:0.85\n", - "step: 3732, Red action: do-nothing, Blue reward:0.85\n", - "step: 3733, Red action: do-nothing, Blue reward:0.85\n", - "step: 3734, Red action: do-nothing, Blue reward:0.85\n", - "step: 3735, Red action: do-nothing, Blue reward:0.85\n", - "step: 3736, Red action: do-nothing, Blue reward:0.85\n", - "step: 3737, Red action: do-nothing, Blue reward:0.85\n", - "step: 3738, Red action: do-nothing, Blue reward:0.85\n", - "step: 3739, Red action: do-nothing, Blue reward:0.85\n", - "step: 3740, Red action: do-nothing, Blue reward:0.85\n", - "step: 3741, Red action: do-nothing, Blue reward:0.85\n", - "step: 3742, Red action: do-nothing, Blue reward:0.85\n", - "step: 3743, Red action: do-nothing, Blue reward:0.85\n", - "step: 3744, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3745, Red action: do-nothing, Blue reward:0.85\n", - "step: 3746, Red action: do-nothing, Blue reward:0.85\n", - "step: 3747, Red action: do-nothing, Blue reward:0.85\n", - "step: 3748, Red action: do-nothing, Blue reward:0.85\n", - "step: 3749, Red action: do-nothing, Blue reward:0.85\n", - "step: 3750, Red action: do-nothing, Blue reward:0.85\n", - "step: 3751, Red action: do-nothing, Blue reward:0.85\n", - "step: 3752, Red action: do-nothing, Blue reward:0.85\n", - "step: 3753, Red action: do-nothing, Blue reward:0.85\n", - "step: 3754, Red action: do-nothing, Blue reward:0.85\n", - "step: 3755, Red action: do-nothing, Blue reward:0.85\n", - "step: 3756, Red action: do-nothing, Blue reward:0.85\n", - "step: 3757, Red action: do-nothing, Blue reward:0.85\n", - "step: 3758, Red action: do-nothing, Blue reward:0.85\n", - "step: 3759, Red action: do-nothing, Blue reward:0.85\n", - "step: 3760, Red action: do-nothing, Blue reward:0.85\n", - "step: 3761, Red action: do-nothing, Blue reward:0.85\n", - "step: 3762, Red action: do-nothing, Blue reward:0.85\n", - "step: 3763, Red action: do-nothing, Blue reward:0.85\n", - "step: 3764, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3765, Red action: do-nothing, Blue reward:0.85\n", - "step: 3766, Red action: do-nothing, Blue reward:0.85\n", - "step: 3767, Red action: do-nothing, Blue reward:0.85\n", - "step: 3768, Red action: do-nothing, Blue reward:0.85\n", - "step: 3769, Red action: do-nothing, Blue reward:0.85\n", - "step: 3770, Red action: do-nothing, Blue reward:0.85\n", - "step: 3771, Red action: do-nothing, Blue reward:0.85\n", - "step: 3772, Red action: do-nothing, Blue reward:0.85\n", - "step: 3773, Red action: do-nothing, Blue reward:0.85\n", - "step: 3774, Red action: do-nothing, Blue reward:0.85\n", - "step: 3775, Red action: do-nothing, Blue reward:0.85\n", - "step: 3776, Red action: do-nothing, Blue reward:0.85\n", - "step: 3777, Red action: do-nothing, Blue reward:0.85\n", - "step: 3778, Red action: do-nothing, Blue reward:0.85\n", - "step: 3779, Red action: do-nothing, Blue reward:0.85\n", - "step: 3780, Red action: do-nothing, Blue reward:0.85\n", - "step: 3781, Red action: do-nothing, Blue reward:0.85\n", - "step: 3782, Red action: do-nothing, Blue reward:0.85\n", - "step: 3783, Red action: do-nothing, Blue reward:0.85\n", - "step: 3784, Red action: node-application-execute, Blue reward:0.85\n", - "step: 3785, Red action: do-nothing, Blue reward:0.85\n", - "step: 3786, Red action: do-nothing, Blue reward:0.85\n", - "step: 3787, Red action: do-nothing, Blue reward:0.85\n", - "step: 3788, Red action: do-nothing, Blue reward:0.85\n", - "step: 3789, Red action: do-nothing, Blue reward:0.85\n", - "step: 3790, Red action: do-nothing, Blue reward:0.85\n", - "step: 3791, Red action: do-nothing, Blue reward:0.85\n", - "step: 3792, Red action: do-nothing, Blue reward:0.85\n", - "step: 3793, Red action: do-nothing, Blue reward:0.85\n", - "step: 3794, Red action: do-nothing, Blue reward:0.85\n", - "step: 3795, Red action: do-nothing, Blue reward:0.85\n", - "step: 3796, Red action: do-nothing, Blue reward:0.85\n", - "step: 3797, Red action: do-nothing, Blue reward:0.85\n", - "step: 3798, Red action: do-nothing, Blue reward:0.85\n", - "step: 3799, Red action: do-nothing, Blue reward:0.85\n", - "step: 3800, Red action: do-nothing, Blue reward:0.85\n", - "step: 3801, Red action: do-nothing, Blue reward:0.85\n", - "step: 3802, Red action: do-nothing, Blue reward:0.85\n" - ] - }, - { - "ename": "KeyboardInterrupt", - "evalue": "", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[11], line 11\u001b[0m\n\u001b[1;32m 8\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mstep: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00menv\u001b[38;5;241m.\u001b[39mgame\u001b[38;5;241m.\u001b[39mstep_counter\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m, Red action: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00minfo[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124magent_actions\u001b[39m\u001b[38;5;124m'\u001b[39m][\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mdata_manipulation_attacker\u001b[39m\u001b[38;5;124m'\u001b[39m]\u001b[38;5;241m.\u001b[39maction\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m, Blue reward:\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mreward\u001b[38;5;132;01m:\u001b[39;00m\u001b[38;5;124m.2f\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m )\n\u001b[1;32m 10\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28mabs\u001b[39m(reward \u001b[38;5;241m-\u001b[39m \u001b[38;5;241m0.8\u001b[39m) \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m1e-5\u001b[39m:\n\u001b[0;32m---> 11\u001b[0m obs, reward, terminated, truncated, info \u001b[38;5;241m=\u001b[39m \u001b[43menv\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstep\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m)\u001b[49m \u001b[38;5;66;03m# do nothing\u001b[39;00m\n\u001b[1;32m 12\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mstep: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00menv\u001b[38;5;241m.\u001b[39mgame\u001b[38;5;241m.\u001b[39mstep_counter\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m, Red action: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00minfo[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124magent_actions\u001b[39m\u001b[38;5;124m'\u001b[39m][\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mdata_manipulation_attacker\u001b[39m\u001b[38;5;124m'\u001b[39m]\u001b[38;5;241m.\u001b[39maction\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m, Blue reward:\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mreward\u001b[38;5;132;01m:\u001b[39;00m\u001b[38;5;124m.2f\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m )\n\u001b[1;32m 13\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m env\u001b[38;5;241m.\u001b[39mgame\u001b[38;5;241m.\u001b[39mstep_counter \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m10000\u001b[39m:\n", - "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/session/environment.py:111\u001b[0m, in \u001b[0;36mPrimaiteGymEnv.step\u001b[0;34m(self, action)\u001b[0m\n\u001b[1;32m 109\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgame\u001b[38;5;241m.\u001b[39mapply_agent_actions()\n\u001b[1;32m 110\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgame\u001b[38;5;241m.\u001b[39madvance_timestep()\n\u001b[0;32m--> 111\u001b[0m state \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgame\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_sim_state\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 112\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgame\u001b[38;5;241m.\u001b[39mupdate_agents(state)\n\u001b[1;32m 114\u001b[0m next_obs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_get_obs() \u001b[38;5;66;03m# this doesn't update observation, just gets the current observation\u001b[39;00m\n", - "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/game/game.py:151\u001b[0m, in \u001b[0;36mPrimaiteGame.get_sim_state\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 149\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mget_sim_state\u001b[39m(\u001b[38;5;28mself\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Dict:\n\u001b[1;32m 150\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"Get the current state of the simulation.\"\"\"\u001b[39;00m\n\u001b[0;32m--> 151\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msimulation\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdescribe_state\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/simulator/sim_container.py:57\u001b[0m, in \u001b[0;36mSimulation.describe_state\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 46\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 47\u001b[0m \u001b[38;5;124;03mProduce a dictionary describing the current state of this object.\u001b[39;00m\n\u001b[1;32m 48\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 52\u001b[0m \u001b[38;5;124;03m:rtype: Dict\u001b[39;00m\n\u001b[1;32m 53\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 54\u001b[0m state \u001b[38;5;241m=\u001b[39m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39mdescribe_state()\n\u001b[1;32m 55\u001b[0m state\u001b[38;5;241m.\u001b[39mupdate(\n\u001b[1;32m 56\u001b[0m {\n\u001b[0;32m---> 57\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnetwork\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mnetwork\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdescribe_state\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m,\n\u001b[1;32m 58\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdomain\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdomain\u001b[38;5;241m.\u001b[39mdescribe_state(),\n\u001b[1;32m 59\u001b[0m }\n\u001b[1;32m 60\u001b[0m )\n\u001b[1;32m 61\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m state\n", - "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/simulator/network/container.py:260\u001b[0m, in \u001b[0;36mNetwork.describe_state\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 252\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 253\u001b[0m \u001b[38;5;124;03mProduce a dictionary describing the current state of the Network.\u001b[39;00m\n\u001b[1;32m 254\u001b[0m \n\u001b[1;32m 255\u001b[0m \u001b[38;5;124;03m:return: A dictionary capturing the current state of the Network and its child objects.\u001b[39;00m\n\u001b[1;32m 256\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 257\u001b[0m state \u001b[38;5;241m=\u001b[39m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39mdescribe_state()\n\u001b[1;32m 258\u001b[0m state\u001b[38;5;241m.\u001b[39mupdate(\n\u001b[1;32m 259\u001b[0m {\n\u001b[0;32m--> 260\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnodes\u001b[39m\u001b[38;5;124m\"\u001b[39m: {node\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mhostname: node\u001b[38;5;241m.\u001b[39mdescribe_state() \u001b[38;5;28;01mfor\u001b[39;00m node \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnodes\u001b[38;5;241m.\u001b[39mvalues()},\n\u001b[1;32m 261\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mlinks\u001b[39m\u001b[38;5;124m\"\u001b[39m: {},\n\u001b[1;32m 262\u001b[0m }\n\u001b[1;32m 263\u001b[0m )\n\u001b[1;32m 264\u001b[0m \u001b[38;5;66;03m# Update the links one-by-one. The key is a 4-tuple of `hostname_a, port_a, hostname_b, port_b`\u001b[39;00m\n\u001b[1;32m 265\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m _, link \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlinks\u001b[38;5;241m.\u001b[39mitems():\n", - "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/simulator/network/container.py:260\u001b[0m, in \u001b[0;36m\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 252\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 253\u001b[0m \u001b[38;5;124;03mProduce a dictionary describing the current state of the Network.\u001b[39;00m\n\u001b[1;32m 254\u001b[0m \n\u001b[1;32m 255\u001b[0m \u001b[38;5;124;03m:return: A dictionary capturing the current state of the Network and its child objects.\u001b[39;00m\n\u001b[1;32m 256\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 257\u001b[0m state \u001b[38;5;241m=\u001b[39m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39mdescribe_state()\n\u001b[1;32m 258\u001b[0m state\u001b[38;5;241m.\u001b[39mupdate(\n\u001b[1;32m 259\u001b[0m {\n\u001b[0;32m--> 260\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnodes\u001b[39m\u001b[38;5;124m\"\u001b[39m: {node\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mhostname: \u001b[43mnode\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdescribe_state\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mfor\u001b[39;00m node \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnodes\u001b[38;5;241m.\u001b[39mvalues()},\n\u001b[1;32m 261\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mlinks\u001b[39m\u001b[38;5;124m\"\u001b[39m: {},\n\u001b[1;32m 262\u001b[0m }\n\u001b[1;32m 263\u001b[0m )\n\u001b[1;32m 264\u001b[0m \u001b[38;5;66;03m# Update the links one-by-one. The key is a 4-tuple of `hostname_a, port_a, hostname_b, port_b`\u001b[39;00m\n\u001b[1;32m 265\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m _, link \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlinks\u001b[38;5;241m.\u001b[39mitems():\n", - "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/simulator/network/hardware/base.py:1900\u001b[0m, in \u001b[0;36mNode.describe_state\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1881\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 1882\u001b[0m \u001b[38;5;124;03mProduce a dictionary describing the current state of this object.\u001b[39;00m\n\u001b[1;32m 1883\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1887\u001b[0m \u001b[38;5;124;03m:rtype: Dict\u001b[39;00m\n\u001b[1;32m 1888\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 1889\u001b[0m state \u001b[38;5;241m=\u001b[39m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39mdescribe_state()\n\u001b[1;32m 1890\u001b[0m state\u001b[38;5;241m.\u001b[39mupdate(\n\u001b[1;32m 1891\u001b[0m {\n\u001b[1;32m 1892\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhostname\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mhostname,\n\u001b[1;32m 1893\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moperating_state\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39moperating_state\u001b[38;5;241m.\u001b[39mvalue,\n\u001b[1;32m 1894\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mNICs\u001b[39m\u001b[38;5;124m\"\u001b[39m: {\n\u001b[1;32m 1895\u001b[0m eth_num: network_interface\u001b[38;5;241m.\u001b[39mdescribe_state()\n\u001b[1;32m 1896\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m eth_num, network_interface \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnetwork_interface\u001b[38;5;241m.\u001b[39mitems()\n\u001b[1;32m 1897\u001b[0m },\n\u001b[1;32m 1898\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfile_system\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfile_system\u001b[38;5;241m.\u001b[39mdescribe_state(),\n\u001b[1;32m 1899\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mapplications\u001b[39m\u001b[38;5;124m\"\u001b[39m: {app\u001b[38;5;241m.\u001b[39mname: app\u001b[38;5;241m.\u001b[39mdescribe_state() \u001b[38;5;28;01mfor\u001b[39;00m app \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mapplications\u001b[38;5;241m.\u001b[39mvalues()},\n\u001b[0;32m-> 1900\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mservices\u001b[39m\u001b[38;5;124m\"\u001b[39m: {svc\u001b[38;5;241m.\u001b[39mname: svc\u001b[38;5;241m.\u001b[39mdescribe_state() \u001b[38;5;28;01mfor\u001b[39;00m svc \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mservices\u001b[38;5;241m.\u001b[39mvalues()},\n\u001b[1;32m 1901\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mprocess\u001b[39m\u001b[38;5;124m\"\u001b[39m: {proc\u001b[38;5;241m.\u001b[39mname: proc\u001b[38;5;241m.\u001b[39mdescribe_state() \u001b[38;5;28;01mfor\u001b[39;00m proc \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprocesses\u001b[38;5;241m.\u001b[39mvalues()},\n\u001b[1;32m 1902\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mrevealed_to_red\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mrevealed_to_red,\n\u001b[1;32m 1903\u001b[0m }\n\u001b[1;32m 1904\u001b[0m )\n\u001b[1;32m 1905\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m state\n", - "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/simulator/network/hardware/base.py:1900\u001b[0m, in \u001b[0;36m\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 1881\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 1882\u001b[0m \u001b[38;5;124;03mProduce a dictionary describing the current state of this object.\u001b[39;00m\n\u001b[1;32m 1883\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1887\u001b[0m \u001b[38;5;124;03m:rtype: Dict\u001b[39;00m\n\u001b[1;32m 1888\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 1889\u001b[0m state \u001b[38;5;241m=\u001b[39m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39mdescribe_state()\n\u001b[1;32m 1890\u001b[0m state\u001b[38;5;241m.\u001b[39mupdate(\n\u001b[1;32m 1891\u001b[0m {\n\u001b[1;32m 1892\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhostname\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mhostname,\n\u001b[1;32m 1893\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moperating_state\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39moperating_state\u001b[38;5;241m.\u001b[39mvalue,\n\u001b[1;32m 1894\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mNICs\u001b[39m\u001b[38;5;124m\"\u001b[39m: {\n\u001b[1;32m 1895\u001b[0m eth_num: network_interface\u001b[38;5;241m.\u001b[39mdescribe_state()\n\u001b[1;32m 1896\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m eth_num, network_interface \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnetwork_interface\u001b[38;5;241m.\u001b[39mitems()\n\u001b[1;32m 1897\u001b[0m },\n\u001b[1;32m 1898\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfile_system\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfile_system\u001b[38;5;241m.\u001b[39mdescribe_state(),\n\u001b[1;32m 1899\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mapplications\u001b[39m\u001b[38;5;124m\"\u001b[39m: {app\u001b[38;5;241m.\u001b[39mname: app\u001b[38;5;241m.\u001b[39mdescribe_state() \u001b[38;5;28;01mfor\u001b[39;00m app \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mapplications\u001b[38;5;241m.\u001b[39mvalues()},\n\u001b[0;32m-> 1900\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mservices\u001b[39m\u001b[38;5;124m\"\u001b[39m: {svc\u001b[38;5;241m.\u001b[39mname: \u001b[43msvc\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdescribe_state\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mfor\u001b[39;00m svc \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mservices\u001b[38;5;241m.\u001b[39mvalues()},\n\u001b[1;32m 1901\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mprocess\u001b[39m\u001b[38;5;124m\"\u001b[39m: {proc\u001b[38;5;241m.\u001b[39mname: proc\u001b[38;5;241m.\u001b[39mdescribe_state() \u001b[38;5;28;01mfor\u001b[39;00m proc \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprocesses\u001b[38;5;241m.\u001b[39mvalues()},\n\u001b[1;32m 1902\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mrevealed_to_red\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mrevealed_to_red,\n\u001b[1;32m 1903\u001b[0m }\n\u001b[1;32m 1904\u001b[0m )\n\u001b[1;32m 1905\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m state\n", - "File \u001b[0;32m~/arcd/PrimAITE/src/primaite/simulator/system/services/ntp/ntp_client.py:61\u001b[0m, in \u001b[0;36mNTPClient.describe_state\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 50\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdescribe_state\u001b[39m(\u001b[38;5;28mself\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Dict:\n\u001b[1;32m 51\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 52\u001b[0m \u001b[38;5;124;03m Describes the current state of the software.\u001b[39;00m\n\u001b[1;32m 53\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 59\u001b[0m \u001b[38;5;124;03m :rtype: Dict\u001b[39;00m\n\u001b[1;32m 60\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m---> 61\u001b[0m state \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241m.\u001b[39mdescribe_state()\n\u001b[1;32m 62\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m state\n", - "\u001b[0;31mKeyboardInterrupt\u001b[0m: " - ] - } - ], + "outputs": [], "source": [ "env.step(13) # Patch the database\n", "print(f\"step: {env.game.step_counter}, Red action: {info['agent_actions']['data_manipulation_attacker'].action}, Blue reward:{reward:.2f}\" )\n", From 8c7f8cd0ecb257df594825201dce6b598f35f5cf Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Tue, 18 Feb 2025 11:36:22 +0000 Subject: [PATCH 19/24] #3075: Fix final two demo notebooks --- .../create-simulation_demo.ipynb | 49 ++++++++++++++++--- .../network_simulator_demo.ipynb | 16 +----- src/primaite/simulator/network/networks.py | 2 +- 3 files changed, 45 insertions(+), 22 deletions(-) diff --git a/src/primaite/simulator/_package_data/create-simulation_demo.ipynb b/src/primaite/simulator/_package_data/create-simulation_demo.ipynb index 690e7856..46f03be6 100644 --- a/src/primaite/simulator/_package_data/create-simulation_demo.ipynb +++ b/src/primaite/simulator/_package_data/create-simulation_demo.ipynb @@ -6,7 +6,7 @@ "source": [ "# Build a simulation using the Python API\n", "\n", - "© Crown-owned copyright 2024, Defence Science and Technology Laboratory UK\n", + "© Crown-owned copyright 2025, Defence Science and Technology Laboratory UK\n", "\n", "Currently, this notebook manipulates the simulation by directly placing objects inside of the attributes of the network and domain. It should be refactored when proper methods exist for adding these objects." ] @@ -70,9 +70,23 @@ "metadata": {}, "outputs": [], "source": [ - "my_pc = Computer(hostname=\"Computer\", ip_address=\"192.168.1.10\", subnet_mask=\"255.255.255.0\")\n", + "my_pc = Computer.from_config(\n", + " config={\n", + " \"type\": \"computer\",\n", + " \"hostname\":\"pc_1\",\n", + " \"ip_address\":\"192.168.1.10\",\n", + " \"subnet_mask\":\"255.255.255.0\",\n", + " }\n", + " )\n", "net.add_node(my_pc)\n", - "my_server = Server(hostname=\"Server\", ip_address=\"192.168.1.11\", subnet_mask=\"255.255.255.0\")\n", + "my_server = Server.from_config(\n", + " config={\n", + " \"type\": \"server\",\n", + " \"hostname\":\"Server\",\n", + " \"ip_address\":\"192.168.1.11\",\n", + " \"subnet_mask\":\"255.255.255.0\"\n", + " }\n", + ")\n", "net.add_node(my_server)\n" ] }, @@ -99,7 +113,13 @@ "metadata": {}, "outputs": [], "source": [ - "my_switch = Switch(hostname=\"switch1\", num_ports=12)\n", + "my_switch = Switch.from_config(\n", + " config = {\n", + " \"type\":\"switch\",\n", + " \"hostname\":\"switch1\",\n", + " \"num_ports\":12\n", + " }\n", + ")\n", "net.add_node(my_switch)\n", "\n", "pc_nic = NIC(ip_address=\"130.1.1.1\", gateway=\"130.1.1.255\", subnet_mask=\"255.255.255.0\")\n", @@ -163,16 +183,30 @@ "metadata": {}, "outputs": [], "source": [ + "from pydantic import Field\n", + "\n", "from pathlib import Path\n", "from primaite.simulator.system.applications.application import Application, ApplicationOperatingState\n", "from primaite.simulator.system.software import SoftwareHealthState, SoftwareCriticality\n", "from primaite.simulator.file_system.file_system import FileSystem\n", "from primaite.utils.validation.ip_protocol import PROTOCOL_LOOKUP\n", "from primaite.utils.validation.port import PORT_LOOKUP\n", + "from primaite.simulator.system.core.sys_log import SysLog\n", "\n", "\n", "# no applications exist yet so we will create our own.\n", "class MSPaint(Application, discriminator=\"MSPaint\"):\n", + " class ConfigSchema(Application.ConfigSchema):\n", + " type: str = \"MSPaint\"\n", + "\n", + " config: ConfigSchema = Field(default_factory=lambda: MSPaint.ConfigSchema())\n", + "\n", + " def __init__(self, **kwargs):\n", + " kwargs[\"name\"] = \"MSPaint\"\n", + " kwargs[\"port\"] = PORT_LOOKUP[\"HTTP\"]\n", + " kwargs[\"protocol\"] = PROTOCOL_LOOKUP[\"NONE\"]\n", + " super().__init__(**kwargs)\n", + "\n", " def describe_state(self):\n", " return super().describe_state()" ] @@ -183,7 +217,8 @@ "metadata": {}, "outputs": [], "source": [ - "mspaint = MSPaint(name = \"mspaint\", health_state_actual=SoftwareHealthState.GOOD, health_state_visible=SoftwareHealthState.GOOD, criticality=SoftwareCriticality.MEDIUM, port=PORT_LOOKUP[\"HTTP\"], protocol = PROTOCOL_LOOKUP[\"NONE\"],operating_state=ApplicationOperatingState.RUNNING,execution_control_status='manual', file_system=FileSystem(sys_log=SysLog(hostname=\"Test\"), sim_root=Path(__name__).parent),)" + "my_pc.software_manager.install(MSPaint)\n", + "mspaint = my_pc.software_manager.software.get(\"MSPaint\")" ] }, { @@ -250,7 +285,7 @@ ], "metadata": { "kernelspec": { - "display_name": ".venv", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -264,7 +299,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.11" + "version": "3.10.12" } }, "nbformat": 4, diff --git a/src/primaite/simulator/_package_data/network_simulator_demo.ipynb b/src/primaite/simulator/_package_data/network_simulator_demo.ipynb index 58c07fe5..be930ac0 100644 --- a/src/primaite/simulator/_package_data/network_simulator_demo.ipynb +++ b/src/primaite/simulator/_package_data/network_simulator_demo.ipynb @@ -7,7 +7,7 @@ "source": [ "# PrimAITE Router Simulation Demo\n", "\n", - "© Crown-owned copyright 2024, Defence Science and Technology Laboratory UK\n", + "© Crown-owned copyright 2025, Defence Science and Technology Laboratory UK\n", "\n", "This demo uses a modified version of the ARCD Use Case 2 Network (seen below) to demonstrate the capabilities of the Network simulator in PrimAITE." ] @@ -650,21 +650,9 @@ ], "metadata": { "kernelspec": { - "display_name": ".venv", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.11" } }, "nbformat": 4, diff --git a/src/primaite/simulator/network/networks.py b/src/primaite/simulator/network/networks.py index 5e677679..5d558e80 100644 --- a/src/primaite/simulator/network/networks.py +++ b/src/primaite/simulator/network/networks.py @@ -295,7 +295,7 @@ def arcd_uc2_network() -> Network: # Security Suite security_suite_cfg = { "type": "server", - "hostname": "backup_server", + "hostname": "security_suite", "ip_address": "192.168.1.110", "subnet_mask": "255.255.255.0", "default_gateway": "192.168.1.1", From 72f80e55da0a17766975bf1d73aaff75786ec303 Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Tue, 18 Feb 2025 11:51:35 +0000 Subject: [PATCH 20/24] #3075: Move demo notebook to correct folder. --- .../_package_data => notebooks}/create-simulation_demo.ipynb | 0 .../_package_data => notebooks}/network_simulator_demo.ipynb | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename src/primaite/{simulator/_package_data => notebooks}/create-simulation_demo.ipynb (100%) rename src/primaite/{simulator/_package_data => notebooks}/network_simulator_demo.ipynb (100%) diff --git a/src/primaite/simulator/_package_data/create-simulation_demo.ipynb b/src/primaite/notebooks/create-simulation_demo.ipynb similarity index 100% rename from src/primaite/simulator/_package_data/create-simulation_demo.ipynb rename to src/primaite/notebooks/create-simulation_demo.ipynb diff --git a/src/primaite/simulator/_package_data/network_simulator_demo.ipynb b/src/primaite/notebooks/network_simulator_demo.ipynb similarity index 100% rename from src/primaite/simulator/_package_data/network_simulator_demo.ipynb rename to src/primaite/notebooks/network_simulator_demo.ipynb From 90d7af3ff9fb0bc5fd60515b3d30752d7da27493 Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Tue, 18 Feb 2025 12:42:33 +0000 Subject: [PATCH 21/24] #3075: Remove references to src/primaite/simulator/_package_data after notebook move. --- .azure/azure-ci-build-pipeline.yaml | 4 ---- MANIFEST.in | 1 - 2 files changed, 5 deletions(-) diff --git a/.azure/azure-ci-build-pipeline.yaml b/.azure/azure-ci-build-pipeline.yaml index 624c9ca4..20c5b239 100644 --- a/.azure/azure-ci-build-pipeline.yaml +++ b/.azure/azure-ci-build-pipeline.yaml @@ -113,8 +113,6 @@ stages: - script: | pytest --nbmake -n=auto src/primaite/notebooks --junit-xml=./notebook-tests/notebooks.xml notebooks_exit_code=$? - pytest --nbmake -n=auto src/primaite/simulator/_package_data --junit-xml=./notebook-tests/package-notebooks.xml - package_notebooks_exit_code=$? # Fail step if either of these do not have exit code 0 if [ $notebooks_exit_code -ne 0 ] || [ $package_notebooks_exit_code -ne 0 ]; then exit 1 @@ -126,8 +124,6 @@ stages: - script: | pytest --nbmake -n=auto src/primaite/notebooks --junit-xml=./notebook-tests/notebooks.xml set notebooks_exit_code=%ERRORLEVEL% - pytest --nbmake -n=auto src/primaite/simulator/_package_data --junit-xml=./notebook-tests/package-notebooks.xml - set package_notebooks_exit_code=%ERRORLEVEL% rem Fail step if either of these do not have exit code 0 if %notebooks_exit_code% NEQ 0 exit /b 1 if %package_notebooks_exit_code% NEQ 0 exit /b 1 diff --git a/MANIFEST.in b/MANIFEST.in index 2ac7b306..51ae4ddf 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,2 @@ include src/primaite/setup/_package_data/primaite_config.yaml include src/primaite/config/_package_data/*.yaml -include src/primaite/simulator/_package_data/*.ipynb From f5cfaeedab633498274bbc05e34679dea19fec7a Mon Sep 17 00:00:00 2001 From: Archer Bowen Date: Fri, 21 Feb 2025 14:57:48 +0000 Subject: [PATCH 22/24] #3075 Cleaned up yaml snippets that were rendering as a strings. It seems that adding '...' to yaml snippets seems to end up causing the snippets to render as strings rather than actual yaml. --- ...ommand-and-Control-E2E-Demonstration.ipynb | 66 ++++++++----------- 1 file changed, 28 insertions(+), 38 deletions(-) diff --git a/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb b/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb index 6dc8e077..f187c8d5 100644 --- a/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb +++ b/src/primaite/notebooks/Command-and-Control-E2E-Demonstration.ipynb @@ -120,6 +120,9 @@ " options:\n", " node_name: web_server\n", " c2_server_ip_address: 192.168.10.21\n", + " keep_alive_frequency: 10\n", + " masquerade_protocol: tcp\n", + " masquerade_port: dns\n", " 9:\n", " action: configure-c2-beacon\n", " options:\n", @@ -235,23 +238,18 @@ "The yaml snippet below shows all the relevant agent options for this action:\n", "\n", "```yaml\n", + "\n", " action_space:\n", - " options:\n", - " nodes: # Node List\n", - " - node_name: web_server\n", - " ...\n", - " ...\n", " action_map:\n", - " ...\n", " 2:\n", " action: configure-c2-beacon\n", " options:\n", - " node_id: 0 # Node Index\n", - " config: # Further information about these config options can be found at the bottom of this notebook.\n", - " c2_server_ip_address: 192.168.10.21\n", - " keep_alive_frequency:\n", - " masquerade_protocol:\n", - " masquerade_port:\n", + " node_name: web_server\n", + " c2_server_ip_address: 192.168.10.21 # Further information about these config options can be found at the bottom of this notebook.\n", + " keep_alive_frequency:\n", + " masquerade_protocol:\n", + " masquerade_port:\n", + "\n", "```" ] }, @@ -279,20 +277,12 @@ "\n", "```yaml\n", " action_space:\n", - " options:\n", - " nodes: # Node List\n", - " - node_name: web_server\n", - " applications: \n", - " - application_name: c2-beacon\n", - " ...\n", - " ...\n", " action_map:\n", - " ...\n", " 3:\n", " action: node-application-execute\n", " options:\n", - " node_id: 0\n", - " application_id: 0\n", + " node_name: web_server\n", + " application_name: c2-beacon\n", "```" ] }, @@ -346,7 +336,6 @@ "``` yaml\n", " action_space:\n", " action_map:\n", - " ...\n", " 4:\n", " action: c2-server-terminal-command\n", " options:\n", @@ -396,14 +385,12 @@ "``` yaml\n", " action_space:\n", " action_map:\n", - " ...\n", " 5:\n", " action: c2-server-ransomware-configure\n", " options:\n", - " node_id: 1\n", - " config:\n", - " server_ip_address: 192.168.1.14\n", - " payload: ENCRYPT\n", + " node_name: client_1\n", + " server_ip_address: 192.168.1.14\n", + " payload: ENCRYPT\n", "```\n" ] }, @@ -442,7 +429,6 @@ "``` yaml\n", " action_space:\n", " action_map:\n", - " ...\n", " 6:\n", " action: c2-server-data-exfiltrate\n", " options:\n", @@ -451,7 +437,7 @@ " target_folder_name: \"database\"\n", " exfiltration_folder_name: \"spoils\"\n", " target_ip_address: \"192.168.1.14\"\n", - " username: \"admin\",\n", + " username: \"admin\"\n", " password: \"admin\"\n", "\n", "```" @@ -500,7 +486,6 @@ "\n", "``` yaml\n", " action_space:\n", - " ...\n", " action_map:\n", " 7:\n", " action: c2-server-ransomware-launch\n", @@ -1313,13 +1298,18 @@ "source": [ "As demonstrated earlier, red agents can use the ``configure-c2-beacon`` action to configure these settings mid episode through the configuration options:\n", "\n", - "``` YAML\n", - "...\n", - " action: configure-c2-beacon\n", - " options:\n", - " node_name: web_server\n", - " config:\n", + "```YAML\n", + "\n", + " action_space:\n", + " action_map:\n", + " 8:\n", + " action: configure-c2-beacon\n", + " options:\n", + " node_name: web_server\n", " c2_server_ip_address: 192.168.10.21\n", + " keep_alive_frequency: 10\n", + " masquerade_protocol: tcp\n", + " masquerade_port: dns\n", "```" ] }, @@ -1684,7 +1674,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": ".venv", "language": "python", "name": "python3" }, From 892cdb82b91d621b92328dc0b2014348a08aac94 Mon Sep 17 00:00:00 2001 From: Nick Todd Date: Tue, 25 Feb 2025 14:25:27 +0000 Subject: [PATCH 23/24] #3075: Code review changes. --- .azure/azure-ci-build-pipeline.yaml | 7 +++---- .../Data-Manipulation-Customising-Red-Agent.ipynb | 4 ++-- tests/assets/configs/basic_switched_network.yaml | 4 ++-- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.azure/azure-ci-build-pipeline.yaml b/.azure/azure-ci-build-pipeline.yaml index 20c5b239..b6f24777 100644 --- a/.azure/azure-ci-build-pipeline.yaml +++ b/.azure/azure-ci-build-pipeline.yaml @@ -113,8 +113,8 @@ stages: - script: | pytest --nbmake -n=auto src/primaite/notebooks --junit-xml=./notebook-tests/notebooks.xml notebooks_exit_code=$? - # Fail step if either of these do not have exit code 0 - if [ $notebooks_exit_code -ne 0 ] || [ $package_notebooks_exit_code -ne 0 ]; then + # Fail step if exit code not equal to 0 + if [ $notebooks_exit_code -ne 0 ]; then exit 1 fi displayName: 'Run notebooks on Linux and macOS' @@ -124,9 +124,8 @@ stages: - script: | pytest --nbmake -n=auto src/primaite/notebooks --junit-xml=./notebook-tests/notebooks.xml set notebooks_exit_code=%ERRORLEVEL% - rem Fail step if either of these do not have exit code 0 + rem Fail step if exit code not equal to 0 if %notebooks_exit_code% NEQ 0 exit /b 1 - if %package_notebooks_exit_code% NEQ 0 exit /b 1 displayName: 'Run notebooks on Windows' condition: eq(variables['Agent.OS'], 'Windows_NT') diff --git a/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb b/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb index 1eb814e3..5982407b 100644 --- a/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb +++ b/src/primaite/notebooks/Data-Manipulation-Customising-Red-Agent.ipynb @@ -289,10 +289,10 @@ " action_space:\n", " action_map:\n", " 0:\n", - " action: do_nothing\n", + " action: do-nothing\n", " options: {}\n", " 1:\n", - " action: node_application_execute\n", + " action: node-application-execute\n", " options:\n", " node_name: client_1\n", " application_name: DataManipulationBot\n", diff --git a/tests/assets/configs/basic_switched_network.yaml b/tests/assets/configs/basic_switched_network.yaml index 2af48ea2..7339c34c 100644 --- a/tests/assets/configs/basic_switched_network.yaml +++ b/tests/assets/configs/basic_switched_network.yaml @@ -37,10 +37,10 @@ agents: action_space: action_map: 0: - action: do_nothing + action: do-nothing options: {} 1: - action: node_application_execute + action: node-application-execute options: node_name: client_2 application_name: web-browser From 1605b0989962a9c00c44cd63dc470a17b432e35e Mon Sep 17 00:00:00 2001 From: Marek Wolan Date: Wed, 26 Feb 2025 10:49:01 +0000 Subject: [PATCH 24/24] Apply suggestions from code review --- tests/assets/configs/basic_switched_network.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/assets/configs/basic_switched_network.yaml b/tests/assets/configs/basic_switched_network.yaml index 7339c34c..b57ed3e0 100644 --- a/tests/assets/configs/basic_switched_network.yaml +++ b/tests/assets/configs/basic_switched_network.yaml @@ -112,7 +112,7 @@ agents: action_space: action_map: 0: - action: do_nothing + action: do-nothing options: {} reward_function: