3086 UC7 Migration - All YAMLS, tests and notebooks. A few lingering issues such as the OS-SCAN not working and agent logs not appearing.

2025-02-10 14:39:28 +00:00
parent 0d1edf0362
commit d8c8aa40a4
118 changed files with 21789 additions and 368 deletions
--- a/docs/Makefile
+++ b/docs/Makefile
@@ -1,4 +1,4 @@
-# © Crown-owned copyright 2024, Defence Science and Technology Laboratory UK
+# © Crown-owned copyright 2025, Defence Science and Technology Laboratory UK
 # Minimal makefile for Sphinx documentation
 # You can set these variables from the command line, and also
 # from the environment for the first two.
--- a/docs/build-sphinx-docs-to-github-pages.sh
+++ b/docs/build-sphinx-docs-to-github-pages.sh
@@ -1,4 +1,4 @@
-# © Crown-owned copyright 2024, Defence Science and Technology Laboratory UK
+# © Crown-owned copyright 2025, Defence Science and Technology Laboratory UK
 #!/bin/bash
 set -x

--- a/docs/source/configuration/agents.rst
+++ b/docs/source/configuration/agents.rst
@@ -21,7 +21,7 @@ Agents can be scripted (deterministic and stochastic), or controlled by a reinfo
            team: GREEN
            type: probabilistic-agent
            observation_space:
-                type: UC2GreenObservation
+                type: UC2GreenObservation  # TODO: what
            action_space:
            reward_function:
                reward_components:
@@ -160,3 +160,4 @@ If ``True``, gymnasium flattening will be performed on the observation space bef
 -----------------

 Agents will record their action log for each step. This is a summary of what the agent did, along with response information from requests within the simulation.
+A summary of the actions taken by the agent can be viewed using the `show_history()` function. By default, this will display all actions taken apart from ``DONOTHING``.
--- a/docs/source/configuration/simulation/nodes/common/common_node_attributes.rst
+++ b/docs/source/configuration/simulation/nodes/common/common_node_attributes.rst
@@ -54,6 +54,39 @@ Optional. Default value is ``3``.

 The number of time steps required to occur in order for the node to cycle from ``ON`` to ``SHUTTING_DOWN`` and then finally ``OFF``.

+``file_system``
+---------------
+
+Optional.
+
+The file system of the node. This configuration allows nodes to be initialised with files and/or folders.
+
+The file system takes a list of folders and files.
+
+Example:
+
+.. code-block:: yaml
+
+    simulation:
+      network:
+        nodes:
+        - hostname: client_1
+          type: computer
+          ip_address: 192.168.10.11
+          subnet_mask: 255.255.255.0
+          default_gateway: 192.168.10.1
+          file_system:
+            - empty_folder  # example of an empty folder
+            - downloads:
+              - "test_1.txt"  # files in the downloads folder
+              - "test_2.txt"
+            - root:
+              - passwords:  # example of file with size and type
+                  size: 69  # size in bytes
+                  type: TXT  # See FileType for list of available file types
+
+List of file types: :py:mod:`primaite.simulator.file_system.file_type.FileType`
+
 ``users``
 ---------

--- a/docs/source/configuration/simulation/nodes/network_examples.rst
+++ b/docs/source/configuration/simulation/nodes/network_examples.rst
@@ -1177,8 +1177,8 @@ ACLs permitting or denying traffic as per our configured ACL rules.
    some_tech_storage_srv = network.get_node_by_hostname("some_tech_storage_srv")
    some_tech_storage_srv.file_system.create_file(file_name="test.png")

-    pc_1_ftp_client: FTPClient = network.get_node_by_hostname("pc_1").software_manager.software["FTPClient"]
-    pc_2_ftp_client: FTPClient = network.get_node_by_hostname("pc_2").software_manager.software["FTPClient"]
+    pc_1_ftp_client: FTPClient = network.get_node_by_hostname("pc_1").software_manager.software["ftp-client"]
+    pc_2_ftp_client: FTPClient = network.get_node_by_hostname("pc_2").software_manager.software["ftp-client"]

    assert not pc_1_ftp_client.request_file(
        dest_ip_address=some_tech_storage_srv.network_interface[1].ip_address,
@@ -1224,7 +1224,7 @@ ACLs permitting or denying traffic as per our configured ACL rules.

    web_server: Server = network.get_node_by_hostname("some_tech_web_srv")

-    web_ftp_client: FTPClient = web_server.software_manager.software["FTPClient"]
+    web_ftp_client: FTPClient = web_server.software_manager.software["ftp-client"]

    assert not web_ftp_client.request_file(
        dest_ip_address=some_tech_storage_srv.network_interface[1].ip_address,
@@ -1269,7 +1269,7 @@ ACLs permitting or denying traffic as per our configured ACL rules.
    some_tech_storage_srv.file_system.create_file(file_name="test.png")

    some_tech_snr_dev_pc: Computer = network.get_node_by_hostname("some_tech_snr_dev_pc")
-    snr_dev_ftp_client: FTPClient = some_tech_snr_dev_pc.software_manager.software["FTPClient"]
+    snr_dev_ftp_client: FTPClient = some_tech_snr_dev_pc.software_manager.software["ftp-client"]

    assert snr_dev_ftp_client.request_file(
        dest_ip_address=some_tech_storage_srv.network_interface[1].ip_address,
@@ -1294,7 +1294,7 @@ ACLs permitting or denying traffic as per our configured ACL rules.
    some_tech_storage_srv.file_system.create_file(file_name="test.png")

    some_tech_jnr_dev_pc: Computer = network.get_node_by_hostname("some_tech_jnr_dev_pc")
-    jnr_dev_ftp_client: FTPClient = some_tech_jnr_dev_pc.software_manager.software["FTPClient"]
+    jnr_dev_ftp_client: FTPClient = some_tech_jnr_dev_pc.software_manager.software["ftp-client"]

    assert not jnr_dev_ftp_client.request_file(
        dest_ip_address=some_tech_storage_srv.network_interface[1].ip_address,
@@ -1337,7 +1337,7 @@ ACLs permitting or denying traffic as per our configured ACL rules.
    some_tech_storage_srv.file_system.create_file(file_name="test.png")

    some_tech_hr_pc: Computer = network.get_node_by_hostname("some_tech_hr_1")
-    hr_ftp_client: FTPClient = some_tech_hr_pc.software_manager.software["FTPClient"]
+    hr_ftp_client: FTPClient = some_tech_hr_pc.software_manager.software["ftp-client"]

    assert not hr_ftp_client.request_file(
        dest_ip_address=some_tech_storage_srv.network_interface[1].ip_address,
--- a/docs/source/configuration/simulation/nodes/router.rst
+++ b/docs/source/configuration/simulation/nodes/router.rst
@@ -74,7 +74,7 @@ The subnet mask setting for the port.
 ``acl``
 -------

-Sets up the ACL rules for the router.
+Sets up the ACL rules for the router to apply to layer-3 traffic. These are not applied to layer-2 traffic such as ARP.

 e.g.

@@ -85,10 +85,6 @@ e.g.
        ...
        acl:
            1:
-                action: PERMIT
-                src_port: ARP
-                dst_port: ARP
-            2:
                action: PERMIT
                protocol: ICMP

--- a/docs/source/how_to_guides/extensible_agents.rst
+++ b/docs/source/how_to_guides/extensible_agents.rst
@@ -46,17 +46,13 @@ The core features that should be implemented in any new agent are detailed below

 		- ref: example_green_agent
 		  team: GREEN
-		  type: ExampleAgent
+		  type: example-agent

 		  action_space:
 			action_map:
 				0:
 					action: do-nothing
 					options: {}
-		  reward_function:
-		  	reward_components:
-				- type: dummy
-
 		  agent_settings:
 			start_step: 25
 			frequency: 20
--- a/docs/source/how_to_guides/extensible_nodes.rst
+++ b/docs/source/how_to_guides/extensible_nodes.rst
@@ -26,9 +26,9 @@ class Router(NetworkNode, identifier="router"):
    """ Represents a network router within the simulation, managing routing and forwarding of IP packets across network interfaces."""

    SYSTEM_SOFTWARE: ClassVar[Dict] = {
-        "UserSessionManager": UserSessionManager,
-        "UserManager": UserManager,
-        "Terminal": Terminal,
+        "user-session-manager": UserSessionManager,
+        "user-manager": UserManager,
+        "terminal": Terminal,
    }

    network_interfaces: Dict[str, RouterInterface] = {}
@@ -52,4 +52,4 @@ class Router(NetworkNode, identifier="router"):
 Changes to YAML file.
 =====================

-While effort has been made to ensure that nodes defined within configuration YAML files for use with PrimAITE 3.X remain compatible with PrimAITE v4+, it is encouraged to review for minor changes needed. 
+While effort has been made to ensure that nodes defined within configuration YAML files for use with PrimAITE 3.X remain compatible with PrimAITE v4+, it is encouraged to review for minor changes needed.
--- a/docs/source/request_system.rst
+++ b/docs/source/request_system.rst
@@ -2,6 +2,8 @@

    © Crown-owned copyright 2025, Defence Science and Technology Laboratory UK

+.. _request_system:
+
 Request System
 **************

--- a/docs/source/simulation_components/network/network.rst
+++ b/docs/source/simulation_components/network/network.rst
@@ -97,19 +97,19 @@ we'll use the following Network that has a client, server, two switches, and a r
    network.connect(endpoint_a=switch_2.network_interface[1], endpoint_b=client_1.network_interface[1])
    network.connect(endpoint_a=switch_1.network_interface[1], endpoint_b=server_1.network_interface[1])

-8. Add ACL rules on the Router to allow ARP and ICMP traffic.
+8. Add an ACL rule on the Router to allow ICMP traffic.

 .. code-block:: python

    router_1.acl.add_rule(
        action=ACLAction.PERMIT,
-        src_port=Port["ARP"],
-        dst_port=Port["ARP"],
+        src_port=PORT_LOOKUP["ARP"],
+        dst_port=PORT_LOOKUP["ARP"],
        position=22
    )

    router_1.acl.add_rule(
        action=ACLAction.PERMIT,
-        protocol=IPProtocol["ICMP"],
+        protocol=PROTOCOL_LOOKUP["ICMP"],
        position=23
    )
--- a/docs/source/simulation_components/network/nodes/wireless_router.rst
+++ b/docs/source/simulation_components/network/nodes/wireless_router.rst
@@ -102,8 +102,8 @@ ICMP traffic, ensuring basic network connectivity and ping functionality.
    network.connect(pc_a.network_interface[1], router_1.router_interface)

    # Configure Router 1 ACLs
-    router_1.acl.add_rule(action=ACLAction.PERMIT, src_port=Port["ARP"], dst_port=Port["ARP"], position=22)
-    router_1.acl.add_rule(action=ACLAction.PERMIT, protocol=IPProtocol["ICMP"], position=23)
+    router_1.acl.add_rule(action=ACLAction.PERMIT, src_port=PORT_LOOKUP["ARP"], dst_port=PORT_LOOKUP["ARP"], position=22)
+    router_1.acl.add_rule(action=ACLAction.PERMIT, protocol=PROTOCOL_LOOKUP["ICMP"], position=23)

    # Configure PC B
    pc_b = Computer(
--- a/docs/source/simulation_components/system/applications/c2_suite.rst
+++ b/docs/source/simulation_components/system/applications/c2_suite.rst
@@ -183,7 +183,7 @@ Python
    # Example command: Installing and configuring Ransomware:

    ransomware_installation_command = { "commands": [
-            ["software_manager","application","install","RansomwareScript"],
+            ["software_manager","application","install","ransomware-script"],
        ],
        "username": "admin",
        "password": "admin",
--- a/docs/source/simulation_components/system/applications/data_manipulation_bot.rst
+++ b/docs/source/simulation_components/system/applications/data_manipulation_bot.rst
@@ -77,7 +77,7 @@ Python
    network.connect(endpoint_b=client_1.network_interface[1], endpoint_a=switch_2.network_interface[1])
    client_1.software_manager.install(DatabaseClient)
    client_1.software_manager.install(DataManipulationBot)
-    data_manipulation_bot: DataManipulationBot = client_1.software_manager.software.get("DataManipulationBot")
+    data_manipulation_bot: DataManipulationBot = client_1.software_manager.software.get("data-manipulation-bot")
    data_manipulation_bot.configure(server_ip_address=IPv4Address("192.168.1.14"), payload="DELETE")
    data_manipulation_bot.run()

@@ -98,7 +98,7 @@ If not using the data manipulation bot manually, it needs to be used with a data
          type: red-database-corrupting-agent

          observation_space:
-            type: UC2RedObservation
+            type: uc2-red-observation #TODO what
            options:
              nodes:
                - node_name: client_1
--- a/docs/source/simulation_components/system/applications/database_client.rst
+++ b/docs/source/simulation_components/system/applications/database_client.rst
@@ -59,7 +59,7 @@ Python
    # install DatabaseClient
    client.software_manager.install(DatabaseClient)

-    database_client: DatabaseClient = client.software_manager.software.get("DatabaseClient")
+    database_client: DatabaseClient = client.software_manager.software.get("database-sclient")

    # Configure the DatabaseClient
    database_client.configure(server_ip_address=IPv4Address("192.168.0.1"))  # address of the DatabaseService
--- a/docs/source/simulation_components/system/applications/ransomware_script.rst
+++ b/docs/source/simulation_components/system/applications/ransomware_script.rst
@@ -62,7 +62,7 @@ Python
    network.connect(endpoint_b=client_1.network_interface[1], endpoint_a=switch_2.network_interface[1])
    client_1.software_manager.install(DatabaseClient)
    client_1.software_manager.install(RansomwareScript)
-    RansomwareScript: RansomwareScript = client_1.software_manager.software.get("RansomwareScript")
+    RansomwareScript: RansomwareScript = client_1.software_manager.software.get("ransomware-script")
    RansomwareScript.configure(server_ip_address=IPv4Address("192.168.1.14"))
    RansomwareScript.execute()

--- a/docs/source/simulation_components/system/applications/web_browser.rst
+++ b/docs/source/simulation_components/system/applications/web_browser.rst
@@ -61,7 +61,7 @@ The :ref:`DNSClient` must be configured to use the :ref:`DNSServer`. The :ref:`D

    # Install WebBrowser on computer
    computer.software_manager.install(WebBrowser)
-    web_browser: WebBrowser = computer.software_manager.software.get("WebBrowser")
+    web_browser: WebBrowser = computer.software_manager.software.get("web-browser")
    web_browser.run()

    # configure the WebBrowser
--- a/docs/source/simulation_components/system/services/database_service.rst
+++ b/docs/source/simulation_components/system/services/database_service.rst
@@ -66,7 +66,7 @@ Python

    # Install DatabaseService on server
    server.software_manager.install(DatabaseService)
-    db_service: DatabaseService = server.software_manager.software.get("DatabaseService")
+    db_service: DatabaseService = server.software_manager.software.get("database-service")
    db_service.start()

    # configure DatabaseService
--- a/docs/source/simulation_components/system/services/dns_client.rst
+++ b/docs/source/simulation_components/system/services/dns_client.rst
@@ -56,7 +56,7 @@ Python

    # Install DNSClient on server
    server.software_manager.install(DNSClient)
-    dns_client: DNSClient = server.software_manager.software.get("DNSClient")
+    dns_client: DNSClient = server.software_manager.software.get("dns-client")
    dns_client.start()

    # configure DatabaseService
--- a/docs/source/simulation_components/system/services/dns_server.rst
+++ b/docs/source/simulation_components/system/services/dns_server.rst
@@ -53,7 +53,7 @@ Python

    # Install DNSServer on server
    server.software_manager.install(DNSServer)
-    dns_server: DNSServer = server.software_manager.software.get("DNSServer")
+    dns_server: DNSServer = server.software_manager.software.get("dns-server")
    dns_server.start()

    # configure DatabaseService
--- a/docs/source/simulation_components/system/services/ftp_client.rst
+++ b/docs/source/simulation_components/system/services/ftp_client.rst
@@ -60,7 +60,7 @@ Python

    # Install FTPClient on server
    server.software_manager.install(FTPClient)
-    ftp_client: FTPClient = server.software_manager.software.get("FTPClient")
+    ftp_client: FTPClient = server.software_manager.software.get("ftp-client")
    ftp_client.start()


--- a/docs/source/simulation_components/system/services/ftp_server.rst
+++ b/docs/source/simulation_components/system/services/ftp_server.rst
@@ -55,7 +55,7 @@ Python

    # Install FTPServer on server
    server.software_manager.install(FTPServer)
-    ftp_server: FTPServer = server.software_manager.software.get("FTPServer")
+    ftp_server: FTPServer = server.software_manager.software.get("ftp-server")
    ftp_server.start()

    ftp_server.server_password = "test"
--- a/docs/source/simulation_components/system/services/ntp_client.rst
+++ b/docs/source/simulation_components/system/services/ntp_client.rst
@@ -53,7 +53,7 @@ Python

    # Install NTPClient on server
    server.software_manager.install(NTPClient)
-    ntp_client: NTPClient = server.software_manager.software.get("NTPClient")
+    ntp_client: NTPClient = server.software_manager.software.get("ntp-client")
    ntp_client.start()

    ntp_client.configure(ntp_server_ip_address=IPv4Address("192.168.0.10"))
--- a/docs/source/simulation_components/system/services/ntp_server.rst
+++ b/docs/source/simulation_components/system/services/ntp_server.rst
@@ -55,7 +55,7 @@ Python

    # Install NTPServer on server
    server.software_manager.install(NTPServer)
-    ntp_server: NTPServer = server.software_manager.software.get("NTPServer")
+    ntp_server: NTPServer = server.software_manager.software.get("ntp-server")
    ntp_server.start()


--- a/docs/source/simulation_components/system/services/terminal.rst
+++ b/docs/source/simulation_components/system/services/terminal.rst
@@ -23,6 +23,14 @@ Key capabilities
 - Simulates common Terminal processes/commands.
 - Leverages the Service base class for install/uninstall, status tracking etc.

+Usage
+"""""
+
+ - Pre-Installs on any `Node` component (with the exception of `Switches`).
+ - Terminal Clients connect, execute commands and disconnect from remote nodes.
+ - Ensures that users are logged in to the component before executing any commands.
+ - Service runs on SSH port 22 by default.
+ - Enables Agents to send commands both remotely and locally.

 Implementation
 """"""""""""""
@@ -30,19 +38,112 @@ Implementation
 - Manages remote connections in a dictionary by session ID.
 - Processes commands, forwarding to the ``RequestManager`` or ``SessionManager`` where appropriate.
 - Extends Service class.
-  - A detailed guide on the implementation and functionality of the Terminal class can be found in the "Terminal-Processing" jupyter notebook.
+
+A detailed guide on the implementation and functionality of the Terminal class can be found in the "Terminal-Processing" jupyter notebook.
+
+Command Format
+^^^^^^^^^^^^^^
+
+Terminals implement their commands through leveraging the pre-existing :ref:`request_system`.
+
+Due to this Terminals will only accept commands passed within the ``RequestFormat``.
+
+:py:class:`primaite.game.interface.RequestFormat`
+
+For example, ``terminal`` command actions when used in ``yaml`` format are formatted as follows:
+
+.. code-block:: yaml
+
+    command:
+        - "file_system"
+        - "create"
+        - "file"
+        - "downloads"
+        - "cat.png"
+        - "False
+
+This is then loaded from yaml into a dictionary containing the terminal command:
+
+.. code-block:: python
+
+    {"command":["file_system", "create", "file", "downloads", "cat.png", "False"]}
+
+Which is then passed to the ``Terminals`` Request Manager to be executed.
+
+Game Layer Usage (Agents)
+========================
+
+The below code examples demonstrate how to use terminal related actions in yaml files.
+
+yaml
+""""
+
+``node-send-local-command``
+"""""""""""""""""""""""""""
+
+Agents can execute local commands without needing to perform a separate remote login action (``node-session-remote-login``).
+
+.. code-block:: yaml
+
+    ...
+        ...
+          action: node-send-local-command
+          options:
+            node_id: 0
+            username: admin
+            password: admin
+            command: # Example command - Creates a file called 'cat.png' in the downloads folder.
+              - "file_system"
+              - "create"
+              - "file"
+              - "downloads"
+              - "cat.png"
+              - "False"


-Usage
-"""""
+``node-session-remote-login``
+"""""""""""""""""

- - Pre-Installs on all ``Nodes`` (with the exception of ``Switches``).
- - Terminal Clients connect, execute commands and disconnect from remote nodes.
- - Ensures that users are logged in to the component before executing any commands.
- - Service runs on SSH port 22 by default.
+Agents are able to use the terminal to login into remote nodes via ``SSH`` which allows for agents to execute commands on remote hosts.
+
+.. code-block:: yaml
+
+    ...
+        ...
+          action: node-session-remote-login
+          options:
+            node_id: 0
+            username: admin
+            password: admin
+            remote_ip: 192.168.0.10 # Example Ip Address. (The remote host's IP that will be used by ssh)
+
+
+``node-send-remote-command``
+""""""""""""""""""""""""""""
+
+After remotely logging into another host, an agent can use the ``node-send-remote-command`` to execute commands across the network remotely.
+
+.. code-block:: yaml
+
+    ...
+        ...
+          action: node-send-remote-command
+          options:
+            node_id: 0
+            remote_ip: 192.168.0.10
+            command:
+              - "file_system"
+              - "create"
+              - "file"
+              - "downloads"
+              - "cat.png"
+              - "False"
+
+
+
+Simulation Layer Usage
+======================

-Usage
-=====

 The below code examples demonstrate how to create a terminal, a remote terminal, and how to send a basic application install command to a remote node.

@@ -65,7 +166,7 @@ Python
        operating_state=NodeOperatingState.ON,
    )

-    terminal: Terminal = client.software_manager.software.get("Terminal")
+    terminal: Terminal = client.software_manager.software.get("terminal")

 Creating Remote Terminal Connection
 """""""""""""""""""""""""""""""""""
@@ -86,7 +187,7 @@ Creating Remote Terminal Connection
    node_b.power_on()
    network.connect(node_a.network_interface[1], node_b.network_interface[1])

-    terminal_a: Terminal = node_a.software_manager.software.get("Terminal")
+    terminal_a: Terminal = node_a.software_manager.software.get("terminal")


    term_a_term_b_remote_connection: RemoteTerminalConnection = terminal_a.login(username="admin", password="Admin123!", ip_address="192.168.0.11")
@@ -112,12 +213,12 @@ Executing a basic application install command
    node_b.power_on()
    network.connect(node_a.network_interface[1], node_b.network_interface[1])

-    terminal_a: Terminal = node_a.software_manager.software.get("Terminal")
+    terminal_a: Terminal = node_a.software_manager.software.get("terminal")


    term_a_term_b_remote_connection: RemoteTerminalConnection = terminal_a.login(username="admin", password="Admin123!", ip_address="192.168.0.11")

-    term_a_term_b_remote_connection.execute(["software_manager", "application", "install", "RansomwareScript"])
+    term_a_term_b_remote_connection.execute(["software_manager", "application", "install", "ransomware-script"])



@@ -140,7 +241,7 @@ Creating a folder on a remote node
    node_b.power_on()
    network.connect(node_a.network_interface[1], node_b.network_interface[1])

-    terminal_a: Terminal = node_a.software_manager.software.get("Terminal")
+    terminal_a: Terminal = node_a.software_manager.software.get("terminal")


    term_a_term_b_remote_connection: RemoteTerminalConnection = terminal_a.login(username="admin", password="Admin123!", ip_address="192.168.0.11")
@@ -167,7 +268,7 @@ Disconnect from Remote Node
    node_b.power_on()
    network.connect(node_a.network_interface[1], node_b.network_interface[1])

-    terminal_a: Terminal = node_a.software_manager.software.get("Terminal")
+    terminal_a: Terminal = node_a.software_manager.software.get("terminal")


    term_a_term_b_remote_connection: RemoteTerminalConnection = terminal_a.login(username="admin", password="Admin123!", ip_address="192.168.0.11")
--- a/docs/source/simulation_components/system/services/web_server.rst
+++ b/docs/source/simulation_components/system/services/web_server.rst
@@ -56,7 +56,7 @@ Python

    # Install WebServer on server
    server.software_manager.install(WebServer)
-    web_server: WebServer = server.software_manager.software.get("WebServer")
+    web_server: WebServer = server.software_manager.software.get("web-server")
    web_server.start()

 Via Configuration
--- a/docs/source/simulation_components/system/software.rst
+++ b/docs/source/simulation_components/system/software.rst
@@ -30,7 +30,7 @@ See :ref:`Node Start up and Shut down`

    node.software_manager.install(WebServer)

-    web_server: WebServer = node.software_manager.software.get("WebServer")
+    web_server: WebServer = node.software_manager.software.get("web-server")
    assert web_server.operating_state is ServiceOperatingState.RUNNING # service is immediately ran after install

    node.power_off()