Merged PR 523: #2782: initial impl of files in nodes

## Summary
Added ability to add folders and files to nodes via configuration

## Test process
https://dev.azure.com/ma-dev-uk/PrimAITE/_git/PrimAITE/pullrequest/523?_a=files&path=/tests/integration_tests/configuration_file_parsing/test_node_file_system_config.py

## Checklist
- [X] PR is linked to a **work item**
- [X] **acceptance criteria** of linked ticket are met
- [X] performed **self-review** of the code
- [X] written **tests** for any new functionality added with this PR
- [X] updated the **documentation** if this PR changes or adds functionality
- [ ] written/updated **design docs** if this PR implements new functionality
- [ ] updated the **change log**
- [X] ran **pre-commit** checks for code style
- [ ] attended to any **TO-DOs** left in the code

#2782: initial impl of files in nodes

Related work items: #2782

docs/source/configuration/simulation/nodes/common/common_node_attributes.rst

@@ -54,6 +54,39 @@ Optional. Default value is ``3``.
 
 The number of time steps required to occur in order for the node to cycle from ``ON`` to ``SHUTTING_DOWN`` and then finally ``OFF``.
 
+``file_system``
+---------------
+
+Optional.
+
+The file system of the node. This configuration allows nodes to be initialised with files and/or folders.
+
+The file system takes a list of folders and files.
+
+Example:
+
+.. code-block:: yaml
+
+    simulation:
+      network:
+        nodes:
+        - hostname: client_1
+          type: computer
+          ip_address: 192.168.10.11
+          subnet_mask: 255.255.255.0
+          default_gateway: 192.168.10.1
+          file_system:
+            - empty_folder  # example of an empty folder
+            - downloads:
+              - "test_1.txt"  # files in the downloads folder
+              - "test_2.txt"
+            - root:
+              - passwords:  # example of file with size and type
+                  size: 69  # size in bytes
+                  type: TXT  # See FileType for list of available file types
+
+List of file types: :py:mod:`primaite.simulator.file_system.file_type.FileType`
+
 ``users``
 ---------
 

src/primaite/config/_package_data/data_manipulation.yaml

@@ -843,7 +843,6 @@ simulation:
       dns_server: 192.168.1.10
       services:
       - type: FTPServer
-
     - hostname: security_suite
       type: server
       ip_address: 192.168.1.110

src/primaite/game/game.py

@@ -17,6 +17,7 @@ from primaite.game.agent.scripted_agents.random_agent import PeriodicAgent
 from primaite.game.agent.scripted_agents.tap001 import TAP001
 from primaite.game.science import graph_has_cycle, topological_sort
 from primaite.simulator import SIM_OUTPUT
+from primaite.simulator.file_system.file_type import FileType
 from primaite.simulator.network.airspace import AirSpaceFrequency
 from primaite.simulator.network.hardware.base import NetworkInterface, NodeOperatingState, UserManager
 from primaite.simulator.network.hardware.nodes.host.computer import Computer
@@ -330,6 +331,26 @@ class PrimaiteGame:
                 _LOGGER.error(msg)
                 raise ValueError(msg)
 
+            # handle node file system
+            if node_cfg.get("file_system"):
+                for folder_idx, folder_obj in enumerate(node_cfg.get("file_system")):
+                    # if the folder is not a Dict, create an empty folder
+                    if not isinstance(folder_obj, Dict):
+                        new_node.file_system.create_folder(folder_name=folder_obj)
+                    else:
+                        folder_name = next(iter(folder_obj))
+                        for file_idx, file_obj in enumerate(node_cfg["file_system"][folder_idx][folder_name]):
+                            if not isinstance(file_obj, Dict):
+                                new_node.file_system.create_file(folder_name=folder_name, file_name=file_obj)
+                            else:
+                                file_name = next(iter(file_obj))
+                                new_node.file_system.create_file(
+                                    folder_name=folder_name,
+                                    file_name=file_name,
+                                    size=file_obj[file_name].get("size", 0),
+                                    file_type=FileType[file_obj[file_name].get("type", "UNKNOWN").upper()],
+                                )
+
             if "users" in node_cfg and new_node.software_manager.software.get("UserManager"):
                 user_manager: UserManager = new_node.software_manager.software["UserManager"]  # noqa
                 for user_cfg in node_cfg["users"]:

tests/assets/configs/nodes_with_initial_files.yaml

@@ -0,0 +1,259 @@
+#  Basic Switched network
+#
+#  --------------      --------------      --------------
+#  |  client_1  |------|  switch_1  |------|  client_2  |
+#  --------------      --------------      --------------
+#
+io_settings:
+  save_step_metadata: false
+  save_pcap_logs: true
+  save_sys_logs: true
+  sys_log_level: WARNING
+  agent_log_level: INFO
+  save_agent_logs: true
+  write_agent_log_to_terminal: True
+
+
+game:
+  max_episode_length: 256
+  ports:
+  - ARP
+  - DNS
+  - HTTP
+  - POSTGRES_SERVER
+  protocols:
+  - ICMP
+  - TCP
+  - UDP
+
+agents:
+  - ref: client_2_green_user
+    team: GREEN
+    type: ProbabilisticAgent
+    observation_space: null
+    action_space:
+      action_list:
+        - type: DONOTHING
+        - type: NODE_APPLICATION_EXECUTE
+      action_map:
+        0:
+          action: DONOTHING
+          options: {}
+        1:
+          action: NODE_APPLICATION_EXECUTE
+          options:
+            node_id: 0
+            application_id: 0
+      options:
+        nodes:
+        - node_name: client_2
+          applications:
+            - application_name: WebBrowser
+        max_folders_per_node: 1
+        max_files_per_folder: 1
+        max_services_per_node: 1
+        max_applications_per_node: 1
+
+    reward_function:
+      reward_components:
+        - type: DUMMY
+
+    agent_settings:
+      start_settings:
+        start_step: 5
+        frequency: 4
+        variance: 3
+
+
+
+  - ref: defender
+    team: BLUE
+    type: ProxyAgent
+
+    observation_space:
+      type: CUSTOM
+      options:
+        components:
+          - type: NODES
+            label: NODES
+            options:
+              hosts:
+                - hostname: client_1
+                - hostname: client_2
+                - hostname: client_3
+              num_services: 1
+              num_applications: 0
+              num_folders: 1
+              num_files: 1
+              num_nics: 2
+              include_num_access: false
+              monitored_traffic:
+                icmp:
+                    - NONE
+                tcp:
+                    - DNS
+              include_nmne: false
+              routers:
+                - hostname: router_1
+              num_ports: 0
+              ip_list:
+                - 192.168.10.21
+                - 192.168.10.22
+                - 192.168.10.23
+              wildcard_list:
+                - 0.0.0.1
+              port_list:
+                - 80
+                - 5432
+              protocol_list:
+                - ICMP
+                - TCP
+                - UDP
+              num_rules: 10
+
+          - type: LINKS
+            label: LINKS
+            options:
+              link_references:
+                - switch_1:eth-1<->client_1:eth-1
+                - switch_1:eth-2<->client_2:eth-1
+          - type: "NONE"
+            label: ICS
+            options: {}
+
+    action_space:
+      action_list:
+        - type: DONOTHING
+
+      action_map:
+          0:
+            action: DONOTHING
+            options: {}
+      options:
+        nodes:
+        - node_name: switch
+        - node_name: client_1
+        - node_name: client_2
+        - node_name: client_3
+        max_folders_per_node: 2
+        max_files_per_folder: 2
+        max_services_per_node: 2
+        max_nics_per_node: 8
+        max_acl_rules: 10
+        ip_list:
+          - 192.168.10.21
+          - 192.168.10.22
+          - 192.168.10.23
+
+    reward_function:
+      reward_components:
+        - type: DATABASE_FILE_INTEGRITY
+          weight: 0.5
+          options:
+            node_hostname: database_server
+            folder_name: database
+            file_name: database.db
+
+
+        - type: WEB_SERVER_404_PENALTY
+          weight: 0.5
+          options:
+            node_hostname: web_server
+            service_name: web_server_web_service
+
+
+    agent_settings:
+      flatten_obs: true
+
+simulation:
+  network:
+    nodes:
+
+    - type: switch
+      hostname: switch_1
+      num_ports: 8
+
+    - hostname: client_1
+      type: computer
+      ip_address: 192.168.10.21
+      subnet_mask: 255.255.255.0
+      default_gateway: 192.168.10.1
+      dns_server: 192.168.1.10
+      applications:
+      - type: RansomwareScript
+      - type: WebBrowser
+        options:
+          target_url: http://arcd.com/users/
+      - type: DatabaseClient
+        options:
+          db_server_ip: 192.168.1.10
+          server_password: arcd
+      - type: DataManipulationBot
+        options:
+          port_scan_p_of_success: 0.8
+          data_manipulation_p_of_success: 0.8
+          payload: "DELETE"
+          server_ip: 192.168.1.21
+          server_password: arcd
+      - type: DoSBot
+        options:
+          target_ip_address: 192.168.10.21
+          payload: SPOOF DATA
+          port_scan_p_of_success: 0.8
+      services:
+      - type: DNSClient
+        options:
+          dns_server: 192.168.1.10
+      - type: DNSServer
+        options:
+          domain_mapping:
+            arcd.com: 192.168.1.10
+      - type: DatabaseService
+        options:
+          backup_server_ip: 192.168.1.10
+      - type: WebServer
+      - type: FTPServer
+        options:
+          server_password: arcd
+      - type: NTPClient
+        options:
+          ntp_server_ip: 192.168.1.10
+      - type: NTPServer
+    - hostname: client_2
+      type: computer
+      ip_address: 192.168.10.22
+      subnet_mask: 255.255.255.0
+      default_gateway: 192.168.10.1
+      dns_server: 192.168.1.10
+      file_system:
+        - empty_folder
+        - downloads:
+            - "test.txt"
+            - "suh_con.dn"
+        - root:
+            - passwords:
+                size: 69
+                type: TXT
+      # pre installed services and applications
+    - hostname: client_3
+      type: computer
+      ip_address: 192.168.10.23
+      subnet_mask: 255.255.255.0
+      default_gateway: 192.168.10.1
+      dns_server: 192.168.1.10
+      start_up_duration: 0
+      shut_down_duration: 0
+      operating_state: "OFF"
+      # pre installed services and applications
+
+    links:
+    - endpoint_a_hostname: switch_1
+      endpoint_a_port: 1
+      endpoint_b_hostname: client_1
+      endpoint_b_port: 1
+      bandwidth: 200
+    - endpoint_a_hostname: switch_1
+      endpoint_a_port: 2
+      endpoint_b_hostname: client_2
+      endpoint_b_port: 1
+      bandwidth: 200

tests/integration_tests/configuration_file_parsing/test_node_file_system_config.py

@@ -0,0 +1,64 @@
+# © Crown-owned copyright 2024, Defence Science and Technology Laboratory UK
+from pathlib import Path
+from typing import Union
+
+import yaml
+
+from primaite.game.game import PrimaiteGame
+from primaite.simulator.file_system.file_type import FileType
+from tests import TEST_ASSETS_ROOT
+
+BASIC_CONFIG = TEST_ASSETS_ROOT / "configs/nodes_with_initial_files.yaml"
+
+
+def load_config(config_path: Union[str, Path]) -> PrimaiteGame:
+    """Returns a PrimaiteGame object which loads the contents of a given yaml path."""
+    with open(config_path, "r") as f:
+        cfg = yaml.safe_load(f)
+
+    return PrimaiteGame.from_config(cfg)
+
+
+def test_node_file_system_from_config():
+    """Test that the appropriate files are instantiated in nodes when loaded from config."""
+    game = load_config(BASIC_CONFIG)
+
+    client_1 = game.simulation.network.get_node_by_hostname("client_1")
+
+    assert client_1.software_manager.software.get("DatabaseService")  # database service should be installed
+    assert client_1.file_system.get_file(folder_name="database", file_name="database.db")  # database files should exist
+
+    assert client_1.software_manager.software.get("WebServer")  # web server should be installed
+    assert client_1.file_system.get_file(folder_name="primaite", file_name="index.html")  # web files should exist
+
+    client_2 = game.simulation.network.get_node_by_hostname("client_2")
+
+    # database service should not be installed
+    assert client_2.software_manager.software.get("DatabaseService") is None
+    # database files should not exist
+    assert client_2.file_system.get_file(folder_name="database", file_name="database.db") is None
+
+    # web server should not be installed
+    assert client_2.software_manager.software.get("WebServer") is None
+    # web files should not exist
+    assert client_2.file_system.get_file(folder_name="primaite", file_name="index.html") is None
+
+    empty_folder = client_2.file_system.get_folder(folder_name="empty_folder")
+    assert empty_folder
+    assert len(empty_folder.files) == 0  # should have no files
+
+    password_file = client_2.file_system.get_file(folder_name="root", file_name="passwords.txt")
+    assert password_file  # should exist
+    assert password_file.file_type is FileType.TXT
+    assert password_file.size is 69
+
+    downloads_folder = client_2.file_system.get_folder(folder_name="downloads")
+    assert downloads_folder  # downloads folder should exist
+
+    test_txt = downloads_folder.get_file(file_name="test.txt")
+    assert test_txt  # test.txt should exist
+    assert test_txt.file_type is FileType.TXT
+
+    unknown_file_type = downloads_folder.get_file(file_name="suh_con.dn")
+    assert unknown_file_type  # unknown_file_type should exist
+    assert unknown_file_type.file_type is FileType.UNKNOWN