From a7f00c668dc75932f6cb72de9f8709ce672b58f2 Mon Sep 17 00:00:00 2001 From: Czar Echavez Date: Mon, 2 Sep 2024 15:15:45 +0100 Subject: [PATCH 1/5] #2782: initial impl of files in nodes --- src/primaite/game/game.py | 6 + .../configs/nodes_with_initial_files.yaml | 256 ++++++++++++++++++ .../test_node_file_system_config.py | 47 ++++ 3 files changed, 309 insertions(+) create mode 100644 tests/assets/configs/nodes_with_initial_files.yaml create mode 100644 tests/integration_tests/configuration_file_parsing/test_node_file_system_config.py diff --git a/src/primaite/game/game.py b/src/primaite/game/game.py index 045b2467..befa4032 100644 --- a/src/primaite/game/game.py +++ b/src/primaite/game/game.py @@ -329,6 +329,12 @@ class PrimaiteGame: _LOGGER.error(msg) raise ValueError(msg) + # handle node file system + if node_cfg.get("file_system") is not None and len(node_cfg.get("file_system")) > 0: + for folder in node_cfg.get("file_system"): + for file in node_cfg["file_system"][folder]: + new_node.file_system.create_file(folder_name=folder, file_name=file) + if "users" in node_cfg and new_node.software_manager.software.get("UserManager"): user_manager: UserManager = new_node.software_manager.software["UserManager"] # noqa for user_cfg in node_cfg["users"]: diff --git a/tests/assets/configs/nodes_with_initial_files.yaml b/tests/assets/configs/nodes_with_initial_files.yaml new file mode 100644 index 00000000..3213098b --- /dev/null +++ b/tests/assets/configs/nodes_with_initial_files.yaml @@ -0,0 +1,256 @@ +# Basic Switched network +# +# -------------- -------------- -------------- +# | client_1 |------| switch_1 |------| client_2 | +# -------------- -------------- -------------- +# +io_settings: + save_step_metadata: false + save_pcap_logs: true + save_sys_logs: true + sys_log_level: WARNING + agent_log_level: INFO + save_agent_logs: true + write_agent_log_to_terminal: True + + +game: + max_episode_length: 256 + ports: + - ARP + - DNS + - HTTP + - POSTGRES_SERVER + protocols: + - ICMP + - TCP + - UDP + +agents: + - ref: client_2_green_user + team: GREEN + type: ProbabilisticAgent + observation_space: null + action_space: + action_list: + - type: DONOTHING + - type: NODE_APPLICATION_EXECUTE + action_map: + 0: + action: DONOTHING + options: {} + 1: + action: NODE_APPLICATION_EXECUTE + options: + node_id: 0 + application_id: 0 + options: + nodes: + - node_name: client_2 + applications: + - application_name: WebBrowser + max_folders_per_node: 1 + max_files_per_folder: 1 + max_services_per_node: 1 + max_applications_per_node: 1 + + reward_function: + reward_components: + - type: DUMMY + + agent_settings: + start_settings: + start_step: 5 + frequency: 4 + variance: 3 + + + + - ref: defender + team: BLUE + type: ProxyAgent + + observation_space: + type: CUSTOM + options: + components: + - type: NODES + label: NODES + options: + hosts: + - hostname: client_1 + - hostname: client_2 + - hostname: client_3 + num_services: 1 + num_applications: 0 + num_folders: 1 + num_files: 1 + num_nics: 2 + include_num_access: false + monitored_traffic: + icmp: + - NONE + tcp: + - DNS + include_nmne: false + routers: + - hostname: router_1 + num_ports: 0 + ip_list: + - 192.168.10.21 + - 192.168.10.22 + - 192.168.10.23 + wildcard_list: + - 0.0.0.1 + port_list: + - 80 + - 5432 + protocol_list: + - ICMP + - TCP + - UDP + num_rules: 10 + + - type: LINKS + label: LINKS + options: + link_references: + - switch_1:eth-1<->client_1:eth-1 + - switch_1:eth-2<->client_2:eth-1 + - type: "NONE" + label: ICS + options: {} + + action_space: + action_list: + - type: DONOTHING + + action_map: + 0: + action: DONOTHING + options: {} + options: + nodes: + - node_name: switch + - node_name: client_1 + - node_name: client_2 + - node_name: client_3 + max_folders_per_node: 2 + max_files_per_folder: 2 + max_services_per_node: 2 + max_nics_per_node: 8 + max_acl_rules: 10 + ip_list: + - 192.168.10.21 + - 192.168.10.22 + - 192.168.10.23 + + reward_function: + reward_components: + - type: DATABASE_FILE_INTEGRITY + weight: 0.5 + options: + node_hostname: database_server + folder_name: database + file_name: database.db + + + - type: WEB_SERVER_404_PENALTY + weight: 0.5 + options: + node_hostname: web_server + service_name: web_server_web_service + + + agent_settings: + flatten_obs: true + +simulation: + network: + nodes: + + - type: switch + hostname: switch_1 + num_ports: 8 + + - hostname: client_1 + type: computer + ip_address: 192.168.10.21 + subnet_mask: 255.255.255.0 + default_gateway: 192.168.10.1 + dns_server: 192.168.1.10 + applications: + - type: RansomwareScript + - type: WebBrowser + options: + target_url: http://arcd.com/users/ + - type: DatabaseClient + options: + db_server_ip: 192.168.1.10 + server_password: arcd + - type: DataManipulationBot + options: + port_scan_p_of_success: 0.8 + data_manipulation_p_of_success: 0.8 + payload: "DELETE" + server_ip: 192.168.1.21 + server_password: arcd + - type: DoSBot + options: + target_ip_address: 192.168.10.21 + payload: SPOOF DATA + port_scan_p_of_success: 0.8 + services: + - type: DNSClient + options: + dns_server: 192.168.1.10 + - type: DNSServer + options: + domain_mapping: + arcd.com: 192.168.1.10 + - type: DatabaseService + options: + backup_server_ip: 192.168.1.10 + - type: WebServer + - type: FTPServer + options: + server_password: arcd + - type: NTPClient + options: + ntp_server_ip: 192.168.1.10 + - type: NTPServer + - hostname: client_2 + type: computer + ip_address: 192.168.10.22 + subnet_mask: 255.255.255.0 + default_gateway: 192.168.10.1 + dns_server: 192.168.1.10 + file_system: + downloads: + - "test.txt" + - "suh_con.dn" + root: + - "passwords.txt" + # pre installed services and applications + - hostname: client_3 + type: computer + ip_address: 192.168.10.23 + subnet_mask: 255.255.255.0 + default_gateway: 192.168.10.1 + dns_server: 192.168.1.10 + start_up_duration: 0 + shut_down_duration: 0 + operating_state: "OFF" + # pre installed services and applications + + links: + - endpoint_a_hostname: switch_1 + endpoint_a_port: 1 + endpoint_b_hostname: client_1 + endpoint_b_port: 1 + bandwidth: 200 + - endpoint_a_hostname: switch_1 + endpoint_a_port: 2 + endpoint_b_hostname: client_2 + endpoint_b_port: 1 + bandwidth: 200 diff --git a/tests/integration_tests/configuration_file_parsing/test_node_file_system_config.py b/tests/integration_tests/configuration_file_parsing/test_node_file_system_config.py new file mode 100644 index 00000000..05ef7275 --- /dev/null +++ b/tests/integration_tests/configuration_file_parsing/test_node_file_system_config.py @@ -0,0 +1,47 @@ +# © Crown-owned copyright 2024, Defence Science and Technology Laboratory UK +from pathlib import Path +from typing import Union + +import yaml + +from primaite.game.game import PrimaiteGame +from tests import TEST_ASSETS_ROOT + +BASIC_CONFIG = TEST_ASSETS_ROOT / "configs/nodes_with_initial_files.yaml" + + +def load_config(config_path: Union[str, Path]) -> PrimaiteGame: + """Returns a PrimaiteGame object which loads the contents of a given yaml path.""" + with open(config_path, "r") as f: + cfg = yaml.safe_load(f) + + return PrimaiteGame.from_config(cfg) + + +def test_node_file_system_from_config(): + """Test that the appropriate files are instantiated in nodes when loaded from config.""" + game = load_config(BASIC_CONFIG) + + client_1 = game.simulation.network.get_node_by_hostname("client_1") + + assert client_1.software_manager.software.get("DatabaseService") # database service should be installed + assert client_1.file_system.get_file(folder_name="database", file_name="database.db") # database files should exist + + assert client_1.software_manager.software.get("WebServer") # web server should be installed + assert client_1.file_system.get_file(folder_name="primaite", file_name="index.html") # web files should exist + + client_2 = game.simulation.network.get_node_by_hostname("client_2") + + # database service should not be installed + assert client_2.software_manager.software.get("DatabaseService") is None + # database files should not exist + assert client_2.file_system.get_file(folder_name="database", file_name="database.db") is None + + # web server should not be installed + assert client_2.software_manager.software.get("WebServer") is None + # web files should not exist + assert client_2.file_system.get_file(folder_name="primaite", file_name="index.html") is None + + # TODO file sizes and file types + # TODO assert that files and folders created: + # TODO create empty folders From 8e6b9f39707e1236f3d5f9e8f85d962b33f0e1d5 Mon Sep 17 00:00:00 2001 From: Czar Echavez Date: Tue, 3 Sep 2024 11:53:23 +0100 Subject: [PATCH 2/5] #2782: added ability to create empty folders + create files with size and types + tests --- src/primaite/game/game.py | 21 ++++++++++++++--- .../configs/nodes_with_initial_files.yaml | 9 +++++--- .../test_node_file_system_config.py | 23 ++++++++++++++++--- 3 files changed, 44 insertions(+), 9 deletions(-) diff --git a/src/primaite/game/game.py b/src/primaite/game/game.py index befa4032..d11f6a19 100644 --- a/src/primaite/game/game.py +++ b/src/primaite/game/game.py @@ -17,6 +17,7 @@ from primaite.game.agent.scripted_agents.random_agent import PeriodicAgent from primaite.game.agent.scripted_agents.tap001 import TAP001 from primaite.game.science import graph_has_cycle, topological_sort from primaite.simulator import SIM_OUTPUT +from primaite.simulator.file_system.file_type import FileType from primaite.simulator.network.airspace import AirSpaceFrequency from primaite.simulator.network.hardware.base import NetworkInterface, NodeOperatingState, UserManager from primaite.simulator.network.hardware.nodes.host.computer import Computer @@ -331,9 +332,23 @@ class PrimaiteGame: # handle node file system if node_cfg.get("file_system") is not None and len(node_cfg.get("file_system")) > 0: - for folder in node_cfg.get("file_system"): - for file in node_cfg["file_system"][folder]: - new_node.file_system.create_file(folder_name=folder, file_name=file) + for folder_idx, folder_obj in enumerate(node_cfg.get("file_system")): + # if the folder is not a Dict, create an empty folder + if not isinstance(folder_obj, Dict): + new_node.file_system.create_folder(folder_name=folder_obj) + else: + folder_name = next(iter(folder_obj)) + for file_idx, file_obj in enumerate(node_cfg["file_system"][folder_idx][folder_name]): + if not isinstance(file_obj, Dict): + new_node.file_system.create_file(folder_name=folder_name, file_name=file_obj) + else: + file_name = next(iter(file_obj)) + new_node.file_system.create_file( + folder_name=folder_name, + file_name=file_name, + size=file_obj[file_name].get("size", 0), + file_type=FileType[file_obj[file_name].get("type", "UNKNOWN").upper()], + ) if "users" in node_cfg and new_node.software_manager.software.get("UserManager"): user_manager: UserManager = new_node.software_manager.software["UserManager"] # noqa diff --git a/tests/assets/configs/nodes_with_initial_files.yaml b/tests/assets/configs/nodes_with_initial_files.yaml index 3213098b..fad6cffd 100644 --- a/tests/assets/configs/nodes_with_initial_files.yaml +++ b/tests/assets/configs/nodes_with_initial_files.yaml @@ -226,11 +226,14 @@ simulation: default_gateway: 192.168.10.1 dns_server: 192.168.1.10 file_system: - downloads: + - empty_folder + - downloads: - "test.txt" - "suh_con.dn" - root: - - "passwords.txt" + - root: + - passwords: + size: 69 + type: TXT # pre installed services and applications - hostname: client_3 type: computer diff --git a/tests/integration_tests/configuration_file_parsing/test_node_file_system_config.py b/tests/integration_tests/configuration_file_parsing/test_node_file_system_config.py index 05ef7275..49e90b54 100644 --- a/tests/integration_tests/configuration_file_parsing/test_node_file_system_config.py +++ b/tests/integration_tests/configuration_file_parsing/test_node_file_system_config.py @@ -5,6 +5,7 @@ from typing import Union import yaml from primaite.game.game import PrimaiteGame +from primaite.simulator.file_system.file_type import FileType from tests import TEST_ASSETS_ROOT BASIC_CONFIG = TEST_ASSETS_ROOT / "configs/nodes_with_initial_files.yaml" @@ -42,6 +43,22 @@ def test_node_file_system_from_config(): # web files should not exist assert client_2.file_system.get_file(folder_name="primaite", file_name="index.html") is None - # TODO file sizes and file types - # TODO assert that files and folders created: - # TODO create empty folders + empty_folder = client_2.file_system.get_folder(folder_name="empty_folder") + assert empty_folder + assert len(empty_folder.files) == 0 # should have no files + + password_file = client_2.file_system.get_file(folder_name="root", file_name="passwords.txt") + assert password_file # should exist + assert password_file.file_type is FileType.TXT + assert password_file.size is 69 + + downloads_folder = client_2.file_system.get_folder(folder_name="downloads") + assert downloads_folder # downloads folder should exist + + test_txt = downloads_folder.get_file(file_name="test.txt") + assert test_txt # test.txt should exist + assert test_txt.file_type is FileType.TXT + + unknown_file_type = downloads_folder.get_file(file_name="suh_con.dn") + assert unknown_file_type # unknown_file_type should exist + assert unknown_file_type.file_type is FileType.UNKNOWN From 26a56bf3608d2f0c7930d8e0b6e5faa0830e092f Mon Sep 17 00:00:00 2001 From: Czar Echavez Date: Tue, 3 Sep 2024 12:37:39 +0100 Subject: [PATCH 3/5] #2782: documentation + adding example to data_manipulation.yaml --- .../nodes/common/common_node_attributes.rst | 33 +++++++++++++++++++ .../_package_data/data_manipulation.yaml | 9 ++++- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/docs/source/configuration/simulation/nodes/common/common_node_attributes.rst b/docs/source/configuration/simulation/nodes/common/common_node_attributes.rst index 7cf11eb4..056422ca 100644 --- a/docs/source/configuration/simulation/nodes/common/common_node_attributes.rst +++ b/docs/source/configuration/simulation/nodes/common/common_node_attributes.rst @@ -54,6 +54,39 @@ Optional. Default value is ``3``. The number of time steps required to occur in order for the node to cycle from ``ON`` to ``SHUTTING_DOWN`` and then finally ``OFF``. +``file_system`` +--------------- + +Optional. + +The file system of the node. This configuration allows nodes to be initialised with files and/or folders. + +The file system takes a list of folders and files. + +Example: + +.. code-block:: yaml + + simulation: + network: + nodes: + - hostname: client_1 + type: computer + ip_address: 192.168.10.11 + subnet_mask: 255.255.255.0 + default_gateway: 192.168.10.1 + file_system: + - empty_folder # example of an empty folder + - downloads: + - "test_1.txt" # files in the downloads folder + - "test_2.txt" + - root: + - passwords: # example of file with size and type + size: 69 # size in bytes + type: TXT # See FileType for list of available file types + +List of file types: :py:mod:`primaite.simulator.file_system.file_type.FileType` + ``users`` --------- diff --git a/src/primaite/config/_package_data/data_manipulation.yaml b/src/primaite/config/_package_data/data_manipulation.yaml index 97442903..2d03609a 100644 --- a/src/primaite/config/_package_data/data_manipulation.yaml +++ b/src/primaite/config/_package_data/data_manipulation.yaml @@ -843,7 +843,14 @@ simulation: dns_server: 192.168.1.10 services: - type: FTPServer - + file_system: + - root: + - backup_script.sh: # example file in backup server + size: 400 + type: SH + - downloads: + - "ChromeSetup.exe" # another example file + - "New Folder" # example of an empty folder - hostname: security_suite type: server ip_address: 192.168.1.110 From 1374a23e14fb9fea35c346747eb8d9edd303c2ca Mon Sep 17 00:00:00 2001 From: Czar Echavez Date: Wed, 4 Sep 2024 10:17:33 +0100 Subject: [PATCH 4/5] #2782: fix spacing in data_manipulation yaml + documentation --- .../simulation/nodes/common/common_node_attributes.rst | 4 ++-- src/primaite/config/_package_data/data_manipulation.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/source/configuration/simulation/nodes/common/common_node_attributes.rst b/docs/source/configuration/simulation/nodes/common/common_node_attributes.rst index 056422ca..c94344fd 100644 --- a/docs/source/configuration/simulation/nodes/common/common_node_attributes.rst +++ b/docs/source/configuration/simulation/nodes/common/common_node_attributes.rst @@ -82,8 +82,8 @@ Example: - "test_2.txt" - root: - passwords: # example of file with size and type - size: 69 # size in bytes - type: TXT # See FileType for list of available file types + size: 69 # size in bytes + type: TXT # See FileType for list of available file types List of file types: :py:mod:`primaite.simulator.file_system.file_type.FileType` diff --git a/src/primaite/config/_package_data/data_manipulation.yaml b/src/primaite/config/_package_data/data_manipulation.yaml index 2d03609a..b36ec707 100644 --- a/src/primaite/config/_package_data/data_manipulation.yaml +++ b/src/primaite/config/_package_data/data_manipulation.yaml @@ -846,8 +846,8 @@ simulation: file_system: - root: - backup_script.sh: # example file in backup server - size: 400 - type: SH + size: 400 + type: SH - downloads: - "ChromeSetup.exe" # another example file - "New Folder" # example of an empty folder From 2391c485698a645a035333208252c39209c1a9da Mon Sep 17 00:00:00 2001 From: Czar Echavez Date: Thu, 5 Sep 2024 10:18:35 +0100 Subject: [PATCH 5/5] #2782: apply suggestions --- src/primaite/config/_package_data/data_manipulation.yaml | 8 -------- src/primaite/game/game.py | 2 +- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/src/primaite/config/_package_data/data_manipulation.yaml b/src/primaite/config/_package_data/data_manipulation.yaml index b36ec707..2a069971 100644 --- a/src/primaite/config/_package_data/data_manipulation.yaml +++ b/src/primaite/config/_package_data/data_manipulation.yaml @@ -843,14 +843,6 @@ simulation: dns_server: 192.168.1.10 services: - type: FTPServer - file_system: - - root: - - backup_script.sh: # example file in backup server - size: 400 - type: SH - - downloads: - - "ChromeSetup.exe" # another example file - - "New Folder" # example of an empty folder - hostname: security_suite type: server ip_address: 192.168.1.110 diff --git a/src/primaite/game/game.py b/src/primaite/game/game.py index d11f6a19..8e4d4513 100644 --- a/src/primaite/game/game.py +++ b/src/primaite/game/game.py @@ -331,7 +331,7 @@ class PrimaiteGame: raise ValueError(msg) # handle node file system - if node_cfg.get("file_system") is not None and len(node_cfg.get("file_system")) > 0: + if node_cfg.get("file_system"): for folder_idx, folder_obj in enumerate(node_cfg.get("file_system")): # if the folder is not a Dict, create an empty folder if not isinstance(folder_obj, Dict):