oxbrbtx/PrimAITE
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/dev' into feature/2840_LOCAL_TERMINAL_COMMAND

Browse Source
This commit is contained in:
Archer Bowen
2024-09-06 10:04:49 +01:00
parent fd3d3812f6 1234b2dc4a
commit 7562f6fc5c
8 changed files with 426 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
CHANGELOG.md
View File

@@ -8,6 +8,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
- New ``NODE_SEND_LOCAL_COMMAND`` action implemented which grants agents the ability to execute commands locally. (Previously limited to remote only)
### Added
- Log observation space data by episode and step.
## [3.3.0] - 2024-09-04
### Added
- Random Number Generator Seeding by specifying a random number seed in the config file.
- Implemented Terminal service class, providing a generic terminal simulation.

33
docs/source/configuration/simulation/nodes/common/common_node_attributes.rst
View File

@@ -54,6 +54,39 @@ Optional. Default value is ``3``.
The number of time steps required to occur in order for the node to cycle from ``ON`` to ``SHUTTING_DOWN`` and then finally ``OFF``.
``file_system``
---------------
Optional.
The file system of the node. This configuration allows nodes to be initialised with files and/or folders.
The file system takes a list of folders and files.
Example:
.. code-block:: yaml
simulation:
network:
nodes:
- hostname: client_1
type: computer
ip_address: 192.168.10.11
subnet_mask: 255.255.255.0
default_gateway: 192.168.10.1
file_system:
- empty_folder # example of an empty folder
- downloads:
- "test_1.txt" # files in the downloads folder
- "test_2.txt"
- root:
- passwords: # example of file with size and type
size: 69 # size in bytes
type: TXT # See FileType for list of available file types
List of file types: :py:mod:`primaite.simulator.file_system.file_type.FileType`
``users``
---------

1
src/primaite/config/_package_data/data_manipulation.yaml
View File

@@ -843,7 +843,6 @@ simulation:
dns_server: 192.168.1.10
services:
- type: FTPServer
- hostname: security_suite
type: server
ip_address: 192.168.1.110

18
src/primaite/game/agent/interface.py
View File

@@ -38,6 +38,9 @@ class AgentHistoryItem(BaseModel):
reward_info: Dict[str, Any] = {}
observation: Optional[ObsType] = None
"""The observation space data for this step."""
class AgentStartSettings(BaseModel):
"""Configuration values for when an agent starts performing actions."""
@@ -169,12 +172,23 @@ class AbstractAgent(ABC):
return request
def process_action_response(
self, timestep: int, action: str, parameters: Dict[str, Any], request: RequestFormat, response: RequestResponse
self,
timestep: int,
action: str,
parameters: Dict[str, Any],
request: RequestFormat,
response: RequestResponse,
observation: ObsType,
) -> None:
"""Process the response from the most recent action."""
self.history.append(
AgentHistoryItem(
timestep=timestep, action=action, parameters=parameters, request=request, response=response
timestep=timestep,
action=action,
parameters=parameters,
request=request,
response=response,
observation=observation,
)
)

22
src/primaite/game/game.py
View File

@@ -17,6 +17,7 @@ from primaite.game.agent.scripted_agents.random_agent import PeriodicAgent
from primaite.game.agent.scripted_agents.tap001 import TAP001
from primaite.game.science import graph_has_cycle, topological_sort
from primaite.simulator import SIM_OUTPUT
from primaite.simulator.file_system.file_type import FileType
from primaite.simulator.network.airspace import AirSpaceFrequency
from primaite.simulator.network.hardware.base import NetworkInterface, NodeOperatingState, UserManager
from primaite.simulator.network.hardware.nodes.host.computer import Computer
@@ -186,6 +187,7 @@ class PrimaiteGame:
parameters=parameters,
request=request,
response=response,
observation=obs,
)
def pre_timestep(self) -> None:
@@ -329,6 +331,26 @@ class PrimaiteGame:
_LOGGER.error(msg)
raise ValueError(msg)
# handle node file system
if node_cfg.get("file_system"):
for folder_idx, folder_obj in enumerate(node_cfg.get("file_system")):
# if the folder is not a Dict, create an empty folder
if not isinstance(folder_obj, Dict):
new_node.file_system.create_folder(folder_name=folder_obj)
else:
folder_name = next(iter(folder_obj))
for file_idx, file_obj in enumerate(node_cfg["file_system"][folder_idx][folder_name]):
if not isinstance(file_obj, Dict):
new_node.file_system.create_file(folder_name=folder_name, file_name=file_obj)
else:
file_name = next(iter(file_obj))
new_node.file_system.create_file(
folder_name=folder_name,
file_name=file_name,
size=file_obj[file_name].get("size", 0),
file_type=FileType[file_obj[file_name].get("type", "UNKNOWN").upper()],
)
if "users" in node_cfg and new_node.software_manager.software.get("UserManager"):
user_manager: UserManager = new_node.software_manager.software["UserManager"] # noqa
for user_cfg in node_cfg["users"]:

259
tests/assets/configs/nodes_with_initial_files.yaml Normal file
View File

@@ -0,0 +1,259 @@
# Basic Switched network
#
# -------------- -------------- --------------
# | client_1 |------| switch_1 |------| client_2 |
# -------------- -------------- --------------
#
io_settings:
save_step_metadata: false
save_pcap_logs: true
save_sys_logs: true
sys_log_level: WARNING
agent_log_level: INFO
save_agent_logs: true
write_agent_log_to_terminal: True
game:
max_episode_length: 256
ports:
- ARP
- DNS
- HTTP
- POSTGRES_SERVER
protocols:
- ICMP
- TCP
- UDP
agents:
- ref: client_2_green_user
team: GREEN
type: ProbabilisticAgent
observation_space: null
action_space:
action_list:
- type: DONOTHING
- type: NODE_APPLICATION_EXECUTE
action_map:
0:
action: DONOTHING
options: {}
1:
action: NODE_APPLICATION_EXECUTE
options:
node_id: 0
application_id: 0
options:
nodes:
- node_name: client_2
applications:
- application_name: WebBrowser
max_folders_per_node: 1
max_files_per_folder: 1
max_services_per_node: 1
max_applications_per_node: 1
reward_function:
reward_components:
- type: DUMMY
agent_settings:
start_settings:
start_step: 5
frequency: 4
variance: 3
- ref: defender
team: BLUE
type: ProxyAgent
observation_space:
type: CUSTOM
options:
components:
- type: NODES
label: NODES
options:
hosts:
- hostname: client_1
- hostname: client_2
- hostname: client_3
num_services: 1
num_applications: 0
num_folders: 1
num_files: 1
num_nics: 2
include_num_access: false
monitored_traffic:
icmp:
- NONE
tcp:
- DNS
include_nmne: false
routers:
- hostname: router_1
num_ports: 0
ip_list:
- 192.168.10.21
- 192.168.10.22
- 192.168.10.23
wildcard_list:
- 0.0.0.1
port_list:
- 80
- 5432
protocol_list:
- ICMP
- TCP
- UDP
num_rules: 10
- type: LINKS
label: LINKS
options:
link_references:
- switch_1:eth-1<->client_1:eth-1
- switch_1:eth-2<->client_2:eth-1
- type: "NONE"
label: ICS
options: {}
action_space:
action_list:
- type: DONOTHING
action_map:
0:
action: DONOTHING
options: {}
options:
nodes:
- node_name: switch
- node_name: client_1
- node_name: client_2
- node_name: client_3
max_folders_per_node: 2
max_files_per_folder: 2
max_services_per_node: 2
max_nics_per_node: 8
max_acl_rules: 10
ip_list:
- 192.168.10.21
- 192.168.10.22
- 192.168.10.23
reward_function:
reward_components:
- type: DATABASE_FILE_INTEGRITY
weight: 0.5
options:
node_hostname: database_server
folder_name: database
file_name: database.db
- type: WEB_SERVER_404_PENALTY
weight: 0.5
options:
node_hostname: web_server
service_name: web_server_web_service
agent_settings:
flatten_obs: true
simulation:
network:
nodes:
- type: switch
hostname: switch_1
num_ports: 8
- hostname: client_1
type: computer
ip_address: 192.168.10.21
subnet_mask: 255.255.255.0
default_gateway: 192.168.10.1
dns_server: 192.168.1.10
applications:
- type: RansomwareScript
- type: WebBrowser
options:
target_url: http://arcd.com/users/
- type: DatabaseClient
options:
db_server_ip: 192.168.1.10
server_password: arcd
- type: DataManipulationBot
options:
port_scan_p_of_success: 0.8
data_manipulation_p_of_success: 0.8
payload: "DELETE"
server_ip: 192.168.1.21
server_password: arcd
- type: DoSBot
options:
target_ip_address: 192.168.10.21
payload: SPOOF DATA
port_scan_p_of_success: 0.8
services:
- type: DNSClient
options:
dns_server: 192.168.1.10
- type: DNSServer
options:
domain_mapping:
arcd.com: 192.168.1.10
- type: DatabaseService
options:
backup_server_ip: 192.168.1.10
- type: WebServer
- type: FTPServer
options:
server_password: arcd
- type: NTPClient
options:
ntp_server_ip: 192.168.1.10
- type: NTPServer
- hostname: client_2
type: computer
ip_address: 192.168.10.22
subnet_mask: 255.255.255.0
default_gateway: 192.168.10.1
dns_server: 192.168.1.10
file_system:
- empty_folder
- downloads:
- "test.txt"
- "suh_con.dn"
- root:
- passwords:
size: 69
type: TXT
# pre installed services and applications
- hostname: client_3
type: computer
ip_address: 192.168.10.23
subnet_mask: 255.255.255.0
default_gateway: 192.168.10.1
dns_server: 192.168.1.10
start_up_duration: 0
shut_down_duration: 0
operating_state: "OFF"
# pre installed services and applications
links:
- endpoint_a_hostname: switch_1
endpoint_a_port: 1
endpoint_b_hostname: client_1
endpoint_b_port: 1
bandwidth: 200
- endpoint_a_hostname: switch_1
endpoint_a_port: 2
endpoint_b_hostname: client_2
endpoint_b_port: 1
bandwidth: 200

64
tests/integration_tests/configuration_file_parsing/test_node_file_system_config.py Normal file
View File

@@ -0,0 +1,64 @@
# © Crown-owned copyright 2024, Defence Science and Technology Laboratory UK
from pathlib import Path
from typing import Union
import yaml
from primaite.game.game import PrimaiteGame
from primaite.simulator.file_system.file_type import FileType
from tests import TEST_ASSETS_ROOT
BASIC_CONFIG = TEST_ASSETS_ROOT / "configs/nodes_with_initial_files.yaml"
def load_config(config_path: Union[str, Path]) -> PrimaiteGame:
"""Returns a PrimaiteGame object which loads the contents of a given yaml path."""
with open(config_path, "r") as f:
cfg = yaml.safe_load(f)
return PrimaiteGame.from_config(cfg)
def test_node_file_system_from_config():
"""Test that the appropriate files are instantiated in nodes when loaded from config."""
game = load_config(BASIC_CONFIG)
client_1 = game.simulation.network.get_node_by_hostname("client_1")
assert client_1.software_manager.software.get("DatabaseService") # database service should be installed
assert client_1.file_system.get_file(folder_name="database", file_name="database.db") # database files should exist
assert client_1.software_manager.software.get("WebServer") # web server should be installed
assert client_1.file_system.get_file(folder_name="primaite", file_name="index.html") # web files should exist
client_2 = game.simulation.network.get_node_by_hostname("client_2")
# database service should not be installed
assert client_2.software_manager.software.get("DatabaseService") is None
# database files should not exist
assert client_2.file_system.get_file(folder_name="database", file_name="database.db") is None
# web server should not be installed
assert client_2.software_manager.software.get("WebServer") is None
# web files should not exist
assert client_2.file_system.get_file(folder_name="primaite", file_name="index.html") is None
empty_folder = client_2.file_system.get_folder(folder_name="empty_folder")
assert empty_folder
assert len(empty_folder.files) == 0 # should have no files
password_file = client_2.file_system.get_file(folder_name="root", file_name="passwords.txt")
assert password_file # should exist
assert password_file.file_type is FileType.TXT
assert password_file.size is 69
downloads_folder = client_2.file_system.get_folder(folder_name="downloads")
assert downloads_folder # downloads folder should exist
test_txt = downloads_folder.get_file(file_name="test.txt")
assert test_txt # test.txt should exist
assert test_txt.file_type is FileType.TXT
unknown_file_type = downloads_folder.get_file(file_name="suh_con.dn")
assert unknown_file_type # unknown_file_type should exist
assert unknown_file_type.file_type is FileType.UNKNOWN

28
tests/integration_tests/game_layer/observations/test_obs_data_capture.py Normal file
View File

@@ -0,0 +1,28 @@
# © Crown-owned copyright 2024, Defence Science and Technology Laboratory UK
import json
from primaite.session.environment import PrimaiteGymEnv
from primaite.session.io import PrimaiteIO
from tests import TEST_ASSETS_ROOT
DATA_MANIPULATION_CONFIG = TEST_ASSETS_ROOT / "configs" / "data_manipulation.yaml"
def test_obs_data_in_log_file():
"""Create a log file of AgentHistoryItems and check observation data is
included. Assumes that data_manipulation.yaml has an agent labelled
'defender' with a non-null observation space.
The log file will be in:
primaite/VERSION/sessions/YYYY-MM-DD/HH-MM-SS/agent_actions
"""
env = PrimaiteGymEnv(DATA_MANIPULATION_CONFIG)
env.reset()
for _ in range(10):
env.step(0)
env.reset()
io = PrimaiteIO()
path = io.generate_agent_actions_save_path(episode=1)
with open(path, "r") as f:
j = json.load(f)
assert type(j["0"]["defender"]["observation"]) == dict