From 388176b8bddd748c0a0fe9a252201db1ca4b831a Mon Sep 17 00:00:00 2001 From: Chris McCarthy Date: Mon, 11 Sep 2023 09:30:40 +0100 Subject: [PATCH] #1816 - Added full documentation on the database client/server, and the internal frame processing process --- docs/source/simulation.rst | 1 + .../network/database_client_server.rst | 70 +++++++++++++ .../network/internal_frame_processing.rst | 98 +++++++++++++++++++ .../network/software.rst | 18 ++++ .../simulator/file_system/file_type.py | 5 +- .../simulator/network/hardware/base.py | 1 + src/primaite/simulator/network/networks.py | 2 +- .../system/applications/application.py | 4 +- .../system/applications/database_client.py | 34 +++++++ .../simulator/system/core/software_manager.py | 17 +++- .../{database.py => database_service.py} | 2 - .../system/test_database_on_node.py | 2 +- .../_system/_services/test_database.py | 2 +- 13 files changed, 246 insertions(+), 10 deletions(-) create mode 100644 docs/source/simulation_components/network/database_client_server.rst create mode 100644 docs/source/simulation_components/network/software.rst rename src/primaite/simulator/system/services/{database.py => database_service.py} (98%) diff --git a/docs/source/simulation.rst b/docs/source/simulation.rst index 7e9fe77f..ab4530f1 100644 --- a/docs/source/simulation.rst +++ b/docs/source/simulation.rst @@ -21,3 +21,4 @@ Contents simulation_components/network/router simulation_components/network/switch simulation_components/network/network + simulation_components/internal_frame_processing diff --git a/docs/source/simulation_components/network/database_client_server.rst b/docs/source/simulation_components/network/database_client_server.rst new file mode 100644 index 00000000..99bbe25e --- /dev/null +++ b/docs/source/simulation_components/network/database_client_server.rst @@ -0,0 +1,70 @@ +.. only:: comment + + © Crown-owned copyright 2023, Defence Science and Technology Laboratory UK + + +Database Client Server +====================== + +Database Service +---------------- + +The ``DatabaseService`` provides a SQL database server simulation by extending the base Service class. + +Key capabilities +^^^^^^^^^^^^^^^^ + +- Initialises a SQLite database file in the ``Node``'s ``FileSystem`` upon creation. +- Handles connecting clients by maintaining a dictionary of connections mapped to session IDs. +- Authenticates connections using a configurable password. +- Executes SQL queries against the SQLite database. +- Returns query results and status codes back to clients. +- Leverages the Service base class for install/uninstall, status tracking, etc. + +Usage +^^^^^ +- Install on a Node via the ``SoftwareManager`` to start the database service. +- Clients connect, execute queries, and disconnect. +- Service runs on TCP port 5432 by default. + +Implementation +^^^^^^^^^^^^^^ + +- Uses SQLite for persistent storage. +- Creates the database file within the node's file system. +- Manages client connections in a dictionary by session ID. +- Processes SQL queries via the SQLite cursor and connection. +- Returns results and status codes in a standard dictionary format. +- Extends Service class for integration with ``SoftwareManager``. + +Database Client +--------------- + +The DatabaseClient provides a client interface for connecting to the ``DatabaseService``. + +Key features +^^^^^^^^^^^^ + +- Connects to the ``DatabaseService`` via the ``SoftwareManager``. +- Executes SQL queries and retrieves result sets. +- Handles connecting, querying, and disconnecting. +- Provides a simple ``query`` method for running SQL. + + +Usage +^^^^^ + +- Initialise with server IP address and optional password. +- Connect to the ``DatabaseService`` with ``connect``. +- Execute SQL queries via ``query``. +- Retrieve results in a dictionary. +- Disconnect when finished. + +Implementation +^^^^^^^^^^^^^^ + +- Leverages ``SoftwareManager`` for sending payloads over the network. +- Connect and disconnect methods manage sessions. +- Provides easy interface for applications to query database. +- Payloads serialised as dictionaries for transmission. +- Extends base Application class. diff --git a/docs/source/simulation_components/network/internal_frame_processing.rst b/docs/source/simulation_components/network/internal_frame_processing.rst index e69de29b..e173a3ac 100644 --- a/docs/source/simulation_components/network/internal_frame_processing.rst +++ b/docs/source/simulation_components/network/internal_frame_processing.rst @@ -0,0 +1,98 @@ +.. only:: comment + + © Crown-owned copyright 2023, Defence Science and Technology Laboratory UK + +.. _about: + +Internal Frame Processing +========================= + +Inbound +------- + +At the NIC +^^^^^^^^^^ +When a Frame is received on the Node's NIC: + +- The NIC checks if it is enabled. If so, it will process the Frame. +- The Frame's received timestamp is set. +- The Frame is captured by the NIC's PacketCapture if configured. +- The NIC decrements the IP Packet's TTL by 1. +- The NIC calls the Node's ``receive_frame`` method, passing itself as the receiving NIC and the Frame. + + +At the Node +^^^^^^^^^^^ + +When ``receive_frame`` is called on the Node: + +- The source IP address is added to the ARP cache if not already present. +- The Frame's protocol is checked: + - If ARP or ICMP, the Frame is passed to that protocol's handler method. + - Otherwise it is passed to the SessionManager's ``receive_frame`` method. + +At the SessionManager +^^^^^^^^^^^^^^^^^^^^^ + +When ``receive_frame`` is called on the SessionManager: + +- It extracts the key session details from the Frame: + - Protocol (TCP, UDP, etc) + - Source IP + - Destination IP + - Source Port + - Destination Port +- It checks if an existing Session matches these details. +- If no match, a new Session is created to represent this exchange. +- The payload and new/existing Session ID are passed to the SoftwareManager's ``receive_payload_from_session_manager`` method. + +At the SoftwareManager +^^^^^^^^^^^^^^^^^^^^^^ + +Inside ``receive_payload_from_session_manager``: + +- The SoftwareManager checks its port/protocol mapping to find which Service or Application is listening on the destination port and protocol. +- The payload and Session ID are forwarded to that receiver Service/Application instance via their ``receive`` method. +- The Service/Application can then process the payload as needed. + +Outbound +-------- + +At the Service/Application +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +When a Service or Application needs to send a payload: + +- It calls the SoftwareManager's ``send_payload_to_session_manager`` method. +- Passes the payload, and either destination IP and destination port for new payloads, or session id for existing sessions. + +At the SoftwareManager +^^^^^^^^^^^^^^^^^^^^^^ + +Inside ``send_payload_to_session_manager``: + +- The SoftwareManager forwards the payload and details through to to the SessionManager's ``receive_payload_from_software_manager`` method. + +At the SessionManager +^^^^^^^^^^^^^^^^^^^^^ + +When ``receive_payload_from_software_manager`` is called: + +- If a Session ID was provided, it looks up the Session. +- Gets the destination MAC address by checking the ARP cache. +- If no Session ID was provided, the destination Port, IP address and Mac Address are used along with the outbound IP Address and Mac Address to create a new Session. +- Calls `send_payload_to_nic`` to construct and send the Frame. + +When ``send_payload_to_nic`` is called: + +- It constructs a new Frame with the payload, using the source NIC's MAC, source IP, destination MAC, etc. +- The outbound NIC is looked up via the ARP cache based on destination IP. +- The constructed Frame is passed to the outbound NIC's ``send_frame`` method. + +At the NIC +^^^^^^^^^^ + +When ``send_frame`` is called: + +- The NIC checks if it is enabled before sending. +- If enabled, it sends the Frame out to the connected Link. diff --git a/docs/source/simulation_components/network/software.rst b/docs/source/simulation_components/network/software.rst new file mode 100644 index 00000000..0dcb1d63 --- /dev/null +++ b/docs/source/simulation_components/network/software.rst @@ -0,0 +1,18 @@ +.. only:: comment + + © Crown-owned copyright 2023, Defence Science and Technology Laboratory UK + + +Software +======== + + + + +Contents +######## + +.. toctree:: + :maxdepth: 8 + + database_client_server diff --git a/src/primaite/simulator/file_system/file_type.py b/src/primaite/simulator/file_system/file_type.py index 140cd0e7..f87cd86f 100644 --- a/src/primaite/simulator/file_system/file_type.py +++ b/src/primaite/simulator/file_system/file_type.py @@ -2,6 +2,7 @@ from __future__ import annotations from enum import Enum from random import choice +from typing import Any class FileType(Enum): @@ -95,7 +96,7 @@ class FileType(Enum): "Generic DB file. Used by sqlite3." @classmethod - def _missing_(cls, value): + def _missing_(cls, value: Any) -> FileType: return cls.UNKNOWN @classmethod @@ -118,7 +119,7 @@ class FileType(Enum): return size if size else 0 -def get_file_type_from_extension(file_type_extension: str): +def get_file_type_from_extension(file_type_extension: str) -> FileType: """ Get a FileType from a file type extension. diff --git a/src/primaite/simulator/network/hardware/base.py b/src/primaite/simulator/network/hardware/base.py index efc1e251..5b9cdf5b 100644 --- a/src/primaite/simulator/network/hardware/base.py +++ b/src/primaite/simulator/network/hardware/base.py @@ -960,6 +960,7 @@ class Node(SimComponent): return state def show(self, markdown: bool = False, component: Literal["NIC", "OPEN_PORTS"] = "NIC"): + """A multi-use .show function that accepts either NIC or OPEN_PORTS.""" if component == "NIC": self._show_nic(markdown) elif component == "OPEN_PORTS": diff --git a/src/primaite/simulator/network/networks.py b/src/primaite/simulator/network/networks.py index a364abea..b9554cb9 100644 --- a/src/primaite/simulator/network/networks.py +++ b/src/primaite/simulator/network/networks.py @@ -7,7 +7,7 @@ from primaite.simulator.network.hardware.nodes.switch import Switch from primaite.simulator.network.transmission.network_layer import IPProtocol from primaite.simulator.network.transmission.transport_layer import Port from primaite.simulator.system.applications.database_client import DatabaseClient -from primaite.simulator.system.services.database import DatabaseService +from primaite.simulator.system.services.database_service import DatabaseService def client_server_routed() -> Network: diff --git a/src/primaite/simulator/system/applications/application.py b/src/primaite/simulator/system/applications/application.py index 2a3013e1..30efd5b7 100644 --- a/src/primaite/simulator/system/applications/application.py +++ b/src/primaite/simulator/system/applications/application.py @@ -54,13 +54,13 @@ class Application(IOSoftware): return state def run(self) -> None: - """Open the Application""" + """Open the Application.""" if self.operating_state == ApplicationOperatingState.CLOSED: self.sys_log.info(f"Running Application {self.name}") self.operating_state = ApplicationOperatingState.RUNNING def close(self) -> None: - """Close the Application""" + """Close the Application.""" if self.operating_state == ApplicationOperatingState.RUNNING: self.sys_log.info(f"Closed Application{self.name}") self.operating_state = ApplicationOperatingState.CLOSED diff --git a/src/primaite/simulator/system/applications/database_client.py b/src/primaite/simulator/system/applications/database_client.py index 38ce3c7f..bbcde00f 100644 --- a/src/primaite/simulator/system/applications/database_client.py +++ b/src/primaite/simulator/system/applications/database_client.py @@ -10,6 +10,15 @@ from primaite.simulator.system.core.software_manager import SoftwareManager class DatabaseClient(Application): + """ + A DatabaseClient application. + + Extends the Application class to provide functionality for connecting, querying, and disconnecting from a + Database Service. It mainly operates over TCP protocol. + + :ivar server_ip_address: The IPv4 address of the Database Service server, defaults to None. + """ + server_ip_address: Optional[IPv4Address] = None connected: bool = False @@ -20,9 +29,21 @@ class DatabaseClient(Application): super().__init__(**kwargs) def describe_state(self) -> Dict: + """ + Describes the current state of the ACLRule. + + :return: A dictionary representing the current state. + """ + pass return super().describe_state() def connect(self, server_ip_address: IPv4Address, password: Optional[str] = None) -> bool: + """ + Connect to a Database Service. + + :param server_ip_address: The IPv4 Address of the Node the Database Service is running on. + :param password: The Database Service password. Is optional and has a default value of None. + """ if not self.connected and self.operating_state.RUNNING: return self._connect(server_ip_address, password) @@ -44,6 +65,7 @@ class DatabaseClient(Application): return self._connect(server_ip_address, password, True) def disconnect(self): + """Disconnect from the Database Service.""" if self.connected and self.operating_state.RUNNING: software_manager: SoftwareManager = self.software_manager software_manager.send_payload_to_session_manager( @@ -54,6 +76,11 @@ class DatabaseClient(Application): self.server_ip_address = None def query(self, sql: str): + """ + Send a query to the Database Service. + + :param sql: The SQL query. + """ if self.connected and self.operating_state.RUNNING: software_manager: SoftwareManager = self.software_manager software_manager.send_payload_to_session_manager( @@ -75,6 +102,13 @@ class DatabaseClient(Application): print(table) def receive(self, payload: Any, session_id: str, **kwargs) -> bool: + """ + Receive a payload from the Software Manager. + + :param payload: A payload to receive. + :param session_id: The session id the payload relates to. + :return: True. + """ if isinstance(payload, dict) and payload.get("type"): if payload["type"] == "connect_response": self.connected = payload["response"] == True diff --git a/src/primaite/simulator/system/core/software_manager.py b/src/primaite/simulator/system/core/software_manager.py index c3fe29fd..6860ebc2 100644 --- a/src/primaite/simulator/system/core/software_manager.py +++ b/src/primaite/simulator/system/core/software_manager.py @@ -9,7 +9,7 @@ from primaite.simulator.network.transmission.transport_layer import Port from primaite.simulator.system.applications.application import Application, ApplicationOperatingState from primaite.simulator.system.core.sys_log import SysLog from primaite.simulator.system.services.service import Service, ServiceOperatingState -from primaite.simulator.system.software import IOSoftware, SoftwareType +from primaite.simulator.system.software import IOSoftware if TYPE_CHECKING: from primaite.simulator.system.core.session_manager import SessionManager @@ -37,6 +37,11 @@ class SoftwareManager: self.file_system: FileSystem = file_system def get_open_ports(self) -> List[Port]: + """ + Get a list of open ports. + + :return: A list of all open ports on the Node. + """ open_ports = [Port.ARP] for software in self.port_protocol_mapping.values(): if software.operating_state in {ApplicationOperatingState.RUNNING, ServiceOperatingState.RUNNING}: @@ -45,6 +50,11 @@ class SoftwareManager: return open_ports def install(self, software_class: Type[IOSoftwareClass]): + """ + Install an Application or Service. + + :param software_class: The software class. + """ if software_class in self._software_class_to_name_map: self.sys_log.info(f"Cannot install {software_class} as it is already installed") return @@ -59,6 +69,11 @@ class SoftwareManager: software.operating_state = ApplicationOperatingState.CLOSED def uninstall(self, software_name: str): + """ + Uninstall an Application or Service. + + :param software_name: The software name. + """ if software_name in self.software: software = self.software.pop(software_name) # noqa del software diff --git a/src/primaite/simulator/system/services/database.py b/src/primaite/simulator/system/services/database_service.py similarity index 98% rename from src/primaite/simulator/system/services/database.py rename to src/primaite/simulator/system/services/database_service.py index e34f06fa..d4289c08 100644 --- a/src/primaite/simulator/system/services/database.py +++ b/src/primaite/simulator/system/services/database_service.py @@ -1,6 +1,5 @@ import sqlite3 from datetime import datetime -from ipaddress import IPv4Address from sqlite3 import OperationalError from typing import Any, Dict, List, Optional, Union @@ -9,7 +8,6 @@ from prettytable import MARKDOWN, PrettyTable from primaite.simulator.file_system.file_system import File from primaite.simulator.network.transmission.network_layer import IPProtocol from primaite.simulator.network.transmission.transport_layer import Port -from primaite.simulator.system.core.session_manager import Session from primaite.simulator.system.core.software_manager import SoftwareManager from primaite.simulator.system.services.service import Service, ServiceOperatingState from primaite.simulator.system.software import SoftwareHealthState diff --git a/tests/integration_tests/system/test_database_on_node.py b/tests/integration_tests/system/test_database_on_node.py index 7562b29b..b360907e 100644 --- a/tests/integration_tests/system/test_database_on_node.py +++ b/tests/integration_tests/system/test_database_on_node.py @@ -2,7 +2,7 @@ from ipaddress import IPv4Address from primaite.simulator.network.hardware.nodes.server import Server from primaite.simulator.system.applications.database_client import DatabaseClient -from primaite.simulator.system.services.database import DatabaseService +from primaite.simulator.system.services.database_service import DatabaseService def test_database_client_server_connection(uc2_network): diff --git a/tests/unit_tests/_primaite/_simulator/_system/_services/test_database.py b/tests/unit_tests/_primaite/_simulator/_system/_services/test_database.py index f3751f27..db33908d 100644 --- a/tests/unit_tests/_primaite/_simulator/_system/_services/test_database.py +++ b/tests/unit_tests/_primaite/_simulator/_system/_services/test_database.py @@ -3,7 +3,7 @@ import json import pytest from primaite.simulator.network.hardware.base import Node -from primaite.simulator.system.services.database import DatabaseService +from primaite.simulator.system.services.database_service import DatabaseService DDL = """ CREATE TABLE IF NOT EXISTS user (