Merged PR 187: Database Service backup using FTP

## Summary
- Used FTP changes to allow the database service to create a copy of the database.db file in the backup server
- Used FTP changes to allow the database service to restore the database.db file from a copy made in the backup server

## Test process
*How have you tested this (if applicable)?*

## Checklist
- [x] PR is linked to a **work item**
- [x] **acceptance criteria** of linked ticket are met
- [X] performed **self-review** of the code
- [x] written **tests** for any new functionality added with this PR
- [X] updated the **documentation** if this PR changes or adds functionality
- [ ] written/updated **design docs** if this PR implements new functionality
- [X] updated the **change log**
- [x] ran **pre-commit** checks for code style
- [x] attended to any **TO-DOs** left in the code

Related work items: #1752, #1796, #1816, #1866, #1916

CHANGELOG.md

@@ -27,6 +27,9 @@ SessionManager.
 - File System - ability to emulate a node's file system during a simulation
 - Example notebooks - There is currently 1 jupyter notebook which walks through using PrimAITE
   1. Creating a simulation - this notebook explains how to build up a simulation using the Python package. (WIP)
+- Database:
+  - `DatabaseClient` and `DatabaseService` created to allow emulation of database actions
+  - Ability for `DatabaseService` to backup its data to another server via FTP and restore data from backup
 - Red Agent Services:
   - Data Manipulator Bot - A red agent service which sends a payload to a target machine. (By default this payload is a SQL query that breaks a database)
 - DNS Services: `DNSClient` and `DNSServer`

docs/source/simulation_components/system/database_client_server.rst

@@ -60,6 +60,12 @@ Usage
 - Retrieve results in a dictionary.
 - Disconnect when finished.
 
+To create database backups:
+
+- Configure the backup server on the ``DatabaseService`` by providing the Backup server ``IPv4Address`` with ``configure_backup``
+- Create a backup using ``backup_database``. This fails if the backup server is not configured.
+- Restore a backup using ``restore_backup``. By default, this uses the database created via ``backup_database``.
+
 Implementation
 ^^^^^^^^^^^^^^
 

docs/source/simulation_components/system/ftp_client_server.rst

@@ -94,11 +94,11 @@ Example peer to peer network
 Install the FTP Server
 ^^^^^^^^^^^^^^^^^^^^^^
 
+FTP Client should be pre installed on nodes
+
 .. code-block:: python
 
     srv.software_manager.install(FTPServer)
-    pc1.software_manager.install(FTPClient)
-    client: FTPClient = pc1.software_manager.software['FTPClient']
     ftpserv: FTPServer = srv.software_manager.software['FTPServer']
 
 Setting up the FTP Server

src/primaite/simulator/network/hardware/base.py

@@ -496,7 +496,9 @@ class Link(SimComponent):
     def _can_transmit(self, frame: Frame) -> bool:
         if self.is_up:
             frame_size_Mbits = frame.size_Mbits  # noqa - Leaving it as Mbits as this is how they're expressed
-            return self.current_load + frame_size_Mbits <= self.bandwidth
+            # return self.current_load + frame_size_Mbits <= self.bandwidth
+            # TODO: re add this check once packet size limiting and MTU checks are implemented
+            return True
         return False
 
     def transmit_frame(self, sender_nic: Union[NIC, SwitchPort], frame: Frame) -> bool:
@@ -937,6 +939,12 @@ class Node(SimComponent):
         self.arp.nics = self.nics
         self.session_manager.software_manager = self.software_manager
 
+        self._install_system_software()
+
+    def _install_system_software(self):
+        """Install System Software - software that is usually provided with the OS."""
+        pass
+
     def describe_state(self) -> Dict:
         """
         Produce a dictionary describing the current state of this object.

src/primaite/simulator/network/hardware/nodes/computer.py

@@ -1,4 +1,6 @@
 from primaite.simulator.network.hardware.base import NIC, Node
+from primaite.simulator.system.services.dns.dns_client import DNSClient
+from primaite.simulator.system.services.ftp.ftp_client import FTPClient
 
 
 class Computer(Node):
@@ -36,3 +38,14 @@ class Computer(Node):
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
         self.connect_nic(NIC(ip_address=kwargs["ip_address"], subnet_mask=kwargs["subnet_mask"]))
+        self._install_system_software()
+
+    def _install_system_software(self):
+        """Install System Software - software that is usually provided with the OS."""
+        # DNS Client
+        self.software_manager.install(DNSClient)
+
+        # FTP
+        self.software_manager.install(FTPClient)
+
+        super()._install_system_software()

src/primaite/simulator/network/networks.py

@@ -10,9 +10,7 @@ from primaite.simulator.network.transmission.network_layer import IPProtocol
 from primaite.simulator.network.transmission.transport_layer import Port
 from primaite.simulator.system.applications.database_client import DatabaseClient
 from primaite.simulator.system.services.database.database_service import DatabaseService
-from primaite.simulator.system.services.dns.dns_client import DNSClient
 from primaite.simulator.system.services.dns.dns_server import DNSServer
-from primaite.simulator.system.services.ftp.ftp_client import FTPClient
 from primaite.simulator.system.services.ftp.ftp_server import FTPServer
 from primaite.simulator.system.services.red_services.data_manipulation_bot import DataManipulationBot
 
@@ -137,14 +135,11 @@ def arcd_uc2_network() -> Network:
         dns_server=IPv4Address("192.168.1.10"),
     )
     client_1.power_on()
-    client_1.software_manager.install(DNSClient)
     network.connect(endpoint_b=client_1.ethernet_port[1], endpoint_a=switch_2.switch_ports[1])
     client_1.software_manager.install(DataManipulationBot)
     db_manipulation_bot: DataManipulationBot = client_1.software_manager.software["DataManipulationBot"]
     db_manipulation_bot.configure(server_ip_address=IPv4Address("192.168.1.14"), payload="DROP TABLE IF EXISTS user;")
 
-    client_1.software_manager.install(FTPClient)
-
     # Client 2
     client_2 = Computer(
         hostname="client_2",
@@ -154,11 +149,8 @@ def arcd_uc2_network() -> Network:
         dns_server=IPv4Address("192.168.1.10"),
     )
     client_2.power_on()
-    client_2.software_manager.install(DNSClient)
     network.connect(endpoint_b=client_2.ethernet_port[1], endpoint_a=switch_2.switch_ports[2])
 
-    client_2.software_manager.install(FTPClient)
-
     # Domain Controller
     domain_controller = Server(
         hostname="domain_controller",
@@ -239,6 +231,7 @@ def arcd_uc2_network() -> Network:
     database_server.software_manager.install(DatabaseService)
     database_service: DatabaseService = database_server.software_manager.software["DatabaseService"]  # noqa
     database_service.start()
+    database_service.configure_backup(backup_server=IPv4Address("192.168.1.16"))
     database_service._process_sql(ddl, None)  # noqa
     for insert_statement in user_insert_statements:
         database_service._process_sql(insert_statement, None)  # noqa

src/primaite/simulator/system/services/database/database_service.py

@@ -1,5 +1,6 @@
 import sqlite3
 from datetime import datetime
+from ipaddress import IPv4Address
 from sqlite3 import OperationalError
 from typing import Any, Dict, List, Optional, Union
 
@@ -9,6 +10,7 @@ from primaite.simulator.file_system.file_system import File
 from primaite.simulator.network.transmission.network_layer import IPProtocol
 from primaite.simulator.network.transmission.transport_layer import Port
 from primaite.simulator.system.core.software_manager import SoftwareManager
+from primaite.simulator.system.services.ftp.ftp_client import FTPClient
 from primaite.simulator.system.services.service import Service, ServiceOperatingState
 from primaite.simulator.system.software import SoftwareHealthState
 
@@ -23,6 +25,15 @@ class DatabaseService(Service):
     password: Optional[str] = None
     connections: Dict[str, datetime] = {}
 
+    backup_server: IPv4Address = None
+    """IP address of the backup server."""
+
+    latest_backup_directory: str = None
+    """Directory of latest backup."""
+
+    latest_backup_file_name: str = None
+    """File name of latest backup."""
+
     def __init__(self, **kwargs):
         kwargs["name"] = "DatabaseService"
         kwargs["port"] = Port.POSTGRES_SERVER
@@ -30,6 +41,9 @@ class DatabaseService(Service):
         super().__init__(**kwargs)
         self._db_file: File
         self._create_db_file()
+        self._connect()
+
+    def _connect(self):
         self._conn = sqlite3.connect(self._db_file.sim_path)
         self._cursor = self._conn.cursor()
 
@@ -40,8 +54,10 @@ class DatabaseService(Service):
         :return: List of table names.
         """
         sql = "SELECT name FROM sqlite_master WHERE type='table' AND name != 'sqlite_sequence';"
-        results = self._process_sql(sql)
+        results = self._process_sql(sql, None)
-        return [row[0] for row in results["data"]]
+        if isinstance(results["data"], dict):
+            return list(results["data"].keys())
+        return []
 
     def show(self, markdown: bool = False):
         """
@@ -58,6 +74,72 @@ class DatabaseService(Service):
             table.add_row([row])
         print(table)
 
+    def configure_backup(self, backup_server: IPv4Address):
+        """
+        Set up the database backup.
+
+        :param: backup_server_ip: The IP address of the backup server
+        """
+        self.backup_server = backup_server
+
+    def backup_database(self) -> bool:
+        """Create a backup of the database to the configured backup server."""
+        # check if the backup server was configured
+        if self.backup_server is None:
+            self.sys_log.error(f"{self.name} - {self.sys_log.hostname}: not configured.")
+            return False
+
+        self._conn.close()
+
+        software_manager: SoftwareManager = self.software_manager
+        ftp_client_service: FTPClient = software_manager.software["FTPClient"]
+
+        # send backup copy of database file to FTP server
+        response = ftp_client_service.send_file(
+            dest_ip_address=self.backup_server,
+            src_file_name=self._db_file.name,
+            src_folder_name=self._db_file.folder.name,
+            dest_folder_name=str(self.uuid),
+            dest_file_name="database.db",
+            real_file_path=self._db_file.sim_path,
+        )
+        self._connect()
+
+        if response:
+            return True
+
+        self.sys_log.error("Unable to create database backup.")
+        return False
+
+    def restore_backup(self) -> bool:
+        """Restore a backup from backup server."""
+        software_manager: SoftwareManager = self.software_manager
+        ftp_client_service: FTPClient = software_manager.software["FTPClient"]
+
+        # retrieve backup file from backup server
+        response = ftp_client_service.request_file(
+            src_folder_name=str(self.uuid),
+            src_file_name="database.db",
+            dest_folder_name="downloads",
+            dest_file_name="database.db",
+            dest_ip_address=self.backup_server,
+        )
+
+        if response:
+            self._conn.close()
+            # replace db file
+            self.file_system.delete_file(folder_name=self.folder.name, file_name="downloads.db")
+            self.file_system.move_file(
+                src_folder_name="downloads", src_file_name="database.db", dst_folder_name=self.folder.name
+            )
+            self._db_file = self.file_system.get_file(folder_name=self.folder.name, file_name="database.db")
+            self._connect()
+
+            return self._db_file is not None
+
+        self.sys_log.error("Unable to restore database backup.")
+        return False
+
     def _create_db_file(self):
         """Creates the Simulation File and sqlite file in the file system."""
         self._db_file: File = self.file_system.create_file(folder_name="database", file_name="database.db", real=True)

src/primaite/simulator/system/services/ftp/ftp_client.py

@@ -119,6 +119,7 @@ class FTPClient(FTPServiceABC):
         dest_folder_name: str,
         dest_file_name: str,
         dest_port: Optional[Port] = Port.FTP,
+        real_file_path: Optional[str] = None,
     ) -> bool:
         """
         Send a file to a target IP address.
@@ -159,18 +160,19 @@ class FTPClient(FTPServiceABC):
         if not self.connected:
             return False
         else:
+            self.sys_log.info(f"Sending file {src_folder_name}/{src_file_name} to {str(dest_ip_address)}")
             # send STOR request
-            self._send_data(
+            if self._send_data(
                 file=file_to_transfer,
                 dest_folder_name=dest_folder_name,
                 dest_file_name=dest_file_name,
                 dest_ip_address=dest_ip_address,
                 dest_port=dest_port,
-            )
+            ):
-
-            # send disconnect
                 return self._disconnect_from_server(dest_ip_address=dest_ip_address, dest_port=dest_port)
 
+            return False
+
     def request_file(
         self,
         dest_ip_address: IPv4Address,
@@ -222,14 +224,12 @@ class FTPClient(FTPServiceABC):
                     "dest_folder_name": dest_folder_name,
                 },
             )
+            self.sys_log.info(f"Requesting file {src_folder_name}/{src_file_name} from {str(dest_ip_address)}")
             software_manager: SoftwareManager = self.software_manager
             software_manager.send_payload_to_session_manager(
                 payload=payload, dest_ip_address=dest_ip_address, dest_port=dest_port
             )
 
-            # send disconnect
-            self._disconnect_from_server(dest_ip_address=dest_ip_address, dest_port=dest_port)
-
             # the payload should have ok status code
             if payload.status_code == FTPStatusCode.OK:
                 self.sys_log.info(f"File {src_folder_name}/{src_file_name} found in FTP server.")

src/primaite/simulator/system/services/ftp/ftp_server.py

@@ -55,6 +55,9 @@ class FTPServer(FTPServiceABC):
         if session_id:
             session_details = self._get_session_details(session_id)
 
+        if payload.ftp_command is not None:
+            self.sys_log.info(f"Received FTP {payload.ftp_command.name} command.")
+
         # process server specific commands, otherwise call super
         if payload.ftp_command == FTPCommand.PORT:
             # check that the port is valid

src/primaite/simulator/system/services/ftp/ftp_service.py

@@ -1,3 +1,4 @@
+import shutil
 from abc import ABC
 from ipaddress import IPv4Address
 from typing import Optional
@@ -25,6 +26,9 @@ class FTPServiceABC(Service, ABC):
         :param: session_id: session ID linked to the FTP Packet. Optional.
         :type: session_id: Optional[str]
         """
+        if payload.ftp_command is not None:
+            self.sys_log.info(f"Received FTP {payload.ftp_command.name} command.")
+
         # handle STOR request
         if payload.ftp_command == FTPCommand.STOR:
             # check that the file is created in the computed hosting the FTP server
@@ -48,15 +52,17 @@ class FTPServiceABC(Service, ABC):
             file_name = payload.ftp_command_args["dest_file_name"]
             folder_name = payload.ftp_command_args["dest_folder_name"]
             file_size = payload.ftp_command_args["file_size"]
-            self.file_system.create_file(
+            real_file_path = payload.ftp_command_args.get("real_file_path")
-                file_name=file_name,
+            is_real = real_file_path is not None
-                folder_name=folder_name,
+            file = self.file_system.create_file(
-                size=file_size,
+                file_name=file_name, folder_name=folder_name, size=file_size, real=is_real
             )
             self.sys_log.info(
                 f"Created item in {self.sys_log.hostname}: {payload.ftp_command_args['dest_folder_name']}/"
                 f"{payload.ftp_command_args['dest_file_name']}"
             )
+            if is_real:
+                shutil.copy(real_file_path, file.sim_path)
             # file should exist
             return self.file_system.get_file(file_name=file_name, folder_name=folder_name) is not None
         except Exception as e:
@@ -100,6 +106,7 @@ class FTPServiceABC(Service, ABC):
                 "dest_folder_name": dest_folder_name,
                 "dest_file_name": dest_file_name,
                 "file_size": file.sim_size,
+                "real_file_path": file.sim_path if file.real else None,
             },
             packet_payload_size=file.sim_size,
         )

tests/e2e_integration_tests/test_uc2_data_manipulation_scenario.py

@@ -6,6 +6,7 @@ from primaite.simulator.system.services.red_services.data_manipulation_bot impor
 
 
 def test_data_manipulation(uc2_network):
+    """Tests the UC2 data manipulation scenario end-to-end. Is a work in progress."""
     client_1: Computer = uc2_network.get_node_by_hostname("client_1")
     db_manipulation_bot: DataManipulationBot = client_1.software_manager.software["DataManipulationBot"]
 
@@ -15,6 +16,8 @@ def test_data_manipulation(uc2_network):
     web_server: Server = uc2_network.get_node_by_hostname("web_server")
     db_client: DatabaseClient = web_server.software_manager.software["DatabaseClient"]
 
+    db_service.backup_database()
+
     # First check that the DB client on the web_server can successfully query the users table on the database
     assert db_client.query("SELECT * FROM user;")
 
@@ -23,3 +26,9 @@ def test_data_manipulation(uc2_network):
 
     # Now check that the DB client on the web_server cannot query the users table on the database
     assert not db_client.query("SELECT * FROM user;")
+
+    # Now restore the database
+    db_service.restore_backup()
+
+    # Now check that the DB client on the web_server can successfully query the users table on the database
+    assert db_client.query("SELECT * FROM user;")

tests/integration_tests/system/test_database_on_node.py

@@ -3,6 +3,7 @@ from ipaddress import IPv4Address
 from primaite.simulator.network.hardware.nodes.server import Server
 from primaite.simulator.system.applications.database_client import DatabaseClient
 from primaite.simulator.system.services.database.database_service import DatabaseService
+from primaite.simulator.system.services.ftp.ftp_server import FTPServer
 
 
 def test_database_client_server_connection(uc2_network):
@@ -57,3 +58,37 @@ def test_database_client_query(uc2_network):
     db_client.connect()
 
     assert db_client.query("SELECT * FROM user;")
+
+
+def test_create_database_backup(uc2_network):
+    """Run the backup_database method and check if the FTP server has the relevant file."""
+    db_server: Server = uc2_network.get_node_by_hostname("database_server")
+    db_service: DatabaseService = db_server.software_manager.software["DatabaseService"]
+
+    # back up should be created
+    assert db_service.backup_database() is True
+
+    backup_server: Server = uc2_network.get_node_by_hostname("backup_server")
+    ftp_server: FTPServer = backup_server.software_manager.software["FTPServer"]
+
+    # backup file should exist in the backup server
+    assert ftp_server.file_system.get_file(folder_name=db_service.uuid, file_name="database.db") is not None
+
+
+def test_restore_backup(uc2_network):
+    """Run the restore_backup method and check if the backup is properly restored."""
+    db_server: Server = uc2_network.get_node_by_hostname("database_server")
+    db_service: DatabaseService = db_server.software_manager.software["DatabaseService"]
+
+    # create a back up
+    assert db_service.backup_database() is True
+
+    # delete database locally
+    db_service.file_system.delete_file(folder_name="database", file_name="database.db")
+
+    assert db_service.file_system.get_file(folder_name="database", file_name="database.db") is None
+
+    # back up should be restored
+    assert db_service.restore_backup() is True
+
+    assert db_service.file_system.get_file(folder_name="database", file_name="database.db") is not None

tests/integration_tests/system/test_ftp_client_server.py

@@ -23,7 +23,7 @@ def test_ftp_client_store_file_in_server(uc2_network):
     # create file on ftp client
     ftp_client.file_system.create_file(file_name="test_file.txt")
 
-    ftp_client.send_file(
+    assert ftp_client.send_file(
         src_folder_name="root",
         src_file_name="test_file.txt",
         dest_folder_name="client_1_backup",
@@ -50,7 +50,7 @@ def test_ftp_client_retrieve_file_from_server(uc2_network):
     # create file on ftp server
     ftp_server.file_system.create_file(file_name="test_file.txt", folder_name="file_share")
 
-    ftp_client.request_file(
+    assert ftp_client.request_file(
         src_folder_name="file_share",
         src_file_name="test_file.txt",
         dest_folder_name="downloads",

tests/unit_tests/_primaite/_simulator/_system/_services/test_dns.py

@@ -3,6 +3,8 @@ from ipaddress import IPv4Address
 import pytest
 
 from primaite.simulator.network.hardware.base import Node
+from primaite.simulator.network.hardware.nodes.computer import Computer
+from primaite.simulator.network.hardware.nodes.server import Server
 from primaite.simulator.network.protocols.dns import DNSPacket, DNSReply, DNSRequest
 from primaite.simulator.network.transmission.network_layer import IPProtocol
 from primaite.simulator.network.transmission.transport_layer import Port
@@ -12,7 +14,9 @@ from primaite.simulator.system.services.dns.dns_server import DNSServer
 
 @pytest.fixture(scope="function")
 def dns_server() -> Node:
-    node = Node(hostname="dns_server")
+    node = Server(
+        hostname="dns_server", ip_address="192.168.1.10", subnet_mask="255.255.255.0", default_gateway="192.168.1.1"
+    )
     node.software_manager.install(software_class=DNSServer)
     node.software_manager.software["DNSServer"].start()
     return node
@@ -20,9 +24,9 @@ def dns_server() -> Node:
 
 @pytest.fixture(scope="function")
 def dns_client() -> Node:
-    node = Node(hostname="dns_client")
+    node = Computer(
-    node.software_manager.install(software_class=DNSClient)
+        hostname="dns_client", ip_address="192.168.1.11", subnet_mask="255.255.255.0", default_gateway="192.168.1.1"
-    node.software_manager.software["DNSClient"].start()
+    )
     return node
 
 

tests/unit_tests/_primaite/_simulator/_system/_services/test_ftp.py

@@ -3,6 +3,8 @@ from ipaddress import IPv4Address
 import pytest
 
 from primaite.simulator.network.hardware.base import Node
+from primaite.simulator.network.hardware.nodes.computer import Computer
+from primaite.simulator.network.hardware.nodes.server import Server
 from primaite.simulator.network.protocols.ftp import FTPCommand, FTPPacket
 from primaite.simulator.network.transmission.network_layer import IPProtocol
 from primaite.simulator.network.transmission.transport_layer import Port
@@ -12,7 +14,9 @@ from primaite.simulator.system.services.ftp.ftp_server import FTPServer
 
 @pytest.fixture(scope="function")
 def ftp_server() -> Node:
-    node = Node(hostname="ftp_server")
+    node = Server(
+        hostname="ftp_server", ip_address="192.168.1.10", subnet_mask="255.255.255.0", default_gateway="192.168.1.1"
+    )
     node.software_manager.install(software_class=FTPServer)
     node.software_manager.software["FTPServer"].start()
     return node
@@ -20,9 +24,9 @@ def ftp_server() -> Node:
 
 @pytest.fixture(scope="function")
 def ftp_client() -> Node:
-    node = Node(hostname="ftp_client")
+    node = Computer(
-    node.software_manager.install(software_class=FTPClient)
+        hostname="ftp_client", ip_address="192.168.1.11", subnet_mask="255.255.255.0", default_gateway="192.168.1.1"
-    node.software_manager.software["FTPClient"].start()
+    )
     return node