Merge 'origin/feature/1812' into feature/1924

This commit is contained in:
Marek Wolan
2023-10-09 18:25:46 +01:00
43 changed files with 1362 additions and 879 deletions

View File

@@ -1,5 +1,6 @@
import sqlite3
from datetime import datetime
from ipaddress import IPv4Address
from sqlite3 import OperationalError
from typing import Any, Dict, List, Optional, Union
@@ -9,6 +10,7 @@ from primaite.simulator.file_system.file_system import File
from primaite.simulator.network.transmission.network_layer import IPProtocol
from primaite.simulator.network.transmission.transport_layer import Port
from primaite.simulator.system.core.software_manager import SoftwareManager
from primaite.simulator.system.services.ftp.ftp_client import FTPClient
from primaite.simulator.system.services.service import Service, ServiceOperatingState
from primaite.simulator.system.software import SoftwareHealthState
@@ -23,6 +25,15 @@ class DatabaseService(Service):
password: Optional[str] = None
connections: Dict[str, datetime] = {}
backup_server: IPv4Address = None
"""IP address of the backup server."""
latest_backup_directory: str = None
"""Directory of latest backup."""
latest_backup_file_name: str = None
"""File name of latest backup."""
def __init__(self, **kwargs):
kwargs["name"] = "DatabaseService"
kwargs["port"] = Port.POSTGRES_SERVER
@@ -30,6 +41,9 @@ class DatabaseService(Service):
super().__init__(**kwargs)
self._db_file: File
self._create_db_file()
self._connect()
def _connect(self):
self._conn = sqlite3.connect(self._db_file.sim_path)
self._cursor = self._conn.cursor()
@@ -40,8 +54,10 @@ class DatabaseService(Service):
:return: List of table names.
"""
sql = "SELECT name FROM sqlite_master WHERE type='table' AND name != 'sqlite_sequence';"
results = self._process_sql(sql)
return [row[0] for row in results["data"]]
results = self._process_sql(sql, None)
if isinstance(results["data"], dict):
return list(results["data"].keys())
return []
def show(self, markdown: bool = False):
"""
@@ -58,6 +74,72 @@ class DatabaseService(Service):
table.add_row([row])
print(table)
def configure_backup(self, backup_server: IPv4Address):
"""
Set up the database backup.
:param: backup_server_ip: The IP address of the backup server
"""
self.backup_server = backup_server
def backup_database(self) -> bool:
"""Create a backup of the database to the configured backup server."""
# check if the backup server was configured
if self.backup_server is None:
self.sys_log.error(f"{self.name} - {self.sys_log.hostname}: not configured.")
return False
self._conn.close()
software_manager: SoftwareManager = self.software_manager
ftp_client_service: FTPClient = software_manager.software["FTPClient"]
# send backup copy of database file to FTP server
response = ftp_client_service.send_file(
dest_ip_address=self.backup_server,
src_file_name=self._db_file.name,
src_folder_name=self._db_file.folder.name,
dest_folder_name=str(self.uuid),
dest_file_name="database.db",
real_file_path=self._db_file.sim_path,
)
self._connect()
if response:
return True
self.sys_log.error("Unable to create database backup.")
return False
def restore_backup(self) -> bool:
"""Restore a backup from backup server."""
software_manager: SoftwareManager = self.software_manager
ftp_client_service: FTPClient = software_manager.software["FTPClient"]
# retrieve backup file from backup server
response = ftp_client_service.request_file(
src_folder_name=str(self.uuid),
src_file_name="database.db",
dest_folder_name="downloads",
dest_file_name="database.db",
dest_ip_address=self.backup_server,
)
if response:
self._conn.close()
# replace db file
self.file_system.delete_file(folder_name=self.folder.name, file_name="downloads.db")
self.file_system.move_file(
src_folder_name="downloads", src_file_name="database.db", dst_folder_name=self.folder.name
)
self._db_file = self.file_system.get_file(folder_name=self.folder.name, file_name="database.db")
self._connect()
return self._db_file is not None
self.sys_log.error("Unable to restore database backup.")
return False
def _create_db_file(self):
"""Creates the Simulation File and sqlite file in the file system."""
self._db_file: File = self.file_system.create_file(folder_name="database", file_name="database.db", real=True)

View File

@@ -27,6 +27,7 @@ class DNSClient(Service):
# TCP for now
kwargs["protocol"] = IPProtocol.TCP
super().__init__(**kwargs)
self.start()
def describe_state(self) -> Dict:
"""

View File

@@ -26,6 +26,7 @@ class DNSServer(Service):
# TCP for now
kwargs["protocol"] = IPProtocol.TCP
super().__init__(**kwargs)
self.start()
def describe_state(self) -> Dict:
"""

View File

@@ -0,0 +1,256 @@
from ipaddress import IPv4Address
from typing import Optional
from primaite.simulator.file_system.file_system import File
from primaite.simulator.network.protocols.ftp import FTPCommand, FTPPacket, FTPStatusCode
from primaite.simulator.network.transmission.network_layer import IPProtocol
from primaite.simulator.network.transmission.transport_layer import Port
from primaite.simulator.system.core.software_manager import SoftwareManager
from primaite.simulator.system.services.ftp.ftp_service import FTPServiceABC
from primaite.simulator.system.services.service import ServiceOperatingState
class FTPClient(FTPServiceABC):
"""
A class for simulating an FTP client service.
This class inherits from the `Service` class and provides methods to emulate FTP
RFC 959: https://datatracker.ietf.org/doc/html/rfc959
"""
connected: bool = False
"""Keeps track of whether or not the FTP client is connected to an FTP server."""
def __init__(self, **kwargs):
kwargs["name"] = "FTPClient"
kwargs["port"] = Port.FTP
kwargs["protocol"] = IPProtocol.TCP
super().__init__(**kwargs)
self.start()
def _process_ftp_command(self, payload: FTPPacket, session_id: Optional[str] = None, **kwargs) -> FTPPacket:
"""
Process the command in the FTP Packet.
:param: payload: The FTP Packet to process
:type: payload: FTPPacket
:param: session_id: session ID linked to the FTP Packet. Optional.
:type: session_id: Optional[str]
"""
# if client service is down, return error
if self.operating_state != ServiceOperatingState.RUNNING:
payload.status_code = FTPStatusCode.ERROR
return payload
# process client specific commands, otherwise call super
return super()._process_ftp_command(payload=payload, session_id=session_id, **kwargs)
def _connect_to_server(
self,
dest_ip_address: Optional[IPv4Address] = None,
dest_port: Optional[Port] = Port.FTP,
is_reattempt: Optional[bool] = False,
) -> bool:
"""
Connects the client to a given FTP server.
:param: dest_ip_address: IP address of the FTP server the client needs to connect to. Optional.
:type: dest_ip_address: Optional[IPv4Address]
:param: dest_port: Port of the FTP server the client needs to connect to. Optional.
:type: dest_port: Optional[Port]
:param: is_reattempt: Set to True if attempt to connect to FTP Server has been attempted. Default False.
:type: is_reattempt: Optional[bool]
"""
# make sure the service is running before attempting
if self.operating_state != ServiceOperatingState.RUNNING:
self.sys_log.error(f"FTPClient not running for {self.sys_log.hostname}")
return False
# normally FTP will choose a random port for the transfer, but using the FTP command port will do for now
# create FTP packet
payload: FTPPacket = FTPPacket(
ftp_command=FTPCommand.PORT,
ftp_command_args=Port.FTP,
)
software_manager: SoftwareManager = self.software_manager
software_manager.send_payload_to_session_manager(
payload=payload, dest_ip_address=dest_ip_address, dest_port=dest_port
)
if payload.status_code == FTPStatusCode.OK:
return True
else:
if is_reattempt:
# reattempt failed
return False
else:
# try again
self._connect_to_server(dest_ip_address=dest_ip_address, dest_port=dest_port, is_reattempt=True)
def _disconnect_from_server(
self, dest_ip_address: Optional[IPv4Address] = None, dest_port: Optional[Port] = Port.FTP
) -> bool:
"""
Connects the client from a given FTP server.
:param: dest_ip_address: IP address of the FTP server the client needs to disconnect from. Optional.
:type: dest_ip_address: Optional[IPv4Address]
:param: dest_port: Port of the FTP server the client needs to disconnect from. Optional.
:type: dest_port: Optional[Port]
:param: is_reattempt: Set to True if attempt to disconnect from FTP Server has been attempted. Default False.
:type: is_reattempt: Optional[bool]
"""
# send a disconnect request payload to FTP server
payload: FTPPacket = FTPPacket(ftp_command=FTPCommand.QUIT)
software_manager: SoftwareManager = self.software_manager
software_manager.send_payload_to_session_manager(
payload=payload, dest_ip_address=dest_ip_address, dest_port=dest_port
)
if payload.status_code == FTPStatusCode.OK:
self.connected = False
return True
return False
def send_file(
self,
dest_ip_address: IPv4Address,
src_folder_name: str,
src_file_name: str,
dest_folder_name: str,
dest_file_name: str,
dest_port: Optional[Port] = Port.FTP,
real_file_path: Optional[str] = None,
) -> bool:
"""
Send a file to a target IP address.
The function checks if the file exists in the FTP Client host.
The STOR command is then sent to the FTP Server.
:param: dest_ip_address: The IP address of the machine that hosts the FTP Server.
:type: dest_ip_address: IPv4Address
:param: src_folder_name: The name of the folder that contains the file to send to the FTP Server.
:type: src_folder_name: str
:param: src_file_name: The name of the file to send to the FTP Server.
:type: src_file_name: str
:param: dest_folder_name: The name of the folder where the file will be stored in the FTP Server.
:type: dest_folder_name: str
:param: dest_file_name: The name of the file to be saved on the FTP Server.
:type: dest_file_name: str
:param: dest_port: The open port of the machine that hosts the FTP Server. Default is Port.FTP.
:type: dest_port: Optional[Port]
"""
# check if the file to transfer exists on the client
file_to_transfer: File = self.file_system.get_file(folder_name=src_folder_name, file_name=src_file_name)
if not file_to_transfer:
self.sys_log.error(f"Unable to send file that does not exist: {src_folder_name}/{src_file_name}")
return False
# check if FTP is currently connected to IP
self.connected = self._connect_to_server(
dest_ip_address=dest_ip_address,
dest_port=dest_port,
)
if not self.connected:
return False
else:
self.sys_log.info(f"Sending file {src_folder_name}/{src_file_name} to {str(dest_ip_address)}")
# send STOR request
if self._send_data(
file=file_to_transfer,
dest_folder_name=dest_folder_name,
dest_file_name=dest_file_name,
dest_ip_address=dest_ip_address,
dest_port=dest_port,
):
return self._disconnect_from_server(dest_ip_address=dest_ip_address, dest_port=dest_port)
return False
def request_file(
self,
dest_ip_address: IPv4Address,
src_folder_name: str,
src_file_name: str,
dest_folder_name: str,
dest_file_name: str,
dest_port: Optional[Port] = Port.FTP,
) -> bool:
"""
Request a file from a target IP address.
Sends a RETR command to the FTP Server.
:param: dest_ip_address: The IP address of the machine that hosts the FTP Server.
:type: dest_ip_address: IPv4Address
:param: src_folder_name: The name of the folder that contains the file to send to the FTP Server.
:type: src_folder_name: str
:param: src_file_name: The name of the file to send to the FTP Server.
:type: src_file_name: str
:param: dest_folder_name: The name of the folder where the file will be stored in the FTP Server.
:type: dest_folder_name: str
:param: dest_file_name: The name of the file to be saved on the FTP Server.
:type: dest_file_name: str
:param: dest_port: The open port of the machine that hosts the FTP Server. Default is Port.FTP.
:type: dest_port: Optional[Port]
"""
# check if FTP is currently connected to IP
self.connected = self._connect_to_server(
dest_ip_address=dest_ip_address,
dest_port=dest_port,
)
if not self.connected:
return False
else:
# send retrieve request
payload: FTPPacket = FTPPacket(
ftp_command=FTPCommand.RETR,
ftp_command_args={
"src_folder_name": src_folder_name,
"src_file_name": src_file_name,
"dest_file_name": dest_file_name,
"dest_folder_name": dest_folder_name,
},
)
self.sys_log.info(f"Requesting file {src_folder_name}/{src_file_name} from {str(dest_ip_address)}")
software_manager: SoftwareManager = self.software_manager
software_manager.send_payload_to_session_manager(
payload=payload, dest_ip_address=dest_ip_address, dest_port=dest_port
)
# the payload should have ok status code
if payload.status_code == FTPStatusCode.OK:
self.sys_log.info(f"File {src_folder_name}/{src_file_name} found in FTP server.")
return True
else:
self.sys_log.error(f"File {src_folder_name}/{src_file_name} does not exist in FTP server")
return False
def receive(self, payload: FTPPacket, session_id: Optional[str] = None, **kwargs) -> bool:
"""
Receives a payload from the SessionManager.
:param: payload: FTPPacket payload.
:type: payload: FTPPacket
:param: session_id: ID of the session. Optional.
:type: session_id: Optional[str]
"""
if not isinstance(payload, FTPPacket):
self.sys_log.error(f"{payload} is not an FTP packet")
return False
self._process_ftp_command(payload=payload, session_id=session_id)
return True

View File

@@ -0,0 +1,83 @@
from ipaddress import IPv4Address
from typing import Any, Dict, Optional
from primaite.simulator.network.protocols.ftp import FTPCommand, FTPPacket, FTPStatusCode
from primaite.simulator.network.transmission.network_layer import IPProtocol
from primaite.simulator.network.transmission.transport_layer import Port
from primaite.simulator.system.core.session_manager import Session
from primaite.simulator.system.services.ftp.ftp_service import FTPServiceABC
from primaite.simulator.system.services.service import ServiceOperatingState
class FTPServer(FTPServiceABC):
"""
A class for simulating an FTP server service.
This class inherits from the `Service` class and provides methods to emulate FTP
RFC 959: https://datatracker.ietf.org/doc/html/rfc959
"""
server_password: Optional[str] = None
"""Password needed to connect to FTP server. Default is None."""
connections: Dict[str, IPv4Address] = {}
"""Current active connections to the FTP server."""
def __init__(self, **kwargs):
kwargs["name"] = "FTPServer"
kwargs["port"] = Port.FTP
kwargs["protocol"] = IPProtocol.TCP
super().__init__(**kwargs)
self.start()
def _get_session_details(self, session_id: str) -> Session:
"""
Returns the Session object from the given session id.
:param: session_id: ID of the session that needs details retrieved
"""
return self.software_manager.session_manager.sessions_by_uuid[session_id]
def _process_ftp_command(self, payload: FTPPacket, session_id: Optional[str] = None, **kwargs) -> FTPPacket:
"""
Process the command in the FTP Packet.
:param: payload: The FTP Packet to process
:type: payload: FTPPacket
:param: session_id: session ID linked to the FTP Packet. Optional.
:type: session_id: Optional[str]
"""
# if server service is down, return error
if self.operating_state != ServiceOperatingState.RUNNING:
payload.status_code = FTPStatusCode.ERROR
return payload
if session_id:
session_details = self._get_session_details(session_id)
if payload.ftp_command is not None:
self.sys_log.info(f"Received FTP {payload.ftp_command.name} command.")
# process server specific commands, otherwise call super
if payload.ftp_command == FTPCommand.PORT:
# check that the port is valid
if isinstance(payload.ftp_command_args, Port) and payload.ftp_command_args.value in range(0, 65535):
# return successful connection
self.connections[session_id] = session_details.with_ip_address
payload.status_code = FTPStatusCode.OK
return payload
if payload.ftp_command == FTPCommand.QUIT:
self.connections.pop(session_id)
payload.status_code = FTPStatusCode.OK
return super()._process_ftp_command(payload=payload, session_id=session_id, **kwargs)
def receive(self, payload: Any, session_id: Optional[str] = None, **kwargs) -> bool:
"""Receives a payload from the SessionManager."""
if not isinstance(payload, FTPPacket):
self.sys_log.error(f"{payload} is not an FTP packet")
return False
self.send(self._process_ftp_command(payload=payload, session_id=session_id), session_id)
return True

View File

@@ -0,0 +1,155 @@
import shutil
from abc import ABC
from ipaddress import IPv4Address
from typing import Optional
from primaite.simulator.file_system.file_system import File
from primaite.simulator.network.protocols.ftp import FTPCommand, FTPPacket, FTPStatusCode
from primaite.simulator.network.transmission.transport_layer import Port
from primaite.simulator.system.core.software_manager import SoftwareManager
from primaite.simulator.system.services.service import Service
class FTPServiceABC(Service, ABC):
"""
Abstract Base Class for FTP Client and Service.
Contains shared methods between both classes.
"""
def _process_ftp_command(self, payload: FTPPacket, session_id: Optional[str] = None, **kwargs) -> FTPPacket:
"""
Process the command in the FTP Packet.
:param: payload: The FTP Packet to process
:type: payload: FTPPacket
:param: session_id: session ID linked to the FTP Packet. Optional.
:type: session_id: Optional[str]
"""
if payload.ftp_command is not None:
self.sys_log.info(f"Received FTP {payload.ftp_command.name} command.")
# handle STOR request
if payload.ftp_command == FTPCommand.STOR:
# check that the file is created in the computed hosting the FTP server
if self._store_data(payload=payload):
payload.status_code = FTPStatusCode.OK
if payload.ftp_command == FTPCommand.RETR:
if self._retrieve_data(payload=payload, session_id=session_id):
payload.status_code = FTPStatusCode.OK
return payload
def _store_data(self, payload: FTPPacket) -> bool:
"""
Stores the data in the FTP Service's host machine.
:param: payload: The FTP Packet that contains the file data
:type: FTPPacket
"""
try:
file_name = payload.ftp_command_args["dest_file_name"]
folder_name = payload.ftp_command_args["dest_folder_name"]
file_size = payload.ftp_command_args["file_size"]
real_file_path = payload.ftp_command_args.get("real_file_path")
is_real = real_file_path is not None
file = self.file_system.create_file(
file_name=file_name, folder_name=folder_name, size=file_size, real=is_real
)
self.sys_log.info(
f"Created item in {self.sys_log.hostname}: {payload.ftp_command_args['dest_folder_name']}/"
f"{payload.ftp_command_args['dest_file_name']}"
)
if is_real:
shutil.copy(real_file_path, file.sim_path)
# file should exist
return self.file_system.get_file(file_name=file_name, folder_name=folder_name) is not None
except Exception as e:
self.sys_log.error(f"Unable to create file in {self.sys_log.hostname}: {e}")
return False
def _send_data(
self,
file: File,
dest_folder_name: str,
dest_file_name: str,
dest_ip_address: Optional[IPv4Address] = None,
dest_port: Optional[Port] = None,
session_id: Optional[str] = None,
) -> bool:
"""
Sends data from the host FTP Service's machine to another FTP Service's host machine.
:param: file: File to send to the target FTP Service.
:type: file: File
:param: dest_folder_name: The name of the folder where the file will be stored in the FTP Server.
:type: dest_folder_name: str
:param: dest_file_name: The name of the file to be saved on the FTP Server.
:type: dest_file_name: str
:param: dest_ip_address: The IP address of the machine that hosts the FTP Server.
:type: dest_ip_address: Optional[IPv4Address]
:param: dest_port: The open port of the machine that hosts the FTP Server. Default is Port.FTP.
:type: dest_port: Optional[Port]
:param: session_id: session ID linked to the FTP Packet. Optional.
:type: session_id: Optional[str]
"""
# send STOR request
payload: FTPPacket = FTPPacket(
ftp_command=FTPCommand.STOR,
ftp_command_args={
"dest_folder_name": dest_folder_name,
"dest_file_name": dest_file_name,
"file_size": file.sim_size,
"real_file_path": file.sim_path if file.real else None,
},
packet_payload_size=file.sim_size,
)
software_manager: SoftwareManager = self.software_manager
software_manager.send_payload_to_session_manager(
payload=payload, dest_ip_address=dest_ip_address, dest_port=dest_port, session_id=session_id
)
if payload.status_code == FTPStatusCode.OK:
return True
return False
def _retrieve_data(self, payload: FTPPacket, session_id: Optional[str] = None) -> bool:
"""
Handle the transfer of data from Server to Client.
:param: payload: The FTP Packet that contains the file data
:type: FTPPacket
"""
try:
# find the file
file_name = payload.ftp_command_args["src_file_name"]
folder_name = payload.ftp_command_args["src_folder_name"]
dest_folder_name = payload.ftp_command_args["dest_folder_name"]
dest_file_name = payload.ftp_command_args["dest_file_name"]
retrieved_file: File = self.file_system.get_file(folder_name=folder_name, file_name=file_name)
# if file does not exist, return an error
if not retrieved_file:
self.sys_log.error(
f"File {payload.ftp_command_args['dest_folder_name']}/"
f"{payload.ftp_command_args['dest_file_name']} does not exist in {self.sys_log.hostname}"
)
return False
else:
# send requested data
return self._send_data(
file=retrieved_file,
dest_file_name=dest_file_name,
dest_folder_name=dest_folder_name,
session_id=session_id,
)
except Exception as e:
self.sys_log.error(f"Unable to retrieve file from {self.sys_log.hostname}: {e}")
return False

View File

@@ -2,7 +2,7 @@ from enum import Enum
from typing import Any, Dict, Optional
from primaite import getLogger
from primaite.simulator.core import Action, ActionManager
from primaite.simulator.core import RequestManager, RequestType
from primaite.simulator.system.software import IOSoftware
_LOGGER = getLogger(__name__)
@@ -39,15 +39,15 @@ class Service(IOSoftware):
_restart_countdown: Optional[int] = None
"If currently restarting, how many timesteps remain until the restart is finished."
def _init_action_manager(self) -> ActionManager:
am = super()._init_action_manager()
am.add_action("stop", Action(func=lambda request, context: self.stop()))
am.add_action("start", Action(func=lambda request, context: self.start()))
am.add_action("pause", Action(func=lambda request, context: self.pause()))
am.add_action("resume", Action(func=lambda request, context: self.resume()))
am.add_action("restart", Action(func=lambda request, context: self.restart()))
am.add_action("disable", Action(func=lambda request, context: self.disable()))
am.add_action("enable", Action(func=lambda request, context: self.enable()))
def _init_request_manager(self) -> RequestManager:
am = super()._init_request_manager()
am.add_request("stop", RequestType(func=lambda request, context: self.stop()))
am.add_request("start", RequestType(func=lambda request, context: self.start()))
am.add_request("pause", RequestType(func=lambda request, context: self.pause()))
am.add_request("resume", RequestType(func=lambda request, context: self.resume()))
am.add_request("restart", RequestType(func=lambda request, context: self.restart()))
am.add_request("disable", RequestType(func=lambda request, context: self.disable()))
am.add_request("enable", RequestType(func=lambda request, context: self.enable()))
return am
def describe_state(self) -> Dict: