From c6b6753aeb81f9c9c180738763c1daf22a6148fd Mon Sep 17 00:00:00 2001 From: Marek Wolan Date: Wed, 8 May 2024 11:13:32 +0000 Subject: [PATCH 1/2] Updated VERSION --- src/primaite/VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/primaite/VERSION b/src/primaite/VERSION index f08efbdc..9e0b71d0 100644 --- a/src/primaite/VERSION +++ b/src/primaite/VERSION @@ -1 +1 @@ -3.0.0b9dev +3.0.0b9 From ebbdea517e91ad6075e689b49d17a79c265b2eab Mon Sep 17 00:00:00 2001 From: Chris McCarthy Date: Mon, 13 May 2024 07:30:33 +0100 Subject: [PATCH 2/2] 2536 - Removed 'real files' from FileSystem and FTP --- src/primaite/simulator/file_system/file.py | 40 +++---------------- .../simulator/file_system/file_system.py | 13 ------ .../system/services/ftp/ftp_client.py | 1 - .../system/services/ftp/ftp_service.py | 10 ++--- .../_simulator/_file_system/test_file.py | 14 ------- .../_file_system/test_file_system.py | 2 +- .../_simulator/_file_system/test_folder.py | 18 --------- 7 files changed, 10 insertions(+), 88 deletions(-) diff --git a/src/primaite/simulator/file_system/file.py b/src/primaite/simulator/file_system/file.py index a4d54e79..328b052b 100644 --- a/src/primaite/simulator/file_system/file.py +++ b/src/primaite/simulator/file_system/file.py @@ -2,9 +2,7 @@ from __future__ import annotations import hashlib import json -import os.path import warnings -from pathlib import Path from typing import Dict, Optional from primaite import getLogger @@ -21,8 +19,6 @@ class File(FileSystemItemABC): :ivar Folder folder: The folder in which the file resides. :ivar FileType file_type: The type of the file. :ivar Optional[int] sim_size: The simulated file size. - :ivar bool real: Indicates if the file is actually a real file in the Node sim fs output. - :ivar Optional[Path] sim_path: The path if the file is real. """ folder_id: str @@ -33,12 +29,6 @@ class File(FileSystemItemABC): "The type of File." sim_size: Optional[int] = None "The simulated file size." - real: bool = False - "Indicates whether the File is actually a real file in the Node sim fs output." - sim_path: Optional[Path] = None - "The Path if real is True." - sim_root: Optional[Path] = None - "Root path of the simulation." num_access: int = 0 "Number of times the file was accessed in the current step." @@ -67,13 +57,6 @@ class File(FileSystemItemABC): if not kwargs.get("sim_size"): kwargs["sim_size"] = kwargs["file_type"].default_size super().__init__(**kwargs) - if self.real: - self.sim_path = self.sim_root / self.path - if not self.sim_path.exists(): - self.sim_path.parent.mkdir(exist_ok=True, parents=True) - with open(self.sim_path, mode="a"): - pass - self.sys_log.info(f"Created file /{self.path} (id: {self.uuid})") @property @@ -92,8 +75,6 @@ class File(FileSystemItemABC): :return: The size of the file in bytes. """ - if self.real: - return os.path.getsize(self.sim_path) return self.sim_size def apply_timestep(self, timestep: int) -> None: @@ -127,7 +108,7 @@ class File(FileSystemItemABC): self.num_access += 1 # file was accessed path = self.folder.name + "/" + self.name - self.sys_log.info(f"Scanning file {self.sim_path if self.sim_path else path}") + self.sys_log.info(f"Scanning file {path}") self.visible_health_status = self.health_status return True @@ -155,17 +136,8 @@ class File(FileSystemItemABC): return False current_hash = None - # if file is real, read the file contents - if self.real: - with open(self.sim_path, "rb") as f: - file_hash = hashlib.blake2b() - while chunk := f.read(8192): - file_hash.update(chunk) - - current_hash = file_hash.hexdigest() - else: - # otherwise get describe_state dict and hash that - current_hash = hashlib.blake2b(json.dumps(self.describe_state(), sort_keys=True).encode()).hexdigest() + # otherwise get describe_state dict and hash that + current_hash = hashlib.blake2b(json.dumps(self.describe_state(), sort_keys=True).encode()).hexdigest() # if the previous hash is None, set the current hash to previous if self.previous_hash is None: @@ -188,7 +160,7 @@ class File(FileSystemItemABC): self.num_access += 1 # file was accessed path = self.folder.name + "/" + self.name - self.sys_log.info(f"Repaired file {self.sim_path if self.sim_path else path}") + self.sys_log.info(f"Repaired file {path}") return True def corrupt(self) -> bool: @@ -203,7 +175,7 @@ class File(FileSystemItemABC): self.num_access += 1 # file was accessed path = self.folder.name + "/" + self.name - self.sys_log.info(f"Corrupted file {self.sim_path if self.sim_path else path}") + self.sys_log.info(f"Corrupted file {path}") return True def restore(self) -> bool: @@ -217,7 +189,7 @@ class File(FileSystemItemABC): self.num_access += 1 # file was accessed path = self.folder.name + "/" + self.name - self.sys_log.info(f"Restored file {self.sim_path if self.sim_path else path}") + self.sys_log.info(f"Restored file {path}") return True def delete(self) -> bool: diff --git a/src/primaite/simulator/file_system/file_system.py b/src/primaite/simulator/file_system/file_system.py index 0eae6009..cf4a4721 100644 --- a/src/primaite/simulator/file_system/file_system.py +++ b/src/primaite/simulator/file_system/file_system.py @@ -1,6 +1,5 @@ from __future__ import annotations -import shutil from pathlib import Path from typing import Dict, Optional @@ -230,7 +229,6 @@ class FileSystem(SimComponent): size: Optional[int] = None, file_type: Optional[FileType] = None, folder_name: Optional[str] = None, - real: bool = False, ) -> File: """ Creates a File and adds it to the list of files. @@ -239,7 +237,6 @@ class FileSystem(SimComponent): :param size: The size the file takes on disk in bytes. :param file_type: The type of the file. :param folder_name: The folder to add the file to. - :param real: "Indicates whether the File is actually a real file in the Node sim fs output." """ if folder_name: # check if file with name already exists @@ -258,8 +255,6 @@ class FileSystem(SimComponent): file_type=file_type, folder_id=folder.uuid, folder_name=folder.name, - real=real, - sim_path=self.sim_root if real else None, sim_root=self.sim_root, sys_log=self.sys_log, ) @@ -368,11 +363,6 @@ class FileSystem(SimComponent): # add file to dst dst_folder.add_file(file) self.num_file_creations += 1 - if file.real: - old_sim_path = file.sim_path - file.sim_path = file.sim_root / file.path - file.sim_path.parent.mkdir(exist_ok=True) - shutil.move(old_sim_path, file.sim_path) def copy_file(self, src_folder_name: str, src_file_name: str, dst_folder_name: str): """ @@ -401,9 +391,6 @@ class FileSystem(SimComponent): dst_folder.add_file(file_copy, force=True) - if file.real: - file_copy.sim_path.parent.mkdir(exist_ok=True) - shutil.copy2(file.sim_path, file_copy.sim_path) else: self.sys_log.error(f"Unable to copy file. {src_file_name} does not exist.") diff --git a/src/primaite/simulator/system/services/ftp/ftp_client.py b/src/primaite/simulator/system/services/ftp/ftp_client.py index 22a583e4..4eb18f6a 100644 --- a/src/primaite/simulator/system/services/ftp/ftp_client.py +++ b/src/primaite/simulator/system/services/ftp/ftp_client.py @@ -126,7 +126,6 @@ class FTPClient(FTPServiceABC): dest_file_name: str, dest_port: Optional[Port] = Port.FTP, session_id: Optional[str] = None, - real_file_path: Optional[str] = None, ) -> bool: """ Send a file to a target IP address. diff --git a/src/primaite/simulator/system/services/ftp/ftp_service.py b/src/primaite/simulator/system/services/ftp/ftp_service.py index 70ba74d7..b89ae1a2 100644 --- a/src/primaite/simulator/system/services/ftp/ftp_service.py +++ b/src/primaite/simulator/system/services/ftp/ftp_service.py @@ -1,4 +1,3 @@ -import shutil from abc import ABC from ipaddress import IPv4Address from typing import Dict, Optional @@ -55,19 +54,17 @@ class FTPServiceABC(Service, ABC): file_name = payload.ftp_command_args["dest_file_name"] folder_name = payload.ftp_command_args["dest_folder_name"] file_size = payload.ftp_command_args["file_size"] - real_file_path = payload.ftp_command_args.get("real_file_path") health_status = payload.ftp_command_args["health_status"] - is_real = real_file_path is not None file = self.file_system.create_file( - file_name=file_name, folder_name=folder_name, size=file_size, real=is_real + file_name=file_name, + folder_name=folder_name, + size=file_size, ) file.health_status = health_status self.sys_log.info( f"{self.name}: Created item in {self.sys_log.hostname}: {payload.ftp_command_args['dest_folder_name']}/" f"{payload.ftp_command_args['dest_file_name']}" ) - if is_real: - shutil.copy(real_file_path, file.sim_path) # file should exist return self.file_system.get_file(file_name=file_name, folder_name=folder_name) is not None except Exception as e: @@ -115,7 +112,6 @@ class FTPServiceABC(Service, ABC): "dest_folder_name": dest_folder_name, "dest_file_name": dest_file_name, "file_size": file.sim_size, - "real_file_path": file.sim_path if file.real else None, "health_status": file.health_status, }, packet_payload_size=file.sim_size, diff --git a/tests/unit_tests/_primaite/_simulator/_file_system/test_file.py b/tests/unit_tests/_primaite/_simulator/_file_system/test_file.py index 9d9228d8..72a5889e 100644 --- a/tests/unit_tests/_primaite/_simulator/_file_system/test_file.py +++ b/tests/unit_tests/_primaite/_simulator/_file_system/test_file.py @@ -57,20 +57,6 @@ def test_simulated_file_check_hash(file_system): assert file.health_status == FileSystemItemHealthStatus.CORRUPT -@pytest.mark.skip(reason="NODE_FILE_CHECKHASH not implemented") -def test_real_file_check_hash(file_system): - file: File = file_system.create_file(file_name="test_file.txt", real=True) - - file.check_hash() - assert file.health_status == FileSystemItemHealthStatus.GOOD - # change file content - with open(file.sim_path, "a") as f: - f.write("get hacked scrub lol xD\n") - - file.check_hash() - assert file.health_status == FileSystemItemHealthStatus.CORRUPT - - def test_file_corrupt_repair_restore(file_system): """Test the ability to corrupt and repair files.""" file: File = file_system.create_file(file_name="test_file.txt", folder_name="test_folder") diff --git a/tests/unit_tests/_primaite/_simulator/_file_system/test_file_system.py b/tests/unit_tests/_primaite/_simulator/_file_system/test_file_system.py index 9b2ecf45..0cb7dce7 100644 --- a/tests/unit_tests/_primaite/_simulator/_file_system/test_file_system.py +++ b/tests/unit_tests/_primaite/_simulator/_file_system/test_file_system.py @@ -191,7 +191,7 @@ def test_copy_file(file_system): file_system.create_folder(folder_name="src_folder") file_system.create_folder(folder_name="dst_folder") - file = file_system.create_file(file_name="test_file.txt", size=10, folder_name="src_folder", real=True) + file = file_system.create_file(file_name="test_file.txt", size=10, folder_name="src_folder") assert file_system.num_file_creations == 1 original_uuid = file.uuid diff --git a/tests/unit_tests/_primaite/_simulator/_file_system/test_folder.py b/tests/unit_tests/_primaite/_simulator/_file_system/test_folder.py index e00c002d..1d9f2d9c 100644 --- a/tests/unit_tests/_primaite/_simulator/_file_system/test_folder.py +++ b/tests/unit_tests/_primaite/_simulator/_file_system/test_folder.py @@ -132,21 +132,3 @@ def test_simulated_folder_check_hash(file_system): file.sim_size = 0 folder.check_hash() assert folder.health_status == FileSystemItemHealthStatus.CORRUPT - - -@pytest.mark.skip(reason="NODE_FILE_CHECKHASH not implemented") -def test_real_folder_check_hash(file_system): - folder: Folder = file_system.create_folder(folder_name="test_folder") - file_system.create_file(file_name="test_file.txt", folder_name="test_folder", real=True) - - folder.check_hash() - assert folder.health_status == FileSystemItemHealthStatus.GOOD - # change simulated file size - file = folder.get_file(file_name="test_file.txt") - - # change file content - with open(file.sim_path, "a") as f: - f.write("get hacked scrub lol xD\n") - - folder.check_hash() - assert folder.health_status == FileSystemItemHealthStatus.CORRUPT