From 3abe39aa10632f8d21909002daf1c827fb37f90a Mon Sep 17 00:00:00 2001 From: Marek Wolan Date: Tue, 4 Jul 2023 10:55:07 +0100 Subject: [PATCH 01/41] Add Favicon --- docs/conf.py | 1 + docs/source/primaite.ico | Bin 0 -> 3126 bytes 2 files changed, 1 insertion(+) create mode 100644 docs/source/primaite.ico diff --git a/docs/conf.py b/docs/conf.py index 51b745cf..4e22ebc8 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -55,3 +55,4 @@ exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] html_theme = "furo" html_static_path = ["_static"] +html_favicon = 'source/primaite.ico' \ No newline at end of file diff --git a/docs/source/primaite.ico b/docs/source/primaite.ico new file mode 100644 index 0000000000000000000000000000000000000000..4f8a9cbe34b83692f839d84fb1765a62fc0a2bf2 GIT binary patch literal 3126 zcmb7G2~<;O7Phtyh>mrkRz*S7A}R_3WeICQmh5}pd&x^)5+EByHj%}MfFetA#sx(X zizu8Zhyo(Gp~xnJ3Rp(8RgWNb!#d8`4%RJmG1gOiY(3s{-k*QV_uu{A0EXMdF|WxD zMtk_?zWF{GGXrKoZ_Z5%(`ft{`t>z^%s)PGcX)WXs;Vj}DT&MFQYaKAlL=#Ra4<~b zl?vtrK}aMLSONtE1VltcT)K4W zPhj4&Ju)()*XvElLZMJ5lZ$Z)PlU5EF++%Pr33@T5JIL13&(K@ilPF603=eY)lZ*3 zeGlBbTRlBJ2!e1p92kWNA|+%hjfOzE4XfpCN^buKO$XDbS;jEPIBYHxK`~4uHVFb^ z1_lOpb#=Xq_s;6nsZ%T#3-|%HMPjiwGKxzJ;jeJ^bDZa3ZN1!Pw$pS=r`a&3_5Ls z8P(NBjgVDBvQ`wRV0&2o-F%7743&u5T%H`pCTUc128)BDnCXf3_V(XU-~2v$^oY%7 z12T{zETZ*z+4dbfUOet9*r1Fh=)pd&W5t8ufNV5A zC`u>R#)dT>Euk^kpfJ-DPo6ya{S#WP78D4~<6^w=^wA3~HRT7Yo{!wxmAA%c*<5q; z2{WgC?y`6u$!Vdd<3cApn=hwKGB+Eatyi5nURju_Bm|USe!3b~*=N|hEjN=y zBAq;0yGku?*p~@7J|xegP5O*DNwkC(EukhT=t^PeSGKkm=HreV@?w>k8lzd5kFVXo zlNs#GU@$}?5dcD@0RG_MAdN-?aFIwHr4Bz+Q)D%1ERW4dTB+EwUY8!lbzb72(?kfE zA!3FM$&(FjtviZ#=4EB%rN>2} zK~ip@Qo{FGz6>sWGG>ge_3U6eJ#|xVjeb>+*TO=w2SZax@p>Fv4u_%Zmy%cnOsrxIEzH8DoTqf=&CPMI`u zg8Q5=>ij&~=S=Tc(l1y~|JHpu8<&{yA@Rb5Eyj2Id-T!WyOF?ltErs=-%V*6K8q@!!v1zIt`1yY)a>enOHC%ScP~a2VS(@0&beu-R!{pLBDh-?dfDAR zpXA2DanMh6baYBe3iv7y6Ky~^S1@v~hT82i|KN^|V4f(gMyC!-UmbO@IOB)GfnT0I z_FQDGR7ks8s`c@Ty7GSz3=+-XC6w$d;?ee|tAB_M>LfW{qq$}%_&ihu_RY!3DJ?C9 z%mevPjEirl$*yHZTo48ma*>y}_nPFSjJ1i)bpze1GBk zyL~mgv$Y6?B;vE3mn_z3@yptvPEUva#j`NE3;@ADtE;Q~`}?8ffeGkhyxw(DAKkAy z7ImV(9}=7srJQY^El7vlIRJ1kF1}!Biu@>(}WCOemncA44eDrkcg8!}+g^8kA80 z2QnB8kTgI9ng}XzJkhW=HGcT#;alNBhtwh-G3qeDb5m232_HtlhfT=dGy!^EOuotX z-L`LEM}L3Mz>f_zWtJA7PM$bnzV(;9Kv$gVv0#p+)yx?)ET@P<-MX(_DK6Prz-8Rj zkh_$=xOOGD4OkY+@mp}1K(_#;4Mc!oUDudb!_^5C84uSTAY+#2}T@&1kj7fx5A0H`=i;cx)a$p?d2sl4{`0#tdO`9bpC7>vf5d;wk_*4o7++a^>@uiDR z1J_$`^d4zBvSrWa`1Py9VzqLmQVD$!Tn5{L&Y{PHQur6(rlV`ut^xCJDkh2oK99$z z(_x3j;_%rV0V@ Date: Tue, 4 Jul 2023 10:57:00 +0100 Subject: [PATCH 02/41] fix formatting on Observation docs --- src/primaite/environment/observations.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/primaite/environment/observations.py b/src/primaite/environment/observations.py index 23bc4a39..f8b42e1c 100644 --- a/src/primaite/environment/observations.py +++ b/src/primaite/environment/observations.py @@ -275,11 +275,11 @@ class LinkTrafficLevels(AbstractObservationComponent): For each link, total traffic or traffic per service is encoded into a categorical value. For example, if ``quantisation_levels=5``, the traffic levels represent these values: - 0 = No traffic (0% of bandwidth) - 1 = No traffic (0%-33% of bandwidth) - 2 = No traffic (33%-66% of bandwidth) - 3 = No traffic (66%-100% of bandwidth) - 4 = No traffic (100% of bandwidth) + * 0 = No traffic (0% of bandwidth) + * 1 = No traffic (0%-33% of bandwidth) + * 2 = No traffic (33%-66% of bandwidth) + * 3 = No traffic (66%-100% of bandwidth) + * 4 = No traffic (100% of bandwidth) .. note:: The lowest category always corresponds to no traffic and the highest category to the link being at max capacity. @@ -288,10 +288,10 @@ class LinkTrafficLevels(AbstractObservationComponent): :param env: The environment that forms the basis of the observations :type env: Primaite :param combine_service_traffic: Whether to consider total traffic on the link, or each protocol individually, - defaults to False + defaults to False :type combine_service_traffic: bool, optional :param quantisation_levels: How many bands to consider when converting the traffic amount to a categorical value , - defaults to 5 + defaults to 5 :type quantisation_levels: int, optional """ From 5e270c76735d9a499e00477dd3d474ead094388d Mon Sep 17 00:00:00 2001 From: Marek Wolan Date: Tue, 4 Jul 2023 11:11:52 +0100 Subject: [PATCH 03/41] Format docstrings --- docs/conf.py | 2 +- src/primaite/__init__.py | 6 +- src/primaite/acl/access_control_list.py | 15 ++-- src/primaite/acl/acl_rule.py | 21 ++---- src/primaite/agents/agent.py | 29 +++----- src/primaite/agents/hardcoded_acl.py | 24 +++---- src/primaite/agents/hardcoded_node.py | 3 +- src/primaite/agents/rllib.py | 6 +- src/primaite/agents/sb3.py | 6 +- src/primaite/agents/simple.py | 15 ++-- src/primaite/agents/utils.py | 57 +++++---------- src/primaite/cli.py | 18 ++--- src/primaite/common/protocol.py | 12 ++-- src/primaite/common/service.py | 3 +- src/primaite/config/lay_down_config.py | 21 ++---- src/primaite/config/training_config.py | 28 +++----- src/primaite/data_viz/session_plots.py | 3 +- src/primaite/environment/observations.py | 4 +- src/primaite/environment/primaite_env.py | 71 ++++++------------- src/primaite/environment/reward.py | 15 ++-- src/primaite/links/link.py | 27 +++---- src/primaite/nodes/active_node.py | 18 ++--- src/primaite/nodes/node.py | 3 +- .../nodes/node_state_instruction_green.py | 21 ++---- .../nodes/node_state_instruction_red.py | 33 +++------ src/primaite/nodes/passive_node.py | 6 +- src/primaite/nodes/service_node.py | 30 +++----- src/primaite/notebooks/__init__.py | 3 +- src/primaite/pol/green_pol.py | 6 +- src/primaite/pol/ier.py | 39 ++++------ src/primaite/pol/red_agent_pol.py | 9 +-- src/primaite/primaite_session.py | 15 ++-- src/primaite/setup/reset_demo_notebooks.py | 6 +- src/primaite/setup/reset_example_configs.py | 6 +- src/primaite/setup/setup_app_dirs.py | 3 +- src/primaite/transactions/transaction.py | 15 ++-- src/primaite/utils/package_data.py | 3 +- src/primaite/utils/session_output_reader.py | 6 +- src/primaite/utils/session_output_writer.py | 9 +-- 39 files changed, 208 insertions(+), 409 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 4e22ebc8..d6923446 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -55,4 +55,4 @@ exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] html_theme = "furo" html_static_path = ["_static"] -html_favicon = 'source/primaite.ico' \ No newline at end of file +html_favicon = "source/primaite.ico" diff --git a/src/primaite/__init__.py b/src/primaite/__init__.py index 030860d8..a2d157c6 100644 --- a/src/primaite/__init__.py +++ b/src/primaite/__init__.py @@ -66,8 +66,7 @@ Users PrimAITE Sessions are stored at: ``~/primaite/sessions``. # region Setup Logging class _LevelFormatter(Formatter): - """ - A custom level-specific formatter. + """A custom level-specific formatter. Credit to: https://stackoverflow.com/a/68154386 """ @@ -135,8 +134,7 @@ _LOGGER.addHandler(_FILE_HANDLER) def getLogger(name: str) -> Logger: # noqa - """ - Get a PrimAITE logger. + """Get a PrimAITE logger. :param name: The logger name. Use ``__name__``. :return: An instance of :py:class:`logging.Logger` with the PrimAITE diff --git a/src/primaite/acl/access_control_list.py b/src/primaite/acl/access_control_list.py index 3b0e9234..42460b94 100644 --- a/src/primaite/acl/access_control_list.py +++ b/src/primaite/acl/access_control_list.py @@ -13,8 +13,7 @@ class AccessControlList: self.acl: Dict[str, AccessControlList] = {} # A dictionary of ACL Rules def check_address_match(self, _rule, _source_ip_address, _dest_ip_address): - """ - Checks for IP address matches. + """Checks for IP address matches. Args: _rule: The rule being checked @@ -35,8 +34,7 @@ class AccessControlList: return False def is_blocked(self, _source_ip_address, _dest_ip_address, _protocol, _port): - """ - Checks for rules that block a protocol / port. + """Checks for rules that block a protocol / port. Args: _source_ip_address: the source IP address to check @@ -62,8 +60,7 @@ class AccessControlList: return True def add_rule(self, _permission, _source_ip, _dest_ip, _protocol, _port): - """ - Adds a new rule. + """Adds a new rule. Args: _permission: the permission value (e.g. "ALLOW" or "DENY") @@ -77,8 +74,7 @@ class AccessControlList: self.acl[hash_value] = new_rule def remove_rule(self, _permission, _source_ip, _dest_ip, _protocol, _port): - """ - Removes a rule. + """Removes a rule. Args: _permission: the permission value (e.g. "ALLOW" or "DENY") @@ -100,8 +96,7 @@ class AccessControlList: self.acl.clear() def get_dictionary_hash(self, _permission, _source_ip, _dest_ip, _protocol, _port): - """ - Produces a hash value for a rule. + """Produces a hash value for a rule. Args: _permission: the permission value (e.g. "ALLOW" or "DENY") diff --git a/src/primaite/acl/acl_rule.py b/src/primaite/acl/acl_rule.py index 05daecc4..29f52f88 100644 --- a/src/primaite/acl/acl_rule.py +++ b/src/primaite/acl/acl_rule.py @@ -6,8 +6,7 @@ class ACLRule: """Access Control List Rule class.""" def __init__(self, _permission, _source_ip, _dest_ip, _protocol, _port): - """ - Init. + """Init. Args: _permission: The permission (ALLOW or DENY) @@ -23,8 +22,7 @@ class ACLRule: self.port = _port def __hash__(self): - """ - Override the hash function. + """Override the hash function. Returns: Returns hash of core parameters. @@ -40,8 +38,7 @@ class ACLRule: ) def get_permission(self): - """ - Gets the permission attribute. + """Gets the permission attribute. Returns: Returns permission attribute @@ -49,8 +46,7 @@ class ACLRule: return self.permission def get_source_ip(self): - """ - Gets the source IP address attribute. + """Gets the source IP address attribute. Returns: Returns source IP address attribute @@ -58,8 +54,7 @@ class ACLRule: return self.source_ip def get_dest_ip(self): - """ - Gets the desintation IP address attribute. + """Gets the desintation IP address attribute. Returns: Returns destination IP address attribute @@ -67,8 +62,7 @@ class ACLRule: return self.dest_ip def get_protocol(self): - """ - Gets the protocol attribute. + """Gets the protocol attribute. Returns: Returns protocol attribute @@ -76,8 +70,7 @@ class ACLRule: return self.protocol def get_port(self): - """ - Gets the port attribute. + """Gets the port attribute. Returns: Returns port attribute diff --git a/src/primaite/agents/agent.py b/src/primaite/agents/agent.py index 685fe776..a43f2d0b 100644 --- a/src/primaite/agents/agent.py +++ b/src/primaite/agents/agent.py @@ -21,8 +21,7 @@ _LOGGER = getLogger(__name__) def get_session_path(session_timestamp: datetime) -> Path: - """ - Get the directory path the session will output to. + """Get the directory path the session will output to. This is set in the format of: ~/primaite/sessions//_. @@ -39,11 +38,10 @@ def get_session_path(session_timestamp: datetime) -> Path: class AgentSessionABC(ABC): - """ - An ABC that manages training and/or evaluation of agents in PrimAITE. + """An ABC that manages training and/or evaluation of agents in PrimAITE. - This class cannot be directly instantiated and must be inherited from - with all implemented abstract methods implemented. + This class cannot be directly instantiated and must be inherited from with all implemented abstract methods + implemented. """ @abstractmethod @@ -186,8 +184,7 @@ class AgentSessionABC(ABC): self, **kwargs, ): - """ - Train the agent. + """Train the agent. :param kwargs: Any agent-specific key-word args to be passed. """ @@ -204,8 +201,7 @@ class AgentSessionABC(ABC): self, **kwargs, ): - """ - Evaluate the agent. + """Evaluate the agent. :param kwargs: Any agent-specific key-word args to be passed. """ @@ -293,11 +289,10 @@ class AgentSessionABC(ABC): class HardCodedAgentSessionABC(AgentSessionABC): - """ - An Agent Session ABC for evaluation deterministic agents. + """An Agent Session ABC for evaluation deterministic agents. - This class cannot be directly instantiated and must be inherited from - with all implemented abstract methods implemented. + This class cannot be directly instantiated and must be inherited from with all implemented abstract methods + implemented. """ def __init__(self, training_config_path, lay_down_config_path): @@ -325,8 +320,7 @@ class HardCodedAgentSessionABC(AgentSessionABC): self, **kwargs, ): - """ - Train the agent. + """Train the agent. :param kwargs: Any agent-specific key-word args to be passed. """ @@ -340,8 +334,7 @@ class HardCodedAgentSessionABC(AgentSessionABC): self, **kwargs, ): - """ - Evaluate the agent. + """Evaluate the agent. :param kwargs: Any agent-specific key-word args to be passed. """ diff --git a/src/primaite/agents/hardcoded_acl.py b/src/primaite/agents/hardcoded_acl.py index 263ccbdc..9ed9fd28 100644 --- a/src/primaite/agents/hardcoded_acl.py +++ b/src/primaite/agents/hardcoded_acl.py @@ -23,8 +23,7 @@ class HardCodedACLAgent(HardCodedAgentSessionABC): return self._calculate_action_full_view(obs) def get_blocked_green_iers(self, green_iers, acl, nodes): - """ - Get blocked green IERs. + """Get blocked green IERs. TODO: Add params and return in docstring. TODO: Typehint params and return. @@ -46,8 +45,7 @@ class HardCodedACLAgent(HardCodedAgentSessionABC): return blocked_green_iers def get_matching_acl_rules_for_ier(self, ier, acl, nodes): - """ - Get matching ACL rules for an IER. + """Get matching ACL rules for an IER. TODO: Add params and return in docstring. TODO: Typehint params and return. @@ -63,8 +61,7 @@ class HardCodedACLAgent(HardCodedAgentSessionABC): return matching_rules def get_blocking_acl_rules_for_ier(self, ier, acl, nodes): - """ - Get blocking ACL rules for an IER. + """Get blocking ACL rules for an IER. .. warning:: Can return empty dict but IER can still be blocked by default @@ -83,8 +80,7 @@ class HardCodedACLAgent(HardCodedAgentSessionABC): return blocked_rules def get_allow_acl_rules_for_ier(self, ier, acl, nodes): - """ - Get all allowing ACL rules for an IER. + """Get all allowing ACL rules for an IER. TODO: Add params and return in docstring. TODO: Typehint params and return. @@ -108,8 +104,7 @@ class HardCodedACLAgent(HardCodedAgentSessionABC): nodes, services_list, ): - """ - Get matching ACL rules. + """Get matching ACL rules. TODO: Add params and return in docstring. TODO: Typehint params and return. @@ -140,8 +135,7 @@ class HardCodedACLAgent(HardCodedAgentSessionABC): nodes, services_list, ): - """ - Get the ALLOW ACL rules. + """Get the ALLOW ACL rules. TODO: Add params and return in docstring. TODO: Typehint params and return. @@ -173,8 +167,7 @@ class HardCodedACLAgent(HardCodedAgentSessionABC): nodes, services_list, ): - """ - Get the DENY ACL rules. + """Get the DENY ACL rules. TODO: Add params and return in docstring. TODO: Typehint params and return. @@ -197,8 +190,7 @@ class HardCodedACLAgent(HardCodedAgentSessionABC): return allowed_rules def _calculate_action_full_view(self, obs): - """ - Calculate a good acl-based action for the blue agent to take. + """Calculate a good acl-based action for the blue agent to take. Knowledge of just the observation space is insufficient for a perfect solution, as we need to know: diff --git a/src/primaite/agents/hardcoded_node.py b/src/primaite/agents/hardcoded_node.py index 310fc178..27a2a823 100644 --- a/src/primaite/agents/hardcoded_node.py +++ b/src/primaite/agents/hardcoded_node.py @@ -6,8 +6,7 @@ class HardCodedNodeAgent(HardCodedAgentSessionABC): """An Agent Session class that implements a deterministic Node agent.""" def _calculate_action(self, obs): - """ - Calculate a good node-based action for the blue agent to take. + """Calculate a good node-based action for the blue agent to take. TODO: Add params and return in docstring. TODO: Typehint params and return. diff --git a/src/primaite/agents/rllib.py b/src/primaite/agents/rllib.py index d851ba9c..20503459 100644 --- a/src/primaite/agents/rllib.py +++ b/src/primaite/agents/rllib.py @@ -128,8 +128,7 @@ class RLlibAgent(AgentSessionABC): self, **kwargs, ): - """ - Evaluate the agent. + """Evaluate the agent. :param kwargs: Any agent-specific key-word args to be passed. """ @@ -147,8 +146,7 @@ class RLlibAgent(AgentSessionABC): self, **kwargs, ): - """ - Evaluate the agent. + """Evaluate the agent. :param kwargs: Any agent-specific key-word args to be passed. """ diff --git a/src/primaite/agents/sb3.py b/src/primaite/agents/sb3.py index f5ac44cb..58148d1f 100644 --- a/src/primaite/agents/sb3.py +++ b/src/primaite/agents/sb3.py @@ -77,8 +77,7 @@ class SB3Agent(AgentSessionABC): self, **kwargs, ): - """ - Train the agent. + """Train the agent. :param kwargs: Any agent-specific key-word args to be passed. """ @@ -98,8 +97,7 @@ class SB3Agent(AgentSessionABC): deterministic: bool = True, **kwargs, ): - """ - Evaluate the agent. + """Evaluate the agent. :param deterministic: Whether the evaluation is deterministic. :param kwargs: Any agent-specific key-word args to be passed. diff --git a/src/primaite/agents/simple.py b/src/primaite/agents/simple.py index 5a6c9da5..df93e56d 100644 --- a/src/primaite/agents/simple.py +++ b/src/primaite/agents/simple.py @@ -3,8 +3,7 @@ from primaite.agents.utils import get_new_action, transform_action_acl_enum, tra class RandomAgent(HardCodedAgentSessionABC): - """ - A Random Agent. + """A Random Agent. Get a completely random action from the action space. """ @@ -14,11 +13,9 @@ class RandomAgent(HardCodedAgentSessionABC): class DummyAgent(HardCodedAgentSessionABC): - """ - A Dummy Agent. + """A Dummy Agent. - All action spaces setup so dummy action is always 0 regardless of action - type used. + All action spaces setup so dummy action is always 0 regardless of action type used. """ def _calculate_action(self, obs): @@ -26,8 +23,7 @@ class DummyAgent(HardCodedAgentSessionABC): class DoNothingACLAgent(HardCodedAgentSessionABC): - """ - A do nothing ACL agent. + """A do nothing ACL agent. A valid ACL action that has no effect; does nothing. """ @@ -41,8 +37,7 @@ class DoNothingACLAgent(HardCodedAgentSessionABC): class DoNothingNodeAgent(HardCodedAgentSessionABC): - """ - A do nothing Node agent. + """A do nothing Node agent. A valid Node action that has no effect; does nothing. """ diff --git a/src/primaite/agents/utils.py b/src/primaite/agents/utils.py index 8c59faf7..8b3b57f5 100644 --- a/src/primaite/agents/utils.py +++ b/src/primaite/agents/utils.py @@ -11,8 +11,7 @@ from primaite.common.enums import ( def transform_action_node_readable(action): - """ - Convert a node action from enumerated format to readable format. + """Convert a node action from enumerated format to readable format. example: [1, 3, 1, 0] -> [1, 'SERVICE', 'PATCHING', 0] @@ -34,8 +33,7 @@ def transform_action_node_readable(action): def transform_action_acl_readable(action): - """ - Transform an ACL action to a more readable format. + """Transform an ACL action to a more readable format. example: [0, 1, 2, 5, 0, 1] -> ['NONE', 'ALLOW', 2, 5, 'ANY', 1] @@ -94,8 +92,7 @@ def is_valid_node_action(action): def is_valid_acl_action(action): - """ - Is the ACL action an actual valid action. + """Is the ACL action an actual valid action. Only uses information about the action to determine if the action has an effect. @@ -127,8 +124,7 @@ def is_valid_acl_action(action): def is_valid_acl_action_extra(action): - """ - Harsher version of valid acl actions, does not allow action. + """Harsher version of valid acl actions, does not allow action. TODO: Add params and return in docstring. TODO: Typehint params and return. @@ -151,8 +147,7 @@ def is_valid_acl_action_extra(action): def transform_change_obs_readable(obs): - """ - Transform list of transactions to readable list of each observation property. + """Transform list of transactions to readable list of each observation property. example: np.array([[1,2,1,3],[2,1,1,1]]) -> [[1, 2], ['OFF', 'ON'], ['GOOD', 'GOOD'], ['COMPROMISED', 'GOOD']] @@ -174,8 +169,7 @@ def transform_change_obs_readable(obs): def transform_obs_readable(obs): - """ - Transform observation to readable format. + """Transform observation to readable format. np.array([[1,2,1,3],[2,1,1,1]]) -> [[1, 'OFF', 'GOOD', 'COMPROMISED'], [2, 'ON', 'GOOD', 'GOOD']] @@ -191,8 +185,7 @@ def transform_obs_readable(obs): def convert_to_new_obs(obs, num_nodes=10): - """ - Convert original gym Box observation space to new multiDiscrete observation space. + """Convert original gym Box observation space to new multiDiscrete observation space. TODO: Add params and return in docstring. TODO: Typehint params and return. @@ -203,8 +196,7 @@ def convert_to_new_obs(obs, num_nodes=10): def convert_to_old_obs(obs, num_nodes=10, num_links=10, num_services=1): - """ - Convert to old observation. + """Convert to old observation. Links filled with 0's as no information is included in new observation space. @@ -240,8 +232,7 @@ def convert_to_old_obs(obs, num_nodes=10, num_links=10, num_services=1): def describe_obs_change(obs1, obs2, num_nodes=10, num_links=10, num_services=1): - """ - Return string describing change between two observations. + """Return string describing change between two observations. example: obs_1 = array([[1, 1, 1, 1, 3], [2, 1, 1, 1, 1]]) @@ -269,8 +260,7 @@ def describe_obs_change(obs1, obs2, num_nodes=10, num_links=10, num_services=1): def _describe_obs_change_helper(obs_change, is_link): - """ - Helper funcion to describe what has changed. + """Helper funcion to describe what has changed. example: [ 1 -1 -1 -1 1] -> "ID 1: Service 1 changed to GOOD" @@ -305,8 +295,7 @@ def _describe_obs_change_helper(obs_change, is_link): def transform_action_node_enum(action): - """ - Convert a node action from readable string format, to enumerated format. + """Convert a node action from readable string format, to enumerated format. example: [1, 'SERVICE', 'PATCHING', 0] -> [1, 3, 1, 0] @@ -337,8 +326,7 @@ def transform_action_node_enum(action): def transform_action_node_readable(action): - """ - Convert a node action from enumerated format to readable format. + """Convert a node action from enumerated format to readable format. example: [1, 3, 1, 0] -> [1, 'SERVICE', 'PATCHING', 0] @@ -360,8 +348,7 @@ def transform_action_node_readable(action): def node_action_description(action): - """ - Generate string describing a node-based action. + """Generate string describing a node-based action. TODO: Add params and return in docstring. TODO: Typehint params and return. @@ -388,8 +375,7 @@ def node_action_description(action): def transform_action_acl_enum(action): - """ - Convert acl action from readable str format, to enumerated format. + """Convert acl action from readable str format, to enumerated format. TODO: Add params and return in docstring. TODO: Typehint params and return. @@ -411,8 +397,7 @@ def transform_action_acl_enum(action): def acl_action_description(action): - """ - Generate string describing an acl-based action. + """Generate string describing an acl-based action. TODO: Add params and return in docstring. TODO: Typehint params and return. @@ -432,8 +417,7 @@ def acl_action_description(action): def get_node_of_ip(ip, node_dict): - """ - Get the node ID of an IP address. + """Get the node ID of an IP address. node_dict: dictionary of nodes where key is ID, and value is the node (can be ontained from env.nodes) @@ -480,8 +464,7 @@ def is_valid_node_action(action): def is_valid_acl_action(action): - """ - Is the ACL action an actual valid action. + """Is the ACL action an actual valid action. Only uses information about the action to determine if the action has an effect @@ -513,8 +496,7 @@ def is_valid_acl_action(action): def is_valid_acl_action_extra(action): - """ - Harsher version of valid acl actions, does not allow action. + """Harsher version of valid acl actions, does not allow action. TODO: Add params and return in docstring. TODO: Typehint params and return. @@ -537,8 +519,7 @@ def is_valid_acl_action_extra(action): def get_new_action(old_action, action_dict): - """ - Get new action (e.g. 32) from old action e.g. [1,1,1,0]. + """Get new action (e.g. 32) from old action e.g. [1,1,1,0]. Old_action can be either node or acl action type diff --git a/src/primaite/cli.py b/src/primaite/cli.py index 40e8cf0d..42825144 100644 --- a/src/primaite/cli.py +++ b/src/primaite/cli.py @@ -28,8 +28,7 @@ def build_dirs(): @app.command() def reset_notebooks(overwrite: bool = True): - """ - Force a reset of the demo notebooks in the users notebooks directory. + """Force a reset of the demo notebooks in the users notebooks directory. :param overwrite: If True, will overwrite existing demo notebooks. """ @@ -40,8 +39,7 @@ def reset_notebooks(overwrite: bool = True): @app.command() def logs(last_n: Annotated[int, typer.Option("-n")]): - """ - Print the PrimAITE log file. + """Print the PrimAITE log file. :param last_n: The number of lines to print. Default value is 10. """ @@ -61,8 +59,7 @@ _LogLevel = Enum("LogLevel", {k: k for k in logging._levelToName.values()}) # n @app.command() def log_level(level: Annotated[Optional[_LogLevel], typer.Argument()] = None): - """ - View or set the PrimAITE Log Level. + """View or set the PrimAITE Log Level. To View, simply call: primaite log-level @@ -113,8 +110,7 @@ def clean_up(): @app.command() def setup(overwrite_existing: bool = True): - """ - Perform the PrimAITE first-time setup. + """Perform the PrimAITE first-time setup. WARNING: All user-data will be lost. """ @@ -152,8 +148,7 @@ def setup(overwrite_existing: bool = True): @app.command() def session(tc: Optional[str] = None, ldc: Optional[str] = None): - """ - Run a PrimAITE session. + """Run a PrimAITE session. tc: The training config filepath. Optional. If no value is passed then example default training config is used from: @@ -178,8 +173,7 @@ def session(tc: Optional[str] = None, ldc: Optional[str] = None): @app.command() def plotly_template(template: Annotated[Optional[PlotlyTemplate], typer.Argument()] = None): - """ - View or set the plotly template for Session plots. + """View or set the plotly template for Session plots. To View, simply call: primaite plotly-template diff --git a/src/primaite/common/protocol.py b/src/primaite/common/protocol.py index 2e3683e8..ebda1fcf 100644 --- a/src/primaite/common/protocol.py +++ b/src/primaite/common/protocol.py @@ -6,8 +6,7 @@ class Protocol(object): """Protocol class.""" def __init__(self, _name): - """ - Init. + """Init. Args: _name: The protocol name @@ -16,8 +15,7 @@ class Protocol(object): self.load = 0 # bps def get_name(self): - """ - Gets the protocol name. + """Gets the protocol name. Returns: The protocol name @@ -25,8 +23,7 @@ class Protocol(object): return self.name def get_load(self): - """ - Gets the protocol load. + """Gets the protocol load. Returns: The protocol load (bps) @@ -34,8 +31,7 @@ class Protocol(object): return self.load def add_load(self, _load): - """ - Adds load to the protocol. + """Adds load to the protocol. Args: _load: The load to add diff --git a/src/primaite/common/service.py b/src/primaite/common/service.py index 51403879..c381f51f 100644 --- a/src/primaite/common/service.py +++ b/src/primaite/common/service.py @@ -8,8 +8,7 @@ class Service(object): """Service class.""" def __init__(self, name: str, port: str, software_state: SoftwareState): - """ - Init. + """Init. :param name: The service name. :param port: The service port. diff --git a/src/primaite/config/lay_down_config.py b/src/primaite/config/lay_down_config.py index 08f77b2f..587997b7 100644 --- a/src/primaite/config/lay_down_config.py +++ b/src/primaite/config/lay_down_config.py @@ -12,8 +12,7 @@ _EXAMPLE_LAY_DOWN: Final[Path] = USERS_CONFIG_DIR / "example_config" / "lay_down def convert_legacy_lay_down_config_dict(legacy_config_dict: Dict[str, Any]) -> Dict[str, Any]: - """ - Convert a legacy lay down config dict to the new format. + """Convert a legacy lay down config dict to the new format. :param legacy_config_dict: A legacy lay down config dict. """ @@ -22,12 +21,10 @@ def convert_legacy_lay_down_config_dict(legacy_config_dict: Dict[str, Any]) -> D def load(file_path: Union[str, Path], legacy_file: bool = False) -> Dict: - """ - Read in a lay down config yaml file. + """Read in a lay down config yaml file. :param file_path: The config file path. - :param legacy_file: True if the config file is legacy format, otherwise - False. + :param legacy_file: True if the config file is legacy format, otherwise False. :return: The lay down config as a dict. :raises ValueError: If the file_path does not exist. """ @@ -53,8 +50,7 @@ def load(file_path: Union[str, Path], legacy_file: bool = False) -> Dict: def ddos_basic_one_config_path() -> Path: - """ - The path to the example lay_down_config_1_DDOS_basic.yaml file. + """The path to the example lay_down_config_1_DDOS_basic.yaml file. :return: The file path. """ @@ -68,8 +64,7 @@ def ddos_basic_one_config_path() -> Path: def ddos_basic_two_config_path() -> Path: - """ - The path to the example lay_down_config_2_DDOS_basic.yaml file. + """The path to the example lay_down_config_2_DDOS_basic.yaml file. :return: The file path. """ @@ -83,8 +78,7 @@ def ddos_basic_two_config_path() -> Path: def dos_very_basic_config_path() -> Path: - """ - The path to the example lay_down_config_3_DOS_very_basic.yaml file. + """The path to the example lay_down_config_3_DOS_very_basic.yaml file. :return: The file path. """ @@ -98,8 +92,7 @@ def dos_very_basic_config_path() -> Path: def data_manipulation_config_path() -> Path: - """ - The path to the example lay_down_config_5_data_manipulation.yaml file. + """The path to the example lay_down_config_5_data_manipulation.yaml file. :return: The file path. """ diff --git a/src/primaite/config/training_config.py b/src/primaite/config/training_config.py index bd73f65b..040ef6fa 100644 --- a/src/primaite/config/training_config.py +++ b/src/primaite/config/training_config.py @@ -24,8 +24,7 @@ _EXAMPLE_TRAINING: Final[Path] = USERS_CONFIG_DIR / "example_config" / "training def main_training_config_path() -> Path: - """ - The path to the example training_config_main.yaml file. + """The path to the example training_config_main.yaml file. :return: The file path. """ @@ -180,8 +179,7 @@ class TrainingConfig: @classmethod def from_dict(cls, config_dict: Dict[str, Union[str, int, bool]]) -> TrainingConfig: - """ - Create an instance of TrainingConfig from a dict. + """Create an instance of TrainingConfig from a dict. :param config_dict: The training config dict. :return: The instance of TrainingConfig. @@ -236,8 +234,7 @@ class TrainingConfig: def load(file_path: Union[str, Path], legacy_file: bool = False) -> TrainingConfig: - """ - Read in a training config yaml file. + """Read in a training config yaml file. :param file_path: The config file path. :param legacy_file: True if the config file is legacy format, otherwise @@ -281,18 +278,14 @@ def convert_legacy_training_config_dict( action_type: ActionType = ActionType.ANY, num_steps: int = 256, ) -> Dict[str, Any]: - """ - Convert a legacy training config dict to the new format. + """Convert a legacy training config dict to the new format. :param legacy_config_dict: A legacy training config dict. - :param agent_framework: The agent framework to use as legacy training - configs don't have agent_framework values. - :param agent_identifier: The red agent identifier to use as legacy - training configs don't have agent_identifier values. - :param action_type: The action space type to set as legacy training configs - don't have action_type values. - :param num_steps: The number of steps to set as legacy training configs - don't have num_steps values. + :param agent_framework: The agent framework to use as legacy training configs don't have agent_framework values. + :param agent_identifier: The red agent identifier to use as legacy training configs don't have agent_identifier + values. + :param action_type: The action space type to set as legacy training configs don't have action_type values. + :param num_steps: The number of steps to set as legacy training configs don't have num_steps values. :return: The converted training config dict. """ config_dict = { @@ -312,8 +305,7 @@ def convert_legacy_training_config_dict( def _get_new_key_from_legacy(legacy_key: str) -> str: - """ - Maps legacy training config keys to the new format keys. + """Maps legacy training config keys to the new format keys. :param legacy_key: A legacy training config key. :return: The mapped key. diff --git a/src/primaite/data_viz/session_plots.py b/src/primaite/data_viz/session_plots.py index 245b9774..542c6677 100644 --- a/src/primaite/data_viz/session_plots.py +++ b/src/primaite/data_viz/session_plots.py @@ -22,8 +22,7 @@ def plot_av_reward_per_episode( title: Optional[str] = None, subtitle: Optional[str] = None, ) -> Figure: - """ - Plot the average reward per episode from a csv session output. + """Plot the average reward per episode from a csv session output. :param av_reward_per_episode_csv: The average reward per episode csv file path. diff --git a/src/primaite/environment/observations.py b/src/primaite/environment/observations.py index f8b42e1c..e347a65c 100644 --- a/src/primaite/environment/observations.py +++ b/src/primaite/environment/observations.py @@ -376,8 +376,8 @@ class LinkTrafficLevels(AbstractObservationComponent): class ObservationsHandler: """Component-based observation space handler. - This allows users to configure observation spaces by mixing and matching components. - Each component can also define further parameters to make them more flexible. + This allows users to configure observation spaces by mixing and matching components. Each component can also define + further parameters to make them more flexible. """ _REGISTRY: Final[Dict[str, type]] = { diff --git a/src/primaite/environment/primaite_env.py b/src/primaite/environment/primaite_env.py index 03c23f93..29662988 100644 --- a/src/primaite/environment/primaite_env.py +++ b/src/primaite/environment/primaite_env.py @@ -67,14 +67,12 @@ class Primaite(Env): session_path: Path, timestamp_str: str, ): - """ - The Primaite constructor. + """The Primaite constructor. :param training_config_path: The training config filepath. :param lay_down_config_path: The lay down config filepath. :param session_path: The directory path the session is writing to. - :param timestamp_str: The session timestamp in the format: - _. + :param timestamp_str: The session timestamp in the format: _. """ self.session_path: Final[Path] = session_path self.timestamp_str: Final[str] = timestamp_str @@ -256,8 +254,7 @@ class Primaite(Env): self.total_step_count = 0 def reset(self): - """ - AI Gym Reset function. + """AI Gym Reset function. Returns: Environment observation space (reset) @@ -293,8 +290,7 @@ class Primaite(Env): return self.env_obs def step(self, action): - """ - AI Gym Step function. + """AI Gym Step function. Args: action: Action space from agent @@ -432,8 +428,7 @@ class Primaite(Env): print(" Protocol: " + protocol.get_name().name + ", Load: " + str(protocol.get_load())) def interpret_action_and_apply(self, _action): - """ - Applies agent actions to the nodes and Access Control List. + """Applies agent actions to the nodes and Access Control List. Args: _action: The action space from the agent @@ -452,8 +447,7 @@ class Primaite(Env): logging.error("Invalid action type found") def apply_actions_to_nodes(self, _action): - """ - Applies agent actions to the nodes. + """Applies agent actions to the nodes. Args: _action: The action space from the agent @@ -540,8 +534,7 @@ class Primaite(Env): return def apply_actions_to_acl(self, _action): - """ - Applies agent actions to the Access Control List [TO DO]. + """Applies agent actions to the Access Control List [TO DO]. Args: _action: The action space from the agent @@ -618,8 +611,7 @@ class Primaite(Env): return def apply_time_based_updates(self): - """ - Updates anything that needs to count down and then change state. + """Updates anything that needs to count down and then change state. e.g. reset / patching status """ @@ -716,8 +708,7 @@ class Primaite(Env): print("Environment configuration loaded") def create_node(self, item): - """ - Creates a node from config data. + """Creates a node from config data. Args: item: A config data item @@ -797,8 +788,7 @@ class Primaite(Env): self.network_reference.add_nodes_from([node_ref]) def create_link(self, item: Dict): - """ - Creates a link from config data. + """Creates a link from config data. Args: item: A config data item @@ -841,8 +831,7 @@ class Primaite(Env): ) def create_green_ier(self, item): - """ - Creates a green IER from config data. + """Creates a green IER from config data. Args: item: A config data item @@ -882,8 +871,7 @@ class Primaite(Env): ) def create_red_ier(self, item): - """ - Creates a red IER from config data. + """Creates a red IER from config data. Args: item: A config data item @@ -912,8 +900,7 @@ class Primaite(Env): ) def create_green_pol(self, item): - """ - Creates a green PoL object from config data. + """Creates a green PoL object from config data. Args: item: A config data item @@ -946,8 +933,7 @@ class Primaite(Env): ) def create_red_pol(self, item): - """ - Creates a red PoL object from config data. + """Creates a red PoL object from config data. Args: item: A config data item @@ -987,8 +973,7 @@ class Primaite(Env): ) def create_acl_rule(self, item): - """ - Creates an ACL rule from config data. + """Creates an ACL rule from config data. Args: item: A config data item @@ -1008,8 +993,7 @@ class Primaite(Env): ) def create_services_list(self, services): - """ - Creates a list of services (enum) from config data. + """Creates a list of services (enum) from config data. Args: item: A config data item representing the services @@ -1024,8 +1008,7 @@ class Primaite(Env): self.num_services = len(self.services_list) def create_ports_list(self, ports): - """ - Creates a list of ports from config data. + """Creates a list of ports from config data. Args: item: A config data item representing the ports @@ -1048,8 +1031,7 @@ class Primaite(Env): self.observation_type = ObservationType[observation_info["type"]] def get_action_info(self, action_info): - """ - Extracts action_info. + """Extracts action_info. Args: item: A config data item representing action info @@ -1069,11 +1051,9 @@ class Primaite(Env): self.obs_config = obs_config def reset_environment(self): - """ - # Resets environment. + """# Resets environment. - Uses config data config data in order to build the environment - configuration. + Uses config data config data in order to build the environment configuration. """ for item in self.lay_down_config: if item["item_type"] == "NODE": @@ -1095,8 +1075,7 @@ class Primaite(Env): ier_value.set_is_running(False) def reset_node(self, item): - """ - Resets the statuses of a node. + """Resets the statuses of a node. Args: item: A config data item @@ -1143,8 +1122,7 @@ class Primaite(Env): pass def create_node_action_dict(self): - """ - Creates a dictionary mapping each possible discrete action to more readable multidiscrete action. + """Creates a dictionary mapping each possible discrete action to more readable multidiscrete action. Note: Only actions that have the potential to change the state exist in the mapping (except for key 0) @@ -1157,7 +1135,6 @@ class Primaite(Env): 5: [1, 3, 1, 0], ... } - """ # reserve 0 action to be a nothing action actions = {0: [1, 0, 0, 0]} @@ -1209,11 +1186,9 @@ class Primaite(Env): return actions def create_node_and_acl_action_dict(self): - """ - Create a dictionary mapping each possible discrete action to a more readable mutlidiscrete action. + """Create a dictionary mapping each possible discrete action to a more readable mutlidiscrete action. The dictionary contains actions of both Node and ACL action types. - """ node_action_dict = self.create_node_action_dict() acl_action_dict = self.create_acl_action_dict() diff --git a/src/primaite/environment/reward.py b/src/primaite/environment/reward.py index 19094a18..5cef47ef 100644 --- a/src/primaite/environment/reward.py +++ b/src/primaite/environment/reward.py @@ -21,8 +21,7 @@ def calculate_reward_function( step_count, config_values, ): - """ - Compares the states of the initial and final nodes/links to get a reward. + """Compares the states of the initial and final nodes/links to get a reward. Args: initial_nodes: The nodes before red and blue agents take effect @@ -95,8 +94,7 @@ def calculate_reward_function( def score_node_operating_state(final_node, initial_node, reference_node, config_values): - """ - Calculates score relating to the hardware state of a node. + """Calculates score relating to the hardware state of a node. Args: final_node: The node after red and blue agents take effect @@ -144,8 +142,7 @@ def score_node_operating_state(final_node, initial_node, reference_node, config_ def score_node_os_state(final_node, initial_node, reference_node, config_values): - """ - Calculates score relating to the Software State of a node. + """Calculates score relating to the Software State of a node. Args: final_node: The node after red and blue agents take effect @@ -195,8 +192,7 @@ def score_node_os_state(final_node, initial_node, reference_node, config_values) def score_node_service_state(final_node, initial_node, reference_node, config_values): - """ - Calculates score relating to the service state(s) of a node. + """Calculates score relating to the service state(s) of a node. Args: final_node: The node after red and blue agents take effect @@ -267,8 +263,7 @@ def score_node_service_state(final_node, initial_node, reference_node, config_va def score_node_file_system(final_node, initial_node, reference_node, config_values): - """ - Calculates score relating to the file system state of a node. + """Calculates score relating to the file system state of a node. Args: final_node: The node after red and blue agents take effect diff --git a/src/primaite/links/link.py b/src/primaite/links/link.py index 90235e9f..e8901b3d 100644 --- a/src/primaite/links/link.py +++ b/src/primaite/links/link.py @@ -9,8 +9,7 @@ class Link(object): """Link class.""" def __init__(self, _id, _bandwidth, _source_node_name, _dest_node_name, _services): - """ - Init. + """Init. Args: _id: The IER id @@ -30,8 +29,7 @@ class Link(object): self.add_protocol(protocol_name) def add_protocol(self, _protocol): - """ - Adds a new protocol to the list of protocols on this link. + """Adds a new protocol to the list of protocols on this link. Args: _protocol: The protocol to be added (enum) @@ -39,8 +37,7 @@ class Link(object): self.protocol_list.append(Protocol(_protocol)) def get_id(self): - """ - Gets link ID. + """Gets link ID. Returns: Link ID @@ -48,8 +45,7 @@ class Link(object): return self.id def get_source_node_name(self): - """ - Gets source node name. + """Gets source node name. Returns: Source node name @@ -57,8 +53,7 @@ class Link(object): return self.source_node_name def get_dest_node_name(self): - """ - Gets destination node name. + """Gets destination node name. Returns: Destination node name @@ -66,8 +61,7 @@ class Link(object): return self.dest_node_name def get_bandwidth(self): - """ - Gets bandwidth of link. + """Gets bandwidth of link. Returns: Link bandwidth (bps) @@ -75,8 +69,7 @@ class Link(object): return self.bandwidth def get_protocol_list(self): - """ - Gets list of protocols on this link. + """Gets list of protocols on this link. Returns: List of protocols on this link @@ -84,8 +77,7 @@ class Link(object): return self.protocol_list def get_current_load(self): - """ - Gets current total load on this link. + """Gets current total load on this link. Returns: Total load on this link (bps) @@ -96,8 +88,7 @@ class Link(object): return total_load def add_protocol_load(self, _protocol, _load): - """ - Adds a loading to a protocol on this link. + """Adds a loading to a protocol on this link. Args: _protocol: The protocol to load diff --git a/src/primaite/nodes/active_node.py b/src/primaite/nodes/active_node.py index 07a0ea0a..588ccd93 100644 --- a/src/primaite/nodes/active_node.py +++ b/src/primaite/nodes/active_node.py @@ -25,8 +25,7 @@ class ActiveNode(Node): file_system_state: FileSystemState, config_values: TrainingConfig, ): - """ - Init. + """Init. :param node_id: The node ID :param name: The node name @@ -52,8 +51,7 @@ class ActiveNode(Node): @property def software_state(self) -> SoftwareState: - """ - Get the software_state. + """Get the software_state. :return: The software_state. """ @@ -61,8 +59,7 @@ class ActiveNode(Node): @software_state.setter def software_state(self, software_state: SoftwareState): - """ - Get the software_state. + """Get the software_state. :param software_state: Software State. """ @@ -80,8 +77,7 @@ class ActiveNode(Node): ) def set_software_state_if_not_compromised(self, software_state: SoftwareState): - """ - Sets Software State if the node is not compromised. + """Sets Software State if the node is not compromised. Args: software_state: Software State @@ -107,8 +103,7 @@ class ActiveNode(Node): self._software_state = SoftwareState.GOOD def set_file_system_state(self, file_system_state: FileSystemState): - """ - Sets the file system state (actual and observed). + """Sets the file system state (actual and observed). Args: file_system_state: File system state @@ -134,8 +129,7 @@ class ActiveNode(Node): ) def set_file_system_state_if_not_compromised(self, file_system_state: FileSystemState): - """ - Sets the file system state (actual and observed) if not in a compromised state. + """Sets the file system state (actual and observed) if not in a compromised state. Use for green PoL to prevent it overturning a compromised state diff --git a/src/primaite/nodes/node.py b/src/primaite/nodes/node.py index bac1792d..40f6328f 100644 --- a/src/primaite/nodes/node.py +++ b/src/primaite/nodes/node.py @@ -18,8 +18,7 @@ class Node: hardware_state: HardwareState, config_values: TrainingConfig, ): - """ - Init. + """Init. :param node_id: The node id. :param name: The name of the node. diff --git a/src/primaite/nodes/node_state_instruction_green.py b/src/primaite/nodes/node_state_instruction_green.py index 2b1d94be..e1244144 100644 --- a/src/primaite/nodes/node_state_instruction_green.py +++ b/src/primaite/nodes/node_state_instruction_green.py @@ -15,8 +15,7 @@ class NodeStateInstructionGreen(object): _service_name, _state, ): - """ - Init. + """Init. Args: _id: The node state instruction id @@ -36,8 +35,7 @@ class NodeStateInstructionGreen(object): self.state = _state def get_start_step(self): - """ - Gets the start step. + """Gets the start step. Returns: The start step @@ -45,8 +43,7 @@ class NodeStateInstructionGreen(object): return self.start_step def get_end_step(self): - """ - Gets the end step. + """Gets the end step. Returns: The end step @@ -54,8 +51,7 @@ class NodeStateInstructionGreen(object): return self.end_step def get_node_id(self): - """ - Gets the node ID. + """Gets the node ID. Returns: The node ID @@ -63,8 +59,7 @@ class NodeStateInstructionGreen(object): return self.node_id def get_node_pol_type(self): - """ - Gets the node pattern of life type (enum). + """Gets the node pattern of life type (enum). Returns: The node pattern of life type (enum) @@ -72,8 +67,7 @@ class NodeStateInstructionGreen(object): return self.node_pol_type def get_service_name(self): - """ - Gets the service name. + """Gets the service name. Returns: The service name @@ -81,8 +75,7 @@ class NodeStateInstructionGreen(object): return self.service_name def get_state(self): - """ - Gets the state (node or service). + """Gets the state (node or service). Returns: The state (node or service) diff --git a/src/primaite/nodes/node_state_instruction_red.py b/src/primaite/nodes/node_state_instruction_red.py index 4272ce24..3e2e734d 100644 --- a/src/primaite/nodes/node_state_instruction_red.py +++ b/src/primaite/nodes/node_state_instruction_red.py @@ -23,8 +23,7 @@ class NodeStateInstructionRed(object): _pol_source_node_service, _pol_source_node_service_state, ): - """ - Init. + """Init. Args: _id: The node state instruction id @@ -52,8 +51,7 @@ class NodeStateInstructionRed(object): self.source_node_service_state = _pol_source_node_service_state def get_start_step(self): - """ - Gets the start step. + """Gets the start step. Returns: The start step @@ -61,8 +59,7 @@ class NodeStateInstructionRed(object): return self.start_step def get_end_step(self): - """ - Gets the end step. + """Gets the end step. Returns: The end step @@ -70,8 +67,7 @@ class NodeStateInstructionRed(object): return self.end_step def get_target_node_id(self): - """ - Gets the node ID. + """Gets the node ID. Returns: The node ID @@ -79,8 +75,7 @@ class NodeStateInstructionRed(object): return self.target_node_id def get_initiator(self): - """ - Gets the initiator. + """Gets the initiator. Returns: The initiator @@ -88,8 +83,7 @@ class NodeStateInstructionRed(object): return self.initiator def get_pol_type(self) -> NodePOLType: - """ - Gets the node pattern of life type (enum). + """Gets the node pattern of life type (enum). Returns: The node pattern of life type (enum) @@ -97,8 +91,7 @@ class NodeStateInstructionRed(object): return self.pol_type def get_service_name(self): - """ - Gets the service name. + """Gets the service name. Returns: The service name @@ -106,8 +99,7 @@ class NodeStateInstructionRed(object): return self.service_name def get_state(self): - """ - Gets the state (node or service). + """Gets the state (node or service). Returns: The state (node or service) @@ -115,8 +107,7 @@ class NodeStateInstructionRed(object): return self.state def get_source_node_id(self): - """ - Gets the source node id (used for initiator type SERVICE). + """Gets the source node id (used for initiator type SERVICE). Returns: The source node id @@ -124,8 +115,7 @@ class NodeStateInstructionRed(object): return self.source_node_id def get_source_node_service(self): - """ - Gets the source node service (used for initiator type SERVICE). + """Gets the source node service (used for initiator type SERVICE). Returns: The source node service @@ -133,8 +123,7 @@ class NodeStateInstructionRed(object): return self.source_node_service def get_source_node_service_state(self): - """ - Gets the source node service state (used for initiator type SERVICE). + """Gets the source node service state (used for initiator type SERVICE). Returns: The source node service state diff --git a/src/primaite/nodes/passive_node.py b/src/primaite/nodes/passive_node.py index 9aa5c7d7..188b4ee3 100644 --- a/src/primaite/nodes/passive_node.py +++ b/src/primaite/nodes/passive_node.py @@ -17,8 +17,7 @@ class PassiveNode(Node): hardware_state: HardwareState, config_values: TrainingConfig, ): - """ - Init. + """Init. :param node_id: The node id. :param name: The name of the node. @@ -32,8 +31,7 @@ class PassiveNode(Node): @property def ip_address(self) -> str: - """ - Gets the node IP address as an empty string. + """Gets the node IP address as an empty string. No concept of IP address for passive nodes for now. diff --git a/src/primaite/nodes/service_node.py b/src/primaite/nodes/service_node.py index 5d69df92..0114f507 100644 --- a/src/primaite/nodes/service_node.py +++ b/src/primaite/nodes/service_node.py @@ -26,8 +26,7 @@ class ServiceNode(ActiveNode): file_system_state: FileSystemState, config_values: TrainingConfig, ): - """ - Init. + """Init. :param node_id: The node ID :param name: The node name @@ -53,16 +52,14 @@ class ServiceNode(ActiveNode): self.services: Dict[str, Service] = {} def add_service(self, service: Service): - """ - Adds a service to the node. + """Adds a service to the node. :param service: The service to add """ self.services[service.name] = service def has_service(self, protocol_name: str) -> bool: - """ - Indicates whether a service is on a node. + """Indicates whether a service is on a node. :param protocol_name: The service (protocol)e. :return: True if service (protocol) is on the node, otherwise False. @@ -73,12 +70,10 @@ class ServiceNode(ActiveNode): return False def service_running(self, protocol_name: str) -> bool: - """ - Indicates whether a service is in a running state on the node. + """Indicates whether a service is in a running state on the node. :param protocol_name: The service (protocol) - :return: True if service (protocol) is in a running state on the - node, otherwise False. + :return: True if service (protocol) is in a running state on the node, otherwise False. """ for service_key, service_value in self.services.items(): if service_key == protocol_name: @@ -89,12 +84,10 @@ class ServiceNode(ActiveNode): return False def service_is_overwhelmed(self, protocol_name: str) -> bool: - """ - Indicates whether a service is in an overwhelmed state on the node. + """Indicates whether a service is in an overwhelmed state on the node. :param protocol_name: The service (protocol) - :return: True if service (protocol) is in an overwhelmed state on the - node, otherwise False. + :return: True if service (protocol) is in an overwhelmed state on the node, otherwise False. """ for service_key, service_value in self.services.items(): if service_key == protocol_name: @@ -105,8 +98,7 @@ class ServiceNode(ActiveNode): return False def set_service_state(self, protocol_name: str, software_state: SoftwareState): - """ - Sets the software_state of a service (protocol) on the node. + """Sets the software_state of a service (protocol) on the node. :param protocol_name: The service (protocol). :param software_state: The software_state. @@ -134,8 +126,7 @@ class ServiceNode(ActiveNode): ) def set_service_state_if_not_compromised(self, protocol_name: str, software_state: SoftwareState): - """ - Sets the software_state of a service (protocol) on the node. + """Sets the software_state of a service (protocol) on the node. Done if the software_state is not "compromised". @@ -161,8 +152,7 @@ class ServiceNode(ActiveNode): ) def get_service_state(self, protocol_name): - """ - Gets the state of a service. + """Gets the state of a service. :return: The software_state of the service. """ diff --git a/src/primaite/notebooks/__init__.py b/src/primaite/notebooks/__init__.py index 0e81e581..0730312e 100644 --- a/src/primaite/notebooks/__init__.py +++ b/src/primaite/notebooks/__init__.py @@ -10,8 +10,7 @@ _LOGGER = getLogger(__name__) def start_jupyter_session(): - """ - Starts a new Jupyter notebook session in the app notebooks directory. + """Starts a new Jupyter notebook session in the app notebooks directory. Currently only works on Windows OS. diff --git a/src/primaite/pol/green_pol.py b/src/primaite/pol/green_pol.py index e9dfef8c..91a6f787 100644 --- a/src/primaite/pol/green_pol.py +++ b/src/primaite/pol/green_pol.py @@ -25,8 +25,7 @@ def apply_iers( acl: AccessControlList, step: int, ): - """ - Applies IERs to the links (link pattern of life). + """Applies IERs to the links (link pattern of life). Args: network: The network modelled in the environment @@ -218,8 +217,7 @@ def apply_node_pol( node_pol: Dict[any, Union[NodeStateInstructionGreen, NodeStateInstructionRed]], step: int, ): - """ - Applies node pattern of life. + """Applies node pattern of life. Args: nodes: The nodes within the environment diff --git a/src/primaite/pol/ier.py b/src/primaite/pol/ier.py index daa49727..09f32aeb 100644 --- a/src/primaite/pol/ier.py +++ b/src/primaite/pol/ier.py @@ -1,6 +1,5 @@ # Crown Copyright (C) Dstl 2022. DEFCON 703. Shared in confidence. -""" -Information Exchange Requirements for APE. +"""Information Exchange Requirements for APE. Used to represent an information flow from source to destination. """ @@ -22,8 +21,7 @@ class IER(object): _mission_criticality, _running=False, ): - """ - Init. + """Init. Args: _id: The IER id @@ -49,8 +47,7 @@ class IER(object): self.running = _running def get_id(self): - """ - Gets IER ID. + """Gets IER ID. Returns: IER ID @@ -58,8 +55,7 @@ class IER(object): return self.id def get_start_step(self): - """ - Gets IER start step. + """Gets IER start step. Returns: IER start step @@ -67,8 +63,7 @@ class IER(object): return self.start_step def get_end_step(self): - """ - Gets IER end step. + """Gets IER end step. Returns: IER end step @@ -76,8 +71,7 @@ class IER(object): return self.end_step def get_load(self): - """ - Gets IER load. + """Gets IER load. Returns: IER load @@ -85,8 +79,7 @@ class IER(object): return self.load def get_protocol(self): - """ - Gets IER protocol. + """Gets IER protocol. Returns: IER protocol @@ -94,8 +87,7 @@ class IER(object): return self.protocol def get_port(self): - """ - Gets IER port. + """Gets IER port. Returns: IER port @@ -103,8 +95,7 @@ class IER(object): return self.port def get_source_node_id(self): - """ - Gets IER source node ID. + """Gets IER source node ID. Returns: IER source node ID @@ -112,8 +103,7 @@ class IER(object): return self.source_node_id def get_dest_node_id(self): - """ - Gets IER destination node ID. + """Gets IER destination node ID. Returns: IER destination node ID @@ -121,8 +111,7 @@ class IER(object): return self.dest_node_id def get_is_running(self): - """ - Informs whether the IER is currently running. + """Informs whether the IER is currently running. Returns: True if running @@ -130,8 +119,7 @@ class IER(object): return self.running def set_is_running(self, _value): - """ - Sets the running state of the IER. + """Sets the running state of the IER. Args: _value: running status @@ -139,8 +127,7 @@ class IER(object): self.running = _value def get_mission_criticality(self): - """ - Gets the IER mission criticality (used in the reward function). + """Gets the IER mission criticality (used in the reward function). Returns: Mission criticality value (0 lowest to 5 highest) diff --git a/src/primaite/pol/red_agent_pol.py b/src/primaite/pol/red_agent_pol.py index bff19bf8..86482903 100644 --- a/src/primaite/pol/red_agent_pol.py +++ b/src/primaite/pol/red_agent_pol.py @@ -24,8 +24,7 @@ def apply_red_agent_iers( acl: AccessControlList, step: int, ): - """ - Applies IERs to the links (link POL) resulting from red agent attack. + """Applies IERs to the links (link POL) resulting from red agent attack. Args: network: The network modelled in the environment @@ -214,8 +213,7 @@ def apply_red_agent_node_pol( node_pol: Dict[str, NodeStateInstructionRed], step: int, ): - """ - Applies node pattern of life. + """Applies node pattern of life. Args: nodes: The nodes within the environment @@ -297,8 +295,7 @@ def apply_red_agent_node_pol( def is_red_ier_incoming(node, iers, node_pol_type): - """ - Checks if the RED IER is incoming. + """Checks if the RED IER is incoming. TODO: Write more descriptive docstring with params and returns. """ diff --git a/src/primaite/primaite_session.py b/src/primaite/primaite_session.py index df3ebec1..ed2b9bf1 100644 --- a/src/primaite/primaite_session.py +++ b/src/primaite/primaite_session.py @@ -18,11 +18,9 @@ _LOGGER = getLogger(__name__) class PrimaiteSession: - """ - The PrimaiteSession class. + """The PrimaiteSession class. - Provides a single learning and evaluation entry point for all training - and lay down configurations. + Provides a single learning and evaluation entry point for all training and lay down configurations. """ def __init__( @@ -30,8 +28,7 @@ class PrimaiteSession: training_config_path: Union[str, Path], lay_down_config_path: Union[str, Path], ): - """ - The PrimaiteSession constructor. + """The PrimaiteSession constructor. :param training_config_path: The training config path. :param lay_down_config_path: The lay down config path. @@ -125,8 +122,7 @@ class PrimaiteSession: self, **kwargs, ): - """ - Train the agent. + """Train the agent. :param kwargs: Any agent-framework specific key word args. """ @@ -137,8 +133,7 @@ class PrimaiteSession: self, **kwargs, ): - """ - Evaluate the agent. + """Evaluate the agent. :param kwargs: Any agent-framework specific key word args. """ diff --git a/src/primaite/setup/reset_demo_notebooks.py b/src/primaite/setup/reset_demo_notebooks.py index 7fa96783..8d2a94c7 100644 --- a/src/primaite/setup/reset_demo_notebooks.py +++ b/src/primaite/setup/reset_demo_notebooks.py @@ -12,11 +12,9 @@ _LOGGER = getLogger(__name__) def run(overwrite_existing: bool = True): - """ - Resets the demo jupyter notebooks in the users app notebooks directory. + """Resets the demo jupyter notebooks in the users app notebooks directory. - :param overwrite_existing: A bool to toggle replacing existing edited - notebooks on or off. + :param overwrite_existing: A bool to toggle replacing existing edited notebooks on or off. """ notebooks_package_data_root = pkg_resources.resource_filename("primaite", "notebooks/_package_data") for subdir, dirs, files in os.walk(notebooks_package_data_root): diff --git a/src/primaite/setup/reset_example_configs.py b/src/primaite/setup/reset_example_configs.py index 5d62298c..a2e1f2c9 100644 --- a/src/primaite/setup/reset_example_configs.py +++ b/src/primaite/setup/reset_example_configs.py @@ -11,11 +11,9 @@ _LOGGER = getLogger(__name__) def run(overwrite_existing=True): - """ - Resets the example config files in the users app config directory. + """Resets the example config files in the users app config directory. - :param overwrite_existing: A bool to toggle replacing existing edited - config on or off. + :param overwrite_existing: A bool to toggle replacing existing edited config on or off. """ configs_package_data_root = pkg_resources.resource_filename("primaite", "config/_package_data") diff --git a/src/primaite/setup/setup_app_dirs.py b/src/primaite/setup/setup_app_dirs.py index 693b11c1..bf7dbe59 100644 --- a/src/primaite/setup/setup_app_dirs.py +++ b/src/primaite/setup/setup_app_dirs.py @@ -5,8 +5,7 @@ _LOGGER = getLogger(__name__) def run(): - """ - Handles creation of application directories and user directories. + """Handles creation of application directories and user directories. Uses `platformdirs.PlatformDirs` and `pathlib.Path` to create the required app directories in the correct locations based on the users OS. diff --git a/src/primaite/transactions/transaction.py b/src/primaite/transactions/transaction.py index 7db2444a..21d4ee05 100644 --- a/src/primaite/transactions/transaction.py +++ b/src/primaite/transactions/transaction.py @@ -10,8 +10,7 @@ class Transaction(object): """Transaction class.""" def __init__(self, agent_identifier: AgentIdentifier, episode_number: int, step_number: int): - """ - Transaction constructor. + """Transaction constructor. :param agent_identifier: An identifier for the agent in use :param episode_number: The episode number @@ -39,8 +38,7 @@ class Transaction(object): "The env observation space description" def as_csv_data(self) -> Tuple[List, List]: - """ - Converts the Transaction to a csv data row and provides a header. + """Converts the Transaction to a csv data row and provides a header. :return: A tuple consisting of (header, data). """ @@ -69,8 +67,7 @@ class Transaction(object): def _turn_action_space_to_array(action_space) -> List[str]: - """ - Turns action space into a string array so it can be saved to csv. + """Turns action space into a string array so it can be saved to csv. :param action_space: The action space :return: The action space as an array of strings @@ -82,12 +79,10 @@ def _turn_action_space_to_array(action_space) -> List[str]: def _turn_obs_space_to_array(obs_space, obs_assets, obs_features) -> List[str]: - """ - Turns observation space into a string array so it can be saved to csv. + """Turns observation space into a string array so it can be saved to csv. :param obs_space: The observation space - :param obs_assets: The number of assets (i.e. nodes or links) in the - observation space + :param obs_assets: The number of assets (i.e. nodes or links) in the observation space :param obs_features: The number of features associated with the asset :return: The observation space as an array of strings """ diff --git a/src/primaite/utils/package_data.py b/src/primaite/utils/package_data.py index 59f36851..463a4309 100644 --- a/src/primaite/utils/package_data.py +++ b/src/primaite/utils/package_data.py @@ -10,8 +10,7 @@ _LOGGER = getLogger(__name__) def get_file_path(path: str) -> Path: - """ - Get PrimAITE package data. + """Get PrimAITE package data. :Example: diff --git a/src/primaite/utils/session_output_reader.py b/src/primaite/utils/session_output_reader.py index d04f375e..6b5cfdc3 100644 --- a/src/primaite/utils/session_output_reader.py +++ b/src/primaite/utils/session_output_reader.py @@ -7,11 +7,9 @@ import polars as pl def av_rewards_dict(av_rewards_csv_file: Union[str, Path]) -> Dict[int, float]: - """ - Read an average rewards per episode csv file and return as a dict. + """Read an average rewards per episode csv file and return as a dict. - The dictionary keys are the episode number, and the values are the mean - reward that episode. + The dictionary keys are the episode number, and the values are the mean reward that episode. :param av_rewards_csv_file: The average rewards per episode csv file path. :return: The average rewards per episode cdv as a dict. diff --git a/src/primaite/utils/session_output_writer.py b/src/primaite/utils/session_output_writer.py index a05b0453..939ebdb5 100644 --- a/src/primaite/utils/session_output_writer.py +++ b/src/primaite/utils/session_output_writer.py @@ -12,8 +12,7 @@ _LOGGER: Logger = getLogger(__name__) class SessionOutputWriter: - """ - A session output writer class. + """A session output writer class. Is used to write session outputs to csv file. """ @@ -65,11 +64,9 @@ class SessionOutputWriter: _LOGGER.debug(f"Finished writing file: {self._csv_file_path}") def write(self, data: Union[Tuple, Transaction]): - """ - Write a row of session data. + """Write a row of session data. - :param data: The row of data to write. Can be a Tuple or an instance - of Transaction. + :param data: The row of data to write. Can be a Tuple or an instance of Transaction. """ if isinstance(data, Transaction): header, data = data.as_csv_data() From d41e2ad59094abe1794f96dbcfa25f6d4ac86664 Mon Sep 17 00:00:00 2001 From: Marek Wolan Date: Tue, 4 Jul 2023 11:34:36 +0100 Subject: [PATCH 04/41] Resolve remaining build warnings for docs --- docs/api.rst | 16 ---------------- docs/index.rst | 1 + src/primaite/environment/observations.py | 8 ++++++-- 3 files changed, 7 insertions(+), 18 deletions(-) delete mode 100644 docs/api.rst diff --git a/docs/api.rst b/docs/api.rst deleted file mode 100644 index df2bc193..00000000 --- a/docs/api.rst +++ /dev/null @@ -1,16 +0,0 @@ -.. - DO NOT DELETE THIS FILE! It contains the all-important `.. autosummary::` directive with `:recursive:` option, without - which API documentation wouldn't get extracted from docstrings by the `sphinx.ext.autosummary` engine. It is hidden - (not declared in any toctree) to remove an unnecessary intermediate page; index.rst instead points directly to the - package page. DO NOT REMOVE THIS FILE! - - Credit to https://github.com/JamesALeedham/Sphinx-Autosummary-Recursion for the custom templates. -.. - -.. autosummary:: - :toctree: source/_autosummary - :template: custom-module-template.rst - :recursive: - - primaite - tests diff --git a/docs/index.rst b/docs/index.rst index 17dae2c9..72e91b07 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -43,6 +43,7 @@ The best place to start is :ref:`about` :caption: Project Links: :hidden: +.. #Code <> #Issues <> #Pull Requests <> diff --git a/src/primaite/environment/observations.py b/src/primaite/environment/observations.py index e347a65c..4d027326 100644 --- a/src/primaite/environment/observations.py +++ b/src/primaite/environment/observations.py @@ -49,6 +49,7 @@ class NodeLinkTable(AbstractObservationComponent): This will create the observation space formatted as a table of integers. There is one row per node, followed by one row per link. The number of columns is 4 plus one per service. They are: + * node/link ID * node hardware status / 0 for links * node operating system status (if active/service) / 0 for links @@ -175,6 +176,7 @@ class NodeStatuses(AbstractObservationComponent): integers. Each node has 3 elements plus 1 per service. It will have the following structure: .. code-block:: + [ node1 hardware state, node1 OS state, @@ -275,6 +277,7 @@ class LinkTrafficLevels(AbstractObservationComponent): For each link, total traffic or traffic per service is encoded into a categorical value. For example, if ``quantisation_levels=5``, the traffic levels represent these values: + * 0 = No traffic (0% of bandwidth) * 1 = No traffic (0%-33% of bandwidth) * 2 = No traffic (33%-66% of bandwidth) @@ -426,7 +429,7 @@ class ObservationsHandler: """Remove a component from this handler. :param obs_component: Which component to remove. It must exist within this object's - ``registered_obs_components`` attribute. + ``registered_obs_components`` attribute. :type obs_component: AbstractObservationComponent """ self.registered_obs_components.remove(obs_component) @@ -470,7 +473,8 @@ class ObservationsHandler: The expected format for the config dictionary is: - ..code-block::python + .. code-block:: python + config = { components: [ { From 7bdcee5c46f3f1a80db1c1eb572481c69be27005 Mon Sep 17 00:00:00 2001 From: Marek Wolan Date: Tue, 4 Jul 2023 11:57:10 +0100 Subject: [PATCH 05/41] remove primaite dependencies as it's autogenerated --- .gitignore | 1 + docs/_static/.gitkeep | 0 docs/source/primaite-dependencies.rst | 429 -------------------------- src/primaite/agents/agent.py | 2 +- 4 files changed, 2 insertions(+), 430 deletions(-) create mode 100644 docs/_static/.gitkeep delete mode 100644 docs/source/primaite-dependencies.rst diff --git a/.gitignore b/.gitignore index b65d1fd8..f223391e 100644 --- a/.gitignore +++ b/.gitignore @@ -139,3 +139,4 @@ dmypy.json cython_debug/ .idea/ +docs/source/primaite-dependencies.rst diff --git a/docs/_static/.gitkeep b/docs/_static/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/docs/source/primaite-dependencies.rst b/docs/source/primaite-dependencies.rst deleted file mode 100644 index 48f835fe..00000000 --- a/docs/source/primaite-dependencies.rst +++ /dev/null @@ -1,429 +0,0 @@ -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| Name | Version | License | URL | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| Babel | 2.12.1 | BSD License | https://babel.pocoo.org/ | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| GPUtil | 1.4.0 | MIT | https://github.com/anderskm/gputil | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| Gymnasium | 0.26.3 | MIT | https://gymnasium.farama.org/ | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| Jinja2 | 3.1.2 | BSD License | https://palletsprojects.com/p/jinja/ | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| Markdown | 3.4.3 | BSD License | https://Python-Markdown.github.io/ | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| MarkupSafe | 2.1.2 | BSD License | https://palletsprojects.com/p/markupsafe/ | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| Pillow | 9.5.0 | Historical Permission Notice and Disclaimer (HPND) | https://python-pillow.org | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| PyWavelets | 1.4.1 | MIT License | https://github.com/PyWavelets/pywt | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| PyYAML | 6.0 | MIT License | https://pyyaml.org/ | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| Pygments | 2.15.1 | BSD License | https://pygments.org | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| Send2Trash | 1.8.2 | BSD License | https://github.com/arsenetar/send2trash | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| Sphinx | 6.1.3 | BSD License | https://www.sphinx-doc.org/ | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| Werkzeug | 2.3.4 | BSD License | UNKNOWN | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| absl-py | 1.4.0 | Apache Software License | https://github.com/abseil/abseil-py | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| aiofiles | 22.1.0 | Apache Software License | https://github.com/Tinche/aiofiles | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| aiosignal | 1.3.1 | Apache Software License | https://github.com/aio-libs/aiosignal | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| aiosqlite | 0.19.0 | MIT License | UNKNOWN | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| alabaster | 0.7.13 | BSD License | https://alabaster.readthedocs.io | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| anyio | 3.7.0 | MIT License | https://anyio.readthedocs.io/en/stable/versionhistory.html | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| argon2-cffi | 21.3.0 | MIT License | https://github.com/hynek/argon2-cffi/blob/main/CHANGELOG.md | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| argon2-cffi-bindings | 21.2.0 | MIT License | https://github.com/hynek/argon2-cffi-bindings | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| arrow | 1.2.3 | Apache Software License | https://arrow.readthedocs.io | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| asttokens | 2.2.1 | Apache 2.0 | https://github.com/gristlabs/asttokens | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| astunparse | 1.6.3 | BSD License | https://github.com/simonpercivall/astunparse | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| attrs | 23.1.0 | MIT License | https://www.attrs.org/en/stable/changelog.html | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| backcall | 0.2.0 | BSD License | https://github.com/takluyver/backcall | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| beautifulsoup4 | 4.12.2 | MIT License | https://www.crummy.com/software/BeautifulSoup/bs4/ | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| bleach | 6.0.0 | Apache Software License | https://github.com/mozilla/bleach | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| build | 0.10.0 | MIT License | UNKNOWN | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| cachetools | 5.3.0 | MIT License | https://github.com/tkem/cachetools/ | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| certifi | 2023.5.7 | Mozilla Public License 2.0 (MPL 2.0) | https://github.com/certifi/python-certifi | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| cffi | 1.15.1 | MIT License | http://cffi.readthedocs.org | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| cfgv | 3.3.1 | MIT License | https://github.com/asottile/cfgv | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| charset-normalizer | 3.1.0 | MIT License | https://github.com/Ousret/charset_normalizer | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| click | 8.1.3 | BSD License | https://palletsprojects.com/p/click/ | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| cloudpickle | 2.2.1 | BSD License | https://github.com/cloudpipe/cloudpickle | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| colorama | 0.4.6 | BSD License | https://github.com/tartley/colorama | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| comm | 0.1.3 | BSD License | https://github.com/ipython/comm | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| contourpy | 1.0.7 | BSD License | UNKNOWN | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| coverage | 7.2.6 | Apache Software License | https://github.com/nedbat/coveragepy | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| cycler | 0.11.0 | BSD License | https://github.com/matplotlib/cycler | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| debugpy | 1.6.7 | Eclipse Public License 2.0 (EPL-2.0); MIT License | https://aka.ms/debugpy | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| decorator | 5.1.1 | BSD License | https://github.com/micheles/decorator | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| defusedxml | 0.7.1 | Python Software Foundation License | https://github.com/tiran/defusedxml | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| distlib | 0.3.6 | Python Software Foundation License | https://github.com/pypa/distlib | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| dm-tree | 0.1.8 | Apache Software License | https://github.com/deepmind/tree | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| docutils | 0.19 | BSD License; GNU General Public License (GPL); Public Domain; Python Software Foundation License | https://docutils.sourceforge.io/ | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| exceptiongroup | 1.1.1 | MIT License | https://github.com/agronholm/exceptiongroup/blob/main/CHANGES.rst | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| executing | 1.2.0 | MIT License | https://github.com/alexmojaki/executing | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| fastjsonschema | 2.17.1 | BSD License | https://github.com/horejsek/python-fastjsonschema | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| filelock | 3.12.0 | The Unlicense (Unlicense) | https://github.com/tox-dev/py-filelock | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| flake8 | 6.0.0 | MIT License | https://github.com/pycqa/flake8 | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| flatbuffers | 23.5.26 | Apache Software License | https://google.github.io/flatbuffers/ | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| fonttools | 4.39.4 | MIT License | http://github.com/fonttools/fonttools | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| fqdn | 1.5.1 | Mozilla Public License 2.0 (MPL 2.0) | https://github.com/ypcrts/fqdn | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| frozenlist | 1.3.3 | Apache Software License | https://github.com/aio-libs/frozenlist | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| furo | 2023.3.27 | MIT License | UNKNOWN | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| gast | 0.4.0 | BSD License | https://github.com/serge-sans-paille/gast/ | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| google-auth | 2.19.0 | Apache Software License | https://github.com/googleapis/google-auth-library-python | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| google-auth-oauthlib | 0.4.6 | Apache Software License | https://github.com/GoogleCloudPlatform/google-auth-library-python-oauthlib | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| google-pasta | 0.2.0 | Apache Software License | https://github.com/google/pasta | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| grpcio | 1.51.3 | Apache Software License | https://grpc.io | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| gym | 0.21.0 | UNKNOWN | https://github.com/openai/gym | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| gymnasium-notices | 0.0.1 | MIT License | https://github.com/Farama-Foundation/gym-notices | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| h5py | 3.9.0 | BSD License | https://www.h5py.org/ | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| identify | 2.5.24 | MIT License | https://github.com/pre-commit/identify | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| idna | 3.4 | BSD License | https://github.com/kjd/idna | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| imageio | 2.29.0 | BSD License | https://github.com/imageio/imageio | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| imagesize | 1.4.1 | MIT License | https://github.com/shibukawa/imagesize_py | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| importlib-metadata | 4.13.0 | Apache Software License | https://github.com/python/importlib_metadata | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| iniconfig | 2.0.0 | MIT License | https://github.com/pytest-dev/iniconfig | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| ipykernel | 6.23.1 | BSD License | https://ipython.org | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| ipython | 8.13.2 | BSD License | https://ipython.org | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| ipython-genutils | 0.2.0 | BSD License | http://ipython.org | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| isoduration | 20.11.0 | ISC License (ISCL) | https://github.com/bolsote/isoduration | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| jax | 0.4.12 | Apache-2.0 | https://github.com/google/jax | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| jedi | 0.18.2 | MIT License | https://github.com/davidhalter/jedi | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| json5 | 0.9.14 | Apache Software License | https://github.com/dpranke/pyjson5 | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| jsonpointer | 2.3 | BSD License | https://github.com/stefankoegl/python-json-pointer | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| jsonschema | 4.17.3 | MIT License | https://github.com/python-jsonschema/jsonschema | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| jupyter-events | 0.6.3 | BSD License | http://jupyter.org | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| jupyter-server | 1.24.0 | BSD License | https://jupyter-server.readthedocs.io | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| jupyter-ydoc | 0.2.4 | BSD 3-Clause License | https://jupyter.org | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| jupyter_client | 8.2.0 | BSD License | https://jupyter.org | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| jupyter_core | 5.3.0 | BSD License | https://jupyter.org | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| jupyter_server_fileid | 0.9.0 | BSD License | UNKNOWN | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| jupyter_server_terminals | 0.4.4 | BSD License | https://jupyter.org | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| jupyter_server_ydoc | 0.6.1 | BSD License | https://jupyter.org | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| jupyterlab | 3.6.1 | BSD License | https://jupyter.org | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| jupyterlab-pygments | 0.2.2 | BSD | https://github.com/jupyterlab/jupyterlab_pygments | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| jupyterlab_server | 2.22.1 | BSD License | https://jupyterlab-server.readthedocs.io | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| keras | 2.12.0 | Apache Software License | https://keras.io/ | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| kiwisolver | 1.4.4 | BSD License | UNKNOWN | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| lazy_loader | 0.2 | BSD License | https://github.com/scientific-python/lazy_loader | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| libclang | 16.0.0 | Apache Software License | https://github.com/sighingnow/libclang | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| lz4 | 4.3.2 | BSD License | https://github.com/python-lz4/python-lz4 | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| markdown-it-py | 2.2.0 | MIT License | https://github.com/executablebooks/markdown-it-py | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| matplotlib | 3.7.1 | Python Software Foundation License | https://matplotlib.org | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| matplotlib-inline | 0.1.6 | BSD 3-Clause | https://github.com/ipython/matplotlib-inline | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| mavizstyle | 1.0.0 | UNKNOWN | UNKNOWN | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| mccabe | 0.7.0 | MIT License | https://github.com/pycqa/mccabe | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| mdurl | 0.1.2 | MIT License | https://github.com/executablebooks/mdurl | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| mistune | 2.0.5 | BSD License | https://github.com/lepture/mistune | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| ml-dtypes | 0.2.0 | Apache Software License | UNKNOWN | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| mock | 5.0.2 | BSD License | http://mock.readthedocs.org/en/latest/ | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| mpmath | 1.3.0 | BSD License | http://mpmath.org/ | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| msgpack | 1.0.5 | Apache Software License | https://msgpack.org/ | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| nbclassic | 0.5.6 | BSD License | https://github.com/jupyter/nbclassic | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| nbclient | 0.8.0 | BSD License | https://jupyter.org | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| nbconvert | 7.4.0 | BSD License | https://jupyter.org | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| nbformat | 5.9.0 | BSD License | https://jupyter.org | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| nest-asyncio | 1.5.6 | BSD License | https://github.com/erdewit/nest_asyncio | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| networkx | 3.1 | BSD License | https://networkx.org/ | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| nodeenv | 1.8.0 | BSD License | https://github.com/ekalinin/nodeenv | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| notebook | 6.5.4 | BSD License | http://jupyter.org | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| notebook_shim | 0.2.3 | BSD License | UNKNOWN | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| numpy | 1.23.5 | BSD License | https://www.numpy.org | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| oauthlib | 3.2.2 | BSD License | https://github.com/oauthlib/oauthlib | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| opt-einsum | 3.3.0 | MIT | https://github.com/dgasmith/opt_einsum | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| overrides | 7.3.1 | Apache License, Version 2.0 | https://github.com/mkorpela/overrides | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| packaging | 23.1 | Apache Software License; BSD License | https://github.com/pypa/packaging | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| pandas | 2.0.1 | BSD License | UNKNOWN | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| pandocfilters | 1.5.0 | BSD License | http://github.com/jgm/pandocfilters | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| parso | 0.8.3 | MIT License | https://github.com/davidhalter/parso | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| pickleshare | 0.7.5 | MIT License | https://github.com/pickleshare/pickleshare | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| platformdirs | 3.5.1 | MIT License | https://github.com/platformdirs/platformdirs | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| plotly | 5.15.0 | MIT License | https://plotly.com/python/ | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| pluggy | 1.0.0 | MIT License | https://github.com/pytest-dev/pluggy | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| pre-commit | 2.20.0 | MIT License | https://github.com/pre-commit/pre-commit | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| primaite | 2.0.0rc1 | GFX | UNKNOWN | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| primaite | 2.0.0rc1 | GFX | UNKNOWN | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| primaite | 2.0.0rc1 | GFX | UNKNOWN | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| prometheus-client | 0.17.0 | Apache Software License | https://github.com/prometheus/client_python | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| prompt-toolkit | 3.0.38 | BSD License | https://github.com/prompt-toolkit/python-prompt-toolkit | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| protobuf | 3.20.3 | BSD-3-Clause | https://developers.google.com/protocol-buffers/ | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| psutil | 5.9.5 | BSD License | https://github.com/giampaolo/psutil | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| pure-eval | 0.2.2 | MIT License | http://github.com/alexmojaki/pure_eval | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| pyasn1 | 0.5.0 | BSD License | https://github.com/pyasn1/pyasn1 | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| pyasn1-modules | 0.3.0 | BSD License | https://github.com/pyasn1/pyasn1-modules | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| pycodestyle | 2.10.0 | MIT License | https://pycodestyle.pycqa.org/ | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| pycparser | 2.21 | BSD License | https://github.com/eliben/pycparser | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| pyflakes | 3.0.1 | MIT License | https://github.com/PyCQA/pyflakes | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| pyparsing | 3.0.9 | MIT License | https://github.com/pyparsing/pyparsing/ | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| pyproject_hooks | 1.0.0 | MIT License | https://github.com/pypa/pyproject-hooks | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| pyrsistent | 0.19.3 | MIT License | https://github.com/tobgu/pyrsistent/ | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| pytest | 7.2.0 | MIT License | https://docs.pytest.org/en/latest/ | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| pytest-cov | 4.0.0 | MIT License | https://github.com/pytest-dev/pytest-cov | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| pytest-flake8 | 1.1.1 | BSD License | https://github.com/tholo/pytest-flake8 | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| python-dateutil | 2.8.2 | Apache Software License; BSD License | https://github.com/dateutil/dateutil | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| python-json-logger | 2.0.7 | BSD License | http://github.com/madzak/python-json-logger | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| pytz | 2023.3 | MIT License | http://pythonhosted.org/pytz | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| pywin32 | 306 | Python Software Foundation License | https://github.com/mhammond/pywin32 | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| pywinpty | 2.0.10 | MIT | UNKNOWN | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| pyzmq | 25.1.0 | BSD License; GNU Library or Lesser General Public License (LGPL) | https://pyzmq.readthedocs.org | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| ray | 2.2.0 | Apache 2.0 | https://github.com/ray-project/ray | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| requests | 2.31.0 | Apache Software License | https://requests.readthedocs.io | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| requests-oauthlib | 1.3.1 | BSD License | https://github.com/requests/requests-oauthlib | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| rfc3339-validator | 0.1.4 | MIT License | https://github.com/naimetti/rfc3339-validator | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| rfc3986-validator | 0.1.1 | MIT License | https://github.com/naimetti/rfc3986-validator | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| rich | 13.3.5 | MIT License | https://github.com/Textualize/rich | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| rsa | 4.9 | Apache Software License | https://stuvel.eu/rsa | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| ruff | 0.0.272 | MIT License | https://github.com/charliermarsh/ruff | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| scikit-image | 0.20.0 | BSD License | https://scikit-image.org | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| scipy | 1.10.1 | BSD License | https://scipy.org/ | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| shellingham | 1.5.0.post1 | ISC License (ISCL) | https://github.com/sarugaku/shellingham | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| six | 1.16.0 | MIT License | https://github.com/benjaminp/six | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| sniffio | 1.3.0 | Apache Software License; MIT License | https://github.com/python-trio/sniffio | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| snowballstemmer | 2.2.0 | BSD License | https://github.com/snowballstem/snowball | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| soupsieve | 2.4.1 | MIT License | https://github.com/facelessuser/soupsieve | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| sphinx-basic-ng | 1.0.0b1 | MIT License | https://github.com/pradyunsg/sphinx-basic-ng | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| sphinx-code-tabs | 0.5.3 | The Unlicense (Unlicense) | https://github.com/coldfix/sphinx-code-tabs | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| sphinx-copybutton | 0.5.2 | MIT License | https://github.com/executablebooks/sphinx-copybutton | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| sphinxcontrib-applehelp | 1.0.4 | BSD License | https://www.sphinx-doc.org/ | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| sphinxcontrib-devhelp | 1.0.2 | BSD License | http://sphinx-doc.org/ | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| sphinxcontrib-htmlhelp | 2.0.1 | BSD License | https://www.sphinx-doc.org/ | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| sphinxcontrib-jsmath | 1.0.1 | BSD License | http://sphinx-doc.org/ | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| sphinxcontrib-qthelp | 1.0.3 | BSD License | http://sphinx-doc.org/ | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| sphinxcontrib-serializinghtml | 1.1.5 | BSD License | http://sphinx-doc.org/ | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| stable-baselines3 | 1.6.2 | MIT | https://github.com/DLR-RM/stable-baselines3 | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| stack-data | 0.6.2 | MIT License | http://github.com/alexmojaki/stack_data | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| sympy | 1.12 | BSD License | https://sympy.org | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| tabulate | 0.9.0 | MIT License | https://github.com/astanin/python-tabulate | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| tenacity | 8.2.2 | Apache Software License | https://github.com/jd/tenacity | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| tensorboard | 2.11.2 | Apache Software License | https://github.com/tensorflow/tensorboard | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| tensorboard-data-server | 0.6.1 | Apache Software License | https://github.com/tensorflow/tensorboard/tree/master/tensorboard/data/server | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| tensorboard-plugin-wit | 1.8.1 | Apache 2.0 | https://whatif-tool.dev | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| tensorboardX | 2.6 | MIT License | https://github.com/lanpa/tensorboardX | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| tensorflow | 2.12.0 | Apache Software License | https://www.tensorflow.org/ | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| tensorflow-estimator | 2.12.0 | Apache Software License | https://www.tensorflow.org/ | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| tensorflow-intel | 2.12.0 | Apache Software License | https://www.tensorflow.org/ | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| tensorflow-io-gcs-filesystem | 0.31.0 | Apache Software License | https://github.com/tensorflow/io | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| termcolor | 2.3.0 | MIT License | https://github.com/termcolor/termcolor | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| terminado | 0.17.1 | BSD License | https://github.com/jupyter/terminado | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| tifffile | 2023.4.12 | BSD License | https://www.cgohlke.com | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| tinycss2 | 1.2.1 | BSD License | https://www.courtbouillon.org/tinycss2 | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| toml | 0.10.2 | MIT License | https://github.com/uiri/toml | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| tomli | 2.0.1 | MIT License | https://github.com/hukkin/tomli | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| torch | 2.0.1 | BSD License | https://pytorch.org/ | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| tornado | 6.3.2 | Apache Software License | http://www.tornadoweb.org/ | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| traitlets | 5.9.0 | BSD License | https://github.com/ipython/traitlets | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| typer | 0.9.0 | MIT License | https://github.com/tiangolo/typer | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| typing_extensions | 4.6.2 | Python Software Foundation License | https://github.com/python/typing_extensions/issues | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| tzdata | 2023.3 | Apache Software License | https://github.com/python/tzdata | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| uri-template | 1.2.0 | MIT License | https://github.com/plinss/uri_template/ | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| urllib3 | 1.26.16 | MIT License | https://urllib3.readthedocs.io/ | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| virtualenv | 20.21.0 | MIT License | https://github.com/pypa/virtualenv | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| webcolors | 1.13 | BSD License | UNKNOWN | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| webencodings | 0.5.1 | BSD License | https://github.com/SimonSapin/python-webencodings | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| websocket-client | 1.5.2 | Apache Software License | https://github.com/websocket-client/websocket-client.git | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| wrapt | 1.14.1 | BSD License | https://github.com/GrahamDumpleton/wrapt | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| y-py | 0.5.9 | MIT License | https://github.com/y-crdt/ypy | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| ypy-websocket | 0.8.2 | UNKNOWN | https://github.com/y-crdt/ypy-websocket | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| zipp | 3.15.0 | MIT License | https://github.com/jaraco/zipp | -+-------------------------------+-------------+--------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ diff --git a/src/primaite/agents/agent.py b/src/primaite/agents/agent.py index a43f2d0b..c68b6df0 100644 --- a/src/primaite/agents/agent.py +++ b/src/primaite/agents/agent.py @@ -53,7 +53,7 @@ class AgentSessionABC(ABC): if not isinstance(lay_down_config_path, Path): lay_down_config_path = Path(lay_down_config_path) - self._lay_down_config_path: Final[Union[Path]] = lay_down_config_path + self._lay_down_config_path: Final[Union[Path, str]] = lay_down_config_path self._lay_down_config: Dict = lay_down_config.load(self._lay_down_config_path) self.sb3_output_verbose_level = self._training_config.sb3_output_verbose_level From 0756e61e5d1cc6d0021041cafb6b0922b4a374ed Mon Sep 17 00:00:00 2001 From: Marek Wolan Date: Tue, 4 Jul 2023 13:11:06 +0100 Subject: [PATCH 06/41] add module level docstrings --- src/primaite/acl/__init__.py | 1 + src/primaite/agents/__init__.py | 1 + src/primaite/common/__init__.py | 1 + src/primaite/data_viz/__init__.py | 2 +- src/primaite/environment/__init__.py | 1 + src/primaite/links/__init__.py | 1 + src/primaite/nodes/__init__.py | 1 + src/primaite/notebooks/__init__.py | 1 + src/primaite/pol/__init__.py | 1 + src/primaite/primaite_session.py | 5 +++-- src/primaite/setup/__init__.py | 1 + src/primaite/transactions/__init__.py | 1 + src/primaite/utils/__init__.py | 1 + 13 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/primaite/acl/__init__.py b/src/primaite/acl/__init__.py index 63f825c2..434e5899 100644 --- a/src/primaite/acl/__init__.py +++ b/src/primaite/acl/__init__.py @@ -1 +1,2 @@ # Crown Copyright (C) Dstl 2022. DEFCON 703. Shared in confidence. +"""Access Control List. Models firewall functionality.""" \ No newline at end of file diff --git a/src/primaite/agents/__init__.py b/src/primaite/agents/__init__.py index e69de29b..90d719b4 100644 --- a/src/primaite/agents/__init__.py +++ b/src/primaite/agents/__init__.py @@ -0,0 +1 @@ +"""Common interface between RL agents from different libraries and PrimAITE.""" \ No newline at end of file diff --git a/src/primaite/common/__init__.py b/src/primaite/common/__init__.py index 63f825c2..e79054cf 100644 --- a/src/primaite/common/__init__.py +++ b/src/primaite/common/__init__.py @@ -1 +1,2 @@ # Crown Copyright (C) Dstl 2022. DEFCON 703. Shared in confidence. +"""Objects which are shared between many PrimAITE modules.""" \ No newline at end of file diff --git a/src/primaite/data_viz/__init__.py b/src/primaite/data_viz/__init__.py index a7cc3e8b..ca28900e 100644 --- a/src/primaite/data_viz/__init__.py +++ b/src/primaite/data_viz/__init__.py @@ -1,5 +1,5 @@ from enum import Enum - +"""Utility to generate plots of sessions metrics after PrimAITE.""" class PlotlyTemplate(Enum): """The built-in plotly templates.""" diff --git a/src/primaite/environment/__init__.py b/src/primaite/environment/__init__.py index 63f825c2..6f82aec1 100644 --- a/src/primaite/environment/__init__.py +++ b/src/primaite/environment/__init__.py @@ -1 +1,2 @@ # Crown Copyright (C) Dstl 2022. DEFCON 703. Shared in confidence. +"""Gym/Gymnasium environment for RL agents consisting of a simulated computer network.""" \ No newline at end of file diff --git a/src/primaite/links/__init__.py b/src/primaite/links/__init__.py index 63f825c2..919c35ac 100644 --- a/src/primaite/links/__init__.py +++ b/src/primaite/links/__init__.py @@ -1 +1,2 @@ # Crown Copyright (C) Dstl 2022. DEFCON 703. Shared in confidence. +"""Network connections between nodes in the simulation.""" \ No newline at end of file diff --git a/src/primaite/nodes/__init__.py b/src/primaite/nodes/__init__.py index 63f825c2..5db9f9b5 100644 --- a/src/primaite/nodes/__init__.py +++ b/src/primaite/nodes/__init__.py @@ -1 +1,2 @@ # Crown Copyright (C) Dstl 2022. DEFCON 703. Shared in confidence. +"""Nodes represent network hosts in the simulation.""" \ No newline at end of file diff --git a/src/primaite/notebooks/__init__.py b/src/primaite/notebooks/__init__.py index 0730312e..da65da38 100644 --- a/src/primaite/notebooks/__init__.py +++ b/src/primaite/notebooks/__init__.py @@ -1,3 +1,4 @@ +"""Contains default jupyter notebooks which demonstrate PrimAITE functionality.""" # Crown Copyright (C) Dstl 2022. DEFCON 703. Shared in confidence. import importlib.util import os diff --git a/src/primaite/pol/__init__.py b/src/primaite/pol/__init__.py index 63f825c2..cba4b28b 100644 --- a/src/primaite/pol/__init__.py +++ b/src/primaite/pol/__init__.py @@ -1 +1,2 @@ +"""Pattern of Life- Represents the actions of users on the network.""" # Crown Copyright (C) Dstl 2022. DEFCON 703. Shared in confidence. diff --git a/src/primaite/primaite_session.py b/src/primaite/primaite_session.py index ed2b9bf1..4aa8476a 100644 --- a/src/primaite/primaite_session.py +++ b/src/primaite/primaite_session.py @@ -1,3 +1,4 @@ +"""Main entry point to PrimAITE. Configure training/evaluation experiments and input/output.""" from __future__ import annotations from pathlib import Path @@ -35,12 +36,12 @@ class PrimaiteSession: """ if not isinstance(training_config_path, Path): training_config_path = Path(training_config_path) - self._training_config_path: Final[Union[Path]] = training_config_path + self._training_config_path: Final[Union[Path, str]] = training_config_path self._training_config: Final[TrainingConfig] = training_config.load(self._training_config_path) if not isinstance(lay_down_config_path, Path): lay_down_config_path = Path(lay_down_config_path) - self._lay_down_config_path: Final[Union[Path]] = lay_down_config_path + self._lay_down_config_path: Final[Union[Path, str]] = lay_down_config_path self._lay_down_config: Dict = lay_down_config.load(self._lay_down_config_path) self._agent_session: AgentSessionABC = None # noqa diff --git a/src/primaite/setup/__init__.py b/src/primaite/setup/__init__.py index 63f825c2..3c0bfe14 100644 --- a/src/primaite/setup/__init__.py +++ b/src/primaite/setup/__init__.py @@ -1 +1,2 @@ +"""Utilities to prepare the user's data folders.""" # Crown Copyright (C) Dstl 2022. DEFCON 703. Shared in confidence. diff --git a/src/primaite/transactions/__init__.py b/src/primaite/transactions/__init__.py index 63f825c2..45315b22 100644 --- a/src/primaite/transactions/__init__.py +++ b/src/primaite/transactions/__init__.py @@ -1 +1,2 @@ +"""Record data of the system's state and agent's observations and actions.""" # Crown Copyright (C) Dstl 2022. DEFCON 703. Shared in confidence. diff --git a/src/primaite/utils/__init__.py b/src/primaite/utils/__init__.py index e69de29b..878f1d75 100644 --- a/src/primaite/utils/__init__.py +++ b/src/primaite/utils/__init__.py @@ -0,0 +1 @@ +"""Utilities for PrimAITE.""" \ No newline at end of file From eac79e09419394e7d640ff03475765e628bbe6c3 Mon Sep 17 00:00:00 2001 From: Marek Wolan Date: Wed, 5 Jul 2023 09:19:58 +0100 Subject: [PATCH 07/41] Add missing module level docstrings. --- src/primaite/config/__init__.py | 1 + src/primaite/data_viz/__init__.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/primaite/config/__init__.py b/src/primaite/config/__init__.py index e69de29b..185ae069 100644 --- a/src/primaite/config/__init__.py +++ b/src/primaite/config/__init__.py @@ -0,0 +1 @@ +"""Configuration parameters for running experiments.""" \ No newline at end of file diff --git a/src/primaite/data_viz/__init__.py b/src/primaite/data_viz/__init__.py index ca28900e..78aa423a 100644 --- a/src/primaite/data_viz/__init__.py +++ b/src/primaite/data_viz/__init__.py @@ -1,5 +1,5 @@ -from enum import Enum """Utility to generate plots of sessions metrics after PrimAITE.""" +from enum import Enum class PlotlyTemplate(Enum): """The built-in plotly templates.""" From cda9819e72928925803ef903f334eddfaf9c3759 Mon Sep 17 00:00:00 2001 From: Marek Wolan Date: Wed, 5 Jul 2023 09:22:49 +0100 Subject: [PATCH 08/41] Add blank lines at the end of file. --- src/primaite/acl/__init__.py | 2 +- src/primaite/agents/__init__.py | 2 +- src/primaite/common/__init__.py | 2 +- src/primaite/config/__init__.py | 2 +- src/primaite/data_viz/__init__.py | 1 + src/primaite/environment/__init__.py | 2 +- src/primaite/links/__init__.py | 2 +- src/primaite/nodes/__init__.py | 2 +- src/primaite/utils/__init__.py | 2 +- 9 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/primaite/acl/__init__.py b/src/primaite/acl/__init__.py index 434e5899..2623efbc 100644 --- a/src/primaite/acl/__init__.py +++ b/src/primaite/acl/__init__.py @@ -1,2 +1,2 @@ # Crown Copyright (C) Dstl 2022. DEFCON 703. Shared in confidence. -"""Access Control List. Models firewall functionality.""" \ No newline at end of file +"""Access Control List. Models firewall functionality.""" diff --git a/src/primaite/agents/__init__.py b/src/primaite/agents/__init__.py index 90d719b4..89580145 100644 --- a/src/primaite/agents/__init__.py +++ b/src/primaite/agents/__init__.py @@ -1 +1 @@ -"""Common interface between RL agents from different libraries and PrimAITE.""" \ No newline at end of file +"""Common interface between RL agents from different libraries and PrimAITE.""" diff --git a/src/primaite/common/__init__.py b/src/primaite/common/__init__.py index e79054cf..5f47b0b5 100644 --- a/src/primaite/common/__init__.py +++ b/src/primaite/common/__init__.py @@ -1,2 +1,2 @@ # Crown Copyright (C) Dstl 2022. DEFCON 703. Shared in confidence. -"""Objects which are shared between many PrimAITE modules.""" \ No newline at end of file +"""Objects which are shared between many PrimAITE modules.""" diff --git a/src/primaite/config/__init__.py b/src/primaite/config/__init__.py index 185ae069..03ed4cf1 100644 --- a/src/primaite/config/__init__.py +++ b/src/primaite/config/__init__.py @@ -1 +1 @@ -"""Configuration parameters for running experiments.""" \ No newline at end of file +"""Configuration parameters for running experiments.""" diff --git a/src/primaite/data_viz/__init__.py b/src/primaite/data_viz/__init__.py index 78aa423a..db6ce6c8 100644 --- a/src/primaite/data_viz/__init__.py +++ b/src/primaite/data_viz/__init__.py @@ -1,6 +1,7 @@ """Utility to generate plots of sessions metrics after PrimAITE.""" from enum import Enum + class PlotlyTemplate(Enum): """The built-in plotly templates.""" diff --git a/src/primaite/environment/__init__.py b/src/primaite/environment/__init__.py index 6f82aec1..8b0060c0 100644 --- a/src/primaite/environment/__init__.py +++ b/src/primaite/environment/__init__.py @@ -1,2 +1,2 @@ # Crown Copyright (C) Dstl 2022. DEFCON 703. Shared in confidence. -"""Gym/Gymnasium environment for RL agents consisting of a simulated computer network.""" \ No newline at end of file +"""Gym/Gymnasium environment for RL agents consisting of a simulated computer network.""" diff --git a/src/primaite/links/__init__.py b/src/primaite/links/__init__.py index 919c35ac..6257f282 100644 --- a/src/primaite/links/__init__.py +++ b/src/primaite/links/__init__.py @@ -1,2 +1,2 @@ # Crown Copyright (C) Dstl 2022. DEFCON 703. Shared in confidence. -"""Network connections between nodes in the simulation.""" \ No newline at end of file +"""Network connections between nodes in the simulation.""" diff --git a/src/primaite/nodes/__init__.py b/src/primaite/nodes/__init__.py index 5db9f9b5..19347372 100644 --- a/src/primaite/nodes/__init__.py +++ b/src/primaite/nodes/__init__.py @@ -1,2 +1,2 @@ # Crown Copyright (C) Dstl 2022. DEFCON 703. Shared in confidence. -"""Nodes represent network hosts in the simulation.""" \ No newline at end of file +"""Nodes represent network hosts in the simulation.""" diff --git a/src/primaite/utils/__init__.py b/src/primaite/utils/__init__.py index 878f1d75..55e8a6ba 100644 --- a/src/primaite/utils/__init__.py +++ b/src/primaite/utils/__init__.py @@ -1 +1 @@ -"""Utilities for PrimAITE.""" \ No newline at end of file +"""Utilities for PrimAITE.""" From 3ced1a19137ae40d287b8b785cf6d2aa0b5af70d Mon Sep 17 00:00:00 2001 From: Marek Wolan Date: Wed, 5 Jul 2023 09:54:50 +0100 Subject: [PATCH 09/41] Update some param descriptions for hardcoded agent --- src/primaite/agents/hardcoded_acl.py | 60 ++++++++++++++++++++-------- 1 file changed, 43 insertions(+), 17 deletions(-) diff --git a/src/primaite/agents/hardcoded_acl.py b/src/primaite/agents/hardcoded_acl.py index 263ccbdc..430c8b54 100644 --- a/src/primaite/agents/hardcoded_acl.py +++ b/src/primaite/agents/hardcoded_acl.py @@ -1,4 +1,6 @@ +from typing import Any, Dict import numpy as np +from primaite.acl.access_control_list import AccessControlList from primaite.agents.agent import HardCodedAgentSessionABC from primaite.agents.utils import ( @@ -7,7 +9,9 @@ from primaite.agents.utils import ( transform_action_acl_enum, transform_change_obs_readable, ) +from primaite.common.custom_typing import NodeUnion from primaite.common.enums import HardCodedAgentView +from primaite.pol.ier import IER class HardCodedACLAgent(HardCodedAgentSessionABC): @@ -22,12 +26,17 @@ class HardCodedACLAgent(HardCodedAgentSessionABC): # history and reward feedback return self._calculate_action_full_view(obs) - def get_blocked_green_iers(self, green_iers, acl, nodes): - """ - Get blocked green IERs. + def get_blocked_green_iers(self, green_iers:Dict[str, IER], acl:AccessControlList, nodes:Dict[str,NodeUnion]): + """Get blocked green IERs. - TODO: Add params and return in docstring. - TODO: Typehint params and return. + :param green_iers: Green IERs to check for being + :type green_iers: Dict[str, IER] + :param acl: Firewall rules + :type acl: AccessControlList + :param nodes: Nodes in the network + :type nodes: Dict[str,NodeUnion] + :return: Same as `green_iers` input dict, but filtered to only contain the blocked ones. + :rtype: Dict[str, IER] """ blocked_green_iers = {} @@ -45,12 +54,17 @@ class HardCodedACLAgent(HardCodedAgentSessionABC): return blocked_green_iers - def get_matching_acl_rules_for_ier(self, ier, acl, nodes): - """ - Get matching ACL rules for an IER. + def get_matching_acl_rules_for_ier(self, ier:IER, acl:AccessControlList, nodes:Dict[str,NodeUnion]): + """Get list of ACL rules which are relevant to an IER - TODO: Add params and return in docstring. - TODO: Typehint params and return. + :param ier: Information Exchange Request to query against the ACL list + :type ier: IER + :param acl: Firewall rules + :type acl: AccessControlList + :param nodes: Nodes in the network + :type nodes: Dict[str,NodeUnion] + :return: _description_ + :rtype: _type_ """ source_node_id = ier.get_source_node_id() source_node_address = nodes[source_node_id].ip_address @@ -58,11 +72,11 @@ class HardCodedACLAgent(HardCodedAgentSessionABC): dest_node_address = nodes[dest_node_id].ip_address protocol = ier.get_protocol() # e.g. 'TCP' port = ier.get_port() - + # I can't find where this function 'get_relevant_rules' is defined... matching_rules = acl.get_relevant_rules(source_node_address, dest_node_address, protocol, port) return matching_rules - def get_blocking_acl_rules_for_ier(self, ier, acl, nodes): + def get_blocking_acl_rules_for_ier(self, ier:IER, acl:AccessControlList, nodes:Dict[str,NodeUnion])->Dict[str,Any]: """ Get blocking ACL rules for an IER. @@ -70,8 +84,14 @@ class HardCodedACLAgent(HardCodedAgentSessionABC): Can return empty dict but IER can still be blocked by default (No ALLOW rule, therefore blocked). - TODO: Add params and return in docstring. - TODO: Typehint params and return. + :param ier: Information Exchange Request to query against the ACL list + :type ier: IER + :param acl: Firewall rules + :type acl: AccessControlList + :param nodes: Nodes in the network + :type nodes: Dict[str,NodeUnion] + :return: _description_ + :rtype: _type_ """ matching_rules = self.get_matching_acl_rules_for_ier(ier, acl, nodes) @@ -82,12 +102,18 @@ class HardCodedACLAgent(HardCodedAgentSessionABC): return blocked_rules - def get_allow_acl_rules_for_ier(self, ier, acl, nodes): + def get_allow_acl_rules_for_ier(self, ier:IER, acl:AccessControlList, nodes:Dict[str,NodeUnion])->Dict[str,Any]: """ Get all allowing ACL rules for an IER. - TODO: Add params and return in docstring. - TODO: Typehint params and return. + :param ier: Information Exchange Request to query against the ACL list + :type ier: IER + :param acl: Firewall rules + :type acl: AccessControlList + :param nodes: Nodes in the network + :type nodes: Dict[str,NodeUnion] + :return: _description_ + :rtype: _type_ """ matching_rules = self.get_matching_acl_rules_for_ier(ier, acl, nodes) From ea01e2209b9c4b3d51ed2265df228f5ca79a4160 Mon Sep 17 00:00:00 2001 From: Marek Wolan Date: Wed, 5 Jul 2023 09:00:41 +0000 Subject: [PATCH 10/41] Updated access_control_list.py --- src/primaite/acl/access_control_list.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/primaite/acl/access_control_list.py b/src/primaite/acl/access_control_list.py index 42460b94..f6ae3fad 100644 --- a/src/primaite/acl/access_control_list.py +++ b/src/primaite/acl/access_control_list.py @@ -9,7 +9,6 @@ class AccessControlList: """Access Control List class.""" def __init__(self): - """Init.""" self.acl: Dict[str, AccessControlList] = {} # A dictionary of ACL Rules def check_address_match(self, _rule, _source_ip_address, _dest_ip_address): From 38a3666e8eb0cdbd560963a1fbb9bb0447bd2402 Mon Sep 17 00:00:00 2001 From: Marek Wolan Date: Wed, 5 Jul 2023 10:14:16 +0100 Subject: [PATCH 11/41] Move class docstrings out of init function. --- src/primaite/acl/access_control_list.py | 1 - src/primaite/acl/acl_rule.py | 18 +++++------- src/primaite/common/protocol.py | 10 +++---- src/primaite/common/service.py | 13 ++++----- src/primaite/links/link.py | 18 +++++------- src/primaite/nodes/active_node.py | 26 ++++++++--------- src/primaite/nodes/node.py | 19 ++++++------ .../nodes/node_state_instruction_green.py | 22 +++++++------- .../nodes/node_state_instruction_red.py | 29 +++++++++---------- src/primaite/nodes/passive_node.py | 19 ++++++------ src/primaite/nodes/service_node.py | 25 ++++++++-------- src/primaite/pol/ier.py | 28 +++++++++--------- 12 files changed, 105 insertions(+), 123 deletions(-) diff --git a/src/primaite/acl/access_control_list.py b/src/primaite/acl/access_control_list.py index 42460b94..f6ae3fad 100644 --- a/src/primaite/acl/access_control_list.py +++ b/src/primaite/acl/access_control_list.py @@ -9,7 +9,6 @@ class AccessControlList: """Access Control List class.""" def __init__(self): - """Init.""" self.acl: Dict[str, AccessControlList] = {} # A dictionary of ACL Rules def check_address_match(self, _rule, _source_ip_address, _dest_ip_address): diff --git a/src/primaite/acl/acl_rule.py b/src/primaite/acl/acl_rule.py index 29f52f88..49aebc1b 100644 --- a/src/primaite/acl/acl_rule.py +++ b/src/primaite/acl/acl_rule.py @@ -3,18 +3,16 @@ class ACLRule: - """Access Control List Rule class.""" + """Access Control List Rule class. + + :param _permission: The permission (ALLOW or DENY) + :param _source_ip: The source IP address + :param _dest_ip: The destination IP address + :param _protocol: The rule protocol + :param _port: The rule port + """ def __init__(self, _permission, _source_ip, _dest_ip, _protocol, _port): - """Init. - - Args: - _permission: The permission (ALLOW or DENY) - _source_ip: The source IP address - _dest_ip: The destination IP address - _protocol: The rule protocol - _port: The rule port - """ self.permission = _permission self.source_ip = _source_ip self.dest_ip = _dest_ip diff --git a/src/primaite/common/protocol.py b/src/primaite/common/protocol.py index ebda1fcf..ec67caa3 100644 --- a/src/primaite/common/protocol.py +++ b/src/primaite/common/protocol.py @@ -3,14 +3,12 @@ class Protocol(object): - """Protocol class.""" + """Protocol class. + + :param _name: The protocol name + """ def __init__(self, _name): - """Init. - - Args: - _name: The protocol name - """ self.name = _name self.load = 0 # bps diff --git a/src/primaite/common/service.py b/src/primaite/common/service.py index c381f51f..2d08a3c5 100644 --- a/src/primaite/common/service.py +++ b/src/primaite/common/service.py @@ -5,15 +5,14 @@ from primaite.common.enums import SoftwareState class Service(object): - """Service class.""" + """Service class. + + :param name: The service name. + :param port: The service port. + :param software_state: The service SoftwareState. + """ def __init__(self, name: str, port: str, software_state: SoftwareState): - """Init. - - :param name: The service name. - :param port: The service port. - :param software_state: The service SoftwareState. - """ self.name = name self.port = port self.software_state = software_state diff --git a/src/primaite/links/link.py b/src/primaite/links/link.py index e8901b3d..ff73ccc8 100644 --- a/src/primaite/links/link.py +++ b/src/primaite/links/link.py @@ -6,18 +6,16 @@ from primaite.common.protocol import Protocol class Link(object): - """Link class.""" + """Link class. + + :param _id: The IER id + :param _bandwidth: The bandwidth of the link (bps) + :param _source_node_name: The name of the source node + :param _dest_node_name: The name of the destination node + :param _protocols: The protocols to add to the link + """ def __init__(self, _id, _bandwidth, _source_node_name, _dest_node_name, _services): - """Init. - - Args: - _id: The IER id - _bandwidth: The bandwidth of the link (bps) - _source_node_name: The name of the source node - _dest_node_name: The name of the destination node - _protocols: The protocols to add to the link - """ self.id = _id self.bandwidth = _bandwidth self.source_node_name = _source_node_name diff --git a/src/primaite/nodes/active_node.py b/src/primaite/nodes/active_node.py index 588ccd93..e20ce0e0 100644 --- a/src/primaite/nodes/active_node.py +++ b/src/primaite/nodes/active_node.py @@ -11,7 +11,19 @@ _LOGGER: Final[logging.Logger] = logging.getLogger(__name__) class ActiveNode(Node): - """Active Node class.""" + """Active Node class. + + :param node_id: The node ID + :param name: The node name + :param node_type: The node type (enum) + :param priority: The node priority (enum) + :param hardware_state: The node Hardware State + :param ip_address: The node IP address + :param software_state: The node Software State + :param file_system_state: The node file system state + :param config_values: The config values + + """ def __init__( self, @@ -25,18 +37,6 @@ class ActiveNode(Node): file_system_state: FileSystemState, config_values: TrainingConfig, ): - """Init. - - :param node_id: The node ID - :param name: The node name - :param node_type: The node type (enum) - :param priority: The node priority (enum) - :param hardware_state: The node Hardware State - :param ip_address: The node IP address - :param software_state: The node Software State - :param file_system_state: The node file system state - :param config_values: The config values - """ super().__init__(node_id, name, node_type, priority, hardware_state, config_values) self.ip_address: str = ip_address # Related to Software diff --git a/src/primaite/nodes/node.py b/src/primaite/nodes/node.py index 40f6328f..b54989bf 100644 --- a/src/primaite/nodes/node.py +++ b/src/primaite/nodes/node.py @@ -7,7 +7,15 @@ from primaite.config.training_config import TrainingConfig class Node: - """Node class.""" + """Node class. + + :param node_id: The node id. + :param name: The name of the node. + :param node_type: The type of the node. + :param priority: The priority of the node. + :param hardware_state: The state of the node. + :param config_values: Config values. + """ def __init__( self, @@ -18,15 +26,6 @@ class Node: hardware_state: HardwareState, config_values: TrainingConfig, ): - """Init. - - :param node_id: The node id. - :param name: The name of the node. - :param node_type: The type of the node. - :param priority: The priority of the node. - :param hardware_state: The state of the node. - :param config_values: Config values. - """ self.node_id: Final[str] = node_id self.name: Final[str] = name self.node_type: Final[NodeType] = node_type diff --git a/src/primaite/nodes/node_state_instruction_green.py b/src/primaite/nodes/node_state_instruction_green.py index e1244144..fcf5268c 100644 --- a/src/primaite/nodes/node_state_instruction_green.py +++ b/src/primaite/nodes/node_state_instruction_green.py @@ -3,7 +3,16 @@ class NodeStateInstructionGreen(object): - """The Node State Instruction class.""" + """The Node State Instruction class.# + + :param _id: The node state instruction id + :param _start_step: The start step of the instruction + :param _end_step: The end step of the instruction + :param _node_id: The id of the associated node + :param _node_pol_type: The pattern of life type + :param _service_name: The service name + :param _state: The state (node or service) + """ def __init__( self, @@ -15,17 +24,6 @@ class NodeStateInstructionGreen(object): _service_name, _state, ): - """Init. - - Args: - _id: The node state instruction id - _start_step: The start step of the instruction - _end_step: The end step of the instruction - _node_id: The id of the associated node - _node_pol_type: The pattern of life type - _service_name: The service name - _state: The state (node or service) - """ self.id = _id self.start_step = _start_step self.end_step = _end_step diff --git a/src/primaite/nodes/node_state_instruction_red.py b/src/primaite/nodes/node_state_instruction_red.py index 3e2e734d..a6cae2bd 100644 --- a/src/primaite/nodes/node_state_instruction_red.py +++ b/src/primaite/nodes/node_state_instruction_red.py @@ -7,7 +7,19 @@ from primaite.common.enums import NodePOLType @dataclass() class NodeStateInstructionRed(object): - """The Node State Instruction class.""" + """The Node State Instruction class. + :param _id: The node state instruction id + :param _start_step: The start step of the instruction + :param _end_step: The end step of the instruction + :param _target_node_id: The id of the associated node + :param -pol_initiator: The way the PoL is applied (DIRECT, IER or SERVICE) + :param _pol_type: The pattern of life type + :param pol_protocol: The pattern of life protocol/service affected + :param _pol_state: The state (node or service) + :param _pol_source_node_id: The source node Id (used for initiator type SERVICE) + :param _pol_source_node_service: The source node service (used for initiator type SERVICE) + :param _pol_source_node_service_state: The source node service state (used for initiator type SERVICE) + """ def __init__( self, @@ -23,21 +35,6 @@ class NodeStateInstructionRed(object): _pol_source_node_service, _pol_source_node_service_state, ): - """Init. - - Args: - _id: The node state instruction id - _start_step: The start step of the instruction - _end_step: The end step of the instruction - _target_node_id: The id of the associated node - -pol_initiator: The way the PoL is applied (DIRECT, IER or SERVICE) - _pol_type: The pattern of life type - -pol_protocol: The pattern of life protocol/service affected - _pol_state: The state (node or service) - _pol_source_node_id: The source node Id (used for initiator type SERVICE) - _pol_source_node_service: The source node service (used for initiator type SERVICE) - _pol_source_node_service_state: The source node service state (used for initiator type SERVICE) - """ self.id = _id self.start_step = _start_step self.end_step = _end_step diff --git a/src/primaite/nodes/passive_node.py b/src/primaite/nodes/passive_node.py index 188b4ee3..fa289593 100644 --- a/src/primaite/nodes/passive_node.py +++ b/src/primaite/nodes/passive_node.py @@ -6,7 +6,15 @@ from primaite.nodes.node import Node class PassiveNode(Node): - """The Passive Node class.""" + """The Passive Node class. + + :param node_id: The node id. + :param name: The name of the node. + :param node_type: The type of the node. + :param priority: The priority of the node. + :param hardware_state: The state of the node. + :param config_values: Config values. + """ def __init__( self, @@ -17,15 +25,6 @@ class PassiveNode(Node): hardware_state: HardwareState, config_values: TrainingConfig, ): - """Init. - - :param node_id: The node id. - :param name: The name of the node. - :param node_type: The type of the node. - :param priority: The priority of the node. - :param hardware_state: The state of the node. - :param config_values: Config values. - """ # Pass through to Super for now super().__init__(node_id, name, node_type, priority, hardware_state, config_values) diff --git a/src/primaite/nodes/service_node.py b/src/primaite/nodes/service_node.py index 0114f507..db435c7d 100644 --- a/src/primaite/nodes/service_node.py +++ b/src/primaite/nodes/service_node.py @@ -12,7 +12,18 @@ _LOGGER: Final[logging.Logger] = logging.getLogger(__name__) class ServiceNode(ActiveNode): - """ServiceNode class.""" + """ServiceNode class. + + :param node_id: The node ID + :param name: The node name + :param node_type: The node type (enum) + :param priority: The node priority (enum) + :param hardware_state: The node Hardware State + :param ip_address: The node IP address + :param software_state: The node Software State + :param file_system_state: The node file system state + :param config_values: The config values + """ def __init__( self, @@ -26,18 +37,6 @@ class ServiceNode(ActiveNode): file_system_state: FileSystemState, config_values: TrainingConfig, ): - """Init. - - :param node_id: The node ID - :param name: The node name - :param node_type: The node type (enum) - :param priority: The node priority (enum) - :param hardware_state: The node Hardware State - :param ip_address: The node IP address - :param software_state: The node Software State - :param file_system_state: The node file system state - :param config_values: The config values - """ super().__init__( node_id, name, diff --git a/src/primaite/pol/ier.py b/src/primaite/pol/ier.py index 09f32aeb..bfbc9a31 100644 --- a/src/primaite/pol/ier.py +++ b/src/primaite/pol/ier.py @@ -6,7 +6,19 @@ Used to represent an information flow from source to destination. class IER(object): - """Information Exchange Requirement class.""" + """Information Exchange Requirement class. + + :param _id: The IER id + :param _start_step: The step when this IER should start + :param _end_step: The step when this IER should end + :param _load: The load this IER should put on a link (bps) + :param _protocol: The protocol of this IER + :param _port: The port this IER runs on + :param _source_node_id: The source node ID + :param _dest_node_id: The destination node ID + :param _mission_criticality: Criticality of this IER to the mission (0 none, 5 mission critical) + :param _running: Indicates whether the IER is currently running + """ def __init__( self, @@ -21,20 +33,6 @@ class IER(object): _mission_criticality, _running=False, ): - """Init. - - Args: - _id: The IER id - _start_step: The step when this IER should start - _end_step: The step when this IER should end - _load: The load this IER should put on a link (bps) - _protocol: The protocol of this IER - _port: The port this IER runs on - _source_node_id: The source node ID - _dest_node_id: The destination node ID - _mission_criticality: Criticality of this IER to the mission (0 none, 5 mission critical) - _running: Indicates whether the IER is currently running - """ self.id = _id self.start_step = _start_step self.end_step = _end_step From b3d4eb4ec0920201291daa8c1feecc17ffb5baca Mon Sep 17 00:00:00 2001 From: Marek Wolan Date: Wed, 5 Jul 2023 13:58:46 +0100 Subject: [PATCH 12/41] Changed hardcoded agent helper for new obs space --- src/primaite/agents/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/primaite/agents/utils.py b/src/primaite/agents/utils.py index 8c59faf7..b5a3c673 100644 --- a/src/primaite/agents/utils.py +++ b/src/primaite/agents/utils.py @@ -165,7 +165,8 @@ def transform_change_obs_readable(obs): os_states = [SoftwareState(i).name for i in obs[:, 2]] new_obs = [ids, operating_states, os_states] - for service in range(3, obs.shape[1]): + # changed range(3,...) to range(4,...) because we added file system which was new since ADSP + for service in range(4, obs.shape[1]): # Links bit/s don't have a service state service_states = [SoftwareState(i).name if i <= 4 else i for i in obs[:, service]] new_obs.append(service_states) From 171b5cb58e0e56f242d18cc98f423a98e6291007 Mon Sep 17 00:00:00 2001 From: Marek Wolan Date: Wed, 5 Jul 2023 14:10:52 +0100 Subject: [PATCH 13/41] Imported ADSP function for ACL --- src/primaite/acl/access_control_list.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/primaite/acl/access_control_list.py b/src/primaite/acl/access_control_list.py index 3b0e9234..c155deed 100644 --- a/src/primaite/acl/access_control_list.py +++ b/src/primaite/acl/access_control_list.py @@ -116,3 +116,27 @@ class AccessControlList: rule = ACLRule(_permission, _source_ip, _dest_ip, _protocol, str(_port)) hash_value = hash(rule) return hash_value + + def get_relevant_rules(self, _source_ip_address, _dest_ip_address, _protocol, _port): + """Get all ACL rules that relate to the given arguments + + :param _source_ip_address: the source IP address to check + :param _dest_ip_address: the destination IP address to check + :param _protocol: the protocol to check + :param _port: the port to check + :return: Dictionary of all ACL rules that relate to the given arguments + :rtype: Dict[str, ACLRule] + """ + relevant_rules = {} + + for rule_key, rule_value in self.acl.items(): + if self.check_address_match(rule_value, _source_ip_address, _dest_ip_address): + if ( + rule_value.get_protocol() == _protocol or rule_value.get_protocol() == "ANY" or _protocol == "ANY" + ) and ( + str(rule_value.get_port()) == str(_port) or rule_value.get_port() == "ANY" or str(_port) == "ANY" + ): + # There's a matching rule. + relevant_rules[rule_key] = rule_value + + return relevant_rules From f62b2aef1c17276bba6c4b17d61a7d493e6924f5 Mon Sep 17 00:00:00 2001 From: Marek Wolan Date: Wed, 5 Jul 2023 14:13:43 +0100 Subject: [PATCH 14/41] Fix minor typos in docstrings --- src/primaite/nodes/node_state_instruction_green.py | 2 +- src/primaite/nodes/node_state_instruction_red.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/primaite/nodes/node_state_instruction_green.py b/src/primaite/nodes/node_state_instruction_green.py index fcf5268c..0faef627 100644 --- a/src/primaite/nodes/node_state_instruction_green.py +++ b/src/primaite/nodes/node_state_instruction_green.py @@ -3,7 +3,7 @@ class NodeStateInstructionGreen(object): - """The Node State Instruction class.# + """The Node State Instruction class. :param _id: The node state instruction id :param _start_step: The start step of the instruction diff --git a/src/primaite/nodes/node_state_instruction_red.py b/src/primaite/nodes/node_state_instruction_red.py index a6cae2bd..8308a1c0 100644 --- a/src/primaite/nodes/node_state_instruction_red.py +++ b/src/primaite/nodes/node_state_instruction_red.py @@ -8,6 +8,7 @@ from primaite.common.enums import NodePOLType @dataclass() class NodeStateInstructionRed(object): """The Node State Instruction class. + :param _id: The node state instruction id :param _start_step: The start step of the instruction :param _end_step: The end step of the instruction From 7482aead76d702b70527e550995fc3a7c25a9b98 Mon Sep 17 00:00:00 2001 From: Marek Wolan Date: Wed, 5 Jul 2023 14:50:03 +0100 Subject: [PATCH 15/41] typo --- src/primaite/acl/access_control_list.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/primaite/acl/access_control_list.py b/src/primaite/acl/access_control_list.py index c155deed..7e90724a 100644 --- a/src/primaite/acl/access_control_list.py +++ b/src/primaite/acl/access_control_list.py @@ -118,7 +118,7 @@ class AccessControlList: return hash_value def get_relevant_rules(self, _source_ip_address, _dest_ip_address, _protocol, _port): - """Get all ACL rules that relate to the given arguments + """Get all ACL rules that relate to the given arguments. :param _source_ip_address: the source IP address to check :param _dest_ip_address: the destination IP address to check From 5c167293e3833dd558294cec3dafd19c8e915c96 Mon Sep 17 00:00:00 2001 From: Marek Wolan Date: Wed, 5 Jul 2023 16:19:43 +0100 Subject: [PATCH 16/41] Add docstrings and type hints. --- src/primaite/acl/access_control_list.py | 22 ++-- src/primaite/agents/hardcoded_acl.py | 160 ++++++++++++++++-------- src/primaite/agents/hardcoded_node.py | 10 +- src/primaite/agents/utils.py | 125 +++++++++++------- 4 files changed, 208 insertions(+), 109 deletions(-) diff --git a/src/primaite/acl/access_control_list.py b/src/primaite/acl/access_control_list.py index 7e90724a..f5f70d6c 100644 --- a/src/primaite/acl/access_control_list.py +++ b/src/primaite/acl/access_control_list.py @@ -10,19 +10,19 @@ class AccessControlList: def __init__(self): """Init.""" - self.acl: Dict[str, AccessControlList] = {} # A dictionary of ACL Rules + self.acl: Dict[str, ACLRule] = {} # A dictionary of ACL Rules - def check_address_match(self, _rule, _source_ip_address, _dest_ip_address): - """ - Checks for IP address matches. + def check_address_match(self, _rule: ACLRule, _source_ip_address: str, _dest_ip_address: str) -> bool: + """Checks for IP address matches. - Args: - _rule: The rule being checked - _source_ip_address: the source IP address to compare - _dest_ip_address: the destination IP address to compare - - Returns: - True if match; False otherwise. + :param _rule: The rule object to check + :type _rule: ACLRule + :param _source_ip_address: Source IP address to compare + :type _source_ip_address: str + :param _dest_ip_address: Destination IP address to compare + :type _dest_ip_address: str + :return: True if there is a match, otherwise False. + :rtype: bool """ if ( (_rule.get_source_ip() == _source_ip_address and _rule.get_dest_ip() == _dest_ip_address) diff --git a/src/primaite/agents/hardcoded_acl.py b/src/primaite/agents/hardcoded_acl.py index 430c8b54..a724e10e 100644 --- a/src/primaite/agents/hardcoded_acl.py +++ b/src/primaite/agents/hardcoded_acl.py @@ -1,7 +1,9 @@ -from typing import Any, Dict -import numpy as np -from primaite.acl.access_control_list import AccessControlList +from typing import Any, Dict, List, Union +import numpy as np + +from primaite.acl.access_control_list import AccessControlList +from primaite.acl.acl_rule import ACLRule from primaite.agents.agent import HardCodedAgentSessionABC from primaite.agents.utils import ( get_new_action, @@ -11,13 +13,15 @@ from primaite.agents.utils import ( ) from primaite.common.custom_typing import NodeUnion from primaite.common.enums import HardCodedAgentView +from primaite.nodes.active_node import ActiveNode +from primaite.nodes.service_node import ServiceNode from primaite.pol.ier import IER class HardCodedACLAgent(HardCodedAgentSessionABC): """An Agent Session class that implements a deterministic ACL agent.""" - def _calculate_action(self, obs): + def _calculate_action(self, obs: np.ndarray) -> int: if self._training_config.hard_coded_agent_view == HardCodedAgentView.BASIC: # Basic view action using only the current observation return self._calculate_action_basic_view(obs) @@ -26,7 +30,9 @@ class HardCodedACLAgent(HardCodedAgentSessionABC): # history and reward feedback return self._calculate_action_full_view(obs) - def get_blocked_green_iers(self, green_iers:Dict[str, IER], acl:AccessControlList, nodes:Dict[str,NodeUnion]): + def get_blocked_green_iers( + self, green_iers: Dict[str, IER], acl: AccessControlList, nodes: Dict[str, NodeUnion] + ) -> Dict[Any, Any]: """Get blocked green IERs. :param green_iers: Green IERs to check for being @@ -54,8 +60,8 @@ class HardCodedACLAgent(HardCodedAgentSessionABC): return blocked_green_iers - def get_matching_acl_rules_for_ier(self, ier:IER, acl:AccessControlList, nodes:Dict[str,NodeUnion]): - """Get list of ACL rules which are relevant to an IER + def get_matching_acl_rules_for_ier(self, ier: IER, acl: AccessControlList, nodes: Dict[str, NodeUnion]): + """Get list of ACL rules which are relevant to an IER. :param ier: Information Exchange Request to query against the ACL list :type ier: IER @@ -76,7 +82,9 @@ class HardCodedACLAgent(HardCodedAgentSessionABC): matching_rules = acl.get_relevant_rules(source_node_address, dest_node_address, protocol, port) return matching_rules - def get_blocking_acl_rules_for_ier(self, ier:IER, acl:AccessControlList, nodes:Dict[str,NodeUnion])->Dict[str,Any]: + def get_blocking_acl_rules_for_ier( + self, ier: IER, acl: AccessControlList, nodes: Dict[str, NodeUnion] + ) -> Dict[str, Any]: """ Get blocking ACL rules for an IER. @@ -102,9 +110,10 @@ class HardCodedACLAgent(HardCodedAgentSessionABC): return blocked_rules - def get_allow_acl_rules_for_ier(self, ier:IER, acl:AccessControlList, nodes:Dict[str,NodeUnion])->Dict[str,Any]: - """ - Get all allowing ACL rules for an IER. + def get_allow_acl_rules_for_ier( + self, ier: IER, acl: AccessControlList, nodes: Dict[str, NodeUnion] + ) -> Dict[str, Any]: + """Get all allowing ACL rules for an IER. :param ier: Information Exchange Request to query against the ACL list :type ier: IER @@ -126,19 +135,32 @@ class HardCodedACLAgent(HardCodedAgentSessionABC): def get_matching_acl_rules( self, - source_node_id, - dest_node_id, - protocol, - port, - acl, - nodes, - services_list, - ): - """ - Get matching ACL rules. + source_node_id: str, + dest_node_id: str, + protocol: str, + port: str, + acl: AccessControlList, + nodes: Dict[str, Union[ServiceNode, ActiveNode]], + services_list: List[str], + ) -> Dict[str, ACLRule]: + """Filter ACL rules to only those which are relevant to the specified nodes. - TODO: Add params and return in docstring. - TODO: Typehint params and return. + :param source_node_id: Source node + :type source_node_id: str + :param dest_node_id: Destination nodes + :type dest_node_id: str + :param protocol: Network protocol + :type protocol: str + :param port: Network port + :type port: str + :param acl: Access Control list which will be filtered + :type acl: AccessControlList + :param nodes: The environment's node directory. + :type nodes: Dict[str, Union[ServiceNode, ActiveNode]] + :param services_list: List of services registered for the environment. + :type services_list: List[str] + :return: Filtered version of 'acl' + :rtype: Dict[str, ACLRule] """ if source_node_id != "ANY": source_node_address = nodes[str(source_node_id)].ip_address @@ -158,19 +180,33 @@ class HardCodedACLAgent(HardCodedAgentSessionABC): def get_allow_acl_rules( self, - source_node_id, - dest_node_id, - protocol, - port, - acl, - nodes, - services_list, - ): - """ - Get the ALLOW ACL rules. + source_node_id: int, + dest_node_id: str, + protocol: int, + port: str, + acl: AccessControlList, + nodes: Dict[str, NodeUnion], + services_list: List[str], + ) -> Dict[str, ACLRule]: + """List ALLOW rules relating to specified nodes. - TODO: Add params and return in docstring. - TODO: Typehint params and return. + :param source_node_id: Source node id + :type source_node_id: int + :param dest_node_id: Destination node + :type dest_node_id: str + :param protocol: Network protocol + :type protocol: int + :param port: Port + :type port: str + :param acl: Firewall ruleset which is applied to the network + :type acl: AccessControlList + :param nodes: The simulation's node store + :type nodes: Dict[str, NodeUnion] + :param services_list: Services list + :type services_list: List[str] + :return: Filtered ACL Rule directory which includes only those rules which affect the specified source and + desination nodes + :rtype: Dict[str, ACLRule] """ matching_rules = self.get_matching_acl_rules( source_node_id, @@ -191,19 +227,33 @@ class HardCodedACLAgent(HardCodedAgentSessionABC): def get_deny_acl_rules( self, - source_node_id, - dest_node_id, - protocol, - port, - acl, - nodes, - services_list, - ): - """ - Get the DENY ACL rules. + source_node_id: int, + dest_node_id: str, + protocol: int, + port: str, + acl: AccessControlList, + nodes: Dict[str, NodeUnion], + services_list: List[str], + ) -> Dict[str, ACLRule]: + """List DENY rules relating to specified nodes. - TODO: Add params and return in docstring. - TODO: Typehint params and return. + :param source_node_id: Source node id + :type source_node_id: int + :param dest_node_id: Destination node + :type dest_node_id: str + :param protocol: Network protocol + :type protocol: int + :param port: Port + :type port: str + :param acl: Firewall ruleset which is applied to the network + :type acl: AccessControlList + :param nodes: The simulation's node store + :type nodes: Dict[str, NodeUnion] + :param services_list: Services list + :type services_list: List[str] + :return: Filtered ACL Rule directory which includes only those rules which affect the specified source and + desination nodes + :rtype: Dict[str, ACLRule] """ matching_rules = self.get_matching_acl_rules( source_node_id, @@ -222,7 +272,7 @@ class HardCodedACLAgent(HardCodedAgentSessionABC): return allowed_rules - def _calculate_action_full_view(self, obs): + def _calculate_action_full_view(self, obs: np.ndarray) -> int: """ Calculate a good acl-based action for the blue agent to take. @@ -250,8 +300,10 @@ class HardCodedACLAgent(HardCodedAgentSessionABC): nodes once a service becomes overwhelmed. However currently the ACL action space has no way of reversing an overwhelmed state, so we don't do this. - TODO: Add params and return in docstring. - TODO: Typehint params and return. + :param obs: current observation from the gym environment + :type obs: np.ndarray + :return: Optimal action to take in the environment (chosen from the discrete action space) + :rtype: int """ # obs = convert_to_old_obs(obs) r_obs = transform_change_obs_readable(obs) @@ -387,7 +439,7 @@ class HardCodedACLAgent(HardCodedAgentSessionABC): action = get_new_action(action, self._env.action_dict) return action - def _calculate_action_basic_view(self, obs): + def _calculate_action_basic_view(self, obs: np.ndarray) -> int: """Calculate a good acl-based action for the blue agent to take. Uses ONLY information from the current observation with NO knowledge @@ -405,8 +457,10 @@ class HardCodedACLAgent(HardCodedAgentSessionABC): Currently, a deny rule does not overwrite an allow rule. The allow rules must be deleted. - TODO: Add params and return in docstring. - TODO: Typehint params and return. + :param obs: current observation from the gym environment + :type obs: np.ndarray + :return: Optimal action to take in the environment (chosen from the discrete action space) + :rtype: int """ action_dict = self._env.action_dict r_obs = transform_change_obs_readable(obs) diff --git a/src/primaite/agents/hardcoded_node.py b/src/primaite/agents/hardcoded_node.py index 310fc178..c00cf421 100644 --- a/src/primaite/agents/hardcoded_node.py +++ b/src/primaite/agents/hardcoded_node.py @@ -1,3 +1,5 @@ +import numpy as np + from primaite.agents.agent import HardCodedAgentSessionABC from primaite.agents.utils import get_new_action, transform_action_node_enum, transform_change_obs_readable @@ -5,12 +7,14 @@ from primaite.agents.utils import get_new_action, transform_action_node_enum, tr class HardCodedNodeAgent(HardCodedAgentSessionABC): """An Agent Session class that implements a deterministic Node agent.""" - def _calculate_action(self, obs): + def _calculate_action(self, obs: np.ndarray) -> int: """ Calculate a good node-based action for the blue agent to take. - TODO: Add params and return in docstring. - TODO: Typehint params and return. + :param obs: current observation from the gym environment + :type obs: np.ndarray + :return: Optimal action to take in the environment (chosen from the discrete action space) + :rtype: int """ action_dict = self._env.action_dict r_obs = transform_change_obs_readable(obs) diff --git a/src/primaite/agents/utils.py b/src/primaite/agents/utils.py index b5a3c673..bffdbf43 100644 --- a/src/primaite/agents/utils.py +++ b/src/primaite/agents/utils.py @@ -1,3 +1,5 @@ +from typing import List, Union + import numpy as np from primaite.common.enums import ( @@ -10,15 +12,17 @@ from primaite.common.enums import ( ) -def transform_action_node_readable(action): - """ - Convert a node action from enumerated format to readable format. +def transform_action_node_readable(action: List[int]) -> List[Union[int, str]]: + """Convert a node action from enumerated format to readable format. example: [1, 3, 1, 0] -> [1, 'SERVICE', 'PATCHING', 0] - TODO: Add params and return in docstring. - TODO: Typehint params and return. + :param action: Agent action, formatted as a list of ints, for more information check out + `primaite.environment.primaite_env.Primaite` + :type action: List[int] + :return: The same action list, but with the encodings translated back into meaningful labels + :rtype: List[Union[int,str]] """ action_node_property = NodePOLType(action[1]).name @@ -33,15 +37,18 @@ def transform_action_node_readable(action): return new_action -def transform_action_acl_readable(action): +def transform_action_acl_readable(action: List[str]) -> List[Union[str, int]]: """ Transform an ACL action to a more readable format. example: [0, 1, 2, 5, 0, 1] -> ['NONE', 'ALLOW', 2, 5, 'ANY', 1] - TODO: Add params and return in docstring. - TODO: Typehint params and return. + :param action: Agent action, formatted as a list of ints, for more information check out + `primaite.environment.primaite_env.Primaite` + :type action: List[int] + :return: The same action list, but with the encodings translated back into meaningful labels + :rtype: List[Union[int,str]] """ action_decisions = {0: "NONE", 1: "CREATE", 2: "DELETE"} action_permissions = {0: "DENY", 1: "ALLOW"} @@ -58,7 +65,7 @@ def transform_action_acl_readable(action): return new_action -def is_valid_node_action(action): +def is_valid_node_action(action: List[int]) -> bool: """Is the node action an actual valid action. Only uses information about the action to determine if the action has an effect @@ -67,8 +74,11 @@ def is_valid_node_action(action): - Node ID not valid to perform an operation - e.g. selected node has no service so cannot patch - Node already being in that state (turning an ON node ON) - TODO: Add params and return in docstring. - TODO: Typehint params and return. + :param action: Agent action, formatted as a list of ints, for more information check out + `primaite.environment.primaite_env.Primaite` + :type action: List[int] + :return: Whether the action is valid + :rtype: bool """ action_r = transform_action_node_readable(action) @@ -93,7 +103,7 @@ def is_valid_node_action(action): return True -def is_valid_acl_action(action): +def is_valid_acl_action(action: List[int]) -> bool: """ Is the ACL action an actual valid action. @@ -103,8 +113,11 @@ def is_valid_acl_action(action): - Trying to create identical rules - Trying to create a rule which is a subset of another rule (caused by "ANY") - TODO: Add params and return in docstring. - TODO: Typehint params and return. + :param action: Agent action, formatted as a list of ints, for more information check out + `primaite.environment.primaite_env.Primaite` + :type action: List[int] + :return: Whether the action is valid + :rtype: bool """ action_r = transform_action_acl_readable(action) @@ -126,12 +139,15 @@ def is_valid_acl_action(action): return True -def is_valid_acl_action_extra(action): +def is_valid_acl_action_extra(action: List[int]) -> bool: """ Harsher version of valid acl actions, does not allow action. - TODO: Add params and return in docstring. - TODO: Typehint params and return. + :param action: Agent action, formatted as a list of ints, for more information check out + `primaite.environment.primaite_env.Primaite` + :type action: List[int] + :return: Whether the action is valid + :rtype: bool """ if is_valid_acl_action(action) is False: return False @@ -150,15 +166,16 @@ def is_valid_acl_action_extra(action): return True -def transform_change_obs_readable(obs): - """ - Transform list of transactions to readable list of each observation property. +def transform_change_obs_readable(obs: np.ndarray) -> List[List[Union[str, int]]]: + """Transform list of transactions to readable list of each observation property. example: np.array([[1,2,1,3],[2,1,1,1]]) -> [[1, 2], ['OFF', 'ON'], ['GOOD', 'GOOD'], ['COMPROMISED', 'GOOD']] - TODO: Add params and return in docstring. - TODO: Typehint params and return. + :param obs: Raw observation from the environment. + :type obs: np.ndarray + :return: The same observation, but the encoded integer values are replaced with readable names. + :rtype: List[List[Union[str, int]]] """ ids = [i for i in obs[:, 0]] operating_states = [HardwareState(i).name for i in obs[:, 1]] @@ -174,14 +191,16 @@ def transform_change_obs_readable(obs): return new_obs -def transform_obs_readable(obs): - """ - Transform observation to readable format. +def transform_obs_readable(obs: np.ndarray) -> List[List[Union[str, int]]]: + """Transform observation to readable format. + example np.array([[1,2,1,3],[2,1,1,1]]) -> [[1, 'OFF', 'GOOD', 'COMPROMISED'], [2, 'ON', 'GOOD', 'GOOD']] - TODO: Add params and return in docstring. - TODO: Typehint params and return. + :param obs: Raw observation from the environment. + :type obs: np.ndarray + :return: The same observation, but the encoded integer values are replaced with readable names. + :rtype: List[List[Union[str, int]]] """ changed_obs = transform_change_obs_readable(obs) new_obs = list(zip(*changed_obs)) @@ -191,21 +210,23 @@ def transform_obs_readable(obs): return new_obs -def convert_to_new_obs(obs, num_nodes=10): - """ - Convert original gym Box observation space to new multiDiscrete observation space. +def convert_to_new_obs(obs: np.ndarray, num_nodes: int = 10) -> np.ndarray: + """Convert original gym Box observation space to new multiDiscrete observation space. - TODO: Add params and return in docstring. - TODO: Typehint params and return. + :param obs: observation in the 'old' (NodeLinkTable) format + :type obs: np.ndarray + :param num_nodes: number of nodes in the network, defaults to 10 + :type num_nodes: int, optional + :return: reformatted observation + :rtype: np.ndarray """ # Remove ID columns, remove links and flatten to MultiDiscrete observation space new_obs = obs[:num_nodes, 1:].flatten() return new_obs -def convert_to_old_obs(obs, num_nodes=10, num_links=10, num_services=1): - """ - Convert to old observation. +def convert_to_old_obs(obs: np.ndarray, num_nodes: int = 10, num_links: int = 10, num_services: int = 1) -> np.ndarray: + """Convert to old observation. Links filled with 0's as no information is included in new observation space. @@ -217,8 +238,17 @@ def convert_to_old_obs(obs, num_nodes=10, num_links=10, num_services=1): [ 3, 1, 1, 1], ... [20, 0, 0, 0]]) - TODO: Add params and return in docstring. - TODO: Typehint params and return. + + :param obs: observation in the 'new' (MultiDiscrete) format + :type obs: np.ndarray + :param num_nodes: number of nodes in the network, defaults to 10 + :type num_nodes: int, optional + :param num_links: number of links in the network, defaults to 10 + :type num_links: int, optional + :param num_services: number of services on the network, defaults to 1 + :type num_services: int, optional + :return: 2-d BOX observation space, in the same format as NodeLinkTable + :rtype: np.ndarray """ # Convert back to more readable, original format reshaped_nodes = obs[:-num_links].reshape(num_nodes, num_services + 2) @@ -240,17 +270,28 @@ def convert_to_old_obs(obs, num_nodes=10, num_links=10, num_services=1): return new_obs -def describe_obs_change(obs1, obs2, num_nodes=10, num_links=10, num_services=1): - """ - Return string describing change between two observations. +def describe_obs_change( + obs1: np.ndarray, obs2: np.ndarray, num_nodes: int = 10, num_links: int = 10, num_services: int = 1 +) -> str: + """Build a string describing the difference between two observations. example: obs_1 = array([[1, 1, 1, 1, 3], [2, 1, 1, 1, 1]]) obs_2 = array([[1, 1, 1, 1, 1], [2, 1, 1, 1, 1]]) output = 'ID 1: SERVICE 2 set to GOOD' - TODO: Add params and return in docstring. - TODO: Typehint params and return. + :param obs1: First observation + :type obs1: np.ndarray + :param obs2: Second observation + :type obs2: np.ndarray + :param num_nodes: How many nodes are in the network laydown, defaults to 10 + :type num_nodes: int, optional + :param num_links: How many links are in the network laydown, defaults to 10 + :type num_links: int, optional + :param num_services: How many services are configured for this scenario, defaults to 1 + :type num_services: int, optional + :return: A multi-line string with a human-readable description of the difference. + :rtype: str """ obs1 = convert_to_old_obs(obs1, num_nodes, num_links, num_services) obs2 = convert_to_old_obs(obs2, num_nodes, num_links, num_services) From b426d5802e6769f3dab3abca409271fc4fe94abd Mon Sep 17 00:00:00 2001 From: Marek Wolan Date: Wed, 5 Jul 2023 16:46:23 +0100 Subject: [PATCH 17/41] Updated docstrings --- src/primaite/agents/utils.py | 132 ++++++++++++++++-------------- src/primaite/pol/red_agent_pol.py | 14 +++- 2 files changed, 82 insertions(+), 64 deletions(-) diff --git a/src/primaite/agents/utils.py b/src/primaite/agents/utils.py index bffdbf43..58b422d0 100644 --- a/src/primaite/agents/utils.py +++ b/src/primaite/agents/utils.py @@ -1,4 +1,4 @@ -from typing import List, Union +from typing import Dict, List, Union import numpy as np @@ -346,15 +346,15 @@ def _describe_obs_change_helper(obs_change, is_link): return desc -def transform_action_node_enum(action): - """ - Convert a node action from readable string format, to enumerated format. +def transform_action_node_enum(action: List[Union[str, int]]) -> List[int]: + """Convert a node action from readable string format, to enumerated format. example: [1, 'SERVICE', 'PATCHING', 0] -> [1, 3, 1, 0] - - TODO: Add params and return in docstring. - TODO: Typehint params and return. + :param action: Action in 'readable' format + :type action: List[Union[str,int]] + :return: Action with verbs encoded as ints + :rtype: List[int] """ action_node_id = action[0] action_node_property = NodePOLType[action[1]].value @@ -378,15 +378,16 @@ def transform_action_node_enum(action): return new_action -def transform_action_node_readable(action): - """ - Convert a node action from enumerated format to readable format. +def transform_action_node_readable(action: List[int]) -> List[Union[int, str]]: + """Convert a node action from enumerated format to readable format. example: [1, 3, 1, 0] -> [1, 'SERVICE', 'PATCHING', 0] - TODO: Add params and return in docstring. - TODO: Typehint params and return. + :param action: Raw action with integer encodings + :type action: List[int] + :return: Human-redable version of the action + :rtype: List[Union[int,str]] """ action_node_property = NodePOLType(action[1]).name @@ -401,32 +402,33 @@ def transform_action_node_readable(action): return new_action -def node_action_description(action): - """ - Generate string describing a node-based action. +# unused +# def node_action_description(action): +# """ +# Generate string describing a node-based action. - TODO: Add params and return in docstring. - TODO: Typehint params and return. - """ - if isinstance(action[1], (int, np.int64)): - # transform action to readable format - action = transform_action_node_readable(action) +# TO#DO: Add params and return in docstring. +# TO#DO: Typehint params and return. +# """ +# if isinstance(action[1], (int, np.int64)): +# # transform action to readable format +# action = transform_action_node_readable(action) - node_id = action[0] - node_property = action[1] - property_action = action[2] - service_id = action[3] +# node_id = action[0] +# node_property = action[1] +# property_action = action[2] +# service_id = action[3] - if property_action == "NONE": - return "" - if node_property == "OPERATING" or node_property == "OS": - description = f"NODE {node_id}, {node_property}, SET TO {property_action}" - elif node_property == "SERVICE": - description = f"NODE {node_id} FROM SERVICE {service_id}, SET TO {property_action}" - else: - return "" +# if property_action == "NONE": +# return "" +# if node_property == "OPERATING" or node_property == "OS": +# description = f"NODE {node_id}, {node_property}, SET TO {property_action}" +# elif node_property == "SERVICE": +# description = f"NODE {node_id} FROM SERVICE {service_id}, SET TO {property_action}" +# else: +# return "" - return description +# return description def transform_action_acl_enum(action): @@ -452,25 +454,26 @@ def transform_action_acl_enum(action): return new_action -def acl_action_description(action): - """ - Generate string describing an acl-based action. +# unused +# def acl_action_description(action): +# """ +# Generate string describing an acl-based action. - TODO: Add params and return in docstring. - TODO: Typehint params and return. - """ - if isinstance(action[0], (int, np.int64)): - # transform action to readable format - action = transform_action_acl_readable(action) - if action[0] == "NONE": - description = "NO ACL RULE APPLIED" - else: - description = ( - f"{action[0]} RULE: {action[1]} traffic from IP {action[2]} to IP {action[3]}," - f" for protocol/service index {action[4]} on port index {action[5]}" - ) +# TODO: Add params and return in docstring. +# TODO: Typehint params and return. +# """ +# if isinstance(action[0], (int, np.int64)): +# # transform action to readable format +# action = transform_action_acl_readable(action) +# if action[0] == "NONE": +# description = "NO ACL RULE APPLIED" +# else: +# description = ( +# f"{action[0]} RULE: {action[1]} traffic from IP {action[2]} to IP {action[3]}," +# f" for protocol/service index {action[4]} on port index {action[5]}" +# ) - return description +# return description def get_node_of_ip(ip, node_dict): @@ -521,7 +524,7 @@ def is_valid_node_action(action): return True -def is_valid_acl_action(action): +def is_valid_acl_action(action: List[int]) -> bool: """ Is the ACL action an actual valid action. @@ -531,8 +534,11 @@ def is_valid_acl_action(action): - Trying to create identical rules - Trying to create a rule which is a subset of another rule (caused by "ANY") - TODO: Add params and return in docstring. - TODO: Typehint params and return. + + :param action: Action to check + :type action: List[int] + :return: Whether the action is valid + :rtype: bool """ action_r = transform_action_acl_readable(action) @@ -554,12 +560,14 @@ def is_valid_acl_action(action): return True -def is_valid_acl_action_extra(action): +def is_valid_acl_action_extra(action: List[int]) -> bool: """ Harsher version of valid acl actions, does not allow action. - TODO: Add params and return in docstring. - TODO: Typehint params and return. + :param action: Input action + :type action: List[int] + :return: Whether the action is a valid ACL action + :rtype: bool """ if is_valid_acl_action(action) is False: return False @@ -578,14 +586,18 @@ def is_valid_acl_action_extra(action): return True -def get_new_action(old_action, action_dict): +def get_new_action(old_action: np.ndarray, action_dict: Dict[int, List]) -> int: """ Get new action (e.g. 32) from old action e.g. [1,1,1,0]. Old_action can be either node or acl action type - TODO: Add params and return in docstring. - TODO: Typehint params and return. + :param old_action: Action expressed as a list of choices, eg. [1,1,1,0] + :type old_action: np.ndarray + :param action_dict: Dictionary for translating the multidiscrete actions into the list-based actions. + :type action_dict: Dict[int,List] + :return: Action key correspoinding to the input `old_action` + :rtype: int """ for key, val in action_dict.items(): if list(val) == list(old_action): diff --git a/src/primaite/pol/red_agent_pol.py b/src/primaite/pol/red_agent_pol.py index bff19bf8..1a8bd406 100644 --- a/src/primaite/pol/red_agent_pol.py +++ b/src/primaite/pol/red_agent_pol.py @@ -296,11 +296,17 @@ def apply_red_agent_node_pol( pass -def is_red_ier_incoming(node, iers, node_pol_type): - """ - Checks if the RED IER is incoming. +def is_red_ier_incoming(node: NodeUnion, iers: Dict[str, IER], node_pol_type: NodePOLType) -> bool: + """Checks if the RED IER is incoming. - TODO: Write more descriptive docstring with params and returns. + :param node: Destination node of the IER + :type node: NodeUnion + :param iers: Directory of IERs + :type iers: Dict[str,IER] + :param node_pol_type: Type of Pattern-Of-Life + :type node_pol_type: NodePOLType + :return: Whether the RED IER is incoming. + :rtype: bool """ node_id = node.node_id From c38dda34b9b3f4917c921876d9b2fc7f71939220 Mon Sep 17 00:00:00 2001 From: Marek Wolan Date: Thu, 6 Jul 2023 10:23:14 +0100 Subject: [PATCH 18/41] Removed duplicated function definitions --- src/primaite/agents/utils.py | 204 ++++------------------------------- 1 file changed, 23 insertions(+), 181 deletions(-) diff --git a/src/primaite/agents/utils.py b/src/primaite/agents/utils.py index 58b422d0..5f5261e0 100644 --- a/src/primaite/agents/utils.py +++ b/src/primaite/agents/utils.py @@ -2,6 +2,7 @@ from typing import Dict, List, Union import numpy as np +from primaite.common.custom_typing import NodeUnion from primaite.common.enums import ( HardwareState, LinkStatus, @@ -310,7 +311,7 @@ def describe_obs_change( return change_string -def _describe_obs_change_helper(obs_change, is_link): +def _describe_obs_change_helper(obs_change: List[int], is_link: bool) -> str: """ Helper funcion to describe what has changed. @@ -319,8 +320,14 @@ def _describe_obs_change_helper(obs_change, is_link): Handles multiple changes e.g. 'ID 1: SERVICE 1 changed to PATCHING. SERVICE 2 set to GOOD.' - TODO: Add params and return in docstring. - TODO: Typehint params and return. + :param obs_change: List of integers generated within the `describe_obs_change` function. It should correspond to one + row of the observation table, and have `-1` at locations where the observation hasn't changed, and the new + status where it has changed. + :type obs_change: List[int] + :param is_link: Whether the row of the observation space corresponds to a link. False means it represents a node. + :type is_link: bool + :return: A human-readable description of the difference between the two observation rows. + :rtype: str """ # Indexes where a change has occured, not including 0th index index_changed = [i for i in range(1, len(obs_change)) if obs_change[i] != -1] @@ -378,65 +385,14 @@ def transform_action_node_enum(action: List[Union[str, int]]) -> List[int]: return new_action -def transform_action_node_readable(action: List[int]) -> List[Union[int, str]]: - """Convert a node action from enumerated format to readable format. - - example: - [1, 3, 1, 0] -> [1, 'SERVICE', 'PATCHING', 0] - - :param action: Raw action with integer encodings - :type action: List[int] - :return: Human-redable version of the action - :rtype: List[Union[int,str]] - """ - action_node_property = NodePOLType(action[1]).name - - if action_node_property == "OPERATING": - property_action = NodeHardwareAction(action[2]).name - elif (action_node_property == "OS" or action_node_property == "SERVICE") and action[2] <= 1: - property_action = NodeSoftwareAction(action[2]).name - else: - property_action = "NONE" - - new_action = [action[0], action_node_property, property_action, action[3]] - return new_action - - -# unused -# def node_action_description(action): -# """ -# Generate string describing a node-based action. - -# TO#DO: Add params and return in docstring. -# TO#DO: Typehint params and return. -# """ -# if isinstance(action[1], (int, np.int64)): -# # transform action to readable format -# action = transform_action_node_readable(action) - -# node_id = action[0] -# node_property = action[1] -# property_action = action[2] -# service_id = action[3] - -# if property_action == "NONE": -# return "" -# if node_property == "OPERATING" or node_property == "OS": -# description = f"NODE {node_id}, {node_property}, SET TO {property_action}" -# elif node_property == "SERVICE": -# description = f"NODE {node_id} FROM SERVICE {service_id}, SET TO {property_action}" -# else: -# return "" - -# return description - - -def transform_action_acl_enum(action): +def transform_action_acl_enum(action: List[Union[int, str]]) -> np.ndarray: """ Convert acl action from readable str format, to enumerated format. - TODO: Add params and return in docstring. - TODO: Typehint params and return. + :param action: ACL-based action expressed as a list of human-readable ints and strings + :type action: List[Union[int,str]] + :return: The same action but encoded to contain only integers. + :rtype: np.ndarray """ action_decisions = {"NONE": 0, "CREATE": 1, "DELETE": 2} action_permissions = {"DENY": 0, "ALLOW": 1} @@ -454,36 +410,17 @@ def transform_action_acl_enum(action): return new_action -# unused -# def acl_action_description(action): -# """ -# Generate string describing an acl-based action. - -# TODO: Add params and return in docstring. -# TODO: Typehint params and return. -# """ -# if isinstance(action[0], (int, np.int64)): -# # transform action to readable format -# action = transform_action_acl_readable(action) -# if action[0] == "NONE": -# description = "NO ACL RULE APPLIED" -# else: -# description = ( -# f"{action[0]} RULE: {action[1]} traffic from IP {action[2]} to IP {action[3]}," -# f" for protocol/service index {action[4]} on port index {action[5]}" -# ) - -# return description - - -def get_node_of_ip(ip, node_dict): - """ - Get the node ID of an IP address. +def get_node_of_ip(ip: str, node_dict: Dict[str, NodeUnion]) -> str: + """Get the node ID of an IP address. node_dict: dictionary of nodes where key is ID, and value is the node (can be ontained from env.nodes) - TODO: Add params and return in docstring. - TODO: Typehint params and return. + :param ip: The IP address of the node whose ID is required + :type ip: str + :param node_dict: The environment's node registry dictionary + :type node_dict: Dict[str,NodeUnion] + :return: The key from the registry dict that corresponds to the node with the IP adress provided by `ip` + :rtype: str """ for node_key, node_value in node_dict.items(): node_ip = node_value.ip_address @@ -491,101 +428,6 @@ def get_node_of_ip(ip, node_dict): return node_key -def is_valid_node_action(action): - """Is the node action an actual valid action. - - Only uses information about the action to determine if the action has an effect - - Does NOT consider: - - Node ID not valid to perform an operation - e.g. selected node has no service so cannot patch - - Node already being in that state (turning an ON node ON) - - TODO: Add params and return in docstring. - TODO: Typehint params and return. - """ - action_r = transform_action_node_readable(action) - - node_property = action_r[1] - node_action = action_r[2] - - if node_property == "NONE": - return False - if node_action == "NONE": - return False - if node_property == "OPERATING" and node_action == "PATCHING": - # Operating State cannot PATCH - return False - if node_property != "OPERATING" and node_action not in [ - "NONE", - "PATCHING", - ]: - # Software States can only do Nothing or Patch - return False - return True - - -def is_valid_acl_action(action: List[int]) -> bool: - """ - Is the ACL action an actual valid action. - - Only uses information about the action to determine if the action has an effect - - Does NOT consider: - - Trying to create identical rules - - Trying to create a rule which is a subset of another rule (caused by "ANY") - - - :param action: Action to check - :type action: List[int] - :return: Whether the action is valid - :rtype: bool - """ - action_r = transform_action_acl_readable(action) - - action_decision = action_r[0] - action_permission = action_r[1] - action_source_id = action_r[2] - action_destination_id = action_r[3] - - if action_decision == "NONE": - return False - if action_source_id == action_destination_id and action_source_id != "ANY" and action_destination_id != "ANY": - # ACL rule towards itself - return False - if action_permission == "DENY": - # DENY is unnecessary, we can create and delete allow rules instead - # No allow rule = blocked/DENY by feault. ALLOW overrides existing DENY. - return False - - return True - - -def is_valid_acl_action_extra(action: List[int]) -> bool: - """ - Harsher version of valid acl actions, does not allow action. - - :param action: Input action - :type action: List[int] - :return: Whether the action is a valid ACL action - :rtype: bool - """ - if is_valid_acl_action(action) is False: - return False - - action_r = transform_action_acl_readable(action) - action_protocol = action_r[4] - action_port = action_r[5] - - # Don't allow protocols or ports to be ANY - # in the future we might want to do the opposite, and only have ANY option for ports and service - if action_protocol == "ANY": - return False - if action_port == "ANY": - return False - - return True - - def get_new_action(old_action: np.ndarray, action_dict: Dict[int, List]) -> int: """ Get new action (e.g. 32) from old action e.g. [1,1,1,0]. From 87bdaa1ec3f24adbd9b38d22c3a6c69359b6ffbd Mon Sep 17 00:00:00 2001 From: Marek Wolan Date: Thu, 6 Jul 2023 10:34:27 +0100 Subject: [PATCH 19/41] Updated documentation --- src/primaite/acl/access_control_list.py | 2 +- src/primaite/agents/hardcoded_acl.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/primaite/acl/access_control_list.py b/src/primaite/acl/access_control_list.py index f5f70d6c..78ea8a36 100644 --- a/src/primaite/acl/access_control_list.py +++ b/src/primaite/acl/access_control_list.py @@ -34,7 +34,7 @@ class AccessControlList: else: return False - def is_blocked(self, _source_ip_address, _dest_ip_address, _protocol, _port): + def is_blocked(self, _source_ip_address: str, _dest_ip_address: str, _protocol: str, _port: str) -> bool: """ Checks for rules that block a protocol / port. diff --git a/src/primaite/agents/hardcoded_acl.py b/src/primaite/agents/hardcoded_acl.py index a724e10e..f8c571c9 100644 --- a/src/primaite/agents/hardcoded_acl.py +++ b/src/primaite/agents/hardcoded_acl.py @@ -78,7 +78,6 @@ class HardCodedACLAgent(HardCodedAgentSessionABC): dest_node_address = nodes[dest_node_id].ip_address protocol = ier.get_protocol() # e.g. 'TCP' port = ier.get_port() - # I can't find where this function 'get_relevant_rules' is defined... matching_rules = acl.get_relevant_rules(source_node_address, dest_node_address, protocol, port) return matching_rules From 1ade92f55cad37a408d25ea9772ff9fe9029f926 Mon Sep 17 00:00:00 2001 From: Marek Wolan Date: Thu, 6 Jul 2023 15:04:46 +0100 Subject: [PATCH 20/41] Deleted icon --- docs/source/primaite.ico | Bin 3126 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 docs/source/primaite.ico diff --git a/docs/source/primaite.ico b/docs/source/primaite.ico deleted file mode 100644 index 4f8a9cbe34b83692f839d84fb1765a62fc0a2bf2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3126 zcmb7G2~<;O7Phtyh>mrkRz*S7A}R_3WeICQmh5}pd&x^)5+EByHj%}MfFetA#sx(X zizu8Zhyo(Gp~xnJ3Rp(8RgWNb!#d8`4%RJmG1gOiY(3s{-k*QV_uu{A0EXMdF|WxD zMtk_?zWF{GGXrKoZ_Z5%(`ft{`t>z^%s)PGcX)WXs;Vj}DT&MFQYaKAlL=#Ra4<~b zl?vtrK}aMLSONtE1VltcT)K4W zPhj4&Ju)()*XvElLZMJ5lZ$Z)PlU5EF++%Pr33@T5JIL13&(K@ilPF603=eY)lZ*3 zeGlBbTRlBJ2!e1p92kWNA|+%hjfOzE4XfpCN^buKO$XDbS;jEPIBYHxK`~4uHVFb^ z1_lOpb#=Xq_s;6nsZ%T#3-|%HMPjiwGKxzJ;jeJ^bDZa3ZN1!Pw$pS=r`a&3_5Ls z8P(NBjgVDBvQ`wRV0&2o-F%7743&u5T%H`pCTUc128)BDnCXf3_V(XU-~2v$^oY%7 z12T{zETZ*z+4dbfUOet9*r1Fh=)pd&W5t8ufNV5A zC`u>R#)dT>Euk^kpfJ-DPo6ya{S#WP78D4~<6^w=^wA3~HRT7Yo{!wxmAA%c*<5q; z2{WgC?y`6u$!Vdd<3cApn=hwKGB+Eatyi5nURju_Bm|USe!3b~*=N|hEjN=y zBAq;0yGku?*p~@7J|xegP5O*DNwkC(EukhT=t^PeSGKkm=HreV@?w>k8lzd5kFVXo zlNs#GU@$}?5dcD@0RG_MAdN-?aFIwHr4Bz+Q)D%1ERW4dTB+EwUY8!lbzb72(?kfE zA!3FM$&(FjtviZ#=4EB%rN>2} zK~ip@Qo{FGz6>sWGG>ge_3U6eJ#|xVjeb>+*TO=w2SZax@p>Fv4u_%Zmy%cnOsrxIEzH8DoTqf=&CPMI`u zg8Q5=>ij&~=S=Tc(l1y~|JHpu8<&{yA@Rb5Eyj2Id-T!WyOF?ltErs=-%V*6K8q@!!v1zIt`1yY)a>enOHC%ScP~a2VS(@0&beu-R!{pLBDh-?dfDAR zpXA2DanMh6baYBe3iv7y6Ky~^S1@v~hT82i|KN^|V4f(gMyC!-UmbO@IOB)GfnT0I z_FQDGR7ks8s`c@Ty7GSz3=+-XC6w$d;?ee|tAB_M>LfW{qq$}%_&ihu_RY!3DJ?C9 z%mevPjEirl$*yHZTo48ma*>y}_nPFSjJ1i)bpze1GBk zyL~mgv$Y6?B;vE3mn_z3@yptvPEUva#j`NE3;@ADtE;Q~`}?8ffeGkhyxw(DAKkAy z7ImV(9}=7srJQY^El7vlIRJ1kF1}!Biu@>(}WCOemncA44eDrkcg8!}+g^8kA80 z2QnB8kTgI9ng}XzJkhW=HGcT#;alNBhtwh-G3qeDb5m232_HtlhfT=dGy!^EOuotX z-L`LEM}L3Mz>f_zWtJA7PM$bnzV(;9Kv$gVv0#p+)yx?)ET@P<-MX(_DK6Prz-8Rj zkh_$=xOOGD4OkY+@mp}1K(_#;4Mc!oUDudb!_^5C84uSTAY+#2}T@&1kj7fx5A0H`=i;cx)a$p?d2sl4{`0#tdO`9bpC7>vf5d;wk_*4o7++a^>@uiDR z1J_$`^d4zBvSrWa`1Py9VzqLmQVD$!Tn5{L&Y{PHQur6(rlV`ut^xCJDkh2oK99$z z(_x3j;_%rV0V@ Date: Thu, 6 Jul 2023 15:05:39 +0100 Subject: [PATCH 21/41] undeleted api (lol) --- docs/api.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 docs/api.rst diff --git a/docs/api.rst b/docs/api.rst new file mode 100644 index 00000000..df2bc193 --- /dev/null +++ b/docs/api.rst @@ -0,0 +1,16 @@ +.. + DO NOT DELETE THIS FILE! It contains the all-important `.. autosummary::` directive with `:recursive:` option, without + which API documentation wouldn't get extracted from docstrings by the `sphinx.ext.autosummary` engine. It is hidden + (not declared in any toctree) to remove an unnecessary intermediate page; index.rst instead points directly to the + package page. DO NOT REMOVE THIS FILE! + + Credit to https://github.com/JamesALeedham/Sphinx-Autosummary-Recursion for the custom templates. +.. + +.. autosummary:: + :toctree: source/_autosummary + :template: custom-module-template.rst + :recursive: + + primaite + tests From 82a5122276f40d9a1fe9f0e7b4d13d594fd26201 Mon Sep 17 00:00:00 2001 From: Marek Wolan Date: Thu, 6 Jul 2023 15:18:33 +0100 Subject: [PATCH 22/41] Add __init__ to class special members doc --- docs/_templates/custom-class-template.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/_templates/custom-class-template.rst b/docs/_templates/custom-class-template.rst index 01e5299d..8a539bc9 100644 --- a/docs/_templates/custom-class-template.rst +++ b/docs/_templates/custom-class-template.rst @@ -9,7 +9,7 @@ :members: :show-inheritance: :inherited-members: - :special-members: __call__, __add__, __mul__ + :special-members: __init__, __call__, __add__, __mul__ {% block methods %} {% if methods %} From 2a08d3a2a5c6bb13b6ce790e371d97e8f0071bbd Mon Sep 17 00:00:00 2001 From: Marek Wolan Date: Thu, 6 Jul 2023 15:18:49 +0100 Subject: [PATCH 23/41] Removed reference to file that no longer exists --- docs/conf.py | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index d6923446..51b745cf 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -55,4 +55,3 @@ exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] html_theme = "furo" html_static_path = ["_static"] -html_favicon = "source/primaite.ico" From 86725064ec9e085f3e94b6ad0f98161417da16f7 Mon Sep 17 00:00:00 2001 From: Marek Wolan Date: Thu, 6 Jul 2023 16:08:51 +0100 Subject: [PATCH 24/41] Added docstrings to class intialisers --- src/primaite/acl/access_control_list.py | 4 +- src/primaite/acl/acl_rule.py | 18 ++++---- src/primaite/agents/agent.py | 24 ++++++++++- src/primaite/agents/hardcoded_acl.py | 3 +- src/primaite/agents/rllib.py | 12 ++++++ src/primaite/agents/sb3.py | 12 ++++++ src/primaite/agents/simple.py | 12 ++++-- src/primaite/common/protocol.py | 20 +++++---- src/primaite/common/service.py | 14 ++++--- src/primaite/config/training_config.py | 12 ++++-- src/primaite/environment/observations.py | 41 ++++++++++++++----- src/primaite/environment/primaite_env.py | 3 +- src/primaite/links/link.py | 18 ++++---- src/primaite/nodes/active_node.py | 27 ++++++------ src/primaite/nodes/node.py | 20 +++++---- .../nodes/node_state_instruction_green.py | 22 +++++----- .../nodes/node_state_instruction_red.py | 30 +++++++------- src/primaite/nodes/passive_node.py | 20 +++++---- src/primaite/nodes/service_node.py | 26 ++++++------ src/primaite/pol/ier.py | 28 +++++++------ src/primaite/primaite_session.py | 6 ++- src/primaite/transactions/transaction.py | 3 +- src/primaite/utils/session_output_writer.py | 15 ++++++- 23 files changed, 253 insertions(+), 137 deletions(-) diff --git a/src/primaite/acl/access_control_list.py b/src/primaite/acl/access_control_list.py index f6ae3fad..e1d6aa74 100644 --- a/src/primaite/acl/access_control_list.py +++ b/src/primaite/acl/access_control_list.py @@ -9,10 +9,12 @@ class AccessControlList: """Access Control List class.""" def __init__(self): + """Initialise an empty AccessControlList.""" self.acl: Dict[str, AccessControlList] = {} # A dictionary of ACL Rules def check_address_match(self, _rule, _source_ip_address, _dest_ip_address): - """Checks for IP address matches. + """ + Checks for IP address matches. Args: _rule: The rule being checked diff --git a/src/primaite/acl/acl_rule.py b/src/primaite/acl/acl_rule.py index 49aebc1b..117c9457 100644 --- a/src/primaite/acl/acl_rule.py +++ b/src/primaite/acl/acl_rule.py @@ -3,16 +3,18 @@ class ACLRule: - """Access Control List Rule class. - - :param _permission: The permission (ALLOW or DENY) - :param _source_ip: The source IP address - :param _dest_ip: The destination IP address - :param _protocol: The rule protocol - :param _port: The rule port - """ + """Access Control List Rule class.""" def __init__(self, _permission, _source_ip, _dest_ip, _protocol, _port): + """ + Initialise an ACL Rule. + + :param _permission: The permission (ALLOW or DENY) + :param _source_ip: The source IP address + :param _dest_ip: The destination IP address + :param _protocol: The rule protocol + :param _port: The rule port + """ self.permission = _permission self.source_ip = _source_ip self.dest_ip = _dest_ip diff --git a/src/primaite/agents/agent.py b/src/primaite/agents/agent.py index c68b6df0..7073d795 100644 --- a/src/primaite/agents/agent.py +++ b/src/primaite/agents/agent.py @@ -38,7 +38,8 @@ def get_session_path(session_timestamp: datetime) -> Path: class AgentSessionABC(ABC): - """An ABC that manages training and/or evaluation of agents in PrimAITE. + """ + An ABC that manages training and/or evaluation of agents in PrimAITE. This class cannot be directly instantiated and must be inherited from with all implemented abstract methods implemented. @@ -46,6 +47,15 @@ class AgentSessionABC(ABC): @abstractmethod def __init__(self, training_config_path, lay_down_config_path): + """ + Initialise an agent session from config files. + + :param training_config_path: YAML file containing configurable items defined in + `primaite.config.training_config.TrainingConfig` + :type training_config_path: Union[path, str] + :param lay_down_config_path: YAML file containing configurable items for generating network laydown. + :type lay_down_config_path: Union[path, str] + """ if not isinstance(training_config_path, Path): training_config_path = Path(training_config_path) self._training_config_path: Final[Union[Path, str]] = training_config_path @@ -289,13 +299,23 @@ class AgentSessionABC(ABC): class HardCodedAgentSessionABC(AgentSessionABC): - """An Agent Session ABC for evaluation deterministic agents. + """ + An Agent Session ABC for evaluation deterministic agents. This class cannot be directly instantiated and must be inherited from with all implemented abstract methods implemented. """ def __init__(self, training_config_path, lay_down_config_path): + """ + Initialise a hardcoded agent session. + + :param training_config_path: YAML file containing configurable items defined in + `primaite.config.training_config.TrainingConfig` + :type training_config_path: Union[path, str] + :param lay_down_config_path: YAML file containing configurable items for generating network laydown. + :type lay_down_config_path: Union[path, str] + """ super().__init__(training_config_path, lay_down_config_path) self._setup() diff --git a/src/primaite/agents/hardcoded_acl.py b/src/primaite/agents/hardcoded_acl.py index 9ed9fd28..5cc06bdc 100644 --- a/src/primaite/agents/hardcoded_acl.py +++ b/src/primaite/agents/hardcoded_acl.py @@ -23,7 +23,8 @@ class HardCodedACLAgent(HardCodedAgentSessionABC): return self._calculate_action_full_view(obs) def get_blocked_green_iers(self, green_iers, acl, nodes): - """Get blocked green IERs. + """ + Get blocked green IERs. TODO: Add params and return in docstring. TODO: Typehint params and return. diff --git a/src/primaite/agents/rllib.py b/src/primaite/agents/rllib.py index 20503459..044b760f 100644 --- a/src/primaite/agents/rllib.py +++ b/src/primaite/agents/rllib.py @@ -42,6 +42,18 @@ class RLlibAgent(AgentSessionABC): """An AgentSession class that implements a Ray RLlib agent.""" def __init__(self, training_config_path, lay_down_config_path): + """ + Initialise the RLLib Agent training session. + + :param training_config_path: YAML file containing configurable items defined in + `primaite.config.training_config.TrainingConfig` + :type training_config_path: Union[path, str] + :param lay_down_config_path: YAML file containing configurable items for generating network laydown. + :type lay_down_config_path: Union[path, str] + :raises ValueError: If the training config contains an unexpected value for agent_framework (should be "RLLIB") + :raises ValueError: If the training config contains an unexpected value for agent_identifies (should be `PPO` + or `A2C`) + """ super().__init__(training_config_path, lay_down_config_path) if not self._training_config.agent_framework == AgentFramework.RLLIB: msg = f"Expected RLLIB agent_framework, " f"got {self._training_config.agent_framework}" diff --git a/src/primaite/agents/sb3.py b/src/primaite/agents/sb3.py index 58148d1f..b81a0a18 100644 --- a/src/primaite/agents/sb3.py +++ b/src/primaite/agents/sb3.py @@ -19,6 +19,18 @@ class SB3Agent(AgentSessionABC): """An AgentSession class that implements a Stable Baselines3 agent.""" def __init__(self, training_config_path, lay_down_config_path): + """ + Initialise the SB3 Agent training session. + + :param training_config_path: YAML file containing configurable items defined in + `primaite.config.training_config.TrainingConfig` + :type training_config_path: Union[path, str] + :param lay_down_config_path: YAML file containing configurable items for generating network laydown. + :type lay_down_config_path: Union[path, str] + :raises ValueError: If the training config contains an unexpected value for agent_framework (should be "SB3") + :raises ValueError: If the training config contains an unexpected value for agent_identifies (should be `PPO` + or `A2C`) + """ super().__init__(training_config_path, lay_down_config_path) if not self._training_config.agent_framework == AgentFramework.SB3: msg = f"Expected SB3 agent_framework, " f"got {self._training_config.agent_framework}" diff --git a/src/primaite/agents/simple.py b/src/primaite/agents/simple.py index df93e56d..b429a2f5 100644 --- a/src/primaite/agents/simple.py +++ b/src/primaite/agents/simple.py @@ -3,7 +3,8 @@ from primaite.agents.utils import get_new_action, transform_action_acl_enum, tra class RandomAgent(HardCodedAgentSessionABC): - """A Random Agent. + """ + A Random Agent. Get a completely random action from the action space. """ @@ -13,7 +14,8 @@ class RandomAgent(HardCodedAgentSessionABC): class DummyAgent(HardCodedAgentSessionABC): - """A Dummy Agent. + """ + A Dummy Agent. All action spaces setup so dummy action is always 0 regardless of action type used. """ @@ -23,7 +25,8 @@ class DummyAgent(HardCodedAgentSessionABC): class DoNothingACLAgent(HardCodedAgentSessionABC): - """A do nothing ACL agent. + """ + A do nothing ACL agent. A valid ACL action that has no effect; does nothing. """ @@ -37,7 +40,8 @@ class DoNothingACLAgent(HardCodedAgentSessionABC): class DoNothingNodeAgent(HardCodedAgentSessionABC): - """A do nothing Node agent. + """ + A do nothing Node agent. A valid Node action that has no effect; does nothing. """ diff --git a/src/primaite/common/protocol.py b/src/primaite/common/protocol.py index ec67caa3..ad6a1d83 100644 --- a/src/primaite/common/protocol.py +++ b/src/primaite/common/protocol.py @@ -3,17 +3,21 @@ class Protocol(object): - """Protocol class. - - :param _name: The protocol name - """ + """Protocol class.""" def __init__(self, _name): + """ + Initialise a protocol. + + :param _name: The name of the protocol + :type _name: str + """ self.name = _name self.load = 0 # bps def get_name(self): - """Gets the protocol name. + """ + Gets the protocol name. Returns: The protocol name @@ -21,7 +25,8 @@ class Protocol(object): return self.name def get_load(self): - """Gets the protocol load. + """ + Gets the protocol load. Returns: The protocol load (bps) @@ -29,7 +34,8 @@ class Protocol(object): return self.load def add_load(self, _load): - """Adds load to the protocol. + """ + Adds load to the protocol. Args: _load: The load to add diff --git a/src/primaite/common/service.py b/src/primaite/common/service.py index 2d08a3c5..258ac8f9 100644 --- a/src/primaite/common/service.py +++ b/src/primaite/common/service.py @@ -5,14 +5,16 @@ from primaite.common.enums import SoftwareState class Service(object): - """Service class. - - :param name: The service name. - :param port: The service port. - :param software_state: The service SoftwareState. - """ + """Service class.""" def __init__(self, name: str, port: str, software_state: SoftwareState): + """ + Initialise a service. + + :param name: The service name. + :param port: The service port. + :param software_state: The service SoftwareState. + """ self.name = name self.port = port self.software_state = software_state diff --git a/src/primaite/config/training_config.py b/src/primaite/config/training_config.py index 040ef6fa..7bdf7995 100644 --- a/src/primaite/config/training_config.py +++ b/src/primaite/config/training_config.py @@ -24,7 +24,8 @@ _EXAMPLE_TRAINING: Final[Path] = USERS_CONFIG_DIR / "example_config" / "training def main_training_config_path() -> Path: - """The path to the example training_config_main.yaml file. + """ + The path to the example training_config_main.yaml file. :return: The file path. """ @@ -234,7 +235,8 @@ class TrainingConfig: def load(file_path: Union[str, Path], legacy_file: bool = False) -> TrainingConfig: - """Read in a training config yaml file. + """ + Read in a training config yaml file. :param file_path: The config file path. :param legacy_file: True if the config file is legacy format, otherwise @@ -278,7 +280,8 @@ def convert_legacy_training_config_dict( action_type: ActionType = ActionType.ANY, num_steps: int = 256, ) -> Dict[str, Any]: - """Convert a legacy training config dict to the new format. + """ + Convert a legacy training config dict to the new format. :param legacy_config_dict: A legacy training config dict. :param agent_framework: The agent framework to use as legacy training configs don't have agent_framework values. @@ -305,7 +308,8 @@ def convert_legacy_training_config_dict( def _get_new_key_from_legacy(legacy_key: str) -> str: - """Maps legacy training config keys to the new format keys. + """ + Maps legacy training config keys to the new format keys. :param legacy_key: A legacy training config key. :return: The mapped key. diff --git a/src/primaite/environment/observations.py b/src/primaite/environment/observations.py index 4d027326..28e85b7f 100644 --- a/src/primaite/environment/observations.py +++ b/src/primaite/environment/observations.py @@ -25,6 +25,12 @@ class AbstractObservationComponent(ABC): @abstractmethod def __init__(self, env: "Primaite"): + """ + Initialise observation component. + + :param env: Primaite training environment. + :type env: Primaite + """ _LOGGER.info(f"Initialising {self} observation component") self.env: "Primaite" = env self.space: spaces.Space @@ -68,6 +74,11 @@ class NodeLinkTable(AbstractObservationComponent): _DATA_TYPE: type = np.int64 def __init__(self, env: "Primaite"): + """Initialise a NodeLinkTable observation space component. + + :param env: Training environment. + :type env: Primaite + """ super().__init__(env) # 1. Define the shape of your observation space component @@ -192,14 +203,17 @@ class NodeStatuses(AbstractObservationComponent): node2 serviceN state (one for each service), ... ] - - :param env: The environment that forms the basis of the observations - :type env: Primaite """ _DATA_TYPE: type = np.int64 def __init__(self, env: "Primaite"): + """ + Initialise a NodeStatuses observation component. + + :param env: Training environment. + :type env: Primaite + """ super().__init__(env) # 1. Define the shape of your observation space component @@ -288,14 +302,6 @@ class LinkTrafficLevels(AbstractObservationComponent): The lowest category always corresponds to no traffic and the highest category to the link being at max capacity. Any amount of traffic between 0% and 100% (exclusive) is divided evenly into the remaining categories. - :param env: The environment that forms the basis of the observations - :type env: Primaite - :param combine_service_traffic: Whether to consider total traffic on the link, or each protocol individually, - defaults to False - :type combine_service_traffic: bool, optional - :param quantisation_levels: How many bands to consider when converting the traffic amount to a categorical value , - defaults to 5 - :type quantisation_levels: int, optional """ _DATA_TYPE: type = np.int64 @@ -306,6 +312,18 @@ class LinkTrafficLevels(AbstractObservationComponent): combine_service_traffic: bool = False, quantisation_levels: int = 5, ): + """ + Initialise a LinkTrafficLevels observation component. + + :param env: The environment that forms the basis of the observations + :type env: Primaite + :param combine_service_traffic: Whether to consider total traffic on the link, or each protocol individually, + defaults to False + :type combine_service_traffic: bool, optional + :param quantisation_levels: How many bands to consider when converting the traffic amount to a categorical + value, defaults to 5 + :type quantisation_levels: int, optional + """ if quantisation_levels < 3: _msg = ( f"quantisation_levels must be 3 or more because the lowest and highest levels are " @@ -390,6 +408,7 @@ class ObservationsHandler: } def __init__(self): + """Initialise the observation handler.""" self.registered_obs_components: List[AbstractObservationComponent] = [] # internal the observation space (unflattened version of space if flatten=True) diff --git a/src/primaite/environment/primaite_env.py b/src/primaite/environment/primaite_env.py index 29662988..825818fd 100644 --- a/src/primaite/environment/primaite_env.py +++ b/src/primaite/environment/primaite_env.py @@ -67,7 +67,8 @@ class Primaite(Env): session_path: Path, timestamp_str: str, ): - """The Primaite constructor. + """ + The Primaite constructor. :param training_config_path: The training config filepath. :param lay_down_config_path: The lay down config filepath. diff --git a/src/primaite/links/link.py b/src/primaite/links/link.py index ff73ccc8..5892b8e2 100644 --- a/src/primaite/links/link.py +++ b/src/primaite/links/link.py @@ -6,16 +6,18 @@ from primaite.common.protocol import Protocol class Link(object): - """Link class. - - :param _id: The IER id - :param _bandwidth: The bandwidth of the link (bps) - :param _source_node_name: The name of the source node - :param _dest_node_name: The name of the destination node - :param _protocols: The protocols to add to the link - """ + """Link class.""" def __init__(self, _id, _bandwidth, _source_node_name, _dest_node_name, _services): + """ + Initialise a Link within the simulated network. + + :param _id: The IER id + :param _bandwidth: The bandwidth of the link (bps) + :param _source_node_name: The name of the source node + :param _dest_node_name: The name of the destination node + :param _protocols: The protocols to add to the link + """ self.id = _id self.bandwidth = _bandwidth self.source_node_name = _source_node_name diff --git a/src/primaite/nodes/active_node.py b/src/primaite/nodes/active_node.py index e20ce0e0..3789b7a4 100644 --- a/src/primaite/nodes/active_node.py +++ b/src/primaite/nodes/active_node.py @@ -11,19 +11,7 @@ _LOGGER: Final[logging.Logger] = logging.getLogger(__name__) class ActiveNode(Node): - """Active Node class. - - :param node_id: The node ID - :param name: The node name - :param node_type: The node type (enum) - :param priority: The node priority (enum) - :param hardware_state: The node Hardware State - :param ip_address: The node IP address - :param software_state: The node Software State - :param file_system_state: The node file system state - :param config_values: The config values - - """ + """Active Node class.""" def __init__( self, @@ -37,6 +25,19 @@ class ActiveNode(Node): file_system_state: FileSystemState, config_values: TrainingConfig, ): + """ + Initialise an active node. + + :param node_id: The node ID + :param name: The node name + :param node_type: The node type (enum) + :param priority: The node priority (enum) + :param hardware_state: The node Hardware State + :param ip_address: The node IP address + :param software_state: The node Software State + :param file_system_state: The node file system state + :param config_values: The config values + """ super().__init__(node_id, name, node_type, priority, hardware_state, config_values) self.ip_address: str = ip_address # Related to Software diff --git a/src/primaite/nodes/node.py b/src/primaite/nodes/node.py index b54989bf..9fd5b719 100644 --- a/src/primaite/nodes/node.py +++ b/src/primaite/nodes/node.py @@ -7,15 +7,7 @@ from primaite.config.training_config import TrainingConfig class Node: - """Node class. - - :param node_id: The node id. - :param name: The name of the node. - :param node_type: The type of the node. - :param priority: The priority of the node. - :param hardware_state: The state of the node. - :param config_values: Config values. - """ + """Node class.""" def __init__( self, @@ -26,6 +18,16 @@ class Node: hardware_state: HardwareState, config_values: TrainingConfig, ): + """ + Initialise a node. + + :param node_id: The node id. + :param name: The name of the node. + :param node_type: The type of the node. + :param priority: The priority of the node. + :param hardware_state: The state of the node. + :param config_values: Config values. + """ self.node_id: Final[str] = node_id self.name: Final[str] = name self.node_type: Final[NodeType] = node_type diff --git a/src/primaite/nodes/node_state_instruction_green.py b/src/primaite/nodes/node_state_instruction_green.py index 0faef627..da4be35e 100644 --- a/src/primaite/nodes/node_state_instruction_green.py +++ b/src/primaite/nodes/node_state_instruction_green.py @@ -3,16 +3,7 @@ class NodeStateInstructionGreen(object): - """The Node State Instruction class. - - :param _id: The node state instruction id - :param _start_step: The start step of the instruction - :param _end_step: The end step of the instruction - :param _node_id: The id of the associated node - :param _node_pol_type: The pattern of life type - :param _service_name: The service name - :param _state: The state (node or service) - """ + """The Node State Instruction class.""" def __init__( self, @@ -24,6 +15,17 @@ class NodeStateInstructionGreen(object): _service_name, _state, ): + """ + Initialise the Node State Instruction. + + :param _id: The node state instruction id + :param _start_step: The start step of the instruction + :param _end_step: The end step of the instruction + :param _node_id: The id of the associated node + :param _node_pol_type: The pattern of life type + :param _service_name: The service name + :param _state: The state (node or service) + """ self.id = _id self.start_step = _start_step self.end_step = _end_step diff --git a/src/primaite/nodes/node_state_instruction_red.py b/src/primaite/nodes/node_state_instruction_red.py index 8308a1c0..f8ce4e74 100644 --- a/src/primaite/nodes/node_state_instruction_red.py +++ b/src/primaite/nodes/node_state_instruction_red.py @@ -7,20 +7,7 @@ from primaite.common.enums import NodePOLType @dataclass() class NodeStateInstructionRed(object): - """The Node State Instruction class. - - :param _id: The node state instruction id - :param _start_step: The start step of the instruction - :param _end_step: The end step of the instruction - :param _target_node_id: The id of the associated node - :param -pol_initiator: The way the PoL is applied (DIRECT, IER or SERVICE) - :param _pol_type: The pattern of life type - :param pol_protocol: The pattern of life protocol/service affected - :param _pol_state: The state (node or service) - :param _pol_source_node_id: The source node Id (used for initiator type SERVICE) - :param _pol_source_node_service: The source node service (used for initiator type SERVICE) - :param _pol_source_node_service_state: The source node service state (used for initiator type SERVICE) - """ + """The Node State Instruction class.""" def __init__( self, @@ -36,6 +23,21 @@ class NodeStateInstructionRed(object): _pol_source_node_service, _pol_source_node_service_state, ): + """ + Initialise the Node State Instruction for the red agent. + + :param _id: The node state instruction id + :param _start_step: The start step of the instruction + :param _end_step: The end step of the instruction + :param _target_node_id: The id of the associated node + :param -pol_initiator: The way the PoL is applied (DIRECT, IER or SERVICE) + :param _pol_type: The pattern of life type + :param pol_protocol: The pattern of life protocol/service affected + :param _pol_state: The state (node or service) + :param _pol_source_node_id: The source node Id (used for initiator type SERVICE) + :param _pol_source_node_service: The source node service (used for initiator type SERVICE) + :param _pol_source_node_service_state: The source node service state (used for initiator type SERVICE) + """ self.id = _id self.start_step = _start_step self.end_step = _end_step diff --git a/src/primaite/nodes/passive_node.py b/src/primaite/nodes/passive_node.py index fa289593..13b2d6ad 100644 --- a/src/primaite/nodes/passive_node.py +++ b/src/primaite/nodes/passive_node.py @@ -6,15 +6,7 @@ from primaite.nodes.node import Node class PassiveNode(Node): - """The Passive Node class. - - :param node_id: The node id. - :param name: The name of the node. - :param node_type: The type of the node. - :param priority: The priority of the node. - :param hardware_state: The state of the node. - :param config_values: Config values. - """ + """The Passive Node class.""" def __init__( self, @@ -25,6 +17,16 @@ class PassiveNode(Node): hardware_state: HardwareState, config_values: TrainingConfig, ): + """ + Initialise a passive node. + + :param node_id: The node id. + :param name: The name of the node. + :param node_type: The type of the node. + :param priority: The priority of the node. + :param hardware_state: The state of the node. + :param config_values: Config values. + """ # Pass through to Super for now super().__init__(node_id, name, node_type, priority, hardware_state, config_values) diff --git a/src/primaite/nodes/service_node.py b/src/primaite/nodes/service_node.py index db435c7d..7632e944 100644 --- a/src/primaite/nodes/service_node.py +++ b/src/primaite/nodes/service_node.py @@ -12,18 +12,7 @@ _LOGGER: Final[logging.Logger] = logging.getLogger(__name__) class ServiceNode(ActiveNode): - """ServiceNode class. - - :param node_id: The node ID - :param name: The node name - :param node_type: The node type (enum) - :param priority: The node priority (enum) - :param hardware_state: The node Hardware State - :param ip_address: The node IP address - :param software_state: The node Software State - :param file_system_state: The node file system state - :param config_values: The config values - """ + """ServiceNode class.""" def __init__( self, @@ -37,6 +26,19 @@ class ServiceNode(ActiveNode): file_system_state: FileSystemState, config_values: TrainingConfig, ): + """ + Initialise a Service Node. + + :param node_id: The node ID + :param name: The node name + :param node_type: The node type (enum) + :param priority: The node priority (enum) + :param hardware_state: The node Hardware State + :param ip_address: The node IP address + :param software_state: The node Software State + :param file_system_state: The node file system state + :param config_values: The config values + """ super().__init__( node_id, name, diff --git a/src/primaite/pol/ier.py b/src/primaite/pol/ier.py index bfbc9a31..913a06da 100644 --- a/src/primaite/pol/ier.py +++ b/src/primaite/pol/ier.py @@ -6,19 +6,7 @@ Used to represent an information flow from source to destination. class IER(object): - """Information Exchange Requirement class. - - :param _id: The IER id - :param _start_step: The step when this IER should start - :param _end_step: The step when this IER should end - :param _load: The load this IER should put on a link (bps) - :param _protocol: The protocol of this IER - :param _port: The port this IER runs on - :param _source_node_id: The source node ID - :param _dest_node_id: The destination node ID - :param _mission_criticality: Criticality of this IER to the mission (0 none, 5 mission critical) - :param _running: Indicates whether the IER is currently running - """ + """Information Exchange Requirement class.""" def __init__( self, @@ -33,6 +21,20 @@ class IER(object): _mission_criticality, _running=False, ): + """ + Initialise an Information Exchange Request. + + :param _id: The IER id + :param _start_step: The step when this IER should start + :param _end_step: The step when this IER should end + :param _load: The load this IER should put on a link (bps) + :param _protocol: The protocol of this IER + :param _port: The port this IER runs on + :param _source_node_id: The source node ID + :param _dest_node_id: The destination node ID + :param _mission_criticality: Criticality of this IER to the mission (0 none, 5 mission critical) + :param _running: Indicates whether the IER is currently running + """ self.id = _id self.start_step = _start_step self.end_step = _end_step diff --git a/src/primaite/primaite_session.py b/src/primaite/primaite_session.py index 4aa8476a..1bfb7403 100644 --- a/src/primaite/primaite_session.py +++ b/src/primaite/primaite_session.py @@ -19,7 +19,8 @@ _LOGGER = getLogger(__name__) class PrimaiteSession: - """The PrimaiteSession class. + """ + The PrimaiteSession class. Provides a single learning and evaluation entry point for all training and lay down configurations. """ @@ -29,7 +30,8 @@ class PrimaiteSession: training_config_path: Union[str, Path], lay_down_config_path: Union[str, Path], ): - """The PrimaiteSession constructor. + """ + The PrimaiteSession constructor. :param training_config_path: The training config path. :param lay_down_config_path: The lay down config path. diff --git a/src/primaite/transactions/transaction.py b/src/primaite/transactions/transaction.py index 21d4ee05..a74ef4f9 100644 --- a/src/primaite/transactions/transaction.py +++ b/src/primaite/transactions/transaction.py @@ -10,7 +10,8 @@ class Transaction(object): """Transaction class.""" def __init__(self, agent_identifier: AgentIdentifier, episode_number: int, step_number: int): - """Transaction constructor. + """ + Transaction constructor. :param agent_identifier: An identifier for the agent in use :param episode_number: The episode number diff --git a/src/primaite/utils/session_output_writer.py b/src/primaite/utils/session_output_writer.py index 939ebdb5..5852a84d 100644 --- a/src/primaite/utils/session_output_writer.py +++ b/src/primaite/utils/session_output_writer.py @@ -12,7 +12,8 @@ _LOGGER: Logger = getLogger(__name__) class SessionOutputWriter: - """A session output writer class. + """ + A session output writer class. Is used to write session outputs to csv file. """ @@ -28,6 +29,18 @@ class SessionOutputWriter: transaction_writer: bool = False, learning_session: bool = True, ): + """ + Initialise the Session Output Writer. + + :param env: PrimAITE gym environment. + :type env: Primaite + :param transaction_writer: If `true`, this will output a full account of every transaction taken by the agent. + If `false` it will output the average reward per episode, defaults to False + :type transaction_writer: bool, optional + :param learning_session: Set to `true` to indicate that the current session is a training session. This + determines the name of the folder which contains the final output csv. Defaults to True + :type learning_session: bool, optional + """ self._env = env self.transaction_writer = transaction_writer self.learning_session = learning_session From f4b98542b664de64e4f362c76a8ee8d680582387 Mon Sep 17 00:00:00 2001 From: Marek Wolan Date: Fri, 7 Jul 2023 10:28:00 +0100 Subject: [PATCH 25/41] Standardise docstring summary line placement. --- src/primaite/__init__.py | 6 +- src/primaite/acl/access_control_list.py | 12 ++-- src/primaite/acl/acl_rule.py | 18 +++-- src/primaite/agents/agent.py | 15 ++-- src/primaite/agents/hardcoded_acl.py | 24 ++++--- src/primaite/agents/hardcoded_node.py | 3 +- src/primaite/agents/rllib.py | 6 +- src/primaite/agents/sb3.py | 6 +- src/primaite/agents/utils.py | 63 +++++++++++------ src/primaite/cli.py | 18 +++-- src/primaite/config/lay_down_config.py | 18 +++-- src/primaite/config/training_config.py | 3 +- src/primaite/data_viz/session_plots.py | 3 +- src/primaite/environment/observations.py | 36 ++++++---- src/primaite/environment/primaite_env.py | 69 ++++++++++++------- src/primaite/environment/reward.py | 15 ++-- src/primaite/links/link.py | 24 ++++--- src/primaite/main.py | 3 +- src/primaite/nodes/active_node.py | 15 ++-- .../nodes/node_state_instruction_green.py | 18 +++-- .../nodes/node_state_instruction_red.py | 30 +++++--- src/primaite/nodes/passive_node.py | 3 +- src/primaite/nodes/service_node.py | 21 ++++-- src/primaite/notebooks/__init__.py | 3 +- src/primaite/pol/green_pol.py | 6 +- src/primaite/pol/ier.py | 36 ++++++---- src/primaite/pol/red_agent_pol.py | 9 ++- src/primaite/primaite_session.py | 6 +- src/primaite/setup/reset_demo_notebooks.py | 3 +- src/primaite/setup/reset_example_configs.py | 3 +- src/primaite/setup/setup_app_dirs.py | 3 +- src/primaite/transactions/transaction.py | 9 ++- src/primaite/utils/package_data.py | 3 +- src/primaite/utils/session_output_reader.py | 3 +- src/primaite/utils/session_output_writer.py | 3 +- tests/test_observation_space.py | 9 ++- 36 files changed, 350 insertions(+), 175 deletions(-) diff --git a/src/primaite/__init__.py b/src/primaite/__init__.py index a2d157c6..030860d8 100644 --- a/src/primaite/__init__.py +++ b/src/primaite/__init__.py @@ -66,7 +66,8 @@ Users PrimAITE Sessions are stored at: ``~/primaite/sessions``. # region Setup Logging class _LevelFormatter(Formatter): - """A custom level-specific formatter. + """ + A custom level-specific formatter. Credit to: https://stackoverflow.com/a/68154386 """ @@ -134,7 +135,8 @@ _LOGGER.addHandler(_FILE_HANDLER) def getLogger(name: str) -> Logger: # noqa - """Get a PrimAITE logger. + """ + Get a PrimAITE logger. :param name: The logger name. Use ``__name__``. :return: An instance of :py:class:`logging.Logger` with the PrimAITE diff --git a/src/primaite/acl/access_control_list.py b/src/primaite/acl/access_control_list.py index e1d6aa74..3ac9a8af 100644 --- a/src/primaite/acl/access_control_list.py +++ b/src/primaite/acl/access_control_list.py @@ -35,7 +35,8 @@ class AccessControlList: return False def is_blocked(self, _source_ip_address, _dest_ip_address, _protocol, _port): - """Checks for rules that block a protocol / port. + """ + Checks for rules that block a protocol / port. Args: _source_ip_address: the source IP address to check @@ -61,7 +62,8 @@ class AccessControlList: return True def add_rule(self, _permission, _source_ip, _dest_ip, _protocol, _port): - """Adds a new rule. + """ + Adds a new rule. Args: _permission: the permission value (e.g. "ALLOW" or "DENY") @@ -75,7 +77,8 @@ class AccessControlList: self.acl[hash_value] = new_rule def remove_rule(self, _permission, _source_ip, _dest_ip, _protocol, _port): - """Removes a rule. + """ + Removes a rule. Args: _permission: the permission value (e.g. "ALLOW" or "DENY") @@ -97,7 +100,8 @@ class AccessControlList: self.acl.clear() def get_dictionary_hash(self, _permission, _source_ip, _dest_ip, _protocol, _port): - """Produces a hash value for a rule. + """ + Produces a hash value for a rule. Args: _permission: the permission value (e.g. "ALLOW" or "DENY") diff --git a/src/primaite/acl/acl_rule.py b/src/primaite/acl/acl_rule.py index 117c9457..a1fd93f2 100644 --- a/src/primaite/acl/acl_rule.py +++ b/src/primaite/acl/acl_rule.py @@ -22,7 +22,8 @@ class ACLRule: self.port = _port def __hash__(self): - """Override the hash function. + """ + Override the hash function. Returns: Returns hash of core parameters. @@ -38,7 +39,8 @@ class ACLRule: ) def get_permission(self): - """Gets the permission attribute. + """ + Gets the permission attribute. Returns: Returns permission attribute @@ -46,7 +48,8 @@ class ACLRule: return self.permission def get_source_ip(self): - """Gets the source IP address attribute. + """ + Gets the source IP address attribute. Returns: Returns source IP address attribute @@ -54,7 +57,8 @@ class ACLRule: return self.source_ip def get_dest_ip(self): - """Gets the desintation IP address attribute. + """ + Gets the desintation IP address attribute. Returns: Returns destination IP address attribute @@ -62,7 +66,8 @@ class ACLRule: return self.dest_ip def get_protocol(self): - """Gets the protocol attribute. + """ + Gets the protocol attribute. Returns: Returns protocol attribute @@ -70,7 +75,8 @@ class ACLRule: return self.protocol def get_port(self): - """Gets the port attribute. + """ + Gets the port attribute. Returns: Returns port attribute diff --git a/src/primaite/agents/agent.py b/src/primaite/agents/agent.py index 7073d795..3b093f86 100644 --- a/src/primaite/agents/agent.py +++ b/src/primaite/agents/agent.py @@ -21,7 +21,8 @@ _LOGGER = getLogger(__name__) def get_session_path(session_timestamp: datetime) -> Path: - """Get the directory path the session will output to. + """ + Get the directory path the session will output to. This is set in the format of: ~/primaite/sessions//_. @@ -194,7 +195,8 @@ class AgentSessionABC(ABC): self, **kwargs, ): - """Train the agent. + """ + Train the agent. :param kwargs: Any agent-specific key-word args to be passed. """ @@ -211,7 +213,8 @@ class AgentSessionABC(ABC): self, **kwargs, ): - """Evaluate the agent. + """ + Evaluate the agent. :param kwargs: Any agent-specific key-word args to be passed. """ @@ -340,7 +343,8 @@ class HardCodedAgentSessionABC(AgentSessionABC): self, **kwargs, ): - """Train the agent. + """ + Train the agent. :param kwargs: Any agent-specific key-word args to be passed. """ @@ -354,7 +358,8 @@ class HardCodedAgentSessionABC(AgentSessionABC): self, **kwargs, ): - """Evaluate the agent. + """ + Evaluate the agent. :param kwargs: Any agent-specific key-word args to be passed. """ diff --git a/src/primaite/agents/hardcoded_acl.py b/src/primaite/agents/hardcoded_acl.py index 5cc06bdc..c26bcacf 100644 --- a/src/primaite/agents/hardcoded_acl.py +++ b/src/primaite/agents/hardcoded_acl.py @@ -46,7 +46,8 @@ class HardCodedACLAgent(HardCodedAgentSessionABC): return blocked_green_iers def get_matching_acl_rules_for_ier(self, ier, acl, nodes): - """Get matching ACL rules for an IER. + """ + Get matching ACL rules for an IER. TODO: Add params and return in docstring. TODO: Typehint params and return. @@ -62,7 +63,8 @@ class HardCodedACLAgent(HardCodedAgentSessionABC): return matching_rules def get_blocking_acl_rules_for_ier(self, ier, acl, nodes): - """Get blocking ACL rules for an IER. + """ + Get blocking ACL rules for an IER. .. warning:: Can return empty dict but IER can still be blocked by default @@ -81,7 +83,8 @@ class HardCodedACLAgent(HardCodedAgentSessionABC): return blocked_rules def get_allow_acl_rules_for_ier(self, ier, acl, nodes): - """Get all allowing ACL rules for an IER. + """ + Get all allowing ACL rules for an IER. TODO: Add params and return in docstring. TODO: Typehint params and return. @@ -105,7 +108,8 @@ class HardCodedACLAgent(HardCodedAgentSessionABC): nodes, services_list, ): - """Get matching ACL rules. + """ + Get matching ACL rules. TODO: Add params and return in docstring. TODO: Typehint params and return. @@ -136,7 +140,8 @@ class HardCodedACLAgent(HardCodedAgentSessionABC): nodes, services_list, ): - """Get the ALLOW ACL rules. + """ + Get the ALLOW ACL rules. TODO: Add params and return in docstring. TODO: Typehint params and return. @@ -168,7 +173,8 @@ class HardCodedACLAgent(HardCodedAgentSessionABC): nodes, services_list, ): - """Get the DENY ACL rules. + """ + Get the DENY ACL rules. TODO: Add params and return in docstring. TODO: Typehint params and return. @@ -191,7 +197,8 @@ class HardCodedACLAgent(HardCodedAgentSessionABC): return allowed_rules def _calculate_action_full_view(self, obs): - """Calculate a good acl-based action for the blue agent to take. + """ + Calculate a good acl-based action for the blue agent to take. Knowledge of just the observation space is insufficient for a perfect solution, as we need to know: @@ -355,7 +362,8 @@ class HardCodedACLAgent(HardCodedAgentSessionABC): return action def _calculate_action_basic_view(self, obs): - """Calculate a good acl-based action for the blue agent to take. + """ + Calculate a good acl-based action for the blue agent to take. Uses ONLY information from the current observation with NO knowledge of previous actions taken and NO reward feedback. diff --git a/src/primaite/agents/hardcoded_node.py b/src/primaite/agents/hardcoded_node.py index 27a2a823..310fc178 100644 --- a/src/primaite/agents/hardcoded_node.py +++ b/src/primaite/agents/hardcoded_node.py @@ -6,7 +6,8 @@ class HardCodedNodeAgent(HardCodedAgentSessionABC): """An Agent Session class that implements a deterministic Node agent.""" def _calculate_action(self, obs): - """Calculate a good node-based action for the blue agent to take. + """ + Calculate a good node-based action for the blue agent to take. TODO: Add params and return in docstring. TODO: Typehint params and return. diff --git a/src/primaite/agents/rllib.py b/src/primaite/agents/rllib.py index 044b760f..bd5c8585 100644 --- a/src/primaite/agents/rllib.py +++ b/src/primaite/agents/rllib.py @@ -140,7 +140,8 @@ class RLlibAgent(AgentSessionABC): self, **kwargs, ): - """Evaluate the agent. + """ + Evaluate the agent. :param kwargs: Any agent-specific key-word args to be passed. """ @@ -158,7 +159,8 @@ class RLlibAgent(AgentSessionABC): self, **kwargs, ): - """Evaluate the agent. + """ + Evaluate the agent. :param kwargs: Any agent-specific key-word args to be passed. """ diff --git a/src/primaite/agents/sb3.py b/src/primaite/agents/sb3.py index b81a0a18..90a24ee2 100644 --- a/src/primaite/agents/sb3.py +++ b/src/primaite/agents/sb3.py @@ -89,7 +89,8 @@ class SB3Agent(AgentSessionABC): self, **kwargs, ): - """Train the agent. + """ + Train the agent. :param kwargs: Any agent-specific key-word args to be passed. """ @@ -109,7 +110,8 @@ class SB3Agent(AgentSessionABC): deterministic: bool = True, **kwargs, ): - """Evaluate the agent. + """ + Evaluate the agent. :param deterministic: Whether the evaluation is deterministic. :param kwargs: Any agent-specific key-word args to be passed. diff --git a/src/primaite/agents/utils.py b/src/primaite/agents/utils.py index 8b3b57f5..0d4a8e2a 100644 --- a/src/primaite/agents/utils.py +++ b/src/primaite/agents/utils.py @@ -11,7 +11,8 @@ from primaite.common.enums import ( def transform_action_node_readable(action): - """Convert a node action from enumerated format to readable format. + """ + Convert a node action from enumerated format to readable format. example: [1, 3, 1, 0] -> [1, 'SERVICE', 'PATCHING', 0] @@ -33,7 +34,8 @@ def transform_action_node_readable(action): def transform_action_acl_readable(action): - """Transform an ACL action to a more readable format. + """ + Transform an ACL action to a more readable format. example: [0, 1, 2, 5, 0, 1] -> ['NONE', 'ALLOW', 2, 5, 'ANY', 1] @@ -57,7 +59,8 @@ def transform_action_acl_readable(action): def is_valid_node_action(action): - """Is the node action an actual valid action. + """ + Is the node action an actual valid action. Only uses information about the action to determine if the action has an effect @@ -92,7 +95,8 @@ def is_valid_node_action(action): def is_valid_acl_action(action): - """Is the ACL action an actual valid action. + """ + Is the ACL action an actual valid action. Only uses information about the action to determine if the action has an effect. @@ -124,7 +128,8 @@ def is_valid_acl_action(action): def is_valid_acl_action_extra(action): - """Harsher version of valid acl actions, does not allow action. + """ + Harsher version of valid acl actions, does not allow action. TODO: Add params and return in docstring. TODO: Typehint params and return. @@ -147,7 +152,8 @@ def is_valid_acl_action_extra(action): def transform_change_obs_readable(obs): - """Transform list of transactions to readable list of each observation property. + """ + Transform list of transactions to readable list of each observation property. example: np.array([[1,2,1,3],[2,1,1,1]]) -> [[1, 2], ['OFF', 'ON'], ['GOOD', 'GOOD'], ['COMPROMISED', 'GOOD']] @@ -169,7 +175,8 @@ def transform_change_obs_readable(obs): def transform_obs_readable(obs): - """Transform observation to readable format. + """ + Transform observation to readable format. np.array([[1,2,1,3],[2,1,1,1]]) -> [[1, 'OFF', 'GOOD', 'COMPROMISED'], [2, 'ON', 'GOOD', 'GOOD']] @@ -185,7 +192,8 @@ def transform_obs_readable(obs): def convert_to_new_obs(obs, num_nodes=10): - """Convert original gym Box observation space to new multiDiscrete observation space. + """ + Convert original gym Box observation space to new multiDiscrete observation space. TODO: Add params and return in docstring. TODO: Typehint params and return. @@ -196,7 +204,8 @@ def convert_to_new_obs(obs, num_nodes=10): def convert_to_old_obs(obs, num_nodes=10, num_links=10, num_services=1): - """Convert to old observation. + """ + Convert to old observation. Links filled with 0's as no information is included in new observation space. @@ -232,7 +241,8 @@ def convert_to_old_obs(obs, num_nodes=10, num_links=10, num_services=1): def describe_obs_change(obs1, obs2, num_nodes=10, num_links=10, num_services=1): - """Return string describing change between two observations. + """ + Return string describing change between two observations. example: obs_1 = array([[1, 1, 1, 1, 3], [2, 1, 1, 1, 1]]) @@ -260,7 +270,8 @@ def describe_obs_change(obs1, obs2, num_nodes=10, num_links=10, num_services=1): def _describe_obs_change_helper(obs_change, is_link): - """Helper funcion to describe what has changed. + """ + Helper funcion to describe what has changed. example: [ 1 -1 -1 -1 1] -> "ID 1: Service 1 changed to GOOD" @@ -295,7 +306,8 @@ def _describe_obs_change_helper(obs_change, is_link): def transform_action_node_enum(action): - """Convert a node action from readable string format, to enumerated format. + """ + Convert a node action from readable string format, to enumerated format. example: [1, 'SERVICE', 'PATCHING', 0] -> [1, 3, 1, 0] @@ -326,7 +338,8 @@ def transform_action_node_enum(action): def transform_action_node_readable(action): - """Convert a node action from enumerated format to readable format. + """ + Convert a node action from enumerated format to readable format. example: [1, 3, 1, 0] -> [1, 'SERVICE', 'PATCHING', 0] @@ -348,7 +361,8 @@ def transform_action_node_readable(action): def node_action_description(action): - """Generate string describing a node-based action. + """ + Generate string describing a node-based action. TODO: Add params and return in docstring. TODO: Typehint params and return. @@ -375,7 +389,8 @@ def node_action_description(action): def transform_action_acl_enum(action): - """Convert acl action from readable str format, to enumerated format. + """ + Convert acl action from readable str format, to enumerated format. TODO: Add params and return in docstring. TODO: Typehint params and return. @@ -397,7 +412,8 @@ def transform_action_acl_enum(action): def acl_action_description(action): - """Generate string describing an acl-based action. + """ + Generate string describing an acl-based action. TODO: Add params and return in docstring. TODO: Typehint params and return. @@ -417,7 +433,8 @@ def acl_action_description(action): def get_node_of_ip(ip, node_dict): - """Get the node ID of an IP address. + """ + Get the node ID of an IP address. node_dict: dictionary of nodes where key is ID, and value is the node (can be ontained from env.nodes) @@ -431,7 +448,8 @@ def get_node_of_ip(ip, node_dict): def is_valid_node_action(action): - """Is the node action an actual valid action. + """ + Is the node action an actual valid action. Only uses information about the action to determine if the action has an effect @@ -464,7 +482,8 @@ def is_valid_node_action(action): def is_valid_acl_action(action): - """Is the ACL action an actual valid action. + """ + Is the ACL action an actual valid action. Only uses information about the action to determine if the action has an effect @@ -496,7 +515,8 @@ def is_valid_acl_action(action): def is_valid_acl_action_extra(action): - """Harsher version of valid acl actions, does not allow action. + """ + Harsher version of valid acl actions, does not allow action. TODO: Add params and return in docstring. TODO: Typehint params and return. @@ -519,7 +539,8 @@ def is_valid_acl_action_extra(action): def get_new_action(old_action, action_dict): - """Get new action (e.g. 32) from old action e.g. [1,1,1,0]. + """ + Get new action (e.g. 32) from old action e.g. [1,1,1,0]. Old_action can be either node or acl action type diff --git a/src/primaite/cli.py b/src/primaite/cli.py index 42825144..40e8cf0d 100644 --- a/src/primaite/cli.py +++ b/src/primaite/cli.py @@ -28,7 +28,8 @@ def build_dirs(): @app.command() def reset_notebooks(overwrite: bool = True): - """Force a reset of the demo notebooks in the users notebooks directory. + """ + Force a reset of the demo notebooks in the users notebooks directory. :param overwrite: If True, will overwrite existing demo notebooks. """ @@ -39,7 +40,8 @@ def reset_notebooks(overwrite: bool = True): @app.command() def logs(last_n: Annotated[int, typer.Option("-n")]): - """Print the PrimAITE log file. + """ + Print the PrimAITE log file. :param last_n: The number of lines to print. Default value is 10. """ @@ -59,7 +61,8 @@ _LogLevel = Enum("LogLevel", {k: k for k in logging._levelToName.values()}) # n @app.command() def log_level(level: Annotated[Optional[_LogLevel], typer.Argument()] = None): - """View or set the PrimAITE Log Level. + """ + View or set the PrimAITE Log Level. To View, simply call: primaite log-level @@ -110,7 +113,8 @@ def clean_up(): @app.command() def setup(overwrite_existing: bool = True): - """Perform the PrimAITE first-time setup. + """ + Perform the PrimAITE first-time setup. WARNING: All user-data will be lost. """ @@ -148,7 +152,8 @@ def setup(overwrite_existing: bool = True): @app.command() def session(tc: Optional[str] = None, ldc: Optional[str] = None): - """Run a PrimAITE session. + """ + Run a PrimAITE session. tc: The training config filepath. Optional. If no value is passed then example default training config is used from: @@ -173,7 +178,8 @@ def session(tc: Optional[str] = None, ldc: Optional[str] = None): @app.command() def plotly_template(template: Annotated[Optional[PlotlyTemplate], typer.Argument()] = None): - """View or set the plotly template for Session plots. + """ + View or set the plotly template for Session plots. To View, simply call: primaite plotly-template diff --git a/src/primaite/config/lay_down_config.py b/src/primaite/config/lay_down_config.py index 587997b7..3a85b9da 100644 --- a/src/primaite/config/lay_down_config.py +++ b/src/primaite/config/lay_down_config.py @@ -12,7 +12,8 @@ _EXAMPLE_LAY_DOWN: Final[Path] = USERS_CONFIG_DIR / "example_config" / "lay_down def convert_legacy_lay_down_config_dict(legacy_config_dict: Dict[str, Any]) -> Dict[str, Any]: - """Convert a legacy lay down config dict to the new format. + """ + Convert a legacy lay down config dict to the new format. :param legacy_config_dict: A legacy lay down config dict. """ @@ -21,7 +22,8 @@ def convert_legacy_lay_down_config_dict(legacy_config_dict: Dict[str, Any]) -> D def load(file_path: Union[str, Path], legacy_file: bool = False) -> Dict: - """Read in a lay down config yaml file. + """ + Read in a lay down config yaml file. :param file_path: The config file path. :param legacy_file: True if the config file is legacy format, otherwise False. @@ -50,7 +52,8 @@ def load(file_path: Union[str, Path], legacy_file: bool = False) -> Dict: def ddos_basic_one_config_path() -> Path: - """The path to the example lay_down_config_1_DDOS_basic.yaml file. + """ + The path to the example lay_down_config_1_DDOS_basic.yaml file. :return: The file path. """ @@ -64,7 +67,8 @@ def ddos_basic_one_config_path() -> Path: def ddos_basic_two_config_path() -> Path: - """The path to the example lay_down_config_2_DDOS_basic.yaml file. + """ + The path to the example lay_down_config_2_DDOS_basic.yaml file. :return: The file path. """ @@ -78,7 +82,8 @@ def ddos_basic_two_config_path() -> Path: def dos_very_basic_config_path() -> Path: - """The path to the example lay_down_config_3_DOS_very_basic.yaml file. + """ + The path to the example lay_down_config_3_DOS_very_basic.yaml file. :return: The file path. """ @@ -92,7 +97,8 @@ def dos_very_basic_config_path() -> Path: def data_manipulation_config_path() -> Path: - """The path to the example lay_down_config_5_data_manipulation.yaml file. + """ + The path to the example lay_down_config_5_data_manipulation.yaml file. :return: The file path. """ diff --git a/src/primaite/config/training_config.py b/src/primaite/config/training_config.py index 7bdf7995..30edb79b 100644 --- a/src/primaite/config/training_config.py +++ b/src/primaite/config/training_config.py @@ -180,7 +180,8 @@ class TrainingConfig: @classmethod def from_dict(cls, config_dict: Dict[str, Union[str, int, bool]]) -> TrainingConfig: - """Create an instance of TrainingConfig from a dict. + """ + Create an instance of TrainingConfig from a dict. :param config_dict: The training config dict. :return: The instance of TrainingConfig. diff --git a/src/primaite/data_viz/session_plots.py b/src/primaite/data_viz/session_plots.py index 542c6677..245b9774 100644 --- a/src/primaite/data_viz/session_plots.py +++ b/src/primaite/data_viz/session_plots.py @@ -22,7 +22,8 @@ def plot_av_reward_per_episode( title: Optional[str] = None, subtitle: Optional[str] = None, ) -> Figure: - """Plot the average reward per episode from a csv session output. + """ + Plot the average reward per episode from a csv session output. :param av_reward_per_episode_csv: The average reward per episode csv file path. diff --git a/src/primaite/environment/observations.py b/src/primaite/environment/observations.py index 28e85b7f..53c173fd 100644 --- a/src/primaite/environment/observations.py +++ b/src/primaite/environment/observations.py @@ -50,7 +50,8 @@ class AbstractObservationComponent(ABC): class NodeLinkTable(AbstractObservationComponent): - """Table with nodes and links as rows and hardware/software status as cols. + """ + Table with nodes and links as rows and hardware/software status as cols. This will create the observation space formatted as a table of integers. There is one row per node, followed by one row per link. @@ -74,7 +75,8 @@ class NodeLinkTable(AbstractObservationComponent): _DATA_TYPE: type = np.int64 def __init__(self, env: "Primaite"): - """Initialise a NodeLinkTable observation space component. + """ + Initialise a NodeLinkTable observation space component. :param env: Training environment. :type env: Primaite @@ -100,7 +102,8 @@ class NodeLinkTable(AbstractObservationComponent): self.structure = self.generate_structure() def update(self): - """Update the observation based on current environment state. + """ + Update the observation based on current environment state. The structure of the observation space is described in :class:`.NodeLinkTable` """ @@ -181,7 +184,8 @@ class NodeLinkTable(AbstractObservationComponent): class NodeStatuses(AbstractObservationComponent): - """Flat list of nodes' hardware, OS, file system, and service states. + """ + Flat list of nodes' hardware, OS, file system, and service states. The MultiDiscrete observation space can be though of as a one-dimensional vector of discrete states, represented by integers. @@ -234,7 +238,8 @@ class NodeStatuses(AbstractObservationComponent): self.structure = self.generate_structure() def update(self): - """Update the observation based on current environment state. + """ + Update the observation based on current environment state. The structure of the observation space is described in :class:`.NodeStatuses` """ @@ -287,7 +292,8 @@ class NodeStatuses(AbstractObservationComponent): class LinkTrafficLevels(AbstractObservationComponent): - """Flat list of traffic levels encoded into banded categories. + """ + Flat list of traffic levels encoded into banded categories. For each link, total traffic or traffic per service is encoded into a categorical value. For example, if ``quantisation_levels=5``, the traffic levels represent these values: @@ -354,7 +360,8 @@ class LinkTrafficLevels(AbstractObservationComponent): self.structure = self.generate_structure() def update(self): - """Update the observation based on current environment state. + """ + Update the observation based on current environment state. The structure of the observation space is described in :class:`.LinkTrafficLevels` """ @@ -395,7 +402,8 @@ class LinkTrafficLevels(AbstractObservationComponent): class ObservationsHandler: - """Component-based observation space handler. + """ + Component-based observation space handler. This allows users to configure observation spaces by mixing and matching components. Each component can also define further parameters to make them more flexible. @@ -436,7 +444,8 @@ class ObservationsHandler: self._flat_observation = spaces.flatten(self._space, self._observation) def register(self, obs_component: AbstractObservationComponent): - """Add a component for this handler to track. + """ + Add a component for this handler to track. :param obs_component: The component to add. :type obs_component: AbstractObservationComponent @@ -445,7 +454,8 @@ class ObservationsHandler: self.update_space() def deregister(self, obs_component: AbstractObservationComponent): - """Remove a component from this handler. + """ + Remove a component from this handler. :param obs_component: Which component to remove. It must exist within this object's ``registered_obs_components`` attribute. @@ -488,7 +498,8 @@ class ObservationsHandler: @classmethod def from_config(cls, env: "Primaite", obs_space_config: dict): - """Parse a config dictinary, return a new observation handler populated with new observation component objects. + """ + Parse a config dictinary, return a new observation handler populated with new observation component objects. The expected format for the config dictionary is: @@ -533,7 +544,8 @@ class ObservationsHandler: return handler def describe_structure(self): - """Create a list of names for the features of the obs space. + """ + Create a list of names for the features of the obs space. The order of labels follows the flattened version of the space. """ diff --git a/src/primaite/environment/primaite_env.py b/src/primaite/environment/primaite_env.py index 825818fd..9a5df13a 100644 --- a/src/primaite/environment/primaite_env.py +++ b/src/primaite/environment/primaite_env.py @@ -255,7 +255,8 @@ class Primaite(Env): self.total_step_count = 0 def reset(self): - """AI Gym Reset function. + """ + AI Gym Reset function. Returns: Environment observation space (reset) @@ -291,7 +292,8 @@ class Primaite(Env): return self.env_obs def step(self, action): - """AI Gym Step function. + """ + AI Gym Step function. Args: action: Action space from agent @@ -429,7 +431,8 @@ class Primaite(Env): print(" Protocol: " + protocol.get_name().name + ", Load: " + str(protocol.get_load())) def interpret_action_and_apply(self, _action): - """Applies agent actions to the nodes and Access Control List. + """ + Applies agent actions to the nodes and Access Control List. Args: _action: The action space from the agent @@ -448,7 +451,8 @@ class Primaite(Env): logging.error("Invalid action type found") def apply_actions_to_nodes(self, _action): - """Applies agent actions to the nodes. + """ + Applies agent actions to the nodes. Args: _action: The action space from the agent @@ -535,7 +539,8 @@ class Primaite(Env): return def apply_actions_to_acl(self, _action): - """Applies agent actions to the Access Control List [TO DO]. + """ + Applies agent actions to the Access Control List [TO DO]. Args: _action: The action space from the agent @@ -612,7 +617,8 @@ class Primaite(Env): return def apply_time_based_updates(self): - """Updates anything that needs to count down and then change state. + """ + Updates anything that needs to count down and then change state. e.g. reset / patching status """ @@ -653,7 +659,8 @@ class Primaite(Env): pass def init_observations(self) -> Tuple[spaces.Space, np.ndarray]: - """Create the environment's observation handler. + """ + Create the environment's observation handler. :return: The observation space, initial observation (zeroed out array with the correct shape) :rtype: Tuple[spaces.Space, np.ndarray] @@ -709,7 +716,8 @@ class Primaite(Env): print("Environment configuration loaded") def create_node(self, item): - """Creates a node from config data. + """ + Creates a node from config data. Args: item: A config data item @@ -789,7 +797,8 @@ class Primaite(Env): self.network_reference.add_nodes_from([node_ref]) def create_link(self, item: Dict): - """Creates a link from config data. + """ + Creates a link from config data. Args: item: A config data item @@ -832,7 +841,8 @@ class Primaite(Env): ) def create_green_ier(self, item): - """Creates a green IER from config data. + """ + Creates a green IER from config data. Args: item: A config data item @@ -872,7 +882,8 @@ class Primaite(Env): ) def create_red_ier(self, item): - """Creates a red IER from config data. + """ + Creates a red IER from config data. Args: item: A config data item @@ -901,7 +912,8 @@ class Primaite(Env): ) def create_green_pol(self, item): - """Creates a green PoL object from config data. + """ + Creates a green PoL object from config data. Args: item: A config data item @@ -934,7 +946,8 @@ class Primaite(Env): ) def create_red_pol(self, item): - """Creates a red PoL object from config data. + """ + Creates a red PoL object from config data. Args: item: A config data item @@ -974,7 +987,8 @@ class Primaite(Env): ) def create_acl_rule(self, item): - """Creates an ACL rule from config data. + """ + Creates an ACL rule from config data. Args: item: A config data item @@ -994,7 +1008,8 @@ class Primaite(Env): ) def create_services_list(self, services): - """Creates a list of services (enum) from config data. + """ + Creates a list of services (enum) from config data. Args: item: A config data item representing the services @@ -1009,7 +1024,8 @@ class Primaite(Env): self.num_services = len(self.services_list) def create_ports_list(self, ports): - """Creates a list of ports from config data. + """ + Creates a list of ports from config data. Args: item: A config data item representing the ports @@ -1024,7 +1040,8 @@ class Primaite(Env): self.num_ports = len(self.ports_list) def get_observation_info(self, observation_info): - """Extracts observation_info. + """ + Extracts observation_info. :param observation_info: Config item that defines which type of observation space to use :type observation_info: str @@ -1032,7 +1049,8 @@ class Primaite(Env): self.observation_type = ObservationType[observation_info["type"]] def get_action_info(self, action_info): - """Extracts action_info. + """ + Extracts action_info. Args: item: A config data item representing action info @@ -1040,7 +1058,8 @@ class Primaite(Env): self.action_type = ActionType[action_info["type"]] def save_obs_config(self, obs_config: dict): - """Cache the config for the observation space. + """ + Cache the config for the observation space. This is necessary as the observation space can't be built while reading the config, it must be done after all the nodes, links, and services have been initialised. @@ -1052,7 +1071,8 @@ class Primaite(Env): self.obs_config = obs_config def reset_environment(self): - """# Resets environment. + """ + Resets environment. Uses config data config data in order to build the environment configuration. """ @@ -1076,7 +1096,8 @@ class Primaite(Env): ier_value.set_is_running(False) def reset_node(self, item): - """Resets the statuses of a node. + """ + Resets the statuses of a node. Args: item: A config data item @@ -1123,7 +1144,8 @@ class Primaite(Env): pass def create_node_action_dict(self): - """Creates a dictionary mapping each possible discrete action to more readable multidiscrete action. + """ + Creates a dictionary mapping each possible discrete action to more readable multidiscrete action. Note: Only actions that have the potential to change the state exist in the mapping (except for key 0) @@ -1187,7 +1209,8 @@ class Primaite(Env): return actions def create_node_and_acl_action_dict(self): - """Create a dictionary mapping each possible discrete action to a more readable mutlidiscrete action. + """ + Create a dictionary mapping each possible discrete action to a more readable mutlidiscrete action. The dictionary contains actions of both Node and ACL action types. """ diff --git a/src/primaite/environment/reward.py b/src/primaite/environment/reward.py index 5cef47ef..19094a18 100644 --- a/src/primaite/environment/reward.py +++ b/src/primaite/environment/reward.py @@ -21,7 +21,8 @@ def calculate_reward_function( step_count, config_values, ): - """Compares the states of the initial and final nodes/links to get a reward. + """ + Compares the states of the initial and final nodes/links to get a reward. Args: initial_nodes: The nodes before red and blue agents take effect @@ -94,7 +95,8 @@ def calculate_reward_function( def score_node_operating_state(final_node, initial_node, reference_node, config_values): - """Calculates score relating to the hardware state of a node. + """ + Calculates score relating to the hardware state of a node. Args: final_node: The node after red and blue agents take effect @@ -142,7 +144,8 @@ def score_node_operating_state(final_node, initial_node, reference_node, config_ def score_node_os_state(final_node, initial_node, reference_node, config_values): - """Calculates score relating to the Software State of a node. + """ + Calculates score relating to the Software State of a node. Args: final_node: The node after red and blue agents take effect @@ -192,7 +195,8 @@ def score_node_os_state(final_node, initial_node, reference_node, config_values) def score_node_service_state(final_node, initial_node, reference_node, config_values): - """Calculates score relating to the service state(s) of a node. + """ + Calculates score relating to the service state(s) of a node. Args: final_node: The node after red and blue agents take effect @@ -263,7 +267,8 @@ def score_node_service_state(final_node, initial_node, reference_node, config_va def score_node_file_system(final_node, initial_node, reference_node, config_values): - """Calculates score relating to the file system state of a node. + """ + Calculates score relating to the file system state of a node. Args: final_node: The node after red and blue agents take effect diff --git a/src/primaite/links/link.py b/src/primaite/links/link.py index 5892b8e2..f61281cd 100644 --- a/src/primaite/links/link.py +++ b/src/primaite/links/link.py @@ -29,7 +29,8 @@ class Link(object): self.add_protocol(protocol_name) def add_protocol(self, _protocol): - """Adds a new protocol to the list of protocols on this link. + """ + Adds a new protocol to the list of protocols on this link. Args: _protocol: The protocol to be added (enum) @@ -37,7 +38,8 @@ class Link(object): self.protocol_list.append(Protocol(_protocol)) def get_id(self): - """Gets link ID. + """ + Gets link ID. Returns: Link ID @@ -45,7 +47,8 @@ class Link(object): return self.id def get_source_node_name(self): - """Gets source node name. + """ + Gets source node name. Returns: Source node name @@ -53,7 +56,8 @@ class Link(object): return self.source_node_name def get_dest_node_name(self): - """Gets destination node name. + """ + Gets destination node name. Returns: Destination node name @@ -61,7 +65,8 @@ class Link(object): return self.dest_node_name def get_bandwidth(self): - """Gets bandwidth of link. + """ + Gets bandwidth of link. Returns: Link bandwidth (bps) @@ -69,7 +74,8 @@ class Link(object): return self.bandwidth def get_protocol_list(self): - """Gets list of protocols on this link. + """ + Gets list of protocols on this link. Returns: List of protocols on this link @@ -77,7 +83,8 @@ class Link(object): return self.protocol_list def get_current_load(self): - """Gets current total load on this link. + """ + Gets current total load on this link. Returns: Total load on this link (bps) @@ -88,7 +95,8 @@ class Link(object): return total_load def add_protocol_load(self, _protocol, _load): - """Adds a loading to a protocol on this link. + """ + Adds a loading to a protocol on this link. Args: _protocol: The protocol to load diff --git a/src/primaite/main.py b/src/primaite/main.py index 7b1d7ab3..f2d1b9c2 100644 --- a/src/primaite/main.py +++ b/src/primaite/main.py @@ -14,7 +14,8 @@ def run( training_config_path: Union[str, Path], lay_down_config_path: Union[str, Path], ): - """Run the PrimAITE Session. + """ + Run the PrimAITE Session. :param training_config_path: The training config filepath. :param lay_down_config_path: The lay down config filepath. diff --git a/src/primaite/nodes/active_node.py b/src/primaite/nodes/active_node.py index 3789b7a4..f86f818b 100644 --- a/src/primaite/nodes/active_node.py +++ b/src/primaite/nodes/active_node.py @@ -52,7 +52,8 @@ class ActiveNode(Node): @property def software_state(self) -> SoftwareState: - """Get the software_state. + """ + Get the software_state. :return: The software_state. """ @@ -60,7 +61,8 @@ class ActiveNode(Node): @software_state.setter def software_state(self, software_state: SoftwareState): - """Get the software_state. + """ + Get the software_state. :param software_state: Software State. """ @@ -78,7 +80,8 @@ class ActiveNode(Node): ) def set_software_state_if_not_compromised(self, software_state: SoftwareState): - """Sets Software State if the node is not compromised. + """ + Sets Software State if the node is not compromised. Args: software_state: Software State @@ -104,7 +107,8 @@ class ActiveNode(Node): self._software_state = SoftwareState.GOOD def set_file_system_state(self, file_system_state: FileSystemState): - """Sets the file system state (actual and observed). + """ + Sets the file system state (actual and observed). Args: file_system_state: File system state @@ -130,7 +134,8 @@ class ActiveNode(Node): ) def set_file_system_state_if_not_compromised(self, file_system_state: FileSystemState): - """Sets the file system state (actual and observed) if not in a compromised state. + """ + Sets the file system state (actual and observed) if not in a compromised state. Use for green PoL to prevent it overturning a compromised state diff --git a/src/primaite/nodes/node_state_instruction_green.py b/src/primaite/nodes/node_state_instruction_green.py index da4be35e..7ebe3886 100644 --- a/src/primaite/nodes/node_state_instruction_green.py +++ b/src/primaite/nodes/node_state_instruction_green.py @@ -35,7 +35,8 @@ class NodeStateInstructionGreen(object): self.state = _state def get_start_step(self): - """Gets the start step. + """ + Gets the start step. Returns: The start step @@ -43,7 +44,8 @@ class NodeStateInstructionGreen(object): return self.start_step def get_end_step(self): - """Gets the end step. + """ + Gets the end step. Returns: The end step @@ -51,7 +53,8 @@ class NodeStateInstructionGreen(object): return self.end_step def get_node_id(self): - """Gets the node ID. + """ + Gets the node ID. Returns: The node ID @@ -59,7 +62,8 @@ class NodeStateInstructionGreen(object): return self.node_id def get_node_pol_type(self): - """Gets the node pattern of life type (enum). + """ + Gets the node pattern of life type (enum). Returns: The node pattern of life type (enum) @@ -67,7 +71,8 @@ class NodeStateInstructionGreen(object): return self.node_pol_type def get_service_name(self): - """Gets the service name. + """ + Gets the service name. Returns: The service name @@ -75,7 +80,8 @@ class NodeStateInstructionGreen(object): return self.service_name def get_state(self): - """Gets the state (node or service). + """ + Gets the state (node or service). Returns: The state (node or service) diff --git a/src/primaite/nodes/node_state_instruction_red.py b/src/primaite/nodes/node_state_instruction_red.py index f8ce4e74..540625cc 100644 --- a/src/primaite/nodes/node_state_instruction_red.py +++ b/src/primaite/nodes/node_state_instruction_red.py @@ -51,7 +51,8 @@ class NodeStateInstructionRed(object): self.source_node_service_state = _pol_source_node_service_state def get_start_step(self): - """Gets the start step. + """ + Gets the start step. Returns: The start step @@ -59,7 +60,8 @@ class NodeStateInstructionRed(object): return self.start_step def get_end_step(self): - """Gets the end step. + """ + Gets the end step. Returns: The end step @@ -67,7 +69,8 @@ class NodeStateInstructionRed(object): return self.end_step def get_target_node_id(self): - """Gets the node ID. + """ + Gets the node ID. Returns: The node ID @@ -75,7 +78,8 @@ class NodeStateInstructionRed(object): return self.target_node_id def get_initiator(self): - """Gets the initiator. + """ + Gets the initiator. Returns: The initiator @@ -83,7 +87,8 @@ class NodeStateInstructionRed(object): return self.initiator def get_pol_type(self) -> NodePOLType: - """Gets the node pattern of life type (enum). + """ + Gets the node pattern of life type (enum). Returns: The node pattern of life type (enum) @@ -91,7 +96,8 @@ class NodeStateInstructionRed(object): return self.pol_type def get_service_name(self): - """Gets the service name. + """ + Gets the service name. Returns: The service name @@ -99,7 +105,8 @@ class NodeStateInstructionRed(object): return self.service_name def get_state(self): - """Gets the state (node or service). + """ + Gets the state (node or service). Returns: The state (node or service) @@ -107,7 +114,8 @@ class NodeStateInstructionRed(object): return self.state def get_source_node_id(self): - """Gets the source node id (used for initiator type SERVICE). + """ + Gets the source node id (used for initiator type SERVICE). Returns: The source node id @@ -115,7 +123,8 @@ class NodeStateInstructionRed(object): return self.source_node_id def get_source_node_service(self): - """Gets the source node service (used for initiator type SERVICE). + """ + Gets the source node service (used for initiator type SERVICE). Returns: The source node service @@ -123,7 +132,8 @@ class NodeStateInstructionRed(object): return self.source_node_service def get_source_node_service_state(self): - """Gets the source node service state (used for initiator type SERVICE). + """ + Gets the source node service state (used for initiator type SERVICE). Returns: The source node service state diff --git a/src/primaite/nodes/passive_node.py b/src/primaite/nodes/passive_node.py index 13b2d6ad..afe4e2d1 100644 --- a/src/primaite/nodes/passive_node.py +++ b/src/primaite/nodes/passive_node.py @@ -32,7 +32,8 @@ class PassiveNode(Node): @property def ip_address(self) -> str: - """Gets the node IP address as an empty string. + """ + Gets the node IP address as an empty string. No concept of IP address for passive nodes for now. diff --git a/src/primaite/nodes/service_node.py b/src/primaite/nodes/service_node.py index 7632e944..4ad52a1e 100644 --- a/src/primaite/nodes/service_node.py +++ b/src/primaite/nodes/service_node.py @@ -53,14 +53,16 @@ class ServiceNode(ActiveNode): self.services: Dict[str, Service] = {} def add_service(self, service: Service): - """Adds a service to the node. + """ + Adds a service to the node. :param service: The service to add """ self.services[service.name] = service def has_service(self, protocol_name: str) -> bool: - """Indicates whether a service is on a node. + """ + Indicates whether a service is on a node. :param protocol_name: The service (protocol)e. :return: True if service (protocol) is on the node, otherwise False. @@ -71,7 +73,8 @@ class ServiceNode(ActiveNode): return False def service_running(self, protocol_name: str) -> bool: - """Indicates whether a service is in a running state on the node. + """ + Indicates whether a service is in a running state on the node. :param protocol_name: The service (protocol) :return: True if service (protocol) is in a running state on the node, otherwise False. @@ -85,7 +88,8 @@ class ServiceNode(ActiveNode): return False def service_is_overwhelmed(self, protocol_name: str) -> bool: - """Indicates whether a service is in an overwhelmed state on the node. + """ + Indicates whether a service is in an overwhelmed state on the node. :param protocol_name: The service (protocol) :return: True if service (protocol) is in an overwhelmed state on the node, otherwise False. @@ -99,7 +103,8 @@ class ServiceNode(ActiveNode): return False def set_service_state(self, protocol_name: str, software_state: SoftwareState): - """Sets the software_state of a service (protocol) on the node. + """ + Sets the software_state of a service (protocol) on the node. :param protocol_name: The service (protocol). :param software_state: The software_state. @@ -127,7 +132,8 @@ class ServiceNode(ActiveNode): ) def set_service_state_if_not_compromised(self, protocol_name: str, software_state: SoftwareState): - """Sets the software_state of a service (protocol) on the node. + """ + Sets the software_state of a service (protocol) on the node. Done if the software_state is not "compromised". @@ -153,7 +159,8 @@ class ServiceNode(ActiveNode): ) def get_service_state(self, protocol_name): - """Gets the state of a service. + """ + Gets the state of a service. :return: The software_state of the service. """ diff --git a/src/primaite/notebooks/__init__.py b/src/primaite/notebooks/__init__.py index da65da38..6ca1d3f6 100644 --- a/src/primaite/notebooks/__init__.py +++ b/src/primaite/notebooks/__init__.py @@ -11,7 +11,8 @@ _LOGGER = getLogger(__name__) def start_jupyter_session(): - """Starts a new Jupyter notebook session in the app notebooks directory. + """ + Starts a new Jupyter notebook session in the app notebooks directory. Currently only works on Windows OS. diff --git a/src/primaite/pol/green_pol.py b/src/primaite/pol/green_pol.py index 91a6f787..e9dfef8c 100644 --- a/src/primaite/pol/green_pol.py +++ b/src/primaite/pol/green_pol.py @@ -25,7 +25,8 @@ def apply_iers( acl: AccessControlList, step: int, ): - """Applies IERs to the links (link pattern of life). + """ + Applies IERs to the links (link pattern of life). Args: network: The network modelled in the environment @@ -217,7 +218,8 @@ def apply_node_pol( node_pol: Dict[any, Union[NodeStateInstructionGreen, NodeStateInstructionRed]], step: int, ): - """Applies node pattern of life. + """ + Applies node pattern of life. Args: nodes: The nodes within the environment diff --git a/src/primaite/pol/ier.py b/src/primaite/pol/ier.py index 913a06da..2de8fe6f 100644 --- a/src/primaite/pol/ier.py +++ b/src/primaite/pol/ier.py @@ -1,5 +1,6 @@ # Crown Copyright (C) Dstl 2022. DEFCON 703. Shared in confidence. -"""Information Exchange Requirements for APE. +""" +Information Exchange Requirements for APE. Used to represent an information flow from source to destination. """ @@ -47,7 +48,8 @@ class IER(object): self.running = _running def get_id(self): - """Gets IER ID. + """ + Gets IER ID. Returns: IER ID @@ -55,7 +57,8 @@ class IER(object): return self.id def get_start_step(self): - """Gets IER start step. + """ + Gets IER start step. Returns: IER start step @@ -63,7 +66,8 @@ class IER(object): return self.start_step def get_end_step(self): - """Gets IER end step. + """ + Gets IER end step. Returns: IER end step @@ -71,7 +75,8 @@ class IER(object): return self.end_step def get_load(self): - """Gets IER load. + """ + Gets IER load. Returns: IER load @@ -79,7 +84,8 @@ class IER(object): return self.load def get_protocol(self): - """Gets IER protocol. + """ + Gets IER protocol. Returns: IER protocol @@ -87,7 +93,8 @@ class IER(object): return self.protocol def get_port(self): - """Gets IER port. + """ + Gets IER port. Returns: IER port @@ -95,7 +102,8 @@ class IER(object): return self.port def get_source_node_id(self): - """Gets IER source node ID. + """ + Gets IER source node ID. Returns: IER source node ID @@ -103,7 +111,8 @@ class IER(object): return self.source_node_id def get_dest_node_id(self): - """Gets IER destination node ID. + """ + Gets IER destination node ID. Returns: IER destination node ID @@ -111,7 +120,8 @@ class IER(object): return self.dest_node_id def get_is_running(self): - """Informs whether the IER is currently running. + """ + Informs whether the IER is currently running. Returns: True if running @@ -119,7 +129,8 @@ class IER(object): return self.running def set_is_running(self, _value): - """Sets the running state of the IER. + """ + Sets the running state of the IER. Args: _value: running status @@ -127,7 +138,8 @@ class IER(object): self.running = _value def get_mission_criticality(self): - """Gets the IER mission criticality (used in the reward function). + """ + Gets the IER mission criticality (used in the reward function). Returns: Mission criticality value (0 lowest to 5 highest) diff --git a/src/primaite/pol/red_agent_pol.py b/src/primaite/pol/red_agent_pol.py index 86482903..bff19bf8 100644 --- a/src/primaite/pol/red_agent_pol.py +++ b/src/primaite/pol/red_agent_pol.py @@ -24,7 +24,8 @@ def apply_red_agent_iers( acl: AccessControlList, step: int, ): - """Applies IERs to the links (link POL) resulting from red agent attack. + """ + Applies IERs to the links (link POL) resulting from red agent attack. Args: network: The network modelled in the environment @@ -213,7 +214,8 @@ def apply_red_agent_node_pol( node_pol: Dict[str, NodeStateInstructionRed], step: int, ): - """Applies node pattern of life. + """ + Applies node pattern of life. Args: nodes: The nodes within the environment @@ -295,7 +297,8 @@ def apply_red_agent_node_pol( def is_red_ier_incoming(node, iers, node_pol_type): - """Checks if the RED IER is incoming. + """ + Checks if the RED IER is incoming. TODO: Write more descriptive docstring with params and returns. """ diff --git a/src/primaite/primaite_session.py b/src/primaite/primaite_session.py index 1bfb7403..caa85e9e 100644 --- a/src/primaite/primaite_session.py +++ b/src/primaite/primaite_session.py @@ -125,7 +125,8 @@ class PrimaiteSession: self, **kwargs, ): - """Train the agent. + """ + Train the agent. :param kwargs: Any agent-framework specific key word args. """ @@ -136,7 +137,8 @@ class PrimaiteSession: self, **kwargs, ): - """Evaluate the agent. + """ + Evaluate the agent. :param kwargs: Any agent-framework specific key word args. """ diff --git a/src/primaite/setup/reset_demo_notebooks.py b/src/primaite/setup/reset_demo_notebooks.py index 8d2a94c7..793f9ade 100644 --- a/src/primaite/setup/reset_demo_notebooks.py +++ b/src/primaite/setup/reset_demo_notebooks.py @@ -12,7 +12,8 @@ _LOGGER = getLogger(__name__) def run(overwrite_existing: bool = True): - """Resets the demo jupyter notebooks in the users app notebooks directory. + """ + Resets the demo jupyter notebooks in the users app notebooks directory. :param overwrite_existing: A bool to toggle replacing existing edited notebooks on or off. """ diff --git a/src/primaite/setup/reset_example_configs.py b/src/primaite/setup/reset_example_configs.py index a2e1f2c9..599de8dc 100644 --- a/src/primaite/setup/reset_example_configs.py +++ b/src/primaite/setup/reset_example_configs.py @@ -11,7 +11,8 @@ _LOGGER = getLogger(__name__) def run(overwrite_existing=True): - """Resets the example config files in the users app config directory. + """ + Resets the example config files in the users app config directory. :param overwrite_existing: A bool to toggle replacing existing edited config on or off. """ diff --git a/src/primaite/setup/setup_app_dirs.py b/src/primaite/setup/setup_app_dirs.py index bf7dbe59..693b11c1 100644 --- a/src/primaite/setup/setup_app_dirs.py +++ b/src/primaite/setup/setup_app_dirs.py @@ -5,7 +5,8 @@ _LOGGER = getLogger(__name__) def run(): - """Handles creation of application directories and user directories. + """ + Handles creation of application directories and user directories. Uses `platformdirs.PlatformDirs` and `pathlib.Path` to create the required app directories in the correct locations based on the users OS. diff --git a/src/primaite/transactions/transaction.py b/src/primaite/transactions/transaction.py index a74ef4f9..3a5a13db 100644 --- a/src/primaite/transactions/transaction.py +++ b/src/primaite/transactions/transaction.py @@ -39,7 +39,8 @@ class Transaction(object): "The env observation space description" def as_csv_data(self) -> Tuple[List, List]: - """Converts the Transaction to a csv data row and provides a header. + """ + Converts the Transaction to a csv data row and provides a header. :return: A tuple consisting of (header, data). """ @@ -68,7 +69,8 @@ class Transaction(object): def _turn_action_space_to_array(action_space) -> List[str]: - """Turns action space into a string array so it can be saved to csv. + """ + Turns action space into a string array so it can be saved to csv. :param action_space: The action space :return: The action space as an array of strings @@ -80,7 +82,8 @@ def _turn_action_space_to_array(action_space) -> List[str]: def _turn_obs_space_to_array(obs_space, obs_assets, obs_features) -> List[str]: - """Turns observation space into a string array so it can be saved to csv. + """ + Turns observation space into a string array so it can be saved to csv. :param obs_space: The observation space :param obs_assets: The number of assets (i.e. nodes or links) in the observation space diff --git a/src/primaite/utils/package_data.py b/src/primaite/utils/package_data.py index 463a4309..59f36851 100644 --- a/src/primaite/utils/package_data.py +++ b/src/primaite/utils/package_data.py @@ -10,7 +10,8 @@ _LOGGER = getLogger(__name__) def get_file_path(path: str) -> Path: - """Get PrimAITE package data. + """ + Get PrimAITE package data. :Example: diff --git a/src/primaite/utils/session_output_reader.py b/src/primaite/utils/session_output_reader.py index 6b5cfdc3..e70c98e2 100644 --- a/src/primaite/utils/session_output_reader.py +++ b/src/primaite/utils/session_output_reader.py @@ -7,7 +7,8 @@ import polars as pl def av_rewards_dict(av_rewards_csv_file: Union[str, Path]) -> Dict[int, float]: - """Read an average rewards per episode csv file and return as a dict. + """ + Read an average rewards per episode csv file and return as a dict. The dictionary keys are the episode number, and the values are the mean reward that episode. diff --git a/src/primaite/utils/session_output_writer.py b/src/primaite/utils/session_output_writer.py index 5852a84d..104acc62 100644 --- a/src/primaite/utils/session_output_writer.py +++ b/src/primaite/utils/session_output_writer.py @@ -77,7 +77,8 @@ class SessionOutputWriter: _LOGGER.debug(f"Finished writing file: {self._csv_file_path}") def write(self, data: Union[Tuple, Transaction]): - """Write a row of session data. + """ + Write a row of session data. :param data: The row of data to write. Can be a Tuple or an instance of Transaction. """ diff --git a/tests/test_observation_space.py b/tests/test_observation_space.py index d1082049..d5844fd9 100644 --- a/tests/test_observation_space.py +++ b/tests/test_observation_space.py @@ -75,7 +75,8 @@ class TestNodeLinkTable: assert env.env_obs.shape == (5, 6) def test_value(self, temp_primaite_session): - """Test that the observation is generated correctly. + """ + Test that the observation is generated correctly. The laydown has: * 3 nodes (2 service nodes and 1 active node) @@ -157,7 +158,8 @@ class TestNodeStatuses: assert env.env_obs.shape == (15,) def test_values(self, temp_primaite_session): - """Test that the hardware and software states are encoded correctly. + """ + Test that the hardware and software states are encoded correctly. The laydown has: * one node with a compromised operating system state @@ -213,7 +215,8 @@ class TestLinkTrafficLevels: assert env.env_obs.shape == (2 * 2,) def test_values(self, temp_primaite_session): - """Test that traffic values are encoded correctly. + """ + Test that traffic values are encoded correctly. The laydown has: * two services From bbb305d561c3fc771789c4450d2f5d3374ad7dee Mon Sep 17 00:00:00 2001 From: Marek Wolan Date: Fri, 7 Jul 2023 13:52:14 +0100 Subject: [PATCH 26/41] Update observation space documentation --- docs/source/about.rst | 3 +-- docs/source/config.rst | 60 ++++++++++++++++++++++++------------------ 2 files changed, 36 insertions(+), 27 deletions(-) diff --git a/docs/source/about.rst b/docs/source/about.rst index 1f4669fe..a599829a 100644 --- a/docs/source/about.rst +++ b/docs/source/about.rst @@ -186,7 +186,7 @@ Observation Spaces ****************** The observation space provides the blue agent with information about the current status of nodes and links. -PrimAITE builds on top of Gym Spaces to create an observation space that is easily configurable for users. It's made up of components which are managed by the :py:class:`primaite.environment.observations.ObservationHandler`. Each training scenario can define its own observation space, and the user can choose which information to inlude, and how it should be formatted. +PrimAITE builds on top of Gym Spaces to create an observation space that is easily configurable for users. It's made up of components which are managed by the :py:class:`primaite.environment.observations.ObservationsHandler`. Each training scenario can define its own observation space, and the user can choose which information to inlude, and how it should be formatted. NodeLinkTable component ----------------------- @@ -416,4 +416,3 @@ The PrimAITE project has an ambition to include the following enhancements in fu * Integration with a suitable standardised framework to allow multi-agent integration * Integration with external threat emulation tools, either using off-line data, or integrating at runtime -* Provision of data such that agents can construct alternative observation spaces (as an alternative to the default PrimAITE observation space) diff --git a/docs/source/config.rst b/docs/source/config.rst index c80baa3c..35e83771 100644 --- a/docs/source/config.rst +++ b/docs/source/config.rst @@ -48,6 +48,41 @@ The environment config file consists of the following attributes: Determines whether a NODE, ACL, or ANY (combined NODE & ACL) action space format is adopted for the session +* **OBSERVATION_SPACE** [dict] + + Allows for user to configure observation space by combining one or more observation components. List of available + components is in :py:mod:`primaite.environment.observations`. + + The observation space config item should have a ``components`` key which is a list of components. Each component + config must have a ``name`` key, and can optionally have an ``options`` key. The ``options`` are passed to the + component while it is being initialised. + + This example illustrates the correct format for the observation space config item + + .. code-block:: yaml + + observation_space: + flatten: true + components: + - name: NODE_LINK_TABLE + - name: NODE_STATUSES + - name: LINK_TRAFFIC_LEVELS + options: + combine_service_traffic : False + quantisation_levels: 99 + + + Currently available components are: + + * :py:mod:`NODE_LINK_TABLE` this does not accept any additional options + * :py:mod:`NODE_STATUSES`, this does not accept any additional options + * :py:mod:`LINK_TRAFFIC_LEVELS`, this accepts the following options: + + * ``combine_service_traffic`` - whether to consider bandwidth use separately for each network protocol or combine them into a single bandwidth reading (boolean) + * ``quantisation_levels`` - how many discrete bandwidth usage levels to use for encoding. This can be an integer equal to or greater than 3. + + The other configurable item is ``flatten`` which is false by default. When set to true, the observation space is flattened (turned into a 1-D vector). You should use this if your RL agent does not natively support the chosen shape of the observation space. + * **num_episodes** [int] This defines the number of episodes that the agent will train or be evaluated over. @@ -321,31 +356,6 @@ The Lay Down Config The lay down config file consists of the following attributes: -* **itemType: ACTIONS** [enum] - - Determines whether a NODE or ACL action space format is adopted for the session - -* **itemType: OBSERVATION_SPACE** [dict] - - Allows for user to configure observation space by combining one or more observation components. List of available - components is is :py:mod:'primaite.environment.observations'. - - The observation space config item should have a ``components`` key which is a list of components. Each component - config must have a ``name`` key, and can optionally have an ``options`` key. The ``options`` are passed to the - component while it is being initialised. - - This example illustrates the correct format for the observation space config item - -.. code-block::yaml - - - item_type: OBSERVATION_SPACE - components: - - name: LINK_TRAFFIC_LEVELS - options: - combine_service_traffic: false - quantisation_levels: 8 - - name: NODE_STATUSES - - name: LINK_TRAFFIC_LEVELS * **itemType: STEPS** [int] From d7bf678b1f6dded2d53b438be89adaf368af35f1 Mon Sep 17 00:00:00 2001 From: Marek Wolan Date: Fri, 7 Jul 2023 14:24:37 +0100 Subject: [PATCH 27/41] Reworded observation description --- docs/source/config.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/config.rst b/docs/source/config.rst index 35e83771..a28f0ec1 100644 --- a/docs/source/config.rst +++ b/docs/source/config.rst @@ -81,7 +81,7 @@ The environment config file consists of the following attributes: * ``combine_service_traffic`` - whether to consider bandwidth use separately for each network protocol or combine them into a single bandwidth reading (boolean) * ``quantisation_levels`` - how many discrete bandwidth usage levels to use for encoding. This can be an integer equal to or greater than 3. - The other configurable item is ``flatten`` which is false by default. When set to true, the observation space is flattened (turned into a 1-D vector). You should use this if your RL agent does not natively support the chosen shape of the observation space. + The other configurable item is ``flatten`` which is false by default. When set to true, the observation space is flattened (turned into a 1-D vector). You should use this if your RL agent does not natively support observation space types like ``gym.Spaces.Tuple``. * **num_episodes** [int] From 17894376c6ae17229ac57a05c8ac381f7995d639 Mon Sep 17 00:00:00 2001 From: Marek Wolan Date: Sun, 9 Jul 2023 18:07:21 +0100 Subject: [PATCH 28/41] Removed comment --- src/primaite/agents/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/primaite/agents/utils.py b/src/primaite/agents/utils.py index b5a3c673..acc70cc4 100644 --- a/src/primaite/agents/utils.py +++ b/src/primaite/agents/utils.py @@ -165,7 +165,6 @@ def transform_change_obs_readable(obs): os_states = [SoftwareState(i).name for i in obs[:, 2]] new_obs = [ids, operating_states, os_states] - # changed range(3,...) to range(4,...) because we added file system which was new since ADSP for service in range(4, obs.shape[1]): # Links bit/s don't have a service state service_states = [SoftwareState(i).name if i <= 4 else i for i in obs[:, service]] From ca737e080fe3466c56d4c876088c0ee55be5af27 Mon Sep 17 00:00:00 2001 From: Marek Wolan Date: Mon, 10 Jul 2023 10:25:26 +0000 Subject: [PATCH 29/41] Changed build pipeline experimentally. --- .azure/azure-ci-build-pipeline.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.azure/azure-ci-build-pipeline.yaml b/.azure/azure-ci-build-pipeline.yaml index 902eb38d..fe50fb32 100644 --- a/.azure/azure-ci-build-pipeline.yaml +++ b/.azure/azure-ci-build-pipeline.yaml @@ -11,21 +11,27 @@ strategy: UbuntuPython38: python.version: '3.8' imageName: 'ubuntu-latest' + condition: eq(variables['Build.Reason'], 'PullRequest') UbuntuPython310: python.version: '3.10' imageName: 'ubuntu-latest' WindowsPython38: python.version: '3.8' imageName: 'windows-latest' + condition: eq(variables['Build.Reason'], 'PullRequest') WindowsPython310: python.version: '3.10' imageName: 'windows-latest' + condition: eq(variables['Build.Reason'], 'PullRequest') MacOSPython38: python.version: '3.8' imageName: 'macOS-latest' + condition: eq(variables['Build.Reason'], 'PullRequest') MacOSPython310: python.version: '3.10' imageName: 'macOS-latest' + condition: eq(variables['Build.Reason'], 'PullRequest') + # pretty sure this does not support a 'condition' parameter but worth a try. Otherwise a more complicated solution might be warranted. pool: vmImage: $(imageName) From 831469d01c59ea14e0a5bdaea859cc2dd92e1991 Mon Sep 17 00:00:00 2001 From: Marek Wolan Date: Tue, 11 Jul 2023 07:16:11 +0000 Subject: [PATCH 30/41] Built matrix conditionally --- .azure/azure-ci-build-pipeline.yaml | 47 ++++++++++++++--------------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/.azure/azure-ci-build-pipeline.yaml b/.azure/azure-ci-build-pipeline.yaml index fe50fb32..bb9c03fa 100644 --- a/.azure/azure-ci-build-pipeline.yaml +++ b/.azure/azure-ci-build-pipeline.yaml @@ -8,30 +8,29 @@ trigger: strategy: matrix: - UbuntuPython38: - python.version: '3.8' - imageName: 'ubuntu-latest' - condition: eq(variables['Build.Reason'], 'PullRequest') - UbuntuPython310: - python.version: '3.10' - imageName: 'ubuntu-latest' - WindowsPython38: - python.version: '3.8' - imageName: 'windows-latest' - condition: eq(variables['Build.Reason'], 'PullRequest') - WindowsPython310: - python.version: '3.10' - imageName: 'windows-latest' - condition: eq(variables['Build.Reason'], 'PullRequest') - MacOSPython38: - python.version: '3.8' - imageName: 'macOS-latest' - condition: eq(variables['Build.Reason'], 'PullRequest') - MacOSPython310: - python.version: '3.10' - imageName: 'macOS-latest' - condition: eq(variables['Build.Reason'], 'PullRequest') - # pretty sure this does not support a 'condition' parameter but worth a try. Otherwise a more complicated solution might be warranted. + ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + UbuntuPython38: + python.version: '3.8' + imageName: 'ubuntu-latest' + UbuntuPython310: + python.version: '3.10' + imageName: 'ubuntu-latest' + WindowsPython38: + python.version: '3.8' + imageName: 'windows-latest' + WindowsPython310: + python.version: '3.10' + imageName: 'windows-latest' + MacOSPython38: + python.version: '3.8' + imageName: 'macOS-latest' + MacOSPython310: + python.version: '3.10' + imageName: 'macOS-latest' + ${{ if not(eq(variables['Build.Reason'], 'PullRequest')) }}: + UbuntuPython310: + python.version: '3.10' + imageName: 'ubuntu-latest' pool: vmImage: $(imageName) From d8cfbc104211334935f91b383ead0821edc4e8ad Mon Sep 17 00:00:00 2001 From: Marek Wolan Date: Tue, 11 Jul 2023 07:19:58 +0000 Subject: [PATCH 31/41] Updated azure-ci-build-pipeline.yaml --- .azure/azure-ci-build-pipeline.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.azure/azure-ci-build-pipeline.yaml b/.azure/azure-ci-build-pipeline.yaml index bb9c03fa..9c21577b 100644 --- a/.azure/azure-ci-build-pipeline.yaml +++ b/.azure/azure-ci-build-pipeline.yaml @@ -7,8 +7,8 @@ trigger: - release/* strategy: - matrix: - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + matrix: UbuntuPython38: python.version: '3.8' imageName: 'ubuntu-latest' @@ -27,7 +27,8 @@ strategy: MacOSPython310: python.version: '3.10' imageName: 'macOS-latest' - ${{ if not(eq(variables['Build.Reason'], 'PullRequest')) }}: + ${{ if not(eq(variables['Build.Reason'], 'PullRequest')) }}: + matrix: UbuntuPython310: python.version: '3.10' imageName: 'ubuntu-latest' From 548ecf8e0824da7c161fd6d8da5bf11f7daa651f Mon Sep 17 00:00:00 2001 From: Marek Wolan Date: Tue, 11 Jul 2023 08:05:38 +0000 Subject: [PATCH 32/41] Edit pipeline to use runtime parameters https://stackoverflow.com/a/70046417 --- .azure/azure-ci-build-pipeline.yaml | 124 +++++++++++++++------------- 1 file changed, 66 insertions(+), 58 deletions(-) diff --git a/.azure/azure-ci-build-pipeline.yaml b/.azure/azure-ci-build-pipeline.yaml index 9c21577b..b1557b7b 100644 --- a/.azure/azure-ci-build-pipeline.yaml +++ b/.azure/azure-ci-build-pipeline.yaml @@ -6,75 +6,83 @@ trigger: - bugfix/* - release/* -strategy: - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: - matrix: +parameters: + # https://stackoverflow.com/a/70046417 + - name: matrix + type: object + default: UbuntuPython38: - python.version: '3.8' - imageName: 'ubuntu-latest' + py: '3.8' + img: 'ubuntu-latest' + only_pr: false UbuntuPython310: - python.version: '3.10' - imageName: 'ubuntu-latest' + py: '3.10' + img: 'ubuntu-latest' + only_pr: true WindowsPython38: - python.version: '3.8' - imageName: 'windows-latest' + py: '3.8' + img: 'windows-latest' + only_pr: false WindowsPython310: - python.version: '3.10' - imageName: 'windows-latest' + py: '3.10' + img: 'windows-latest' + only_pr: false MacOSPython38: - python.version: '3.8' - imageName: 'macOS-latest' + py: '3.8' + img: 'macOS-latest' + only_pr: false MacOSPython310: - python.version: '3.10' - imageName: 'macOS-latest' - ${{ if not(eq(variables['Build.Reason'], 'PullRequest')) }}: - matrix: - UbuntuPython310: - python.version: '3.10' - imageName: 'ubuntu-latest' + py: '3.10' + img: 'macOS-latest' + only_pr: false -pool: - vmImage: $(imageName) +stages: + - stage: Test + jobs: + - ${{ each item in parameters.matrix }}: + ${{ if or(eq(variables['Build.Reason'], 'PullRequest'), item.value.on_pr) }}: + - job: ${{ item.Key }} + pool: + vmImage: ${{ item.Value.img }} + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: ${{ item.Value.py }} + displayName: 'Use Python ${{ item.Value.py }}' -steps: -- task: UsePythonVersion@0 - inputs: - versionSpec: '$(python.version)' - displayName: 'Use Python $(python.version)' + - script: | + python -m pip install pre-commit + pre-commit install + pre-commit run --all-files + displayName: 'Run pre-commits' -- script: | - python -m pip install pre-commit - pre-commit install - pre-commit run --all-files - displayName: 'Run pre-commits' + - script: | + python -m pip install --upgrade pip==23.0.1 + pip install wheel==0.38.4 --upgrade + pip install setuptools==66 --upgrade + pip install build==0.10.0 + pip install pytest-azurepipelines + displayName: 'Install build dependencies' -- script: | - python -m pip install --upgrade pip==23.0.1 - pip install wheel==0.38.4 --upgrade - pip install setuptools==66 --upgrade - pip install build==0.10.0 - pip install pytest-azurepipelines - displayName: 'Install build dependencies' + - script: | + python -m build + displayName: 'Build PrimAITE' -- script: | - python -m build - displayName: 'Build PrimAITE' + - script: | + PRIMAITE_WHEEL=$(ls ./dist/primaite*.whl) + python -m pip install $PRIMAITE_WHEEL[dev] + displayName: 'Install PrimAITE' + condition: or(eq( variables['Agent.OS'], 'Linux' ), eq( variables['Agent.OS'], 'Darwin' )) -- script: | - PRIMAITE_WHEEL=$(ls ./dist/primaite*.whl) - python -m pip install $PRIMAITE_WHEEL[dev] - displayName: 'Install PrimAITE' - condition: or(eq( variables['Agent.OS'], 'Linux' ), eq( variables['Agent.OS'], 'Darwin' )) + - script: | + forfiles /p dist\ /m *.whl /c "cmd /c python -m pip install @file[dev]" + displayName: 'Install PrimAITE' + condition: eq( variables['Agent.OS'], 'Windows_NT' ) -- script: | - forfiles /p dist\ /m *.whl /c "cmd /c python -m pip install @file[dev]" - displayName: 'Install PrimAITE' - condition: eq( variables['Agent.OS'], 'Windows_NT' ) + - script: | + primaite setup + displayName: 'Perform PrimAITE Setup' -- script: | - primaite setup - displayName: 'Perform PrimAITE Setup' - -- script: | - pytest tests/ - displayName: 'Run tests' + - script: | + pytest tests/ + displayName: 'Run tests' From d555584e90a06082e6631a8794890fe8a6fc995c Mon Sep 17 00:00:00 2001 From: Marek Wolan Date: Tue, 11 Jul 2023 08:08:29 +0000 Subject: [PATCH 33/41] Potentially fix syntax error --- .azure/azure-ci-build-pipeline.yaml | 68 ++++++++++++++--------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/.azure/azure-ci-build-pipeline.yaml b/.azure/azure-ci-build-pipeline.yaml index b1557b7b..ba2c6d51 100644 --- a/.azure/azure-ci-build-pipeline.yaml +++ b/.azure/azure-ci-build-pipeline.yaml @@ -45,44 +45,44 @@ stages: pool: vmImage: ${{ item.Value.img }} steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: ${{ item.Value.py }} - displayName: 'Use Python ${{ item.Value.py }}' + - task: UsePythonVersion@0 + inputs: + versionSpec: ${{ item.Value.py }} + displayName: 'Use Python ${{ item.Value.py }}' - - script: | - python -m pip install pre-commit - pre-commit install - pre-commit run --all-files - displayName: 'Run pre-commits' + - script: | + python -m pip install pre-commit + pre-commit install + pre-commit run --all-files + displayName: 'Run pre-commits' - - script: | - python -m pip install --upgrade pip==23.0.1 - pip install wheel==0.38.4 --upgrade - pip install setuptools==66 --upgrade - pip install build==0.10.0 - pip install pytest-azurepipelines - displayName: 'Install build dependencies' + - script: | + python -m pip install --upgrade pip==23.0.1 + pip install wheel==0.38.4 --upgrade + pip install setuptools==66 --upgrade + pip install build==0.10.0 + pip install pytest-azurepipelines + displayName: 'Install build dependencies' - - script: | - python -m build - displayName: 'Build PrimAITE' + - script: | + python -m build + displayName: 'Build PrimAITE' - - script: | - PRIMAITE_WHEEL=$(ls ./dist/primaite*.whl) - python -m pip install $PRIMAITE_WHEEL[dev] - displayName: 'Install PrimAITE' - condition: or(eq( variables['Agent.OS'], 'Linux' ), eq( variables['Agent.OS'], 'Darwin' )) + - script: | + PRIMAITE_WHEEL=$(ls ./dist/primaite*.whl) + python -m pip install $PRIMAITE_WHEEL[dev] + displayName: 'Install PrimAITE' + condition: or(eq( variables['Agent.OS'], 'Linux' ), eq( variables['Agent.OS'], 'Darwin' )) - - script: | - forfiles /p dist\ /m *.whl /c "cmd /c python -m pip install @file[dev]" - displayName: 'Install PrimAITE' - condition: eq( variables['Agent.OS'], 'Windows_NT' ) + - script: | + forfiles /p dist\ /m *.whl /c "cmd /c python -m pip install @file[dev]" + displayName: 'Install PrimAITE' + condition: eq( variables['Agent.OS'], 'Windows_NT' ) - - script: | - primaite setup - displayName: 'Perform PrimAITE Setup' + - script: | + primaite setup + displayName: 'Perform PrimAITE Setup' - - script: | - pytest tests/ - displayName: 'Run tests' + - script: | + pytest tests/ + displayName: 'Run tests' From c8191e60ba0c9c1800a5ffc9c41777e7f2576c96 Mon Sep 17 00:00:00 2001 From: Marek Wolan Date: Tue, 11 Jul 2023 08:14:08 +0000 Subject: [PATCH 34/41] Typo in word only --- .azure/azure-ci-build-pipeline.yaml | 78 ++++++++++++++--------------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/.azure/azure-ci-build-pipeline.yaml b/.azure/azure-ci-build-pipeline.yaml index ba2c6d51..276f0ef1 100644 --- a/.azure/azure-ci-build-pipeline.yaml +++ b/.azure/azure-ci-build-pipeline.yaml @@ -40,49 +40,49 @@ stages: - stage: Test jobs: - ${{ each item in parameters.matrix }}: - ${{ if or(eq(variables['Build.Reason'], 'PullRequest'), item.value.on_pr) }}: - - job: ${{ item.Key }} - pool: - vmImage: ${{ item.Value.img }} - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: ${{ item.Value.py }} - displayName: 'Use Python ${{ item.Value.py }}' + ${{ if or(eq(variables['Build.Reason'], 'PullRequest'), item.value.only_pr) }}: + - job: ${{ item.Key }} + pool: + vmImage: ${{ item.Value.img }} + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: ${{ item.Value.py }} + displayName: 'Use Python ${{ item.Value.py }}' - - script: | - python -m pip install pre-commit - pre-commit install - pre-commit run --all-files - displayName: 'Run pre-commits' + - script: | + python -m pip install pre-commit + pre-commit install + pre-commit run --all-files + displayName: 'Run pre-commits' - - script: | - python -m pip install --upgrade pip==23.0.1 - pip install wheel==0.38.4 --upgrade - pip install setuptools==66 --upgrade - pip install build==0.10.0 - pip install pytest-azurepipelines - displayName: 'Install build dependencies' + - script: | + python -m pip install --upgrade pip==23.0.1 + pip install wheel==0.38.4 --upgrade + pip install setuptools==66 --upgrade + pip install build==0.10.0 + pip install pytest-azurepipelines + displayName: 'Install build dependencies' - - script: | - python -m build - displayName: 'Build PrimAITE' + - script: | + python -m build + displayName: 'Build PrimAITE' - - script: | - PRIMAITE_WHEEL=$(ls ./dist/primaite*.whl) - python -m pip install $PRIMAITE_WHEEL[dev] - displayName: 'Install PrimAITE' - condition: or(eq( variables['Agent.OS'], 'Linux' ), eq( variables['Agent.OS'], 'Darwin' )) + - script: | + PRIMAITE_WHEEL=$(ls ./dist/primaite*.whl) + python -m pip install $PRIMAITE_WHEEL[dev] + displayName: 'Install PrimAITE' + condition: or(eq( variables['Agent.OS'], 'Linux' ), eq( variables['Agent.OS'], 'Darwin' )) - - script: | - forfiles /p dist\ /m *.whl /c "cmd /c python -m pip install @file[dev]" - displayName: 'Install PrimAITE' - condition: eq( variables['Agent.OS'], 'Windows_NT' ) + - script: | + forfiles /p dist\ /m *.whl /c "cmd /c python -m pip install @file[dev]" + displayName: 'Install PrimAITE' + condition: eq( variables['Agent.OS'], 'Windows_NT' ) - - script: | - primaite setup - displayName: 'Perform PrimAITE Setup' + - script: | + primaite setup + displayName: 'Perform PrimAITE Setup' - - script: | - pytest tests/ - displayName: 'Run tests' + - script: | + pytest tests/ + displayName: 'Run tests' From 7f64d06ad42f9230e922c96b42d5649f6ead420a Mon Sep 17 00:00:00 2001 From: Marek Wolan Date: Tue, 11 Jul 2023 08:14:34 +0000 Subject: [PATCH 35/41] Fix indent --- .azure/azure-ci-build-pipeline.yaml | 78 ++++++++++++++--------------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/.azure/azure-ci-build-pipeline.yaml b/.azure/azure-ci-build-pipeline.yaml index 276f0ef1..e1a26924 100644 --- a/.azure/azure-ci-build-pipeline.yaml +++ b/.azure/azure-ci-build-pipeline.yaml @@ -40,49 +40,49 @@ stages: - stage: Test jobs: - ${{ each item in parameters.matrix }}: - ${{ if or(eq(variables['Build.Reason'], 'PullRequest'), item.value.only_pr) }}: - - job: ${{ item.Key }} - pool: - vmImage: ${{ item.Value.img }} - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: ${{ item.Value.py }} - displayName: 'Use Python ${{ item.Value.py }}' + ${{ if or(eq(variables['Build.Reason'], 'PullRequest'), item.value.only_pr) }}: + - job: ${{ item.Key }} + pool: + vmImage: ${{ item.Value.img }} + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: ${{ item.Value.py }} + displayName: 'Use Python ${{ item.Value.py }}' - - script: | - python -m pip install pre-commit - pre-commit install - pre-commit run --all-files - displayName: 'Run pre-commits' + - script: | + python -m pip install pre-commit + pre-commit install + pre-commit run --all-files + displayName: 'Run pre-commits' - - script: | - python -m pip install --upgrade pip==23.0.1 - pip install wheel==0.38.4 --upgrade - pip install setuptools==66 --upgrade - pip install build==0.10.0 - pip install pytest-azurepipelines - displayName: 'Install build dependencies' + - script: | + python -m pip install --upgrade pip==23.0.1 + pip install wheel==0.38.4 --upgrade + pip install setuptools==66 --upgrade + pip install build==0.10.0 + pip install pytest-azurepipelines + displayName: 'Install build dependencies' - - script: | - python -m build - displayName: 'Build PrimAITE' + - script: | + python -m build + displayName: 'Build PrimAITE' - - script: | - PRIMAITE_WHEEL=$(ls ./dist/primaite*.whl) - python -m pip install $PRIMAITE_WHEEL[dev] - displayName: 'Install PrimAITE' - condition: or(eq( variables['Agent.OS'], 'Linux' ), eq( variables['Agent.OS'], 'Darwin' )) + - script: | + PRIMAITE_WHEEL=$(ls ./dist/primaite*.whl) + python -m pip install $PRIMAITE_WHEEL[dev] + displayName: 'Install PrimAITE' + condition: or(eq( variables['Agent.OS'], 'Linux' ), eq( variables['Agent.OS'], 'Darwin' )) - - script: | - forfiles /p dist\ /m *.whl /c "cmd /c python -m pip install @file[dev]" - displayName: 'Install PrimAITE' - condition: eq( variables['Agent.OS'], 'Windows_NT' ) + - script: | + forfiles /p dist\ /m *.whl /c "cmd /c python -m pip install @file[dev]" + displayName: 'Install PrimAITE' + condition: eq( variables['Agent.OS'], 'Windows_NT' ) - - script: | - primaite setup - displayName: 'Perform PrimAITE Setup' + - script: | + primaite setup + displayName: 'Perform PrimAITE Setup' - - script: | - pytest tests/ - displayName: 'Run tests' + - script: | + pytest tests/ + displayName: 'Run tests' From c641f67914024afdd6600df797bfbf5751f12bf4 Mon Sep 17 00:00:00 2001 From: Marek Wolan Date: Tue, 11 Jul 2023 08:15:16 +0000 Subject: [PATCH 36/41] Capitalisation error in value --- .azure/azure-ci-build-pipeline.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure/azure-ci-build-pipeline.yaml b/.azure/azure-ci-build-pipeline.yaml index e1a26924..49fd7174 100644 --- a/.azure/azure-ci-build-pipeline.yaml +++ b/.azure/azure-ci-build-pipeline.yaml @@ -40,7 +40,7 @@ stages: - stage: Test jobs: - ${{ each item in parameters.matrix }}: - ${{ if or(eq(variables['Build.Reason'], 'PullRequest'), item.value.only_pr) }}: + ${{ if or(eq(variables['Build.Reason'], 'PullRequest'), item.Value.only_pr) }}: - job: ${{ item.Key }} pool: vmImage: ${{ item.Value.img }} From 81a8058836ac30259ca570e0c912e2b385cdbc55 Mon Sep 17 00:00:00 2001 From: Marek Wolan Date: Tue, 11 Jul 2023 08:22:30 +0000 Subject: [PATCH 37/41] Change parameter matrix to list instead of dict --- .azure/azure-ci-build-pipeline.yaml | 128 ++++++++++++++-------------- 1 file changed, 64 insertions(+), 64 deletions(-) diff --git a/.azure/azure-ci-build-pipeline.yaml b/.azure/azure-ci-build-pipeline.yaml index 49fd7174..de760316 100644 --- a/.azure/azure-ci-build-pipeline.yaml +++ b/.azure/azure-ci-build-pipeline.yaml @@ -11,78 +11,78 @@ parameters: - name: matrix type: object default: - UbuntuPython38: - py: '3.8' - img: 'ubuntu-latest' - only_pr: false - UbuntuPython310: - py: '3.10' - img: 'ubuntu-latest' - only_pr: true - WindowsPython38: - py: '3.8' - img: 'windows-latest' - only_pr: false - WindowsPython310: - py: '3.10' - img: 'windows-latest' - only_pr: false - MacOSPython38: - py: '3.8' - img: 'macOS-latest' - only_pr: false - MacOSPython310: - py: '3.10' - img: 'macOS-latest' - only_pr: false + - job_name: 'UbuntuPython38' + py: '3.8' + img: 'ubuntu-latest' + only_pr: false + - job_name: 'UbuntuPython310' + py: '3.10' + img: 'ubuntu-latest' + only_pr: true + - job_name: 'WindowsPython38' + py: '3.8' + img: 'windows-latest' + only_pr: false + - job_name: 'WindowsPython310' + py: '3.10' + img: 'windows-latest' + only_pr: false + - job_name: 'MacOSPython38' + py: '3.8' + img: 'macOS-latest' + only_pr: false + - job_name: 'MacOSPython310' + py: '3.10' + img: 'macOS-latest' + only_pr: false stages: - stage: Test jobs: - - ${{ each item in parameters.matrix }}: - ${{ if or(eq(variables['Build.Reason'], 'PullRequest'), item.Value.only_pr) }}: - - job: ${{ item.Key }} - pool: - vmImage: ${{ item.Value.img }} - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: ${{ item.Value.py }} - displayName: 'Use Python ${{ item.Value.py }}' + - ${{ each item in parameters.matrix }}: + ${{ if or(eq(variables['Build.Reason'], 'PullRequest'), item.only_pr) }}: + - job: ${{ item.job_name }} + pool: + vmImage: ${{ item.img }} + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: ${{ item.py }} + displayName: 'Use Python ${{ item.py }}' - - script: | - python -m pip install pre-commit - pre-commit install - pre-commit run --all-files - displayName: 'Run pre-commits' + - script: | + python -m pip install pre-commit + pre-commit install + pre-commit run --all-files + displayName: 'Run pre-commits' - - script: | - python -m pip install --upgrade pip==23.0.1 - pip install wheel==0.38.4 --upgrade - pip install setuptools==66 --upgrade - pip install build==0.10.0 - pip install pytest-azurepipelines - displayName: 'Install build dependencies' + - script: | + python -m pip install --upgrade pip==23.0.1 + pip install wheel==0.38.4 --upgrade + pip install setuptools==66 --upgrade + pip install build==0.10.0 + pip install pytest-azurepipelines + displayName: 'Install build dependencies' - - script: | - python -m build - displayName: 'Build PrimAITE' + - script: | + python -m build + displayName: 'Build PrimAITE' - - script: | - PRIMAITE_WHEEL=$(ls ./dist/primaite*.whl) - python -m pip install $PRIMAITE_WHEEL[dev] - displayName: 'Install PrimAITE' - condition: or(eq( variables['Agent.OS'], 'Linux' ), eq( variables['Agent.OS'], 'Darwin' )) + - script: | + PRIMAITE_WHEEL=$(ls ./dist/primaite*.whl) + python -m pip install $PRIMAITE_WHEEL[dev] + displayName: 'Install PrimAITE' + condition: or(eq( variables['Agent.OS'], 'Linux' ), eq( variables['Agent.OS'], 'Darwin' )) - - script: | - forfiles /p dist\ /m *.whl /c "cmd /c python -m pip install @file[dev]" - displayName: 'Install PrimAITE' - condition: eq( variables['Agent.OS'], 'Windows_NT' ) + - script: | + forfiles /p dist\ /m *.whl /c "cmd /c python -m pip install @file[dev]" + displayName: 'Install PrimAITE' + condition: eq( variables['Agent.OS'], 'Windows_NT' ) - - script: | - primaite setup - displayName: 'Perform PrimAITE Setup' + - script: | + primaite setup + displayName: 'Perform PrimAITE Setup' - - script: | - pytest tests/ - displayName: 'Run tests' + - script: | + pytest tests/ + displayName: 'Run tests' From a303e9096a8cf6a1d31698a7a9d841b270969317 Mon Sep 17 00:00:00 2001 From: Marek Wolan Date: Tue, 11 Jul 2023 08:53:37 +0000 Subject: [PATCH 38/41] Changed structure of build pipeline yaml --- .azure/azure-ci-build-pipeline.yaml | 92 +++++++++++++++-------------- 1 file changed, 47 insertions(+), 45 deletions(-) diff --git a/.azure/azure-ci-build-pipeline.yaml b/.azure/azure-ci-build-pipeline.yaml index de760316..d8568d46 100644 --- a/.azure/azure-ci-build-pipeline.yaml +++ b/.azure/azure-ci-build-pipeline.yaml @@ -14,75 +14,77 @@ parameters: - job_name: 'UbuntuPython38' py: '3.8' img: 'ubuntu-latest' - only_pr: false + every_time: false - job_name: 'UbuntuPython310' py: '3.10' img: 'ubuntu-latest' - only_pr: true + every_time: true - job_name: 'WindowsPython38' py: '3.8' img: 'windows-latest' - only_pr: false + every_time: false - job_name: 'WindowsPython310' py: '3.10' img: 'windows-latest' - only_pr: false + every_time: false - job_name: 'MacOSPython38' py: '3.8' img: 'macOS-latest' - only_pr: false + every_time: false - job_name: 'MacOSPython310' py: '3.10' img: 'macOS-latest' - only_pr: false + every_time: false stages: - stage: Test jobs: - ${{ each item in parameters.matrix }}: - ${{ if or(eq(variables['Build.Reason'], 'PullRequest'), item.only_pr) }}: - - job: ${{ item.job_name }} - pool: - vmImage: ${{ item.img }} - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: ${{ item.py }} - displayName: 'Use Python ${{ item.py }}' + - job: ${{ item.job_name }} + pool: + vmImage: ${{ item.img }} + + condition: or( eq(variables['BuildReason'], 'PullRequest'), item.every_time ) - - script: | - python -m pip install pre-commit - pre-commit install - pre-commit run --all-files - displayName: 'Run pre-commits' + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: ${{ item.py }} + displayName: 'Use Python ${{ item.py }}' - - script: | - python -m pip install --upgrade pip==23.0.1 - pip install wheel==0.38.4 --upgrade - pip install setuptools==66 --upgrade - pip install build==0.10.0 - pip install pytest-azurepipelines - displayName: 'Install build dependencies' + - script: | + python -m pip install pre-commit + pre-commit install + pre-commit run --all-files + displayName: 'Run pre-commits' - - script: | - python -m build - displayName: 'Build PrimAITE' + - script: | + python -m pip install --upgrade pip==23.0.1 + pip install wheel==0.38.4 --upgrade + pip install setuptools==66 --upgrade + pip install build==0.10.0 + pip install pytest-azurepipelines + displayName: 'Install build dependencies' - - script: | - PRIMAITE_WHEEL=$(ls ./dist/primaite*.whl) - python -m pip install $PRIMAITE_WHEEL[dev] - displayName: 'Install PrimAITE' - condition: or(eq( variables['Agent.OS'], 'Linux' ), eq( variables['Agent.OS'], 'Darwin' )) + - script: | + python -m build + displayName: 'Build PrimAITE' - - script: | - forfiles /p dist\ /m *.whl /c "cmd /c python -m pip install @file[dev]" - displayName: 'Install PrimAITE' - condition: eq( variables['Agent.OS'], 'Windows_NT' ) + - script: | + PRIMAITE_WHEEL=$(ls ./dist/primaite*.whl) + python -m pip install $PRIMAITE_WHEEL[dev] + displayName: 'Install PrimAITE' + condition: or(eq( variables['Agent.OS'], 'Linux' ), eq( variables['Agent.OS'], 'Darwin' )) - - script: | - primaite setup - displayName: 'Perform PrimAITE Setup' + - script: | + forfiles /p dist\ /m *.whl /c "cmd /c python -m pip install @file[dev]" + displayName: 'Install PrimAITE' + condition: eq( variables['Agent.OS'], 'Windows_NT' ) - - script: | - pytest tests/ - displayName: 'Run tests' + - script: | + primaite setup + displayName: 'Perform PrimAITE Setup' + + - script: | + pytest tests/ + displayName: 'Run tests' From dcf5bfddfaee41081a14fd8acc3b622a2f06bace Mon Sep 17 00:00:00 2001 From: Marek Wolan Date: Tue, 11 Jul 2023 08:54:22 +0000 Subject: [PATCH 39/41] Fix syntax --- .azure/azure-ci-build-pipeline.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure/azure-ci-build-pipeline.yaml b/.azure/azure-ci-build-pipeline.yaml index d8568d46..e3f11b7f 100644 --- a/.azure/azure-ci-build-pipeline.yaml +++ b/.azure/azure-ci-build-pipeline.yaml @@ -44,7 +44,7 @@ stages: pool: vmImage: ${{ item.img }} - condition: or( eq(variables['BuildReason'], 'PullRequest'), item.every_time ) + condition: or( eq(variables['BuildReason'], 'PullRequest'), ${{ item.every_time }} ) steps: - task: UsePythonVersion@0 From 6c7ec6216680f7043e30939da7a0b7ecc2312edc Mon Sep 17 00:00:00 2001 From: Marek Wolan Date: Tue, 11 Jul 2023 09:57:27 +0100 Subject: [PATCH 40/41] Fixed formatting with pre-commit --- .azure/azure-ci-build-pipeline.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure/azure-ci-build-pipeline.yaml b/.azure/azure-ci-build-pipeline.yaml index e3f11b7f..4001b1c5 100644 --- a/.azure/azure-ci-build-pipeline.yaml +++ b/.azure/azure-ci-build-pipeline.yaml @@ -43,7 +43,7 @@ stages: - job: ${{ item.job_name }} pool: vmImage: ${{ item.img }} - + condition: or( eq(variables['BuildReason'], 'PullRequest'), ${{ item.every_time }} ) steps: From 1633900ce76cbf9164d1c0a44c78852f49a0c36d Mon Sep 17 00:00:00 2001 From: Marek Wolan Date: Tue, 11 Jul 2023 09:01:43 +0000 Subject: [PATCH 41/41] Fix typo in `Build.Reason` --- .azure/azure-ci-build-pipeline.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure/azure-ci-build-pipeline.yaml b/.azure/azure-ci-build-pipeline.yaml index 4001b1c5..066c66b2 100644 --- a/.azure/azure-ci-build-pipeline.yaml +++ b/.azure/azure-ci-build-pipeline.yaml @@ -44,7 +44,7 @@ stages: pool: vmImage: ${{ item.img }} - condition: or( eq(variables['BuildReason'], 'PullRequest'), ${{ item.every_time }} ) + condition: or( eq(variables['Build.Reason'], 'PullRequest'), ${{ item.every_time }} ) steps: - task: UsePythonVersion@0