#2628: committing to be reviewed

.gitignore

@@ -166,4 +166,5 @@ sandbox/
 sandbox.ipynb
 
 # benchmarking
-**/benchmark_session/
+**/benchmark/sessions/
+**/benchmark/output/

benchmark/primaite_benchmark.py

@@ -13,7 +13,7 @@ from primaite.config.load import data_manipulation_config_path
 
 _LOGGER = primaite.getLogger(__name__)
 
-_BENCHMARK_ROOT = Path(__file__).parent / "benchmark_session"
+_BENCHMARK_ROOT = Path(__file__).parent
 _RESULTS_ROOT: Final[Path] = _BENCHMARK_ROOT / "results"
 _RESULTS_ROOT.mkdir(exist_ok=True, parents=True)
 
@@ -33,9 +33,15 @@ class BenchmarkSession:
     num_episodes: int
     """Number of episodes to run the training session."""
 
+    num_steps: int
+    """Number of steps to run the training session."""
+
     batch_size: int
     """Number of steps for each episode."""
 
+    learning_rate: float
+    """Learning rate for the model."""
+
     start_time: datetime
     """Start time for the session."""
 
@@ -45,11 +51,15 @@ class BenchmarkSession:
     session_metadata: Dict
     """Dict containing the metadata for the session - used to generate benchmark report."""
 
-    def __init__(self, gym_env: BenchmarkPrimaiteGymEnv, num_episodes: int, batch_size: int):
+    def __init__(
+        self, gym_env: BenchmarkPrimaiteGymEnv, num_episodes: int, num_steps: int, batch_size: int, learning_rate: float
+    ):
         """Initialise the BenchmarkSession."""
         self.gym_env = gym_env
         self.num_episodes = num_episodes
+        self.num_steps = num_steps
         self.batch_size = batch_size
+        self.learning_rate = learning_rate
 
     def train(self):
         """Run the training session."""
@@ -59,10 +69,11 @@ class BenchmarkSession:
         model = PPO(
             policy="MlpPolicy",
             env=self.gym_env,
-            batch_size=self.batch_size,
-            n_steps=self.batch_size * self.num_episodes,
+            learning_rate=self.learning_rate,
+            n_steps=self.num_steps * self.num_episodes,
+            batch_size=self.num_steps * self.num_episodes,
         )
-        model.learn(total_timesteps=self.num_episodes * self.gym_env.game.options.max_episode_length)
+        model.learn(total_timesteps=self.num_episodes * self.num_steps)
 
         # end timer for session
         self.end_time = datetime.now()
@@ -108,14 +119,13 @@ class BenchmarkSession:
         }
 
 
-def _get_benchmark_primaite_environment(num_timesteps: int) -> BenchmarkPrimaiteGymEnv:
+def _get_benchmark_primaite_environment() -> BenchmarkPrimaiteGymEnv:
     """
     Create an instance of the BenchmarkPrimaiteGymEnv.
 
     This environment will be used to train the agents on.
     """
     env = BenchmarkPrimaiteGymEnv(env_config=data_manipulation_config_path())
-    env.game.options.max_episode_length = num_timesteps
     return env
 
 
@@ -132,7 +142,11 @@ def _prepare_session_directory():
 
 
 def run(
-    number_of_sessions: int = 3, num_episodes: int = 3, num_timesteps: int = 128, batch_size: int = 128
+    number_of_sessions: int = 10,
+    num_episodes: int = 1000,
+    num_timesteps: int = 128,
+    batch_size: int = 1280,
+    learning_rate: float = 3e-4,
 ) -> None:  # 10  # 1000  # 256
     """Run the PrimAITE benchmark."""
     benchmark_start_time = datetime.now()
@@ -145,8 +159,14 @@ def run(
     for i in range(1, number_of_sessions + 1):
         print(f"Starting Benchmark Session: {i}")
 
-        with _get_benchmark_primaite_environment(num_timesteps=num_timesteps) as gym_env:
-            session = BenchmarkSession(gym_env=gym_env, num_episodes=num_episodes, batch_size=batch_size)
+        with _get_benchmark_primaite_environment() as gym_env:
+            session = BenchmarkSession(
+                gym_env=gym_env,
+                num_episodes=num_episodes,
+                num_steps=num_timesteps,
+                batch_size=batch_size,
+                learning_rate=learning_rate,
+            )
             session.train()
             session_metadata_dict[i] = session.session_metadata
 

benchmark/results/v3.0.0/PrimAITE v3.0.0 Learning Benchmark.tex

@@ -0,0 +1,122 @@
+\documentclass{article}%
+\usepackage[T1]{fontenc}%
+\usepackage[utf8]{inputenc}%
+\usepackage{lmodern}%
+\usepackage{textcomp}%
+\usepackage{lastpage}%
+\usepackage{geometry}%
+\geometry{tmargin=2.5cm,rmargin=2.5cm,bmargin=2.5cm,lmargin=2.5cm}%
+\usepackage{graphicx}%
+%
+\title{PrimAITE 3.0.0 Learning Benchmark}%
+\author{PrimAITE Dev Team}%
+\date{2024{-}06{-}01}%
+%
+\begin{document}%
+\normalsize%
+\maketitle%
+\section{Introduction}%
+\label{sec:Introduction}%
+PrimAITE v3.0.0 was benchmarked automatically upon release. Learning rate metrics were captured to be referenced during system{-}level testing and user acceptance testing (UAT).%
+\newline%
+The benchmarking process consists of running 10 training session using the same config file. Each session trains an agent for 1000 episodes, with each episode consisting of 128 steps.%
+\newline%
+The mean reward per episode from each session is captured. This is then used to calculate a combined average reward per episode from the 10 individual sessions for smoothing. Finally, a 25{-}widow rolling average of the combined average reward per session is calculated for further smoothing.
+
+%
+\section{System Information}%
+\label{sec:SystemInformation}%
+\subsection{Python}%
+\label{subsec:Python}%
+\begin{tabular}{|l|l|}%
+\hline%
+\textbf{Version}&3.8.10 (tags/v3.8.10:3d8993a, May  3 2021, 11:48:03) {[}MSC v.1928 64 bit (AMD64){]}\\%
+\hline%
+\end{tabular}
+
+%
+\subsection{System}%
+\label{subsec:System}%
+\begin{tabular}{|l|l|}%
+\hline%
+\textbf{OS}&Windows\\%
+\hline%
+\textbf{OS Version}&10.0.19045\\%
+\hline%
+\textbf{Machine}&AMD64\\%
+\hline%
+\textbf{Processor}&Intel64 Family 6 Model 85 Stepping 4, GenuineIntel\\%
+\hline%
+\end{tabular}
+
+%
+\subsection{CPU}%
+\label{subsec:CPU}%
+\begin{tabular}{|l|l|}%
+\hline%
+\textbf{Physical Cores}&6\\%
+\hline%
+\textbf{Total Cores}&12\\%
+\hline%
+\textbf{Max Frequency}&3600.00Mhz\\%
+\hline%
+\end{tabular}
+
+%
+\subsection{Memory}%
+\label{subsec:Memory}%
+\begin{tabular}{|l|l|}%
+\hline%
+\textbf{Total}&63.52GB\\%
+\hline%
+\textbf{Swap Total}&9.50GB\\%
+\hline%
+\end{tabular}
+
+%
+\section{Stats}%
+\label{sec:Stats}%
+\subsection{Benchmark Results}%
+\label{subsec:BenchmarkResults}%
+\begin{tabular}{|l|l|}%
+\hline%
+\textbf{Total Sessions}&10\\%
+\hline%
+\textbf{Total Episodes}&10010\\%
+\hline%
+\textbf{Total Steps}&1280000\\%
+\hline%
+\textbf{Av Session Duration (s)}&1569.8775\\%
+\hline%
+\textbf{Av Step Duration (s)}&0.0012\\%
+\hline%
+\textbf{Av Duration per 100 Steps per 10 Nodes (s)}&0.1226\\%
+\hline%
+\end{tabular}
+
+%
+\section{Graphs}%
+\label{sec:Graphs}%
+\subsection{PrimAITE 3.0.0 Learning Benchmark Plot}%
+\label{subsec:PrimAITE3.0.0LearningBenchmarkPlot}%
+
+
+\begin{figure}[h!]%
+\centering%
+\includegraphics[width=0.8\textwidth]{D:/Projects/ARCD/PrimAITE/PrimAITE/benchmark/results/v3.0.0/PrimAITE v3.0.0 Learning Benchmark.png}%
+\caption{PrimAITE 3.0.0 Learning Benchmark Plot}%
+\end{figure}
+
+%
+\subsection{PrimAITE All Versions Learning Benchmark Plot}%
+\label{subsec:PrimAITEAllVersionsLearningBenchmarkPlot}%
+
+
+\begin{figure}[h!]%
+\centering%
+\includegraphics[width=0.8\textwidth]{D:/Projects/ARCD/PrimAITE/PrimAITE/benchmark/results/PrimAITE Versions Learning Benchmark.png}%
+\caption{PrimAITE All Versions Learning Benchmark Plot}%
+\end{figure}
+
+%
+\end{document}

src/primaite/VERSION

@@ -1 +1 @@
-3.0.0b9
+3.0.0