Merge remote-tracking branch 'refs/remotes/origin/release/3.2.0' into dev

# Conflicts:
#	src/primaite/VERSION

benchmark/report.py

@@ -17,8 +17,6 @@ PLOT_CONFIG = {
     "size": {"auto_size": False, "width": 1500, "height": 900},
     "template": "plotly_white",
     "range_slider": False,
-    "av_s_per_100_steps_10_nodes_benchmark_threshold": 5,
-    "benchmark_line_color": "grey",
 }
 
 
@@ -229,10 +227,7 @@ def _plot_av_s_per_100_steps_10_nodes(
     """
     Creates a bar chart visualising the performance of each version of PrimAITE.
 
-    Performance is based on the average training time per 100 steps on 10 nodes. The function also includes a benchmark
-    line indicating the target maximum time.
-
-    Versions that perform under this time are marked in green, and those over are marked in red.
+    Performance is based on the average training time per 100 steps on 10 nodes.
 
     :param version_times_dict: A dictionary with software versions as keys and average times as values.
     :return: A Plotly figure object representing the bar chart of the performance metrics.
@@ -256,7 +251,6 @@ def _plot_av_s_per_100_steps_10_nodes(
     versions = sorted(list(version_times_dict.keys()))
     times = [version_times_dict[version] for version in versions]
     av_s_per_100_steps_10_nodes_benchmark_threshold = PLOT_CONFIG["av_s_per_100_steps_10_nodes_benchmark_threshold"]
-    benchmark_line_color = PLOT_CONFIG["benchmark_line_color"]
 
     # Calculate the appropriate maximum y-axis value
     max_y_axis_value = max(max(times), av_s_per_100_steps_10_nodes_benchmark_threshold) + 1
@@ -265,28 +259,11 @@ def _plot_av_s_per_100_steps_10_nodes(
         go.Bar(
             x=versions,
             y=times,
-            marker_color=[
-                "green" if time < av_s_per_100_steps_10_nodes_benchmark_threshold else "red" for time in times
-            ],
             text=times,
             textposition="auto",
         )
     )
 
-    # Add a horizontal line for the benchmark
-    fig.add_shape(
-        type="line",
-        x0=-0.5,  # start slightly before the first bar
-        x1=len(versions) - 0.5,  # end slightly after the last bar
-        y0=av_s_per_100_steps_10_nodes_benchmark_threshold,
-        y1=av_s_per_100_steps_10_nodes_benchmark_threshold,
-        line=dict(
-            color=benchmark_line_color,
-            width=2,
-            dash="dot",
-        ),
-    )
-
     fig.update_layout(
         xaxis_title="PrimAITE Version",
         yaxis_title="Avg Time per 100 Steps on 10 Nodes (seconds)",

benchmark/results/v3/v3.1.0/session_metadata/1.json

@@ -1006,4 +1006,4 @@
         "999": 112.00000000000009,
         "1000": 102.55000000000008
     }
-}
+}

benchmark/results/v3/v3.1.0/session_metadata/2.json

@@ -1006,4 +1006,4 @@
         "999": 110.30000000000007,
         "1000": 118.05000000000011
     }
-}
+}

benchmark/results/v3/v3.1.0/session_metadata/3.json

@@ -1006,4 +1006,4 @@
         "999": 118.44999999999996,
         "1000": 108.20000000000029
     }
-}
+}

benchmark/results/v3/v3.1.0/session_metadata/4.json

@@ -1006,4 +1006,4 @@
         "999": 99.50000000000009,
         "1000": 98.70000000000006
     }
-}
+}

benchmark/results/v3/v3.1.0/session_metadata/5.json

@@ -1006,4 +1006,4 @@
         "999": 118.45000000000007,
         "1000": 116.3000000000001
     }
-}
+}

benchmark/results/v3/v3.2.0/PrimAITE v3.2.0 Benchmark Report.md

@@ -0,0 +1,38 @@
+# PrimAITE v3.2.0 Learning Benchmark
+## PrimAITE Dev Team
+### 2024-07-21
+
+---
+## 1 Introduction
+PrimAITE v3.2.0 was benchmarked automatically upon release. Learning rate metrics were captured to be referenced during system-level testing and user acceptance testing (UAT).
+The benchmarking process consists of running 5 training session using the same config file. Each session trains an agent for 1000 episodes, with each episode consisting of 128 steps.
+The total reward per episode from each session is captured. This is then used to calculate an caverage total reward per episode from the 5 individual sessions for smoothing. Finally, a 25-widow rolling average of the average total reward per session is calculated for further smoothing.
+## 2 System Information
+### 2.1 Python
+**Version:** 3.10.14 (main, Apr  6 2024, 18:45:05) [GCC 9.4.0]
+### 2.2 System
+- **OS:** Linux
+- **OS Version:** #76~20.04.1-Ubuntu SMP Thu Jun 13 18:00:23 UTC 2024
+- **Machine:** x86_64
+- **Processor:** x86_64
+### 2.3 CPU
+- **Physical Cores:** 2
+- **Total Cores:** 4
+- **Max Frequency:** 0.00Mhz
+### 2.4 Memory
+- **Total:** 15.62GB
+- **Swap Total:** 0.00B
+## 3 Stats
+- **Total Sessions:** 5
+- **Total Episodes:** 5005
+- **Total Steps:** 640000
+- **Av Session Duration (s):** 1691.5034
+- **Av Step Duration (s):** 0.0529
+- **Av Duration per 100 Steps per 10 Nodes (s):** 5.2859
+## 4 Graphs
+### 4.1 v3.2.0 Learning Benchmark Plot
+![PrimAITE 3.2.0 Learning Benchmark Plot](PrimAITE v3.2.0 Learning Benchmark.png)
+### 4.2 Learning Benchmark of Minor and Bugfix Releases for Major Version 3
+![Learning Benchmark of Minor and Bugfix Releases for Major Version 3](PrimAITE Learning Benchmark of Minor and Bugfix Releases for Major Version 3.png)
+### 4.3 Performance of Minor and Bugfix Releases for Major Version 3
+![Performance of Minor and Bugfix Releases for Major Version 3](PrimAITE Performance of Minor and Bugfix Releases for Major Version 3.png)