Merge remote-tracking branch 'origin/release/3.2.0' into backport-3.2

.azure/azure-benchmark-pipeline.yaml

@@ -1,9 +1,5 @@
 trigger:
-  branches:
-    exclude:
-    - '*'
-    include:
-    - 'refs/heads/release/*'
+- release/*
 
 schedules:
 - cron: "0 2 * * 1-5"  # Run at 2 AM every weekday
@@ -19,13 +15,18 @@ jobs:
 - job: PrimAITE_Benchmark
   timeoutInMinutes: 360 # 6-hour maximum
   pool:
-    vmImage: ubuntu-latest
+    name: 'Imaginary Yak Pool'
   workspace:
     clean: all
+
   steps:
   - checkout: self
     persistCredentials: true
 
+  - script: |
+      python3.10 -m venv venv
+    displayName: 'Create venv'
+
   - script: |
       VERSION=$(cat src/primaite/VERSION | tr -d '\n')
       if [[ "$(Build.SourceBranch)" == "refs/heads/dev" ]]; then
@@ -41,41 +42,21 @@ jobs:
       echo "##vso[task.setvariable variable=MAJOR_VERSION]$MAJOR_VERSION"
     displayName: 'Set Version Variables'
 
-  - task: UsePythonVersion@0
-    inputs:
-      versionSpec: '3.11'
-      addToPath: true
-
   - script: |
-      python -m pip install --upgrade pip
+      source venv/bin/activate
+      pip install --upgrade pip
       pip install -e .[dev,rl]
       primaite setup
     displayName: 'Install Dependencies'
 
   - script: |
       set -e
+      source venv/bin/activate
       cd benchmark
-      python3 primaite_benchmark.py
+      python primaite_benchmark.py
       cd ..
     displayName: 'Run Benchmarking Script'
 
-  - script: |
-      git config --global user.email "oss@dstl.gov.uk"
-      git config --global user.name "Defence Science and Technology Laboratory UK"
-    workingDirectory: $(System.DefaultWorkingDirectory)
-    displayName: 'Configure Git'
-    condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release'))
-
-  - script: |
-      git add benchmark/results/v$(MAJOR_VERSION)/v$(VERSION)/*
-      git commit -m "Automated benchmark output commit for version $(VERSION)"
-      git push origin HEAD:refs/heads/$(Build.SourceBranchName)
-    displayName: 'Commit and Push Benchmark Results'
-    workingDirectory: $(System.DefaultWorkingDirectory)
-    env:
-      GIT_CREDENTIALS: $(System.AccessToken)
-    condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/heads/release'))
-
   - script: |
       tar czf primaite_v$(VERSION)_benchmark.tar.gz benchmark/results/v$(MAJOR_VERSION)/v$(VERSION)
     displayName: 'Prepare Artifacts for Publishing'
@@ -83,6 +64,45 @@ jobs:
   - task: PublishPipelineArtifact@1
     inputs:
       targetPath: primaite_v$(VERSION)_benchmark.tar.gz
-      artifactName: 'benchmark-output'
+      artifactName: 'benchmark-zip-output'
       publishLocation: 'pipeline'
-    displayName: 'Publish Benchmark Output as Artifact'
+    displayName: 'Publish Benchmark Output zip as Artifact'
+
+  - script: |
+      git config --global user.email "oss@dstl.gov.uk"
+      git config --global user.name "Defence Science and Technology Laboratory UK"
+    workingDirectory: $(System.DefaultWorkingDirectory)
+    displayName: 'Configure Git'
+    condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/heads/release'))
+
+  - script: |
+      echo "Fetching all branches..."
+      git fetch --all --prune
+
+      echo "Stashing files..."
+      git stash push -u
+
+      echo "Resolving branch name..."
+      # Extracting just the branch name from the full ref path
+      branch_name=$(echo "$(Build.SourceBranch)" | sed 's|refs/heads/||')
+      echo "Branch Name: $branch_name"
+
+      echo "Checking out branch $branch_name..."
+      git checkout $branch_name
+
+      echo "Popping stash..."
+      git stash pop
+
+      echo "Adding benchmark results..."
+      git add benchmark/results/v$(MAJOR_VERSION)/v$(VERSION)/*
+
+      echo "Committing changes..."
+      git commit -m "Automated benchmark output commit for version $(VERSION) [skip ci]"
+
+      echo "Pushing to remote..."
+      git push origin $branch_name
+    displayName: 'Commit and Push Benchmark Results'
+    workingDirectory: $(System.DefaultWorkingDirectory)
+    env:
+      GIT_CREDENTIALS: $(System.AccessToken)
+    condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/heads/release'))

benchmark/primaite_benchmark.py

@@ -5,7 +5,7 @@ from datetime import datetime
 from pathlib import Path
 from typing import Any, Dict, Final, Tuple
 
-from report import build_benchmark_latex_report
+from report import build_benchmark_md_report
 from stable_baselines3 import PPO
 
 import primaite
@@ -188,7 +188,7 @@ def run(
         with open(_SESSION_METADATA_ROOT / f"{i}.json", "r") as file:
             session_metadata_dict[i] = json.load(file)
     # generate report
-    build_benchmark_latex_report(
+    build_benchmark_md_report(
         benchmark_start_time=benchmark_start_time,
         session_metadata=session_metadata_dict,
         config_path=data_manipulation_config_path(),

benchmark/report.py

@@ -21,6 +21,14 @@ PLOT_CONFIG = {
 
 
 def _build_benchmark_results_dict(start_datetime: datetime, metadata_dict: Dict, config: Dict) -> dict:
+    """
+    Constructs a dictionary aggregating benchmark results from multiple sessions.
+
+    :param start_datetime: The datetime when the benchmarking started.
+    :param metadata_dict: Dictionary containing metadata for each session.
+    :param config: Configuration settings used during the benchmarking.
+    :return: A dictionary containing aggregated data and metadata from the benchmarking sessions.
+    """
     num_sessions = len(metadata_dict)  # number of sessions
 
     averaged_data = {
@@ -53,6 +61,12 @@ def _build_benchmark_results_dict(start_datetime: datetime, metadata_dict: Dict,
 
 
 def _get_df_from_episode_av_reward_dict(data: Dict) -> pl.DataFrame:
+    """
+    Converts a dictionary of episode average rewards into a Polars DataFrame.
+
+    :param data: Dictionary with episodes as keys and average rewards as values.
+    :return: Polars DataFrame with episodes and average rewards, including a rolling average.
+    """
     data: Dict = {"episode": data.keys(), "av_reward": data.values()}
 
     return (
@@ -67,6 +81,14 @@ def _plot_benchmark_metadata(
     title: Optional[str] = None,
     subtitle: Optional[str] = None,
 ) -> Figure:
+    """
+    Plots benchmark metadata as a line graph using Plotly.
+
+    :param benchmark_metadata_dict: Dictionary containing the total reward per episode and session.
+    :param title: Optional title for the graph.
+    :param subtitle: Optional subtitle for the graph.
+    :return: Plotly figure object representing the benchmark metadata plot.
+    """
     if title:
         if subtitle:
             title = f"{title} <br>{subtitle}</sup>"
@@ -136,7 +158,7 @@ def _plot_all_benchmarks_combined_session_av(results_directory: Path) -> Figure:
     converted into a polars dataframe, and plotted as a scatter line in plotly.
     """
     major_v = primaite.__version__.split(".")[0]
-    title = f"Learning Benchmarking of All Released Versions under Major v{major_v}.#.#"
+    title = f"Learning Benchmark of Minor and Bugfix Releases for Major Version {major_v}"
     subtitle = "Rolling Av (Combined Session Av)"
     if title:
         if subtitle:
@@ -176,10 +198,97 @@ def _plot_all_benchmarks_combined_session_av(results_directory: Path) -> Figure:
     return fig
 
 
-def build_benchmark_latex_report(
+def _get_performance_benchmark_for_all_version_dict(results_directory: Path) -> Dict[str, float]:
+    """
+    Gathers performance benchmarks for all versions of the software stored in a specified directory.
+
+    This function iterates through each directory within the specified results directory,
+    extracts the av_s_per_100_steps_10_nodes from the benchmark_metadata.json files, and aggregates it into a
+    dictionary.
+
+    :param results_directory: The directory containing subdirectories for each version's benchmark data.
+    :return: A dictionary with version numbers as keys and their corresponding average performance benchmark
+        (average time per 100 steps on 10 nodes) as values.
+    """
+    performance_benchmark_dict = {}
+    for dir in results_directory.iterdir():
+        if dir.is_dir():
+            metadata_file = dir / f"{dir.name}_benchmark_metadata.json"
+            with open(metadata_file, "r") as file:
+                metadata_dict = json.load(file)
+                version = metadata_dict["primaite_version"]
+                performance_benchmark_dict[version] = metadata_dict["av_s_per_100_steps_10_nodes"]
+    return performance_benchmark_dict
+
+
+def _plot_av_s_per_100_steps_10_nodes(
+    version_times_dict: Dict[str, float],
+) -> Figure:
+    """
+    Creates a bar chart visualising the performance of each version of PrimAITE.
+
+    Performance is based on the average training time per 100 steps on 10 nodes.
+
+    :param version_times_dict: A dictionary with software versions as keys and average times as values.
+    :return: A Plotly figure object representing the bar chart of the performance metrics.
+    """
+    major_v = primaite.__version__.split(".")[0]
+    title = f"Performance of Minor and Bugfix Releases for Major Version {major_v}"
+    subtitle = (
+        f"Average Training Time per 100 Steps on 10 Nodes "
+        f"(target: <= {PLOT_CONFIG['av_s_per_100_steps_10_nodes_benchmark_threshold']} seconds)"
+    )
+    title = f"{title} <br><sub>{subtitle}</sub>"
+
+    layout = go.Layout(
+        autosize=PLOT_CONFIG["size"]["auto_size"],
+        width=PLOT_CONFIG["size"]["width"],
+        height=PLOT_CONFIG["size"]["height"],
+    )
+    fig = go.Figure(layout=layout)
+    fig.update_layout(template=PLOT_CONFIG["template"])
+
+    versions = sorted(list(version_times_dict.keys()))
+    times = [version_times_dict[version] for version in versions]
+    av_s_per_100_steps_10_nodes_benchmark_threshold = PLOT_CONFIG["av_s_per_100_steps_10_nodes_benchmark_threshold"]
+
+    # Calculate the appropriate maximum y-axis value
+    max_y_axis_value = max(max(times), av_s_per_100_steps_10_nodes_benchmark_threshold) + 1
+
+    fig.add_trace(
+        go.Bar(
+            x=versions,
+            y=times,
+            text=times,
+            textposition="auto",
+        )
+    )
+
+    fig.update_layout(
+        xaxis_title="PrimAITE Version",
+        yaxis_title="Avg Time per 100 Steps on 10 Nodes (seconds)",
+        yaxis=dict(range=[0, max_y_axis_value]),
+        title=title,
+    )
+
+    return fig
+
+
+def build_benchmark_md_report(
     benchmark_start_time: datetime, session_metadata: Dict, config_path: Path, results_root_path: Path
 ) -> None:
-    """Generates a latex report of the benchmark run."""
+    """
+    Generates a Markdown report for a benchmarking session, documenting performance metrics and graphs.
+
+    This function orchestrates the creation of several graphs depicting various performance benchmarks and aggregates
+    them into a markdown document that includes comprehensive system and benchmark information.
+
+    :param benchmark_start_time: The datetime object representing when the benchmarking process was initiated.
+    :param session_metadata: A dictionary containing metadata for each benchmarking session.
+    :param config_path: A pathlib.Path object pointing to the configuration file used for the benchmark sessions.
+    :param results_root_path: A pathlib.Path object pointing to the directory where the results and graphs should be
+        saved.
+    """
     # generate report folder
     v_str = f"v{primaite.__version__}"
 
@@ -204,13 +313,21 @@ def build_benchmark_latex_report(
 
     fig = _plot_all_benchmarks_combined_session_av(results_directory=results_root_path)
 
-    all_version_plot_path = version_result_dir / "PrimAITE Versions Learning Benchmark.png"
+    filename = f"PrimAITE Learning Benchmark of Minor and Bugfix Releases for Major Version {major_v}.png"
+
+    all_version_plot_path = version_result_dir / filename
     fig.write_image(all_version_plot_path)
 
+    performance_benchmark_dict = _get_performance_benchmark_for_all_version_dict(results_directory=results_root_path)
+    fig = _plot_av_s_per_100_steps_10_nodes(performance_benchmark_dict)
+    filename = f"PrimAITE Performance of Minor and Bugfix Releases for Major Version {major_v}.png"
+    performance_benchmark_plot_path = version_result_dir / filename
+    fig.write_image(performance_benchmark_plot_path)
+
     data = benchmark_metadata_dict
     primaite_version = data["primaite_version"]
 
-    with open(version_result_dir / f"PrimAITE v{primaite_version} Learning Benchmark.md", "w") as file:
+    with open(version_result_dir / f"PrimAITE v{primaite_version} Benchmark Report.md", "w") as file:
         # Title
         file.write(f"# PrimAITE v{primaite_version} Learning Benchmark\n")
         file.write("## PrimAITE Dev Team\n")
@@ -273,8 +390,14 @@ def build_benchmark_latex_report(
         file.write(f"### 4.1 v{primaite_version} Learning Benchmark Plot\n")
         file.write(f"![PrimAITE {primaite_version} Learning Benchmark Plot]({this_version_plot_path.name})\n")
 
-        file.write(f"### 4.2 Learning Benchmarking of All Released Versions under Major v{major_v}.#.#\n")
+        file.write(f"### 4.2 Learning Benchmark of Minor and Bugfix Releases for Major Version {major_v}\n")
         file.write(
-            f"![Learning Benchmarking of All Released Versions under "
-            f"Major v{major_v}.#.#]({all_version_plot_path.name})\n"
+            f"![Learning Benchmark of Minor and Bugfix Releases for Major Version {major_v}]"
+            f"({all_version_plot_path.name})\n"
+        )
+
+        file.write(f"### 4.3 Performance of Minor and Bugfix Releases for Major Version {major_v}\n")
+        file.write(
+            f"![Performance of Minor and Bugfix Releases for Major Version {major_v}]"
+            f"({performance_benchmark_plot_path.name})\n"
         )

benchmark/results/v3/v3.0.0/PrimAITE v3.0.0 Benchmark Report.md

@@ -0,0 +1,38 @@
+# PrimAITE v3.0.0 Learning Benchmark
+## PrimAITE Dev Team
+### 2024-07-20
+
+---
+## 1 Introduction
+PrimAITE v3.0.0 was benchmarked automatically upon release. Learning rate metrics were captured to be referenced during system-level testing and user acceptance testing (UAT).
+The benchmarking process consists of running 5 training session using the same config file. Each session trains an agent for 1000 episodes, with each episode consisting of 128 steps.
+The total reward per episode from each session is captured. This is then used to calculate an caverage total reward per episode from the 5 individual sessions for smoothing. Finally, a 25-widow rolling average of the average total reward per session is calculated for further smoothing.
+## 2 System Information
+### 2.1 Python
+**Version:** 3.10.14 (main, Apr  6 2024, 18:45:05) [GCC 9.4.0]
+### 2.2 System
+- **OS:** Linux
+- **OS Version:** #76~20.04.1-Ubuntu SMP Thu Jun 13 18:00:23 UTC 2024
+- **Machine:** x86_64
+- **Processor:** x86_64
+### 2.3 CPU
+- **Physical Cores:** 2
+- **Total Cores:** 4
+- **Max Frequency:** 0.00Mhz
+### 2.4 Memory
+- **Total:** 15.62GB
+- **Swap Total:** 0.00B
+## 3 Stats
+- **Total Sessions:** 5
+- **Total Episodes:** 5005
+- **Total Steps:** 640000
+- **Av Session Duration (s):** 1452.5910
+- **Av Step Duration (s):** 0.0454
+- **Av Duration per 100 Steps per 10 Nodes (s):** 4.5393
+## 4 Graphs
+### 4.1 v3.0.0 Learning Benchmark Plot
+![PrimAITE 3.0.0 Learning Benchmark Plot](PrimAITE v3.0.0 Learning Benchmark.png)
+### 4.2 Learning Benchmark of Minor and Bugfix Releases for Major Version 3
+![Learning Benchmark of Minor and Bugfix Releases for Major Version 3](PrimAITE Learning Benchmark of Minor and Bugfix Releases for Major Version 3.png)
+### 4.3 Performance of Minor and Bugfix Releases for Major Version 3
+![Performance of Minor and Bugfix Releases for Major Version 3](PrimAITE Performance of Minor and Bugfix Releases for Major Version 3.png)

benchmark/results/v3/v3.1.0/PrimAITE v3.1.0 Benchmark Report.md

@@ -0,0 +1,38 @@
+# PrimAITE v3.1.0 Learning Benchmark
+## PrimAITE Dev Team
+### 2024-07-20
+
+---
+## 1 Introduction
+PrimAITE v3.1.0 was benchmarked automatically upon release. Learning rate metrics were captured to be referenced during system-level testing and user acceptance testing (UAT).
+The benchmarking process consists of running 5 training session using the same config file. Each session trains an agent for 1000 episodes, with each episode consisting of 128 steps.
+The total reward per episode from each session is captured. This is then used to calculate an caverage total reward per episode from the 5 individual sessions for smoothing. Finally, a 25-widow rolling average of the average total reward per session is calculated for further smoothing.
+## 2 System Information
+### 2.1 Python
+**Version:** 3.10.14 (main, Apr  6 2024, 18:45:05) [GCC 9.4.0]
+### 2.2 System
+- **OS:** Linux
+- **OS Version:** #76~20.04.1-Ubuntu SMP Thu Jun 13 18:00:23 UTC 2024
+- **Machine:** x86_64
+- **Processor:** x86_64
+### 2.3 CPU
+- **Physical Cores:** 2
+- **Total Cores:** 4
+- **Max Frequency:** 0.00Mhz
+### 2.4 Memory
+- **Total:** 15.62GB
+- **Swap Total:** 0.00B
+## 3 Stats
+- **Total Sessions:** 5
+- **Total Episodes:** 5005
+- **Total Steps:** 640000
+- **Av Session Duration (s):** 1632.8888
+- **Av Step Duration (s):** 0.0510
+- **Av Duration per 100 Steps per 10 Nodes (s):** 5.1028
+## 4 Graphs
+### 4.1 v3.1.0 Learning Benchmark Plot
+![PrimAITE 3.1.0 Learning Benchmark Plot](PrimAITE v3.1.0 Learning Benchmark.png)
+### 4.2 Learning Benchmark of Minor and Bugfix Releases for Major Version 3
+![Learning Benchmark of Minor and Bugfix Releases for Major Version 3](PrimAITE Learning Benchmark of Minor and Bugfix Releases for Major Version 3.png)
+### 4.3 Performance of Minor and Bugfix Releases for Major Version 3
+![Performance of Minor and Bugfix Releases for Major Version 3](PrimAITE Performance of Minor and Bugfix Releases for Major Version 3.png)

benchmark/results/v3/v3.2.0/PrimAITE v3.2.0 Benchmark Report.md

@@ -0,0 +1,38 @@
+# PrimAITE v3.2.0 Learning Benchmark
+## PrimAITE Dev Team
+### 2024-07-21
+
+---
+## 1 Introduction
+PrimAITE v3.2.0 was benchmarked automatically upon release. Learning rate metrics were captured to be referenced during system-level testing and user acceptance testing (UAT).
+The benchmarking process consists of running 5 training session using the same config file. Each session trains an agent for 1000 episodes, with each episode consisting of 128 steps.
+The total reward per episode from each session is captured. This is then used to calculate an caverage total reward per episode from the 5 individual sessions for smoothing. Finally, a 25-widow rolling average of the average total reward per session is calculated for further smoothing.
+## 2 System Information
+### 2.1 Python
+**Version:** 3.10.14 (main, Apr  6 2024, 18:45:05) [GCC 9.4.0]
+### 2.2 System
+- **OS:** Linux
+- **OS Version:** #76~20.04.1-Ubuntu SMP Thu Jun 13 18:00:23 UTC 2024
+- **Machine:** x86_64
+- **Processor:** x86_64
+### 2.3 CPU
+- **Physical Cores:** 2
+- **Total Cores:** 4
+- **Max Frequency:** 0.00Mhz
+### 2.4 Memory
+- **Total:** 15.62GB
+- **Swap Total:** 0.00B
+## 3 Stats
+- **Total Sessions:** 5
+- **Total Episodes:** 5005
+- **Total Steps:** 640000
+- **Av Session Duration (s):** 1691.5034
+- **Av Step Duration (s):** 0.0529
+- **Av Duration per 100 Steps per 10 Nodes (s):** 5.2859
+## 4 Graphs
+### 4.1 v3.2.0 Learning Benchmark Plot
+![PrimAITE 3.2.0 Learning Benchmark Plot](PrimAITE v3.2.0 Learning Benchmark.png)
+### 4.2 Learning Benchmark of Minor and Bugfix Releases for Major Version 3
+![Learning Benchmark of Minor and Bugfix Releases for Major Version 3](PrimAITE Learning Benchmark of Minor and Bugfix Releases for Major Version 3.png)
+### 4.3 Performance of Minor and Bugfix Releases for Major Version 3
+![Performance of Minor and Bugfix Releases for Major Version 3](PrimAITE Performance of Minor and Bugfix Releases for Major Version 3.png)