Merged PR 502: Add PDF benchmark output

## Summary
Uses md2pdf (MIT License) to convert the markdown file to html then to PDF.
I updated the formatting of the charts to be more legible too.
This requires WeasyPrint which is a system package available on linux, mac and windows.

## Test process
installed weasyprint on our build machine and tried the benchmark script and it worked.

https://dev.azure.com/ma-dev-uk/PrimAITE/_git/PrimAITE?version=GBrelease/fake-release-for-testing-purposes&path=/benchmark/results/v3/v3.3.0-dev0/PrimAITE%20v3.3.0-dev0%20Benchmark%20Report.pdf

## Checklist
- [X] PR is linked to a **work item**
- [X] **acceptance criteria** of linked ticket are met
- [X] performed **self-review** of the code
- [ ] written **tests** for any new functionality added with this PR
- [ ] updated the **documentation** if this PR changes or adds functionality
- [ ] written/updated **design docs** if this PR implements new functionality
- [ ] updated the **change log**
- [X] ran **pre-commit** checks for code style
- [X] attended to any **TO-DOs** left in the code

Related work items: #2772

benchmark/primaite_benchmark.py

@@ -5,7 +5,7 @@ from datetime import datetime
 from pathlib import Path
 from typing import Any, Dict, Final, Tuple
 
-from report import build_benchmark_md_report
+from report import build_benchmark_md_report, md2pdf
 from stable_baselines3 import PPO
 
 import primaite
@@ -159,6 +159,13 @@ def run(
     learning_rate: float = 3e-4,
 ) -> None:
     """Run the PrimAITE benchmark."""
+    # generate report folder
+    v_str = f"v{primaite.__version__}"
+
+    version_result_dir = _RESULTS_ROOT / v_str
+    version_result_dir.mkdir(exist_ok=True, parents=True)
+    output_path = version_result_dir / f"PrimAITE {v_str} Benchmark Report.md"
+
     benchmark_start_time = datetime.now()
 
     session_metadata_dict = {}
@@ -193,6 +200,12 @@ def run(
         session_metadata=session_metadata_dict,
         config_path=data_manipulation_config_path(),
         results_root_path=_RESULTS_ROOT,
+        output_path=output_path,
+    )
+    md2pdf(
+        md_path=output_path,
+        pdf_path=str(output_path).replace(".md", ".pdf"),
+        css_path="static/styles.css",
     )
 
 

benchmark/report.py

@@ -2,6 +2,7 @@
 import json
 import sys
 from datetime import datetime
+from os import PathLike
 from pathlib import Path
 from typing import Dict, Optional
 
@@ -14,7 +15,7 @@ from utils import _get_system_info
 import primaite
 
 PLOT_CONFIG = {
-    "size": {"auto_size": False, "width": 1500, "height": 900},
+    "size": {"auto_size": False, "width": 800, "height": 640},
     "template": "plotly_white",
     "range_slider": False,
 }
@@ -144,6 +145,20 @@ def _plot_benchmark_metadata(
         yaxis={"title": "Total Reward"},
         title=title,
     )
+    fig.update_layout(
+        legend=dict(
+            yanchor="top",
+            y=0.99,
+            xanchor="left",
+            x=0.01,
+            bgcolor="rgba(255,255,255,0.3)",
+        )
+    )
+    for trace in fig["data"]:
+        if trace["name"].startswith("Session"):
+            trace["showlegend"] = False
+    fig["data"][0]["name"] = "Individual Sessions"
+    fig["data"][0]["showlegend"] = True
 
     return fig
 
@@ -194,6 +209,7 @@ def _plot_all_benchmarks_combined_session_av(results_directory: Path) -> Figure:
         title=title,
     )
     fig["data"][0]["showlegend"] = True
+    fig.update_layout(legend=dict(yanchor="top", y=-0.2, xanchor="left", x=0.01, orientation="h"))
 
     return fig
 
@@ -248,14 +264,7 @@ def _plot_av_s_per_100_steps_10_nodes(
     versions = sorted(list(version_times_dict.keys()))
     times = [version_times_dict[version] for version in versions]
 
-    fig.add_trace(
-        go.Bar(
-            x=versions,
-            y=times,
-            text=times,
-            textposition="auto",
-        )
-    )
+    fig.add_trace(go.Bar(x=versions, y=times, text=times, textposition="auto", texttemplate="%{y:.3f}"))
 
     fig.update_layout(
         xaxis_title="PrimAITE Version",
@@ -267,7 +276,11 @@ def _plot_av_s_per_100_steps_10_nodes(
 
 
 def build_benchmark_md_report(
-    benchmark_start_time: datetime, session_metadata: Dict, config_path: Path, results_root_path: Path
+    benchmark_start_time: datetime,
+    session_metadata: Dict,
+    config_path: Path,
+    results_root_path: Path,
+    output_path: PathLike,
 ) -> None:
     """
     Generates a Markdown report for a benchmarking session, documenting performance metrics and graphs.
@@ -319,7 +332,7 @@ def build_benchmark_md_report(
     data = benchmark_metadata_dict
     primaite_version = data["primaite_version"]
 
-    with open(version_result_dir / f"PrimAITE v{primaite_version} Benchmark Report.md", "w") as file:
+    with open(output_path, "w") as file:
         # Title
         file.write(f"# PrimAITE v{primaite_version} Learning Benchmark\n")
         file.write("## PrimAITE Dev Team\n")
@@ -393,3 +406,15 @@ def build_benchmark_md_report(
             f"![Performance of Minor and Bugfix Releases for Major Version {major_v}]"
             f"({performance_benchmark_plot_path.name})\n"
         )
+
+
+def md2pdf(md_path: PathLike, pdf_path: PathLike, css_path: PathLike) -> None:
+    """Generate PDF version of Markdown report."""
+    from md2pdf.core import md2pdf
+
+    md2pdf(
+        pdf_file_path=pdf_path,
+        md_file_path=md_path,
+        base_url=Path(md_path).parent,
+        css_file_path=css_path,
+    )

benchmark/static/styles.css

@@ -0,0 +1,34 @@
+body {
+    font-family: 'Arial', sans-serif;
+    line-height: 1.6;
+    /* margin: 1cm; */
+}
+h1, h2, h3, h4, h5, h6 {
+    font-weight: bold;
+    /* margin: 1em 0; */
+}
+p {
+    /* margin: 0.5em 0; */
+}
+ul, ol {
+    margin: 1em 0;
+    padding-left: 1.5em;
+}
+pre {
+    background: #f4f4f4;
+    padding: 0.5em;
+    overflow-x: auto;
+}
+img {
+    max-width: 100%;
+    height: auto;
+}
+table {
+    width: 100%;
+    border-collapse: collapse;
+    margin: 1em 0;
+}
+th, td {
+    padding: 0.5em;
+    border: 1px solid #ddd;
+}

pyproject.toml

@@ -75,7 +75,8 @@ dev = [
     "wheel==0.38.4",
     "nbsphinx==0.9.4",
     "nbmake==1.5.4",
-    "pytest-xdist==3.3.1"
+    "pytest-xdist==3.3.1",
+    "md2pdf",
 ]
 
 [project.scripts]