diff --git a/benchmark/primaite_benchmark.py b/benchmark/primaite_benchmark.py index 0e6c2acc..86ed22a9 100644 --- a/benchmark/primaite_benchmark.py +++ b/benchmark/primaite_benchmark.py @@ -5,7 +5,7 @@ from datetime import datetime from pathlib import Path from typing import Any, Dict, Final, Tuple -from report import build_benchmark_md_report +from report import build_benchmark_md_report, md2pdf from stable_baselines3 import PPO import primaite @@ -159,6 +159,13 @@ def run( learning_rate: float = 3e-4, ) -> None: """Run the PrimAITE benchmark.""" + # generate report folder + v_str = f"v{primaite.__version__}" + + version_result_dir = _RESULTS_ROOT / v_str + version_result_dir.mkdir(exist_ok=True, parents=True) + output_path = version_result_dir / f"PrimAITE {v_str} Benchmark Report.md" + benchmark_start_time = datetime.now() session_metadata_dict = {} @@ -193,6 +200,12 @@ def run( session_metadata=session_metadata_dict, config_path=data_manipulation_config_path(), results_root_path=_RESULTS_ROOT, + output_path=output_path, + ) + md2pdf( + md_path=output_path, + pdf_path=str(output_path).replace(".md", ".pdf"), + css_path="static/styles.css", ) diff --git a/benchmark/report.py b/benchmark/report.py index e1ff46b9..4035ceca 100644 --- a/benchmark/report.py +++ b/benchmark/report.py @@ -2,6 +2,7 @@ import json import sys from datetime import datetime +from os import PathLike from pathlib import Path from typing import Dict, Optional @@ -14,7 +15,7 @@ from utils import _get_system_info import primaite PLOT_CONFIG = { - "size": {"auto_size": False, "width": 1500, "height": 900}, + "size": {"auto_size": False, "width": 800, "height": 640}, "template": "plotly_white", "range_slider": False, } @@ -144,6 +145,20 @@ def _plot_benchmark_metadata( yaxis={"title": "Total Reward"}, title=title, ) + fig.update_layout( + legend=dict( + yanchor="top", + y=0.99, + xanchor="left", + x=0.01, + bgcolor="rgba(255,255,255,0.3)", + ) + ) + for trace in fig["data"]: + if trace["name"].startswith("Session"): + trace["showlegend"] = False + fig["data"][0]["name"] = "Individual Sessions" + fig["data"][0]["showlegend"] = True return fig @@ -194,6 +209,7 @@ def _plot_all_benchmarks_combined_session_av(results_directory: Path) -> Figure: title=title, ) fig["data"][0]["showlegend"] = True + fig.update_layout(legend=dict(yanchor="top", y=-0.2, xanchor="left", x=0.01, orientation="h")) return fig @@ -248,14 +264,7 @@ def _plot_av_s_per_100_steps_10_nodes( versions = sorted(list(version_times_dict.keys())) times = [version_times_dict[version] for version in versions] - fig.add_trace( - go.Bar( - x=versions, - y=times, - text=times, - textposition="auto", - ) - ) + fig.add_trace(go.Bar(x=versions, y=times, text=times, textposition="auto", texttemplate="%{y:.3f}")) fig.update_layout( xaxis_title="PrimAITE Version", @@ -267,7 +276,11 @@ def _plot_av_s_per_100_steps_10_nodes( def build_benchmark_md_report( - benchmark_start_time: datetime, session_metadata: Dict, config_path: Path, results_root_path: Path + benchmark_start_time: datetime, + session_metadata: Dict, + config_path: Path, + results_root_path: Path, + output_path: PathLike, ) -> None: """ Generates a Markdown report for a benchmarking session, documenting performance metrics and graphs. @@ -319,7 +332,7 @@ def build_benchmark_md_report( data = benchmark_metadata_dict primaite_version = data["primaite_version"] - with open(version_result_dir / f"PrimAITE v{primaite_version} Benchmark Report.md", "w") as file: + with open(output_path, "w") as file: # Title file.write(f"# PrimAITE v{primaite_version} Learning Benchmark\n") file.write("## PrimAITE Dev Team\n") @@ -393,3 +406,15 @@ def build_benchmark_md_report( f"![Performance of Minor and Bugfix Releases for Major Version {major_v}]" f"({performance_benchmark_plot_path.name})\n" ) + + +def md2pdf(md_path: PathLike, pdf_path: PathLike, css_path: PathLike) -> None: + """Generate PDF version of Markdown report.""" + from md2pdf.core import md2pdf + + md2pdf( + pdf_file_path=pdf_path, + md_file_path=md_path, + base_url=Path(md_path).parent, + css_file_path=css_path, + ) diff --git a/benchmark/static/styles.css b/benchmark/static/styles.css new file mode 100644 index 00000000..4fbb9bd5 --- /dev/null +++ b/benchmark/static/styles.css @@ -0,0 +1,34 @@ +body { + font-family: 'Arial', sans-serif; + line-height: 1.6; + /* margin: 1cm; */ +} +h1, h2, h3, h4, h5, h6 { + font-weight: bold; + /* margin: 1em 0; */ +} +p { + /* margin: 0.5em 0; */ +} +ul, ol { + margin: 1em 0; + padding-left: 1.5em; +} +pre { + background: #f4f4f4; + padding: 0.5em; + overflow-x: auto; +} +img { + max-width: 100%; + height: auto; +} +table { + width: 100%; + border-collapse: collapse; + margin: 1em 0; +} +th, td { + padding: 0.5em; + border: 1px solid #ddd; +} diff --git a/pyproject.toml b/pyproject.toml index c9b7c062..354df8b2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -75,7 +75,8 @@ dev = [ "wheel==0.38.4", "nbsphinx==0.9.4", "nbmake==1.5.4", - "pytest-xdist==3.3.1" + "pytest-xdist==3.3.1", + "md2pdf", ] [project.scripts]