From 824729276ec60f9dda1ebc2f62f159376c9c749f Mon Sep 17 00:00:00 2001 From: Chris McCarthy Date: Tue, 25 Jun 2024 16:58:39 +0100 Subject: [PATCH 01/14] #2648 - updated benchmark process to output markdown file instead of LaTeX. Added pipeline that runs benchmarking at 2am on a weekday and automatically upon creation of release branch --- .azure/azure-benchmark-pipeline.yaml | 83 +++++++++++++++++ benchmark/primaite_benchmark.py | 4 +- benchmark/report.py | 133 +++++++++++---------------- pyproject.toml | 1 - 4 files changed, 139 insertions(+), 82 deletions(-) create mode 100644 .azure/azure-benchmark-pipeline.yaml diff --git a/.azure/azure-benchmark-pipeline.yaml b/.azure/azure-benchmark-pipeline.yaml new file mode 100644 index 00000000..ac52ce2b --- /dev/null +++ b/.azure/azure-benchmark-pipeline.yaml @@ -0,0 +1,83 @@ +trigger: + branches: + exclude: + - '*' + include: + - 'refs/heads/release/*' # Trigger on creation of release branches + +schedules: +- cron: "0 2 * * 1-5" # Run at 2 AM every weekday + displayName: "Weekday Schedule" + branches: + include: + - dev + +pool: + vmImage: ubuntu-latest + +variables: + VERSION: '' + MAJOR_VERSION: '' + +steps: +- checkout: self + persistCredentials: true + +- script: | + VERSION=$(cat src/primaite/VERSION | tr -d '\n') + MAJOR_VERSION=$(echo $VERSION | cut -d. -f1) + echo "##vso[task.setvariable variable=VERSION]$VERSION" + echo "##vso[task.setvariable variable=MAJOR_VERSION]$MAJOR_VERSION" + displayName: 'Set Version Variables' + +- script: | + if [[ "$(Build.SourceBranch)" == "refs/heads/dev" ]]; then + DATE=$(date +%Y%m%d%H%M) + echo "${VERSION}+dev.${DATE}" > src/primaite/VERSION + fi + displayName: 'Update VERSION file for Dev Benchmark' + +- task: UsePythonVersion@0 + inputs: + versionSpec: '3.11' + addToPath: true + +- script: | + python -m pip install --upgrade pip + pip install -e . + primaite setup + displayName: 'Install Dependencies' + +- script: | + mkdir -p benchmark/results/v$(MAJOR_VERSION)/v$(VERSION) + python benchmark.py --output benchmark/results/v$(MAJOR_VERSION)/v$(VERSION) + displayName: 'Run Benchmarking Script' + +- script: | + git config --global user.email "oss@dstl.gov.uk" + git config --global user.name "Defence Science and Technology Laboratory UK" + workingDirectory: $(System.DefaultWorkingDirectory) + displayName: 'Configure Git' + condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release/')) + +- script: | + git add benchmark/results/v$(MAJOR_VERSION)/v$(VERSION)/* + git commit -m "Automated benchmark output commit for version $(VERSION)" + git push origin HEAD:$(Build.SourceBranchName) + displayName: 'Commit and Push Benchmark Results' + workingDirectory: $(System.DefaultWorkingDirectory) + env: + GIT_CREDENTIALS: $(System.AccessToken) + condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/heads/release/')) + +- script: | + mkdir -p artifact_output/benchmark/results/v$(MAJOR_VERSION) + cp -r benchmark/results/v$(MAJOR_VERSION)/v$(VERSION) artifact_output/benchmark/results/v$(MAJOR_VERSION)/ + displayName: 'Prepare Artifacts for Publishing' + +- task: PublishPipelineArtifact@1 + inputs: + targetPath: 'artifact_output/benchmark/results' # Path to the files you want to publish + artifactName: 'benchmark-output' # Name of the artifact + publishLocation: 'pipeline' + displayName: 'Publish Benchmark Output as Artifact' diff --git a/benchmark/primaite_benchmark.py b/benchmark/primaite_benchmark.py index b19dbb16..f3d0a10c 100644 --- a/benchmark/primaite_benchmark.py +++ b/benchmark/primaite_benchmark.py @@ -117,14 +117,14 @@ class BenchmarkSession: def generate_learn_metadata_dict(self) -> Dict[str, Any]: """Metadata specific to the learning session.""" total_s, s_per_step, s_per_100_steps_10_nodes = self._learn_benchmark_durations() - self.gym_env.average_reward_per_episode.pop(0) # remove episode 0 + self.gym_env.total_reward_per_episode.pop(0) # remove episode 0 return { "total_episodes": self.gym_env.episode_counter, "total_time_steps": self.gym_env.total_time_steps, "total_s": total_s, "s_per_step": s_per_step, "s_per_100_steps_10_nodes": s_per_100_steps_10_nodes, - "av_reward_per_episode": self.gym_env.average_reward_per_episode, + "av_reward_per_episode": self.gym_env.total_reward_per_episode, } diff --git a/benchmark/report.py b/benchmark/report.py index ca3d03a3..6a71ef57 100644 --- a/benchmark/report.py +++ b/benchmark/report.py @@ -9,10 +9,6 @@ import plotly.graph_objects as go import polars as pl import yaml from plotly.graph_objs import Figure -from pylatex import Command, Document -from pylatex import Figure as LatexFigure -from pylatex import Section, Subsection, Tabular -from pylatex.utils import bold from utils import _get_system_info import primaite @@ -140,7 +136,7 @@ def _plot_all_benchmarks_combined_session_av(results_directory: Path) -> Figure: converted into a polars dataframe, and plotted as a scatter line in plotly. """ major_v = primaite.__version__.split(".")[0] - title = f"Learning Benchmarking of All Released Versions under Major v{major_v}.*.*" + title = f"Learning Benchmarking of All Released Versions under Major v{major_v}.#.#" subtitle = "Rolling Av (Combined Session Av)" if title: if subtitle: @@ -208,98 +204,77 @@ def build_benchmark_latex_report( fig = _plot_all_benchmarks_combined_session_av(results_directory=results_root_path) - all_version_plot_path = results_root_path / "PrimAITE Versions Learning Benchmark.png" + all_version_plot_path = version_result_dir / "PrimAITE Versions Learning Benchmark.png" fig.write_image(all_version_plot_path) - geometry_options = {"tmargin": "2.5cm", "rmargin": "2.5cm", "bmargin": "2.5cm", "lmargin": "2.5cm"} data = benchmark_metadata_dict primaite_version = data["primaite_version"] - # Create a new document - doc = Document("report", geometry_options=geometry_options) - # Title - doc.preamble.append(Command("title", f"PrimAITE {primaite_version} Learning Benchmark")) - doc.preamble.append(Command("author", "PrimAITE Dev Team")) - doc.preamble.append(Command("date", datetime.now().date())) - doc.append(Command("maketitle")) + with open(version_result_dir / f"PrimAITE v{primaite_version} Learning Benchmark.md", "w") as file: + # Title + file.write(f"# PrimAITE v{primaite_version} Learning Benchmark\n") + file.write("## PrimAITE Dev Team\n") + file.write(f"### {datetime.now().date()}\n") + file.write("\n---\n") - sessions = data["total_sessions"] - episodes = session_metadata[1]["total_episodes"] - 1 - steps = data["config"]["game"]["max_episode_length"] + sessions = data["total_sessions"] + episodes = session_metadata[1]["total_episodes"] - 1 + steps = data["config"]["game"]["max_episode_length"] - # Body - with doc.create(Section("Introduction")): - doc.append( + # Body + file.write("## 1 Introduction\n") + file.write( f"PrimAITE v{primaite_version} was benchmarked automatically upon release. Learning rate metrics " - f"were captured to be referenced during system-level testing and user acceptance testing (UAT)." + f"were captured to be referenced during system-level testing and user acceptance testing (UAT).\n" ) - doc.append( - f"\nThe benchmarking process consists of running {sessions} training session using the same " + file.write( + f"The benchmarking process consists of running {sessions} training session using the same " f"config file. Each session trains an agent for {episodes} episodes, " - f"with each episode consisting of {steps} steps." + f"with each episode consisting of {steps} steps.\n" ) - doc.append( - f"\nThe total reward per episode from each session is captured. This is then used to calculate an " + file.write( + f"The total reward per episode from each session is captured. This is then used to calculate an " f"caverage total reward per episode from the {sessions} individual sessions for smoothing. " f"Finally, a 25-widow rolling average of the average total reward per session is calculated for " - f"further smoothing." + f"further smoothing.\n" ) - with doc.create(Section("System Information")): - with doc.create(Subsection("Python")): - with doc.create(Tabular("|l|l|")) as table: - table.add_hline() - table.add_row((bold("Version"), sys.version)) - table.add_hline() + file.write("## 2 System Information\n") + i = 1 + file.write(f"### 2.{i} Python\n") + file.write(f"**Version:** {sys.version}\n") + for section, section_data in data["system_info"].items(): + i += 1 if section_data: - with doc.create(Subsection(section)): - if isinstance(section_data, dict): - with doc.create(Tabular("|l|l|")) as table: - table.add_hline() - for key, value in section_data.items(): - table.add_row((bold(key), value)) - table.add_hline() - elif isinstance(section_data, list): - headers = section_data[0].keys() - tabs_str = "|".join(["l" for _ in range(len(headers))]) - tabs_str = f"|{tabs_str}|" - with doc.create(Tabular(tabs_str)) as table: - table.add_hline() - table.add_row([bold(h) for h in headers]) - table.add_hline() - for item in section_data: - table.add_row(item.values()) - table.add_hline() + file.write(f"### 2.{i} {section}\n") + if isinstance(section_data, dict): + for key, value in section_data.items(): + file.write(f"- **{key}:** {value}\n") - headers_map = { - "total_sessions": "Total Sessions", - "total_episodes": "Total Episodes", - "total_time_steps": "Total Steps", - "av_s_per_session": "Av Session Duration (s)", - "av_s_per_step": "Av Step Duration (s)", - "av_s_per_100_steps_10_nodes": "Av Duration per 100 Steps per 10 Nodes (s)", - } - with doc.create(Section("Stats")): - with doc.create(Subsection("Benchmark Results")): - with doc.create(Tabular("|l|l|")) as table: - table.add_hline() - for section, header in headers_map.items(): - if section.startswith("av_"): - table.add_row((bold(header), f"{data[section]:.4f}")) - else: - table.add_row((bold(header), data[section])) - table.add_hline() + headers_map = { + "total_sessions": "Total Sessions", + "total_episodes": "Total Episodes", + "total_time_steps": "Total Steps", + "av_s_per_session": "Av Session Duration (s)", + "av_s_per_step": "Av Step Duration (s)", + "av_s_per_100_steps_10_nodes": "Av Duration per 100 Steps per 10 Nodes (s)", + } - with doc.create(Section("Graphs")): - with doc.create(Subsection(f"v{primaite_version} Learning Benchmark Plot")): - with doc.create(LatexFigure(position="h!")) as pic: - pic.add_image(str(this_version_plot_path)) - pic.add_caption(f"PrimAITE {primaite_version} Learning Benchmark Plot") + file.write("## 3 Stats\n") + for section, header in headers_map.items(): + if section.startswith("av_"): + file.write(f"- **{header}:** {data[section]:.4f}\n") + else: + file.write(f"- **{header}:** {data[section]}\n") - with doc.create(Subsection(f"Learning Benchmarking of All Released Versions under Major v{major_v}.*.*")): - with doc.create(LatexFigure(position="h!")) as pic: - pic.add_image(str(all_version_plot_path)) - pic.add_caption(f"Learning Benchmarking of All Released Versions under Major v{major_v}.*.*") + file.write("## 4 Graphs\n") - doc.generate_pdf(str(this_version_plot_path).replace(".png", ""), clean_tex=True) + file.write(f"### 4.1 v{primaite_version} Learning Benchmark Plot\n") + file.write(f"![PrimAITE {primaite_version} Learning Benchmark Plot]({this_version_plot_path.name})\n") + + file.write(f"### 4.2 Learning Benchmarking of All Released Versions under Major v{major_v}.#.#\n") + file.write( + f"![Learning Benchmarking of All Released Versions under " + f"Major v{major_v}.#.#]({all_version_plot_path.name})\n" + ) diff --git a/pyproject.toml b/pyproject.toml index 31ce5312..9d53e961 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,7 +64,6 @@ dev = [ "gputil==1.4.0", "pip-licenses==4.3.0", "pre-commit==2.20.0", - "pylatex==1.4.1", "pytest==7.2.0", "pytest-xdist==3.3.1", "pytest-cov==4.0.0", From 1033e696cd94e342bbc66bd4b298cc09161e75db Mon Sep 17 00:00:00 2001 From: Christopher McCarthy Date: Tue, 25 Jun 2024 16:01:18 +0000 Subject: [PATCH 02/14] Set up CI with Azure Pipelines [skip ci] --- .azure/azure-benchmark-pipeline.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.azure/azure-benchmark-pipeline.yaml b/.azure/azure-benchmark-pipeline.yaml index ac52ce2b..53df7155 100644 --- a/.azure/azure-benchmark-pipeline.yaml +++ b/.azure/azure-benchmark-pipeline.yaml @@ -10,7 +10,7 @@ schedules: displayName: "Weekday Schedule" branches: include: - - dev + - feature/2648_Automate-the-benchmarking-process pool: vmImage: ubuntu-latest @@ -31,7 +31,7 @@ steps: displayName: 'Set Version Variables' - script: | - if [[ "$(Build.SourceBranch)" == "refs/heads/dev" ]]; then + if [[ "$(Build.SourceBranch)" == "refs/heads/feature/2648_Automate-the-benchmarking-process" ]]; then DATE=$(date +%Y%m%d%H%M) echo "${VERSION}+dev.${DATE}" > src/primaite/VERSION fi From 55d69d6568bfd97950989df56db011cba33dc05c Mon Sep 17 00:00:00 2001 From: Chris McCarthy Date: Tue, 25 Jun 2024 17:23:04 +0100 Subject: [PATCH 03/14] #2648 - updated benchmark run command --- .azure/azure-benchmark-pipeline.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.azure/azure-benchmark-pipeline.yaml b/.azure/azure-benchmark-pipeline.yaml index ac52ce2b..43140d7d 100644 --- a/.azure/azure-benchmark-pipeline.yaml +++ b/.azure/azure-benchmark-pipeline.yaml @@ -49,8 +49,9 @@ steps: displayName: 'Install Dependencies' - script: | - mkdir -p benchmark/results/v$(MAJOR_VERSION)/v$(VERSION) - python benchmark.py --output benchmark/results/v$(MAJOR_VERSION)/v$(VERSION) + cd benchmark + python3 primaite_benchmark.py + cd .. displayName: 'Run Benchmarking Script' - script: | From 4249314672062c6edc8854e2ec9980075a3f6091 Mon Sep 17 00:00:00 2001 From: Chris McCarthy Date: Tue, 25 Jun 2024 17:28:05 +0100 Subject: [PATCH 04/14] #2648 - reduced the benchmark sessions and episodes for fail-fast speed while testing --- benchmark/primaite_benchmark.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmark/primaite_benchmark.py b/benchmark/primaite_benchmark.py index f3d0a10c..92c9bf0a 100644 --- a/benchmark/primaite_benchmark.py +++ b/benchmark/primaite_benchmark.py @@ -151,8 +151,8 @@ def _prepare_session_directory(): def run( - number_of_sessions: int = 5, - num_episodes: int = 1000, + number_of_sessions: int = 1, + num_episodes: int = 25, episode_len: int = 128, n_steps: int = 1280, batch_size: int = 32, From 0ee243a242b47dc109905b2a089d42406dea89c7 Mon Sep 17 00:00:00 2001 From: Chris McCarthy Date: Wed, 26 Jun 2024 09:09:16 +0100 Subject: [PATCH 05/14] #2648 - trying to fix the artifacts publish stage. currently creating tar.gz and publishing that --- .azure/azure-benchmark-pipeline.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.azure/azure-benchmark-pipeline.yaml b/.azure/azure-benchmark-pipeline.yaml index 133de713..f0c7c063 100644 --- a/.azure/azure-benchmark-pipeline.yaml +++ b/.azure/azure-benchmark-pipeline.yaml @@ -72,13 +72,13 @@ steps: condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/heads/release/')) - script: | - mkdir -p artifact_output/benchmark/results/v$(MAJOR_VERSION) - cp -r benchmark/results/v$(MAJOR_VERSION)/v$(VERSION) artifact_output/benchmark/results/v$(MAJOR_VERSION)/ + ls benchmark/results/v$(MAJOR_VERSION)/v$(VERSION) + tar czf benchmark.tar.gz benchmark/results/v$(MAJOR_VERSION)/v$(VERSION) displayName: 'Prepare Artifacts for Publishing' - task: PublishPipelineArtifact@1 inputs: - targetPath: 'artifact_output/benchmark/results' # Path to the files you want to publish + targetPath: benchmark.tar.gz # Path to the files you want to publish artifactName: 'benchmark-output' # Name of the artifact publishLocation: 'pipeline' displayName: 'Publish Benchmark Output as Artifact' From ad2b132a10b2b701e3337a7a11ad8539c5b8cbae Mon Sep 17 00:00:00 2001 From: Chris McCarthy Date: Wed, 26 Jun 2024 09:13:55 +0100 Subject: [PATCH 06/14] #2648 - added dev and rl extras to the pip install step --- .azure/azure-benchmark-pipeline.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure/azure-benchmark-pipeline.yaml b/.azure/azure-benchmark-pipeline.yaml index f0c7c063..c16a21cf 100644 --- a/.azure/azure-benchmark-pipeline.yaml +++ b/.azure/azure-benchmark-pipeline.yaml @@ -44,7 +44,7 @@ steps: - script: | python -m pip install --upgrade pip - pip install -e . + pip install -e .[dev,rl] primaite setup displayName: 'Install Dependencies' From 2192516c9aa360bec42c0ebda21dbae8a71d89a1 Mon Sep 17 00:00:00 2001 From: Chris McCarthy Date: Wed, 26 Jun 2024 09:40:16 +0100 Subject: [PATCH 07/14] #2648 - reordered steps so that dev version is set first before version variables are set --- .azure/azure-benchmark-pipeline.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.azure/azure-benchmark-pipeline.yaml b/.azure/azure-benchmark-pipeline.yaml index c16a21cf..1ad28197 100644 --- a/.azure/azure-benchmark-pipeline.yaml +++ b/.azure/azure-benchmark-pipeline.yaml @@ -23,13 +23,6 @@ steps: - checkout: self persistCredentials: true -- script: | - VERSION=$(cat src/primaite/VERSION | tr -d '\n') - MAJOR_VERSION=$(echo $VERSION | cut -d. -f1) - echo "##vso[task.setvariable variable=VERSION]$VERSION" - echo "##vso[task.setvariable variable=MAJOR_VERSION]$MAJOR_VERSION" - displayName: 'Set Version Variables' - - script: | if [[ "$(Build.SourceBranch)" == "refs/heads/feature/2648_Automate-the-benchmarking-process" ]]; then DATE=$(date +%Y%m%d%H%M) @@ -37,6 +30,13 @@ steps: fi displayName: 'Update VERSION file for Dev Benchmark' +- script: | + VERSION=$(cat src/primaite/VERSION | tr -d '\n') + MAJOR_VERSION=$(echo $VERSION | cut -d. -f1) + echo "##vso[task.setvariable variable=VERSION]$VERSION" + echo "##vso[task.setvariable variable=MAJOR_VERSION]$MAJOR_VERSION" + displayName: 'Set Version Variables' + - task: UsePythonVersion@0 inputs: versionSpec: '3.11' From 112f116a89a208b88502ef63302d828cdede33b5 Mon Sep 17 00:00:00 2001 From: Chris McCarthy Date: Wed, 26 Jun 2024 09:46:05 +0100 Subject: [PATCH 08/14] #2648 - fixed error whereby VERSION variable isn't set before setting the dev build version in the VERSION file --- .azure/azure-benchmark-pipeline.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.azure/azure-benchmark-pipeline.yaml b/.azure/azure-benchmark-pipeline.yaml index 1ad28197..06ef72a4 100644 --- a/.azure/azure-benchmark-pipeline.yaml +++ b/.azure/azure-benchmark-pipeline.yaml @@ -24,6 +24,7 @@ steps: persistCredentials: true - script: | + VERSION=$(cat src/primaite/VERSION | tr -d '\n') if [[ "$(Build.SourceBranch)" == "refs/heads/feature/2648_Automate-the-benchmarking-process" ]]; then DATE=$(date +%Y%m%d%H%M) echo "${VERSION}+dev.${DATE}" > src/primaite/VERSION From 08ca4d8889fc386b49863ebc6a788ef9c5acbc62 Mon Sep 17 00:00:00 2001 From: Chris McCarthy Date: Wed, 26 Jun 2024 10:50:05 +0100 Subject: [PATCH 09/14] #2648 - now testing the release benchmark auto commit and push --- .azure/azure-benchmark-pipeline.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.azure/azure-benchmark-pipeline.yaml b/.azure/azure-benchmark-pipeline.yaml index 06ef72a4..5d84fc8c 100644 --- a/.azure/azure-benchmark-pipeline.yaml +++ b/.azure/azure-benchmark-pipeline.yaml @@ -3,7 +3,7 @@ trigger: exclude: - '*' include: - - 'refs/heads/release/*' # Trigger on creation of release branches + - feature/2648_Automate-the-benchmarking-process # Trigger on creation of release branches schedules: - cron: "0 2 * * 1-5" # Run at 2 AM every weekday @@ -60,7 +60,7 @@ steps: git config --global user.name "Defence Science and Technology Laboratory UK" workingDirectory: $(System.DefaultWorkingDirectory) displayName: 'Configure Git' - condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release/')) + condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual'), startsWith(variables['Build.SourceBranch'], 'refs/heads/feature/2648_Automate-the-benchmarking-process')) - script: | git add benchmark/results/v$(MAJOR_VERSION)/v$(VERSION)/* @@ -70,7 +70,7 @@ steps: workingDirectory: $(System.DefaultWorkingDirectory) env: GIT_CREDENTIALS: $(System.AccessToken) - condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/heads/release/')) + condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/heads/feature/2648_Automate-the-benchmarking-process')) - script: | ls benchmark/results/v$(MAJOR_VERSION)/v$(VERSION) From 3b118fa0add4f9696b531f84e29aab4d96b781c5 Mon Sep 17 00:00:00 2001 From: Chris McCarthy Date: Wed, 26 Jun 2024 10:52:23 +0100 Subject: [PATCH 10/14] #2648 - now testing the release benchmark auto commit and push --- .azure/azure-benchmark-pipeline.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure/azure-benchmark-pipeline.yaml b/.azure/azure-benchmark-pipeline.yaml index 5d84fc8c..a6118570 100644 --- a/.azure/azure-benchmark-pipeline.yaml +++ b/.azure/azure-benchmark-pipeline.yaml @@ -3,7 +3,7 @@ trigger: exclude: - '*' include: - - feature/2648_Automate-the-benchmarking-process # Trigger on creation of release branches + - 'refs/heads/feature/2648_Automate-the-benchmarking-process' schedules: - cron: "0 2 * * 1-5" # Run at 2 AM every weekday From 20c9719f1e70849b2823142b4d1720007ae7de55 Mon Sep 17 00:00:00 2001 From: Chris McCarthy Date: Wed, 26 Jun 2024 11:32:17 +0100 Subject: [PATCH 11/14] #2648 - added full branch name reference for push --- .azure/azure-benchmark-pipeline.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure/azure-benchmark-pipeline.yaml b/.azure/azure-benchmark-pipeline.yaml index a6118570..bc123454 100644 --- a/.azure/azure-benchmark-pipeline.yaml +++ b/.azure/azure-benchmark-pipeline.yaml @@ -65,7 +65,7 @@ steps: - script: | git add benchmark/results/v$(MAJOR_VERSION)/v$(VERSION)/* git commit -m "Automated benchmark output commit for version $(VERSION)" - git push origin HEAD:$(Build.SourceBranchName) + git push origin HEAD:refs/heads/$(Build.SourceBranchName) displayName: 'Commit and Push Benchmark Results' workingDirectory: $(System.DefaultWorkingDirectory) env: From 795b5a80fbf69add363f60cedc2928c1f6fb2e45 Mon Sep 17 00:00:00 2001 From: Chris McCarthy Date: Wed, 26 Jun 2024 12:00:56 +0100 Subject: [PATCH 12/14] #2648 - testing the new benchmark artifact name --- .azure/azure-benchmark-pipeline.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.azure/azure-benchmark-pipeline.yaml b/.azure/azure-benchmark-pipeline.yaml index bc123454..cd8703ec 100644 --- a/.azure/azure-benchmark-pipeline.yaml +++ b/.azure/azure-benchmark-pipeline.yaml @@ -74,12 +74,12 @@ steps: - script: | ls benchmark/results/v$(MAJOR_VERSION)/v$(VERSION) - tar czf benchmark.tar.gz benchmark/results/v$(MAJOR_VERSION)/v$(VERSION) + tar czf primaite_v$(VERSION)_benchmark.tar.gz benchmark/results/v$(MAJOR_VERSION)/v$(VERSION) displayName: 'Prepare Artifacts for Publishing' - task: PublishPipelineArtifact@1 inputs: - targetPath: benchmark.tar.gz # Path to the files you want to publish - artifactName: 'benchmark-output' # Name of the artifact + targetPath: primaite_v$(VERSION)_benchmark.tar.gz + artifactName: 'benchmark-output' publishLocation: 'pipeline' displayName: 'Publish Benchmark Output as Artifact' From e7f979b78e566139be8ccd50f89b8f27be0f9c04 Mon Sep 17 00:00:00 2001 From: Chris McCarthy Date: Wed, 26 Jun 2024 12:52:29 +0100 Subject: [PATCH 13/14] #2648 - reverted temp changes to benchmark durations and branch names for testing purposes --- .azure/azure-benchmark-pipeline.yaml | 13 ++++++------- benchmark/primaite_benchmark.py | 4 ++-- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/.azure/azure-benchmark-pipeline.yaml b/.azure/azure-benchmark-pipeline.yaml index cd8703ec..1f7b8ebe 100644 --- a/.azure/azure-benchmark-pipeline.yaml +++ b/.azure/azure-benchmark-pipeline.yaml @@ -3,14 +3,14 @@ trigger: exclude: - '*' include: - - 'refs/heads/feature/2648_Automate-the-benchmarking-process' + - 'refs/heads/release/*' schedules: - cron: "0 2 * * 1-5" # Run at 2 AM every weekday displayName: "Weekday Schedule" branches: include: - - feature/2648_Automate-the-benchmarking-process + - 'refs/heads/dev' pool: vmImage: ubuntu-latest @@ -25,8 +25,8 @@ steps: - script: | VERSION=$(cat src/primaite/VERSION | tr -d '\n') - if [[ "$(Build.SourceBranch)" == "refs/heads/feature/2648_Automate-the-benchmarking-process" ]]; then - DATE=$(date +%Y%m%d%H%M) + if [[ "$(Build.SourceBranch)" == "refs/heads/dev" ]]; then + DATE=$(date +%Y%m%d) echo "${VERSION}+dev.${DATE}" > src/primaite/VERSION fi displayName: 'Update VERSION file for Dev Benchmark' @@ -60,7 +60,7 @@ steps: git config --global user.name "Defence Science and Technology Laboratory UK" workingDirectory: $(System.DefaultWorkingDirectory) displayName: 'Configure Git' - condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual'), startsWith(variables['Build.SourceBranch'], 'refs/heads/feature/2648_Automate-the-benchmarking-process')) + condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release')) - script: | git add benchmark/results/v$(MAJOR_VERSION)/v$(VERSION)/* @@ -70,10 +70,9 @@ steps: workingDirectory: $(System.DefaultWorkingDirectory) env: GIT_CREDENTIALS: $(System.AccessToken) - condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/heads/feature/2648_Automate-the-benchmarking-process')) + condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/heads/release')) - script: | - ls benchmark/results/v$(MAJOR_VERSION)/v$(VERSION) tar czf primaite_v$(VERSION)_benchmark.tar.gz benchmark/results/v$(MAJOR_VERSION)/v$(VERSION) displayName: 'Prepare Artifacts for Publishing' diff --git a/benchmark/primaite_benchmark.py b/benchmark/primaite_benchmark.py index 92c9bf0a..f3d0a10c 100644 --- a/benchmark/primaite_benchmark.py +++ b/benchmark/primaite_benchmark.py @@ -151,8 +151,8 @@ def _prepare_session_directory(): def run( - number_of_sessions: int = 1, - num_episodes: int = 25, + number_of_sessions: int = 5, + num_episodes: int = 1000, episode_len: int = 128, n_steps: int = 1280, batch_size: int = 32, From 7f0a0c562fd89858fc1d39ef96fcef04da775063 Mon Sep 17 00:00:00 2001 From: Chris McCarthy Date: Wed, 26 Jun 2024 13:26:18 +0100 Subject: [PATCH 14/14] #2648 - changed av_reward_per_episode to total_reward_per_episode in primaite_benchmark.py and report.py --- benchmark/primaite_benchmark.py | 2 +- benchmark/report.py | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/benchmark/primaite_benchmark.py b/benchmark/primaite_benchmark.py index f3d0a10c..27e25a0c 100644 --- a/benchmark/primaite_benchmark.py +++ b/benchmark/primaite_benchmark.py @@ -124,7 +124,7 @@ class BenchmarkSession: "total_s": total_s, "s_per_step": s_per_step, "s_per_100_steps_10_nodes": s_per_100_steps_10_nodes, - "av_reward_per_episode": self.gym_env.total_reward_per_episode, + "total_reward_per_episode": self.gym_env.total_reward_per_episode, } diff --git a/benchmark/report.py b/benchmark/report.py index 6a71ef57..dc8e51e4 100644 --- a/benchmark/report.py +++ b/benchmark/report.py @@ -35,19 +35,19 @@ def _build_benchmark_results_dict(start_datetime: datetime, metadata_dict: Dict, "av_s_per_step": sum(d["s_per_step"] for d in metadata_dict.values()) / num_sessions, "av_s_per_100_steps_10_nodes": sum(d["s_per_100_steps_10_nodes"] for d in metadata_dict.values()) / num_sessions, - "combined_av_reward_per_episode": {}, - "session_av_reward_per_episode": {k: v["av_reward_per_episode"] for k, v in metadata_dict.items()}, + "combined_total_reward_per_episode": {}, + "session_total_reward_per_episode": {k: v["total_reward_per_episode"] for k, v in metadata_dict.items()}, "config": config, } # find the average of each episode across all sessions - episodes = metadata_dict[1]["av_reward_per_episode"].keys() + episodes = metadata_dict[1]["total_reward_per_episode"].keys() for episode in episodes: combined_av_reward = ( - sum(metadata_dict[k]["av_reward_per_episode"][episode] for k in metadata_dict.keys()) / num_sessions + sum(metadata_dict[k]["total_reward_per_episode"][episode] for k in metadata_dict.keys()) / num_sessions ) - averaged_data["combined_av_reward_per_episode"][episode] = combined_av_reward + averaged_data["combined_total_reward_per_episode"][episode] = combined_av_reward return averaged_data @@ -83,7 +83,7 @@ def _plot_benchmark_metadata( fig = go.Figure(layout=layout) fig.update_layout(template=PLOT_CONFIG["template"]) - for session, av_reward_dict in benchmark_metadata_dict["session_av_reward_per_episode"].items(): + for session, av_reward_dict in benchmark_metadata_dict["session_total_reward_per_episode"].items(): df = _get_df_from_episode_av_reward_dict(av_reward_dict) fig.add_trace( go.Scatter( @@ -96,7 +96,7 @@ def _plot_benchmark_metadata( ) ) - df = _get_df_from_episode_av_reward_dict(benchmark_metadata_dict["combined_av_reward_per_episode"]) + df = _get_df_from_episode_av_reward_dict(benchmark_metadata_dict["combined_total_reward_per_episode"]) fig.add_trace( go.Scatter( x=df["episode"], y=df["av_reward"], mode="lines", name="Combined Session Av", line={"color": "#FF0000"} @@ -132,7 +132,7 @@ def _plot_all_benchmarks_combined_session_av(results_directory: Path) -> Figure: Does this by iterating over the ``benchmark/results`` directory and extracting the benchmark metadata json for each version that has been - benchmarked. The combined_av_reward_per_episode is extracted from each, + benchmarked. The combined_total_reward_per_episode is extracted from each, converted into a polars dataframe, and plotted as a scatter line in plotly. """ major_v = primaite.__version__.split(".")[0] @@ -158,7 +158,7 @@ def _plot_all_benchmarks_combined_session_av(results_directory: Path) -> Figure: metadata_file = dir / f"{dir.name}_benchmark_metadata.json" with open(metadata_file, "r") as file: metadata_dict = json.load(file) - df = _get_df_from_episode_av_reward_dict(metadata_dict["combined_av_reward_per_episode"]) + df = _get_df_from_episode_av_reward_dict(metadata_dict["combined_total_reward_per_episode"]) fig.add_trace(go.Scatter(x=df["episode"], y=df["rolling_av_reward"], mode="lines", name=dir.name))