From 7173c329b09c8ccf95f2d9fe2aedc2ba72bdedb4 Mon Sep 17 00:00:00 2001 From: Chris McCarthy Date: Fri, 5 Jul 2024 16:56:07 +0100 Subject: [PATCH 1/6] #2739 - Updated azure-benchmark-pipeline.yaml to allow it to run for unlimited time on the Imaginary Yak Pool --- .azure/azure-benchmark-pipeline.yaml | 119 ++++++++++++++------------- 1 file changed, 61 insertions(+), 58 deletions(-) diff --git a/.azure/azure-benchmark-pipeline.yaml b/.azure/azure-benchmark-pipeline.yaml index 1f7b8ebe..e764c3c1 100644 --- a/.azure/azure-benchmark-pipeline.yaml +++ b/.azure/azure-benchmark-pipeline.yaml @@ -11,74 +11,77 @@ schedules: branches: include: - 'refs/heads/dev' - -pool: - vmImage: ubuntu-latest - variables: VERSION: '' MAJOR_VERSION: '' -steps: -- checkout: self - persistCredentials: true +jobs: +- job: PrimAITE Benchmark + timeoutInMinutes: 0 # Set to unlimited timeout + pool: + name: 'Imaginary Yak Pool' + workspace: + clean: all + steps: + - checkout: self + persistCredentials: true -- script: | - VERSION=$(cat src/primaite/VERSION | tr -d '\n') - if [[ "$(Build.SourceBranch)" == "refs/heads/dev" ]]; then - DATE=$(date +%Y%m%d) - echo "${VERSION}+dev.${DATE}" > src/primaite/VERSION - fi - displayName: 'Update VERSION file for Dev Benchmark' + - script: | + VERSION=$(cat src/primaite/VERSION | tr -d '\n') + if [[ "$(Build.SourceBranch)" == "refs/heads/dev" ]]; then + DATE=$(date +%Y%m%d) + echo "${VERSION}+dev.${DATE}" > src/primaite/VERSION + fi + displayName: 'Update VERSION file for Dev Benchmark' -- script: | - VERSION=$(cat src/primaite/VERSION | tr -d '\n') - MAJOR_VERSION=$(echo $VERSION | cut -d. -f1) - echo "##vso[task.setvariable variable=VERSION]$VERSION" - echo "##vso[task.setvariable variable=MAJOR_VERSION]$MAJOR_VERSION" - displayName: 'Set Version Variables' + - script: | + VERSION=$(cat src/primaite/VERSION | tr -d '\n') + MAJOR_VERSION=$(echo $VERSION | cut -d. -f1) + echo "##vso[task.setvariable variable=VERSION]$VERSION" + echo "##vso[task.setvariable variable=MAJOR_VERSION]$MAJOR_VERSION" + displayName: 'Set Version Variables' -- task: UsePythonVersion@0 - inputs: - versionSpec: '3.11' - addToPath: true + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.11' + addToPath: true -- script: | - python -m pip install --upgrade pip - pip install -e .[dev,rl] - primaite setup - displayName: 'Install Dependencies' + - script: | + python -m pip install --upgrade pip + pip install -e .[dev,rl] + primaite setup + displayName: 'Install Dependencies' -- script: | - cd benchmark - python3 primaite_benchmark.py - cd .. - displayName: 'Run Benchmarking Script' + - script: | + cd benchmark + python3 primaite_benchmark.py + cd .. + displayName: 'Run Benchmarking Script' -- script: | - git config --global user.email "oss@dstl.gov.uk" - git config --global user.name "Defence Science and Technology Laboratory UK" - workingDirectory: $(System.DefaultWorkingDirectory) - displayName: 'Configure Git' - condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release')) + - script: | + git config --global user.email "oss@dstl.gov.uk" + git config --global user.name "Defence Science and Technology Laboratory UK" + workingDirectory: $(System.DefaultWorkingDirectory) + displayName: 'Configure Git' + condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release')) -- script: | - git add benchmark/results/v$(MAJOR_VERSION)/v$(VERSION)/* - git commit -m "Automated benchmark output commit for version $(VERSION)" - git push origin HEAD:refs/heads/$(Build.SourceBranchName) - displayName: 'Commit and Push Benchmark Results' - workingDirectory: $(System.DefaultWorkingDirectory) - env: - GIT_CREDENTIALS: $(System.AccessToken) - condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/heads/release')) + - script: | + git add benchmark/results/v$(MAJOR_VERSION)/v$(VERSION)/* + git commit -m "Automated benchmark output commit for version $(VERSION)" + git push origin HEAD:refs/heads/$(Build.SourceBranchName) + displayName: 'Commit and Push Benchmark Results' + workingDirectory: $(System.DefaultWorkingDirectory) + env: + GIT_CREDENTIALS: $(System.AccessToken) + condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/heads/release')) -- script: | - tar czf primaite_v$(VERSION)_benchmark.tar.gz benchmark/results/v$(MAJOR_VERSION)/v$(VERSION) - displayName: 'Prepare Artifacts for Publishing' + - script: | + tar czf primaite_v$(VERSION)_benchmark.tar.gz benchmark/results/v$(MAJOR_VERSION)/v$(VERSION) + displayName: 'Prepare Artifacts for Publishing' -- task: PublishPipelineArtifact@1 - inputs: - targetPath: primaite_v$(VERSION)_benchmark.tar.gz - artifactName: 'benchmark-output' - publishLocation: 'pipeline' - displayName: 'Publish Benchmark Output as Artifact' + - task: PublishPipelineArtifact@1 + inputs: + targetPath: primaite_v$(VERSION)_benchmark.tar.gz + artifactName: 'benchmark-output' + publishLocation: 'pipeline' + displayName: 'Publish Benchmark Output as Artifact' From 5e8343ca9154870b5eb6c5a2f88cd75a64903dd3 Mon Sep 17 00:00:00 2001 From: Chris McCarthy Date: Fri, 5 Jul 2024 16:57:40 +0100 Subject: [PATCH 2/6] #2739 - update job name in azure-benchmark-pipeline.yaml --- .azure/azure-benchmark-pipeline.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure/azure-benchmark-pipeline.yaml b/.azure/azure-benchmark-pipeline.yaml index e764c3c1..e8e6ec9c 100644 --- a/.azure/azure-benchmark-pipeline.yaml +++ b/.azure/azure-benchmark-pipeline.yaml @@ -16,7 +16,7 @@ variables: MAJOR_VERSION: '' jobs: -- job: PrimAITE Benchmark +- job: PrimAITE_Benchmark timeoutInMinutes: 0 # Set to unlimited timeout pool: name: 'Imaginary Yak Pool' From c14699230714843406e8538f0e0d39bfb7babf7f Mon Sep 17 00:00:00 2001 From: Chris McCarthy Date: Fri, 5 Jul 2024 17:02:35 +0100 Subject: [PATCH 3/6] #2739 - updated azure-benchmark-pipeline.yaml to run on ubuntu-latest while we wait for authorisation to use the yak pool --- .azure/azure-benchmark-pipeline.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure/azure-benchmark-pipeline.yaml b/.azure/azure-benchmark-pipeline.yaml index e8e6ec9c..350123e5 100644 --- a/.azure/azure-benchmark-pipeline.yaml +++ b/.azure/azure-benchmark-pipeline.yaml @@ -19,7 +19,7 @@ jobs: - job: PrimAITE_Benchmark timeoutInMinutes: 0 # Set to unlimited timeout pool: - name: 'Imaginary Yak Pool' + vmImage: ubuntu-latest workspace: clean: all steps: From 2a003eece9d9845e4cbad1ea5bb6c02f7435af20 Mon Sep 17 00:00:00 2001 From: Christopher McCarthy Date: Mon, 8 Jul 2024 08:25:05 +0000 Subject: [PATCH 4/6] Updated v3.0.0_benchmark_metadata.json so that combined_av_reward_per_episode is now named combined_total_reward_per_episode to match the new script --- benchmark/results/v3/v3.0.0/v3.0.0_benchmark_metadata.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/results/v3/v3.0.0/v3.0.0_benchmark_metadata.json b/benchmark/results/v3/v3.0.0/v3.0.0_benchmark_metadata.json index b6780eac..ed3ea4eb 100644 --- a/benchmark/results/v3/v3.0.0/v3.0.0_benchmark_metadata.json +++ b/benchmark/results/v3/v3.0.0/v3.0.0_benchmark_metadata.json @@ -26,7 +26,7 @@ "av_s_per_session": 3205.6340542, "av_s_per_step": 0.10017606419375, "av_s_per_100_steps_10_nodes": 10.017606419375, - "combined_av_reward_per_episode": { + "combined_total_reward_per_episode": { "1": -53.42999999999999, "2": -25.18000000000001, "3": -42.00000000000002, From 2d2f2df360d12e1e27c38b3ea8f2baa05090ffb8 Mon Sep 17 00:00:00 2001 From: Christopher McCarthy Date: Mon, 8 Jul 2024 08:26:52 +0000 Subject: [PATCH 5/6] Added set -e to the Run Benchmark Script step in azure-benchmark-pipeline.yaml so that the pipeline fails if the python script fails. --- .azure/azure-benchmark-pipeline.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.azure/azure-benchmark-pipeline.yaml b/.azure/azure-benchmark-pipeline.yaml index 350123e5..7eab2114 100644 --- a/.azure/azure-benchmark-pipeline.yaml +++ b/.azure/azure-benchmark-pipeline.yaml @@ -53,6 +53,7 @@ jobs: displayName: 'Install Dependencies' - script: | + set -e cd benchmark python3 primaite_benchmark.py cd .. From 2eb9d970bf5a84be9736a11a34c0047392983c36 Mon Sep 17 00:00:00 2001 From: Christopher McCarthy Date: Tue, 9 Jul 2024 19:25:13 +0000 Subject: [PATCH 6/6] Apply suggestions from code review --- .azure/azure-benchmark-pipeline.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure/azure-benchmark-pipeline.yaml b/.azure/azure-benchmark-pipeline.yaml index 7eab2114..8bd7d08e 100644 --- a/.azure/azure-benchmark-pipeline.yaml +++ b/.azure/azure-benchmark-pipeline.yaml @@ -17,7 +17,7 @@ variables: jobs: - job: PrimAITE_Benchmark - timeoutInMinutes: 0 # Set to unlimited timeout + timeoutInMinutes: 360 # 6-hour maximum pool: vmImage: ubuntu-latest workspace: