diff --git a/.azure/azure-benchmark-pipeline.yaml b/.azure/azure-benchmark-pipeline.yaml index 1f7b8ebe..8bd7d08e 100644 --- a/.azure/azure-benchmark-pipeline.yaml +++ b/.azure/azure-benchmark-pipeline.yaml @@ -11,74 +11,78 @@ schedules: branches: include: - 'refs/heads/dev' - -pool: - vmImage: ubuntu-latest - variables: VERSION: '' MAJOR_VERSION: '' -steps: -- checkout: self - persistCredentials: true +jobs: +- job: PrimAITE_Benchmark + timeoutInMinutes: 360 # 6-hour maximum + pool: + vmImage: ubuntu-latest + workspace: + clean: all + steps: + - checkout: self + persistCredentials: true -- script: | - VERSION=$(cat src/primaite/VERSION | tr -d '\n') - if [[ "$(Build.SourceBranch)" == "refs/heads/dev" ]]; then - DATE=$(date +%Y%m%d) - echo "${VERSION}+dev.${DATE}" > src/primaite/VERSION - fi - displayName: 'Update VERSION file for Dev Benchmark' + - script: | + VERSION=$(cat src/primaite/VERSION | tr -d '\n') + if [[ "$(Build.SourceBranch)" == "refs/heads/dev" ]]; then + DATE=$(date +%Y%m%d) + echo "${VERSION}+dev.${DATE}" > src/primaite/VERSION + fi + displayName: 'Update VERSION file for Dev Benchmark' -- script: | - VERSION=$(cat src/primaite/VERSION | tr -d '\n') - MAJOR_VERSION=$(echo $VERSION | cut -d. -f1) - echo "##vso[task.setvariable variable=VERSION]$VERSION" - echo "##vso[task.setvariable variable=MAJOR_VERSION]$MAJOR_VERSION" - displayName: 'Set Version Variables' + - script: | + VERSION=$(cat src/primaite/VERSION | tr -d '\n') + MAJOR_VERSION=$(echo $VERSION | cut -d. -f1) + echo "##vso[task.setvariable variable=VERSION]$VERSION" + echo "##vso[task.setvariable variable=MAJOR_VERSION]$MAJOR_VERSION" + displayName: 'Set Version Variables' -- task: UsePythonVersion@0 - inputs: - versionSpec: '3.11' - addToPath: true + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.11' + addToPath: true -- script: | - python -m pip install --upgrade pip - pip install -e .[dev,rl] - primaite setup - displayName: 'Install Dependencies' + - script: | + python -m pip install --upgrade pip + pip install -e .[dev,rl] + primaite setup + displayName: 'Install Dependencies' -- script: | - cd benchmark - python3 primaite_benchmark.py - cd .. - displayName: 'Run Benchmarking Script' + - script: | + set -e + cd benchmark + python3 primaite_benchmark.py + cd .. + displayName: 'Run Benchmarking Script' -- script: | - git config --global user.email "oss@dstl.gov.uk" - git config --global user.name "Defence Science and Technology Laboratory UK" - workingDirectory: $(System.DefaultWorkingDirectory) - displayName: 'Configure Git' - condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release')) + - script: | + git config --global user.email "oss@dstl.gov.uk" + git config --global user.name "Defence Science and Technology Laboratory UK" + workingDirectory: $(System.DefaultWorkingDirectory) + displayName: 'Configure Git' + condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release')) -- script: | - git add benchmark/results/v$(MAJOR_VERSION)/v$(VERSION)/* - git commit -m "Automated benchmark output commit for version $(VERSION)" - git push origin HEAD:refs/heads/$(Build.SourceBranchName) - displayName: 'Commit and Push Benchmark Results' - workingDirectory: $(System.DefaultWorkingDirectory) - env: - GIT_CREDENTIALS: $(System.AccessToken) - condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/heads/release')) + - script: | + git add benchmark/results/v$(MAJOR_VERSION)/v$(VERSION)/* + git commit -m "Automated benchmark output commit for version $(VERSION)" + git push origin HEAD:refs/heads/$(Build.SourceBranchName) + displayName: 'Commit and Push Benchmark Results' + workingDirectory: $(System.DefaultWorkingDirectory) + env: + GIT_CREDENTIALS: $(System.AccessToken) + condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/heads/release')) -- script: | - tar czf primaite_v$(VERSION)_benchmark.tar.gz benchmark/results/v$(MAJOR_VERSION)/v$(VERSION) - displayName: 'Prepare Artifacts for Publishing' + - script: | + tar czf primaite_v$(VERSION)_benchmark.tar.gz benchmark/results/v$(MAJOR_VERSION)/v$(VERSION) + displayName: 'Prepare Artifacts for Publishing' -- task: PublishPipelineArtifact@1 - inputs: - targetPath: primaite_v$(VERSION)_benchmark.tar.gz - artifactName: 'benchmark-output' - publishLocation: 'pipeline' - displayName: 'Publish Benchmark Output as Artifact' + - task: PublishPipelineArtifact@1 + inputs: + targetPath: primaite_v$(VERSION)_benchmark.tar.gz + artifactName: 'benchmark-output' + publishLocation: 'pipeline' + displayName: 'Publish Benchmark Output as Artifact' diff --git a/benchmark/results/v3/v3.0.0/v3.0.0_benchmark_metadata.json b/benchmark/results/v3/v3.0.0/v3.0.0_benchmark_metadata.json index b6780eac..ed3ea4eb 100644 --- a/benchmark/results/v3/v3.0.0/v3.0.0_benchmark_metadata.json +++ b/benchmark/results/v3/v3.0.0/v3.0.0_benchmark_metadata.json @@ -26,7 +26,7 @@ "av_s_per_session": 3205.6340542, "av_s_per_step": 0.10017606419375, "av_s_per_100_steps_10_nodes": 10.017606419375, - "combined_av_reward_per_episode": { + "combined_total_reward_per_episode": { "1": -53.42999999999999, "2": -25.18000000000001, "3": -42.00000000000002,