From 2ffd0887ae038ab26f511b5a1cb0e16be744690c Mon Sep 17 00:00:00 2001 From: Matt Thalman Date: Fri, 29 Mar 2024 10:22:54 -0500 Subject: [PATCH] [release/8.0.1xx] Migrate VMR pipeline to 1ES templates (#19073) --- eng/pipelines/templates/jobs/vmr-build.yml | 35 ++-- eng/pipelines/templates/stages/vmr-build.yml | 157 ++++++++---------- .../templates/variables/vmr-build.yml | 24 +++ src/SourceBuild/content/eng/pipelines/ci.yml | 28 +++- .../pipelines/templates/stages/vmr-scan.yml | 11 +- 5 files changed, 145 insertions(+), 110 deletions(-) create mode 100644 eng/pipelines/templates/variables/vmr-build.yml diff --git a/eng/pipelines/templates/jobs/vmr-build.yml b/eng/pipelines/templates/jobs/vmr-build.yml index 9c5e58dbe..bc986e9f4 100644 --- a/eng/pipelines/templates/jobs/vmr-build.yml +++ b/eng/pipelines/templates/jobs/vmr-build.yml @@ -67,7 +67,10 @@ jobs: condition: succeededOrFailed() dependsOn: ${{ parameters.reuseBuildArtifactsFrom }}_${{ parameters.architecture }} variables: - - template: /eng/common/templates/variables/pool-providers.yml + - ${{ if eq(variables['System.TeamProject'], 'public') }}: + - template: /eng/common/templates/variables/pool-providers.yml + - ${{ else }}: + - template: /eng/common/templates-official/variables/pool-providers.yml - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - group: AzureDevOps-Artifact-Feeds-Pats - ${{ if and(not(parameters.isBuiltFromVmr), eq(variables['System.TeamProject'], 'internal'), not(startswith(parameters.vmrBranch, 'internal/release/')), not(eq(variables['Build.Reason'], 'PullRequest'))) }}: @@ -87,8 +90,21 @@ jobs: - name: sourcesPath value: ${{ parameters.vmrPath }} + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish BuildLogs' + condition: succeededOrFailed() + targetPath: '$(Build.StagingDirectory)/BuildLogs' + artifactName: $(Agent.JobName)_BuildLogs_Attempt$(System.JobAttempt) + - output: pipelineArtifact + displayName: 'Publish Artifacts' + condition: succeededOrFailed() + targetPath: '${{ variables.sourcesPath }}/artifacts/${{ parameters.architecture }}/Release/' + artifactName: $(Agent.JobName)_Artifacts + steps: - - template: ../steps/vmr-prepare.yml + - template: ../steps/vmr-prepare.yml@self parameters: ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: vmrBranch: $(System.PullRequest.TargetBranch) @@ -99,7 +115,7 @@ jobs: # Synchronize new content in the VMR during PRs (we expect this to come - ${{ if and(not(parameters.isBuiltFromVmr), eq(variables['Build.Reason'], 'PullRequest')) }}: - - template: ../steps/vmr-pull-updates.yml + - template: ../steps/vmr-pull-updates.yml@self parameters: vmrPath: ${{ parameters.vmrPath }} vmrBranch: ${{ parameters.vmrBranch }} @@ -250,13 +266,6 @@ jobs: displayName: Prepare BuildLogs staging directory continueOnError: true condition: succeededOrFailed() - - - publish: '$(Build.StagingDirectory)/BuildLogs' - artifact: $(Agent.JobName)_BuildLogs_Attempt$(System.JobAttempt) - displayName: Publish BuildLogs - continueOnError: true - condition: succeededOrFailed() - - task: PublishTestResults@2 displayName: Publish Test Results condition: succeededOrFailed() @@ -268,9 +277,3 @@ jobs: mergeTestResults: true publishRunAttachments: true testRunTitle: SourceBuild_SmokeTests_$(Agent.JobName) - - - publish: '${{ variables.sourcesPath }}/artifacts/${{ parameters.architecture }}/Release/' - artifact: $(Agent.JobName)_Artifacts - displayName: Publish Artifacts - condition: succeededOrFailed() - continueOnError: true diff --git a/eng/pipelines/templates/stages/vmr-build.yml b/eng/pipelines/templates/stages/vmr-build.yml index 43f0c8633..2f9692f22 100644 --- a/eng/pipelines/templates/stages/vmr-build.yml +++ b/eng/pipelines/templates/stages/vmr-build.yml @@ -2,38 +2,61 @@ ### If run in a PR, new changes are applied to a local copy of the VMR, then it is source-built and tested parameters: - dependsOn: [] - condition: always() +- name: dependsOn + type: object + default: [] +- name: condition + type: string + default: always() # Branch of the VMR to use (to push to for internal builds) - vmrBranch: $(Build.SourceBranch) +- name: vmrBranch + type: string + default: $(Build.SourceBranch) # True when build is running from dotnet/dotnet directly - isBuiltFromVmr: +- name: isBuiltFromVmr + type: boolean + default: false # The following parameters aren't expected to be passed in rather they are used for encapsulation # ----------------------------------------------------------------------------------------------- - alpine317Container: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.17 - centOSStream8Container: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8 - centOSStream9Container: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9 - fedora38Container: mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-38 - ubuntu2204Container: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04 - ubuntu2204ArmContainer: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-arm64 +- name: alpine317Container + type: string + default: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.17 +- name: centOSStream8Container + type: string + default: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8 +- name: centOSStream9Container + type: string + default: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9 +- name: fedora38Container + type: string + default: mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-38 +- name: ubuntu2204Container + type: string + default: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04 +- name: ubuntu2204ArmContainer + type: string + default: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-arm64 - # Internal builds - poolInternalAmd64: - name: NetCore1ESPool-Svc-Internal - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 - poolInternalAmd64PR: - name: NetCore1ESPool-Internal-XL - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 - poolInternalArm64: - name: Docker-Linux-Arm-Internal +# These are not expected to be passed it but rather just object variables reused below +- name: pool_Linux + type: object + default: + name: $(defaultPoolName) + image: $(poolImage_Linux) + demands: ImageOverride -equals $(poolImage_Linux) + os: linux - # Public builds / PRs - poolPublicAmd64: - name: NetCore-Public-XL - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open +- name: pool_LinuxArm64 + type: object + default: + name: $(poolName_LinuxArm64) + image: $(poolImage_LinuxArm64) + demands: ImageOverride -equals $(poolImage_Linux) + hostArchitecture: Arm64 + os: linux stages: - stage: VMR_Source_Build @@ -41,23 +64,7 @@ stages: dependsOn: ${{ parameters.dependsOn }} condition: ${{ parameters.condition }} variables: - - ${{ if eq(variables['System.TeamProject'], 'public') }}: - - name: defaultPoolName - value: ${{ parameters.poolPublicAmd64.name }} - - name: defaultPoolDemands - value: ${{ parameters.poolPublicAmd64.demands }} - - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: - - name: defaultPoolName - value: ${{ parameters.poolInternalAmd64PR.name }} - - name: defaultPoolDemands - value: ${{ parameters.poolInternalAmd64PR.demands }} - - ${{ else }}: - - name: defaultPoolName - value: ${{ parameters.poolInternalAmd64.name }} - - name: defaultPoolDemands - value: ${{ parameters.poolInternalAmd64.demands }} - + - template: ../variables/vmr-build.yml@self - ${{ if ne(parameters.vmrBranch, '') }}: - name: VmrBranch value: ${{ parameters.vmrBranch }} @@ -69,16 +76,14 @@ stages: # PR and CI legs ------------------------------------ - - template: ../jobs/vmr-build.yml + - template: ../jobs/vmr-build.yml@self parameters: # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline buildName: CentOSStream8_Online_MsftSdk isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} architecture: x64 - pool: - name: ${{ variables.defaultPoolName }} - demands: ${{ variables.defaultPoolDemands }} + pool: ${{ parameters.pool_Linux }} container: ${{ parameters.centOSStream8Container }} buildFromArchive: false # 🚫 enablePoison: false # 🚫 @@ -87,16 +92,14 @@ stages: useMonoRuntime: false # 🚫 withPreviousSDK: false # 🚫 - - template: ../jobs/vmr-build.yml + - template: ../jobs/vmr-build.yml@self parameters: # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline buildName: CentOSStream8_Offline_MsftSdk isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} architecture: x64 - pool: - name: ${{ variables.defaultPoolName }} - demands: ${{ variables.defaultPoolDemands }} + pool: ${{ parameters.pool_Linux }} container: ${{ parameters.centOSStream8Container }} buildFromArchive: true # ✅ enablePoison: false # 🚫 @@ -109,16 +112,14 @@ stages: # CI - Stage 1 x64 legs ------------------------------------ - - template: ../jobs/vmr-build.yml + - template: ../jobs/vmr-build.yml@self parameters: # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline buildName: Alpine317_Offline_MsftSdk isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} architecture: x64 - pool: - name: ${{ variables.defaultPoolName }} - demands: ${{ variables.defaultPoolDemands }} + pool: ${{ parameters.pool_Linux }} container: ${{ parameters.alpine317Container }} buildFromArchive: false # ✅ enablePoison: false # 🚫 @@ -127,16 +128,14 @@ stages: useMonoRuntime: false # 🚫 withPreviousSDK: false # 🚫 - - template: ../jobs/vmr-build.yml + - template: ../jobs/vmr-build.yml@self parameters: # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline buildName: CentOSStream8_Online_PreviousSourceBuiltSdk isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} architecture: x64 - pool: - name: ${{ variables.defaultPoolName }} - demands: ${{ variables.defaultPoolDemands }} + pool: ${{ parameters.pool_Linux }} container: ${{ parameters.centOSStream8Container }} buildFromArchive: false # 🚫 enablePoison: false # 🚫 @@ -145,16 +144,14 @@ stages: useMonoRuntime: false # 🚫 withPreviousSDK: true # ✅ - - template: ../jobs/vmr-build.yml + - template: ../jobs/vmr-build.yml@self parameters: # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline buildName: CentOSStream8_Offline_PreviousSourceBuiltSdk isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} architecture: x64 - pool: - name: ${{ variables.defaultPoolName }} - demands: ${{ variables.defaultPoolDemands }} + pool: ${{ parameters.pool_Linux }} container: ${{ parameters.centOSStream8Container }} buildFromArchive: false # 🚫 enablePoison: false # 🚫 @@ -163,16 +160,14 @@ stages: useMonoRuntime: false # 🚫 withPreviousSDK: true # ✅ - - template: ../jobs/vmr-build.yml + - template: ../jobs/vmr-build.yml@self parameters: # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline buildName: CentOSStream8_Mono_Offline_MsftSdk isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} architecture: x64 - pool: - name: ${{ variables.defaultPoolName }} - demands: ${{ variables.defaultPoolDemands }} + pool: ${{ parameters.pool_Linux }} container: ${{ parameters.centOSStream8Container }} buildFromArchive: true # ✅ enablePoison: false # 🚫 @@ -181,16 +176,14 @@ stages: useMonoRuntime: true # ✅ withPreviousSDK: false # 🚫 - - template: ../jobs/vmr-build.yml + - template: ../jobs/vmr-build.yml@self parameters: # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline buildName: CentOSStream9_Offline_MsftSdk isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} architecture: x64 - pool: - name: ${{ variables.defaultPoolName }} - demands: ${{ variables.defaultPoolDemands }} + pool: ${{ parameters.pool_Linux }} container: ${{ parameters.centOSStream9Container }} buildFromArchive: true # ✅ enablePoison: false # 🚫 @@ -199,16 +192,14 @@ stages: useMonoRuntime: false # 🚫 withPreviousSDK: false # 🚫 - - template: ../jobs/vmr-build.yml + - template: ../jobs/vmr-build.yml@self parameters: # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline buildName: Fedora38_Offline_MsftSdk isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} architecture: x64 - pool: - name: ${{ variables.defaultPoolName }} - demands: ${{ variables.defaultPoolDemands }} + pool: ${{ parameters.pool_Linux }} container: ${{ parameters.fedora38Container }} buildFromArchive: true # ✅ enablePoison: true # ✅ @@ -217,16 +208,14 @@ stages: useMonoRuntime: false # 🚫 withPreviousSDK: false # 🚫 - - template: ../jobs/vmr-build.yml + - template: ../jobs/vmr-build.yml@self parameters: # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline buildName: Ubuntu2204_Offline_MsftSdk isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} architecture: x64 - pool: - name: ${{ variables.defaultPoolName }} - demands: ${{ variables.defaultPoolDemands }} + pool: ${{ parameters.pool_Linux }} container: ${{ parameters.ubuntu2204Container }} buildFromArchive: false # 🚫 enablePoison: false # 🚫 @@ -237,14 +226,14 @@ stages: # CI - Stage 1 arm64 Legs ------------------------------------ - - template: ../jobs/vmr-build.yml + - template: ../jobs/vmr-build.yml@self parameters: # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline buildName: Ubuntu2204Arm64_Offline_MsftSdk isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} architecture: arm64 - pool: ${{ parameters.poolInternalArm64 }} + pool: ${{ parameters.pool_LinuxArm64 }} container: ${{ parameters.ubuntu2204ArmContainer }} buildFromArchive: false # 🚫 enablePoison: false # 🚫 @@ -255,16 +244,14 @@ stages: # CI - Stage 2 x64 Legs ------------------------------------ - - template: ../jobs/vmr-build.yml + - template: ../jobs/vmr-build.yml@self parameters: # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline buildName: CentOSStream8_Online_CurrentSourceBuiltSdk isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} architecture: x64 - pool: - name: ${{ variables.defaultPoolName }} - demands: ${{ variables.defaultPoolDemands }} + pool: ${{ parameters.pool_Linux }} container: ${{ parameters.centOSStream8Container }} buildFromArchive: false # 🚫 enablePoison: false # 🚫 @@ -274,16 +261,14 @@ stages: withPreviousSDK: false # 🚫 reuseBuildArtifactsFrom: CentOSStream8_Online_MsftSdk - - template: ../jobs/vmr-build.yml + - template: ../jobs/vmr-build.yml@self parameters: # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline buildName: Fedora38_Offline_CurrentSourceBuiltSdk isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} architecture: x64 - pool: - name: ${{ variables.defaultPoolName }} - demands: ${{ variables.defaultPoolDemands }} + pool: ${{ parameters.pool_Linux }} container: ${{ parameters.fedora38Container }} buildFromArchive: false # 🚫 enablePoison: false # 🚫 diff --git a/eng/pipelines/templates/variables/vmr-build.yml b/eng/pipelines/templates/variables/vmr-build.yml new file mode 100644 index 000000000..04873069a --- /dev/null +++ b/eng/pipelines/templates/variables/vmr-build.yml @@ -0,0 +1,24 @@ +variables: + +- ${{ if eq(variables['System.TeamProject'], 'public') }}: + - name: defaultPoolName + value: NetCore-Public-XL + - name: poolImage_Linux + value: 1es-ubuntu-2004-open + - name: poolImage_LinuxArm64 + value: Mariner-2-Docker-ARM64 + - name: poolName_LinuxArm64 + value: Docker-Linux-Arm-Public +- ${{ else }}: + - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: + - name: defaultPoolName + value: NetCore1ESPool-Internal-XL + - ${{ else }}: + - name: defaultPoolName + value: $(DncEngInternalBuildPool) + - name: poolImage_Linux + value: 1es-ubuntu-2204 + - name: poolImage_LinuxArm64 + value: Mariner-2-Docker-ARM64 + - name: poolName_LinuxArm64 + value: Docker-Linux-Arm-Internal diff --git a/src/SourceBuild/content/eng/pipelines/ci.yml b/src/SourceBuild/content/eng/pipelines/ci.yml index 55b6a212d..883ea980f 100644 --- a/src/SourceBuild/content/eng/pipelines/ci.yml +++ b/src/SourceBuild/content/eng/pipelines/ci.yml @@ -15,10 +15,28 @@ pr: - release/* - internal/release/* -stages: -- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - template: templates/stages/vmr-scan.yml +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release -- template: /src/installer/eng/pipelines/templates/stages/vmr-build.yml +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates parameters: - isBuiltFromVmr: true + sdl: + sourceAnalysisPool: + name: NetCore1ESPool-Internal + image: 1es-windows-2022 + os: windows + + customBuildTags: + - ES365AIMigrationTooling + + stages: + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - template: /eng/pipelines/templates/stages/vmr-scan.yml@self + - template: /src/installer/eng/pipelines/templates/stages/vmr-build.yml@self + parameters: + isBuiltFromVmr: true diff --git a/src/SourceBuild/content/eng/pipelines/templates/stages/vmr-scan.yml b/src/SourceBuild/content/eng/pipelines/templates/stages/vmr-scan.yml index 6024ce182..28541d572 100644 --- a/src/SourceBuild/content/eng/pipelines/templates/stages/vmr-scan.yml +++ b/src/SourceBuild/content/eng/pipelines/templates/stages/vmr-scan.yml @@ -3,17 +3,22 @@ stages: displayName: Tag & Scan dependsOn: [] variables: - - template: /eng/common/templates/variables/pool-providers.yml + - ${{ if eq(variables['System.TeamProject'], 'public') }}: + - template: /eng/common/templates/variables/pool-providers.yml + - ${{ else }}: + - template: /eng/common/templates-official/variables/pool-providers.yml jobs: - job: Tag_n_Scan displayName: Tag & Scan pool: ${{ if eq(variables['System.TeamProject'], 'public') }}: name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals Build.Ubuntu.2004.Amd64.Open + image: 1es-ubuntu-2004-open + os: linux ${{ if eq(variables['System.TeamProject'], 'internal') }}: name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals Build.Ubuntu.2004.Amd64 + image: 1es-ubuntu-2004 + os: linux steps: - checkout: self