From ce1921c17de51cfaba45b488d883a4564a7c353c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emek=20Vysok=C3=BD?= Date: Thu, 15 Dec 2022 09:33:09 +0100 Subject: [PATCH] Run VMR's pipelines from the VMR directly (#15124) --- .vsts-ci.yml | 2 +- eng/build.yml | 57 ++++++++++++------- eng/pipelines/templates/jobs/vmr-build.yml | 19 +++---- .../templates/jobs/vmr-synchronization.yml | 10 +--- eng/pipelines/templates/stages/vmr-build.yml | 24 ++++---- eng/pipelines/templates/steps/vmr-prepare.yml | 36 +++++------- eng/pipelines/vmr-ci.yml | 21 ------- .../tarball/content/eng/pipelines/ci.yml | 52 +++++++++++++++++ 8 files changed, 130 insertions(+), 91 deletions(-) delete mode 100644 eng/pipelines/vmr-ci.yml create mode 100644 src/SourceBuild/tarball/content/eng/pipelines/ci.yml diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 7e9bc79af..59526b0ca 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -377,7 +377,7 @@ stages: - template: eng/pipelines/templates/stages/vmr-build.yml parameters: vmrBranch: ${{ parameters.vmrBranch }} - installerBuildResourceId: current + isBuiltFromVmr: false - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - stage: Publish diff --git a/eng/build.yml b/eng/build.yml index 246bf679e..121f81771 100644 --- a/eng/build.yml +++ b/eng/build.yml @@ -63,6 +63,11 @@ parameters: type: boolean default: false +- name: isBuiltFromVmr + displayName: True when build is running from dotnet/dotnet + type: boolean + default: false + jobs: - template: common/templates/job/job.yml parameters: @@ -96,9 +101,12 @@ jobs: vmImage: 'macOS-latest' timeoutInMinutes: ${{ parameters.timeoutInMinutes }} - enablePublishBuildAssets: true - enablePublishUsingPipelines: true - enableTelemetry: true + ${{ if parameters.isBuiltFromVmr }}: + enableSbom: false + ${{ else }}: + enablePublishBuildAssets: true + enablePublishUsingPipelines: true + enableTelemetry: true helixRepo: dotnet/installer workspace: clean: all @@ -139,29 +147,40 @@ jobs: - _SignType: test - _BuildArgs: '/p:DotNetSignType=$(_SignType) $(_PgoInstrument)' - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: DotNet-HelixApi-Access - - _PushToVSFeed: true - - _SignType: real + - ${{ if parameters.isBuiltFromVmr }}: + - installerRoot: '$(Build.SourcesDirectory)/src/installer' + - _SignType: test + - _PushToVSFeed: false - _BuildArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) - /p:DotNetSignType=$(_SignType) /p:TeamName=$(_TeamName) - /p:DotNetPublishUsingPipelines=$(_PublishUsingPipelines) + /p:DotNetPublishUsingPipelines=true + /p:PublishToSymbolServer=false $(_PgoInstrument) + - ${{ else }}: + - installerRoot: '$(Build.SourcesDirectory)' + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - group: DotNet-HelixApi-Access + - _PushToVSFeed: true + - _SignType: real + - _BuildArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) + /p:DotNetSignType=$(_SignType) + /p:TeamName=$(_TeamName) + /p:DotNetPublishUsingPipelines=$(_PublishUsingPipelines) + $(_PgoInstrument) steps: - checkout: self clean: true - ${{ if eq(parameters.agentOs, 'Windows_NT') }}: - - ${{ if ne(variables['System.TeamProject'], 'public') }}: + - ${{ if and(not(parameters.isBuiltFromVmr), ne(variables['System.TeamProject'], 'public')) }}: - task: PowerShell@2 displayName: Setup Private Feeds Credentials inputs: - filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 - arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token + filePath: $(installerRoot)/eng/common/SetupNugetSources.ps1 + arguments: -ConfigFile $(installerRoot)/NuGet.config -Password $Env:Token env: Token: $(dn-bot-dnceng-artifact-feeds-rw) - - script: build.cmd + - script: $(installerRoot)/build.cmd $(_TestArg) $(_PackArg) -publish -ci -sign -Configuration ${{ parameters.buildConfiguration }} @@ -174,16 +193,16 @@ jobs: DOTNET_CLI_UI_LANGUAGE: ${{ parameters.dotnetCLIUILanguage }} - ${{ if ne(parameters.agentOs, 'Windows_NT') }}: - - ${{ if ne(variables['System.TeamProject'], 'public') }}: + - ${{ if and(not(parameters.isBuiltFromVmr), ne(variables['System.TeamProject'], 'public')) }}: - task: Bash@3 displayName: Setup Private Feeds Credentials inputs: - filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh - arguments: $(Build.SourcesDirectory)/NuGet.config $Token + filePath: $(installerRoot)/eng/common/SetupNugetSources.sh + arguments: $(installerRoot)/NuGet.config $Token env: Token: $(dn-bot-dnceng-artifact-feeds-rw) - ${{ if eq(parameters.agentOs, 'Linux') }}: - - script: ./build.sh + - script: $(installerRoot)/build.sh $(_TestArg) $(_PackArg) --publish --ci --noprettyprint @@ -197,7 +216,7 @@ jobs: displayName: Build - ${{ if or(eq(parameters.agentOs, 'Darwin'), eq(parameters.agentOs, 'FreeBSD')) }}: - - script: ./build.sh + - script: $(installerRoot)/build.sh $(_TestArg) --pack --publish --ci --noprettyprint @@ -222,7 +241,7 @@ jobs: - task: CopyFiles@2 displayName: Gather Logs inputs: - SourceFolder: '$(Build.SourcesDirectory)/artifacts' + SourceFolder: '$(installerRoot)/artifacts' Contents: | log/${{ parameters.buildConfiguration }}/**/* TestResults/${{ parameters.buildConfiguration }}/**/* diff --git a/eng/pipelines/templates/jobs/vmr-build.yml b/eng/pipelines/templates/jobs/vmr-build.yml index d7337f38e..5c21c1bf5 100644 --- a/eng/pipelines/templates/jobs/vmr-build.yml +++ b/eng/pipelines/templates/jobs/vmr-build.yml @@ -2,6 +2,9 @@ ### If run in a PR, new changes are applied to a local copy of the VMR, then it is source-built and tested parameters: +- name: isBuiltFromVmr + displayName: True when build is running from dotnet/dotnet directly + type: boolean - name: vmrPath type: string @@ -12,10 +15,6 @@ parameters: type: string default: $(Build.SourceBranchName) -# Resource id of the installer build to retrieve MSFT SDK from -- name: installerBuildResourceId - type: string - - name: buildName type: string @@ -63,7 +62,7 @@ jobs: dependsOn: ${{ parameters.reuseBuildArtifactsFrom }}_${{ parameters.architecture }} variables: - template: /eng/common/templates/variables/pool-providers.yml - - ${{ if eq(variables['System.TeamProject'], 'internal') }}: + - ${{ if and(not(parameters.isBuiltFromVmr), eq(variables['System.TeamProject'], 'internal')) }}: - group: DotNetBot-GitHub - group: AzureDevOps-Artifact-Feeds-Pats - ${{ else }}: @@ -80,11 +79,11 @@ jobs: - template: ../steps/vmr-prepare.yml parameters: vmrBranch: ${{ parameters.vmrBranch }} - installerBuildResourceId: ${{ parameters.installerBuildResourceId }} + isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} skipComponentGovernanceDetection: true # Synchronize new content in the VMR during PRs (we expect this to come - - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + - ${{ if and(not(parameters.isBuiltFromVmr), eq(variables['Build.Reason'], 'PullRequest')) }}: - template: ../steps/vmr-pull-updates.yml parameters: vmrPath: ${{ parameters.vmrPath }} @@ -104,7 +103,7 @@ jobs: Token: $(dn-bot-dnceng-artifact-feeds-rw) - ${{ if ne(parameters.excludeSdkContentTests, 'true') }}: - - download: ${{ parameters.installerBuildResourceId }} + - download: current artifact: BlobArtifacts patterns: '**/dotnet-sdk-+([0-9]).+([0-9]).+([0-9])?(-@(alpha|preview|rc|rtm)*)-linux-${{ parameters.architecture }}.tar.gz' displayName: Download MSFT SDK @@ -167,8 +166,8 @@ jobs: poisonArg='' if [[ '${{ parameters.excludeSdkContentTests }}' != 'True' ]]; then - dockerVolumeArgs+=" -v $(Pipeline.Workspace)/${{ parameters.installerBuildResourceId }}/BlobArtifacts/:/BlobArtifacts" - msftSdkTarballName=$(find "$(Pipeline.Workspace)/${{ parameters.installerBuildResourceId }}/BlobArtifacts/" -name "dotnet-sdk-*-linux-${{ parameters.architecture }}.tar.gz" -exec basename {} \;) + dockerVolumeArgs+=" -v $(Pipeline.Workspace)/BlobArtifacts/:/BlobArtifacts" + msftSdkTarballName=$(find "$(Pipeline.Workspace)/BlobArtifacts/" -name "dotnet-sdk-*-linux-${{ parameters.architecture }}.tar.gz" -exec basename {} \;) dockerEnvArgs+=" -e SMOKE_TESTS_MSFT_SDK_TARBALL_PATH=/BlobArtifacts/$msftSdkTarballName" fi diff --git a/eng/pipelines/templates/jobs/vmr-synchronization.yml b/eng/pipelines/templates/jobs/vmr-synchronization.yml index 012b804a0..4befbafd9 100644 --- a/eng/pipelines/templates/jobs/vmr-synchronization.yml +++ b/eng/pipelines/templates/jobs/vmr-synchronization.yml @@ -43,6 +43,7 @@ jobs: - template: ../steps/vmr-prepare.yml parameters: vmrBranch: ${{ parameters.vmrBranch }} + isBuiltFromVmr: false - template: ../steps/vmr-pull-updates.yml parameters: @@ -60,14 +61,5 @@ jobs: git branch ${{ parameters.vmrBranch }} git branch --set-upstream-to=dotnet/${{ parameters.vmrBranch }} ${{ parameters.vmrBranch }} || echo 'Branch ${{ parameters.vmrBranch }} not found in remote' git push dotnet ${{ parameters.vmrBranch }} - - # Export the SHA as an artifact to be reused by the official VMR build - sha=`git rev-parse HEAD` - mkdir -p "$(Build.ArtifactStagingDirectory)/VmrRevision" - echo $sha > "$(Build.ArtifactStagingDirectory)/VmrRevision/VmrRevision.txt" displayName: Push changes to dotnet/dotnet workingDirectory: $(vmrPath) - - - publish: $(Build.ArtifactStagingDirectory)/VmrRevision - artifact: VmrRevision - displayName: Publish new SHA diff --git a/eng/pipelines/templates/stages/vmr-build.yml b/eng/pipelines/templates/stages/vmr-build.yml index 4a3db6eaf..8d052af5d 100644 --- a/eng/pipelines/templates/stages/vmr-build.yml +++ b/eng/pipelines/templates/stages/vmr-build.yml @@ -2,12 +2,15 @@ ### If run in a PR, new changes are applied to a local copy of the VMR, then it is source-built and tested parameters: - # Resource id of the installer build to retrieve source/product tarball's from - installerBuildResourceId: current + dependsOn: [] + condition: always() # Branch of the VMR to use (to push to for internal builds) vmrBranch: $(Build.SourceBranchName) + # True when build is running from dotnet/dotnet directly + isBuiltFromVmr: + # The following parameters aren't expected to be passed in rather they are used for encapsulation # ----------------------------------------------------------------------------------------------- centOSStream8Container: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8 @@ -34,7 +37,8 @@ parameters: stages: - stage: VMR_Source_Build displayName: VMR Source-Build - dependsOn: [] + dependsOn: ${{ parameters.dependsOn }} + condition: ${{ parameters.condition }} variables: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: defaultPoolName @@ -57,7 +61,7 @@ stages: - template: ../jobs/vmr-build.yml parameters: buildName: CentOSStream8_Online - installerBuildResourceId: ${{ parameters.installerBuildResourceId }} + isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ parameters.vmrBranch }} architecture: x64 pool: @@ -74,7 +78,7 @@ stages: - template: ../jobs/vmr-build.yml parameters: buildName: CentOSStream8_Offline - installerBuildResourceId: ${{ parameters.installerBuildResourceId }} + isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ parameters.vmrBranch }} architecture: x64 pool: @@ -90,7 +94,7 @@ stages: - template: ../jobs/vmr-build.yml parameters: buildName: CentOSStream9_Offline - installerBuildResourceId: ${{ parameters.installerBuildResourceId }} + isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ parameters.vmrBranch }} architecture: x64 pool: @@ -106,7 +110,7 @@ stages: - template: ../jobs/vmr-build.yml parameters: buildName: Fedora36_Offline - installerBuildResourceId: ${{ parameters.installerBuildResourceId }} + isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ parameters.vmrBranch }} architecture: x64 pool: @@ -122,7 +126,7 @@ stages: - template: ../jobs/vmr-build.yml parameters: buildName: Ubuntu2004_Offline - installerBuildResourceId: ${{ parameters.installerBuildResourceId }} + isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ parameters.vmrBranch }} architecture: x64 pool: @@ -138,7 +142,7 @@ stages: - template: ../jobs/vmr-build.yml parameters: buildName: Debian11_Offline - installerBuildResourceId: ${{ parameters.installerBuildResourceId }} + isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ parameters.vmrBranch }} architecture: arm64 pool: ${{ parameters.poolInternalArm64 }} @@ -152,7 +156,7 @@ stages: - template: ../jobs/vmr-build.yml parameters: buildName: Fedora36_Offline_Using_Previous - installerBuildResourceId: ${{ parameters.installerBuildResourceId }} + isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ parameters.vmrBranch }} architecture: x64 pool: diff --git a/eng/pipelines/templates/steps/vmr-prepare.yml b/eng/pipelines/templates/steps/vmr-prepare.yml index 2786cbfad..d6aaa13da 100644 --- a/eng/pipelines/templates/steps/vmr-prepare.yml +++ b/eng/pipelines/templates/steps/vmr-prepare.yml @@ -2,39 +2,33 @@ ### Component Governance scan is also triggered over the VMR's non-repo sources parameters: +- name: isBuiltFromVmr + displayName: True when build is running from dotnet/dotnet directly + type: boolean + - name: vmrBranch displayName: dotnet/dotnet branch to use type: string default: $(Build.SourceBranchName) -- name: installerBuildResourceId - displayName: Installer build to get the commit hash to check out from - type: string - default: current - - name: skipComponentGovernanceDetection type: boolean default: false steps: -- checkout: vmr - displayName: Clone dotnet/dotnet - path: vmr - clean: true - fetchDepth: 0 # We need this so that we can check out the new commit +- ${{ if parameters.isBuiltFromVmr }}: + - checkout: self + displayName: Clone dotnet/dotnet + path: vmr + clean: true -- ${{ if ne(parameters.installerBuildResourceId, 'current') }}: - - download: ${{ parameters.installerBuildResourceId }} - artifact: VmrRevision - displayName: Download VmrRevision - - - script: | - set -ex - sha=`head -n 1 $(Pipeline.Workspace)/${{ parameters.installerBuildResourceId }}/VmrRevision/VmrRevision.txt` - git checkout $sha - displayName: Check out dotnet/dotnet commit - workingDirectory: $(Agent.BuildDirectory)/vmr - ${{ else }}: + - checkout: vmr + displayName: Clone dotnet/dotnet + path: vmr + clean: true + +- ${{ if or(not(parameters.isBuiltFromVmr), eq(variables['System.TeamProject'], 'internal')) }}: - script: | git switch -c ${{ parameters.vmrBranch }} displayName: Checkout ${{ parameters.vmrBranch }} diff --git a/eng/pipelines/vmr-ci.yml b/eng/pipelines/vmr-ci.yml deleted file mode 100644 index a63858e14..000000000 --- a/eng/pipelines/vmr-ci.yml +++ /dev/null @@ -1,21 +0,0 @@ -trigger: none -pr: none -resources: - pipelines: - - pipeline: installer-build-resource - source: dotnet-installer-official-ci - trigger: - branches: - include: - - main - - repositories: - - repository: vmr - type: github - name: dotnet/dotnet - endpoint: dotnet - -stages: -- template: templates/stages/vmr-build.yml - parameters: - installerBuildResourceId: installer-build-resource diff --git a/src/SourceBuild/tarball/content/eng/pipelines/ci.yml b/src/SourceBuild/tarball/content/eng/pipelines/ci.yml new file mode 100644 index 000000000..9e74ea872 --- /dev/null +++ b/src/SourceBuild/tarball/content/eng/pipelines/ci.yml @@ -0,0 +1,52 @@ +# This is the main build definition (PR+CI) for dotnet/dotnet + +trigger: + batch: true + branches: + include: + - main + - release/* + - internal/release/* + +pr: + branches: + include: + - main + - release/* + - internal/release/* + +stages: +# For rolling builds we want to build the MSFT SDK first so that we can +# compare the contents with the source-built one later. +# This only works because we don't run this test in PRs. If we decided +# to run this test in PRs, we would need to build the MSFT SDK there too. +- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - stage: Build_MSFT_SDK + displayName: Build MSFT SDK + jobs: + - template: /src/installer/eng/build.yml + parameters: + agentOs: Linux + jobName: Build_LinuxPortable_Release_x64 + buildConfiguration: Release + buildArchitecture: x64 + linuxPortable: true + runTests: false + isBuiltFromVmr: true + - template: /src/installer/eng/build.yml + parameters: + agentOs: Linux + jobName: Build_Arm64_Release + buildConfiguration: Release + buildArchitecture: arm64 + runtimeIdentifier: 'linux-arm64' + linuxPortable: true + runTests: false + isBuiltFromVmr: true + +- template: /src/installer/eng/pipelines/templates/stages/vmr-build.yml + parameters: + isBuiltFromVmr: true + ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + dependsOn: Build_MSFT_SDK + condition: always()