Run VMR's pipelines from the VMR directly (#15124)

2022-12-15 09:33:09 +01:00 · 2022-12-15 09:33:09 +01:00 · ce1921c17d
commit ce1921c17d
parent dd0bce7b3f
8 changed files with 130 additions and 91 deletions
--- a/.vsts-ci.yml
+++ b/.vsts-ci.yml
@ -377,7 +377,7 @@ stages:
  - template: eng/pipelines/templates/stages/vmr-build.yml
    parameters:
      vmrBranch: ${{ parameters.vmrBranch }}
-      installerBuildResourceId: current
+      isBuiltFromVmr: false

 - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
  - stage: Publish
--- a/eng/build.yml
+++ b/eng/build.yml
@ -63,6 +63,11 @@ parameters:
  type: boolean
  default: false

+- name: isBuiltFromVmr
+  displayName: True when build is running from dotnet/dotnet
+  type: boolean
+  default: false
+
 jobs:
 - template: common/templates/job/job.yml
  parameters:
@ -96,9 +101,12 @@ jobs:
        vmImage: 'macOS-latest'

    timeoutInMinutes: ${{ parameters.timeoutInMinutes }}
-    enablePublishBuildAssets: true
-    enablePublishUsingPipelines: true
-    enableTelemetry: true
+    ${{ if parameters.isBuiltFromVmr }}:
+      enableSbom: false
+    ${{ else }}:
+      enablePublishBuildAssets: true
+      enablePublishUsingPipelines: true
+      enableTelemetry: true
    helixRepo: dotnet/installer
    workspace:
      clean: all
@ -139,29 +147,40 @@ jobs:
    - _SignType: test
    - _BuildArgs: '/p:DotNetSignType=$(_SignType) $(_PgoInstrument)'

-    - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
-      - group: DotNet-HelixApi-Access
-      - _PushToVSFeed: true
-      - _SignType: real
+    - ${{ if parameters.isBuiltFromVmr }}:
+      - installerRoot: '$(Build.SourcesDirectory)/src/installer'
+      - _SignType: test
+      - _PushToVSFeed: false
      - _BuildArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER)
-                /p:DotNetSignType=$(_SignType)
                /p:TeamName=$(_TeamName)
-                /p:DotNetPublishUsingPipelines=$(_PublishUsingPipelines)
+                /p:DotNetPublishUsingPipelines=true
+                /p:PublishToSymbolServer=false
                $(_PgoInstrument)
+    - ${{ else }}:
+      - installerRoot: '$(Build.SourcesDirectory)'
+      - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+        - group: DotNet-HelixApi-Access
+        - _PushToVSFeed: true
+        - _SignType: real
+        - _BuildArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER)
+                  /p:DotNetSignType=$(_SignType)
+                  /p:TeamName=$(_TeamName)
+                  /p:DotNetPublishUsingPipelines=$(_PublishUsingPipelines)
+                  $(_PgoInstrument)

    steps:
    - checkout: self
      clean: true
    - ${{ if eq(parameters.agentOs, 'Windows_NT') }}:
-      - ${{ if ne(variables['System.TeamProject'], 'public') }}:
+      - ${{ if and(not(parameters.isBuiltFromVmr), ne(variables['System.TeamProject'], 'public')) }}:
        - task: PowerShell@2
          displayName: Setup Private Feeds Credentials
          inputs:
-            filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1
-            arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token
+            filePath: $(installerRoot)/eng/common/SetupNugetSources.ps1
+            arguments: -ConfigFile $(installerRoot)/NuGet.config -Password $Env:Token
          env:
            Token: $(dn-bot-dnceng-artifact-feeds-rw)
-      - script: build.cmd
+      - script: $(installerRoot)/build.cmd
                  $(_TestArg) $(_PackArg)
                  -publish -ci -sign
                  -Configuration ${{ parameters.buildConfiguration }}
@ -174,16 +193,16 @@ jobs:
          DOTNET_CLI_UI_LANGUAGE: ${{ parameters.dotnetCLIUILanguage }}

    - ${{ if ne(parameters.agentOs, 'Windows_NT') }}:
-      - ${{ if ne(variables['System.TeamProject'], 'public') }}:
+      - ${{ if and(not(parameters.isBuiltFromVmr), ne(variables['System.TeamProject'], 'public')) }}:
        - task: Bash@3
          displayName: Setup Private Feeds Credentials
          inputs:
-            filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh
-            arguments: $(Build.SourcesDirectory)/NuGet.config $Token
+            filePath: $(installerRoot)/eng/common/SetupNugetSources.sh
+            arguments: $(installerRoot)/NuGet.config $Token
          env:
            Token: $(dn-bot-dnceng-artifact-feeds-rw)
    - ${{ if eq(parameters.agentOs, 'Linux') }}:
-      - script: ./build.sh
+      - script: $(installerRoot)/build.sh
                  $(_TestArg) $(_PackArg)
                  --publish --ci
                  --noprettyprint
@ -197,7 +216,7 @@ jobs:
        displayName: Build

    - ${{ if or(eq(parameters.agentOs, 'Darwin'), eq(parameters.agentOs, 'FreeBSD')) }}:
-      - script: ./build.sh
+      - script: $(installerRoot)/build.sh
                  $(_TestArg)
                  --pack --publish --ci
                  --noprettyprint
@ -222,7 +241,7 @@ jobs:
    - task: CopyFiles@2
      displayName: Gather Logs
      inputs:
-        SourceFolder: '$(Build.SourcesDirectory)/artifacts'
+        SourceFolder: '$(installerRoot)/artifacts'
        Contents: |
         log/${{ parameters.buildConfiguration }}/**/*
         TestResults/${{ parameters.buildConfiguration }}/**/*
--- a/eng/pipelines/templates/jobs/vmr-build.yml
+++ b/eng/pipelines/templates/jobs/vmr-build.yml
@ -2,6 +2,9 @@
 ### If run in a PR, new changes are applied to a local copy of the VMR, then it is source-built and tested

 parameters:
+- name: isBuiltFromVmr
+  displayName: True when build is running from dotnet/dotnet directly
+  type: boolean

 - name: vmrPath
  type: string
@ -12,10 +15,6 @@ parameters:
  type: string
  default: $(Build.SourceBranchName)

-# Resource id of the installer build to retrieve MSFT SDK from
- name: installerBuildResourceId
-  type: string
-
 - name: buildName
  type: string

@ -63,7 +62,7 @@ jobs:
    dependsOn: ${{ parameters.reuseBuildArtifactsFrom }}_${{ parameters.architecture }}
  variables:
  - template: /eng/common/templates/variables/pool-providers.yml
-  - ${{ if eq(variables['System.TeamProject'], 'internal') }}:
+  - ${{ if and(not(parameters.isBuiltFromVmr), eq(variables['System.TeamProject'], 'internal')) }}:
    - group: DotNetBot-GitHub
    - group: AzureDevOps-Artifact-Feeds-Pats
  - ${{ else }}:
@ -80,11 +79,11 @@ jobs:
  - template: ../steps/vmr-prepare.yml
    parameters:
      vmrBranch: ${{ parameters.vmrBranch }}
-      installerBuildResourceId: ${{ parameters.installerBuildResourceId }}
+      isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }}
      skipComponentGovernanceDetection: true

  # Synchronize new content in the VMR during PRs (we expect this to come
-  - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
+  - ${{ if and(not(parameters.isBuiltFromVmr), eq(variables['Build.Reason'], 'PullRequest')) }}:
    - template: ../steps/vmr-pull-updates.yml
      parameters:
        vmrPath: ${{ parameters.vmrPath }}
@ -104,7 +103,7 @@ jobs:
        Token: $(dn-bot-dnceng-artifact-feeds-rw)

  - ${{ if ne(parameters.excludeSdkContentTests, 'true') }}:
-    - download: ${{ parameters.installerBuildResourceId }}
+    - download: current
      artifact: BlobArtifacts
      patterns: '**/dotnet-sdk-+([0-9]).+([0-9]).+([0-9])?(-@(alpha|preview|rc|rtm)*)-linux-${{ parameters.architecture }}.tar.gz'
      displayName: Download MSFT SDK
@ -167,8 +166,8 @@ jobs:
      poisonArg=''

      if [[ '${{ parameters.excludeSdkContentTests }}' != 'True' ]]; then
-        dockerVolumeArgs+=" -v $(Pipeline.Workspace)/${{ parameters.installerBuildResourceId }}/BlobArtifacts/:/BlobArtifacts"
-        msftSdkTarballName=$(find "$(Pipeline.Workspace)/${{ parameters.installerBuildResourceId }}/BlobArtifacts/" -name "dotnet-sdk-*-linux-${{ parameters.architecture }}.tar.gz" -exec basename {} \;)
+        dockerVolumeArgs+=" -v $(Pipeline.Workspace)/BlobArtifacts/:/BlobArtifacts"
+        msftSdkTarballName=$(find "$(Pipeline.Workspace)/BlobArtifacts/" -name "dotnet-sdk-*-linux-${{ parameters.architecture }}.tar.gz" -exec basename {} \;)
        dockerEnvArgs+=" -e SMOKE_TESTS_MSFT_SDK_TARBALL_PATH=/BlobArtifacts/$msftSdkTarballName"
      fi

--- a/eng/pipelines/templates/jobs/vmr-synchronization.yml
+++ b/eng/pipelines/templates/jobs/vmr-synchronization.yml
@ -43,6 +43,7 @@ jobs:
  - template: ../steps/vmr-prepare.yml
    parameters:
      vmrBranch: ${{ parameters.vmrBranch }}
+      isBuiltFromVmr: false

  - template: ../steps/vmr-pull-updates.yml
    parameters:
@ -60,14 +61,5 @@ jobs:
        git branch ${{ parameters.vmrBranch }}
        git branch --set-upstream-to=dotnet/${{ parameters.vmrBranch }} ${{ parameters.vmrBranch }} || echo 'Branch ${{ parameters.vmrBranch }} not found in remote'
        git push dotnet ${{ parameters.vmrBranch }}
-
-        # Export the SHA as an artifact to be reused by the official VMR build
-        sha=`git rev-parse HEAD`
-        mkdir -p "$(Build.ArtifactStagingDirectory)/VmrRevision"
-        echo $sha > "$(Build.ArtifactStagingDirectory)/VmrRevision/VmrRevision.txt"
      displayName: Push changes to dotnet/dotnet
      workingDirectory: $(vmrPath)
-
-    - publish: $(Build.ArtifactStagingDirectory)/VmrRevision
-      artifact: VmrRevision
-      displayName: Publish new SHA
--- a/eng/pipelines/templates/stages/vmr-build.yml
+++ b/eng/pipelines/templates/stages/vmr-build.yml
@ -2,12 +2,15 @@
 ### If run in a PR, new changes are applied to a local copy of the VMR, then it is source-built and tested

 parameters:
-  # Resource id of the installer build to retrieve source/product tarball's from
-  installerBuildResourceId: current
+  dependsOn: []
+  condition: always()

  # Branch of the VMR to use (to push to for internal builds)
  vmrBranch: $(Build.SourceBranchName)

+  # True when build is running from dotnet/dotnet directly
+  isBuiltFromVmr:
+
  # The following parameters aren't expected to be passed in rather they are used for encapsulation
  # -----------------------------------------------------------------------------------------------
  centOSStream8Container: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8
@ -34,7 +37,8 @@ parameters:
 stages:
 - stage: VMR_Source_Build
  displayName: VMR Source-Build
-  dependsOn: []
+  dependsOn: ${{ parameters.dependsOn }}
+  condition: ${{ parameters.condition }}
  variables:
  - ${{ if eq(variables['System.TeamProject'], 'public') }}:
    - name: defaultPoolName
@ -57,7 +61,7 @@ stages:
  - template: ../jobs/vmr-build.yml
    parameters:
      buildName: CentOSStream8_Online
-      installerBuildResourceId: ${{ parameters.installerBuildResourceId }}
+      isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }}
      vmrBranch: ${{ parameters.vmrBranch }}
      architecture: x64
      pool:
@ -74,7 +78,7 @@ stages:
    - template: ../jobs/vmr-build.yml
      parameters:
        buildName: CentOSStream8_Offline
-        installerBuildResourceId: ${{ parameters.installerBuildResourceId }}
+        isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }}
        vmrBranch: ${{ parameters.vmrBranch }}
        architecture: x64
        pool:
@ -90,7 +94,7 @@ stages:
    - template: ../jobs/vmr-build.yml
      parameters:
        buildName: CentOSStream9_Offline
-        installerBuildResourceId: ${{ parameters.installerBuildResourceId }}
+        isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }}
        vmrBranch: ${{ parameters.vmrBranch }}
        architecture: x64
        pool:
@ -106,7 +110,7 @@ stages:
    - template: ../jobs/vmr-build.yml
      parameters:
        buildName: Fedora36_Offline
-        installerBuildResourceId: ${{ parameters.installerBuildResourceId }}
+        isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }}
        vmrBranch: ${{ parameters.vmrBranch }}
        architecture: x64
        pool:
@ -122,7 +126,7 @@ stages:
    - template: ../jobs/vmr-build.yml
      parameters:
        buildName: Ubuntu2004_Offline
-        installerBuildResourceId: ${{ parameters.installerBuildResourceId }}
+        isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }}
        vmrBranch: ${{ parameters.vmrBranch }}
        architecture: x64
        pool:
@ -138,7 +142,7 @@ stages:
    - template: ../jobs/vmr-build.yml
      parameters:
        buildName: Debian11_Offline
-        installerBuildResourceId: ${{ parameters.installerBuildResourceId }}
+        isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }}
        vmrBranch: ${{ parameters.vmrBranch }}
        architecture: arm64
        pool: ${{ parameters.poolInternalArm64 }}
@ -152,7 +156,7 @@ stages:
    - template: ../jobs/vmr-build.yml
      parameters:
        buildName: Fedora36_Offline_Using_Previous
-        installerBuildResourceId: ${{ parameters.installerBuildResourceId }}
+        isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }}
        vmrBranch: ${{ parameters.vmrBranch }}
        architecture: x64
        pool:
--- a/eng/pipelines/templates/steps/vmr-prepare.yml
+++ b/eng/pipelines/templates/steps/vmr-prepare.yml
@ -2,39 +2,33 @@
 ### Component Governance scan is also triggered over the VMR's non-repo sources

 parameters:
+- name: isBuiltFromVmr
+  displayName: True when build is running from dotnet/dotnet directly
+  type: boolean
+
 - name: vmrBranch
  displayName: dotnet/dotnet branch to use
  type: string
  default: $(Build.SourceBranchName)

- name: installerBuildResourceId
-  displayName: Installer build to get the commit hash to check out from
-  type: string
-  default: current
-
 - name: skipComponentGovernanceDetection
  type: boolean
  default: false

 steps:
- checkout: vmr
-  displayName: Clone dotnet/dotnet
-  path: vmr
-  clean: true
-  fetchDepth: 0 # We need this so that we can check out the new commit
+- ${{ if parameters.isBuiltFromVmr }}:
+  - checkout: self
+    displayName: Clone dotnet/dotnet
+    path: vmr
+    clean: true

- ${{ if ne(parameters.installerBuildResourceId, 'current') }}:
-  - download: ${{ parameters.installerBuildResourceId }}
-    artifact: VmrRevision
-    displayName: Download VmrRevision
-
-  - script: |
-      set -ex
-      sha=`head -n 1 $(Pipeline.Workspace)/${{ parameters.installerBuildResourceId }}/VmrRevision/VmrRevision.txt`
-      git checkout $sha
-    displayName: Check out dotnet/dotnet commit
-    workingDirectory: $(Agent.BuildDirectory)/vmr
 - ${{ else }}:
+  - checkout: vmr
+    displayName: Clone dotnet/dotnet
+    path: vmr
+    clean: true
+
+- ${{ if or(not(parameters.isBuiltFromVmr), eq(variables['System.TeamProject'], 'internal')) }}:
  - script: |
      git switch -c ${{ parameters.vmrBranch }}
    displayName: Checkout ${{ parameters.vmrBranch }}
--- a/eng/pipelines/vmr-ci.yml
+++ b/eng/pipelines/vmr-ci.yml
@ -1,21 +0,0 @@
-trigger: none
-pr: none
-resources:
-  pipelines:
-  - pipeline: installer-build-resource
-    source: dotnet-installer-official-ci
-    trigger:
-      branches:
-        include:
-        - main
-
-  repositories:
-  - repository: vmr
-    type: github
-    name: dotnet/dotnet
-    endpoint: dotnet
-
-stages:
- template: templates/stages/vmr-build.yml
-  parameters:
-    installerBuildResourceId: installer-build-resource
--- a/src/SourceBuild/tarball/content/eng/pipelines/ci.yml
+++ b/src/SourceBuild/tarball/content/eng/pipelines/ci.yml
@ -0,0 +1,52 @@
+# This is the main build definition (PR+CI) for dotnet/dotnet
+
+trigger:
+  batch: true
+  branches:
+    include:
+    - main
+    - release/*
+    - internal/release/*
+
+pr:
+  branches:
+    include:
+    - main
+    - release/*
+    - internal/release/*
+
+stages:
+# For rolling builds we want to build the MSFT SDK first so that we can
+# compare the contents with the source-built one later.
+# This only works because we don't run this test in PRs. If we decided
+# to run this test in PRs, we would need to build the MSFT SDK there too.
+- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+  - stage: Build_MSFT_SDK
+    displayName: Build MSFT SDK
+    jobs:
+    - template: /src/installer/eng/build.yml
+      parameters:
+        agentOs: Linux
+        jobName: Build_LinuxPortable_Release_x64
+        buildConfiguration: Release
+        buildArchitecture: x64
+        linuxPortable: true
+        runTests: false
+        isBuiltFromVmr: true
+    - template: /src/installer/eng/build.yml
+      parameters:
+        agentOs: Linux
+        jobName: Build_Arm64_Release
+        buildConfiguration: Release
+        buildArchitecture: arm64
+        runtimeIdentifier: 'linux-arm64'
+        linuxPortable: true
+        runTests: false
+        isBuiltFromVmr: true
+
+- template: /src/installer/eng/pipelines/templates/stages/vmr-build.yml
+  parameters:
+    isBuiltFromVmr: true
+    ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+      dependsOn: Build_MSFT_SDK
+      condition: always()