Run VMR's pipelines from the VMR directly (#15124)

.vsts-ci.yml

@@ -377,7 +377,7 @@ stages:
   - template: eng/pipelines/templates/stages/vmr-build.yml
     parameters:
       vmrBranch: ${{ parameters.vmrBranch }}
-      installerBuildResourceId: current
+      isBuiltFromVmr: false
 
 - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
   - stage: Publish

eng/build.yml

@@ -63,6 +63,11 @@ parameters:
   type: boolean
   default: false
 
+- name: isBuiltFromVmr
+  displayName: True when build is running from dotnet/dotnet
+  type: boolean
+  default: false
+
 jobs:
 - template: common/templates/job/job.yml
   parameters:
@@ -96,6 +101,9 @@ jobs:
         vmImage: 'macOS-latest'
 
     timeoutInMinutes: ${{ parameters.timeoutInMinutes }}
+    ${{ if parameters.isBuiltFromVmr }}:
+      enableSbom: false
+    ${{ else }}:
       enablePublishBuildAssets: true
       enablePublishUsingPipelines: true
       enableTelemetry: true
@@ -139,6 +147,17 @@ jobs:
     - _SignType: test
     - _BuildArgs: '/p:DotNetSignType=$(_SignType) $(_PgoInstrument)'
 
+    - ${{ if parameters.isBuiltFromVmr }}:
+      - installerRoot: '$(Build.SourcesDirectory)/src/installer'
+      - _SignType: test
+      - _PushToVSFeed: false
+      - _BuildArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER)
+                /p:TeamName=$(_TeamName)
+                /p:DotNetPublishUsingPipelines=true
+                /p:PublishToSymbolServer=false
+                $(_PgoInstrument)
+    - ${{ else }}:
+      - installerRoot: '$(Build.SourcesDirectory)'
       - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
         - group: DotNet-HelixApi-Access
         - _PushToVSFeed: true
@@ -153,15 +172,15 @@ jobs:
     - checkout: self
       clean: true
     - ${{ if eq(parameters.agentOs, 'Windows_NT') }}:
-      - ${{ if ne(variables['System.TeamProject'], 'public') }}:
+      - ${{ if and(not(parameters.isBuiltFromVmr), ne(variables['System.TeamProject'], 'public')) }}:
         - task: PowerShell@2
           displayName: Setup Private Feeds Credentials
           inputs:
-            filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1
+            filePath: $(installerRoot)/eng/common/SetupNugetSources.ps1
-            arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token
+            arguments: -ConfigFile $(installerRoot)/NuGet.config -Password $Env:Token
           env:
             Token: $(dn-bot-dnceng-artifact-feeds-rw)
-      - script: build.cmd
+      - script: $(installerRoot)/build.cmd
                   $(_TestArg) $(_PackArg)
                   -publish -ci -sign
                   -Configuration ${{ parameters.buildConfiguration }}
@@ -174,16 +193,16 @@ jobs:
           DOTNET_CLI_UI_LANGUAGE: ${{ parameters.dotnetCLIUILanguage }}
 
     - ${{ if ne(parameters.agentOs, 'Windows_NT') }}:
-      - ${{ if ne(variables['System.TeamProject'], 'public') }}:
+      - ${{ if and(not(parameters.isBuiltFromVmr), ne(variables['System.TeamProject'], 'public')) }}:
         - task: Bash@3
           displayName: Setup Private Feeds Credentials
           inputs:
-            filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh
+            filePath: $(installerRoot)/eng/common/SetupNugetSources.sh
-            arguments: $(Build.SourcesDirectory)/NuGet.config $Token
+            arguments: $(installerRoot)/NuGet.config $Token
           env:
             Token: $(dn-bot-dnceng-artifact-feeds-rw)
     - ${{ if eq(parameters.agentOs, 'Linux') }}:
-      - script: ./build.sh
+      - script: $(installerRoot)/build.sh
                   $(_TestArg) $(_PackArg)
                   --publish --ci
                   --noprettyprint
@@ -197,7 +216,7 @@ jobs:
         displayName: Build
 
     - ${{ if or(eq(parameters.agentOs, 'Darwin'), eq(parameters.agentOs, 'FreeBSD')) }}:
-      - script: ./build.sh
+      - script: $(installerRoot)/build.sh
                   $(_TestArg)
                   --pack --publish --ci
                   --noprettyprint
@@ -222,7 +241,7 @@ jobs:
     - task: CopyFiles@2
       displayName: Gather Logs
       inputs:
-        SourceFolder: '$(Build.SourcesDirectory)/artifacts'
+        SourceFolder: '$(installerRoot)/artifacts'
         Contents: |
          log/${{ parameters.buildConfiguration }}/**/*
          TestResults/${{ parameters.buildConfiguration }}/**/*

eng/pipelines/templates/jobs/vmr-build.yml

@@ -2,6 +2,9 @@
 ### If run in a PR, new changes are applied to a local copy of the VMR, then it is source-built and tested
 
 parameters:
+- name: isBuiltFromVmr
+  displayName: True when build is running from dotnet/dotnet directly
+  type: boolean
 
 - name: vmrPath
   type: string
@@ -12,10 +15,6 @@ parameters:
   type: string
   default: $(Build.SourceBranchName)
 
-# Resource id of the installer build to retrieve MSFT SDK from
-- name: installerBuildResourceId
-  type: string
-
 - name: buildName
   type: string
 
@@ -63,7 +62,7 @@ jobs:
     dependsOn: ${{ parameters.reuseBuildArtifactsFrom }}_${{ parameters.architecture }}
   variables:
   - template: /eng/common/templates/variables/pool-providers.yml
-  - ${{ if eq(variables['System.TeamProject'], 'internal') }}:
+  - ${{ if and(not(parameters.isBuiltFromVmr), eq(variables['System.TeamProject'], 'internal')) }}:
     - group: DotNetBot-GitHub
     - group: AzureDevOps-Artifact-Feeds-Pats
   - ${{ else }}:
@@ -80,11 +79,11 @@ jobs:
   - template: ../steps/vmr-prepare.yml
     parameters:
       vmrBranch: ${{ parameters.vmrBranch }}
-      installerBuildResourceId: ${{ parameters.installerBuildResourceId }}
+      isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }}
       skipComponentGovernanceDetection: true
 
   # Synchronize new content in the VMR during PRs (we expect this to come
-  - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
+  - ${{ if and(not(parameters.isBuiltFromVmr), eq(variables['Build.Reason'], 'PullRequest')) }}:
     - template: ../steps/vmr-pull-updates.yml
       parameters:
         vmrPath: ${{ parameters.vmrPath }}
@@ -104,7 +103,7 @@ jobs:
         Token: $(dn-bot-dnceng-artifact-feeds-rw)
 
   - ${{ if ne(parameters.excludeSdkContentTests, 'true') }}:
-    - download: ${{ parameters.installerBuildResourceId }}
+    - download: current
       artifact: BlobArtifacts
       patterns: '**/dotnet-sdk-+([0-9]).+([0-9]).+([0-9])?(-@(alpha|preview|rc|rtm)*)-linux-${{ parameters.architecture }}.tar.gz'
       displayName: Download MSFT SDK
@@ -167,8 +166,8 @@ jobs:
       poisonArg=''
 
       if [[ '${{ parameters.excludeSdkContentTests }}' != 'True' ]]; then
-        dockerVolumeArgs+=" -v $(Pipeline.Workspace)/${{ parameters.installerBuildResourceId }}/BlobArtifacts/:/BlobArtifacts"
+        dockerVolumeArgs+=" -v $(Pipeline.Workspace)/BlobArtifacts/:/BlobArtifacts"
-        msftSdkTarballName=$(find "$(Pipeline.Workspace)/${{ parameters.installerBuildResourceId }}/BlobArtifacts/" -name "dotnet-sdk-*-linux-${{ parameters.architecture }}.tar.gz" -exec basename {} \;)
+        msftSdkTarballName=$(find "$(Pipeline.Workspace)/BlobArtifacts/" -name "dotnet-sdk-*-linux-${{ parameters.architecture }}.tar.gz" -exec basename {} \;)
         dockerEnvArgs+=" -e SMOKE_TESTS_MSFT_SDK_TARBALL_PATH=/BlobArtifacts/$msftSdkTarballName"
       fi
 

eng/pipelines/templates/jobs/vmr-synchronization.yml

@@ -43,6 +43,7 @@ jobs:
   - template: ../steps/vmr-prepare.yml
     parameters:
       vmrBranch: ${{ parameters.vmrBranch }}
+      isBuiltFromVmr: false
 
   - template: ../steps/vmr-pull-updates.yml
     parameters:
@@ -60,14 +61,5 @@ jobs:
         git branch ${{ parameters.vmrBranch }}
         git branch --set-upstream-to=dotnet/${{ parameters.vmrBranch }} ${{ parameters.vmrBranch }} || echo 'Branch ${{ parameters.vmrBranch }} not found in remote'
         git push dotnet ${{ parameters.vmrBranch }}
-
-        # Export the SHA as an artifact to be reused by the official VMR build
-        sha=`git rev-parse HEAD`
-        mkdir -p "$(Build.ArtifactStagingDirectory)/VmrRevision"
-        echo $sha > "$(Build.ArtifactStagingDirectory)/VmrRevision/VmrRevision.txt"
       displayName: Push changes to dotnet/dotnet
       workingDirectory: $(vmrPath)
-
-    - publish: $(Build.ArtifactStagingDirectory)/VmrRevision
-      artifact: VmrRevision
-      displayName: Publish new SHA

eng/pipelines/templates/stages/vmr-build.yml

@@ -2,12 +2,15 @@
 ### If run in a PR, new changes are applied to a local copy of the VMR, then it is source-built and tested
 
 parameters:
-  # Resource id of the installer build to retrieve source/product tarball's from
+  dependsOn: []
-  installerBuildResourceId: current
+  condition: always()
 
   # Branch of the VMR to use (to push to for internal builds)
   vmrBranch: $(Build.SourceBranchName)
 
+  # True when build is running from dotnet/dotnet directly
+  isBuiltFromVmr:
+
   # The following parameters aren't expected to be passed in rather they are used for encapsulation
   # -----------------------------------------------------------------------------------------------
   centOSStream8Container: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8
@@ -34,7 +37,8 @@ parameters:
 stages:
 - stage: VMR_Source_Build
   displayName: VMR Source-Build
-  dependsOn: []
+  dependsOn: ${{ parameters.dependsOn }}
+  condition: ${{ parameters.condition }}
   variables:
   - ${{ if eq(variables['System.TeamProject'], 'public') }}:
     - name: defaultPoolName
@@ -57,7 +61,7 @@ stages:
   - template: ../jobs/vmr-build.yml
     parameters:
       buildName: CentOSStream8_Online
-      installerBuildResourceId: ${{ parameters.installerBuildResourceId }}
+      isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }}
       vmrBranch: ${{ parameters.vmrBranch }}
       architecture: x64
       pool:
@@ -74,7 +78,7 @@ stages:
     - template: ../jobs/vmr-build.yml
       parameters:
         buildName: CentOSStream8_Offline
-        installerBuildResourceId: ${{ parameters.installerBuildResourceId }}
+        isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }}
         vmrBranch: ${{ parameters.vmrBranch }}
         architecture: x64
         pool:
@@ -90,7 +94,7 @@ stages:
     - template: ../jobs/vmr-build.yml
       parameters:
         buildName: CentOSStream9_Offline
-        installerBuildResourceId: ${{ parameters.installerBuildResourceId }}
+        isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }}
         vmrBranch: ${{ parameters.vmrBranch }}
         architecture: x64
         pool:
@@ -106,7 +110,7 @@ stages:
     - template: ../jobs/vmr-build.yml
       parameters:
         buildName: Fedora36_Offline
-        installerBuildResourceId: ${{ parameters.installerBuildResourceId }}
+        isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }}
         vmrBranch: ${{ parameters.vmrBranch }}
         architecture: x64
         pool:
@@ -122,7 +126,7 @@ stages:
     - template: ../jobs/vmr-build.yml
       parameters:
         buildName: Ubuntu2004_Offline
-        installerBuildResourceId: ${{ parameters.installerBuildResourceId }}
+        isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }}
         vmrBranch: ${{ parameters.vmrBranch }}
         architecture: x64
         pool:
@@ -138,7 +142,7 @@ stages:
     - template: ../jobs/vmr-build.yml
       parameters:
         buildName: Debian11_Offline
-        installerBuildResourceId: ${{ parameters.installerBuildResourceId }}
+        isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }}
         vmrBranch: ${{ parameters.vmrBranch }}
         architecture: arm64
         pool: ${{ parameters.poolInternalArm64 }}
@@ -152,7 +156,7 @@ stages:
     - template: ../jobs/vmr-build.yml
       parameters:
         buildName: Fedora36_Offline_Using_Previous
-        installerBuildResourceId: ${{ parameters.installerBuildResourceId }}
+        isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }}
         vmrBranch: ${{ parameters.vmrBranch }}
         architecture: x64
         pool:

eng/pipelines/templates/steps/vmr-prepare.yml

@@ -2,39 +2,33 @@
 ### Component Governance scan is also triggered over the VMR's non-repo sources
 
 parameters:
+- name: isBuiltFromVmr
+  displayName: True when build is running from dotnet/dotnet directly
+  type: boolean
+
 - name: vmrBranch
   displayName: dotnet/dotnet branch to use
   type: string
   default: $(Build.SourceBranchName)
 
-- name: installerBuildResourceId
-  displayName: Installer build to get the commit hash to check out from
-  type: string
-  default: current
-
 - name: skipComponentGovernanceDetection
   type: boolean
   default: false
 
 steps:
+- ${{ if parameters.isBuiltFromVmr }}:
+  - checkout: self
+    displayName: Clone dotnet/dotnet
+    path: vmr
+    clean: true
+
+- ${{ else }}:
   - checkout: vmr
     displayName: Clone dotnet/dotnet
     path: vmr
     clean: true
-  fetchDepth: 0 # We need this so that we can check out the new commit
 
-- ${{ if ne(parameters.installerBuildResourceId, 'current') }}:
+- ${{ if or(not(parameters.isBuiltFromVmr), eq(variables['System.TeamProject'], 'internal')) }}:
-  - download: ${{ parameters.installerBuildResourceId }}
-    artifact: VmrRevision
-    displayName: Download VmrRevision
-
-  - script: |
-      set -ex
-      sha=`head -n 1 $(Pipeline.Workspace)/${{ parameters.installerBuildResourceId }}/VmrRevision/VmrRevision.txt`
-      git checkout $sha
-    displayName: Check out dotnet/dotnet commit
-    workingDirectory: $(Agent.BuildDirectory)/vmr
-- ${{ else }}:
   - script: |
       git switch -c ${{ parameters.vmrBranch }}
     displayName: Checkout ${{ parameters.vmrBranch }}

eng/pipelines/vmr-ci.yml

@@ -1,21 +0,0 @@
-trigger: none
-pr: none
-resources:
-  pipelines:
-  - pipeline: installer-build-resource
-    source: dotnet-installer-official-ci
-    trigger:
-      branches:
-        include:
-        - main
-
-  repositories:
-  - repository: vmr
-    type: github
-    name: dotnet/dotnet
-    endpoint: dotnet
-
-stages:
-- template: templates/stages/vmr-build.yml
-  parameters:
-    installerBuildResourceId: installer-build-resource

src/SourceBuild/tarball/content/eng/pipelines/ci.yml

@@ -0,0 +1,52 @@
+# This is the main build definition (PR+CI) for dotnet/dotnet
+
+trigger:
+  batch: true
+  branches:
+    include:
+    - main
+    - release/*
+    - internal/release/*
+
+pr:
+  branches:
+    include:
+    - main
+    - release/*
+    - internal/release/*
+
+stages:
+# For rolling builds we want to build the MSFT SDK first so that we can
+# compare the contents with the source-built one later.
+# This only works because we don't run this test in PRs. If we decided
+# to run this test in PRs, we would need to build the MSFT SDK there too.
+- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+  - stage: Build_MSFT_SDK
+    displayName: Build MSFT SDK
+    jobs:
+    - template: /src/installer/eng/build.yml
+      parameters:
+        agentOs: Linux
+        jobName: Build_LinuxPortable_Release_x64
+        buildConfiguration: Release
+        buildArchitecture: x64
+        linuxPortable: true
+        runTests: false
+        isBuiltFromVmr: true
+    - template: /src/installer/eng/build.yml
+      parameters:
+        agentOs: Linux
+        jobName: Build_Arm64_Release
+        buildConfiguration: Release
+        buildArchitecture: arm64
+        runtimeIdentifier: 'linux-arm64'
+        linuxPortable: true
+        runTests: false
+        isBuiltFromVmr: true
+
+- template: /src/installer/eng/pipelines/templates/stages/vmr-build.yml
+  parameters:
+    isBuiltFromVmr: true
+    ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+      dependsOn: Build_MSFT_SDK
+      condition: always()