Add a VMR synchronization PR validation (#14918)

Adds a new pipeline that will run during installer's PRs that verifies that the current VMR tooling can synchronize that particular change into the VMR.

eng/pipelines/templates/jobs/vmr-synchronization.yml

@@ -1,103 +0,0 @@
-### These steps synchronize code from product repositories into the VMR (https://github.com/dotnet/dotnet)
-
-parameters:
-- name: targetRef
-  displayName: Target revision in dotnet/installer to synchronize
-  type: string
-  default: $(Build.SourceVersion)
-
-- name: vmrBranch
-  displayName: dotnet/dotnet branch to use
-  type: string
-  default: $(Build.SourceBranchName)
-
-- name: vmrToken
-  displayName: PAT that allows pushing to dotnet/dotnet
-  type: string
-
-- name: noPush
-  displayName: Don't push changes to dotnet/dotnet
-  type: boolean
-  default: false
-
-jobs:
-- job: Synchronize
-  displayName: Synchronize dotnet/dotnet
-  timeoutInMinutes: 120
-
-  variables:
-  - name: vmrPath
-    value: $(Agent.BuildDirectory)/vmr
-
-  pool:
-    ${{ if eq(variables['System.TeamProject'], 'public') }}:
-      name: NetCore-Public
-      demands:
-      - ImageOverride -equals Build.Ubuntu.2004.Amd64.Open
-    ${{ else }}:
-      name: NetCore1ESPool-Internal
-      demands:
-      - ImageOverride -equals Build.Ubuntu.2004.Amd64
-
-  steps:
-  - checkout: self
-    clean: true
-    displayName: Checkout dotnet/installer
-    fetchDepth: 0
-    path: installer
-
-  - checkout: vmr
-    clean: true
-    displayName: Checkout dotnet/dotnet
-    fetchDepth: 0
-    path: vmr
-
-  # TODO (https://github.com/dotnet/arcade/issues/11332): Allow full CG?
-  # Currently, we ignore dirs of individual repos - they have been scanned before
-  - ${{ if and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['System.TeamProject'], 'internal')) }}:
-    - task: ComponentGovernanceComponentDetection@0
-      inputs:
-        sourceScanPath: $(vmrPath)
-        ignoreDirectories: $(vmrPath)/src
-
-  - script: |-
-      source ./eng/common/tools.sh
-      InitializeDotNetCli true
-      ./.dotnet/dotnet tool restore
-    displayName: Restore toolset
-    workingDirectory: $(Agent.BuildDirectory)/installer
-
-  # TODO (https://github.com/dotnet/arcade/issues/11386): Remove this step
-  # This step is here so that darc can find the PR commit (which might be in some fork)
-  # We need to make darc understand that it needs to look in the fork from this PR
-  - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
-    - script: |-
-        cp -r $(Agent.BuildDirectory)/installer $(Agent.TempDirectory)/installer
-      displayName: '[PR Only] Prepare dotnet/installer clone'
-
-  - script: git checkout -B ${{ parameters.vmrBranch }}
-    displayName: Prepare branch ${{ parameters.vmrBranch }}
-    workingDirectory: $(vmrPath)
-
-  - script: >
-      $(Agent.BuildDirectory)/installer/.dotnet/dotnet darc vmr update
-      --vmr $(vmrPath)
-      --tmp $(Agent.TempDirectory)
-      --azdev-pat $(System.AccessToken)
-      --github-pat ${{ parameters.vmrToken }}
-      --recursive
-      --verbose
-      installer:${{ parameters.targetRef }}
-    displayName: Synchronize dotnet/dotnet
-    workingDirectory: $(Agent.BuildDirectory)/installer
-
-  - ${{ if and(not(parameters.noPush), ne(variables['Build.Reason'], 'PullRequest'), eq(variables['System.TeamProject'], 'internal')) }}:
-    - script: |-
-        set -x
-        git config --global user.email 'dotnet-maestro[bot]@users.noreply.github.com' && git config --global user.name 'dotnet-maestro[bot]'
-        git remote add dotnet 'https://${{ parameters.vmrToken }}@github.com/dotnet/dotnet.git'
-        git fetch dotnet
-        git branch --set-upstream-to=dotnet/${{ parameters.vmrBranch }} ${{ parameters.vmrBranch }} || echo 'Branch ${{ parameters.vmrBranch }} not found in remote'
-        git push dotnet ${{ parameters.vmrBranch }}
-      displayName: Push changes to dotnet/dotnet
-      workingDirectory: $(vmrPath)

eng/pipelines/templates/steps/vmr-prepare.yml

@@ -0,0 +1,28 @@
+### These steps clone the VMR (https://github.com/dotnet/dotnet) into $(Agent.BuildDirectory)/vmr
+### Component Governance scan is also triggered over the VMR's non-repo sources
+
+parameters:
+- name: vmrBranch
+  displayName: dotnet/dotnet branch to use
+  type: string
+  default: $(Build.SourceBranchName)
+
+steps:
+- checkout: vmr
+  clean: true
+  displayName: Clone dotnet/dotnet
+  fetchDepth: 0
+  path: vmr
+
+- script: |
+    git switch -c ${{ parameters.vmrBranch }}
+  displayName: Checkout ${{ parameters.vmrBranch }}
+  workingDirectory: $(Agent.BuildDirectory)/vmr
+
+# TODO (https://github.com/dotnet/arcade/issues/11332): Allow full CG?
+# Currently, we ignore dirs of individual repos - they have been scanned before
+- ${{ if and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['System.TeamProject'], 'internal')) }}:
+  - task: ComponentGovernanceComponentDetection@0
+    inputs:
+      sourceScanPath: $(Agent.BuildDirectory)/vmr
+      ignoreDirectories: $(Agent.BuildDirectory)/vmr/src

eng/pipelines/templates/steps/vmr-pull-updates.yml

@@ -0,0 +1,60 @@
+### These steps synchronize new code from product repositories into the VMR (https://github.com/dotnet/dotnet).
+### They initialize the darc CLI and pull the new updates.
+### Changes are applied locally onto the already cloned VMR (located in $vmrPath).
+
+parameters:
+- name: vmrBranch
+  displayName: dotnet/dotnet branch to use
+  type: string
+
+- name: targetRef
+  displayName: Target revision in dotnet/installer to synchronize
+  type: string
+  default: $(Build.SourceVersion)
+
+- name: vmrToken
+  displayName: PAT that allows pushing to dotnet/dotnet
+  type: string
+
+- name: vmrPath
+  displayName: Path where the dotnet/dotnet is checked out to
+  type: string
+  default: $(Agent.BuildDirectory)/vmr
+
+steps:
+- checkout: self
+  clean: true
+  displayName: Clone dotnet/installer
+  fetchDepth: 0
+  path: installer
+
+- script: |
+    source ./eng/common/tools.sh
+    InitializeDotNetCli true
+    ./.dotnet/dotnet tool restore
+  displayName: Restore darc
+  workingDirectory: $(Agent.BuildDirectory)/installer
+
+- script: git checkout -B ${{ parameters.vmrBranch }}
+  displayName: Prepare branch ${{ parameters.vmrBranch }}
+  workingDirectory: $(vmrPath)
+
+# TODO (https://github.com/dotnet/arcade/issues/11386): Remove this step
+# This step is here so that darc can find the PR commit (which might be in some fork)
+# We need to make darc understand that it needs to look in the fork from this PR
+- ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
+  - script: |
+      cp -r $(Agent.BuildDirectory)/installer $(Agent.TempDirectory)/497F233644435505515AF465112B48CD
+    displayName: Prepare dotnet/installer clone
+
+- script: >
+    $(Agent.BuildDirectory)/installer/.dotnet/dotnet darc vmr update
+    --vmr ${{ parameters.vmrPath }}
+    --tmp $(Agent.TempDirectory)
+    --azdev-pat $(System.AccessToken)
+    --github-pat ${{ parameters.vmrToken }}
+    --recursive
+    --verbose
+    installer:${{ parameters.targetRef }}
+  displayName: Synchronize dotnet/dotnet
+  workingDirectory: $(Agent.BuildDirectory)/installer

eng/pipelines/templates/variables/vmr-variables.yml

@@ -0,0 +1,9 @@
+variables:
+- name: vmrPath
+  value: $(Agent.BuildDirectory)/vmr
+
+- ${{ if eq(variables['System.TeamProject'], 'internal') }}:
+  - group: DotNetBot-GitHub
+- ${{ else }}:
+  - name: BotAccount-dotnet-bot-repo-PAT
+    value: N/A

eng/pipelines/vmr-synchronization.yml

@@ -0,0 +1,74 @@
+### This pipeline synchronizes code from product repositories into the VMR (https://github.com/dotnet/dotnet)
+
+parameters:
+- name: targetRef
+  displayName: Target revision in dotnet/installer to synchronize to
+  type: string
+  default: $(Build.SourceVersion)
+
+- name: vmrBranch
+  displayName: dotnet/dotnet branch to use
+  type: string
+  default: $(Build.SourceBranchName)
+
+- name: noPush
+  displayName: Don't push changes to dotnet/dotnet
+  type: boolean
+  default: false
+
+trigger:
+  branches:
+    include:
+    - main
+    - release/*
+    - internal/release/*
+
+pr: none
+
+resources:
+  repositories:
+  - repository: vmr
+    type: github
+    name: dotnet/dotnet
+    endpoint: dotnet
+
+variables:
+- template: /eng/common/templates/variables/pool-providers.yml
+- template: templates/variables/vmr-variables.yml
+
+jobs:
+- job: Synchronize
+  displayName: Synchronize dotnet/dotnet
+  timeoutInMinutes: 120
+
+  pool:
+    ${{ if eq(variables['System.TeamProject'], 'public') }}:
+      name: $(DncEngPublicBuildPool)
+      demands: ImageOverride -equals Build.Ubuntu.2004.Amd64.Open
+    ${{ else }}:
+      name: $(DncEngInternalBuildPool)
+      demands: ImageOverride -equals Build.Ubuntu.2004.Amd64
+
+  steps:
+  - template: templates/steps/vmr-prepare.yml
+    parameters:
+      vmrBranch: ${{ parameters.vmrBranch }}
+
+  - template: templates/steps/vmr-pull-updates.yml
+    parameters:
+      vmrPath: $(vmrPath)
+      vmrBranch: ${{ parameters.vmrBranch }}
+      targetRef: ${{ parameters.targetRef }}
+      vmrToken: $(BotAccount-dotnet-bot-repo-PAT)
+
+  - ${{ if and(not(parameters.noPush), ne(variables['Build.Reason'], 'PullRequest'), eq(variables['System.TeamProject'], 'internal')) }}:
+    - script: |-
+        set -x
+        git config --global user.email 'dotnet-maestro[bot]@users.noreply.github.com' && git config --global user.name 'dotnet-maestro[bot]'
+        git remote add dotnet 'https://$(BotAccount-dotnet-bot-repo-PAT)@github.com/dotnet/dotnet.git'
+        git fetch dotnet
+        git branch ${{ parameters.vmrBranch }}
+        git branch --set-upstream-to=dotnet/${{ parameters.vmrBranch }} ${{ parameters.vmrBranch }} || echo 'Branch ${{ parameters.vmrBranch }} not found in remote'
+        git push dotnet ${{ parameters.vmrBranch }}
+      displayName: Push changes to dotnet/dotnet
+      workingDirectory: $(vmrPath)

eng/pipelines/vmr-validation.yml

@@ -0,0 +1,51 @@
+### This pipeline validates new changes of the VMR (https://github.com/dotnet/dotnet)
+### VMR is cloned and updates are applied locally
+
+parameters:
+- name: vmrBranch
+  displayName: dotnet/dotnet branch to use
+  type: string
+  default: $(System.PullRequest.TargetBranch)
+
+trigger: none
+
+pr:
+  branches:
+    include:
+    - main
+    - release/*
+    - internal/release/*
+
+resources:
+  repositories:
+  - repository: vmr
+    type: github
+    name: dotnet/dotnet
+    endpoint: public
+
+variables:
+- template: /eng/common/templates/variables/pool-providers.yml
+- template: templates/variables/vmr-variables.yml
+
+jobs:
+- job: Validate
+  displayName: Test VMR synchronization
+  pool:
+    ${{ if eq(variables['System.TeamProject'], 'public') }}:
+      name: $(DncEngPublicBuildPool)
+      demands: ImageOverride -equals Build.Ubuntu.2004.Amd64.Open
+    ${{ else }}:
+      name: $(DncEngInternalBuildPool)
+      demands: ImageOverride -equals Build.Ubuntu.2004.Amd64
+
+  steps:
+  - template: templates/steps/vmr-prepare.yml
+    parameters:
+      vmrBranch: ${{ parameters.vmrBranch }}
+
+  - template: templates/steps/vmr-pull-updates.yml
+    parameters:
+      vmrPath: $(vmrPath)
+      vmrBranch: ${{ parameters.vmrBranch }}
+      targetRef: $(Build.SourceVersion)
+      vmrToken: $(BotAccount-dotnet-bot-repo-PAT)

eng/vmr-synchronization.yml

@@ -1,47 +0,0 @@
-### This pipeline synchronizes code from product repositories into the VMR (https://github.com/dotnet/dotnet)
-
-parameters:
-- name: targetRef
-  displayName: Target revision in dotnet/installer to synchronize
-  type: string
-  default: $(Build.SourceVersion)
-
-- name: vmrBranch
-  displayName: dotnet/dotnet branch to use
-  type: string
-  default: $(Build.SourceBranchName)
-
-- name: noPush
-  displayName: Don't push changes to dotnet/dotnet
-  type: boolean
-  default: false
-
-trigger:
-  branches:
-    include:
-    - main
-
-pr: none
-
-resources:
-  repositories:
-  - repository: vmr
-    type: github
-    name: dotnet/dotnet
-    endpoint: dotnet
-
-variables:
-- ${{ if eq(variables['System.TeamProject'], 'internal') }}:
-  - group: DotNetBot-GitHub
-
-- ${{ else }}:
-  - name: BotAccount-dotnet-bot-repo-PAT
-    value: N/A
-
-jobs:
-- template: ./pipelines/templates/jobs/vmr-synchronization.yml
-  parameters:
-    targetRef: ${{ parameters.targetRef }}
-    vmrBranch: ${{ parameters.vmrBranch }}
-    vmrToken: $(BotAccount-dotnet-bot-repo-PAT)
-    noPush: ${{ parameters.noPush }}