From 6b20a1eae03b4a06b4f33e49f495084d78e2c4e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emek=20Vysok=C3=BD?= Date: Tue, 8 Nov 2022 17:47:32 +0100 Subject: [PATCH] Add a VMR synchronization PR validation (#14918) Adds a new pipeline that will run during installer's PRs that verifies that the current VMR tooling can synchronize that particular change into the VMR. --- .../templates/jobs/vmr-synchronization.yml | 103 ------------------ eng/pipelines/templates/steps/vmr-prepare.yml | 28 +++++ .../templates/steps/vmr-pull-updates.yml | 60 ++++++++++ .../templates/variables/vmr-variables.yml | 9 ++ eng/pipelines/vmr-synchronization.yml | 74 +++++++++++++ eng/pipelines/vmr-validation.yml | 51 +++++++++ eng/vmr-synchronization.yml | 47 -------- 7 files changed, 222 insertions(+), 150 deletions(-) delete mode 100644 eng/pipelines/templates/jobs/vmr-synchronization.yml create mode 100644 eng/pipelines/templates/steps/vmr-prepare.yml create mode 100644 eng/pipelines/templates/steps/vmr-pull-updates.yml create mode 100644 eng/pipelines/templates/variables/vmr-variables.yml create mode 100644 eng/pipelines/vmr-synchronization.yml create mode 100644 eng/pipelines/vmr-validation.yml delete mode 100644 eng/vmr-synchronization.yml diff --git a/eng/pipelines/templates/jobs/vmr-synchronization.yml b/eng/pipelines/templates/jobs/vmr-synchronization.yml deleted file mode 100644 index 6b4ee5bb6..000000000 --- a/eng/pipelines/templates/jobs/vmr-synchronization.yml +++ /dev/null @@ -1,103 +0,0 @@ -### These steps synchronize code from product repositories into the VMR (https://github.com/dotnet/dotnet) - -parameters: -- name: targetRef - displayName: Target revision in dotnet/installer to synchronize - type: string - default: $(Build.SourceVersion) - -- name: vmrBranch - displayName: dotnet/dotnet branch to use - type: string - default: $(Build.SourceBranchName) - -- name: vmrToken - displayName: PAT that allows pushing to dotnet/dotnet - type: string - -- name: noPush - displayName: Don't push changes to dotnet/dotnet - type: boolean - default: false - -jobs: -- job: Synchronize - displayName: Synchronize dotnet/dotnet - timeoutInMinutes: 120 - - variables: - - name: vmrPath - value: $(Agent.BuildDirectory)/vmr - - pool: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: NetCore-Public - demands: - - ImageOverride -equals Build.Ubuntu.2004.Amd64.Open - ${{ else }}: - name: NetCore1ESPool-Internal - demands: - - ImageOverride -equals Build.Ubuntu.2004.Amd64 - - steps: - - checkout: self - clean: true - displayName: Checkout dotnet/installer - fetchDepth: 0 - path: installer - - - checkout: vmr - clean: true - displayName: Checkout dotnet/dotnet - fetchDepth: 0 - path: vmr - - # TODO (https://github.com/dotnet/arcade/issues/11332): Allow full CG? - # Currently, we ignore dirs of individual repos - they have been scanned before - - ${{ if and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['System.TeamProject'], 'internal')) }}: - - task: ComponentGovernanceComponentDetection@0 - inputs: - sourceScanPath: $(vmrPath) - ignoreDirectories: $(vmrPath)/src - - - script: |- - source ./eng/common/tools.sh - InitializeDotNetCli true - ./.dotnet/dotnet tool restore - displayName: Restore toolset - workingDirectory: $(Agent.BuildDirectory)/installer - - # TODO (https://github.com/dotnet/arcade/issues/11386): Remove this step - # This step is here so that darc can find the PR commit (which might be in some fork) - # We need to make darc understand that it needs to look in the fork from this PR - - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: - - script: |- - cp -r $(Agent.BuildDirectory)/installer $(Agent.TempDirectory)/installer - displayName: '[PR Only] Prepare dotnet/installer clone' - - - script: git checkout -B ${{ parameters.vmrBranch }} - displayName: Prepare branch ${{ parameters.vmrBranch }} - workingDirectory: $(vmrPath) - - - script: > - $(Agent.BuildDirectory)/installer/.dotnet/dotnet darc vmr update - --vmr $(vmrPath) - --tmp $(Agent.TempDirectory) - --azdev-pat $(System.AccessToken) - --github-pat ${{ parameters.vmrToken }} - --recursive - --verbose - installer:${{ parameters.targetRef }} - displayName: Synchronize dotnet/dotnet - workingDirectory: $(Agent.BuildDirectory)/installer - - - ${{ if and(not(parameters.noPush), ne(variables['Build.Reason'], 'PullRequest'), eq(variables['System.TeamProject'], 'internal')) }}: - - script: |- - set -x - git config --global user.email 'dotnet-maestro[bot]@users.noreply.github.com' && git config --global user.name 'dotnet-maestro[bot]' - git remote add dotnet 'https://${{ parameters.vmrToken }}@github.com/dotnet/dotnet.git' - git fetch dotnet - git branch --set-upstream-to=dotnet/${{ parameters.vmrBranch }} ${{ parameters.vmrBranch }} || echo 'Branch ${{ parameters.vmrBranch }} not found in remote' - git push dotnet ${{ parameters.vmrBranch }} - displayName: Push changes to dotnet/dotnet - workingDirectory: $(vmrPath) diff --git a/eng/pipelines/templates/steps/vmr-prepare.yml b/eng/pipelines/templates/steps/vmr-prepare.yml new file mode 100644 index 000000000..5a18630f9 --- /dev/null +++ b/eng/pipelines/templates/steps/vmr-prepare.yml @@ -0,0 +1,28 @@ +### These steps clone the VMR (https://github.com/dotnet/dotnet) into $(Agent.BuildDirectory)/vmr +### Component Governance scan is also triggered over the VMR's non-repo sources + +parameters: +- name: vmrBranch + displayName: dotnet/dotnet branch to use + type: string + default: $(Build.SourceBranchName) + +steps: +- checkout: vmr + clean: true + displayName: Clone dotnet/dotnet + fetchDepth: 0 + path: vmr + +- script: | + git switch -c ${{ parameters.vmrBranch }} + displayName: Checkout ${{ parameters.vmrBranch }} + workingDirectory: $(Agent.BuildDirectory)/vmr + +# TODO (https://github.com/dotnet/arcade/issues/11332): Allow full CG? +# Currently, we ignore dirs of individual repos - they have been scanned before +- ${{ if and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['System.TeamProject'], 'internal')) }}: + - task: ComponentGovernanceComponentDetection@0 + inputs: + sourceScanPath: $(Agent.BuildDirectory)/vmr + ignoreDirectories: $(Agent.BuildDirectory)/vmr/src diff --git a/eng/pipelines/templates/steps/vmr-pull-updates.yml b/eng/pipelines/templates/steps/vmr-pull-updates.yml new file mode 100644 index 000000000..bef0af718 --- /dev/null +++ b/eng/pipelines/templates/steps/vmr-pull-updates.yml @@ -0,0 +1,60 @@ +### These steps synchronize new code from product repositories into the VMR (https://github.com/dotnet/dotnet). +### They initialize the darc CLI and pull the new updates. +### Changes are applied locally onto the already cloned VMR (located in $vmrPath). + +parameters: +- name: vmrBranch + displayName: dotnet/dotnet branch to use + type: string + +- name: targetRef + displayName: Target revision in dotnet/installer to synchronize + type: string + default: $(Build.SourceVersion) + +- name: vmrToken + displayName: PAT that allows pushing to dotnet/dotnet + type: string + +- name: vmrPath + displayName: Path where the dotnet/dotnet is checked out to + type: string + default: $(Agent.BuildDirectory)/vmr + +steps: +- checkout: self + clean: true + displayName: Clone dotnet/installer + fetchDepth: 0 + path: installer + +- script: | + source ./eng/common/tools.sh + InitializeDotNetCli true + ./.dotnet/dotnet tool restore + displayName: Restore darc + workingDirectory: $(Agent.BuildDirectory)/installer + +- script: git checkout -B ${{ parameters.vmrBranch }} + displayName: Prepare branch ${{ parameters.vmrBranch }} + workingDirectory: $(vmrPath) + +# TODO (https://github.com/dotnet/arcade/issues/11386): Remove this step +# This step is here so that darc can find the PR commit (which might be in some fork) +# We need to make darc understand that it needs to look in the fork from this PR +- ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + - script: | + cp -r $(Agent.BuildDirectory)/installer $(Agent.TempDirectory)/497F233644435505515AF465112B48CD + displayName: Prepare dotnet/installer clone + +- script: > + $(Agent.BuildDirectory)/installer/.dotnet/dotnet darc vmr update + --vmr ${{ parameters.vmrPath }} + --tmp $(Agent.TempDirectory) + --azdev-pat $(System.AccessToken) + --github-pat ${{ parameters.vmrToken }} + --recursive + --verbose + installer:${{ parameters.targetRef }} + displayName: Synchronize dotnet/dotnet + workingDirectory: $(Agent.BuildDirectory)/installer diff --git a/eng/pipelines/templates/variables/vmr-variables.yml b/eng/pipelines/templates/variables/vmr-variables.yml new file mode 100644 index 000000000..0d6af8fff --- /dev/null +++ b/eng/pipelines/templates/variables/vmr-variables.yml @@ -0,0 +1,9 @@ +variables: +- name: vmrPath + value: $(Agent.BuildDirectory)/vmr + +- ${{ if eq(variables['System.TeamProject'], 'internal') }}: + - group: DotNetBot-GitHub +- ${{ else }}: + - name: BotAccount-dotnet-bot-repo-PAT + value: N/A \ No newline at end of file diff --git a/eng/pipelines/vmr-synchronization.yml b/eng/pipelines/vmr-synchronization.yml new file mode 100644 index 000000000..cf624ab12 --- /dev/null +++ b/eng/pipelines/vmr-synchronization.yml @@ -0,0 +1,74 @@ +### This pipeline synchronizes code from product repositories into the VMR (https://github.com/dotnet/dotnet) + +parameters: +- name: targetRef + displayName: Target revision in dotnet/installer to synchronize to + type: string + default: $(Build.SourceVersion) + +- name: vmrBranch + displayName: dotnet/dotnet branch to use + type: string + default: $(Build.SourceBranchName) + +- name: noPush + displayName: Don't push changes to dotnet/dotnet + type: boolean + default: false + +trigger: + branches: + include: + - main + - release/* + - internal/release/* + +pr: none + +resources: + repositories: + - repository: vmr + type: github + name: dotnet/dotnet + endpoint: dotnet + +variables: +- template: /eng/common/templates/variables/pool-providers.yml +- template: templates/variables/vmr-variables.yml + +jobs: +- job: Synchronize + displayName: Synchronize dotnet/dotnet + timeoutInMinutes: 120 + + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals Build.Ubuntu.2004.Amd64.Open + ${{ else }}: + name: $(DncEngInternalBuildPool) + demands: ImageOverride -equals Build.Ubuntu.2004.Amd64 + + steps: + - template: templates/steps/vmr-prepare.yml + parameters: + vmrBranch: ${{ parameters.vmrBranch }} + + - template: templates/steps/vmr-pull-updates.yml + parameters: + vmrPath: $(vmrPath) + vmrBranch: ${{ parameters.vmrBranch }} + targetRef: ${{ parameters.targetRef }} + vmrToken: $(BotAccount-dotnet-bot-repo-PAT) + + - ${{ if and(not(parameters.noPush), ne(variables['Build.Reason'], 'PullRequest'), eq(variables['System.TeamProject'], 'internal')) }}: + - script: |- + set -x + git config --global user.email 'dotnet-maestro[bot]@users.noreply.github.com' && git config --global user.name 'dotnet-maestro[bot]' + git remote add dotnet 'https://$(BotAccount-dotnet-bot-repo-PAT)@github.com/dotnet/dotnet.git' + git fetch dotnet + git branch ${{ parameters.vmrBranch }} + git branch --set-upstream-to=dotnet/${{ parameters.vmrBranch }} ${{ parameters.vmrBranch }} || echo 'Branch ${{ parameters.vmrBranch }} not found in remote' + git push dotnet ${{ parameters.vmrBranch }} + displayName: Push changes to dotnet/dotnet + workingDirectory: $(vmrPath) diff --git a/eng/pipelines/vmr-validation.yml b/eng/pipelines/vmr-validation.yml new file mode 100644 index 000000000..bdd0bbd31 --- /dev/null +++ b/eng/pipelines/vmr-validation.yml @@ -0,0 +1,51 @@ +### This pipeline validates new changes of the VMR (https://github.com/dotnet/dotnet) +### VMR is cloned and updates are applied locally + +parameters: +- name: vmrBranch + displayName: dotnet/dotnet branch to use + type: string + default: $(System.PullRequest.TargetBranch) + +trigger: none + +pr: + branches: + include: + - main + - release/* + - internal/release/* + +resources: + repositories: + - repository: vmr + type: github + name: dotnet/dotnet + endpoint: public + +variables: +- template: /eng/common/templates/variables/pool-providers.yml +- template: templates/variables/vmr-variables.yml + +jobs: +- job: Validate + displayName: Test VMR synchronization + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals Build.Ubuntu.2004.Amd64.Open + ${{ else }}: + name: $(DncEngInternalBuildPool) + demands: ImageOverride -equals Build.Ubuntu.2004.Amd64 + + steps: + - template: templates/steps/vmr-prepare.yml + parameters: + vmrBranch: ${{ parameters.vmrBranch }} + + - template: templates/steps/vmr-pull-updates.yml + parameters: + vmrPath: $(vmrPath) + vmrBranch: ${{ parameters.vmrBranch }} + targetRef: $(Build.SourceVersion) + vmrToken: $(BotAccount-dotnet-bot-repo-PAT) diff --git a/eng/vmr-synchronization.yml b/eng/vmr-synchronization.yml deleted file mode 100644 index 84e52e3f3..000000000 --- a/eng/vmr-synchronization.yml +++ /dev/null @@ -1,47 +0,0 @@ -### This pipeline synchronizes code from product repositories into the VMR (https://github.com/dotnet/dotnet) - -parameters: -- name: targetRef - displayName: Target revision in dotnet/installer to synchronize - type: string - default: $(Build.SourceVersion) - -- name: vmrBranch - displayName: dotnet/dotnet branch to use - type: string - default: $(Build.SourceBranchName) - -- name: noPush - displayName: Don't push changes to dotnet/dotnet - type: boolean - default: false - -trigger: - branches: - include: - - main - -pr: none - -resources: - repositories: - - repository: vmr - type: github - name: dotnet/dotnet - endpoint: dotnet - -variables: -- ${{ if eq(variables['System.TeamProject'], 'internal') }}: - - group: DotNetBot-GitHub - -- ${{ else }}: - - name: BotAccount-dotnet-bot-repo-PAT - value: N/A - -jobs: -- template: ./pipelines/templates/jobs/vmr-synchronization.yml - parameters: - targetRef: ${{ parameters.targetRef }} - vmrBranch: ${{ parameters.vmrBranch }} - vmrToken: $(BotAccount-dotnet-bot-repo-PAT) - noPush: ${{ parameters.noPush }}