Support sync of a single repository into the VMR (#15569)

.config/dotnet-tools.json

@@ -3,7 +3,7 @@
   "isRoot": true,
   "tools": {
     "microsoft.dotnet.darc": {
-      "version": "1.1.0-beta.23107.1",
+      "version": "1.1.0-beta.23116.2",
       "commands": [
         "darc"
       ]

.devcontainer/vmr-source-build/README.md

@@ -14,6 +14,8 @@ This Codespace can help you debug the source build of .NET. In case you have run
 the VMR which is what the VMR leg in the installer PR build doing. This build takes about 45
 minutes and, after completion, produces an archived .NET SDK in `/workspaces/artifacts/x64/Release`.
 
+## Build the SDK
+
 To build the VMR, run following:
 ```bash
 cd /workspaces/dotnet
@@ -22,6 +24,38 @@ unset RepositoryName
 ```
 
 > Please note that, at this time, the build modifies some of the checked-in sources so it might
-be preferential to rebuild the Codespace between attempts.
+be preferential to rebuild the Codespace between attempts (or reset the working tree changes).
 
 For more details, see the instructions at https://github.com/dotnet/dotnet.
+
+## Synchronize your changes in locally
+
+When debugging the build, you have two options how to test your changes in this environment.
+
+### Making changes to the VMR directly
+
+You can make the changes directly to the local checkout of the VMR at `/workspaces/dotnet`. You
+can then try to build the VMR and see if the change works for you.
+
+### Pull changes into the local VMR
+
+You can also make a fix in the individual source repository (e.g. `dotnet/runtime`) and push the
+fix into a branch (can be in your fork too). Once you have the commit pushed, you can pull this
+version of the repository into the VMR locally.
+
+Let's consider you pushed a commit with SHA `abcdef` into your fork at `github.com/yourfork/runtime`.
+You can now bring this version of runtime into the local VMR in this Codespace by running:
+
+```bash
+cd /workspaces/installer
+./eng/vmr-sync.sh             \
+  --vmr /workspaces/dotnet    \
+  --tmp /workspaces/tmp       \
+  --repository runtime:abcdef \
+  --remote runtime:https://github.com/yourfork/runtime
+```
+
+You can now proceed building the VMR in the Codespace using instructions above. You can repeat
+this process and sync a new commit from your fork. Only note that, at this time, Source-Build
+modifies some of the checked-in sources so you'll need to revert the working tree changes
+between attempts.

.devcontainer/vmr-source-build/init.sh

@@ -17,7 +17,11 @@ git -C "$installer_dir" fetch --all --unshallow
 
 # We will try to figure out, which branch is the current (PR) branch based off of
 # We need this to figure out, which VMR branch to use
-vmr_branch=$(git log --pretty=format:'%D' HEAD^ | grep 'origin/' | head -n1 | sed 's@origin/@@' | sed 's@,.*@@')
+vmr_branch=$(git -C "$installer_dir" log --pretty=format:'%D' HEAD^ \
+  | grep 'origin/' \
+  | head -n1 \
+  | sed 's@origin/@@' \
+  | sed 's@,.*@@')
 
 pushd "$installer_dir"
   "./eng/vmr-sync.sh"      \

eng/Version.Details.xml

@@ -208,13 +208,13 @@
       <Uri>https://github.com/dotnet/arcade</Uri>
       <Sha>02980a654d20995c38eca6ceabf729c4c13412c4</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Darc" Version="1.1.0-beta.23107.1">
+    <Dependency Name="Microsoft.DotNet.Darc" Version="1.1.0-beta.23116.2">
       <Uri>https://github.com/dotnet/arcade-services</Uri>
-      <Sha>c6f3ec3b751020f757695bcd351b1cbb179e21bd</Sha>
+      <Sha>babaa0d787b3921bae3e97d053f6b73c74e750b0</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.DarcLib" Version="1.1.0-beta.23107.1">
+    <Dependency Name="Microsoft.DotNet.DarcLib" Version="1.1.0-beta.23116.2">
       <Uri>https://github.com/dotnet/arcade-services</Uri>
-      <Sha>c6f3ec3b751020f757695bcd351b1cbb179e21bd</Sha>
+      <Sha>babaa0d787b3921bae3e97d053f6b73c74e750b0</Sha>
     </Dependency>
     <Dependency Name="Microsoft.Extensions.Logging.Console" Version="8.0.0-alpha.1.22557.12">
       <Uri>https://github.com/dotnet/runtime</Uri>

eng/Versions.props

@@ -30,7 +30,7 @@
   </PropertyGroup>
   <PropertyGroup>
     <!-- Dependency from https://github.com/dotnet/arcade-services -->
-    <MicrosoftDotNetDarcLibVersion>1.1.0-beta.23107.1</MicrosoftDotNetDarcLibVersion>
+    <MicrosoftDotNetDarcLibVersion>1.1.0-beta.23116.2</MicrosoftDotNetDarcLibVersion>
   </PropertyGroup>
   <PropertyGroup>
     <!-- Dependency from https://github.com/dotnet/winforms -->

eng/pipelines/templates/steps/vmr-pull-updates.yml

@@ -33,7 +33,8 @@ steps:
     --vmr ${{ parameters.vmrPath }}
     --tmp $(Agent.TempDirectory)
     --branch ${{ parameters.vmrBranch }}
-    --target-ref ${{ parameters.targetRef }}
+    --repository "installer:${{ parameters.targetRef }}"
+    --recursive
     --readme-template $(Agent.BuildDirectory)/installer/src/VirtualMonoRepo/README.template.md
     --tpn-template $(Agent.BuildDirectory)/installer/src/VirtualMonoRepo/THIRD-PARTY-NOTICES.template.txt
     --debug

eng/vmr-sync.sh

@@ -1,63 +1,58 @@
 #!/bin/bash
 
-### This script is used for synchronizing the dotnet/dotnet repository locally
+### This script is used for synchronizing the dotnet/dotnet VMR locally. This means pulling new
-### It is used during CI to ingest new code based on dotnet/installer
+### code from various repositories into the 'dotnet/dotnet' repository.
-### I can also help for reproducing potential failures during installer's PRs,
-### namely during errors during the 'Synchronize dotnet/dotnet' build step from
-### the 'VMR Source-Build'.
-### The following scenario is assumed:
-### - There is a PR in dotnet/installer
-### - The PR is failing on the 'VMR Source-Build' job in the 'Synchronize dotnet/dotnet' step
 ###
-### There are a few possible reasons but usually this happens because there is a mismatch between
+### The script is used during CI to ingest new code based on dotnet/installer but it can also help
-### source patches that are applied on top of the files that are being synchronized from repositories
+### for reproducing potential failures during installer's PRs, namely to fix the Source-Build.
-### into the 'dotnet/dotnet' repo and new changes in the repositories.
+### Another usecase is to try manually synchronizing a given commit of some repo into the VMR and
-### This manifests as the following error:
+### trying to Source-Build the VMR. This can help when fixing the Source-Build but using a commit
-###     fail: Failed to synchronize repo installer
+### from a not-yet merged branch (or fork) to test the fix will help.
-###           Failed to apply the patch for src/aspnetcore
-###           Exit code: 1
-###           Std err:
-###           error: patch failed: src/aspnetcore/eng/SourceBuild.props:55
-###           error: src/aspnetcore/eng/SourceBuild.props: patch does not apply
 ###
-### where 'src/aspnetcore/eng/SourceBuild.props' would be the file that is being patched and new
+### The tooling that synchronizes the VMR will need to clone the various repositories into a temporary
-### changes to it are conflicting with the patch that is being applied.
+### folder. These clones can be re-used in future synchronizations so it is advised you dedicate a
-###
+### folder to this to speed up your re-runs.
-### The whole process can be reproduced locally easily by running this script.
-### The patches are located in a folder in the 'dotnet/installer' repository.
-### At the moment of writing, the location is 'src/SourceBuild/patches' but to get
-### the up-to-date location, please see the 'patchesPath' property in
-### https://github.com/dotnet/dotnet/blob/main/src/source-mappings.json
-###
-### You will need to compare what is in the patch and what is in the file and fix the patch.
-###
-### The tooling that synchronizes the VMR will need to clone the various repositories.
-### It clones them into a temporary folder and re-uses them on future runs so it is advised
-### you dedicate a folder to this to speed up your re-runs.
-###
-### This script will synchronize the 'dotnet/dotnet' repo locally and let you inspect the changes.
 ###
 ### USAGE:
+###   Synchronize the VMR to the commit of dotnet/installer that is currently being built:
 ###     ./vmr-sync.sh --tmp-dir "$HOME/repos/tmp"
+###
+###   Synchronize the VMR to a specific commit of dotnet/runtime using custom fork:
+###     ./vmr-sync.sh \
+###        --repository runtime:e7e71da303af8dc97df99b098f21f526398c3943 \
+###        --remote runtime:https://github.com/yourfork/runtime \
+###        --tmp-dir "$HOME/repos/tmp"\
+###
 ### Options:
 ###   -t, --tmp, --tmp-dir PATH
 ###       Required. Path to the temporary folder where repositories will be cloned
+###   --repository name:GIT_REF
+###       Optional. Repository + git ref separated by colon to synchronize to.
+###       This can be a specific commit, branch, tag..
+###       When omitted, the script will synchronize the installer commit based on the version of the parent
+###       where this script is stored.
 ###   -v, --vmr, --vmr-dir PATH
 ###       Optional. Path to the dotnet/dotnet repository. When null, gets cloned to the temporary folder
 ###   -b, --branch, --vmr-branch BRANCH_NAME
 ###       Optional. Branch of the 'dotnet/dotnet' repo to synchronize to
-###       This should match the target branch of the PR, defaults to 'main'
+###       This should match the target branch of the PR; defaults to 'main'
-###   --target-ref GIT_REF
+###   --remote name:URI
-###       Optional. Git ref to synchronize to. This can be a specific commit, branch, tag..
+###       Optional. Additional remote to use during the synchronization
-###       Defaults to the revision of the parent installer repo
+###       This can be used to synchronize to a commit from a fork of the repository
-###   --debug
+###       Example: 'runtime:https://github.com/yourfork/runtime'
-###       Optional. Turns on the most verbose logging for the VMR tooling
+###   --recursive
+###       Optional. Recursively synchronize all the source build dependencies (declared in Version.Details.xml)
+###       This is used when performing the full synchronization during installer's CI and the final VMR sync.
+###       Defaults to false unless no repository is supplied in which case a recursive sync of installer is performed.
 ###   --readme-template
-###       Optional. Template for VMRs README.md used for regenerating the file to list the newest versions of components. 
+###       Optional. Template for VMRs README.md used for regenerating the file to list the newest versions of
+###       components.
 ###       Defaults to src/VirtualMonoRepo/README.template.md
 ###   --tpn-template
 ###       Optional. Template for the header of VMRs THIRD-PARTY-NOTICES file.
 ###       Defaults to src/VirtualMonoRepo/THIRD-PARTY-NOTICES.template.txt
+###   --debug
+###       Optional. Turns on the most verbose logging for the VMR tooling
 
 source="${BASH_SOURCE[0]}"
 
@@ -93,10 +88,12 @@ installer_dir=$(realpath "$scriptroot/../")
 tmp_dir=''
 vmr_dir=''
 vmr_branch='main'
-target_ref=''
+repository=''
+recursive=false
 verbosity=verbose
-tpn_template=''
+additional_remotes="installer:$installer_dir"
-readme_template=''
+readme_template="$installer_dir/src/VirtualMonoRepo/README.template.md"
+tpn_template="$installer_dir/src/VirtualMonoRepo/THIRD-PARTY-NOTICES.template.txt"
 
 while [[ $# -gt 0 ]]; do
   opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")"
@@ -113,11 +110,15 @@ while [[ $# -gt 0 ]]; do
       vmr_branch=$2
       shift
       ;;
-    -d|--debug)
+    --repository)
-      verbosity=debug
+      repository=$2
+      shift
       ;;
-    --target-ref)
+    --recursive)
-      target_ref=$2
+      recursive=true
+      ;;
+    --remote)
+      additional_remotes="$additional_remotes,$2"
       shift
       ;;
     --readme-template)
@@ -128,6 +129,9 @@ while [[ $# -gt 0 ]]; do
       tpn_template=$2
       shift
       ;;
+    -d|--debug)
+      verbosity=debug
+      ;;
     -h|--help)
       print_help
       exit 0
@@ -142,19 +146,13 @@ while [[ $# -gt 0 ]]; do
   shift
 done
 
+# Validation
+
 if [[ ! -d "$installer_dir" ]]; then
   fail "Directory '$installer_dir' does not exist. Please specify the path to the dotnet/installer repo"
   exit 1
 fi
 
-if [[ -z "$readme_template" ]]; then
-  readme_template="$installer_dir/src/VirtualMonoRepo/README.template.md"
-fi
-
-if [[ -z "$tpn_template" ]]; then
-  tpn_template="$installer_dir/src/VirtualMonoRepo/THIRD-PARTY-NOTICES.template.txt"
-fi
-
 if [[ -z "$tmp_dir" ]]; then
   fail "Missing --tmp-dir argument. Please specify the path to the temporary folder where the repositories will be cloned"
   exit 1
@@ -170,6 +168,15 @@ if [[ ! -f "$tpn_template" ]]; then
   exit 1
 fi
 
+# Sanitize the input
+
+if [[ -z "$repository" ]]; then
+  current_sha=$(git -C "$installer_dir" rev-parse HEAD)
+  echo "No repository specified, will synchronize installer:$current_sha"
+  repository="installer:$current_sha"
+  recursive=true
+fi
+
 if [[ -z "$vmr_dir" ]]; then
   vmr_dir="$tmp_dir/dotnet"
 fi
@@ -182,6 +189,8 @@ if [[ "$verbosity" == "debug" ]]; then
   set -x
 fi
 
+# Prepare the VMR
+
 if [[ ! -d "$vmr_dir" ]]; then
   highlight "Cloning 'dotnet/dotnet' into $vmr_dir.."
   git clone https://github.com/dotnet/dotnet "$vmr_dir"
@@ -200,24 +209,37 @@ fi
 set -e
 
 # Prepare darc
+
 highlight 'Installing .NET, preparing the tooling..'
 source "$scriptroot/common/tools.sh"
 InitializeDotNetCli true
 dotnet="$scriptroot/../.dotnet/dotnet"
 "$dotnet" tool restore
 
-# Run the sync
+highlight "Starting the synchronization to '$repository'.."
-if [[ -z "$target_ref" ]]; then
-  target_ref=$(git -C "$installer_dir" rev-parse HEAD)
-fi
-
-highlight "Starting the synchronization to '$target_ref'.."
 set +e
 
-if "$dotnet" darc vmr update --vmr "$vmr_dir" --tmp "$tmp_dir" --$verbosity --recursive --readme-template "$readme_template" --tpn-template "$tpn_template" --additional-remotes "installer:$installer_dir" "installer:$target_ref"; then
+recursive_arg=''
+if [[ "$recursive" == "true" ]]; then
+  recursive_arg="--recursive"
+fi
+
+# Synchronize the VMR
+
+"$dotnet" darc vmr update                    \
+  --vmr "$vmr_dir"                           \
+  --tmp "$tmp_dir"                           \
+  --$verbosity                               \
+  $recursive_arg                             \
+  --readme-template "$readme_template"       \
+  --tpn-template "$tpn_template"             \
+  --additional-remotes "$additional_remotes" \
+  "$repository"
+
+if [[ $? == 0 ]]; then
   highlight "Synchronization succeeded"
 else
-  fail "Synchronization of dotnet/dotnet to '$target_ref' failed!"
+  fail "Synchronization of dotnet/dotnet to '$repository' failed!"
   fail "'$vmr_dir' is left in its last state (re-run of this script will reset it)."
   fail "Please inspect the logs which contain path to the failing patch file (use --debug to get all the details)."
   fail "Once you make changes to the conflicting VMR patch, commit it locally and re-run this script."