[main] Update dependencies from dotnet/arcade-services (#15211)

Co-authored-by: dotnet-maestro[bot] <dotnet-maestro[bot]@users.noreply.github.com>
Co-authored-by: Premek Vysoky <premek.vysoky@microsoft.com>
Co-authored-by: MilenaHristova <mhristova@microsoft.com>

.config/dotnet-tools.json

@@ -3,7 +3,7 @@
   "isRoot": true,
   "tools": {
     "microsoft.dotnet.darc": {
-      "version": "1.1.0-beta.23053.1",
+      "version": "1.1.0-beta.23056.9",
       "commands": [
         "darc"
       ]

eng/Version.Details.xml

@@ -205,13 +205,13 @@
       <Uri>https://github.com/dotnet/arcade</Uri>
       <Sha>5a5ae41c42e475e280342b668e668ff34539e36d</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Darc" Version="1.1.0-beta.23053.1">
+    <Dependency Name="Microsoft.DotNet.Darc" Version="1.1.0-beta.23056.9">
       <Uri>https://github.com/dotnet/arcade-services</Uri>
-      <Sha>11dfc0ff33280c18db03a2108673a41ade41fd55</Sha>
+      <Sha>2e6a975bf4753c44a1a98f77282e64ea3a0e31ab</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.DarcLib" Version="1.1.0-beta.23053.1">
+    <Dependency Name="Microsoft.DotNet.DarcLib" Version="1.1.0-beta.23056.9">
       <Uri>https://github.com/dotnet/arcade-services</Uri>
-      <Sha>11dfc0ff33280c18db03a2108673a41ade41fd55</Sha>
+      <Sha>2e6a975bf4753c44a1a98f77282e64ea3a0e31ab</Sha>
     </Dependency>
     <Dependency Name="Microsoft.Extensions.Logging.Console" Version="8.0.0-alpha.1.22557.12">
       <Uri>https://github.com/dotnet/runtime</Uri>

eng/Versions.props

@@ -30,7 +30,7 @@
   </PropertyGroup>
   <PropertyGroup>
     <!-- Dependency from https://github.com/dotnet/arcade-services -->
-    <MicrosoftDotNetDarcLibVersion>1.1.0-beta.23053.1</MicrosoftDotNetDarcLibVersion>
+    <MicrosoftDotNetDarcLibVersion>1.1.0-beta.23056.9</MicrosoftDotNetDarcLibVersion>
   </PropertyGroup>
   <PropertyGroup>
     <!-- Dependency from https://github.com/dotnet/winforms -->

eng/pipelines/templates/steps/vmr-pull-updates.yml

@@ -22,36 +22,17 @@ steps:
   displayName: Clone dotnet/installer
   path: installer
 
-- script: |
-    source ./eng/common/tools.sh
-    InitializeDotNetCli true
-    ./.dotnet/dotnet tool restore
-  displayName: Restore darc
-  workingDirectory: $(Agent.BuildDirectory)/installer
-
 - script: |
     git checkout -B ${{ parameters.vmrBranch }}
     echo "##vso[task.setvariable variable=vmrBranch]${{ parameters.vmrBranch }}"
   displayName: Prepare branch ${{ parameters.vmrBranch }}
   workingDirectory: ${{ parameters.vmrPath }}
 
-# This step make sure that darc finds the PR commit (which might be in a fork of installer)
-# The cloned folder does have the commit so we copy it into temp where the tooling will find it
-# We also cannot keep the AzDO remote as pulling from it would require a token
-- script: |
-    cp -r $(Agent.BuildDirectory)/installer $(Agent.TempDirectory)/03298978DFFFCD23
-    cd $(Agent.TempDirectory)/03298978DFFFCD23
-    git remote remove origin
-    git remote add origin https://github.com/dotnet/installer
-  displayName: Prepare dotnet/installer clone
-
 - script: >
-    $(Agent.BuildDirectory)/installer/.dotnet/dotnet darc vmr update
+    ./eng/vmr-sync.sh
     --vmr ${{ parameters.vmrPath }}
     --tmp $(Agent.TempDirectory)
-    --github-pat $(BotAccount-dotnet-bot-repo-PAT)
+    --branch ${{ parameters.vmrBranch }}
-    --recursive
+    --target-ref ${{ parameters.targetRef }}
-    --verbose
-    installer:${{ parameters.targetRef }}
   displayName: Synchronize dotnet/dotnet
   workingDirectory: $(Agent.BuildDirectory)/installer

eng/vmr-sync.sh

@@ -1,7 +1,10 @@
 #!/bin/bash
 
-### This script helps to reproduce potential failures of the 'Synchronize dotnet/dotnet'
+### This script is used for synchronizing the dotnet/dotnet repository locally
-### build step from the **VMR Source-Build** job locally.
+### It is used during CI to ingest new code based on dotnet/installer
+### I can also help for reproducing potential failures during installer's PRs,
+### namely during errors during the 'Synchronize dotnet/dotnet' build step from
+### the 'VMR Source-Build'.
 ### The following scenario is assumed:
 ### - There is a PR in dotnet/installer
 ### - The PR is failing on the 'VMR Source-Build' job in the 'Synchronize dotnet/dotnet' step
@@ -44,6 +47,9 @@
 ###   -b, --branch, --vmr-branch BRANCH_NAME
 ###       Optional. Branch of the 'dotnet/dotnet' repo to synchronize to
 ###       This should match the target branch of the PR, defaults to 'main'
+###   --target-ref GIT_REF
+###       Optional. Git ref to synchronize to. This can be a specific commit, branch, tag..
+###       Defaults to the revision of the parent installer repo
 ###   --debug
 ###       Optional. Turns on the most verbose logging for the VMR tooling
 
@@ -81,9 +87,8 @@ installer_dir="$scriptroot/../"
 tmp_dir=''
 vmr_dir=''
 vmr_branch='main'
+target_ref=''
 verbosity=verbose
-# hashed name coming from the VMR tooling
-INSTALLER_TMP_DIR_NAME='03298978DFFFCD23'
 
 while [[ $# -gt 0 ]]; do
   opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")"
@@ -103,6 +108,10 @@ while [[ $# -gt 0 ]]; do
     -d|--debug)
       verbosity=debug
       ;;
+    --target-ref)
+      target_ref=$2
+      shift
+      ;;
     -h|--help)
       print_help
       exit 0
@@ -135,6 +144,10 @@ if [[ ! -d "$tmp_dir" ]]; then
   mkdir -p "$tmp_dir"
 fi
 
+if [[ "$verbosity" == "debug" ]]; then
+  set -x
+fi
+
 if [[ ! -d "$vmr_dir" ]]; then
   highlight "Cloning 'dotnet/dotnet' into $vmr_dir.."
   git clone https://github.com/dotnet/dotnet "$vmr_dir"
@@ -151,12 +164,6 @@ fi
 
 set -e
 
-# These lines makes sure the temp dir (which the tooling would clone)
-# has the synchronized commit inside as well
-highlight 'Preparing the temporary directory..'
-rm -rf "${tmp_dir:?}/$INSTALLER_TMP_DIR_NAME"
-git clone "$installer_dir" "${tmp_dir:?}/$INSTALLER_TMP_DIR_NAME"
-
 # Prepare darc
 highlight 'Installing .NET, preparing the tooling..'
 source "$scriptroot/common/tools.sh"
@@ -165,15 +172,22 @@ dotnet="$scriptroot/../.dotnet/dotnet"
 "$dotnet" tool restore
 
 # Run the sync
-target_sha=$(git -C "$installer_dir" rev-parse HEAD)
+if [[ -z "$target_ref" ]]; then
-highlight "Starting the synchronization to $target_sha.."
+  target_ref=$(git -C "$installer_dir" rev-parse HEAD)
+fi
+
+highlight "Starting the synchronization to '$target_ref'.."
 set +e
 
-if "$dotnet" darc vmr update --vmr "$vmr_dir" --tmp "$tmp_dir" --$verbosity --recursive installer:$target_sha; then
+# Temporary workaround while we fix fetching commits
+rm -rf "$tmp_dir/installer"
+cp -r "$installer_dir" "$tmp_dir/installer"
+
+if "$dotnet" darc vmr update --vmr "$vmr_dir" --tmp "$tmp_dir" --$verbosity --recursive --additional-remotes "installer:$installer_dir" "installer:$target_ref"; then
   highlight "Synchronization succeeded"
 else
-  fail "Synchronization of dotnet/dotnet to $target_sha failed!"
+  fail "Synchronization of dotnet/dotnet to '$target_ref' failed!"
-  fail "$vmr_dir is left in its last state (re-run of this script will reset it)."
+  fail "'$vmr_dir' is left in its last state (re-run of this script will reset it)."
   fail "Please inspect the logs which contain path to the failing patch file (use --debug to get all the details)."
   fail "Once you make changes to the conflicting VMR patch, commit it locally and re-run this script."
   exit 1

src/VirtualMonoRepo/InitializeVMR.proj

@@ -7,16 +7,6 @@
     <VmrDir Condition=" '$(VmrDir)' != '' ">$([MSBuild]::EnsureTrailingSlash('$(VmrDir)'))</VmrDir>
     <TmpDir Condition=" '$(TmpDir)' == '' ">$(ArtifactsDir)tmp/</TmpDir>
     <VmrDefaultBranch Condition=" '$(VmrDefaultBranch)' == '' ">main</VmrDefaultBranch>
-
-    <!-- We set this for the source build targets to work well -->
-    <TarballDir>$(VmrDir)</TarballDir>
-    <InitializeVMR>true</InitializeVMR>
-  </PropertyGroup>
-
-  <!-- Pull back in some properties from the tarball defaults -->
-  <PropertyGroup>
-    <VmrGitInfoDir>$(TarballGitInfoDir)</VmrGitInfoDir>
-    <VmrSourceDir>$(TarballSourceDir)</VmrSourceDir>
   </PropertyGroup>
 
   <UsingTask TaskName="Microsoft.DotNet.VirtualMonoRepo.Tasks.VirtualMonoRepo_Initialize" AssemblyFile="$(VirtualMonoRepoTasksAssembly)" />
@@ -36,14 +26,11 @@
   <Target Name="InitializeCleanVmr">
     <RemoveDir Directories='$(VmrDir)' Condition=" EXISTS('$(VmrDir)') " />
     <MakeDir Directories="$(VmrDir)" />
-    <MakeDir Directories="$(VmrGitInfoDir)" Condition=" '$(VmrGitInfoDir)' != '' " />
     <MakeDir Directories="$(TmpDir)" Condition=" !EXISTS('$(TmpDir)') " />
-    <MakeDir Directories="$(VmrDir)/src" />
     <Copy SourceFiles="$(MSBuildThisFileDirectory)/source-mappings.json" DestinationFolder="$(VmrDir)/src" />
 
     <Exec WorkingDirectory="$(VmrDir)" Command="git init -b $(VmrDefaultBranch)" />
-    <Exec WorkingDirectory="$(VmrDir)" Command="git add -A" />
+    <Exec WorkingDirectory="$(VmrDir)" Command="git commit --allow-empty -m 'Initial commit of the VMR%0D%0A%0D%0A[[ commit created by automation ]]'" />
-    <Exec WorkingDirectory="$(VmrDir)" Command="git commit -m 'Initial commit of the VMR%0D%0A%0D%0A[[ commit created by automation ]]'" />
   </Target>
 
   <Target Name="InitializeRepoAndDependentsRecursive">
@@ -68,7 +55,8 @@
       PackageVersion="8.0.100"
       Recursive="true"
       VmrPath="$(VmrDir)"
-      TmpPath="$(TmpDir)" />
+      TmpPath="$(TmpDir)"
+      SourceMappingsPath="$(MSBuildThisFileDirectory)/source-mappings.json" />
 
     <Message Text=" -> Done initializing individual repositories recursively" Importance="High" />
   </Target>

src/VirtualMonoRepo/Tasks/VirtualMonoRepo_Initialize.cs

@@ -7,6 +7,7 @@ using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.Build.Framework;
+using Microsoft.DotNet.DarcLib.Helpers;
 using Microsoft.DotNet.DarcLib.VirtualMonoRepo;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
@@ -26,6 +27,9 @@ public class VirtualMonoRepo_Initialize : Build.Utilities.Task, ICancelableTask
     [Required]
     public string Repository { get; set; }
 
+    [Required]
+    public string SourceMappingsPath { get; set; }
+
     [Required]
     public string VmrPath { get; set; }
 
@@ -51,7 +55,14 @@ public class VirtualMonoRepo_Initialize : Build.Utilities.Task, ICancelableTask
         TmpPath = Path.GetFullPath(TmpPath);
 
         var vmrInitializer = _serviceProvider.Value.GetRequiredService<IVmrInitializer>();
-        await vmrInitializer.InitializeRepository(Repository, Revision, PackageVersion, Recursive, _cancellationToken.Token);
+        await vmrInitializer.InitializeRepository(
+            Repository,
+            Revision,
+            PackageVersion,
+            Recursive,
+            new NativePath(SourceMappingsPath),
+            Array.Empty<AdditionalRemote>(),
+            _cancellationToken.Token);
         return true;
     }
 

src/VirtualMonoRepo/source-mappings.json

@@ -12,6 +12,8 @@
     // These patches are applied on top of the code synchronized into the VMR
     "patchesPath": "src/installer/src/SourceBuild/patches",
 
+    "sourceMappingsPath": "src/installer/src/VirtualMonoRepo/source-mappings.json",
+
     // Some files are copied outside of the src/ directory into other locations
     // When files in the source paths are changed, they are automatically synchronized too
     "additionalMappings": [
@@ -22,6 +24,10 @@
         {
             "source": "src/installer/eng/common",
             "destination": "eng/common"
+        },
+        {
+            "source": "src/installer/src/VirtualMonoRepo/source-mappings.json",
+            "destination": "src"
         }
     ],