[release/6.0.3xx] Handle NuGet package source mapping in source-build tasks. (#13733)

src/SourceBuild/Arcade/eng/common/templates/job/source-build-run-tarball-build.yml

@@ -44,13 +44,13 @@ jobs:
         _EnablePoison: false
         _ExcludeOmniSharpTests: true
         _RunOnline: true
-      CentOS7-Offline:
-        _BootstrapPrep: false
-        _Container: ${{ parameters.centOS7Container }}
-        _EnablePoison: false
-        _ExcludeOmniSharpTests: true
-        _RunOnline: false
       ${{ if ne(variables['Build.Reason'], 'PullRequest') }}:
+        CentOS7-Offline:
+          _BootstrapPrep: false
+          _Container: ${{ parameters.centOS7Container }}
+          _EnablePoison: false
+          _ExcludeOmniSharpTests: true
+          _RunOnline: false
         CentOS8-Offline:
           _BootstrapPrep: false
           _Container: ${{ parameters.centOS8Container }}

src/SourceBuild/tarball/content/repos/Directory.Build.targets

@@ -11,6 +11,7 @@
   <UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="RemoveInternetSourcesFromNuGetConfig" />
   <UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="ReplaceFeedsInNuGetConfig" />
   <UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="UpdateJson" />
+  <UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="UpdateNuGetConfigPackageSourcesMappings" />
   <UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="ValidateUsageAgainstBaseline" />
   <UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="WriteBuildOutputProps" />
   <UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="WritePackageUsageData" />
@@ -176,6 +177,19 @@
           Condition="'$(NuGetConfigFile)' != '' OR '@(NuGetConfigFiles)' != ''"
           Inputs="$(MSBuildProjectFullPath)"
           Outputs="$(RepoCompletedSemaphorePath)UpdateNuGetConfig.complete">
+
+    <PropertyGroup>
+      <PrebuiltNuGetSourceName>prebuilt</PrebuiltNuGetSourceName>
+      <PreviouslySourceBuiltNuGetSourceName>previously-source-built</PreviouslySourceBuiltNuGetSourceName>
+      <ReferencePackagesNuGetSourceName>reference-packages</ReferencePackagesNuGetSourceName>
+      <SourceBuiltNuGetSourceName>source-built</SourceBuiltNuGetSourceName>
+      <ExtraSourcesNuGetSourceName>ExtraSources</ExtraSourcesNuGetSourceName>
+      <DotNet5InternalTransportNuGetSourceName>dotnet5-internal-transport</DotNet5InternalTransportNuGetSourceName>
+      <SourceBuildSources>$(PrebuiltNuGetSourceName);$(PreviouslySourceBuiltNuGetSourceName);$(ReferencePackagesNuGetSourceName);$(SourceBuiltNuGetSourceName)</SourceBuildSources>
+      <SourceBuildSources Condition="'$(ExtraRestoreSourcePath)' != ''">$(SourceBuildSources);$(ExtraSourcesNuGetSourceName)</SourceBuildSources>
+      <SourceBuildSources Condition="'$(VSS_NUGET_EXTERNAL_FEED_ENDPOINTS)' != '' and '$(SetUpInternalTransportFeed)' == 'true'">$(SourceBuildSources);$(DotNet5InternalTransportNuGetSourceName)</SourceBuildSources>
+    </PropertyGroup>
+
     <!-- Update the detected or manually specified NuGetConfigFile, but also allow multiple. -->
     <ItemGroup>
       <NuGetConfigFiles Include="$(NuGetConfigFile)" />
@@ -187,26 +201,26 @@
       KeepFeedPrefixes="@(KeepFeedPrefixes)" />
 
     <AddSourceToNuGetConfig NuGetConfigFile="%(NuGetConfigFiles.Identity)"
-                            SourceName="prebuilt"
+                            SourceName="$(PrebuiltNuGetSourceName)"
                             SourcePath="$(PrebuiltPackagesPath)"
                             />
 
     <AddSourceToNuGetConfig NuGetConfigFile="%(NuGetConfigFiles.Identity)"
-                            SourceName="previously-source-built"
+                            SourceName="$(PreviouslySourceBuiltNuGetSourceName)"
                             SourcePath="$(PrebuiltSourceBuiltPackagesPath)"
                             />
 
     <AddSourceToNuGetConfig NuGetConfigFile="%(NuGetConfigFiles.Identity)"
-                            SourceName="reference-packages"
+                            SourceName="$(ReferencePackagesNuGetSourceName)"
                             SourcePath="$(ReferencePackagesDir)"
                             />
 
     <AddSourceToNuGetConfig NuGetConfigFile="%(NuGetConfigFiles.Identity)"
-                            SourceName="source-built"
+                            SourceName="$(SourceBuiltNuGetSourceName)"
                             SourcePath="$(SourceBuiltPackagesPath)" />
 
     <AddSourceToNuGetConfig NuGetConfigFile="%(NuGetConfigFiles.Identity)"
-                            SourceName="ExtraSources"
+                            SourceName="$(ExtraSourcesNuGetSourceName)"
                             SourcePath="$(ExtraRestoreSourcePath)"
                             Condition="'$(ExtraRestoreSourcePath)' != ''" />
 
@@ -219,9 +233,14 @@
         '$(VSS_NUGET_EXTERNAL_FEED_ENDPOINTS)' != '' and
         '$(SetUpInternalTransportFeed)' == 'true'"
       NuGetConfigFile="%(NuGetConfigFiles.Identity)"
-      SourceName="dotnet5-internal-transport"
+      SourceName="$(DotNet5InternalTransportNuGetSourceName)"
       SourcePath="https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal-transport/nuget/v3/index.json" />
 
+    <UpdateNuGetConfigPackageSourcesMappings
+      NuGetConfigFile="%(NuGetConfigFiles.Identity)"
+      BuildWithOnlineSources="$(BuildWithOnlineSources)"
+      SourceBuildSources="$(SourceBuildSources)" />
+
     <!-- Update NuGet.Config files that have deprecated myget feeds -->
     <ItemGroup>
       <LegacyFeedMapping

src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UpdateNuGetConfigPackageSourcesMappings.cs

@@ -0,0 +1,91 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Xml;
+using System.Xml.Linq;
+using Microsoft.Build.Framework;
+using Microsoft.Build.Utilities;
+
+namespace Microsoft.DotNet.Build.Tasks
+{
+    /*
+     * This task updates the package source mappings in the NuGet.Config.
+     * If package source mappings are used, source-build packages sources will be added with the cumulative package patterns
+     * for all of the existing package sources. When building offline, the existing package source mappings will be removed;
+     * otherwise they will be preserved after the source-build sources.
+     */
+    public class UpdateNuGetConfigPackageSourcesMappings : Task
+    {
+        [Required]
+        public string NuGetConfigFile { get; set; }
+
+        /// <summary>
+        /// Whether to work in offline mode (remove all internet sources) or online mode (remove only authenticated sources)
+        /// </summary>
+        public bool BuildWithOnlineSources { get; set; }
+
+        /// <summary>
+        /// A list of all source-build specific NuGet sources.
+        /// </summary>
+        public string[] SourceBuildSources { get; set; }
+
+        public override bool Execute()
+        {
+            string xml = File.ReadAllText(NuGetConfigFile);
+            string newLineChars = FileUtilities.DetectNewLineChars(xml);
+            XDocument document = XDocument.Parse(xml);
+            XElement pkgSrcMappingElement = document.Root.Descendants().FirstOrDefault(e => e.Name == "packageSourceMapping");
+
+            if (pkgSrcMappingElement == null)
+            {
+                return true;
+            }
+
+            // Union all package sources to get the distinct list.  These will get added to the source-build sources.
+            IEnumerable<string> packagePatterns = pkgSrcMappingElement.Descendants()
+                .Where(e => e.Name == "packageSource")
+                .SelectMany(e => e.Descendants().Where(e => e.Name == "package"))
+                .Select(e => e.Attribute("pattern").Value)
+                .Distinct();
+
+            if (!BuildWithOnlineSources)
+            {
+                // When building offline remove all packageSourceMappings.
+                pkgSrcMappingElement?.ReplaceNodes(new XElement("clear"));
+            }
+
+            XElement pkgSrcMappingClearElement = pkgSrcMappingElement.Descendants().FirstOrDefault(e => e.Name == "clear");
+
+            foreach (string packageSource in SourceBuildSources)
+            {
+                XElement pkgSrc = new XElement("packageSource", new XAttribute("key", packageSource));
+                foreach (string packagePattern in packagePatterns)
+                {
+                    pkgSrc.Add(new XElement("package", new XAttribute("pattern", packagePattern)));
+                }
+
+                if (pkgSrcMappingClearElement != null)
+                {
+                    pkgSrcMappingClearElement.AddAfterSelf(pkgSrc);
+                }
+                else
+                {
+                    pkgSrcMappingElement.AddFirst(pkgSrc);
+                    pkgSrcMappingElement.AddFirst(new XElement("clear"));
+                }
+            }
+
+            using (var writer = XmlWriter.Create(NuGetConfigFile, new XmlWriterSettings { NewLineChars = newLineChars, Indent = true }))
+            {
+                document.Save(writer);
+            }
+
+            return true;
+        }
+    }
+}

src/SourceBuild/tarball/patches/nuget-client/0001-Pin-Microsoft.Build-and-Microsoft.Extensions.Command.patch

@@ -0,0 +1,33 @@
+From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
+From: MichaelSimons <msimons@microsoft.com>
+Date: Wed, 11 May 2022 21:16:46 +0000
+Subject: [PATCH] Pin Microsoft.Build and
+ Microsoft.Extensions.CommandLineUtils.Sources versions in source-build
+
+Update package version reference to not pick up the source-build PVP versions and instead utilize SBRP versions 
+so that source-build builds closer to the normal repo build.
+---
+ build/packages.targets | 4 ++--
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+
+diff --git a/build/packages.targets b/build/packages.targets
+index 0ca4b1309..dc3db5b8e 100644
+--- a/build/packages.targets
++++ b/build/packages.targets
+@@ -1,6 +1,6 @@
+ <Project>
+     <PropertyGroup>
+-        <MicrosoftBuildPackageVersion Condition="'$(MicrosoftBuildPackageVersion)' == ''">16.8.0</MicrosoftBuildPackageVersion>
++        <MicrosoftBuildPackageVersion>16.8.0</MicrosoftBuildPackageVersion>
+         <NewtonsoftJsonPackageVersion Condition="$(NewtonsoftJsonPackageVersion) == ''">13.0.1</NewtonsoftJsonPackageVersion>
+         <MicrosoftWebXdtPackageVersion Condition="'$(MicrosoftWebXdtPackageVersion)' == ''">3.0.0</MicrosoftWebXdtPackageVersion>
+         <SystemComponentModelCompositionPackageVersion Condition="'$(SystemComponentModelCompositionPackageVersion)' == ''">4.5.0</SystemComponentModelCompositionPackageVersion>
+@@ -10,7 +10,7 @@
+         <CryptographyPackagesVersion>5.0.0</CryptographyPackagesVersion>
+         <NuGetCoreV2Version>2.14.0-rtm-832</NuGetCoreV2Version>
+         <ProjectSystemManagedVersion>17.0.0-beta1-10402-05</ProjectSystemManagedVersion>
+-        <MicrosoftExtensionsCommandLineUtilsSourcesPackageVersion Condition="'$(MicrosoftExtensionsCommandLineUtilsSourcesPackageVersion)' == ''">3.0.0-preview6.19253.5</MicrosoftExtensionsCommandLineUtilsSourcesPackageVersion>
++        <MicrosoftExtensionsCommandLineUtilsSourcesPackageVersion>3.0.0-preview6.19253.5</MicrosoftExtensionsCommandLineUtilsSourcesPackageVersion>
+     </PropertyGroup>
+ 
+     <!-- Test and package versions -->