[main] Handle NuGet package source mapping in source-build tasks. (#13788)

2022-05-13 14:52:42 -05:00 · 2022-05-13 14:52:42 -05:00 · b648ad31b1
commit b648ad31b1
parent f05c1cde28
4 changed files with 150 additions and 7 deletions
--- a/src/SourceBuild/tarball/content/repos/Directory.Build.targets
+++ b/src/SourceBuild/tarball/content/repos/Directory.Build.targets
@ -11,6 +11,7 @@
  <UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="RemoveInternetSourcesFromNuGetConfig" />
  <UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="ReplaceFeedsInNuGetConfig" />
  <UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="UpdateJson" />
+  <UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="UpdateNuGetConfigPackageSourcesMappings" />
  <UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="ValidateUsageAgainstBaseline" />
  <UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="WriteBuildOutputProps" />
  <UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="WritePackageUsageData" />
@ -176,6 +177,19 @@
          Condition="'$(NuGetConfigFile)' != '' OR '@(NuGetConfigFiles)' != ''"
          Inputs="$(MSBuildProjectFullPath)"
          Outputs="$(RepoCompletedSemaphorePath)UpdateNuGetConfig.complete">
+
+    <PropertyGroup>
+      <PrebuiltNuGetSourceName>prebuilt</PrebuiltNuGetSourceName>
+      <PreviouslySourceBuiltNuGetSourceName>previously-source-built</PreviouslySourceBuiltNuGetSourceName>
+      <ReferencePackagesNuGetSourceName>reference-packages</ReferencePackagesNuGetSourceName>
+      <SourceBuiltNuGetSourceName>source-built</SourceBuiltNuGetSourceName>
+      <ExtraSourcesNuGetSourceName>ExtraSources</ExtraSourcesNuGetSourceName>
+      <DotNet5InternalTransportNuGetSourceName>dotnet5-internal-transport</DotNet5InternalTransportNuGetSourceName>
+      <SourceBuildSources>$(PrebuiltNuGetSourceName);$(PreviouslySourceBuiltNuGetSourceName);$(ReferencePackagesNuGetSourceName);$(SourceBuiltNuGetSourceName)</SourceBuildSources>
+      <SourceBuildSources Condition="'$(ExtraRestoreSourcePath)' != ''">$(SourceBuildSources);$(ExtraSourcesNuGetSourceName)</SourceBuildSources>
+      <SourceBuildSources Condition="'$(VSS_NUGET_EXTERNAL_FEED_ENDPOINTS)' != '' and '$(SetUpInternalTransportFeed)' == 'true'">$(SourceBuildSources);$(DotNet5InternalTransportNuGetSourceName)</SourceBuildSources>
+    </PropertyGroup>
+
    <!-- Update the detected or manually specified NuGetConfigFile, but also allow multiple. -->
    <ItemGroup>
      <NuGetConfigFiles Include="$(NuGetConfigFile)" />
@ -187,26 +201,26 @@
      KeepFeedPrefixes="@(KeepFeedPrefixes)" />

    <AddSourceToNuGetConfig NuGetConfigFile="%(NuGetConfigFiles.Identity)"
-                            SourceName="prebuilt"
+                            SourceName="$(PrebuiltNuGetSourceName)"
                            SourcePath="$(PrebuiltPackagesPath)"
                            />

    <AddSourceToNuGetConfig NuGetConfigFile="%(NuGetConfigFiles.Identity)"
-                            SourceName="previously-source-built"
+                            SourceName="$(PreviouslySourceBuiltNuGetSourceName)"
                            SourcePath="$(PrebuiltSourceBuiltPackagesPath)"
                            />

    <AddSourceToNuGetConfig NuGetConfigFile="%(NuGetConfigFiles.Identity)"
-                            SourceName="reference-packages"
+                            SourceName="$(ReferencePackagesNuGetSourceName)"
                            SourcePath="$(ReferencePackagesDir)"
                            />

    <AddSourceToNuGetConfig NuGetConfigFile="%(NuGetConfigFiles.Identity)"
-                            SourceName="source-built"
+                            SourceName="$(SourceBuiltNuGetSourceName)"
                            SourcePath="$(SourceBuiltPackagesPath)" />

    <AddSourceToNuGetConfig NuGetConfigFile="%(NuGetConfigFiles.Identity)"
-                            SourceName="ExtraSources"
+                            SourceName="$(ExtraSourcesNuGetSourceName)"
                            SourcePath="$(ExtraRestoreSourcePath)"
                            Condition="'$(ExtraRestoreSourcePath)' != ''" />

@ -219,9 +233,14 @@
        '$(VSS_NUGET_EXTERNAL_FEED_ENDPOINTS)' != '' and
        '$(SetUpInternalTransportFeed)' == 'true'"
      NuGetConfigFile="%(NuGetConfigFiles.Identity)"
-      SourceName="dotnet5-internal-transport"
+      SourceName="$(DotNet5InternalTransportNuGetSourceName)"
      SourcePath="https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal-transport/nuget/v3/index.json" />

+    <UpdateNuGetConfigPackageSourcesMappings
+      NuGetConfigFile="%(NuGetConfigFiles.Identity)"
+      BuildWithOnlineSources="$(BuildWithOnlineSources)"
+      SourceBuildSources="$(SourceBuildSources)" />
+
    <!-- Update NuGet.Config files that have deprecated myget feeds -->
    <ItemGroup>
      <LegacyFeedMapping
--- a/src/SourceBuild/tarball/content/repos/known-good.proj
+++ b/src/SourceBuild/tarball/content/repos/known-good.proj
@ -53,7 +53,7 @@
        <!-- <RepositoryReference Include="aspnetcore" /> -->
        <RepositoryReference Include="deployment-tools" />
        <!-- <RepositoryReference Include="format" /> -->
-        <!-- <RepositoryReference Include="nuget-client" /> -->
+        <RepositoryReference Include="nuget-client" />
        <RepositoryReference Include="templating" />
        <RepositoryReference Include="test-templates" />

--- a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UpdateNuGetConfigPackageSourcesMappings.cs
+++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UpdateNuGetConfigPackageSourcesMappings.cs
@ -0,0 +1,91 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Xml;
+using System.Xml.Linq;
+using Microsoft.Build.Framework;
+using Microsoft.Build.Utilities;
+
+namespace Microsoft.DotNet.Build.Tasks
+{
+    /*
+     * This task updates the package source mappings in the NuGet.Config.
+     * If package source mappings are used, source-build packages sources will be added with the cumulative package patterns
+     * for all of the existing package sources. When building offline, the existing package source mappings will be removed;
+     * otherwise they will be preserved after the source-build sources.
+     */
+    public class UpdateNuGetConfigPackageSourcesMappings : Task
+    {
+        [Required]
+        public string NuGetConfigFile { get; set; }
+
+        /// <summary>
+        /// Whether to work in offline mode (remove all internet sources) or online mode (remove only authenticated sources)
+        /// </summary>
+        public bool BuildWithOnlineSources { get; set; }
+
+        /// <summary>
+        /// A list of all source-build specific NuGet sources.
+        /// </summary>
+        public string[] SourceBuildSources { get; set; }
+
+        public override bool Execute()
+        {
+            string xml = File.ReadAllText(NuGetConfigFile);
+            string newLineChars = FileUtilities.DetectNewLineChars(xml);
+            XDocument document = XDocument.Parse(xml);
+            XElement pkgSrcMappingElement = document.Root.Descendants().FirstOrDefault(e => e.Name == "packageSourceMapping");
+
+            if (pkgSrcMappingElement == null)
+            {
+                return true;
+            }
+
+            // Union all package sources to get the distinct list.  These will get added to the source-build sources.
+            IEnumerable<string> packagePatterns = pkgSrcMappingElement.Descendants()
+                .Where(e => e.Name == "packageSource")
+                .SelectMany(e => e.Descendants().Where(e => e.Name == "package"))
+                .Select(e => e.Attribute("pattern").Value)
+                .Distinct();
+
+            if (!BuildWithOnlineSources)
+            {
+                // When building offline remove all packageSourceMappings.
+                pkgSrcMappingElement?.ReplaceNodes(new XElement("clear"));
+            }
+
+            XElement pkgSrcMappingClearElement = pkgSrcMappingElement.Descendants().FirstOrDefault(e => e.Name == "clear");
+
+            foreach (string packageSource in SourceBuildSources)
+            {
+                XElement pkgSrc = new XElement("packageSource", new XAttribute("key", packageSource));
+                foreach (string packagePattern in packagePatterns)
+                {
+                    pkgSrc.Add(new XElement("package", new XAttribute("pattern", packagePattern)));
+                }
+
+                if (pkgSrcMappingClearElement != null)
+                {
+                    pkgSrcMappingClearElement.AddAfterSelf(pkgSrc);
+                }
+                else
+                {
+                    pkgSrcMappingElement.AddFirst(pkgSrc);
+                    pkgSrcMappingElement.AddFirst(new XElement("clear"));
+                }
+            }
+
+            using (var writer = XmlWriter.Create(NuGetConfigFile, new XmlWriterSettings { NewLineChars = newLineChars, Indent = true }))
+            {
+                document.Save(writer);
+            }
+
+            return true;
+        }
+    }
+}
--- a/src/SourceBuild/tarball/patches/nuget-client/0001-Pin-Microsoft.Build-and-Microsoft.Extensions.Command.patch
+++ b/src/SourceBuild/tarball/patches/nuget-client/0001-Pin-Microsoft.Build-and-Microsoft.Extensions.Command.patch
@ -0,0 +1,33 @@
+From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
+From: MichaelSimons <msimons@microsoft.com>
+Date: Wed, 11 May 2022 21:16:46 +0000
+Subject: [PATCH] Pin Microsoft.Build and
+ Microsoft.Extensions.CommandLineUtils.Sources versions in source-build
+
+Update package version reference to not pick up the source-build PVP versions and instead utilize SBRP versions 
+so that source-build builds closer to the normal repo build.
+---
+ build/packages.targets | 4 ++--
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+
+diff --git a/build/packages.targets b/build/packages.targets
+index 0ca4b1309..dc3db5b8e 100644
+--- a/build/packages.targets
+++ b/build/packages.targets
+@@ -1,6 +1,6 @@
+ <Project>
+     <PropertyGroup>
+-        <MicrosoftBuildPackageVersion Condition="'$(MicrosoftBuildPackageVersion)' == ''">16.8.0</MicrosoftBuildPackageVersion>
+        <MicrosoftBuildPackageVersion>16.8.0</MicrosoftBuildPackageVersion>
+         <NewtonsoftJsonPackageVersion Condition="$(NewtonsoftJsonPackageVersion) == ''">13.0.1</NewtonsoftJsonPackageVersion>
+         <MicrosoftWebXdtPackageVersion Condition="'$(MicrosoftWebXdtPackageVersion)' == ''">3.0.0</MicrosoftWebXdtPackageVersion>
+         <SystemComponentModelCompositionPackageVersion Condition="'$(SystemComponentModelCompositionPackageVersion)' == ''">4.5.0</SystemComponentModelCompositionPackageVersion>
+@@ -10,7 +10,7 @@
+         <CryptographyPackagesVersion>5.0.0</CryptographyPackagesVersion>
+         <NuGetCoreV2Version>2.14.0-rtm-832</NuGetCoreV2Version>
+         <ProjectSystemManagedVersion>17.0.0-beta1-10402-05</ProjectSystemManagedVersion>
+-        <MicrosoftExtensionsCommandLineUtilsSourcesPackageVersion Condition="'$(MicrosoftExtensionsCommandLineUtilsSourcesPackageVersion)' == ''">3.0.0-preview6.19253.5</MicrosoftExtensionsCommandLineUtilsSourcesPackageVersion>
+        <MicrosoftExtensionsCommandLineUtilsSourcesPackageVersion>3.0.0-preview6.19253.5</MicrosoftExtensionsCommandLineUtilsSourcesPackageVersion>
+     </PropertyGroup>
+ 
+     <!-- Test and package versions -->