From b648ad31b10960fbfa405e2b95b31b5d6427b2d0 Mon Sep 17 00:00:00 2001 From: Chris Rummel Date: Fri, 13 May 2022 14:52:42 -0500 Subject: [PATCH] [main] Handle NuGet package source mapping in source-build tasks. (#13788) --- .../content/repos/Directory.Build.targets | 31 +++++-- .../tarball/content/repos/known-good.proj | 2 +- ...UpdateNuGetConfigPackageSourcesMappings.cs | 91 +++++++++++++++++++ ...ild-and-Microsoft.Extensions.Command.patch | 33 +++++++ 4 files changed, 150 insertions(+), 7 deletions(-) create mode 100644 src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UpdateNuGetConfigPackageSourcesMappings.cs create mode 100644 src/SourceBuild/tarball/patches/nuget-client/0001-Pin-Microsoft.Build-and-Microsoft.Extensions.Command.patch diff --git a/src/SourceBuild/tarball/content/repos/Directory.Build.targets b/src/SourceBuild/tarball/content/repos/Directory.Build.targets index 102b1e063..6ba562b3c 100644 --- a/src/SourceBuild/tarball/content/repos/Directory.Build.targets +++ b/src/SourceBuild/tarball/content/repos/Directory.Build.targets @@ -11,6 +11,7 @@ + @@ -176,6 +177,19 @@ Condition="'$(NuGetConfigFile)' != '' OR '@(NuGetConfigFiles)' != ''" Inputs="$(MSBuildProjectFullPath)" Outputs="$(RepoCompletedSemaphorePath)UpdateNuGetConfig.complete"> + + + prebuilt + previously-source-built + reference-packages + source-built + ExtraSources + dotnet5-internal-transport + $(PrebuiltNuGetSourceName);$(PreviouslySourceBuiltNuGetSourceName);$(ReferencePackagesNuGetSourceName);$(SourceBuiltNuGetSourceName) + $(SourceBuildSources);$(ExtraSourcesNuGetSourceName) + $(SourceBuildSources);$(DotNet5InternalTransportNuGetSourceName) + + @@ -187,26 +201,26 @@ KeepFeedPrefixes="@(KeepFeedPrefixes)" /> @@ -219,9 +233,14 @@ '$(VSS_NUGET_EXTERNAL_FEED_ENDPOINTS)' != '' and '$(SetUpInternalTransportFeed)' == 'true'" NuGetConfigFile="%(NuGetConfigFiles.Identity)" - SourceName="dotnet5-internal-transport" + SourceName="$(DotNet5InternalTransportNuGetSourceName)" SourcePath="https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal-transport/nuget/v3/index.json" /> + + --> - + diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UpdateNuGetConfigPackageSourcesMappings.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UpdateNuGetConfigPackageSourcesMappings.cs new file mode 100644 index 000000000..9172566b3 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UpdateNuGetConfigPackageSourcesMappings.cs @@ -0,0 +1,91 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Xml; +using System.Xml.Linq; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +namespace Microsoft.DotNet.Build.Tasks +{ + /* + * This task updates the package source mappings in the NuGet.Config. + * If package source mappings are used, source-build packages sources will be added with the cumulative package patterns + * for all of the existing package sources. When building offline, the existing package source mappings will be removed; + * otherwise they will be preserved after the source-build sources. + */ + public class UpdateNuGetConfigPackageSourcesMappings : Task + { + [Required] + public string NuGetConfigFile { get; set; } + + /// + /// Whether to work in offline mode (remove all internet sources) or online mode (remove only authenticated sources) + /// + public bool BuildWithOnlineSources { get; set; } + + /// + /// A list of all source-build specific NuGet sources. + /// + public string[] SourceBuildSources { get; set; } + + public override bool Execute() + { + string xml = File.ReadAllText(NuGetConfigFile); + string newLineChars = FileUtilities.DetectNewLineChars(xml); + XDocument document = XDocument.Parse(xml); + XElement pkgSrcMappingElement = document.Root.Descendants().FirstOrDefault(e => e.Name == "packageSourceMapping"); + + if (pkgSrcMappingElement == null) + { + return true; + } + + // Union all package sources to get the distinct list. These will get added to the source-build sources. + IEnumerable packagePatterns = pkgSrcMappingElement.Descendants() + .Where(e => e.Name == "packageSource") + .SelectMany(e => e.Descendants().Where(e => e.Name == "package")) + .Select(e => e.Attribute("pattern").Value) + .Distinct(); + + if (!BuildWithOnlineSources) + { + // When building offline remove all packageSourceMappings. + pkgSrcMappingElement?.ReplaceNodes(new XElement("clear")); + } + + XElement pkgSrcMappingClearElement = pkgSrcMappingElement.Descendants().FirstOrDefault(e => e.Name == "clear"); + + foreach (string packageSource in SourceBuildSources) + { + XElement pkgSrc = new XElement("packageSource", new XAttribute("key", packageSource)); + foreach (string packagePattern in packagePatterns) + { + pkgSrc.Add(new XElement("package", new XAttribute("pattern", packagePattern))); + } + + if (pkgSrcMappingClearElement != null) + { + pkgSrcMappingClearElement.AddAfterSelf(pkgSrc); + } + else + { + pkgSrcMappingElement.AddFirst(pkgSrc); + pkgSrcMappingElement.AddFirst(new XElement("clear")); + } + } + + using (var writer = XmlWriter.Create(NuGetConfigFile, new XmlWriterSettings { NewLineChars = newLineChars, Indent = true })) + { + document.Save(writer); + } + + return true; + } + } +} diff --git a/src/SourceBuild/tarball/patches/nuget-client/0001-Pin-Microsoft.Build-and-Microsoft.Extensions.Command.patch b/src/SourceBuild/tarball/patches/nuget-client/0001-Pin-Microsoft.Build-and-Microsoft.Extensions.Command.patch new file mode 100644 index 000000000..1d284c47e --- /dev/null +++ b/src/SourceBuild/tarball/patches/nuget-client/0001-Pin-Microsoft.Build-and-Microsoft.Extensions.Command.patch @@ -0,0 +1,33 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: MichaelSimons +Date: Wed, 11 May 2022 21:16:46 +0000 +Subject: [PATCH] Pin Microsoft.Build and + Microsoft.Extensions.CommandLineUtils.Sources versions in source-build + +Update package version reference to not pick up the source-build PVP versions and instead utilize SBRP versions +so that source-build builds closer to the normal repo build. +--- + build/packages.targets | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/build/packages.targets b/build/packages.targets +index 0ca4b1309..dc3db5b8e 100644 +--- a/build/packages.targets ++++ b/build/packages.targets +@@ -1,6 +1,6 @@ + + +- 16.8.0 ++ 16.8.0 + 13.0.1 + 3.0.0 + 4.5.0 +@@ -10,7 +10,7 @@ + 5.0.0 + 2.14.0-rtm-832 + 17.0.0-beta1-10402-05 +- 3.0.0-preview6.19253.5 ++ 3.0.0-preview6.19253.5 + + +