diff --git a/src/SourceBuild/content/Directory.Build.props b/src/SourceBuild/content/Directory.Build.props index 3fe574c31..f8977b54b 100644 --- a/src/SourceBuild/content/Directory.Build.props +++ b/src/SourceBuild/content/Directory.Build.props @@ -119,9 +119,6 @@ $(LocalBlobStorageRoot)Sdk/ $(LocalBlobStorageRoot)Runtime/ $(LocalBlobStorageRoot)aspnetcore/Runtime/ - $(IntermediatePath)RestoreSources.props - $(IntermediatePath)PackageVersions.props - $(IntermediatePath)CurrentSourceBuiltPackageVersions.props $(BaseOutputPath)logs/ $(BaseOutputPath)msbuild-debug/ $(BaseOutputPath)roslyn-debug/ diff --git a/src/SourceBuild/content/eng/tools/init-build.proj b/src/SourceBuild/content/eng/tools/init-build.proj index c2ecc5f35..319333041 100644 --- a/src/SourceBuild/content/eng/tools/init-build.proj +++ b/src/SourceBuild/content/eng/tools/init-build.proj @@ -9,7 +9,6 @@ - @@ -23,7 +22,6 @@ - - - - - $(IntermediatePath)PackageVersions.props - - - - - - -]]> - - - - - @@ -107,16 +86,6 @@ - - - - - - /// Source usage data JSON file. @@ -26,7 +26,7 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.UsageReport public string DataFile { get; set; } /// - /// A set of "PackageVersions.props.pre.{repo}.xml" files. They are analyzed to find + /// A set of "PackageVersions.{repo}.Current.props" files. They are analyzed to find /// packages built during source-build, and which repo built them. This info is added to the /// report. New packages are associated to a repo by going through each PVP in ascending /// file modification order. @@ -114,7 +114,7 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.UsageReport string id = usage.PackageIdentity.Id; string version = usage.PackageIdentity.Version.OriginalVersion; - string pvpIdent = WriteBuildOutputProps.GetPropertyName(id); + string pvpIdent = WritePackageVersionsProps.GetPropertyName(id, WritePackageVersionsProps.VersionPropertySuffix); var sourceBuildCreator = new StringBuilder(); foreach (RepoOutput output in sourceBuildRepoOutputs) @@ -200,7 +200,7 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.UsageReport // Get the creation time element. ?.Element(snapshot.Xml .GetDefaultNamespace() - .GetName(WriteBuildOutputProps.CreationTimePropertyName)) + .GetName(WritePackageVersionsProps.CreationTimePropertyName)) ?.Value; if (string.IsNullOrEmpty(creationTime)) diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/WriteBuildOutputProps.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/WriteBuildOutputProps.cs deleted file mode 100644 index fb1d2fe4d..000000000 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/WriteBuildOutputProps.cs +++ /dev/null @@ -1,142 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; -using NuGet.Packaging; -using NuGet.Packaging.Core; -using NuGet.Versioning; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text; -using System.Text.RegularExpressions; - -namespace Microsoft.DotNet.Build.Tasks -{ - public class WriteBuildOutputProps : Task - { - private static readonly Regex InvalidElementNameCharRegex = new Regex(@"(^|[^A-Za-z0-9])(?.)"); - - public const string CreationTimePropertyName = "BuildOutputPropsCreationTime"; - - [Required] - public ITaskItem[] NuGetPackages { get; set; } - - [Required] - public string OutputPath { get; set; } - - /// - /// Adds a second PropertyGroup to the output XML containing a property with the time of - /// creation in UTC DateTime Ticks. This can be used to track creation time in situations - /// where file metadata isn't reliable or preserved. - /// - public bool IncludeCreationTimeProperty { get; set; } - - /// - /// Properties to add to the build output props, which may not exist as nupkgs. - /// FOr example, this is used to pass the version of the CLI toolset archives. - /// - /// %(Identity): Package identity. - /// %(Version): Package version. - /// - public ITaskItem[] ExtraProperties { get; set; } - - /// - /// Additional assets to be added to the build output props. - /// i.e. /bin/obj/x64/Release/blobs/Toolset/3.0.100 - /// This parameter is the / portion only, and the asset - /// must be in a / folder. - /// - public string[] AdditionalAssetDirs { get; set; } - - public override bool Execute() - { - PackageIdentity[] latestPackages = NuGetPackages - .Select(item => - { - using (var reader = new PackageArchiveReader(item.GetMetadata("FullPath"))) - { - return reader.GetIdentity(); - } - }) - .GroupBy(identity => identity.Id) - .Select(g => g.OrderBy(id => id.Version).Last()) - .OrderBy(id => id.Id) - .ToArray(); - - var additionalAssets = (AdditionalAssetDirs ?? new string[0]) - .Where(Directory.Exists) - .Where(dir => Directory.GetDirectories(dir).Count() > 0) - .Select(dir => new { - Name = new DirectoryInfo(dir).Name + "Version", - Version = new DirectoryInfo(Directory.EnumerateDirectories(dir).OrderBy(s => s).Last()).Name - }).ToArray(); - - Directory.CreateDirectory(Path.GetDirectoryName(OutputPath)); - - using (var outStream = File.Open(OutputPath, FileMode.Create)) - using (var sw = new StreamWriter(outStream, new UTF8Encoding(false))) - { - sw.WriteLine(@""); - sw.WriteLine(@""); - sw.WriteLine(@" "); - foreach (PackageIdentity packageIdentity in latestPackages) - { - string propertyName = GetPropertyName(packageIdentity.Id); - sw.WriteLine($" <{propertyName}>{packageIdentity.Version}"); - - propertyName = GetAlternatePropertyName(packageIdentity.Id); - sw.WriteLine($" <{propertyName}>{packageIdentity.Version}"); - } - foreach (var extraProp in ExtraProperties ?? Enumerable.Empty()) - { - string propertyName = extraProp.GetMetadata("Identity"); - bool doNotOverwrite = false; - string overwriteCondition = string.Empty; - if (bool.TryParse(extraProp.GetMetadata("DoNotOverwrite"), out doNotOverwrite) && doNotOverwrite) - { - overwriteCondition = $" Condition=\"'$({propertyName})' == ''\""; - } - sw.WriteLine($" <{propertyName}{overwriteCondition}>{extraProp.GetMetadata("Version")}"); - } - foreach (var additionalAsset in additionalAssets) - { - sw.WriteLine($" <{additionalAsset.Name}>{additionalAsset.Version}"); - } - sw.WriteLine(@" "); - if (IncludeCreationTimeProperty) - { - sw.WriteLine(@" "); - sw.WriteLine($@" <{CreationTimePropertyName}>{DateTime.UtcNow.Ticks}"); - sw.WriteLine(@" "); - } - sw.WriteLine(@""); - } - - return true; - } - - public static string GetPropertyName(string id) - { - string formattedId = InvalidElementNameCharRegex.Replace( - id, - match => match.Groups?["FirstPartChar"].Value.ToUpperInvariant() - ?? string.Empty); - - return $"{formattedId}PackageVersion"; - } - - public static string GetAlternatePropertyName(string id) - { - string formattedId = InvalidElementNameCharRegex.Replace( - id, - match => match.Groups?["FirstPartChar"].Value.ToUpperInvariant() - ?? string.Empty); - - return $"{formattedId}Version"; - } - } -} diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/WritePackageVersionProps.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/WritePackageVersionProps.cs new file mode 100644 index 000000000..bfc98e487 --- /dev/null +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/WritePackageVersionProps.cs @@ -0,0 +1,326 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using NuGet.Packaging; +using NuGet.Packaging.Core; +using NuGet.Versioning; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using System.Xml; + +namespace Microsoft.DotNet.Build.Tasks +{ + public class VersionEntry + { + public string Name; + public string Version; + } + + /// + /// Creates a props file that is used as the input for a repo-level build. The props file + /// includes package version numbers that should be used by the repo build and additional special properties. + /// + /// There are two types of input props that can be written: + /// - Versions of union of all packages produced by the builds are added. (AllPackages) + /// - Only versions of packages that are listed as dependencies of a repo are added. (DependenciesOnly) + /// + /// The former represents the current way that source build works for most repos. The latter represents the desired + /// methodology (PVP Flow). PVP flow closely matches how the product is built in non-source-build mode. + /// + public class WritePackageVersionsProps : Microsoft.Build.Utilities.Task + { + private static readonly Regex InvalidElementNameCharRegex = new Regex(@"(^|[^A-Za-z0-9])(?.)"); + + public const string CreationTimePropertyName = "BuildOutputPropsCreationTime"; + public const string VersionPropertySuffix = "Version"; + private const string VersionPropertyAlternateSuffix = "PackageVersion"; + private const string PinnedAttributeName = "Pinned"; + private const string DependencyAttributeName = "Dependency"; + private const string NameAttributeName = "Name"; + + private const string AllPackagesVersionPropsFlowType = "AllPackages"; + private const string DependenciesOnlyVersionPropsFlowType = "DependenciesOnly"; + private const string DefaultVersionPropsFlowType = AllPackagesVersionPropsFlowType; + + /// + /// Set of input nuget package files to generate version properties for. + /// + [Required] + public ITaskItem[] NuGetPackages { get; set; } + + /// + /// File where the version properties should be written. + /// + [Required] + public string OutputPath { get; set; } + + /// + /// Adds a second PropertyGroup to the output XML containing a property with the time of + /// creation in UTC DateTime Ticks. This can be used to track creation time in situations + /// where file metadata isn't reliable or preserved. + /// + public bool IncludeCreationTimeProperty { get; set; } + + /// + /// Properties to add to the build output props, which may not exist as nupkgs. + /// FOr example, this is used to pass the version of the CLI toolset archives. + /// + /// %(Identity): Package identity. + /// %(Version): Package version. + /// + public ITaskItem[] ExtraProperties { get; set; } + + /// + /// Additional assets to be added to the build output props. + /// i.e. /bin/obj/x64/Release/blobs/Toolset/3.0.100 + /// This parameter is the / portion only, and the asset + /// must be in a / folder. + /// + public string[] AdditionalAssetDirs { get; set; } + + /// + /// Indicates which properties will be written into the Version props file. + /// If AllPackages (Default), all packages from previously built repos will be written. + /// If DependenciesOnly, then only those packages appearing as dependencies in + /// Version.Details.xml will show up. The VersionsDetails property must be set to a + /// valid Version.Details.xml path when DependenciesOnly is used. + /// + public string VersionPropsFlowType { get; set; } = DefaultVersionPropsFlowType; + + /// + /// If VersionPropsFlowType is set to DependenciesOnly, should be the path to the Version.Detail.xml file for the repo. + /// + public string VersionDetails { get; set; } + + /// + /// Retrieve the set of the dependencies from the repo's Version.Details.Xml file. + /// + /// Hash set of dependency names. + private HashSet GetDependences() + { + XmlDocument document = new XmlDocument(); + + try + { + document.Load(VersionDetails); + } + catch (Exception e) + { + Log.LogErrorFromException(e); + return null; + } + + HashSet dependencyNames = new HashSet(StringComparer.OrdinalIgnoreCase); + + // Load the nodes, filter those that are not pinned, and + XmlNodeList dependencyNodes = document.DocumentElement.SelectNodes($"//{DependencyAttributeName}"); + + foreach (XmlNode dependency in dependencyNodes) + { + if (dependency.NodeType == XmlNodeType.Comment || dependency.NodeType == XmlNodeType.Whitespace) + { + continue; + } + + bool isPinned = false; + XmlAttribute pinnedAttribute = dependency.Attributes[PinnedAttributeName]; + if (pinnedAttribute != null && !bool.TryParse(pinnedAttribute.Value, out isPinned)) + { + Log.LogError($"The '{PinnedAttributeName}' attribute is set but the value " + + $"'{pinnedAttribute.Value}' is not a valid boolean..."); + return null; + } + + if (isPinned) + { + continue; + } + + var name = dependency.Attributes[NameAttributeName]?.Value?.Trim(); + + if (string.IsNullOrEmpty(name)) + { + Log.LogError($"The '{NameAttributeName}' attribute must be specified."); + return null; + } + + dependencyNames.Add(name); + } + + return dependencyNames; + } + + /// + /// Filter a set of input dependencies to those that appear in + /// + /// Input set of entries + /// Set of dependencies + /// Set of that appears in + private IEnumerable FilterNonDependencies(IEnumerable input, HashSet dependencies) + { + return input.Where(entry => dependencies.Contains(entry.Name)); + } + + public override bool Execute() + { + if (VersionPropsFlowType != AllPackagesVersionPropsFlowType && + VersionPropsFlowType != DependenciesOnlyVersionPropsFlowType) + { + Log.LogError($"Valid version flow types are '{DependenciesOnlyVersionPropsFlowType}' and '{AllPackagesVersionPropsFlowType}'"); + return !Log.HasLoggedErrors; + } + + if (VersionPropsFlowType == DependenciesOnlyVersionPropsFlowType && (string.IsNullOrEmpty(VersionDetails) || !File.Exists(VersionDetails))) + { + Log.LogError($"When version flow type is DependenciesOnly, the VersionDetails task parameter must point to a valid path to the Version.Details.xml file for the repo. " + + "Provided file path '{VersionDetails}' does not exist."); + return !Log.HasLoggedErrors; + } + + // First, obtain version information from the packages and additional assets that + // are provided. + var latestPackages = NuGetPackages + .Select(item => + { + using (var reader = new PackageArchiveReader(item.GetMetadata("FullPath"))) + { + return reader.GetIdentity(); + } + }) + .GroupBy(identity => identity.Id) + .Select(g => g.OrderBy(id => id.Version).Last()) + .OrderBy(id => id.Id) + .Select(identity => new VersionEntry() + { + Name = identity.Id, + Version = identity.Version.ToString() + }); + + var additionalAssets = (AdditionalAssetDirs ?? new string[0]) + .Where(Directory.Exists) + .Where(dir => Directory.GetDirectories(dir).Count() > 0) + .Select(dir => new VersionEntry() + { + Name = new DirectoryInfo(dir).Name, + Version = new DirectoryInfo(Directory.EnumerateDirectories(dir).OrderBy(s => s).Last()).Name + }); + + var packageElementsToWrite = latestPackages; + var additionalAssetElementsToWrite = additionalAssets; + + // Then, if version flow type is "DependenciesOnly", filter those + // dependencies that do not appear in the version.details.xml file. + if (VersionPropsFlowType == DependenciesOnlyVersionPropsFlowType) + { + var dependencies = GetDependences(); + + if (Log.HasLoggedErrors) + { + return false; + } + + packageElementsToWrite = FilterNonDependencies(packageElementsToWrite, dependencies); + additionalAssetElementsToWrite = FilterNonDependencies(additionalAssetElementsToWrite, dependencies); + } + + Directory.CreateDirectory(Path.GetDirectoryName(OutputPath)); + + using (var outStream = File.Open(OutputPath, FileMode.Create)) + using (var sw = new StreamWriter(outStream, new UTF8Encoding(false))) + { + sw.WriteLine(@""); + sw.WriteLine(@""); + + WriteVersionEntries(sw, packageElementsToWrite, "packages"); + WriteExtraProperties(sw); + WriteVersionEntries(sw, additionalAssetElementsToWrite, "additional assets"); + + if (IncludeCreationTimeProperty) + { + sw.WriteLine(@" "); + sw.WriteLine($@" <{CreationTimePropertyName}>{DateTime.UtcNow.Ticks}"); + sw.WriteLine(@" "); + } + + sw.WriteLine(@""); + } + + return !Log.HasLoggedErrors; + } + + /// + /// Write properties specified in the "ExtraProperties task parameter + /// + /// Stream writer + private void WriteExtraProperties(StreamWriter sw) + { + if (ExtraProperties == null) + { + return; + } + + sw.WriteLine(@" "); + sw.WriteLine(@" "); + + foreach (var extraProp in ExtraProperties ?? Enumerable.Empty()) + { + string propertyName = extraProp.GetMetadata("Identity"); + bool doNotOverwrite = false; + string overwriteCondition = string.Empty; + if (bool.TryParse(extraProp.GetMetadata("DoNotOverwrite"), out doNotOverwrite) && doNotOverwrite) + { + overwriteCondition = $" Condition=\"'$({propertyName})' == ''\""; + } + sw.WriteLine($" <{propertyName}{overwriteCondition}>{extraProp.GetMetadata("Version")}"); + } + + sw.WriteLine(@" "); + } + + /// + /// Write properties for the version numbers required for this repo. + /// + /// Stream writer + /// Version entries + private void WriteVersionEntries(StreamWriter sw, IEnumerable entries, string entryType) + { + if (!entries.Any()) + { + return; + } + + sw.WriteLine($" "); + if (VersionPropsFlowType == DependenciesOnlyVersionPropsFlowType) + { + sw.WriteLine(@" "); + } + sw.WriteLine(@" "); + foreach (var package in entries) + { + string propertyName = GetPropertyName(package.Name, VersionPropertySuffix); + string alternatePropertyName = GetPropertyName(package.Name, VersionPropertyAlternateSuffix); + + sw.WriteLine($" <{propertyName}>{package.Version}"); + sw.WriteLine($" <{alternatePropertyName}>{package.Version}"); + } + sw.WriteLine(@" "); + } + + public static string GetPropertyName(string id, string suffix) + { + string formattedId = InvalidElementNameCharRegex.Replace( + id, + match => match.Groups?["FirstPartChar"].Value.ToUpperInvariant() + ?? string.Empty); + + return $"{formattedId}{suffix}"; + } + } +} diff --git a/src/SourceBuild/content/repo-projects/Directory.Build.props b/src/SourceBuild/content/repo-projects/Directory.Build.props index ef66ba647..337b9df49 100644 --- a/src/SourceBuild/content/repo-projects/Directory.Build.props +++ b/src/SourceBuild/content/repo-projects/Directory.Build.props @@ -22,6 +22,12 @@ Repo specific semaphore path for incremental build --> $(CompletedSemaphorePath)$(RepositoryName)/ + + + $(IntermediatePath)PackageVersions.$(RepositoryName).Current.props + $(IntermediatePath)PackageVersions.$(RepositoryName).Previous.props + $(IntermediatePath)PackageVersions.$(RepositoryName).props + AllPackages diff --git a/src/SourceBuild/content/repo-projects/Directory.Build.targets b/src/SourceBuild/content/repo-projects/Directory.Build.targets index 64745b5d3..d10278261 100644 --- a/src/SourceBuild/content/repo-projects/Directory.Build.targets +++ b/src/SourceBuild/content/repo-projects/Directory.Build.targets @@ -13,12 +13,13 @@ - + + @@ -232,36 +232,67 @@ - + - - <_PackageVersionPropsBackupPath>$(PackageVersionPropsPath).pre.$(RepositoryName).xml - + Outputs="$(RepoCompletedSemaphorePath)CreateBuildInputProps.complete"> - - <_AdditionalAssetDirs Include="$(SourceBuiltToolsetDir)" Condition="Exists('$(SourceBuiltToolsetDir)')" /> + <_CurrentSourceBuiltPackages Include="$(SourceBuiltPackagesPath)*.nupkg" + Exclude="$(SourceBuiltPackagesPath)*.symbols.nupkg" /> + <_PreviouslyBuiltSourceBuiltPackages Include="$(PrebuiltSourceBuiltPackagesPath)*.nupkg" /> + + <_CurrentAdditionalAssetDirs Include="$(SourceBuiltToolsetDir)" Condition="Exists('$(SourceBuiltToolsetDir)')" /> - + + + <_VersionDetailsXml Condition="'$(PackageVersionPropsFlowType)' == 'DependenciesOnly'">$(ProjectDirectory)/eng/Version.Details.xml + + + + - + + - + + - - - + + + + + + + + +]]> + + - + + + + + <_PackagesNotCreatedReason Include="^ There may have been a silent failure in the submodule build. To confirm, check the build log file for undetected errors that may have prevented package creation: $(RepoConsoleLogFile)" /> <_PackagesNotCreatedReason Include="^ This error might be a false positive if $(RepositoryName) intentionally builds no nuget packages. If so, set the SkipEnsurePackagesCreated property to true in $(MSBuildProjectFullPath)" /> - <_PackagesNotCreatedReason Include="^ The 'bin' directory might be dirty from a previous build and the package files already existed. If so, perform a clean build, or check which packages were already in 'bin' by opening $(_PackageVersionPropsBackupPath)" /> + <_PackagesNotCreatedReason Include="^ The 'bin' directory might be dirty from a previous build and the package files already existed. If so, perform a clean build, or check which packages were already in 'bin' by opening $(CurrentSourceBuiltPackageVersionPropsPath)" /> <_PackagesNotCreatedReason Include="^ The packages may have been written to an unexpected directory. For example, some repos used bin/ and changed to artifacts/ to match Arcade. Check PackagesOutput in $(MSBuildProjectFullPath) (currently '$(PackagesOutput)')" />