diff --git a/eng/pipelines/templates/jobs/vmr-build.yml b/eng/pipelines/templates/jobs/vmr-build.yml index 1d2f411a6..b59f2e9c5 100644 --- a/eng/pipelines/templates/jobs/vmr-build.yml +++ b/eng/pipelines/templates/jobs/vmr-build.yml @@ -441,3 +441,24 @@ jobs: mergeTestResults: true publishRunAttachments: true testRunTitle: SourceBuild_SmokeTests_$(Agent.JobName) + + - task: CopyFiles@2 + inputs: + SourceFolder: $(sourcesPath)/artifacts + Contents: | + VerticalManifest.xml + assets/** + TargetFolder: $(Build.ArtifactStagingDirectory)/publishing + displayName: Copy artifacts to Artifact Staging Directory + + # When building from source, the Private.SourceBuilt.Artifacts archive already contains the nuget packages + - ${{ if ne(parameters.buildSourceOnly, 'true') }}: + - task: CopyFiles@2 + inputs: + SourceFolder: $(sourcesPath)/artifacts/packages + TargetFolder: $(Build.ArtifactStagingDirectory)/publishing/packages + displayName: Copy packages to Artifact Staging Directory + + - publish: $(Build.ArtifactStagingDirectory)/publishing + artifact: $(Agent.JobName)_Artifacts + displayName: Publish Artifacts diff --git a/src/SourceBuild/content/Directory.Build.props b/src/SourceBuild/content/Directory.Build.props index cdc0c6e87..cf2727cee 100644 --- a/src/SourceBuild/content/Directory.Build.props +++ b/src/SourceBuild/content/Directory.Build.props @@ -206,6 +206,7 @@ $([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'Microsoft.DotNet.SourceBuild.Tasks.XPlat', '$(Configuration)', 'Microsoft.DotNet.SourceBuild.Tasks.XPlat.dll')) $([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'Microsoft.DotNet.SourceBuild.Tasks.LeakDetection', '$(Configuration)', 'Microsoft.DotNet.SourceBuild.Tasks.LeakDetection.dll')) $([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff', '$(Configuration)', 'Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff.dll')) + $([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'Microsoft.DotNet.SourceBuild.Tasks.MergeAssetManifests', '$(Configuration)', 'Microsoft.DotNet.SourceBuild.Tasks.MergeAssetManifests.dll')) @@ -214,6 +215,12 @@ $(PackageReportDir)poisoned.txt + + + $(BUILD_BUILDNUMBER)" + + diff --git a/src/SourceBuild/content/build.proj b/src/SourceBuild/content/build.proj index 0f0a2081c..0f3b6948c 100644 --- a/src/SourceBuild/content/build.proj +++ b/src/SourceBuild/content/build.proj @@ -9,6 +9,8 @@ + + source-build @@ -25,6 +27,21 @@ StopOnFirstFailure="true" /> + + + $(ArtifactsDir)VerticalManifest.xml + + + + + + + + + diff --git a/src/SourceBuild/content/eng/tools/init-build.proj b/src/SourceBuild/content/eng/tools/init-build.proj index 63e0492ab..eba7880ea 100644 --- a/src/SourceBuild/content/eng/tools/init-build.proj +++ b/src/SourceBuild/content/eng/tools/init-build.proj @@ -17,6 +17,7 @@ BuildMSBuildSdkResolver; BuildSdkArchiveDiff; BuildLeakDetection; + BuildMergeAssetManifests; ExtractToolPackage; GenerateRootFs; PoisonPrebuiltPackages" /> @@ -126,6 +127,24 @@ Targets="Build" /> + + + + + + + + + + + + diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.MergeAssetManifests/MergeAssetManifests.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.MergeAssetManifests/MergeAssetManifests.cs new file mode 100644 index 000000000..8695fa3ca --- /dev/null +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.MergeAssetManifests/MergeAssetManifests.cs @@ -0,0 +1,103 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Build.Utilities; +using Microsoft.Build.Framework; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Xml.Linq; + +namespace Microsoft.DotNet.SourceBuild.Tasks +{ + public class MergeAssetManifests : Task + { + /// + /// AssetManifest paths + /// + [Required] + public required ITaskItem[] AssetManifest { get; init; } + + /// + /// Merged asset manifest output path + /// + [Required] + public required string MergedAssetManifestOutputPath { get; init; } + + /// + /// Azure DevOps build number + /// + [Required] + public required string VmrBuildNumber { get; init; } + + private static readonly string _buildIdAttribute = "BuildId"; + private static readonly string _azureDevOpsBuildNumberAttribute = "AzureDevOpsBuildNumber"; + private static readonly string[] _ignoredAttributes = [_buildIdAttribute, _azureDevOpsBuildNumberAttribute, "IsReleaseOnlyPackageVersion"]; + + public override bool Execute() + { + List assetManifestXmls = AssetManifest.Select(xmlPath => XDocument.Load(xmlPath.ItemSpec)).ToList(); + + VerifyAssetManifests(assetManifestXmls); + + XElement mergedManifestRoot = assetManifestXmls.First().Root + ?? throw new ArgumentException("The root element of the asset manifest is null."); + + // Set the BuildId and AzureDevOpsBuildNumber attributes to the value of VmrBuildNumber + mergedManifestRoot.SetAttributeValue(_buildIdAttribute, VmrBuildNumber); + mergedManifestRoot.SetAttributeValue(_azureDevOpsBuildNumberAttribute, VmrBuildNumber); + + List packageElements = new(); + List blobElements = new(); + + foreach (var assetManifestXml in assetManifestXmls) + { + packageElements.AddRange(assetManifestXml.Descendants("Package")); + blobElements.AddRange(assetManifestXml.Descendants("Blob")); + } + + packageElements = packageElements.OrderBy(packageElement => packageElement.Attribute("Id")?.Value).ToList(); + blobElements = blobElements.OrderBy(blobElement => blobElement.Attribute("Id")?.Value).ToList(); + + XDocument verticalManifest = new(new XElement(mergedManifestRoot.Name, mergedManifestRoot.Attributes(), packageElements, blobElements)); + + File.WriteAllText(MergedAssetManifestOutputPath, verticalManifest.ToString()); + + return !Log.HasLoggedErrors; + } + + private static void VerifyAssetManifests(IReadOnlyList assetManifestXmls) + { + if (assetManifestXmls.Count == 0) + { + throw new ArgumentException("No asset manifests were provided."); + } + + HashSet rootAttributes = assetManifestXmls + .First() + .Root? + .Attributes() + .Select(attribute => attribute.ToString()) + .ToHashSet() + ?? throw new ArgumentException("The root element of the asset manifest is null."); + + if (assetManifestXmls.Skip(1).Any(manifest => manifest.Root?.Attributes().Count() != rootAttributes.Count)) + { + throw new ArgumentException("The asset manifests do not have the same number of root attributes."); + } + + if (assetManifestXmls.Skip(1).Any(assetManifestXml => + !assetManifestXml.Root?.Attributes().Select(attribute => attribute.ToString()) + .All(attribute => + // Ignore BuildId and AzureDevOpsBuildNumber attributes, they're different for different repos, + // TODO this should be fixed with https://github.com/dotnet/source-build/issues/3934 + _ignoredAttributes.Any(ignoredAttribute => attribute.StartsWith(ignoredAttribute)) || rootAttributes.Contains(attribute)) + ?? false)) + { + throw new ArgumentException("The asset manifests do not have the same root attributes."); + } + } + } +} diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.MergeAssetManifests/Microsoft.DotNet.SourceBuild.Tasks.MergeAssetManifests.csproj b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.MergeAssetManifests/Microsoft.DotNet.SourceBuild.Tasks.MergeAssetManifests.csproj new file mode 100644 index 000000000..25d4fd1b9 --- /dev/null +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.MergeAssetManifests/Microsoft.DotNet.SourceBuild.Tasks.MergeAssetManifests.csproj @@ -0,0 +1,16 @@ + + + + $(NetCurrent) + enable + + + + + + + + + + +