Use the repo asset manifests to discover dependency versions (#19389)

This commit is contained in:
Jeremy Koritzinsky 2024-04-16 07:43:32 -07:00 committed by GitHub
parent 028595d80c
commit 9b33ce6b38
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 123 additions and 78 deletions

View file

@ -20,7 +20,7 @@ namespace Microsoft.DotNet.UnifiedBuild.Tasks
public class VersionEntry
{
public string Name;
public string Version;
public NuGetVersion Version;
}
/// <summary>
@ -52,9 +52,16 @@ namespace Microsoft.DotNet.UnifiedBuild.Tasks
/// <summary>
/// Set of input nuget package files to generate version properties for.
/// </summary>
[Required]
public ITaskItem[] NuGetPackages { get; set; }
/// <summary>
/// Set of packages built by dependencies of this repo during this build.
///
/// %(Identity): Package identity.
/// %(Version): Package version.
/// </summary>
public ITaskItem[] KnownPackages { get; set; }
/// <summary>
/// File where the version properties should be written.
/// </summary>
@ -63,7 +70,7 @@ namespace Microsoft.DotNet.UnifiedBuild.Tasks
/// <summary>
/// Properties to add to the build output props, which may not exist as nupkgs.
/// FOr example, this is used to pass the version of the CLI toolset archives.
/// For example, this is used to pass the version of the CLI toolset archives.
///
/// %(Identity): Package identity.
/// %(Version): Package version.
@ -169,6 +176,9 @@ namespace Microsoft.DotNet.UnifiedBuild.Tasks
return !Log.HasLoggedErrors;
}
NuGetPackages ??= Array.Empty<ITaskItem>();
KnownPackages ??= Array.Empty<ITaskItem>();
// First, obtain version information from the packages and additional assets that
// are provided.
var latestPackages = NuGetPackages
@ -179,16 +189,26 @@ namespace Microsoft.DotNet.UnifiedBuild.Tasks
return reader.GetIdentity();
}
})
.GroupBy(identity => identity.Id)
.Select(g => g.OrderBy(id => id.Version).Last())
.OrderBy(id => id.Id)
.Select(identity => new VersionEntry()
{
Name = identity.Id,
Version = identity.Version
});
var knownPackages = KnownPackages
.Select(item => new VersionEntry()
{
Name = identity.Id,
Version = identity.Version.ToString()
Name = item.GetMetadata("Identity"),
Version = new NuGetVersion(item.GetMetadata("Version"))
});
var packageElementsToWrite = latestPackages;
// We may have multiple versions of the same package. We'll keep the latest one.
// This can even happen in the KnownPackages list, as a repo (such as source-build-reference-packages)
// may have multiple versions of the same package.
IEnumerable<VersionEntry> packageElementsToWrite = latestPackages.Concat(knownPackages)
.GroupBy(identity => identity.Name)
.Select(g => g.OrderByDescending(id => id.Version).First())
.OrderBy(id => id.Name);
// Then, if version flow type is "DependenciesOnly", filter those
// dependencies that do not appear in the version.details.xml file.

View file

@ -182,31 +182,14 @@
</ItemGroup>
<!-- If we're using the bootstrapped arcade, we can set the override here. -->
<ItemGroup Condition="'$(UseBootstrapArcade)' == 'true'">
<SourceBuiltSdkOverride Include="Microsoft.DotNet.Arcade.Sdk"
Group="ARCADE"
Version="$(ArcadeBootstrapVersion)"
Location="$(BootstrapPackagesDir)microsoft.dotnet.arcade.sdk/$(ArcadeBootstrapVersion)" />
</ItemGroup>
<ItemGroup>
<!-- Configure the bootstrapped Arcade version here. Repositories that build before arcade (and arcade itself) will use the bootstrapped SDK. -->
<BootstrapArcadeSdkOverride Include="Microsoft.DotNet.Arcade.Sdk"
Group="ARCADE"
Version="$(ArcadeBootstrapVersion)"
Location="$(BootstrapPackagesDir)microsoft.dotnet.arcade.sdk/$(ArcadeBootstrapVersion)" />
<!-- Make the WindowsDesktop SDK override opt-in for repos that need it. -->
<WindowsDesktopSdkOverride Include="Microsoft.Net.Sdk.WindowsDesktop" Group="WINDOWS_DESKTOP" Location="$(ToolsDir)EmptySdk" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
<!--
Add Microsoft.Build.NoTargets and Microsoft.Build.Traversal SDK overrides, for all repos,
except source-build-reference-packages repo, that builds these packages and utility projects.
We need to override all SDK packages that are used by repos, to prevent NuGet SDK Resolver from trying to
restore these packages from online feeds. In offline build we only want to use local source-build feeds.
Therefore, we create a copy of repo's NuGet.config file, modify it, and pass it into repo build using
RestoreConfigFile property. MSBuild uses NuGetSdkResolver to restore missing SDKs, which doesn't honor
RestoreConfigFile property and uses NuGet.config file from repo's root which has online feeds.
-->
<NoTargetsSdkOverride Include="Microsoft.Build.NoTargets" Group="NOTARGETS" Version="$(NOTARGETS_BOOTSTRAP_VERSION)" />
<TraversalSdkOverride Include="Microsoft.Build.Traversal" Group="TRAVERSAL" Version="$(TRAVERSAL_BOOTSTRAP_VERSION)" />
<SourceBuiltSdkOverride Include="@(NoTargetsSdkOverride)" Condition="'$(DotNetBuildSourceOnly)' == 'true' and '$(RepositoryName)' != 'source-build-reference-packages'" />
<SourceBuiltSdkOverride Include="@(TraversalSdkOverride)" Condition="'$(DotNetBuildSourceOnly)' == 'true' and '$(RepositoryName)' != 'source-build-reference-packages'" />
</ItemGroup>
<!-- CLI internal version is statically set by us to a version that will never show up in the wild.

View file

@ -36,7 +36,7 @@
</PropertyGroup>
<!-- Returns the repository references of this project and all the projects this project references, recursively -->
<Target Name="GetTransitiveRepositoryReferences" Outputs="@(TransitiveRepositoryReference)">
<Target Name="GetTransitiveRepositoryReferences" Returns="@(TransitiveRepositoryReference)">
<ItemGroup>
<_TransitiveRepositoryReference Include="@(RepositoryReference)" />
</ItemGroup>
@ -80,8 +80,6 @@
<ItemGroup Condition="'@(RepositoryReferenceInfo)' != ''">
<DependentRepoSourceName Include="@(RepositoryReferenceInfo->'%(ShippingSourceName)')" />
<DependentRepoSourceName Include="@(RepositoryReferenceInfo->'%(NonShippingSourceName)')" />
<DependentRepoPackageFile Include="%(RepositoryReferenceInfo.ShippingPackagesPath)**" />
<DependentRepoPackageFile Include="%(RepositoryReferenceInfo.NonShippingPackagesPath)**" />
</ItemGroup>
</Target>
@ -258,13 +256,10 @@
There are 3 files generated -->
<UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.WritePackageVersionsProps" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" TaskFactory="TaskHostFactory" />
<Target Name="CreateBuildInputProps"
DependsOnTargets="GetRepositoryReferenceInfo"
DependsOnTargets="GetProducedPackagesFromTransitiveReferences"
Inputs="$(MSBuildProjectFullPath)"
Outputs="$(BaseIntermediateOutputPath)CreateBuildInputProps.complete">
<ItemGroup>
<!-- Collect all the NuGet packages from dependent repos except for the symbols -->
<_CurrentSourceBuiltPackages Include="@(DependentRepoPackageFile)"
Condition="!$([System.String]::Copy('%(Identity)').EndsWith('.symbols.nupkg'))" />
<_PreviouslyBuiltSourceBuiltPackages Include="$(PrebuiltSourceBuiltPackagesPath)*.nupkg" />
</ItemGroup>
@ -276,7 +271,7 @@
</PropertyGroup>
<!-- Write the build input properties, then save off a copy that will be used for generating usage reports later -->
<WritePackageVersionsProps NuGetPackages="@(_CurrentSourceBuiltPackages)"
<WritePackageVersionsProps KnownPackages="@(_DependencyProducedPackage->WithMetadataValue('ReferenceOnly', 'false'))"
ExtraProperties="@(ExtraPackageVersionPropsPackageInfo)"
VersionPropsFlowType="$(PackageVersionPropsFlowType)"
VersionDetails="$(_VersionDetailsXml)"
@ -297,7 +292,7 @@
A B and C.
A key element of this algorith is that we must write the full package version props and not the filtered version. -->
<WritePackageVersionsProps NuGetPackages="@(_CurrentSourceBuiltPackages)"
<WritePackageVersionsProps KnownPackages="@(_DependencyProducedPackage->WithMetadataValue('ReferenceOnly', 'false'))"
VersionPropsFlowType="AllPackages"
OutputPath="$(SnapshotPackageVersionPropsPath)" />
@ -342,40 +337,45 @@
RunEachTargetSeparately="$(RunEachTargetSeparately)" />
</Target>
<!-- Discover the Arcade SDKs built from the live Arcade source and set them as overrides here. -->
<Target Name="DiscoverArcadeSourceBuiltSdkOverrides"
DependsOnTargets="BuildRepoReferences;GetRepositoryReferenceInfo"
Condition="'$(UseBootstrapArcade)' != 'true'">
<PropertyGroup>
<_ArcadeSdkName>Microsoft.DotNet.Arcade.Sdk</_ArcadeSdkName>
</PropertyGroup>
<Target Name="DiscoverBuiltSdkOverrides"
DependsOnTargets="BuildRepoReferences;GetProducedPackagesFromTransitiveReferences">
<!--
Discover the version of the Arcade SDK NuGet package by matching the name of the nupkg file and extracting the version.
In the future, we should use the asset manifests to discover this information.
Discover the Arcade SDKs built from the live Arcade source and set them as overrides here.
This will automatically no-op for Arcade and SBRP, as these packages will not have been produced yet.
-->
<ItemGroup>
<_BuiltArcadeSdkPackage Include="@(DependentRepoPackageFile)"
Condition="!$([System.String]::new('%(Identity)').EndsWith('.symbols.nupkg')) and $([System.String]::new('%(Identity)').Contains($(_ArcadeSdkName)))" />
<ArcadeBuiltArcadeSdk Include="@(_DependencyProducedPackage->WithMetadataValue('Identity', 'Microsoft.DotNet.Arcade.Sdk'))" />
</ItemGroup>
<PropertyGroup>
<_BuiltArcadeSdkPackageVersion>$([System.String]::new('%(_BuiltArcadeSdkPackage.FileName)').Substring($([MSBuild]::Add($(_ArcadeSdkName.Length), 1))))</_BuiltArcadeSdkPackageVersion>
</PropertyGroup>
<Error Text="The source-built Arcade SDK should be available for any repository that does not explicitly set UseBootstrapArcade to true." Condition="'@(ArcadeBuiltArcadeSdk)' == '' and '$(UseBootstrapArcade)' != 'true'" />
<ItemGroup Condition="'@(ArcadeBuiltArcadeSdk)' == ''">
<SourceBuiltSdkOverride Include="@(BootstrapArcadeSdkOverride)" />
</ItemGroup>
<ItemGroup>
<SourceBuiltSdkOverride Include="Microsoft.DotNet.Arcade.Sdk" Group="ARCADE" Version="$(_BuiltArcadeSdkPackageVersion)" />
<SourceBuiltSdkOverride Include="Microsoft.DotNet.SharedFramework.Sdk" Group="ARCADE_SHARED_FX_SDK" Version="$(_BuiltArcadeSdkPackageVersion)" />
<SourceBuiltSdkOverride Include="Microsoft.DotNet.CMake.Sdk" Group="ARCADE_CMAKE_SDK" Version="$(_BuiltArcadeSdkPackageVersion)" />
<SourceBuiltSdkOverride Include="@(ArcadeSdkOverride)" />
<SourceBuiltSdkOverride Include="@(ArcadeCMakeSdkOverride)" />
<SourceBuiltSdkOverride Include="@(ArcadeSharedFrameworkSdkOverride)" />
<SourceBuiltSdkOverride Include="@(ArcadeBuiltArcadeSdk)" Group="ARCADE" />
<SourceBuiltSdkOverride Include="@(_DependencyProducedPackage->WithMetadataValue('Identity', 'Microsoft.DotNet.SharedFramework.Sdk'))" Group="ARCADE_SHARED_FX_SDK" />
<SourceBuiltSdkOverride Include="@(_DependencyProducedPackage->WithMetadataValue('Identity', 'Microsoft.DotNet.CMake.Sdk'))" Group="ARCADE_CMAKE_SDK" />
</ItemGroup>
<!--
Add Microsoft.Build.NoTargets and Microsoft.Build.Traversal SDK overrides, for all repos that have it built (ie. excluding SBRP as it builds the SDKs).
We need to override all SDK packages that are used by repos, to prevent NuGet SDK Resolver from trying to
restore these packages from online feeds. In offline build we only want to use local source-build feeds.
Therefore, we create a copy of repo's NuGet.config file, modify it, and pass it into repo build using
RestoreConfigFile property. MSBuild uses NuGetSdkResolver to restore missing SDKs, which doesn't honor
RestoreConfigFile property and uses NuGet.config file from repo's root which has online feeds.
-->
<ItemGroup>
<SourceBuiltSdkOverride Include="@(_DependencyProducedPackage->WithMetadataValue('Identity', 'Microsoft.Build.Traversal'))" Group="TRAVERSAL" />
<SourceBuiltSdkOverride Include="@(_DependencyProducedPackage->WithMetadataValue('Identity', 'Microsoft.Build.NoTargets'))" Group="NOTARGETS" />
</ItemGroup>
</Target>
<Target Name="SetSourceBuiltSdkOverrides"
DependsOnTargets="DiscoverArcadeSourceBuiltSdkOverrides"
DependsOnTargets="DiscoverBuiltSdkOverrides"
Condition="'@(SourceBuiltSdkOverride)' != ''">
<ItemGroup>
<!-- Set the environment variables for MSBuild to look for our additional SDK Resolvers and or our resolver to find our source-built SDKs. -->
@ -396,7 +396,7 @@
UpdateGlobalJsonVersions;
UpdateEngCommonFiles;
CreateBuildInputProps;
DiscoverArcadeSourceBuiltSdkOverrides;
DiscoverBuiltSdkOverrides;
SetSourceBuiltSdkOverrides">
<Message Importance="High" Text="[$([System.DateTime]::Now.ToString('HH:mm:ss.ff'))] Building $(RepositoryName)" />
<Message Importance="High" Text="Running command:" />
@ -449,14 +449,16 @@
<Message Importance="High" Text="See '$(RepoConsoleLogFile)' for more information." Condition="Exists('$(RepoConsoleLogFile)') and '$(MinimalConsoleLogOutput)' == 'true'" />
</Target>
<!-- Log the new repo artifacts -->
<Target Name="LogRepoArtifacts"
DependsOnTargets="RepoBuild"
Condition="'$(IsUtilityProject)' != 'true'">
<Target Name="GetRepoAssetManifests" DependsOnTargets="RepoBuild" Returns="@(RepoAssetManifest)">
<ItemGroup>
<RepoAssetManifest Include="$(RepoAssetManifestsDir)*.xml" />
</ItemGroup>
</Target>
<!-- Log the new repo artifacts -->
<Target Name="LogRepoArtifacts"
DependsOnTargets="GetRepoAssetManifests"
Condition="'$(IsUtilityProject)' != 'true'">
<Error Text="Repo manifest files don't exist." Condition="'@(RepoAssetManifest)' == ''" />
<XmlPeek XmlInputPath="%(RepoAssetManifest.Identity)"
@ -468,6 +470,36 @@
<Message Importance="High" Text=" -> %(RepoManifestArtifact.Identity)" />
</Target>
<!-- Discover the produced packages from all repo asset manifests for this repository. -->
<Target Name="DiscoverProducedPackages" DependsOnTargets="GetRepoAssetManifests" Inputs="@(RepoAssetManifest)" Outputs="%(Identity).ForBatching">
<XmlPeek XmlInputPath="%(RepoAssetManifest.Identity)" Query="/Build/*[self::Package]">
<Output TaskParameter="Result" ItemName="ProducedPackageEntry" />
</XmlPeek>
</Target>
<Target Name="ProcessPackageEntries" DependsOnTargets="DiscoverProducedPackages" Inputs="@(ProducedPackageEntry)" Outputs="%(Identity).ForBatching">
<XmlPeek XmlContent="%(ProducedPackageEntry.Identity)" Query="/Package/@Id">
<Output TaskParameter="Result" PropertyName="ProducedPackageId" />
</XmlPeek>
<XmlPeek XmlContent="%(ProducedPackageEntry.Identity)" Query="/Package/@Version">
<Output TaskParameter="Result" PropertyName="ProducedPackageVersion" />
</XmlPeek>
<ItemGroup>
<ProducedPackage Include="$(ProducedPackageId)" Version="$(ProducedPackageVersion)" ReferenceOnly="$([MSBuild]::ValueOrDefault('$(ReferenceOnlyRepoArtifacts)', 'false'))" />
</ItemGroup>
</Target>
<Target Name="GetProducedPackages" DependsOnTargets="GetRepoAssetManifests;DiscoverProducedPackages;ProcessPackageEntries" Returns="@(ProducedPackage)" />
<Target Name="GetProducedPackagesFromTransitiveReferences" DependsOnTargets="GetTransitiveRepositoryReferences" Returns="@(_DependencyProducedPackage)">
<MSBuild Projects="@(TransitiveRepositoryReference->'%(Identity).proj')"
Targets="GetProducedPackages"
BuildInParallel="true">
<Output TaskParameter="TargetOutputs" ItemName="_DependencyProducedPackage" />
</MSBuild>
</Target>
<!-- Copy restored packages from inner build to ensure they're included in the
main build prebuilt check -->
<Target Name="CopyInnerBuildRestoredPackages"
@ -540,8 +572,16 @@
<Exec Command="df -h $(RepoRoot)" Condition="'$(BuildOS)' != 'windows'and '@(DirsToDeleteWithTrailingSeparator)' != ''" />
</Target>
<Target Name="ExtractToolPackage"
DependsOnTargets="RepoBuild"
<Target Name="DiscoverToolPackageVersions" DependsOnTargets="GetProducedPackages">
<JoinItems Left="@(ProducedPackage)"
Right="@(BuiltSdkPackage)"
LeftMetadata="*">
<Output TaskParameter="JoinResult" ItemName="BuiltSdkPackageOverride" />
</JoinItems>
</Target>
<Target Name="ExtractToolPackageCore"
DependsOnTargets="RepoBuild;DiscoverToolPackageVersions"
Condition="'@(BuiltSdkPackageOverride)' != ''"
Inputs="$(MSBuildProjectFullPath)"
Outputs="$(BaseIntermediateOutputPath)ExtractToolPackage.complete">
@ -578,6 +618,9 @@
</Touch>
</Target>
<Target Name="ExtractToolPackage"
DependsOnTargets="DiscoverToolPackageVersions;ExtractToolPackageCore" />
<Target Name="Build"
DependsOnTargets="
BuildRepoReferences;
@ -727,7 +770,7 @@
<!-- Recursively walks the repo dependency graph gathering each repo's dependencies which are returned as a YAML string representation -->
<Target Name="GetDependencyGraphString"
DependsOnTargets="GetTransitiveRepositoryReferences"
Outputs="$(DependencyGraphString)">
Returns="$(DependencyGraphString)">
<PropertyGroup>
<!-- Each step deeper in the graph builds up the indentation used for the YAML representation.
Use '_' as a placeholder for the space ' ' character since trailing spaces aren't handled well as property values. -->

View file

@ -9,14 +9,13 @@
<RepositoryReference Include="source-build-reference-packages" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
</ItemGroup>
<ItemGroup>
<BuiltSdkPackageOverride Include="Microsoft.DotNet.Arcade.Sdk" Version="$(OutputPackageVersion)" />
<BuiltSdkPackageOverride Include="Microsoft.DotNet.SharedFramework.Sdk" Version="$(OutputPackageVersion)" />
<BuiltSdkPackageOverride Include="Microsoft.DotNet.CMake.Sdk" Version="$(OutputPackageVersion)" />
</ItemGroup>
<ItemGroup>
<ExtraPackageVersionPropsPackageInfo Include="NuGetVersion" Version="%24(NuGetPackagingVersion)" />
</ItemGroup>
<ItemGroup>
<BuiltSdkPackage Include="Microsoft.DotNet.Arcade.Sdk" />
<BuiltSdkPackage Include="Microsoft.DotNet.SharedFramework.Sdk" />
<BuiltSdkPackage Include="Microsoft.DotNet.CMake.Sdk" />
</ItemGroup>
</Project>

View file

@ -15,8 +15,8 @@
</PropertyGroup>
<ItemGroup>
<BuiltSdkPackageOverride Include="@(NoTargetsSdkOverride)" />
<BuiltSdkPackageOverride Include="@(TraversalSdkOverride)" />
<BuiltSdkPackage Include="Microsoft.Build.NoTargets" />
<BuiltSdkPackage Include="Microsoft.Build.Traversal" />
</ItemGroup>
<Target Name="AddLocalNuGetPackageCacheDirectory"