Poison work - reordering and exclusion of non-shipping packages (#15347)

This commit is contained in:
Nikola Milosavljevic 2023-01-25 22:19:45 -08:00 committed by GitHub
commit b389ab61bd
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 284 additions and 4 deletions

View file

@ -125,6 +125,9 @@
<PackageReportDir>$(BaseOutputPath)prebuilt-report/</PackageReportDir>
<RepoProjectsDir>$(ProjectDir)/repo-projects/</RepoProjectsDir>
<ResultingPrebuiltPackagesDir>$(PackageReportDir)prebuilt-packages/</ResultingPrebuiltPackagesDir>
<PackageListsDir>$(PackageReportDir)packagelists/</PackageListsDir>
<!-- The prefix needs to match what's defined in Arcade's source-build infra. Consider using a single property, in the future. -->
<NonShippingPackagesListPrefix>NonShipping.Packages.</NonShippingPackagesListPrefix>
<PackageReportDataFile>$(PackageReportDir)prebuilt-usage.xml</PackageReportDataFile>
<PoisonUsageReportFile>$(PackageReportDir)poison-usage.xml</PoisonUsageReportFile>
<PoisonReportDataFile>$(PackageReportDir)poison-catalog.xml</PoisonReportDataFile>

View file

@ -70,10 +70,15 @@
<Message Importance="High" Text="[$([System.DateTime]::Now.ToString('HH:mm:ss.ff'))] Checking @(FinalCliTarball) for poisoned files." />
<ItemGroup>
<NonShippingPackagesList Include="$(PackageListsDir)**/$(NonShippingPackagesListPrefix)*" />
</ItemGroup>
<CheckForPoison FilesToCheck="@(FinalCliTarball)"
HashCatalogFilePath="$(PoisonReportDataFile)"
MarkerFileName="$(PoisonMarkerFile)"
PoisonReportOutputFilePath="$(PoisonUsageReportFile)" />
PoisonReportOutputFilePath="$(PoisonUsageReportFile)"
NonShippingPackagesListFiles="@(NonShippingPackagesList)" />
<WriteLinesToFile File="$(CompletedSemaphorePath)ReportPoisonUsage.complete" Overwrite="true" />
<Message Importance="High" Text="[$([System.DateTime]::Now.ToString('HH:mm:ss.ff'))] Done checking for poison." />

View file

@ -0,0 +1,100 @@
<!-- Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this file to you under the MIT license. -->
<Project>
<!-- Repo extensibility point. -->
<Import Project="$(RepositoryEngineeringDir)SourceBuild.props" Condition="Exists('$(RepositoryEngineeringDir)SourceBuild.props')" />
<PropertyGroup>
<SourceBuildOutputDir Condition="'$(SourceBuildOutputDir)' == ''">$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'source-build'))</SourceBuildOutputDir>
<SourceBuildSelfDir>$([MSBuild]::NormalizeDirectory('$(SourceBuildOutputDir)', 'self'))</SourceBuildSelfDir>
<CurrentRepoSourceBuildSourceDir>$([MSBuild]::NormalizeDirectory('$(SourceBuildSelfDir)', 'src'))</CurrentRepoSourceBuildSourceDir>
<CurrentRepoSourceBuildPackageCache>$([MSBuild]::NormalizeDirectory('$(SourceBuildSelfDir)', 'package-cache'))</CurrentRepoSourceBuildPackageCache>
<SourceBuildSelfPrebuiltReportDir>$([MSBuild]::NormalizeDirectory('$(SourceBuildSelfDir)', 'prebuilt-report'))</SourceBuildSelfPrebuiltReportDir>
<!--
Keep artifacts/ inside source dir so that ancestor-based file lookups find the inner repo, not
the outer repo. The inner repo global.json and NuGet.config files may have been modified by
source-build, and we want projects inside the artifacts/ dir to respect that.
-->
<CurrentRepoSourceBuildArtifactsDir>$([MSBuild]::NormalizeDirectory('$(CurrentRepoSourceBuildSourceDir)', 'artifacts'))</CurrentRepoSourceBuildArtifactsDir>
<CurrentRepoSourceBuildArtifactsPackagesDir>$([MSBuild]::NormalizeDirectory('$(CurrentRepoSourceBuildArtifactsDir)', 'packages', '$(Configuration)'))</CurrentRepoSourceBuildArtifactsPackagesDir>
<CurrentRepoSourceBuiltNupkgCacheDir>$([MSBuild]::NormalizeDirectory('$(CurrentRepoSourceBuildArtifactsDir)', 'obj', 'source-built-upstream-cache'))</CurrentRepoSourceBuiltNupkgCacheDir>
<PrebuiltBaselineDataFileDefault>$(RepositoryEngineeringDir)SourceBuildPrebuiltBaseline.xml</PrebuiltBaselineDataFileDefault>
<PrebuiltBaselineDataFile Condition="Exists('$(PrebuiltBaselineDataFileDefault)')">$(PrebuiltBaselineDataFileDefault)</PrebuiltBaselineDataFile>
<!-- By default, use the license file from the root of the repo for the intermediate nupkg. -->
<DetectSourceBuildIntermediateNupkgLicense Condition="'$(DetectSourceBuildIntermediateNupkgLicense)' == ''">true</DetectSourceBuildIntermediateNupkgLicense>
<EnableDefaultSourceBuildIntermediateItems Condition="'$(EnableDefaultSourceBuildIntermediateItems)' == ''">true</EnableDefaultSourceBuildIntermediateItems>
</PropertyGroup>
<Target Name="GetSourceBuildIntermediateNupkgNameConvention">
<PropertyGroup Condition="'$(SourceBuildNonPortable)' == 'true' and '$(TargetRid)' == ''">
<TargetRid>$([System.Runtime.InteropServices.RuntimeInformation]::RuntimeIdentifier)</TargetRid>
</PropertyGroup>
<PropertyGroup Condition="'$(TargetRid)' == ''">
<HostArchitecture>$([System.Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture.ToString().ToLowerInvariant())</HostArchitecture>
<SourceBuildTargetArchitecture Condition="'$(SourceBuildTargetArchitecture)' == ''">$(HostArchitecture)</SourceBuildTargetArchitecture>
<SourceBuildTargetPortableOSPlatform Condition="$([MSBuild]::IsOSPlatform('windows'))">win</SourceBuildTargetPortableOSPlatform>
<SourceBuildTargetPortableOSPlatform Condition="$([MSBuild]::IsOSPlatform('osx'))">osx</SourceBuildTargetPortableOSPlatform>
<SourceBuildTargetPortableOSPlatform Condition="$([MSBuild]::IsOSPlatform('linux'))">linux</SourceBuildTargetPortableOSPlatform>
<SourceBuildTargetPortableOSPlatform Condition="'$(SourceBuildTargetPortableOSPlatform)'==''">linux</SourceBuildTargetPortableOSPlatform>
<!-- By default, build for the portable OS platform with host machine architecture. -->
<TargetRid>$(SourceBuildTargetPortableOSPlatform)-$(SourceBuildTargetArchitecture)</TargetRid>
</PropertyGroup>
<!-- If this repo builds only managed outputs, no RID is used for the intermediate nupkg. -->
<PropertyGroup Condition="'$(SourceBuildManagedOnly)' != 'true'">
<SourceBuildIntermediateNupkgRid Condition="'$(SourceBuildIntermediateNupkgRid)' == ''">$(TargetRid)</SourceBuildIntermediateNupkgRid>
</PropertyGroup>
<PropertyGroup>
<SourceBuildIntermediateNupkgPrefix>Microsoft.SourceBuild.Intermediate.</SourceBuildIntermediateNupkgPrefix>
<SourceBuildIntermediateNupkgSuffix Condition="'$(SourceBuildIntermediateNupkgRid)' != ''">.$(SourceBuildIntermediateNupkgRid)</SourceBuildIntermediateNupkgSuffix>
</PropertyGroup>
</Target>
<!--
Get the list of nupkg contents, categorized into supplemental categories if necessary. By
default, all non-symbol-package nupkg files and tar.gz files in
'artifacts/packages/{configuration}' are packed in the intermediate nupkg.
To configure this, add a target to eng/SourceBuild.props with
'BeforeTargets="GetCategorizedIntermediateNupkgContents"' that sets up
'IntermediateNupkgArtifactFile' items with optional 'Category' metadata.
When 'Category' is set, this tooling produces one supplemental intermediate nupkg per named
category. All files not in a category are put into the "main" intermediate nupkg.
-->
<Target Name="GetCategorizedIntermediateNupkgContents">
<ItemGroup Condition="'$(EnableDefaultSourceBuildIntermediateItems)' == 'true'">
<!-- Catch-all: anything not in a category gets packed in the 'main' intermediate nupkg. -->
<IntermediateNupkgArtifactFile Include="$(CurrentRepoSourceBuildArtifactsPackagesDir)**\*.nupkg" />
<IntermediateNupkgArtifactFile Include="$(CurrentRepoSourceBuildArtifactsPackagesDir)**\*.tar.gz" />
<!-- Don't pack any symbol packages: not needed for downstream source-build CI. -->
<IntermediateNupkgArtifactFile Remove="$(CurrentRepoSourceBuildArtifactsPackagesDir)**\*.symbols.nupkg" />
</ItemGroup>
<ItemGroup>
<IntermediateNupkgFile Include="@(IntermediateNupkgArtifactFile)" PackagePath="artifacts" />
<!-- Report goes into the 'main' intermediate nupkg. -->
<IntermediateNupkgFile Include="$(SourceBuildSelfPrebuiltReportDir)**\*" PackagePath="prebuilt-report" />
</ItemGroup>
<RemoveDuplicates Inputs="@(IntermediateNupkgFile)">
<Output TaskParameter="Filtered" ItemName="IntermediatePackageFile" />
</RemoveDuplicates>
<ItemGroup>
<IntermediateNonShippingNupkgFile Include="@(IntermediatePackageFile)" Condition="$([System.Text.RegularExpressions.Regex]::Match(%(Identity),'[\\\\/]NonShipping[\\\\/]').Success)"/>
<SupplementalIntermediateNupkgCategory Include="%(IntermediatePackageFile.Category)" />
</ItemGroup>
</Target>
</Project>

View file

@ -0,0 +1,108 @@
<!-- Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this file to you under the MIT license. -->
<Project>
<!-- Shield this project from nonstandard Directory.Build.props/targets. -->
<PropertyGroup>
<ImportDirectoryPackagesProps>false</ImportDirectoryPackagesProps>
<ImportDirectoryBuildProps>false</ImportDirectoryBuildProps>
<ImportDirectoryBuildTargets>false</ImportDirectoryBuildTargets>
</PropertyGroup>
<Import Project="Sdk.props" Sdk="Microsoft.DotNet.Arcade.Sdk" />
<Import Project="Sdk.props" Sdk="Microsoft.NET.Sdk" />
<!-- Import common functionality, including repo extension point. -->
<Import Project="$(SourceBuildArcadeTargetsFile)" />
<PropertyGroup>
<Copyright Condition="'$(Copyright)' == ''">$(CopyrightNetFoundation)</Copyright>
<IsPackable>true</IsPackable>
<IsShipping>false</IsShipping>
<SuppressDependenciesWhenPacking>true</SuppressDependenciesWhenPacking>
<IncludeBuildOutput>false</IncludeBuildOutput>
<!-- NuGet excludes nupkgs by default: disable this behavior. -->
<NoDefaultExcludes>true</NoDefaultExcludes>
<!-- Arbitrary TargetFramework to appease SDK. -->
<TargetFramework>netstandard2.0</TargetFramework>
</PropertyGroup>
<PropertyGroup Condition="'$(SourceBuildIntermediateNupkgLicenseFile)' != ''">
<PackageLicenseFile>$([System.IO.Path]::GetFileName('$(SourceBuildIntermediateNupkgLicenseFile)'))</PackageLicenseFile>
</PropertyGroup>
<ItemGroup Condition="'$(SourceBuildIntermediateNupkgLicenseFile)' != ''">
<None Include="$(SourceBuildIntermediateNupkgLicenseFile)" Pack="true" PackagePath="$(PackageLicenseFile)" />
</ItemGroup>
<Target Name="InitializeSourceBuildIntermediatePackageId"
DependsOnTargets="GetSourceBuildIntermediateNupkgNameConvention"
BeforeTargets="GenerateNuspec;InitializeStandardNuspecProperties">
<Error Condition="'$(GitHubRepositoryName)' == ''" Text="GitHubRepositoryName property is not defined." />
<PropertyGroup>
<PackageId>$(GitHubRepositoryName)</PackageId>
<PackageId Condition="'$(SupplementalIntermediateNupkgCategory)' != ''">$(PackageId).$(SupplementalIntermediateNupkgCategory)</PackageId>
<PackageId>$(SourceBuildIntermediateNupkgPrefix)$(PackageId)$(SourceBuildIntermediateNupkgSuffix)</PackageId>
</PropertyGroup>
</Target>
<Target Name="GetIntermediateNupkgArtifactFiles"
DependsOnTargets="
GetCategorizedIntermediateNupkgContents;
GetSupplementalIntermediateNupkgManifest;
CreateNonShippingNupkgList"
BeforeTargets="_GetPackageFiles">
<ItemGroup>
<Content Include="@(IntermediatePackageFile->WithMetadataValue('Category', '$(SupplementalIntermediateNupkgCategory)'))" />
</ItemGroup>
</Target>
<Target Name="GetSupplementalIntermediateNupkgManifest"
Condition="'$(SupplementalIntermediateNupkgCategory)' == ''"
DependsOnTargets="InitializeSourceBuildIntermediatePackageId">
<PropertyGroup>
<SupplementalIntermediateNupkgManifestFile>$(BaseOutputPath)SupplementalIntermediatePackages.txt</SupplementalIntermediateNupkgManifestFile>
</PropertyGroup>
<WriteLinesToFile
File="$(SupplementalIntermediateNupkgManifestFile)"
Lines="@(SupplementalIntermediateNupkgCategory->'$(SourceBuildIntermediateNupkgPrefix)$(GitHubRepositoryName).%(Identity)$(SourceBuildIntermediateNupkgSuffix)', '%0A')"
Overwrite="true" />
<ItemGroup>
<!-- The list of supplemental package ids goes into the "main" intermediate nupkg. -->
<Content Include="$(SupplementalIntermediateNupkgManifestFile)" PackagePath="." />
</ItemGroup>
</Target>
<!-- Create a list of non-shipping packages and include it in the intermediate package. -->
<Target Name="CreateNonShippingNupkgList"
Condition="'@(IntermediateNonShippingNupkgFile)' != ''">
<PropertyGroup>
<!-- The prefix needs to match what's defined in tarball source-build infra. Consider using a single property, in the future. -->
<NonShippingPackagesListPrefix>NonShipping.Packages.</NonShippingPackagesListPrefix>
<NonShippingPackagesList>$(CurrentRepoSourceBuildArtifactsPackagesDir)$(NonShippingPackagesListPrefix)$(GitHubRepositoryName).lst</NonShippingPackagesList>
</PropertyGroup>
<WriteLinesToFile
File="$(NonShippingPackagesList)"
Lines="@(IntermediateNonShippingNupkgFile->'%(Filename)%(Extension)')"
Overwrite="true" />
<ItemGroup>
<!-- The list of non-shipping packages goes into the "main" intermediate nupkg. -->
<Content Include="$(NonShippingPackagesList)" PackagePath="." />
</ItemGroup>
</Target>
<Import Project="Sdk.targets" Sdk="Microsoft.NET.Sdk" />
<Import Project="Sdk.targets" Sdk="Microsoft.DotNet.Arcade.Sdk" />
<Target Name="Build" />
</Project>

View file

@ -57,6 +57,11 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
/// </summary>
public string OverrideTempPath { get; set; }
/// <summary>
/// Array of files containing lists of non-shipping packages
/// </summary>
public ITaskItem[] NonShippingPackagesListFiles { get; set; }
private static readonly string[] ZipFileExtensions =
{
".zip",
@ -173,6 +178,7 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
/// <returns>List of poisoned packages and files found and reasons for each</returns>
internal IEnumerable<PoisonedFileEntry> GetPoisonedFiles(IEnumerable<string> initialCandidates, string catalogedPackagesFilePath, string markerFileName)
{
IEnumerable<string> nonShippingPackages = GetAllNonShippingPackages();
IEnumerable<CatalogPackageEntry> catalogedPackages = ReadCatalog(catalogedPackagesFilePath);
var poisons = new List<PoisonedFileEntry>();
var candidateQueue = new Queue<string>(initialCandidates);
@ -191,6 +197,14 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
// add its contents to the list to be checked.
if (ZipFileExtensions.Concat(TarFileExtensions).Concat(TarGzFileExtensions).Any(e => checking.ToLowerInvariant().EndsWith(e)))
{
Log.LogMessage($"Zip or NuPkg file to check: {checking}");
// Skip non-shipping packages
if (nonShippingPackages.Contains(Path.GetFileName(checking), StringComparer.OrdinalIgnoreCase))
{
continue;
}
var tempCheckingDir = Path.Combine(tempDir.FullName, Path.GetFileNameWithoutExtension(checking));
PoisonedFileEntry result = ExtractAndCheckZipFileOnly(catalogedPackages, checking, markerFileName, tempCheckingDir, candidateQueue);
if (result != null)
@ -213,6 +227,21 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
return poisons;
}
private IEnumerable<string> GetAllNonShippingPackages()
{
if (NonShippingPackagesListFiles != null)
{
return NonShippingPackagesListFiles
.SelectMany(item => File.ReadAllLines(item.ItemSpec))
.Distinct()
.ToList();
}
else
{
return Enumerable.Empty<string>();
}
}
private static PoisonedFileEntry CheckSingleFile(IEnumerable<CatalogPackageEntry> catalogedPackages, string rootPath, string fileToCheck)
{
// skip some common files that get copied verbatim from nupkgs - LICENSE, _._, etc as well as

View file

@ -355,6 +355,13 @@
<SourceBuiltNupkgFiles Include="$(SourceBuiltPackagesPath)extractArtifacts/**/artifacts/*.nupkg" />
</ItemGroup>
<ItemGroup Condition="'@(_BuiltIntermediatePackages)' != ''">
<NonShippingPackageLists Include="$(SourceBuiltPackagesPath)extractArtifacts/**/$(NonShippingPackagesListPrefix)*" />
</ItemGroup>
<!-- Copy lists of NonShipping packages to prebuilt-report dir -->
<Copy SourceFiles="@(NonShippingPackageLists)" DestinationFolder="$(PackageReportDir)packagelists/" />
<Move
Condition="'@(SourceBuiltNupkgFiles)' != ''"
SourceFiles="@(SourceBuiltNupkgFiles)"

View file

@ -36,6 +36,7 @@
<RepositoryReference Include="diagnostics" />
<RepositoryReference Include="razor" />
<RepositoryReference Include="xliff-tasks" />
<RepositoryReference Include="runtime" />
<RepositoryReference Include="roslyn" />
<RepositoryReference Include="source-build-externals" />
<RepositoryReference Include="symreader" />
@ -43,7 +44,6 @@
<!-- Tier 2 -->
<RepositoryReference Include="linker" />
<RepositoryReference Include="runtime" />
<RepositoryReference Include="msbuild" />
<!-- Tier 3 -->

View file

@ -18,6 +18,10 @@
<NuGetConfigFiles Include="$(ProjectDirectory)/external/cecil/NuGet.config" />
</ItemGroup>
<ItemGroup>
<RepositoryReference Include="runtime" />
</ItemGroup>
<ItemGroup>
<UseSourceBuiltSdkOverride Include="@(ArcadeSdkOverride)" />
</ItemGroup>

View file

@ -26,6 +26,7 @@
<RepositoryReference Include="command-line-api" />
<RepositoryReference Include="source-build-externals" />
<RepositoryReference Include="xliff-tasks" />
<RepositoryReference Include="runtime" />
</ItemGroup>
<ItemGroup>

View file

@ -23,6 +23,7 @@
<BuildCommandArgs>$(BuildCommandArgs) /p:RuntimeOS=$(RuntimeOS)</BuildCommandArgs>
<BuildCommandArgs>$(BuildCommandArgs) /p:BaseOS=$(BaseOS)</BuildCommandArgs>
<BuildCommandArgs>$(BuildCommandArgs) /p:SourceBuildNonPortable=true</BuildCommandArgs>
<BuildCommandArgs>$(BuildCommandArgs) /p:UsingToolMicrosoftNetCompilers=false</BuildCommandArgs>
<BuildCommand>$(StandardSourceBuildCommand) $(BuildCommandArgs)</BuildCommand>
</PropertyGroup>
@ -58,9 +59,7 @@
<ItemGroup>
<RepositoryReference Include="arcade" />
<RepositoryReference Include="symreader" />
<RepositoryReference Include="linker" />
<RepositoryReference Include="source-build-externals" />
<RepositoryReference Include="roslyn" />
</ItemGroup>
<UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="AddRidToRuntimeJson" />

View file

@ -0,0 +1,24 @@
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: Nikola Milosavljevic <nikolam@microsoft.com>
Date: Wed, 25 Jan 2023 20:21:17 +0000
Subject: [PATCH] Allow source-build to set UsingToolMicrosoftNetCompilers
property
backport: https://github.com/dotnet/runtime/pull/81180
---
eng/Versions.props | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/eng/Versions.props b/eng/Versions.props
index a57587c74a8..15f65bc269b 100644
--- a/eng/Versions.props
+++ b/eng/Versions.props
@@ -22,7 +22,7 @@
<UsingToolIbcOptimization>false</UsingToolIbcOptimization>
<UsingToolXliff>false</UsingToolXliff>
<LastReleasedStableAssemblyVersion>$(AssemblyVersion)</LastReleasedStableAssemblyVersion>
- <UsingToolMicrosoftNetCompilers>true</UsingToolMicrosoftNetCompilers>
+ <UsingToolMicrosoftNetCompilers Condition="'$(DotNetBuildFromSource)' != 'true'">true</UsingToolMicrosoftNetCompilers>
</PropertyGroup>
<ItemGroup>
<WorkloadSdkBandVersions Include="$(SdkBandVersion)" SupportsMachineArch="true" />