mirrors/dotnet-installer
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

Poison work - reordering and non-shipping packages

Browse source
This commit is contained in:
Nikola Milosavljevic 2023-01-25 18:59:16 +00:00
parent 1c38575cf1
commit 8d07426e10
11 changed files with 101 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
src/SourceBuild/content/Directory.Build.props
View file

@ -125,6 +125,9 @@
<PackageReportDir>$(BaseOutputPath)prebuilt-report/</PackageReportDir>
<RepoProjectsDir>$(ProjectDir)/repo-projects/</RepoProjectsDir>
<ResultingPrebuiltPackagesDir>$(PackageReportDir)prebuilt-packages/</ResultingPrebuiltPackagesDir>
<PackageListsDir>$(PackageReportDir)packagelists/</PackageListsDir>
<!-- The prefix needs to match what's defined in Arcade's source-build infra. Consider using a single property, in the future. -->
<NonShippingPackagesListPrefix>NonShipping.Packages.</NonShippingPackagesListPrefix>
<PackageReportDataFile>$(PackageReportDir)prebuilt-usage.xml</PackageReportDataFile>
<PoisonUsageReportFile>$(PackageReportDir)poison-usage.xml</PoisonUsageReportFile>
<PoisonReportDataFile>$(PackageReportDir)poison-catalog.xml</PoisonReportDataFile>

7
src/SourceBuild/content/build.proj
View file

@ -70,10 +70,15 @@
<Message Importance="High" Text="[$([System.DateTime]::Now.ToString('HH:mm:ss.ff'))] Checking @(FinalCliTarball) for poisoned files." />
<ItemGroup>
<NonShippingPackagesList Include="$(PackageListsDir)**/$(NonShippingPackagesListPrefix)*" />
</ItemGroup>
<CheckForPoison FilesToCheck="@(FinalCliTarball)"
HashCatalogFilePath="$(PoisonReportDataFile)"
MarkerFileName="$(PoisonMarkerFile)"
PoisonReportOutputFilePath="$(PoisonUsageReportFile)" />
PoisonReportOutputFilePath="$(PoisonUsageReportFile)"
NonShippingPackagesListFiles="@(NonShippingPackagesList)" />
<WriteLinesToFile File="$(CompletedSemaphorePath)ReportPoisonUsage.complete" Overwrite="true" />
<Message Importance="High" Text="[$([System.DateTime]::Now.ToString('HH:mm:ss.ff'))] Done checking for poison." />

1
src/SourceBuild/content/eng/arcade-overrides/SourceBuildArcade.targets
View file

@ -92,6 +92,7 @@
</RemoveDuplicates>
<ItemGroup>
<IntermediateNonShippingNupkgFile Include="@(IntermediatePackageFile)" Condition="$([System.Text.RegularExpressions.Regex]::Match(%(Identity),'[\\\\/]NonShipping[\\\\/]').Success)"/>
<SupplementalIntermediateNupkgCategory Include="%(IntermediatePackageFile.Category)" />
</ItemGroup>
</Target>

24
src/SourceBuild/content/eng/arcade-overrides/SourceBuildIntermediate.proj
View file

@ -53,7 +53,8 @@
<Target Name="GetIntermediateNupkgArtifactFiles"
DependsOnTargets="
GetCategorizedIntermediateNupkgContents;
GetSupplementalIntermediateNupkgManifest"
GetSupplementalIntermediateNupkgManifest;
CreateNonShippingNupkgList"
BeforeTargets="_GetPackageFiles">
<ItemGroup>
<Content Include="@(IntermediatePackageFile->WithMetadataValue('Category', '$(SupplementalIntermediateNupkgCategory)'))" />
@ -78,6 +79,27 @@
</ItemGroup>
</Target>
<!-- Create a list of non-shipping packages and include it in the intermediate package. -->
<Target Name="CreateNonShippingNupkgList"
Condition="'@(IntermediateNonShippingNupkgFile)' != ''">
<PropertyGroup>
<!-- The prefix needs to match what's defined in tarball source-build infra. Consider using a single property, in the future. -->
<NonShippingPackagesListPrefix>NonShipping.Packages.</NonShippingPackagesListPrefix>
<NonShippingPackagesList>$(CurrentRepoSourceBuildArtifactsPackagesDir)$(NonShippingPackagesListPrefix)$(GitHubRepositoryName).lst</NonShippingPackagesList>
</PropertyGroup>
<WriteLinesToFile
File="$(NonShippingPackagesList)"
Lines="@(IntermediateNonShippingNupkgFile->'%(Filename)%(Extension)')"
Overwrite="true" />
<ItemGroup>
<!-- The list of non-shipping packages goes into the "main" intermediate nupkg. -->
<Content Include="$(NonShippingPackagesList)" PackagePath="." />
</ItemGroup>
</Target>
<Import Project="Sdk.targets" Sdk="Microsoft.NET.Sdk" />
<Import Project="Sdk.targets" Sdk="Microsoft.DotNet.Arcade.Sdk" />

28
src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CheckForPoison.cs
View file

@ -57,6 +57,11 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
/// </summary>
public string OverrideTempPath { get; set; }
/// <summary>
/// Array of files containing lists of non-shipping packages
/// </summary>
public ITaskItem[] NonShippingPackagesListFiles { get; set; }
private static readonly string[] ZipFileExtensions =
{
".zip",
@ -173,6 +178,7 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
/// <returns>List of poisoned packages and files found and reasons for each</returns>
internal IEnumerable<PoisonedFileEntry> GetPoisonedFiles(IEnumerable<string> initialCandidates, string catalogedPackagesFilePath, string markerFileName)
{
IEnumerable<string> nonShippingPackages = GetAllNonShippingPackages();
IEnumerable<CatalogPackageEntry> catalogedPackages = ReadCatalog(catalogedPackagesFilePath);
var poisons = new List<PoisonedFileEntry>();
var candidateQueue = new Queue<string>(initialCandidates);
@ -191,6 +197,14 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
// add its contents to the list to be checked.
if (ZipFileExtensions.Concat(TarFileExtensions).Concat(TarGzFileExtensions).Any(e => checking.ToLowerInvariant().EndsWith(e)))
{
Log.LogMessage($"Zip or NuPkg file to check: {checking}");
// Skip non-shipping packages
if (nonShippingPackages.Contains(Path.GetFileName(checking), StringComparer.OrdinalIgnoreCase))
{
continue;
}
var tempCheckingDir = Path.Combine(tempDir.FullName, Path.GetFileNameWithoutExtension(checking));
PoisonedFileEntry result = ExtractAndCheckZipFileOnly(catalogedPackages, checking, markerFileName, tempCheckingDir, candidateQueue);
if (result != null)
@ -213,6 +227,20 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
return poisons;
}
private IEnumerable<string> GetAllNonShippingPackages()
{
if (NonShippingPackagesListFiles != null)
{
return NonShippingPackagesListFiles
.SelectMany(item => File.ReadAllLines(item.ItemSpec))
.Distinct();
}
else
{
return Enumerable.Empty<string>();
}
}
private static PoisonedFileEntry CheckSingleFile(IEnumerable<CatalogPackageEntry> catalogedPackages, string rootPath, string fileToCheck)
{
// skip some common files that get copied verbatim from nupkgs - LICENSE, _._, etc as well as

7
src/SourceBuild/content/repo-projects/Directory.Build.targets
View file

@ -355,6 +355,13 @@
<SourceBuiltNupkgFiles Include="$(SourceBuiltPackagesPath)extractArtifacts/**/artifacts/*.nupkg" />
</ItemGroup>
<ItemGroup Condition="'@(_BuiltIntermediatePackages)' != ''">
<NonShippingPackageLists Include="$(SourceBuiltPackagesPath)extractArtifacts/**/$(NonShippingPackagesListPrefix)*" />
</ItemGroup>
<!-- Copy lists of NonShipping packages to prebuilt-report dir -->
<Copy SourceFiles="@(NonShippingPackageLists)" DestinationFolder="$(PackageReportDir)packagelists/" />
<Move
Condition="'@(SourceBuiltNupkgFiles)' != ''"
SourceFiles="@(SourceBuiltNupkgFiles)"

2
src/SourceBuild/content/repo-projects/known-good.proj
View file

@ -36,6 +36,7 @@
<RepositoryReference Include="diagnostics" />
<RepositoryReference Include="razor" />
<RepositoryReference Include="xliff-tasks" />
<RepositoryReference Include="runtime" />
<RepositoryReference Include="roslyn" />
<RepositoryReference Include="source-build-externals" />
<RepositoryReference Include="symreader" />
@ -43,7 +44,6 @@
<!-- Tier 2 -->
<RepositoryReference Include="linker" />
<RepositoryReference Include="runtime" />
<RepositoryReference Include="msbuild" />
<!-- Tier 3 -->

4
src/SourceBuild/content/repo-projects/linker.proj
View file

@ -18,6 +18,10 @@
<NuGetConfigFiles Include="$(ProjectDirectory)/external/cecil/NuGet.config" />
</ItemGroup>
<ItemGroup>
<RepositoryReference Include="runtime" />
</ItemGroup>
<ItemGroup>
<UseSourceBuiltSdkOverride Include="@(ArcadeSdkOverride)" />
</ItemGroup>

1
src/SourceBuild/content/repo-projects/roslyn.proj
View file

@ -26,6 +26,7 @@
<RepositoryReference Include="command-line-api" />
<RepositoryReference Include="source-build-externals" />
<RepositoryReference Include="xliff-tasks" />
<RepositoryReference Include="runtime" />
</ItemGroup>
<ItemGroup>

3
src/SourceBuild/content/repo-projects/runtime.proj
View file

@ -23,6 +23,7 @@
<BuildCommandArgs>$(BuildCommandArgs) /p:RuntimeOS=$(RuntimeOS)</BuildCommandArgs>
<BuildCommandArgs>$(BuildCommandArgs) /p:BaseOS=$(BaseOS)</BuildCommandArgs>
<BuildCommandArgs>$(BuildCommandArgs) /p:SourceBuildNonPortable=true</BuildCommandArgs>
<BuildCommandArgs>$(BuildCommandArgs) /p:UsingToolMicrosoftNetCompilers=false</BuildCommandArgs>
<BuildCommand>$(StandardSourceBuildCommand) $(BuildCommandArgs)</BuildCommand>
</PropertyGroup>
@ -58,9 +59,7 @@
<ItemGroup>
<RepositoryReference Include="arcade" />
<RepositoryReference Include="symreader" />
<RepositoryReference Include="linker" />
<RepositoryReference Include="source-build-externals" />
<RepositoryReference Include="roslyn" />
</ItemGroup>
<UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="AddRidToRuntimeJson" />

26
src/SourceBuild/patches/0001-Allow-source-build-to-set-UsingToolMicrosoftNetCompi.patch Normal file
View file

@ -0,0 +1,26 @@
From ed89288f65df0db02817a9d7f72b768b51b4b57a Mon Sep 17 00:00:00 2001
From: Nikola Milosavljevic <nikolam@microsoft.com>
Date: Tue, 24 Jan 2023 18:03:12 +0000
Subject: [PATCH] Allow source-build to set UsingToolMicrosoftNetCompilers
property
---
eng/Versions.props | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/eng/Versions.props b/eng/Versions.props
index 3a97f80fd74..6148f6c6ae5 100644
--- a/eng/Versions.props
+++ b/eng/Versions.props
@@ -22,7 +22,7 @@
<UsingToolIbcOptimization>false</UsingToolIbcOptimization>
<UsingToolXliff>false</UsingToolXliff>
<LastReleasedStableAssemblyVersion>$(AssemblyVersion)</LastReleasedStableAssemblyVersion>
- <UsingToolMicrosoftNetCompilers>true</UsingToolMicrosoftNetCompilers>
+ <UsingToolMicrosoftNetCompilers Condition="'$(DotNetBuildFromSource)' != 'true'">true</UsingToolMicrosoftNetCompilers>
</PropertyGroup>
<ItemGroup>
<WorkloadSdkBandVersions Include="$(SdkBandVersion)" SupportsMachineArch="true" />
--
2.29.2