Poison work - reordering and non-shipping packages

2023-01-25 18:59:16 +00:00 · 2023-01-25 18:59:16 +00:00 · 8d07426e10
commit 8d07426e10
parent 1c38575cf1
11 changed files with 101 additions and 5 deletions
--- a/src/SourceBuild/content/Directory.Build.props
+++ b/src/SourceBuild/content/Directory.Build.props
@ -125,6 +125,9 @@
    <PackageReportDir>$(BaseOutputPath)prebuilt-report/</PackageReportDir>
    <RepoProjectsDir>$(ProjectDir)/repo-projects/</RepoProjectsDir>
    <ResultingPrebuiltPackagesDir>$(PackageReportDir)prebuilt-packages/</ResultingPrebuiltPackagesDir>
+    <PackageListsDir>$(PackageReportDir)packagelists/</PackageListsDir>
+    <!-- The prefix needs to match what's defined in Arcade's source-build infra. Consider using a single property, in the future. -->
+    <NonShippingPackagesListPrefix>NonShipping.Packages.</NonShippingPackagesListPrefix>
    <PackageReportDataFile>$(PackageReportDir)prebuilt-usage.xml</PackageReportDataFile>
    <PoisonUsageReportFile>$(PackageReportDir)poison-usage.xml</PoisonUsageReportFile>
    <PoisonReportDataFile>$(PackageReportDir)poison-catalog.xml</PoisonReportDataFile>
--- a/src/SourceBuild/content/build.proj
+++ b/src/SourceBuild/content/build.proj
@ -70,10 +70,15 @@

    <Message Importance="High" Text="[$([System.DateTime]::Now.ToString('HH:mm:ss.ff'))] Checking @(FinalCliTarball) for poisoned files." />

+    <ItemGroup>
+      <NonShippingPackagesList Include="$(PackageListsDir)**/$(NonShippingPackagesListPrefix)*" />
+    </ItemGroup>
+
    <CheckForPoison FilesToCheck="@(FinalCliTarball)"
                    HashCatalogFilePath="$(PoisonReportDataFile)"
                    MarkerFileName="$(PoisonMarkerFile)"
-                    PoisonReportOutputFilePath="$(PoisonUsageReportFile)" />
+                    PoisonReportOutputFilePath="$(PoisonUsageReportFile)"
+                    NonShippingPackagesListFiles="@(NonShippingPackagesList)" />

    <WriteLinesToFile File="$(CompletedSemaphorePath)ReportPoisonUsage.complete" Overwrite="true" />
    <Message Importance="High" Text="[$([System.DateTime]::Now.ToString('HH:mm:ss.ff'))] Done checking for poison." />
--- a/src/SourceBuild/content/eng/arcade-overrides/SourceBuildArcade.targets
+++ b/src/SourceBuild/content/eng/arcade-overrides/SourceBuildArcade.targets
@ -92,6 +92,7 @@
    </RemoveDuplicates>

    <ItemGroup>
+      <IntermediateNonShippingNupkgFile Include="@(IntermediatePackageFile)" Condition="$([System.Text.RegularExpressions.Regex]::Match(%(Identity),'[\\\\/]NonShipping[\\\\/]').Success)"/>
      <SupplementalIntermediateNupkgCategory Include="%(IntermediatePackageFile.Category)" />
    </ItemGroup>
  </Target>
--- a/src/SourceBuild/content/eng/arcade-overrides/SourceBuildIntermediate.proj
+++ b/src/SourceBuild/content/eng/arcade-overrides/SourceBuildIntermediate.proj
@ -53,7 +53,8 @@
  <Target Name="GetIntermediateNupkgArtifactFiles"
          DependsOnTargets="
            GetCategorizedIntermediateNupkgContents;
-            GetSupplementalIntermediateNupkgManifest"
+            GetSupplementalIntermediateNupkgManifest;
+            CreateNonShippingNupkgList"
          BeforeTargets="_GetPackageFiles">
    <ItemGroup>
      <Content Include="@(IntermediatePackageFile->WithMetadataValue('Category', '$(SupplementalIntermediateNupkgCategory)'))" />
@ -78,6 +79,27 @@
    </ItemGroup>
  </Target>

+  <!-- Create a list of non-shipping packages and include it in the intermediate package. -->
+  <Target Name="CreateNonShippingNupkgList"
+          Condition="'@(IntermediateNonShippingNupkgFile)' != ''">
+    <PropertyGroup>
+      <!-- The prefix needs to match what's defined in tarball source-build infra. Consider using a single property, in the future. -->
+      <NonShippingPackagesListPrefix>NonShipping.Packages.</NonShippingPackagesListPrefix>
+      <NonShippingPackagesList>$(CurrentRepoSourceBuildArtifactsPackagesDir)$(NonShippingPackagesListPrefix)$(GitHubRepositoryName).lst</NonShippingPackagesList>
+    </PropertyGroup>
+
+    <WriteLinesToFile
+      File="$(NonShippingPackagesList)"
+      Lines="@(IntermediateNonShippingNupkgFile->'%(Filename)%(Extension)')"
+      Overwrite="true" />
+
+    <ItemGroup>
+      <!-- The list of non-shipping packages goes into the "main" intermediate nupkg. -->
+      <Content Include="$(NonShippingPackagesList)" PackagePath="." />
+    </ItemGroup>
+  </Target>
+
+
  <Import Project="Sdk.targets" Sdk="Microsoft.NET.Sdk" />
  <Import Project="Sdk.targets" Sdk="Microsoft.DotNet.Arcade.Sdk" />

--- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CheckForPoison.cs
+++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CheckForPoison.cs
@ -57,6 +57,11 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
        /// </summary>
        public string OverrideTempPath { get; set; }

+        /// <summary>
+        /// Array of files containing lists of non-shipping packages
+        /// </summary>
+        public ITaskItem[] NonShippingPackagesListFiles { get; set; }
+
        private static readonly string[] ZipFileExtensions =
        {
            ".zip",
@ -173,6 +178,7 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
        /// <returns>List of poisoned packages and files found and reasons for each</returns>
        internal IEnumerable<PoisonedFileEntry> GetPoisonedFiles(IEnumerable<string> initialCandidates, string catalogedPackagesFilePath, string markerFileName)
        {
+            IEnumerable<string> nonShippingPackages = GetAllNonShippingPackages();
            IEnumerable<CatalogPackageEntry> catalogedPackages = ReadCatalog(catalogedPackagesFilePath);
            var poisons = new List<PoisonedFileEntry>();
            var candidateQueue = new Queue<string>(initialCandidates);
@ -191,6 +197,14 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
                // add its contents to the list to be checked.
                if (ZipFileExtensions.Concat(TarFileExtensions).Concat(TarGzFileExtensions).Any(e => checking.ToLowerInvariant().EndsWith(e)))
                {
+                    Log.LogMessage($"Zip or NuPkg file to check: {checking}");
+
+                    // Skip non-shipping packages
+                    if (nonShippingPackages.Contains(Path.GetFileName(checking), StringComparer.OrdinalIgnoreCase))
+                    {
+                        continue;
+                    }
+
                    var tempCheckingDir = Path.Combine(tempDir.FullName, Path.GetFileNameWithoutExtension(checking));
                    PoisonedFileEntry result = ExtractAndCheckZipFileOnly(catalogedPackages, checking, markerFileName, tempCheckingDir, candidateQueue);
                    if (result != null)
@ -213,6 +227,20 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
            return poisons;
        }

+        private IEnumerable<string> GetAllNonShippingPackages()
+        {
+            if (NonShippingPackagesListFiles != null)
+            {
+                return NonShippingPackagesListFiles
+                    .SelectMany(item => File.ReadAllLines(item.ItemSpec))
+                    .Distinct();
+            }
+            else
+            {
+                return Enumerable.Empty<string>();
+            }
+        }
+
        private static PoisonedFileEntry CheckSingleFile(IEnumerable<CatalogPackageEntry> catalogedPackages, string rootPath, string fileToCheck)
        {
            // skip some common files that get copied verbatim from nupkgs - LICENSE, _._, etc as well as
--- a/src/SourceBuild/content/repo-projects/Directory.Build.targets
+++ b/src/SourceBuild/content/repo-projects/Directory.Build.targets
@ -355,6 +355,13 @@
      <SourceBuiltNupkgFiles Include="$(SourceBuiltPackagesPath)extractArtifacts/**/artifacts/*.nupkg" />
    </ItemGroup>

+    <ItemGroup Condition="'@(_BuiltIntermediatePackages)' != ''">
+      <NonShippingPackageLists Include="$(SourceBuiltPackagesPath)extractArtifacts/**/$(NonShippingPackagesListPrefix)*" />
+    </ItemGroup>
+
+    <!-- Copy lists of NonShipping packages to prebuilt-report dir -->
+    <Copy SourceFiles="@(NonShippingPackageLists)" DestinationFolder="$(PackageReportDir)packagelists/" />
+
    <Move
      Condition="'@(SourceBuiltNupkgFiles)' != ''"
      SourceFiles="@(SourceBuiltNupkgFiles)"
--- a/src/SourceBuild/content/repo-projects/known-good.proj
+++ b/src/SourceBuild/content/repo-projects/known-good.proj
@ -36,6 +36,7 @@
        <RepositoryReference Include="diagnostics" />
        <RepositoryReference Include="razor" />
        <RepositoryReference Include="xliff-tasks" />
+        <RepositoryReference Include="runtime" />
        <RepositoryReference Include="roslyn" />
        <RepositoryReference Include="source-build-externals" />
        <RepositoryReference Include="symreader" />
@ -43,7 +44,6 @@

        <!-- Tier 2 -->
        <RepositoryReference Include="linker" />
-        <RepositoryReference Include="runtime" />
        <RepositoryReference Include="msbuild" />

        <!-- Tier 3 -->
--- a/src/SourceBuild/content/repo-projects/linker.proj
+++ b/src/SourceBuild/content/repo-projects/linker.proj
@ -18,6 +18,10 @@
    <NuGetConfigFiles Include="$(ProjectDirectory)/external/cecil/NuGet.config" />
  </ItemGroup>

+  <ItemGroup>
+    <RepositoryReference Include="runtime" />
+  </ItemGroup>
+
  <ItemGroup>
    <UseSourceBuiltSdkOverride Include="@(ArcadeSdkOverride)" />
  </ItemGroup>
--- a/src/SourceBuild/content/repo-projects/roslyn.proj
+++ b/src/SourceBuild/content/repo-projects/roslyn.proj
@ -26,6 +26,7 @@
    <RepositoryReference Include="command-line-api" />
    <RepositoryReference Include="source-build-externals" />
    <RepositoryReference Include="xliff-tasks" />
+    <RepositoryReference Include="runtime" />
  </ItemGroup>

  <ItemGroup>
--- a/src/SourceBuild/content/repo-projects/runtime.proj
+++ b/src/SourceBuild/content/repo-projects/runtime.proj
@ -23,6 +23,7 @@
    <BuildCommandArgs>$(BuildCommandArgs) /p:RuntimeOS=$(RuntimeOS)</BuildCommandArgs>
    <BuildCommandArgs>$(BuildCommandArgs) /p:BaseOS=$(BaseOS)</BuildCommandArgs>
    <BuildCommandArgs>$(BuildCommandArgs) /p:SourceBuildNonPortable=true</BuildCommandArgs>
+    <BuildCommandArgs>$(BuildCommandArgs) /p:UsingToolMicrosoftNetCompilers=false</BuildCommandArgs>
    <BuildCommand>$(StandardSourceBuildCommand) $(BuildCommandArgs)</BuildCommand>
  </PropertyGroup>

@ -58,9 +59,7 @@
  <ItemGroup>
      <RepositoryReference Include="arcade" />
      <RepositoryReference Include="symreader" />
-      <RepositoryReference Include="linker" />
      <RepositoryReference Include="source-build-externals" />
-      <RepositoryReference Include="roslyn" />
  </ItemGroup>

  <UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="AddRidToRuntimeJson" />
--- a/src/SourceBuild/patches/0001-Allow-source-build-to-set-UsingToolMicrosoftNetCompi.patch
+++ b/src/SourceBuild/patches/0001-Allow-source-build-to-set-UsingToolMicrosoftNetCompi.patch
@ -0,0 +1,26 @@
+From ed89288f65df0db02817a9d7f72b768b51b4b57a Mon Sep 17 00:00:00 2001
+From: Nikola Milosavljevic <nikolam@microsoft.com>
+Date: Tue, 24 Jan 2023 18:03:12 +0000
+Subject: [PATCH] Allow source-build to set UsingToolMicrosoftNetCompilers
+ property
+
+---
+ eng/Versions.props | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/eng/Versions.props b/eng/Versions.props
+index 3a97f80fd74..6148f6c6ae5 100644
+--- a/eng/Versions.props
+++ b/eng/Versions.props
+@@ -22,7 +22,7 @@
+     <UsingToolIbcOptimization>false</UsingToolIbcOptimization>
+     <UsingToolXliff>false</UsingToolXliff>
+     <LastReleasedStableAssemblyVersion>$(AssemblyVersion)</LastReleasedStableAssemblyVersion>
+-    <UsingToolMicrosoftNetCompilers>true</UsingToolMicrosoftNetCompilers>
+    <UsingToolMicrosoftNetCompilers Condition="'$(DotNetBuildFromSource)' != 'true'">true</UsingToolMicrosoftNetCompilers>
+   </PropertyGroup>
+   <ItemGroup>
+     <WorkloadSdkBandVersions Include="$(SdkBandVersion)" SupportsMachineArch="true" />
+-- 
+2.29.2
+