Poison work - reordering and non-shipping packages

src/SourceBuild/content/Directory.Build.props

@@ -125,6 +125,9 @@
     <PackageReportDir>$(BaseOutputPath)prebuilt-report/</PackageReportDir>
     <RepoProjectsDir>$(ProjectDir)/repo-projects/</RepoProjectsDir>
     <ResultingPrebuiltPackagesDir>$(PackageReportDir)prebuilt-packages/</ResultingPrebuiltPackagesDir>
+    <PackageListsDir>$(PackageReportDir)packagelists/</PackageListsDir>
+    <!-- The prefix needs to match what's defined in Arcade's source-build infra. Consider using a single property, in the future. -->
+    <NonShippingPackagesListPrefix>NonShipping.Packages.</NonShippingPackagesListPrefix>
     <PackageReportDataFile>$(PackageReportDir)prebuilt-usage.xml</PackageReportDataFile>
     <PoisonUsageReportFile>$(PackageReportDir)poison-usage.xml</PoisonUsageReportFile>
     <PoisonReportDataFile>$(PackageReportDir)poison-catalog.xml</PoisonReportDataFile>

src/SourceBuild/content/build.proj

@@ -70,10 +70,15 @@
 
     <Message Importance="High" Text="[$([System.DateTime]::Now.ToString('HH:mm:ss.ff'))] Checking @(FinalCliTarball) for poisoned files." />
 
+    <ItemGroup>
+      <NonShippingPackagesList Include="$(PackageListsDir)**/$(NonShippingPackagesListPrefix)*" />
+    </ItemGroup>
+
     <CheckForPoison FilesToCheck="@(FinalCliTarball)"
                     HashCatalogFilePath="$(PoisonReportDataFile)"
                     MarkerFileName="$(PoisonMarkerFile)"
-                    PoisonReportOutputFilePath="$(PoisonUsageReportFile)" />
+                    PoisonReportOutputFilePath="$(PoisonUsageReportFile)"
+                    NonShippingPackagesListFiles="@(NonShippingPackagesList)" />
 
     <WriteLinesToFile File="$(CompletedSemaphorePath)ReportPoisonUsage.complete" Overwrite="true" />
     <Message Importance="High" Text="[$([System.DateTime]::Now.ToString('HH:mm:ss.ff'))] Done checking for poison." />

src/SourceBuild/content/eng/arcade-overrides/SourceBuildArcade.targets

@@ -92,6 +92,7 @@
     </RemoveDuplicates>
 
     <ItemGroup>
+      <IntermediateNonShippingNupkgFile Include="@(IntermediatePackageFile)" Condition="$([System.Text.RegularExpressions.Regex]::Match(%(Identity),'[\\\\/]NonShipping[\\\\/]').Success)"/>
       <SupplementalIntermediateNupkgCategory Include="%(IntermediatePackageFile.Category)" />
     </ItemGroup>
   </Target>

src/SourceBuild/content/eng/arcade-overrides/SourceBuildIntermediate.proj

@@ -53,7 +53,8 @@
   <Target Name="GetIntermediateNupkgArtifactFiles"
           DependsOnTargets="
             GetCategorizedIntermediateNupkgContents;
-            GetSupplementalIntermediateNupkgManifest"
+            GetSupplementalIntermediateNupkgManifest;
+            CreateNonShippingNupkgList"
           BeforeTargets="_GetPackageFiles">
     <ItemGroup>
       <Content Include="@(IntermediatePackageFile->WithMetadataValue('Category', '$(SupplementalIntermediateNupkgCategory)'))" />
@@ -78,6 +79,27 @@
     </ItemGroup>
   </Target>
 
+  <!-- Create a list of non-shipping packages and include it in the intermediate package. -->
+  <Target Name="CreateNonShippingNupkgList"
+          Condition="'@(IntermediateNonShippingNupkgFile)' != ''">
+    <PropertyGroup>
+      <!-- The prefix needs to match what's defined in tarball source-build infra. Consider using a single property, in the future. -->
+      <NonShippingPackagesListPrefix>NonShipping.Packages.</NonShippingPackagesListPrefix>
+      <NonShippingPackagesList>$(CurrentRepoSourceBuildArtifactsPackagesDir)$(NonShippingPackagesListPrefix)$(GitHubRepositoryName).lst</NonShippingPackagesList>
+    </PropertyGroup>
+
+    <WriteLinesToFile
+      File="$(NonShippingPackagesList)"
+      Lines="@(IntermediateNonShippingNupkgFile->'%(Filename)%(Extension)')"
+      Overwrite="true" />
+
+    <ItemGroup>
+      <!-- The list of non-shipping packages goes into the "main" intermediate nupkg. -->
+      <Content Include="$(NonShippingPackagesList)" PackagePath="." />
+    </ItemGroup>
+  </Target>
+
+
   <Import Project="Sdk.targets" Sdk="Microsoft.NET.Sdk" />
   <Import Project="Sdk.targets" Sdk="Microsoft.DotNet.Arcade.Sdk" />
 

src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CheckForPoison.cs

@@ -57,6 +57,11 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
         /// </summary>
         public string OverrideTempPath { get; set; }
 
+        /// <summary>
+        /// Array of files containing lists of non-shipping packages
+        /// </summary>
+        public ITaskItem[] NonShippingPackagesListFiles { get; set; }
+
         private static readonly string[] ZipFileExtensions =
         {
             ".zip",
@@ -173,6 +178,7 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
         /// <returns>List of poisoned packages and files found and reasons for each</returns>
         internal IEnumerable<PoisonedFileEntry> GetPoisonedFiles(IEnumerable<string> initialCandidates, string catalogedPackagesFilePath, string markerFileName)
         {
+            IEnumerable<string> nonShippingPackages = GetAllNonShippingPackages();
             IEnumerable<CatalogPackageEntry> catalogedPackages = ReadCatalog(catalogedPackagesFilePath);
             var poisons = new List<PoisonedFileEntry>();
             var candidateQueue = new Queue<string>(initialCandidates);
@@ -191,6 +197,14 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
                 // add its contents to the list to be checked.
                 if (ZipFileExtensions.Concat(TarFileExtensions).Concat(TarGzFileExtensions).Any(e => checking.ToLowerInvariant().EndsWith(e)))
                 {
+                    Log.LogMessage($"Zip or NuPkg file to check: {checking}");
+
+                    // Skip non-shipping packages
+                    if (nonShippingPackages.Contains(Path.GetFileName(checking), StringComparer.OrdinalIgnoreCase))
+                    {
+                        continue;
+                    }
+
                     var tempCheckingDir = Path.Combine(tempDir.FullName, Path.GetFileNameWithoutExtension(checking));
                     PoisonedFileEntry result = ExtractAndCheckZipFileOnly(catalogedPackages, checking, markerFileName, tempCheckingDir, candidateQueue);
                     if (result != null)
@@ -213,6 +227,20 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
             return poisons;
         }
 
+        private IEnumerable<string> GetAllNonShippingPackages()
+        {
+            if (NonShippingPackagesListFiles != null)
+            {
+                return NonShippingPackagesListFiles
+                    .SelectMany(item => File.ReadAllLines(item.ItemSpec))
+                    .Distinct();
+            }
+            else
+            {
+                return Enumerable.Empty<string>();
+            }
+        }
+
         private static PoisonedFileEntry CheckSingleFile(IEnumerable<CatalogPackageEntry> catalogedPackages, string rootPath, string fileToCheck)
         {
             // skip some common files that get copied verbatim from nupkgs - LICENSE, _._, etc as well as

src/SourceBuild/content/repo-projects/Directory.Build.targets

@@ -355,6 +355,13 @@
       <SourceBuiltNupkgFiles Include="$(SourceBuiltPackagesPath)extractArtifacts/**/artifacts/*.nupkg" />
     </ItemGroup>
 
+    <ItemGroup Condition="'@(_BuiltIntermediatePackages)' != ''">
+      <NonShippingPackageLists Include="$(SourceBuiltPackagesPath)extractArtifacts/**/$(NonShippingPackagesListPrefix)*" />
+    </ItemGroup>
+
+    <!-- Copy lists of NonShipping packages to prebuilt-report dir -->
+    <Copy SourceFiles="@(NonShippingPackageLists)" DestinationFolder="$(PackageReportDir)packagelists/" />
+
     <Move
       Condition="'@(SourceBuiltNupkgFiles)' != ''"
       SourceFiles="@(SourceBuiltNupkgFiles)"

src/SourceBuild/content/repo-projects/known-good.proj

@@ -36,6 +36,7 @@
         <RepositoryReference Include="diagnostics" />
         <RepositoryReference Include="razor" />
         <RepositoryReference Include="xliff-tasks" />
+        <RepositoryReference Include="runtime" />
         <RepositoryReference Include="roslyn" />
         <RepositoryReference Include="source-build-externals" />
         <RepositoryReference Include="symreader" />
@@ -43,7 +44,6 @@
 
         <!-- Tier 2 -->
         <RepositoryReference Include="linker" />
-        <RepositoryReference Include="runtime" />
         <RepositoryReference Include="msbuild" />
 
         <!-- Tier 3 -->

src/SourceBuild/content/repo-projects/linker.proj

@@ -18,6 +18,10 @@
     <NuGetConfigFiles Include="$(ProjectDirectory)/external/cecil/NuGet.config" />
   </ItemGroup>
 
+  <ItemGroup>
+    <RepositoryReference Include="runtime" />
+  </ItemGroup>
+
   <ItemGroup>
     <UseSourceBuiltSdkOverride Include="@(ArcadeSdkOverride)" />
   </ItemGroup>

src/SourceBuild/content/repo-projects/roslyn.proj

@@ -26,6 +26,7 @@
     <RepositoryReference Include="command-line-api" />
     <RepositoryReference Include="source-build-externals" />
     <RepositoryReference Include="xliff-tasks" />
+    <RepositoryReference Include="runtime" />
   </ItemGroup>
 
   <ItemGroup>

src/SourceBuild/content/repo-projects/runtime.proj

@@ -23,6 +23,7 @@
     <BuildCommandArgs>$(BuildCommandArgs) /p:RuntimeOS=$(RuntimeOS)</BuildCommandArgs>
     <BuildCommandArgs>$(BuildCommandArgs) /p:BaseOS=$(BaseOS)</BuildCommandArgs>
     <BuildCommandArgs>$(BuildCommandArgs) /p:SourceBuildNonPortable=true</BuildCommandArgs>
+    <BuildCommandArgs>$(BuildCommandArgs) /p:UsingToolMicrosoftNetCompilers=false</BuildCommandArgs>
     <BuildCommand>$(StandardSourceBuildCommand) $(BuildCommandArgs)</BuildCommand>
   </PropertyGroup>
 
@@ -58,9 +59,7 @@
   <ItemGroup>
       <RepositoryReference Include="arcade" />
       <RepositoryReference Include="symreader" />
-      <RepositoryReference Include="linker" />
       <RepositoryReference Include="source-build-externals" />
-      <RepositoryReference Include="roslyn" />
   </ItemGroup>
 
   <UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="AddRidToRuntimeJson" />

src/SourceBuild/patches/0001-Allow-source-build-to-set-UsingToolMicrosoftNetCompi.patch

@@ -0,0 +1,26 @@
+From ed89288f65df0db02817a9d7f72b768b51b4b57a Mon Sep 17 00:00:00 2001
+From: Nikola Milosavljevic <nikolam@microsoft.com>
+Date: Tue, 24 Jan 2023 18:03:12 +0000
+Subject: [PATCH] Allow source-build to set UsingToolMicrosoftNetCompilers
+ property
+
+---
+ eng/Versions.props | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/eng/Versions.props b/eng/Versions.props
+index 3a97f80fd74..6148f6c6ae5 100644
+--- a/eng/Versions.props
++++ b/eng/Versions.props
+@@ -22,7 +22,7 @@
+     <UsingToolIbcOptimization>false</UsingToolIbcOptimization>
+     <UsingToolXliff>false</UsingToolXliff>
+     <LastReleasedStableAssemblyVersion>$(AssemblyVersion)</LastReleasedStableAssemblyVersion>
+-    <UsingToolMicrosoftNetCompilers>true</UsingToolMicrosoftNetCompilers>
++    <UsingToolMicrosoftNetCompilers Condition="'$(DotNetBuildFromSource)' != 'true'">true</UsingToolMicrosoftNetCompilers>
+   </PropertyGroup>
+   <ItemGroup>
+     <WorkloadSdkBandVersions Include="$(SdkBandVersion)" SupportsMachineArch="true" />
+-- 
+2.29.2
+