From 1c38575cf1f64f6aac0d1d4bb8ac6ed6921ae8b9 Mon Sep 17 00:00:00 2001 From: Nikola Milosavljevic Date: Wed, 25 Jan 2023 18:21:26 +0000 Subject: [PATCH 1/4] Add arcade overrides from baseline - 3600aa80a01e90f38a7b86b9d7c1264e091aa5a8 --- .../SourceBuildArcade.targets | 99 +++++++++++++++++++ .../SourceBuildIntermediate.proj | 86 ++++++++++++++++ 2 files changed, 185 insertions(+) create mode 100644 src/SourceBuild/content/eng/arcade-overrides/SourceBuildArcade.targets create mode 100644 src/SourceBuild/content/eng/arcade-overrides/SourceBuildIntermediate.proj diff --git a/src/SourceBuild/content/eng/arcade-overrides/SourceBuildArcade.targets b/src/SourceBuild/content/eng/arcade-overrides/SourceBuildArcade.targets new file mode 100644 index 000000000..dc0a01b4b --- /dev/null +++ b/src/SourceBuild/content/eng/arcade-overrides/SourceBuildArcade.targets @@ -0,0 +1,99 @@ + + + + + + + + $([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'source-build')) + $([MSBuild]::NormalizeDirectory('$(SourceBuildOutputDir)', 'self')) + $([MSBuild]::NormalizeDirectory('$(SourceBuildSelfDir)', 'src')) + $([MSBuild]::NormalizeDirectory('$(SourceBuildSelfDir)', 'package-cache')) + $([MSBuild]::NormalizeDirectory('$(SourceBuildSelfDir)', 'prebuilt-report')) + + + $([MSBuild]::NormalizeDirectory('$(CurrentRepoSourceBuildSourceDir)', 'artifacts')) + $([MSBuild]::NormalizeDirectory('$(CurrentRepoSourceBuildArtifactsDir)', 'packages', '$(Configuration)')) + + $([MSBuild]::NormalizeDirectory('$(CurrentRepoSourceBuildArtifactsDir)', 'obj', 'source-built-upstream-cache')) + + $(RepositoryEngineeringDir)SourceBuildPrebuiltBaseline.xml + $(PrebuiltBaselineDataFileDefault) + + + true + + true + + + + + $([System.Runtime.InteropServices.RuntimeInformation]::RuntimeIdentifier) + + + + $([System.Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture.ToString().ToLowerInvariant()) + $(HostArchitecture) + + win + osx + linux + linux + + + $(SourceBuildTargetPortableOSPlatform)-$(SourceBuildTargetArchitecture) + + + + + $(TargetRid) + + + + Microsoft.SourceBuild.Intermediate. + .$(SourceBuildIntermediateNupkgRid) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/SourceBuild/content/eng/arcade-overrides/SourceBuildIntermediate.proj b/src/SourceBuild/content/eng/arcade-overrides/SourceBuildIntermediate.proj new file mode 100644 index 000000000..383496e98 --- /dev/null +++ b/src/SourceBuild/content/eng/arcade-overrides/SourceBuildIntermediate.proj @@ -0,0 +1,86 @@ + + + + + + false + false + false + + + + + + + + + + $(CopyrightNetFoundation) + + true + false + true + false + + + true + + + netstandard2.0 + + + + $([System.IO.Path]::GetFileName('$(SourceBuildIntermediateNupkgLicenseFile)')) + + + + + + + + + + + $(GitHubRepositoryName) + $(PackageId).$(SupplementalIntermediateNupkgCategory) + + $(SourceBuildIntermediateNupkgPrefix)$(PackageId)$(SourceBuildIntermediateNupkgSuffix) + + + + + + + + + + + + $(BaseOutputPath)SupplementalIntermediatePackages.txt + + + + + + + + + + + + + + + + From 8d07426e10aba73050bee678553c17cc52cc5e20 Mon Sep 17 00:00:00 2001 From: Nikola Milosavljevic Date: Wed, 25 Jan 2023 18:59:16 +0000 Subject: [PATCH 2/4] Poison work - reordering and non-shipping packages --- src/SourceBuild/content/Directory.Build.props | 3 ++ src/SourceBuild/content/build.proj | 7 ++++- .../SourceBuildArcade.targets | 1 + .../SourceBuildIntermediate.proj | 24 +++++++++++++++- .../CheckForPoison.cs | 28 +++++++++++++++++++ .../repo-projects/Directory.Build.targets | 7 +++++ .../content/repo-projects/known-good.proj | 2 +- .../content/repo-projects/linker.proj | 4 +++ .../content/repo-projects/roslyn.proj | 1 + .../content/repo-projects/runtime.proj | 3 +- ...ld-to-set-UsingToolMicrosoftNetCompi.patch | 26 +++++++++++++++++ 11 files changed, 101 insertions(+), 5 deletions(-) create mode 100644 src/SourceBuild/patches/0001-Allow-source-build-to-set-UsingToolMicrosoftNetCompi.patch diff --git a/src/SourceBuild/content/Directory.Build.props b/src/SourceBuild/content/Directory.Build.props index 21c49e867..a4649f591 100644 --- a/src/SourceBuild/content/Directory.Build.props +++ b/src/SourceBuild/content/Directory.Build.props @@ -125,6 +125,9 @@ $(BaseOutputPath)prebuilt-report/ $(ProjectDir)/repo-projects/ $(PackageReportDir)prebuilt-packages/ + $(PackageReportDir)packagelists/ + + NonShipping.Packages. $(PackageReportDir)prebuilt-usage.xml $(PackageReportDir)poison-usage.xml $(PackageReportDir)poison-catalog.xml diff --git a/src/SourceBuild/content/build.proj b/src/SourceBuild/content/build.proj index f4fd4ae62..efcb226fe 100644 --- a/src/SourceBuild/content/build.proj +++ b/src/SourceBuild/content/build.proj @@ -70,10 +70,15 @@ + + + + + PoisonReportOutputFilePath="$(PoisonUsageReportFile)" + NonShippingPackagesListFiles="@(NonShippingPackagesList)" /> diff --git a/src/SourceBuild/content/eng/arcade-overrides/SourceBuildArcade.targets b/src/SourceBuild/content/eng/arcade-overrides/SourceBuildArcade.targets index dc0a01b4b..2d1c499f8 100644 --- a/src/SourceBuild/content/eng/arcade-overrides/SourceBuildArcade.targets +++ b/src/SourceBuild/content/eng/arcade-overrides/SourceBuildArcade.targets @@ -92,6 +92,7 @@ + diff --git a/src/SourceBuild/content/eng/arcade-overrides/SourceBuildIntermediate.proj b/src/SourceBuild/content/eng/arcade-overrides/SourceBuildIntermediate.proj index 383496e98..a017397be 100644 --- a/src/SourceBuild/content/eng/arcade-overrides/SourceBuildIntermediate.proj +++ b/src/SourceBuild/content/eng/arcade-overrides/SourceBuildIntermediate.proj @@ -53,7 +53,8 @@ @@ -78,6 +79,27 @@ + + + + + NonShipping.Packages. + $(CurrentRepoSourceBuildArtifactsPackagesDir)$(NonShippingPackagesListPrefix)$(GitHubRepositoryName).lst + + + + + + + + + + + diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CheckForPoison.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CheckForPoison.cs index 1ac545724..9c0a1bc57 100644 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CheckForPoison.cs +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CheckForPoison.cs @@ -57,6 +57,11 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection /// public string OverrideTempPath { get; set; } + /// + /// Array of files containing lists of non-shipping packages + /// + public ITaskItem[] NonShippingPackagesListFiles { get; set; } + private static readonly string[] ZipFileExtensions = { ".zip", @@ -173,6 +178,7 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection /// List of poisoned packages and files found and reasons for each internal IEnumerable GetPoisonedFiles(IEnumerable initialCandidates, string catalogedPackagesFilePath, string markerFileName) { + IEnumerable nonShippingPackages = GetAllNonShippingPackages(); IEnumerable catalogedPackages = ReadCatalog(catalogedPackagesFilePath); var poisons = new List(); var candidateQueue = new Queue(initialCandidates); @@ -191,6 +197,14 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection // add its contents to the list to be checked. if (ZipFileExtensions.Concat(TarFileExtensions).Concat(TarGzFileExtensions).Any(e => checking.ToLowerInvariant().EndsWith(e))) { + Log.LogMessage($"Zip or NuPkg file to check: {checking}"); + + // Skip non-shipping packages + if (nonShippingPackages.Contains(Path.GetFileName(checking), StringComparer.OrdinalIgnoreCase)) + { + continue; + } + var tempCheckingDir = Path.Combine(tempDir.FullName, Path.GetFileNameWithoutExtension(checking)); PoisonedFileEntry result = ExtractAndCheckZipFileOnly(catalogedPackages, checking, markerFileName, tempCheckingDir, candidateQueue); if (result != null) @@ -213,6 +227,20 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection return poisons; } + private IEnumerable GetAllNonShippingPackages() + { + if (NonShippingPackagesListFiles != null) + { + return NonShippingPackagesListFiles + .SelectMany(item => File.ReadAllLines(item.ItemSpec)) + .Distinct(); + } + else + { + return Enumerable.Empty(); + } + } + private static PoisonedFileEntry CheckSingleFile(IEnumerable catalogedPackages, string rootPath, string fileToCheck) { // skip some common files that get copied verbatim from nupkgs - LICENSE, _._, etc as well as diff --git a/src/SourceBuild/content/repo-projects/Directory.Build.targets b/src/SourceBuild/content/repo-projects/Directory.Build.targets index 3b9a171c2..3a68fd5f1 100644 --- a/src/SourceBuild/content/repo-projects/Directory.Build.targets +++ b/src/SourceBuild/content/repo-projects/Directory.Build.targets @@ -355,6 +355,13 @@ + + + + + + + + @@ -43,7 +44,6 @@ - diff --git a/src/SourceBuild/content/repo-projects/linker.proj b/src/SourceBuild/content/repo-projects/linker.proj index db14ef098..a5c79e1c1 100644 --- a/src/SourceBuild/content/repo-projects/linker.proj +++ b/src/SourceBuild/content/repo-projects/linker.proj @@ -18,6 +18,10 @@ + + + + diff --git a/src/SourceBuild/content/repo-projects/roslyn.proj b/src/SourceBuild/content/repo-projects/roslyn.proj index bdc856b21..f4e3b8095 100644 --- a/src/SourceBuild/content/repo-projects/roslyn.proj +++ b/src/SourceBuild/content/repo-projects/roslyn.proj @@ -26,6 +26,7 @@ + diff --git a/src/SourceBuild/content/repo-projects/runtime.proj b/src/SourceBuild/content/repo-projects/runtime.proj index c477000a6..ddf743231 100644 --- a/src/SourceBuild/content/repo-projects/runtime.proj +++ b/src/SourceBuild/content/repo-projects/runtime.proj @@ -23,6 +23,7 @@ $(BuildCommandArgs) /p:RuntimeOS=$(RuntimeOS) $(BuildCommandArgs) /p:BaseOS=$(BaseOS) $(BuildCommandArgs) /p:SourceBuildNonPortable=true + $(BuildCommandArgs) /p:UsingToolMicrosoftNetCompilers=false $(StandardSourceBuildCommand) $(BuildCommandArgs) @@ -58,9 +59,7 @@ - - diff --git a/src/SourceBuild/patches/0001-Allow-source-build-to-set-UsingToolMicrosoftNetCompi.patch b/src/SourceBuild/patches/0001-Allow-source-build-to-set-UsingToolMicrosoftNetCompi.patch new file mode 100644 index 000000000..8012f3028 --- /dev/null +++ b/src/SourceBuild/patches/0001-Allow-source-build-to-set-UsingToolMicrosoftNetCompi.patch @@ -0,0 +1,26 @@ +From ed89288f65df0db02817a9d7f72b768b51b4b57a Mon Sep 17 00:00:00 2001 +From: Nikola Milosavljevic +Date: Tue, 24 Jan 2023 18:03:12 +0000 +Subject: [PATCH] Allow source-build to set UsingToolMicrosoftNetCompilers + property + +--- + eng/Versions.props | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/eng/Versions.props b/eng/Versions.props +index 3a97f80fd74..6148f6c6ae5 100644 +--- a/eng/Versions.props ++++ b/eng/Versions.props +@@ -22,7 +22,7 @@ + false + false + $(AssemblyVersion) +- true ++ true + + + +-- +2.29.2 + From a506f64358bda686ec12daeee0e0482fd2617873 Mon Sep 17 00:00:00 2001 From: Nikola Milosavljevic Date: Wed, 25 Jan 2023 20:24:59 +0000 Subject: [PATCH 3/4] Update runtime patch and move it to correct location --- ...-source-build-to-set-UsingToolMicrosoftNetCompi.patch | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) rename src/SourceBuild/patches/{ => runtime}/0001-Allow-source-build-to-set-UsingToolMicrosoftNetCompi.patch (85%) diff --git a/src/SourceBuild/patches/0001-Allow-source-build-to-set-UsingToolMicrosoftNetCompi.patch b/src/SourceBuild/patches/runtime/0001-Allow-source-build-to-set-UsingToolMicrosoftNetCompi.patch similarity index 85% rename from src/SourceBuild/patches/0001-Allow-source-build-to-set-UsingToolMicrosoftNetCompi.patch rename to src/SourceBuild/patches/runtime/0001-Allow-source-build-to-set-UsingToolMicrosoftNetCompi.patch index 8012f3028..94f348560 100644 --- a/src/SourceBuild/patches/0001-Allow-source-build-to-set-UsingToolMicrosoftNetCompi.patch +++ b/src/SourceBuild/patches/runtime/0001-Allow-source-build-to-set-UsingToolMicrosoftNetCompi.patch @@ -1,6 +1,6 @@ -From ed89288f65df0db02817a9d7f72b768b51b4b57a Mon Sep 17 00:00:00 2001 +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 From: Nikola Milosavljevic -Date: Tue, 24 Jan 2023 18:03:12 +0000 +Date: Wed, 25 Jan 2023 20:21:17 +0000 Subject: [PATCH] Allow source-build to set UsingToolMicrosoftNetCompilers property @@ -9,7 +9,7 @@ Subject: [PATCH] Allow source-build to set UsingToolMicrosoftNetCompilers 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/Versions.props b/eng/Versions.props -index 3a97f80fd74..6148f6c6ae5 100644 +index a57587c74a8..15f65bc269b 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -22,7 +22,7 @@ @@ -21,6 +21,3 @@ index 3a97f80fd74..6148f6c6ae5 100644 --- -2.29.2 - From 1648da6165a06405c85731304b3965b296c6f222 Mon Sep 17 00:00:00 2001 From: Nikola Milosavljevic Date: Wed, 25 Jan 2023 22:29:07 +0000 Subject: [PATCH 4/4] Addressing review comments --- .../CheckForPoison.cs | 3 ++- ...-Allow-source-build-to-set-UsingToolMicrosoftNetCompi.patch | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CheckForPoison.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CheckForPoison.cs index 9c0a1bc57..27ff93efb 100644 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CheckForPoison.cs +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CheckForPoison.cs @@ -233,7 +233,8 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection { return NonShippingPackagesListFiles .SelectMany(item => File.ReadAllLines(item.ItemSpec)) - .Distinct(); + .Distinct() + .ToList(); } else { diff --git a/src/SourceBuild/patches/runtime/0001-Allow-source-build-to-set-UsingToolMicrosoftNetCompi.patch b/src/SourceBuild/patches/runtime/0001-Allow-source-build-to-set-UsingToolMicrosoftNetCompi.patch index 94f348560..7e48a742d 100644 --- a/src/SourceBuild/patches/runtime/0001-Allow-source-build-to-set-UsingToolMicrosoftNetCompi.patch +++ b/src/SourceBuild/patches/runtime/0001-Allow-source-build-to-set-UsingToolMicrosoftNetCompi.patch @@ -4,6 +4,7 @@ Date: Wed, 25 Jan 2023 20:21:17 +0000 Subject: [PATCH] Allow source-build to set UsingToolMicrosoftNetCompilers property +backport: https://github.com/dotnet/runtime/pull/81180 --- eng/Versions.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)