diff --git a/src/SourceBuild/content/Directory.Build.props b/src/SourceBuild/content/Directory.Build.props index 21c49e867..a4649f591 100644 --- a/src/SourceBuild/content/Directory.Build.props +++ b/src/SourceBuild/content/Directory.Build.props @@ -125,6 +125,9 @@ $(BaseOutputPath)prebuilt-report/ $(ProjectDir)/repo-projects/ $(PackageReportDir)prebuilt-packages/ + $(PackageReportDir)packagelists/ + + NonShipping.Packages. $(PackageReportDir)prebuilt-usage.xml $(PackageReportDir)poison-usage.xml $(PackageReportDir)poison-catalog.xml diff --git a/src/SourceBuild/content/build.proj b/src/SourceBuild/content/build.proj index f4fd4ae62..efcb226fe 100644 --- a/src/SourceBuild/content/build.proj +++ b/src/SourceBuild/content/build.proj @@ -70,10 +70,15 @@ + + + + + PoisonReportOutputFilePath="$(PoisonUsageReportFile)" + NonShippingPackagesListFiles="@(NonShippingPackagesList)" /> diff --git a/src/SourceBuild/content/eng/arcade-overrides/SourceBuildArcade.targets b/src/SourceBuild/content/eng/arcade-overrides/SourceBuildArcade.targets new file mode 100644 index 000000000..2d1c499f8 --- /dev/null +++ b/src/SourceBuild/content/eng/arcade-overrides/SourceBuildArcade.targets @@ -0,0 +1,100 @@ + + + + + + + + $([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'source-build')) + $([MSBuild]::NormalizeDirectory('$(SourceBuildOutputDir)', 'self')) + $([MSBuild]::NormalizeDirectory('$(SourceBuildSelfDir)', 'src')) + $([MSBuild]::NormalizeDirectory('$(SourceBuildSelfDir)', 'package-cache')) + $([MSBuild]::NormalizeDirectory('$(SourceBuildSelfDir)', 'prebuilt-report')) + + + $([MSBuild]::NormalizeDirectory('$(CurrentRepoSourceBuildSourceDir)', 'artifacts')) + $([MSBuild]::NormalizeDirectory('$(CurrentRepoSourceBuildArtifactsDir)', 'packages', '$(Configuration)')) + + $([MSBuild]::NormalizeDirectory('$(CurrentRepoSourceBuildArtifactsDir)', 'obj', 'source-built-upstream-cache')) + + $(RepositoryEngineeringDir)SourceBuildPrebuiltBaseline.xml + $(PrebuiltBaselineDataFileDefault) + + + true + + true + + + + + $([System.Runtime.InteropServices.RuntimeInformation]::RuntimeIdentifier) + + + + $([System.Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture.ToString().ToLowerInvariant()) + $(HostArchitecture) + + win + osx + linux + linux + + + $(SourceBuildTargetPortableOSPlatform)-$(SourceBuildTargetArchitecture) + + + + + $(TargetRid) + + + + Microsoft.SourceBuild.Intermediate. + .$(SourceBuildIntermediateNupkgRid) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/SourceBuild/content/eng/arcade-overrides/SourceBuildIntermediate.proj b/src/SourceBuild/content/eng/arcade-overrides/SourceBuildIntermediate.proj new file mode 100644 index 000000000..a017397be --- /dev/null +++ b/src/SourceBuild/content/eng/arcade-overrides/SourceBuildIntermediate.proj @@ -0,0 +1,108 @@ + + + + + + false + false + false + + + + + + + + + + $(CopyrightNetFoundation) + + true + false + true + false + + + true + + + netstandard2.0 + + + + $([System.IO.Path]::GetFileName('$(SourceBuildIntermediateNupkgLicenseFile)')) + + + + + + + + + + + $(GitHubRepositoryName) + $(PackageId).$(SupplementalIntermediateNupkgCategory) + + $(SourceBuildIntermediateNupkgPrefix)$(PackageId)$(SourceBuildIntermediateNupkgSuffix) + + + + + + + + + + + + $(BaseOutputPath)SupplementalIntermediatePackages.txt + + + + + + + + + + + + + + + NonShipping.Packages. + $(CurrentRepoSourceBuildArtifactsPackagesDir)$(NonShippingPackagesListPrefix)$(GitHubRepositoryName).lst + + + + + + + + + + + + + + + + + diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CheckForPoison.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CheckForPoison.cs index 1ac545724..27ff93efb 100644 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CheckForPoison.cs +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CheckForPoison.cs @@ -57,6 +57,11 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection /// public string OverrideTempPath { get; set; } + /// + /// Array of files containing lists of non-shipping packages + /// + public ITaskItem[] NonShippingPackagesListFiles { get; set; } + private static readonly string[] ZipFileExtensions = { ".zip", @@ -173,6 +178,7 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection /// List of poisoned packages and files found and reasons for each internal IEnumerable GetPoisonedFiles(IEnumerable initialCandidates, string catalogedPackagesFilePath, string markerFileName) { + IEnumerable nonShippingPackages = GetAllNonShippingPackages(); IEnumerable catalogedPackages = ReadCatalog(catalogedPackagesFilePath); var poisons = new List(); var candidateQueue = new Queue(initialCandidates); @@ -191,6 +197,14 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection // add its contents to the list to be checked. if (ZipFileExtensions.Concat(TarFileExtensions).Concat(TarGzFileExtensions).Any(e => checking.ToLowerInvariant().EndsWith(e))) { + Log.LogMessage($"Zip or NuPkg file to check: {checking}"); + + // Skip non-shipping packages + if (nonShippingPackages.Contains(Path.GetFileName(checking), StringComparer.OrdinalIgnoreCase)) + { + continue; + } + var tempCheckingDir = Path.Combine(tempDir.FullName, Path.GetFileNameWithoutExtension(checking)); PoisonedFileEntry result = ExtractAndCheckZipFileOnly(catalogedPackages, checking, markerFileName, tempCheckingDir, candidateQueue); if (result != null) @@ -213,6 +227,21 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection return poisons; } + private IEnumerable GetAllNonShippingPackages() + { + if (NonShippingPackagesListFiles != null) + { + return NonShippingPackagesListFiles + .SelectMany(item => File.ReadAllLines(item.ItemSpec)) + .Distinct() + .ToList(); + } + else + { + return Enumerable.Empty(); + } + } + private static PoisonedFileEntry CheckSingleFile(IEnumerable catalogedPackages, string rootPath, string fileToCheck) { // skip some common files that get copied verbatim from nupkgs - LICENSE, _._, etc as well as diff --git a/src/SourceBuild/content/repo-projects/Directory.Build.targets b/src/SourceBuild/content/repo-projects/Directory.Build.targets index 3b9a171c2..3a68fd5f1 100644 --- a/src/SourceBuild/content/repo-projects/Directory.Build.targets +++ b/src/SourceBuild/content/repo-projects/Directory.Build.targets @@ -355,6 +355,13 @@ + + + + + + + + @@ -43,7 +44,6 @@ - diff --git a/src/SourceBuild/content/repo-projects/linker.proj b/src/SourceBuild/content/repo-projects/linker.proj index db14ef098..a5c79e1c1 100644 --- a/src/SourceBuild/content/repo-projects/linker.proj +++ b/src/SourceBuild/content/repo-projects/linker.proj @@ -18,6 +18,10 @@ + + + + diff --git a/src/SourceBuild/content/repo-projects/roslyn.proj b/src/SourceBuild/content/repo-projects/roslyn.proj index bdc856b21..f4e3b8095 100644 --- a/src/SourceBuild/content/repo-projects/roslyn.proj +++ b/src/SourceBuild/content/repo-projects/roslyn.proj @@ -26,6 +26,7 @@ + diff --git a/src/SourceBuild/content/repo-projects/runtime.proj b/src/SourceBuild/content/repo-projects/runtime.proj index c477000a6..ddf743231 100644 --- a/src/SourceBuild/content/repo-projects/runtime.proj +++ b/src/SourceBuild/content/repo-projects/runtime.proj @@ -23,6 +23,7 @@ $(BuildCommandArgs) /p:RuntimeOS=$(RuntimeOS) $(BuildCommandArgs) /p:BaseOS=$(BaseOS) $(BuildCommandArgs) /p:SourceBuildNonPortable=true + $(BuildCommandArgs) /p:UsingToolMicrosoftNetCompilers=false $(StandardSourceBuildCommand) $(BuildCommandArgs) @@ -58,9 +59,7 @@ - - diff --git a/src/SourceBuild/patches/runtime/0001-Allow-source-build-to-set-UsingToolMicrosoftNetCompi.patch b/src/SourceBuild/patches/runtime/0001-Allow-source-build-to-set-UsingToolMicrosoftNetCompi.patch new file mode 100644 index 000000000..7e48a742d --- /dev/null +++ b/src/SourceBuild/patches/runtime/0001-Allow-source-build-to-set-UsingToolMicrosoftNetCompi.patch @@ -0,0 +1,24 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Nikola Milosavljevic +Date: Wed, 25 Jan 2023 20:21:17 +0000 +Subject: [PATCH] Allow source-build to set UsingToolMicrosoftNetCompilers + property + +backport: https://github.com/dotnet/runtime/pull/81180 +--- + eng/Versions.props | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/eng/Versions.props b/eng/Versions.props +index a57587c74a8..15f65bc269b 100644 +--- a/eng/Versions.props ++++ b/eng/Versions.props +@@ -22,7 +22,7 @@ + false + false + $(AssemblyVersion) +- true ++ true + + +