diff --git a/eng/common/sdk-task.ps1 b/eng/common/sdk-task.ps1 index 91f8196cc..73828dd30 100644 --- a/eng/common/sdk-task.ps1 +++ b/eng/common/sdk-task.ps1 @@ -64,7 +64,7 @@ try { $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty } if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) { - $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.7.2-1" -MemberType NoteProperty + $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.8.1-2" -MemberType NoteProperty } if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") { $xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 84cfe7cd9..fdd0cbb91 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -384,8 +384,8 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = # If the version of msbuild is going to be xcopied, # use this version. Version matches a package here: - # https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet-eng/NuGet/RoslynTools.MSBuild/versions/17.7.2-1 - $defaultXCopyMSBuildVersion = '17.7.2-1' + # https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet-eng/NuGet/RoslynTools.MSBuild/versions/17.8.1-2 + $defaultXCopyMSBuildVersion = '17.8.1-2' if (!$vsRequirements) { if (Get-Member -InputObject $GlobalJson.tools -Name 'vs') { diff --git a/src/SourceBuild/content/build.proj b/src/SourceBuild/content/build.proj index 193bbd6f0..7d074d880 100644 --- a/src/SourceBuild/content/build.proj +++ b/src/SourceBuild/content/build.proj @@ -161,6 +161,7 @@ + /// The path of the project directory to the FilesToCheck. + /// + [Required] + public string ProjectDirPath { get; set; } + /// /// The output path for an XML poison report, if desired. /// @@ -141,6 +147,8 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection private const string PoisonMarker = "POISONED"; + private record CandidateFileEntry(string ExtractedPath, string DisplayPath); + public override bool Execute() { IEnumerable poisons = GetPoisonedFiles(FilesToCheck.Select(f => f.ItemSpec), HashCatalogFilePath, MarkerFileName); @@ -176,7 +184,9 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection IEnumerable nonShippingPackages = GetAllNonShippingPackages(); IEnumerable catalogedPackages = ReadCatalog(catalogedPackagesFilePath); var poisons = new List(); - var candidateQueue = new Queue(initialCandidates); + var candidateQueue = new Queue(initialCandidates.Select(candidate => + new CandidateFileEntry(candidate, Utility.MakeRelativePath(candidate, ProjectDirPath)))); + if (!string.IsNullOrWhiteSpace(OverrideTempPath)) { Directory.CreateDirectory(OverrideTempPath); @@ -186,22 +196,22 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection while (candidateQueue.Any()) { - var checking = candidateQueue.Dequeue(); + var candidate = candidateQueue.Dequeue(); // if this is a zip or NuPkg, extract it, check for the poison marker, and // add its contents to the list to be checked. - if (ZipFileExtensions.Concat(TarFileExtensions).Concat(TarGzFileExtensions).Any(e => checking.ToLowerInvariant().EndsWith(e))) + if (ZipFileExtensions.Concat(TarFileExtensions).Concat(TarGzFileExtensions).Any(e => candidate.ExtractedPath.ToLowerInvariant().EndsWith(e))) { - Log.LogMessage($"Zip or NuPkg file to check: {checking}"); + Log.LogMessage($"Zip or NuPkg file to check: {candidate.ExtractedPath}"); // Skip non-shipping packages - if (nonShippingPackages.Contains(Path.GetFileName(checking), StringComparer.OrdinalIgnoreCase)) + if (nonShippingPackages.Contains(Path.GetFileName(candidate.ExtractedPath), StringComparer.OrdinalIgnoreCase)) { continue; } - var tempCheckingDir = Path.Combine(tempDir.FullName, Path.GetFileNameWithoutExtension(checking)); - PoisonedFileEntry result = ExtractAndCheckZipFileOnly(catalogedPackages, checking, markerFileName, tempCheckingDir, candidateQueue); + var tempCheckingDir = Path.Combine(tempDir.FullName, Path.GetFileNameWithoutExtension(candidate.ExtractedPath)); + PoisonedFileEntry result = ExtractAndCheckZipFileOnly(catalogedPackages, candidate, markerFileName, tempCheckingDir, candidateQueue); if (result != null) { poisons.Add(result); @@ -209,7 +219,7 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection } else { - PoisonedFileEntry result = CheckSingleFile(catalogedPackages, tempDir.FullName, checking); + PoisonedFileEntry result = CheckSingleFile(catalogedPackages, candidate); if (result != null) { poisons.Add(result); @@ -237,10 +247,12 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection } } - private static PoisonedFileEntry CheckSingleFile(IEnumerable catalogedPackages, string rootPath, string fileToCheck) + private static PoisonedFileEntry CheckSingleFile(IEnumerable catalogedPackages, CandidateFileEntry candidate) { // skip some common files that get copied verbatim from nupkgs - LICENSE, _._, etc as well as // file types that we never care about - text files, .gitconfig, etc. + var fileToCheck = candidate.ExtractedPath; + if (FileNamesToSkip.Any(f => Path.GetFileName(fileToCheck).ToLowerInvariant() == f.ToLowerInvariant()) || FileExtensionsToSkip.Any(e => Path.GetExtension(fileToCheck).ToLowerInvariant() == e.ToLowerInvariant()) || (new FileInfo(fileToCheck).Length == 0)) @@ -249,7 +261,7 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection } var poisonEntry = new PoisonedFileEntry(); - poisonEntry.Path = Utility.MakeRelativePath(fileToCheck, rootPath); + poisonEntry.Path = candidate.DisplayPath; // There seems to be some weird issues with using file streams both for hashing and assembly loading. // Copy everything into a memory stream to avoid these problems. @@ -320,9 +332,10 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection return false; } - private static PoisonedFileEntry ExtractAndCheckZipFileOnly(IEnumerable catalogedPackages, string zipToCheck, string markerFileName, string tempDir, Queue futureFilesToCheck) + private static PoisonedFileEntry ExtractAndCheckZipFileOnly(IEnumerable catalogedPackages, CandidateFileEntry candidate, string markerFileName, string tempDir, Queue futureFilesToCheck) { var poisonEntry = new PoisonedFileEntry(); + var zipToCheck = candidate.ExtractedPath; poisonEntry.Path = zipToCheck; using (var sha = SHA256.Create()) @@ -375,8 +388,9 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection foreach (var child in Directory.EnumerateFiles(tempDir, "*", SearchOption.AllDirectories)) { - // also add anything in this zip/package for checking - futureFilesToCheck.Enqueue(child); + string displayPath = $"{candidate.DisplayPath}/{child.Replace(tempDir, string.Empty).TrimStart(Path.DirectorySeparatorChar)}"; + + futureFilesToCheck.Enqueue(new CandidateFileEntry(child, displayPath)); } return poisonEntry.Type != PoisonType.None ? poisonEntry : null; diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/DotNetFormatTests.cs b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/DotNetFormatTests.cs index 4ec0ad325..bfd1a8766 100644 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/DotNetFormatTests.cs +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/DotNetFormatTests.cs @@ -22,12 +22,6 @@ public class DotNetFormatTests : SdkTests [Fact] public void FormatProject() { - if (Config.TargetRid.Contains("alpine")) - { - // Skipping this test on Alpine due to https://github.com/dotnet/format/issues/1945 - return; - } - string unformattedCsFilePath = Path.Combine(BaselineHelper.GetAssetsDirectory(), UnformattedFileName); string projectDirectory = DotNetHelper.ExecuteNew("console", nameof(FormatProject), "C#"); diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/LicenseScanTests.cs b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/LicenseScanTests.cs index 6ddf668c5..8a088ba4d 100644 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/LicenseScanTests.cs +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/LicenseScanTests.cs @@ -73,6 +73,7 @@ public class LicenseScanTests : TestBase "lgpl-2.0-plus", // https://opensource.org/license/lgpl-2-0/ "lgpl-2.1", // https://opensource.org/license/lgpl-2-1/ "lgpl-2.1-plus", // https://opensource.org/license/lgpl-2-1/ + "lzma-sdk-9.22", // https://github.com/nexB/scancode-toolkit/blob/develop/src/licensedcode/data/licenses/lzma-sdk-9.22.LICENSE "mit", // https://opensource.org/license/mit/ "mit-addition", // https://github.com/nexB/scancode-toolkit/blob/develop/src/licensedcode/data/licenses/mit-addition.LICENSE "ms-patent-promise", // https://github.com/nexB/scancode-toolkit/blob/develop/src/licensedcode/data/licenses/ms-patent-promise.LICENSE @@ -141,17 +142,16 @@ public class LicenseScanTests : TestBase { Assert.NotNull(Config.LicenseScanPath); - string OriginalScancodeResultsPath = Path.Combine(LogsDirectory, "scancode-results-original.json"); - string FilteredScancodeResultsPath = Path.Combine(LogsDirectory, "scancode-results-filtered.json"); + string scancodeResultsPath = Path.Combine(LogsDirectory, "scancode-results.json"); // Scancode Doc: https://scancode-toolkit.readthedocs.io/en/latest/index.html string ignoreOptions = string.Join(" ", s_ignoredFilePatterns.Select(pattern => $"--ignore {pattern}")); ExecuteHelper.ExecuteProcessValidateExitCode( "scancode", - $"--license --strip-root --only-findings {ignoreOptions} --json-pp {OriginalScancodeResultsPath} {Config.LicenseScanPath}", + $"--license --strip-root --only-findings {ignoreOptions} --json-pp {scancodeResultsPath} {Config.LicenseScanPath}", OutputHelper); - JsonDocument doc = JsonDocument.Parse(File.ReadAllText(OriginalScancodeResultsPath)); + JsonDocument doc = JsonDocument.Parse(File.ReadAllText(scancodeResultsPath)); ScancodeResults? scancodeResults = doc.Deserialize(); Assert.NotNull(scancodeResults); @@ -162,7 +162,6 @@ public class LicenseScanTests : TestBase WriteIndented = true }; string json = JsonSerializer.Serialize(scancodeResults, options); - File.WriteAllText(FilteredScancodeResultsPath, json); string baselineName = $"Licenses.{_targetRepo}.json"; diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/LicenseExclusions.txt b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/LicenseExclusions.txt index e0efc77dd..2722333d5 100644 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/LicenseExclusions.txt +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/LicenseExclusions.txt @@ -167,7 +167,7 @@ src/runtime/src/installer/pkg/THIRD-PARTY-NOTICES.TXT # False positive src/runtime/src/libraries/System.ComponentModel.TypeConverter/src/System/ComponentModel/LicFileLicenseProvider.cs|proprietary-license -src/runtime/src/libraries/System.Configuration.ConfigurationManager/tests/Mono/LongValidatorTest.cs|json +src/runtime/src/libraries/System.Configuration.ConfigurationManager/tests/Mono/LongValidatorTest.cs|embedthis-extension src/runtime/src/libraries/System.Net.Sockets/tests/FunctionalTests/Connect.cs|other-permissive src/runtime/src/libraries/System.Net.Sockets/tests/FunctionalTests/UdpClientTest.cs|other-permissive src/runtime/src/libraries/System.Net.Sockets/tests/FunctionalTests/SendReceive/SendReceive.cs|other-permissive @@ -226,10 +226,11 @@ src/source-build-reference-packages/src/targetPacks/ILsrc/netstandard.library/2. src/source-build-reference-packages/src/targetPacks/ILsrc/netstandard.library.ref/2.1.0/THIRD-PARTY-NOTICES.TXT|codesourcery-2004 src/source-build-reference-packages/src/textOnlyPackages/src/microsoft.codeanalysis.collections/4.2.0-1.22102.8/ThirdPartyNotices.rtf|json src/source-build-reference-packages/src/textOnlyPackages/src/microsoft.netcore.*/1.*/ThirdPartyNotices.txt|unknown +src/source-build-reference-packages/src/textOnlyPackages/src/microsoft.private.intellisense/8.0.*/IntellisenseFiles/*/1033/System.Security.Permissions.xml|unknown-license-reference # Contains references to licenses which are not applicable to the source src/source-build-reference-packages/src/packageSourceGenerator/PackageSourceGeneratorTask/RewriteNuspec.cs|unknown-license-reference,ms-net-library-2018-11 -src/source-build-reference-packages/src/textOnlyPackages/src/microsoft.private.intellisense/8.0.0-preview-20230918.1/IntellisenseFiles/windowsdesktop/1033/PresentationCore.xml|proprietary-license +src/source-build-reference-packages/src/textOnlyPackages/src/microsoft.private.intellisense/8.0.*/IntellisenseFiles/windowsdesktop/1033/PresentationCore.xml|proprietary-license # # sourcelink