From 91455a8f313e0e3da957306b674ae2105b663710 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Thu, 19 Oct 2023 16:00:42 +0000 Subject: [PATCH 1/4] [release/8.0.1xx] Update dependencies from dotnet/arcade (#17577) [release/8.0.1xx] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 12 ++++++------ eng/Versions.props | 2 +- eng/common/sdk-task.ps1 | 2 +- eng/common/tools.ps1 | 4 ++-- global.json | 4 ++-- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index f4b9e0ccc..369888944 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -205,18 +205,18 @@ - + https://github.com/dotnet/arcade - e6be64c3e27aeb29f93f6aa751fad972e4ef2d52 + 39042b4048580366d35a7c1c4f4ce8fc0dbea4b4 - + https://github.com/dotnet/arcade - e6be64c3e27aeb29f93f6aa751fad972e4ef2d52 + 39042b4048580366d35a7c1c4f4ce8fc0dbea4b4 - + https://github.com/dotnet/arcade - e6be64c3e27aeb29f93f6aa751fad972e4ef2d52 + 39042b4048580366d35a7c1c4f4ce8fc0dbea4b4 https://github.com/dotnet/arcade-services diff --git a/eng/Versions.props b/eng/Versions.props index cb9518d31..af4a2baa2 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -40,7 +40,7 @@ - 8.0.0-beta.23505.1 + 8.0.0-beta.23516.4 diff --git a/eng/common/sdk-task.ps1 b/eng/common/sdk-task.ps1 index 91f8196cc..73828dd30 100644 --- a/eng/common/sdk-task.ps1 +++ b/eng/common/sdk-task.ps1 @@ -64,7 +64,7 @@ try { $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty } if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) { - $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.7.2-1" -MemberType NoteProperty + $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.8.1-2" -MemberType NoteProperty } if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") { $xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 84cfe7cd9..fdd0cbb91 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -384,8 +384,8 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = # If the version of msbuild is going to be xcopied, # use this version. Version matches a package here: - # https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet-eng/NuGet/RoslynTools.MSBuild/versions/17.7.2-1 - $defaultXCopyMSBuildVersion = '17.7.2-1' + # https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet-eng/NuGet/RoslynTools.MSBuild/versions/17.8.1-2 + $defaultXCopyMSBuildVersion = '17.8.1-2' if (!$vsRequirements) { if (Get-Member -InputObject $GlobalJson.tools -Name 'vs') { diff --git a/global.json b/global.json index b53315c16..10f733dd5 100644 --- a/global.json +++ b/global.json @@ -11,7 +11,7 @@ "cmake": "3.21.0" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23505.1", - "Microsoft.DotNet.CMake.Sdk": "8.0.0-beta.23505.1" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23516.4", + "Microsoft.DotNet.CMake.Sdk": "8.0.0-beta.23516.4" } } From 2526514cd4b7c67706b86a1328ae1771dc9c5c0d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 19 Oct 2023 17:36:41 +0000 Subject: [PATCH 2/4] [release/8.0.1xx] Include full path in poison report (#17580) Co-authored-by: Ella Hathaway --- src/SourceBuild/content/build.proj | 1 + .../CheckForPoison.cs | 40 +++++++++++++------ 2 files changed, 28 insertions(+), 13 deletions(-) diff --git a/src/SourceBuild/content/build.proj b/src/SourceBuild/content/build.proj index 193bbd6f0..7d074d880 100644 --- a/src/SourceBuild/content/build.proj +++ b/src/SourceBuild/content/build.proj @@ -161,6 +161,7 @@ + /// The path of the project directory to the FilesToCheck. + /// + [Required] + public string ProjectDirPath { get; set; } + /// /// The output path for an XML poison report, if desired. /// @@ -141,6 +147,8 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection private const string PoisonMarker = "POISONED"; + private record CandidateFileEntry(string ExtractedPath, string DisplayPath); + public override bool Execute() { IEnumerable poisons = GetPoisonedFiles(FilesToCheck.Select(f => f.ItemSpec), HashCatalogFilePath, MarkerFileName); @@ -176,7 +184,9 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection IEnumerable nonShippingPackages = GetAllNonShippingPackages(); IEnumerable catalogedPackages = ReadCatalog(catalogedPackagesFilePath); var poisons = new List(); - var candidateQueue = new Queue(initialCandidates); + var candidateQueue = new Queue(initialCandidates.Select(candidate => + new CandidateFileEntry(candidate, Utility.MakeRelativePath(candidate, ProjectDirPath)))); + if (!string.IsNullOrWhiteSpace(OverrideTempPath)) { Directory.CreateDirectory(OverrideTempPath); @@ -186,22 +196,22 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection while (candidateQueue.Any()) { - var checking = candidateQueue.Dequeue(); + var candidate = candidateQueue.Dequeue(); // if this is a zip or NuPkg, extract it, check for the poison marker, and // add its contents to the list to be checked. - if (ZipFileExtensions.Concat(TarFileExtensions).Concat(TarGzFileExtensions).Any(e => checking.ToLowerInvariant().EndsWith(e))) + if (ZipFileExtensions.Concat(TarFileExtensions).Concat(TarGzFileExtensions).Any(e => candidate.ExtractedPath.ToLowerInvariant().EndsWith(e))) { - Log.LogMessage($"Zip or NuPkg file to check: {checking}"); + Log.LogMessage($"Zip or NuPkg file to check: {candidate.ExtractedPath}"); // Skip non-shipping packages - if (nonShippingPackages.Contains(Path.GetFileName(checking), StringComparer.OrdinalIgnoreCase)) + if (nonShippingPackages.Contains(Path.GetFileName(candidate.ExtractedPath), StringComparer.OrdinalIgnoreCase)) { continue; } - var tempCheckingDir = Path.Combine(tempDir.FullName, Path.GetFileNameWithoutExtension(checking)); - PoisonedFileEntry result = ExtractAndCheckZipFileOnly(catalogedPackages, checking, markerFileName, tempCheckingDir, candidateQueue); + var tempCheckingDir = Path.Combine(tempDir.FullName, Path.GetFileNameWithoutExtension(candidate.ExtractedPath)); + PoisonedFileEntry result = ExtractAndCheckZipFileOnly(catalogedPackages, candidate, markerFileName, tempCheckingDir, candidateQueue); if (result != null) { poisons.Add(result); @@ -209,7 +219,7 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection } else { - PoisonedFileEntry result = CheckSingleFile(catalogedPackages, tempDir.FullName, checking); + PoisonedFileEntry result = CheckSingleFile(catalogedPackages, candidate); if (result != null) { poisons.Add(result); @@ -237,10 +247,12 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection } } - private static PoisonedFileEntry CheckSingleFile(IEnumerable catalogedPackages, string rootPath, string fileToCheck) + private static PoisonedFileEntry CheckSingleFile(IEnumerable catalogedPackages, CandidateFileEntry candidate) { // skip some common files that get copied verbatim from nupkgs - LICENSE, _._, etc as well as // file types that we never care about - text files, .gitconfig, etc. + var fileToCheck = candidate.ExtractedPath; + if (FileNamesToSkip.Any(f => Path.GetFileName(fileToCheck).ToLowerInvariant() == f.ToLowerInvariant()) || FileExtensionsToSkip.Any(e => Path.GetExtension(fileToCheck).ToLowerInvariant() == e.ToLowerInvariant()) || (new FileInfo(fileToCheck).Length == 0)) @@ -249,7 +261,7 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection } var poisonEntry = new PoisonedFileEntry(); - poisonEntry.Path = Utility.MakeRelativePath(fileToCheck, rootPath); + poisonEntry.Path = candidate.DisplayPath; // There seems to be some weird issues with using file streams both for hashing and assembly loading. // Copy everything into a memory stream to avoid these problems. @@ -320,9 +332,10 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection return false; } - private static PoisonedFileEntry ExtractAndCheckZipFileOnly(IEnumerable catalogedPackages, string zipToCheck, string markerFileName, string tempDir, Queue futureFilesToCheck) + private static PoisonedFileEntry ExtractAndCheckZipFileOnly(IEnumerable catalogedPackages, CandidateFileEntry candidate, string markerFileName, string tempDir, Queue futureFilesToCheck) { var poisonEntry = new PoisonedFileEntry(); + var zipToCheck = candidate.ExtractedPath; poisonEntry.Path = zipToCheck; using (var sha = SHA256.Create()) @@ -375,8 +388,9 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection foreach (var child in Directory.EnumerateFiles(tempDir, "*", SearchOption.AllDirectories)) { - // also add anything in this zip/package for checking - futureFilesToCheck.Enqueue(child); + string displayPath = $"{candidate.DisplayPath}/{child.Replace(tempDir, string.Empty).TrimStart(Path.DirectorySeparatorChar)}"; + + futureFilesToCheck.Enqueue(new CandidateFileEntry(child, displayPath)); } return poisonEntry.Type != PoisonType.None ? poisonEntry : null; From fe64b50ba301f8f88c2ab50b08ae22b151afd47a Mon Sep 17 00:00:00 2001 From: Nikola Milosavljevic Date: Thu, 19 Oct 2023 13:05:08 -0700 Subject: [PATCH 3/4] Re-enable dotnet-format tests on Alpine (#17582) --- .../DotNetFormatTests.cs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/DotNetFormatTests.cs b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/DotNetFormatTests.cs index 4ec0ad325..bfd1a8766 100644 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/DotNetFormatTests.cs +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/DotNetFormatTests.cs @@ -22,12 +22,6 @@ public class DotNetFormatTests : SdkTests [Fact] public void FormatProject() { - if (Config.TargetRid.Contains("alpine")) - { - // Skipping this test on Alpine due to https://github.com/dotnet/format/issues/1945 - return; - } - string unformattedCsFilePath = Path.Combine(BaselineHelper.GetAssetsDirectory(), UnformattedFileName); string projectDirectory = DotNetHelper.ExecuteNew("console", nameof(FormatProject), "C#"); From 7c50a5c83c2651f41a1333bd86c2e45a31943ccc Mon Sep 17 00:00:00 2001 From: Matt Thalman Date: Thu, 19 Oct 2023 15:57:34 -0500 Subject: [PATCH 4/4] License scan updates (#17561) --- .../LicenseScanTests.cs | 9 ++++----- .../assets/LicenseExclusions.txt | 5 +++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/LicenseScanTests.cs b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/LicenseScanTests.cs index 6ddf668c5..8a088ba4d 100644 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/LicenseScanTests.cs +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/LicenseScanTests.cs @@ -73,6 +73,7 @@ public class LicenseScanTests : TestBase "lgpl-2.0-plus", // https://opensource.org/license/lgpl-2-0/ "lgpl-2.1", // https://opensource.org/license/lgpl-2-1/ "lgpl-2.1-plus", // https://opensource.org/license/lgpl-2-1/ + "lzma-sdk-9.22", // https://github.com/nexB/scancode-toolkit/blob/develop/src/licensedcode/data/licenses/lzma-sdk-9.22.LICENSE "mit", // https://opensource.org/license/mit/ "mit-addition", // https://github.com/nexB/scancode-toolkit/blob/develop/src/licensedcode/data/licenses/mit-addition.LICENSE "ms-patent-promise", // https://github.com/nexB/scancode-toolkit/blob/develop/src/licensedcode/data/licenses/ms-patent-promise.LICENSE @@ -141,17 +142,16 @@ public class LicenseScanTests : TestBase { Assert.NotNull(Config.LicenseScanPath); - string OriginalScancodeResultsPath = Path.Combine(LogsDirectory, "scancode-results-original.json"); - string FilteredScancodeResultsPath = Path.Combine(LogsDirectory, "scancode-results-filtered.json"); + string scancodeResultsPath = Path.Combine(LogsDirectory, "scancode-results.json"); // Scancode Doc: https://scancode-toolkit.readthedocs.io/en/latest/index.html string ignoreOptions = string.Join(" ", s_ignoredFilePatterns.Select(pattern => $"--ignore {pattern}")); ExecuteHelper.ExecuteProcessValidateExitCode( "scancode", - $"--license --strip-root --only-findings {ignoreOptions} --json-pp {OriginalScancodeResultsPath} {Config.LicenseScanPath}", + $"--license --strip-root --only-findings {ignoreOptions} --json-pp {scancodeResultsPath} {Config.LicenseScanPath}", OutputHelper); - JsonDocument doc = JsonDocument.Parse(File.ReadAllText(OriginalScancodeResultsPath)); + JsonDocument doc = JsonDocument.Parse(File.ReadAllText(scancodeResultsPath)); ScancodeResults? scancodeResults = doc.Deserialize(); Assert.NotNull(scancodeResults); @@ -162,7 +162,6 @@ public class LicenseScanTests : TestBase WriteIndented = true }; string json = JsonSerializer.Serialize(scancodeResults, options); - File.WriteAllText(FilteredScancodeResultsPath, json); string baselineName = $"Licenses.{_targetRepo}.json"; diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/LicenseExclusions.txt b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/LicenseExclusions.txt index e0efc77dd..2722333d5 100644 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/LicenseExclusions.txt +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/LicenseExclusions.txt @@ -167,7 +167,7 @@ src/runtime/src/installer/pkg/THIRD-PARTY-NOTICES.TXT # False positive src/runtime/src/libraries/System.ComponentModel.TypeConverter/src/System/ComponentModel/LicFileLicenseProvider.cs|proprietary-license -src/runtime/src/libraries/System.Configuration.ConfigurationManager/tests/Mono/LongValidatorTest.cs|json +src/runtime/src/libraries/System.Configuration.ConfigurationManager/tests/Mono/LongValidatorTest.cs|embedthis-extension src/runtime/src/libraries/System.Net.Sockets/tests/FunctionalTests/Connect.cs|other-permissive src/runtime/src/libraries/System.Net.Sockets/tests/FunctionalTests/UdpClientTest.cs|other-permissive src/runtime/src/libraries/System.Net.Sockets/tests/FunctionalTests/SendReceive/SendReceive.cs|other-permissive @@ -226,10 +226,11 @@ src/source-build-reference-packages/src/targetPacks/ILsrc/netstandard.library/2. src/source-build-reference-packages/src/targetPacks/ILsrc/netstandard.library.ref/2.1.0/THIRD-PARTY-NOTICES.TXT|codesourcery-2004 src/source-build-reference-packages/src/textOnlyPackages/src/microsoft.codeanalysis.collections/4.2.0-1.22102.8/ThirdPartyNotices.rtf|json src/source-build-reference-packages/src/textOnlyPackages/src/microsoft.netcore.*/1.*/ThirdPartyNotices.txt|unknown +src/source-build-reference-packages/src/textOnlyPackages/src/microsoft.private.intellisense/8.0.*/IntellisenseFiles/*/1033/System.Security.Permissions.xml|unknown-license-reference # Contains references to licenses which are not applicable to the source src/source-build-reference-packages/src/packageSourceGenerator/PackageSourceGeneratorTask/RewriteNuspec.cs|unknown-license-reference,ms-net-library-2018-11 -src/source-build-reference-packages/src/textOnlyPackages/src/microsoft.private.intellisense/8.0.0-preview-20230918.1/IntellisenseFiles/windowsdesktop/1033/PresentationCore.xml|proprietary-license +src/source-build-reference-packages/src/textOnlyPackages/src/microsoft.private.intellisense/8.0.*/IntellisenseFiles/windowsdesktop/1033/PresentationCore.xml|proprietary-license # # sourcelink