mirrors/dotnet-installer
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

Merge branch 'release/8.0.1xx' into release/8.0.2xx

Browse source
This commit is contained in:
Jason Zhai 2023-10-19 23:06:42 -07:00
commit fa47055732
7 changed files with 38 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
eng/common/sdk-task.ps1
View file

@ -64,7 +64,7 @@ try {
$GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty
} }
if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) { if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) {
$GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.7.2-1" -MemberType NoteProperty $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.8.1-2" -MemberType NoteProperty
} }
if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") { if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") {
$xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true $xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true

4
eng/common/tools.ps1
View file

@ -384,8 +384,8 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements =
# If the version of msbuild is going to be xcopied, # If the version of msbuild is going to be xcopied,
# use this version. Version matches a package here: # use this version. Version matches a package here:
# https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet-eng/NuGet/RoslynTools.MSBuild/versions/17.7.2-1 # https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet-eng/NuGet/RoslynTools.MSBuild/versions/17.8.1-2
$defaultXCopyMSBuildVersion = '17.7.2-1' $defaultXCopyMSBuildVersion = '17.8.1-2'
if (!$vsRequirements) { if (!$vsRequirements) {
if (Get-Member -InputObject $GlobalJson.tools -Name 'vs') { if (Get-Member -InputObject $GlobalJson.tools -Name 'vs') {

1
src/SourceBuild/content/build.proj
View file

@ -161,6 +161,7 @@
</ItemGroup> </ItemGroup>
<CheckForPoison FilesToCheck="@(FinalCliTarball)" <CheckForPoison FilesToCheck="@(FinalCliTarball)"
ProjectDirPath="$(ProjectDir)"
HashCatalogFilePath="$(PoisonReportDataFile)" HashCatalogFilePath="$(PoisonReportDataFile)"
MarkerFileName="$(PoisonMarkerFile)" MarkerFileName="$(PoisonMarkerFile)"
PoisonReportOutputFilePath="$(PoisonUsageReportFile)" PoisonReportOutputFilePath="$(PoisonUsageReportFile)"

40
src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CheckForPoison.cs
View file

@ -29,6 +29,12 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
[Required] [Required]
public ITaskItem[] FilesToCheck { get; set; } public ITaskItem[] FilesToCheck { get; set; }
/// <summary>
/// The path of the project directory to the FilesToCheck.
/// </summary>
[Required]
public string ProjectDirPath { get; set; }
/// <summary> /// <summary>
/// The output path for an XML poison report, if desired. /// The output path for an XML poison report, if desired.
/// </summary> /// </summary>
@ -141,6 +147,8 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
private const string PoisonMarker = "POISONED"; private const string PoisonMarker = "POISONED";
private record CandidateFileEntry(string ExtractedPath, string DisplayPath);
public override bool Execute() public override bool Execute()
{ {
IEnumerable<PoisonedFileEntry> poisons = GetPoisonedFiles(FilesToCheck.Select(f => f.ItemSpec), HashCatalogFilePath, MarkerFileName); IEnumerable<PoisonedFileEntry> poisons = GetPoisonedFiles(FilesToCheck.Select(f => f.ItemSpec), HashCatalogFilePath, MarkerFileName);
@ -176,7 +184,9 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
IEnumerable<string> nonShippingPackages = GetAllNonShippingPackages(); IEnumerable<string> nonShippingPackages = GetAllNonShippingPackages();
IEnumerable<CatalogPackageEntry> catalogedPackages = ReadCatalog(catalogedPackagesFilePath); IEnumerable<CatalogPackageEntry> catalogedPackages = ReadCatalog(catalogedPackagesFilePath);
var poisons = new List<PoisonedFileEntry>(); var poisons = new List<PoisonedFileEntry>();
var candidateQueue = new Queue<string>(initialCandidates); var candidateQueue = new Queue<CandidateFileEntry>(initialCandidates.Select(candidate =>
new CandidateFileEntry(candidate, Utility.MakeRelativePath(candidate, ProjectDirPath))));
if (!string.IsNullOrWhiteSpace(OverrideTempPath)) if (!string.IsNullOrWhiteSpace(OverrideTempPath))
{ {
Directory.CreateDirectory(OverrideTempPath); Directory.CreateDirectory(OverrideTempPath);
@ -186,22 +196,22 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
while (candidateQueue.Any()) while (candidateQueue.Any())
{ {
var checking = candidateQueue.Dequeue(); var candidate = candidateQueue.Dequeue();
// if this is a zip or NuPkg, extract it, check for the poison marker, and // if this is a zip or NuPkg, extract it, check for the poison marker, and
// add its contents to the list to be checked. // add its contents to the list to be checked.
if (ZipFileExtensions.Concat(TarFileExtensions).Concat(TarGzFileExtensions).Any(e => checking.ToLowerInvariant().EndsWith(e))) if (ZipFileExtensions.Concat(TarFileExtensions).Concat(TarGzFileExtensions).Any(e => candidate.ExtractedPath.ToLowerInvariant().EndsWith(e)))
{ {
Log.LogMessage($"Zip or NuPkg file to check: {checking}"); Log.LogMessage($"Zip or NuPkg file to check: {candidate.ExtractedPath}");
// Skip non-shipping packages // Skip non-shipping packages
if (nonShippingPackages.Contains(Path.GetFileName(checking), StringComparer.OrdinalIgnoreCase)) if (nonShippingPackages.Contains(Path.GetFileName(candidate.ExtractedPath), StringComparer.OrdinalIgnoreCase))
{ {
continue; continue;
} }
var tempCheckingDir = Path.Combine(tempDir.FullName, Path.GetFileNameWithoutExtension(checking)); var tempCheckingDir = Path.Combine(tempDir.FullName, Path.GetFileNameWithoutExtension(candidate.ExtractedPath));
PoisonedFileEntry result = ExtractAndCheckZipFileOnly(catalogedPackages, checking, markerFileName, tempCheckingDir, candidateQueue); PoisonedFileEntry result = ExtractAndCheckZipFileOnly(catalogedPackages, candidate, markerFileName, tempCheckingDir, candidateQueue);
if (result != null) if (result != null)
{ {
poisons.Add(result); poisons.Add(result);
@ -209,7 +219,7 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
} }
else else
{ {
PoisonedFileEntry result = CheckSingleFile(catalogedPackages, tempDir.FullName, checking); PoisonedFileEntry result = CheckSingleFile(catalogedPackages, candidate);
if (result != null) if (result != null)
{ {
poisons.Add(result); poisons.Add(result);
@ -237,10 +247,12 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
} }
} }
private static PoisonedFileEntry CheckSingleFile(IEnumerable<CatalogPackageEntry> catalogedPackages, string rootPath, string fileToCheck) private static PoisonedFileEntry CheckSingleFile(IEnumerable<CatalogPackageEntry> catalogedPackages, CandidateFileEntry candidate)
{ {
// skip some common files that get copied verbatim from nupkgs - LICENSE, _._, etc as well as // skip some common files that get copied verbatim from nupkgs - LICENSE, _._, etc as well as
// file types that we never care about - text files, .gitconfig, etc. // file types that we never care about - text files, .gitconfig, etc.
var fileToCheck = candidate.ExtractedPath;
if (FileNamesToSkip.Any(f => Path.GetFileName(fileToCheck).ToLowerInvariant() == f.ToLowerInvariant()) || if (FileNamesToSkip.Any(f => Path.GetFileName(fileToCheck).ToLowerInvariant() == f.ToLowerInvariant()) ||
FileExtensionsToSkip.Any(e => Path.GetExtension(fileToCheck).ToLowerInvariant() == e.ToLowerInvariant()) || FileExtensionsToSkip.Any(e => Path.GetExtension(fileToCheck).ToLowerInvariant() == e.ToLowerInvariant()) ||
(new FileInfo(fileToCheck).Length == 0)) (new FileInfo(fileToCheck).Length == 0))
@ -249,7 +261,7 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
} }
var poisonEntry = new PoisonedFileEntry(); var poisonEntry = new PoisonedFileEntry();
poisonEntry.Path = Utility.MakeRelativePath(fileToCheck, rootPath); poisonEntry.Path = candidate.DisplayPath;
// There seems to be some weird issues with using file streams both for hashing and assembly loading. // There seems to be some weird issues with using file streams both for hashing and assembly loading.
// Copy everything into a memory stream to avoid these problems. // Copy everything into a memory stream to avoid these problems.
@ -320,9 +332,10 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
return false; return false;
} }
private static PoisonedFileEntry ExtractAndCheckZipFileOnly(IEnumerable<CatalogPackageEntry> catalogedPackages, string zipToCheck, string markerFileName, string tempDir, Queue<string> futureFilesToCheck) private static PoisonedFileEntry ExtractAndCheckZipFileOnly(IEnumerable<CatalogPackageEntry> catalogedPackages, CandidateFileEntry candidate, string markerFileName, string tempDir, Queue<CandidateFileEntry> futureFilesToCheck)
{ {
var poisonEntry = new PoisonedFileEntry(); var poisonEntry = new PoisonedFileEntry();
var zipToCheck = candidate.ExtractedPath;
poisonEntry.Path = zipToCheck; poisonEntry.Path = zipToCheck;
using (var sha = SHA256.Create()) using (var sha = SHA256.Create())
@ -375,8 +388,9 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
foreach (var child in Directory.EnumerateFiles(tempDir, "*", SearchOption.AllDirectories)) foreach (var child in Directory.EnumerateFiles(tempDir, "*", SearchOption.AllDirectories))
{ {
// also add anything in this zip/package for checking string displayPath = $"{candidate.DisplayPath}/{child.Replace(tempDir, string.Empty).TrimStart(Path.DirectorySeparatorChar)}";
futureFilesToCheck.Enqueue(child);
futureFilesToCheck.Enqueue(new CandidateFileEntry(child, displayPath));
} }
return poisonEntry.Type != PoisonType.None ? poisonEntry : null; return poisonEntry.Type != PoisonType.None ? poisonEntry : null;

6
src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/DotNetFormatTests.cs
View file

@ -22,12 +22,6 @@ public class DotNetFormatTests : SdkTests
[Fact] [Fact]
public void FormatProject() public void FormatProject()
{ {
if (Config.TargetRid.Contains("alpine"))
{
// Skipping this test on Alpine due to https://github.com/dotnet/format/issues/1945
return;
}
string unformattedCsFilePath = Path.Combine(BaselineHelper.GetAssetsDirectory(), UnformattedFileName); string unformattedCsFilePath = Path.Combine(BaselineHelper.GetAssetsDirectory(), UnformattedFileName);
string projectDirectory = DotNetHelper.ExecuteNew("console", nameof(FormatProject), "C#"); string projectDirectory = DotNetHelper.ExecuteNew("console", nameof(FormatProject), "C#");

9
src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/LicenseScanTests.cs
View file

@ -73,6 +73,7 @@ public class LicenseScanTests : TestBase
"lgpl-2.0-plus", // https://opensource.org/license/lgpl-2-0/ "lgpl-2.0-plus", // https://opensource.org/license/lgpl-2-0/
"lgpl-2.1", // https://opensource.org/license/lgpl-2-1/ "lgpl-2.1", // https://opensource.org/license/lgpl-2-1/
"lgpl-2.1-plus", // https://opensource.org/license/lgpl-2-1/ "lgpl-2.1-plus", // https://opensource.org/license/lgpl-2-1/
"lzma-sdk-9.22", // https://github.com/nexB/scancode-toolkit/blob/develop/src/licensedcode/data/licenses/lzma-sdk-9.22.LICENSE
"mit", // https://opensource.org/license/mit/ "mit", // https://opensource.org/license/mit/
"mit-addition", // https://github.com/nexB/scancode-toolkit/blob/develop/src/licensedcode/data/licenses/mit-addition.LICENSE "mit-addition", // https://github.com/nexB/scancode-toolkit/blob/develop/src/licensedcode/data/licenses/mit-addition.LICENSE
"ms-patent-promise", // https://github.com/nexB/scancode-toolkit/blob/develop/src/licensedcode/data/licenses/ms-patent-promise.LICENSE "ms-patent-promise", // https://github.com/nexB/scancode-toolkit/blob/develop/src/licensedcode/data/licenses/ms-patent-promise.LICENSE
@ -141,17 +142,16 @@ public class LicenseScanTests : TestBase
{ {
Assert.NotNull(Config.LicenseScanPath); Assert.NotNull(Config.LicenseScanPath);
string OriginalScancodeResultsPath = Path.Combine(LogsDirectory, "scancode-results-original.json"); string scancodeResultsPath = Path.Combine(LogsDirectory, "scancode-results.json");
string FilteredScancodeResultsPath = Path.Combine(LogsDirectory, "scancode-results-filtered.json");
// Scancode Doc: https://scancode-toolkit.readthedocs.io/en/latest/index.html // Scancode Doc: https://scancode-toolkit.readthedocs.io/en/latest/index.html
string ignoreOptions = string.Join(" ", s_ignoredFilePatterns.Select(pattern => $"--ignore {pattern}")); string ignoreOptions = string.Join(" ", s_ignoredFilePatterns.Select(pattern => $"--ignore {pattern}"));
ExecuteHelper.ExecuteProcessValidateExitCode( ExecuteHelper.ExecuteProcessValidateExitCode(
"scancode", "scancode",
$"--license --strip-root --only-findings {ignoreOptions} --json-pp {OriginalScancodeResultsPath} {Config.LicenseScanPath}", $"--license --strip-root --only-findings {ignoreOptions} --json-pp {scancodeResultsPath} {Config.LicenseScanPath}",
OutputHelper); OutputHelper);
JsonDocument doc = JsonDocument.Parse(File.ReadAllText(OriginalScancodeResultsPath)); JsonDocument doc = JsonDocument.Parse(File.ReadAllText(scancodeResultsPath));
ScancodeResults? scancodeResults = doc.Deserialize<ScancodeResults>(); ScancodeResults? scancodeResults = doc.Deserialize<ScancodeResults>();
Assert.NotNull(scancodeResults); Assert.NotNull(scancodeResults);
@ -162,7 +162,6 @@ public class LicenseScanTests : TestBase
WriteIndented = true WriteIndented = true
}; };
string json = JsonSerializer.Serialize(scancodeResults, options); string json = JsonSerializer.Serialize(scancodeResults, options);
File.WriteAllText(FilteredScancodeResultsPath, json);
string baselineName = $"Licenses.{_targetRepo}.json"; string baselineName = $"Licenses.{_targetRepo}.json";

5
src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/LicenseExclusions.txt
View file

@ -167,7 +167,7 @@ src/runtime/src/installer/pkg/THIRD-PARTY-NOTICES.TXT
# False positive # False positive
src/runtime/src/libraries/System.ComponentModel.TypeConverter/src/System/ComponentModel/LicFileLicenseProvider.cs|proprietary-license src/runtime/src/libraries/System.ComponentModel.TypeConverter/src/System/ComponentModel/LicFileLicenseProvider.cs|proprietary-license
src/runtime/src/libraries/System.Configuration.ConfigurationManager/tests/Mono/LongValidatorTest.cs|json src/runtime/src/libraries/System.Configuration.ConfigurationManager/tests/Mono/LongValidatorTest.cs|embedthis-extension
src/runtime/src/libraries/System.Net.Sockets/tests/FunctionalTests/Connect.cs|other-permissive src/runtime/src/libraries/System.Net.Sockets/tests/FunctionalTests/Connect.cs|other-permissive
src/runtime/src/libraries/System.Net.Sockets/tests/FunctionalTests/UdpClientTest.cs|other-permissive src/runtime/src/libraries/System.Net.Sockets/tests/FunctionalTests/UdpClientTest.cs|other-permissive
src/runtime/src/libraries/System.Net.Sockets/tests/FunctionalTests/SendReceive/SendReceive.cs|other-permissive src/runtime/src/libraries/System.Net.Sockets/tests/FunctionalTests/SendReceive/SendReceive.cs|other-permissive
@ -226,10 +226,11 @@ src/source-build-reference-packages/src/targetPacks/ILsrc/netstandard.library/2.
src/source-build-reference-packages/src/targetPacks/ILsrc/netstandard.library.ref/2.1.0/THIRD-PARTY-NOTICES.TXT|codesourcery-2004 src/source-build-reference-packages/src/targetPacks/ILsrc/netstandard.library.ref/2.1.0/THIRD-PARTY-NOTICES.TXT|codesourcery-2004
src/source-build-reference-packages/src/textOnlyPackages/src/microsoft.codeanalysis.collections/4.2.0-1.22102.8/ThirdPartyNotices.rtf|json src/source-build-reference-packages/src/textOnlyPackages/src/microsoft.codeanalysis.collections/4.2.0-1.22102.8/ThirdPartyNotices.rtf|json
src/source-build-reference-packages/src/textOnlyPackages/src/microsoft.netcore.*/1.*/ThirdPartyNotices.txt|unknown src/source-build-reference-packages/src/textOnlyPackages/src/microsoft.netcore.*/1.*/ThirdPartyNotices.txt|unknown
src/source-build-reference-packages/src/textOnlyPackages/src/microsoft.private.intellisense/8.0.*/IntellisenseFiles/*/1033/System.Security.Permissions.xml|unknown-license-reference
# Contains references to licenses which are not applicable to the source # Contains references to licenses which are not applicable to the source
src/source-build-reference-packages/src/packageSourceGenerator/PackageSourceGeneratorTask/RewriteNuspec.cs|unknown-license-reference,ms-net-library-2018-11 src/source-build-reference-packages/src/packageSourceGenerator/PackageSourceGeneratorTask/RewriteNuspec.cs|unknown-license-reference,ms-net-library-2018-11
src/source-build-reference-packages/src/textOnlyPackages/src/microsoft.private.intellisense/8.0.0-preview-20230918.1/IntellisenseFiles/windowsdesktop/1033/PresentationCore.xml|proprietary-license src/source-build-reference-packages/src/textOnlyPackages/src/microsoft.private.intellisense/8.0.*/IntellisenseFiles/windowsdesktop/1033/PresentationCore.xml|proprietary-license
# #
# sourcelink # sourcelink