mirrors/dotnet-installer
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions 1

Merge branch 'release/8.0.1xx' into release/8.0.2xx

Browse source
This commit is contained in:
Jason Zhai 2023-10-19 23:06:42 -07:00
commit fa47055732
7 changed files with 38 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
eng/common/sdk-task.ps1
View file

@ -64,7 +64,7 @@ try {
$GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty
}
if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) {
$GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.7.2-1" -MemberType NoteProperty
$GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.8.1-2" -MemberType NoteProperty
}
if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") {
$xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true

4
eng/common/tools.ps1
View file

@ -384,8 +384,8 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements =
# If the version of msbuild is going to be xcopied,
# use this version. Version matches a package here:
# https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet-eng/NuGet/RoslynTools.MSBuild/versions/17.7.2-1
$defaultXCopyMSBuildVersion = '17.7.2-1'
# https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet-eng/NuGet/RoslynTools.MSBuild/versions/17.8.1-2
$defaultXCopyMSBuildVersion = '17.8.1-2'
if (!$vsRequirements) {
if (Get-Member -InputObject $GlobalJson.tools -Name 'vs') {

1
src/SourceBuild/content/build.proj
View file

@ -161,6 +161,7 @@
</ItemGroup>
<CheckForPoison FilesToCheck="@(FinalCliTarball)"
ProjectDirPath="$(ProjectDir)"
HashCatalogFilePath="$(PoisonReportDataFile)"
MarkerFileName="$(PoisonMarkerFile)"
PoisonReportOutputFilePath="$(PoisonUsageReportFile)"

40
src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CheckForPoison.cs
View file

@ -29,6 +29,12 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
[Required]
public ITaskItem[] FilesToCheck { get; set; }
/// <summary>
/// The path of the project directory to the FilesToCheck.
/// </summary>
[Required]
public string ProjectDirPath { get; set; }
/// <summary>
/// The output path for an XML poison report, if desired.
/// </summary>
@ -141,6 +147,8 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
private const string PoisonMarker = "POISONED";
private record CandidateFileEntry(string ExtractedPath, string DisplayPath);
public override bool Execute()
{
IEnumerable<PoisonedFileEntry> poisons = GetPoisonedFiles(FilesToCheck.Select(f => f.ItemSpec), HashCatalogFilePath, MarkerFileName);
@ -176,7 +184,9 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
IEnumerable<string> nonShippingPackages = GetAllNonShippingPackages();
IEnumerable<CatalogPackageEntry> catalogedPackages = ReadCatalog(catalogedPackagesFilePath);
var poisons = new List<PoisonedFileEntry>();
var candidateQueue = new Queue<string>(initialCandidates);
var candidateQueue = new Queue<CandidateFileEntry>(initialCandidates.Select(candidate =>
new CandidateFileEntry(candidate, Utility.MakeRelativePath(candidate, ProjectDirPath))));
if (!string.IsNullOrWhiteSpace(OverrideTempPath))
{
Directory.CreateDirectory(OverrideTempPath);
@ -186,22 +196,22 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
while (candidateQueue.Any())
{
var checking = candidateQueue.Dequeue();
var candidate = candidateQueue.Dequeue();
// if this is a zip or NuPkg, extract it, check for the poison marker, and
// add its contents to the list to be checked.
if (ZipFileExtensions.Concat(TarFileExtensions).Concat(TarGzFileExtensions).Any(e => checking.ToLowerInvariant().EndsWith(e)))
if (ZipFileExtensions.Concat(TarFileExtensions).Concat(TarGzFileExtensions).Any(e => candidate.ExtractedPath.ToLowerInvariant().EndsWith(e)))
{
Log.LogMessage($"Zip or NuPkg file to check: {checking}");
Log.LogMessage($"Zip or NuPkg file to check: {candidate.ExtractedPath}");
// Skip non-shipping packages
if (nonShippingPackages.Contains(Path.GetFileName(checking), StringComparer.OrdinalIgnoreCase))
if (nonShippingPackages.Contains(Path.GetFileName(candidate.ExtractedPath), StringComparer.OrdinalIgnoreCase))
{
continue;
}
var tempCheckingDir = Path.Combine(tempDir.FullName, Path.GetFileNameWithoutExtension(checking));
PoisonedFileEntry result = ExtractAndCheckZipFileOnly(catalogedPackages, checking, markerFileName, tempCheckingDir, candidateQueue);
var tempCheckingDir = Path.Combine(tempDir.FullName, Path.GetFileNameWithoutExtension(candidate.ExtractedPath));
PoisonedFileEntry result = ExtractAndCheckZipFileOnly(catalogedPackages, candidate, markerFileName, tempCheckingDir, candidateQueue);
if (result != null)
{
poisons.Add(result);
@ -209,7 +219,7 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
}
else
{
PoisonedFileEntry result = CheckSingleFile(catalogedPackages, tempDir.FullName, checking);
PoisonedFileEntry result = CheckSingleFile(catalogedPackages, candidate);
if (result != null)
{
poisons.Add(result);
@ -237,10 +247,12 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
}
}
private static PoisonedFileEntry CheckSingleFile(IEnumerable<CatalogPackageEntry> catalogedPackages, string rootPath, string fileToCheck)
private static PoisonedFileEntry CheckSingleFile(IEnumerable<CatalogPackageEntry> catalogedPackages, CandidateFileEntry candidate)
{
// skip some common files that get copied verbatim from nupkgs - LICENSE, _._, etc as well as
// file types that we never care about - text files, .gitconfig, etc.
var fileToCheck = candidate.ExtractedPath;
if (FileNamesToSkip.Any(f => Path.GetFileName(fileToCheck).ToLowerInvariant() == f.ToLowerInvariant()) ||
FileExtensionsToSkip.Any(e => Path.GetExtension(fileToCheck).ToLowerInvariant() == e.ToLowerInvariant()) ||
(new FileInfo(fileToCheck).Length == 0))
@ -249,7 +261,7 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
}
var poisonEntry = new PoisonedFileEntry();
poisonEntry.Path = Utility.MakeRelativePath(fileToCheck, rootPath);
poisonEntry.Path = candidate.DisplayPath;
// There seems to be some weird issues with using file streams both for hashing and assembly loading.
// Copy everything into a memory stream to avoid these problems.
@ -320,9 +332,10 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
return false;
}
private static PoisonedFileEntry ExtractAndCheckZipFileOnly(IEnumerable<CatalogPackageEntry> catalogedPackages, string zipToCheck, string markerFileName, string tempDir, Queue<string> futureFilesToCheck)
private static PoisonedFileEntry ExtractAndCheckZipFileOnly(IEnumerable<CatalogPackageEntry> catalogedPackages, CandidateFileEntry candidate, string markerFileName, string tempDir, Queue<CandidateFileEntry> futureFilesToCheck)
{
var poisonEntry = new PoisonedFileEntry();
var zipToCheck = candidate.ExtractedPath;
poisonEntry.Path = zipToCheck;
using (var sha = SHA256.Create())
@ -375,8 +388,9 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
foreach (var child in Directory.EnumerateFiles(tempDir, "*", SearchOption.AllDirectories))
{
// also add anything in this zip/package for checking
futureFilesToCheck.Enqueue(child);
string displayPath = $"{candidate.DisplayPath}/{child.Replace(tempDir, string.Empty).TrimStart(Path.DirectorySeparatorChar)}";
futureFilesToCheck.Enqueue(new CandidateFileEntry(child, displayPath));
}
return poisonEntry.Type != PoisonType.None ? poisonEntry : null;

6
src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/DotNetFormatTests.cs
View file

@ -22,12 +22,6 @@ public class DotNetFormatTests : SdkTests
[Fact]
public void FormatProject()
{
if (Config.TargetRid.Contains("alpine"))
{
// Skipping this test on Alpine due to https://github.com/dotnet/format/issues/1945
return;
}
string unformattedCsFilePath = Path.Combine(BaselineHelper.GetAssetsDirectory(), UnformattedFileName);
string projectDirectory = DotNetHelper.ExecuteNew("console", nameof(FormatProject), "C#");

9
src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/LicenseScanTests.cs
View file

@ -73,6 +73,7 @@ public class LicenseScanTests : TestBase
"lgpl-2.0-plus", // https://opensource.org/license/lgpl-2-0/
"lgpl-2.1", // https://opensource.org/license/lgpl-2-1/
"lgpl-2.1-plus", // https://opensource.org/license/lgpl-2-1/
"lzma-sdk-9.22", // https://github.com/nexB/scancode-toolkit/blob/develop/src/licensedcode/data/licenses/lzma-sdk-9.22.LICENSE
"mit", // https://opensource.org/license/mit/
"mit-addition", // https://github.com/nexB/scancode-toolkit/blob/develop/src/licensedcode/data/licenses/mit-addition.LICENSE
"ms-patent-promise", // https://github.com/nexB/scancode-toolkit/blob/develop/src/licensedcode/data/licenses/ms-patent-promise.LICENSE
@ -141,17 +142,16 @@ public class LicenseScanTests : TestBase
{
Assert.NotNull(Config.LicenseScanPath);
string OriginalScancodeResultsPath = Path.Combine(LogsDirectory, "scancode-results-original.json");
string FilteredScancodeResultsPath = Path.Combine(LogsDirectory, "scancode-results-filtered.json");
string scancodeResultsPath = Path.Combine(LogsDirectory, "scancode-results.json");
// Scancode Doc: https://scancode-toolkit.readthedocs.io/en/latest/index.html
string ignoreOptions = string.Join(" ", s_ignoredFilePatterns.Select(pattern => $"--ignore {pattern}"));
ExecuteHelper.ExecuteProcessValidateExitCode(
"scancode",
$"--license --strip-root --only-findings {ignoreOptions} --json-pp {OriginalScancodeResultsPath} {Config.LicenseScanPath}",
$"--license --strip-root --only-findings {ignoreOptions} --json-pp {scancodeResultsPath} {Config.LicenseScanPath}",
OutputHelper);
JsonDocument doc = JsonDocument.Parse(File.ReadAllText(OriginalScancodeResultsPath));
JsonDocument doc = JsonDocument.Parse(File.ReadAllText(scancodeResultsPath));
ScancodeResults? scancodeResults = doc.Deserialize<ScancodeResults>();
Assert.NotNull(scancodeResults);
@ -162,7 +162,6 @@ public class LicenseScanTests : TestBase
WriteIndented = true
};
string json = JsonSerializer.Serialize(scancodeResults, options);
File.WriteAllText(FilteredScancodeResultsPath, json);
string baselineName = $"Licenses.{_targetRepo}.json";

5
src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/LicenseExclusions.txt
View file

@ -167,7 +167,7 @@ src/runtime/src/installer/pkg/THIRD-PARTY-NOTICES.TXT
# False positive
src/runtime/src/libraries/System.ComponentModel.TypeConverter/src/System/ComponentModel/LicFileLicenseProvider.cs|proprietary-license
src/runtime/src/libraries/System.Configuration.ConfigurationManager/tests/Mono/LongValidatorTest.cs|json
src/runtime/src/libraries/System.Configuration.ConfigurationManager/tests/Mono/LongValidatorTest.cs|embedthis-extension
src/runtime/src/libraries/System.Net.Sockets/tests/FunctionalTests/Connect.cs|other-permissive
src/runtime/src/libraries/System.Net.Sockets/tests/FunctionalTests/UdpClientTest.cs|other-permissive
src/runtime/src/libraries/System.Net.Sockets/tests/FunctionalTests/SendReceive/SendReceive.cs|other-permissive
@ -226,10 +226,11 @@ src/source-build-reference-packages/src/targetPacks/ILsrc/netstandard.library/2.
src/source-build-reference-packages/src/targetPacks/ILsrc/netstandard.library.ref/2.1.0/THIRD-PARTY-NOTICES.TXT|codesourcery-2004
src/source-build-reference-packages/src/textOnlyPackages/src/microsoft.codeanalysis.collections/4.2.0-1.22102.8/ThirdPartyNotices.rtf|json
src/source-build-reference-packages/src/textOnlyPackages/src/microsoft.netcore.*/1.*/ThirdPartyNotices.txt|unknown
src/source-build-reference-packages/src/textOnlyPackages/src/microsoft.private.intellisense/8.0.*/IntellisenseFiles/*/1033/System.Security.Permissions.xml|unknown-license-reference
# Contains references to licenses which are not applicable to the source
src/source-build-reference-packages/src/packageSourceGenerator/PackageSourceGeneratorTask/RewriteNuspec.cs|unknown-license-reference,ms-net-library-2018-11
src/source-build-reference-packages/src/textOnlyPackages/src/microsoft.private.intellisense/8.0.0-preview-20230918.1/IntellisenseFiles/windowsdesktop/1033/PresentationCore.xml|proprietary-license
src/source-build-reference-packages/src/textOnlyPackages/src/microsoft.private.intellisense/8.0.*/IntellisenseFiles/windowsdesktop/1033/PresentationCore.xml|proprietary-license
#
# sourcelink