mirrors/dotnet-installer
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

Include full path in poison report (#17438)

Browse source
This commit is contained in:
Ella Hathaway 2023-09-29 06:34:42 -07:00 • committed by GitHub
parent ac9a10eadb
commit d7ad055f7f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 28 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
src/SourceBuild/content/build.proj
View file

@ -83,6 +83,7 @@
</ItemGroup> </ItemGroup>
<CheckForPoison FilesToCheck="@(FinalCliTarball)" <CheckForPoison FilesToCheck="@(FinalCliTarball)"
ProjectDirPath="$(ProjectDir)"
HashCatalogFilePath="$(PoisonReportDataFile)" HashCatalogFilePath="$(PoisonReportDataFile)"
MarkerFileName="$(PoisonMarkerFile)" MarkerFileName="$(PoisonMarkerFile)"
PoisonReportOutputFilePath="$(PoisonUsageReportFile)" PoisonReportOutputFilePath="$(PoisonUsageReportFile)"

40
src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CheckForPoison.cs
View file

@ -29,6 +29,12 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
[Required] [Required]
public ITaskItem[] FilesToCheck { get; set; } public ITaskItem[] FilesToCheck { get; set; }
/// <summary>
/// The path of the project directory to the FilesToCheck.
/// </summary>
[Required]
public string ProjectDirPath { get; set; }
/// <summary> /// <summary>
/// The output path for an XML poison report, if desired. /// The output path for an XML poison report, if desired.
/// </summary> /// </summary>
@ -141,6 +147,8 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
private const string PoisonMarker = "POISONED"; private const string PoisonMarker = "POISONED";
private record CandidateFileEntry(string ExtractedPath, string DisplayPath);
public override bool Execute() public override bool Execute()
{ {
IEnumerable<PoisonedFileEntry> poisons = GetPoisonedFiles(FilesToCheck.Select(f => f.ItemSpec), HashCatalogFilePath, MarkerFileName); IEnumerable<PoisonedFileEntry> poisons = GetPoisonedFiles(FilesToCheck.Select(f => f.ItemSpec), HashCatalogFilePath, MarkerFileName);
@ -176,7 +184,9 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
IEnumerable<string> nonShippingPackages = GetAllNonShippingPackages(); IEnumerable<string> nonShippingPackages = GetAllNonShippingPackages();
IEnumerable<CatalogPackageEntry> catalogedPackages = ReadCatalog(catalogedPackagesFilePath); IEnumerable<CatalogPackageEntry> catalogedPackages = ReadCatalog(catalogedPackagesFilePath);
var poisons = new List<PoisonedFileEntry>(); var poisons = new List<PoisonedFileEntry>();
var candidateQueue = new Queue<string>(initialCandidates); var candidateQueue = new Queue<CandidateFileEntry>(initialCandidates.Select(candidate =>
new CandidateFileEntry(candidate, Utility.MakeRelativePath(candidate, ProjectDirPath))));
if (!string.IsNullOrWhiteSpace(OverrideTempPath)) if (!string.IsNullOrWhiteSpace(OverrideTempPath))
{ {
Directory.CreateDirectory(OverrideTempPath); Directory.CreateDirectory(OverrideTempPath);
@ -186,22 +196,22 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
while (candidateQueue.Any()) while (candidateQueue.Any())
{ {
var checking = candidateQueue.Dequeue(); var candidate = candidateQueue.Dequeue();
// if this is a zip or NuPkg, extract it, check for the poison marker, and // if this is a zip or NuPkg, extract it, check for the poison marker, and
// add its contents to the list to be checked. // add its contents to the list to be checked.
if (ZipFileExtensions.Concat(TarFileExtensions).Concat(TarGzFileExtensions).Any(e => checking.ToLowerInvariant().EndsWith(e))) if (ZipFileExtensions.Concat(TarFileExtensions).Concat(TarGzFileExtensions).Any(e => candidate.ExtractedPath.ToLowerInvariant().EndsWith(e)))
{ {
Log.LogMessage($"Zip or NuPkg file to check: {checking}"); Log.LogMessage($"Zip or NuPkg file to check: {candidate.ExtractedPath}");
// Skip non-shipping packages // Skip non-shipping packages
if (nonShippingPackages.Contains(Path.GetFileName(checking), StringComparer.OrdinalIgnoreCase)) if (nonShippingPackages.Contains(Path.GetFileName(candidate.ExtractedPath), StringComparer.OrdinalIgnoreCase))
{ {
continue; continue;
} }
var tempCheckingDir = Path.Combine(tempDir.FullName, Path.GetFileNameWithoutExtension(checking)); var tempCheckingDir = Path.Combine(tempDir.FullName, Path.GetFileNameWithoutExtension(candidate.ExtractedPath));
PoisonedFileEntry result = ExtractAndCheckZipFileOnly(catalogedPackages, checking, markerFileName, tempCheckingDir, candidateQueue); PoisonedFileEntry result = ExtractAndCheckZipFileOnly(catalogedPackages, candidate, markerFileName, tempCheckingDir, candidateQueue);
if (result != null) if (result != null)
{ {
poisons.Add(result); poisons.Add(result);
@ -209,7 +219,7 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
} }
else else
{ {
PoisonedFileEntry result = CheckSingleFile(catalogedPackages, tempDir.FullName, checking); PoisonedFileEntry result = CheckSingleFile(catalogedPackages, candidate);
if (result != null) if (result != null)
{ {
poisons.Add(result); poisons.Add(result);
@ -237,10 +247,12 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
} }
} }
private static PoisonedFileEntry CheckSingleFile(IEnumerable<CatalogPackageEntry> catalogedPackages, string rootPath, string fileToCheck) private static PoisonedFileEntry CheckSingleFile(IEnumerable<CatalogPackageEntry> catalogedPackages, CandidateFileEntry candidate)
{ {
// skip some common files that get copied verbatim from nupkgs - LICENSE, _._, etc as well as // skip some common files that get copied verbatim from nupkgs - LICENSE, _._, etc as well as
// file types that we never care about - text files, .gitconfig, etc. // file types that we never care about - text files, .gitconfig, etc.
var fileToCheck = candidate.ExtractedPath;
if (FileNamesToSkip.Any(f => Path.GetFileName(fileToCheck).ToLowerInvariant() == f.ToLowerInvariant()) || if (FileNamesToSkip.Any(f => Path.GetFileName(fileToCheck).ToLowerInvariant() == f.ToLowerInvariant()) ||
FileExtensionsToSkip.Any(e => Path.GetExtension(fileToCheck).ToLowerInvariant() == e.ToLowerInvariant()) || FileExtensionsToSkip.Any(e => Path.GetExtension(fileToCheck).ToLowerInvariant() == e.ToLowerInvariant()) ||
(new FileInfo(fileToCheck).Length == 0)) (new FileInfo(fileToCheck).Length == 0))
@ -249,7 +261,7 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
} }
var poisonEntry = new PoisonedFileEntry(); var poisonEntry = new PoisonedFileEntry();
poisonEntry.Path = Utility.MakeRelativePath(fileToCheck, rootPath); poisonEntry.Path = candidate.DisplayPath;
// There seems to be some weird issues with using file streams both for hashing and assembly loading. // There seems to be some weird issues with using file streams both for hashing and assembly loading.
// Copy everything into a memory stream to avoid these problems. // Copy everything into a memory stream to avoid these problems.
@ -320,9 +332,10 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
return false; return false;
} }
private static PoisonedFileEntry ExtractAndCheckZipFileOnly(IEnumerable<CatalogPackageEntry> catalogedPackages, string zipToCheck, string markerFileName, string tempDir, Queue<string> futureFilesToCheck) private static PoisonedFileEntry ExtractAndCheckZipFileOnly(IEnumerable<CatalogPackageEntry> catalogedPackages, CandidateFileEntry candidate, string markerFileName, string tempDir, Queue<CandidateFileEntry> futureFilesToCheck)
{ {
var poisonEntry = new PoisonedFileEntry(); var poisonEntry = new PoisonedFileEntry();
var zipToCheck = candidate.ExtractedPath;
poisonEntry.Path = zipToCheck; poisonEntry.Path = zipToCheck;
using (var sha = SHA256.Create()) using (var sha = SHA256.Create())
@ -375,8 +388,9 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
foreach (var child in Directory.EnumerateFiles(tempDir, "*", SearchOption.AllDirectories)) foreach (var child in Directory.EnumerateFiles(tempDir, "*", SearchOption.AllDirectories))
{ {
// also add anything in this zip/package for checking string displayPath = $"{candidate.DisplayPath}/{child.Replace(tempDir, string.Empty).TrimStart(Path.DirectorySeparatorChar)}";
futureFilesToCheck.Enqueue(child);
futureFilesToCheck.Enqueue(new CandidateFileEntry(child, displayPath));
} }
return poisonEntry.Type != PoisonType.None ? poisonEntry : null; return poisonEntry.Type != PoisonType.None ? poisonEntry : null;