Merge branch 'release/8.0.1xx' into release/8.0.2xx

This commit is contained in:
Jason Zhai 2023-10-24 23:40:53 -07:00
commit 8a6870faa1
9 changed files with 127 additions and 70 deletions

View file

@ -12,8 +12,11 @@ using System.IO;
using System.IO.Compression; using System.IO.Compression;
using System.Linq; using System.Linq;
using System.Reflection; using System.Reflection;
using System.Reflection.Metadata;
using System.Reflection.PortableExecutable;
using System.Security.Cryptography; using System.Security.Cryptography;
using System.Text; using System.Text;
using System.Text.RegularExpressions;
using System.Xml; using System.Xml;
using System.Xml.Linq; using System.Xml.Linq;
@ -147,6 +150,10 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
private const string PoisonMarker = "POISONED"; private const string PoisonMarker = "POISONED";
private const string SbrpAttributeType = "System.Reflection.AssemblyMetadataAttribute";
private const string SbrpAttributeValuePattern = "source\\s?source\\-build\\-reference\\-packages";
private record CandidateFileEntry(string ExtractedPath, string DisplayPath); private record CandidateFileEntry(string ExtractedPath, string DisplayPath);
public override bool Execute() public override bool Execute()
@ -298,7 +305,11 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
try try
{ {
AssemblyName asm = AssemblyName.GetAssemblyName(fileToCheck); AssemblyName asm = AssemblyName.GetAssemblyName(fileToCheck);
if (IsAssemblyPoisoned(fileToCheck)) if (!candidate.DisplayPath.Contains("SourceBuildReferencePackages") && IsAssemblyFromSbrp(fileToCheck))
{
poisonEntry.Type |= PoisonType.SourceBuildReferenceAssembly;
}
else if (IsAssemblyPoisoned(fileToCheck))
{ {
poisonEntry.Type |= PoisonType.AssemblyAttribute; poisonEntry.Type |= PoisonType.AssemblyAttribute;
} }
@ -332,6 +343,41 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
return false; return false;
} }
private static bool IsAssemblyFromSbrp(string assemblyPath)
{
using var stream = new FileStream(assemblyPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
using var peReader = new PEReader(stream);
MetadataReader reader = peReader.GetMetadataReader();
return reader.CustomAttributes.Select(attrHandle => reader.GetCustomAttribute(attrHandle))
.Any(attr => IsAttributeSbrp(reader, attr));
}
private static bool IsAttributeSbrp(MetadataReader reader, CustomAttribute attr)
{
string attributeType = string.Empty;
if (attr.Constructor.Kind == HandleKind.MemberReference)
{
MemberReference mref = reader.GetMemberReference((MemberReferenceHandle)attr.Constructor);
if (mref.Parent.Kind == HandleKind.TypeReference)
{
TypeReference tref = reader.GetTypeReference((TypeReferenceHandle)mref.Parent);
attributeType = $"{reader.GetString(tref.Namespace)}.{reader.GetString(tref.Name)}";
}
}
if (attributeType == SbrpAttributeType)
{
BlobReader blobReader = reader.GetBlobReader(attr.Value);
string attributeValue = Encoding.UTF8.GetString(blobReader.ReadBytes(blobReader.Length));
attributeValue = Regex.Replace(attributeValue, @"\p{C}+", string.Empty);
return Regex.IsMatch(attributeValue, SbrpAttributeValuePattern);
}
return false;
}
private static PoisonedFileEntry ExtractAndCheckZipFileOnly(IEnumerable<CatalogPackageEntry> catalogedPackages, CandidateFileEntry candidate, string markerFileName, string tempDir, Queue<CandidateFileEntry> futureFilesToCheck) private static PoisonedFileEntry ExtractAndCheckZipFileOnly(IEnumerable<CatalogPackageEntry> catalogedPackages, CandidateFileEntry candidate, string markerFileName, string tempDir, Queue<CandidateFileEntry> futureFilesToCheck)
{ {
var poisonEntry = new PoisonedFileEntry(); var poisonEntry = new PoisonedFileEntry();

View file

@ -11,5 +11,6 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
Hash = 1, Hash = 1,
AssemblyAttribute = 2, AssemblyAttribute = 2,
NupkgFile = 4, NupkgFile = 4,
SourceBuildReferenceAssembly = 8,
} }
} }

View file

@ -28,8 +28,9 @@ src/arcade/src/Microsoft.DotNet.Build.Tasks.Installers/build/wix/eula.rtf
# aspnetcore # aspnetcore
# #
# Line 1 is a generic statement about license applicability that is being detected as "unknown" # A generic statement about license applicability that is being detected as "unknown"
src/aspnetcore/src/Components/THIRD-PARTY-NOTICES.txt|unknown src/aspnetcore/src/Components/THIRD-PARTY-NOTICES.txt|unknown
src/aspnetcore/THIRD-PARTY-NOTICES.txt|unknown
# Windows installer files that have a reference to a URL for license # Windows installer files that have a reference to a URL for license
src/aspnetcore/src/Installers/Windows/**/*.wxl|unknown-license-reference src/aspnetcore/src/Installers/Windows/**/*.wxl|unknown-license-reference

View file

@ -48,6 +48,7 @@ index ------------
./packs/NETStandard.Library.Ref/x.y.z/ref/netstandard2.1/System.Xml.XPath.XDocument.dll ./packs/NETStandard.Library.Ref/x.y.z/ref/netstandard2.1/System.Xml.XPath.XDocument.dll
./sdk-manifests/ ./sdk-manifests/
./sdk-manifests/x.y.z/ ./sdk-manifests/x.y.z/
-./sdk-manifests/x.y.z/
-./sdk-manifests/x.y.z/ -./sdk-manifests/x.y.z/
./sdk-manifests/x.y.z/microsoft.net.workload.emscripten.current/ ./sdk-manifests/x.y.z/microsoft.net.workload.emscripten.current/
./sdk-manifests/x.y.z/microsoft.net.workload.emscripten.current/x.y.z/ ./sdk-manifests/x.y.z/microsoft.net.workload.emscripten.current/x.y.z/

View file

@ -1 +1,14 @@
<PrebuiltLeakReport /> <PrebuiltLeakReport>
<File Path="artifacts/x64/Release/dotnet-sdk-x.y.z-banana-rid.tar.gz/sdk/x.y.z/DotnetTools/dotnet-format/Microsoft.Bcl.AsyncInterfaces.dll">
<Type>SourceBuildReferenceAssembly</Type>
</File>
<File Path="artifacts/x64/Release/Private.SourceBuilt.Artifacts.x.y.z/dotnet-format.x.y.z.nupkg/tools/netx.y/any/Microsoft.Bcl.AsyncInterfaces.dll">
<Type>SourceBuildReferenceAssembly</Type>
</File>
<File Path="artifacts/x64/Release/Private.SourceBuilt.Artifacts.x.y.z/Microsoft.TestPlatform.CLI.x.y.z/contentFiles/any/netx.y/Microsoft.Extensions.DependencyModel.dll">
<Type>SourceBuildReferenceAssembly</Type>
</File>
<File Path="artifacts/x64/Release/Private.SourceBuilt.Artifacts.x.y.z/Microsoft.TestPlatform.CLI.x.y.z/contentFiles/any/netx.y/Microsoft.Extensions.FileSystemGlobbing.dll">
<Type>SourceBuildReferenceAssembly</Type>
</File>
</PrebuiltLeakReport>

View file

@ -22,18 +22,24 @@ namespace Microsoft.DotNet.Build.Tasks
/// <summary> /// <summary>
/// Replaces files that have the same content with hard links. /// Replaces files that have the same content with hard links.
/// </summary> /// </summary>
public sealed class ReplaceDuplicateFilesWithHardLinks : Task public sealed class ReplaceFilesWithSymbolicLinks : Task
{ {
/// <summary> /// <summary>
/// The path to the directory. /// The path to the directory to recursively search for files to replace with symbolic links.
/// </summary> /// </summary>
[Required] [Required]
public string Directory { get; set; } = ""; public string Directory { get; set; } = "";
/// <summary>
/// The path to the directory with files to link to.
/// </summary>
[Required]
public string LinkToFilesFrom { get; set; } = "";
#if NETFRAMEWORK #if NETFRAMEWORK
public override bool Execute() public override bool Execute()
{ {
Log.LogError($"{nameof(ReplaceDuplicateFilesWithHardLinks)} is not supported on .NET Framework."); Log.LogError($"{nameof(ReplaceFilesWithSymbolicLinks)} is not supported on .NET Framework.");
return false; return false;
} }
#else #else
@ -41,7 +47,7 @@ namespace Microsoft.DotNet.Build.Tasks
{ {
if (OperatingSystem.IsWindows()) if (OperatingSystem.IsWindows())
{ {
Log.LogError($"{nameof(ReplaceDuplicateFilesWithHardLinks)} is not supported on Windows."); Log.LogError($"{nameof(ReplaceFilesWithSymbolicLinks)} is not supported on Windows.");
return false; return false;
} }
@ -51,52 +57,36 @@ namespace Microsoft.DotNet.Build.Tasks
return false; return false;
} }
if (!System.IO.Directory.Exists(LinkToFilesFrom))
{
Log.LogError($"'{LinkToFilesFrom}' does not exist.");
return false;
}
// Find all non-empty, non-symbolic link files. // Find all non-empty, non-symbolic link files.
IEnumerable<FileInfo> fse = new FileSystemEnumerable<FileInfo>( string[] files = new FileSystemEnumerable<string>(
Directory, Directory,
(ref FileSystemEntry entry) => (FileInfo)entry.ToFileSystemInfo(), (ref FileSystemEntry entry) => entry.ToFullPath(),
new EnumerationOptions() new EnumerationOptions()
{ {
AttributesToSkip = FileAttributes.ReparsePoint, AttributesToSkip = FileAttributes.ReparsePoint,
RecurseSubdirectories = true RecurseSubdirectories = true
}) })
{ {
ShouldIncludePredicate = (ref FileSystemEntry entry) => !entry.IsDirectory ShouldIncludePredicate = (ref FileSystemEntry entry) => !entry.IsDirectory
&& entry.Length > 0 && entry.Length > 0
}; }.ToArray();
// Group them by file size. foreach (var file in files)
IEnumerable<string?[]> filesGroupedBySize = fse.GroupBy(file => file.Length,
file => file.FullName,
(size, files) => files.ToArray());
// Replace files with same content with hard link.
foreach (var files in filesGroupedBySize)
{ {
for (int i = 0; i < files.Length; i++) string fileName = Path.GetFileName(file);
// Look for a file with the same name in LinkToFilesFrom
// and replace it with a symbolic link if it has the same content.
string targetFile = Path.Combine(LinkToFilesFrom, fileName);
if (File.Exists(targetFile) && FilesHaveSameContent(file, targetFile))
{ {
string? path1 = files[i]; ReplaceByLinkTo(file, targetFile);
if (path1 is null)
{
continue; // already linked.
}
for (int j = i + 1; j < files.Length; j++)
{
string? path2 = files[j];
if (path2 is null)
{
continue; // already linked.
}
// note: There's no public API we can use to see if paths are already linked.
// We treat those paths as unlinked files, and link them again.
if (FilesHaveSameContent(path1, path2))
{
ReplaceByLink(path1, path2);
files[j] = null;
}
}
} }
} }
@ -138,34 +128,30 @@ namespace Microsoft.DotNet.Build.Tasks
} }
} }
void ReplaceByLink(string path1, string path2) void ReplaceByLinkTo(string path, string pathToTarget)
{ {
// To link, the target mustn't exist. Make a backup, so we can restore it when linking fails. // To link, the target mustn't exist. Make a backup, so we can restore it when linking fails.
string path2Backup = $"{path2}.pre_link_backup"; string backupFile = $"{path}.pre_link_backup";
File.Move(path2, path2Backup); File.Move(path, backupFile);
int rv = SystemNative_Link(path1, path2); try
if (rv != 0)
{ {
var ex = new Win32Exception(); // Captures the LastError. string relativePath = Path.GetRelativePath(Path.GetDirectoryName(path)!, pathToTarget);
File.CreateSymbolicLink(path, relativePath);
Log.LogError($"Unable to link '{path2}' to '{path1}.': {ex}"); File.Delete(backupFile);
File.Move(path2Backup, path2); Log.LogMessage(MessageImportance.Normal, $"Linked '{path}' to '{relativePath}'.");
throw ex;
} }
else catch (Exception ex)
{ {
File.Delete(path2Backup); Log.LogError($"Unable to link '{path}' to '{pathToTarget}.': {ex}");
Log.LogMessage(MessageImportance.Normal, $"Linked '{path1}' and '{path2}'."); File.Move(backupFile, path);
throw;
} }
} }
// This native method is used by the runtime to create hard links. It is not exposed through a public .NET API.
[DllImport("libSystem.Native", SetLastError = true)]
static extern int SystemNative_Link(string source, string link);
#endif #endif
} }
} }

View file

@ -40,6 +40,6 @@
<UsingTask TaskName="CollatePackageDownloads" AssemblyFile="$(CoreSdkTaskDll)"/> <UsingTask TaskName="CollatePackageDownloads" AssemblyFile="$(CoreSdkTaskDll)"/>
<UsingTask TaskName="GenerateSdkRuntimeIdentifierChain" AssemblyFile="$(CoreSdkTaskDll)"/> <UsingTask TaskName="GenerateSdkRuntimeIdentifierChain" AssemblyFile="$(CoreSdkTaskDll)"/>
<UsingTask TaskName="GetDependencyInfo" AssemblyFile="$(CoreSdkTaskDll)"/> <UsingTask TaskName="GetDependencyInfo" AssemblyFile="$(CoreSdkTaskDll)"/>
<UsingTask TaskName="ReplaceDuplicateFilesWithHardLinks" AssemblyFile="$(CoreSdkTaskDll)"/> <UsingTask TaskName="ReplaceFilesWithSymbolicLinks" AssemblyFile="$(CoreSdkTaskDll)"/>
</Project> </Project>

View file

@ -569,11 +569,12 @@
<ResolveAssemblyReference AssemblyFiles="@(AssembliesToResolve)" Silent="$(ResolveAssemblyReferencesSilent)" AssemblyInformationCacheOutputPath="$(RedistLayoutPath)sdk\$(Version)\SDKPrecomputedAssemblyReferences.cache" SearchPaths="{RawFileName}" WarnOrErrorOnTargetArchitectureMismatch="$(ResolveAssemblyWarnOrErrorOnTargetArchitectureMismatch)" /> <ResolveAssemblyReference AssemblyFiles="@(AssembliesToResolve)" Silent="$(ResolveAssemblyReferencesSilent)" AssemblyInformationCacheOutputPath="$(RedistLayoutPath)sdk\$(Version)\SDKPrecomputedAssemblyReferences.cache" SearchPaths="{RawFileName}" WarnOrErrorOnTargetArchitectureMismatch="$(ResolveAssemblyWarnOrErrorOnTargetArchitectureMismatch)" />
</Target> </Target>
<!-- Replace duplicate files with hard links so that when the same files from a runtime pack <!-- Replace files from the runtime packs with symbolic links to the corresponding shared framework files (and hostfxr) to reduce the size of the runtime pack directories. -->
and the corresponding shared frameworks are included in a distro package their data is shared instead of duplicated. --> <Target Name="ReplaceBundledRuntimePackFilesWithSymbolicLinks" DependsOnTargets="LayoutBundledComponents"
<Target Name="ReplaceDuplicateFilesWithHardLinks" DependsOnTargets="LayoutBundledComponents" Condition="'$(BundleRuntimePacks)' == 'true' and !$([MSBuild]::IsOSPlatform('WINDOWS'))">
Condition="'$(BundleRuntimePacks)' == 'true' and !$([MSBuild]::IsOSPlatform('WINDOWS'))"> <ReplaceFilesWithSymbolicLinks Directory="$(RedistLayoutPath)/packs/Microsoft.NETCore.App.Runtime.$(SharedFrameworkRid)/$(MicrosoftNETCoreAppRuntimePackageVersion)" LinkToFilesFrom="$(RedistLayoutPath)/shared/Microsoft.NETCore.App/$(MicrosoftNETCoreAppRuntimePackageVersion)" />
<ReplaceDuplicateFilesWithHardLinks Directory="$(RedistLayoutPath)" /> <ReplaceFilesWithSymbolicLinks Directory="$(RedistLayoutPath)/packs/Microsoft.NETCore.App.Runtime.$(SharedFrameworkRid)/$(MicrosoftNETCoreAppRuntimePackageVersion)" LinkToFilesFrom="$(RedistLayoutPath)/host/fxr/$(MicrosoftNETCoreAppRuntimePackageVersion)" />
<ReplaceFilesWithSymbolicLinks Directory="$(RedistLayoutPath)/packs/Microsoft.AspNetCore.App.Runtime.$(SharedFrameworkRid)/$(MicrosoftAspNetCoreAppRuntimePackageVersion)" LinkToFilesFrom="$(RedistLayoutPath)/shared/Microsoft.AspNetCore.App/$(MicrosoftAspNetCoreAppRuntimePackageVersion)" />
</Target> </Target>
<Target Name="GenerateLayout" <Target Name="GenerateLayout"
@ -593,7 +594,7 @@
CrossgenLayout; CrossgenLayout;
LayoutAppHostTemplate; LayoutAppHostTemplate;
GeneratePrecomputedRarCache; GeneratePrecomputedRarCache;
ReplaceDuplicateFilesWithHardLinks" ReplaceBundledRuntimePackFilesWithSymbolicLinks"
BeforeTargets="AfterBuild"> BeforeTargets="AfterBuild">
</Target> </Target>

View file

@ -245,6 +245,14 @@
Skip="true" Skip="true"
Issue="" Issue=""
Reason="Cannot run with non-existent LastRuntimeFrameworkVersion"/> Reason="Cannot run with non-existent LastRuntimeFrameworkVersion"/>
<Method Name="Microsoft.NET.Publish.Tests.GivenThatWeWantToRunILLink.IsTrimmable_warns_when_expected_for_not_correctly_multitargeted_libraries"
Skip="true"
Issue=""
Reason="Cannot run with non-existent LastRuntimeFrameworkVersion"/>
<Method Name="Microsoft.NET.Publish.Tests.GivenThatWeWantToPublishASingleFileApp.EnableSingleFile_warns_when_expected_for_not_correctly_multitargeted_libraries"
Skip="true"
Issue=""
Reason="Cannot run with non-existent LastRuntimeFrameworkVersion"/>
<Method Name="Microsoft.NET.Publish.Tests.GivenThatWeWantToRunILLink.ILLink_can_use_latest_with_unsupported_target_framework" <Method Name="Microsoft.NET.Publish.Tests.GivenThatWeWantToRunILLink.ILLink_can_use_latest_with_unsupported_target_framework"
Skip="true" Skip="true"
Issue="" Issue=""