mirrors/dotnet-installer
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

Merge branch 'release/8.0.1xx' into release/8.0.2xx

Browse source
This commit is contained in:
Jason Zhai 2023-10-24 23:40:53 -07:00
commit 8a6870faa1
9 changed files with 127 additions and 70 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

48
src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CheckForPoison.cs
View file

@ -12,8 +12,11 @@ using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Reflection;
using System.Reflection.Metadata;
using System.Reflection.PortableExecutable;
using System.Security.Cryptography;
using System.Text;
using System.Text.RegularExpressions;
using System.Xml;
using System.Xml.Linq;
@ -147,6 +150,10 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
private const string PoisonMarker = "POISONED";
private const string SbrpAttributeType = "System.Reflection.AssemblyMetadataAttribute";
private const string SbrpAttributeValuePattern = "source\\s?source\\-build\\-reference\\-packages";
private record CandidateFileEntry(string ExtractedPath, string DisplayPath);
public override bool Execute()
@ -298,7 +305,11 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
try
{
AssemblyName asm = AssemblyName.GetAssemblyName(fileToCheck);
if (IsAssemblyPoisoned(fileToCheck))
if (!candidate.DisplayPath.Contains("SourceBuildReferencePackages") && IsAssemblyFromSbrp(fileToCheck))
{
poisonEntry.Type |= PoisonType.SourceBuildReferenceAssembly;
}
else if (IsAssemblyPoisoned(fileToCheck))
{
poisonEntry.Type |= PoisonType.AssemblyAttribute;
}
@ -332,6 +343,41 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
return false;
}
private static bool IsAssemblyFromSbrp(string assemblyPath)
{
using var stream = new FileStream(assemblyPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
using var peReader = new PEReader(stream);
MetadataReader reader = peReader.GetMetadataReader();
return reader.CustomAttributes.Select(attrHandle => reader.GetCustomAttribute(attrHandle))
.Any(attr => IsAttributeSbrp(reader, attr));
}
private static bool IsAttributeSbrp(MetadataReader reader, CustomAttribute attr)
{
string attributeType = string.Empty;
if (attr.Constructor.Kind == HandleKind.MemberReference)
{
MemberReference mref = reader.GetMemberReference((MemberReferenceHandle)attr.Constructor);
if (mref.Parent.Kind == HandleKind.TypeReference)
{
TypeReference tref = reader.GetTypeReference((TypeReferenceHandle)mref.Parent);
attributeType = $"{reader.GetString(tref.Namespace)}.{reader.GetString(tref.Name)}";
}
}
if (attributeType == SbrpAttributeType)
{
BlobReader blobReader = reader.GetBlobReader(attr.Value);
string attributeValue = Encoding.UTF8.GetString(blobReader.ReadBytes(blobReader.Length));
attributeValue = Regex.Replace(attributeValue, @"\p{C}+", string.Empty);
return Regex.IsMatch(attributeValue, SbrpAttributeValuePattern);
}
return false;
}
private static PoisonedFileEntry ExtractAndCheckZipFileOnly(IEnumerable<CatalogPackageEntry> catalogedPackages, CandidateFileEntry candidate, string markerFileName, string tempDir, Queue<CandidateFileEntry> futureFilesToCheck)
{
var poisonEntry = new PoisonedFileEntry();

1
src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/PoisonType.cs
View file

@ -11,5 +11,6 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
Hash = 1,
AssemblyAttribute = 2,
NupkgFile = 4,
SourceBuildReferenceAssembly = 8,
}
}

3
src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/LicenseExclusions.txt
View file

@ -28,8 +28,9 @@ src/arcade/src/Microsoft.DotNet.Build.Tasks.Installers/build/wix/eula.rtf
# aspnetcore
#
# Line 1 is a generic statement about license applicability that is being detected as "unknown"
# A generic statement about license applicability that is being detected as "unknown"
src/aspnetcore/src/Components/THIRD-PARTY-NOTICES.txt|unknown
src/aspnetcore/THIRD-PARTY-NOTICES.txt|unknown
# Windows installer files that have a reference to a URL for license
src/aspnetcore/src/Installers/Windows/**/*.wxl|unknown-license-reference

1
src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/baselines/MsftToSbSdkFiles.diff
View file

@ -48,6 +48,7 @@ index ------------
./packs/NETStandard.Library.Ref/x.y.z/ref/netstandard2.1/System.Xml.XPath.XDocument.dll
./sdk-manifests/
./sdk-manifests/x.y.z/
-./sdk-manifests/x.y.z/
-./sdk-manifests/x.y.z/
./sdk-manifests/x.y.z/microsoft.net.workload.emscripten.current/
./sdk-manifests/x.y.z/microsoft.net.workload.emscripten.current/x.y.z/

15
src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/baselines/PoisonUsage.txt
View file

@ -1 +1,14 @@
<PrebuiltLeakReport />
<PrebuiltLeakReport>
<File Path="artifacts/x64/Release/dotnet-sdk-x.y.z-banana-rid.tar.gz/sdk/x.y.z/DotnetTools/dotnet-format/Microsoft.Bcl.AsyncInterfaces.dll">
<Type>SourceBuildReferenceAssembly</Type>
</File>
<File Path="artifacts/x64/Release/Private.SourceBuilt.Artifacts.x.y.z/dotnet-format.x.y.z.nupkg/tools/netx.y/any/Microsoft.Bcl.AsyncInterfaces.dll">
<Type>SourceBuildReferenceAssembly</Type>
</File>
<File Path="artifacts/x64/Release/Private.SourceBuilt.Artifacts.x.y.z/Microsoft.TestPlatform.CLI.x.y.z/contentFiles/any/netx.y/Microsoft.Extensions.DependencyModel.dll">
<Type>SourceBuildReferenceAssembly</Type>
</File>
<File Path="artifacts/x64/Release/Private.SourceBuilt.Artifacts.x.y.z/Microsoft.TestPlatform.CLI.x.y.z/contentFiles/any/netx.y/Microsoft.Extensions.FileSystemGlobbing.dll">
<Type>SourceBuildReferenceAssembly</Type>
</File>
</PrebuiltLeakReport>

106
src/core-sdk-tasks/ReplaceDuplicateFilesWithHardLinks.cs → src/core-sdk-tasks/ReplaceFilesWithSymbolicLinks.cs
View file

@ -22,18 +22,24 @@ namespace Microsoft.DotNet.Build.Tasks
/// <summary>
/// Replaces files that have the same content with hard links.
/// </summary>
public sealed class ReplaceDuplicateFilesWithHardLinks : Task
public sealed class ReplaceFilesWithSymbolicLinks : Task
{
/// <summary>
/// The path to the directory.
/// The path to the directory to recursively search for files to replace with symbolic links.
/// </summary>
[Required]
public string Directory { get; set; } = "";
/// <summary>
/// The path to the directory with files to link to.
/// </summary>
[Required]
public string LinkToFilesFrom { get; set; } = "";
#if NETFRAMEWORK
public override bool Execute()
{
Log.LogError($"{nameof(ReplaceDuplicateFilesWithHardLinks)} is not supported on .NET Framework.");
Log.LogError($"{nameof(ReplaceFilesWithSymbolicLinks)} is not supported on .NET Framework.");
return false;
}
#else
@ -41,7 +47,7 @@ namespace Microsoft.DotNet.Build.Tasks
{
if (OperatingSystem.IsWindows())
{
Log.LogError($"{nameof(ReplaceDuplicateFilesWithHardLinks)} is not supported on Windows.");
Log.LogError($"{nameof(ReplaceFilesWithSymbolicLinks)} is not supported on Windows.");
return false;
}
@ -51,52 +57,36 @@ namespace Microsoft.DotNet.Build.Tasks
return false;
}
if (!System.IO.Directory.Exists(LinkToFilesFrom))
{
Log.LogError($"'{LinkToFilesFrom}' does not exist.");
return false;
}
// Find all non-empty, non-symbolic link files.
IEnumerable<FileInfo> fse = new FileSystemEnumerable<FileInfo>(
Directory,
(ref FileSystemEntry entry) => (FileInfo)entry.ToFileSystemInfo(),
new EnumerationOptions()
{
AttributesToSkip = FileAttributes.ReparsePoint,
RecurseSubdirectories = true
})
string[] files = new FileSystemEnumerable<string>(
Directory,
(ref FileSystemEntry entry) => entry.ToFullPath(),
new EnumerationOptions()
{
AttributesToSkip = FileAttributes.ReparsePoint,
RecurseSubdirectories = true
})
{
ShouldIncludePredicate = (ref FileSystemEntry entry) => !entry.IsDirectory
&& entry.Length > 0
};
}.ToArray();
// Group them by file size.
IEnumerable<string?[]> filesGroupedBySize = fse.GroupBy(file => file.Length,
file => file.FullName,
(size, files) => files.ToArray());
// Replace files with same content with hard link.
foreach (var files in filesGroupedBySize)
foreach (var file in files)
{
for (int i = 0; i < files.Length; i++)
string fileName = Path.GetFileName(file);
// Look for a file with the same name in LinkToFilesFrom
// and replace it with a symbolic link if it has the same content.
string targetFile = Path.Combine(LinkToFilesFrom, fileName);
if (File.Exists(targetFile) && FilesHaveSameContent(file, targetFile))
{
string? path1 = files[i];
if (path1 is null)
{
continue; // already linked.
}
for (int j = i + 1; j < files.Length; j++)
{
string? path2 = files[j];
if (path2 is null)
{
continue; // already linked.
}
// note: There's no public API we can use to see if paths are already linked.
// We treat those paths as unlinked files, and link them again.
if (FilesHaveSameContent(path1, path2))
{
ReplaceByLink(path1, path2);
files[j] = null;
}
}
ReplaceByLinkTo(file, targetFile);
}
}
@ -138,34 +128,30 @@ namespace Microsoft.DotNet.Build.Tasks
}
}
void ReplaceByLink(string path1, string path2)
void ReplaceByLinkTo(string path, string pathToTarget)
{
// To link, the target mustn't exist. Make a backup, so we can restore it when linking fails.
string path2Backup = $"{path2}.pre_link_backup";
File.Move(path2, path2Backup);
string backupFile = $"{path}.pre_link_backup";
File.Move(path, backupFile);
int rv = SystemNative_Link(path1, path2);
if (rv != 0)
try
{
var ex = new Win32Exception(); // Captures the LastError.
string relativePath = Path.GetRelativePath(Path.GetDirectoryName(path)!, pathToTarget);
File.CreateSymbolicLink(path, relativePath);
Log.LogError($"Unable to link '{path2}' to '{path1}.': {ex}");
File.Delete(backupFile);
File.Move(path2Backup, path2);
throw ex;
Log.LogMessage(MessageImportance.Normal, $"Linked '{path}' to '{relativePath}'.");
}
else
catch (Exception ex)
{
File.Delete(path2Backup);
Log.LogError($"Unable to link '{path}' to '{pathToTarget}.': {ex}");
Log.LogMessage(MessageImportance.Normal, $"Linked '{path1}' and '{path2}'.");
File.Move(backupFile, path);
throw;
}
}
// This native method is used by the runtime to create hard links. It is not exposed through a public .NET API.
[DllImport("libSystem.Native", SetLastError = true)]
static extern int SystemNative_Link(string source, string link);
#endif
}
}

2
src/redist/targets/BuildCoreSdkTasks.targets
View file

@ -40,6 +40,6 @@
<UsingTask TaskName="CollatePackageDownloads" AssemblyFile="$(CoreSdkTaskDll)"/>
<UsingTask TaskName="GenerateSdkRuntimeIdentifierChain" AssemblyFile="$(CoreSdkTaskDll)"/>
<UsingTask TaskName="GetDependencyInfo" AssemblyFile="$(CoreSdkTaskDll)"/>
<UsingTask TaskName="ReplaceDuplicateFilesWithHardLinks" AssemblyFile="$(CoreSdkTaskDll)"/>
<UsingTask TaskName="ReplaceFilesWithSymbolicLinks" AssemblyFile="$(CoreSdkTaskDll)"/>
</Project>

13
src/redist/targets/GenerateLayout.targets
View file

@ -569,11 +569,12 @@
<ResolveAssemblyReference AssemblyFiles="@(AssembliesToResolve)" Silent="$(ResolveAssemblyReferencesSilent)" AssemblyInformationCacheOutputPath="$(RedistLayoutPath)sdk\$(Version)\SDKPrecomputedAssemblyReferences.cache" SearchPaths="{RawFileName}" WarnOrErrorOnTargetArchitectureMismatch="$(ResolveAssemblyWarnOrErrorOnTargetArchitectureMismatch)" />
</Target>
<!-- Replace duplicate files with hard links so that when the same files from a runtime pack
and the corresponding shared frameworks are included in a distro package their data is shared instead of duplicated. -->
<Target Name="ReplaceDuplicateFilesWithHardLinks" DependsOnTargets="LayoutBundledComponents"
Condition="'$(BundleRuntimePacks)' == 'true' and !$([MSBuild]::IsOSPlatform('WINDOWS'))">
<ReplaceDuplicateFilesWithHardLinks Directory="$(RedistLayoutPath)" />
<!-- Replace files from the runtime packs with symbolic links to the corresponding shared framework files (and hostfxr) to reduce the size of the runtime pack directories. -->
<Target Name="ReplaceBundledRuntimePackFilesWithSymbolicLinks" DependsOnTargets="LayoutBundledComponents"
Condition="'$(BundleRuntimePacks)' == 'true' and !$([MSBuild]::IsOSPlatform('WINDOWS'))">
<ReplaceFilesWithSymbolicLinks Directory="$(RedistLayoutPath)/packs/Microsoft.NETCore.App.Runtime.$(SharedFrameworkRid)/$(MicrosoftNETCoreAppRuntimePackageVersion)" LinkToFilesFrom="$(RedistLayoutPath)/shared/Microsoft.NETCore.App/$(MicrosoftNETCoreAppRuntimePackageVersion)" />
<ReplaceFilesWithSymbolicLinks Directory="$(RedistLayoutPath)/packs/Microsoft.NETCore.App.Runtime.$(SharedFrameworkRid)/$(MicrosoftNETCoreAppRuntimePackageVersion)" LinkToFilesFrom="$(RedistLayoutPath)/host/fxr/$(MicrosoftNETCoreAppRuntimePackageVersion)" />
<ReplaceFilesWithSymbolicLinks Directory="$(RedistLayoutPath)/packs/Microsoft.AspNetCore.App.Runtime.$(SharedFrameworkRid)/$(MicrosoftAspNetCoreAppRuntimePackageVersion)" LinkToFilesFrom="$(RedistLayoutPath)/shared/Microsoft.AspNetCore.App/$(MicrosoftAspNetCoreAppRuntimePackageVersion)" />
</Target>
<Target Name="GenerateLayout"
@ -593,7 +594,7 @@
CrossgenLayout;
LayoutAppHostTemplate;
GeneratePrecomputedRarCache;
ReplaceDuplicateFilesWithHardLinks"
ReplaceBundledRuntimePackFilesWithSymbolicLinks"
BeforeTargets="AfterBuild">
</Target>

8
test/SdkTests/TestConfig.xml
View file

@ -245,6 +245,14 @@
Skip="true"
Issue=""
Reason="Cannot run with non-existent LastRuntimeFrameworkVersion"/>
<Method Name="Microsoft.NET.Publish.Tests.GivenThatWeWantToRunILLink.IsTrimmable_warns_when_expected_for_not_correctly_multitargeted_libraries"
Skip="true"
Issue=""
Reason="Cannot run with non-existent LastRuntimeFrameworkVersion"/>
<Method Name="Microsoft.NET.Publish.Tests.GivenThatWeWantToPublishASingleFileApp.EnableSingleFile_warns_when_expected_for_not_correctly_multitargeted_libraries"
Skip="true"
Issue=""
Reason="Cannot run with non-existent LastRuntimeFrameworkVersion"/>
<Method Name="Microsoft.NET.Publish.Tests.GivenThatWeWantToRunILLink.ILLink_can_use_latest_with_unsupported_target_framework"
Skip="true"
Issue=""