mirrors/dotnet-installer
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

Use symbolic links instead of hard links in runtime pack directories. (#17534)

Browse source
This commit is contained in:
Daniel Plaisted 2023-10-20 18:47:29 -04:00 committed by GitHub
commit 2068c00c14
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 54 additions and 67 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

94
src/core-sdk-tasks/ReplaceDuplicateFilesWithHardLinks.cs → src/core-sdk-tasks/ReplaceFilesWithSymbolicLinks.cs
View file

@ -22,18 +22,24 @@ namespace Microsoft.DotNet.Build.Tasks
/// <summary>
/// Replaces files that have the same content with hard links.
/// </summary>
public sealed class ReplaceDuplicateFilesWithHardLinks : Task
public sealed class ReplaceFilesWithSymbolicLinks : Task
{
/// <summary>
/// The path to the directory.
/// The path to the directory to recursively search for files to replace with symbolic links.
/// </summary>
[Required]
public string Directory { get; set; } = "";
/// <summary>
/// The path to the directory with files to link to.
/// </summary>
[Required]
public string LinkToFilesFrom { get; set; } = "";
#if NETFRAMEWORK
public override bool Execute()
{
Log.LogError($"{nameof(ReplaceDuplicateFilesWithHardLinks)} is not supported on .NET Framework.");
Log.LogError($"{nameof(ReplaceFilesWithSymbolicLinks)} is not supported on .NET Framework.");
return false;
}
#else
@ -41,7 +47,7 @@ namespace Microsoft.DotNet.Build.Tasks
{
if (OperatingSystem.IsWindows())
{
Log.LogError($"{nameof(ReplaceDuplicateFilesWithHardLinks)} is not supported on Windows.");
Log.LogError($"{nameof(ReplaceFilesWithSymbolicLinks)} is not supported on Windows.");
return false;
}
@ -51,10 +57,16 @@ namespace Microsoft.DotNet.Build.Tasks
return false;
}
if (!System.IO.Directory.Exists(LinkToFilesFrom))
{
Log.LogError($"'{LinkToFilesFrom}' does not exist.");
return false;
}
// Find all non-empty, non-symbolic link files.
IEnumerable<FileInfo> fse = new FileSystemEnumerable<FileInfo>(
string[] files = new FileSystemEnumerable<string>(
Directory,
(ref FileSystemEntry entry) => (FileInfo)entry.ToFileSystemInfo(),
(ref FileSystemEntry entry) => entry.ToFullPath(),
new EnumerationOptions()
{
AttributesToSkip = FileAttributes.ReparsePoint,
@ -63,40 +75,18 @@ namespace Microsoft.DotNet.Build.Tasks
{
ShouldIncludePredicate = (ref FileSystemEntry entry) => !entry.IsDirectory
&& entry.Length > 0
};
}.ToArray();
// Group them by file size.
IEnumerable<string?[]> filesGroupedBySize = fse.GroupBy(file => file.Length,
file => file.FullName,
(size, files) => files.ToArray());
foreach (var file in files)
{
string fileName = Path.GetFileName(file);
// Replace files with same content with hard link.
foreach (var files in filesGroupedBySize)
// Look for a file with the same name in LinkToFilesFrom
// and replace it with a symbolic link if it has the same content.
string targetFile = Path.Combine(LinkToFilesFrom, fileName);
if (File.Exists(targetFile) && FilesHaveSameContent(file, targetFile))
{
for (int i = 0; i < files.Length; i++)
{
string? path1 = files[i];
if (path1 is null)
{
continue; // already linked.
}
for (int j = i + 1; j < files.Length; j++)
{
string? path2 = files[j];
if (path2 is null)
{
continue; // already linked.
}
// note: There's no public API we can use to see if paths are already linked.
// We treat those paths as unlinked files, and link them again.
if (FilesHaveSameContent(path1, path2))
{
ReplaceByLink(path1, path2);
files[j] = null;
}
}
ReplaceByLinkTo(file, targetFile);
}
}
@ -138,34 +128,30 @@ namespace Microsoft.DotNet.Build.Tasks
}
}
void ReplaceByLink(string path1, string path2)
void ReplaceByLinkTo(string path, string pathToTarget)
{
// To link, the target mustn't exist. Make a backup, so we can restore it when linking fails.
string path2Backup = $"{path2}.pre_link_backup";
File.Move(path2, path2Backup);
string backupFile = $"{path}.pre_link_backup";
File.Move(path, backupFile);
int rv = SystemNative_Link(path1, path2);
if (rv != 0)
try
{
var ex = new Win32Exception(); // Captures the LastError.
string relativePath = Path.GetRelativePath(Path.GetDirectoryName(path)!, pathToTarget);
File.CreateSymbolicLink(path, relativePath);
Log.LogError($"Unable to link '{path2}' to '{path1}.': {ex}");
File.Delete(backupFile);
File.Move(path2Backup, path2);
throw ex;
Log.LogMessage(MessageImportance.Normal, $"Linked '{path}' to '{relativePath}'.");
}
else
catch (Exception ex)
{
File.Delete(path2Backup);
Log.LogError($"Unable to link '{path}' to '{pathToTarget}.': {ex}");
Log.LogMessage(MessageImportance.Normal, $"Linked '{path1}' and '{path2}'.");
File.Move(backupFile, path);
throw;
}
}
// This native method is used by the runtime to create hard links. It is not exposed through a public .NET API.
[DllImport("libSystem.Native", SetLastError = true)]
static extern int SystemNative_Link(string source, string link);
#endif
}
}

2
src/redist/targets/BuildCoreSdkTasks.targets
View file

@ -40,6 +40,6 @@
<UsingTask TaskName="CollatePackageDownloads" AssemblyFile="$(CoreSdkTaskDll)"/>
<UsingTask TaskName="GenerateSdkRuntimeIdentifierChain" AssemblyFile="$(CoreSdkTaskDll)"/>
<UsingTask TaskName="GetDependencyInfo" AssemblyFile="$(CoreSdkTaskDll)"/>
<UsingTask TaskName="ReplaceDuplicateFilesWithHardLinks" AssemblyFile="$(CoreSdkTaskDll)"/>
<UsingTask TaskName="ReplaceFilesWithSymbolicLinks" AssemblyFile="$(CoreSdkTaskDll)"/>
</Project>

11
src/redist/targets/GenerateLayout.targets
View file

@ -569,11 +569,12 @@
<ResolveAssemblyReference AssemblyFiles="@(AssembliesToResolve)" Silent="$(ResolveAssemblyReferencesSilent)" AssemblyInformationCacheOutputPath="$(RedistLayoutPath)sdk\$(Version)\SDKPrecomputedAssemblyReferences.cache" SearchPaths="{RawFileName}" WarnOrErrorOnTargetArchitectureMismatch="$(ResolveAssemblyWarnOrErrorOnTargetArchitectureMismatch)" />
</Target>
<!-- Replace duplicate files with hard links so that when the same files from a runtime pack
and the corresponding shared frameworks are included in a distro package their data is shared instead of duplicated. -->
<Target Name="ReplaceDuplicateFilesWithHardLinks" DependsOnTargets="LayoutBundledComponents"
<!-- Replace files from the runtime packs with symbolic links to the corresponding shared framework files (and hostfxr) to reduce the size of the runtime pack directories. -->
<Target Name="ReplaceBundledRuntimePackFilesWithSymbolicLinks" DependsOnTargets="LayoutBundledComponents"
Condition="'$(BundleRuntimePacks)' == 'true' and !$([MSBuild]::IsOSPlatform('WINDOWS'))">
<ReplaceDuplicateFilesWithHardLinks Directory="$(RedistLayoutPath)" />
<ReplaceFilesWithSymbolicLinks Directory="$(RedistLayoutPath)/packs/Microsoft.NETCore.App.Runtime.$(SharedFrameworkRid)/$(MicrosoftNETCoreAppRuntimePackageVersion)" LinkToFilesFrom="$(RedistLayoutPath)/shared/Microsoft.NETCore.App/$(MicrosoftNETCoreAppRuntimePackageVersion)" />
<ReplaceFilesWithSymbolicLinks Directory="$(RedistLayoutPath)/packs/Microsoft.NETCore.App.Runtime.$(SharedFrameworkRid)/$(MicrosoftNETCoreAppRuntimePackageVersion)" LinkToFilesFrom="$(RedistLayoutPath)/host/fxr/$(MicrosoftNETCoreAppRuntimePackageVersion)" />
<ReplaceFilesWithSymbolicLinks Directory="$(RedistLayoutPath)/packs/Microsoft.AspNetCore.App.Runtime.$(SharedFrameworkRid)/$(MicrosoftAspNetCoreAppRuntimePackageVersion)" LinkToFilesFrom="$(RedistLayoutPath)/shared/Microsoft.AspNetCore.App/$(MicrosoftAspNetCoreAppRuntimePackageVersion)" />
</Target>
<Target Name="GenerateLayout"
@ -593,7 +594,7 @@
CrossgenLayout;
LayoutAppHostTemplate;
GeneratePrecomputedRarCache;
ReplaceDuplicateFilesWithHardLinks"
ReplaceBundledRuntimePackFilesWithSymbolicLinks"
BeforeTargets="AfterBuild">
</Target>