Use symbolic links instead of hard links in runtime pack directories. (#17534)
This commit is contained in:
Daniel Plaisted
GitHub
commit
2068c00c14
3 changed files with 54 additions and 67 deletions
|
|
@ -22,18 +22,24 @@ namespace Microsoft.DotNet.Build.Tasks
|
|||
/// <summary>
|
||||
/// Replaces files that have the same content with hard links.
|
||||
/// </summary>
|
||||
public sealed class ReplaceDuplicateFilesWithHardLinks : Task
|
||||
public sealed class ReplaceFilesWithSymbolicLinks : Task
|
||||
{
|
||||
/// <summary>
|
||||
/// The path to the directory.
|
||||
/// The path to the directory to recursively search for files to replace with symbolic links.
|
||||
/// </summary>
|
||||
[Required]
|
||||
public string Directory { get; set; } = "";
|
||||
|
||||
/// <summary>
|
||||
/// The path to the directory with files to link to.
|
||||
/// </summary>
|
||||
[Required]
|
||||
public string LinkToFilesFrom { get; set; } = "";
|
||||
|
||||
#if NETFRAMEWORK
|
||||
public override bool Execute()
|
||||
{
|
||||
Log.LogError($"{nameof(ReplaceDuplicateFilesWithHardLinks)} is not supported on .NET Framework.");
|
||||
Log.LogError($"{nameof(ReplaceFilesWithSymbolicLinks)} is not supported on .NET Framework.");
|
||||
return false;
|
||||
}
|
||||
#else
|
||||
|
|
@ -41,7 +47,7 @@ namespace Microsoft.DotNet.Build.Tasks
|
|||
{
|
||||
if (OperatingSystem.IsWindows())
|
||||
{
|
||||
Log.LogError($"{nameof(ReplaceDuplicateFilesWithHardLinks)} is not supported on Windows.");
|
||||
Log.LogError($"{nameof(ReplaceFilesWithSymbolicLinks)} is not supported on Windows.");
|
||||
return false;
|
||||
}
|
||||
|
||||
|
|
@ -51,52 +57,36 @@ namespace Microsoft.DotNet.Build.Tasks
|
|||
return false;
|
||||
}
|
||||
|
||||
if (!System.IO.Directory.Exists(LinkToFilesFrom))
|
||||
{
|
||||
Log.LogError($"'{LinkToFilesFrom}' does not exist.");
|
||||
return false;
|
||||
}
|
||||
|
||||
// Find all non-empty, non-symbolic link files.
|
||||
IEnumerable<FileInfo> fse = new FileSystemEnumerable<FileInfo>(
|
||||
Directory,
|
||||
(ref FileSystemEntry entry) => (FileInfo)entry.ToFileSystemInfo(),
|
||||
new EnumerationOptions()
|
||||
{
|
||||
AttributesToSkip = FileAttributes.ReparsePoint,
|
||||
RecurseSubdirectories = true
|
||||
})
|
||||
string[] files = new FileSystemEnumerable<string>(
|
||||
Directory,
|
||||
(ref FileSystemEntry entry) => entry.ToFullPath(),
|
||||
new EnumerationOptions()
|
||||
{
|
||||
AttributesToSkip = FileAttributes.ReparsePoint,
|
||||
RecurseSubdirectories = true
|
||||
})
|
||||
{
|
||||
ShouldIncludePredicate = (ref FileSystemEntry entry) => !entry.IsDirectory
|
||||
&& entry.Length > 0
|
||||
};
|
||||
}.ToArray();
|
||||
|
||||
// Group them by file size.
|
||||
IEnumerable<string?[]> filesGroupedBySize = fse.GroupBy(file => file.Length,
|
||||
file => file.FullName,
|
||||
(size, files) => files.ToArray());
|
||||
|
||||
// Replace files with same content with hard link.
|
||||
foreach (var files in filesGroupedBySize)
|
||||
foreach (var file in files)
|
||||
{
|
||||
for (int i = 0; i < files.Length; i++)
|
||||
string fileName = Path.GetFileName(file);
|
||||
|
||||
// Look for a file with the same name in LinkToFilesFrom
|
||||
// and replace it with a symbolic link if it has the same content.
|
||||
string targetFile = Path.Combine(LinkToFilesFrom, fileName);
|
||||
if (File.Exists(targetFile) && FilesHaveSameContent(file, targetFile))
|
||||
{
|
||||
string? path1 = files[i];
|
||||
if (path1 is null)
|
||||
{
|
||||
continue; // already linked.
|
||||
}
|
||||
for (int j = i + 1; j < files.Length; j++)
|
||||
{
|
||||
string? path2 = files[j];
|
||||
if (path2 is null)
|
||||
{
|
||||
continue; // already linked.
|
||||
}
|
||||
|
||||
// note: There's no public API we can use to see if paths are already linked.
|
||||
// We treat those paths as unlinked files, and link them again.
|
||||
if (FilesHaveSameContent(path1, path2))
|
||||
{
|
||||
ReplaceByLink(path1, path2);
|
||||
|
||||
files[j] = null;
|
||||
}
|
||||
}
|
||||
ReplaceByLinkTo(file, targetFile);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -138,34 +128,30 @@ namespace Microsoft.DotNet.Build.Tasks
|
|||
}
|
||||
}
|
||||
|
||||
void ReplaceByLink(string path1, string path2)
|
||||
void ReplaceByLinkTo(string path, string pathToTarget)
|
||||
{
|
||||
// To link, the target mustn't exist. Make a backup, so we can restore it when linking fails.
|
||||
string path2Backup = $"{path2}.pre_link_backup";
|
||||
File.Move(path2, path2Backup);
|
||||
string backupFile = $"{path}.pre_link_backup";
|
||||
File.Move(path, backupFile);
|
||||
|
||||
int rv = SystemNative_Link(path1, path2);
|
||||
if (rv != 0)
|
||||
try
|
||||
{
|
||||
var ex = new Win32Exception(); // Captures the LastError.
|
||||
string relativePath = Path.GetRelativePath(Path.GetDirectoryName(path)!, pathToTarget);
|
||||
File.CreateSymbolicLink(path, relativePath);
|
||||
|
||||
Log.LogError($"Unable to link '{path2}' to '{path1}.': {ex}");
|
||||
File.Delete(backupFile);
|
||||
|
||||
File.Move(path2Backup, path2);
|
||||
|
||||
throw ex;
|
||||
Log.LogMessage(MessageImportance.Normal, $"Linked '{path}' to '{relativePath}'.");
|
||||
}
|
||||
else
|
||||
catch (Exception ex)
|
||||
{
|
||||
File.Delete(path2Backup);
|
||||
Log.LogError($"Unable to link '{path}' to '{pathToTarget}.': {ex}");
|
||||
|
||||
Log.LogMessage(MessageImportance.Normal, $"Linked '{path1}' and '{path2}'.");
|
||||
File.Move(backupFile, path);
|
||||
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
// This native method is used by the runtime to create hard links. It is not exposed through a public .NET API.
|
||||
[DllImport("libSystem.Native", SetLastError = true)]
|
||||
static extern int SystemNative_Link(string source, string link);
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
|
@ -40,6 +40,6 @@
|
|||
<UsingTask TaskName="CollatePackageDownloads" AssemblyFile="$(CoreSdkTaskDll)"/>
|
||||
<UsingTask TaskName="GenerateSdkRuntimeIdentifierChain" AssemblyFile="$(CoreSdkTaskDll)"/>
|
||||
<UsingTask TaskName="GetDependencyInfo" AssemblyFile="$(CoreSdkTaskDll)"/>
|
||||
<UsingTask TaskName="ReplaceDuplicateFilesWithHardLinks" AssemblyFile="$(CoreSdkTaskDll)"/>
|
||||
<UsingTask TaskName="ReplaceFilesWithSymbolicLinks" AssemblyFile="$(CoreSdkTaskDll)"/>
|
||||
|
||||
</Project>
|
||||
|
|
|
|||
|
|
@ -569,11 +569,12 @@
|
|||
<ResolveAssemblyReference AssemblyFiles="@(AssembliesToResolve)" Silent="$(ResolveAssemblyReferencesSilent)" AssemblyInformationCacheOutputPath="$(RedistLayoutPath)sdk\$(Version)\SDKPrecomputedAssemblyReferences.cache" SearchPaths="{RawFileName}" WarnOrErrorOnTargetArchitectureMismatch="$(ResolveAssemblyWarnOrErrorOnTargetArchitectureMismatch)" />
|
||||
</Target>
|
||||
|
||||
<!-- Replace duplicate files with hard links so that when the same files from a runtime pack
|
||||
and the corresponding shared frameworks are included in a distro package their data is shared instead of duplicated. -->
|
||||
<Target Name="ReplaceDuplicateFilesWithHardLinks" DependsOnTargets="LayoutBundledComponents"
|
||||
Condition="'$(BundleRuntimePacks)' == 'true' and !$([MSBuild]::IsOSPlatform('WINDOWS'))">
|
||||
<ReplaceDuplicateFilesWithHardLinks Directory="$(RedistLayoutPath)" />
|
||||
<!-- Replace files from the runtime packs with symbolic links to the corresponding shared framework files (and hostfxr) to reduce the size of the runtime pack directories. -->
|
||||
<Target Name="ReplaceBundledRuntimePackFilesWithSymbolicLinks" DependsOnTargets="LayoutBundledComponents"
|
||||
Condition="'$(BundleRuntimePacks)' == 'true' and !$([MSBuild]::IsOSPlatform('WINDOWS'))">
|
||||
<ReplaceFilesWithSymbolicLinks Directory="$(RedistLayoutPath)/packs/Microsoft.NETCore.App.Runtime.$(SharedFrameworkRid)/$(MicrosoftNETCoreAppRuntimePackageVersion)" LinkToFilesFrom="$(RedistLayoutPath)/shared/Microsoft.NETCore.App/$(MicrosoftNETCoreAppRuntimePackageVersion)" />
|
||||
<ReplaceFilesWithSymbolicLinks Directory="$(RedistLayoutPath)/packs/Microsoft.NETCore.App.Runtime.$(SharedFrameworkRid)/$(MicrosoftNETCoreAppRuntimePackageVersion)" LinkToFilesFrom="$(RedistLayoutPath)/host/fxr/$(MicrosoftNETCoreAppRuntimePackageVersion)" />
|
||||
<ReplaceFilesWithSymbolicLinks Directory="$(RedistLayoutPath)/packs/Microsoft.AspNetCore.App.Runtime.$(SharedFrameworkRid)/$(MicrosoftAspNetCoreAppRuntimePackageVersion)" LinkToFilesFrom="$(RedistLayoutPath)/shared/Microsoft.AspNetCore.App/$(MicrosoftAspNetCoreAppRuntimePackageVersion)" />
|
||||
</Target>
|
||||
|
||||
<Target Name="GenerateLayout"
|
||||
|
|
@ -593,7 +594,7 @@
|
|||
CrossgenLayout;
|
||||
LayoutAppHostTemplate;
|
||||
GeneratePrecomputedRarCache;
|
||||
ReplaceDuplicateFilesWithHardLinks"
|
||||
ReplaceBundledRuntimePackFilesWithSymbolicLinks"
|
||||
BeforeTargets="AfterBuild">
|
||||
|
||||
</Target>
|
||||
|
|
|
|||
Loading…
Reference in a new issue