Source-build symbols repackaging (#17454)
Co-authored-by: Michael Simons <msimons@microsoft.com>
This commit is contained in:
Nikola Milosavljevic
GitHub
parent
44e6cb59e0
commit
77a7628585
3 changed files with 236 additions and 2 deletions
|
|
@ -127,7 +127,7 @@ jobs:
|
|||
artifact: ${{ parameters.reuseBuildArtifactsFrom }}_${{ parameters.architecture }}_Artifacts
|
||||
patterns: |
|
||||
**/Private.SourceBuilt.Artifacts.*.tar.gz
|
||||
**/dotnet-sdk-*.tar.gz
|
||||
**/dotnet-sdk-+([0-9]).+([0-9]).+([0-9])*.tar.gz
|
||||
displayName: Download Previous Build
|
||||
|
||||
- task: CopyFiles@2
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@
|
|||
<UsingTask AssemblyFile="$(LeakDetectionTasksAssembly)" TaskName="CheckForPoison" />
|
||||
<UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="WriteUsageBurndownData" />
|
||||
<UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="ReplaceTextInFile" />
|
||||
<UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="CreateSdkSymbolsLayout" />
|
||||
|
||||
<Import Project="Sdk.targets" Sdk="Microsoft.NET.Sdk" />
|
||||
|
||||
|
|
@ -57,6 +58,84 @@
|
|||
<MSBuild Projects="$(RepoProjectsDir)$(RootRepo).proj" Targets="WritePrebuiltUsageData;ReportPrebuiltUsage" />
|
||||
</Target>
|
||||
|
||||
<Target Name="DiscoverSymbolsTarballs"
|
||||
AfterTargets="Build">
|
||||
<ItemGroup>
|
||||
<SymbolsTarball Include="$(OutputPath)Symbols.*.tar.gz" />
|
||||
</ItemGroup>
|
||||
</Target>
|
||||
|
||||
<Target Name="ExtractSymbolsTarballs"
|
||||
AfterTargets="Build"
|
||||
DependsOnTargets="DiscoverSymbolsTarballs"
|
||||
Outputs="%(SymbolsTarball.Identity)">
|
||||
|
||||
<PropertyGroup>
|
||||
<Filename>$([System.IO.Path]::GetFileName('%(SymbolsTarball.Identity)'))</Filename>
|
||||
<RepositoryName>$(Filename.Split('.')[1])</RepositoryName>
|
||||
<UnifiedSymbolsLayout>$(ArtifactsTmpDir)Symbols</UnifiedSymbolsLayout>
|
||||
<DestinationFolder>$(UnifiedSymbolsLayout)/$(RepositoryName)</DestinationFolder>
|
||||
</PropertyGroup>
|
||||
|
||||
<MakeDir Directories="$(DestinationFolder)" />
|
||||
<Exec Command="tar -xzf %(SymbolsTarball.Identity) -C $(DestinationFolder)"
|
||||
WorkingDirectory="$(SymbolsRoot)" />
|
||||
|
||||
<Delete Files="%(SymbolsTarball.Identity)" />
|
||||
</Target>
|
||||
|
||||
<!-- After building, repackage symbols into a single tarball. -->
|
||||
<Target Name="RepackageSymbols"
|
||||
AfterTargets="Build"
|
||||
DependsOnTargets="
|
||||
DetermineMicrosoftSourceBuildIntermediateInstallerVersion;
|
||||
DiscoverSymbolsTarballs;
|
||||
ExtractSymbolsTarballs">
|
||||
<PropertyGroup>
|
||||
<UnifiedSymbolsTarball>$(OutputPath)dotnet-symbols-$(MicrosoftSourceBuildIntermediateInstallerVersion)-$(TargetRid).tar.gz</UnifiedSymbolsTarball>
|
||||
</PropertyGroup>
|
||||
|
||||
<Exec Command="tar --numeric-owner -czf $(UnifiedSymbolsTarball) *"
|
||||
WorkingDirectory="$(UnifiedSymbolsLayout)" />
|
||||
|
||||
<Message Importance="High" Text="Packaged all symbols in '$(UnifiedSymbolsTarball)'" />
|
||||
</Target>
|
||||
|
||||
<!-- After building, create the sdk symbols tarball. -->
|
||||
<Target Name="CreateSdkSymbolsTarball"
|
||||
AfterTargets="Build"
|
||||
DependsOnTargets="RepackageSymbols">
|
||||
<ItemGroup>
|
||||
<SdkTarballItem Include="$(OutputPath)dotnet-sdk-*$(TarBallExtension)"
|
||||
Exclude="$(OutputPath)dotnet-sdk-symbols-*$(TarBallExtension)" />
|
||||
</ItemGroup>
|
||||
|
||||
<PropertyGroup>
|
||||
<SdkSymbolsLayout>$(ArtifactsTmpDir)SdkSymbols</SdkSymbolsLayout>
|
||||
<SdkSymbolsTarball>$(OutputPath)dotnet-sdk-symbols-$(MicrosoftSourceBuildIntermediateInstallerVersion)-$(TargetRid).tar.gz</SdkSymbolsTarball>
|
||||
<SdkLayout>$(ArtifactsTmpDir)Sdk</SdkLayout>
|
||||
<SdkTarball>%(SdkTarballItem.Identity)</SdkTarball>
|
||||
</PropertyGroup>
|
||||
|
||||
<MakeDir Directories="$(SdkLayout)" />
|
||||
<Exec Command="tar -xzf $(SdkTarball) -C $(SdkLayout)"
|
||||
WorkingDirectory="$(OutputPath)" />
|
||||
|
||||
<CreateSdkSymbolsLayout SdkLayoutPath="$(SdkLayout)"
|
||||
AllSymbolsPath="$(UnifiedSymbolsLayout)"
|
||||
SdkSymbolsLayoutPath="$(SdkSymbolsLayout)"
|
||||
FailOnMissingPDBs="true" />
|
||||
|
||||
<Exec Command="tar --numeric-owner -czf $(SdkSymbolsTarball) *"
|
||||
WorkingDirectory="$(SdkSymbolsLayout)" />
|
||||
|
||||
<Message Importance="High" Text="Packaged sdk symbols in '$(SdkSymbolsTarball)'" />
|
||||
|
||||
<RemoveDir Directories="$(UnifiedSymbolsLayout)" />
|
||||
<RemoveDir Directories="$(SdkSymbolsLayout)" />
|
||||
<RemoveDir Directories="$(SdkLayout)" />
|
||||
</Target>
|
||||
|
||||
<!--
|
||||
Dev scenario: rewrite a prebuilt-report. This makes it easy to add data to an existing
|
||||
prebuilt report without performing another full build. This doesn't reevalutate which packages
|
||||
|
|
@ -104,7 +183,8 @@
|
|||
|
||||
<Target Name="RunSmokeTest">
|
||||
<ItemGroup>
|
||||
<SdkTarballItem Include="$(SourceBuiltTarBallPath)**/dotnet-sdk*$(TarBallExtension)" />
|
||||
<SdkTarballItem Include="$(SourceBuiltTarBallPath)**/dotnet-sdk*$(TarBallExtension)"
|
||||
Exclude="$(SourceBuiltTarBallPath)**/dotnet-sdk-symbols*$(TarBallExtension)" />
|
||||
<SourceBuiltArtifactsItem Include="$(SourceBuiltTarBallPath)**/Private.SourceBuilt.Artifacts.*$(TarBallExtension)" />
|
||||
</ItemGroup>
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,154 @@
|
|||
// Licensed to the .NET Foundation under one or more agreements.
|
||||
// The .NET Foundation licenses this file to you under the MIT license.
|
||||
// See the LICENSE file in the project root for more information.
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Reflection.Metadata;
|
||||
using System.Reflection.PortableExecutable;
|
||||
using Microsoft.Build.Framework;
|
||||
using Microsoft.Build.Utilities;
|
||||
|
||||
namespace Microsoft.DotNet.Build.Tasks
|
||||
{
|
||||
// Creates a symbols layout that matches the SDK layout
|
||||
public class CreateSdkSymbolsLayout : Task
|
||||
{
|
||||
/// <summary>
|
||||
/// Path to SDK layout.
|
||||
/// </summary>
|
||||
[Required]
|
||||
public string SdkLayoutPath { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Path to all source-built symbols, flat or with folder hierarchy.
|
||||
/// </summary>
|
||||
[Required]
|
||||
public string AllSymbolsPath { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Path to SDK symbols layout - will be created if it doesn't exist.
|
||||
/// </summary>
|
||||
[Required]
|
||||
public string SdkSymbolsLayoutPath { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// If true, fails the build if any PDBs are missing.
|
||||
/// </summary>
|
||||
public bool FailOnMissingPDBs { get; set; }
|
||||
|
||||
public override bool Execute()
|
||||
{
|
||||
IList<string> filesWithoutPDBs = GenerateSymbolsLayout(IndexAllSymbols());
|
||||
if (filesWithoutPDBs.Count > 0)
|
||||
{
|
||||
LogErrorOrWarning(FailOnMissingPDBs, $"Did not find PDBs for the following SDK files:");
|
||||
foreach (string file in filesWithoutPDBs)
|
||||
{
|
||||
LogErrorOrWarning(FailOnMissingPDBs, file);
|
||||
}
|
||||
}
|
||||
|
||||
return !Log.HasLoggedErrors;
|
||||
}
|
||||
|
||||
private void LogErrorOrWarning(bool isError, string message)
|
||||
{
|
||||
if (FailOnMissingPDBs)
|
||||
{
|
||||
Log.LogError(message);
|
||||
}
|
||||
else
|
||||
{
|
||||
Log.LogWarning(message);
|
||||
}
|
||||
}
|
||||
|
||||
private IList<string> GenerateSymbolsLayout(Hashtable allPdbGuids)
|
||||
{
|
||||
List<string> filesWithoutPDBs = new List<string>();
|
||||
|
||||
if (Directory.Exists(SdkSymbolsLayoutPath))
|
||||
{
|
||||
Directory.Delete(SdkSymbolsLayoutPath, true);
|
||||
}
|
||||
|
||||
foreach (string file in Directory.GetFiles(SdkLayoutPath, "*", SearchOption.AllDirectories))
|
||||
{
|
||||
if (file.EndsWith(".dll", StringComparison.InvariantCultureIgnoreCase) &&
|
||||
!file.EndsWith(".resources.dll", StringComparison.InvariantCultureIgnoreCase))
|
||||
{
|
||||
string guid = string.Empty;
|
||||
using var pdbStream = File.OpenRead(file);
|
||||
using var peReader = new PEReader(pdbStream);
|
||||
try
|
||||
{
|
||||
// Check if pdb is embedded
|
||||
if (peReader.ReadDebugDirectory().Any(entry => entry.Type == DebugDirectoryEntryType.EmbeddedPortablePdb))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var debugDirectory = peReader.ReadDebugDirectory().First(entry => entry.Type == DebugDirectoryEntryType.CodeView);
|
||||
var codeViewData = peReader.ReadCodeViewDebugDirectoryData(debugDirectory);
|
||||
guid = $"{codeViewData.Guid.ToString("N").Replace("-", string.Empty)}";
|
||||
}
|
||||
catch (Exception e) when (e is BadImageFormatException || e is InvalidOperationException)
|
||||
{
|
||||
// Ignore binaries without debug info
|
||||
continue;
|
||||
}
|
||||
|
||||
if (guid != string.Empty)
|
||||
{
|
||||
if (!allPdbGuids.ContainsKey(guid))
|
||||
{
|
||||
filesWithoutPDBs.Add(file.Substring(SdkLayoutPath.Length + 1));
|
||||
}
|
||||
else
|
||||
{
|
||||
// Copy matching pdb to symbols path, preserving sdk binary's hierarchy
|
||||
string sourcePath = (string)allPdbGuids[guid]!;
|
||||
string destinationPath =
|
||||
file.Replace(SdkLayoutPath, SdkSymbolsLayoutPath)
|
||||
.Replace(Path.GetFileName(file), Path.GetFileName(sourcePath));
|
||||
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(destinationPath)!);
|
||||
File.Copy(sourcePath, destinationPath, true);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return filesWithoutPDBs;
|
||||
}
|
||||
|
||||
public Hashtable IndexAllSymbols()
|
||||
{
|
||||
Hashtable allPdbGuids = new Hashtable();
|
||||
|
||||
foreach (string file in Directory.GetFiles(AllSymbolsPath, "*.pdb", SearchOption.AllDirectories))
|
||||
{
|
||||
using var pdbFileStream = File.OpenRead(file);
|
||||
var metadataProvider = MetadataReaderProvider.FromPortablePdbStream(pdbFileStream);
|
||||
var metadataReader = metadataProvider.GetMetadataReader();
|
||||
if (metadataReader.DebugMetadataHeader == null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var id = new BlobContentId(metadataReader.DebugMetadataHeader.Id);
|
||||
string guid = $"{id.Guid:N}";
|
||||
if (!string.IsNullOrEmpty(guid) && !allPdbGuids.ContainsKey(guid))
|
||||
{
|
||||
allPdbGuids.Add(guid, file);
|
||||
}
|
||||
}
|
||||
|
||||
return allPdbGuids;
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue