Source-build symbols repackaging (#17454)

Co-authored-by: Michael Simons <msimons@microsoft.com>

eng/pipelines/templates/jobs/vmr-build.yml

@@ -127,7 +127,7 @@ jobs:
       artifact: ${{ parameters.reuseBuildArtifactsFrom }}_${{ parameters.architecture }}_Artifacts
       patterns: |
         **/Private.SourceBuilt.Artifacts.*.tar.gz
-        **/dotnet-sdk-*.tar.gz
+        **/dotnet-sdk-+([0-9]).+([0-9]).+([0-9])*.tar.gz
       displayName: Download Previous Build
 
     - task: CopyFiles@2

src/SourceBuild/content/build.proj

@@ -4,6 +4,7 @@
   <UsingTask AssemblyFile="$(LeakDetectionTasksAssembly)" TaskName="CheckForPoison" />
   <UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="WriteUsageBurndownData" />
   <UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="ReplaceTextInFile" />
+  <UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="CreateSdkSymbolsLayout" />
 
   <Import Project="Sdk.targets" Sdk="Microsoft.NET.Sdk" />
 
@@ -57,6 +58,84 @@
     <MSBuild Projects="$(RepoProjectsDir)$(RootRepo).proj" Targets="WritePrebuiltUsageData;ReportPrebuiltUsage" />
   </Target>
 
+  <Target Name="DiscoverSymbolsTarballs"
+          AfterTargets="Build">
+    <ItemGroup>
+      <SymbolsTarball Include="$(OutputPath)Symbols.*.tar.gz" />
+    </ItemGroup>
+  </Target>
+
+  <Target Name="ExtractSymbolsTarballs"
+          AfterTargets="Build"
+          DependsOnTargets="DiscoverSymbolsTarballs"
+          Outputs="%(SymbolsTarball.Identity)">
+
+    <PropertyGroup>
+      <Filename>$([System.IO.Path]::GetFileName('%(SymbolsTarball.Identity)'))</Filename>
+      <RepositoryName>$(Filename.Split('.')[1])</RepositoryName>
+      <UnifiedSymbolsLayout>$(ArtifactsTmpDir)Symbols</UnifiedSymbolsLayout>
+      <DestinationFolder>$(UnifiedSymbolsLayout)/$(RepositoryName)</DestinationFolder>
+    </PropertyGroup>
+
+    <MakeDir Directories="$(DestinationFolder)" />
+    <Exec Command="tar -xzf %(SymbolsTarball.Identity) -C $(DestinationFolder)"
+          WorkingDirectory="$(SymbolsRoot)" />
+
+    <Delete Files="%(SymbolsTarball.Identity)" />
+  </Target>
+
+  <!-- After building, repackage symbols into a single tarball. -->
+  <Target Name="RepackageSymbols"
+          AfterTargets="Build"
+          DependsOnTargets="
+            DetermineMicrosoftSourceBuildIntermediateInstallerVersion;
+            DiscoverSymbolsTarballs;
+            ExtractSymbolsTarballs">
+    <PropertyGroup>
+      <UnifiedSymbolsTarball>$(OutputPath)dotnet-symbols-$(MicrosoftSourceBuildIntermediateInstallerVersion)-$(TargetRid).tar.gz</UnifiedSymbolsTarball>
+    </PropertyGroup>
+
+    <Exec Command="tar --numeric-owner -czf $(UnifiedSymbolsTarball) *"
+          WorkingDirectory="$(UnifiedSymbolsLayout)" />
+
+    <Message Importance="High" Text="Packaged all symbols in '$(UnifiedSymbolsTarball)'" />
+  </Target>
+
+  <!-- After building, create the sdk symbols tarball. -->
+  <Target Name="CreateSdkSymbolsTarball"
+          AfterTargets="Build"
+          DependsOnTargets="RepackageSymbols">
+    <ItemGroup>
+      <SdkTarballItem Include="$(OutputPath)dotnet-sdk-*$(TarBallExtension)"
+                      Exclude="$(OutputPath)dotnet-sdk-symbols-*$(TarBallExtension)" />
+    </ItemGroup>
+
+    <PropertyGroup>
+      <SdkSymbolsLayout>$(ArtifactsTmpDir)SdkSymbols</SdkSymbolsLayout>
+      <SdkSymbolsTarball>$(OutputPath)dotnet-sdk-symbols-$(MicrosoftSourceBuildIntermediateInstallerVersion)-$(TargetRid).tar.gz</SdkSymbolsTarball>
+      <SdkLayout>$(ArtifactsTmpDir)Sdk</SdkLayout>
+      <SdkTarball>%(SdkTarballItem.Identity)</SdkTarball>
+    </PropertyGroup>
+
+    <MakeDir Directories="$(SdkLayout)" />
+    <Exec Command="tar -xzf $(SdkTarball) -C $(SdkLayout)"
+          WorkingDirectory="$(OutputPath)" />
+
+    <CreateSdkSymbolsLayout SdkLayoutPath="$(SdkLayout)"
+                            AllSymbolsPath="$(UnifiedSymbolsLayout)"
+                            SdkSymbolsLayoutPath="$(SdkSymbolsLayout)"
+                            FailOnMissingPDBs="true" />
+
+    <Exec Command="tar --numeric-owner -czf $(SdkSymbolsTarball) *"
+          WorkingDirectory="$(SdkSymbolsLayout)" />
+
+    <Message Importance="High" Text="Packaged sdk symbols in '$(SdkSymbolsTarball)'" />
+
+    <RemoveDir Directories="$(UnifiedSymbolsLayout)" />
+    <RemoveDir Directories="$(SdkSymbolsLayout)" />
+    <RemoveDir Directories="$(SdkLayout)" />
+  </Target>
+
   <!--
     Dev scenario: rewrite a prebuilt-report. This makes it easy to add data to an existing
     prebuilt report without performing another full build. This doesn't reevalutate which packages
@@ -104,7 +183,8 @@
 
   <Target Name="RunSmokeTest">
     <ItemGroup>
-      <SdkTarballItem Include="$(SourceBuiltTarBallPath)**/dotnet-sdk*$(TarBallExtension)" />
+      <SdkTarballItem Include="$(SourceBuiltTarBallPath)**/dotnet-sdk*$(TarBallExtension)"
+                      Exclude="$(SourceBuiltTarBallPath)**/dotnet-sdk-symbols*$(TarBallExtension)" />
       <SourceBuiltArtifactsItem Include="$(SourceBuiltTarBallPath)**/Private.SourceBuilt.Artifacts.*$(TarBallExtension)" />
     </ItemGroup>
 

src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/CreateSdkSymbolsLayout.cs

@@ -0,0 +1,154 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Reflection.Metadata;
+using System.Reflection.PortableExecutable;
+using Microsoft.Build.Framework;
+using Microsoft.Build.Utilities;
+
+namespace Microsoft.DotNet.Build.Tasks
+{
+    // Creates a symbols layout that matches the SDK layout
+    public class CreateSdkSymbolsLayout : Task
+    {
+        /// <summary>
+        /// Path to SDK layout.
+        /// </summary>
+        [Required]
+        public string SdkLayoutPath { get; set; }
+
+        /// <summary>
+        /// Path to all source-built symbols, flat or with folder hierarchy.
+        /// </summary>
+        [Required]
+        public string AllSymbolsPath { get; set; }
+
+        /// <summary>
+        /// Path to SDK symbols layout - will be created if it doesn't exist.
+        /// </summary>
+        [Required]
+        public string SdkSymbolsLayoutPath { get; set; }
+
+        /// <summary>
+        /// If true, fails the build if any PDBs are missing.
+        /// </summary>
+        public bool FailOnMissingPDBs { get; set; }
+
+        public override bool Execute()
+        {
+            IList<string> filesWithoutPDBs = GenerateSymbolsLayout(IndexAllSymbols());
+            if (filesWithoutPDBs.Count > 0)
+            {
+                LogErrorOrWarning(FailOnMissingPDBs, $"Did not find PDBs for the following SDK files:");
+                foreach (string file in filesWithoutPDBs)
+                {
+                    LogErrorOrWarning(FailOnMissingPDBs, file);
+                }
+           }
+
+            return !Log.HasLoggedErrors;
+        }
+
+        private void LogErrorOrWarning(bool isError, string message)
+        {
+            if (FailOnMissingPDBs)
+            {
+                Log.LogError(message);
+            }
+            else
+            {
+                Log.LogWarning(message);
+            }
+        }
+
+        private IList<string> GenerateSymbolsLayout(Hashtable allPdbGuids)
+        {
+            List<string> filesWithoutPDBs = new List<string>();
+
+            if (Directory.Exists(SdkSymbolsLayoutPath))
+            {
+                Directory.Delete(SdkSymbolsLayoutPath, true);
+            }
+
+            foreach (string file in Directory.GetFiles(SdkLayoutPath, "*", SearchOption.AllDirectories))
+            {
+                if (file.EndsWith(".dll", StringComparison.InvariantCultureIgnoreCase) &&
+                    !file.EndsWith(".resources.dll", StringComparison.InvariantCultureIgnoreCase))
+                {
+                    string guid = string.Empty;
+                    using var pdbStream = File.OpenRead(file);
+                    using var peReader = new PEReader(pdbStream);
+                    try
+                    {
+                        // Check if pdb is embedded
+                        if (peReader.ReadDebugDirectory().Any(entry => entry.Type == DebugDirectoryEntryType.EmbeddedPortablePdb))
+                        {
+                            continue;
+                        }
+
+                        var debugDirectory = peReader.ReadDebugDirectory().First(entry => entry.Type == DebugDirectoryEntryType.CodeView);
+                        var codeViewData = peReader.ReadCodeViewDebugDirectoryData(debugDirectory);
+                        guid = $"{codeViewData.Guid.ToString("N").Replace("-", string.Empty)}";
+                    }
+                    catch (Exception e) when (e is BadImageFormatException || e is InvalidOperationException)
+                    {
+                        // Ignore binaries without debug info
+                        continue;
+                    }
+
+                    if (guid != string.Empty)
+                    {
+                        if (!allPdbGuids.ContainsKey(guid))
+                        {
+                            filesWithoutPDBs.Add(file.Substring(SdkLayoutPath.Length + 1));
+                        }
+                        else
+                        {
+                            // Copy matching pdb to symbols path, preserving sdk binary's hierarchy
+                            string sourcePath = (string)allPdbGuids[guid]!;
+                            string destinationPath =
+                                file.Replace(SdkLayoutPath, SdkSymbolsLayoutPath)
+                                    .Replace(Path.GetFileName(file), Path.GetFileName(sourcePath));
+
+                            Directory.CreateDirectory(Path.GetDirectoryName(destinationPath)!);
+                            File.Copy(sourcePath, destinationPath, true);
+                        }
+                    }
+                }
+            }
+
+            return filesWithoutPDBs;
+        }
+
+        public Hashtable IndexAllSymbols()
+        {
+            Hashtable allPdbGuids = new Hashtable();
+
+            foreach (string file in Directory.GetFiles(AllSymbolsPath, "*.pdb", SearchOption.AllDirectories))
+            {
+                using var pdbFileStream = File.OpenRead(file);
+                var metadataProvider = MetadataReaderProvider.FromPortablePdbStream(pdbFileStream);
+                var metadataReader = metadataProvider.GetMetadataReader();
+                if (metadataReader.DebugMetadataHeader == null)
+                {
+                    continue;
+                }
+
+                var id = new BlobContentId(metadataReader.DebugMetadataHeader.Id);
+                string guid = $"{id.Guid:N}";
+                if (!string.IsNullOrEmpty(guid) && !allPdbGuids.ContainsKey(guid))
+                {
+                    allPdbGuids.Add(guid, file);
+                }
+            }
+
+            return allPdbGuids;
+        }
+    }
+}