Copy symbols to intermediates (#19303)

2024-04-05 12:11:26 -07:00 · 2024-04-05 12:11:26 -07:00 · 112c8f4a45
commit 112c8f4a45
parent 5adceb727b
4 changed files with 133 additions and 38 deletions
--- a/src/SourceBuild/content/eng/build.sourcebuild.targets
+++ b/src/SourceBuild/content/eng/build.sourcebuild.targets
@ -15,13 +15,28 @@
    <MSBuild Projects="$(RepoProjectsDir)$(RootRepo).proj" Targets="WritePrebuiltUsageData;ReportPrebuiltUsage" />
  </Target>

-  <!-- After building, repackage symbols into a single tarball. -->
-  <Target Name="RepackageSymbols"
+  <!--
+    Determine symbols tarball names and discover all intermediate symbols,
+    to be used as inputs and outputs of symbols repackaging targets.
+  -->
+  <Target Name="DetermineSymbolsTargetsInputsAndOutputs"
          AfterTargets="Build"
          DependsOnTargets="DetermineSourceBuiltSdkVersion">
    <PropertyGroup>
      <UnifiedSymbolsTarball>$(ArtifactsAssetsDir)dotnet-symbols-all-$(SourceBuiltSdkVersion)-$(TargetRid)$(ArchiveExtension)</UnifiedSymbolsTarball>
+      <SdkSymbolsTarball>$(ArtifactsAssetsDir)dotnet-symbols-sdk-$(SourceBuiltSdkVersion)-$(TargetRid)$(ArchiveExtension)</SdkSymbolsTarball>
    </PropertyGroup>
+    <ItemGroup>
+      <IntermediateSymbol Include="$(IntermediateSymbolsRootDir)**/*" />
+    </ItemGroup>
+  </Target>
+ 
+  <!-- After building, repackage symbols into a single tarball. -->
+  <Target Name="RepackageSymbols"
+          AfterTargets="Build"
+          DependsOnTargets="DetermineSymbolsTargetsInputsAndOutputs"
+          Inputs="@(IntermediateSymbol)"
+          Outputs="$(UnifiedSymbolsTarball)">

    <MakeDir Directories="$([System.IO.Path]::GetDirectoryName('$(UnifiedSymbolsTarball)'))" />
    <Exec Command="tar --numeric-owner -czf $(UnifiedSymbolsTarball) *"
@ -34,10 +49,10 @@
  <UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.CreateSdkSymbolsLayout" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" />
  <Target Name="CreateSdkSymbolsTarball"
          AfterTargets="Build"
-          DependsOnTargets="RepackageSymbols">
+          DependsOnTargets="RepackageSymbols"
+          Inputs="@(IntermediateSymbol);$(SdkTarballPath)"
+          Outputs="$(SdkSymbolsTarball)">
    <PropertyGroup>
-      <SdkSymbolsTarball>$(ArtifactsAssetsDir)dotnet-symbols-sdk-$(SourceBuiltSdkVersion)-$(TargetRid)$(ArchiveExtension)</SdkSymbolsTarball>
-
      <IntermediateSdkSymbolsLayout>$(BaseIntermediateOutputPath)SdkSymbols</IntermediateSdkSymbolsLayout>
      <IntermediateSdkLayout>$(BaseIntermediateOutputPath)Sdk</IntermediateSdkLayout>
    </PropertyGroup>
@ -55,7 +70,6 @@

    <Message Importance="High" Text="Packaged sdk symbols in '$(SdkSymbolsTarball)'" />

-    <RemoveDir Directories="$(IntermediateSymbolsRootDir)" />
    <RemoveDir Directories="$(IntermediateSdkSymbolsLayout)" />
    <RemoveDir Directories="$(IntermediateSdkLayout)" />
  </Target>
--- a/src/SourceBuild/content/repo-projects/Directory.Build.props
+++ b/src/SourceBuild/content/repo-projects/Directory.Build.props
@ -39,6 +39,7 @@
    <NuGetConfigFile Condition="'$(OriginalNuGetConfigFile)' != ''">$(BaseIntermediateOutputPath)$([System.IO.Path]::GetFileName('$(OriginalNuGetConfigFile)'))</NuGetConfigFile>

    <RepoAssetManifestsDir>$([MSBuild]::NormalizeDirectory('$(AssetManifestsIntermediateDir)', '$(RepositoryName)'))</RepoAssetManifestsDir>
+    <IntermediateSymbolsRepoDir>$([MSBuild]::NormalizeDirectory('$(IntermediateSymbolsRootDir)', '$(RepositoryName)'))</IntermediateSymbolsRepoDir>

    <SourceBuiltSdksDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'source-built-sdks'))</SourceBuiltSdksDir>

@ -92,6 +93,7 @@
  </PropertyGroup>

  <PropertyGroup Condition="'$(DotNetBuildSourceOnly)' == 'true'">
+    <BuildArgs>$(BuildArgs) /p:SourceBuiltSymbolsDir=$(IntermediateSymbolsRepoDir)</BuildArgs>
    <BuildArgs>$(BuildArgs) /p:ArcadeBuildFromSource=true</BuildArgs>
    <BuildArgs>$(BuildArgs) /p:DotNetBuildSourceOnly=true</BuildArgs>
    <BuildArgs>$(BuildArgs) /p:PreviouslySourceBuiltNupkgCacheDir="$(PrebuiltSourceBuiltPackagesPath)"</BuildArgs>
--- a/src/SourceBuild/content/repo-projects/Directory.Build.targets
+++ b/src/SourceBuild/content/repo-projects/Directory.Build.targets
@ -24,8 +24,6 @@
    <PackageReportDataFile>$(PackageReportDir)prebuilt-usage.xml</PackageReportDataFile>
    <ProjectAssetsJsonArchiveFile>$(PackageReportDir)all-project-assets-json-files.zip</ProjectAssetsJsonArchiveFile>
    <ProdConManifestFile>$(PackageReportDir)prodcon-build.xml</ProdConManifestFile>
-
-    <IntermediateSymbolsRepoDir>$(IntermediateSymbolsRootDir)$(RepositoryName)</IntermediateSymbolsRepoDir>
  </PropertyGroup>

  <!-- Returns the repository references of this project and all the projects this project references, recursively -->
@ -538,34 +536,6 @@
    </Touch>
  </Target>

-  <Target Name="DiscoverSymbolsArchiveFile"
-          Condition="'$(DotNetBuildSourceOnly)' == 'true' and
-                     '$(IsUtilityProject)' != 'true'">
-    <PropertyGroup>
-      <RepoAssetsSymbolsArchive>$(ArtifactsAssetsSymbolsDir)Symbols.$(RepositoryName)$(ArchiveExtension)</RepoAssetsSymbolsArchive>
-    </PropertyGroup>
-
-    <ItemGroup>
-      <RepoSymbolsArchive Include="$(RepoAssetsSymbolsArchive)" Condition="Exists('$(RepoAssetsSymbolsArchive)')" />
-    </ItemGroup>
-  </Target>
-
-  <Target Name="ExtractSymbolsArchiveFile"
-          DependsOnTargets="RepoBuild;DiscoverSymbolsArchiveFile"
-          Condition="'$(DotNetBuildSourceOnly)' == 'true' and
-                     '$(IsUtilityProject)' != 'true' and
-                     '@(RepoSymbolsArchive)' != ''"
-          Inputs="@(RepoSymbolsArchive)"
-          Outputs="$(IntermediateSymbolsRepoDir)">
-
-    <Error Text="There must only be a single repository symbols archive file. Found: @(RepoSymbolsArchive)"
-           Condition="@(RepoSymbolsArchive->Count()) &gt; 1" />
-
-    <!-- Extract repo symbols tarball -->
-    <MakeDir Directories="$(IntermediateSymbolsRepoDir)" />
-    <Exec Command="tar -xzf %(RepoSymbolsArchive.Identity) -C $(IntermediateSymbolsRepoDir)" />
-  </Target>
-
  <Target Name="Build"
          DependsOnTargets="
    BuildRepoReferences;
@ -573,8 +543,6 @@
    LogRepoArtifacts;
    CopyInnerBuildRestoredPackages;
    ExtractToolPackage;
-    DiscoverSymbolsArchiveFile;
-    ExtractSymbolsArchiveFile;
    CleanupRepo" />

  <UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.UsageReport.WritePackageUsageData" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" />
--- a/src/SourceBuild/patches/arcade/0001-Copy-repo-symbols-to-intermediates.patch
+++ b/src/SourceBuild/patches/arcade/0001-Copy-repo-symbols-to-intermediates.patch
@ -0,0 +1,111 @@
+From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
+From: Nikola Milosavljevic <nikolam@microsoft.com>
+Date: Wed, 3 Apr 2024 21:23:27 +0000
+Subject: [PATCH] Copy repo symbols to intermediates
+
+Backport: https://github.com/dotnet/arcade/pull/14681
+---
+ .../tools/Publish.proj                        |  8 -----
+ .../tools/SourceBuild/AfterSourceBuild.proj   |  2 +-
+ .../SourceBuild/SourceBuildArcade.targets     | 35 +++++++++++++++----
+ 3 files changed, 29 insertions(+), 16 deletions(-)
+
+diff --git a/src/Microsoft.DotNet.Arcade.Sdk/tools/Publish.proj b/src/Microsoft.DotNet.Arcade.Sdk/tools/Publish.proj
+index 43fadc322..bec1a4963 100644
+--- a/src/Microsoft.DotNet.Arcade.Sdk/tools/Publish.proj
+++ b/src/Microsoft.DotNet.Arcade.Sdk/tools/Publish.proj
+@@ -118,14 +118,6 @@
+       <ItemsToSignPostBuild Include="@(SymbolPackagesToGenerate->'%(Filename)%(Extension)')" Condition="'$(PostBuildSign)' == 'true'" />
+     </ItemGroup>
+ 
+-    <!-- Include Symbols.<repo>.tar.gz, if running in inner source-only build -->
+-    <ItemGroup Condition="'$(DotNetBuildSourceOnly)' == 'true' and '$(DotNetBuildInnerRepo)' == 'true'">
+-      <UnifiedSymbolsPackage Include="$(ArtifactsNonShippingPackagesDir)Symbols.*.tar.gz" />
+-      <Artifact Include="@(UnifiedSymbolsPackage)"
+-                RelativeBlobPath="Symbols/%(Filename)%(Extension)"
+-                IsShipping="false" />
+-    </ItemGroup>
+-
+     <!--
+       If a symbol package doesn't exist yet we assume that the regular package contains Portable PDBs.
+       Such packages can act as symbol packages since they have the same structure.
+diff --git a/src/Microsoft.DotNet.Arcade.Sdk/tools/SourceBuild/AfterSourceBuild.proj b/src/Microsoft.DotNet.Arcade.Sdk/tools/SourceBuild/AfterSourceBuild.proj
+index c2ba0644b..6a1fd449d 100644
+--- a/src/Microsoft.DotNet.Arcade.Sdk/tools/SourceBuild/AfterSourceBuild.proj
+++ b/src/Microsoft.DotNet.Arcade.Sdk/tools/SourceBuild/AfterSourceBuild.proj
+@@ -14,7 +14,7 @@
+   <Target Name="AfterSourceBuildInnerBuild"
+           Condition="('$(ArcadeInnerBuildFromSource)' == 'true' or '$(DotNetBuildInnerRepo)' == 'true') and
+                      '$(DotNetBuildOrchestrator)' == 'true'"
+-          DependsOnTargets="CreateRepoSymbolsArchive;ClearPreviousSBRP" />
+          DependsOnTargets="CopyRepoSymbolsToIntermediates;ClearPreviousSBRP" />
+ 
+   <Target Name="ClearPreviousSBRP"
+           Condition="'$(GitHubRepositoryName)' == 'source-build-reference-packages'">
+diff --git a/src/Microsoft.DotNet.Arcade.Sdk/tools/SourceBuild/SourceBuildArcade.targets b/src/Microsoft.DotNet.Arcade.Sdk/tools/SourceBuild/SourceBuildArcade.targets
+index 41817c3a9..f06d33095 100644
+--- a/src/Microsoft.DotNet.Arcade.Sdk/tools/SourceBuild/SourceBuildArcade.targets
+++ b/src/Microsoft.DotNet.Arcade.Sdk/tools/SourceBuild/SourceBuildArcade.targets
+@@ -159,20 +159,20 @@
+ 
+     Conditioning out for Windows as the tar execution below doesn't work cross-plat.
+   -->
+
+   <Target Name="CreateRepoSymbolsArchive"
+          Condition="'$(OS)' != 'Windows_NT' and ('$(DotNetBuildFromSourceFlavor)' == 'Product' or '$(DotNetBuildOrchestrator)' == 'true' or '$(SupplementalIntermediateNupkgCategory)' == '$(SymbolsIntermediateNupkgCategory)')"
+          DependsOnTargets="$(CreateRepoSymbolsArchiveDependsOn);
+                            DiscoverRepoSymbols;
+                            PackageRepoSymbols"/>
+
+  <Target Name="DiscoverRepoSymbols"
+           Condition="'$(OS)' != 'Windows_NT' and ('$(DotNetBuildFromSourceFlavor)' == 'Product' or '$(DotNetBuildOrchestrator)' == 'true' or '$(SupplementalIntermediateNupkgCategory)' == '$(SymbolsIntermediateNupkgCategory)')"
+           DependsOnTargets="$(CreateRepoSymbolsArchiveDependsOn)">
+     <PropertyGroup>
+       <SymbolsRoot>$(CurrentRepoSourceBuildArtifactsDir)</SymbolsRoot>
+       <!-- Fall back to repo root for source-build-externals or repos that don't have the regular SymbolsRoot as defined above -->
+       <SymbolsRoot Condition="!Exists('$(SymbolsRoot)') or '$(GitHubRepositoryName)' == 'source-build-externals'">$(RepoRoot)</SymbolsRoot>
+-      <SymbolsArchiveLocation>$(CurrentRepoSourceBuildArtifactsNonShippingPackagesDir)</SymbolsArchiveLocation>
+-      <SymbolsArchiveLocation Condition="'$(GitHubRepositoryName)' == 'nuget-client' and '$(PackageOutputPath)' != ''">$([MSBuild]::EnsureTrailingSlash('$(PackageOutputPath)'))</SymbolsArchiveLocation>
+-      <SymbolsList>$([MSBuild]::NormalizePath('$(SymbolsArchiveLocation)', 'symbols.lst'))</SymbolsList>
+-      <SymbolsArchivePrefix>Symbols.</SymbolsArchivePrefix>
+-      <!-- $(Version) and $(TargetRid) are only available when target is executed as part of intermediate package creation. -->
+-      <SymbolsArchiveSuffix Condition="'$(CreateIntermediatePackage)' == 'true'">.$(Version).$(TargetRid)</SymbolsArchiveSuffix>
+-      <SymbolsArchiveFile>$(SymbolsArchiveLocation)$(SymbolsArchivePrefix)$(GitHubRepositoryName)$(SymbolsArchiveSuffix)$(ArchiveExtension)</SymbolsArchiveFile>
+     </PropertyGroup>
+ 
+     <ItemGroup>
+@@ -181,6 +181,20 @@
+         <RelativePath>$([MSBuild]::MakeRelative($(SymbolsRoot), %(FullPath)))</RelativePath>
+       </AbsoluteSymbolPath>
+     </ItemGroup>
+  </Target>
+
+  <Target Name="PackageRepoSymbols"
+          Condition="'$(OS)' != 'Windows_NT' and ('$(DotNetBuildFromSourceFlavor)' == 'Product' or '$(DotNetBuildOrchestrator)' == 'true' or '$(SupplementalIntermediateNupkgCategory)' == '$(SymbolsIntermediateNupkgCategory)')"
+          DependsOnTargets="$(CreateRepoSymbolsArchiveDependsOn);DiscoverRepoSymbols">
+    <PropertyGroup>
+      <SymbolsArchiveLocation>$(CurrentRepoSourceBuildArtifactsNonShippingPackagesDir)</SymbolsArchiveLocation>
+      <SymbolsArchiveLocation Condition="'$(GitHubRepositoryName)' == 'nuget-client' and '$(PackageOutputPath)' != ''">$([MSBuild]::EnsureTrailingSlash('$(PackageOutputPath)'))</SymbolsArchiveLocation>
+      <SymbolsList>$([MSBuild]::NormalizePath('$(SymbolsArchiveLocation)', 'symbols.lst'))</SymbolsList>
+      <SymbolsArchivePrefix>Symbols.</SymbolsArchivePrefix>
+      <!-- $(Version) and $(TargetRid) are only available when target is executed as part of intermediate package creation. -->
+      <SymbolsArchiveSuffix Condition="'$(CreateIntermediatePackage)' == 'true'">.$(Version).$(TargetRid)</SymbolsArchiveSuffix>
+      <SymbolsArchiveFile>$(SymbolsArchiveLocation)$(SymbolsArchivePrefix)$(GitHubRepositoryName)$(SymbolsArchiveSuffix)$(ArchiveExtension)</SymbolsArchiveFile>
+    </PropertyGroup>
+ 
+     <WriteLinesToFile
+       File="$(SymbolsList)"
+@@ -203,6 +217,13 @@
+     <Delete Files="$(SymbolsList)" Condition="Exists($(SymbolsList))" />
+   </Target>
+ 
+  <Target Name="CopyRepoSymbolsToIntermediates"
+          DependsOnTargets="DiscoverRepoSymbols">
+    <MakeDir Directories="$(SourceBuiltSymbolsDir)" />
+    <Copy
+      SourceFiles="@(AbsoluteSymbolPath)"
+      DestinationFolder="$(SourceBuiltSymbolsDir)%(RecursiveDir)" />
+  </Target>
+   <!--
+     This target can be removed once we enable standard repo assets manifests and SB orchestrator
+     starts using it - https://github.com/dotnet/source-build/issues/3970