Isolate package flow between repos (#18557)

As a prerequisite to enabling parallel builds of the repos in the VMR, it's necessary to isolate the package flow between repos. This ensures that repo dependencies are defined correctly. Prior to these changes, the packages outputted by repos went to the same directory (Shipping or NonShipping) and those two directories were used as the package feed inputs to dependent repos. This would be dangerous in the context of running builds in parallel across repos because any ill-defined dependencies could lead to race conditions. For example, consider the msbuild repo's dependency on System.Text.Json from the runtime repo. If runtime was not defined as a dependency of msbuild, then there's no guarantee that System.Text.Json will exist when it restores it. Based on timing, it may exist in one build and then not exist in another build.

To solve this problem, the packages output from a repos build are placed in a repo-specific package location as a sub-directory of the package location (Shipping or NonShipping). Previously, all repos would output their packages to `artifacts/packages/Release/[NonShipping|Shipping]`. With these changes, they output to `artifacts/packages/Release/[NonShipping|Shipping]/<repo-name>`. This isolates all packages on a per-repo basis. The next step is to provide access to these packages based on dependencies. Going back to msbuild's dependency on runtime, this is accomplished by modifying msbuild's nuget.config file to include feeds specific to runtime:

```xml
<add key="source-built-runtime" value="/vmr/artifacts/packages/Release/Shipping/runtime/" />
<add key="source-built-transport-runtime" value="/vmr/artifacts/packages/Release/NonShipping/runtime/" />
```

src/SourceBuild/content/eng/tools/generate-graphviz/generate-graphviz.proj

@@ -1,49 +0,0 @@
-<Project Sdk="Microsoft.Build.NoTargets">
-
-  <PropertyGroup>
-    <TargetFramework>$(NetCurrent)</TargetFramework>
-  </PropertyGroup>
-
-  <Target Name="GenerateGraphViz"
-          AfterTargets="Build">
-    <ItemGroup>
-      <RepoProject Include="$(RepoProjectsDir)*.proj"
-                   Exclude="$(RepoProjectsDir)dotnet.proj;
-                            $(RepoProjectsDir)package-source-build.proj" />
-    </ItemGroup>
-
-    <MSBuild Projects="@(RepoProject)"
-             Targets="GetRepositoryReferences">
-      <Output TaskParameter="TargetOutputs" ItemName="RepoReference" />
-    </MSBuild>
-
-    <ItemGroup>
-      <RepoLink Include="%(RepoReference.MSBuildSourceProjectFile)" SourceRepo="%(RepoReference.Identity)" />
-      <RepoLink TargetRepo="%(Filename)" />
-      <RepoLink Text="&quot;%(SourceRepo)&quot; -> &quot;%(TargetRepo)&quot;" />
-    </ItemGroup>
-
-    <PropertyGroup>
-      <GraphVizFile>$(BaseIntermediateOutputPath)graphviz.dot</GraphVizFile>
-      <GraphVizPngFile>$(BaseIntermediateOutputPath)graphviz.png</GraphVizPngFile>
-      <GraphVizContent>digraph {
-graph [ dpi = 150 ]
-@(RepoLink -> '%(Text)')
-}</GraphVizContent>
-    </PropertyGroup>
-
-    <WriteLinesToFile Lines="$(GraphVizContent)"
-                      File="$(GraphVizFile)"
-                      Overwrite="True" />
-
-    <Message Text="$(MSBuildProjectName) -> $(GraphVizFile)" Importance="High" />
-
-    <Exec Command="$([MSBuild]::NormalizePath('$(GraphVizDir)', 'dot')) $(GraphVizFile) -Tpng:cairo -o $(GraphVizPngFile)"
-          Condition="'$(GraphVizDir)' != ''" />
-
-    <Message Text="$(MSBuildProjectName) -> $(GraphVizPngFile)"
-             Importance="High"
-             Condition="'$(GraphVizDir)' != ''" />
-  </Target>
-
-</Project>

src/SourceBuild/content/repo-projects/Directory.Build.props

@@ -33,6 +33,11 @@
     <NuGetConfigFile Condition="'$(OriginalNuGetConfigFile)' != ''">$(BaseIntermediateOutputPath)$([System.IO.Path]::GetFileName('$(OriginalNuGetConfigFile)'))</NuGetConfigFile>
 
     <SourceBuiltSdksDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'source-built-sdks'))</SourceBuiltSdksDir>
+    <RepoArtifactsShippingPackagesDir Condition="'$(RepositoryName)' != 'source-build-reference-packages'">$([MSBuild]::NormalizeDirectory('$(ArtifactsShippingPackagesDir)', '$(RepositoryName)'))</RepoArtifactsShippingPackagesDir>
+    <RepoArtifactsShippingPackagesDir Condition="'$(RepositoryName)' == 'source-build-reference-packages'">$(ReferencePackagesDir)</RepoArtifactsShippingPackagesDir>
+    <RepoArtifactsNonShippingPackagesDir Condition="'$(RepositoryName)' != 'source-build-reference-packages'">$([MSBuild]::NormalizeDirectory('$(ArtifactsNonShippingPackagesDir)', '$(RepositoryName)'))</RepoArtifactsNonShippingPackagesDir>
+    <RepoArtifactsNonShippingPackagesDir Condition="'$(RepositoryName)' == 'source-build-reference-packages'">$(ReferencePackagesDir)</RepoArtifactsNonShippingPackagesDir>
+    
 
     <!-- Set the bootstrap version to the VMR's version if empty. (no bootstrap set). -->
     <ArcadeBootstrapVersion>$([MSBuild]::ValueOrDefault('$(ARCADE_BOOTSTRAP_VERSION)', '$(ArcadeSdkVersion)'))</ArcadeBootstrapVersion>

src/SourceBuild/content/repo-projects/Directory.Build.targets

@@ -32,19 +32,6 @@
     <IntermediateSymbolsRepoDir>$(IntermediateSymbolsRootDir)$(RepositoryName)</IntermediateSymbolsRepoDir>
   </PropertyGroup>
 
-  <!-- Exclude repositories that currently don't build when not building source-only. -->
-  <ItemGroup Condition="'$(DotNetBuildSourceOnly)' != 'true'">
-    <RepositoryReference Remove="roslyn" />
-    <RepositoryReference Remove="msbuild" />
-    <RepositoryReference Remove="aspnetcore" />
-    <RepositoryReference Remove="razor" />
-    <RepositoryReference Remove="format" />
-    <RepositoryReference Remove="nuget-client" />
-    <RepositoryReference Remove="fsharp" />
-    <RepositoryReference Remove="vstest" />
-    <RepositoryReference Remove="installer" />
-  </ItemGroup>
-
   <UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="AddSourceToNuGetConfig" />
   <UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="RemoveInternetSourcesFromNuGetConfig" />
   <UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="UpdateJson" />
@@ -53,6 +40,28 @@
   <UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="WritePackageVersionsProps" />
   <UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="WritePackageUsageData" />
   <UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="WriteUsageReports" />
+  
+  <!-- Wraps the transitive repo references with additional metadata -->
+  <Target Name="GetRepositoryReferenceInfo"
+          DependsOnTargets="GetTransitiveRepositoryReferences">
+    <!-- SBRP is explicitly excluded since it is output to its own special directory, $(ReferencePackagesDir). -->
+    <ItemGroup Condition="'@(TransitiveRepositoryReference)' != ''">
+      <RepositoryReferenceInfo Include="@(TransitiveRepositoryReference)"
+                               Exclude="source-build-reference-packages">
+        <ShippingSourceName>source-built-%(Identity)</ShippingSourceName>
+        <NonShippingSourceName>source-built-transport-%(Identity)</NonShippingSourceName>
+        <ShippingPackagesPath>$(ArtifactsShippingPackagesDir)/%(Identity)/</ShippingPackagesPath>
+        <NonShippingPackagesPath>$(ArtifactsNonShippingPackagesDir)/%(Identity)/</NonShippingPackagesPath>
+      </RepositoryReferenceInfo>
+    </ItemGroup>
+
+    <ItemGroup Condition="'@(RepositoryReferenceInfo)' != ''">
+      <DependentRepoSourceName Include="@(RepositoryReferenceInfo->'%(ShippingSourceName)')" />
+      <DependentRepoSourceName Include="@(RepositoryReferenceInfo->'%(NonShippingSourceName)')" />
+      <DependentRepoPackageFile Include="%(RepositoryReferenceInfo.ShippingPackagesPath)**" />
+      <DependentRepoPackageFile Include="%(RepositoryReferenceInfo.NonShippingPackagesPath)**" />
+    </ItemGroup>
+  </Target>
 
   <Target Name="CopyNuGetConfig"
           Condition="'$(NuGetConfigFile)' != ''"
@@ -69,29 +78,32 @@
   </Target>
 
   <Target Name="UpdateNuGetConfig"
-          DependsOnTargets="CopyNuGetConfig"
+          DependsOnTargets="CopyNuGetConfig;GetRepositoryReferenceInfo"
           Condition="'$(NuGetConfigFile)' != ''"
           Inputs="$(MSBuildProjectFullPath)"
           Outputs="$(BaseIntermediateOutputPath)UpdateNuGetConfig.complete">
     <PropertyGroup>
-      <!-- Feed for the shipping live built packages -->
-      <SourceBuiltShippingNuGetSourceName>source-built</SourceBuiltShippingNuGetSourceName>
-      <!-- Feed for the non-shipping live built packages -->
-      <SourceBuiltNonShippingNuGetSourceName>source-built-transport</SourceBuiltNonShippingNuGetSourceName>
-      <SourceBuildSources>$(SourceBuiltShippingNuGetSourceName);$(SourceBuiltNonShippingNuGetSourceName)</SourceBuildSources>
-
       <!-- Dev innerloop opt-in feed: /p:ExtraRestoreSourcePath=... -->
       <ExtraSourcesNuGetSourceName>ExtraSources</ExtraSourcesNuGetSourceName>
-      <SourceBuildSources Condition="'$(ExtraRestoreSourcePath)' != ''">$(SourceBuildSources);$(ExtraSourcesNuGetSourceName)</SourceBuildSources>
     </PropertyGroup>
   
     <PropertyGroup Condition="'$(DotNetBuildSourceOnly)' == 'true'">
       <PrebuiltNuGetSourceName>prebuilt</PrebuiltNuGetSourceName>
       <PreviouslySourceBuiltNuGetSourceName>previously-source-built</PreviouslySourceBuiltNuGetSourceName>
       <ReferencePackagesNuGetSourceName>reference-packages</ReferencePackagesNuGetSourceName>
-      <SourceBuildSources>$(SourceBuildSources);$(PrebuiltNuGetSourceName);$(PreviouslySourceBuiltNuGetSourceName);$(ReferencePackagesNuGetSourceName)</SourceBuildSources>
     </PropertyGroup>
 
+    <ItemGroup>
+      <_CommonBuildSources Include="@(DependentRepoSourceName)" />
+      <_CommonBuildSources Include="$(ExtraSourcesNuGetSourceName)" Condition="'$(ExtraRestoreSourcePath)' != ''" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <_BuildSources Condition="'$(DotNetBuildSourceOnly)' == 'true'"
+                     Include="$(PrebuiltNuGetSourceName);$(PreviouslySourceBuiltNuGetSourceName);$(ReferencePackagesNuGetSourceName)" />
+      <_BuildSources Include="@(_CommonBuildSources)" />
+    </ItemGroup>
+    
     <PropertyGroup Condition="'$(DotNetBuildSourceOnly)' != 'true'">
       <!-- When not building source-only, repositories lagging behind on the .NET SDK (e.g. tooling repos)
           that need to utilize a VS-aligned version for development purposes may not have the latest product
@@ -130,12 +142,14 @@
                             Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
 
     <AddSourceToNuGetConfig NuGetConfigFile="$(NuGetConfigFile)"
-                            SourceName="$(SourceBuiltShippingNuGetSourceName)"
-                            SourcePath="$(ArtifactsShippingPackagesDir)" />
+                            SourceName="%(RepositoryReferenceInfo.ShippingSourceName)"
+                            SourcePath="%(RepositoryReferenceInfo.ShippingPackagesPath)"
+                            Condition="Exists('%(RepositoryReferenceInfo.ShippingPackagesPath)')" />
 
     <AddSourceToNuGetConfig NuGetConfigFile="$(NuGetConfigFile)"
-                            SourceName="$(SourceBuiltNonShippingNuGetSourceName)"
-                            SourcePath="$(ArtifactsNonShippingPackagesDir)" />
+                            SourceName="%(RepositoryReferenceInfo.NonShippingSourceName)"
+                            SourcePath="%(RepositoryReferenceInfo.NonShippingPackagesPath)"
+                            Condition="Exists('%(RepositoryReferenceInfo.NonShippingPackagesPath)')" />
 
     <AddSourceToNuGetConfig NuGetConfigFile="$(NuGetConfigFile)"
                             SourceName="$(ExtraSourcesNuGetSourceName)"
@@ -151,12 +165,18 @@
     <UpdateNuGetConfigPackageSourcesMappings
       NuGetConfigFile="$(NuGetConfigFile)"
       BuildWithOnlineFeeds="$(DotNetBuildWithOnlineFeeds)"
-      SourceBuildSources="$(SourceBuildSources)" />
+      SourceBuildSources="@(_BuildSources)" />
 
     <MakeDir Directories="$(BaseIntermediateOutputPath)" />
     <Touch Files="$(BaseIntermediateOutputPath)UpdateNuGetConfig.complete" AlwaysCreate="true">
       <Output TaskParameter="TouchedFiles" ItemName="FileWrites" />
     </Touch>
+
+    <ItemGroup>
+      <!-- This is defined in this target to ensure the NuGet.config file is updated before it gets embedded. If this item
+           was define globally, it would immediately get embedded with the original, unmodified version of the file. -->
+      <EmbedInBinlog Include="$(NuGetConfigFile)" />
+    </ItemGroup>
   </Target>
 
   <!-- Update the SDK version in the repo's global.json file.
@@ -201,10 +221,12 @@
   <!-- Before a repository builds, set up the version property files that override the repo's defaults.
        There are 3 files generated -->
   <Target Name="CreateBuildInputProps"
+          DependsOnTargets="GetRepositoryReferenceInfo"
           Inputs="$(MSBuildProjectFullPath)"
           Outputs="$(BaseIntermediateOutputPath)CreateBuildInputProps.complete">
     <ItemGroup>
-      <_CurrentSourceBuiltPackages Include="$(ArtifactsPackagesDir)**\*.nupkg"
+      <!-- Collect all the NuGet packages from dependent repos except for the symbols -->
+      <_CurrentSourceBuiltPackages Include="@(DependentRepoPackageFile)"
                                    Condition="!$([System.String]::Copy('%(Identity)').EndsWith('.symbols.nupkg'))" />
       <_PreviouslyBuiltSourceBuiltPackages Include="$(PrebuiltSourceBuiltPackagesPath)*.nupkg" />
     </ItemGroup>
@@ -268,11 +290,12 @@
 
   <!-- SkipRepoReferences is a developer innerloop switch to skip building dependencies. -->
   <Target Name="BuildRepoReferences"
+          DependsOnTargets="GetTransitiveRepositoryReferences"
           Condition="'@(RepositoryReference)' != '' and '$(SkipRepoReferences)' != 'true'">
-    <Message Importance="High" Text="Building dependencies [@(RepositoryReference)] needed by '$(RepositoryName)'." />
+    <Message Importance="High" Text="Building dependencies [@(TransitiveRepositoryReference)] needed by '$(RepositoryName)'." />
 
     <ItemGroup>
-      <_DependentProject Include="@(RepositoryReference -> '%(Identity).proj')" />
+      <_DependentProject Include="@(TransitiveRepositoryReference -> '%(Identity).proj')" />
     </ItemGroup>
 
     <MSBuild Projects="@(_DependentProject)"
@@ -324,8 +347,6 @@
     </PropertyGroup>
 
     <MakeDir Directories="$([System.IO.Path]::GetDirectoryName('$(RepoConsoleLogFile)'));
-                          $(ArtifactsShippingPackagesDir);
-                          $(ArtifactsNonShippingPackagesDir);
                           $(ArtifactsAssetsDir)" />
 
     <!-- Create directories for extra debugging. -->
@@ -397,14 +418,12 @@
     <ItemGroup>
       <RepoManifestNonShippingPackage Include="@(RepoManifestPackage)"
                                       Condition="$([System.String]::Copy('%(Identity)').Contains('$([System.IO.Path]::DirectorySeparatorChar)NonShipping$([System.IO.Path]::DirectorySeparatorChar)'))">
-        <DestinationFolder Condition="'$(RepositoryName)' != 'source-build-reference-packages'">$(ArtifactsNonShippingPackagesDir)</DestinationFolder>
-        <DestinationFolder Condition="'$(RepositoryName)' == 'source-build-reference-packages'">$(ReferencePackagesDir)</DestinationFolder>
+        <DestinationFolder>$(RepoArtifactsNonShippingPackagesDir)</DestinationFolder>
       </RepoManifestNonShippingPackage>
 
       <RepoManifestShippingPackage Include="@(RepoManifestPackage)"
                                    Exclude="@(RepoManifestNonShippingPackage)">
-        <DestinationFolder Condition="'$(RepositoryName)' != 'source-build-reference-packages'">$(ArtifactsShippingPackagesDir)</DestinationFolder>
-        <DestinationFolder Condition="'$(RepositoryName)' == 'source-build-reference-packages'">$(ReferencePackagesDir)</DestinationFolder>
+        <DestinationFolder>$(RepoArtifactsShippingPackagesDir)</DestinationFolder>
       </RepoManifestShippingPackage>
     </ItemGroup>
 
@@ -505,8 +524,7 @@
           Inputs="$(MSBuildProjectFullPath)"
           Outputs="$(BaseIntermediateOutputPath)ExtractToolPackage.complete">
     <PropertyGroup>
-      <_ToolPackagesRoot Condition="'$(RepositoryName)' != 'source-build-reference-packages'">$(ArtifactsNonShippingPackagesDir)</_ToolPackagesRoot>
-      <_ToolPackagesRoot Condition="'$(RepositoryName)' == 'source-build-reference-packages'">$(ReferencePackagesDir)</_ToolPackagesRoot>
+      <_ToolPackagesRoot>$(RepoArtifactsNonShippingPackagesDir)</_ToolPackagesRoot>
     </PropertyGroup>
 
     <ItemGroup>
@@ -693,4 +711,78 @@
   <Target Name="GetProjectDirectory" Outputs="$(ProjectDirectory)" />
   <Target Name="GetRepositoryReferences" Outputs="@(RepositoryReference)" />
 
+  <!-- Returns the repository references of this project and all the projects this project references, recursively -->
+  <Target Name="GetTransitiveRepositoryReferences" Outputs="@(TransitiveRepositoryReference)">
+    <ItemGroup>
+      <_TransitiveRepositoryReference Include="@(RepositoryReference)" />
+    </ItemGroup>
+
+    <MSBuild Projects="@(RepositoryReference->'%(Identity).proj')"
+             Targets="GetTransitiveRepositoryReferences"
+             BuildInParallel="true">
+      <Output TaskParameter="TargetOutputs" ItemName="_DependencyTransitiveRepositoryReference" />
+    </MSBuild>
+
+    <!-- When items are transferred between projects, they get metadata indicating their source (e.g. MSBuildSourceProjectFile).
+         This is problematic because it causes introduces distinctness on the items. For example, an arcade RepositoryReference
+         that comes from both msbuild and source-build-externals will be treated as two separate items. But we only want one arcade
+         reference. To prevent this from happening, we need to remove the extra metadata so the two references can be seen as duplicates. -->
+    <ItemGroup>
+      <_TransitiveRepositoryReference Include="@(_DependencyTransitiveRepositoryReference)"
+                                      RemoveMetadata="MSBuildSourceProjectFile;MSBuildSourceTargetName;OriginalItemSpec" />
+      <TransitiveRepositoryReference Include="@(_TransitiveRepositoryReference)" KeepDuplicates="false" />
+    </ItemGroup>
+
+    <!-- Exclude repositories that currently don't build when not building source-only. -->
+    <ItemGroup Condition="'$(DotNetBuildSourceOnly)' != 'true'">
+      <TransitiveRepositoryReference Remove="roslyn" />
+      <TransitiveRepositoryReference Remove="msbuild" />
+      <TransitiveRepositoryReference Remove="aspnetcore" />
+      <TransitiveRepositoryReference Remove="razor" />
+      <TransitiveRepositoryReference Remove="format" />
+      <TransitiveRepositoryReference Remove="nuget-client" />
+      <TransitiveRepositoryReference Remove="fsharp" />
+      <TransitiveRepositoryReference Remove="vstest" />
+      <TransitiveRepositoryReference Remove="installer" />
+    </ItemGroup>
+  </Target>
+
+  <!-- Outputs a dependency graph of the repos, in YAML form, to the console. -->
+  <Target Name="ShowDependencyGraph">
+    <MSBuild Projects="$(MSBuildProjectFullPath)"
+             Targets="GetDependencyGraphString"
+             Properties="DependencyGraphIndent=">
+      <Output TaskParameter="TargetOutputs" PropertyName="DependencyGraphString" />
+    </MSBuild>
+
+    <!-- Replace the '_' placeholder with spaces -->
+    <Message Importance="High" Text="$(DependencyGraphString.Replace('_', ' ').Trim())" />
+  </Target>
+
+  <!-- Recursively walks the repo dependency graph gathering each repo's dependencies which are returned as a YAML string representation -->
+  <Target Name="GetDependencyGraphString"
+          DependsOnTargets="GetTransitiveRepositoryReferences"
+          Outputs="$(DependencyGraphString)">
+    <PropertyGroup>
+      <!-- Each step deeper in the graph builds up the indentation used for the YAML representation.
+           Use '_' as a placeholder for the space ' ' character since trailing spaces aren't handled well as property values. -->
+      <_NextIndent>$(DependencyGraphIndent)__</_NextIndent>
+    </PropertyGroup>
+    
+    <!-- For each of the current project's direct repo dependencies, recursively call the target to get each of there dependency graphs -->
+    <MSBuild Projects="@(RepositoryReference->'%(Identity).proj')"
+             Targets="GetDependencyGraphString"
+             Properties="DependencyGraphIndent=$(_NextIndent)">
+      <Output TaskParameter="TargetOutputs" ItemName="_DependencyGraphString" />
+    </MSBuild>
+
+    <!-- Append all the separate dependency graph snippets together -->
+    <PropertyGroup>
+      <!-- Ensure the item `;` separators are removed from the the target output -->
+      <_DependencyGraphString>@(_DependencyGraphString, '')</_DependencyGraphString>
+      <_LineBreak>%0a</_LineBreak>
+      <DependencyGraphString>$(DependencyGraphIndent)-_$(RepositoryName)$(_LineBreak)$(_DependencyGraphString)</DependencyGraphString>
+    </PropertyGroup>
+  </Target>
+
 </Project>

src/SourceBuild/content/repo-projects/aspnetcore.proj

@@ -22,11 +22,16 @@
 
   <ItemGroup>
     <RepositoryReference Include="arcade" />
-    <RepositoryReference Include="source-build-externals" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
-    <RepositoryReference Include="runtime" />
-    <RepositoryReference Include="msbuild" />
     <RepositoryReference Include="roslyn" />
-    <RepositoryReference Include="roslyn-analyzers" />
+    <RepositoryReference Include="runtime" />
+    <RepositoryReference Include="xdt" />
+  </ItemGroup>
+
+  <ItemGroup Condition="'$(DotNetBuildSourceOnly)' == 'true'">
+    <RepositoryReference Include="nuget-client" />
+    <RepositoryReference Include="source-build-externals" />
+    <RepositoryReference Include="source-build-reference-packages" />
+    <RepositoryReference Include="symreader" />
   </ItemGroup>
 
   <ItemGroup>

src/SourceBuild/content/repo-projects/cecil.proj

@@ -4,4 +4,8 @@
     <RepositoryReference Include="arcade" />
   </ItemGroup>
 
+  <ItemGroup Condition="'$(DotNetBuildSourceOnly)' == 'true'">
+    <RepositoryReference Include="source-build-reference-packages" />
+  </ItemGroup>
+
 </Project>

src/SourceBuild/content/repo-projects/command-line-api.proj

@@ -10,4 +10,8 @@
     <RepositoryReference Include="arcade" />
   </ItemGroup>
 
+  <ItemGroup Condition="'$(DotNetBuildSourceOnly)' == 'true'">
+    <RepositoryReference Include="source-build-reference-packages" />
+  </ItemGroup>
+
 </Project>

src/SourceBuild/content/repo-projects/deployment-tools.proj

@@ -2,7 +2,11 @@
 
   <ItemGroup>
     <RepositoryReference Include="arcade" />
-    <RepositoryReference Include="runtime" />
+  </ItemGroup>
+
+  <ItemGroup Condition="'$(DotNetBuildSourceOnly)' == 'true'">
+    <RepositoryReference Include="source-build-externals" />
+    <RepositoryReference Include="source-build-reference-packages" />
   </ItemGroup>
 
   <ItemGroup>

src/SourceBuild/content/repo-projects/diagnostics.proj

@@ -4,4 +4,8 @@
     <RepositoryReference Include="arcade" />
   </ItemGroup>
 
+  <ItemGroup Condition="'$(DotNetBuildSourceOnly)' == 'true'">
+    <RepositoryReference Include="source-build-reference-packages" />
+  </ItemGroup>
+
 </Project>

src/SourceBuild/content/repo-projects/dotnet.proj

@@ -11,42 +11,12 @@
        - If we have a repo that is not in sdk's dependency tree, we can still build it by including it here.  -->
 
   <ItemGroup>
-    <!-- Toolsets -->
-    <RepositoryReference Include="source-build-reference-packages" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
-    <RepositoryReference Include="arcade" />
-
-    <!-- Product Repos -->
-    <RepositoryReference Include="command-line-api" />
-    <RepositoryReference Include="sourcelink" />
-    <RepositoryReference Include="diagnostics" />
-    <RepositoryReference Include="emsdk" />
-    <RepositoryReference Include="cecil" />
-    <RepositoryReference Include="symreader" />
-    <RepositoryReference Include="source-build-externals" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
-    <RepositoryReference Include="runtime" />
-    <RepositoryReference Include="roslyn" />
-    <RepositoryReference Include="windowsdesktop" Condition="'$(TargetOS)' == 'windows'" />
-    <RepositoryReference Include="xdt" />
-    <RepositoryReference Include="msbuild" />
-    <RepositoryReference Include="roslyn-analyzers" />
-    <RepositoryReference Include="aspnetcore" />
-    <RepositoryReference Include="razor" />
-    <RepositoryReference Include="deployment-tools" />
-    <RepositoryReference Include="format" />
-    <RepositoryReference Include="nuget-client" />
-    <RepositoryReference Include="templating" />
-    <RepositoryReference Include="test-templates" />
-    <RepositoryReference Include="fsharp" />
-    <RepositoryReference Include="vstest" />
-    <RepositoryReference Include="sdk" />
-    <RepositoryReference Include="aspire" />
     <RepositoryReference Include="installer" />
-
-    <!-- Package source-build artifacts -->
-    <RepositoryReference Include="package-source-build" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
-
-    <!-- Testing. -->
     <RepositoryReference Include="scenario-tests" />
   </ItemGroup>
 
+  <ItemGroup Condition="'$(DotNetBuildSourceOnly)' == 'true'">
+    <RepositoryReference Include="package-source-build" />
+  </ItemGroup>
+
 </Project>

src/SourceBuild/content/repo-projects/emsdk.proj

@@ -14,4 +14,8 @@
     <RepositoryReference Include="arcade" />
   </ItemGroup>
 
+  <ItemGroup Condition="'$(DotNetBuildSourceOnly)' == 'true'">
+    <RepositoryReference Include="source-build-reference-packages" />
+  </ItemGroup>
+
 </Project>

src/SourceBuild/content/repo-projects/format.proj

@@ -2,9 +2,17 @@
 
   <ItemGroup>
     <RepositoryReference Include="arcade" />
+    <RepositoryReference Include="command-line-api" />
     <RepositoryReference Include="roslyn" />
+    <RepositoryReference Include="runtime" />
+  </ItemGroup>
+
+  <ItemGroup Condition="'$(DotNetBuildSourceOnly)' == 'true'">
+    <RepositoryReference Include="msbuild" />
     <RepositoryReference Include="roslyn-analyzers" />
-    <RepositoryReference Include="source-build-externals" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
+    <RepositoryReference Include="source-build-externals" />
+    <RepositoryReference Include="source-build-reference-packages" />
+    <RepositoryReference Include="symreader" />
   </ItemGroup>
 
 </Project>

src/SourceBuild/content/repo-projects/fsharp.proj

@@ -20,8 +20,12 @@
 
   <ItemGroup>
     <RepositoryReference Include="arcade" />
-    <RepositoryReference Include="msbuild" /> 
-    <RepositoryReference Include="source-build-externals" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
+  </ItemGroup>
+
+  <ItemGroup Condition="'$(DotNetBuildSourceOnly)' == 'true'">
+    <RepositoryReference Include="msbuild" />
+    <RepositoryReference Include="runtime" />
+    <RepositoryReference Include="source-build-reference-packages" />
   </ItemGroup>
 
 </Project>

src/SourceBuild/content/repo-projects/installer.proj

@@ -37,17 +37,26 @@
 
   <ItemGroup>
     <RepositoryReference Include="arcade" />
+    <RepositoryReference Include="aspire" />
     <RepositoryReference Include="aspnetcore" />
+    <RepositoryReference Include="command-line-api" />
+    <RepositoryReference Include="deployment-tools" />
     <RepositoryReference Include="emsdk" />
     <RepositoryReference Include="fsharp" />
     <RepositoryReference Include="msbuild" />
-    <RepositoryReference Include="source-build-externals" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
     <RepositoryReference Include="nuget-client" />
     <RepositoryReference Include="roslyn" />
     <RepositoryReference Include="runtime" />
     <RepositoryReference Include="sdk" />
+    <RepositoryReference Include="symreader" />
     <RepositoryReference Include="test-templates" />
     <RepositoryReference Include="vstest" />
+    <RepositoryReference Include="windowsdesktop" Condition="'$(TargetOS)' == 'windows'" />
+  </ItemGroup>
+
+  <ItemGroup Condition="'$(DotNetBuildSourceOnly)' == 'true'">
+    <RepositoryReference Include="source-build-externals" />
+    <RepositoryReference Include="source-build-reference-packages" />
   </ItemGroup>
 
   <!--

src/SourceBuild/content/repo-projects/msbuild.proj

@@ -9,8 +9,12 @@
 
   <ItemGroup>
     <RepositoryReference Include="arcade" />
-    <RepositoryReference Include="runtime" />
+  </ItemGroup>
+
+  <ItemGroup Condition="'$(DotNetBuildSourceOnly)' == 'true'">
     <RepositoryReference Include="roslyn" />
+    <RepositoryReference Include="runtime" />
+    <RepositoryReference Include="source-build-reference-packages" />
   </ItemGroup>
 
 </Project>

src/SourceBuild/content/repo-projects/nuget-client.proj

@@ -8,9 +8,9 @@
     <BuildScript>$([MSBuild]::NormalizePath('$(ProjectDirectory)', 'eng', 'source-build', 'build$(ShellExtension)'))</BuildScript>
   </PropertyGroup>
 
-  <ItemGroup>
-    <RepositoryReference Include="source-build-externals" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
+  <ItemGroup Condition="'$(DotNetBuildSourceOnly)' == 'true'">
     <RepositoryReference Include="msbuild" />
+    <RepositoryReference Include="source-build-externals" />
     <RepositoryReference Include="xdt" />
   </ItemGroup>
 

src/SourceBuild/content/repo-projects/package-source-build.proj

@@ -14,56 +14,51 @@
   <Target Name="CustomRepoBuild"
           AfterTargets="RepoBuild"
           DependsOnTargets="CreateBuildInputProps;DetermineSourceBuiltSdkVersion">
-    <PropertyGroup>
-      <SourceBuildTarballStagingDir>$([MSBuild]::NormalizeDirectory('$(BaseIntermediateOutputPath)', '$(SourceBuiltArtifactsTarballName)'))</SourceBuildTarballStagingDir>
-      <SourceBuildReferencePackagesDestination>$([MSBuild]::NormalizeDirectory('$(SourceBuildTarballStagingDir)', 'SourceBuildReferencePackages'))</SourceBuildReferencePackagesDestination>
-    </PropertyGroup>
-
-    <!-- Copy PVP to staging dir in order to package them together. -->
-    <Copy SourceFiles="$(CurrentSourceBuiltPackageVersionPropsPath)"
-          DestinationFiles="$(SourceBuildTarballStagingDir)PackageVersions.props" />
-
-    <ItemGroup>
-      <SourceBuildReferencePackagesNupkgFiles Include="$(ReferencePackagesDir)**/*.nupkg" />
-    </ItemGroup>
-
-    <!-- Copy reference packages from ReferencePackagesDir to staging dir. -->
-    <Copy
-      SourceFiles="@(SourceBuildReferencePackagesNupkgFiles)"
-      DestinationFiles="@(SourceBuildReferencePackagesNupkgFiles -> '$(SourceBuildReferencePackagesDestination)%(Filename)%(Extension)')"
-      Condition="'@(SourceBuildReferencePackagesNupkgFiles)' != ''" />
-
-    <ItemGroup>
-      <ArtifactsPackage Include="$(ArtifactsPackagesDir)**\*.nupkg" />
-    </ItemGroup>
-
-    <!-- Copy packages to staging dir. -->
-    <Copy SourceFiles="@(ArtifactsPackage)"
-          DestinationFolder="$(SourceBuildTarballStagingDir)"
-          Condition="'@(ArtifactsPackage)' != ''" />
-
     <PropertyGroup>
       <SourceBuiltTarballName>$(ArtifactsAssetsDir)$(SourceBuiltArtifactsTarballName).$(SourceBuiltSdkVersion).$(TargetRid)$(ArchiveExtension)</SourceBuiltTarballName>
       <SourceBuiltVersionFileName>.version</SourceBuiltVersionFileName>
     </PropertyGroup>
 
-    <!-- Content of the .version file to include in the tarball. Write to staging dir. -->
+    <!-- Content of the .version file to include in the tarball -->
     <ItemGroup>
       <VersionFileContent Include="$(RepositoryCommit);$(SourceBuiltSdkVersion)" />
     </ItemGroup>
 
     <WriteLinesToFile
-      File="$(SourceBuildTarballStagingDir)$(SourceBuiltVersionFileName)"
+      File="$(BaseIntermediateOutputPath)$(SourceBuiltVersionFileName)"
       Lines="@(VersionFileContent)"
       Overwrite="true" />
 
-    <MakeDir Directories="$([System.IO.Path]::GetDirectoryName('$(SourceBuiltTarballName)'))" />
-    <Exec Command="tar --numeric-owner -czf $(SourceBuiltTarballName) $(SourceBuiltVersionFileName) *.nupkg *.props SourceBuildReferencePackages/"
-          WorkingDirectory="$(SourceBuildTarballStagingDir)" />
+    <ItemGroup>
+      <_AllRepoFiles Include="$(ArtifactsShippingPackagesDir)/**" />
+      <_AllRepoFiles Include="$(ArtifactsNonShippingPackagesDir)/**" />
+      <_ReferencePackageFiles Include="$(ReferencePackagesDir)**" />
+    </ItemGroup>
+
+    <!-- Create a layout directory for the files that are to be included in the artifacts tarball. Since there are a large number of files, this will use symlinks
+         instead of copying files to make this execute quickly. -->
+    <PropertyGroup>
+      <_SourceBuiltLayoutDir>$([MSBuild]::NormalizeDirectory('$(BaseIntermediateOutputPath)', 'artifacts-layout'))</_SourceBuiltLayoutDir>
+      <SourceBuildReferencePackagesDestinationDirName>SourceBuildReferencePackages</SourceBuildReferencePackagesDestinationDirName>
+    </PropertyGroup>
+    
+    <Copy SourceFiles="@(_AllRepoFiles)"
+          DestinationFolder="$(_SourceBuiltLayoutDir)"
+          UseSymbolicLinksIfPossible="true" />
+    <Copy SourceFiles="$(BaseIntermediateOutputPath)$(SourceBuiltVersionFileName)"
+          DestinationFolder="$(_SourceBuiltLayoutDir)"
+          UseSymbolicLinksIfPossible="true" />
+    <Copy SourceFiles="$(CurrentSourceBuiltPackageVersionPropsPath)"
+          DestinationFiles="$(_SourceBuiltLayoutDir)PackageVersions.props"
+          UseSymbolicLinksIfPossible="true" />
+    <Copy SourceFiles="@(_ReferencePackageFiles)"
+          DestinationFolder="$(_SourceBuiltLayoutDir)$(SourceBuildReferencePackagesDestinationDirName)"
+          UseSymbolicLinksIfPossible="true" />
+
+    <Exec Command="tar --numeric-owner -czhf $(SourceBuiltTarballName) $(SourceBuiltVersionFileName) *"
+          WorkingDirectory="$(_SourceBuiltLayoutDir)" />
 
     <Message Importance="High" Text="Packaged source-built artifacts to $(SourceBuiltTarballName)" />
-
-    <RemoveDir Directories="$(SourceBuildTarballStagingDir)" />
   </Target>
 
 </Project>

src/SourceBuild/content/repo-projects/razor.proj

@@ -2,8 +2,11 @@
 
   <ItemGroup>
     <RepositoryReference Include="arcade" />
-    <RepositoryReference Include="runtime" />
+  </ItemGroup>
+
+  <ItemGroup Condition="'$(DotNetBuildSourceOnly)' == 'true'">
     <RepositoryReference Include="aspnetcore" />
+    <RepositoryReference Include="source-build-reference-packages" />
   </ItemGroup>
 
 </Project>

src/SourceBuild/content/repo-projects/roslyn-analyzers.proj

@@ -8,8 +8,12 @@
 
   <ItemGroup>
     <RepositoryReference Include="arcade" />
+  </ItemGroup>
+
+  <ItemGroup Condition="'$(DotNetBuildSourceOnly)' == 'true'">
     <RepositoryReference Include="runtime" />
-    <RepositoryReference Include="roslyn" />
+    <RepositoryReference Include="source-build-externals" />
+    <RepositoryReference Include="source-build-reference-packages" />
   </ItemGroup>
 
 </Project>

src/SourceBuild/content/repo-projects/roslyn.proj

@@ -28,9 +28,12 @@
 
   <ItemGroup>
     <RepositoryReference Include="arcade" />
-    <RepositoryReference Include="command-line-api" />
-    <RepositoryReference Include="source-build-externals" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
+  </ItemGroup>
+
+  <ItemGroup Condition="'$(DotNetBuildSourceOnly)' == 'true'">
     <RepositoryReference Include="runtime" />
+    <RepositoryReference Include="source-build-externals" />
+    <RepositoryReference Include="source-build-reference-packages" />
   </ItemGroup>
 
   <ItemGroup>

src/SourceBuild/content/repo-projects/runtime.proj

@@ -33,8 +33,13 @@
   <ItemGroup>
     <RepositoryReference Include="arcade" />
     <RepositoryReference Include="cecil" />
-    <RepositoryReference Include="symreader" />
-    <RepositoryReference Include="source-build-externals" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
+    <RepositoryReference Include="command-line-api" />
+    <RepositoryReference Include="emsdk" />
+  </ItemGroup>
+
+  <ItemGroup Condition="'$(DotNetBuildSourceOnly)' == 'true'">
+    <RepositoryReference Include="source-build-externals" />
+    <RepositoryReference Include="source-build-reference-packages" />
   </ItemGroup>
 
   <ItemGroup Condition="'$(DotNetBuildSourceOnly)' == 'true'">

src/SourceBuild/content/repo-projects/scenario-tests.proj

@@ -1,9 +1,13 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <ItemGroup>
-    <RepositoryReference Include="source-build-externals" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
-    <RepositoryReference Include="source-build-reference-packages" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
     <RepositoryReference Include="arcade" />
+    <RepositoryReference Include="command-line-api" />
+  </ItemGroup>
+
+  <ItemGroup Condition="'$(DotNetBuildSourceOnly)' == 'true'">
+    <RepositoryReference Include="source-build-externals" />
+    <RepositoryReference Include="source-build-reference-packages" />
   </ItemGroup>
 
 </Project>

src/SourceBuild/content/repo-projects/sdk.proj

@@ -14,17 +14,28 @@
 
   <ItemGroup>
     <RepositoryReference Include="arcade" />
-    <RepositoryReference Include="runtime" />
-    <RepositoryReference Include="msbuild" />
-    <RepositoryReference Include="source-build-externals" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
-    <RepositoryReference Include="nuget-client" />
-    <RepositoryReference Include="roslyn-analyzers" />
-    <RepositoryReference Include="vstest" />
-    <RepositoryReference Include="fsharp" />
-    <RepositoryReference Include="format" />
+    <RepositoryReference Include="aspnetcore" />
+    <RepositoryReference Include="command-line-api" />
     <RepositoryReference Include="deployment-tools" />
+    <RepositoryReference Include="emsdk" />
+    <RepositoryReference Include="format" />
+    <RepositoryReference Include="fsharp" />
+    <RepositoryReference Include="msbuild" />
+    <RepositoryReference Include="nuget-client" />
     <RepositoryReference Include="razor" />
-    <RepositoryReference Include="windowsdesktop" Condition="'$(TargetOS)' == 'windows'" />
+    <RepositoryReference Include="roslyn" />
+    <RepositoryReference Include="roslyn-analyzers" />
+    <RepositoryReference Include="runtime" />
+    <RepositoryReference Include="sourcelink" />
+    <RepositoryReference Include="symreader" />
+    <RepositoryReference Include="templating" />
+    <RepositoryReference Include="vstest" />
+    <RepositoryReference Include="xdt" />
+  </ItemGroup>
+
+  <ItemGroup Condition="'$(DotNetBuildSourceOnly)' == 'true'">
+    <RepositoryReference Include="source-build-externals" />
+    <RepositoryReference Include="source-build-reference-packages" />
   </ItemGroup>
 
 </Project>

src/SourceBuild/content/repo-projects/source-build-externals.proj

@@ -10,4 +10,9 @@
     <UseInnerClone>true</UseInnerClone>
   </PropertyGroup>
 
+  <ItemGroup>
+    <RepositoryReference Include="arcade" />
+    <RepositoryReference Include="source-build-reference-packages" />
+  </ItemGroup>
+
 </Project>

src/SourceBuild/content/repo-projects/sourcelink.proj

@@ -12,4 +12,8 @@
     <RepositoryReference Include="command-line-api" />
   </ItemGroup>
 
+  <ItemGroup Condition="'$(DotNetBuildSourceOnly)' == 'true'">
+    <RepositoryReference Include="source-build-reference-packages" />
+  </ItemGroup>
+
 </Project>

src/SourceBuild/content/repo-projects/symreader.proj

@@ -4,4 +4,8 @@
     <RepositoryReference Include="arcade" />
   </ItemGroup>
 
+  <ItemGroup Condition="'$(DotNetBuildSourceOnly)' == 'true'">
+    <RepositoryReference Include="source-build-reference-packages" />
+  </ItemGroup>
+
 </Project>

src/SourceBuild/content/repo-projects/templating.proj

@@ -9,8 +9,12 @@
 
   <ItemGroup>
     <RepositoryReference Include="arcade" />
-    <RepositoryReference Include="nuget-client" />
-    <RepositoryReference Include="source-build-externals" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
+    <RepositoryReference Include="runtime" />
+  </ItemGroup>
+
+  <ItemGroup Condition="'$(DotNetBuildSourceOnly)' == 'true'">
+    <RepositoryReference Include="source-build-externals" />
+    <RepositoryReference Include="source-build-reference-packages" />
   </ItemGroup>
 
 </Project>

src/SourceBuild/content/repo-projects/test-templates.proj

@@ -6,7 +6,10 @@
 
   <ItemGroup>
     <RepositoryReference Include="arcade" />
-    <RepositoryReference Include="templating" />
+  </ItemGroup>
+
+  <ItemGroup Condition="'$(DotNetBuildSourceOnly)' == 'true'">
+    <RepositoryReference Include="source-build-reference-packages" />
   </ItemGroup>
 
 </Project>

src/SourceBuild/content/repo-projects/vstest.proj

@@ -9,8 +9,12 @@
   <ItemGroup>
     <RepositoryReference Include="arcade" />
     <RepositoryReference Include="diagnostics" />
+  </ItemGroup>
+
+  <ItemGroup Condition="'$(DotNetBuildSourceOnly)' == 'true'">
     <RepositoryReference Include="runtime" />
-    <RepositoryReference Include="source-build-externals" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
+    <RepositoryReference Include="source-build-externals" />
+    <RepositoryReference Include="source-build-reference-packages" />
   </ItemGroup>
 
 </Project>