Put artifacts in well-known locations (#18591)

.devcontainer/vmr-source-build/README.md

@@ -13,7 +13,7 @@ This Codespace can help you debug the source build of .NET. In case you have run
 `/workspaces/dotnet` with the PR changes pulled into it. You can then attempt to source-build
 the VMR which is what the VMR leg in the installer PR build doing. This build takes about 45
 minutes and, after completion, produces an archived .NET SDK located in
-`/workspaces/dotnet/artifacts/x64/Release`.
+`/workspaces/dotnet/artifacts/assets/Release`.
 
 ## Build the SDK
 

eng/pipelines/templates/jobs/vmr-build.yml

@@ -139,7 +139,6 @@ jobs:
           vmrPath: $(vmrPath)
           vmrBranch: ${{ parameters.vmrBranch }}
           targetRef: $(Build.SourceVersion) # Synchronize the current installer commit
-          architecture: ${{ parameters.architecture }}
 
   - ${{ if parameters.buildFromArchive }}:
     - script: |
@@ -348,7 +347,6 @@ jobs:
 
         CopyWithRelativeFolders "artifacts/"                 $targetFolder "*.binlog"
         CopyWithRelativeFolders "artifacts/"                 $targetFolder "*.log"
-        CopyWithRelativeFolders "artifacts/prebuilt-report/" $targetFolder
         CopyWithRelativeFolders "src/"                       $targetFolder "*.binlog"
         CopyWithRelativeFolders "src/"                       $targetFolder "*.log"
         CopyWithRelativeFolders "test/"                      $targetFolder "*.binlog"
@@ -370,7 +368,9 @@ jobs:
         cd "$(sourcesPath)"
         find artifacts/ -type f -name "*.binlog" -exec rsync -R {} -t ${targetFolder} \;
         find artifacts/ -type f -name "*.log" -exec rsync -R {} -t ${targetFolder} \;
-        find artifacts/prebuilt-report/ -exec rsync -R {} -t ${targetFolder} \;
+        if [[ "${{ parameters.buildSourceOnly }}" == "True" ]]; then
+          find artifacts/prebuilt-report/ -exec rsync -R {} -t ${targetFolder} \;
+        fi
         find src/ -type f -name "*.binlog" -exec rsync -R {} -t ${targetFolder} \;
         find src/ -type f -name "*.log" -exec rsync -R {} -t ${targetFolder} \;
         find test/ -type f -name "*.binlog" -exec rsync -R {} -t ${targetFolder} \;
@@ -400,8 +400,14 @@ jobs:
         publishRunAttachments: true
         testRunTitle: SourceBuild_SmokeTests_$(Agent.JobName)
 
-  - publish: '$(sourcesPath)/artifacts/${{ parameters.architecture }}/Release/'
+  - publish: '$(sourcesPath)/artifacts/assets/Release/'
     artifact: $(Agent.JobName)_Artifacts
     displayName: Publish Artifacts
     condition: succeededOrFailed()
     continueOnError: true
+
+  # When building from source, the Private.SourceBuilt.Artifacts archive already contains the nuget packages
+  - ${{ if ne(parameters.buildSourceOnly, 'true') }}:
+    - publish: '$(sourcesPath)/artifacts/packages/Release/'
+      artifact: $(Agent.JobName)_Packages
+      displayName: Publish Packages

eng/pipelines/templates/steps/vmr-pull-updates.yml

@@ -17,11 +17,6 @@ parameters:
   type: string
   default: $(Agent.BuildDirectory)/vmr
 
-- name: architecture
-  displayName: Architecture used for build artifacts
-  type: string
-  default: 'x64'
-
 steps:
 - checkout: self
   displayName: Clone dotnet/installer
@@ -41,16 +36,6 @@ steps:
   displayName: Prepare branch ${{ parameters.vmrBranch }}
   workingDirectory: ${{ parameters.vmrPath }}
 
-- script: |
-    mkdir -p ${{ parameters.vmrPath }}/artifacts/${{ parameters.architecture }}/Release
-  displayName: Create artifacts folder (Unix)
-  condition: ne(variables['Agent.OS'], 'Windows_NT')
-
-- powershell: |
-    New-Item -ItemType Directory -Path ${{ parameters.vmrPath }}/artifacts/${{ parameters.architecture }}/Release
-  displayName: Create artifacts folder (Windows)
-  condition: eq(variables['Agent.OS'], 'Windows_NT')
-
 - script: |
     git config --global user.name "dotnet-maestro[bot]"
     git config --global user.email "dotnet-maestro[bot]@users.noreply.github.com"

src/SourceBuild/content/.devcontainer/README.md

@@ -10,7 +10,7 @@
 
 This Codespace can help you debug the source build of .NET. This build takes about
 45 minutes and, after completion, produces an archived .NET SDK located in
-`/workspaces/dotnet/artifacts/x64/Release`. In case you selected the `prebuilt-sdk`
+`/workspaces/dotnet/artifacts/assets/Release`. In case you selected the `prebuilt-sdk`
 Codespace, the built-from-source SDK will already be there.
 
 ## Build the SDK

src/SourceBuild/content/Directory.Build.props

@@ -93,6 +93,9 @@
     <ArtifactsObjDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'obj'))</ArtifactsObjDir>
     <ArtifactsBinDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'bin'))</ArtifactsBinDir>
     <ArtifactsLogDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'log', '$(Configuration)'))</ArtifactsLogDir>
+    <ArtifactsPackagesDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'packages', '$(Configuration)'))</ArtifactsPackagesDir>
+    <ArtifactsNonShippingPackagesDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsPackagesDir)', 'NonShipping'))</ArtifactsNonShippingPackagesDir>
+    <ArtifactsShippingPackagesDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsPackagesDir)', 'Shipping'))</ArtifactsShippingPackagesDir>
     <RepositoryEngineeringDir>$([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'eng'))</RepositoryEngineeringDir>
 
     <!-- ProjectLayout.props -->
@@ -165,14 +168,8 @@
     <ArcadeBootstrapPackageDir Condition="'$(DotNetBuildSourceOnly)' != 'true'">$(NuGetPackageRoot)</ArcadeBootstrapPackageDir>
 
     <VSMSBuildSdkResolversDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'toolset', 'VSSdkResolvers'))</VSMSBuildSdkResolversDir>
-
+    <IntermediateSymbolsRootDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsObjDir)', 'Symbols'))</IntermediateSymbolsRootDir>
-    <!-- Shared output and intermediate output path folders that are architecture and configuration specific. -->
+    <ArtifactsAssetsDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'assets', '$(Configuration)'))</ArtifactsAssetsDir>
-    <SharedOutputPath>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', '$(TargetArchitecture)', '$(Configuration)'))</SharedOutputPath>
-    <SharedIntermediateOutputPath>$([MSBuild]::NormalizeDirectory('$(ArtifactsObjDir)', '$(TargetArchitecture)', '$(Configuration)'))</SharedIntermediateOutputPath>
-
-    <SourceBuiltBlobFeedDir>$([MSBuild]::NormalizeDirectory('$(SharedIntermediateOutputPath)', 'blob-feed'))</SourceBuiltBlobFeedDir>
-    <SourceBuiltPackagesPath>$([MSBuild]::NormalizeDirectory('$(SourceBuiltBlobFeedDir)', 'packages'))</SourceBuiltPackagesPath>
-    <SourceBuiltAssetsDir>$([MSBuild]::NormalizeDirectory('$(SourceBuiltBlobFeedDir)', 'assets'))</SourceBuiltAssetsDir>
 
     <PrebuiltPackagesPath>$([MSBuild]::NormalizeDirectory('$(PrereqsPackagesDir)', 'prebuilt'))</PrebuiltPackagesPath>
     <PreviouslyRestoredPackagesPath>$([MSBuild]::NormalizeDirectory('$(PrereqsPackagesDir)', 'previouslyRestored'))</PreviouslyRestoredPackagesPath>

src/SourceBuild/content/Directory.Build.targets

@@ -7,7 +7,7 @@
       <SdkFilenamePrefix>dotnet-sdk-</SdkFilenamePrefix>
     </PropertyGroup>
     <ItemGroup>
-      <SdkTarballItem Include="$(SourceBuiltAssetsDir)$(SdkFilenamePrefix)*$(ArchiveExtension)" />
+      <SdkTarballItem Include="$(ArtifactsAssetsDir)$(SdkFilenamePrefix)*$(ArchiveExtension)" />
     </ItemGroup>
 
     <!--

src/SourceBuild/content/README.md

@@ -109,21 +109,21 @@ In case you don't want to / cannot prepare your environment per the requirements
         ./build.sh -sb --clean-while-building
         ```
 
-    The resulting SDK is placed at `artifacts/[your-arch]/Release/dotnet-sdk-9.0.100-[your-RID].tar.gz` (for Unix) or `artifacts/[your-arch]/Release/dotnet-sdk-9.0.100-[your-RID].zip` (for Windows).
+    The resulting SDK is placed at `artifacts/assets/Release/dotnet-sdk-9.0.100-[your-RID].tar.gz` (for Unix) or `artifacts/assets/Release/dotnet-sdk-9.0.100-[your-RID].zip` (for Windows).
 
 4. *(Optional)* **Unpack and install the .NET SDK**
     
     For Unix:
     ```bash
     mkdir -p $HOME/dotnet
-    tar zxf artifacts/[your-arch]/Release/dotnet-sdk-9.0.100-[your-RID].tar.gz -C $HOME/dotnet
+    tar zxf artifacts/assets/Release/dotnet-sdk-9.0.100-[your-RID].tar.gz -C $HOME/dotnet
     ln -s $HOME/dotnet/dotnet /usr/bin/dotnet
     ```
 
     For Windows:
     ```cmd
     mkdir %userprofile%\dotnet
-    tar -xf artifacts/<arch>/Release/dotnet-sdk-9.0.100-[your RID].zip -C %userprofile%\dotnet
+    tar -xf artifacts/assets/Release/dotnet-sdk-9.0.100-[your RID].zip -C %userprofile%\dotnet
     set "PATH=%userprofile%\dotnet;%PATH%"
     ```
     
@@ -152,7 +152,7 @@ git clone https://github.com/dotnet/dotnet .
 ./prep.sh && ./build.sh -sb --clean-while-building
 
 mkdir -p $HOME/.dotnet
-tar -zxf artifacts/x64/Release/dotnet-sdk-9.0.100-centos.8-x64.tar.gz -C $HOME/.dotnet
+tar -zxf artifacts/assets/Release/dotnet-sdk-9.0.100-centos.8-x64.tar.gz -C $HOME/.dotnet
 ln -s $HOME/.dotnet/dotnet /usr/bin/dotnet
 ```
 

src/SourceBuild/content/build.proj

@@ -19,21 +19,6 @@
              StopOnFirstFailure="true" />
   </Target>
 
-  <!-- Copies the output assets of the builds to the output path. -->
-  <Target Name="CopyBinariesToBinFolder"
-          AfterTargets="Build">
-    <ItemGroup>
-      <BinaryToCopy Include="$(SourceBuiltAssetsDir)*.*"
-                    Exclude="$(SourceBuiltAssetsDir)*.nupkg;
-                             $(SourceBuiltAssetsDir)*.requires_nupkg_signing" />
-    </ItemGroup>
-
-    <Copy SourceFiles="@(BinaryToCopy)"
-          DestinationFolder="$(SharedOutputPath)"
-          SkipUnchangedFiles="true"
-          Condition="'@(BinaryToCopy)'!=''" />
-  </Target>
-
   <Import Project="$(RepositoryEngineeringDir)build.sourcebuild.targets" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
 
 </Project>

src/SourceBuild/content/eng/build.sourcebuild.targets

@@ -15,45 +15,17 @@
     <MSBuild Projects="$(RepoProjectsDir)$(RootRepo).proj" Targets="WritePrebuiltUsageData;ReportPrebuiltUsage" />
   </Target>
 
-  <Target Name="DiscoverSymbolsTarballs"
-          AfterTargets="Build">
-    <ItemGroup>
-      <SymbolsTarball Include="$(SharedOutputPath)Symbols.*$(ArchiveExtension)" />
-    </ItemGroup>
-  </Target>
-
-  <Target Name="ExtractSymbolsTarballs"
-          AfterTargets="Build"
-          DependsOnTargets="DiscoverSymbolsTarballs"
-          Outputs="%(SymbolsTarball.Identity)">
-
-    <PropertyGroup>
-      <Filename>$([System.IO.Path]::GetFileName('%(SymbolsTarball.Identity)'))</Filename>
-      <RepositoryName>$(Filename.Split('.')[1])</RepositoryName>
-      <UnifiedSymbolsLayout>$(ArtifactsTmpDir)Symbols</UnifiedSymbolsLayout>
-      <DestinationFolder>$(UnifiedSymbolsLayout)/$(RepositoryName)</DestinationFolder>
-    </PropertyGroup>
-
-    <MakeDir Directories="$(DestinationFolder)" />
-    <Exec Command="tar -xzf %(SymbolsTarball.Identity) -C $(DestinationFolder)"
-          WorkingDirectory="$(SymbolsRoot)" />
-
-    <Delete Files="%(SymbolsTarball.Identity)" />
-  </Target>
-
   <!-- After building, repackage symbols into a single tarball. -->
   <Target Name="RepackageSymbols"
           AfterTargets="Build"
-          DependsOnTargets="
+          DependsOnTargets="DetermineSourceBuiltSdkVersion">
-            DetermineSourceBuiltSdkVersion;
-            DiscoverSymbolsTarballs;
-            ExtractSymbolsTarballs">
     <PropertyGroup>
-      <UnifiedSymbolsTarball>$(SharedOutputPath)dotnet-symbols-all-$(SourceBuiltSdkVersion)-$(TargetRid)$(ArchiveExtension)</UnifiedSymbolsTarball>
+      <UnifiedSymbolsTarball>$(ArtifactsAssetsDir)dotnet-symbols-all-$(SourceBuiltSdkVersion)-$(TargetRid)$(ArchiveExtension)</UnifiedSymbolsTarball>
     </PropertyGroup>
 
+    <MakeDir Directories="$([System.IO.Path]::GetDirectoryName('$(UnifiedSymbolsTarball)'))" />
     <Exec Command="tar --numeric-owner -czf $(UnifiedSymbolsTarball) *"
-          WorkingDirectory="$(UnifiedSymbolsLayout)" />
+          WorkingDirectory="$(IntermediateSymbolsRootDir)" />
 
     <Message Importance="High" Text="Packaged all symbols in '$(UnifiedSymbolsTarball)'" />
   </Target>
@@ -64,33 +36,33 @@
           AfterTargets="Build"
           DependsOnTargets="RepackageSymbols">
     <ItemGroup>
-      <SdkTarballItem Include="$(SharedOutputPath)dotnet-sdk-*$(ArchiveExtension)" />
+      <SdkTarballItem Include="$(ArtifactsAssetsDir)dotnet-sdk-*$(ArchiveExtension)" />
     </ItemGroup>
 
     <PropertyGroup>
-      <SdkSymbolsLayout>$(ArtifactsTmpDir)SdkSymbols</SdkSymbolsLayout>
+      <SdkSymbolsTarball>$(ArtifactsAssetsDir)dotnet-symbols-sdk-$(SourceBuiltSdkVersion)-$(TargetRid)$(ArchiveExtension)</SdkSymbolsTarball>
-      <SdkSymbolsTarball>$(SharedOutputPath)dotnet-symbols-sdk-$(SourceBuiltSdkVersion)-$(TargetRid)$(ArchiveExtension)</SdkSymbolsTarball>
+
-      <SdkLayout>$(ArtifactsTmpDir)Sdk</SdkLayout>
+      <IntermediateSdkSymbolsLayout>$(BaseIntermediateOutputPath)SdkSymbols</IntermediateSdkSymbolsLayout>
+      <IntermediateSdkLayout>$(BaseIntermediateOutputPath)Sdk</IntermediateSdkLayout>
       <SdkTarball>%(SdkTarballItem.Identity)</SdkTarball>
     </PropertyGroup>
 
-    <MakeDir Directories="$(SdkLayout)" />
+    <MakeDir Directories="$(IntermediateSdkLayout)" />
-    <Exec Command="tar -xzf $(SdkTarball) -C $(SdkLayout)"
+    <Exec Command="tar -xzf $(SdkTarball) -C $(IntermediateSdkLayout)" />
-          WorkingDirectory="$(SharedOutputPath)" />
 
-    <CreateSdkSymbolsLayout SdkLayoutPath="$(SdkLayout)"
+    <CreateSdkSymbolsLayout SdkLayoutPath="$(IntermediateSdkLayout)"
-                            AllSymbolsPath="$(UnifiedSymbolsLayout)"
+                            AllSymbolsPath="$(IntermediateSymbolsRootDir)"
-                            SdkSymbolsLayoutPath="$(SdkSymbolsLayout)"
+                            SdkSymbolsLayoutPath="$(IntermediateSdkSymbolsLayout)"
                             FailOnMissingPDBs="false" />
 
     <Exec Command="tar --numeric-owner -czf $(SdkSymbolsTarball) *"
-          WorkingDirectory="$(SdkSymbolsLayout)" />
+          WorkingDirectory="$(IntermediateSdkSymbolsLayout)" />
 
     <Message Importance="High" Text="Packaged sdk symbols in '$(SdkSymbolsTarball)'" />
 
-    <RemoveDir Directories="$(UnifiedSymbolsLayout)" />
+    <RemoveDir Directories="$(IntermediateSymbolsRootDir)" />
-    <RemoveDir Directories="$(SdkSymbolsLayout)" />
+    <RemoveDir Directories="$(IntermediateSdkSymbolsLayout)" />
-    <RemoveDir Directories="$(SdkLayout)" />
+    <RemoveDir Directories="$(IntermediateSdkLayout)" />
   </Target>
 
   <!--
@@ -111,21 +83,22 @@
           Inputs="$(MSBuildProjectFullPath)"
           Outputs="$(BaseIntermediateOutputPath)ReportPoisonUsage.complete" >
     <ItemGroup>
-      <FinalCliTarball Include="$(SharedOutputPath)**/*$(ArchiveExtension)" />
+      <!-- Exclude the Private.SourceBuilt.Artifacts archive from poison usage scan. -->
+      <PoisonFileToCheck Include="$(ArtifactsAssetsDir)*$(ArchiveExtension)" />
+      <PoisonFileToCheck Remove="$(ArtifactsAssetsDir)$(SourceBuiltArtifactsTarballName)*" />
+      <!-- Include shipping nuget packages. -->
+      <PoisonFileToCheck Include="$(ArtifactsShippingPackagesDir)*.nupkg" />
+      <!-- Add and mark SBRP packages to validate that they have the correct poison attribute. -->
+      <PoisonFileToCheck Include="$(ReferencePackagesDir)**\*.nupkg" IsSourceBuildReferencePackage="true" />
     </ItemGroup>
 
-    <Message Importance="High" Text="[$([System.DateTime]::Now.ToString('HH:mm:ss.ff'))] Checking @(FinalCliTarball) for poisoned files." />
+    <Message Importance="High" Text="[$([System.DateTime]::Now.ToString('HH:mm:ss.ff'))] Checking @(PoisonFileToCheck) for poisoned files." />
 
-    <ItemGroup>
+    <CheckForPoison FilesToCheck="@(PoisonFileToCheck)"
-      <NonShippingPackagesList Include="$(PackageListsDir)**/$(NonShippingPackagesListPrefix)*" />
-    </ItemGroup>
-
-    <CheckForPoison FilesToCheck="@(FinalCliTarball)"
                     ProjectDirPath="$(RepoRoot)"
                     HashCatalogFilePath="$(PoisonReportDataFile)"
                     MarkerFileName="$(PoisonMarkerFile)"
-                    PoisonReportOutputFilePath="$(PoisonUsageReportFile)"
+                    PoisonReportOutputFilePath="$(PoisonUsageReportFile)" />
-                    NonShippingPackagesListFiles="@(NonShippingPackagesList)" />
 
     <Message Importance="High" Text="[$([System.DateTime]::Now.ToString('HH:mm:ss.ff'))] Done checking for poison." />
 
@@ -155,8 +128,8 @@
 
   <Target Name="RunSmokeTest">
     <ItemGroup>
-      <SdkTarballItem Include="$(SharedOutputPath)**/dotnet-sdk*$(ArchiveExtension)" />
+      <SdkTarballItem Include="$(ArtifactsAssetsDir)dotnet-sdk*$(ArchiveExtension)" />
-      <SourceBuiltArtifactsItem Include="$(SharedOutputPath)**/$(SourceBuiltArtifactsTarballName).*$(ArchiveExtension)" />
+      <SourceBuiltArtifactsItem Include="$(ArtifactsAssetsDir)$(SourceBuiltArtifactsTarballName).*$(ArchiveExtension)" />
     </ItemGroup>
 
     <PropertyGroup>
@@ -203,17 +176,18 @@
           DependsOnTargets="DetermineSourceBuiltSdkVersion"
           Condition="'@(SmokeTestsPrereqs->Count())' != '0'">
     <PropertyGroup>
-      <SmokeTestPrereqsTarballName>$(SharedOutputPath)dotnet-smoke-test-prereqs.$(SourceBuiltSdkVersion).$(TargetRid)$(ArchiveExtension)</SmokeTestPrereqsTarballName>
+      <SmokeTestPrereqsTarball>$(ArtifactsAssetsDir)dotnet-smoke-test-prereqs.$(SourceBuiltSdkVersion).$(TargetRid)$(ArchiveExtension)</SmokeTestPrereqsTarball>
       <SmokeTestsPrereqPackagesDir>$(SmokeTestsArtifactsDir)prereq-packages/</SmokeTestsPrereqPackagesDir>
     </PropertyGroup>
 
     <Copy SourceFiles="@(SmokeTestsPrereqs)"
           DestinationFolder="$(SmokeTestsPrereqPackagesDir)" />
 
-    <Exec Command="tar --numeric-owner -czf $(SmokeTestPrereqsTarballName) ."
+    <MakeDir Directories="$([System.IO.Path]::GetDirectoryName('$(SmokeTestPrereqsTarball)'))" />
+    <Exec Command="tar --numeric-owner -czf $(SmokeTestPrereqsTarball) ."
           WorkingDirectory="$(SmokeTestsPrereqPackagesDir)"/>
 
-    <Message Importance="High" Text="Packaged smoke-test prereqs in '$(SmokeTestPrereqsTarballName)'" />
+    <Message Importance="High" Text="Packaged smoke-test prereqs in '$(SmokeTestPrereqsTarball)'" />
   </Target>
 
   <Target Name="CreatePrebuiltsTarball"
@@ -236,13 +210,14 @@
           DependsOnTargets="DetermineSourceBuiltSdkVersion"
           Condition="'@(PrebuiltFile->Count())' != '0'">
     <PropertyGroup>
-      <TarballFilePath>$(SharedOutputPath)$(SourceBuiltPrebuiltsTarballName).$(SourceBuiltSdkVersion).$(TargetRid)$(ArchiveExtension)</TarballFilePath>
+      <PrebuiltsTarball>$(ArtifactsAssetsDir)$(SourceBuiltPrebuiltsTarballName).$(SourceBuiltSdkVersion).$(TargetRid)$(ArchiveExtension)</PrebuiltsTarball>
-      <TarballWorkingDir>$(ResultingPrebuiltPackagesDir)</TarballWorkingDir>
+      <PrebuiltsTarballWorkingDir>$(ResultingPrebuiltPackagesDir)</PrebuiltsTarballWorkingDir>
     </PropertyGroup>
 
-    <Exec Command="tar --numeric-owner -zcf $(TarballFilePath) -C $(TarballWorkingDir) ." />
+    <MakeDir Directories="$([System.IO.Path]::GetDirectoryName('$(PrebuiltsTarball)'))" />
+    <Exec Command="tar --numeric-owner -zcf $(PrebuiltsTarball) -C $(PrebuiltsTarballWorkingDir) ." />
 
-    <Message Text="Tarball '$(TarballFilePath)' was successfully created from '$(TarballWorkingDir)'" Importance="High" />
+    <Message Text="Tarball '$(PrebuiltsTarball)' was successfully created from '$(PrebuiltsTarballWorkingDir)'" Importance="High" />
   </Target>
 
   <Target Name="ErrorOnPrebuilts"

src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CheckForPoison.cs

@@ -28,6 +28,8 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
         /// The files to check for poison and/or hash matches.  Zips and
         /// nupkgs will be extracted and checked recursively.
         /// %(Identity): Path to the initial set of files.
+        /// Add SourceBuildReferencePackage metadata and set to true to
+        /// indicate that the file comes from SBRP.
         /// </summary>
         [Required]
         public ITaskItem[] FilesToCheck { get; set; }
@@ -66,11 +68,6 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
         /// </summary>
         public string OverrideTempPath { get; set; }
 
-        /// <summary>
-        /// Array of files containing lists of non-shipping packages
-        /// </summary>
-        public ITaskItem[] NonShippingPackagesListFiles { get; set; }
-
         private static readonly string[] ZipFileExtensions =
         {
             ".zip",
@@ -152,11 +149,13 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
 
         private const string SbrpAttributeType = "System.Reflection.AssemblyMetadataAttribute";
 
-        private record CandidateFileEntry(string ExtractedPath, string DisplayPath);
+        private const string SbrpMetadataName = "IsSourceBuildReferencePackage";
+
+        private record CandidateFileEntry(string ExtractedPath, string DisplayPath, bool IsSourceBuildReferencePackage);
 
         public override bool Execute()
         {
-            IEnumerable<PoisonedFileEntry> poisons = GetPoisonedFiles(FilesToCheck.Select(f => f.ItemSpec), HashCatalogFilePath, MarkerFileName);
+            IEnumerable<PoisonedFileEntry> poisons = GetPoisonedFiles(FilesToCheck, HashCatalogFilePath, MarkerFileName);
 
             // if we should write out the poison report, do that
             if (!string.IsNullOrWhiteSpace(PoisonReportOutputFilePath))
@@ -184,13 +183,14 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
         /// <param name="catalogedPackagesFilePath">File path to the file hash catalog</param>
         /// <param name="markerFileName">Marker file name to check for in poisoned nupkgs</param>
         /// <returns>List of poisoned packages and files found and reasons for each</returns>
-        internal IEnumerable<PoisonedFileEntry> GetPoisonedFiles(IEnumerable<string> initialCandidates, string catalogedPackagesFilePath, string markerFileName)
+        internal IEnumerable<PoisonedFileEntry> GetPoisonedFiles(IEnumerable<ITaskItem> initialCandidates, string catalogedPackagesFilePath, string markerFileName)
         {
-            IEnumerable<string> nonShippingPackages = GetAllNonShippingPackages();
             IEnumerable<CatalogPackageEntry> catalogedPackages = ReadCatalog(catalogedPackagesFilePath);
             var poisons = new List<PoisonedFileEntry>();
             var candidateQueue = new Queue<CandidateFileEntry>(initialCandidates.Select(candidate =>
-                new CandidateFileEntry(candidate, Utility.MakeRelativePath(candidate, ProjectDirPath))));
+                new CandidateFileEntry(candidate.ItemSpec,
+                    Utility.MakeRelativePath(candidate.ItemSpec, ProjectDirPath),
+                    candidate.GetMetadata(SbrpMetadataName) == "true")));
 
             if (!string.IsNullOrWhiteSpace(OverrideTempPath))
             {
@@ -209,12 +209,6 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
                 {
                     Log.LogMessage($"Zip or NuPkg file to check: {candidate.ExtractedPath}");
 
-                    // Skip non-shipping packages
-                    if (nonShippingPackages.Contains(Path.GetFileName(candidate.ExtractedPath), StringComparer.OrdinalIgnoreCase))
-                    {
-                        continue;
-                    }
-
                     var tempCheckingDir = Path.Combine(tempDir.FullName, Path.GetFileNameWithoutExtension(candidate.ExtractedPath));
                     PoisonedFileEntry result = ExtractAndCheckZipFileOnly(catalogedPackages, candidate, markerFileName, tempCheckingDir, candidateQueue);
                     if (result != null)
@@ -237,21 +231,6 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
             return poisons;
         }
 
-        private IEnumerable<string> GetAllNonShippingPackages()
-        {
-            if (NonShippingPackagesListFiles != null)
-            {
-                return NonShippingPackagesListFiles
-                    .SelectMany(item => File.ReadAllLines(item.ItemSpec))
-                    .Distinct()
-                    .ToList();
-            }
-            else
-            {
-                return Enumerable.Empty<string>();
-            }
-        }
-
         private static PoisonedFileEntry CheckSingleFile(IEnumerable<CatalogPackageEntry> catalogedPackages, CandidateFileEntry candidate)
         {
             // skip some common files that get copied verbatim from nupkgs - LICENSE, _._, etc as well as
@@ -303,7 +282,7 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
             try
             {
                 AssemblyName asm = AssemblyName.GetAssemblyName(fileToCheck);
-                if (!candidate.DisplayPath.Contains("SourceBuildReferencePackages") && IsAssemblyFromSbrp(fileToCheck))
+                if (!candidate.IsSourceBuildReferencePackage && IsAssemblyFromSbrp(fileToCheck))
                 {
                     poisonEntry.Type |= PoisonType.SourceBuildReferenceAssembly;
                 }
@@ -451,7 +430,7 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
             {
                 string displayPath = $"{candidate.DisplayPath}/{child.Replace(tempDir, string.Empty).TrimStart(Path.DirectorySeparatorChar)}";
 
-                futureFilesToCheck.Enqueue(new CandidateFileEntry(child, displayPath));
+                futureFilesToCheck.Enqueue(new CandidateFileEntry(child, displayPath, candidate.IsSourceBuildReferencePackage));
             }
 
             return poisonEntry.Type != PoisonType.None ? poisonEntry : null;

src/SourceBuild/content/repo-projects/Directory.Build.props

@@ -17,10 +17,11 @@
     <ProjectDirectory>$([MSBuild]::NormalizeDirectory('$(SrcDir)', '$(RepositoryName)'))</ProjectDirectory>
 
     <!-- Paths to the version props files -->
-    <PackageVersionPropsPath>$(SharedIntermediateOutputPath)PackageVersions.$(RepositoryName).props</PackageVersionPropsPath>
+    <PackageVersionsDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsObjDir)', 'PackageVersions'))</PackageVersionsDir>
-    <CurrentSourceBuiltPackageVersionPropsPath>$(SharedIntermediateOutputPath)PackageVersions.$(RepositoryName).Current.props</CurrentSourceBuiltPackageVersionPropsPath>
+    <PackageVersionPropsPath>$(PackageVersionsDir)PackageVersions.$(RepositoryName).props</PackageVersionPropsPath>
-    <PreviouslySourceBuiltPackageVersionPropsPath>$(SharedIntermediateOutputPath)PackageVersions.$(RepositoryName).Previous.props</PreviouslySourceBuiltPackageVersionPropsPath>
+    <CurrentSourceBuiltPackageVersionPropsPath>$(PackageVersionsDir)PackageVersions.$(RepositoryName).Current.props</CurrentSourceBuiltPackageVersionPropsPath>
-    <SnapshotPackageVersionPropsPath>$(SharedIntermediateOutputPath)PackageVersions.$(RepositoryName).Snapshot.props</SnapshotPackageVersionPropsPath>
+    <PreviouslySourceBuiltPackageVersionPropsPath>$(PackageVersionsDir)PackageVersions.$(RepositoryName).Previous.props</PreviouslySourceBuiltPackageVersionPropsPath>
+    <SnapshotPackageVersionPropsPath>$(PackageVersionsDir)PackageVersions.$(RepositoryName).Snapshot.props</SnapshotPackageVersionPropsPath>
     <PackageVersionPropsFlowType>DependenciesOnly</PackageVersionPropsFlowType>
 
     <GlobalJsonFile Condition="'$(GlobalJsonFile)' == '' and Exists('$(ProjectDirectory)global.json')">$(ProjectDirectory)global.json</GlobalJsonFile>

src/SourceBuild/content/repo-projects/Directory.Build.targets

@@ -28,6 +28,8 @@
     <ProjectAssetsJsonArchiveFile>$(PackageReportDir)all-project-assets-json-files.zip</ProjectAssetsJsonArchiveFile>
     <ProdConManifestFile>$(PackageReportDir)prodcon-build.xml</ProdConManifestFile>
     <RepoManifestFile>$([MSBuild]::NormalizePath('$(ProjectDirectory)', 'artifacts', 'RepoManifest.xml'))</RepoManifestFile>
+
+    <IntermediateSymbolsRepoDir>$(IntermediateSymbolsRootDir)$(RepositoryName)</IntermediateSymbolsRepoDir>
   </PropertyGroup>
 
   <!-- Exclude repositories that currently don't build when not building source-only. -->
@@ -48,7 +50,6 @@
   </ItemGroup>
 
   <UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="AddSourceToNuGetConfig" />
-  <UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="ReadNuGetPackageInfos" />
   <UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="RemoveInternetSourcesFromNuGetConfig" />
   <UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="UpdateJson" />
   <UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="UpdateNuGetConfigPackageSourcesMappings" />
@@ -77,9 +78,14 @@
           Inputs="$(MSBuildProjectFullPath)"
           Outputs="$(BaseIntermediateOutputPath)UpdateNuGetConfig.complete">
     <PropertyGroup>
-      <SourceBuiltNuGetSourceName>source-built</SourceBuiltNuGetSourceName>
+      <!-- Feed for the shipping live built packages -->
+      <SourceBuiltShippingNuGetSourceName>source-built</SourceBuiltShippingNuGetSourceName>
+      <!-- Feed for the non-shipping live built packages -->
+      <SourceBuiltNonShippingNuGetSourceName>source-built-transport</SourceBuiltNonShippingNuGetSourceName>
+      <SourceBuildSources>$(SourceBuiltShippingNuGetSourceName);$(SourceBuiltNonShippingNuGetSourceName)</SourceBuildSources>
+
+      <!-- Dev innerloop opt-in feed: /p:ExtraRestoreSourcePath=... -->
       <ExtraSourcesNuGetSourceName>ExtraSources</ExtraSourcesNuGetSourceName>
-      <SourceBuildSources>$(SourceBuiltNuGetSourceName)</SourceBuildSources>
       <SourceBuildSources Condition="'$(ExtraRestoreSourcePath)' != ''">$(SourceBuildSources);$(ExtraSourcesNuGetSourceName)</SourceBuildSources>
     </PropertyGroup>
   
@@ -128,8 +134,12 @@
                             Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
 
     <AddSourceToNuGetConfig NuGetConfigFile="$(NuGetConfigFile)"
-                            SourceName="$(SourceBuiltNuGetSourceName)"
+                            SourceName="$(SourceBuiltShippingNuGetSourceName)"
-                            SourcePath="$(SourceBuiltPackagesPath)" />
+                            SourcePath="$(ArtifactsShippingPackagesDir)" />
+
+    <AddSourceToNuGetConfig NuGetConfigFile="$(NuGetConfigFile)"
+                            SourceName="$(SourceBuiltNonShippingNuGetSourceName)"
+                            SourcePath="$(ArtifactsNonShippingPackagesDir)" />
 
     <AddSourceToNuGetConfig NuGetConfigFile="$(NuGetConfigFile)"
                             SourceName="$(ExtraSourcesNuGetSourceName)"
@@ -198,8 +208,8 @@
           Inputs="$(MSBuildProjectFullPath)"
           Outputs="$(BaseIntermediateOutputPath)CreateBuildInputProps.complete">
     <ItemGroup>
-      <_CurrentSourceBuiltPackages Include="$(SourceBuiltPackagesPath)*.nupkg"
+      <_CurrentSourceBuiltPackages Include="$(ArtifactsPackagesDir)**\*.nupkg"
-                                   Exclude="$(SourceBuiltPackagesPath)*.symbols.nupkg" />
+                                   Condition="!$([System.String]::Copy('%(Identity)').EndsWith('.symbols.nupkg'))" />
       <_PreviouslyBuiltSourceBuiltPackages Include="$(PrebuiltSourceBuiltPackagesPath)*.nupkg" />
     </ItemGroup>
 
@@ -275,6 +285,18 @@
              StopOnFirstFailure="true" />
   </Target>
 
+  <Target Name="SetSourceBuiltSdkOverrides"
+          Condition="'@(SourceBuiltSdkOverride)' != ''">
+    <ItemGroup>
+      <!-- Set the environment variables for MSBuild to look for our additional SDK Resolvers and or our resolver to find our source-built SDKs. -->
+      <EnvironmentVariables Include="MSBUILDADDITIONALSDKRESOLVERSFOLDER=$(VSMSBuildSdkResolversDir)" />
+      <EnvironmentVariables Include="SOURCE_BUILT_SDK_ID_%(SourceBuiltSdkOverride.Group)=%(SourceBuiltSdkOverride.Identity)" />
+      <EnvironmentVariables Include="SOURCE_BUILT_SDK_VERSION_%(SourceBuiltSdkOverride.Group)=%(SourceBuiltSdkOverride.Version)" />
+      <EnvironmentVariables Condition="'%(SourceBuiltSdkOverride.Location)' != ''" Include="SOURCE_BUILT_SDK_DIR_%(SourceBuiltSdkOverride.Group)=%(SourceBuiltSdkOverride.Location)/" />
+      <EnvironmentVariables Condition="'%(SourceBuiltSdkOverride.Location)' == ''" Include="SOURCE_BUILT_SDK_DIR_%(SourceBuiltSdkOverride.Group)=$(SourceBuiltSdksDir)%(SourceBuiltSdkOverride.Identity)/" />
+    </ItemGroup>
+  </Target>
+
   <Target Name="RepoBuild"
           Inputs="$(MSBuildProjectFullPath)"
           Outputs="$(BaseIntermediateOutputPath)Build.complete"
@@ -282,8 +304,8 @@
           DependsOnTargets="BuildRepoReferences;
                             UpdateNuGetConfig;
                             UpdateGlobalJsonVersions;
-                            CreateBuildInputProps;
                             UpdateEngCommonFiles;
+                            CreateBuildInputProps;
                             SetSourceBuiltSdkOverrides">
     <Exec Command="$(DotnetTool) build-server shutdown" />
 
@@ -306,7 +328,9 @@
     </PropertyGroup>
 
     <MakeDir Directories="$([System.IO.Path]::GetDirectoryName('$(RepoConsoleLogFile)'));
-                          $(SourceBuiltPackagesPath)" />
+                          $(ArtifactsShippingPackagesDir);
+                          $(ArtifactsNonShippingPackagesDir);
+                          $(ArtifactsAssetsDir)" />
 
     <!-- Create directories for extra debugging. -->
     <MakeDir Directories="$(MSBuildDebugPathTargetDir);
@@ -335,21 +359,11 @@
     <Message Importance="High" Text="See '$(RepoConsoleLogFile)' for more information." Condition="Exists('$(RepoConsoleLogFile)') and '$(MinimalConsoleLogOutput)' == 'true'" />
   </Target>
 
-  <Target Name="Build" DependsOnTargets="BuildRepoReferences;RepoBuild" />
-
-  <Target Name="Package"
-          AfterTargets="Build"
-          DependsOnTargets="
-    CopyRepoArtifacts;
-    CopyInnerBuildRestoredPackages;
-    EnsurePackagesCreated;
-    ExtractToolPackage;
-    CleanupRepo" />
-
   <Target Name="CopyRepoArtifacts"
+          DependsOnTargets="RepoBuild"
           Condition="'$(IsUtilityProject)' != 'true'"
           Inputs="$(RepoManifestFile)"
-          Outputs="$(BaseIntermediateOutputPath)ArtifactsCopy.complete">
+          Outputs="$(BaseIntermediateOutputPath)CopyRepoArtifacts.complete">
     <Error Text="Repo manifest file doesn't exist." Condition="!Exists('$(RepoManifestFile)')" />
 
     <XmlPeek XmlInputPath="$(RepoManifestFile)"
@@ -357,55 +371,77 @@
         <Output TaskParameter="Result" ItemName="RepoManifestArtifact" />
     </XmlPeek>
 
-    <ItemGroup>
+   <!-- Error if a repository doesn't produce any shipping or non-shipping artifacts. -->
-      <RepoManifestAsset Include="@(RepoManifestArtifact)" Condition="'%(Extension)' != '.nupkg'"/>
+    <ItemGroup Condition="'@(RepoManifestArtifact)' == ''">
-      <RepoManifestPackage Include="@(RepoManifestArtifact)" Condition="'%(Extension)' == '.nupkg'"/>
+      <_ArtifactsNotCreatedReason Include="^ There may have been a silent failure in the repo build." />
-      <RepoManifestNonShippingPackage Include="@(RepoManifestPackage)" Condition="$([System.String]::Copy('%(Identity)').Contains('$([System.IO.Path]::DirectorySeparatorChar)NonShipping$([System.IO.Path]::DirectorySeparatorChar)'))"/>
+      <_ArtifactsNotCreatedReason Include="^ To confirm, check the build log file for undetected errors that may have prevented package creation: $(RepoConsoleLogFile)" Condition="'$(MinimalConsoleLogOutput)' == 'true'" />
+      <_ArtifactsNotCreatedReason Include="^ The packages may have been written to an unexpected directory. For example, some repos used bin/ and changed to artifacts/ to match Arcade. Check RepoManifestFile in $(MSBuildProjectFullPath) (currently '$(RepoManifestFile)')" />
     </ItemGroup>
 
-    <PropertyGroup>
+    <Error Text="$(RepositoryName) produced no new source-built package identities. Known possible causes:%0A@(_ArtifactsNotCreatedReason, '%0A')"
-      <_NupkgsDestination>$(SourceBuiltPackagesPath)</_NupkgsDestination>
+           Condition="'@(RepoManifestArtifact)' == ''" />
-      <_NupkgsDestination Condition="'$(RepositoryName)' == 'source-build-reference-packages'">$(ReferencePackagesDir)</_NupkgsDestination>
+
-    </PropertyGroup>
+    <!-- The repo symbol archive shouldn't be copied. -->
+    <ItemGroup>
+      <RepoSymbolsArchive Include="@(RepoManifestArtifact)"
+                          Condition="$([System.String]::Copy('%(Identity)').EndsWith('Symbols.$(RepositoryName)$(ArchiveExtension)'))" />
+      <RepoManifestArtifact Remove="@(RepoSymbolsArchive)" />
+    </ItemGroup>
+
+    <!-- Log the new repo artifacts -->
+    <Message Importance="High" Text="New artifact(s) after building $(RepositoryName):" />
+    <Message Importance="High" Text="  -> %(RepoManifestArtifact.Filename)%(RepoManifestArtifact.Extension)" />
+
+    <ItemGroup>
+      <RepoManifestPackage Include="@(RepoManifestArtifact->WithMetadataValue('Extension', '.nupkg'))" />
+      <RepoManifestAsset Include="@(RepoManifestArtifact)" Exclude="@(RepoManifestPackage)" />
+    </ItemGroup>
+
+    <!-- Prepare the repo manifest packages and group them by shipping and non-shipping. -->
+    <ItemGroup>
+      <RepoManifestNonShippingPackage Include="@(RepoManifestPackage)"
+                                      Condition="$([System.String]::Copy('%(Identity)').Contains('$([System.IO.Path]::DirectorySeparatorChar)NonShipping$([System.IO.Path]::DirectorySeparatorChar)'))">
+        <DestinationFolder Condition="'$(RepositoryName)' != 'source-build-reference-packages'">$(ArtifactsNonShippingPackagesDir)</DestinationFolder>
+        <DestinationFolder Condition="'$(RepositoryName)' == 'source-build-reference-packages'">$(ReferencePackagesDir)</DestinationFolder>
+      </RepoManifestNonShippingPackage>
+
+      <RepoManifestShippingPackage Include="@(RepoManifestPackage)"
+                                   Exclude="@(RepoManifestNonShippingPackage)">
+        <DestinationFolder Condition="'$(RepositoryName)' != 'source-build-reference-packages'">$(ArtifactsShippingPackagesDir)</DestinationFolder>
+        <DestinationFolder Condition="'$(RepositoryName)' == 'source-build-reference-packages'">$(ReferencePackagesDir)</DestinationFolder>
+      </RepoManifestShippingPackage>
+    </ItemGroup>
 
     <!-- Building SBRP: At this point the References directory contains the previously-source-built SBRPs,
          clear it before copying the current SBRPs.  This ensures n-1 SBRPs aren't required to build the product repos. -->
-    <RemoveDir
+    <RemoveDir Directories="$(ReferencePackagesDir)"
-      Condition="'$(RepositoryName)' == 'source-build-reference-packages'"
+               Condition="'$(RepositoryName)' == 'source-build-reference-packages'" />
-      Directories="$(ReferencePackagesDir)" />
 
-    <!-- Copy nupkgs -->
+    <!-- Copy shipping packages -->
-    <Copy
+    <Copy SourceFiles="@(RepoManifestShippingPackage)"
-      Condition="'@(RepoManifestPackage)' != ''"
+          DestinationFolder="%(RepoManifestShippingPackage.DestinationFolder)"
-      SourceFiles="@(RepoManifestPackage)"
+          Condition="'@(RepoManifestShippingPackage)' != ''" />
-      DestinationFolder="$(_NupkgsDestination)" />
+
+    <!-- Copy non-shipping packages -->
+    <Copy SourceFiles="@(RepoManifestNonShippingPackage)"
+          DestinationFolder="%(RepoManifestNonShippingPackage.DestinationFolder)"
+          Condition="'@(RepoManifestNonShippingPackage)' != ''" />
 
     <!-- Copy assets -->
-    <Copy
+    <Copy SourceFiles="@(RepoManifestAsset)"
-      Condition="'@(RepoManifestAsset)' != ''"
+          DestinationFolder="$(ArtifactsAssetsDir)"
-      SourceFiles="@(RepoManifestAsset)"
+          Condition="'@(RepoManifestAsset)' != ''" />
-      DestinationFolder="$(SourceBuiltAssetsDir)" />
-
-    <!-- Generate non-shipping package list -->
-    <PropertyGroup>
-      <NonShippingPackagesList Condition="'@(RepoManifestNonShippingPackage)' != ''">$(PackageListsDir)$(NonShippingPackagesListPrefix)$(RepositoryName).lst</NonShippingPackagesList>
-    </PropertyGroup>
-
-    <WriteLinesToFile
-      Condition="'$(NonShippingPackagesList)' != ''"
-      File="$(NonShippingPackagesList)"
-      Lines="@(RepoManifestNonShippingPackage->'%(Filename)%(Extension)')"
-      Overwrite="true" />
 
     <MakeDir Directories="$(BaseIntermediateOutputPath)" />
-    <Touch Files="$(BaseIntermediateOutputPath)ArtifactsCopy.complete" AlwaysCreate="true">
+    <Touch Files="$(BaseIntermediateOutputPath)CopyRepoArtifacts.complete" AlwaysCreate="true">
       <Output TaskParameter="TouchedFiles" ItemName="FileWrites" />
     </Touch>
   </Target>
 
   <!-- Copy restored packages from inner build to ensure they're included in the
        main build prebuilt check -->
-  <Target Name="CopyInnerBuildRestoredPackages">
+  <Target Name="CopyInnerBuildRestoredPackages"
+          Condition="'$(IsUtilityProject)' != 'true'">
     <ItemGroup>
       <_InnerPackageCacheFiles Include="$(ProjectDirectory)artifacts/sb/package-cache/**/*" />
     </ItemGroup>
@@ -417,8 +453,9 @@
   </Target>
 
   <Target Name="CleanupRepo"
-          Condition="'$(CleanWhileBuilding)' == 'true' and Exists('$(ProjectDirectory)artifacts')">
+          Condition="'$(IsUtilityProject)' != 'true' and
-
+                     '$(CleanWhileBuilding)' == 'true' and
+                     Exists('$(ProjectDirectory)artifacts')">
     <!--
       Some repositories (WinForms) use source generators that open files manually and keep file handles open on the compiler server (CsWin32).
       These source generators are written incorrectly (a source generator should never do IO itself),
@@ -472,7 +509,7 @@
           Inputs="$(MSBuildProjectFullPath)"
           Outputs="$(BaseIntermediateOutputPath)ExtractToolPackage.complete">
     <PropertyGroup>
-      <_ToolPackagesRoot Condition="'$(RepositoryName)' != 'source-build-reference-packages'">$(SourceBuiltPackagesPath)</_ToolPackagesRoot>
+      <_ToolPackagesRoot Condition="'$(RepositoryName)' != 'source-build-reference-packages'">$(ArtifactsNonShippingPackagesDir)</_ToolPackagesRoot>
       <_ToolPackagesRoot Condition="'$(RepositoryName)' == 'source-build-reference-packages'">$(ReferencePackagesDir)</_ToolPackagesRoot>
     </PropertyGroup>
 
@@ -511,51 +548,31 @@
     </Touch>
   </Target>
 
-  <Target Name="EnsurePackagesCreated"
+  <Target Name="ExtractSymbolsArchiveFile"
           DependsOnTargets="CopyRepoArtifacts"
-          Condition="'$(SkipEnsurePackagesCreated)' != 'true' and '$(IsUtilityProject)' != 'true'"
+          Condition="'$(DotNetBuildSourceOnly)' == 'true' and
-          Inputs="$(MSBuildProjectFullPath)"
+                     '$(IsUtilityProject)' != 'true' and
-          Outputs="$(BaseIntermediateOutputPath)EnsurePackagesCreated.complete">
+                     '@(RepoSymbolsArchive)' != ''"
-    <ItemGroup>
+          Inputs="@(RepoSymbolsArchive)"
-      <JustSourceBuiltPackages
+          Outputs="$(IntermediateSymbolsRepoDir)">
-        Include="$(SourceBuiltPackagesPath)*.nupkg"
-        Exclude="$(SourceBuiltPackagesPath)*.symbols.nupkg" />
-    </ItemGroup>
 
-    <ItemGroup>
+    <Error Text="There must only be a single repository symbols archive file. Found: @(RepoSymbolsArchive)"
-      <_PackagesNotCreatedReason Include="^ There may have been a silent failure in the submodule build. To confirm, check the build log file for undetected errors that may have prevented package creation: $(RepoConsoleLogFile)" />
+           Condition="@(RepoSymbolsArchive->Count()) &gt; 1" />
-      <_PackagesNotCreatedReason Include="^ This error might be a false positive if $(RepositoryName) intentionally builds no nuget packages. If so, set the SkipEnsurePackagesCreated property to true in $(MSBuildProjectFullPath)" />
-      <_PackagesNotCreatedReason Include="^ The 'bin' directory might be dirty from a previous build and the package files already existed. If so, perform a clean build, or check which packages were already in 'bin' by opening $(CurrentSourceBuiltPackageVersionPropsPath)" />
-      <_PackagesNotCreatedReason Include="^ The packages may have been written to an unexpected directory. For example, some repos used bin/ and changed to artifacts/ to match Arcade. Check RepoManifestFile in $(MSBuildProjectFullPath) (currently '$(RepoManifestFile)')" />
-    </ItemGroup>
 
-    <Error Condition="'@(JustSourceBuiltPackages)' == ''"
+    <!-- Extract repo symbols tarball -->
-           Text="$(RepositoryName) produced no new source-built package identities. Known possible causes:%0A@(_PackagesNotCreatedReason, '%0A')" />
+    <MakeDir Directories="$(IntermediateSymbolsRepoDir)" />
-
+    <Exec Command="tar -xzf %(RepoSymbolsArchive.Identity) -C $(IntermediateSymbolsRepoDir)" />
-    <ReadNuGetPackageInfos PackagePaths="@(JustSourceBuiltPackages)">
-      <Output TaskParameter="PackageInfoItems" ItemName="_JustSourceBuiltPackageInfos" />
-    </ReadNuGetPackageInfos>
-
-    <Message Importance="High" Text="New NuGet package(s) after building $(RepositoryName):" />
-    <Message Importance="High" Text="  -> %(_JustSourceBuiltPackageInfos.PackageId) %(_JustSourceBuiltPackageInfos.PackageVersion)" />
-
-    <MakeDir Directories="$(BaseIntermediateOutputPath)" />
-    <Touch Files="$(BaseIntermediateOutputPath)EnsurePackagesCreated.complete" AlwaysCreate="true">
-      <Output TaskParameter="TouchedFiles" ItemName="FileWrites" />
-    </Touch>
   </Target>
 
-  <Target Name="SetSourceBuiltSdkOverrides"
+  <Target Name="Build"
-          Condition="'@(SourceBuiltSdkOverride)' != ''">
+          DependsOnTargets="
-    <ItemGroup>
+    BuildRepoReferences;
-      <!-- Set the environment variables for MSBuild to look for our additional SDK Resolvers and or our resolver to find our source-built SDKs. -->
+    RepoBuild;
-      <EnvironmentVariables Include="MSBUILDADDITIONALSDKRESOLVERSFOLDER=$(VSMSBuildSdkResolversDir)" />
+    CopyRepoArtifacts;
-      <EnvironmentVariables Include="SOURCE_BUILT_SDK_ID_%(SourceBuiltSdkOverride.Group)=%(SourceBuiltSdkOverride.Identity)" />
+    CopyInnerBuildRestoredPackages;
-      <EnvironmentVariables Include="SOURCE_BUILT_SDK_VERSION_%(SourceBuiltSdkOverride.Group)=%(SourceBuiltSdkOverride.Version)" />
+    ExtractToolPackage;
-      <EnvironmentVariables Condition="'%(SourceBuiltSdkOverride.Location)' != ''" Include="SOURCE_BUILT_SDK_DIR_%(SourceBuiltSdkOverride.Group)=%(SourceBuiltSdkOverride.Location)/" />
+    ExtractSymbolsArchiveFile;
-      <EnvironmentVariables Condition="'%(SourceBuiltSdkOverride.Location)' == ''" Include="SOURCE_BUILT_SDK_DIR_%(SourceBuiltSdkOverride.Group)=$(SourceBuiltSdksDir)%(SourceBuiltSdkOverride.Identity)/" />
+    CleanupRepo" />
-    </ItemGroup>
-  </Target>
 
   <Target Name="WritePrebuiltUsageData"
           DependsOnTargets="GetAllProjectDirectories"
@@ -563,7 +580,7 @@
           Outputs="$(BaseIntermediateOutputPath)WritePrebuiltUsageData.complete">
     <!-- Save the PVP snapshot of each build step to be evaluated while building the report. -->
     <ItemGroup>
-      <PackageVersionPropsSnapshotFiles Include="$(SharedIntermediateOutputPath)PackageVersions.*.Snapshot.props" />
+      <PackageVersionPropsSnapshotFiles Include="$(PackageVersionsDir)PackageVersions.*.Snapshot.props" />
     </ItemGroup>
     <Copy SourceFiles="@(PackageVersionPropsSnapshotFiles)" DestinationFolder="$(PackageReportDir)snapshots/" />
 
@@ -573,7 +590,7 @@
       <!-- Only contains packages when building. -->
       <TarballPrebuiltPackageFiles Include="$(PrebuiltPackagesPath)*.nupkg" />
 
-      <SourceBuiltPackageFiles Include="$(SourceBuiltBlobFeedDir)**/*.nupkg" />
+      <SourceBuiltPackageFiles Include="$(ArtifactsPackagesDir)**/*.nupkg" />
       <SourceBuiltPackageFiles Include="$(PrebuiltSourceBuiltPackagesPath)*.nupkg" />
       <ReferencePackageFiles Include="$(ReferencePackagesDir)**/*.nupkg" />
 

src/SourceBuild/content/repo-projects/installer.proj

@@ -29,7 +29,7 @@
     <BuildArgs Condition="'$(TargetOS)' != 'windows'">$(BuildArgs) /p:CoreSetupRid=$(TargetRid)</BuildArgs>
 
     <!-- Consume the source-built Core-Setup and toolset. This line must be removed to source-build CLI without source-building Core-Setup first. -->
-    <BuildArgs>$(BuildArgs) /p:PublicBaseURL=file:%2F%2F$(SourceBuiltAssetsDir)</BuildArgs>
+    <BuildArgs>$(BuildArgs) /p:PublicBaseURL=file:%2F%2F$(ArtifactsAssetsDir)</BuildArgs>
 
     <BuildArgs>$(BuildArgs) /p:UsePortableLinuxSharedFramework=false</BuildArgs>
   </PropertyGroup>

src/SourceBuild/content/repo-projects/package-source-build.proj

@@ -14,43 +14,56 @@
   <Target Name="CustomRepoBuild"
           AfterTargets="RepoBuild"
           DependsOnTargets="CreateBuildInputProps;DetermineSourceBuiltSdkVersion">
-    <!-- Copy PVP to packages dir in order to package them together -->
-    <Copy SourceFiles="$(CurrentSourceBuiltPackageVersionPropsPath)" DestinationFiles="$(SourceBuiltPackagesPath)PackageVersions.props" />
-
     <PropertyGroup>
-      <SourceBuildReferencePackagesDestination>$(SourceBuiltPackagesPath)SourceBuildReferencePackages/</SourceBuildReferencePackagesDestination>
+      <SourceBuildTarballStagingDir>$([MSBuild]::NormalizeDirectory('$(BaseIntermediateOutputPath)', '$(SourceBuiltArtifactsTarballName)'))</SourceBuildTarballStagingDir>
+      <SourceBuildReferencePackagesDestination>$([MSBuild]::NormalizeDirectory('$(SourceBuildTarballStagingDir)', 'SourceBuildReferencePackages'))</SourceBuildReferencePackagesDestination>
     </PropertyGroup>
 
+    <!-- Copy PVP to staging dir in order to package them together. -->
+    <Copy SourceFiles="$(CurrentSourceBuiltPackageVersionPropsPath)"
+          DestinationFiles="$(SourceBuildTarballStagingDir)PackageVersions.props" />
+
     <ItemGroup>
       <SourceBuildReferencePackagesNupkgFiles Include="$(ReferencePackagesDir)**/*.nupkg" />
     </ItemGroup>
 
-    <!-- Copy reference packages from ReferencePackagesDir to blob-feed reference packages path. -->
+    <!-- Copy reference packages from ReferencePackagesDir to staging dir. -->
     <Copy
-      Condition="'@(SourceBuildReferencePackagesNupkgFiles)' != ''"
       SourceFiles="@(SourceBuildReferencePackagesNupkgFiles)"
-      DestinationFiles="@(SourceBuildReferencePackagesNupkgFiles -> '$(SourceBuildReferencePackagesDestination)%(Filename)%(Extension)')" />
+      DestinationFiles="@(SourceBuildReferencePackagesNupkgFiles -> '$(SourceBuildReferencePackagesDestination)%(Filename)%(Extension)')"
+      Condition="'@(SourceBuildReferencePackagesNupkgFiles)' != ''" />
+
+    <ItemGroup>
+      <ArtifactsPackage Include="$(ArtifactsPackagesDir)**\*.nupkg" />
+    </ItemGroup>
+
+    <!-- Copy packages to staging dir. -->
+    <Copy SourceFiles="@(ArtifactsPackage)"
+          DestinationFolder="$(SourceBuildTarballStagingDir)"
+          Condition="'@(ArtifactsPackage)' != ''" />
 
     <PropertyGroup>
-      <SourceBuiltTarballName>$(SharedOutputPath)$(SourceBuiltArtifactsTarballName).$(SourceBuiltSdkVersion).$(TargetRid)$(ArchiveExtension)</SourceBuiltTarballName>
+      <SourceBuiltTarballName>$(ArtifactsAssetsDir)$(SourceBuiltArtifactsTarballName).$(SourceBuiltSdkVersion).$(TargetRid)$(ArchiveExtension)</SourceBuiltTarballName>
       <SourceBuiltVersionFileName>.version</SourceBuiltVersionFileName>
     </PropertyGroup>
 
-    <!-- Content of the .version file to include in the tarball -->
+    <!-- Content of the .version file to include in the tarball. Write to staging dir. -->
     <ItemGroup>
       <VersionFileContent Include="$(RepositoryCommit);$(SourceBuiltSdkVersion)" />
     </ItemGroup>
 
     <WriteLinesToFile
-      File="$(SourceBuiltPackagesPath)$(SourceBuiltVersionFileName)"
+      File="$(SourceBuildTarballStagingDir)$(SourceBuiltVersionFileName)"
       Lines="@(VersionFileContent)"
       Overwrite="true" />
 
-    <MakeDir Directories="$(SharedOutputPath)" />
+    <MakeDir Directories="$([System.IO.Path]::GetDirectoryName('$(SourceBuiltTarballName)'))" />
     <Exec Command="tar --numeric-owner -czf $(SourceBuiltTarballName) $(SourceBuiltVersionFileName) *.nupkg *.props SourceBuildReferencePackages/"
-          WorkingDirectory="$(SourceBuiltPackagesPath)" />
+          WorkingDirectory="$(SourceBuildTarballStagingDir)" />
 
     <Message Importance="High" Text="Packaged source-built artifacts to $(SourceBuiltTarballName)" />
+
+    <RemoveDir Directories="$(SourceBuildTarballStagingDir)" />
   </Target>
 
 </Project>

src/SourceBuild/content/repo-projects/source-build-reference-packages.proj

@@ -1,14 +1,8 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <!--
+    <!-- All packages built in SBRP repo are copied to prereqs/package/reference.
-      All packages built in SBRP repo are copied to prereqs/package/reference.
+         Nothing gets copied to the artifacts/packages folder. -->
-      Nothing gets copied to blob-feed packages cache.
-
-      This would cause an issue with validation in EnsurePackagesCreated target.
-      We need to skip it for SBRP repo.
-    -->
-    <SkipEnsurePackagesCreated>true</SkipEnsurePackagesCreated>
 
     <!-- SBRP builds before Arcade so it also needs the bootstrap Arcade version -->
     <UseBootstrapArcade>true</UseBootstrapArcade>