Enable scenario test execution in VMR build (#19222)

eng/pipelines/templates/jobs/vmr-build.yml

@@ -370,7 +370,7 @@ jobs:
           set -ex
 
           dockerVolumeArgs="-v $(sourcesPath):/vmr"
-          sourceOnlyArgs=''
+          customBuildArgs=''
           extraBuildProperties=''
 
           if [[ ! -z '${{ parameters.targetOS }}' ]]; then
@@ -381,11 +381,15 @@ jobs:
             extraBuildProperties="$extraBuildProperties /p:TargetArchitecture=${{ parameters.targetArchitecture }}"
           fi
 
+          if [[ '${{ parameters.useDevVersions }}' == 'True' ]]; then
+            customBuildArgs="$customBuildArgs --dev"
+          fi
+
           if [[ '${{ parameters.buildSourceOnly }}' == 'True' ]]; then
             if [[ '${{ parameters.enablePoison }}' == 'True' ]]; then
-              sourceOnlyArgs='--poison'
+              customBuildArgs="$customBuildArgs --poison"
             fi
-            sourceOnlyArgs="$sourceOnlyArgs --source-only /p:SmokeTestsWarnOnSdkContentDiffs=true /p:SmokeTestsExcludeOmniSharpTests=${{ parameters.excludeOmniSharpTests }}"
+            customBuildArgs="$customBuildArgs --source-only /p:SmokeTestsWarnOnSdkContentDiffs=true /p:SmokeTestsExcludeOmniSharpTests=${{ parameters.excludeOmniSharpTests }}"
           fi
 
           if [[ -n "${{ parameters.extraProperties }}" ]]; then
@@ -394,10 +398,10 @@ jobs:
 
           # Only use Docker when a container is specified
           if [[ -n "${{ parameters.container }}" ]]; then
-            docker run --rm $dockerVolumeArgs -w /vmr ${{ parameters.container }} ./build.sh /bl:artifacts/log/Release/Test.binlog --test $sourceOnlyArgs $extraBuildProperties $(additionalBuildArgs) 
+            docker run --rm $dockerVolumeArgs -w /vmr ${{ parameters.container }} ./build.sh /bl:artifacts/log/Release/Test.binlog --test $customBuildArgs $extraBuildProperties $(additionalBuildArgs) 
           else
             cd $(sourcesPath)
-            ./build.sh /bl:artifacts/log/Release/Test.binlog --test $sourceOnlyArgs $extraBuildProperties $(additionalBuildArgs)
+            ./build.sh /bl:artifacts/log/Release/Test.binlog --test $customBuildArgs $extraBuildProperties $(additionalBuildArgs)
           fi
 
         displayName: Run Tests
@@ -424,6 +428,10 @@ jobs:
         CopyWithRelativeFolders "src/"                       $targetFolder "*.binlog"
         CopyWithRelativeFolders "src/"                       $targetFolder "*.log"
 
+        if (Test-Path "artifacts/scenario-tests/") {
+            CopyWithRelativeFolders "artifacts/scenario-tests/" $targetFolder "*.binlog"
+        }
+
         if (Test-Path "artifacts/TestResults/*") {
             CopyWithRelativeFolders "artifacts/TestResults/"     $targetFolder "*.binlog"
             CopyWithRelativeFolders "artifacts/TestResults/"     $targetFolder "*.diff"
@@ -451,6 +459,7 @@ jobs:
         cd "$(sourcesPath)"
         find artifacts/log/ -type f -name "*.binlog" -exec rsync -R {} -t ${targetFolder} \;
         find artifacts/log/ -type f -name "*.log" -exec rsync -R {} -t ${targetFolder} \;
+        [ -d "artifacts/scenario-tests/" ] && find artifacts/scenario-tests/ -type f -name "*.binlog" -exec rsync -R {} -t ${targetFolder} \;
         find artifacts/TestResults/ -type f -name "*.binlog" -exec rsync -R {} -t ${targetFolder} \;
         find artifacts/TestResults/ -type f -name "*.diff" -exec rsync -R {} -t ${targetFolder} \;
         find artifacts/TestResults/ -type f -name "Updated*.txt" -exec rsync -R {} -t ${targetFolder} \;
@@ -486,13 +495,25 @@ jobs:
       condition: succeededOrFailed()
       continueOnError: true
       inputs:
-        testRunner: vSTest
+        testRunner: VSTest
         testResultsFiles: 'artifacts/TestResults/Release/*.trx'
         searchFolder: $(sourcesPath)
         mergeTestResults: true
         publishRunAttachments: true
         testRunTitle: Tests_$(Agent.JobName)
 
+    - task: PublishTestResults@2
+      displayName: Publish Scenario Test Results
+      condition: succeededOrFailed()
+      continueOnError: true
+      inputs:
+        testRunner: xUnit
+        testResultsFiles: 'artifacts/TestResults/**/scenario-tests/*.xml'
+        searchFolder: $(sourcesPath)
+        mergeTestResults: true
+        publishRunAttachments: true
+        testRunTitle: ScenarioTests_$(Agent.JobName)
+
   - task: CopyFiles@2
     inputs:
       SourceFolder: $(sourcesPath)/artifacts

eng/pipelines/templates/stages/vmr-build.yml

@@ -123,6 +123,7 @@ stages:
           artifactsRid: alpine.3.19-x64
           pool: ${{ parameters.pool_Linux }}
           container: ${{ variables.alpine319Container }}
+          targetOS: linux-musl
           buildFromArchive: false            # 🚫
           buildSourceOnly: true              # ✅
           enablePoison: true                 # ✅
@@ -143,6 +144,7 @@ stages:
             architecture: x64
             pool: ${{ parameters.pool_Linux }}
             container: ${{ variables.alpine319Container }}
+            targetOS: linux-musl
             buildFromArchive: false            # 🚫
             buildSourceOnly: true              # ✅
             enablePoison: false                # 🚫

src/SourceBuild/content/Directory.Build.props

@@ -160,6 +160,7 @@
     <IntermediateSymbolsRootDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsObjDir)', 'Symbols'))</IntermediateSymbolsRootDir>
     <AssetManifestsIntermediateDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsObjDir)', 'AssetManifests'))</AssetManifestsIntermediateDir>
     <ArtifactsAssetsDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'assets', '$(Configuration)'))</ArtifactsAssetsDir>
+    <DotNetSdkExtractDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsObjDir)', 'extracted-dotnet-sdk'))</DotNetSdkExtractDir>
 
     <PrebuiltPackagesPath>$([MSBuild]::NormalizeDirectory('$(PrereqsPackagesDir)', 'prebuilt'))</PrebuiltPackagesPath>
     <PreviouslyRestoredPackagesPath>$([MSBuild]::NormalizeDirectory('$(PrereqsPackagesDir)', 'previouslyRestored'))</PreviouslyRestoredPackagesPath>

src/SourceBuild/content/build.sh

@@ -162,7 +162,7 @@ while [[ $# > 0 ]]; do
       ;;
 
     # Advanced settings
-    -build-tests)
+    -build-repo-tests)
       properties="$properties /p:DotNetBuildTests=true"
       ;;
     -ci)

src/SourceBuild/content/eng/extract-sdk-archive.proj

@@ -0,0 +1,20 @@
+<Project Sdk="Microsoft.Build.NoTargets">
+
+  <PropertyGroup>
+    <TargetFramework>$(NetCurrent)</TargetFramework>
+  </PropertyGroup>
+
+  <Target Name="ExtractSdkArchive"
+          BeforeTargets="AfterBuild"
+          DependsOnTargets="DetermineSourceBuiltSdkVersion"
+          Inputs="$(SdkTarballPath)"
+          Outputs="$(DotNetSdkExtractDir)">
+    <MakeDir Directories="$(DotNetSdkExtractDir)" />
+    <Exec Condition="'$(ArchiveExtension)' == '.tar.gz'"
+          Command="tar -xzf $(SdkTarballPath) -C $(DotNetSdkExtractDir)" />
+    <Unzip Condition="'$(ArchiveExtension)' == '.zip'"
+           SourceFiles="$(SdkTarballPath)"
+           DestinationFolder="$(DotNetSdkExtractDir)" />
+  </Target>
+
+</Project>

src/SourceBuild/content/eng/finish-source-only.proj

@@ -10,6 +10,7 @@
   <ItemGroup>
     <ProjectReference Include="$(TasksDir)Microsoft.DotNet.UnifiedBuild.Tasks\Microsoft.DotNet.UnifiedBuild.Tasks.csproj" />
     <ProjectReference Include="$(TasksDir)Microsoft.DotNet.SourceBuild.Tasks.LeakDetection\Microsoft.DotNet.SourceBuild.Tasks.LeakDetection.csproj" />
+    <ProjectReference Include="$(RepositoryEngineeringDir)extract-sdk-archive.proj" />
     <ProjectReference Include="$(RepositoryEngineeringDir)merge-asset-manifests.proj" />
   </ItemGroup>
 
@@ -59,13 +60,9 @@
           Outputs="$(SdkSymbolsTarball)">
     <PropertyGroup>
       <IntermediateSdkSymbolsLayout>$(BaseIntermediateOutputPath)SdkSymbols</IntermediateSdkSymbolsLayout>
-      <IntermediateSdkLayout>$(BaseIntermediateOutputPath)Sdk</IntermediateSdkLayout>
     </PropertyGroup>
 
-    <MakeDir Directories="$(IntermediateSdkLayout)" />
+    <CreateSdkSymbolsLayout SdkLayoutPath="$(DotNetSdkExtractDir)"
-    <Exec Command="tar -xzf $(SdkTarballPath) -C $(IntermediateSdkLayout)" />
-
-    <CreateSdkSymbolsLayout SdkLayoutPath="$(IntermediateSdkLayout)"
                             AllSymbolsPath="$(IntermediateSymbolsRootDir)"
                             SdkSymbolsLayoutPath="$(IntermediateSdkSymbolsLayout)"
                             FailOnMissingPDBs="false" />
@@ -76,7 +73,6 @@
     <Message Importance="High" Text="Packaged sdk symbols in '$(SdkSymbolsTarball)'" />
 
     <RemoveDir Directories="$(IntermediateSdkSymbolsLayout)" />
-    <RemoveDir Directories="$(IntermediateSdkLayout)" />
   </Target>
 
   <!--

src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/CreateSdkSymbolsLayout.cs

@@ -89,8 +89,9 @@ namespace Microsoft.DotNet.UnifiedBuild.Tasks
                     {
                         // Copy matching pdb to symbols path, preserving sdk binary's hierarchy
                         string sourcePath = (string)allPdbGuids[debugId]!;
+                        string fileRelativePath = file.Substring(SdkLayoutPath.Length);
                         string destinationPath =
-                            file.Replace(SdkLayoutPath, SdkSymbolsLayoutPath)
+                            Path.Combine(SdkSymbolsLayoutPath, fileRelativePath)
                                 .Replace(Path.GetFileName(file), Path.GetFileName(sourcePath));
 
                         Directory.CreateDirectory(Path.GetDirectoryName(destinationPath)!);

src/SourceBuild/content/repo-projects/Directory.Build.props

@@ -94,6 +94,17 @@
     <BuildArgs>$(BuildArgs) /p:ReferencePackageNupkgCacheDir="$(ReferencePackagesDir)"</BuildArgs>
   </PropertyGroup>
 
+  <PropertyGroup>
+    <TestActions>$(FlagParameterPrefix)restore</TestActions>
+    <TestActions>$(TestActions) $(FlagParameterPrefix)test</TestActions>
+
+    <TestArgs Condition="'$(UseOfficialBuildVersioning)' != 'false'">$(FlagParameterPrefix)ci</TestArgs>
+    <TestArgs>$(TestArgs) $(FlagParameterPrefix)configuration $(Configuration)</TestArgs>
+    <TestArgs>$(TestArgs) /bl:artifacts/log/$(Configuration)/Test.binlog</TestArgs>
+
+    <TestCommand>$(BuildScript) $(TestActions) $(TestArgs)</TestCommand>
+  </PropertyGroup>
+
   <PropertyGroup Condition="'$(EnableExtraDebugging)' == 'true'">
     <MSBuildDebugPathTargetDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'msbuild-debug'))</MSBuildDebugPathTargetDir>
     <RoslynDebugPathTargetDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'roslyn-debug'))</RoslynDebugPathTargetDir>

src/SourceBuild/content/repo-projects/scenario-tests.proj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.Build.NoTargets">
+<Project Sdk="Microsoft.Build.NoTargets" TreatAsLocalProperty="CleanWhileBuilding">
 
   <ItemGroup>
     <RepositoryReference Include="arcade" />
@@ -10,4 +10,57 @@
     <RepositoryReference Include="source-build-reference-packages" />
   </ItemGroup>
 
+  <PropertyGroup>
+    <!-- The scenario-tests repo shouldn't be cleaned after building as we run tests from it. -->
+    <CleanWhileBuilding>false</CleanWhileBuilding>
+
+    <ScenarioTestsArtifactsDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'scenario-tests'))</ScenarioTestsArtifactsDir>
+    <ScenarioTestsResultsDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsTestResultsDir)', 'scenario-tests'))</ScenarioTestsResultsDir>
+    <_ScenarioTestsNuGetConfig>$(ScenarioTestsArtifactsDir)NuGet.config</_ScenarioTestsNuGetConfig>
+    <_InstallerNuGetConfig>$([MSBuild]::NormalizePath('$(SrcDir)', 'installer', 'NuGet.config'))</_InstallerNuGetConfig>
+  </PropertyGroup>
+
+  <Target Name="SetupNuGetConfig"
+          Inputs="$(_InstallerNuGetConfig)"
+          Outputs="$(_ScenarioTestsNuGetConfig)">
+
+    <Copy SourceFiles="$(_InstallerNuGetConfig)"
+          DestinationFiles="$(_ScenarioTestsNuGetConfig)" />
+
+  </Target>
+
+  <Target Name="RunScenarioTests"
+          DependsOnTargets="SetupNuGetConfig;DetermineSourceBuiltSdkVersion">
+    
+    <PropertyGroup>
+      <_CurrentDateTime>$([System.DateTime]::Now.ToString("yyyy-MM-dd_HH_mm_ss"))</_CurrentDateTime>
+      <_TestXmlOutputPath>$(ScenarioTestsResultsDir)$(_CurrentDateTime).xml</_TestXmlOutputPath>
+      <_ScenarioTestsAdditionalArgs>--xml $(_TestXmlOutputPath) --target-rid $(TargetRid) --no-cleanup --no-traits Category=MultiTFM</_ScenarioTestsAdditionalArgs>
+
+      <!-- Define the test root as a sub-directory of the scenario test artifacts directory. It needs to be a sub-directory because the scenario test execution
+           will clean that directory. Since we need the NuGet.config file that we copied in to be preserved, that's stored in the directory above the test root. -->
+      <_TestRoot>$(ScenarioTestsArtifactsDir)artifacts/</_TestRoot>
+
+      <!-- It's necessary to explicitly define the path to the dotnet tool to prevent Arcade from attempting to derive it. Otherwise, it will run the dotnet
+           install script to get a new one. We must use the locally built SDK instead in order to support non-portable RIDs for source build. -->
+      <_DotNetTool>$(DotNetSdkExtractDir)$([System.IO.Path]::GetFileName('$(DotNetTool)'))</_DotNetTool>
+    </PropertyGroup>
+
+    <MakeDir Directories="$(ScenarioTestsResultsDir)" />
+
+    <ItemGroup>
+      <_ScenarioTestEnvVars Include="
+        TestRoot=$(_TestRoot);
+        DotNetRoot=$(DotNetSdkExtractDir);
+        TestSdkVersion=$(SourceBuiltSdkVersion);
+        AdditionalTestArgs=$(_ScenarioTestsAdditionalArgs);
+        DotNetTool=$(_DotNetTool);
+        _InitializeDotNetCli=$(DotNetSdkExtractDir)" />
+    </ItemGroup>
+
+    <Exec Command="$(TestCommand)"
+          WorkingDirectory="$(ProjectDirectory)"
+          EnvironmentVariables="@(_ScenarioTestEnvVars)" />
+  </Target>
+
 </Project>

src/SourceBuild/content/test/scenario-tests/Directory.Build.targets

@@ -0,0 +1,9 @@
+<Project>
+
+  <Import Project="$([MSBuild]::GetPathOfFileAbove(Directory.Build.targets, $(MSBuildThisFileDirectory)..))" />
+
+  <!-- scenario-tests test execution requires a custom test target. -->
+  <Target Name="Test" DependsOnTargets="RunScenarioTests" />
+  <Target Name="VSTest" DependsOnTargets="RunScenarioTests" />
+
+</Project>

src/SourceBuild/content/test/scenario-tests/scenario-tests.proj

@@ -0,0 +1,17 @@
+<Project Sdk="Microsoft.Build.NoTargets">
+
+  <PropertyGroup>
+    <TargetFramework>$(NetCurrent)</TargetFramework>
+    <IsTestProject>true</IsTestProject>
+  </PropertyGroup>
+
+  <!-- Scenarios tests are handled separately from the other test projects. This is because the output of the scenario tests is
+       an executable that is used to run the tests and so using the VSTest runner doesn't work with them. They also require
+       some setup steps to be configured with the VMR. -->
+  <Target Name="RunScenarioTests">
+    <MSBuild Projects="$(RepoProjectsDir)scenario-tests.proj"
+             Properties="Configuration=$(Configuration)"
+             Targets="RunScenarioTests" />
+  </Target>
+
+</Project>

src/SourceBuild/content/test/tests.proj

@@ -1,10 +1,29 @@
 <Project Sdk="Microsoft.Build.Traversal">
 
+  <PropertyGroup>
+    <_RunScenarioTests>true</_RunScenarioTests>
+
+    <!-- Skip scenario tests if the host architecture is different from the target architecture since the tests
+        require the ability to execute the built SDK. But the CLI is not capable of running on a host with a 
+        different architecture (i.e. "cannot execute binary file: Exec format error"). -->
+    <_RunScenarioTests Condition="'$(BuildArchitecture.ToLowerInvariant())' != '$(TargetArchitecture.ToLowerInvariant())'">false</_RunScenarioTests>
+
+    <!-- Skip scenario tests if the portable OS (determined from the host machine) is different from the target OS
+        since the tests require the ability to execute the built SDK. An example of where this would be disabled is
+        cross-build of using Mariner to build for Alpine (linux vs linux-musl). -->
+    <_RunScenarioTests Condition="'$(BuildOS)' != 'windows' and '$(__PortableTargetOS.ToLowerInvariant())' != '$(TargetOS.ToLowerInvariant())'">false</_RunScenarioTests>
+
+    <!-- The scenario tests are not supported when unofficial build versioning is used. -->
+    <_RunScenarioTests Condition="'$(UseOfficialBuildVersioning)' == 'false'">false</_RunScenarioTests>
+  </PropertyGroup>
+
   <ItemGroup>
+    <ProjectReference Include="$(RepositoryEngineeringDir)extract-sdk-archive.proj" />
     <ProjectReference Include="Microsoft.DotNet.SourceBuild.SmokeTests\Microsoft.DotNet.SourceBuild.SmokeTests.csproj"
                       Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
     <ProjectReference Include="Microsoft.DotNet.UnifiedBuild.Tests\Microsoft.DotNet.UnifiedBuild.Tests.csproj"
                       Condition="'$(ShortStack)' != 'true' and '$(PortableBuild)' == 'true' and '$(PgoInstrument)' != 'true'" />
+    <ProjectReference Include="scenario-tests\scenario-tests.proj" Condition="'$(_RunScenarioTests)' == 'true'" />
   </ItemGroup>
 
-</Project>
+</Project>

src/SourceBuild/patches/scenario-tests/0001-Disable-Aspire-scenario-test.patch

@@ -0,0 +1,23 @@
+From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
+From: Matt Thalman <mthalman@microsoft.com>
+Date: Tue, 30 Apr 2024 08:34:08 -0500
+Subject: [PATCH] Disable Aspire scenario test
+
+Backport: https://github.com/dotnet/sdk/pull/40485
+---
+ .../SdkTemplateTests.cs                                         | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/src/Microsoft.DotNet.ScenarioTests.SdkTemplateTests/SdkTemplateTests.cs b/src/Microsoft.DotNet.ScenarioTests.SdkTemplateTests/SdkTemplateTests.cs
+index c43fdd6..bb5c52d 100644
+--- a/src/Microsoft.DotNet.ScenarioTests.SdkTemplateTests/SdkTemplateTests.cs
++++ b/src/Microsoft.DotNet.ScenarioTests.SdkTemplateTests/SdkTemplateTests.cs
+@@ -217,7 +217,7 @@ public class SdkTemplateTests : IClassFixture<ScenarioTestFixture>
+         newTest.Execute(_sdkHelper, _scenarioTestInput.TestRoot, "wasm-tools");
+     }
+ 
+-    [Fact]
++    //[Fact]
+     [Trait("Category", "Workload")]
+     [Trait("Category", "InProgress")]
+     public void VerifyAspireTemplate()

src/SourceBuild/patches/scenario-tests/0002-Disable-WPF-scenario-test.patch

@@ -0,0 +1,23 @@
+From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
+From: Matt Thalman <mthalman@microsoft.com>
+Date: Tue, 30 Apr 2024 10:21:32 -0500
+Subject: [PATCH] Disable WPF scenario test
+
+Backport: https://github.com/dotnet/source-build/issues/4361
+---
+ .../SdkTemplateTests.cs                                         | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/src/Microsoft.DotNet.ScenarioTests.SdkTemplateTests/SdkTemplateTests.cs b/src/Microsoft.DotNet.ScenarioTests.SdkTemplateTests/SdkTemplateTests.cs
+index c43fdd6..35279a9 100644
+--- a/src/Microsoft.DotNet.ScenarioTests.SdkTemplateTests/SdkTemplateTests.cs
++++ b/src/Microsoft.DotNet.ScenarioTests.SdkTemplateTests/SdkTemplateTests.cs
+@@ -84,7 +84,7 @@ public class SdkTemplateTests : IClassFixture<ScenarioTestFixture>
+         newTest.Execute(_sdkHelper, _scenarioTestInput.TestRoot);
+     }
+     
+-    [Theory]
++    //[Theory]
+     [InlineData(DotNetLanguage.CSharp)]
+     [InlineData(DotNetLanguage.VB)]
+     [Trait("Category", "Offline")]