Enable scenario test execution in VMR build (#19222)

2024-05-01 09:16:01 -05:00 · 2024-05-01 09:16:01 -05:00 · 6eadf9fb74
commit 6eadf9fb74
parent e813718a01
14 changed files with 212 additions and 16 deletions
--- a/eng/pipelines/templates/jobs/vmr-build.yml
+++ b/eng/pipelines/templates/jobs/vmr-build.yml
@ -370,7 +370,7 @@ jobs:
          set -ex

          dockerVolumeArgs="-v $(sourcesPath):/vmr"
-          sourceOnlyArgs=''
+          customBuildArgs=''
          extraBuildProperties=''

          if [[ ! -z '${{ parameters.targetOS }}' ]]; then
@ -381,11 +381,15 @@ jobs:
            extraBuildProperties="$extraBuildProperties /p:TargetArchitecture=${{ parameters.targetArchitecture }}"
          fi

+          if [[ '${{ parameters.useDevVersions }}' == 'True' ]]; then
+            customBuildArgs="$customBuildArgs --dev"
+          fi
+
          if [[ '${{ parameters.buildSourceOnly }}' == 'True' ]]; then
            if [[ '${{ parameters.enablePoison }}' == 'True' ]]; then
-              sourceOnlyArgs='--poison'
+              customBuildArgs="$customBuildArgs --poison"
            fi
-            sourceOnlyArgs="$sourceOnlyArgs --source-only /p:SmokeTestsWarnOnSdkContentDiffs=true /p:SmokeTestsExcludeOmniSharpTests=${{ parameters.excludeOmniSharpTests }}"
+            customBuildArgs="$customBuildArgs --source-only /p:SmokeTestsWarnOnSdkContentDiffs=true /p:SmokeTestsExcludeOmniSharpTests=${{ parameters.excludeOmniSharpTests }}"
          fi

          if [[ -n "${{ parameters.extraProperties }}" ]]; then
@ -394,10 +398,10 @@ jobs:

          # Only use Docker when a container is specified
          if [[ -n "${{ parameters.container }}" ]]; then
-            docker run --rm $dockerVolumeArgs -w /vmr ${{ parameters.container }} ./build.sh /bl:artifacts/log/Release/Test.binlog --test $sourceOnlyArgs $extraBuildProperties $(additionalBuildArgs) 
+            docker run --rm $dockerVolumeArgs -w /vmr ${{ parameters.container }} ./build.sh /bl:artifacts/log/Release/Test.binlog --test $customBuildArgs $extraBuildProperties $(additionalBuildArgs) 
          else
            cd $(sourcesPath)
-            ./build.sh /bl:artifacts/log/Release/Test.binlog --test $sourceOnlyArgs $extraBuildProperties $(additionalBuildArgs)
+            ./build.sh /bl:artifacts/log/Release/Test.binlog --test $customBuildArgs $extraBuildProperties $(additionalBuildArgs)
          fi

        displayName: Run Tests
@ -424,6 +428,10 @@ jobs:
        CopyWithRelativeFolders "src/"                       $targetFolder "*.binlog"
        CopyWithRelativeFolders "src/"                       $targetFolder "*.log"

+        if (Test-Path "artifacts/scenario-tests/") {
+            CopyWithRelativeFolders "artifacts/scenario-tests/" $targetFolder "*.binlog"
+        }
+
        if (Test-Path "artifacts/TestResults/*") {
            CopyWithRelativeFolders "artifacts/TestResults/"     $targetFolder "*.binlog"
            CopyWithRelativeFolders "artifacts/TestResults/"     $targetFolder "*.diff"
@ -451,6 +459,7 @@ jobs:
        cd "$(sourcesPath)"
        find artifacts/log/ -type f -name "*.binlog" -exec rsync -R {} -t ${targetFolder} \;
        find artifacts/log/ -type f -name "*.log" -exec rsync -R {} -t ${targetFolder} \;
+        [ -d "artifacts/scenario-tests/" ] && find artifacts/scenario-tests/ -type f -name "*.binlog" -exec rsync -R {} -t ${targetFolder} \;
        find artifacts/TestResults/ -type f -name "*.binlog" -exec rsync -R {} -t ${targetFolder} \;
        find artifacts/TestResults/ -type f -name "*.diff" -exec rsync -R {} -t ${targetFolder} \;
        find artifacts/TestResults/ -type f -name "Updated*.txt" -exec rsync -R {} -t ${targetFolder} \;
@ -486,13 +495,25 @@ jobs:
      condition: succeededOrFailed()
      continueOnError: true
      inputs:
-        testRunner: vSTest
+        testRunner: VSTest
        testResultsFiles: 'artifacts/TestResults/Release/*.trx'
        searchFolder: $(sourcesPath)
        mergeTestResults: true
        publishRunAttachments: true
        testRunTitle: Tests_$(Agent.JobName)

+    - task: PublishTestResults@2
+      displayName: Publish Scenario Test Results
+      condition: succeededOrFailed()
+      continueOnError: true
+      inputs:
+        testRunner: xUnit
+        testResultsFiles: 'artifacts/TestResults/**/scenario-tests/*.xml'
+        searchFolder: $(sourcesPath)
+        mergeTestResults: true
+        publishRunAttachments: true
+        testRunTitle: ScenarioTests_$(Agent.JobName)
+
  - task: CopyFiles@2
    inputs:
      SourceFolder: $(sourcesPath)/artifacts
--- a/eng/pipelines/templates/stages/vmr-build.yml
+++ b/eng/pipelines/templates/stages/vmr-build.yml
@ -123,6 +123,7 @@ stages:
          artifactsRid: alpine.3.19-x64
          pool: ${{ parameters.pool_Linux }}
          container: ${{ variables.alpine319Container }}
+          targetOS: linux-musl
          buildFromArchive: false            # 🚫
          buildSourceOnly: true              # ✅
          enablePoison: true                 # ✅
@ -143,6 +144,7 @@ stages:
            architecture: x64
            pool: ${{ parameters.pool_Linux }}
            container: ${{ variables.alpine319Container }}
+            targetOS: linux-musl
            buildFromArchive: false            # 🚫
            buildSourceOnly: true              # ✅
            enablePoison: false                # 🚫
--- a/src/SourceBuild/content/Directory.Build.props
+++ b/src/SourceBuild/content/Directory.Build.props
@ -160,6 +160,7 @@
    <IntermediateSymbolsRootDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsObjDir)', 'Symbols'))</IntermediateSymbolsRootDir>
    <AssetManifestsIntermediateDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsObjDir)', 'AssetManifests'))</AssetManifestsIntermediateDir>
    <ArtifactsAssetsDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'assets', '$(Configuration)'))</ArtifactsAssetsDir>
+    <DotNetSdkExtractDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsObjDir)', 'extracted-dotnet-sdk'))</DotNetSdkExtractDir>

    <PrebuiltPackagesPath>$([MSBuild]::NormalizeDirectory('$(PrereqsPackagesDir)', 'prebuilt'))</PrebuiltPackagesPath>
    <PreviouslyRestoredPackagesPath>$([MSBuild]::NormalizeDirectory('$(PrereqsPackagesDir)', 'previouslyRestored'))</PreviouslyRestoredPackagesPath>
--- a/src/SourceBuild/content/build.sh
+++ b/src/SourceBuild/content/build.sh
@ -162,7 +162,7 @@ while [[ $# > 0 ]]; do
      ;;

    # Advanced settings
-    -build-tests)
+    -build-repo-tests)
      properties="$properties /p:DotNetBuildTests=true"
      ;;
    -ci)
--- a/src/SourceBuild/content/eng/extract-sdk-archive.proj
+++ b/src/SourceBuild/content/eng/extract-sdk-archive.proj
@ -0,0 +1,20 @@
+<Project Sdk="Microsoft.Build.NoTargets">
+
+  <PropertyGroup>
+    <TargetFramework>$(NetCurrent)</TargetFramework>
+  </PropertyGroup>
+
+  <Target Name="ExtractSdkArchive"
+          BeforeTargets="AfterBuild"
+          DependsOnTargets="DetermineSourceBuiltSdkVersion"
+          Inputs="$(SdkTarballPath)"
+          Outputs="$(DotNetSdkExtractDir)">
+    <MakeDir Directories="$(DotNetSdkExtractDir)" />
+    <Exec Condition="'$(ArchiveExtension)' == '.tar.gz'"
+          Command="tar -xzf $(SdkTarballPath) -C $(DotNetSdkExtractDir)" />
+    <Unzip Condition="'$(ArchiveExtension)' == '.zip'"
+           SourceFiles="$(SdkTarballPath)"
+           DestinationFolder="$(DotNetSdkExtractDir)" />
+  </Target>
+
+</Project>
--- a/src/SourceBuild/content/eng/finish-source-only.proj
+++ b/src/SourceBuild/content/eng/finish-source-only.proj
@ -10,6 +10,7 @@
  <ItemGroup>
    <ProjectReference Include="$(TasksDir)Microsoft.DotNet.UnifiedBuild.Tasks\Microsoft.DotNet.UnifiedBuild.Tasks.csproj" />
    <ProjectReference Include="$(TasksDir)Microsoft.DotNet.SourceBuild.Tasks.LeakDetection\Microsoft.DotNet.SourceBuild.Tasks.LeakDetection.csproj" />
+    <ProjectReference Include="$(RepositoryEngineeringDir)extract-sdk-archive.proj" />
    <ProjectReference Include="$(RepositoryEngineeringDir)merge-asset-manifests.proj" />
  </ItemGroup>

@ -59,13 +60,9 @@
          Outputs="$(SdkSymbolsTarball)">
    <PropertyGroup>
      <IntermediateSdkSymbolsLayout>$(BaseIntermediateOutputPath)SdkSymbols</IntermediateSdkSymbolsLayout>
-      <IntermediateSdkLayout>$(BaseIntermediateOutputPath)Sdk</IntermediateSdkLayout>
    </PropertyGroup>

-    <MakeDir Directories="$(IntermediateSdkLayout)" />
-    <Exec Command="tar -xzf $(SdkTarballPath) -C $(IntermediateSdkLayout)" />
-
-    <CreateSdkSymbolsLayout SdkLayoutPath="$(IntermediateSdkLayout)"
+    <CreateSdkSymbolsLayout SdkLayoutPath="$(DotNetSdkExtractDir)"
                            AllSymbolsPath="$(IntermediateSymbolsRootDir)"
                            SdkSymbolsLayoutPath="$(IntermediateSdkSymbolsLayout)"
                            FailOnMissingPDBs="false" />
@ -76,7 +73,6 @@
    <Message Importance="High" Text="Packaged sdk symbols in '$(SdkSymbolsTarball)'" />

    <RemoveDir Directories="$(IntermediateSdkSymbolsLayout)" />
-    <RemoveDir Directories="$(IntermediateSdkLayout)" />
  </Target>

  <!--
--- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/CreateSdkSymbolsLayout.cs
+++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/CreateSdkSymbolsLayout.cs
@ -89,8 +89,9 @@ namespace Microsoft.DotNet.UnifiedBuild.Tasks
                    {
                        // Copy matching pdb to symbols path, preserving sdk binary's hierarchy
                        string sourcePath = (string)allPdbGuids[debugId]!;
+                        string fileRelativePath = file.Substring(SdkLayoutPath.Length);
                        string destinationPath =
-                            file.Replace(SdkLayoutPath, SdkSymbolsLayoutPath)
+                            Path.Combine(SdkSymbolsLayoutPath, fileRelativePath)
                                .Replace(Path.GetFileName(file), Path.GetFileName(sourcePath));

                        Directory.CreateDirectory(Path.GetDirectoryName(destinationPath)!);
--- a/src/SourceBuild/content/repo-projects/Directory.Build.props
+++ b/src/SourceBuild/content/repo-projects/Directory.Build.props
@ -94,6 +94,17 @@
    <BuildArgs>$(BuildArgs) /p:ReferencePackageNupkgCacheDir="$(ReferencePackagesDir)"</BuildArgs>
  </PropertyGroup>

+  <PropertyGroup>
+    <TestActions>$(FlagParameterPrefix)restore</TestActions>
+    <TestActions>$(TestActions) $(FlagParameterPrefix)test</TestActions>
+
+    <TestArgs Condition="'$(UseOfficialBuildVersioning)' != 'false'">$(FlagParameterPrefix)ci</TestArgs>
+    <TestArgs>$(TestArgs) $(FlagParameterPrefix)configuration $(Configuration)</TestArgs>
+    <TestArgs>$(TestArgs) /bl:artifacts/log/$(Configuration)/Test.binlog</TestArgs>
+
+    <TestCommand>$(BuildScript) $(TestActions) $(TestArgs)</TestCommand>
+  </PropertyGroup>
+
  <PropertyGroup Condition="'$(EnableExtraDebugging)' == 'true'">
    <MSBuildDebugPathTargetDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'msbuild-debug'))</MSBuildDebugPathTargetDir>
    <RoslynDebugPathTargetDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'roslyn-debug'))</RoslynDebugPathTargetDir>
--- a/src/SourceBuild/content/repo-projects/scenario-tests.proj
+++ b/src/SourceBuild/content/repo-projects/scenario-tests.proj
@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.Build.NoTargets">
+<Project Sdk="Microsoft.Build.NoTargets" TreatAsLocalProperty="CleanWhileBuilding">

  <ItemGroup>
    <RepositoryReference Include="arcade" />
@ -10,4 +10,57 @@
    <RepositoryReference Include="source-build-reference-packages" />
  </ItemGroup>

+  <PropertyGroup>
+    <!-- The scenario-tests repo shouldn't be cleaned after building as we run tests from it. -->
+    <CleanWhileBuilding>false</CleanWhileBuilding>
+
+    <ScenarioTestsArtifactsDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'scenario-tests'))</ScenarioTestsArtifactsDir>
+    <ScenarioTestsResultsDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsTestResultsDir)', 'scenario-tests'))</ScenarioTestsResultsDir>
+    <_ScenarioTestsNuGetConfig>$(ScenarioTestsArtifactsDir)NuGet.config</_ScenarioTestsNuGetConfig>
+    <_InstallerNuGetConfig>$([MSBuild]::NormalizePath('$(SrcDir)', 'installer', 'NuGet.config'))</_InstallerNuGetConfig>
+  </PropertyGroup>
+
+  <Target Name="SetupNuGetConfig"
+          Inputs="$(_InstallerNuGetConfig)"
+          Outputs="$(_ScenarioTestsNuGetConfig)">
+
+    <Copy SourceFiles="$(_InstallerNuGetConfig)"
+          DestinationFiles="$(_ScenarioTestsNuGetConfig)" />
+
+  </Target>
+
+  <Target Name="RunScenarioTests"
+          DependsOnTargets="SetupNuGetConfig;DetermineSourceBuiltSdkVersion">
+    
+    <PropertyGroup>
+      <_CurrentDateTime>$([System.DateTime]::Now.ToString("yyyy-MM-dd_HH_mm_ss"))</_CurrentDateTime>
+      <_TestXmlOutputPath>$(ScenarioTestsResultsDir)$(_CurrentDateTime).xml</_TestXmlOutputPath>
+      <_ScenarioTestsAdditionalArgs>--xml $(_TestXmlOutputPath) --target-rid $(TargetRid) --no-cleanup --no-traits Category=MultiTFM</_ScenarioTestsAdditionalArgs>
+
+      <!-- Define the test root as a sub-directory of the scenario test artifacts directory. It needs to be a sub-directory because the scenario test execution
+           will clean that directory. Since we need the NuGet.config file that we copied in to be preserved, that's stored in the directory above the test root. -->
+      <_TestRoot>$(ScenarioTestsArtifactsDir)artifacts/</_TestRoot>
+
+      <!-- It's necessary to explicitly define the path to the dotnet tool to prevent Arcade from attempting to derive it. Otherwise, it will run the dotnet
+           install script to get a new one. We must use the locally built SDK instead in order to support non-portable RIDs for source build. -->
+      <_DotNetTool>$(DotNetSdkExtractDir)$([System.IO.Path]::GetFileName('$(DotNetTool)'))</_DotNetTool>
+    </PropertyGroup>
+
+    <MakeDir Directories="$(ScenarioTestsResultsDir)" />
+
+    <ItemGroup>
+      <_ScenarioTestEnvVars Include="
+        TestRoot=$(_TestRoot);
+        DotNetRoot=$(DotNetSdkExtractDir);
+        TestSdkVersion=$(SourceBuiltSdkVersion);
+        AdditionalTestArgs=$(_ScenarioTestsAdditionalArgs);
+        DotNetTool=$(_DotNetTool);
+        _InitializeDotNetCli=$(DotNetSdkExtractDir)" />
+    </ItemGroup>
+
+    <Exec Command="$(TestCommand)"
+          WorkingDirectory="$(ProjectDirectory)"
+          EnvironmentVariables="@(_ScenarioTestEnvVars)" />
+  </Target>
+
 </Project>
--- a/src/SourceBuild/content/test/scenario-tests/Directory.Build.targets
+++ b/src/SourceBuild/content/test/scenario-tests/Directory.Build.targets
@ -0,0 +1,9 @@
+<Project>
+
+  <Import Project="$([MSBuild]::GetPathOfFileAbove(Directory.Build.targets, $(MSBuildThisFileDirectory)..))" />
+
+  <!-- scenario-tests test execution requires a custom test target. -->
+  <Target Name="Test" DependsOnTargets="RunScenarioTests" />
+  <Target Name="VSTest" DependsOnTargets="RunScenarioTests" />
+
+</Project>
--- a/src/SourceBuild/content/test/scenario-tests/scenario-tests.proj
+++ b/src/SourceBuild/content/test/scenario-tests/scenario-tests.proj
@ -0,0 +1,17 @@
+<Project Sdk="Microsoft.Build.NoTargets">
+
+  <PropertyGroup>
+    <TargetFramework>$(NetCurrent)</TargetFramework>
+    <IsTestProject>true</IsTestProject>
+  </PropertyGroup>
+
+  <!-- Scenarios tests are handled separately from the other test projects. This is because the output of the scenario tests is
+       an executable that is used to run the tests and so using the VSTest runner doesn't work with them. They also require
+       some setup steps to be configured with the VMR. -->
+  <Target Name="RunScenarioTests">
+    <MSBuild Projects="$(RepoProjectsDir)scenario-tests.proj"
+             Properties="Configuration=$(Configuration)"
+             Targets="RunScenarioTests" />
+  </Target>
+
+</Project>
--- a/src/SourceBuild/content/test/tests.proj
+++ b/src/SourceBuild/content/test/tests.proj
@ -1,10 +1,29 @@
 <Project Sdk="Microsoft.Build.Traversal">

+  <PropertyGroup>
+    <_RunScenarioTests>true</_RunScenarioTests>
+
+    <!-- Skip scenario tests if the host architecture is different from the target architecture since the tests
+        require the ability to execute the built SDK. But the CLI is not capable of running on a host with a 
+        different architecture (i.e. "cannot execute binary file: Exec format error"). -->
+    <_RunScenarioTests Condition="'$(BuildArchitecture.ToLowerInvariant())' != '$(TargetArchitecture.ToLowerInvariant())'">false</_RunScenarioTests>
+
+    <!-- Skip scenario tests if the portable OS (determined from the host machine) is different from the target OS
+        since the tests require the ability to execute the built SDK. An example of where this would be disabled is
+        cross-build of using Mariner to build for Alpine (linux vs linux-musl). -->
+    <_RunScenarioTests Condition="'$(BuildOS)' != 'windows' and '$(__PortableTargetOS.ToLowerInvariant())' != '$(TargetOS.ToLowerInvariant())'">false</_RunScenarioTests>
+
+    <!-- The scenario tests are not supported when unofficial build versioning is used. -->
+    <_RunScenarioTests Condition="'$(UseOfficialBuildVersioning)' == 'false'">false</_RunScenarioTests>
+  </PropertyGroup>
+
  <ItemGroup>
+    <ProjectReference Include="$(RepositoryEngineeringDir)extract-sdk-archive.proj" />
    <ProjectReference Include="Microsoft.DotNet.SourceBuild.SmokeTests\Microsoft.DotNet.SourceBuild.SmokeTests.csproj"
                      Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
    <ProjectReference Include="Microsoft.DotNet.UnifiedBuild.Tests\Microsoft.DotNet.UnifiedBuild.Tests.csproj"
                      Condition="'$(ShortStack)' != 'true' and '$(PortableBuild)' == 'true' and '$(PgoInstrument)' != 'true'" />
+    <ProjectReference Include="scenario-tests\scenario-tests.proj" Condition="'$(_RunScenarioTests)' == 'true'" />
  </ItemGroup>

 </Project>
--- a/src/SourceBuild/patches/scenario-tests/0001-Disable-Aspire-scenario-test.patch
+++ b/src/SourceBuild/patches/scenario-tests/0001-Disable-Aspire-scenario-test.patch
@ -0,0 +1,23 @@
+From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
+From: Matt Thalman <mthalman@microsoft.com>
+Date: Tue, 30 Apr 2024 08:34:08 -0500
+Subject: [PATCH] Disable Aspire scenario test
+
+Backport: https://github.com/dotnet/sdk/pull/40485
+---
+ .../SdkTemplateTests.cs                                         | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/src/Microsoft.DotNet.ScenarioTests.SdkTemplateTests/SdkTemplateTests.cs b/src/Microsoft.DotNet.ScenarioTests.SdkTemplateTests/SdkTemplateTests.cs
+index c43fdd6..bb5c52d 100644
+--- a/src/Microsoft.DotNet.ScenarioTests.SdkTemplateTests/SdkTemplateTests.cs
+++ b/src/Microsoft.DotNet.ScenarioTests.SdkTemplateTests/SdkTemplateTests.cs
+@@ -217,7 +217,7 @@ public class SdkTemplateTests : IClassFixture<ScenarioTestFixture>
+         newTest.Execute(_sdkHelper, _scenarioTestInput.TestRoot, "wasm-tools");
+     }
+ 
+-    [Fact]
+    //[Fact]
+     [Trait("Category", "Workload")]
+     [Trait("Category", "InProgress")]
+     public void VerifyAspireTemplate()
--- a/src/SourceBuild/patches/scenario-tests/0002-Disable-WPF-scenario-test.patch
+++ b/src/SourceBuild/patches/scenario-tests/0002-Disable-WPF-scenario-test.patch
@ -0,0 +1,23 @@
+From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
+From: Matt Thalman <mthalman@microsoft.com>
+Date: Tue, 30 Apr 2024 10:21:32 -0500
+Subject: [PATCH] Disable WPF scenario test
+
+Backport: https://github.com/dotnet/source-build/issues/4361
+---
+ .../SdkTemplateTests.cs                                         | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/src/Microsoft.DotNet.ScenarioTests.SdkTemplateTests/SdkTemplateTests.cs b/src/Microsoft.DotNet.ScenarioTests.SdkTemplateTests/SdkTemplateTests.cs
+index c43fdd6..35279a9 100644
+--- a/src/Microsoft.DotNet.ScenarioTests.SdkTemplateTests/SdkTemplateTests.cs
+++ b/src/Microsoft.DotNet.ScenarioTests.SdkTemplateTests/SdkTemplateTests.cs
+@@ -84,7 +84,7 @@ public class SdkTemplateTests : IClassFixture<ScenarioTestFixture>
+         newTest.Execute(_sdkHelper, _scenarioTestInput.TestRoot);
+     }
+     
+-    [Theory]
+    //[Theory]
+     [InlineData(DotNetLanguage.CSharp)]
+     [InlineData(DotNetLanguage.VB)]
+     [Trait("Category", "Offline")]