run sdk comparison tests in a separate pipeline

eng/pipelines/source-build-sdk-content-tests.yml

@@ -0,0 +1,79 @@
+trigger: none
+
+parameters:
+- name: vmrBuildId
+  displayName: dotnet-dotnet build id, defaults to latest build
+  type: string
+  default: ' '
+
+pool:
+  name: NetCore1ESPool-Svc-Internal
+  demands: ImageOverride -equals 1es-ubuntu-2004
+
+jobs:
+- job: Get_Build_Information
+  displayName: Get installer and VMR build ids
+  variables:
+  - template: /eng/common/templates/variables/pipelines.yml
+  - group: Publish-Build-Assets
+  
+  steps:
+  - script: |
+      last_vmr_build=$(az pipelines runs list --organization '$(AZDO_ORG)' --project '$(AZDO_PROJECT)' --pipeline-ids '$(DOTNET_DOTNET_CI_PIPELINE_ID)' --status completed --top 1 --query "[].id" --output tsv)
+      if [[ -z "$last_vmr_build" ]]; then
+        fail "Could not find a completed vmr build"
+        exit 1
+      fi
+      echo "Vmr build id: '$last_vmr_build'"
+
+      last_vmr_build=2108850
+
+      installer_sha=$(az pipelines build tag list --organization '$(AZDO_ORG)' --project '$(AZDO_PROJECT)' --build-id $last_vmr_build --output tsv | sed "s,installer-,,g")
+      installer_build=$(az pipelines runs list --organization '$(AZDO_ORG)' --project '$(AZDO_PROJECT)' --pipeline-ids '$(INSTALLER_OFFICIAL_CI_PIPELINE_ID)' --query "[?sourceVersion == '$installer_sha'].id" --output tsv)
+      if [[ -z "$installer_build" ]]; then
+        fail "Could not find a build of installer for commit '$installer_sha'"
+        exit 1
+      fi
+
+      echo "Installer build id: '$installer_build'"
+      
+      echo "##vso[task.setvariable variable=InstallerBuildId;isoutput=true]$installer_build"
+      echo "##vso[task.setvariable variable=VmrBuildId;isoutput=true]$last_vmr_build"
+    name: Get Build IDs
+    env:
+      AZURE_DEVOPS_EXT_PAT: $(System.AccessToken)
+
+- template: templates/jobs/sdk-comparison-tests.yml
+  parameters:
+    buildName: CentOSStream8_Offline
+    targetRid: centos.8-x64
+    architecture: x64
+    enablePoison: true
+
+- template: templates/jobs/sdk-comparison-tests.yml
+  parameters:
+    buildName: CentOSStream9_Offline
+    targetRid: centos.9-x64
+    architecture: x64
+    enablePoison: true
+
+- template: templates/jobs/sdk-comparison-tests.yml
+  parameters:
+    buildName: Fedora36_Offline
+    targetRid: fedora.36-x64
+    architecture: x64
+    enablePoison: true
+
+- template: templates/jobs/sdk-comparison-tests.yml
+  parameters:
+    buildName: Ubuntu2004_Offline
+    targetRid: ubuntu.2004-x64
+    architecture: x64
+    enablePoison: true
+
+- template: templates/jobs/sdk-comparison-tests.yml
+  parameters:
+    buildName: Debian11_Offline
+    targetRid: debian.11-x64
+    architecture: x64
+    enablePoison: true

eng/pipelines/templates/jobs/sdk-comparison-tests.yml

@@ -0,0 +1,106 @@
+parameters:
+- name: buildName
+  type: string
+
+- name: targetRid
+  type: string
+
+- name: architecture
+  type: string
+
+- name: enablePoison
+  type: boolean
+
+jobs:
+- job: ${{ parameters.buildName }}_${{ parameters.architecture }}
+  dependsOn: Get_Build_Information
+  timeoutInMinutes: 150
+  pool:
+    name: NetCore1ESPool-Svc-Internal
+    demands: ImageOverride -equals 1es-ubuntu-2004
+  variables:
+  - template: /eng/common/templates/variables/pool-providers.yml
+  - template: /eng/common/templates/variables/pipelines.yml
+  - name: InstallerBuildId
+    value: $[ dependencies.Get_Build_Information.outputs['ids.InstallerBuildId'] ]  
+  - name: VmrBuildId
+    value: $[ dependencies.Get_Build_Information.outputs['ids.VmrBuildId'] ] 
+  - name: TestResultsFileName
+    value: ${{ parameters.buildName }}_${{ parameters.architecture }}_SDKComparisonTests.trx
+
+  steps:
+  - task: DownloadPipelineArtifact@2
+    displayName: Download MSFT SDK
+    inputs:
+      buildType: specific
+      buildVersionToDownload: specific
+      project: internal
+      pipeline: $(INSTALLER_OFFICIAL_CI_PIPELINE_ID)
+      buildId: $(InstallerBuildId)
+      artifact: BlobArtifacts
+      patterns: '**/dotnet-sdk-+([0-9]).+([0-9]).+([0-9])?(-@(alpha|preview|rc|rtm)*)-linux-${{ parameters.architecture }}.tar.gz' 
+      allowPartiallySucceededBuilds: true
+      allowFailedBuilds: true
+      downloadPath: $(Pipeline.Workspace)/Artifacts
+      checkDownloadedFiles: true
+
+  - task: DownloadPipelineArtifact@2
+    displayName: Download Source Build SDK
+    inputs:
+      buildType: specific
+      buildVersionToDownload: specific
+      project: internal
+      pipeline: $(DOTNET_DOTNET_CI_PIPELINE_ID)
+      buildId: $(VmrBuildId)
+      artifact: ${{ parameters.buildName }}_${{ parameters.architecture }}_Artifacts
+      patterns: '**/dotnet-sdk-+([0-9]).+([0-9]).+([0-9])?(-@(alpha|preview|rc|rtm)*)-${{ parameters.targetRid }}.tar.gz' 
+      allowPartiallySucceededBuilds: true
+      allowFailedBuilds: true
+      downloadPath: $(Pipeline.Workspace)/Artifacts
+      checkDownloadedFiles: true
+    
+  - script: |
+      set -x
+
+      msftSdkTarballName=$(find "$(Pipeline.Workspace)/Artifacts" -name "dotnet-sdk-*-linux-${{ parameters.architecture }}.tar.gz" -exec basename {} \;)
+
+      if [[ -z $msftSdkTarballName ]]; then
+        fail "Microsoft SDK tarball does not exist in '$(Pipeline.Workspace)/Artifacts'"
+        exit 1
+      fi
+
+      sdkTarballName=$(find "$(Pipeline.Workspace)/Artifacts" -name "dotnet-sdk-*-${{ parameters.targetRid }}.tar.gz" -exec basename {} \;)
+
+      if [[ -z $sdkTarballName ]]; then
+        fail "Source-build SDK tarball does not exist in '$(Pipeline.Workspace)/Artifacts'"
+        exit 1
+      fi
+
+      eng/common/build.sh --projects $(Build.SourcesDirectory)/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Microsoft.DotNet.SourceBuild.SmokeTests.csproj --restore
+
+      envArgs+=" -e SMOKE_TESTS_MSFT_SDK_TARBALL_PATH=$(Pipeline.Workspace)/Artifacts/$msftSdkTarballName"
+      envArgs+=" -e SMOKE_TESTS_SDK_TARBALL_PATH=$(Pipeline.Workspace)/Artifacts/$sdkTarballName"
+      envArgs+=" -e SMOKE_TESTS_SOURCEBUILT_ARTIFACTS_PATH= "
+      envArgs+=" -e SMOKE_TESTS_EXCLUDE_OMNISHARP=true -e SMOKE_TESTS_WARN_SDK_CONTENT_DIFFS=true -e SMOKE_TESTS_RUNNING_IN_CI=true -e SMOKE_TESTS_TARGET_RID=${{ parameters.targetRid }} -e SMOKE_TESTS_PORTABLE_RID=linux-x64 -e SMOKE_TESTS_CUSTOM_PACKAGES_PATH= -e SMOKE_TESTS_INCLUDE=CompareMsftToSb"
+      poisonArg=''
+
+      if [[ '${{ parameters.enablePoison }}' == 'True' ]]; then
+        poisonArg='--poison'
+        dockerEnvArgs+=" -e SMOKE_TESTS_WARN_POISON_DIFFS=true"
+      fi
+       
+      .dotnet/dotnet test $(Build.SourcesDirectory)/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Microsoft.DotNet.SourceBuild.SmokeTests.csproj $envArgs --logger:'trx;LogFileName=$(TestResultsFileName)' --logger:'console;verbosity=detailed'
+    displayName: Run Tests
+    workingDirectory: $(Build.SourcesDirectory)
+
+  - task: PublishTestResults@2
+    displayName: Publish Test Results
+    condition: succeededOrFailed()
+    continueOnError: true
+    inputs:
+      testRunner: vSTest
+      testResultsFiles: '*.trx'
+      searchFolder: $(Build.SourcesDirectory)/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/TestResults
+      mergeTestResults: true
+      publishRunAttachments: true
+      testRunTitle: SourceBuild_SdkComparisonTests_$(Agent.JobName)

src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/BasicScenarioTests.cs

@@ -19,9 +19,9 @@ public class BasicScenarioTests : SmokeTests
 {
     public BasicScenarioTests(ITestOutputHelper outputHelper) : base(outputHelper) { }
 
-    [Theory]
+    [SkippableTheory()]
     [MemberData(nameof(GetScenarioObjects))]
-    public void VerifyScenario(TestScenario scenario) => scenario.Execute(DotNetHelper);
+    public void VerifyBasicScenario(TestScenario scenario) => scenario.Execute(DotNetHelper);
 
     private static IEnumerable<object[]> GetScenarioObjects() => GetScenarios().Select(scenario => new object[] { scenario });
 

src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Config.cs

@@ -11,6 +11,7 @@ internal static class Config
 {
     public const string DotNetDirectoryEnv = "SMOKE_TESTS_DOTNET_DIR";
     public const string ExcludeOmniSharpEnv = "SMOKE_TESTS_EXCLUDE_OMNISHARP";
+    public const string IncludeTestsEnv = "SMOKE_TESTS_INCLUDE";
     public const string MsftSdkTarballPathEnv = "SMOKE_TESTS_MSFT_SDK_TARBALL_PATH";
     public const string PoisonReportPathEnv = "SMOKE_TESTS_POISON_REPORT_PATH";
     public const string PortableRidEnv = "SMOKE_TESTS_PORTABLE_RID";
@@ -45,4 +46,13 @@ internal static class Config
     // Indicates whether the tests are being run in the context of a CI pipeline
     public static bool RunningInCI { get; } =
         bool.TryParse(Environment.GetEnvironmentVariable(RunningInCIEnv), out bool runningInCI) && runningInCI;
+
+    public static string[] IncludedTests
+    {
+        get
+        {
+            var include = Environment.GetEnvironmentVariable(IncludeTestsEnv);
+            return include == null ? Array.Empty<string>() : include.Split(',');
+        }
+    }
 }

src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/DotNetWatchTests.cs

@@ -13,7 +13,7 @@ public class DotNetWatchTests : SmokeTests
 {
     public DotNetWatchTests(ITestOutputHelper outputHelper) : base(outputHelper) { }
 
-    [Fact]
+    [SkippableFact()]
     public void WatchTests()
     {
         string projectDirectory = DotNetHelper.ExecuteNew(DotNetTemplate.Console.GetName(), nameof(DotNetWatchTests));

src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/OmniSharpTests.cs

@@ -34,7 +34,7 @@ public class OmniSharpTests : SmokeTests
     [InlineData(DotNetTemplate.WebApi)]
     [InlineData(DotNetTemplate.Worker)]
     [InlineData(DotNetTemplate.XUnit)]
-    public async void VerifyScenario(DotNetTemplate template)
+    public async void VerifyOmniSharpScenario(DotNetTemplate template)
     {
         await InitializeOmniSharp();
 

src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SkippableFactAttribute.cs

@@ -3,6 +3,8 @@
 // See the LICENSE file in the project root for more information.
 
 using System;
+using System.Linq;
+using System.Runtime.CompilerServices;
 using Xunit;
 
 namespace Microsoft.DotNet.SourceBuild.SmokeTests;
@@ -12,14 +14,29 @@ namespace Microsoft.DotNet.SourceBuild.SmokeTests;
 /// </summary>
 internal class SkippableFactAttribute : FactAttribute
 {
-    public SkippableFactAttribute(string envName, bool skipOnNullOrWhiteSpace = false, bool skipOnTrue = false) =>
+    public SkippableFactAttribute([CallerMemberName] string name = "") =>
-        CheckEnvs(skipOnNullOrWhiteSpace, skipOnTrue, (skip) => Skip = skip, envName);
+        CheckIncluded(name, (skip) => Skip = skip);
 
-    public SkippableFactAttribute(string[] envNames, bool skipOnNullOrWhiteSpace = false, bool skipOnTrue = false) =>
+    public SkippableFactAttribute(string envName, bool skipOnNullOrWhiteSpace = false, bool skipOnTrue = false, [CallerMemberName] string testName = "") =>
-        CheckEnvs(skipOnNullOrWhiteSpace, skipOnTrue, (skip) => Skip = skip, envNames);
+        CheckEnvs(skipOnNullOrWhiteSpace, skipOnTrue, (skip) => Skip = skip, testName, envName);
 
-    public static void CheckEnvs(bool skipOnNullOrWhiteSpace, bool skipOnTrue, Action<string> setSkip, params string[] envNames)
+    public SkippableFactAttribute(string[] envNames, bool skipOnNullOrWhiteSpace = false, bool skipOnTrue = false, [CallerMemberName] string testName = "") =>
+        CheckEnvs(skipOnNullOrWhiteSpace, skipOnTrue, (skip) => Skip = skip, testName, envNames);
+
+    public static void CheckIncluded(string methodName, Action<string> setSkip)
     {
+        var included = Config.IncludedTests;
+        if (included.Length != 0 && !included.Contains(methodName))
+        {
+            setSkip($"Skipping because `{methodName}` is not included");
+            return;
+        }
+    }
+
+    public static void CheckEnvs(bool skipOnNullOrWhiteSpace, bool skipOnTrue, Action<string> setSkip, string testName, params string[] envNames)
+    {
+        CheckIncluded(testName, setSkip);
+
         foreach (string envName in envNames)
         {
             string? envValue = Environment.GetEnvironmentVariable(envName);

src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SkippableTheoryAttribute.cs

@@ -2,6 +2,7 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
+using System.Runtime.CompilerServices;
 using Xunit;
 
 namespace Microsoft.DotNet.SourceBuild.SmokeTests;
@@ -11,9 +12,12 @@ namespace Microsoft.DotNet.SourceBuild.SmokeTests;
 /// </summary>
 internal class SkippableTheoryAttribute : TheoryAttribute
 {
-    public SkippableTheoryAttribute(string envName, bool skipOnNullOrWhiteSpace = false, bool skipOnTrue = false) =>
+    public SkippableTheoryAttribute([CallerMemberName] string testName = "") =>
-        SkippableFactAttribute.CheckEnvs(skipOnNullOrWhiteSpace, skipOnTrue, (skip) => Skip = skip, envName);
+        SkippableFactAttribute.CheckIncluded(testName, (skip) => Skip = skip);
 
-    public SkippableTheoryAttribute(string[] envNames, bool skipOnNullOrWhiteSpace = false, bool skipOnTrue = false) =>
+    public SkippableTheoryAttribute(string envName, bool skipOnNullOrWhiteSpace = false, bool skipOnTrue = false, [CallerMemberName] string testName = "") =>
-        SkippableFactAttribute.CheckEnvs(skipOnNullOrWhiteSpace, skipOnTrue, (skip) => Skip = skip, envNames);
+        SkippableFactAttribute.CheckEnvs(skipOnNullOrWhiteSpace, skipOnTrue, (skip) => Skip = skip, testName, envName);
+
+    public SkippableTheoryAttribute(string[] envNames, bool skipOnNullOrWhiteSpace = false, bool skipOnTrue = false, [CallerMemberName] string testName = "") =>
+        SkippableFactAttribute.CheckEnvs(skipOnNullOrWhiteSpace, skipOnTrue, (skip) => Skip = skip, testName, envNames);
 }

src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/WebScenarioTests.cs

@@ -18,9 +18,9 @@ public class WebScenarioTests : SmokeTests
 {
     public WebScenarioTests(ITestOutputHelper outputHelper) : base(outputHelper) { }
 
-    [Theory]
+    [SkippableTheory()]
     [MemberData(nameof(GetScenarioObjects))]
-    public void VerifyScenario(TestScenario scenario) => scenario.Execute(DotNetHelper);
+    public void VerifyWebScenario(TestScenario scenario) => scenario.Execute(DotNetHelper);
 
     private static IEnumerable<object[]> GetScenarioObjects() => GetScenarios().Select(scenario => new object[] { scenario });