Refactor the MSFT SDK content comparison tests into a separate pipeline (#15686)

eng/pipelines/source-build-sdk-diff-tests.yml

@@ -0,0 +1,56 @@
+schedules:
+- cron: "0 7 * * 1-5"
+  displayName: Run on weekdays at 7am UTC
+  branches:
+    include:
+    - main
+    - release/*
+
+pr: none
+trigger: none
+
+pool:
+  name: NetCore1ESPool-Svc-Internal
+  demands: ImageOverride -equals 1es-ubuntu-2004
+
+parameters:
+- name: dotnetDotnetRunId
+  displayName: 'Specific dotnet-dotnet run ID number (e.g `2108850`)'
+  type: string
+  default: ' '
+
+jobs:
+- template: templates/jobs/sdk-diff-tests.yml
+  parameters:
+    buildName: CentOSStream8_Offline_MsftSdk
+    targetRid: centos.8-x64
+    architecture: x64
+    dotnetDotnetRunId: ${{ parameters.dotnetDotnetRunId }}
+
+- template: templates/jobs/sdk-diff-tests.yml
+  parameters:
+    buildName: CentOSStream9_Offline_MsftSdk
+    targetRid: centos.9-x64
+    architecture: x64
+    dotnetDotnetRunId: ${{ parameters.dotnetDotnetRunId }}
+
+- template: templates/jobs/sdk-diff-tests.yml
+  parameters:
+    buildName: Fedora36_Offline_MsftSdk
+    targetRid: fedora.36-x64
+    architecture: x64
+    dotnetDotnetRunId: ${{ parameters.dotnetDotnetRunId }}
+
+- template: templates/jobs/sdk-diff-tests.yml
+  parameters:
+    buildName: Ubuntu2004_Offline_MsftSdk
+    targetRid: ubuntu.2004-x64
+    architecture: x64
+    dotnetDotnetRunId: ${{ parameters.dotnetDotnetRunId }}
+
+- template: templates/jobs/sdk-diff-tests.yml
+  parameters:
+    buildName: Debian11_Offline_MsftSdk
+    targetRid: debian.11-arm64
+    architecture: arm64
+    dotnetDotnetRunId: ${{ parameters.dotnetDotnetRunId }}

eng/pipelines/templates/jobs/sdk-diff-tests.yml

@@ -0,0 +1,159 @@
+parameters:
+- name: buildName
+  type: string
+
+- name: targetRid
+  type: string
+
+- name: architecture
+  type: string
+
+- name: dotnetDotnetRunId
+  type: string
+
+jobs:
+- job: ${{ parameters.buildName }}_${{ parameters.architecture }}
+  timeoutInMinutes: 150
+  pool:
+    name: NetCore1ESPool-Svc-Internal
+    demands: ImageOverride -equals 1es-ubuntu-2004
+  variables:
+  - template: ../variables/pipelines.yml
+  steps:
+  - script: |
+      dotnet_dotnet_build='${{ replace(parameters.dotnetDotnetRunId, ' ', '') }}'
+
+      if [[ -z "$dotnet_dotnet_build" ]]; then
+        dotnet_dotnet_build=$(az pipelines runs list --branch '$(Build.SourceBranch)' --organization '$(AZDO_ORG)' --project '$(AZDO_PROJECT)' --pipeline-ids '$(DOTNET_DOTNET_CI_PIPELINE_ID)' --status completed --top 1 --query "[].id" --output tsv)
+      fi
+
+      if [[ -z "$dotnet_dotnet_build" ]]; then
+        echo "Could not find a completed dotnet-dotnet build for branch '$(Build.SourceBranch)'"
+        exit 1
+      fi
+      
+      echo "Dotnet-dotnet build: https://dev.azure.com/dnceng/internal/_build/results?buildId=$dotnet_dotnet_build&view=results"
+
+      installer_sha=$(az pipelines build tag list --organization '$(AZDO_ORG)' --project '$(AZDO_PROJECT)' --build-id $dotnet_dotnet_build --output tsv | sed "s,installer-,,g")
+      installer_build=$(az pipelines runs list --organization '$(AZDO_ORG)' --project '$(AZDO_PROJECT)' --pipeline-ids '$(INSTALLER_OFFICIAL_CI_PIPELINE_ID)' --query "[?sourceVersion == '$installer_sha'].id" --output tsv)
+      if [[ -z "$installer_build" ]]; then
+        echo "Could not find a build of installer for commit '$installer_sha'"
+        exit 1
+      fi
+
+      echo "Installer build: https://dev.azure.com/dnceng/internal/_build/results?buildId=$installer_build&view=results"
+
+      echo "##vso[build.addbuildtag]installer-$installer_sha"
+      echo "##vso[task.setvariable variable=InstallerBuildId]$installer_build"
+      echo "##vso[task.setvariable variable=DotnetDotnetBuildId]$dotnet_dotnet_build"
+    displayName: Find associated builds
+    name: Get_Build_Ids
+    env:
+      AZURE_DEVOPS_EXT_PAT: $(System.AccessToken)
+
+  - task: DownloadPipelineArtifact@2
+    displayName: Download MSFT SDK
+    inputs:
+      buildType: specific
+      buildVersionToDownload: specific
+      project: internal
+      pipeline: $(INSTALLER_OFFICIAL_CI_PIPELINE_ID)
+      buildId: $(InstallerBuildId)
+      artifact: BlobArtifacts
+      patterns: '**/dotnet-sdk-+([0-9]).+([0-9]).+([0-9])?(-@(alpha|preview|rc|rtm)*)-linux-${{ parameters.architecture }}.tar.gz' 
+      allowPartiallySucceededBuilds: true
+      allowFailedBuilds: true
+      downloadPath: $(Pipeline.Workspace)/Artifacts
+      checkDownloadedFiles: true
+
+  - task: DownloadPipelineArtifact@2
+    displayName: Download Source Build SDK
+    inputs:
+      buildType: specific
+      buildVersionToDownload: specific
+      project: internal
+      pipeline: $(DOTNET_DOTNET_CI_PIPELINE_ID)
+      buildId: $(DotnetDotnetBuildId)
+      artifact: ${{ parameters.buildName }}_${{ parameters.architecture }}_Artifacts
+      patterns: '**/dotnet-sdk-+([0-9]).+([0-9]).+([0-9])?(-@(alpha|preview|rc|rtm)*)-${{ parameters.targetRid }}.tar.gz' 
+      allowPartiallySucceededBuilds: true
+      allowFailedBuilds: true
+      downloadPath: $(Pipeline.Workspace)/Artifacts
+      checkDownloadedFiles: true
+    
+  - script: |
+      msft_sdk_tarball_name=$(find "$(Pipeline.Workspace)/Artifacts" -name "dotnet-sdk-*-linux-${{ parameters.architecture }}.tar.gz" -exec basename {} \;)
+
+      if [[ -z "$msft_sdk_tarball_name" ]]; then
+        echo "Microsoft SDK tarball does not exist in '$(Pipeline.Workspace)/Artifacts'. The associated build 'https://dev.azure.com/dnceng/internal/_build/results?buildId=$(InstallerBuildId)&view=results' might have failed."
+        exit 1
+      fi
+
+      sdk_tarball_name=$(find "$(Pipeline.Workspace)/Artifacts" -name "dotnet-sdk-*-${{ parameters.targetRid }}.tar.gz" -exec basename {} \;)
+
+      if [[ -z "$sdk_tarball_name" ]]; then
+        echo "Source-build SDK tarball does not exist in '$(Pipeline.Workspace)/Artifacts'. The associated build 'https://dev.azure.com/dnceng/internal/_build/results?buildId=$(DotnetDotnetBuildId)&view=results' might have failed"
+        exit 1
+      fi
+
+      eng/common/build.sh -bl --projects $(Build.SourcesDirectory)/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Microsoft.DotNet.SourceBuild.SmokeTests.csproj --restore
+
+      echo "##vso[task.setvariable variable=MsftSdkTarballPath]$(Pipeline.Workspace)/Artifacts/$msft_sdk_tarball_name"
+      echo "##vso[task.setvariable variable=SdkTarballPath]$(Pipeline.Workspace)/Artifacts/$sdk_tarball_name"
+    displayName: Prepare Tests
+    workingDirectory: $(Build.SourcesDirectory)
+
+  - script: >
+      .dotnet/dotnet test
+      $(Build.SourcesDirectory)/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Microsoft.DotNet.SourceBuild.SmokeTests.csproj
+      --filter "FullyQualifiedName=Microsoft.DotNet.SourceBuild.SmokeTests.SdkContentTests.CompareMsftToSb"
+      --logger:'trx;LogFileName=$(Agent.JobName)_SDKDiffTests.trx'
+      --logger:'console;verbosity=detailed'
+      -c Release
+      -bl:$(Build.SourcesDirectory)/artifacts/log/Debug/BuildTests_$(date +"%m%d%H%M%S").binlog 
+      -flp:LogFile=$(Build.SourcesDirectory)/artifacts/logs/BuildTests_$(date +"%m%d%H%M%S").log
+      -clp:v=m
+      -e SMOKE_TESTS_MSFT_SDK_TARBALL_PATH=$(MsftSdkTarballPath)
+      -e SMOKE_TESTS_SDK_TARBALL_PATH=$(SdkTarballPath)
+      -e SMOKE_TESTS_SOURCEBUILT_ARTIFACTS_PATH= 
+      -e SMOKE_TESTS_WARN_SDK_CONTENT_DIFFS=false
+      -e SMOKE_TESTS_RUNNING_IN_CI=true
+      -e SMOKE_TESTS_TARGET_RID=${{ parameters.targetRid }}
+      -e SMOKE_TESTS_PORTABLE_RID=linux-x64
+      -e SMOKE_TESTS_CUSTOM_PACKAGES_PATH= 
+    displayName: Run Tests
+    workingDirectory: $(Build.SourcesDirectory)
+
+  - script: |
+      set -x
+      targetFolder=$(Build.StagingDirectory)/BuildLogs/
+      mkdir -p ${targetFolder}
+      cd "$(Build.SourcesDirectory)"
+      find artifacts/ -type f -name "BuildTests*.binlog" -exec cp {} --parents -t ${targetFolder} \;
+      find artifacts/ -type f -name "BuildTests*.log" -exec cp {} --parents -t ${targetFolder} \;
+      find artifacts/ -type f -name "Build.binlog" -exec cp {} --parents -t ${targetFolder} \;
+      cd "$(Build.SourcesDirectory)/src/SourceBuild/content"
+      find test/ -type f -name "*.binlog" -exec cp {} --parents -t ${targetFolder} \;
+      find test/ -type f -name "Updated*.diff" -exec cp {} --parents -t ${targetFolder} \;
+      find test/ -type f -name "Updated*.txt" -exec cp {} --parents -t ${targetFolder} \;
+    displayName: Prepare BuildLogs staging directory
+    continueOnError: true
+    condition: succeededOrFailed()
+
+  - publish: '$(Build.StagingDirectory)/BuildLogs'
+    artifact: $(Agent.JobName)_BuildLogs_Attempt$(System.JobAttempt)
+    displayName: Publish BuildLogs
+    continueOnError: true
+    condition: succeededOrFailed()
+
+  - task: PublishTestResults@2
+    displayName: Publish Test Results
+    condition: succeededOrFailed()
+    continueOnError: true
+    inputs:
+      testRunner: vSTest
+      testResultsFiles: '*.trx'
+      searchFolder: $(Build.SourcesDirectory)/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/TestResults
+      mergeTestResults: true
+      publishRunAttachments: true
+      testRunTitle: $(Agent.JobName)

eng/pipelines/templates/jobs/vmr-build.yml

@@ -31,11 +31,6 @@ parameters:
 - name: runOnline
   type: boolean
 
-# Skip running the SDK content smoke-tests
-- name: excludeSdkContentTests
-  type: boolean
-  default: false
-
 # Name of a previous job (from the same template as this) whose output will be used to build this job
 # The SDK from its artifacts is copied to vmr/.dotnet
 - name: reuseBuildArtifactsFrom
@@ -128,12 +123,6 @@ jobs:
       env:
         Token: $(dn-bot-dnceng-artifact-feeds-rw)
 
-  - ${{ if ne(parameters.excludeSdkContentTests, 'true') }}:
-    - download: current
-      artifact: BlobArtifacts
-      patterns: '**/dotnet-sdk-+([0-9]).+([0-9]).+([0-9])?(-@(alpha|preview|rc|rtm)*)-linux-${{ parameters.architecture }}.tar.gz'
-      displayName: Download MSFT SDK
-
   - ${{ if ne(parameters.reuseBuildArtifactsFrom, '') }}:
     - download: current
       artifact: ${{ parameters.reuseBuildArtifactsFrom }}_${{ parameters.architecture }}_Artifacts
@@ -221,12 +210,6 @@ jobs:
       dockerEnvArgs="-e SMOKE_TESTS_EXCLUDE_OMNISHARP=${{ parameters.excludeOmniSharpTests }} -e SMOKE_TESTS_WARN_SDK_CONTENT_DIFFS=true -e SMOKE_TESTS_RUNNING_IN_CI=true"
       poisonArg=''
 
-      if [[ '${{ parameters.excludeSdkContentTests }}' != 'True' ]]; then
-        dockerVolumeArgs+=" -v $(Pipeline.Workspace)/BlobArtifacts/:/BlobArtifacts"
-        msftSdkTarballName=$(find "$(Pipeline.Workspace)/BlobArtifacts/" -name "dotnet-sdk-*-linux-${{ parameters.architecture }}.tar.gz" -exec basename {} \;)
-        dockerEnvArgs+=" -e SMOKE_TESTS_MSFT_SDK_TARBALL_PATH=/BlobArtifacts/$msftSdkTarballName"
-      fi
-
       if [[ '${{ parameters.enablePoison }}' == 'True' ]]; then
         poisonArg='--poison'
         dockerEnvArgs+=" -e SMOKE_TESTS_WARN_POISON_DIFFS=true"

eng/pipelines/templates/stages/vmr-build.yml

@@ -78,7 +78,6 @@ stages:
       buildFromArchive: false            # 🚫
       enablePoison: false                # 🚫
       excludeOmniSharpTests: true        # ✅
-      excludeSdkContentTests: true       # ✅
       overrideDistroDisablingSha1: false # 🚫
       runOnline: true                    # ✅
       withPreviousSDK: false             # 🚫
@@ -115,7 +114,6 @@ stages:
         buildFromArchive: true             # ✅
         enablePoison: false                # 🚫
         excludeOmniSharpTests: true        # ✅
-        excludeSdkContentTests: false      # 🚫
         overrideDistroDisablingSha1: false # 🚫
         runOnline: false                   # 🚫
         withPreviousSDK: false             # 🚫
@@ -133,7 +131,6 @@ stages:
         buildFromArchive: true            # ✅
         enablePoison: false               # 🚫
         excludeOmniSharpTests: false      # 🚫
-        excludeSdkContentTests: false     # 🚫
         overrideDistroDisablingSha1: true # ✅
         runOnline: false                  # 🚫
         withPreviousSDK: false            # 🚫
@@ -151,7 +148,6 @@ stages:
         buildFromArchive: true             # ✅
         enablePoison: true                 # ✅
         excludeOmniSharpTests: false       # 🚫
-        excludeSdkContentTests: false      # 🚫
         overrideDistroDisablingSha1: false # 🚫
         runOnline: false                   # 🚫
         withPreviousSDK: false             # 🚫
@@ -169,7 +165,6 @@ stages:
         buildFromArchive: false            # 🚫
         enablePoison: false                # 🚫
         excludeOmniSharpTests: false       # 🚫
-        excludeSdkContentTests: false      # 🚫
         overrideDistroDisablingSha1: false # 🚫
         runOnline: false                   # 🚫
         withPreviousSDK: false             # 🚫
@@ -185,7 +180,6 @@ stages:
         buildFromArchive: false            # 🚫
         enablePoison: false                # 🚫
         excludeOmniSharpTests: false       # 🚫
-        excludeSdkContentTests: false      # 🚫
         overrideDistroDisablingSha1: false # 🚫
         runOnline: false                   # 🚫
         withPreviousSDK: false             # 🚫
@@ -203,7 +197,6 @@ stages:
         buildFromArchive: false            # 🚫
         enablePoison: false                # 🚫
         excludeOmniSharpTests: false       # 🚫
-        excludeSdkContentTests: true       # ✅
         overrideDistroDisablingSha1: false # 🚫
         runOnline: false                   # 🚫
         withPreviousSDK: false             # 🚫

eng/pipelines/templates/variables/pipelines.yml

@@ -0,0 +1,11 @@
+variables:
+- name: AZDO_PROJECT
+  value: internal
+- name: AZDO_ORG
+  value: https://dev.azure.com/dnceng/
+- name: INSTALLER_OFFICIAL_CI_PIPELINE_ID
+  value: 286
+- name: INSTALLER_TARBALL_BUILD_CI_PIPELINE_ID
+  value: 1011
+- name: DOTNET_DOTNET_CI_PIPELINE_ID
+  value: 1219

src/SourceBuild/content/eng/pipelines/ci.yml

@@ -1,55 +1,24 @@
-# This is the main build definition (PR+CI) for dotnet/dotnet
+# This is the main build definition (PR+CI) for dotnet/dotnet
-
+
-trigger:
+trigger:
-  batch: true
+  batch: true
-  branches:
+  branches:
-    include:
+    include:
-    - main
+    - main
-    - release/*
+    - release/*
-    - internal/release/*
+    - internal/release/*
-
+
-pr:
+pr:
-  branches:
+  branches:
-    include:
+    include:
-    - main
+    - main
-    - release/*
+    - release/*
-    - internal/release/*
+    - internal/release/*
-
+
-stages:
+stages:
-- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
-  - template: templates/stages/vmr-scan.yml
+  - template: templates/stages/vmr-scan.yml
-
+
-  # For rolling builds we want to build the MSFT SDK first so that we can
+- template: /src/installer/eng/pipelines/templates/stages/vmr-build.yml
-  # compare the contents with the source-built one later.
+  parameters:
-  # This only works because we don't run this test in PRs. If we decided
+    isBuiltFromVmr: true
-  # to run this test in PRs, we would need to build the MSFT SDK there too.
-  - stage: Build_MSFT_SDK
-    displayName: Build MSFT SDK
-    dependsOn: []
-    jobs:
-    - template: /src/installer/eng/build.yml
-      parameters:
-        agentOs: Linux
-        jobName: Build_LinuxPortable_Release_x64
-        buildConfiguration: Release
-        buildArchitecture: x64
-        linuxPortable: true
-        runTests: false
-        isBuiltFromVmr: true
-    - template: /src/installer/eng/build.yml
-      parameters:
-        agentOs: Linux
-        jobName: Build_Arm64_Release
-        buildConfiguration: Release
-        buildArchitecture: arm64
-        runtimeIdentifier: 'linux-arm64'
-        linuxPortable: true
-        runTests: false
-        isBuiltFromVmr: true
-
-- template: /src/installer/eng/pipelines/templates/stages/vmr-build.yml
-  parameters:
-    isBuiltFromVmr: true
-    ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
-      dependsOn: Build_MSFT_SDK
-      condition: always()