[main] Add CI for building with previously source-built SDK (#15603)

Co-authored-by: Michael Simons <msimons@microsoft.com>

eng/pipelines/templates/jobs/vmr-build.yml

@@ -58,6 +58,10 @@ parameters:
   type: boolean
   default: false
 
+- name: withPreviousSDK
+  type: boolean
+  default: false
+
 jobs:
 - job: ${{ parameters.buildName }}_${{ parameters.architecture }}
   timeoutInMinutes: 150
@@ -76,12 +80,8 @@ jobs:
   - ${{ else }}:
     - name: BotAccount-dotnet-bot-repo-PAT
       value: N/A
-  - ${{ if eq(parameters.reuseBuildArtifactsFrom, '') }}:
+  - name: additionalBuildArgs
-    - name: additionalBuildArgs
+    value: ''
-      value: ''
-  - ${{ else }}:
-    - name: additionalBuildArgs
-      value: ' --with-sdk /vmr/.dotnet'
 
   # Location of the VMR sources
   # We either build the repo directly, or we extract them outside (which is what partners do)
@@ -147,16 +147,45 @@ jobs:
         Contents: '*.tar.gz'
         TargetFolder: ${{ variables.sourcesPath }}/prereqs/packages/archive/
 
+  - ${{ if eq(parameters.withPreviousSDK, 'true') }}:
+    - script: |
+        set -euo pipefail
+
+        packageVersionsPath="${{ variables.sourcesPath }}/eng/Versions.props"
+        notFoundMessage="No source-built SDK found to download..."
+
+        echo "Looking for source-built SDK to download..."
+        archiveUrlLine=`grep -m 1 "<PrivateSourceBuiltSdkUrl_CentOS8Stream>" "$packageVersionsPath" || :`
+        urlPattern="<PrivateSourceBuiltSdkUrl_CentOS8Stream>(.*)</PrivateSourceBuiltSdkUrl_CentOS8Stream>"
+
+        if [[ $archiveUrlLine =~ $urlPattern ]]; then
+          archiveUrl="${BASH_REMATCH[1]}"
+          downloadDir="$(sourcesPath)/prereqs/packages/archive/"
+          (cd $downloadDir && curl --retry 5 -O $archiveUrl)
+        else
+          echo "$notFoundMessage"
+          exit 1
+        fi
+      displayName: Setup Previously Source-Built SDK
+
   - script: |
       set -x
 
-      if [[ -z '${{ parameters.reuseBuildArtifactsFrom }}' ]]; then
+      if [[ -z '${{ parameters.reuseBuildArtifactsFrom }}' && '${{ parameters.withPreviousSDK }}' != 'True' ]]; then
         docker run --rm -v "$(sourcesPath):/vmr" -w /vmr ${{ parameters.container }} ./prep.sh
       else
         mkdir $(sourcesPath)/.dotnet
         previousSdkPath="$(sourcesPath)/prereqs/packages/archive/dotnet-sdk-*.tar.gz"
         eval tar -ozxf "$previousSdkPath" -C "$(sourcesPath)/.dotnet"
         eval rm -f "$previousSdkPath"
+
+        if [[ '${{ parameters.withPreviousSDK }}' == 'True' ]]; then
+          # Source-built artifacts are from CentOS 8 Stream. We want to download them without
+          # downloading portable versions from the internet.
+          docker run --rm -v "$(sourcesPath):/vmr" -w /vmr ${{ parameters.container }} ./prep.sh --no-bootstrap
+        fi
+
+        echo "##vso[task.setvariable variable=additionalBuildArgs]--with-sdk /vmr/.dotnet"
       fi
     displayName: Prep the Build
 

eng/pipelines/templates/stages/vmr-build.yml

@@ -67,7 +67,7 @@ stages:
   jobs:
   - template: ../jobs/vmr-build.yml
     parameters:
-      buildName: CentOSStream8_Online
+      buildName: CentOSStream8_Online_MsftSdk
       isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }}
       vmrBranch: ${{ variables.VmrBranch }}
       architecture: x64
@@ -81,11 +81,30 @@ stages:
       excludeSdkContentTests: true       # ✅
       overrideDistroDisablingSha1: false # 🚫
       runOnline: true                    # ✅
+      withPreviousSDK: false             # 🚫
 
   - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}:
     - template: ../jobs/vmr-build.yml
       parameters:
-        buildName: CentOSStream8_Offline
+        buildName: CentOSStream8_Online_PreviousSourceBuiltSdk
+        isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }}
+        vmrBranch: ${{ variables.VmrBranch }}
+        architecture: x64
+        pool:
+          name: ${{ variables.defaultPoolName }}
+          demands: ${{ variables.defaultPoolDemands }}
+        container: ${{ parameters.centOSStream8Container }}
+        buildFromArchive: false            # 🚫
+        enablePoison: false                # 🚫
+        excludeOmniSharpTests: true        # ✅
+        excludeSdkContentTests: true       # ✅
+        overrideDistroDisablingSha1: false # 🚫
+        runOnline: true                    # ✅
+        withPreviousSDK: true              # ✅
+
+    - template: ../jobs/vmr-build.yml
+      parameters:
+        buildName: CentOSStream8_Offline_MsftSdk
         isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }}
         vmrBranch: ${{ variables.VmrBranch }}
         architecture: x64
@@ -99,10 +118,11 @@ stages:
         excludeSdkContentTests: false      # 🚫
         overrideDistroDisablingSha1: false # 🚫
         runOnline: false                   # 🚫
+        withPreviousSDK: false             # 🚫
 
     - template: ../jobs/vmr-build.yml
       parameters:
-        buildName: CentOSStream9_Offline
+        buildName: CentOSStream9_Offline_MsftSdk
         isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }}
         vmrBranch: ${{ variables.VmrBranch }}
         architecture: x64
@@ -116,10 +136,11 @@ stages:
         excludeSdkContentTests: false     # 🚫
         overrideDistroDisablingSha1: true # ✅
         runOnline: false                  # 🚫
+        withPreviousSDK: false            # 🚫
 
     - template: ../jobs/vmr-build.yml
       parameters:
-        buildName: Fedora36_Offline
+        buildName: Fedora36_Offline_MsftSdk
         isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }}
         vmrBranch: ${{ variables.VmrBranch }}
         architecture: x64
@@ -133,10 +154,11 @@ stages:
         excludeSdkContentTests: false      # 🚫
         overrideDistroDisablingSha1: false # 🚫
         runOnline: false                   # 🚫
+        withPreviousSDK: false             # 🚫
 
     - template: ../jobs/vmr-build.yml
       parameters:
-        buildName: Ubuntu2004_Offline
+        buildName: Ubuntu2004_Offline_MsftSdk
         isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }}
         vmrBranch: ${{ variables.VmrBranch }}
         architecture: x64
@@ -150,10 +172,11 @@ stages:
         excludeSdkContentTests: false      # 🚫
         overrideDistroDisablingSha1: false # 🚫
         runOnline: false                   # 🚫
+        withPreviousSDK: false             # 🚫
 
     - template: ../jobs/vmr-build.yml
       parameters:
-        buildName: Debian11_Offline
+        buildName: Debian11_Offline_MsftSdk
         isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }}
         vmrBranch: ${{ variables.VmrBranch }}
         architecture: arm64
@@ -165,10 +188,11 @@ stages:
         excludeSdkContentTests: false      # 🚫
         overrideDistroDisablingSha1: false # 🚫
         runOnline: false                   # 🚫
+        withPreviousSDK: false             # 🚫
 
     - template: ../jobs/vmr-build.yml
       parameters:
-        buildName: Fedora36_Offline_Using_Previous
+        buildName: Fedora36_Offline_CurrentSourceBuiltSdk
         isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }}
         vmrBranch: ${{ variables.VmrBranch }}
         architecture: x64
@@ -182,4 +206,5 @@ stages:
         excludeSdkContentTests: true       # ✅
         overrideDistroDisablingSha1: false # 🚫
         runOnline: false                   # 🚫
-        reuseBuildArtifactsFrom: Fedora36_Offline
+        withPreviousSDK: false             # 🚫
+        reuseBuildArtifactsFrom: Fedora36_Offline_MsftSdk

src/SourceBuild/content/eng/Versions.props

@@ -12,19 +12,14 @@
 
   <PropertyGroup>
     <!--
-      Building .NET from source depends on one or two tar.gz files depending on the branch's current
+      Building .NET from source depends on several archives, depending on the branch's current
       source-buildability status.
 
-      PrivateSourceBuiltArtifactsPackageVersion is a tar.gz of .NET build outputs from a previous
+      These URLs can't be composed from their base URL and version as we read them from the
-      build needed to build the current version of .NET. This is always defined, because .NET needs
+      prep.sh and pipeline scripts, outside of MSBuild.
-      to be bootstrappable at any point in time.
-
-      PrivateSourceBuiltPrebuiltsPackageVersion is a tar.gz of assets downloaded from the internet
-      that are needed to build the current version of .NET. Early in the lifecycle of a .NET major
-      or minor release, prebuilts may be needed. When the release is mature, prebuilts are not
-      necessary, and this property is removed from the file.
     -->
-    <PrivateSourceBuiltArtifactsPackageVersion>0.1.0-8.0.100-5.centos.8-x64</PrivateSourceBuiltArtifactsPackageVersion>
+    <PrivateSourceBuiltArtifactsUrl>https://dotnetcli.azureedge.net/source-built-artifacts/assets/Private.SourceBuilt.Artifacts.0.1.0-8.0.100-5.centos.8-x64.tar.gz</PrivateSourceBuiltArtifactsUrl>
-    <PrivateSourceBuiltPrebuiltsPackageVersion>0.1.0-8.0.100-8.centos.8-x64</PrivateSourceBuiltPrebuiltsPackageVersion>
+    <PrivateSourceBuiltPrebuiltsUrl>https://dotnetcli.azureedge.net/source-built-artifacts/assets/Private.SourceBuilt.Prebuilts.0.1.0-8.0.100-8.centos.8-x64.tar.gz</PrivateSourceBuiltPrebuiltsUrl>
+    <PrivateSourceBuiltSdkUrl_CentOS8Stream>https://dotnetcli.azureedge.net/source-built-artifacts/sdks/dotnet-sdk-8.0.100-preview.1.23115.1-centos.8-x64.tar.gz</PrivateSourceBuiltSdkUrl_CentOS8Stream>
   </PropertyGroup>
 </Project>

src/SourceBuild/content/prep.sh

@@ -9,10 +9,12 @@ usage() {
     echo ""
     echo "  Prepares the environment to be built by downloading Private.SourceBuilt.Artifacts.*.tar.gz and"
     echo "  installing the version of dotnet referenced in global.json"
+    echo "options:"
+    echo "  --no-bootstrap Don't replace portable packages in the download source-built artifacts"
     echo ""
 }
 
-buildBootstrap=false
+buildBootstrap=true
 positional_args=()
 while :; do
     if [ $# -le 0 ]; then
@@ -24,6 +26,9 @@ while :; do
             usage
             exit 0
             ;;
+        --no-bootstrap)
+            buildBootstrap=false
+            ;;
         *)
             positional_args+=("$1")
             ;;
@@ -69,15 +74,14 @@ function DownloadArchive {
     baseFileName="$2"
     isRequired="$3"
 
-    sourceBuiltArtifactsTarballUrl="https://dotnetcli.azureedge.net/source-built-artifacts/assets/"
     packageVersionsPath="$SCRIPT_ROOT/eng/Versions.props"
     notFoundMessage="No source-built $archiveType found to download..."
 
     echo "  Looking for source-built $archiveType to download..."
-    archiveVersionLine=`grep -m 1 "<PrivateSourceBuilt${archiveType}PackageVersion>" "$packageVersionsPath" || :`
+    archiveVersionLine=`grep -m 1 "<PrivateSourceBuilt${archiveType}Url>" "$packageVersionsPath" || :`
-    versionPattern="<PrivateSourceBuilt${archiveType}PackageVersion>(.*)</PrivateSourceBuilt${archiveType}PackageVersion>"
+    versionPattern="<PrivateSourceBuilt${archiveType}Url>(.*)</PrivateSourceBuilt${archiveType}Url>"
     if [[ $archiveVersionLine =~ $versionPattern ]]; then
-        archiveUrl="${sourceBuiltArtifactsTarballUrl}${baseFileName}.${BASH_REMATCH[1]}.tar.gz"
+        archiveUrl="${BASH_REMATCH[1]}"
         echo "  Downloading source-built $archiveType from $archiveUrl..."
         (cd $packagesArchiveDir && curl --retry 5 -O $archiveUrl)
     elif [ "$isRequired" == "true" ]; then
@@ -124,7 +128,9 @@ fi
 # Read the eng/Versions.props to get the archives to download and download them
 if [ "$downloadArtifacts" == "true" ]; then
     DownloadArchive "Artifacts" $artifactsBaseFileName "true"
-    BootstrapArtifacts
+    if [ "$buildBootstrap" == "true" ]; then
+        BootstrapArtifacts
+    fi
 fi
 
 if [ "$downloadPrebuilts" == "true" ]; then