[main] Add CI for building with previously source-built SDK (#15603)

Co-authored-by: Michael Simons <msimons@microsoft.com>
2023-03-03 17:53:19 -08:00 · 2023-03-03 17:53:19 -08:00 · 9cf0095d08
commit 9cf0095d08
parent 9fd347ecc6
4 changed files with 87 additions and 32 deletions
--- a/eng/pipelines/templates/jobs/vmr-build.yml
+++ b/eng/pipelines/templates/jobs/vmr-build.yml
@ -58,6 +58,10 @@ parameters:
  type: boolean
  default: false

+- name: withPreviousSDK
+  type: boolean
+  default: false
+
 jobs:
 - job: ${{ parameters.buildName }}_${{ parameters.architecture }}
  timeoutInMinutes: 150
@ -76,12 +80,8 @@ jobs:
  - ${{ else }}:
    - name: BotAccount-dotnet-bot-repo-PAT
      value: N/A
-  - ${{ if eq(parameters.reuseBuildArtifactsFrom, '') }}:
-    - name: additionalBuildArgs
-      value: ''
-  - ${{ else }}:
-    - name: additionalBuildArgs
-      value: ' --with-sdk /vmr/.dotnet'
+  - name: additionalBuildArgs
+    value: ''

  # Location of the VMR sources
  # We either build the repo directly, or we extract them outside (which is what partners do)
@ -147,16 +147,45 @@ jobs:
        Contents: '*.tar.gz'
        TargetFolder: ${{ variables.sourcesPath }}/prereqs/packages/archive/

+  - ${{ if eq(parameters.withPreviousSDK, 'true') }}:
+    - script: |
+        set -euo pipefail
+
+        packageVersionsPath="${{ variables.sourcesPath }}/eng/Versions.props"
+        notFoundMessage="No source-built SDK found to download..."
+
+        echo "Looking for source-built SDK to download..."
+        archiveUrlLine=`grep -m 1 "<PrivateSourceBuiltSdkUrl_CentOS8Stream>" "$packageVersionsPath" || :`
+        urlPattern="<PrivateSourceBuiltSdkUrl_CentOS8Stream>(.*)</PrivateSourceBuiltSdkUrl_CentOS8Stream>"
+
+        if [[ $archiveUrlLine =~ $urlPattern ]]; then
+          archiveUrl="${BASH_REMATCH[1]}"
+          downloadDir="$(sourcesPath)/prereqs/packages/archive/"
+          (cd $downloadDir && curl --retry 5 -O $archiveUrl)
+        else
+          echo "$notFoundMessage"
+          exit 1
+        fi
+      displayName: Setup Previously Source-Built SDK
+
  - script: |
      set -x

-      if [[ -z '${{ parameters.reuseBuildArtifactsFrom }}' ]]; then
+      if [[ -z '${{ parameters.reuseBuildArtifactsFrom }}' && '${{ parameters.withPreviousSDK }}' != 'True' ]]; then
        docker run --rm -v "$(sourcesPath):/vmr" -w /vmr ${{ parameters.container }} ./prep.sh
      else
        mkdir $(sourcesPath)/.dotnet
        previousSdkPath="$(sourcesPath)/prereqs/packages/archive/dotnet-sdk-*.tar.gz"
        eval tar -ozxf "$previousSdkPath" -C "$(sourcesPath)/.dotnet"
        eval rm -f "$previousSdkPath"
+
+        if [[ '${{ parameters.withPreviousSDK }}' == 'True' ]]; then
+          # Source-built artifacts are from CentOS 8 Stream. We want to download them without
+          # downloading portable versions from the internet.
+          docker run --rm -v "$(sourcesPath):/vmr" -w /vmr ${{ parameters.container }} ./prep.sh --no-bootstrap
+        fi
+
+        echo "##vso[task.setvariable variable=additionalBuildArgs]--with-sdk /vmr/.dotnet"
      fi
    displayName: Prep the Build

--- a/eng/pipelines/templates/stages/vmr-build.yml
+++ b/eng/pipelines/templates/stages/vmr-build.yml
@ -67,7 +67,7 @@ stages:
  jobs:
  - template: ../jobs/vmr-build.yml
    parameters:
-      buildName: CentOSStream8_Online
+      buildName: CentOSStream8_Online_MsftSdk
      isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }}
      vmrBranch: ${{ variables.VmrBranch }}
      architecture: x64
@ -81,11 +81,30 @@ stages:
      excludeSdkContentTests: true       # ✅
      overrideDistroDisablingSha1: false # 🚫
      runOnline: true                    # ✅
+      withPreviousSDK: false             # 🚫

  - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}:
    - template: ../jobs/vmr-build.yml
      parameters:
-        buildName: CentOSStream8_Offline
+        buildName: CentOSStream8_Online_PreviousSourceBuiltSdk
+        isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }}
+        vmrBranch: ${{ variables.VmrBranch }}
+        architecture: x64
+        pool:
+          name: ${{ variables.defaultPoolName }}
+          demands: ${{ variables.defaultPoolDemands }}
+        container: ${{ parameters.centOSStream8Container }}
+        buildFromArchive: false            # 🚫
+        enablePoison: false                # 🚫
+        excludeOmniSharpTests: true        # ✅
+        excludeSdkContentTests: true       # ✅
+        overrideDistroDisablingSha1: false # 🚫
+        runOnline: true                    # ✅
+        withPreviousSDK: true              # ✅
+
+    - template: ../jobs/vmr-build.yml
+      parameters:
+        buildName: CentOSStream8_Offline_MsftSdk
        isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }}
        vmrBranch: ${{ variables.VmrBranch }}
        architecture: x64
@ -99,10 +118,11 @@ stages:
        excludeSdkContentTests: false      # 🚫
        overrideDistroDisablingSha1: false # 🚫
        runOnline: false                   # 🚫
+        withPreviousSDK: false             # 🚫

    - template: ../jobs/vmr-build.yml
      parameters:
-        buildName: CentOSStream9_Offline
+        buildName: CentOSStream9_Offline_MsftSdk
        isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }}
        vmrBranch: ${{ variables.VmrBranch }}
        architecture: x64
@ -116,10 +136,11 @@ stages:
        excludeSdkContentTests: false     # 🚫
        overrideDistroDisablingSha1: true # ✅
        runOnline: false                  # 🚫
+        withPreviousSDK: false            # 🚫

    - template: ../jobs/vmr-build.yml
      parameters:
-        buildName: Fedora36_Offline
+        buildName: Fedora36_Offline_MsftSdk
        isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }}
        vmrBranch: ${{ variables.VmrBranch }}
        architecture: x64
@ -133,10 +154,11 @@ stages:
        excludeSdkContentTests: false      # 🚫
        overrideDistroDisablingSha1: false # 🚫
        runOnline: false                   # 🚫
+        withPreviousSDK: false             # 🚫

    - template: ../jobs/vmr-build.yml
      parameters:
-        buildName: Ubuntu2004_Offline
+        buildName: Ubuntu2004_Offline_MsftSdk
        isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }}
        vmrBranch: ${{ variables.VmrBranch }}
        architecture: x64
@ -150,10 +172,11 @@ stages:
        excludeSdkContentTests: false      # 🚫
        overrideDistroDisablingSha1: false # 🚫
        runOnline: false                   # 🚫
+        withPreviousSDK: false             # 🚫

    - template: ../jobs/vmr-build.yml
      parameters:
-        buildName: Debian11_Offline
+        buildName: Debian11_Offline_MsftSdk
        isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }}
        vmrBranch: ${{ variables.VmrBranch }}
        architecture: arm64
@ -165,10 +188,11 @@ stages:
        excludeSdkContentTests: false      # 🚫
        overrideDistroDisablingSha1: false # 🚫
        runOnline: false                   # 🚫
+        withPreviousSDK: false             # 🚫

    - template: ../jobs/vmr-build.yml
      parameters:
-        buildName: Fedora36_Offline_Using_Previous
+        buildName: Fedora36_Offline_CurrentSourceBuiltSdk
        isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }}
        vmrBranch: ${{ variables.VmrBranch }}
        architecture: x64
@ -182,4 +206,5 @@ stages:
        excludeSdkContentTests: true       # ✅
        overrideDistroDisablingSha1: false # 🚫
        runOnline: false                   # 🚫
-        reuseBuildArtifactsFrom: Fedora36_Offline
+        withPreviousSDK: false             # 🚫
+        reuseBuildArtifactsFrom: Fedora36_Offline_MsftSdk
--- a/src/SourceBuild/content/eng/Versions.props
+++ b/src/SourceBuild/content/eng/Versions.props
@ -12,19 +12,14 @@

  <PropertyGroup>
    <!--
-      Building .NET from source depends on one or two tar.gz files depending on the branch's current
+      Building .NET from source depends on several archives, depending on the branch's current
      source-buildability status.

-      PrivateSourceBuiltArtifactsPackageVersion is a tar.gz of .NET build outputs from a previous
-      build needed to build the current version of .NET. This is always defined, because .NET needs
-      to be bootstrappable at any point in time.
-
-      PrivateSourceBuiltPrebuiltsPackageVersion is a tar.gz of assets downloaded from the internet
-      that are needed to build the current version of .NET. Early in the lifecycle of a .NET major
-      or minor release, prebuilts may be needed. When the release is mature, prebuilts are not
-      necessary, and this property is removed from the file.
+      These URLs can't be composed from their base URL and version as we read them from the
+      prep.sh and pipeline scripts, outside of MSBuild.
    -->
-    <PrivateSourceBuiltArtifactsPackageVersion>0.1.0-8.0.100-5.centos.8-x64</PrivateSourceBuiltArtifactsPackageVersion>
-    <PrivateSourceBuiltPrebuiltsPackageVersion>0.1.0-8.0.100-8.centos.8-x64</PrivateSourceBuiltPrebuiltsPackageVersion>
+    <PrivateSourceBuiltArtifactsUrl>https://dotnetcli.azureedge.net/source-built-artifacts/assets/Private.SourceBuilt.Artifacts.0.1.0-8.0.100-5.centos.8-x64.tar.gz</PrivateSourceBuiltArtifactsUrl>
+    <PrivateSourceBuiltPrebuiltsUrl>https://dotnetcli.azureedge.net/source-built-artifacts/assets/Private.SourceBuilt.Prebuilts.0.1.0-8.0.100-8.centos.8-x64.tar.gz</PrivateSourceBuiltPrebuiltsUrl>
+    <PrivateSourceBuiltSdkUrl_CentOS8Stream>https://dotnetcli.azureedge.net/source-built-artifacts/sdks/dotnet-sdk-8.0.100-preview.1.23115.1-centos.8-x64.tar.gz</PrivateSourceBuiltSdkUrl_CentOS8Stream>
  </PropertyGroup>
 </Project>
--- a/src/SourceBuild/content/prep.sh
+++ b/src/SourceBuild/content/prep.sh
@ -9,10 +9,12 @@ usage() {
    echo ""
    echo "  Prepares the environment to be built by downloading Private.SourceBuilt.Artifacts.*.tar.gz and"
    echo "  installing the version of dotnet referenced in global.json"
+    echo "options:"
+    echo "  --no-bootstrap Don't replace portable packages in the download source-built artifacts"
    echo ""
 }

-buildBootstrap=false
+buildBootstrap=true
 positional_args=()
 while :; do
    if [ $# -le 0 ]; then
@ -24,6 +26,9 @@ while :; do
            usage
            exit 0
            ;;
+        --no-bootstrap)
+            buildBootstrap=false
+            ;;
        *)
            positional_args+=("$1")
            ;;
@ -69,15 +74,14 @@ function DownloadArchive {
    baseFileName="$2"
    isRequired="$3"

-    sourceBuiltArtifactsTarballUrl="https://dotnetcli.azureedge.net/source-built-artifacts/assets/"
    packageVersionsPath="$SCRIPT_ROOT/eng/Versions.props"
    notFoundMessage="No source-built $archiveType found to download..."

    echo "  Looking for source-built $archiveType to download..."
-    archiveVersionLine=`grep -m 1 "<PrivateSourceBuilt${archiveType}PackageVersion>" "$packageVersionsPath" || :`
-    versionPattern="<PrivateSourceBuilt${archiveType}PackageVersion>(.*)</PrivateSourceBuilt${archiveType}PackageVersion>"
+    archiveVersionLine=`grep -m 1 "<PrivateSourceBuilt${archiveType}Url>" "$packageVersionsPath" || :`
+    versionPattern="<PrivateSourceBuilt${archiveType}Url>(.*)</PrivateSourceBuilt${archiveType}Url>"
    if [[ $archiveVersionLine =~ $versionPattern ]]; then
-        archiveUrl="${sourceBuiltArtifactsTarballUrl}${baseFileName}.${BASH_REMATCH[1]}.tar.gz"
+        archiveUrl="${BASH_REMATCH[1]}"
        echo "  Downloading source-built $archiveType from $archiveUrl..."
        (cd $packagesArchiveDir && curl --retry 5 -O $archiveUrl)
    elif [ "$isRequired" == "true" ]; then
@ -124,7 +128,9 @@ fi
 # Read the eng/Versions.props to get the archives to download and download them
 if [ "$downloadArtifacts" == "true" ]; then
    DownloadArchive "Artifacts" $artifactsBaseFileName "true"
-    BootstrapArtifacts
+    if [ "$buildBootstrap" == "true" ]; then
+        BootstrapArtifacts
+    fi
 fi

 if [ "$downloadPrebuilts" == "true" ]; then