Split current CI pipline (#17370)

eng/pipelines/templates/jobs/vmr-build.yml

@@ -21,6 +21,10 @@ parameters:
 - name: architecture
   type: string
 
+- name: artifactsRid
+  type: string
+  default: ''
+
 - name: container
   type: string
 
@@ -141,17 +145,25 @@ jobs:
     - script: |
         set -euo pipefail
 
+        if [[ '${{ parameters.artifactsRid }}' == '' ]]; then
+           echo "'artifactsRid' is not specified. Cannot download source-built SDK."
+           exit 1
+        fi
+
         packageVersionsPath="${{ variables.sourcesPath }}/eng/Versions.props"
         notFoundMessage="No source-built SDK found to download..."
 
         echo "Looking for source-built SDK to download..."
-        archiveUrlLine=`grep -m 1 "<PrivateSourceBuiltSdkUrl_CentOS8Stream>" "$packageVersionsPath" || :`
+        archiveVersionLine=$(grep -m 1 "<PrivateSourceBuiltSdkVersion>" "$packageVersionsPath" || :)
-        urlPattern="<PrivateSourceBuiltSdkUrl_CentOS8Stream>(.*)</PrivateSourceBuiltSdkUrl_CentOS8Stream>"
+        versionPattern="<PrivateSourceBuiltSdkVersion>(.*)</PrivateSourceBuiltSdkVersion>"
 
-        if [[ $archiveUrlLine =~ $urlPattern ]]; then
+        if [[ $archiveVersionLine =~ $versionPattern ]]; then
-          archiveUrl="${BASH_REMATCH[1]}"
+          archiveVersion="${BASH_REMATCH[1]}"
+          archiveUrl="https://dotnetcli.azureedge.net/source-built-artifacts/sdks/dotnet-sdk-$archiveVersion-${{ parameters.artifactsRid }}.tar.gz"
           downloadDir="$(sourcesPath)/prereqs/packages/archive/"
-          (cd $downloadDir && curl --retry 5 -O $archiveUrl)
+
+          echo "Downloading source-built SDK from $archiveUrl..."
+          (cd "$downloadDir" && curl --retry 5 -O "$archiveUrl")
         else
           echo "$notFoundMessage"
           exit 1
@@ -163,8 +175,13 @@ jobs:
 
       customPrepArgs=""
       prepSdk=true
+
+      if [[ -n '${{ parameters.artifactsRid }}' ]]; then
+        customPrepArgs="${customPrepArgs} --artifacts-rid ${{ parameters.artifactsRid }}"
+      fi
+      
       if [[ '${{ parameters.withPreviousSDK }}' == 'True' ]]; then
-        # Source-built artifacts are from CentOS 8 Stream. We want to download them without
+        # Source-built artifacts are from CentOS 8 Stream or Alpine 3.17. We want to download them without
         # downloading portable versions from the internet.
         customPrepArgs="${customPrepArgs} --no-sdk --no-bootstrap"
         prepSdk=false

eng/pipelines/templates/stages/vmr-build.yml

@@ -8,6 +8,9 @@ parameters:
   # Branch of the VMR to use (to push to for internal builds)
   vmrBranch: $(Build.SourceBranch)
 
+  # True when the build is a lite build
+  isLiteBuild:
+
   # True when build is running from dotnet/dotnet directly
   isBuiltFromVmr:
 
@@ -67,8 +70,6 @@ stages:
 
   jobs:
 
-  # PR and CI legs ------------------------------------
-
   - template: ../jobs/vmr-build.yml
     parameters:
       # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline
@@ -89,7 +90,45 @@ stages:
 
   - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}:
 
-    # CI - Stage 1 x64 legs ------------------------------------
+    - template: ../jobs/vmr-build.yml
+      parameters:
+        # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline
+        buildName: CentOSStream8_Online_CurrentSourceBuiltSdk
+        isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }}
+        vmrBranch: ${{ variables.VmrBranch }}
+        architecture: x64
+        pool:
+          name: ${{ variables.defaultPoolName }}
+          demands: ${{ variables.defaultPoolDemands }}
+        container: ${{ parameters.centOSStream8Container }}
+        buildFromArchive: false            # 🚫
+        enablePoison: false                # 🚫
+        excludeOmniSharpTests: true        # ✅
+        runOnline: true                    # ✅
+        useMonoRuntime: false              # 🚫
+        withPreviousSDK: false             # 🚫
+        reuseBuildArtifactsFrom: CentOSStream8_Online_MsftSdk
+
+    - template: ../jobs/vmr-build.yml
+      parameters:
+        # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline
+        buildName: Alpine317_Offline_PreviousSourceBuiltSdk
+        isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }}
+        vmrBranch: ${{ variables.VmrBranch }}
+        architecture: x64
+        artifactsRid: alpine.3.17-x64
+        pool:
+          name: ${{ variables.defaultPoolName }}
+          demands: ${{ variables.defaultPoolDemands }}
+        container: ${{ parameters.alpine317Container }}
+        buildFromArchive: false            # 🚫
+        enablePoison: true                 # ✅
+        excludeOmniSharpTests: true        # ✅
+        runOnline: false                   # 🚫
+        useMonoRuntime: false              # 🚫
+        withPreviousSDK: true              # ✅
+
+  - ${{ if eq(parameters.isLiteBuild, false) }}:
 
     - template: ../jobs/vmr-build.yml
       parameters:
@@ -109,24 +148,6 @@ stages:
         useMonoRuntime: false              # 🚫
         withPreviousSDK: false             # 🚫
 
-    - template: ../jobs/vmr-build.yml
-      parameters:
-        # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline
-        buildName: Alpine317_Offline_MsftSdk
-        isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }}
-        vmrBranch: ${{ variables.VmrBranch }}
-        architecture: x64
-        pool:
-          name: ${{ variables.defaultPoolName }}
-          demands: ${{ variables.defaultPoolDemands }}
-        container: ${{ parameters.alpine317Container }}
-        buildFromArchive: false            # ✅
-        enablePoison: false                # 🚫
-        excludeOmniSharpTests: true        # ✅
-        runOnline: false                   # 🚫
-        useMonoRuntime: false              # 🚫
-        withPreviousSDK: false             # 🚫
-
     - template: ../jobs/vmr-build.yml
       parameters:
         # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline
@@ -134,6 +155,7 @@ stages:
         isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }}
         vmrBranch: ${{ variables.VmrBranch }}
         architecture: x64
+        artifactsRid: centos.8-x64
         pool:
           name: ${{ variables.defaultPoolName }}
           demands: ${{ variables.defaultPoolDemands }}
@@ -152,6 +174,7 @@ stages:
         isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }}
         vmrBranch: ${{ variables.VmrBranch }}
         architecture: x64
+        artifactsRid: centos.8-x64
         pool:
           name: ${{ variables.defaultPoolName }}
           demands: ${{ variables.defaultPoolDemands }}
@@ -211,7 +234,7 @@ stages:
           demands: ${{ variables.defaultPoolDemands }}
         container: ${{ parameters.fedora38Container }}
         buildFromArchive: true             # ✅
-        enablePoison: true                 # ✅
+        enablePoison: false                # 🚫
         excludeOmniSharpTests: false       # 🚫
         runOnline: false                   # 🚫
         useMonoRuntime: false              # 🚫
@@ -235,8 +258,6 @@ stages:
         useMonoRuntime: false              # 🚫
         withPreviousSDK: false             # 🚫
 
-    # CI - Stage 1 arm64 Legs ------------------------------------
-
     - template: ../jobs/vmr-build.yml
       parameters:
         # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline
@@ -253,27 +274,6 @@ stages:
         useMonoRuntime: false              # 🚫
         withPreviousSDK: false             # 🚫
 
-    # CI - Stage 2 x64 Legs ------------------------------------
-
-    - template: ../jobs/vmr-build.yml
-      parameters:
-        # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline
-        buildName: CentOSStream8_Online_CurrentSourceBuiltSdk
-        isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }}
-        vmrBranch: ${{ variables.VmrBranch }}
-        architecture: x64
-        pool:
-          name: ${{ variables.defaultPoolName }}
-          demands: ${{ variables.defaultPoolDemands }}
-        container: ${{ parameters.centOSStream8Container }}
-        buildFromArchive: false            # 🚫
-        enablePoison: false                # 🚫
-        excludeOmniSharpTests: true        # ✅
-        runOnline: false                   # 🚫
-        useMonoRuntime: false              # 🚫
-        withPreviousSDK: false             # 🚫
-        reuseBuildArtifactsFrom: CentOSStream8_Online_MsftSdk
-
     - template: ../jobs/vmr-build.yml
       parameters:
         # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline

src/SourceBuild/content/eng/Versions.props

@@ -15,23 +15,21 @@
       Building .NET from source depends on several archives, depending on the branch's current
       source-buildability status.
 
-      PrivateSourceBuiltArtifactsUrl is a tar.gz of .NET build outputs from a previous
+      PrivateSourceBuiltSdkVersion is the version number of a tar.gz of the source build .NET SDK
-      build needed to build the current version of .NET. This is always defined, because .NET needs
+      from the previous release for a particular OS. It is used by CI to validate that the current
-      to be bootstrappable at any point in time.
+      release can be built with it.
 
-      PrivateSourceBuiltPrebuiltsUrl is a tar.gz of assets downloaded from the internet
+      PrivateSourceBuiltArtifactsVersion is the version number of a tar.gz of .NET build outputs
-      that are needed to build the current version of .NET. Early in the lifecycle of a .NET major
+      from a previous build needed to build the current version of .NET. This is always defined
-      or minor release, prebuilts may be needed. When the release is mature, prebuilts are not
+      because .NET needs to be bootstrappable at any point in time.
-      necessary, and this property is removed from the file.
 
-      PrivateSourceBuiltSdkUrl_<abc> is a tar.gz of the source build .NET SDK from the previous release
+      PrivateSourceBuiltPrebuiltsVersion is the version number of a tar.gz of assets downloaded
-      for a particular OS. It is used by CI to validate that the current release can be built with it.
+      from the internet that are needed to build the current version of .NET. Early in the lifecycle
-
+      of a .NET major or minor release, prebuilts may be needed. When the release is mature, prebuilts
-      These URLs can't be composed from their base URL and version as we read them from the
+      are not necessary, and this property is removed from the file.
-      prep.sh and pipeline scripts, outside of MSBuild.
     -->
-    <PrivateSourceBuiltArtifactsUrl>https://dotnetcli.azureedge.net/source-built-artifacts/assets/Private.SourceBuilt.Artifacts.8.0.100-rc.2.23502.1.centos.8-x64.tar.gz</PrivateSourceBuiltArtifactsUrl>
+    <PrivateSourceBuiltSdkVersion>8.0.100-rc.2.23502.1</PrivateSourceBuiltSdkVersion>
-    <PrivateSourceBuiltSdkUrl_CentOS8Stream>https://dotnetcli.azureedge.net/source-built-artifacts/sdks/dotnet-sdk-8.0.100-rc.2.23502.1-centos.8-x64.tar.gz</PrivateSourceBuiltSdkUrl_CentOS8Stream>
+    <PrivateSourceBuiltArtifactsVersion>8.0.100-rc.2.23502.1</PrivateSourceBuiltArtifactsVersion>
-    <PrivateSourceBuiltPrebuiltsUrl>https://dotnetcli.azureedge.net/source-built-artifacts/assets/Private.SourceBuilt.Prebuilts.0.1.0-9.0.100-3.centos.8-x64.tar.gz</PrivateSourceBuiltPrebuiltsUrl>
+    <PrivateSourceBuiltPrebuiltsVersion>0.1.0-9.0.100-3</PrivateSourceBuiltPrebuiltsVersion>
   </PropertyGroup>
 </Project>

src/SourceBuild/content/eng/pipelines/ci-lite.yml

@@ -0,0 +1,22 @@
+# This is the dotnet/dotnet-lite pipeline that is triggered by pushes to main and PRs targetting main and release/*.
+
+trigger:
+  batch: true
+  branches:
+    include:
+    - main
+
+pr:
+  branches:
+    include:
+    - main
+    - release/*
+
+stages:
+- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+  - template: templates/stages/vmr-scan.yml
+
+- template: /src/installer/eng/pipelines/templates/stages/vmr-build.yml
+  parameters:
+    isBuiltFromVmr: true
+    isLiteBuild: true

src/SourceBuild/content/eng/pipelines/ci.yml

@@ -1,31 +1,27 @@
-# This is the main build definition (PR+CI) for dotnet/dotnet
+# This is the dotnet/dotnet pipeline that is triggered every weekday at midnight (00:00 UTC) for "main" and when changes are pushed to release/* and internal/release/* branches.
+
+schedules:
+- cron: '0 0 * * Mon-Fri'
+  displayName: Weekday midnight build
+  branches:
+    include:
+    - main
+  batch: true
 
 trigger:
   batch: true
   branches:
-    include:
+   include:
-    - main
+   - release/*
-    - release/*
+   - internal/release/*
-    - internal/release/*
-    exclude:
-    - release/*.0.2xx
-    - release/*.0.3xx
-    - release/*.0.4xx
-    - internal/release/*.0.2xx
-    - internal/release/*.0.3xx
-    - internal/release/*.0.4xx
 
-pr:
+pr: none
-  branches:
-    include:
-    - main
-    - release/*
-    - internal/release/*
 
 stages:
-- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+- ${{ if ne(variables['Build.Reason'], 'Schedule') }}:
   - template: templates/stages/vmr-scan.yml
 
 - template: /src/installer/eng/pipelines/templates/stages/vmr-build.yml
   parameters:
     isBuiltFromVmr: true
+    isLiteBuild: false

src/SourceBuild/content/prep.sh

@@ -10,6 +10,8 @@
 ###   --no-bootstrap              Don't replace portable packages in the download source-built artifacts
 ###   --no-prebuilts              Exclude the download of the prebuilts archive
 ###   --no-sdk                    Exclude the download of the .NET SDK
+###   --artifacts-rid             The RID of the previously source-built artifacts archive to download
+###                               Default is centos.8-x64
 ###   --runtime-source-feed       URL of a remote server or a local directory, from which SDKs and
 ###                               runtimes can be downloaded
 ###   --runtime-source-feed-key   Key for accessing the above server, if necessary
@@ -24,10 +26,13 @@ function print_help () {
     sed -n '/^### /,/^$/p' "$source" | cut -b 5-
 }
 
+defaultArtifactsRid='centos.8-x64'
+
 buildBootstrap=true
 downloadArtifacts=true
 downloadPrebuilts=true
 installDotnet=true
+artifactsRid=$defaultArtifactsRid
 runtime_source_feed='' # IBM requested these to support s390x scenarios
 runtime_source_feed_key='' # IBM requested these to support s390x scenarios
 positional_args=()
@@ -53,6 +58,9 @@ while :; do
     --no-sdk)
       installDotnet=false
       ;;
+    --artifacts-rid)
+      artifactsRid=$2
+      ;;
     --runtime-source-feed)
       runtime_source_feed=$2
       shift
@@ -107,17 +115,27 @@ fi
 function DownloadArchive {
   archiveType="$1"
   isRequired="$2"
+  artifactsRid="$3"
 
   packageVersionsPath="$SCRIPT_ROOT/eng/Versions.props"
   notFoundMessage="No source-built $archiveType found to download..."
 
   echo "  Looking for source-built $archiveType to download..."
-  archiveVersionLine=$(grep -m 1 "<PrivateSourceBuilt${archiveType}Url>" "$packageVersionsPath" || :)
+  archiveVersionLine=$(grep -m 1 "<PrivateSourceBuilt${archiveType}Version>" "$packageVersionsPath" || :)
-  versionPattern="<PrivateSourceBuilt${archiveType}Url>(.*)</PrivateSourceBuilt${archiveType}Url>"
+  versionPattern="<PrivateSourceBuilt${archiveType}Version>(.*)</PrivateSourceBuilt${archiveType}Version>"
   if [[ $archiveVersionLine =~ $versionPattern ]]; then
-      archiveUrl="${BASH_REMATCH[1]}"
+    archiveVersion="${BASH_REMATCH[1]}"
-      echo "  Downloading source-built $archiveType from $archiveUrl..."
+
-      (cd "$packagesArchiveDir" && curl --retry 5 -O "$archiveUrl")
+    if [ "$archiveType" == "Prebuilts" ]; then
+        archiveRid=$defaultArtifactsRid
+    else
+        archiveRid=$artifactsRid
+    fi
+
+    archiveUrl="https://dotnetcli.azureedge.net/source-built-artifacts/assets/Private.SourceBuilt.$archiveType.$archiveVersion.$archiveRid.tar.gz"
+
+    echo "  Downloading source-built $archiveType from $archiveUrl..."
+    (cd "$packagesArchiveDir" && curl --retry 5 -O "$archiveUrl")
   elif [ "$isRequired" == true ]; then
     echo "  ERROR: $notFoundMessage"
     exit 1
@@ -164,12 +182,12 @@ fi
 
 # Read the eng/Versions.props to get the archives to download and download them
 if [ "$downloadArtifacts" == true ]; then
-  DownloadArchive Artifacts true
+  DownloadArchive Artifacts true $artifactsRid
   if [ "$buildBootstrap" == true ]; then
       BootstrapArtifacts
   fi
 fi
 
 if [ "$downloadPrebuilts" == true ]; then
-  DownloadArchive Prebuilts false
+  DownloadArchive Prebuilts false $artifactsRid
 fi