From eefba512e7fa89d66c233945d723807c3e512c3c Mon Sep 17 00:00:00 2001 From: Ella Hathaway <67609881+ellahathaway@users.noreply.github.com> Date: Tue, 7 Nov 2023 08:54:50 -0800 Subject: [PATCH] Split current CI pipline (#17370) --- eng/pipelines/templates/jobs/vmr-build.yml | 29 ++++-- eng/pipelines/templates/stages/vmr-build.yml | 90 +++++++++---------- src/SourceBuild/content/eng/Versions.props | 28 +++--- .../content/eng/pipelines/ci-lite.yml | 22 +++++ src/SourceBuild/content/eng/pipelines/ci.yml | 34 ++++--- src/SourceBuild/content/prep.sh | 32 +++++-- 6 files changed, 143 insertions(+), 92 deletions(-) create mode 100644 src/SourceBuild/content/eng/pipelines/ci-lite.yml diff --git a/eng/pipelines/templates/jobs/vmr-build.yml b/eng/pipelines/templates/jobs/vmr-build.yml index 041a47580..73c1ce23e 100644 --- a/eng/pipelines/templates/jobs/vmr-build.yml +++ b/eng/pipelines/templates/jobs/vmr-build.yml @@ -21,6 +21,10 @@ parameters: - name: architecture type: string +- name: artifactsRid + type: string + default: '' + - name: container type: string @@ -141,17 +145,25 @@ jobs: - script: | set -euo pipefail + if [[ '${{ parameters.artifactsRid }}' == '' ]]; then + echo "'artifactsRid' is not specified. Cannot download source-built SDK." + exit 1 + fi + packageVersionsPath="${{ variables.sourcesPath }}/eng/Versions.props" notFoundMessage="No source-built SDK found to download..." echo "Looking for source-built SDK to download..." - archiveUrlLine=`grep -m 1 "<PrivateSourceBuiltSdkUrl_CentOS8Stream>" "$packageVersionsPath" || :` - urlPattern="<PrivateSourceBuiltSdkUrl_CentOS8Stream>(.*)</PrivateSourceBuiltSdkUrl_CentOS8Stream>" + archiveVersionLine=$(grep -m 1 "<PrivateSourceBuiltSdkVersion>" "$packageVersionsPath" || :) + versionPattern="<PrivateSourceBuiltSdkVersion>(.*)</PrivateSourceBuiltSdkVersion>" - if [[ $archiveUrlLine =~ $urlPattern ]]; then - archiveUrl="${BASH_REMATCH[1]}" + if [[ $archiveVersionLine =~ $versionPattern ]]; then + archiveVersion="${BASH_REMATCH[1]}" + archiveUrl="https://dotnetcli.azureedge.net/source-built-artifacts/sdks/dotnet-sdk-$archiveVersion-${{ parameters.artifactsRid }}.tar.gz" downloadDir="$(sourcesPath)/prereqs/packages/archive/" - (cd $downloadDir && curl --retry 5 -O $archiveUrl) + + echo "Downloading source-built SDK from $archiveUrl..." + (cd "$downloadDir" && curl --retry 5 -O "$archiveUrl") else echo "$notFoundMessage" exit 1 @@ -163,8 +175,13 @@ jobs: customPrepArgs="" prepSdk=true + + if [[ -n '${{ parameters.artifactsRid }}' ]]; then + customPrepArgs="${customPrepArgs} --artifacts-rid ${{ parameters.artifactsRid }}" + fi + if [[ '${{ parameters.withPreviousSDK }}' == 'True' ]]; then - # Source-built artifacts are from CentOS 8 Stream. We want to download them without + # Source-built artifacts are from CentOS 8 Stream or Alpine 3.17. We want to download them without # downloading portable versions from the internet. customPrepArgs="${customPrepArgs} --no-sdk --no-bootstrap" prepSdk=false diff --git a/eng/pipelines/templates/stages/vmr-build.yml b/eng/pipelines/templates/stages/vmr-build.yml index 59be07b85..619dc243e 100644 --- a/eng/pipelines/templates/stages/vmr-build.yml +++ b/eng/pipelines/templates/stages/vmr-build.yml @@ -8,6 +8,9 @@ parameters: # Branch of the VMR to use (to push to for internal builds) vmrBranch: $(Build.SourceBranch) + # True when the build is a lite build + isLiteBuild: + # True when build is running from dotnet/dotnet directly isBuiltFromVmr: @@ -67,8 +70,6 @@ stages: jobs: - # PR and CI legs ------------------------------------ - - template: ../jobs/vmr-build.yml parameters: # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline @@ -89,7 +90,45 @@ stages: - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: - # CI - Stage 1 x64 legs ------------------------------------ + - template: ../jobs/vmr-build.yml + parameters: + # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline + buildName: CentOSStream8_Online_CurrentSourceBuiltSdk + isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} + vmrBranch: ${{ variables.VmrBranch }} + architecture: x64 + pool: + name: ${{ variables.defaultPoolName }} + demands: ${{ variables.defaultPoolDemands }} + container: ${{ parameters.centOSStream8Container }} + buildFromArchive: false # 🚫 + enablePoison: false # 🚫 + excludeOmniSharpTests: true # ✅ + runOnline: true # ✅ + useMonoRuntime: false # 🚫 + withPreviousSDK: false # 🚫 + reuseBuildArtifactsFrom: CentOSStream8_Online_MsftSdk + + - template: ../jobs/vmr-build.yml + parameters: + # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline + buildName: Alpine317_Offline_PreviousSourceBuiltSdk + isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} + vmrBranch: ${{ variables.VmrBranch }} + architecture: x64 + artifactsRid: alpine.3.17-x64 + pool: + name: ${{ variables.defaultPoolName }} + demands: ${{ variables.defaultPoolDemands }} + container: ${{ parameters.alpine317Container }} + buildFromArchive: false # 🚫 + enablePoison: true # ✅ + excludeOmniSharpTests: true # ✅ + runOnline: false # 🚫 + useMonoRuntime: false # 🚫 + withPreviousSDK: true # ✅ + + - ${{ if eq(parameters.isLiteBuild, false) }}: - template: ../jobs/vmr-build.yml parameters: @@ -109,24 +148,6 @@ stages: useMonoRuntime: false # 🚫 withPreviousSDK: false # 🚫 - - template: ../jobs/vmr-build.yml - parameters: - # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline - buildName: Alpine317_Offline_MsftSdk - isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} - vmrBranch: ${{ variables.VmrBranch }} - architecture: x64 - pool: - name: ${{ variables.defaultPoolName }} - demands: ${{ variables.defaultPoolDemands }} - container: ${{ parameters.alpine317Container }} - buildFromArchive: false # ✅ - enablePoison: false # 🚫 - excludeOmniSharpTests: true # ✅ - runOnline: false # 🚫 - useMonoRuntime: false # 🚫 - withPreviousSDK: false # 🚫 - - template: ../jobs/vmr-build.yml parameters: # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline @@ -134,6 +155,7 @@ stages: isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} architecture: x64 + artifactsRid: centos.8-x64 pool: name: ${{ variables.defaultPoolName }} demands: ${{ variables.defaultPoolDemands }} @@ -152,6 +174,7 @@ stages: isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} architecture: x64 + artifactsRid: centos.8-x64 pool: name: ${{ variables.defaultPoolName }} demands: ${{ variables.defaultPoolDemands }} @@ -211,7 +234,7 @@ stages: demands: ${{ variables.defaultPoolDemands }} container: ${{ parameters.fedora38Container }} buildFromArchive: true # ✅ - enablePoison: true # ✅ + enablePoison: false # 🚫 excludeOmniSharpTests: false # 🚫 runOnline: false # 🚫 useMonoRuntime: false # 🚫 @@ -235,8 +258,6 @@ stages: useMonoRuntime: false # 🚫 withPreviousSDK: false # 🚫 - # CI - Stage 1 arm64 Legs ------------------------------------ - - template: ../jobs/vmr-build.yml parameters: # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline @@ -253,27 +274,6 @@ stages: useMonoRuntime: false # 🚫 withPreviousSDK: false # 🚫 - # CI - Stage 2 x64 Legs ------------------------------------ - - - template: ../jobs/vmr-build.yml - parameters: - # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline - buildName: CentOSStream8_Online_CurrentSourceBuiltSdk - isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} - vmrBranch: ${{ variables.VmrBranch }} - architecture: x64 - pool: - name: ${{ variables.defaultPoolName }} - demands: ${{ variables.defaultPoolDemands }} - container: ${{ parameters.centOSStream8Container }} - buildFromArchive: false # 🚫 - enablePoison: false # 🚫 - excludeOmniSharpTests: true # ✅ - runOnline: false # 🚫 - useMonoRuntime: false # 🚫 - withPreviousSDK: false # 🚫 - reuseBuildArtifactsFrom: CentOSStream8_Online_MsftSdk - - template: ../jobs/vmr-build.yml parameters: # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline diff --git a/src/SourceBuild/content/eng/Versions.props b/src/SourceBuild/content/eng/Versions.props index 07844e239..74f7134c8 100644 --- a/src/SourceBuild/content/eng/Versions.props +++ b/src/SourceBuild/content/eng/Versions.props @@ -15,23 +15,21 @@ Building .NET from source depends on several archives, depending on the branch's current source-buildability status. - PrivateSourceBuiltArtifactsUrl is a tar.gz of .NET build outputs from a previous - build needed to build the current version of .NET. This is always defined, because .NET needs - to be bootstrappable at any point in time. + PrivateSourceBuiltSdkVersion is the version number of a tar.gz of the source build .NET SDK + from the previous release for a particular OS. It is used by CI to validate that the current + release can be built with it. - PrivateSourceBuiltPrebuiltsUrl is a tar.gz of assets downloaded from the internet - that are needed to build the current version of .NET. Early in the lifecycle of a .NET major - or minor release, prebuilts may be needed. When the release is mature, prebuilts are not - necessary, and this property is removed from the file. + PrivateSourceBuiltArtifactsVersion is the version number of a tar.gz of .NET build outputs + from a previous build needed to build the current version of .NET. This is always defined + because .NET needs to be bootstrappable at any point in time. - PrivateSourceBuiltSdkUrl_<abc> is a tar.gz of the source build .NET SDK from the previous release - for a particular OS. It is used by CI to validate that the current release can be built with it. - - These URLs can't be composed from their base URL and version as we read them from the - prep.sh and pipeline scripts, outside of MSBuild. + PrivateSourceBuiltPrebuiltsVersion is the version number of a tar.gz of assets downloaded + from the internet that are needed to build the current version of .NET. Early in the lifecycle + of a .NET major or minor release, prebuilts may be needed. When the release is mature, prebuilts + are not necessary, and this property is removed from the file. --> - <PrivateSourceBuiltArtifactsUrl>https://dotnetcli.azureedge.net/source-built-artifacts/assets/Private.SourceBuilt.Artifacts.8.0.100-rc.2.23502.1.centos.8-x64.tar.gz</PrivateSourceBuiltArtifactsUrl> - <PrivateSourceBuiltSdkUrl_CentOS8Stream>https://dotnetcli.azureedge.net/source-built-artifacts/sdks/dotnet-sdk-8.0.100-rc.2.23502.1-centos.8-x64.tar.gz</PrivateSourceBuiltSdkUrl_CentOS8Stream> - <PrivateSourceBuiltPrebuiltsUrl>https://dotnetcli.azureedge.net/source-built-artifacts/assets/Private.SourceBuilt.Prebuilts.0.1.0-9.0.100-3.centos.8-x64.tar.gz</PrivateSourceBuiltPrebuiltsUrl> + <PrivateSourceBuiltSdkVersion>8.0.100-rc.2.23502.1</PrivateSourceBuiltSdkVersion> + <PrivateSourceBuiltArtifactsVersion>8.0.100-rc.2.23502.1</PrivateSourceBuiltArtifactsVersion> + <PrivateSourceBuiltPrebuiltsVersion>0.1.0-9.0.100-3</PrivateSourceBuiltPrebuiltsVersion> </PropertyGroup> </Project> diff --git a/src/SourceBuild/content/eng/pipelines/ci-lite.yml b/src/SourceBuild/content/eng/pipelines/ci-lite.yml new file mode 100644 index 000000000..6276ec4a2 --- /dev/null +++ b/src/SourceBuild/content/eng/pipelines/ci-lite.yml @@ -0,0 +1,22 @@ +# This is the dotnet/dotnet-lite pipeline that is triggered by pushes to main and PRs targetting main and release/*. + +trigger: + batch: true + branches: + include: + - main + +pr: + branches: + include: + - main + - release/* + +stages: +- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - template: templates/stages/vmr-scan.yml + +- template: /src/installer/eng/pipelines/templates/stages/vmr-build.yml + parameters: + isBuiltFromVmr: true + isLiteBuild: true diff --git a/src/SourceBuild/content/eng/pipelines/ci.yml b/src/SourceBuild/content/eng/pipelines/ci.yml index be751b083..0a8412db5 100644 --- a/src/SourceBuild/content/eng/pipelines/ci.yml +++ b/src/SourceBuild/content/eng/pipelines/ci.yml @@ -1,31 +1,27 @@ -# This is the main build definition (PR+CI) for dotnet/dotnet +# This is the dotnet/dotnet pipeline that is triggered every weekday at midnight (00:00 UTC) for "main" and when changes are pushed to release/* and internal/release/* branches. + +schedules: +- cron: '0 0 * * Mon-Fri' + displayName: Weekday midnight build + branches: + include: + - main + batch: true trigger: batch: true branches: - include: - - main - - release/* - - internal/release/* - exclude: - - release/*.0.2xx - - release/*.0.3xx - - release/*.0.4xx - - internal/release/*.0.2xx - - internal/release/*.0.3xx - - internal/release/*.0.4xx + include: + - release/* + - internal/release/* -pr: - branches: - include: - - main - - release/* - - internal/release/* +pr: none stages: -- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: +- ${{ if ne(variables['Build.Reason'], 'Schedule') }}: - template: templates/stages/vmr-scan.yml - template: /src/installer/eng/pipelines/templates/stages/vmr-build.yml parameters: isBuiltFromVmr: true + isLiteBuild: false diff --git a/src/SourceBuild/content/prep.sh b/src/SourceBuild/content/prep.sh index f9e88d6c4..678cb8a6f 100755 --- a/src/SourceBuild/content/prep.sh +++ b/src/SourceBuild/content/prep.sh @@ -10,6 +10,8 @@ ### --no-bootstrap Don't replace portable packages in the download source-built artifacts ### --no-prebuilts Exclude the download of the prebuilts archive ### --no-sdk Exclude the download of the .NET SDK +### --artifacts-rid The RID of the previously source-built artifacts archive to download +### Default is centos.8-x64 ### --runtime-source-feed URL of a remote server or a local directory, from which SDKs and ### runtimes can be downloaded ### --runtime-source-feed-key Key for accessing the above server, if necessary @@ -24,10 +26,13 @@ function print_help () { sed -n '/^### /,/^$/p' "$source" | cut -b 5- } +defaultArtifactsRid='centos.8-x64' + buildBootstrap=true downloadArtifacts=true downloadPrebuilts=true installDotnet=true +artifactsRid=$defaultArtifactsRid runtime_source_feed='' # IBM requested these to support s390x scenarios runtime_source_feed_key='' # IBM requested these to support s390x scenarios positional_args=() @@ -53,6 +58,9 @@ while :; do --no-sdk) installDotnet=false ;; + --artifacts-rid) + artifactsRid=$2 + ;; --runtime-source-feed) runtime_source_feed=$2 shift @@ -107,17 +115,27 @@ fi function DownloadArchive { archiveType="$1" isRequired="$2" + artifactsRid="$3" packageVersionsPath="$SCRIPT_ROOT/eng/Versions.props" notFoundMessage="No source-built $archiveType found to download..." echo " Looking for source-built $archiveType to download..." - archiveVersionLine=$(grep -m 1 "<PrivateSourceBuilt${archiveType}Url>" "$packageVersionsPath" || :) - versionPattern="<PrivateSourceBuilt${archiveType}Url>(.*)</PrivateSourceBuilt${archiveType}Url>" + archiveVersionLine=$(grep -m 1 "<PrivateSourceBuilt${archiveType}Version>" "$packageVersionsPath" || :) + versionPattern="<PrivateSourceBuilt${archiveType}Version>(.*)</PrivateSourceBuilt${archiveType}Version>" if [[ $archiveVersionLine =~ $versionPattern ]]; then - archiveUrl="${BASH_REMATCH[1]}" - echo " Downloading source-built $archiveType from $archiveUrl..." - (cd "$packagesArchiveDir" && curl --retry 5 -O "$archiveUrl") + archiveVersion="${BASH_REMATCH[1]}" + + if [ "$archiveType" == "Prebuilts" ]; then + archiveRid=$defaultArtifactsRid + else + archiveRid=$artifactsRid + fi + + archiveUrl="https://dotnetcli.azureedge.net/source-built-artifacts/assets/Private.SourceBuilt.$archiveType.$archiveVersion.$archiveRid.tar.gz" + + echo " Downloading source-built $archiveType from $archiveUrl..." + (cd "$packagesArchiveDir" && curl --retry 5 -O "$archiveUrl") elif [ "$isRequired" == true ]; then echo " ERROR: $notFoundMessage" exit 1 @@ -164,12 +182,12 @@ fi # Read the eng/Versions.props to get the archives to download and download them if [ "$downloadArtifacts" == true ]; then - DownloadArchive Artifacts true + DownloadArchive Artifacts true $artifactsRid if [ "$buildBootstrap" == true ]; then BootstrapArtifacts fi fi if [ "$downloadPrebuilts" == true ]; then - DownloadArchive Prebuilts false + DownloadArchive Prebuilts false $artifactsRid fi