From 9cf0095d0892d385a7e26772752d75e3bb68d8e4 Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Fri, 3 Mar 2023 17:53:19 -0800 Subject: [PATCH] [main] Add CI for building with previously source-built SDK (#15603) Co-authored-by: Michael Simons --- eng/pipelines/templates/jobs/vmr-build.yml | 43 ++++++++++++++++---- eng/pipelines/templates/stages/vmr-build.yml | 41 +++++++++++++++---- src/SourceBuild/content/eng/Versions.props | 17 +++----- src/SourceBuild/content/prep.sh | 18 +++++--- 4 files changed, 87 insertions(+), 32 deletions(-) diff --git a/eng/pipelines/templates/jobs/vmr-build.yml b/eng/pipelines/templates/jobs/vmr-build.yml index 625e55eda..3f575ff29 100644 --- a/eng/pipelines/templates/jobs/vmr-build.yml +++ b/eng/pipelines/templates/jobs/vmr-build.yml @@ -58,6 +58,10 @@ parameters: type: boolean default: false +- name: withPreviousSDK + type: boolean + default: false + jobs: - job: ${{ parameters.buildName }}_${{ parameters.architecture }} timeoutInMinutes: 150 @@ -76,12 +80,8 @@ jobs: - ${{ else }}: - name: BotAccount-dotnet-bot-repo-PAT value: N/A - - ${{ if eq(parameters.reuseBuildArtifactsFrom, '') }}: - - name: additionalBuildArgs - value: '' - - ${{ else }}: - - name: additionalBuildArgs - value: ' --with-sdk /vmr/.dotnet' + - name: additionalBuildArgs + value: '' # Location of the VMR sources # We either build the repo directly, or we extract them outside (which is what partners do) @@ -147,16 +147,45 @@ jobs: Contents: '*.tar.gz' TargetFolder: ${{ variables.sourcesPath }}/prereqs/packages/archive/ + - ${{ if eq(parameters.withPreviousSDK, 'true') }}: + - script: | + set -euo pipefail + + packageVersionsPath="${{ variables.sourcesPath }}/eng/Versions.props" + notFoundMessage="No source-built SDK found to download..." + + echo "Looking for source-built SDK to download..." + archiveUrlLine=`grep -m 1 "" "$packageVersionsPath" || :` + urlPattern="(.*)" + + if [[ $archiveUrlLine =~ $urlPattern ]]; then + archiveUrl="${BASH_REMATCH[1]}" + downloadDir="$(sourcesPath)/prereqs/packages/archive/" + (cd $downloadDir && curl --retry 5 -O $archiveUrl) + else + echo "$notFoundMessage" + exit 1 + fi + displayName: Setup Previously Source-Built SDK + - script: | set -x - if [[ -z '${{ parameters.reuseBuildArtifactsFrom }}' ]]; then + if [[ -z '${{ parameters.reuseBuildArtifactsFrom }}' && '${{ parameters.withPreviousSDK }}' != 'True' ]]; then docker run --rm -v "$(sourcesPath):/vmr" -w /vmr ${{ parameters.container }} ./prep.sh else mkdir $(sourcesPath)/.dotnet previousSdkPath="$(sourcesPath)/prereqs/packages/archive/dotnet-sdk-*.tar.gz" eval tar -ozxf "$previousSdkPath" -C "$(sourcesPath)/.dotnet" eval rm -f "$previousSdkPath" + + if [[ '${{ parameters.withPreviousSDK }}' == 'True' ]]; then + # Source-built artifacts are from CentOS 8 Stream. We want to download them without + # downloading portable versions from the internet. + docker run --rm -v "$(sourcesPath):/vmr" -w /vmr ${{ parameters.container }} ./prep.sh --no-bootstrap + fi + + echo "##vso[task.setvariable variable=additionalBuildArgs]--with-sdk /vmr/.dotnet" fi displayName: Prep the Build diff --git a/eng/pipelines/templates/stages/vmr-build.yml b/eng/pipelines/templates/stages/vmr-build.yml index 515ab8dac..e09fdbbe6 100644 --- a/eng/pipelines/templates/stages/vmr-build.yml +++ b/eng/pipelines/templates/stages/vmr-build.yml @@ -67,7 +67,7 @@ stages: jobs: - template: ../jobs/vmr-build.yml parameters: - buildName: CentOSStream8_Online + buildName: CentOSStream8_Online_MsftSdk isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} architecture: x64 @@ -81,11 +81,30 @@ stages: excludeSdkContentTests: true # ✅ overrideDistroDisablingSha1: false # 🚫 runOnline: true # ✅ + withPreviousSDK: false # 🚫 - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: - template: ../jobs/vmr-build.yml parameters: - buildName: CentOSStream8_Offline + buildName: CentOSStream8_Online_PreviousSourceBuiltSdk + isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} + vmrBranch: ${{ variables.VmrBranch }} + architecture: x64 + pool: + name: ${{ variables.defaultPoolName }} + demands: ${{ variables.defaultPoolDemands }} + container: ${{ parameters.centOSStream8Container }} + buildFromArchive: false # 🚫 + enablePoison: false # 🚫 + excludeOmniSharpTests: true # ✅ + excludeSdkContentTests: true # ✅ + overrideDistroDisablingSha1: false # 🚫 + runOnline: true # ✅ + withPreviousSDK: true # ✅ + + - template: ../jobs/vmr-build.yml + parameters: + buildName: CentOSStream8_Offline_MsftSdk isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} architecture: x64 @@ -99,10 +118,11 @@ stages: excludeSdkContentTests: false # 🚫 overrideDistroDisablingSha1: false # 🚫 runOnline: false # 🚫 + withPreviousSDK: false # 🚫 - template: ../jobs/vmr-build.yml parameters: - buildName: CentOSStream9_Offline + buildName: CentOSStream9_Offline_MsftSdk isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} architecture: x64 @@ -116,10 +136,11 @@ stages: excludeSdkContentTests: false # 🚫 overrideDistroDisablingSha1: true # ✅ runOnline: false # 🚫 + withPreviousSDK: false # 🚫 - template: ../jobs/vmr-build.yml parameters: - buildName: Fedora36_Offline + buildName: Fedora36_Offline_MsftSdk isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} architecture: x64 @@ -133,10 +154,11 @@ stages: excludeSdkContentTests: false # 🚫 overrideDistroDisablingSha1: false # 🚫 runOnline: false # 🚫 + withPreviousSDK: false # 🚫 - template: ../jobs/vmr-build.yml parameters: - buildName: Ubuntu2004_Offline + buildName: Ubuntu2004_Offline_MsftSdk isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} architecture: x64 @@ -150,10 +172,11 @@ stages: excludeSdkContentTests: false # 🚫 overrideDistroDisablingSha1: false # 🚫 runOnline: false # 🚫 + withPreviousSDK: false # 🚫 - template: ../jobs/vmr-build.yml parameters: - buildName: Debian11_Offline + buildName: Debian11_Offline_MsftSdk isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} architecture: arm64 @@ -165,10 +188,11 @@ stages: excludeSdkContentTests: false # 🚫 overrideDistroDisablingSha1: false # 🚫 runOnline: false # 🚫 + withPreviousSDK: false # 🚫 - template: ../jobs/vmr-build.yml parameters: - buildName: Fedora36_Offline_Using_Previous + buildName: Fedora36_Offline_CurrentSourceBuiltSdk isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} architecture: x64 @@ -182,4 +206,5 @@ stages: excludeSdkContentTests: true # ✅ overrideDistroDisablingSha1: false # 🚫 runOnline: false # 🚫 - reuseBuildArtifactsFrom: Fedora36_Offline + withPreviousSDK: false # 🚫 + reuseBuildArtifactsFrom: Fedora36_Offline_MsftSdk diff --git a/src/SourceBuild/content/eng/Versions.props b/src/SourceBuild/content/eng/Versions.props index e466a9189..049760016 100644 --- a/src/SourceBuild/content/eng/Versions.props +++ b/src/SourceBuild/content/eng/Versions.props @@ -12,19 +12,14 @@ - 0.1.0-8.0.100-5.centos.8-x64 - 0.1.0-8.0.100-8.centos.8-x64 + https://dotnetcli.azureedge.net/source-built-artifacts/assets/Private.SourceBuilt.Artifacts.0.1.0-8.0.100-5.centos.8-x64.tar.gz + https://dotnetcli.azureedge.net/source-built-artifacts/assets/Private.SourceBuilt.Prebuilts.0.1.0-8.0.100-8.centos.8-x64.tar.gz + https://dotnetcli.azureedge.net/source-built-artifacts/sdks/dotnet-sdk-8.0.100-preview.1.23115.1-centos.8-x64.tar.gz diff --git a/src/SourceBuild/content/prep.sh b/src/SourceBuild/content/prep.sh index 8c03cadc0..ba5415671 100755 --- a/src/SourceBuild/content/prep.sh +++ b/src/SourceBuild/content/prep.sh @@ -9,10 +9,12 @@ usage() { echo "" echo " Prepares the environment to be built by downloading Private.SourceBuilt.Artifacts.*.tar.gz and" echo " installing the version of dotnet referenced in global.json" + echo "options:" + echo " --no-bootstrap Don't replace portable packages in the download source-built artifacts" echo "" } -buildBootstrap=false +buildBootstrap=true positional_args=() while :; do if [ $# -le 0 ]; then @@ -24,6 +26,9 @@ while :; do usage exit 0 ;; + --no-bootstrap) + buildBootstrap=false + ;; *) positional_args+=("$1") ;; @@ -69,15 +74,14 @@ function DownloadArchive { baseFileName="$2" isRequired="$3" - sourceBuiltArtifactsTarballUrl="https://dotnetcli.azureedge.net/source-built-artifacts/assets/" packageVersionsPath="$SCRIPT_ROOT/eng/Versions.props" notFoundMessage="No source-built $archiveType found to download..." echo " Looking for source-built $archiveType to download..." - archiveVersionLine=`grep -m 1 "" "$packageVersionsPath" || :` - versionPattern="(.*)" + archiveVersionLine=`grep -m 1 "" "$packageVersionsPath" || :` + versionPattern="(.*)" if [[ $archiveVersionLine =~ $versionPattern ]]; then - archiveUrl="${sourceBuiltArtifactsTarballUrl}${baseFileName}.${BASH_REMATCH[1]}.tar.gz" + archiveUrl="${BASH_REMATCH[1]}" echo " Downloading source-built $archiveType from $archiveUrl..." (cd $packagesArchiveDir && curl --retry 5 -O $archiveUrl) elif [ "$isRequired" == "true" ]; then @@ -124,7 +128,9 @@ fi # Read the eng/Versions.props to get the archives to download and download them if [ "$downloadArtifacts" == "true" ]; then DownloadArchive "Artifacts" $artifactsBaseFileName "true" - BootstrapArtifacts + if [ "$buildBootstrap" == "true" ]; then + BootstrapArtifacts + fi fi if [ "$downloadPrebuilts" == "true" ]; then