Split current CI pipline (#17370)
This commit is contained in:
Ella Hathaway
GitHub
parent
7ac1022e78
commit
eefba512e7
6 changed files with 143 additions and 92 deletions
|
|
@ -21,6 +21,10 @@ parameters:
|
|||
- name: architecture
|
||||
type: string
|
||||
|
||||
- name: artifactsRid
|
||||
type: string
|
||||
default: ''
|
||||
|
||||
- name: container
|
||||
type: string
|
||||
|
||||
|
|
@ -141,17 +145,25 @@ jobs:
|
|||
- script: |
|
||||
set -euo pipefail
|
||||
|
||||
if [[ '${{ parameters.artifactsRid }}' == '' ]]; then
|
||||
echo "'artifactsRid' is not specified. Cannot download source-built SDK."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
packageVersionsPath="${{ variables.sourcesPath }}/eng/Versions.props"
|
||||
notFoundMessage="No source-built SDK found to download..."
|
||||
|
||||
echo "Looking for source-built SDK to download..."
|
||||
archiveUrlLine=`grep -m 1 "<PrivateSourceBuiltSdkUrl_CentOS8Stream>" "$packageVersionsPath" || :`
|
||||
urlPattern="<PrivateSourceBuiltSdkUrl_CentOS8Stream>(.*)</PrivateSourceBuiltSdkUrl_CentOS8Stream>"
|
||||
archiveVersionLine=$(grep -m 1 "<PrivateSourceBuiltSdkVersion>" "$packageVersionsPath" || :)
|
||||
versionPattern="<PrivateSourceBuiltSdkVersion>(.*)</PrivateSourceBuiltSdkVersion>"
|
||||
|
||||
if [[ $archiveUrlLine =~ $urlPattern ]]; then
|
||||
archiveUrl="${BASH_REMATCH[1]}"
|
||||
if [[ $archiveVersionLine =~ $versionPattern ]]; then
|
||||
archiveVersion="${BASH_REMATCH[1]}"
|
||||
archiveUrl="https://dotnetcli.azureedge.net/source-built-artifacts/sdks/dotnet-sdk-$archiveVersion-${{ parameters.artifactsRid }}.tar.gz"
|
||||
downloadDir="$(sourcesPath)/prereqs/packages/archive/"
|
||||
(cd $downloadDir && curl --retry 5 -O $archiveUrl)
|
||||
|
||||
echo "Downloading source-built SDK from $archiveUrl..."
|
||||
(cd "$downloadDir" && curl --retry 5 -O "$archiveUrl")
|
||||
else
|
||||
echo "$notFoundMessage"
|
||||
exit 1
|
||||
|
|
@ -163,8 +175,13 @@ jobs:
|
|||
|
||||
customPrepArgs=""
|
||||
prepSdk=true
|
||||
|
||||
if [[ -n '${{ parameters.artifactsRid }}' ]]; then
|
||||
customPrepArgs="${customPrepArgs} --artifacts-rid ${{ parameters.artifactsRid }}"
|
||||
fi
|
||||
|
||||
if [[ '${{ parameters.withPreviousSDK }}' == 'True' ]]; then
|
||||
# Source-built artifacts are from CentOS 8 Stream. We want to download them without
|
||||
# Source-built artifacts are from CentOS 8 Stream or Alpine 3.17. We want to download them without
|
||||
# downloading portable versions from the internet.
|
||||
customPrepArgs="${customPrepArgs} --no-sdk --no-bootstrap"
|
||||
prepSdk=false
|
||||
|
|
|
|||
|
|
@ -8,6 +8,9 @@ parameters:
|
|||
# Branch of the VMR to use (to push to for internal builds)
|
||||
vmrBranch: $(Build.SourceBranch)
|
||||
|
||||
# True when the build is a lite build
|
||||
isLiteBuild:
|
||||
|
||||
# True when build is running from dotnet/dotnet directly
|
||||
isBuiltFromVmr:
|
||||
|
||||
|
|
@ -67,8 +70,6 @@ stages:
|
|||
|
||||
jobs:
|
||||
|
||||
# PR and CI legs ------------------------------------
|
||||
|
||||
- template: ../jobs/vmr-build.yml
|
||||
parameters:
|
||||
# Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline
|
||||
|
|
@ -89,7 +90,45 @@ stages:
|
|||
|
||||
- ${{ if ne(variables['Build.Reason'], 'PullRequest') }}:
|
||||
|
||||
# CI - Stage 1 x64 legs ------------------------------------
|
||||
- template: ../jobs/vmr-build.yml
|
||||
parameters:
|
||||
# Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline
|
||||
buildName: CentOSStream8_Online_CurrentSourceBuiltSdk
|
||||
isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }}
|
||||
vmrBranch: ${{ variables.VmrBranch }}
|
||||
architecture: x64
|
||||
pool:
|
||||
name: ${{ variables.defaultPoolName }}
|
||||
demands: ${{ variables.defaultPoolDemands }}
|
||||
container: ${{ parameters.centOSStream8Container }}
|
||||
buildFromArchive: false # 🚫
|
||||
enablePoison: false # 🚫
|
||||
excludeOmniSharpTests: true # ✅
|
||||
runOnline: true # ✅
|
||||
useMonoRuntime: false # 🚫
|
||||
withPreviousSDK: false # 🚫
|
||||
reuseBuildArtifactsFrom: CentOSStream8_Online_MsftSdk
|
||||
|
||||
- template: ../jobs/vmr-build.yml
|
||||
parameters:
|
||||
# Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline
|
||||
buildName: Alpine317_Offline_PreviousSourceBuiltSdk
|
||||
isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }}
|
||||
vmrBranch: ${{ variables.VmrBranch }}
|
||||
architecture: x64
|
||||
artifactsRid: alpine.3.17-x64
|
||||
pool:
|
||||
name: ${{ variables.defaultPoolName }}
|
||||
demands: ${{ variables.defaultPoolDemands }}
|
||||
container: ${{ parameters.alpine317Container }}
|
||||
buildFromArchive: false # 🚫
|
||||
enablePoison: true # ✅
|
||||
excludeOmniSharpTests: true # ✅
|
||||
runOnline: false # 🚫
|
||||
useMonoRuntime: false # 🚫
|
||||
withPreviousSDK: true # ✅
|
||||
|
||||
- ${{ if eq(parameters.isLiteBuild, false) }}:
|
||||
|
||||
- template: ../jobs/vmr-build.yml
|
||||
parameters:
|
||||
|
|
@ -109,24 +148,6 @@ stages:
|
|||
useMonoRuntime: false # 🚫
|
||||
withPreviousSDK: false # 🚫
|
||||
|
||||
- template: ../jobs/vmr-build.yml
|
||||
parameters:
|
||||
# Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline
|
||||
buildName: Alpine317_Offline_MsftSdk
|
||||
isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }}
|
||||
vmrBranch: ${{ variables.VmrBranch }}
|
||||
architecture: x64
|
||||
pool:
|
||||
name: ${{ variables.defaultPoolName }}
|
||||
demands: ${{ variables.defaultPoolDemands }}
|
||||
container: ${{ parameters.alpine317Container }}
|
||||
buildFromArchive: false # ✅
|
||||
enablePoison: false # 🚫
|
||||
excludeOmniSharpTests: true # ✅
|
||||
runOnline: false # 🚫
|
||||
useMonoRuntime: false # 🚫
|
||||
withPreviousSDK: false # 🚫
|
||||
|
||||
- template: ../jobs/vmr-build.yml
|
||||
parameters:
|
||||
# Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline
|
||||
|
|
@ -134,6 +155,7 @@ stages:
|
|||
isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }}
|
||||
vmrBranch: ${{ variables.VmrBranch }}
|
||||
architecture: x64
|
||||
artifactsRid: centos.8-x64
|
||||
pool:
|
||||
name: ${{ variables.defaultPoolName }}
|
||||
demands: ${{ variables.defaultPoolDemands }}
|
||||
|
|
@ -152,6 +174,7 @@ stages:
|
|||
isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }}
|
||||
vmrBranch: ${{ variables.VmrBranch }}
|
||||
architecture: x64
|
||||
artifactsRid: centos.8-x64
|
||||
pool:
|
||||
name: ${{ variables.defaultPoolName }}
|
||||
demands: ${{ variables.defaultPoolDemands }}
|
||||
|
|
@ -211,7 +234,7 @@ stages:
|
|||
demands: ${{ variables.defaultPoolDemands }}
|
||||
container: ${{ parameters.fedora38Container }}
|
||||
buildFromArchive: true # ✅
|
||||
enablePoison: true # ✅
|
||||
enablePoison: false # 🚫
|
||||
excludeOmniSharpTests: false # 🚫
|
||||
runOnline: false # 🚫
|
||||
useMonoRuntime: false # 🚫
|
||||
|
|
@ -235,8 +258,6 @@ stages:
|
|||
useMonoRuntime: false # 🚫
|
||||
withPreviousSDK: false # 🚫
|
||||
|
||||
# CI - Stage 1 arm64 Legs ------------------------------------
|
||||
|
||||
- template: ../jobs/vmr-build.yml
|
||||
parameters:
|
||||
# Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline
|
||||
|
|
@ -253,27 +274,6 @@ stages:
|
|||
useMonoRuntime: false # 🚫
|
||||
withPreviousSDK: false # 🚫
|
||||
|
||||
# CI - Stage 2 x64 Legs ------------------------------------
|
||||
|
||||
- template: ../jobs/vmr-build.yml
|
||||
parameters:
|
||||
# Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline
|
||||
buildName: CentOSStream8_Online_CurrentSourceBuiltSdk
|
||||
isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }}
|
||||
vmrBranch: ${{ variables.VmrBranch }}
|
||||
architecture: x64
|
||||
pool:
|
||||
name: ${{ variables.defaultPoolName }}
|
||||
demands: ${{ variables.defaultPoolDemands }}
|
||||
container: ${{ parameters.centOSStream8Container }}
|
||||
buildFromArchive: false # 🚫
|
||||
enablePoison: false # 🚫
|
||||
excludeOmniSharpTests: true # ✅
|
||||
runOnline: false # 🚫
|
||||
useMonoRuntime: false # 🚫
|
||||
withPreviousSDK: false # 🚫
|
||||
reuseBuildArtifactsFrom: CentOSStream8_Online_MsftSdk
|
||||
|
||||
- template: ../jobs/vmr-build.yml
|
||||
parameters:
|
||||
# Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline
|
||||
|
|
|
|||
|
|
@ -15,23 +15,21 @@
|
|||
Building .NET from source depends on several archives, depending on the branch's current
|
||||
source-buildability status.
|
||||
|
||||
PrivateSourceBuiltArtifactsUrl is a tar.gz of .NET build outputs from a previous
|
||||
build needed to build the current version of .NET. This is always defined, because .NET needs
|
||||
to be bootstrappable at any point in time.
|
||||
PrivateSourceBuiltSdkVersion is the version number of a tar.gz of the source build .NET SDK
|
||||
from the previous release for a particular OS. It is used by CI to validate that the current
|
||||
release can be built with it.
|
||||
|
||||
PrivateSourceBuiltPrebuiltsUrl is a tar.gz of assets downloaded from the internet
|
||||
that are needed to build the current version of .NET. Early in the lifecycle of a .NET major
|
||||
or minor release, prebuilts may be needed. When the release is mature, prebuilts are not
|
||||
necessary, and this property is removed from the file.
|
||||
PrivateSourceBuiltArtifactsVersion is the version number of a tar.gz of .NET build outputs
|
||||
from a previous build needed to build the current version of .NET. This is always defined
|
||||
because .NET needs to be bootstrappable at any point in time.
|
||||
|
||||
PrivateSourceBuiltSdkUrl_<abc> is a tar.gz of the source build .NET SDK from the previous release
|
||||
for a particular OS. It is used by CI to validate that the current release can be built with it.
|
||||
|
||||
These URLs can't be composed from their base URL and version as we read them from the
|
||||
prep.sh and pipeline scripts, outside of MSBuild.
|
||||
PrivateSourceBuiltPrebuiltsVersion is the version number of a tar.gz of assets downloaded
|
||||
from the internet that are needed to build the current version of .NET. Early in the lifecycle
|
||||
of a .NET major or minor release, prebuilts may be needed. When the release is mature, prebuilts
|
||||
are not necessary, and this property is removed from the file.
|
||||
-->
|
||||
<PrivateSourceBuiltArtifactsUrl>https://dotnetcli.azureedge.net/source-built-artifacts/assets/Private.SourceBuilt.Artifacts.8.0.100-rc.2.23502.1.centos.8-x64.tar.gz</PrivateSourceBuiltArtifactsUrl>
|
||||
<PrivateSourceBuiltSdkUrl_CentOS8Stream>https://dotnetcli.azureedge.net/source-built-artifacts/sdks/dotnet-sdk-8.0.100-rc.2.23502.1-centos.8-x64.tar.gz</PrivateSourceBuiltSdkUrl_CentOS8Stream>
|
||||
<PrivateSourceBuiltPrebuiltsUrl>https://dotnetcli.azureedge.net/source-built-artifacts/assets/Private.SourceBuilt.Prebuilts.0.1.0-9.0.100-3.centos.8-x64.tar.gz</PrivateSourceBuiltPrebuiltsUrl>
|
||||
<PrivateSourceBuiltSdkVersion>8.0.100-rc.2.23502.1</PrivateSourceBuiltSdkVersion>
|
||||
<PrivateSourceBuiltArtifactsVersion>8.0.100-rc.2.23502.1</PrivateSourceBuiltArtifactsVersion>
|
||||
<PrivateSourceBuiltPrebuiltsVersion>0.1.0-9.0.100-3</PrivateSourceBuiltPrebuiltsVersion>
|
||||
</PropertyGroup>
|
||||
</Project>
|
||||
|
|
|
|||
22
src/SourceBuild/content/eng/pipelines/ci-lite.yml
Normal file
22
src/SourceBuild/content/eng/pipelines/ci-lite.yml
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
# This is the dotnet/dotnet-lite pipeline that is triggered by pushes to main and PRs targetting main and release/*.
|
||||
|
||||
trigger:
|
||||
batch: true
|
||||
branches:
|
||||
include:
|
||||
- main
|
||||
|
||||
pr:
|
||||
branches:
|
||||
include:
|
||||
- main
|
||||
- release/*
|
||||
|
||||
stages:
|
||||
- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
|
||||
- template: templates/stages/vmr-scan.yml
|
||||
|
||||
- template: /src/installer/eng/pipelines/templates/stages/vmr-build.yml
|
||||
parameters:
|
||||
isBuiltFromVmr: true
|
||||
isLiteBuild: true
|
||||
|
|
@ -1,31 +1,27 @@
|
|||
# This is the main build definition (PR+CI) for dotnet/dotnet
|
||||
# This is the dotnet/dotnet pipeline that is triggered every weekday at midnight (00:00 UTC) for "main" and when changes are pushed to release/* and internal/release/* branches.
|
||||
|
||||
schedules:
|
||||
- cron: '0 0 * * Mon-Fri'
|
||||
displayName: Weekday midnight build
|
||||
branches:
|
||||
include:
|
||||
- main
|
||||
batch: true
|
||||
|
||||
trigger:
|
||||
batch: true
|
||||
branches:
|
||||
include:
|
||||
- main
|
||||
- release/*
|
||||
- internal/release/*
|
||||
exclude:
|
||||
- release/*.0.2xx
|
||||
- release/*.0.3xx
|
||||
- release/*.0.4xx
|
||||
- internal/release/*.0.2xx
|
||||
- internal/release/*.0.3xx
|
||||
- internal/release/*.0.4xx
|
||||
include:
|
||||
- release/*
|
||||
- internal/release/*
|
||||
|
||||
pr:
|
||||
branches:
|
||||
include:
|
||||
- main
|
||||
- release/*
|
||||
- internal/release/*
|
||||
pr: none
|
||||
|
||||
stages:
|
||||
- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
|
||||
- ${{ if ne(variables['Build.Reason'], 'Schedule') }}:
|
||||
- template: templates/stages/vmr-scan.yml
|
||||
|
||||
- template: /src/installer/eng/pipelines/templates/stages/vmr-build.yml
|
||||
parameters:
|
||||
isBuiltFromVmr: true
|
||||
isLiteBuild: false
|
||||
|
|
|
|||
|
|
@ -10,6 +10,8 @@
|
|||
### --no-bootstrap Don't replace portable packages in the download source-built artifacts
|
||||
### --no-prebuilts Exclude the download of the prebuilts archive
|
||||
### --no-sdk Exclude the download of the .NET SDK
|
||||
### --artifacts-rid The RID of the previously source-built artifacts archive to download
|
||||
### Default is centos.8-x64
|
||||
### --runtime-source-feed URL of a remote server or a local directory, from which SDKs and
|
||||
### runtimes can be downloaded
|
||||
### --runtime-source-feed-key Key for accessing the above server, if necessary
|
||||
|
|
@ -24,10 +26,13 @@ function print_help () {
|
|||
sed -n '/^### /,/^$/p' "$source" | cut -b 5-
|
||||
}
|
||||
|
||||
defaultArtifactsRid='centos.8-x64'
|
||||
|
||||
buildBootstrap=true
|
||||
downloadArtifacts=true
|
||||
downloadPrebuilts=true
|
||||
installDotnet=true
|
||||
artifactsRid=$defaultArtifactsRid
|
||||
runtime_source_feed='' # IBM requested these to support s390x scenarios
|
||||
runtime_source_feed_key='' # IBM requested these to support s390x scenarios
|
||||
positional_args=()
|
||||
|
|
@ -53,6 +58,9 @@ while :; do
|
|||
--no-sdk)
|
||||
installDotnet=false
|
||||
;;
|
||||
--artifacts-rid)
|
||||
artifactsRid=$2
|
||||
;;
|
||||
--runtime-source-feed)
|
||||
runtime_source_feed=$2
|
||||
shift
|
||||
|
|
@ -107,17 +115,27 @@ fi
|
|||
function DownloadArchive {
|
||||
archiveType="$1"
|
||||
isRequired="$2"
|
||||
artifactsRid="$3"
|
||||
|
||||
packageVersionsPath="$SCRIPT_ROOT/eng/Versions.props"
|
||||
notFoundMessage="No source-built $archiveType found to download..."
|
||||
|
||||
echo " Looking for source-built $archiveType to download..."
|
||||
archiveVersionLine=$(grep -m 1 "<PrivateSourceBuilt${archiveType}Url>" "$packageVersionsPath" || :)
|
||||
versionPattern="<PrivateSourceBuilt${archiveType}Url>(.*)</PrivateSourceBuilt${archiveType}Url>"
|
||||
archiveVersionLine=$(grep -m 1 "<PrivateSourceBuilt${archiveType}Version>" "$packageVersionsPath" || :)
|
||||
versionPattern="<PrivateSourceBuilt${archiveType}Version>(.*)</PrivateSourceBuilt${archiveType}Version>"
|
||||
if [[ $archiveVersionLine =~ $versionPattern ]]; then
|
||||
archiveUrl="${BASH_REMATCH[1]}"
|
||||
echo " Downloading source-built $archiveType from $archiveUrl..."
|
||||
(cd "$packagesArchiveDir" && curl --retry 5 -O "$archiveUrl")
|
||||
archiveVersion="${BASH_REMATCH[1]}"
|
||||
|
||||
if [ "$archiveType" == "Prebuilts" ]; then
|
||||
archiveRid=$defaultArtifactsRid
|
||||
else
|
||||
archiveRid=$artifactsRid
|
||||
fi
|
||||
|
||||
archiveUrl="https://dotnetcli.azureedge.net/source-built-artifacts/assets/Private.SourceBuilt.$archiveType.$archiveVersion.$archiveRid.tar.gz"
|
||||
|
||||
echo " Downloading source-built $archiveType from $archiveUrl..."
|
||||
(cd "$packagesArchiveDir" && curl --retry 5 -O "$archiveUrl")
|
||||
elif [ "$isRequired" == true ]; then
|
||||
echo " ERROR: $notFoundMessage"
|
||||
exit 1
|
||||
|
|
@ -164,12 +182,12 @@ fi
|
|||
|
||||
# Read the eng/Versions.props to get the archives to download and download them
|
||||
if [ "$downloadArtifacts" == true ]; then
|
||||
DownloadArchive Artifacts true
|
||||
DownloadArchive Artifacts true $artifactsRid
|
||||
if [ "$buildBootstrap" == true ]; then
|
||||
BootstrapArtifacts
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "$downloadPrebuilts" == true ]; then
|
||||
DownloadArchive Prebuilts false
|
||||
DownloadArchive Prebuilts false $artifactsRid
|
||||
fi
|
||||
|
|
|
|||
Loading…
Reference in a new issue