diff --git a/eng/pipelines/templates/jobs/vmr-build.yml b/eng/pipelines/templates/jobs/vmr-build.yml index 0f61dc489..c5e3d243c 100644 --- a/eng/pipelines/templates/jobs/vmr-build.yml +++ b/eng/pipelines/templates/jobs/vmr-build.yml @@ -1,11 +1,39 @@ -### This job source-builds https://github.com/dotnet/dotnet with given parameters -### If run in a PR, new changes are applied to a local copy of the VMR, then it is source-built and tested +### This job builds https://github.com/dotnet/dotnet with given parameters +### If run in a PR, new changes are applied to a local copy of the VMR, then it is built and tested parameters: +- name: architecture + type: string + +- name: artifactsRid + type: string + default: '' + +- name: buildName + type: string + +- name: container + type: string + +- name: crossRootFS + type: string + default: '' + - name: isBuiltFromVmr displayName: True when build is running from dotnet/dotnet directly type: boolean +- name: pool + type: object + +- name: targetOS + type: string + default: '' + +- name: targetArchitecture + type: string + default: '' + - name: vmrPath type: string default: $(Agent.BuildDirectory)/vmr @@ -15,25 +43,25 @@ parameters: type: string default: $(Build.SourceBranch) -- name: buildName - type: string +#### SOURCE-ONLY parameters #### -- name: architecture - type: string - -- name: artifactsRid - type: string - default: '' - -- name: container - type: string - -- name: pool - type: object - -# Allow downloading artifacts from the internet during the build -- name: runOnline +# Instead of building the VMR directly, exports the sources into a tarball and builds from that +- name: buildFromArchive type: boolean + default: false + +# Enable for source-building the VMR +- name: buildSourceOnly + type: boolean + default: false + +- name: enablePoison + type: boolean + default: false + +- name: excludeOmniSharpTests + type: boolean + default: false # Name of a previous job (from the same template as this) whose output will be used to build this job # The SDK from its artifacts is copied to vmr/.dotnet @@ -41,18 +69,12 @@ parameters: type: string default: '' -- name: excludeOmniSharpTests +# Allow downloading artifacts from the internet during the build +- name: runOnline type: boolean + default: true -- name: enablePoison - type: boolean - -# Instead of building the VMR directly, exports the sources into a tarball and builds from that -- name: buildFromArchive - type: boolean - -# Use the previous version's SDK to build the current one -- name: withPreviousSDK +- name: runTests type: boolean default: false @@ -61,22 +83,11 @@ parameters: type: boolean default: false -- name: crossRootFS - type: string - default: '' - -- name: runTests +# Use the previous version's SDK to build the current one +- name: withPreviousSDK type: boolean default: false -- name: targetOS - type: string - default: '' - -- name: targetArchitecture - type: string - default: '' - jobs: - job: ${{ parameters.buildName }}_${{ parameters.architecture }} timeoutInMinutes: 150 @@ -133,7 +144,7 @@ jobs: displayName: Export VMR sources workingDirectory: $(Build.StagingDirectory) - - ${{ if ne(variables['System.TeamProject'], 'public') }}: + - ${{ if and(ne(variables['System.TeamProject'], 'public'), eq(parameters.runTests, 'True')) }}: - script: cp "$(sourcesPath)/src/installer/NuGet.config" "$(sourcesPath)/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/online.NuGet.Config" displayName: Copy Test NuGet Config @@ -188,112 +199,114 @@ jobs: exit 1 fi displayName: Setup Previously Source-Built SDK + + - ${{ if eq(parameters.targetOS, 'windows') }}: + - script: | + call $(sourcesPath)\build.cmd -ci -cleanWhileBuilding -prepareMachine + displayName: Build - - script: | - set -x + - ${{ else }}: + - ${{ if eq(parameters.buildSourceOnly, 'true') }}: + - script: | + set -x - customPrepArgs="" - prepSdk=true + customPrepArgs="" + prepSdk=true - if [[ -n '${{ parameters.artifactsRid }}' ]]; then - customPrepArgs="${customPrepArgs} --artifacts-rid ${{ parameters.artifactsRid }}" - fi + if [[ -n '${{ parameters.artifactsRid }}' ]]; then + customPrepArgs="${customPrepArgs} --artifacts-rid ${{ parameters.artifactsRid }}" + fi - if [[ '${{ parameters.withPreviousSDK }}' == 'True' ]]; then - # Source-built artifacts are from CentOS 8 Stream or Alpine 3.19. We want to download them without - # downloading portable versions from the internet. - customPrepArgs="${customPrepArgs} --no-sdk --no-bootstrap" - prepSdk=false - elif [[ -n '${{ parameters.reuseBuildArtifactsFrom }}' ]]; then - customPrepArgs="${customPrepArgs} --no-sdk --no-artifacts" - prepSdk=false - fi + if [[ '${{ parameters.withPreviousSDK }}' == 'True' ]]; then + # Source-built artifacts are from CentOS 8 Stream or Alpine 3.19. We want to download them without + # downloading portable versions from the internet. + customPrepArgs="${customPrepArgs} --no-sdk --no-bootstrap" + prepSdk=false + elif [[ -n '${{ parameters.reuseBuildArtifactsFrom }}' ]]; then + customPrepArgs="${customPrepArgs} --no-sdk --no-artifacts" + prepSdk=false + fi - if [[ "$prepSdk" == "false" ]]; then - mkdir $(sourcesPath)/.dotnet - previousSdkPath="$(sourcesPath)/prereqs/packages/archive/dotnet-sdk-*.tar.gz" - eval tar -ozxf "$previousSdkPath" -C "$(sourcesPath)/.dotnet" - eval rm -f "$previousSdkPath" + if [[ "$prepSdk" == "false" ]]; then + mkdir $(sourcesPath)/.dotnet + previousSdkPath="$(sourcesPath)/prereqs/packages/archive/dotnet-sdk-*.tar.gz" + eval tar -ozxf "$previousSdkPath" -C "$(sourcesPath)/.dotnet" + eval rm -f "$previousSdkPath" - echo "##vso[task.setvariable variable=additionalBuildArgs]--with-sdk /vmr/.dotnet" - fi + echo "##vso[task.setvariable variable=additionalBuildArgs]--with-sdk /vmr/.dotnet" + fi - # Only use Docker stuff on Linux - if [[ -n "${{ parameters.container }}" ]]; then - docker run --rm -v "$(sourcesPath):/vmr" -w /vmr ${{ parameters.container }} ./prep.sh $customPrepArgs - else - cd $(sourcesPath) - ./prep.sh $customPrepArgs - fi - displayName: Prep the Build + docker run --rm -v "$(sourcesPath):/vmr" -w /vmr ${{ parameters.container }} ./prep.sh $customPrepArgs + displayName: Prep the Build - - script: | - set -x - df -h + - script: | + set -x + df -h - # Allows Arcade to have access to the commit for the build - customEnvVars="BUILD_SOURCEVERSION=$BUILD_SOURCEVERSION" - customBuildArgs="--ci --prepareMachine" - if [[ '${{ parameters.runOnline }}' == 'True' ]]; then - customBuildArgs="$customBuildArgs --online" - fi - - if [[ '${{ parameters.enablePoison }}' == 'True' ]]; then - customBuildArgs="$customBuildArgs --poison" - fi - - if [[ '${{ parameters.buildFromArchive }}' == 'True' ]]; then - customBuildArgs="$customBuildArgs --source-repository https://github.com/dotnet/dotnet" - customBuildArgs="$customBuildArgs --source-version $(git -C "${{ parameters.vmrPath }}" rev-parse HEAD)" - fi - - if [[ '${{ parameters.useMonoRuntime }}' == 'True' ]]; then - customBuildArgs="$customBuildArgs --use-mono-runtime" - fi - - if [[ -n "${{ parameters.container }}" ]]; then - useDocker=true - fi - - if [[ '${{ parameters.useMonoRuntime }}' == 'True' ]]; then - customEnvVars="$customEnvVars CROSSCOMPILE=1" - fi - - if [[ ! -z '${{ parameters.crossRootFs }}' ]]; then - customEnvVars="$customEnvVars ROOTFS_DIR=${{ parameters.crossRootFs}}" - fi - - if [[ ! -z '${{ parameters.targetOS }}' ]]; then - extraBuildProperties="$extraBuildProperties /p:TargetOS=${{ parameters.targetOS }}" - fi - - if [[ ! -z '${{ parameters.targetArchitecture }}' ]]; then - extraBuildProperties="$extraBuildProperties /p:TargetArchitecture=${{ parameters.targetArchitecture }}" - fi - - # Only use Docker stuff on Linux - if [[ "$useDocker" == "true" ]]; then - customDockerRunArgs="" - for envVar in $customEnvVars; do - customDockerRunArgs="$customDockerRunArgs -e $envVar" - done - - if [[ '${{ parameters.runOnline }}' == 'False' ]]; then - customDockerRunArgs="$customDockerRunArgs --network none" + customEnvVars="" + customBuildArgs="--ci --clean-while-building --prepareMachine" + if [[ '${{ parameters.runOnline }}' == 'True' ]]; then + customBuildArgs="$customBuildArgs --online" fi - docker run --rm -v "$(sourcesPath):/vmr" -w /vmr $customDockerRunArgs ${{ parameters.container }} ./build.sh --source-only --clean-while-building $(additionalBuildArgs) $customBuildArgs $extraBuildProperties - else - for envVar in $customEnvVars; do - customEnvVarsWithBashSyntax="$customEnvVarsWithBashSyntax export $envVar;" - done - cd $(sourcesPath) - eval $customEnvVarsWithBashSyntax - ./build.sh --source-only --clean-while-building $(additionalBuildArgs) $customBuildArgs $extraBuildProperties - fi - displayName: Build + if [[ '${{ parameters.enablePoison }}' == 'True' ]]; then + customBuildArgs="$customBuildArgs --poison" + fi - # Don't run tests if overriding RID, we don't support that for now + if [[ '${{ parameters.buildFromArchive }}' == 'True' ]]; then + customBuildArgs="$customBuildArgs --source-repository https://github.com/dotnet/dotnet" + customBuildArgs="$customBuildArgs --source-version $(git -C "${{ parameters.vmrPath }}" rev-parse HEAD)" + fi + + if [[ '${{ parameters.buildSourceOnly }}' == 'True' ]]; then + customBuildArgs="$customBuildArgs --source-only" + fi + + if [[ '${{ parameters.useMonoRuntime }}' == 'True' ]]; then + customBuildArgs="$customBuildArgs --use-mono-runtime" + customEnvVars="$customEnvVars CROSSCOMPILE=1" + fi + + if [[ ! -z '${{ parameters.crossRootFs }}' ]]; then + customEnvVars="$customEnvVars ROOTFS_DIR=${{ parameters.crossRootFs}}" + fi + + if [[ ! -z '${{ parameters.targetOS }}' ]]; then + extraBuildProperties="$extraBuildProperties /p:TargetOS=${{ parameters.targetOS }}" + fi + + if [[ ! -z '${{ parameters.targetArchitecture }}' ]]; then + extraBuildProperties="$extraBuildProperties /p:TargetArchitecture=${{ parameters.targetArchitecture }}" + fi + + buildArgs="$(additionalBuildArgs) $customBuildArgs $extraBuildProperties" + + # Only use Docker when a container is specified + if [[ -n "${{ parameters.container }}" ]]; then + # Allows Arcade to have access to the commit for the build, pass it through to the container + customEnvVars="$customEnvVars BUILD_SOURCEVERSION=$BUILD_SOURCEVERSION" + + customDockerRunArgs="" + for envVar in $customEnvVars; do + customDockerRunArgs="$customDockerRunArgs -e $envVar" + done + + if [[ '${{ parameters.runOnline }}' == 'False' ]]; then + customDockerRunArgs="$customDockerRunArgs --network none" + fi + + docker run --rm -v "$(sourcesPath):/vmr" -w /vmr $customDockerRunArgs ${{ parameters.container }} ./build.sh $buildArgs + else + for envVar in $customEnvVars; do + customEnvVarsWithBashSyntax="$customEnvVarsWithBashSyntax export $envVar;" + done + cd $(sourcesPath) + eval $customEnvVarsWithBashSyntax + ./build.sh $buildArgs + fi + displayName: Build + + # Only run tests if enabled - ${{ if eq(parameters.runTests, 'True') }}: - script: | set -x @@ -309,25 +322,56 @@ jobs: docker run --rm $dockerVolumeArgs -w /vmr $dockerEnvArgs ${{ parameters.container }} ./build.sh --source-only $poisonArg --test $(additionalBuildArgs) /p:SmokeTestConsoleVerbosity=detailed displayName: Run Tests - # Don't use CopyFiles@2 as it encounters permissions issues because it indexes all files in the source directory graph. - - script: | - set -x + - ${{ if eq(parameters.targetOS, 'windows') }}: - targetFolder=$(Build.StagingDirectory)/BuildLogs/ - mkdir -p ${targetFolder} + # Don't use CopyFiles@2 as it encounters permissions issues because it indexes all files in the source directory graph. + - powershell: | + function CopyWithRelativeFolders($sourcePath, $targetFolder, $filter) { + Get-ChildItem -Path $sourcePath -Filter $filter -Recurse | ForEach-Object { + $targetPath = Join-Path $targetFolder (Resolve-Path -Relative $_.FullName) + New-Item -ItemType Directory -Path (Split-Path -Parent $targetPath) -Force | Out-Null + Copy-Item $_.FullName -Destination $targetPath -Force + } + } - cd "$(sourcesPath)" - find artifacts/ -type f -name "*.binlog" -exec cp {} --parents -t ${targetFolder} \; - find artifacts/ -type f -name "*.log" -exec cp {} --parents -t ${targetFolder} \; - find artifacts/prebuilt-report/ -exec cp {} --parents -t ${targetFolder} \; - find src/ -type f -name "*.binlog" -exec cp {} --parents -t ${targetFolder} \; - find src/ -type f -name "*.log" -exec cp {} --parents -t ${targetFolder} \; - find test/ -type f -name "*.binlog" -exec cp {} --parents -t ${targetFolder} \; - find test/ -type f -name "Updated*.diff" -exec cp {} --parents -t ${targetFolder} \; - find test/ -type f -name "Updated*.txt" -exec cp {} --parents -t ${targetFolder} \; - displayName: Prepare BuildLogs staging directory - continueOnError: true - condition: succeededOrFailed() + $targetFolder = "$(Build.StagingDirectory)/BuildLogs/" + New-Item -ItemType Directory -Path $targetFolder -Force | Out-Null + + cd "$(sourcesPath)" + + CopyWithRelativeFolders "artifacts/" $targetFolder "*.binlog" + CopyWithRelativeFolders "artifacts/" $targetFolder "*.log" + CopyWithRelativeFolders "artifacts/prebuilt-report/" $targetFolder + CopyWithRelativeFolders "src/" $targetFolder "*.binlog" + CopyWithRelativeFolders "src/" $targetFolder "*.log" + CopyWithRelativeFolders "test/" $targetFolder "*.binlog" + CopyWithRelativeFolders "test/" $targetFolder "Updated*.diff" + CopyWithRelativeFolders "test/" $targetFolder "Updated*.txt" + displayName: Prepare BuildLogs staging directory + continueOnError: true + condition: succeededOrFailed() + + - ${{ else }}: + + # Don't use CopyFiles@2 as it encounters permissions issues because it indexes all files in the source directory graph. + - script: | + set -x + + targetFolder=$(Build.StagingDirectory)/BuildLogs/ + mkdir -p ${targetFolder} + + cd "$(sourcesPath)" + find artifacts/ -type f -name "*.binlog" -exec rsync -R {} -t ${targetFolder} \; + find artifacts/ -type f -name "*.log" -exec rsync -R {} -t ${targetFolder} \; + find artifacts/prebuilt-report/ -exec rsync -R {} -t ${targetFolder} \; + find src/ -type f -name "*.binlog" -exec rsync -R {} -t ${targetFolder} \; + find src/ -type f -name "*.log" -exec rsync -R {} -t ${targetFolder} \; + find test/ -type f -name "*.binlog" -exec rsync -R {} -t ${targetFolder} \; + find test/ -type f -name "Updated*.diff" -exec rsync -R {} -t ${targetFolder} \; + find test/ -type f -name "Updated*.txt" -exec rsync -R {} -t ${targetFolder} \; + displayName: Prepare BuildLogs staging directory + continueOnError: true + condition: succeededOrFailed() - publish: '$(Build.StagingDirectory)/BuildLogs' artifact: $(Agent.JobName)_BuildLogs_Attempt$(System.JobAttempt) @@ -335,17 +379,19 @@ jobs: continueOnError: true condition: succeededOrFailed() - - task: PublishTestResults@2 - displayName: Publish Test Results - condition: succeededOrFailed() - continueOnError: true - inputs: - testRunner: vSTest - testResultsFiles: 'test/**/*.trx' - searchFolder: ${{ variables.sourcesPath }} - mergeTestResults: true - publishRunAttachments: true - testRunTitle: SourceBuild_SmokeTests_$(Agent.JobName) + # Only upload test results if enabled + - ${{ if eq(parameters.runTests, 'True') }}: + - task: PublishTestResults@2 + displayName: Publish Test Results + condition: succeededOrFailed() + continueOnError: true + inputs: + testRunner: vSTest + testResultsFiles: 'test/**/*.trx' + searchFolder: ${{ variables.sourcesPath }} + mergeTestResults: true + publishRunAttachments: true + testRunTitle: SourceBuild_SmokeTests_$(Agent.JobName) - publish: '${{ variables.sourcesPath }}/artifacts/${{ parameters.architecture }}/Release/' artifact: $(Agent.JobName)_Artifacts diff --git a/eng/pipelines/templates/stages/vmr-build.yml b/eng/pipelines/templates/stages/vmr-build.yml index 1dd31827b..e270bbb14 100644 --- a/eng/pipelines/templates/stages/vmr-build.yml +++ b/eng/pipelines/templates/stages/vmr-build.yml @@ -1,150 +1,58 @@ -### This stage source-builds https://github.com/dotnet/dotnet with varying parameters -### If run in a PR, new changes are applied to a local copy of the VMR, then it is source-built and tested +### This stage builds https://github.com/dotnet/dotnet with varying parameters +### If run in a PR, new changes are applied to a local copy of the VMR, then it is built and tested parameters: - dependsOn: [] - condition: always() +# Branch of the VMR to use (to push to for internal builds) +- name: vmrBranch + type: string + default: $(Build.SourceBranch) - # Branch of the VMR to use (to push to for internal builds) - vmrBranch: $(Build.SourceBranch) +# Scope of jobs which are executed +- name: scope + type: string + values: + # run 1 leg for smoke tests + - ultralite + # run several legs e.g. win/linux/mac for basic testing + - lite + # run everything + - full - # True when the build is a lite build - isLiteBuild: +# True when build is running from dotnet/dotnet directly +- name: isBuiltFromVmr + type: boolean + default: false - # True when build is running from dotnet/dotnet directly - isBuiltFromVmr: - - # The following parameters aren't expected to be passed in rather they are used for encapsulation - # ----------------------------------------------------------------------------------------------- - alpine319Container: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.19-WithNode - centOSStream8Container: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8 - centOSStream9Container: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9 - fedora39Container: mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-39 - ubuntu2204Container: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04 - ubuntu2204ArmContainer: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-arm64 - - # Internal builds - poolInternalAmd64: - name: NetCore1ESPool-Svc-Internal - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 - poolInternalAmd64PR: - name: NetCore1ESPool-Internal-XL - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 - poolInternalArm64: - name: Docker-Linux-Arm-Internal - - # Public builds / PRs - poolPublicAmd64: - name: NetCore-Public-XL - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open +# True when building the VMR in source-only mode +- name: isSourceOnlyBuild + type: boolean + default: false +#### SOURCE-ONLY BUILD #### stages: -- stage: VMR_Source_Build - displayName: VMR Source-Build - dependsOn: ${{ parameters.dependsOn }} - condition: ${{ parameters.condition }} - variables: - - ${{ if eq(variables['System.TeamProject'], 'public') }}: - - name: defaultPoolName - value: ${{ parameters.poolPublicAmd64.name }} - - name: defaultPoolDemands - value: ${{ parameters.poolPublicAmd64.demands }} - - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: - - name: defaultPoolName - value: ${{ parameters.poolInternalAmd64PR.name }} - - name: defaultPoolDemands - value: ${{ parameters.poolInternalAmd64PR.demands }} - - ${{ else }}: - - name: defaultPoolName - value: ${{ parameters.poolInternalAmd64.name }} - - name: defaultPoolDemands - value: ${{ parameters.poolInternalAmd64.demands }} - - - ${{ if ne(parameters.vmrBranch, '') }}: - - name: VmrBranch - value: ${{ parameters.vmrBranch }} - - ${{ else }}: - - name: VmrBranch - value: ${{ replace(replace(variables['Build.SourceBranch'], 'refs/heads/', ''), 'refs/pull/', '') }} - - jobs: - - - template: ../jobs/vmr-build.yml - parameters: - # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline - buildName: CentOSStream8_Online_MsftSdk - isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} - vmrBranch: ${{ variables.VmrBranch }} - architecture: x64 - pool: - name: ${{ variables.defaultPoolName }} - demands: ${{ variables.defaultPoolDemands }} - container: ${{ parameters.centOSStream8Container }} - buildFromArchive: false # 🚫 - enablePoison: false # 🚫 - excludeOmniSharpTests: true # ✅ - runOnline: true # ✅ - runTests: true # ✅ - useMonoRuntime: false # 🚫 - withPreviousSDK: false # 🚫 - - - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: +- ${{ if parameters.isSourceOnlyBuild }}: + - stage: VMR_SourceOnly_Build + displayName: VMR Source-Only Build + variables: + - template: ../variables/vmr-stage.yml + parameters: + vmrBranch: ${{ parameters.vmrBranch }} + jobs: + ### Jobs for ultralite builds ### - template: ../jobs/vmr-build.yml parameters: # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline - buildName: CentOSStream8_Online_CurrentSourceBuiltSdk + buildName: CentOSStream8_Online_MsftSdk isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} architecture: x64 pool: name: ${{ variables.defaultPoolName }} - demands: ${{ variables.defaultPoolDemands }} - container: ${{ parameters.centOSStream8Container }} - buildFromArchive: false # 🚫 - enablePoison: false # 🚫 - excludeOmniSharpTests: true # ✅ - runOnline: true # ✅ - runTests: true # ✅ - useMonoRuntime: false # 🚫 - withPreviousSDK: false # 🚫 - reuseBuildArtifactsFrom: CentOSStream8_Online_MsftSdk - - - template: ../jobs/vmr-build.yml - parameters: - # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline - buildName: Alpine319_Offline_PreviousSourceBuiltSdk - isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} - vmrBranch: ${{ variables.VmrBranch }} - architecture: x64 - artifactsRid: alpine.3.19-x64 - pool: - name: ${{ variables.defaultPoolName }} - demands: ${{ variables.defaultPoolDemands }} - container: ${{ parameters.alpine319Container }} - buildFromArchive: false # 🚫 - enablePoison: true # ✅ - excludeOmniSharpTests: true # ✅ - runOnline: false # 🚫 - runTests: true # ✅ - useMonoRuntime: false # 🚫 - withPreviousSDK: true # ✅ - - - ${{ if eq(parameters.isLiteBuild, false) }}: - - - template: ../jobs/vmr-build.yml - parameters: - # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline - buildName: Alpine319_Online_MsftSdk - isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} - vmrBranch: ${{ variables.VmrBranch }} - architecture: x64 - pool: - name: ${{ variables.defaultPoolName }} - demands: ${{ variables.defaultPoolDemands }} - container: ${{ parameters.alpine319Container }} + demands: ${{ variables.defaultPoolDemandsLinux }} + container: ${{ variables.centOSStream8Container }} buildFromArchive: false # 🚫 + buildSourceOnly: true # ✅ enablePoison: false # 🚫 excludeOmniSharpTests: true # ✅ runOnline: true # ✅ @@ -152,174 +60,344 @@ stages: useMonoRuntime: false # 🚫 withPreviousSDK: false # 🚫 + ### Additional jobs for lite/full builds ### + - ${{ if in(parameters.scope, 'lite', 'full') }}: + + - template: ../jobs/vmr-build.yml + parameters: + # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline + buildName: CentOSStream8_Online_CurrentSourceBuiltSdk + isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} + vmrBranch: ${{ variables.VmrBranch }} + architecture: x64 + pool: + name: ${{ variables.defaultPoolName }} + demands: ${{ variables.defaultPoolDemandsLinux }} + container: ${{ variables.centOSStream8Container }} + buildFromArchive: false # 🚫 + buildSourceOnly: true # ✅ + enablePoison: false # 🚫 + excludeOmniSharpTests: true # ✅ + runOnline: true # ✅ + runTests: true # ✅ + useMonoRuntime: false # 🚫 + withPreviousSDK: false # 🚫 + reuseBuildArtifactsFrom: CentOSStream8_Online_MsftSdk + + - template: ../jobs/vmr-build.yml + parameters: + # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline + buildName: Alpine319_Offline_PreviousSourceBuiltSdk + isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} + vmrBranch: ${{ variables.VmrBranch }} + architecture: x64 + artifactsRid: alpine.3.19-x64 + pool: + name: ${{ variables.defaultPoolName }} + demands: ${{ variables.defaultPoolDemandsLinux }} + container: ${{ variables.alpine319Container }} + buildFromArchive: false # 🚫 + buildSourceOnly: true # ✅ + enablePoison: true # ✅ + excludeOmniSharpTests: true # ✅ + runOnline: false # 🚫 + runTests: true # ✅ + useMonoRuntime: false # 🚫 + withPreviousSDK: true # ✅ + + ### Additional jobs for full build ### + - ${{ if in(parameters.scope, 'full') }}: + + - template: ../jobs/vmr-build.yml + parameters: + # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline + buildName: Alpine319_Online_MsftSdk + isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} + vmrBranch: ${{ variables.VmrBranch }} + architecture: x64 + pool: + name: ${{ variables.defaultPoolName }} + demands: ${{ variables.defaultPoolDemandsLinux }} + container: ${{ variables.alpine319Container }} + buildFromArchive: false # 🚫 + buildSourceOnly: true # ✅ + enablePoison: false # 🚫 + excludeOmniSharpTests: true # ✅ + runOnline: true # ✅ + runTests: true # ✅ + useMonoRuntime: false # 🚫 + withPreviousSDK: false # 🚫 + + - template: ../jobs/vmr-build.yml + parameters: + # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline + buildName: CentOSStream8_Offline_MsftSdk + isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} + vmrBranch: ${{ variables.VmrBranch }} + architecture: x64 + pool: + name: ${{ variables.defaultPoolName }} + demands: ${{ variables.defaultPoolDemandsLinux }} + container: ${{ variables.centOSStream8Container }} + buildFromArchive: true # ✅ + buildSourceOnly: true # ✅ + enablePoison: false # 🚫 + excludeOmniSharpTests: true # ✅ + runOnline: false # 🚫 + runTests: true # ✅ + useMonoRuntime: false # 🚫 + withPreviousSDK: false # 🚫 + + - template: ../jobs/vmr-build.yml + parameters: + # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline + buildName: CentOSStream8_Online_PreviousSourceBuiltSdk + isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} + vmrBranch: ${{ variables.VmrBranch }} + architecture: x64 + artifactsRid: centos.8-x64 + pool: + name: ${{ variables.defaultPoolName }} + demands: ${{ variables.defaultPoolDemandsLinux }} + container: ${{ variables.centOSStream8Container }} + buildFromArchive: false # 🚫 + buildSourceOnly: true # ✅ + enablePoison: false # 🚫 + excludeOmniSharpTests: true # ✅ + runOnline: true # ✅ + runTests: true # ✅ + useMonoRuntime: false # 🚫 + withPreviousSDK: true # ✅ + + - template: ../jobs/vmr-build.yml + parameters: + # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline + buildName: CentOSStream8_Offline_PreviousSourceBuiltSdk + isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} + vmrBranch: ${{ variables.VmrBranch }} + architecture: x64 + artifactsRid: centos.8-x64 + pool: + name: ${{ variables.defaultPoolName }} + demands: ${{ variables.defaultPoolDemandsLinux }} + container: ${{ variables.centOSStream8Container }} + buildFromArchive: false # 🚫 + buildSourceOnly: true # ✅ + enablePoison: false # 🚫 + excludeOmniSharpTests: true # ✅ + runOnline: false # 🚫 + runTests: true # ✅ + useMonoRuntime: false # 🚫 + withPreviousSDK: true # ✅ + + - template: ../jobs/vmr-build.yml + parameters: + # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline + buildName: CentOSStream8_Mono_Offline_MsftSdk + isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} + vmrBranch: ${{ variables.VmrBranch }} + architecture: x64 + pool: + name: ${{ variables.defaultPoolName }} + demands: ${{ variables.defaultPoolDemandsLinux }} + container: ${{ variables.centOSStream8Container }} + buildFromArchive: true # ✅ + buildSourceOnly: true # ✅ + enablePoison: false # 🚫 + excludeOmniSharpTests: true # ✅ + runOnline: false # 🚫 + runTests: true # ✅ + useMonoRuntime: true # ✅ + withPreviousSDK: false # 🚫 + + - template: ../jobs/vmr-build.yml + parameters: + # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline + buildName: CentOSStream9_Offline_MsftSdk + isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} + vmrBranch: ${{ variables.VmrBranch }} + architecture: x64 + pool: + name: ${{ variables.defaultPoolName }} + demands: ${{ variables.defaultPoolDemandsLinux }} + container: ${{ variables.centOSStream9Container }} + buildFromArchive: true # ✅ + buildSourceOnly: true # ✅ + enablePoison: false # 🚫 + excludeOmniSharpTests: false # 🚫 + runOnline: false # 🚫 + runTests: true # ✅ + useMonoRuntime: false # 🚫 + withPreviousSDK: false # 🚫 + + - template: ../jobs/vmr-build.yml + parameters: + # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline + buildName: Fedora39_Offline_MsftSdk + isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} + vmrBranch: ${{ variables.VmrBranch }} + architecture: x64 + pool: + name: ${{ variables.defaultPoolName }} + demands: ${{ variables.defaultPoolDemandsLinux }} + container: ${{ variables.fedora39Container }} + buildFromArchive: true # ✅ + buildSourceOnly: true # ✅ + enablePoison: false # 🚫 + excludeOmniSharpTests: false # 🚫 + runOnline: false # 🚫 + runTests: true # ✅ + useMonoRuntime: false # 🚫 + withPreviousSDK: false # 🚫 + + - template: ../jobs/vmr-build.yml + parameters: + # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline + buildName: Ubuntu2204_Offline_MsftSdk + isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} + vmrBranch: ${{ variables.VmrBranch }} + architecture: x64 + pool: + name: ${{ variables.defaultPoolName }} + demands: ${{ variables.defaultPoolDemandsLinux }} + container: ${{ variables.ubuntu2204Container }} + buildFromArchive: false # 🚫 + buildSourceOnly: true # ✅ + enablePoison: false # 🚫 + excludeOmniSharpTests: false # 🚫 + runOnline: false # 🚫 + runTests: true # ✅ + useMonoRuntime: false # 🚫 + withPreviousSDK: false # 🚫 + + - template: ../jobs/vmr-build.yml + parameters: + # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline + buildName: Ubuntu2204Arm64_Offline_MsftSdk + isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} + vmrBranch: ${{ variables.VmrBranch }} + architecture: arm64 + pool: ${{ variables.defaultPoolNameLinuxArm64 }} + container: ${{ variables.ubuntu2204ArmContainer }} + buildFromArchive: false # 🚫 + buildSourceOnly: true # ✅ + enablePoison: false # 🚫 + excludeOmniSharpTests: false # 🚫 + runOnline: false # 🚫 + runTests: true # ✅ + useMonoRuntime: false # 🚫 + withPreviousSDK: false # 🚫 + + - template: ../jobs/vmr-build.yml + parameters: + # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline + buildName: Fedora39_Offline_CurrentSourceBuiltSdk + isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} + vmrBranch: ${{ variables.VmrBranch }} + architecture: x64 + pool: + name: ${{ variables.defaultPoolName }} + demands: ${{ variables.defaultPoolDemandsLinux }} + container: ${{ variables.fedora39Container }} + buildFromArchive: false # 🚫 + buildSourceOnly: true # ✅ + enablePoison: false # 🚫 + excludeOmniSharpTests: false # 🚫 + runOnline: false # 🚫 + runTests: true # ✅ + useMonoRuntime: false # 🚫 + withPreviousSDK: false # 🚫 + reuseBuildArtifactsFrom: Fedora39_Offline_MsftSdk + +#### VERTICAL BUILD #### +- ${{ if not(parameters.isSourceOnlyBuild) }}: + - stage: VMR_Vertical_Build + displayName: VMR Vertical Build + variables: + - template: ../variables/vmr-stage.yml + parameters: + vmrBranch: ${{ parameters.vmrBranch }} + jobs: + - template: ../jobs/vmr-build.yml parameters: - # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline - buildName: CentOSStream8_Offline_MsftSdk + buildName: Ubuntu2204 isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} architecture: x64 pool: name: ${{ variables.defaultPoolName }} - demands: ${{ variables.defaultPoolDemands }} - container: ${{ parameters.centOSStream8Container }} - buildFromArchive: true # ✅ - enablePoison: false # 🚫 - excludeOmniSharpTests: true # ✅ - runOnline: false # 🚫 - runTests: true # ✅ - useMonoRuntime: false # 🚫 - withPreviousSDK: false # 🚫 + demands: ${{ variables.defaultPoolDemandsLinux }} + container: ${{ variables.ubuntu2204Container }} + targetOS: linux + targetArchitecture: x64 - template: ../jobs/vmr-build.yml parameters: - # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline - buildName: CentOSStream8_Online_PreviousSourceBuiltSdk + buildName: OSX isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} architecture: x64 - artifactsRid: centos.8-x64 pool: - name: ${{ variables.defaultPoolName }} - demands: ${{ variables.defaultPoolDemands }} - container: ${{ parameters.centOSStream8Container }} - buildFromArchive: false # 🚫 - enablePoison: false # 🚫 - excludeOmniSharpTests: true # ✅ - runOnline: true # ✅ - runTests: true # ✅ - useMonoRuntime: false # 🚫 - withPreviousSDK: true # ✅ + vmImage: ${{ variables.defaultPoolNameMac }} + container: '' + targetOS: osx + targetArchitecture: x64 - template: ../jobs/vmr-build.yml parameters: - # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline - buildName: CentOSStream8_Offline_PreviousSourceBuiltSdk - isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} - vmrBranch: ${{ variables.VmrBranch }} - architecture: x64 - artifactsRid: centos.8-x64 - pool: - name: ${{ variables.defaultPoolName }} - demands: ${{ variables.defaultPoolDemands }} - container: ${{ parameters.centOSStream8Container }} - buildFromArchive: false # 🚫 - enablePoison: false # 🚫 - excludeOmniSharpTests: true # ✅ - runOnline: false # 🚫 - runTests: true # ✅ - useMonoRuntime: false # 🚫 - withPreviousSDK: true # ✅ - - - template: ../jobs/vmr-build.yml - parameters: - # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline - buildName: CentOSStream8_Mono_Offline_MsftSdk + buildName: Windows isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} architecture: x64 pool: name: ${{ variables.defaultPoolName }} - demands: ${{ variables.defaultPoolDemands }} - container: ${{ parameters.centOSStream8Container }} - buildFromArchive: true # ✅ - enablePoison: false # 🚫 - excludeOmniSharpTests: true # ✅ - runOnline: false # 🚫 - runTests: true # ✅ - useMonoRuntime: true # ✅ - withPreviousSDK: false # 🚫 + demands: ${{ variables.defaultPoolDemandsWindows }} + container: '' + targetOS: windows + targetArchitecture: x64 - - template: ../jobs/vmr-build.yml - parameters: - # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline - buildName: CentOSStream9_Offline_MsftSdk - isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} - vmrBranch: ${{ variables.VmrBranch }} - architecture: x64 - pool: - name: ${{ variables.defaultPoolName }} - demands: ${{ variables.defaultPoolDemands }} - container: ${{ parameters.centOSStream9Container }} - buildFromArchive: true # ✅ - enablePoison: false # 🚫 - excludeOmniSharpTests: false # 🚫 - runOnline: false # 🚫 - runTests: true # ✅ - useMonoRuntime: false # 🚫 - withPreviousSDK: false # 🚫 + ### Additional jobs for full build ### + # Cross builds are currently failing + # - ${{ if in(parameters.scope, 'full') }}: - - template: ../jobs/vmr-build.yml - parameters: - # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline - buildName: Fedora39_Offline_MsftSdk - isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} - vmrBranch: ${{ variables.VmrBranch }} - architecture: x64 - pool: - name: ${{ variables.defaultPoolName }} - demands: ${{ variables.defaultPoolDemands }} - container: ${{ parameters.fedora39Container }} - buildFromArchive: true # ✅ - enablePoison: false # 🚫 - excludeOmniSharpTests: false # 🚫 - runOnline: false # 🚫 - runTests: true # ✅ - useMonoRuntime: false # 🚫 - withPreviousSDK: false # 🚫 - - template: ../jobs/vmr-build.yml - parameters: - # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline - buildName: Ubuntu2204_Offline_MsftSdk - isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} - vmrBranch: ${{ variables.VmrBranch }} - architecture: x64 - pool: - name: ${{ variables.defaultPoolName }} - demands: ${{ variables.defaultPoolDemands }} - container: ${{ parameters.ubuntu2204Container }} - buildFromArchive: false # 🚫 - enablePoison: false # 🚫 - excludeOmniSharpTests: false # 🚫 - runOnline: false # 🚫 - runTests: true # ✅ - useMonoRuntime: false # 🚫 - withPreviousSDK: false # 🚫 + # - template: ../jobs/vmr-build.yml + # parameters: + # buildName: MarinerCross + # isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} + # vmrBranch: ${{ variables.VmrBranch }} + # architecture: x64 + # pool: + # name: ${{ variables.defaultPoolName }} + # demands: ${{ variables.defaultPoolDemandsLinux }} + # container: ${{ variables.marinerX64CrossContainer }} + # crossRootFs: '/crossrootfs/x64' + # targetOS: linux + # targetArchitecture: x64 - - template: ../jobs/vmr-build.yml - parameters: - # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline - buildName: Ubuntu2204Arm64_Offline_MsftSdk - isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} - vmrBranch: ${{ variables.VmrBranch }} - architecture: arm64 - pool: ${{ parameters.poolInternalArm64 }} - container: ${{ parameters.ubuntu2204ArmContainer }} - buildFromArchive: false # 🚫 - enablePoison: false # 🚫 - excludeOmniSharpTests: false # 🚫 - runOnline: false # 🚫 - runTests: true # ✅ - useMonoRuntime: false # 🚫 - withPreviousSDK: false # 🚫 + # - template: ../jobs/vmr-build.yml + # parameters: + # buildName: MarinerCross + # isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} + # vmrBranch: ${{ variables.VmrBranch }} + # architecture: arm64 + # pool: + # name: ${{ variables.defaultPoolName }} + # demands: ${{ variables.defaultPoolDemandsLinux }} + # container: ${{ variables.marinerArm64CrossContainer }} + # crossRootFs: '/crossrootfs/arm64' + # targetOS: linux + # targetArchitecture: arm64 - - template: ../jobs/vmr-build.yml - parameters: - # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline - buildName: Fedora39_Offline_CurrentSourceBuiltSdk - isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} - vmrBranch: ${{ variables.VmrBranch }} - architecture: x64 - pool: - name: ${{ variables.defaultPoolName }} - demands: ${{ variables.defaultPoolDemands }} - container: ${{ parameters.fedora39Container }} - buildFromArchive: false # 🚫 - enablePoison: false # 🚫 - excludeOmniSharpTests: false # 🚫 - runOnline: false # 🚫 - runTests: true # ✅ - useMonoRuntime: false # 🚫 - withPreviousSDK: false # 🚫 - reuseBuildArtifactsFrom: Fedora39_Offline_MsftSdk + # - template: ../jobs/vmr-build.yml + # parameters: + # buildName: OSXCross + # isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} + # vmrBranch: ${{ variables.VmrBranch }} + # architecture: arm64 + # pool: + # vmImage: ${{ variables.defaultPoolNameMac }} + # container: '' + # targetOS: osx + # targetArchitecture: arm64 diff --git a/eng/pipelines/templates/stages/vmr-cross-build.yml b/eng/pipelines/templates/stages/vmr-cross-build.yml deleted file mode 100644 index 75028b3f1..000000000 --- a/eng/pipelines/templates/stages/vmr-cross-build.yml +++ /dev/null @@ -1,177 +0,0 @@ -### This stage source-builds https://github.com/dotnet/dotnet with varying parameters -### If run in a PR, new changes are applied to a local copy of the VMR, then it is source-built and tested - -parameters: - dependsOn: [] - condition: always() - - # Branch of the VMR to use (to push to for internal builds) - vmrBranch: $(Build.SourceBranch) - - # True when the build is a lite build - isLiteBuild: - - # True when build is running from dotnet/dotnet directly - isBuiltFromVmr: - - # The following parameters aren't expected to be passed in rather they are used for encapsulation - # ----------------------------------------------------------------------------------------------- - marinerX64CrossContainer: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-amd64 - marinerArm64CrossContainer: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm64 - - # Internal builds - poolInternalAmd64: - name: NetCore1ESPool-Svc-Internal - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 - poolInternalAmd64PR: - name: NetCore1ESPool-Internal-XL - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 - poolInternalArm64: - name: Docker-Linux-Arm-Internal - - # Public builds / PRs - poolPublicAmd64: - name: NetCore-Public-XL - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open - - # Mac - poolMac: - vmImage: 'macos-12' - -stages: -- stage: VMR_PoC_Build - displayName: VMR PoC-Build - dependsOn: ${{ parameters.dependsOn }} - condition: ${{ parameters.condition }} - variables: - - ${{ if eq(variables['System.TeamProject'], 'public') }}: - - name: defaultPoolName - value: ${{ parameters.poolPublicAmd64.name }} - - name: defaultPoolDemands - value: ${{ parameters.poolPublicAmd64.demands }} - - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: - - name: defaultPoolName - value: ${{ parameters.poolInternalAmd64PR.name }} - - name: defaultPoolDemands - value: ${{ parameters.poolInternalAmd64PR.demands }} - - ${{ else }}: - - name: defaultPoolName - value: ${{ parameters.poolInternalAmd64.name }} - - name: defaultPoolDemands - value: ${{ parameters.poolInternalAmd64.demands }} - - - ${{ if ne(parameters.vmrBranch, '') }}: - - name: VmrBranch - value: ${{ parameters.vmrBranch }} - - ${{ else }}: - - name: VmrBranch - value: ${{ replace(replace(variables['Build.SourceBranch'], 'refs/heads/', ''), 'refs/pull/', '') }} - - jobs: - - - template: ../jobs/vmr-build.yml - parameters: - # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline - buildName: MarinerCrossX64_Online_MsftSdk - isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} - vmrBranch: ${{ variables.VmrBranch }} - architecture: x64 - pool: - name: ${{ variables.defaultPoolName }} - demands: ${{ variables.defaultPoolDemands }} - container: ${{ parameters.marinerx64CrossContainer }} - buildFromArchive: false # 🚫 - enablePoison: false # 🚫 - excludeOmniSharpTests: true # ✅ - runOnline: true # ✅ - useMonoRuntime: true # ✅ - withPreviousSDK: false # 🚫 - crossRootFs: '/crossrootfs/x64' # 📝 - runTests: false # 🚫 - targetOS: 'linux' # 📝 - targetArchitecture: 'x64' # 📝 - - - template: ../jobs/vmr-build.yml - parameters: - # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline - buildName: MarinerCrossArm64_Online_MsftSdk_Mono - isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} - vmrBranch: ${{ variables.VmrBranch }} - architecture: arm64 - pool: - name: ${{ variables.defaultPoolName }} - demands: ${{ variables.defaultPoolDemands }} - container: ${{ parameters.marinerArm64CrossContainer }} - buildFromArchive: false # 🚫 - enablePoison: false # 🚫 - excludeOmniSharpTests: true # ✅ - runOnline: true # ✅ - useMonoRuntime: true # ✅ - withPreviousSDK: false # 🚫 - crossRootFs: '/crossrootfs/arm64' # 📝 - runTests: false # 🚫 - targetOS: 'linux' # 📝 - targetArchitecture: 'arm64' # 📝 - - - template: ../jobs/vmr-build.yml - parameters: - # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline - buildName: OSXCrossX64_Online_MsftSdk - isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} - vmrBranch: ${{ variables.VmrBranch }} - architecture: x64 - pool: - vmImage: ${{ parameters.poolMac.vmImage }} - container: '' - buildFromArchive: false # 🚫 - enablePoison: false # 🚫 - excludeOmniSharpTests: true # ✅ - runOnline: true # ✅ - useMonoRuntime: true # ✅ - withPreviousSDK: false # 🚫 - runTests: false # 🚫 - targetOS: 'osx' # 📝 - targetArchitecture: 'x64' # 📝 - - - template: ../jobs/vmr-build.yml - parameters: - # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline - buildName: OSXCrossArm64_Online_MsftSdk - isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} - vmrBranch: ${{ variables.VmrBranch }} - architecture: arm64 - pool: - vmImage: ${{ parameters.poolMac.vmImage }} - container: '' - buildFromArchive: false # 🚫 - enablePoison: false # 🚫 - excludeOmniSharpTests: true # ✅ - runOnline: true # ✅ - useMonoRuntime: true # ✅ - withPreviousSDK: false # 🚫 - runTests: false # 🚫 - targetOS: 'osx' # 📝 - targetArchitecture: 'arm64' # 📝 - - - template: ../jobs/vmr-build.yml - parameters: - # Changing the build name requires updating the referenced name in the source-build-sdk-diff-tests.yml pipeline - buildName: MarinerCrossArm64_Online_MsftSdk_CoreCLR - isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} - vmrBranch: ${{ variables.VmrBranch }} - architecture: arm64 - pool: - name: ${{ variables.defaultPoolName }} - demands: ${{ variables.defaultPoolDemands }} - container: ${{ parameters.marinerArm64CrossContainer }} - buildFromArchive: false # 🚫 - enablePoison: false # 🚫 - excludeOmniSharpTests: true # ✅ - runOnline: true # ✅ - useMonoRuntime: false # 🚫 - withPreviousSDK: false # 🚫 - crossRootFs: '/crossrootfs/arm64' # 📝 - runTests: false # 🚫 - targetOS: 'linux' # 📝 - targetArchitecture: 'arm64' # 📝 diff --git a/eng/pipelines/templates/steps/vmr-pull-updates.yml b/eng/pipelines/templates/steps/vmr-pull-updates.yml index 42bcf2aba..74e3fadf7 100644 --- a/eng/pipelines/templates/steps/vmr-pull-updates.yml +++ b/eng/pipelines/templates/steps/vmr-pull-updates.yml @@ -33,10 +33,19 @@ steps: - script: | git checkout -B ${{ parameters.vmrBranch }} echo "##vso[task.setvariable variable=vmrBranch]${{ parameters.vmrBranch }}" - mkdir -p ${{ parameters.vmrPath }}/artifacts/x64/Release displayName: Prepare branch ${{ parameters.vmrBranch }} workingDirectory: ${{ parameters.vmrPath }} +- script: | + mkdir -p ${{ parameters.vmrPath }}/artifacts/x64/Release + displayName: Create artifacts folder (Unix) + condition: ne(variables['Agent.OS'], 'Windows_NT') + +- powershell: | + New-Item -ItemType Directory -Path ${{ parameters.vmrPath }}/artifacts/x64/Release + displayName: Create artifacts folder (Windows) + condition: eq(variables['Agent.OS'], 'Windows_NT') + - script: | git config --global user.name "dotnet-maestro[bot]" git config --global user.email "dotnet-maestro[bot]@users.noreply.github.com" @@ -57,13 +66,36 @@ steps: --branch ${{ parameters.vmrBranch }} --repository "installer:${{ parameters.targetRef }}" --recursive - --remote "installer:$(pwd)" + --remote "installer:$(Agent.BuildDirectory)/installer" --component-template $(Agent.BuildDirectory)/installer/src/VirtualMonoRepo/Component.template.md --tpn-template $(Agent.BuildDirectory)/installer/src/VirtualMonoRepo/THIRD-PARTY-NOTICES.template.txt --debug || (echo "##vso[task.logissue type=error]Failed to synchronize the VMR" && exit 1) - displayName: Synchronize dotnet/dotnet + displayName: Synchronize dotnet/dotnet (Unix) + condition: ne(variables['Agent.OS'], 'Windows_NT') + workingDirectory: $(Agent.BuildDirectory)/installer + +- powershell: > + ./eng/vmr-sync.ps1 ` + -vmr ${{ parameters.vmrPath }} ` + -tmp $(Agent.TempDirectory) ` + -azdevPat '$(dn-bot-all-orgs-code-r)' ` + -branch ${{ parameters.vmrBranch }} ` + -repository "installer:${{ parameters.targetRef }}" ` + -recursive ` + # passing remote fails for some reason, but it is the default anyway + # -remote "installer:$(Agent.BuildDirectory)/installer" + -componentTemplate $(Agent.BuildDirectory)/installer/src/VirtualMonoRepo/Component.template.md ` + -tpnTemplate $(Agent.BuildDirectory)/installer/src/VirtualMonoRepo/THIRD-PARTY-NOTICES.template.txt ` + -debug + + if ($LASTEXITCODE -ne 0) { + echo "##vso[task.logissue type=error]Failed to synchronize the VMR" + exit 1 + } + displayName: Synchronize dotnet/dotnet (Windows) + condition: eq(variables['Agent.OS'], 'Windows_NT') workingDirectory: $(Agent.BuildDirectory)/installer - publish: $(Agent.TempDirectory) diff --git a/eng/pipelines/templates/variables/vmr-stage.yml b/eng/pipelines/templates/variables/vmr-stage.yml new file mode 100644 index 000000000..7e51a24a1 --- /dev/null +++ b/eng/pipelines/templates/variables/vmr-stage.yml @@ -0,0 +1,53 @@ +parameters: +- name: vmrBranch + type: string + +variables: +- ${{ if ne(parameters.vmrBranch, '') }}: + - name: VmrBranch + value: ${{ parameters.vmrBranch }} +- ${{ else }}: + - name: VmrBranch + value: ${{ replace(replace(variables['Build.SourceBranch'], 'refs/heads/', ''), 'refs/pull/', '') }} + +- name: alpine319Container + value: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.19-WithNode +- name: centOSStream8Container + value: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8 +- name: centOSStream9Container + value: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9 +- name: fedora39Container + value: mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-39 +- name: ubuntu2204Container + value: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04 +- name: ubuntu2204ArmContainer + value: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-arm64 +- name: marinerX64CrossContainer + value: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-amd64 +- name: marinerArm64CrossContainer + value: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm64 + +- ${{ if eq(variables['System.TeamProject'], 'public') }}: + - name: defaultPoolName + value: NetCore-Public-XL + - name: defaultPoolDemandsLinux + value: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open + - name: defaultPoolDemandsWindows + value: ImageOverride -equals windows.vs2022.amd64.open +- ${{ if eq(variables['System.TeamProject'], 'internal') }}: + - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: + - name: defaultPoolName + value: NetCore1ESPool-Internal-XL + - ${{ else }}: + - name: defaultPoolName + value: NetCore1ESPool-Svc-Internal + - name: defaultPoolDemandsLinux + value: ImageOverride -equals Build.Ubuntu.1804.Amd64 + - name: defaultPoolDemandsWindows + value: ImageOverride -equals windows.vs2022.amd64 + +- name: defaultPoolNameLinuxArm64 + value: Docker-Linux-Arm-Internal + +- name: defaultPoolNameMac + value: macos-12 diff --git a/eng/pipelines/vmr-build-internal.yml b/eng/pipelines/vmr-build-internal.yml deleted file mode 100644 index d4bd11578..000000000 --- a/eng/pipelines/vmr-build-internal.yml +++ /dev/null @@ -1,53 +0,0 @@ -trigger: none -pr: - branches: - include: - - main - - release/* - - internal/release/* - -parameters: -- name: vmrBranch - displayName: dotnet/dotnet branch to use - type: string - default: ' ' - -- name: disableVmrBuild - displayName: Skip source-building the VMR - type: boolean - default: false - -variables: -- ${{ if ne(parameters.vmrBranch, ' ') }}: - - name: VmrBranch - value: ${{ replace(parameters.vmrBranch, ' ', '') }} -- ${{ else }}: - - name: VmrBranch - value: ${{ replace(replace(variables['System.PullRequest.TargetBranch'], 'refs/heads/', ''), 'refs/pull/', '') }} - -resources: - repositories: - - repository: vmr - type: git - name: dotnet-dotnet - ref: ${{ variables.VmrBranch }} - -stages: -# You can temporarily disable the VMR Build stage by changing the default of disableVmrBuild -- ${{ if not(parameters.disableVmrBuild) }}: - - template: templates/stages/vmr-build.yml - parameters: - vmrBranch: ${{ variables.VmrBranch }} - isBuiltFromVmr: false - -# In case the VMR Build stage is temporarily disabled, the VMR synchronization step is run to validate -# that the PR can be merged and later synchronized into the VMR without problems. -- ${{ else }}: - - stage: Synchronize_VMR - displayName: Synchronize VMR - dependsOn: [] - jobs: - - template: templates/jobs/vmr-synchronization.yml - parameters: - vmrBranch: ${{ variables.VmrBranch }} - noPush: true diff --git a/eng/pipelines/vmr-build.yml b/eng/pipelines/vmr-build.yml index e6a827a51..15cc40fc4 100644 --- a/eng/pipelines/vmr-build.yml +++ b/eng/pipelines/vmr-build.yml @@ -1,3 +1,10 @@ +# This yml is used by these pipelines and triggers: +# +# - installer-source-build (public) +# - PR: release/* and main, ultralite build +# - installer-unified-build (public) +# - PR: release/* and main, lite build + trigger: none pr: branches: @@ -15,8 +22,8 @@ parameters: type: string default: ' ' -- name: disableVmrBuild - displayName: Skip source-building the VMR +- name: disableBuild + displayName: Skip the VMR Build stage type: boolean default: false @@ -28,6 +35,10 @@ variables: - name: VmrBranch value: ${{ replace(replace(variables['System.PullRequest.TargetBranch'], 'refs/heads/', ''), 'refs/pull/', '') }} +# enable source-only build for pipelines with the -source-build suffix +- name: isSourceOnlyBuild + value: ${{ contains(variables['Build.DefinitionName'], '-source-build') }} + resources: repositories: - repository: vmr @@ -37,12 +48,17 @@ resources: ref: ${{ variables.VmrBranch }} stages: -# You can temporarily disable the VMR Build stage by setting the disableVmrBuild variable -- ${{ if not(parameters.disableVmrBuild) }}: +# You can temporarily disable the VMR Build stage by setting the disableBuild variable +- ${{ if not(parameters.disableBuild) }}: - template: templates/stages/vmr-build.yml parameters: vmrBranch: ${{ variables.VmrBranch }} isBuiltFromVmr: false + isSourceOnlyBuild: ${{ variables.isSourceOnlyBuild }} + ${{ if eq(variables.isSourceOnlyBuild, 'true') }}: + scope: ultralite + ${{ else }}: + scope: lite # In case the VMR Build stage is temporarily disabled, the VMR synchronization step is run to validate # that the PR can be merged and later synchronized into the VMR without problems. diff --git a/eng/pipelines/vmr-vertical-build-poc.yml b/eng/pipelines/vmr-vertical-build-poc.yml deleted file mode 100644 index b9ef33cca..000000000 --- a/eng/pipelines/vmr-vertical-build-poc.yml +++ /dev/null @@ -1,52 +0,0 @@ -trigger: none -pr: - branches: - include: - - main - -parameters: -- name: vmrBranch - displayName: dotnet/dotnet branch to push to - type: string - default: ' ' - -- name: disableVmrBuild - displayName: Skip source-building the VMR - type: boolean - default: false - -variables: -- ${{ if ne(parameters.vmrBranch, ' ') }}: - - name: VmrBranch - value: ${{ replace(parameters.vmrBranch, ' ', '') }} -- ${{ else }}: - - name: VmrBranch - value: ${{ replace(replace(variables['System.PullRequest.TargetBranch'], 'refs/heads/', ''), 'refs/pull/', '') }} - -resources: - repositories: - - repository: vmr - type: github - name: dotnet/dotnet - endpoint: dotnet - ref: ${{ variables.VmrBranch }} - -stages: -- ${{ if not(parameters.disableVmrBuild) }}: - - template: templates/stages/vmr-cross-build.yml - parameters: - vmrBranch: ${{ variables.VmrBranch }} - isBuiltFromVmr: false - isLiteBuild: true - -# In case the VMR Build stage is temporarily disabled, the VMR synchronization step is run to validate -# that the PR can be merged and later synchronized into the VMR without problems. -- ${{ else }}: - - stage: Synchronize_VMR - displayName: Synchronize VMR - dependsOn: [] - jobs: - - template: templates/jobs/vmr-synchronization.yml - parameters: - vmrBranch: ${{ variables.VmrBranch }} - noPush: true diff --git a/eng/vmr-sync.ps1 b/eng/vmr-sync.ps1 new file mode 100644 index 000000000..83987fc30 --- /dev/null +++ b/eng/vmr-sync.ps1 @@ -0,0 +1,212 @@ +<# +.SYNOPSIS + +This script is used for synchronizing the dotnet/dotnet VMR locally. This means pulling new +code from various repositories into the 'dotnet/dotnet' repository. + +.DESCRIPTION + +The script is used during CI to ingest new code based on dotnet/installer but it can also help +for reproducing potential failures during installer's PRs, namely to fix the Source-Build. +Another usecase is to try manually synchronizing a given commit of some repo into the VMR and +trying to Source-Build the VMR. This can help when fixing the Source-Build but using a commit +from a not-yet merged branch (or fork) to test the fix will help. + +The tooling that synchronizes the VMR will need to clone the various repositories into a temporary +folder. These clones can be re-used in future synchronizations so it is advised you dedicate a +folder to this to speed up your re-runs. + +.EXAMPLE + Synchronize current installer and all dependencies into a local VMR: + ./vmr-sync.ps1 -vmrDir "$HOME/repos/dotnet" -tmpDir "$HOME/repos/tmp" + + Synchronize the VMR to a specific commit of dotnet/runtime using custom fork: + ./vmr-sync.ps1 ` + -repository runtime:e7e71da303af8dc97df99b098f21f526398c3943 ` + -remote runtime:https://github.com/yourfork/runtime ` + -tmpDir "$HOME/repos/tmp" + +.PARAMETER tmpDir +Required. Path to the temporary folder where repositories will be cloned + +.PARAMETER vmrBranch +Optional. Branch of the 'dotnet/dotnet' repo to synchronize. The VMR will be checked out to this branch + +.PARAMETER component-template +Optional. Template for VMRs Component.md used for regenerating the file to list the newest versions of +components. +Defaults to src/VirtualMonoRepo/Component.template.md + +.PARAMETER recursive +Optional. Recursively synchronize all the source build dependencies (declared in Version.Details.xml) +This is used when performing the full synchronization during installer's CI and the final VMR sync. +Defaults to false unless no repository is supplied in which case a recursive sync of installer is performed. + +.PARAMETER remote +Optional. Additional remote to use during the synchronization +This can be used to synchronize to a commit from a fork of the repository +Example: 'runtime:https://github.com/yourfork/runtime' + +.PARAMETER repository +Optional. Repository + git ref separated by colon to synchronize to. +This can be a specific commit, branch, tag. +If not supplied, the revision of the parent installer repository of this script will be used (recursively). +Example: 'runtime:my-branch-name' + +.PARAMETER tpnTemplate +Optional. Template for the header of VMRs THIRD-PARTY-NOTICES file. +Defaults to src/VirtualMonoRepo/THIRD-PARTY-NOTICES.template.txt + +.PARAMETER azdevPat +Optional. Azure DevOps PAT to use for cloning private repositories. + +.PARAMETER vmrDir +Optional. Path to the dotnet/dotnet repository. When null, gets cloned to the temporary folder +#> +param ( + [Parameter(Mandatory=$true, HelpMessage="Path to the temporary folder where repositories will be cloned")] + [string][Alias('t', 'tmp')]$tmpDir, + [string][Alias('b', 'branch')]$vmrBranch, + [string]$componentTemplate = "src/VirtualMonoRepo/Component.template.md", + [switch]$recursive, + [string]$remote, + [string][Alias('r')]$repository, + [string]$tpnTemplate = "src/VirtualMonoRepo/THIRD-PARTY-NOTICES.template.txt", + [string]$azdevPat, + [string][Alias('v', 'vmr')]$vmrDir +) + +$scriptRoot = $PSScriptRoot + +function Fail { + Write-Host "> $($args[0])" -ForegroundColor 'Red' +} + +function Highlight { + Write-Host "> $($args[0])" -ForegroundColor 'Cyan' +} + +$installerDir = (Split-Path -Parent $scriptRoot) + +# If installer is a repo, we're in an installer and not in the dotnet/dotnet repo +if (Test-Path -Path "$installerDir/.git" -PathType Container) { + $additionalRemotes = "installer:$installerDir" +} + +if ($remote) { + $additionalRemotes = "$additionalRemotes $remote" +} + +$verbosity = 'verbose' +if ($debug) { + $verbosity = 'debug' +} +# Validation + +if (-not (Test-Path -Path $installerDir -PathType Container)) { + Fail "Directory '$installerDir' does not exist. Please specify the path to the dotnet/installer repo" + exit 1 +} + +if (-not $tmpDir) { + Fail "Missing -tmpDir argument. Please specify the path to the temporary folder where the repositories will be cloned" + exit 1 +} + +if (-not (Test-Path -Path $componentTemplate -PathType Leaf)) { + Fail "File '$componentTemplate' does not exist. Please specify a valid path to the component template" + exit 1 +} + +if (-not (Test-Path -Path $tpnTemplate -PathType Leaf)) { + Fail "File '$tpnTemplate' does not exist. Please specify a valid path to the THIRD-PARTY-NOTICES template" + exit 1 +} + +# Sanitize the input + +# Default when no repository is provided +if (-not $repository) { + $repository = "installer:$(git -C $installerDir rev-parse HEAD)" + $recursive = $true +} + +if (-not $vmrDir) { + $vmrDir = Join-Path $tmpDir 'dotnet' +} + +if (-not (Test-Path -Path $tmpDir -PathType Container)) { + New-Item -ItemType Directory -Path $tmpDir | Out-Null +} + +# Prepare the VMR + +if (-not (Test-Path -Path $vmrDir -PathType Container)) { + Highlight "Cloning 'dotnet/dotnet' into $vmrDir.." + git clone https://github.com/dotnet/dotnet $vmrDir + + if ($vmrBranch) { + git -C $vmrDir switch -c $vmrBranch + } +} +else { + if ((git -C $vmrDir diff --quiet) -eq $false) { + Fail "There are changes in the working tree of $vmrDir. Please commit or stash your changes" + exit 1 + } + + if ($vmrBranch) { + Highlight "Preparing $vmrDir" + git -C $vmrDir checkout $vmrBranch + git -C $vmrDir pull + } +} + +Set-StrictMode -Version Latest + +# Prepare darc + +Highlight 'Installing .NET, preparing the tooling..' +. $scriptRoot\common\tools.ps1 +$dotnetRoot = InitializeDotNetCli -install:$true +$dotnet = "$dotnetRoot\dotnet.exe" +& "$dotnet" tool restore + +Highlight "Starting the synchronization of '$repository'.." + +# Synchronize the VMR +$darcArgs = ( + "darc", "vmr", "update", + "--vmr", $vmrDir, + "--tmp", $tmpDir, + "--$verbosity", + "--component-template", $componentTemplate, + "--tpn-template", $tpnTemplate, + "--discard-patches", + $repository +) + +if ($recursive) { + $darcArgs += ("--recursive") +} + +if ($additionalRemotes) { + $darcArgs += ("--additional-remotes", $additionalRemotes) +} + +if ($azdevPat) { + $darcArgs += ("--azdev-pat", $azdevPat) +} + +& "$dotnet" $darcArgs + +if ($LASTEXITCODE -eq 0) { + Highlight "Synchronization succeeded" +} +else { + Fail "Synchronization of dotnet/dotnet to '$repository' failed!" + Fail "'$vmrDir' is left in its last state (re-run of this script will reset it)." + Fail "Please inspect the logs which contain path to the failing patch file (use --debug to get all the details)." + Fail "Once you make changes to the conflicting VMR patch, commit it locally and re-run this script." + exit 1 +} diff --git a/eng/vmr-sync.sh b/eng/vmr-sync.sh index bd453f87c..97c8a1a87 100755 --- a/eng/vmr-sync.sh +++ b/eng/vmr-sync.sh @@ -104,7 +104,7 @@ repository='' additional_remotes='' recursive=false verbosity=verbose -readme_template="$installer_dir/src/VirtualMonoRepo/Component.template.md" +component_template="$installer_dir/src/VirtualMonoRepo/Component.template.md" tpn_template="$installer_dir/src/VirtualMonoRepo/THIRD-PARTY-NOTICES.template.txt" azdev_pat='' @@ -180,8 +180,8 @@ if [[ -z "$tmp_dir" ]]; then exit 1 fi -if [[ ! -f "$readme_template" ]]; then - fail "File '$readme_template' does not exist. Please specify a valid path to the README template" +if [[ ! -f "$component_template" ]]; then + fail "File '$component_template' does not exist. Please specify a valid path to the Component.md template" exit 1 fi diff --git a/src/SourceBuild/content/build.sh b/src/SourceBuild/content/build.sh index becd4547d..ab9c9170d 100755 --- a/src/SourceBuild/content/build.sh +++ b/src/SourceBuild/content/build.sh @@ -185,6 +185,12 @@ while [[ $# > 0 ]]; do shift done +if [[ "$ci" == true ]]; then + if [[ "$exclude_ci_binary_log" == false ]]; then + binary_log=true + fi +fi + . "$scriptroot/eng/common/tools.sh" function Build { diff --git a/src/SourceBuild/content/eng/build.ps1 b/src/SourceBuild/content/eng/build.ps1 index d920e65d9..35a494aa5 100644 --- a/src/SourceBuild/content/eng/build.ps1 +++ b/src/SourceBuild/content/eng/build.ps1 @@ -48,21 +48,17 @@ if ($help) { exit 0 } -$arguments="" -if ($cleanWhileBuilding) { - $arguments += " /p:CleanWhileBuilding=true" -} - function Build { InitializeToolset $bl = if ($binaryLog) { '/bl:' + (Join-Path $LogDir 'Build.binlog') } else { '' } + $cwb = if ($cleanWhileBuilding) { '/p:CleanWhileBuilding=true' } else { '' } $buildProj = Join-Path $RepoRoot 'build.proj' MSBuild $buildProj ` $bl ` /p:Configuration=$configuration ` - $arguments ` + $cwb ` @properties } @@ -75,6 +71,11 @@ try { exit 0 } + if ($ci) { + if (-not $excludeCIBinarylog) { + $binaryLog = $true + } + } Build } diff --git a/src/SourceBuild/content/eng/pipelines/ci-crossbuild.yml b/src/SourceBuild/content/eng/pipelines/ci-crossbuild.yml deleted file mode 100644 index 2fd1764ec..000000000 --- a/src/SourceBuild/content/eng/pipelines/ci-crossbuild.yml +++ /dev/null @@ -1,27 +0,0 @@ -# This is the dotnet/dotnet pipeline that is triggered every weekday at midnight PST (08:00 UTC) for "main" and when changes are pushed to release/* and internal/release/* branches. - -schedules: -- cron: '0 8 * * Mon-Fri' - displayName: Weekday midnight build - branches: - include: - - main - batch: true - -trigger: - batch: true - branches: - include: - - release/* - - internal/release/* - -pr: none - -stages: -- ${{ if ne(variables['Build.Reason'], 'Schedule') }}: - - template: templates/stages/vmr-scan.yml - -- template: /src/installer/eng/pipelines/templates/stages/vmr-cross-build.yml - parameters: - isBuiltFromVmr: true - isLiteBuild: false diff --git a/src/SourceBuild/content/eng/pipelines/ci-lite.yml b/src/SourceBuild/content/eng/pipelines/ci-lite.yml index 6276ec4a2..0f322997f 100644 --- a/src/SourceBuild/content/eng/pipelines/ci-lite.yml +++ b/src/SourceBuild/content/eng/pipelines/ci-lite.yml @@ -1,4 +1,8 @@ -# This is the dotnet/dotnet-lite pipeline that is triggered by pushes to main and PRs targetting main and release/*. +# This yml is used by these pipelines and triggers: +# +# - dotnet-source-build-lite (internal) +# - PR: release/* and main, lite build +# - CI: main only, every batched commit, lite build trigger: batch: true @@ -13,10 +17,11 @@ pr: - release/* stages: -- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: +- ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: - template: templates/stages/vmr-scan.yml - template: /src/installer/eng/pipelines/templates/stages/vmr-build.yml parameters: isBuiltFromVmr: true - isLiteBuild: true + isSourceOnlyBuild: true + scope: lite diff --git a/src/SourceBuild/content/eng/pipelines/ci.yml b/src/SourceBuild/content/eng/pipelines/ci.yml index 14cf59bed..11cb3b272 100644 --- a/src/SourceBuild/content/eng/pipelines/ci.yml +++ b/src/SourceBuild/content/eng/pipelines/ci.yml @@ -1,21 +1,42 @@ -# This is the dotnet/dotnet pipeline that is triggered every weekday at midnight PST (08:00 UTC) for "main" and when changes are pushed to release/* and internal/release/* branches. +# This yml is used by these pipelines and triggers: +# NOTE: the triggers are defined in the Azure DevOps UI as they are too complex +# +# - dotnet-source-build (public) +# - PR: ultralite build +# - CI: release/* only, every batched commit, full build +# - Schedule: main only, full build +# +# - dotnet-unified-build (public) +# - PR: lite build +# - CI: release/* only, every batched commit, full build +# - Schedule: main only, full build +# +# - dotnet-source-build (internal) +# - PR: ultralite build +# - CI: release/* and internal/release/* only, every batched commit, full build +# - CI: main only, every batched commit, lite build +# - Schedule: main only, full build +# +# - dotnet-unified-build (internal) +# - PR: lite build +# - CI: release/*, internal/release/* and main, every batched commit, full build -schedules: -- cron: '0 8 * * Mon-Fri' - displayName: Weekday midnight build - branches: - include: - - main - batch: true +variables: +# enable source-only build for pipelines that contain the -source-build string +- name: isSourceOnlyBuild + value: ${{ contains(variables['Build.DefinitionName'], '-source-build') }} -trigger: - batch: true - branches: - include: - - release/* - - internal/release/* +- name: isInternalBuild + value: ${{ eq(variables['System.TeamProject'], 'internal') }} -pr: none +- name: isMainBranch + value: ${{ eq(variables['Build.SourceBranch'], 'refs/heads/main') }} + +- name: isScheduleTrigger + value: ${{ eq(variables['Build.Reason'], 'Schedule') }} + +- name: isPRTrigger + value: ${{ eq(variables['Build.Reason'], 'PullRequest') }} stages: - template: templates/stages/vmr-scan.yml @@ -23,4 +44,14 @@ stages: - template: /src/installer/eng/pipelines/templates/stages/vmr-build.yml parameters: isBuiltFromVmr: true - isLiteBuild: false + isSourceOnlyBuild: ${{ variables.isSourceOnlyBuild }} + ${{ if eq(variables.isScheduleTrigger, 'true') }}: + scope: full + ${{ elseif and(eq(variables.isPRTrigger, 'true'), eq(variables.isSourceOnlyBuild, 'true')) }}: + scope: ultralite + ${{ elseif and(eq(variables.isPRTrigger, 'true'), not(eq(variables.isSourceOnlyBuild, 'true'))) }}: + scope: lite + ${{ elseif and(not(eq(variables.isPRTrigger, 'true')), eq(variables.isSourceOnlyBuild, 'true'), eq(variables.isInternalBuild, 'true'), eq(variables.isMainBranch, 'true')) }}: + scope: lite + ${{ else }}: + scope: full