From 2e08b0bc5049ca64cae671a1ed79c3eddf758c44 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Tue, 2 Apr 2024 15:02:56 -0700 Subject: [PATCH] [7.0.3xx] Migrate to 1ES templates for internal builds (#19200) Co-authored-by: dotnet-maestro[bot] --- .vsts-ci.yml | 690 +++++++++--------- .vsts-pr.yml | 35 +- eng/build-pr.yml | 156 ++++ eng/build.yml | 14 +- eng/source-build-tarball-build-official.yml | 26 +- eng/source-build-tarball-build.yml | 14 - .../job/source-build-build-tarball-pr.yml | 252 +++++++ .../job/source-build-build-tarball.yml | 14 +- .../job/source-build-create-tarball-pr.yml | 85 +++ .../job/source-build-create-tarball.yml | 20 +- .../job/source-build-run-tarball-build-pr.yml | 147 ++++ .../job/source-build-run-tarball-build.yml | 14 +- 12 files changed, 1057 insertions(+), 410 deletions(-) create mode 100644 eng/build-pr.yml delete mode 100644 eng/source-build-tarball-build.yml create mode 100644 src/SourceBuild/Arcade/eng/common/templates/job/source-build-build-tarball-pr.yml create mode 100644 src/SourceBuild/Arcade/eng/common/templates/job/source-build-create-tarball-pr.yml create mode 100644 src/SourceBuild/Arcade/eng/common/templates/job/source-build-run-tarball-build-pr.yml diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 5e2cdc745..d53d48105 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -1,3 +1,5 @@ +# Pipeline: https://dnceng.visualstudio.com/internal/_build?definitionId=286 + trigger: batch: true branches: @@ -26,7 +28,6 @@ variables: - group: DotNet-Installer-SDLValidation-Params - name: _PublishUsingPipelines value: true - # Default to running tests in PRs and public CI, but not in official builds - name: _WindowsTestArg value: '-test' @@ -37,358 +38,375 @@ variables: value: '' - name: _NonWindowsTestArg value: '' - - name: _InternalRuntimeDownloadArgs value: '' - - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - group: DotNetBuilds storage account read tokens - name: _InternalRuntimeDownloadArgs value: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) /p:dotnetbuilds-internal-container-read-token-base64=$(dotnetbuilds-internal-container-read-token-base64) +- name: DncEngPublicBuildPool + value: NetCore-Svc-Public +- name: DncEngInternalBuildPool + value: NetCore1ESPool-Svc-Internal +# Set the MicroBuild plugin installation directory to the agent temp directory to avoid SDL tool scanning. +- name: MicroBuildOutputFolderOverride + value: $(Agent.TempDirectory) -stages: -- stage: build - jobs: - - job: Publish_Build_Configuration - pool: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: NetCore-Svc-Public - demands: ImageOverride -equals windows.vs2019.amd64.open - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - name: NetCore1ESPool-Svc-Internal - demands: ImageOverride -equals windows.vs2019.amd64 - steps: - - publish: $(Build.SourcesDirectory)\eng\BuildConfiguration - artifact: BuildConfiguration - displayName: Publish Build Config - - template: /eng/build.yml - parameters: - agentOs: Windows_NT - pool: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: NetCore-Svc-Public - demands: ImageOverride -equals windows.vs2019.amd64.open - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - name: NetCore1ESPool-Svc-Internal - demands: ImageOverride -equals windows.vs2019.amd64 - timeoutInMinutes: 180 - strategy: - matrix: - # Public-only builds - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: - Build_Release_x64: - _BuildConfig: Debug - _BuildArchitecture: x64 - _DOTNET_CLI_UI_LANGUAGE: '' - _AdditionalBuildParameters: '/p:PublishInternalAsset=true' - _TestArg: $(_WindowsTestArg) - # Internal-only builds - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - Build_Release_x64: - _BuildConfig: Release - _BuildArchitecture: x64 - _DOTNET_CLI_UI_LANGUAGE: '' - _AdditionalBuildParameters: '/p:PublishInternalAsset=true' - _TestArg: $(_WindowsTestArg) - Build_Release_x86: - _BuildConfig: Release - _BuildArchitecture: x86 - _DOTNET_CLI_UI_LANGUAGE: '' - _AdditionalBuildParameters: '' - _TestArg: $(_WindowsTestArg) - Build_Release_arm64: - _BuildConfig: Release - _BuildArchitecture: arm64 - _DOTNET_CLI_UI_LANGUAGE: '' - _AdditionalBuildParameters: '' - # Never run tests on arm64 - _TestArg: '' +resources: + repositories: + - repository: 1esPipelines + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - template: /eng/build.yml - parameters: - agentOs: Windows_NT +extends: + ${{ if notin(variables['Build.Reason'], 'PullRequest') }}: + template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines + ${{ else }}: + template: v1/1ES.Unofficial.PipelineTemplate.yml@1esPipelines + parameters: + containers: + fedora38: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-38 + sdl: + sourceAnalysisPool: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2019 + os: windows + stages: + - stage: build + jobs: + # Build Retry Configuration + - job: Publish_Build_Configuration pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: $(DncEngPublicBuildPool) + image: 1es-windows-2019-open + os: windows ${{ if eq(variables['System.TeamProject'], 'internal') }}: - name: NetCore1ESPool-Svc-Internal - demands: ImageOverride -equals windows.vs2019.amd64 - timeoutInMinutes: 180 - strategy: - matrix: - Build_Release_x64: - _BuildConfig: Release - _BuildArchitecture: x64 - _DOTNET_CLI_UI_LANGUAGE: '' - _AdditionalBuildParameters: '/p:PublishInternalAsset=true' - # Never run tests on PGO bits - _TestArg: '' - Build_Release_x86: - _BuildConfig: Release - _BuildArchitecture: x86 - _DOTNET_CLI_UI_LANGUAGE: '' - _AdditionalBuildParameters: '' - _TestArg: '' - Build_Release_arm64: - _BuildConfig: Release - _BuildArchitecture: arm64 - _DOTNET_CLI_UI_LANGUAGE: '' - _AdditionalBuildParameters: '' - # Never run tests on arm64 - _TestArg: '' - pgoInstrument: true + name: $(DncEngInternalBuildPool) + image: 1es-windows-2019 + os: windows + steps: + - task: 1ES.PublishPipelineArtifact@1 + displayName: Publish Build Config + inputs: + targetPath: $(Build.SourcesDirectory)\eng\BuildConfiguration + artifactName: BuildConfiguration + - template: /eng/build.yml@self + parameters: + agentOs: Windows_NT + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: $(DncEngPublicBuildPool) + image: 1es-windows-2019-open + os: windows + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2019 + os: windows + timeoutInMinutes: 180 + strategy: + matrix: + # Public-only builds + ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: + Build_Release_x64: + _BuildConfig: Debug + _BuildArchitecture: x64 + _DOTNET_CLI_UI_LANGUAGE: '' + _AdditionalBuildParameters: '/p:PublishInternalAsset=true' + _TestArg: $(_WindowsTestArg) + # Internal-only builds + ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + Build_Release_x64: + _BuildConfig: Release + _BuildArchitecture: x64 + _DOTNET_CLI_UI_LANGUAGE: '' + _AdditionalBuildParameters: '/p:PublishInternalAsset=true' + _TestArg: $(_WindowsTestArg) + Build_Release_x86: + _BuildConfig: Release + _BuildArchitecture: x86 + _DOTNET_CLI_UI_LANGUAGE: '' + _AdditionalBuildParameters: '' + _TestArg: $(_WindowsTestArg) + Build_Release_arm64: + _BuildConfig: Release + _BuildArchitecture: arm64 + _DOTNET_CLI_UI_LANGUAGE: '' + _AdditionalBuildParameters: '' + # Never run tests on arm64 + _TestArg: '' - - template: /eng/build.yml - parameters: - agentOs: Linux - pool: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: NetCore-Svc-Public - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - name: NetCore1ESPool-Svc-Internal - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 - timeoutInMinutes: 180 - strategy: - matrix: - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: - Build_Ubuntu_18_04_Debug_x64: - _BuildConfig: Debug - _DockerParameter: '--docker ubuntu.18.04' - _LinuxPortable: '--linux-portable' - _RuntimeIdentifier: '' - _BuildArchitecture: 'x64' - _TestArg: $(_NonWindowsTestArg) - Build_Fedora_36_Debug_x64: - _BuildConfig: Debug - _DockerParameter: '--docker fedora.36' - _LinuxPortable: '--linux-portable' - _RuntimeIdentifier: '' - _BuildArchitecture: 'x64' - _TestArg: $(_NonWindowsTestArg) - Build_CentOS_7_Debug_x64: - _BuildConfig: Debug - _DockerParameter: '--docker centos' - _LinuxPortable: '' - _RuntimeIdentifier: '' - _BuildArchitecture: 'x64' - _TestArg: $(_NonWindowsTestArg) - Build_Debian_11_Debug_x64: - _BuildConfig: Debug - _DockerParameter: '--docker debian' - _LinuxPortable: '' - _RuntimeIdentifier: '' - _BuildArchitecture: 'x64' - _AdditionalBuildParameters: '/p:BuildSdkDeb=true' - _TestArg: $(_NonWindowsTestArg) - Build_Arm64_Debug: - _BuildConfig: Debug - _DockerParameter: '' - _LinuxPortable: '--linux-portable' - _RuntimeIdentifier: '--runtime-id linux-arm64' - _BuildArchitecture: 'arm64' - # Never run tests on arm64 - _TestArg: '' - Build_Linux_musl_Debug_x64: - _BuildConfig: Debug - _DockerParameter: '--docker alpine.3.15' - _LinuxPortable: '' - _RuntimeIdentifier: '--runtime-id linux-musl-x64' - _BuildArchitecture: 'x64' - # Pass in HostOSName when running on alpine - _AdditionalBuildParameters: '/p:HostOSName="linux-musl"' - _TestArg: $(_NonWindowsTestArg) - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - Build_Arm_Release: - _BuildConfig: Release - _DockerParameter: '' - _LinuxPortable: '--linux-portable' - _RuntimeIdentifier: '--runtime-id linux-arm' - _BuildArchitecture: 'arm' - # Never run tests on arm - _TestArg: '' - Build_Arm64_Release: - _BuildConfig: Release - _DockerParameter: '' - _LinuxPortable: '--linux-portable' - _RuntimeIdentifier: '--runtime-id linux-arm64' - _BuildArchitecture: 'arm64' - # Never run tests on arm64 - _TestArg: '' - Build_Linux_musl_Release_arm: - _BuildConfig: Release - # linux-musl-arm cross gen depends on glibc 2.27 (this OS has it) - _DockerParameter: '--docker ubuntu.18.04' - _LinuxPortable: '' - _RuntimeIdentifier: '--runtime-id linux-musl-arm' - _BuildArchitecture: 'arm' - _AdditionalBuildParameters: '/p:OSName="linux-musl"' - # Never run tests on arm - _TestArg: '' - Build_Linux_musl_Release_arm64: - _BuildConfig: Release - _DockerParameter: '' - _LinuxPortable: '' - _RuntimeIdentifier: '--runtime-id linux-musl-arm64' - _BuildArchitecture: 'arm64' - _AdditionalBuildParameters: '/p:OSName="linux-musl"' - # Never run tests on arm64 - _TestArg: '' - Build_Linux_musl_Release_x64: - _BuildConfig: Release - _DockerParameter: '--docker alpine.3.15' - _LinuxPortable: '' - _RuntimeIdentifier: '--runtime-id linux-musl-x64' - _BuildArchitecture: 'x64' - # Pass in HostOSName when running on alpine - _AdditionalBuildParameters: '/p:HostOSName="linux-musl"' - Build_Linux_Portable_Deb_Release_x64: - _BuildConfig: Release - _DockerParameter: '--docker ubuntu.18.04' - _LinuxPortable: '--linux-portable' - _RuntimeIdentifier: '' - _BuildArchitecture: 'x64' - # Do not publish zips and tarballs. The linux-x64 binaries are - # already published by Build_LinuxPortable_Release_x64 - _AdditionalBuildParameters: '/p:PublishBinariesAndBadge=false /p:BuildSdkDeb=true' - _TestArg: $(_NonWindowsTestArg) - Build_Linux_Portable_Rpm_Release_x64: - _BuildConfig: Release - _DockerParameter: '--docker centos' - _LinuxPortable: '--linux-portable' - _RuntimeIdentifier: '' - _BuildArchitecture: 'x64' - # Do not publish zips and tarballs. The linux-x64 binaries are - # already published by Build_LinuxPortable_Release_x64 - _AdditionalBuildParameters: '/p:PublishBinariesAndBadge=false' - _TestArg: $(_NonWindowsTestArg) - Build_Linux_Portable_Rpm_Release_Arm64: - _BuildConfig: Release - _DockerParameter: '--docker centos' - _LinuxPortable: '--linux-portable' - _RuntimeIdentifier: '--runtime-id linux-arm64' - _BuildArchitecture: 'arm64' - # Do not publish zips and tarballs. The linux-x64 binaries are - # already published by Build_LinuxPortable_Release_x64 - _AdditionalBuildParameters: '/p:PublishBinariesAndBadge=false /p:CLIBUILD_SKIP_TESTS=true' - # Never run tests on arm64 - _TestArg: '' - Build_LinuxPortable_Release_x64: - _BuildConfig: Release - _DockerParameter: '' - _LinuxPortable: '--linux-portable' - _RuntimeIdentifier: '' - _BuildArchitecture: 'x64' - _TestArg: $(_NonWindowsTestArg) + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - template: /eng/build.yml@self + parameters: + agentOs: Windows_NT + pool: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2019 + os: windows + timeoutInMinutes: 180 + strategy: + matrix: + Build_Release_x64: + _BuildConfig: Release + _BuildArchitecture: x64 + _DOTNET_CLI_UI_LANGUAGE: '' + _AdditionalBuildParameters: '/p:PublishInternalAsset=true' + # Never run tests on PGO bits + _TestArg: '' + Build_Release_x86: + _BuildConfig: Release + _BuildArchitecture: x86 + _DOTNET_CLI_UI_LANGUAGE: '' + _AdditionalBuildParameters: '' + _TestArg: '' + Build_Release_arm64: + _BuildConfig: Release + _BuildArchitecture: arm64 + _DOTNET_CLI_UI_LANGUAGE: '' + _AdditionalBuildParameters: '' + # Never run tests on arm64 + _TestArg: '' + pgoInstrument: true - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - template: /eng/build.yml - parameters: - agentOs: Linux - pool: - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - name: NetCore1ESPool-Svc-Internal - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 - timeoutInMinutes: 180 - strategy: - matrix: - Build_LinuxPortable_Release_x64: - _BuildConfig: Release - _DockerParameter: '' - _LinuxPortable: '--linux-portable' - _RuntimeIdentifier: '' - _BuildArchitecture: 'x64' - _TestArg: '' - Build_Release_arm64: - _BuildConfig: Release - _BuildArchitecture: arm64 - _DOTNET_CLI_UI_LANGUAGE: '' - _AdditionalBuildParameters: '' - # Never run tests on arm64 - _TestArg: '' - pgoInstrument: true + - template: /eng/build.yml@self + parameters: + agentOs: Linux + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: $(DncEngPublicBuildPool) + image: 1es-ubuntu-2004-open + os: linux + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: $(DncEngInternalBuildPool) + image: 1es-ubuntu-2004 + os: linux + timeoutInMinutes: 180 + strategy: + matrix: + ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: + Build_Ubuntu_18_04_Debug_x64: + _BuildConfig: Debug + _DockerParameter: '--docker ubuntu.18.04' + _LinuxPortable: '--linux-portable' + _RuntimeIdentifier: '' + _BuildArchitecture: 'x64' + _TestArg: $(_NonWindowsTestArg) + Build_Fedora_36_Debug_x64: + _BuildConfig: Debug + _DockerParameter: '--docker fedora.36' + _LinuxPortable: '--linux-portable' + _RuntimeIdentifier: '' + _BuildArchitecture: 'x64' + _TestArg: $(_NonWindowsTestArg) + Build_CentOS_7_Debug_x64: + _BuildConfig: Debug + _DockerParameter: '--docker centos' + _LinuxPortable: '' + _RuntimeIdentifier: '' + _BuildArchitecture: 'x64' + _TestArg: $(_NonWindowsTestArg) + Build_Debian_11_Debug_x64: + _BuildConfig: Debug + _DockerParameter: '--docker debian' + _LinuxPortable: '' + _RuntimeIdentifier: '' + _BuildArchitecture: 'x64' + _AdditionalBuildParameters: '/p:BuildSdkDeb=true' + _TestArg: $(_NonWindowsTestArg) + Build_Arm64_Debug: + _BuildConfig: Debug + _DockerParameter: '' + _LinuxPortable: '--linux-portable' + _RuntimeIdentifier: '--runtime-id linux-arm64' + _BuildArchitecture: 'arm64' + # Never run tests on arm64 + _TestArg: '' + Build_Linux_musl_Debug_x64: + _BuildConfig: Debug + _DockerParameter: '--docker alpine.3.15' + _LinuxPortable: '' + _RuntimeIdentifier: '--runtime-id linux-musl-x64' + _BuildArchitecture: 'x64' + # Pass in HostOSName when running on alpine + _AdditionalBuildParameters: '/p:HostOSName="linux-musl"' + _TestArg: $(_NonWindowsTestArg) + ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + Build_Arm_Release: + _BuildConfig: Release + _DockerParameter: '' + _LinuxPortable: '--linux-portable' + _RuntimeIdentifier: '--runtime-id linux-arm' + _BuildArchitecture: 'arm' + # Never run tests on arm + _TestArg: '' + Build_Arm64_Release: + _BuildConfig: Release + _DockerParameter: '' + _LinuxPortable: '--linux-portable' + _RuntimeIdentifier: '--runtime-id linux-arm64' + _BuildArchitecture: 'arm64' + # Never run tests on arm64 + _TestArg: '' + Build_Linux_musl_Release_arm: + _BuildConfig: Release + # linux-musl-arm cross gen depends on glibc 2.27 (this OS has it) + _DockerParameter: '--docker ubuntu.18.04' + _LinuxPortable: '' + _RuntimeIdentifier: '--runtime-id linux-musl-arm' + _BuildArchitecture: 'arm' + _AdditionalBuildParameters: '/p:OSName="linux-musl"' + # Never run tests on arm + _TestArg: '' + Build_Linux_musl_Release_arm64: + _BuildConfig: Release + _DockerParameter: '' + _LinuxPortable: '' + _RuntimeIdentifier: '--runtime-id linux-musl-arm64' + _BuildArchitecture: 'arm64' + _AdditionalBuildParameters: '/p:OSName="linux-musl"' + # Never run tests on arm64 + _TestArg: '' + Build_Linux_musl_Release_x64: + _BuildConfig: Release + _DockerParameter: '--docker alpine.3.15' + _LinuxPortable: '' + _RuntimeIdentifier: '--runtime-id linux-musl-x64' + _BuildArchitecture: 'x64' + # Pass in HostOSName when running on alpine + _AdditionalBuildParameters: '/p:HostOSName="linux-musl"' + Build_Linux_Portable_Deb_Release_x64: + _BuildConfig: Release + _DockerParameter: '--docker ubuntu.18.04' + _LinuxPortable: '--linux-portable' + _RuntimeIdentifier: '' + _BuildArchitecture: 'x64' + # Do not publish zips and tarballs. The linux-x64 binaries are + # already published by Build_LinuxPortable_Release_x64 + _AdditionalBuildParameters: '/p:PublishBinariesAndBadge=false /p:BuildSdkDeb=true' + _TestArg: $(_NonWindowsTestArg) + Build_Linux_Portable_Rpm_Release_x64: + _BuildConfig: Release + _DockerParameter: '--docker centos' + _LinuxPortable: '--linux-portable' + _RuntimeIdentifier: '' + _BuildArchitecture: 'x64' + # Do not publish zips and tarballs. The linux-x64 binaries are + # already published by Build_LinuxPortable_Release_x64 + _AdditionalBuildParameters: '/p:PublishBinariesAndBadge=false' + _TestArg: $(_NonWindowsTestArg) + Build_Linux_Portable_Rpm_Release_Arm64: + _BuildConfig: Release + _DockerParameter: '--docker centos' + _LinuxPortable: '--linux-portable' + _RuntimeIdentifier: '--runtime-id linux-arm64' + _BuildArchitecture: 'arm64' + # Do not publish zips and tarballs. The linux-x64 binaries are + # already published by Build_LinuxPortable_Release_x64 + _AdditionalBuildParameters: '/p:PublishBinariesAndBadge=false /p:CLIBUILD_SKIP_TESTS=true' + # Never run tests on arm64 + _TestArg: '' + Build_LinuxPortable_Release_x64: + _BuildConfig: Release + _DockerParameter: '' + _LinuxPortable: '--linux-portable' + _RuntimeIdentifier: '' + _BuildArchitecture: 'x64' + _TestArg: $(_NonWindowsTestArg) - - template: /eng/build.yml - parameters: - agentOs: Darwin - pool: - vmImage: 'macOS-latest' - timeoutInMinutes: 180 - strategy: - matrix: - Build_Release_x64: - _BuildConfig: Release - _RuntimeIdentifier: '' - _BuildArchitecture: 'x64' - _TestArg: $(_NonWindowsTestArg) - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - Build_Release_arm64: - _BuildConfig: Release - _RuntimeIdentifier: '--runtime-id osx-arm64' - _BuildArchitecture: 'arm64' - # Never run tests on arm64 - _TestArg: '' + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - template: /eng/build.yml@self + parameters: + agentOs: Linux + pool: + name: $(DncEngInternalBuildPool) + image: 1es-ubuntu-2004 + os: linux + timeoutInMinutes: 180 + strategy: + matrix: + Build_LinuxPortable_Release_x64: + _BuildConfig: Release + _DockerParameter: '' + _LinuxPortable: '--linux-portable' + _RuntimeIdentifier: '' + _BuildArchitecture: 'x64' + _TestArg: '' + Build_Release_arm64: + _BuildConfig: Release + _BuildArchitecture: arm64 + _DOTNET_CLI_UI_LANGUAGE: '' + _AdditionalBuildParameters: '' + # Never run tests on arm64 + _TestArg: '' + pgoInstrument: true - - template: /eng/common/templates/jobs/source-build.yml + - template: /eng/build.yml@self + parameters: + agentOs: Darwin + pool: + name: Azure Pipelines + image: macOS-latest + os: macOS + timeoutInMinutes: 180 + strategy: + matrix: + Build_Release_x64: + _BuildConfig: Release + _RuntimeIdentifier: '' + _BuildArchitecture: 'x64' + _TestArg: $(_NonWindowsTestArg) + ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + Build_Release_arm64: + _BuildConfig: Release + _RuntimeIdentifier: '--runtime-id osx-arm64' + _BuildArchitecture: 'arm64' + # Never run tests on arm64 + _TestArg: '' - - template: /src/SourceBuild/Arcade/eng/common/templates/job/source-build-create-tarball.yml + - template: /eng/common/templates-official/jobs/source-build.yml@self - # For .NET 7.0, source-build currently supports 7.0.1xx. Disable tarball build validation in all other branches. - # - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: - # - template: /src/SourceBuild/Arcade/eng/common/templates/job/source-build-run-tarball-build.yml - # parameters: - # dependsOn: Source_Build_Create_Tarball + - template: /src/SourceBuild/Arcade/eng/common/templates/job/source-build-create-tarball.yml@self - # https://github.com/dotnet/core-sdk/issues/248 - # - template: /eng/build.yml - # parameters: - # agentOs: FreeBSD - # queue: - # name: dnceng-freebsd-internal - # timeoutInMinutes: 180 - # matrix: - # Build_Release: - # _BuildConfig: Release - # _BuildArchitecture: 'x64' - # _AdditionalBuildParameters: '/p:DisableSourceLink=true /p:DISABLE_CROSSGEN=true' + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - template: /eng/common/templates-official/job/publish-build-assets.yml@self + parameters: + dependsOn: + - Windows_NT + - Linux + - Darwin + - Source_Build_Managed + - Source_Build_Create_Tarball + - PGO_Linux + - PGO_Windows_NT + publishUsingPipelines: true + pool: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2019 + os: windows - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - template: /eng/common/templates/job/publish-build-assets.yml - parameters: - dependsOn: - - Windows_NT - - Linux - - Darwin - - Source_Build_Managed - - Source_Build_Create_Tarball - - PGO_Linux - - PGO_Windows_NT - publishUsingPipelines: true - pool: - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - name: NetCore1ESPool-Svc-Internal - demands: ImageOverride -equals windows.vs2017.amd64 - - -- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - template: eng\common\templates\post-build\post-build.yml - parameters: - publishingInfraVersion: 3 - enableSymbolValidation: false - enableSigningValidation: false - enableNugetValidation: false - enableSourceLinkValidation: false - publishInstallersAndChecksums: true - SDLValidationParameters: - enable: false - params: ' -SourceToolsList @("policheck","credscan") - -TsaInstanceURL $(_TsaInstanceURL) - -TsaProjectName $(_TsaProjectName) - -TsaNotificationEmail $(_TsaNotificationEmail) - -TsaCodebaseAdmin $(_TsaCodebaseAdmin) - -TsaBugAreaPath $(_TsaBugAreaPath) - -TsaIterationPath $(_TsaIterationPath) - -TsaRepositoryName "dotnet-installer" - -TsaCodebaseName "dotnet-installer" - -TsaPublish $True' + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - template: /eng/common/templates-official/post-build/post-build.yml@self + parameters: + publishingInfraVersion: 3 + enableSymbolValidation: false + enableSigningValidation: false + enableNugetValidation: false + enableSourceLinkValidation: false + publishInstallersAndChecksums: true + SDLValidationParameters: + enable: false + params: ' -SourceToolsList @("policheck","credscan") + -TsaInstanceURL $(_TsaInstanceURL) + -TsaProjectName $(_TsaProjectName) + -TsaNotificationEmail $(_TsaNotificationEmail) + -TsaCodebaseAdmin $(_TsaCodebaseAdmin) + -TsaBugAreaPath $(_TsaBugAreaPath) + -TsaIterationPath $(_TsaIterationPath) + -TsaRepositoryName "dotnet-installer" + -TsaCodebaseName "dotnet-installer" + -TsaPublish $True' diff --git a/.vsts-pr.yml b/.vsts-pr.yml index 5e2cdc745..2f14d2054 100644 --- a/.vsts-pr.yml +++ b/.vsts-pr.yml @@ -26,7 +26,6 @@ variables: - group: DotNet-Installer-SDLValidation-Params - name: _PublishUsingPipelines value: true - # Default to running tests in PRs and public CI, but not in official builds - name: _WindowsTestArg value: '-test' @@ -37,10 +36,8 @@ variables: value: '' - name: _NonWindowsTestArg value: '' - - name: _InternalRuntimeDownloadArgs value: '' - - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - group: DotNetBuilds storage account read tokens - name: _InternalRuntimeDownloadArgs @@ -63,7 +60,7 @@ stages: - publish: $(Build.SourcesDirectory)\eng\BuildConfiguration artifact: BuildConfiguration displayName: Publish Build Config - - template: /eng/build.yml + - template: /eng/build-pr.yml parameters: agentOs: Windows_NT pool: @@ -107,7 +104,7 @@ stages: _TestArg: '' - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - template: /eng/build.yml + - template: /eng/build-pr.yml parameters: agentOs: Windows_NT pool: @@ -139,7 +136,7 @@ stages: _TestArg: '' pgoInstrument: true - - template: /eng/build.yml + - template: /eng/build-pr.yml parameters: agentOs: Linux pool: @@ -283,7 +280,7 @@ stages: _TestArg: $(_NonWindowsTestArg) - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - template: /eng/build.yml + - template: /eng/build-pr.yml parameters: agentOs: Linux pool: @@ -309,7 +306,7 @@ stages: _TestArg: '' pgoInstrument: true - - template: /eng/build.yml + - template: /eng/build-pr.yml parameters: agentOs: Darwin pool: @@ -332,26 +329,7 @@ stages: - template: /eng/common/templates/jobs/source-build.yml - - template: /src/SourceBuild/Arcade/eng/common/templates/job/source-build-create-tarball.yml - - # For .NET 7.0, source-build currently supports 7.0.1xx. Disable tarball build validation in all other branches. - # - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: - # - template: /src/SourceBuild/Arcade/eng/common/templates/job/source-build-run-tarball-build.yml - # parameters: - # dependsOn: Source_Build_Create_Tarball - - # https://github.com/dotnet/core-sdk/issues/248 - # - template: /eng/build.yml - # parameters: - # agentOs: FreeBSD - # queue: - # name: dnceng-freebsd-internal - # timeoutInMinutes: 180 - # matrix: - # Build_Release: - # _BuildConfig: Release - # _BuildArchitecture: 'x64' - # _AdditionalBuildParameters: '/p:DisableSourceLink=true /p:DISABLE_CROSSGEN=true' + - template: /src/SourceBuild/Arcade/eng/common/templates/job/source-build-create-tarball-pr.yml - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - template: /eng/common/templates/job/publish-build-assets.yml @@ -370,7 +348,6 @@ stages: name: NetCore1ESPool-Svc-Internal demands: ImageOverride -equals windows.vs2017.amd64 - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - template: eng\common\templates\post-build\post-build.yml parameters: diff --git a/eng/build-pr.yml b/eng/build-pr.yml new file mode 100644 index 000000000..69d1e3f0f --- /dev/null +++ b/eng/build-pr.yml @@ -0,0 +1,156 @@ +parameters: + # Agent OS identifier and used as job name + agentOs: '' + + # Agent pool + pool: {} + + # Additional variables + variables: {} + + # Build strategy - matrix + strategy: {} + + # Job timeout + timeoutInMinutes: 180 + + # Publish using pipelines + enablePublishUsingPipelines: true + +phases: +- template: /eng/common/templates/job/job.yml + parameters: + ${{ if parameters.pgoInstrument }}: + name: PGO_${{ parameters.agentOs }} + ${{ if not(parameters.pgoInstrument) }}: + name: ${{ parameters.agentOs }} + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + ${{ if eq(parameters.agentOs, 'Windows_NT') }}: + enableMicrobuild: true + enablePublishBuildAssets: true + enablePublishUsingPipelines: ${{parameters.enablePublishUsingPipelines}} + enableTelemetry: true + helixRepo: dotnet/installer + pool: ${{ parameters.pool }} + ${{ if ne(parameters.strategy, '') }}: + strategy: ${{ parameters.strategy }} + workspace: + clean: all + + variables: + - _PgoInstrument: '' + - ${{ if eq(parameters.agentOs, 'Windows_NT') }}: + - _PackArg: '-pack' + - ${{ if ne(parameters.agentOs, 'Windows_NT') }}: + - _PackArg: '--pack' + - ${{ if parameters.pgoInstrument }}: + - _PgoInstrument: '/p:PgoInstrument=true' + - _PackArg: '' + - _AgentOSName: ${{ parameters.agentOs }} + - _TeamName: Roslyn-Project-System + - _SignType: test + - _BuildArgs: '/p:DotNetSignType=$(_SignType) $(_PgoInstrument)' + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - group: DotNet-Symbol-Server-PATs + - group: DotNet-HelixApi-Access + - group: DotNet-Blob-Feed + - _DotNetPublishToBlobFeed: true + - _PushToVSFeed: true + - _SignType: real + - _BuildArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) + /p:DotNetSignType=$(_SignType) + /p:TeamName=$(_TeamName) + /p:DotNetPublishUsingPipelines=$(_PublishUsingPipelines) + $(_PgoInstrument) + + steps: + - checkout: self + clean: true + - ${{ if eq(parameters.agentOs, 'Windows_NT') }}: + - ${{ if ne(variables['System.TeamProject'], 'public') }}: + - task: PowerShell@2 + displayName: Setup Private Feeds Credentials + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 + arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token + env: + Token: $(dn-bot-dnceng-artifact-feeds-rw) + - script: build.cmd + $(_TestArg) $(_PackArg) + -publish -ci -sign + -Configuration $(_BuildConfig) + -Architecture $(_BuildArchitecture) + $(_BuildArgs) + $(_AdditionalBuildParameters) + $(_InternalRuntimeDownloadArgs) + displayName: Build + env: + DOTNET_CLI_UI_LANGUAGE: $(_DOTNET_CLI_UI_LANGUAGE) + + - ${{ if ne(parameters.agentOs, 'Windows_NT') }}: + - ${{ if ne(variables['System.TeamProject'], 'public') }}: + - task: Bash@3 + displayName: Setup Private Feeds Credentials + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh + arguments: $(Build.SourcesDirectory)/NuGet.config $Token + env: + Token: $(dn-bot-dnceng-artifact-feeds-rw) + + - ${{ if eq(parameters.agentOs, 'Linux') }}: + - script: ./build.sh + $(_TestArg) $(_PackArg) + --publish --ci + --noprettyprint + --configuration $(_BuildConfig) + $(_DockerParameter) + --architecture $(_BuildArchitecture) + $(_LinuxPortable) + $(_RuntimeIdentifier) + $(_BuildArgs) + $(_AdditionalBuildParameters) + $(_InternalRuntimeDownloadArgs) + displayName: Build + + - ${{ if or(eq(parameters.agentOs, 'Darwin'), eq(parameters.agentOs, 'FreeBSD')) }}: + - script: ./build.sh + $(_TestArg) + --pack --publish --ci + --noprettyprint + --configuration $(_BuildConfig) + --architecture $(_BuildArchitecture) + $(_RuntimeIdentifier) + $(_BuildArgs) + $(_AdditionalBuildParameters) + $(_InternalRuntimeDownloadArgs) + displayName: Build + + - task: PublishTestResults@2 + displayName: Publish Test Results + inputs: + testRunner: XUnit + testResultsFiles: 'artifacts/TestResults/$(_BuildConfig)/*.xml' + testRunTitle: '$(_AgentOSName)_$(Agent.JobName)' + platform: '$(BuildPlatform)' + configuration: '$(_BuildConfig)' + condition: ne(variables['_TestArg'], '') + + - task: CopyFiles@2 + displayName: Gather Logs + inputs: + SourceFolder: '$(Build.SourcesDirectory)/artifacts' + Contents: | + log/$(_BuildConfig)/**/* + TestResults/$(_BuildConfig)/**/* + TargetFolder: '$(Build.ArtifactStagingDirectory)' + continueOnError: true + condition: always() + + - task: PublishBuildArtifacts@1 + displayName: Publish Logs to VSTS + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)' + ArtifactName: '$(_AgentOSName)_$(Agent.JobName)_$(Build.BuildNumber)' + publishLocation: Container + continueOnError: true + condition: always() diff --git a/eng/build.yml b/eng/build.yml index 35c829867..d486c47ec 100644 --- a/eng/build.yml +++ b/eng/build.yml @@ -18,7 +18,7 @@ parameters: enablePublishUsingPipelines: true phases: -- template: /eng/common/templates/job/job.yml +- template: /eng/common/templates-official/job/job.yml parameters: ${{ if parameters.pgoInstrument }}: name: PGO_${{ parameters.agentOs }} @@ -28,9 +28,6 @@ phases: ${{ if eq(parameters.agentOs, 'Windows_NT') }}: enableMicrobuild: true enablePublishBuildAssets: true - # Don't use built-in arcade logic for publishing artifacts and test results, as we manually do this and give better names - #enablePublishBuildArtifacts: true - #enablePublishTestResults: true enablePublishUsingPipelines: ${{parameters.enablePublishUsingPipelines}} enableTelemetry: true helixRepo: dotnet/installer @@ -49,12 +46,10 @@ phases: - ${{ if parameters.pgoInstrument }}: - _PgoInstrument: '/p:PgoInstrument=true' - _PackArg: '' - - _AgentOSName: ${{ parameters.agentOs }} - _TeamName: Roslyn-Project-System - _SignType: test - _BuildArgs: '/p:DotNetSignType=$(_SignType) $(_PgoInstrument)' - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - group: DotNet-Symbol-Server-PATs - group: DotNet-HelixApi-Access @@ -101,6 +96,7 @@ phases: arguments: $(Build.SourcesDirectory)/NuGet.config $Token env: Token: $(dn-bot-dnceng-artifact-feeds-rw) + - ${{ if eq(parameters.agentOs, 'Linux') }}: - script: ./build.sh $(_TestArg) $(_PackArg) @@ -144,13 +140,13 @@ phases: inputs: SourceFolder: '$(Build.SourcesDirectory)/artifacts' Contents: | - log/$(_BuildConfig)/**/* - TestResults/$(_BuildConfig)/**/* + log/$(_BuildConfig)/**/* + TestResults/$(_BuildConfig)/**/* TargetFolder: '$(Build.ArtifactStagingDirectory)' continueOnError: true condition: always() - - task: PublishBuildArtifacts@1 + - task: 1ES.PublishBuildArtifacts@1 displayName: Publish Logs to VSTS inputs: PathtoPublish: '$(Build.ArtifactStagingDirectory)' diff --git a/eng/source-build-tarball-build-official.yml b/eng/source-build-tarball-build-official.yml index aa1de43eb..f93d87fef 100644 --- a/eng/source-build-tarball-build-official.yml +++ b/eng/source-build-tarball-build-official.yml @@ -1,5 +1,6 @@ trigger: none pr: none + resources: pipelines: - pipeline: installer-build-resource @@ -19,10 +20,23 @@ resources: - internal/release/7.0.4xx stages: - build + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release -stages: -- stage: build - jobs: - - template: /src/SourceBuild/Arcade/eng/common/templates/job/source-build-run-tarball-build.yml - parameters: - installerBuildResourceId: installer-build-resource +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + sdl: + sourceAnalysisPool: + name: NetCore1ESPool-Svc-Internal + image: 1es-windows-2022 + os: windows + stages: + - stage: build + jobs: + - template: /src/SourceBuild/Arcade/eng/common/templates/job/source-build-run-tarball-build.yml@self + parameters: + installerBuildResourceId: installer-build-resource diff --git a/eng/source-build-tarball-build.yml b/eng/source-build-tarball-build.yml deleted file mode 100644 index da87d93cf..000000000 --- a/eng/source-build-tarball-build.yml +++ /dev/null @@ -1,14 +0,0 @@ -trigger: none -pr: none -resources: - pipelines: - - pipeline: installer-build-resource - source: installer - trigger: none - -stages: -- stage: build - jobs: - - template: /src/SourceBuild/Arcade/eng/common/templates/job/source-build-run-tarball-build.yml - parameters: - installerBuildResourceId: installer-build-resource diff --git a/src/SourceBuild/Arcade/eng/common/templates/job/source-build-build-tarball-pr.yml b/src/SourceBuild/Arcade/eng/common/templates/job/source-build-build-tarball-pr.yml new file mode 100644 index 000000000..abb9035fd --- /dev/null +++ b/src/SourceBuild/Arcade/eng/common/templates/job/source-build-build-tarball-pr.yml @@ -0,0 +1,252 @@ +# Builds a source-build tarball + +parameters: + # Architecture to build + architecture: + + # Custom condition to apply to the job + condition: + + # Dependent jobs that must be completed before this job will run + dependsOn: + + # Skip running the SDK content smoke-tests + excludeSdkContentTests: false + + # Resource id of the installer build to retrieve source/product tarball's from + installerBuildResourceId: + + # Matrix to run jobs over + matrix: + + # Job name + name: + + # Agent pool to use + pool: + + # Build using the previous source-build artifacts + usePreviousArtifacts: false + +jobs: +- job: ${{ parameters.name }} + condition: ${{ parameters.condition }} + dependsOn: ${{ parameters.dependsOn }} + pool: ${{ parameters.pool }} + strategy: + matrix: ${{ parameters.matrix }} + timeoutInMinutes: 300 + variables: + - ${{ if ne(variables['System.TeamProject'], 'public') }}: + - group: AzureDevOps-Artifact-Feeds-Pats + - ${{ if eq(parameters.usePreviousArtifacts, 'true') }}: + - name: additionalBuildArgs + value: --with-sdk /tarball/.dotnet + - name: tarballDir + ${{ if eq(parameters.installerBuildResourceId, '') }}: + value: $(Build.SourcesDirectory) + ${{ else }}: + value: $(Build.StagingDirectory)/tarball + - name: installerSrcDir + ${{ if eq(parameters.installerBuildResourceId, '') }}: + value: $(Build.SourcesDirectory)/src/installer + ${{ else }}: + value: $(Build.SourcesDirectory) + workspace: + clean: all + + steps: + - checkout: self + clean: true + + - ${{ if ne(parameters.installerBuildResourceId, '') }}: + - download: ${{ parameters.installerBuildResourceId }} + artifact: BlobArtifacts + patterns: '**/dotnet-sdk-source*.tar.gz' + displayName: Download Source Tarball + + - script: | + set -x + + resourceIdPathSegment= + if [[ '${{ parameters.installerBuildResourceId }}' != 'current' ]]; then + resourceIdPathSegment='${{ parameters.installerBuildResourceId }}/' + fi + + mkdir -p "$(tarballDir)" + tarballFilePath="$(PIPELINE.WORKSPACE)/${resourceIdPathSegment}BlobArtifacts/dotnet-sdk-source*.tar.gz" + eval tar -ozxf "$tarballFilePath" -C "$(tarballDir)" + eval rm -f "$tarballFilePath" + displayName: Extract Tarball + + - script: cp $(installerSrcDir)/NuGet.config $(tarballDir)/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/online.NuGet.Config + displayName: Copy Test NuGet Config + + - ${{ if ne(parameters.excludeSdkContentTests, 'true') }}: + - download: ${{ parameters.installerBuildResourceId }} + artifact: BlobArtifacts + patterns: '**/dotnet-sdk-+([0-9]).+([0-9]).+([0-9])?(-@(preview|rc|rtm)*)-linux*-${{ parameters.architecture }}.tar.gz' + displayName: Download MSFT sdk Tarball + + - ${{ if eq(parameters.usePreviousArtifacts, 'true') }}: + - download: current + artifact: $(_PreviousSourceBuildArtifact) + patterns: '*.tar.gz' + displayName: Download Previous Source Build Artifacts + + - task: CopyFiles@2 + displayName: Copy Previous Source Build Artifacts + inputs: + SourceFolder: $(PIPELINE.WORKSPACE)/$(_PreviousSourceBuildArtifact) + Contents: '*.tar.gz' + TargetFolder: $(tarballDir)/packages/archive/ + + - script: | + set -euo pipefail + + sourceBuiltSDKUrl="https://dotnetcli.azureedge.net/source-built-artifacts/sdks/" + packageVersionsPath="$(Build.SourcesDirectory)/eng/Versions.props" + notFoundMessage="No source-built SDK found to download..." + + echo "Looking for source-built SDK to download..." + archiveVersionLine=`grep -m 1 "" "$packageVersionsPath" || :` + versionPattern="(.*)" + + if [[ $archiveVersionLine =~ $versionPattern ]]; then + filename="dotnet-sdk-${BASH_REMATCH[1]}-centos.8-x64.tar.gz" + archiveUrl="${sourceBuiltSDKUrl}${filename}" + downloadDir=$(mktemp -d) + (cd $downloadDir && curl --retry 5 -O $archiveUrl) + + mkdir $(tarballDir)/.dotnet + tar -xzf $downloadDir/$filename -C $(tarballDir)/.dotnet + rm -rf $downloadDir + else + echo "$notFoundMessage" + exit 1 + fi + + echo "##vso[task.setvariable variable=additionalBuildArgs]--with-sdk /tarball/.dotnet" + displayName: Setup Previously Source-Built SDK + condition: eq(variables._WithPreviousSDK, 'true') + + - script: | + set -x + + if [[ '${{ parameters.usePreviousArtifacts }}' != 'true' ]]; then + customPrepArgs= + if [[ '$(_BootstrapPrep)' == 'true' ]]; then + customPrepArgs='--bootstrap' + fi + + docker run --rm -v $(tarballDir):/tarball -w /tarball $(_Container) ./prep.sh ${customPrepArgs} + else + mkdir $(tarballDir)/.dotnet + tarballFilePath="$(tarballDir)/packages/archive/dotnet-sdk-*.tar.gz" + eval tar -ozxf "$tarballFilePath" -C "$(tarballDir)/.dotnet" + eval rm -f "$tarballFilePath" + fi + displayName: Prep Tarball + + - script: | + set -x + df -h + + customRunArgs= + customBuildArgs= + if [[ '$(_RunOnline)' == 'true' ]]; then + customBuildArgs='--online' + else + customRunArgs='--network none' + fi + + # See https://github.com/dotnet/source-build/issues/3712 + if [[ '$(_OverrideDistroDisablingSha1)' == 'true' ]]; then + customRunArgs="$customRunArgs -e OPENSSL_ENABLE_SHA1_SIGNATURES=1" + fi + + if [[ '$(_EnablePoison)' == 'true' ]]; then + customBuildArgs="$customBuildArgs --poison" + fi + + docker run --rm -v $(tarballDir):/tarball -w /tarball ${customRunArgs} $(_Container) ./build.sh --clean-while-building ${customBuildArgs} $(additionalBuildArgs) + displayName: Build Tarball + + - ${{ if ne(variables['System.TeamProject'], 'public') }}: + - task: Bash@3 + displayName: Setup Private Feeds Credentials + inputs: + filePath: $(installerSrcDir)/eng/common/SetupNugetSources.sh + arguments: $(tarballDir)/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/online.NuGet.Config $Token + env: + Token: $(dn-bot-dnceng-artifact-feeds-rw) + + - script: | + set -x + + dockerVolumeArgs="-v $(tarballDir):/tarball" + dockerEnvArgs="-e SMOKE_TESTS_EXCLUDE_OMNISHARP=$(_ExcludeOmniSharpTests) -e SMOKE_TESTS_WARN_SDK_CONTENT_DIFFS=true" + poisonArg='' + + if [[ '${{ parameters.excludeSdkContentTests}}' != 'true' ]]; then + dockerVolumeArgs+=" -v $(PIPELINE.WORKSPACE)/${{ parameters.installerBuildResourceId }}/BlobArtifacts/:/BlobArtifacts" + platform="linux" + docker run --rm $(_Container) /bin/sh -c "cat /etc/os-release | grep -q alpine" + if [[ $? == 0 ]]; then + platform="$platform-musl" + fi + msftSdkTarballName=$(find "$(PIPELINE.WORKSPACE)/${{ parameters.installerBuildResourceId }}/BlobArtifacts/" -name "dotnet-sdk-*-$platform-${{ parameters.architecture }}.tar.gz" -exec basename {} \;) + dockerEnvArgs+=" -e SMOKE_TESTS_MSFT_SDK_TARBALL_PATH=/BlobArtifacts/$msftSdkTarballName" + fi + + if [[ '$(_EnablePoison)' == 'true' ]]; then + poisonArg='--poison' + dockerEnvArgs+=" -e SMOKE_TESTS_WARN_POISON_DIFFS=true" + fi + + docker run --rm $dockerVolumeArgs -w /tarball $dockerEnvArgs $(_Container) ./build.sh $poisonArg --run-smoke-test $(additionalBuildArgs) -- /p:SmokeTestConsoleVerbosity=detailed + displayName: Run Tests + + # Don't use CopyFiles@2 as it encounters permissions issues because it indexes all files in the source directory graph. + - script: | + set -x + + targetFolder=$(Build.StagingDirectory)/BuildLogs/ + mkdir -p ${targetFolder} + + cd "$(tarballDir)" + find artifacts/ -type f -name "*.binlog" -exec cp {} --parents -t ${targetFolder} \; + find artifacts/ -type f -name "*.log" -exec cp {} --parents -t ${targetFolder} \; + find artifacts/prebuilt-report/ -exec cp {} --parents -t ${targetFolder} \; + find src/ -type f -name "*.binlog" -exec cp {} --parents -t ${targetFolder} \; + find src/ -type f -name "*.log" -exec cp {} --parents -t ${targetFolder} \; + find test/ -type f -name "*.binlog" -exec cp {} --parents -t ${targetFolder} \; + find test/ -type f -name "Updated*.diff" -exec cp {} --parents -t ${targetFolder} \; + find test/ -type f -name "Updated*.txt" -exec cp {} --parents -t ${targetFolder} \; + displayName: Prepare BuildLogs staging directory + continueOnError: true + condition: succeededOrFailed() + + - publish: '$(Build.StagingDirectory)/BuildLogs' + artifact: $(Agent.JobName)_BuildLogs_Attempt$(System.JobAttempt) + displayName: Publish BuildLogs + continueOnError: true + condition: succeededOrFailed() + + - task: PublishTestResults@2 + displayName: Publish Test Results + condition: succeededOrFailed() + continueOnError: true + inputs: + testRunner: vSTest + testResultsFiles: 'test/**/*.trx' + searchFolder: $(tarballDir) + mergeTestResults: true + publishRunAttachments: true + testRunTitle: SourceBuild_SmokeTests_$(Agent.JobName) + + - publish: '$(tarballDir)/artifacts/${{ parameters.architecture}}/Release/' + artifact: $(Agent.JobName)_Artifacts + displayName: Publish Artifacts + condition: succeededOrFailed() + continueOnError: true diff --git a/src/SourceBuild/Arcade/eng/common/templates/job/source-build-build-tarball.yml b/src/SourceBuild/Arcade/eng/common/templates/job/source-build-build-tarball.yml index abb9035fd..aa3cf6b96 100644 --- a/src/SourceBuild/Arcade/eng/common/templates/job/source-build-build-tarball.yml +++ b/src/SourceBuild/Arcade/eng/common/templates/job/source-build-build-tarball.yml @@ -227,9 +227,11 @@ jobs: continueOnError: true condition: succeededOrFailed() - - publish: '$(Build.StagingDirectory)/BuildLogs' - artifact: $(Agent.JobName)_BuildLogs_Attempt$(System.JobAttempt) + - task: 1ES.PublishPipelineArtifact@1 displayName: Publish BuildLogs + inputs: + targetPath: $(Build.StagingDirectory)/BuildLogs + artifactName: $(Agent.JobName)_BuildLogs_Attempt$(System.JobAttempt) continueOnError: true condition: succeededOrFailed() @@ -245,8 +247,10 @@ jobs: publishRunAttachments: true testRunTitle: SourceBuild_SmokeTests_$(Agent.JobName) - - publish: '$(tarballDir)/artifacts/${{ parameters.architecture}}/Release/' - artifact: $(Agent.JobName)_Artifacts + - task: 1ES.PublishPipelineArtifact@1 displayName: Publish Artifacts - condition: succeededOrFailed() + inputs: + targetPath: $(tarballDir)/artifacts/${{ parameters.architecture}}/Release/ + artifactName: $(Agent.JobName)_Artifacts continueOnError: true + condition: succeededOrFailed() diff --git a/src/SourceBuild/Arcade/eng/common/templates/job/source-build-create-tarball-pr.yml b/src/SourceBuild/Arcade/eng/common/templates/job/source-build-create-tarball-pr.yml new file mode 100644 index 000000000..347f61cf8 --- /dev/null +++ b/src/SourceBuild/Arcade/eng/common/templates/job/source-build-create-tarball-pr.yml @@ -0,0 +1,85 @@ +# Creates a source-build tarball + +jobs: +- job: Source_Build_Create_Tarball + container: mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-38 + displayName: Source-Build Create Tarball + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + ${{ if in(variables['Build.Reason'], 'PullRequest') }}: + name: NetCore-Public-XL + demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open + ${{ else }}: + name: NetCore-Svc-Public + demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + ${{ if in(variables['Build.Reason'], 'PullRequest') }}: + name: NetCore1ESPool-Internal-XL + demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 + ${{ else }}: + name: NetCore1ESPool-Svc-Internal + demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 + + variables: + - name: _BuildConfig + value: Release + + - ${{ if eq(variables['System.TeamProject'], 'internal') }}: + - group: DotNet-Source-Build-All-Orgs-Source-Access + workspace: + clean: all + + steps: + - ${{ if ne(variables['System.TeamProject'], 'public') }}: + - task: Bash@3 + displayName: Setup Private Feeds Credentials + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh + arguments: $(Build.SourcesDirectory)/NuGet.config $Token + env: + Token: $(dn-bot-dnceng-artifact-feeds-rw) + - script: | + set -x + df -h + + officialBuildArgs= + if [ '${{ and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}' = 'True' ]; then + officialBuildArgs='/p:OfficialBuildId=$(BUILD.BUILDNUMBER)' + fi + + # If building on the internal project, the internal storage variable may be available (usually only if needed) + # In that case, add variables to allow the download of internal runtimes if the specified versions are not found + # in the default public locations. + internalRuntimeDownloadArgs= + if [ '$(dotnetbuilds-internal-container-read-token-base64)' != '$''(dotnetbuilds-internal-container-read-token-base64)' ]; then + internalRuntimeDownloadArgs='/p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) --runtimesourcefeed https://dotnetbuilds.blob.core.windows.net/internal --runtimesourcefeedkey $(dotnetbuilds-internal-container-read-token-base64)' + fi + + ./build.sh \ + --ci \ + --configuration $(_BuildConfig) \ + --publish \ + -bl \ + $officialBuildArgs \ + $internalRuntimeDownloadArgs \ + /p:DotNetPublishUsingPipelines=true \ + /p:ArcadeBuildTarball=true \ + /p:AzDoPat=$(dn-bot-all-orgs-build-rw-code-rw) + displayName: Create Tarball + + - task: CopyFiles@2 + displayName: Prepare BuildLogs staging directory + inputs: + Contents: | + **/*.log + **/*.binlog + TargetFolder: '$(Build.StagingDirectory)/BuildLogs' + CleanTargetFolder: true + continueOnError: true + condition: succeededOrFailed() + + - publish: '$(Build.StagingDirectory)/BuildLogs' + artifact: $(Agent.JobName)_BuildLogs_Attempt$(System.JobAttempt) + displayName: Publish BuildLogs + continueOnError: true + condition: succeededOrFailed() diff --git a/src/SourceBuild/Arcade/eng/common/templates/job/source-build-create-tarball.yml b/src/SourceBuild/Arcade/eng/common/templates/job/source-build-create-tarball.yml index 347f61cf8..7cba0c4ea 100644 --- a/src/SourceBuild/Arcade/eng/common/templates/job/source-build-create-tarball.yml +++ b/src/SourceBuild/Arcade/eng/common/templates/job/source-build-create-tarball.yml @@ -2,23 +2,27 @@ jobs: - job: Source_Build_Create_Tarball - container: mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-38 + container: fedora38 displayName: Source-Build Create Tarball pool: ${{ if eq(variables['System.TeamProject'], 'public') }}: ${{ if in(variables['Build.Reason'], 'PullRequest') }}: name: NetCore-Public-XL - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open + image: 1es-ubuntu-2004-open + os: linux ${{ else }}: name: NetCore-Svc-Public - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open + image: 1es-ubuntu-2004-open + os: linux ${{ if eq(variables['System.TeamProject'], 'internal') }}: ${{ if in(variables['Build.Reason'], 'PullRequest') }}: name: NetCore1ESPool-Internal-XL - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 + image: 1es-ubuntu-2004 + os: linux ${{ else }}: name: NetCore1ESPool-Svc-Internal - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 + image: 1es-ubuntu-2004 + os: linux variables: - name: _BuildConfig @@ -78,8 +82,10 @@ jobs: continueOnError: true condition: succeededOrFailed() - - publish: '$(Build.StagingDirectory)/BuildLogs' - artifact: $(Agent.JobName)_BuildLogs_Attempt$(System.JobAttempt) + - task: 1ES.PublishPipelineArtifact@1 displayName: Publish BuildLogs + inputs: + targetPath: $(Build.StagingDirectory)/BuildLogs + artifactName: $(Agent.JobName)_BuildLogs_Attempt$(System.JobAttempt) continueOnError: true condition: succeededOrFailed() diff --git a/src/SourceBuild/Arcade/eng/common/templates/job/source-build-run-tarball-build-pr.yml b/src/SourceBuild/Arcade/eng/common/templates/job/source-build-run-tarball-build-pr.yml new file mode 100644 index 000000000..6e9c404c5 --- /dev/null +++ b/src/SourceBuild/Arcade/eng/common/templates/job/source-build-run-tarball-build-pr.yml @@ -0,0 +1,147 @@ +# Builds a source-build tarball + +parameters: + # Dependent jobs that must be completed before this job will run + dependsOn: + + # Resource id of the installer build to retrieve source/product tarball's from + installerBuildResourceId: current + + # The following parameters aren't expected to be passed in rather they are used for encapsulation + # ----------------------------------------------------------------------------------------------- + alpine317Container: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.17 + centOSStream8Container: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8 + centOSStream9Container: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9 + debian11Arm64Container: mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-arm64v8 + fedora38Container: mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-38 + ubuntu2004Container: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-20.04 + poolInternalAmd64: + name: NetCore1ESPool-Svc-Internal + demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 + poolInternalAmd64PR: + name: NetCore1ESPool-Internal-XL + demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 + poolInternalArm64: + name: Docker-Linux-Arm-Internal + poolPublicAmd64: + name: NetCore-Public-XL + demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open + +jobs: +- template: /src/SourceBuild/Arcade/eng/common/templates/job/source-build-build-tarball-pr.yml + parameters: + architecture: x64 + dependsOn: ${{ parameters.dependsOn }} + ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: + excludeSdkContentTests: true + installerBuildResourceId: ${{ parameters.installerBuildResourceId }} + matrix: + CentOSStream8-Online: + _BootstrapPrep: true + _Container: ${{ parameters.centOSStream8Container }} + _EnablePoison: false + _ExcludeOmniSharpTests: true + _OverrideDistroDisablingSha1: false + _RunOnline: true + _WithPreviousSDK: false + CentOSStream8-Offline: + _BootstrapPrep: true + _Container: ${{ parameters.centOSStream8Container }} + _EnablePoison: false + _ExcludeOmniSharpTests: true + _RunOnline: false + _WithPreviousSDK: false + ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: + Alpine317-Offline: + _BootstrapPrep: true + _Container: ${{ parameters.alpine317Container }} + _EnablePoison: false + _ExcludeOmniSharpTests: true + _OverrideDistroDisablingSha1: false + _RunOnline: false + _WithPreviousSDK: false + CentOSStream8-WithPreviousSDK: + _BootstrapPrep: false + _Container: ${{ parameters.centOSStream8Container }} + _EnablePoison: false + _ExcludeOmniSharpTests: true + _RunOnline: false + _WithPreviousSDK: true + CentOSStream9-Offline: + _BootstrapPrep: true + _Container: ${{ parameters.centOSStream9Container }} + _EnablePoison: false + _ExcludeOmniSharpTests: false + _OverrideDistroDisablingSha1: true + _RunOnline: false + _WithPreviousSDK: false + Fedora38-Offline: + _BootstrapPrep: true + _Container: ${{ parameters.fedora38Container }} + _EnablePoison: true + _ExcludeOmniSharpTests: false + _OverrideDistroDisablingSha1: false + _RunOnline: false + _WithPreviousSDK: false + Ubuntu2004-Offline: + _BootstrapPrep: true + _Container: ${{ parameters.ubuntu2004Container }} + _EnablePoison: false + _ExcludeOmniSharpTests: false + _OverrideDistroDisablingSha1: false + _RunOnline: false + _WithPreviousSDK: false + name: Build_Tarball_x64 + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + ${{ parameters.poolPublicAmd64 }} + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + ${{ if in(variables['Build.Reason'], 'PullRequest') }}: + ${{ parameters.poolInternalAmd64PR }} + ${{ else }}: + ${{ parameters.poolInternalAmd64 }} + +- ${{ if and(eq(variables['System.TeamProject'], 'internal'), ne(variables['Build.Reason'], 'PullRequest')) }}: + - template: /src/SourceBuild/Arcade/eng/common/templates/job/source-build-build-tarball-pr.yml + parameters: + architecture: arm64 + dependsOn: ${{ parameters.dependsOn }} + installerBuildResourceId: ${{ parameters.installerBuildResourceId }} + matrix: + Debian11-Offline: + _BootstrapPrep: true + _Container: ${{ parameters.debian11Arm64Container }} + _EnablePoison: false + _ExcludeOmniSharpTests: false + _OverrideDistroDisablingSha1: false + _RunOnline: false + _WithPreviousSDK: false + name: Build_Tarball_arm64 + pool: ${{ parameters.poolInternalArm64 }} + +- ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: + - template: /src/SourceBuild/Arcade/eng/common/templates/job/source-build-build-tarball-pr.yml + parameters: + architecture: x64 + # Always attempt to run the bootstrap leg (e.g. even when stage 1 tests fail) in order to get a complete assessment of the build status. + # The bootstrap build will shortcut if the stage 1 build failed. + condition: succeededOrFailed() + dependsOn: Build_Tarball_x64 + excludeSdkContentTests: true + installerBuildResourceId: ${{ parameters.installerBuildResourceId }} + matrix: + Fedora38-Offline: + _PreviousSourceBuildArtifact: Build_Tarball_x64 Fedora38-Offline_Artifacts + _Container: ${{ parameters.fedora38Container }} + _EnablePoison: false + _ExcludeOmniSharpTests: false + _OverrideDistroDisablingSha1: false + _RunOnline: false + _WithPreviousSDK: false + name: Build_Tarball_x64_Using_Previous + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + ${{ parameters.poolPublicAmd64 }} + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + ${{ parameters.poolInternalAmd64 }} + usePreviousArtifacts: true diff --git a/src/SourceBuild/Arcade/eng/common/templates/job/source-build-run-tarball-build.yml b/src/SourceBuild/Arcade/eng/common/templates/job/source-build-run-tarball-build.yml index 307cb7307..f7f9cdb48 100644 --- a/src/SourceBuild/Arcade/eng/common/templates/job/source-build-run-tarball-build.yml +++ b/src/SourceBuild/Arcade/eng/common/templates/job/source-build-run-tarball-build.yml @@ -17,15 +17,21 @@ parameters: ubuntu2004Container: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-20.04 poolInternalAmd64: name: NetCore1ESPool-Svc-Internal - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 + image: 1es-ubuntu-2004 + os: linux poolInternalAmd64PR: name: NetCore1ESPool-Internal-XL - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 + image: 1es-ubuntu-2004 + os: linux poolInternalArm64: name: Docker-Linux-Arm-Internal + image: Mariner-2-Docker-ARM64 + os: linux + hostArchitecture: Arm64 poolPublicAmd64: name: NetCore-Public-XL - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open + image: 1es-ubuntu-2004-open + os: linux jobs: - template: /src/SourceBuild/Arcade/eng/common/templates/job/source-build-build-tarball.yml @@ -123,7 +129,7 @@ jobs: - template: /src/SourceBuild/Arcade/eng/common/templates/job/source-build-build-tarball.yml parameters: architecture: x64 - # Always attempt to run the bootstrap leg (e.g. even when stage 1 tests fail) in order to get a complete accessment of the build status. + # Always attempt to run the bootstrap leg (e.g. even when stage 1 tests fail) in order to get a complete assessment of the build status. # The bootstrap build will shortcut if the stage 1 build failed. condition: succeededOrFailed() dependsOn: Build_Tarball_x64