From 421270b99b0a74728ba8013710ea93653b4f1798 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Fri, 8 Mar 2024 12:38:29 -0800 Subject: [PATCH 01/36] Made the official CI pipeline use 1ES templates from Arcade. Created separate build-pr.yml for PR pipeline to use. Modified CI and normal build.yml to no longer have public conditional logic. Changed to using PT image for migration testing. Updated localization.yml to use the new templates. --- .vsts-ci.yml | 562 +++++++++++++++++++------------------------ .vsts-pr.yml | 58 ++--- eng/build-pr.yml | 261 ++++++++++++++++++++ eng/build.yml | 39 ++- eng/localization.yml | 6 +- 5 files changed, 550 insertions(+), 376 deletions(-) create mode 100644 eng/build-pr.yml diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 429c53be0..7c4acd9d5 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -1,3 +1,5 @@ +# Pipeline: https://dnceng.visualstudio.com/internal/_build?definitionId=286 + trigger: batch: true branches: @@ -8,6 +10,7 @@ trigger: - internal/release/* variables: +# Loose Variables - name: _PublishUsingPipelines value: false - ${{ if or(startswith(variables['Build.SourceBranch'], 'refs/heads/release/'), startswith(variables['Build.SourceBranch'], 'refs/heads/internal/release/'), eq(variables['Build.Reason'], 'Manual')) }}: @@ -23,10 +26,14 @@ variables: - group: DotNet-Installer-SDLValidation-Params - name: _PublishUsingPipelines value: true - - name: _InternalRuntimeDownloadArgs value: '' +# Only used for tracking purposes in MicroBuild tasks. +# See: https://devdiv.visualstudio.com/DevDiv/_wiki/wikis/DevDiv.wiki/650/MicroBuild-Signing?anchor=high-level-steps-to-enable-signing +- name: TeamName + value: DotNetCLI +# Variable Groups - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - group: DotNetBuilds storage account read tokens - name: _InternalRuntimeDownloadArgs @@ -34,328 +41,241 @@ variables: /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) /p:dotnetbuilds-internal-container-read-token-base64=$(dotnetbuilds-internal-container-read-token-base64) -- template: /eng/common/templates/variables/pool-providers.yml +# Variable Templates +- template: /eng/common/templates-official/variables/pool-providers.yml -stages: -- stage: Build - jobs: - # This job is for build retry configuration. - - job: Publish_Build_Configuration - pool: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals windows.vs2022preview.amd64.open - ${{ if eq(variables['System.TeamProject'], 'internal') }}: +resources: + repositories: + - repository: MicroBuildTemplate + type: git + name: 1ESPipelineTemplates/MicroBuildTemplate + ref: refs/tags/release + +extends: + template: azure-pipelines/MicroBuild.1ES.Official.yml@MicroBuildTemplate + parameters: + sdl: + sourceAnalysisPool: name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2022preview.amd64 - steps: - - publish: $(Build.SourcesDirectory)\eng\buildConfiguration - artifact: buildConfiguration - displayName: Publish Build Config - - ## PR-only jobs - - - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: - - ## Windows - - - template: eng/build.yml - parameters: - agentOs: Windows_NT - jobName: Build_Debug_x64 - buildConfiguration: Debug - buildArchitecture: x64 - additionalBuildParameters: '/p:PublishInternalAsset=true' - runTests: true - - ## Linux - - - template: eng/build.yml - parameters: - agentOs: Linux - jobName: Build_Ubuntu_22_04_Debug_x64 - container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04' - buildConfiguration: Debug - buildArchitecture: x64 - linuxPortable: true - runTests: true - - template: eng/build.yml - parameters: - agentOs: Linux - jobName: Build_Fedora_39_Debug_x64 - container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-39' - buildConfiguration: Debug - buildArchitecture: x64 - linuxPortable: true - runTests: true - - template: eng/build.yml - parameters: - agentOs: Linux - jobName: Build_CentOS_8_Stream_Debug_x64 - container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8' - buildConfiguration: Debug - buildArchitecture: x64 - linuxPortable: false - runTests: true - - template: eng/build.yml - parameters: - agentOs: Linux - jobName: Build_Debian_11_Debug_x64 - container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-amd64' - buildConfiguration: Debug - buildArchitecture: x64 - additionalBuildParameters: '/p:BuildSdkDeb=true' - linuxPortable: false - runTests: true - - template: eng/build.yml - parameters: - agentOs: Linux - jobName: Build_Arm64_Debug - buildConfiguration: Debug - buildArchitecture: arm64 - runtimeIdentifier: 'linux-arm64' - linuxPortable: true - # Never run tests on arm64 - runTests: false - - template: eng/build.yml - parameters: - agentOs: Linux - jobName: Build_Linux_musl_Debug_x64 - container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.19-WithNode' - buildConfiguration: Debug - buildArchitecture: x64 - runtimeIdentifier: 'linux-musl-x64' - # Pass in HostOSName when running on alpine - additionalBuildParameters: '/p:HostOSName="linux-musl"' - linuxPortable: false - runTests: true - - template: eng/build.yml - parameters: - agentOs: Linux - jobName: Build_LinuxPortable_Release_x64 - buildConfiguration: Release - buildArchitecture: x64 - linuxPortable: true - runTests: true - - # MacOS - - - template: eng/build.yml - parameters: - agentOs: Darwin - jobName: Build_Release_x64 - buildConfiguration: Release - buildArchitecture: x64 - runTests: true - - ## Official/PGO instrumentation Builds - - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - ## Windows - - - template: eng/build.yml - parameters: - agentOs: Windows_NT - jobName: Build_Release_x64 - buildConfiguration: Release - buildArchitecture: x64 - additionalBuildParameters: '/p:PublishInternalAsset=true' - runTests: false - - template: eng/build.yml - parameters: - agentOs: Windows_NT - jobName: Build_Release_x86 - buildConfiguration: Release - buildArchitecture: x86 - runTests: false - - template: eng/build.yml - parameters: - agentOs: Windows_NT - jobName: Build_Release_arm64 - buildConfiguration: Release - buildArchitecture: arm64 - runTests: false - - ## Linux - - - template: eng/build.yml - parameters: - agentOs: Linux - jobName: Build_Arm_Release - buildConfiguration: Release - buildArchitecture: arm - runtimeIdentifier: 'linux-arm' - linuxPortable: true - runTests: false - - template: eng/build.yml - parameters: - agentOs: Linux - jobName: Build_Arm64_Release - buildConfiguration: Release - buildArchitecture: arm64 - runtimeIdentifier: 'linux-arm64' - linuxPortable: true - runTests: false - - template: eng/build.yml - parameters: - agentOs: Linux - jobName: Build_Linux_musl_Release_arm - container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-arm-alpine' - buildConfiguration: Release - buildArchitecture: arm - runtimeIdentifier: 'linux-musl-arm' - additionalBuildParameters: '/p:OSName="linux-musl"' - linuxPortable: false - runTests: false - - template: eng/build.yml - parameters: - agentOs: Linux - jobName: Build_Linux_musl_Release_arm64 - buildConfiguration: Release - buildArchitecture: arm64 - runtimeIdentifier: 'linux-musl-arm64' - additionalBuildParameters: '/p:OSName="linux-musl"' - linuxPortable: false - runTests: false - - template: eng/build.yml - parameters: - agentOs: Linux - jobName: Build_Linux_musl_Release_x64 - container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.19-WithNode' - buildConfiguration: Release - buildArchitecture: x64 - runtimeIdentifier: 'linux-musl-x64' - # Pass in HostOSName when running on alpine - additionalBuildParameters: '/p:HostOSName="linux-musl"' - linuxPortable: false - runTests: false - - template: eng/build.yml - parameters: - agentOs: Linux - jobName: Build_Linux_Portable_Deb_Release_x64 - container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-debpkg' - buildConfiguration: Release - buildArchitecture: x64 - # Do not publish zips and tarballs. The linux-x64 binaries are - # already published by Build_LinuxPortable_Release_x64 - additionalBuildParameters: '/p:PublishBinariesAndBadge=false /p:BuildSdkDeb=true' - linuxPortable: true - runTests: false - - template: eng/build.yml - parameters: - agentOs: Linux - jobName: Build_Linux_Portable_Rpm_Release_x64 - container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-fpm' - buildConfiguration: Release - buildArchitecture: x64 - # Do not publish zips and tarballs. The linux-x64 binaries are - # already published by Build_LinuxPortable_Release_x64 - additionalBuildParameters: '/p:PublishBinariesAndBadge=false /p:IsRPMBasedDistro=true' - linuxPortable: true - runTests: false - - template: eng/build.yml - parameters: - agentOs: Linux - jobName: Build_Linux_Portable_Rpm_Release_Arm64 - container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-fpm' - buildConfiguration: Release - buildArchitecture: arm64 - runtimeIdentifier: 'linux-arm64' - # Do not publish zips and tarballs. The linux-x64 binaries are - # already published by Build_LinuxPortable_Release_x64 - additionalBuildParameters: '/p:PublishBinariesAndBadge=false /p:CLIBUILD_SKIP_TESTS=true /p:IsRPMBasedDistro=true' - linuxPortable: true - runTests: false - - template: eng/build.yml - parameters: - agentOs: Linux - jobName: Build_LinuxPortable_Release_x64 - buildConfiguration: Release - buildArchitecture: x64 - linuxPortable: true - runTests: false - - # MacOS - - - template: eng/build.yml - parameters: - agentOs: Darwin - jobName: Build_Release_x64 - buildConfiguration: Release - buildArchitecture: x64 - runTests: false - - template: eng/build.yml - parameters: - agentOs: Darwin - jobName: Build_Release_arm64 - runtimeIdentifier: 'osx-arm64' - buildConfiguration: Release - buildArchitecture: arm64 - runTests: false - - ## Windows PGO Instrumentation builds - - - template: eng/build.yml - parameters: - agentOs: Windows_NT - pgoInstrument: true - jobName: Build_Release_x64 - buildConfiguration: Release - buildArchitecture: x64 - additionalBuildParameters: '/p:PublishInternalAsset=true' - runTests: false - - template: eng/build.yml - parameters: - agentOs: Windows_NT - pgoInstrument: true - jobName: Build_Release_x86 - buildConfiguration: Release - buildArchitecture: x86 - runTests: false - - template: eng/build.yml - parameters: - agentOs: Windows_NT - pgoInstrument: true - jobName: Build_Release_arm64 - buildConfiguration: Release - buildArchitecture: arm64 - runTests: false - - ## Linux PGO Instrumentation builds - - - template: eng/build.yml - parameters: - agentOs: Linux - pgoInstrument: true - jobName: Build_LinuxPortable_Release_x64 - buildConfiguration: Release - buildArchitecture: x64 - linuxPortable: true - runTests: false - - - template: eng/build.yml - parameters: - agentOs: Linux - pgoInstrument: true - jobName: Build_Release_arm64 - buildConfiguration: Release - buildArchitecture: arm64 - linuxPortable: true - runTests: false - - - template: /eng/common/templates/jobs/source-build.yml - -- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - stage: Publish - dependsOn: - - Build - jobs: - - template: /eng/common/templates/job/publish-build-assets.yml - parameters: - publishUsingPipelines: true - publishAssetsImmediately: true + image: 1es-windows-2022-pt + os: windows + customBuildTags: + - ES365AIMigrationTooling + stages: + - stage: Build + jobs: + # Build Retry Configuration + - job: Publish_Build_Configuration pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals windows.vs2022preview.amd64.open ${{ if eq(variables['System.TeamProject'], 'internal') }}: name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2022.amd64 + demands: ImageOverride -equals windows.vs2022preview.amd64 + steps: + - publish: $(Build.SourcesDirectory)\eng\buildConfiguration + artifact: buildConfiguration + displayName: Publish Build Config + + # Windows + - template: eng/build.yml + parameters: + agentOs: Windows_NT + jobName: Build_Release_x64 + buildConfiguration: Release + buildArchitecture: x64 + additionalBuildParameters: '/p:PublishInternalAsset=true' + runTests: false + - template: eng/build.yml + parameters: + agentOs: Windows_NT + jobName: Build_Release_x86 + buildConfiguration: Release + buildArchitecture: x86 + runTests: false + - template: eng/build.yml + parameters: + agentOs: Windows_NT + jobName: Build_Release_arm64 + buildConfiguration: Release + buildArchitecture: arm64 + runTests: false + + # Linux + - template: eng/build.yml + parameters: + agentOs: Linux + jobName: Build_Arm_Release + buildConfiguration: Release + buildArchitecture: arm + runtimeIdentifier: 'linux-arm' + linuxPortable: true + runTests: false + - template: eng/build.yml + parameters: + agentOs: Linux + jobName: Build_Arm64_Release + buildConfiguration: Release + buildArchitecture: arm64 + runtimeIdentifier: 'linux-arm64' + linuxPortable: true + runTests: false + - template: eng/build.yml + parameters: + agentOs: Linux + jobName: Build_Linux_musl_Release_arm + container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-arm-alpine' + buildConfiguration: Release + buildArchitecture: arm + runtimeIdentifier: 'linux-musl-arm' + additionalBuildParameters: '/p:OSName="linux-musl"' + linuxPortable: false + runTests: false + - template: eng/build.yml + parameters: + agentOs: Linux + jobName: Build_Linux_musl_Release_arm64 + buildConfiguration: Release + buildArchitecture: arm64 + runtimeIdentifier: 'linux-musl-arm64' + additionalBuildParameters: '/p:OSName="linux-musl"' + linuxPortable: false + runTests: false + - template: eng/build.yml + parameters: + agentOs: Linux + jobName: Build_Linux_musl_Release_x64 + container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.19-WithNode' + buildConfiguration: Release + buildArchitecture: x64 + runtimeIdentifier: 'linux-musl-x64' + # Pass in HostOSName when running on alpine + additionalBuildParameters: '/p:HostOSName="linux-musl"' + linuxPortable: false + runTests: false + - template: eng/build.yml + parameters: + agentOs: Linux + jobName: Build_Linux_Portable_Deb_Release_x64 + container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-debpkg' + buildConfiguration: Release + buildArchitecture: x64 + # Do not publish zips and tarballs. The linux-x64 binaries are + # already published by Build_LinuxPortable_Release_x64 + additionalBuildParameters: '/p:PublishBinariesAndBadge=false /p:BuildSdkDeb=true' + linuxPortable: true + runTests: false + - template: eng/build.yml + parameters: + agentOs: Linux + jobName: Build_Linux_Portable_Rpm_Release_x64 + container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-fpm' + buildConfiguration: Release + buildArchitecture: x64 + # Do not publish zips and tarballs. The linux-x64 binaries are + # already published by Build_LinuxPortable_Release_x64 + additionalBuildParameters: '/p:PublishBinariesAndBadge=false /p:IsRPMBasedDistro=true' + linuxPortable: true + runTests: false + - template: eng/build.yml + parameters: + agentOs: Linux + jobName: Build_Linux_Portable_Rpm_Release_Arm64 + container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-fpm' + buildConfiguration: Release + buildArchitecture: arm64 + runtimeIdentifier: 'linux-arm64' + # Do not publish zips and tarballs. The linux-x64 binaries are + # already published by Build_LinuxPortable_Release_x64 + additionalBuildParameters: '/p:PublishBinariesAndBadge=false /p:CLIBUILD_SKIP_TESTS=true /p:IsRPMBasedDistro=true' + linuxPortable: true + runTests: false + - template: eng/build.yml + parameters: + agentOs: Linux + jobName: Build_LinuxPortable_Release_x64 + buildConfiguration: Release + buildArchitecture: x64 + linuxPortable: true + runTests: false + + # MacOS + - template: eng/build.yml + parameters: + agentOs: Darwin + jobName: Build_Release_x64 + buildConfiguration: Release + buildArchitecture: x64 + runTests: false + - template: eng/build.yml + parameters: + agentOs: Darwin + jobName: Build_Release_arm64 + runtimeIdentifier: 'osx-arm64' + buildConfiguration: Release + buildArchitecture: arm64 + runTests: false + + # Windows PGO Instrumentation + - template: eng/build.yml + parameters: + agentOs: Windows_NT + pgoInstrument: true + jobName: Build_Release_x64 + buildConfiguration: Release + buildArchitecture: x64 + additionalBuildParameters: '/p:PublishInternalAsset=true' + runTests: false + - template: eng/build.yml + parameters: + agentOs: Windows_NT + pgoInstrument: true + jobName: Build_Release_x86 + buildConfiguration: Release + buildArchitecture: x86 + runTests: false + - template: eng/build.yml + parameters: + agentOs: Windows_NT + pgoInstrument: true + jobName: Build_Release_arm64 + buildConfiguration: Release + buildArchitecture: arm64 + runTests: false + + # Linux PGO Instrumentation + - template: eng/build.yml + parameters: + agentOs: Linux + pgoInstrument: true + jobName: Build_LinuxPortable_Release_x64 + buildConfiguration: Release + buildArchitecture: x64 + linuxPortable: true + runTests: false + - template: eng/build.yml + parameters: + agentOs: Linux + pgoInstrument: true + jobName: Build_Release_arm64 + buildConfiguration: Release + buildArchitecture: arm64 + linuxPortable: true + runTests: false + + # Source Build + - template: /eng/common/templates-official/jobs/source-build.yml + + - stage: Publish + dependsOn: + - Build + jobs: + - template: /eng/common/templates-official/job/publish-build-assets.yml + parameters: + publishUsingPipelines: true + publishAssetsImmediately: true + pool: + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: $(DncEngInternalBuildPool) + demands: ImageOverride -equals windows.vs2022.amd64 diff --git a/.vsts-pr.yml b/.vsts-pr.yml index 429c53be0..2a762dd9a 100644 --- a/.vsts-pr.yml +++ b/.vsts-pr.yml @@ -1,3 +1,5 @@ +# Pipeline: https://dev.azure.com/dnceng-public/public/_build?definitionId=20 + trigger: batch: true branches: @@ -59,7 +61,7 @@ stages: ## Windows - - template: eng/build.yml + - template: eng/build-pr.yml parameters: agentOs: Windows_NT jobName: Build_Debug_x64 @@ -70,7 +72,7 @@ stages: ## Linux - - template: eng/build.yml + - template: eng/build-pr.yml parameters: agentOs: Linux jobName: Build_Ubuntu_22_04_Debug_x64 @@ -79,7 +81,7 @@ stages: buildArchitecture: x64 linuxPortable: true runTests: true - - template: eng/build.yml + - template: eng/build-pr.yml parameters: agentOs: Linux jobName: Build_Fedora_39_Debug_x64 @@ -88,7 +90,7 @@ stages: buildArchitecture: x64 linuxPortable: true runTests: true - - template: eng/build.yml + - template: eng/build-pr.yml parameters: agentOs: Linux jobName: Build_CentOS_8_Stream_Debug_x64 @@ -97,7 +99,7 @@ stages: buildArchitecture: x64 linuxPortable: false runTests: true - - template: eng/build.yml + - template: eng/build-pr.yml parameters: agentOs: Linux jobName: Build_Debian_11_Debug_x64 @@ -107,7 +109,7 @@ stages: additionalBuildParameters: '/p:BuildSdkDeb=true' linuxPortable: false runTests: true - - template: eng/build.yml + - template: eng/build-pr.yml parameters: agentOs: Linux jobName: Build_Arm64_Debug @@ -117,7 +119,7 @@ stages: linuxPortable: true # Never run tests on arm64 runTests: false - - template: eng/build.yml + - template: eng/build-pr.yml parameters: agentOs: Linux jobName: Build_Linux_musl_Debug_x64 @@ -129,7 +131,7 @@ stages: additionalBuildParameters: '/p:HostOSName="linux-musl"' linuxPortable: false runTests: true - - template: eng/build.yml + - template: eng/build-pr.yml parameters: agentOs: Linux jobName: Build_LinuxPortable_Release_x64 @@ -140,7 +142,7 @@ stages: # MacOS - - template: eng/build.yml + - template: eng/build-pr.yml parameters: agentOs: Darwin jobName: Build_Release_x64 @@ -154,7 +156,7 @@ stages: ## Windows - - template: eng/build.yml + - template: eng/build-pr.yml parameters: agentOs: Windows_NT jobName: Build_Release_x64 @@ -162,14 +164,14 @@ stages: buildArchitecture: x64 additionalBuildParameters: '/p:PublishInternalAsset=true' runTests: false - - template: eng/build.yml + - template: eng/build-pr.yml parameters: agentOs: Windows_NT jobName: Build_Release_x86 buildConfiguration: Release buildArchitecture: x86 runTests: false - - template: eng/build.yml + - template: eng/build-pr.yml parameters: agentOs: Windows_NT jobName: Build_Release_arm64 @@ -179,7 +181,7 @@ stages: ## Linux - - template: eng/build.yml + - template: eng/build-pr.yml parameters: agentOs: Linux jobName: Build_Arm_Release @@ -188,7 +190,7 @@ stages: runtimeIdentifier: 'linux-arm' linuxPortable: true runTests: false - - template: eng/build.yml + - template: eng/build-pr.yml parameters: agentOs: Linux jobName: Build_Arm64_Release @@ -197,7 +199,7 @@ stages: runtimeIdentifier: 'linux-arm64' linuxPortable: true runTests: false - - template: eng/build.yml + - template: eng/build-pr.yml parameters: agentOs: Linux jobName: Build_Linux_musl_Release_arm @@ -208,7 +210,7 @@ stages: additionalBuildParameters: '/p:OSName="linux-musl"' linuxPortable: false runTests: false - - template: eng/build.yml + - template: eng/build-pr.yml parameters: agentOs: Linux jobName: Build_Linux_musl_Release_arm64 @@ -218,7 +220,7 @@ stages: additionalBuildParameters: '/p:OSName="linux-musl"' linuxPortable: false runTests: false - - template: eng/build.yml + - template: eng/build-pr.yml parameters: agentOs: Linux jobName: Build_Linux_musl_Release_x64 @@ -230,7 +232,7 @@ stages: additionalBuildParameters: '/p:HostOSName="linux-musl"' linuxPortable: false runTests: false - - template: eng/build.yml + - template: eng/build-pr.yml parameters: agentOs: Linux jobName: Build_Linux_Portable_Deb_Release_x64 @@ -242,7 +244,7 @@ stages: additionalBuildParameters: '/p:PublishBinariesAndBadge=false /p:BuildSdkDeb=true' linuxPortable: true runTests: false - - template: eng/build.yml + - template: eng/build-pr.yml parameters: agentOs: Linux jobName: Build_Linux_Portable_Rpm_Release_x64 @@ -254,7 +256,7 @@ stages: additionalBuildParameters: '/p:PublishBinariesAndBadge=false /p:IsRPMBasedDistro=true' linuxPortable: true runTests: false - - template: eng/build.yml + - template: eng/build-pr.yml parameters: agentOs: Linux jobName: Build_Linux_Portable_Rpm_Release_Arm64 @@ -267,7 +269,7 @@ stages: additionalBuildParameters: '/p:PublishBinariesAndBadge=false /p:CLIBUILD_SKIP_TESTS=true /p:IsRPMBasedDistro=true' linuxPortable: true runTests: false - - template: eng/build.yml + - template: eng/build-pr.yml parameters: agentOs: Linux jobName: Build_LinuxPortable_Release_x64 @@ -278,14 +280,14 @@ stages: # MacOS - - template: eng/build.yml + - template: eng/build-pr.yml parameters: agentOs: Darwin jobName: Build_Release_x64 buildConfiguration: Release buildArchitecture: x64 runTests: false - - template: eng/build.yml + - template: eng/build-pr.yml parameters: agentOs: Darwin jobName: Build_Release_arm64 @@ -296,7 +298,7 @@ stages: ## Windows PGO Instrumentation builds - - template: eng/build.yml + - template: eng/build-pr.yml parameters: agentOs: Windows_NT pgoInstrument: true @@ -305,7 +307,7 @@ stages: buildArchitecture: x64 additionalBuildParameters: '/p:PublishInternalAsset=true' runTests: false - - template: eng/build.yml + - template: eng/build-pr.yml parameters: agentOs: Windows_NT pgoInstrument: true @@ -313,7 +315,7 @@ stages: buildConfiguration: Release buildArchitecture: x86 runTests: false - - template: eng/build.yml + - template: eng/build-pr.yml parameters: agentOs: Windows_NT pgoInstrument: true @@ -324,7 +326,7 @@ stages: ## Linux PGO Instrumentation builds - - template: eng/build.yml + - template: eng/build-pr.yml parameters: agentOs: Linux pgoInstrument: true @@ -334,7 +336,7 @@ stages: linuxPortable: true runTests: false - - template: eng/build.yml + - template: eng/build-pr.yml parameters: agentOs: Linux pgoInstrument: true diff --git a/eng/build-pr.yml b/eng/build-pr.yml new file mode 100644 index 000000000..21393e242 --- /dev/null +++ b/eng/build-pr.yml @@ -0,0 +1,261 @@ +parameters: + # Agent OS identifier and used as job name +- name: agentOs + type: string + + # Job name +- name: jobName + type: string + +# Container to run the build in, if any +- name: container + type: string + default: '' + + # Job timeout +- name: timeoutInMinutes + type: number + default: 180 + +# Build configuration (Debug, Release) +- name: buildConfiguration + type: string + values: + - Debug + - Release + +# Build architecture +- name: buildArchitecture + type: string + values: + - arm + - arm64 + - x64 + - x86 + +# Linux portable. If true, passes portable switch to build +- name: linuxPortable + type: boolean + default: false + +# Runtime Identifier +- name: runtimeIdentifier + type: string + default: '' + +# UI lang +- name: dotnetCLIUILanguage + type: string + default: '' + +# Additional parameters +- name: additionalBuildParameters + type: string + default: '' + +# Run tests +- name: runTests + type: boolean + default: true + +# PGO instrumentation jobs +- name: pgoInstrument + type: boolean + default: false + +- name: isBuiltFromVmr + displayName: True when build is running from dotnet/dotnet + type: boolean + default: false + +jobs: +- template: common/templates/job/job.yml + parameters: + # Set up the name of the job. + ${{ if parameters.pgoInstrument }}: + name: PGO_${{ parameters.agentOs }}_${{ parameters.jobName }} + ${{ if not(parameters.pgoInstrument) }}: + name: ${{ parameters.agentOs }}_${{ parameters.jobName }} + + ## Set up the pool/machine info to be used based on the Agent OS + ${{ if eq(parameters.agentOs, 'Windows_NT') }}: + enableMicrobuild: true + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals windows.vs2022.amd64.open + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: $(DncEngInternalBuildPool) + demands: ImageOverride -equals windows.vs2022.amd64 + ${{ if eq(parameters.agentOs, 'Linux') }}: + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: $(DncEngInternalBuildPool) + demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 + container: ${{ parameters.container }} + ${{ if eq(parameters.agentOs, 'Darwin') }}: + pool: + vmImage: 'macOS-latest' + + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + ${{ if parameters.isBuiltFromVmr }}: + enableSbom: false + ${{ else }}: + enablePublishBuildAssets: true + enablePublishUsingPipelines: true + enableTelemetry: true + helixRepo: dotnet/installer + workspace: + clean: all + +# Test parameters + variables: + - ${{ if eq(parameters.agentOs, 'Windows_NT') }}: + - _PackArg: '-pack' + - ${{ if parameters.runTests }}: + - _TestArg: '-test' + - ${{ else }}: + - _TestArg: '' + - ${{ if ne(parameters.agentOs, 'Windows_NT') }}: + - _PackArg: '--pack' + - ${{ if parameters.runTests }}: + - _TestArg: '--test' + - ${{ else }}: + - _TestArg: '' + + - ${{ if parameters.pgoInstrument }}: + - _PgoInstrument: '/p:PgoInstrument=true' + - _PackArg: '' + - ${{ else }}: + - _PgoInstrument: '' + + - ${{ if parameters.linuxPortable }}: + - _LinuxPortable: '--linux-portable' + - ${{ else }}: + - _LinuxPortable: '' + + - ${{ if ne(parameters.runtimeIdentifier, '') }}: + - _RuntimeIdentifier: '--runtime-id ${{ parameters.runtimeIdentifier }}' + - ${{ else }}: + - _RuntimeIdentifier: '' + + - _AgentOSName: ${{ parameters.agentOs }} + - _TeamName: Roslyn-Project-System + - _SignType: test + - _BuildArgs: '/p:DotNetSignType=$(_SignType) $(_PgoInstrument)' + + - ${{ if parameters.isBuiltFromVmr }}: + - installerRoot: '$(Build.SourcesDirectory)/src/installer' + - _SignType: test + - _PushToVSFeed: false + - _BuildArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) + /p:TeamName=$(_TeamName) + /p:DotNetPublishUsingPipelines=true + /p:PublishToSymbolServer=false + $(_PgoInstrument) + - ${{ else }}: + - installerRoot: '$(Build.SourcesDirectory)' + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - group: DotNet-HelixApi-Access + - _PushToVSFeed: true + - _SignType: real + - _BuildArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) + /p:DotNetSignType=$(_SignType) + /p:TeamName=$(_TeamName) + /p:DotNetPublishUsingPipelines=$(_PublishUsingPipelines) + $(_PgoInstrument) + + - template: /eng/common/templates/variables/pool-providers.yml + + steps: + - checkout: self + clean: true + - ${{ if eq(parameters.agentOs, 'Windows_NT') }}: + - ${{ if and(not(parameters.isBuiltFromVmr), ne(variables['System.TeamProject'], 'public')) }}: + - task: PowerShell@2 + displayName: Setup Private Feeds Credentials + inputs: + filePath: $(installerRoot)/eng/common/SetupNugetSources.ps1 + arguments: -ConfigFile $(installerRoot)/NuGet.config -Password $Env:Token + env: + Token: $(dn-bot-dnceng-artifact-feeds-rw) + - script: $(installerRoot)/build.cmd + $(_TestArg) $(_PackArg) + -publish -ci -sign + -Configuration ${{ parameters.buildConfiguration }} + -Architecture ${{ parameters.buildArchitecture }} + $(_BuildArgs) + ${{ parameters.additionalBuildParameters }} + $(_InternalRuntimeDownloadArgs) + displayName: Build + env: + DOTNET_CLI_UI_LANGUAGE: ${{ parameters.dotnetCLIUILanguage }} + + - ${{ if ne(parameters.agentOs, 'Windows_NT') }}: + - ${{ if and(not(parameters.isBuiltFromVmr), ne(variables['System.TeamProject'], 'public')) }}: + - task: Bash@3 + displayName: Setup Private Feeds Credentials + inputs: + filePath: $(installerRoot)/eng/common/SetupNugetSources.sh + arguments: $(installerRoot)/NuGet.config $Token + env: + Token: $(dn-bot-dnceng-artifact-feeds-rw) + - ${{ if eq(parameters.agentOs, 'Linux') }}: + - script: $(installerRoot)/build.sh + $(_TestArg) $(_PackArg) + --publish --ci + --noprettyprint + --configuration ${{ parameters.buildConfiguration }} + --architecture ${{ parameters.buildArchitecture }} + $(_LinuxPortable) + $(_RuntimeIdentifier) + $(_BuildArgs) + ${{ parameters.additionalBuildParameters }} + $(_InternalRuntimeDownloadArgs) + displayName: Build + + - ${{ if or(eq(parameters.agentOs, 'Darwin'), eq(parameters.agentOs, 'FreeBSD')) }}: + - script: $(installerRoot)/build.sh + $(_TestArg) + --pack --publish --ci + --noprettyprint + --configuration ${{ parameters.buildConfiguration }} + --architecture ${{ parameters.buildArchitecture }} + $(_RuntimeIdentifier) + $(_BuildArgs) + ${{ parameters.additionalBuildParameters }} + $(_InternalRuntimeDownloadArgs) + displayName: Build + + - task: PublishTestResults@2 + displayName: Publish Test Results + inputs: + testRunner: XUnit + testResultsFiles: 'artifacts/TestResults/${{ parameters.buildConfiguration }}/*.xml' + testRunTitle: '$(_AgentOSName)_$(Agent.JobName)' + platform: '$(BuildPlatform)' + configuration: '${{ parameters.buildConfiguration }}' + condition: ne(variables['_TestArg'], '') + + - task: CopyFiles@2 + displayName: Gather Logs + inputs: + SourceFolder: '$(installerRoot)/artifacts' + Contents: | + log/${{ parameters.buildConfiguration }}/**/* + TestResults/${{ parameters.buildConfiguration }}/**/* + TargetFolder: '$(Build.ArtifactStagingDirectory)' + continueOnError: true + condition: always() + + - task: PublishBuildArtifacts@1 + displayName: Publish Logs to VSTS + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)' + ArtifactName: '$(_AgentOSName)_$(Agent.JobName)_$(Build.BuildNumber)' + publishLocation: Container + continueOnError: true + condition: always() diff --git a/eng/build.yml b/eng/build.yml index 21393e242..fc9e75699 100644 --- a/eng/build.yml +++ b/eng/build.yml @@ -1,9 +1,9 @@ parameters: - # Agent OS identifier and used as job name +# Agent OS identifier and used as job name - name: agentOs type: string - # Job name +# Job name - name: jobName type: string @@ -12,7 +12,7 @@ parameters: type: string default: '' - # Job timeout +# Job timeout - name: timeoutInMinutes type: number default: 180 @@ -32,7 +32,7 @@ parameters: - arm64 - x64 - x86 - + # Linux portable. If true, passes portable switch to build - name: linuxPortable type: boolean @@ -69,7 +69,7 @@ parameters: default: false jobs: -- template: common/templates/job/job.yml +- template: common/templates-official/job/job.yml parameters: # Set up the name of the job. ${{ if parameters.pgoInstrument }}: @@ -77,24 +77,17 @@ jobs: ${{ if not(parameters.pgoInstrument) }}: name: ${{ parameters.agentOs }}_${{ parameters.jobName }} - ## Set up the pool/machine info to be used based on the Agent OS + # Set up the pool/machine info to be used based on the Agent OS ${{ if eq(parameters.agentOs, 'Windows_NT') }}: enableMicrobuild: true pool: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals windows.vs2022.amd64.open - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2022.amd64 + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022-pt + os: windows ${{ if eq(parameters.agentOs, 'Linux') }}: pool: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 + name: $(DncEngInternalBuildPool) + demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 container: ${{ parameters.container }} ${{ if eq(parameters.agentOs, 'Darwin') }}: pool: @@ -111,8 +104,8 @@ jobs: workspace: clean: all -# Test parameters variables: + # Test variables - ${{ if eq(parameters.agentOs, 'Windows_NT') }}: - _PackArg: '-pack' - ${{ if parameters.runTests }}: @@ -158,7 +151,7 @@ jobs: $(_PgoInstrument) - ${{ else }}: - installerRoot: '$(Build.SourcesDirectory)' - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - ${{ if notin(variables['Build.Reason'], 'PullRequest') }}: - group: DotNet-HelixApi-Access - _PushToVSFeed: true - _SignType: real @@ -168,13 +161,13 @@ jobs: /p:DotNetPublishUsingPipelines=$(_PublishUsingPipelines) $(_PgoInstrument) - - template: /eng/common/templates/variables/pool-providers.yml + - template: /eng/common/templates-official/variables/pool-providers.yml steps: - checkout: self clean: true - ${{ if eq(parameters.agentOs, 'Windows_NT') }}: - - ${{ if and(not(parameters.isBuiltFromVmr), ne(variables['System.TeamProject'], 'public')) }}: + - ${{ if not(parameters.isBuiltFromVmr) }}: - task: PowerShell@2 displayName: Setup Private Feeds Credentials inputs: @@ -195,7 +188,7 @@ jobs: DOTNET_CLI_UI_LANGUAGE: ${{ parameters.dotnetCLIUILanguage }} - ${{ if ne(parameters.agentOs, 'Windows_NT') }}: - - ${{ if and(not(parameters.isBuiltFromVmr), ne(variables['System.TeamProject'], 'public')) }}: + - ${{ if not(parameters.isBuiltFromVmr) }}: - task: Bash@3 displayName: Setup Private Feeds Credentials inputs: diff --git a/eng/localization.yml b/eng/localization.yml index c3ee2613b..dfe5efa11 100644 --- a/eng/localization.yml +++ b/eng/localization.yml @@ -1,6 +1,4 @@ -# -# See https://learn.microsoft.com/en-us/azure/devops/pipelines/yaml-schema for details on this file. -# +# Pipeline: https://dnceng.visualstudio.com/internal/_build?definitionId=1224 schedules: # Cron timezone is UTC. @@ -21,7 +19,7 @@ variables: jobs: - ${{ if and(ne(variables['System.TeamProject'], 'public'), or(eq(variables['Build.SourceBranch'], 'refs/heads/main'), eq(variables['Build.Reason'], 'Manual'))) }}: - - template: /eng/common/templates/job/onelocbuild.yml + - template: /eng/common/templates-official/job/onelocbuild.yml parameters: CreatePr: ${{ ne(variables['Build.Reason'], 'Manual') }} LclPackageId: 'LCL-JUNO-PROD-DOTNETINSTALLER' From 3af7365ff349bfa7db5287c2ce5418e0c04f5ca7 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Fri, 8 Mar 2024 12:49:37 -0800 Subject: [PATCH 02/36] Needed to add @self to reference the eng/build.yml properly from the template. --- .vsts-ci.yml | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 7c4acd9d5..28335ee76 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -79,7 +79,7 @@ extends: displayName: Publish Build Config # Windows - - template: eng/build.yml + - template: eng/build.yml@self parameters: agentOs: Windows_NT jobName: Build_Release_x64 @@ -87,14 +87,14 @@ extends: buildArchitecture: x64 additionalBuildParameters: '/p:PublishInternalAsset=true' runTests: false - - template: eng/build.yml + - template: eng/build.yml@self parameters: agentOs: Windows_NT jobName: Build_Release_x86 buildConfiguration: Release buildArchitecture: x86 runTests: false - - template: eng/build.yml + - template: eng/build.yml@self parameters: agentOs: Windows_NT jobName: Build_Release_arm64 @@ -103,7 +103,7 @@ extends: runTests: false # Linux - - template: eng/build.yml + - template: eng/build.yml@self parameters: agentOs: Linux jobName: Build_Arm_Release @@ -112,7 +112,7 @@ extends: runtimeIdentifier: 'linux-arm' linuxPortable: true runTests: false - - template: eng/build.yml + - template: eng/build.yml@self parameters: agentOs: Linux jobName: Build_Arm64_Release @@ -121,7 +121,7 @@ extends: runtimeIdentifier: 'linux-arm64' linuxPortable: true runTests: false - - template: eng/build.yml + - template: eng/build.yml@self parameters: agentOs: Linux jobName: Build_Linux_musl_Release_arm @@ -132,7 +132,7 @@ extends: additionalBuildParameters: '/p:OSName="linux-musl"' linuxPortable: false runTests: false - - template: eng/build.yml + - template: eng/build.yml@self parameters: agentOs: Linux jobName: Build_Linux_musl_Release_arm64 @@ -142,7 +142,7 @@ extends: additionalBuildParameters: '/p:OSName="linux-musl"' linuxPortable: false runTests: false - - template: eng/build.yml + - template: eng/build.yml@self parameters: agentOs: Linux jobName: Build_Linux_musl_Release_x64 @@ -154,7 +154,7 @@ extends: additionalBuildParameters: '/p:HostOSName="linux-musl"' linuxPortable: false runTests: false - - template: eng/build.yml + - template: eng/build.yml@self parameters: agentOs: Linux jobName: Build_Linux_Portable_Deb_Release_x64 @@ -166,7 +166,7 @@ extends: additionalBuildParameters: '/p:PublishBinariesAndBadge=false /p:BuildSdkDeb=true' linuxPortable: true runTests: false - - template: eng/build.yml + - template: eng/build.yml@self parameters: agentOs: Linux jobName: Build_Linux_Portable_Rpm_Release_x64 @@ -178,7 +178,7 @@ extends: additionalBuildParameters: '/p:PublishBinariesAndBadge=false /p:IsRPMBasedDistro=true' linuxPortable: true runTests: false - - template: eng/build.yml + - template: eng/build.yml@self parameters: agentOs: Linux jobName: Build_Linux_Portable_Rpm_Release_Arm64 @@ -191,7 +191,7 @@ extends: additionalBuildParameters: '/p:PublishBinariesAndBadge=false /p:CLIBUILD_SKIP_TESTS=true /p:IsRPMBasedDistro=true' linuxPortable: true runTests: false - - template: eng/build.yml + - template: eng/build.yml@self parameters: agentOs: Linux jobName: Build_LinuxPortable_Release_x64 @@ -201,14 +201,14 @@ extends: runTests: false # MacOS - - template: eng/build.yml + - template: eng/build.yml@self parameters: agentOs: Darwin jobName: Build_Release_x64 buildConfiguration: Release buildArchitecture: x64 runTests: false - - template: eng/build.yml + - template: eng/build.yml@self parameters: agentOs: Darwin jobName: Build_Release_arm64 @@ -218,7 +218,7 @@ extends: runTests: false # Windows PGO Instrumentation - - template: eng/build.yml + - template: eng/build.yml@self parameters: agentOs: Windows_NT pgoInstrument: true @@ -227,7 +227,7 @@ extends: buildArchitecture: x64 additionalBuildParameters: '/p:PublishInternalAsset=true' runTests: false - - template: eng/build.yml + - template: eng/build.yml@self parameters: agentOs: Windows_NT pgoInstrument: true @@ -235,7 +235,7 @@ extends: buildConfiguration: Release buildArchitecture: x86 runTests: false - - template: eng/build.yml + - template: eng/build.yml@self parameters: agentOs: Windows_NT pgoInstrument: true @@ -245,7 +245,7 @@ extends: runTests: false # Linux PGO Instrumentation - - template: eng/build.yml + - template: eng/build.yml@self parameters: agentOs: Linux pgoInstrument: true @@ -254,7 +254,7 @@ extends: buildArchitecture: x64 linuxPortable: true runTests: false - - template: eng/build.yml + - template: eng/build.yml@self parameters: agentOs: Linux pgoInstrument: true From 40ad2d8bcc21e3478491b990b44b2d68a76319f4 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Fri, 8 Mar 2024 12:52:04 -0800 Subject: [PATCH 03/36] Forgot @self for source-build and publish-build-assets templates. --- .vsts-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 28335ee76..97487766a 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -265,13 +265,13 @@ extends: runTests: false # Source Build - - template: /eng/common/templates-official/jobs/source-build.yml + - template: /eng/common/templates-official/jobs/source-build.yml@self - stage: Publish dependsOn: - Build jobs: - - template: /eng/common/templates-official/job/publish-build-assets.yml + - template: /eng/common/templates-official/job/publish-build-assets.yml@self parameters: publishUsingPipelines: true publishAssetsImmediately: true From 50f28e5441f9c462926b87625417e5d0163b215f Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Fri, 8 Mar 2024 13:16:08 -0800 Subject: [PATCH 04/36] Replace publish task with 1ES.PublishPipelineArtifact task. --- .vsts-ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 97487766a..1d2ac43cf 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -74,9 +74,11 @@ extends: name: $(DncEngInternalBuildPool) demands: ImageOverride -equals windows.vs2022preview.amd64 steps: - - publish: $(Build.SourcesDirectory)\eng\buildConfiguration - artifact: buildConfiguration + - task: 1ES.PublishPipelineArtifact@1 displayName: Publish Build Config + inputs: + targetPath: $(Build.SourcesDirectory)\eng\buildConfiguration + artifactName: buildConfiguration # Windows - template: eng/build.yml@self From 50803471c7516afa49103099587ec61b4c3eb030 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Fri, 8 Mar 2024 13:27:50 -0800 Subject: [PATCH 05/36] Replaced PublishBuildArtifacts with 1ES.PublishBuildArtifacts. --- eng/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/build.yml b/eng/build.yml index fc9e75699..8b9886458 100644 --- a/eng/build.yml +++ b/eng/build.yml @@ -244,7 +244,7 @@ jobs: continueOnError: true condition: always() - - task: PublishBuildArtifacts@1 + - task: 1ES.PublishBuildArtifacts@1 displayName: Publish Logs to VSTS inputs: PathtoPublish: '$(Build.ArtifactStagingDirectory)' From 08038a82051d26063f9d83c0ad1a4cdaab8c2709 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Fri, 8 Mar 2024 13:58:01 -0800 Subject: [PATCH 06/36] Internal-only condition cleanup for some variables. Fixed another Windows agent to PT agent. Fixed Ubuntu agent to PT Ubuntu agent. --- .vsts-ci.yml | 20 +++++++------------- eng/build.yml | 3 ++- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 1d2ac43cf..30edd08f0 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -27,19 +27,16 @@ variables: - name: _PublishUsingPipelines value: true - name: _InternalRuntimeDownloadArgs - value: '' + value: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal + /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) + /p:dotnetbuilds-internal-container-read-token-base64=$(dotnetbuilds-internal-container-read-token-base64) # Only used for tracking purposes in MicroBuild tasks. # See: https://devdiv.visualstudio.com/DevDiv/_wiki/wikis/DevDiv.wiki/650/MicroBuild-Signing?anchor=high-level-steps-to-enable-signing - name: TeamName value: DotNetCLI # Variable Groups -- ${{ if eq(variables['System.TeamProject'], 'internal') }}: - - group: DotNetBuilds storage account read tokens - - name: _InternalRuntimeDownloadArgs - value: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal - /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) - /p:dotnetbuilds-internal-container-read-token-base64=$(dotnetbuilds-internal-container-read-token-base64) +- group: DotNetBuilds storage account read tokens # Variable Templates - template: /eng/common/templates-official/variables/pool-providers.yml @@ -67,12 +64,9 @@ extends: # Build Retry Configuration - job: Publish_Build_Configuration pool: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals windows.vs2022preview.amd64.open - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2022preview.amd64 + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022-pt + os: windows steps: - task: 1ES.PublishPipelineArtifact@1 displayName: Publish Build Config diff --git a/eng/build.yml b/eng/build.yml index 8b9886458..4abc6006b 100644 --- a/eng/build.yml +++ b/eng/build.yml @@ -87,7 +87,8 @@ jobs: ${{ if eq(parameters.agentOs, 'Linux') }}: pool: name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 + image: 1es-ubuntu-2204-pt + os: linux container: ${{ parameters.container }} ${{ if eq(parameters.agentOs, 'Darwin') }}: pool: From a215961e7efd6ef439f5036ce6cff9d5cc898cfe Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Fri, 8 Mar 2024 14:08:32 -0800 Subject: [PATCH 07/36] Using Microsoft hosted pool for the macOS agent. --- eng/build.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/eng/build.yml b/eng/build.yml index 4abc6006b..bd0a25cb3 100644 --- a/eng/build.yml +++ b/eng/build.yml @@ -92,7 +92,9 @@ jobs: container: ${{ parameters.container }} ${{ if eq(parameters.agentOs, 'Darwin') }}: pool: - vmImage: 'macOS-latest' + name: Azure Pipelines + image: macOS-latest + os: macOS timeoutInMinutes: ${{ parameters.timeoutInMinutes }} ${{ if parameters.isBuiltFromVmr }}: From d90fa12bf898ffb03448c0769b8efe8a9531a4a0 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Fri, 8 Mar 2024 15:55:16 -0800 Subject: [PATCH 08/36] Trying the 2004 image of Ubuntu instead of the 2204 image. --- eng/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/build.yml b/eng/build.yml index bd0a25cb3..bf7f5db5b 100644 --- a/eng/build.yml +++ b/eng/build.yml @@ -87,7 +87,7 @@ jobs: ${{ if eq(parameters.agentOs, 'Linux') }}: pool: name: $(DncEngInternalBuildPool) - image: 1es-ubuntu-2204-pt + image: 1es-ubuntu-2004-pt os: linux container: ${{ parameters.container }} ${{ if eq(parameters.agentOs, 'Darwin') }}: From 9be953bf81adf6bdb71c7eeb06e77e657f031dac Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Mon, 11 Mar 2024 12:05:11 -0700 Subject: [PATCH 09/36] Temporarily skip component governance. --- .vsts-ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 30edd08f0..ab4dcc52c 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -34,6 +34,9 @@ variables: # See: https://devdiv.visualstudio.com/DevDiv/_wiki/wikis/DevDiv.wiki/650/MicroBuild-Signing?anchor=high-level-steps-to-enable-signing - name: TeamName value: DotNetCLI +# TODO: TEMPORARY +- name: skipComponentGovernanceDetection + value: true # Variable Groups - group: DotNetBuilds storage account read tokens From bcdca559bbf3e91fe6a4a570b2570a6339e1dc17 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Mon, 11 Mar 2024 13:01:47 -0700 Subject: [PATCH 10/36] NuGetPackageRoot already contains a slash in it so removing extra slash. --- src/redist/targets/Crossgen.targets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/redist/targets/Crossgen.targets b/src/redist/targets/Crossgen.targets index 4a996f927..f7a8cd5cd 100644 --- a/src/redist/targets/Crossgen.targets +++ b/src/redist/targets/Crossgen.targets @@ -8,7 +8,7 @@ microsoft.netcore.app.runtime.$(SharedFrameworkRid) microsoft.netcore.app.crossgen2.$(Crossgen2Rid) - $(NuGetPackageRoot)/$(RuntimeNETCrossgenPackageName)/$(MicrosoftNETCoreAppRuntimePackageVersion)/tools/crossgen2$(ExeExtension) + $(NuGetPackageRoot)$(RuntimeNETCrossgenPackageName)/$(MicrosoftNETCoreAppRuntimePackageVersion)/tools/crossgen2$(ExeExtension) $(RedistLayoutPath)shared/$(SharedFrameworkName)/$(MicrosoftNETCoreAppRuntimePackageVersion) From 0fb9c30132f6856b99a76e2b08545473dd6662c1 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Mon, 11 Mar 2024 13:23:42 -0700 Subject: [PATCH 11/36] Trying to disable build failure for component governance. --- .vsts-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index ab4dcc52c..4afb7e45d 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -34,9 +34,6 @@ variables: # See: https://devdiv.visualstudio.com/DevDiv/_wiki/wikis/DevDiv.wiki/650/MicroBuild-Signing?anchor=high-level-steps-to-enable-signing - name: TeamName value: DotNetCLI -# TODO: TEMPORARY -- name: skipComponentGovernanceDetection - value: true # Variable Groups - group: DotNetBuilds storage account read tokens @@ -59,6 +56,9 @@ extends: name: $(DncEngInternalBuildPool) image: 1es-windows-2022-pt os: windows + # TODO: Temporary + componentgovernance: + failOnAlert: false customBuildTags: - ES365AIMigrationTooling stages: From 7439c0c5c620a58d2b11701167fd7ae07d12c220 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Mon, 11 Mar 2024 15:19:58 -0700 Subject: [PATCH 12/36] Trying not -pt agent images. Trying to turn off component governance. --- .vsts-ci.yml | 4 ++++ eng/build.yml | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 4afb7e45d..bec93a388 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -51,14 +51,18 @@ resources: extends: template: azure-pipelines/MicroBuild.1ES.Official.yml@MicroBuildTemplate parameters: + # TODO: Temporary + enableComponentGovernanceScan: false sdl: sourceAnalysisPool: name: $(DncEngInternalBuildPool) image: 1es-windows-2022-pt os: windows # TODO: Temporary + # https://docs.opensource.microsoft.com/tools/cg/component-detection/variables/ componentgovernance: failOnAlert: false + alertWarningLevel: Never customBuildTags: - ES365AIMigrationTooling stages: diff --git a/eng/build.yml b/eng/build.yml index bf7f5db5b..0c436cc66 100644 --- a/eng/build.yml +++ b/eng/build.yml @@ -82,12 +82,12 @@ jobs: enableMicrobuild: true pool: name: $(DncEngInternalBuildPool) - image: 1es-windows-2022-pt + image: 1es-windows-2022 os: windows ${{ if eq(parameters.agentOs, 'Linux') }}: pool: name: $(DncEngInternalBuildPool) - image: 1es-ubuntu-2004-pt + image: 1es-ubuntu-1804 os: linux container: ${{ parameters.container }} ${{ if eq(parameters.agentOs, 'Darwin') }}: From 96cc91c1413afcb6b248946f65cfe9670afca236 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Mon, 11 Mar 2024 15:24:00 -0700 Subject: [PATCH 13/36] Still trying to skip component governance. --- .vsts-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index bec93a388..6753d8ce7 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -52,7 +52,7 @@ extends: template: azure-pipelines/MicroBuild.1ES.Official.yml@MicroBuildTemplate parameters: # TODO: Temporary - enableComponentGovernanceScan: false + skipSDL: true sdl: sourceAnalysisPool: name: $(DncEngInternalBuildPool) From 9c2ff9dc58269e8914eeb75eaddd2ef675adf200 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Mon, 11 Mar 2024 15:34:22 -0700 Subject: [PATCH 14/36] Ignoring the entire .packages directory for component governance. --- .vsts-ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 6753d8ce7..2fca821d5 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -51,8 +51,6 @@ resources: extends: template: azure-pipelines/MicroBuild.1ES.Official.yml@MicroBuildTemplate parameters: - # TODO: Temporary - skipSDL: true sdl: sourceAnalysisPool: name: $(DncEngInternalBuildPool) @@ -63,6 +61,7 @@ extends: componentgovernance: failOnAlert: false alertWarningLevel: Never + ignoreDirectories: '.packages' customBuildTags: - ES365AIMigrationTooling stages: From 4657a65be1bf604a2e5e06fb8ed88977962e6e8c Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Mon, 11 Mar 2024 15:51:48 -0700 Subject: [PATCH 15/36] Go back to -pt images. Define the linux container used differently. --- eng/build.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/eng/build.yml b/eng/build.yml index 0c436cc66..3923fe78d 100644 --- a/eng/build.yml +++ b/eng/build.yml @@ -82,14 +82,16 @@ jobs: enableMicrobuild: true pool: name: $(DncEngInternalBuildPool) - image: 1es-windows-2022 + image: 1es-windows-2022-pt os: windows ${{ if eq(parameters.agentOs, 'Linux') }}: pool: name: $(DncEngInternalBuildPool) - image: 1es-ubuntu-1804 + image: 1es-ubuntu-2204-pt os: linux - container: ${{ parameters.container }} + containers: + default_linux_container: + image: ${{ parameters.container }} ${{ if eq(parameters.agentOs, 'Darwin') }}: pool: name: Azure Pipelines From 4af3022c82848ece77c9f06c3d9df714244db51b Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Mon, 11 Mar 2024 16:05:46 -0700 Subject: [PATCH 16/36] Trying this 1es-mariner-2-pt image for Linux. --- eng/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/build.yml b/eng/build.yml index 3923fe78d..5c7ca6412 100644 --- a/eng/build.yml +++ b/eng/build.yml @@ -87,7 +87,7 @@ jobs: ${{ if eq(parameters.agentOs, 'Linux') }}: pool: name: $(DncEngInternalBuildPool) - image: 1es-ubuntu-2204-pt + image: 1es-mariner-2-pt os: linux containers: default_linux_container: From d159964710bf385ffe379a836cfbf1c83509c0a9 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Tue, 12 Mar 2024 10:51:51 -0700 Subject: [PATCH 17/36] Attempting to use container aliases. Switched back to ubuntu image. --- .vsts-ci.yml | 19 ++++++++++++++----- eng/build.yml | 6 ++---- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 2fca821d5..33be66921 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -47,6 +47,15 @@ resources: type: git name: 1ESPipelineTemplates/MicroBuildTemplate ref: refs/tags/release + containers: + - container: MuslArm + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-arm-alpine + - container: MuslX64 + image: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.19-WithNode + - container: PortableDeb + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-debpkg + - container: PortableRpm + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-fpm extends: template: azure-pipelines/MicroBuild.1ES.Official.yml@MicroBuildTemplate @@ -127,7 +136,7 @@ extends: parameters: agentOs: Linux jobName: Build_Linux_musl_Release_arm - container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-arm-alpine' + container: MuslArm buildConfiguration: Release buildArchitecture: arm runtimeIdentifier: 'linux-musl-arm' @@ -148,7 +157,7 @@ extends: parameters: agentOs: Linux jobName: Build_Linux_musl_Release_x64 - container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.19-WithNode' + container: MuslX64 buildConfiguration: Release buildArchitecture: x64 runtimeIdentifier: 'linux-musl-x64' @@ -160,7 +169,7 @@ extends: parameters: agentOs: Linux jobName: Build_Linux_Portable_Deb_Release_x64 - container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-debpkg' + container: PortableDeb buildConfiguration: Release buildArchitecture: x64 # Do not publish zips and tarballs. The linux-x64 binaries are @@ -172,7 +181,7 @@ extends: parameters: agentOs: Linux jobName: Build_Linux_Portable_Rpm_Release_x64 - container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-fpm' + container: PortableRpm buildConfiguration: Release buildArchitecture: x64 # Do not publish zips and tarballs. The linux-x64 binaries are @@ -184,7 +193,7 @@ extends: parameters: agentOs: Linux jobName: Build_Linux_Portable_Rpm_Release_Arm64 - container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-fpm' + container: PortableRpm buildConfiguration: Release buildArchitecture: arm64 runtimeIdentifier: 'linux-arm64' diff --git a/eng/build.yml b/eng/build.yml index 5c7ca6412..bd0a25cb3 100644 --- a/eng/build.yml +++ b/eng/build.yml @@ -87,11 +87,9 @@ jobs: ${{ if eq(parameters.agentOs, 'Linux') }}: pool: name: $(DncEngInternalBuildPool) - image: 1es-mariner-2-pt + image: 1es-ubuntu-2204-pt os: linux - containers: - default_linux_container: - image: ${{ parameters.container }} + container: ${{ parameters.container }} ${{ if eq(parameters.agentOs, 'Darwin') }}: pool: name: Azure Pipelines From 9dbffb5ec3d6bfcedb3fd66b94e16c5092852273 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Tue, 12 Mar 2024 11:20:55 -0700 Subject: [PATCH 18/36] Trying to pass the containers directly to the template. --- .vsts-ci.yml | 1 + eng/build.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 33be66921..91ccbf4a1 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -60,6 +60,7 @@ resources: extends: template: azure-pipelines/MicroBuild.1ES.Official.yml@MicroBuildTemplate parameters: + containers: ${{ resources.containers }} sdl: sourceAnalysisPool: name: $(DncEngInternalBuildPool) diff --git a/eng/build.yml b/eng/build.yml index bd0a25cb3..70c0d31f3 100644 --- a/eng/build.yml +++ b/eng/build.yml @@ -76,7 +76,7 @@ jobs: name: PGO_${{ parameters.agentOs }}_${{ parameters.jobName }} ${{ if not(parameters.pgoInstrument) }}: name: ${{ parameters.agentOs }}_${{ parameters.jobName }} - + # Set up the pool/machine info to be used based on the Agent OS ${{ if eq(parameters.agentOs, 'Windows_NT') }}: enableMicrobuild: true From 62ea0e628b96c96ee48aeaf1066914473fa73b80 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Tue, 12 Mar 2024 11:22:07 -0700 Subject: [PATCH 19/36] Redefining the containers node. --- .vsts-ci.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 91ccbf4a1..785366659 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -60,7 +60,15 @@ resources: extends: template: azure-pipelines/MicroBuild.1ES.Official.yml@MicroBuildTemplate parameters: - containers: ${{ resources.containers }} + containers: + - container: MuslArm + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-arm-alpine + - container: MuslX64 + image: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.19-WithNode + - container: PortableDeb + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-debpkg + - container: PortableRpm + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-fpm sdl: sourceAnalysisPool: name: $(DncEngInternalBuildPool) From d4014440fd92c325fc86aca3d958dc6c6ef47eb3 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Tue, 12 Mar 2024 11:24:45 -0700 Subject: [PATCH 20/36] Removed the containers node since it isn't valid. --- .vsts-ci.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 785366659..33be66921 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -60,15 +60,6 @@ resources: extends: template: azure-pipelines/MicroBuild.1ES.Official.yml@MicroBuildTemplate parameters: - containers: - - container: MuslArm - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-arm-alpine - - container: MuslX64 - image: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.19-WithNode - - container: PortableDeb - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-debpkg - - container: PortableRpm - image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-fpm sdl: sourceAnalysisPool: name: $(DncEngInternalBuildPool) From cf782740933415b39ca04c22e9d1831760630630 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Tue, 12 Mar 2024 13:29:59 -0700 Subject: [PATCH 21/36] Attempt to use 1ES template instead of MicroBuild. --- .vsts-ci.yml | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 33be66921..c661922cd 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -47,19 +47,32 @@ resources: type: git name: 1ESPipelineTemplates/MicroBuildTemplate ref: refs/tags/release - containers: - - container: MuslArm - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-arm-alpine - - container: MuslX64 - image: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.19-WithNode - - container: PortableDeb - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-debpkg - - container: PortableRpm - image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-fpm + - repository: 1esPipelines + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + # containers: + # - container: MuslArm + # image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-arm-alpine + # - container: MuslX64 + # image: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.19-WithNode + # - container: PortableDeb + # image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-debpkg + # - container: PortableRpm + # image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-fpm extends: - template: azure-pipelines/MicroBuild.1ES.Official.yml@MicroBuildTemplate + template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines parameters: + containers: + - container: MuslArm + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-arm-alpine + - container: MuslX64 + image: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.19-WithNode + - container: PortableDeb + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-debpkg + - container: PortableRpm + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-fpm sdl: sourceAnalysisPool: name: $(DncEngInternalBuildPool) From cf03de8cf8249793d661f5c8b5d912d304f9d42b Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Tue, 12 Mar 2024 14:28:07 -0700 Subject: [PATCH 22/36] Back to MicroBuild templates and manual container assignment by name directly without aliases. --- .vsts-ci.yml | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index c661922cd..611df5379 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -62,17 +62,17 @@ resources: # image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-fpm extends: - template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines + template: azure-pipelines/MicroBuild.1ES.Official.yml@MicroBuildTemplate parameters: - containers: - - container: MuslArm - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-arm-alpine - - container: MuslX64 - image: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.19-WithNode - - container: PortableDeb - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-debpkg - - container: PortableRpm - image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-fpm + # containers: + # - container: MuslArm + # image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-arm-alpine + # - container: MuslX64 + # image: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.19-WithNode + # - container: PortableDeb + # image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-debpkg + # - container: PortableRpm + # image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-fpm sdl: sourceAnalysisPool: name: $(DncEngInternalBuildPool) @@ -149,7 +149,8 @@ extends: parameters: agentOs: Linux jobName: Build_Linux_musl_Release_arm - container: MuslArm + # container: MuslArm + container: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-arm-alpine buildConfiguration: Release buildArchitecture: arm runtimeIdentifier: 'linux-musl-arm' @@ -170,7 +171,8 @@ extends: parameters: agentOs: Linux jobName: Build_Linux_musl_Release_x64 - container: MuslX64 + # container: MuslX64 + container: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.19-WithNode buildConfiguration: Release buildArchitecture: x64 runtimeIdentifier: 'linux-musl-x64' @@ -182,7 +184,8 @@ extends: parameters: agentOs: Linux jobName: Build_Linux_Portable_Deb_Release_x64 - container: PortableDeb + # container: PortableDeb + container: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-debpkg buildConfiguration: Release buildArchitecture: x64 # Do not publish zips and tarballs. The linux-x64 binaries are @@ -194,7 +197,8 @@ extends: parameters: agentOs: Linux jobName: Build_Linux_Portable_Rpm_Release_x64 - container: PortableRpm + # container: PortableRpm + container: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-fpm buildConfiguration: Release buildArchitecture: x64 # Do not publish zips and tarballs. The linux-x64 binaries are @@ -206,7 +210,8 @@ extends: parameters: agentOs: Linux jobName: Build_Linux_Portable_Rpm_Release_Arm64 - container: PortableRpm + # container: PortableRpm + container: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-fpm buildConfiguration: Release buildArchitecture: arm64 runtimeIdentifier: 'linux-arm64' From 2aef651aafd0a71be914e2860735158826e935df Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Tue, 12 Mar 2024 14:29:25 -0700 Subject: [PATCH 23/36] Opps, left duplicate repository definition. --- .vsts-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 611df5379..b585acce2 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -47,10 +47,10 @@ resources: type: git name: 1ESPipelineTemplates/MicroBuildTemplate ref: refs/tags/release - - repository: 1esPipelines - type: git - name: 1ESPipelineTemplates/1ESPipelineTemplates - ref: refs/tags/release + # - repository: 1esPipelines + # type: git + # name: 1ESPipelineTemplates/1ESPipelineTemplates + # ref: refs/tags/release # containers: # - container: MuslArm # image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-arm-alpine From aad30a266df043874a632d49d254b159aca3635e Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Wed, 13 Mar 2024 08:40:38 -0700 Subject: [PATCH 24/36] Using 1ES template but defining the containers properly. --- .vsts-ci.yml | 40 +++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index b585acce2..889faa993 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -47,10 +47,10 @@ resources: type: git name: 1ESPipelineTemplates/MicroBuildTemplate ref: refs/tags/release - # - repository: 1esPipelines - # type: git - # name: 1ESPipelineTemplates/1ESPipelineTemplates - # ref: refs/tags/release + - repository: 1esPipelines + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release # containers: # - container: MuslArm # image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-arm-alpine @@ -62,7 +62,8 @@ resources: # image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-fpm extends: - template: azure-pipelines/MicroBuild.1ES.Official.yml@MicroBuildTemplate + # template: azure-pipelines/MicroBuild.1ES.Official.yml@MicroBuildTemplate + template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines parameters: # containers: # - container: MuslArm @@ -73,6 +74,15 @@ extends: # image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-debpkg # - container: PortableRpm # image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-fpm + containers: + MuslArm: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-arm-alpine + MuslX64: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.19-WithNode + PortableDeb: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-debpkg + PortableRpm: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-fpm sdl: sourceAnalysisPool: name: $(DncEngInternalBuildPool) @@ -149,8 +159,8 @@ extends: parameters: agentOs: Linux jobName: Build_Linux_musl_Release_arm - # container: MuslArm - container: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-arm-alpine + container: MuslArm + # container: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-arm-alpine buildConfiguration: Release buildArchitecture: arm runtimeIdentifier: 'linux-musl-arm' @@ -171,8 +181,8 @@ extends: parameters: agentOs: Linux jobName: Build_Linux_musl_Release_x64 - # container: MuslX64 - container: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.19-WithNode + container: MuslX64 + # container: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.19-WithNode buildConfiguration: Release buildArchitecture: x64 runtimeIdentifier: 'linux-musl-x64' @@ -184,8 +194,8 @@ extends: parameters: agentOs: Linux jobName: Build_Linux_Portable_Deb_Release_x64 - # container: PortableDeb - container: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-debpkg + container: PortableDeb + # container: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-debpkg buildConfiguration: Release buildArchitecture: x64 # Do not publish zips and tarballs. The linux-x64 binaries are @@ -197,8 +207,8 @@ extends: parameters: agentOs: Linux jobName: Build_Linux_Portable_Rpm_Release_x64 - # container: PortableRpm - container: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-fpm + container: PortableRpm + # container: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-fpm buildConfiguration: Release buildArchitecture: x64 # Do not publish zips and tarballs. The linux-x64 binaries are @@ -210,8 +220,8 @@ extends: parameters: agentOs: Linux jobName: Build_Linux_Portable_Rpm_Release_Arm64 - # container: PortableRpm - container: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-fpm + container: PortableRpm + # container: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-fpm buildConfiguration: Release buildArchitecture: arm64 runtimeIdentifier: 'linux-arm64' From 6c90ec94e58d037f2acc86010b46689e4600b54f Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Wed, 13 Mar 2024 09:03:52 -0700 Subject: [PATCH 25/36] Cleaning up commented code. Sticking with the 1ES templates instead of MicroBuild. Trying only ignoring .packages for component governance. --- .vsts-ci.yml | 32 ++------------------------------ 1 file changed, 2 insertions(+), 30 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 889faa993..78f55f83e 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -43,37 +43,14 @@ variables: resources: repositories: - - repository: MicroBuildTemplate - type: git - name: 1ESPipelineTemplates/MicroBuildTemplate - ref: refs/tags/release - repository: 1esPipelines type: git name: 1ESPipelineTemplates/1ESPipelineTemplates ref: refs/tags/release - # containers: - # - container: MuslArm - # image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-arm-alpine - # - container: MuslX64 - # image: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.19-WithNode - # - container: PortableDeb - # image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-debpkg - # - container: PortableRpm - # image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-fpm extends: - # template: azure-pipelines/MicroBuild.1ES.Official.yml@MicroBuildTemplate template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines parameters: - # containers: - # - container: MuslArm - # image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-arm-alpine - # - container: MuslX64 - # image: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.19-WithNode - # - container: PortableDeb - # image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-debpkg - # - container: PortableRpm - # image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-fpm containers: MuslArm: image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-arm-alpine @@ -91,8 +68,8 @@ extends: # TODO: Temporary # https://docs.opensource.microsoft.com/tools/cg/component-detection/variables/ componentgovernance: - failOnAlert: false - alertWarningLevel: Never + # failOnAlert: false + # alertWarningLevel: Never ignoreDirectories: '.packages' customBuildTags: - ES365AIMigrationTooling @@ -160,7 +137,6 @@ extends: agentOs: Linux jobName: Build_Linux_musl_Release_arm container: MuslArm - # container: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-arm-alpine buildConfiguration: Release buildArchitecture: arm runtimeIdentifier: 'linux-musl-arm' @@ -182,7 +158,6 @@ extends: agentOs: Linux jobName: Build_Linux_musl_Release_x64 container: MuslX64 - # container: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.19-WithNode buildConfiguration: Release buildArchitecture: x64 runtimeIdentifier: 'linux-musl-x64' @@ -195,7 +170,6 @@ extends: agentOs: Linux jobName: Build_Linux_Portable_Deb_Release_x64 container: PortableDeb - # container: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-debpkg buildConfiguration: Release buildArchitecture: x64 # Do not publish zips and tarballs. The linux-x64 binaries are @@ -208,7 +182,6 @@ extends: agentOs: Linux jobName: Build_Linux_Portable_Rpm_Release_x64 container: PortableRpm - # container: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-fpm buildConfiguration: Release buildArchitecture: x64 # Do not publish zips and tarballs. The linux-x64 binaries are @@ -221,7 +194,6 @@ extends: agentOs: Linux jobName: Build_Linux_Portable_Rpm_Release_Arm64 container: PortableRpm - # container: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-fpm buildConfiguration: Release buildArchitecture: arm64 runtimeIdentifier: 'linux-arm64' From d1a64e52d7cd621a710752308b6d9204fc2cd51d Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Wed, 13 Mar 2024 10:20:33 -0700 Subject: [PATCH 26/36] Keeping original PR if expression without internal check. Seeing if TeamName is no longer needed since we're not using MicroBuild templates. --- .vsts-ci.yml | 394 ++++++++++++++++++++++++++------------------------- 1 file changed, 198 insertions(+), 196 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 78f55f83e..af4922444 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -32,8 +32,8 @@ variables: /p:dotnetbuilds-internal-container-read-token-base64=$(dotnetbuilds-internal-container-read-token-base64) # Only used for tracking purposes in MicroBuild tasks. # See: https://devdiv.visualstudio.com/DevDiv/_wiki/wikis/DevDiv.wiki/650/MicroBuild-Signing?anchor=high-level-steps-to-enable-signing -- name: TeamName - value: DotNetCLI +# - name: TeamName +# value: DotNetCLI # Variable Groups - group: DotNetBuilds storage account read tokens @@ -89,204 +89,206 @@ extends: targetPath: $(Build.SourcesDirectory)\eng\buildConfiguration artifactName: buildConfiguration - # Windows - - template: eng/build.yml@self - parameters: - agentOs: Windows_NT - jobName: Build_Release_x64 - buildConfiguration: Release - buildArchitecture: x64 - additionalBuildParameters: '/p:PublishInternalAsset=true' - runTests: false - - template: eng/build.yml@self - parameters: - agentOs: Windows_NT - jobName: Build_Release_x86 - buildConfiguration: Release - buildArchitecture: x86 - runTests: false - - template: eng/build.yml@self - parameters: - agentOs: Windows_NT - jobName: Build_Release_arm64 - buildConfiguration: Release - buildArchitecture: arm64 - runTests: false + - ${{ if notin(variables['Build.Reason'], 'PullRequest') }}: + # Windows + - template: eng/build.yml@self + parameters: + agentOs: Windows_NT + jobName: Build_Release_x64 + buildConfiguration: Release + buildArchitecture: x64 + additionalBuildParameters: '/p:PublishInternalAsset=true' + runTests: false + - template: eng/build.yml@self + parameters: + agentOs: Windows_NT + jobName: Build_Release_x86 + buildConfiguration: Release + buildArchitecture: x86 + runTests: false + - template: eng/build.yml@self + parameters: + agentOs: Windows_NT + jobName: Build_Release_arm64 + buildConfiguration: Release + buildArchitecture: arm64 + runTests: false - # Linux - - template: eng/build.yml@self - parameters: - agentOs: Linux - jobName: Build_Arm_Release - buildConfiguration: Release - buildArchitecture: arm - runtimeIdentifier: 'linux-arm' - linuxPortable: true - runTests: false - - template: eng/build.yml@self - parameters: - agentOs: Linux - jobName: Build_Arm64_Release - buildConfiguration: Release - buildArchitecture: arm64 - runtimeIdentifier: 'linux-arm64' - linuxPortable: true - runTests: false - - template: eng/build.yml@self - parameters: - agentOs: Linux - jobName: Build_Linux_musl_Release_arm - container: MuslArm - buildConfiguration: Release - buildArchitecture: arm - runtimeIdentifier: 'linux-musl-arm' - additionalBuildParameters: '/p:OSName="linux-musl"' - linuxPortable: false - runTests: false - - template: eng/build.yml@self - parameters: - agentOs: Linux - jobName: Build_Linux_musl_Release_arm64 - buildConfiguration: Release - buildArchitecture: arm64 - runtimeIdentifier: 'linux-musl-arm64' - additionalBuildParameters: '/p:OSName="linux-musl"' - linuxPortable: false - runTests: false - - template: eng/build.yml@self - parameters: - agentOs: Linux - jobName: Build_Linux_musl_Release_x64 - container: MuslX64 - buildConfiguration: Release - buildArchitecture: x64 - runtimeIdentifier: 'linux-musl-x64' - # Pass in HostOSName when running on alpine - additionalBuildParameters: '/p:HostOSName="linux-musl"' - linuxPortable: false - runTests: false - - template: eng/build.yml@self - parameters: - agentOs: Linux - jobName: Build_Linux_Portable_Deb_Release_x64 - container: PortableDeb - buildConfiguration: Release - buildArchitecture: x64 - # Do not publish zips and tarballs. The linux-x64 binaries are - # already published by Build_LinuxPortable_Release_x64 - additionalBuildParameters: '/p:PublishBinariesAndBadge=false /p:BuildSdkDeb=true' - linuxPortable: true - runTests: false - - template: eng/build.yml@self - parameters: - agentOs: Linux - jobName: Build_Linux_Portable_Rpm_Release_x64 - container: PortableRpm - buildConfiguration: Release - buildArchitecture: x64 - # Do not publish zips and tarballs. The linux-x64 binaries are - # already published by Build_LinuxPortable_Release_x64 - additionalBuildParameters: '/p:PublishBinariesAndBadge=false /p:IsRPMBasedDistro=true' - linuxPortable: true - runTests: false - - template: eng/build.yml@self - parameters: - agentOs: Linux - jobName: Build_Linux_Portable_Rpm_Release_Arm64 - container: PortableRpm - buildConfiguration: Release - buildArchitecture: arm64 - runtimeIdentifier: 'linux-arm64' - # Do not publish zips and tarballs. The linux-x64 binaries are - # already published by Build_LinuxPortable_Release_x64 - additionalBuildParameters: '/p:PublishBinariesAndBadge=false /p:CLIBUILD_SKIP_TESTS=true /p:IsRPMBasedDistro=true' - linuxPortable: true - runTests: false - - template: eng/build.yml@self - parameters: - agentOs: Linux - jobName: Build_LinuxPortable_Release_x64 - buildConfiguration: Release - buildArchitecture: x64 - linuxPortable: true - runTests: false + # Linux + - template: eng/build.yml@self + parameters: + agentOs: Linux + jobName: Build_Arm_Release + buildConfiguration: Release + buildArchitecture: arm + runtimeIdentifier: 'linux-arm' + linuxPortable: true + runTests: false + - template: eng/build.yml@self + parameters: + agentOs: Linux + jobName: Build_Arm64_Release + buildConfiguration: Release + buildArchitecture: arm64 + runtimeIdentifier: 'linux-arm64' + linuxPortable: true + runTests: false + - template: eng/build.yml@self + parameters: + agentOs: Linux + jobName: Build_Linux_musl_Release_arm + container: MuslArm + buildConfiguration: Release + buildArchitecture: arm + runtimeIdentifier: 'linux-musl-arm' + additionalBuildParameters: '/p:OSName="linux-musl"' + linuxPortable: false + runTests: false + - template: eng/build.yml@self + parameters: + agentOs: Linux + jobName: Build_Linux_musl_Release_arm64 + buildConfiguration: Release + buildArchitecture: arm64 + runtimeIdentifier: 'linux-musl-arm64' + additionalBuildParameters: '/p:OSName="linux-musl"' + linuxPortable: false + runTests: false + - template: eng/build.yml@self + parameters: + agentOs: Linux + jobName: Build_Linux_musl_Release_x64 + container: MuslX64 + buildConfiguration: Release + buildArchitecture: x64 + runtimeIdentifier: 'linux-musl-x64' + # Pass in HostOSName when running on alpine + additionalBuildParameters: '/p:HostOSName="linux-musl"' + linuxPortable: false + runTests: false + - template: eng/build.yml@self + parameters: + agentOs: Linux + jobName: Build_Linux_Portable_Deb_Release_x64 + container: PortableDeb + buildConfiguration: Release + buildArchitecture: x64 + # Do not publish zips and tarballs. The linux-x64 binaries are + # already published by Build_LinuxPortable_Release_x64 + additionalBuildParameters: '/p:PublishBinariesAndBadge=false /p:BuildSdkDeb=true' + linuxPortable: true + runTests: false + - template: eng/build.yml@self + parameters: + agentOs: Linux + jobName: Build_Linux_Portable_Rpm_Release_x64 + container: PortableRpm + buildConfiguration: Release + buildArchitecture: x64 + # Do not publish zips and tarballs. The linux-x64 binaries are + # already published by Build_LinuxPortable_Release_x64 + additionalBuildParameters: '/p:PublishBinariesAndBadge=false /p:IsRPMBasedDistro=true' + linuxPortable: true + runTests: false + - template: eng/build.yml@self + parameters: + agentOs: Linux + jobName: Build_Linux_Portable_Rpm_Release_Arm64 + container: PortableRpm + buildConfiguration: Release + buildArchitecture: arm64 + runtimeIdentifier: 'linux-arm64' + # Do not publish zips and tarballs. The linux-x64 binaries are + # already published by Build_LinuxPortable_Release_x64 + additionalBuildParameters: '/p:PublishBinariesAndBadge=false /p:CLIBUILD_SKIP_TESTS=true /p:IsRPMBasedDistro=true' + linuxPortable: true + runTests: false + - template: eng/build.yml@self + parameters: + agentOs: Linux + jobName: Build_LinuxPortable_Release_x64 + buildConfiguration: Release + buildArchitecture: x64 + linuxPortable: true + runTests: false - # MacOS - - template: eng/build.yml@self - parameters: - agentOs: Darwin - jobName: Build_Release_x64 - buildConfiguration: Release - buildArchitecture: x64 - runTests: false - - template: eng/build.yml@self - parameters: - agentOs: Darwin - jobName: Build_Release_arm64 - runtimeIdentifier: 'osx-arm64' - buildConfiguration: Release - buildArchitecture: arm64 - runTests: false + # MacOS + - template: eng/build.yml@self + parameters: + agentOs: Darwin + jobName: Build_Release_x64 + buildConfiguration: Release + buildArchitecture: x64 + runTests: false + - template: eng/build.yml@self + parameters: + agentOs: Darwin + jobName: Build_Release_arm64 + runtimeIdentifier: 'osx-arm64' + buildConfiguration: Release + buildArchitecture: arm64 + runTests: false - # Windows PGO Instrumentation - - template: eng/build.yml@self - parameters: - agentOs: Windows_NT - pgoInstrument: true - jobName: Build_Release_x64 - buildConfiguration: Release - buildArchitecture: x64 - additionalBuildParameters: '/p:PublishInternalAsset=true' - runTests: false - - template: eng/build.yml@self - parameters: - agentOs: Windows_NT - pgoInstrument: true - jobName: Build_Release_x86 - buildConfiguration: Release - buildArchitecture: x86 - runTests: false - - template: eng/build.yml@self - parameters: - agentOs: Windows_NT - pgoInstrument: true - jobName: Build_Release_arm64 - buildConfiguration: Release - buildArchitecture: arm64 - runTests: false + # Windows PGO Instrumentation + - template: eng/build.yml@self + parameters: + agentOs: Windows_NT + pgoInstrument: true + jobName: Build_Release_x64 + buildConfiguration: Release + buildArchitecture: x64 + additionalBuildParameters: '/p:PublishInternalAsset=true' + runTests: false + - template: eng/build.yml@self + parameters: + agentOs: Windows_NT + pgoInstrument: true + jobName: Build_Release_x86 + buildConfiguration: Release + buildArchitecture: x86 + runTests: false + - template: eng/build.yml@self + parameters: + agentOs: Windows_NT + pgoInstrument: true + jobName: Build_Release_arm64 + buildConfiguration: Release + buildArchitecture: arm64 + runTests: false - # Linux PGO Instrumentation - - template: eng/build.yml@self - parameters: - agentOs: Linux - pgoInstrument: true - jobName: Build_LinuxPortable_Release_x64 - buildConfiguration: Release - buildArchitecture: x64 - linuxPortable: true - runTests: false - - template: eng/build.yml@self - parameters: - agentOs: Linux - pgoInstrument: true - jobName: Build_Release_arm64 - buildConfiguration: Release - buildArchitecture: arm64 - linuxPortable: true - runTests: false + # Linux PGO Instrumentation + - template: eng/build.yml@self + parameters: + agentOs: Linux + pgoInstrument: true + jobName: Build_LinuxPortable_Release_x64 + buildConfiguration: Release + buildArchitecture: x64 + linuxPortable: true + runTests: false + - template: eng/build.yml@self + parameters: + agentOs: Linux + pgoInstrument: true + jobName: Build_Release_arm64 + buildConfiguration: Release + buildArchitecture: arm64 + linuxPortable: true + runTests: false - # Source Build - - template: /eng/common/templates-official/jobs/source-build.yml@self + # Source Build + - template: /eng/common/templates-official/jobs/source-build.yml@self - - stage: Publish - dependsOn: - - Build - jobs: - - template: /eng/common/templates-official/job/publish-build-assets.yml@self - parameters: - publishUsingPipelines: true - publishAssetsImmediately: true - pool: - ${{ if eq(variables['System.TeamProject'], 'internal') }}: + - ${{ if notin(variables['Build.Reason'], 'PullRequest') }}: + - stage: Publish + dependsOn: + - Build + jobs: + - template: /eng/common/templates-official/job/publish-build-assets.yml@self + parameters: + publishUsingPipelines: true + publishAssetsImmediately: true + pool: name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2022.amd64 + image: 1es-windows-2022-pt + os: windows From 9504adc9da28ccf36270ff4830bd9107af545f6d Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Wed, 13 Mar 2024 11:46:44 -0700 Subject: [PATCH 27/36] Re-added PR-specific logic since this pipeline could be used for internal PRs. Reverted adjustments specific to public logic statements. Removed commented code. --- .vsts-ci.yml | 139 ++++++++++++++++++++++++++++++++++++++++---------- eng/build.yml | 6 +-- 2 files changed, 116 insertions(+), 29 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index af4922444..005dc0803 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -10,7 +10,6 @@ trigger: - internal/release/* variables: -# Loose Variables - name: _PublishUsingPipelines value: false - ${{ if or(startswith(variables['Build.SourceBranch'], 'refs/heads/release/'), startswith(variables['Build.SourceBranch'], 'refs/heads/internal/release/'), eq(variables['Build.Reason'], 'Manual')) }}: @@ -26,19 +25,11 @@ variables: - group: DotNet-Installer-SDLValidation-Params - name: _PublishUsingPipelines value: true +- group: DotNetBuilds storage account read tokens - name: _InternalRuntimeDownloadArgs value: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) /p:dotnetbuilds-internal-container-read-token-base64=$(dotnetbuilds-internal-container-read-token-base64) -# Only used for tracking purposes in MicroBuild tasks. -# See: https://devdiv.visualstudio.com/DevDiv/_wiki/wikis/DevDiv.wiki/650/MicroBuild-Signing?anchor=high-level-steps-to-enable-signing -# - name: TeamName -# value: DotNetCLI - -# Variable Groups -- group: DotNetBuilds storage account read tokens - -# Variable Templates - template: /eng/common/templates-official/variables/pool-providers.yml resources: @@ -52,24 +43,29 @@ extends: template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines parameters: containers: - MuslArm: - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-arm-alpine - MuslX64: + alpine319WithNode: image: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.19-WithNode - PortableDeb: - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-debpkg - PortableRpm: + cblMariner20Fpm: image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-fpm + centosStream8: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8 + debian11Amd64: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-amd64 + fedora39: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-39 + ubuntu2204: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04 + ubuntu2204CrossArmAlpine: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-arm-alpine + ubuntu2204DebPkg: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-debpkg sdl: sourceAnalysisPool: name: $(DncEngInternalBuildPool) image: 1es-windows-2022-pt os: windows - # TODO: Temporary # https://docs.opensource.microsoft.com/tools/cg/component-detection/variables/ componentgovernance: - # failOnAlert: false - # alertWarningLevel: Never ignoreDirectories: '.packages' customBuildTags: - ES365AIMigrationTooling @@ -89,7 +85,98 @@ extends: targetPath: $(Build.SourcesDirectory)\eng\buildConfiguration artifactName: buildConfiguration - - ${{ if notin(variables['Build.Reason'], 'PullRequest') }}: + # PR-only jobs + - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: + # Windows + - template: eng/build.yml@self + parameters: + agentOs: Windows_NT + jobName: Build_Debug_x64 + buildConfiguration: Debug + buildArchitecture: x64 + additionalBuildParameters: '/p:PublishInternalAsset=true' + runTests: true + + # Linux + - template: eng/build.yml@self + parameters: + agentOs: Linux + jobName: Build_Ubuntu_22_04_Debug_x64 + container: ubuntu2204 + buildConfiguration: Debug + buildArchitecture: x64 + linuxPortable: true + runTests: true + - template: eng/build.yml@self + parameters: + agentOs: Linux + jobName: Build_Fedora_39_Debug_x64 + container: fedora39 + buildConfiguration: Debug + buildArchitecture: x64 + linuxPortable: true + runTests: true + - template: eng/build.yml@self + parameters: + agentOs: Linux + jobName: Build_CentOS_8_Stream_Debug_x64 + container: centosStream8 + buildConfiguration: Debug + buildArchitecture: x64 + linuxPortable: false + runTests: true + - template: eng/build.yml@self + parameters: + agentOs: Linux + jobName: Build_Debian_11_Debug_x64 + container: debian11Amd64 + buildConfiguration: Debug + buildArchitecture: x64 + additionalBuildParameters: '/p:BuildSdkDeb=true' + linuxPortable: false + runTests: true + - template: eng/build.yml@self + parameters: + agentOs: Linux + jobName: Build_Arm64_Debug + buildConfiguration: Debug + buildArchitecture: arm64 + runtimeIdentifier: 'linux-arm64' + linuxPortable: true + # Never run tests on arm64 + runTests: false + - template: eng/build.yml@self + parameters: + agentOs: Linux + jobName: Build_Linux_musl_Debug_x64 + container: alpine319WithNode + buildConfiguration: Debug + buildArchitecture: x64 + runtimeIdentifier: 'linux-musl-x64' + # Pass in HostOSName when running on alpine + additionalBuildParameters: '/p:HostOSName="linux-musl"' + linuxPortable: false + runTests: true + - template: eng/build.yml@self + parameters: + agentOs: Linux + jobName: Build_LinuxPortable_Release_x64 + buildConfiguration: Release + buildArchitecture: x64 + linuxPortable: true + runTests: true + + # MacOS + - template: eng/build.yml@self + parameters: + agentOs: Darwin + jobName: Build_Release_x64 + buildConfiguration: Release + buildArchitecture: x64 + runTests: true + + # Official/PGO instrumentation Builds + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: # Windows - template: eng/build.yml@self parameters: @@ -137,7 +224,7 @@ extends: parameters: agentOs: Linux jobName: Build_Linux_musl_Release_arm - container: MuslArm + container: ubuntu2204CrossArmAlpine buildConfiguration: Release buildArchitecture: arm runtimeIdentifier: 'linux-musl-arm' @@ -158,7 +245,7 @@ extends: parameters: agentOs: Linux jobName: Build_Linux_musl_Release_x64 - container: MuslX64 + container: alpine319WithNode buildConfiguration: Release buildArchitecture: x64 runtimeIdentifier: 'linux-musl-x64' @@ -170,7 +257,7 @@ extends: parameters: agentOs: Linux jobName: Build_Linux_Portable_Deb_Release_x64 - container: PortableDeb + container: ubuntu2204DebPkg buildConfiguration: Release buildArchitecture: x64 # Do not publish zips and tarballs. The linux-x64 binaries are @@ -182,7 +269,7 @@ extends: parameters: agentOs: Linux jobName: Build_Linux_Portable_Rpm_Release_x64 - container: PortableRpm + container: cblMariner20Fpm buildConfiguration: Release buildArchitecture: x64 # Do not publish zips and tarballs. The linux-x64 binaries are @@ -194,7 +281,7 @@ extends: parameters: agentOs: Linux jobName: Build_Linux_Portable_Rpm_Release_Arm64 - container: PortableRpm + container: cblMariner20Fpm buildConfiguration: Release buildArchitecture: arm64 runtimeIdentifier: 'linux-arm64' @@ -279,7 +366,7 @@ extends: # Source Build - template: /eng/common/templates-official/jobs/source-build.yml@self - - ${{ if notin(variables['Build.Reason'], 'PullRequest') }}: + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - stage: Publish dependsOn: - Build diff --git a/eng/build.yml b/eng/build.yml index 70c0d31f3..b21a62ef3 100644 --- a/eng/build.yml +++ b/eng/build.yml @@ -154,7 +154,7 @@ jobs: $(_PgoInstrument) - ${{ else }}: - installerRoot: '$(Build.SourcesDirectory)' - - ${{ if notin(variables['Build.Reason'], 'PullRequest') }}: + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - group: DotNet-HelixApi-Access - _PushToVSFeed: true - _SignType: real @@ -170,7 +170,7 @@ jobs: - checkout: self clean: true - ${{ if eq(parameters.agentOs, 'Windows_NT') }}: - - ${{ if not(parameters.isBuiltFromVmr) }}: + - ${{ if and(not(parameters.isBuiltFromVmr), ne(variables['System.TeamProject'], 'public')) }}: - task: PowerShell@2 displayName: Setup Private Feeds Credentials inputs: @@ -191,7 +191,7 @@ jobs: DOTNET_CLI_UI_LANGUAGE: ${{ parameters.dotnetCLIUILanguage }} - ${{ if ne(parameters.agentOs, 'Windows_NT') }}: - - ${{ if not(parameters.isBuiltFromVmr) }}: + - ${{ if and(not(parameters.isBuiltFromVmr), ne(variables['System.TeamProject'], 'public')) }}: - task: Bash@3 displayName: Setup Private Feeds Credentials inputs: From dbf7caa71456da921967b9dd8b9dc4a976a36618 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Wed, 13 Mar 2024 11:53:03 -0700 Subject: [PATCH 28/36] Do not update localization.yml since it doesn't use the 1ES templates. --- eng/localization.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/localization.yml b/eng/localization.yml index dfe5efa11..aa253c85c 100644 --- a/eng/localization.yml +++ b/eng/localization.yml @@ -19,7 +19,7 @@ variables: jobs: - ${{ if and(ne(variables['System.TeamProject'], 'public'), or(eq(variables['Build.SourceBranch'], 'refs/heads/main'), eq(variables['Build.Reason'], 'Manual'))) }}: - - template: /eng/common/templates-official/job/onelocbuild.yml + - template: /eng/common/templates/job/onelocbuild.yml parameters: CreatePr: ${{ ne(variables['Build.Reason'], 'Manual') }} LclPackageId: 'LCL-JUNO-PROD-DOTNETINSTALLER' From 2e3dc9538352287c4533a4a5d76625c09ebd37ad Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Thu, 14 Mar 2024 09:28:44 -0700 Subject: [PATCH 29/36] Use non-pt images. --- .vsts-ci.yml | 6 +++--- eng/build.yml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 005dc0803..dee857bf7 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -62,7 +62,7 @@ extends: sdl: sourceAnalysisPool: name: $(DncEngInternalBuildPool) - image: 1es-windows-2022-pt + image: windows.vs2022preview.amd64 os: windows # https://docs.opensource.microsoft.com/tools/cg/component-detection/variables/ componentgovernance: @@ -76,7 +76,7 @@ extends: - job: Publish_Build_Configuration pool: name: $(DncEngInternalBuildPool) - image: 1es-windows-2022-pt + image: windows.vs2022preview.amd64 os: windows steps: - task: 1ES.PublishPipelineArtifact@1 @@ -377,5 +377,5 @@ extends: publishAssetsImmediately: true pool: name: $(DncEngInternalBuildPool) - image: 1es-windows-2022-pt + image: windows.vs2022preview.amd64 os: windows diff --git a/eng/build.yml b/eng/build.yml index b21a62ef3..124863606 100644 --- a/eng/build.yml +++ b/eng/build.yml @@ -82,12 +82,12 @@ jobs: enableMicrobuild: true pool: name: $(DncEngInternalBuildPool) - image: 1es-windows-2022-pt + image: windows.vs2022preview.amd64 os: windows ${{ if eq(parameters.agentOs, 'Linux') }}: pool: name: $(DncEngInternalBuildPool) - image: 1es-ubuntu-2204-pt + image: build.ubuntu.2204.amd64 os: linux container: ${{ parameters.container }} ${{ if eq(parameters.agentOs, 'Darwin') }}: From e5bba6c92a0070ffc7c46bfd25f73592a8f68494 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Thu, 14 Mar 2024 09:33:35 -0700 Subject: [PATCH 30/36] Verifying that the array syntax does not work for containers. --- .vsts-ci.yml | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index dee857bf7..a956bae16 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -43,21 +43,37 @@ extends: template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines parameters: containers: - alpine319WithNode: + # alpine319WithNode: + # image: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.19-WithNode + # cblMariner20Fpm: + # image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-fpm + # centosStream8: + # image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8 + # debian11Amd64: + # image: mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-amd64 + # fedora39: + # image: mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-39 + # ubuntu2204: + # image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04 + # ubuntu2204CrossArmAlpine: + # image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-arm-alpine + # ubuntu2204DebPkg: + # image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-debpkg + - container: alpine319WithNode image: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.19-WithNode - cblMariner20Fpm: + - container: cblMariner20Fpm image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-fpm - centosStream8: + - container: centosStream8 image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8 - debian11Amd64: + - container: debian11Amd64 image: mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-amd64 - fedora39: + - container: fedora39 image: mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-39 - ubuntu2204: + - container: ubuntu2204 image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04 - ubuntu2204CrossArmAlpine: + - container: ubuntu2204CrossArmAlpine image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-arm-alpine - ubuntu2204DebPkg: + - container: ubuntu2204DebPkg image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-debpkg sdl: sourceAnalysisPool: From dab13ec5ec6823c86e49f2d03b1509480a5cc4eb Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Thu, 14 Mar 2024 09:46:32 -0700 Subject: [PATCH 31/36] Readded public vs internal statements since removing them doesn't matter. Trying 1es-ubuntu-2204 since the other is not 1ES PT compatible. --- .vsts-ci.yml | 22 +++++++++++++++------- eng/build.yml | 22 ++++++++++++++++------ 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index a956bae16..6942c2286 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -25,11 +25,14 @@ variables: - group: DotNet-Installer-SDLValidation-Params - name: _PublishUsingPipelines value: true -- group: DotNetBuilds storage account read tokens - name: _InternalRuntimeDownloadArgs - value: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal - /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) - /p:dotnetbuilds-internal-container-read-token-base64=$(dotnetbuilds-internal-container-read-token-base64) + value: '' +- ${{ if eq(variables['System.TeamProject'], 'internal') }}: + - group: DotNetBuilds storage account read tokens + - name: _InternalRuntimeDownloadArgs + value: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal + /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) + /p:dotnetbuilds-internal-container-read-token-base64=$(dotnetbuilds-internal-container-read-token-base64) - template: /eng/common/templates-official/variables/pool-providers.yml resources: @@ -91,9 +94,14 @@ extends: # Build Retry Configuration - job: Publish_Build_Configuration pool: - name: $(DncEngInternalBuildPool) - image: windows.vs2022preview.amd64 - os: windows + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: $(DncEngPublicBuildPool) + image: windows.vs2022preview.amd64.open + os: windows + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: $(DncEngInternalBuildPool) + image: windows.vs2022preview.amd64 + os: windows steps: - task: 1ES.PublishPipelineArtifact@1 displayName: Publish Build Config diff --git a/eng/build.yml b/eng/build.yml index 124863606..dd96b812d 100644 --- a/eng/build.yml +++ b/eng/build.yml @@ -81,14 +81,24 @@ jobs: ${{ if eq(parameters.agentOs, 'Windows_NT') }}: enableMicrobuild: true pool: - name: $(DncEngInternalBuildPool) - image: windows.vs2022preview.amd64 - os: windows + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: $(DncEngPublicBuildPool) + image: windows.vs2022.amd64.open + os: windows + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: $(DncEngInternalBuildPool) + image: windows.vs2022preview.amd64 + os: windows ${{ if eq(parameters.agentOs, 'Linux') }}: pool: - name: $(DncEngInternalBuildPool) - image: build.ubuntu.2204.amd64 - os: linux + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: $(DncEngPublicBuildPool) + image: build.ubuntu.2204.amd64.open + os: linux + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: $(DncEngInternalBuildPool) + image: 1es-ubuntu-2204 + os: linux container: ${{ parameters.container }} ${{ if eq(parameters.agentOs, 'Darwin') }}: pool: From 3fdd59c66e4917ee31eeff9647b81f7fa53c0b70 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Thu, 14 Mar 2024 09:53:46 -0700 Subject: [PATCH 32/36] Untabbing the containers to just be 150% certain they don't work. --- .vsts-ci.yml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 6942c2286..98996b234 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -62,22 +62,22 @@ extends: # image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-arm-alpine # ubuntu2204DebPkg: # image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-debpkg - - container: alpine319WithNode - image: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.19-WithNode - - container: cblMariner20Fpm - image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-fpm - - container: centosStream8 - image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8 - - container: debian11Amd64 - image: mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-amd64 - - container: fedora39 - image: mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-39 - - container: ubuntu2204 - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04 - - container: ubuntu2204CrossArmAlpine - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-arm-alpine - - container: ubuntu2204DebPkg - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-debpkg + - container: alpine319WithNode + image: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.19-WithNode + - container: cblMariner20Fpm + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-fpm + - container: centosStream8 + image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8 + - container: debian11Amd64 + image: mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-amd64 + - container: fedora39 + image: mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-39 + - container: ubuntu2204 + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04 + - container: ubuntu2204CrossArmAlpine + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-arm-alpine + - container: ubuntu2204DebPkg + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-debpkg sdl: sourceAnalysisPool: name: $(DncEngInternalBuildPool) From 10c4260fe80ecda738f48e422a6b1e0cf3be55b7 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Thu, 14 Mar 2024 10:03:43 -0700 Subject: [PATCH 33/36] Use the object syntax for containers again. Use the 1es images for all (except macOS) since the 1es images should all work with 1ES PT. --- .vsts-ci.yml | 56 ++++++++++++++++++--------------------------------- eng/build.yml | 6 +++--- 2 files changed, 23 insertions(+), 39 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 98996b234..5d86f6506 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -46,42 +46,26 @@ extends: template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines parameters: containers: - # alpine319WithNode: - # image: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.19-WithNode - # cblMariner20Fpm: - # image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-fpm - # centosStream8: - # image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8 - # debian11Amd64: - # image: mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-amd64 - # fedora39: - # image: mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-39 - # ubuntu2204: - # image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04 - # ubuntu2204CrossArmAlpine: - # image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-arm-alpine - # ubuntu2204DebPkg: - # image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-debpkg - - container: alpine319WithNode - image: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.19-WithNode - - container: cblMariner20Fpm - image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-fpm - - container: centosStream8 - image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8 - - container: debian11Amd64 - image: mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-amd64 - - container: fedora39 - image: mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-39 - - container: ubuntu2204 - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04 - - container: ubuntu2204CrossArmAlpine - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-arm-alpine - - container: ubuntu2204DebPkg - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-debpkg + alpine319WithNode: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.19-WithNode + cblMariner20Fpm: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-fpm + centosStream8: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8 + debian11Amd64: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-amd64 + fedora39: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-39 + ubuntu2204: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04 + ubuntu2204CrossArmAlpine: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-arm-alpine + ubuntu2204DebPkg: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-debpkg sdl: sourceAnalysisPool: name: $(DncEngInternalBuildPool) - image: windows.vs2022preview.amd64 + image: 1es-windows-2022 os: windows # https://docs.opensource.microsoft.com/tools/cg/component-detection/variables/ componentgovernance: @@ -96,11 +80,11 @@ extends: pool: ${{ if eq(variables['System.TeamProject'], 'public') }}: name: $(DncEngPublicBuildPool) - image: windows.vs2022preview.amd64.open + image: 1es-windows-2022-open os: windows ${{ if eq(variables['System.TeamProject'], 'internal') }}: name: $(DncEngInternalBuildPool) - image: windows.vs2022preview.amd64 + image: 1es-windows-2022 os: windows steps: - task: 1ES.PublishPipelineArtifact@1 @@ -401,5 +385,5 @@ extends: publishAssetsImmediately: true pool: name: $(DncEngInternalBuildPool) - image: windows.vs2022preview.amd64 + image: 1es-windows-2022 os: windows diff --git a/eng/build.yml b/eng/build.yml index dd96b812d..bc6155993 100644 --- a/eng/build.yml +++ b/eng/build.yml @@ -83,17 +83,17 @@ jobs: pool: ${{ if eq(variables['System.TeamProject'], 'public') }}: name: $(DncEngPublicBuildPool) - image: windows.vs2022.amd64.open + image: 1es-windows-2022-open os: windows ${{ if eq(variables['System.TeamProject'], 'internal') }}: name: $(DncEngInternalBuildPool) - image: windows.vs2022preview.amd64 + image: 1es-windows-2022 os: windows ${{ if eq(parameters.agentOs, 'Linux') }}: pool: ${{ if eq(variables['System.TeamProject'], 'public') }}: name: $(DncEngPublicBuildPool) - image: build.ubuntu.2204.amd64.open + image: 1es-ubuntu-2004-open os: linux ${{ if eq(variables['System.TeamProject'], 'internal') }}: name: $(DncEngInternalBuildPool) From a877cf477b6f38a0f38c18f0334e7bc5d777c15c Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Fri, 15 Mar 2024 09:02:42 -0700 Subject: [PATCH 34/36] Seeing if I still need to ignore the .packages directory for component governance. --- .vsts-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 5d86f6506..943af203b 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -68,8 +68,8 @@ extends: image: 1es-windows-2022 os: windows # https://docs.opensource.microsoft.com/tools/cg/component-detection/variables/ - componentgovernance: - ignoreDirectories: '.packages' + # componentgovernance: + # ignoreDirectories: '.packages' customBuildTags: - ES365AIMigrationTooling stages: From f38f790b3112992fc7c6dddf6e0f73c3b1d18bbf Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Fri, 15 Mar 2024 09:58:39 -0700 Subject: [PATCH 35/36] Trying to set failOnAlert to false. --- .vsts-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 943af203b..decf0661b 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -68,7 +68,8 @@ extends: image: 1es-windows-2022 os: windows # https://docs.opensource.microsoft.com/tools/cg/component-detection/variables/ - # componentgovernance: + componentgovernance: + failOnAlert: false # ignoreDirectories: '.packages' customBuildTags: - ES365AIMigrationTooling From dc6a3bc3067fd8a82b3a08d36f4fa2b0bd95b0ab Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Fri, 15 Mar 2024 11:45:35 -0700 Subject: [PATCH 36/36] Back to ignoring .packages directory for component governance. Added comment with tracking issue. --- .vsts-ci.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index decf0661b..50be90f89 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -69,8 +69,11 @@ extends: os: windows # https://docs.opensource.microsoft.com/tools/cg/component-detection/variables/ componentgovernance: - failOnAlert: false - # ignoreDirectories: '.packages' + # This should be uncommented when a fix is available. + # See: https://dev.azure.com/mseng/1ES/_workitems/edit/2159448 + # failOnAlert: false + # This should be removed when failOnAlert works properly. + ignoreDirectories: '.packages' customBuildTags: - ES365AIMigrationTooling stages: