[master] Update dependencies from dotnet/arcade (#7283)

* Update dependencies from https://github.com/dotnet/arcade build 20200421.14

- Microsoft.DotNet.Arcade.Sdk: 5.0.0-beta.20201.2 -> 5.0.0-beta.20221.14

* Update arcade and add dotnet-tools feed

NuGet.config

@@ -5,8 +5,11 @@
   </solution>
   <packageSources>
     <clear />
-    <add key="dotnet-core" value="https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json" />
+    <!-- For custom version of arcade -->
+    <add key="general-testing" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/general-testing/nuget/v3/index.json" />
+    <add key="dotnet-tools" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json" />
     <add key="dotnet-eng" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json" />
+    <add key="dotnet-core" value="https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json" />
     <add key="nuget.org" value="https://api.nuget.org/v3/index.json" />
     <add key="msbuild" value="https://dotnet.myget.org/F/msbuild/api/v3/index.json" />
     <add key="nuget-build" value="https://dotnet.myget.org/F/nuget-build/api/v3/index.json" />

eng/Version.Details.xml

@@ -104,9 +104,9 @@
     </Dependency>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="5.0.0-beta.20201.2">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="5.0.0-beta.20224.2">
-      <Uri>https://github.com/dotnet/arcade</Uri>
+      <Uri>https://dev.azure.com/dnceng/internal/_git/dotnet-arcade</Uri>
-      <Sha>bce0a98620c1c5a110b2bba9912f3d5929069c6b</Sha>
+      <Sha>6764cc0a9b413b2ed861ca93c63352917b15ea4d</Sha>
     </Dependency>
   </ToolsetDependencies>
 </Dependencies>

eng/common/SetupNugetSources.ps1

@@ -83,7 +83,7 @@ function AddCredential($creds, $source, $username, $password) {
     $passwordElement.SetAttribute("value", $Password)
 }
 
-function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Password) {
+function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $Password) {
     $maestroPrivateSources = $Sources.SelectNodes("add[contains(@key,'darc-int')]")
 
     Write-Host "Inserting credentials for $($maestroPrivateSources.Count) Maestro's private feeds."
@@ -123,19 +123,21 @@ if ($creds -eq $null) {
     $doc.DocumentElement.AppendChild($creds) | Out-Null
 }
 
+$userName = "dn-bot"
+
 # Insert credential nodes for Maestro's private feeds
-InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Password $Password
+InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -Password $Password
 
 $dotnet3Source = $sources.SelectSingleNode("add[@key='dotnet3']")
 if ($dotnet3Source -ne $null) {
-    AddPackageSource -Sources $sources -SourceName "dotnet3-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3-internal/nuget/v2" -Creds $creds -Username "dn-bot" -Password $Password
+    AddPackageSource -Sources $sources -SourceName "dotnet3-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password
-    AddPackageSource -Sources $sources -SourceName "dotnet3-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3-internal-transport/nuget/v2" -Creds $creds -Username "dn-bot" -Password $Password
+    AddPackageSource -Sources $sources -SourceName "dotnet3-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password
 }
 
 $dotnet31Source = $sources.SelectSingleNode("add[@key='dotnet3.1']")
 if ($dotnet31Source -ne $null) {
-    AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username "dn-bot" -Password $Password
+    AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password
-    AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username "dn-bot" -Password $Password
+    AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password
 }
 
 $doc.Save($filename)

eng/common/cross/build-rootfs.sh

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+set -e
+
 usage()
 {
     echo "Usage: $0 [BuildArch] [CodeName] [lldbx.y] [--skipunmount] --rootfsdir <directory>]"
@@ -15,6 +17,8 @@ __CodeName=xenial
 __CrossDir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
 __InitialDir=$PWD
 __BuildArch=arm
+__AlpineArch=armv7
+__QEMUArch=arm
 __UbuntuArch=armhf
 __UbuntuRepo="http://ports.ubuntu.com/"
 __LLDB_Package="liblldb-3.9-dev"
@@ -26,9 +30,10 @@ __UbuntuPackages="build-essential"
 __AlpinePackages="alpine-base"
 __AlpinePackages+=" build-base"
 __AlpinePackages+=" linux-headers"
-__AlpinePackagesEdgeTesting=" lldb-dev"
+__AlpinePackagesEdgeCommunity=" lldb-dev"
-__AlpinePackagesEdgeMain=" llvm9-libs"
+__AlpinePackagesEdgeMain=" llvm10-libs"
 __AlpinePackagesEdgeMain+=" python3"
+__AlpinePackagesEdgeMain+=" libedit"
 
 # symlinks fixer
 __UbuntuPackages+=" symlinks"
@@ -55,12 +60,11 @@ __AlpinePackages+=" openssl-dev"
 __AlpinePackages+=" zlib-dev"
 
 __FreeBSDBase="12.1-RELEASE"
-__FreeBSDPkg="1.10.5"
+__FreeBSDPkg="1.12.0"
 __FreeBSDPackages="libunwind"
 __FreeBSDPackages+=" icu"
 __FreeBSDPackages+=" libinotify"
 __FreeBSDPackages+=" lttng-ust"
-__FreeBSDPackages+=" llvm-90"
 __FreeBSDPackages+=" krb5"
 
 __UnprocessedBuildArgs=
@@ -78,7 +82,7 @@ while :; do
         arm)
             __BuildArch=arm
             __UbuntuArch=armhf
-            __AlpineArch=armhf
+            __AlpineArch=armv7
             __QEMUArch=arm
             ;;
         arm64)
@@ -205,7 +209,7 @@ fi
 
 if [ -d "$__RootfsDir" ]; then
     if [ $__SkipUnmount == 0 ]; then
-        umount $__RootfsDir/*
+        umount $__RootfsDir/* || true
     fi
     rm -rf $__RootfsDir
 fi
@@ -231,9 +235,9 @@ if [[ "$__CodeName" == "alpine" ]]; then
       add $__AlpinePackagesEdgeMain
 
     $__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \
-      -X http://dl-cdn.alpinelinux.org/alpine/edge/testing \
+      -X http://dl-cdn.alpinelinux.org/alpine/edge/community \
       -U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \
-      add $__AlpinePackagesEdgeTesting
+      add $__AlpinePackagesEdgeCommunity
 
     rm -r $__ApkToolsDir
 elif [[ "$__CodeName" == "freebsd" ]]; then
@@ -246,7 +250,9 @@ elif [[ "$__CodeName" == "freebsd" ]]; then
     # get and build package manager
     wget -O -  https://github.com/freebsd/pkg/archive/${__FreeBSDPkg}.tar.gz  |  tar -C $__RootfsDir/tmp -zxf -
     cd $__RootfsDir/tmp/pkg-${__FreeBSDPkg}
-    ./autogen.sh && ./configure --prefix=$__RootfsDir/host && make install
+    # needed for install to succeed
+    mkdir -p $__RootfsDir/host/etc
+    ./autogen.sh && ./configure --prefix=$__RootfsDir/host && make && make install
     rm -rf $__RootfsDir/tmp/pkg-${__FreeBSDPkg}
     # install packages we need.
     $__RootfsDir/host/sbin/pkg -r $__RootfsDir -C $__RootfsDir/usr/local/etc/pkg.conf update
@@ -260,7 +266,7 @@ elif [[ -n $__CodeName ]]; then
     chroot $__RootfsDir symlinks -cr /usr
 
     if [ $__SkipUnmount == 0 ]; then
-        umount $__RootfsDir/*
+        umount $__RootfsDir/* || true
     fi
 
     if [[ "$__BuildArch" == "arm" && "$__CodeName" == "trusty" ]]; then

eng/common/post-build/check-channel-consistency.ps1

@@ -6,6 +6,11 @@ param(
 try {
   . $PSScriptRoot\post-build-utils.ps1
 
+  if ($PromoteToChannels -eq "") {
+    Write-PipelineTaskError -Type 'warning' -Message "This build won't publish assets as it's not configured to any Maestro channel. If that wasn't intended use Darc to configure a default channel using add-default-channel for this branch or to promote it to a channel using add-build-to-channel. See https://github.com/dotnet/arcade/blob/master/Documentation/Darc.md#assigning-an-individual-build-to-a-channel for more info."
+    ExitWithExitCode 0
+  }
+
   # Check that every channel that Maestro told to promote the build to 
   # is available in YAML
   $PromoteToChannelsIds = $PromoteToChannels -split "\D" | Where-Object { $_ }

eng/common/post-build/symbols-validation.ps1

@@ -1,7 +1,8 @@
 param(
   [Parameter(Mandatory=$true)][string] $InputPath,              # Full path to directory where NuGet packages to be checked are stored
   [Parameter(Mandatory=$true)][string] $ExtractPath,            # Full path to directory where the packages will be extracted during validation
-  [Parameter(Mandatory=$true)][string] $DotnetSymbolVersion     # Version of dotnet symbol to use
+  [Parameter(Mandatory=$true)][string] $DotnetSymbolVersion,    # Version of dotnet symbol to use
+  [Parameter(Mandatory=$false)][switch] $ContinueOnError        # If we should keep checking symbols after an error
 )
 
 function FirstMatchingSymbolDescriptionOrDefault {
@@ -125,6 +126,8 @@ function CheckSymbolsAvailable {
     Remove-Item $ExtractPath -Force  -Recurse -ErrorAction SilentlyContinue
   }
 
+  $TotalFailures = 0
+
   Get-ChildItem "$InputPath\*.nupkg" |
     ForEach-Object {
       $FileName = $_.Name
@@ -148,11 +151,22 @@ function CheckSymbolsAvailable {
 
       if ($Status -ne 0) {
         Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Missing symbols for $Status modules in the package $FileName"
-        ExitWithExitCode $exitCode
+
+        if ($ContinueOnError) {
+          $TotalFailures++
+        }
+        else {
+          ExitWithExitCode 1
+        }
       }
 
       Write-Host
     }
+
+  if ($TotalFailures -ne 0) {
+    Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Symbols missing for $TotalFailures packages"
+    ExitWithExitCode 1
+  }
 }
 
 function InstallDotnetSymbol {

eng/common/sdk-task.ps1

@@ -57,6 +57,18 @@ try {
     ExitWithExitCode 1
   }
 
+  if( $msbuildEngine -eq "vs") {
+    # Ensure desktop MSBuild is available for sdk tasks.
+    if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "vs" )) {
+      $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.4`" }") -MemberType NoteProperty
+    }
+    if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) {
+      $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "16.4.0-alpha" -MemberType NoteProperty
+    }
+
+    InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true
+  }
+
   $taskProject = GetSdkTaskProject $task
   if (!(Test-Path $taskProject)) {
     Write-PipelineTelemetryError -Category 'Build' -Message "Unknown task: $task" -ForegroundColor Red

eng/common/sdl/NuGet.config

@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="utf-8"?>
+<?xml version="1.0" encoding="utf-8"?>
 <configuration>
   <solution>
     <add key="disableSourceControlIntegration" value="true" />

eng/common/sdl/extract-artifact-packages.ps1

@@ -63,7 +63,7 @@ try {
           }
     }
     catch {
-      Write-Host $_.ScriptStackTrace
+      Write-Host $_
       Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_
       ExitWithExitCode 1
     }
@@ -74,7 +74,7 @@ try {
   Measure-Command { ExtractArtifacts }
 }
 catch {
-  Write-Host $_.ScriptStackTrace
+  Write-Host $_
   Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_
   ExitWithExitCode 1
 }

eng/common/sdl/packages.config

@@ -1,4 +1,4 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
-  <package id="Microsoft.Guardian.Cli" version="0.7.2"/>
+  <package id="Microsoft.Guardian.Cli.win10-x64" version="0.20.1"/>
 </packages>

eng/common/templates/job/execute-sdl.yml

@@ -6,6 +6,7 @@ parameters:
   # 'continueOnError', the parameter value is not correctly picked up.
   # This can also be remedied by the caller (post-build.yml) if it does not use a nested parameter
   sdlContinueOnError: false                                    # optional: determines whether to continue the build if the step errors;
+  downloadArtifacts: true                                      # optional: determines if the artifacts should be dowloaded
   dependsOn: ''                                                # Optional: dependencies of the job
   artifactNames: ''                                            # Optional: patterns supplied to DownloadBuildArtifacts
                                                                # Usage:
@@ -31,8 +32,20 @@ jobs:
   steps:
   - checkout: self
     clean: true
-  - ${{ if ne(parameters.artifactNames, '') }}:
+  - ${{ if ne(parameters.downloadArtifacts, 'false')}}:
-    - ${{ each artifactName in parameters.artifactNames }}:
+    - ${{ if ne(parameters.artifactNames, '') }}:
+      - ${{ each artifactName in parameters.artifactNames }}:
+        - task: DownloadBuildArtifacts@0
+          displayName: Download Build Artifacts
+          inputs:
+            buildType: specific
+            buildVersionToDownload: specific
+            project: $(AzDOProjectName)
+            pipeline: $(AzDOPipelineId)
+            buildId: $(AzDOBuildId)
+            artifactName: ${{ artifactName }}
+            downloadPath: $(Build.ArtifactStagingDirectory)\artifacts
+    - ${{ if eq(parameters.artifactNames, '') }}:
       - task: DownloadBuildArtifacts@0
         displayName: Download Build Artifacts
         inputs:
@@ -41,20 +54,9 @@ jobs:
           project: $(AzDOProjectName)
           pipeline: $(AzDOPipelineId)
           buildId: $(AzDOBuildId)
-          artifactName: ${{ artifactName }}
+          downloadType: specific files
+          itemPattern: "**"
           downloadPath: $(Build.ArtifactStagingDirectory)\artifacts
-  - ${{ if eq(parameters.artifactNames, '') }}:
-    - task: DownloadBuildArtifacts@0
-      displayName: Download Build Artifacts
-      inputs:
-        buildType: specific
-        buildVersionToDownload: specific
-        project: $(AzDOProjectName)
-        pipeline: $(AzDOPipelineId)
-        buildId: $(AzDOBuildId)
-        downloadType: specific files
-        itemPattern: "**"
-        downloadPath: $(Build.ArtifactStagingDirectory)\artifacts
   - powershell: eng/common/sdl/extract-artifact-packages.ps1
       -InputPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts
       -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts
@@ -81,7 +83,7 @@ jobs:
       continueOnError: ${{ parameters.sdlContinueOnError }}
   - ${{ if eq(parameters.overrideParameters, '') }}:
     - powershell: eng/common/sdl/execute-all-sdl-tools.ps1
-        -GuardianPackageName Microsoft.Guardian.Cli.0.7.2
+        -GuardianPackageName Microsoft.Guardian.Cli.win10-x64.0.20.1
         -NugetPackageDirectory $(Build.SourcesDirectory)\.packages
         -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw)
         ${{ parameters.additionalParameters }}

eng/common/templates/job/job.yml

@@ -24,6 +24,7 @@ parameters:
   enablePublishBuildAssets: false
   enablePublishTestResults: false
   enablePublishUsingPipelines: false
+  useBuildManifest: false
   mergeTestResults: false
   testRunTitle: $(AgentOsName)-$(BuildConfiguration)-xunit
   name: ''
@@ -218,3 +219,12 @@ jobs:
         ArtifactName: AssetManifests
       continueOnError: ${{ parameters.continueOnError }}
       condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true'))
+
+  - ${{ if eq(parameters.useBuildManifest, true) }}:
+    - task: PublishBuildArtifacts@1
+      displayName: Publish Build Manifest
+      inputs:
+        PathToPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/manifest.props'
+        PublishLocation: Container
+        ArtifactName: BuildManifests
+      continueOnError: ${{ parameters.continueOnError }}

eng/common/templates/post-build/channels/generic-internal-channel.yml

@@ -23,7 +23,7 @@ stages:
   - job: publish_symbols
     displayName: Symbol Publishing
     dependsOn: setupMaestroVars
-    condition: or(contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.channelId }} )), eq(dependencies.setupMaestroVars.outputs['setReleaseVars.PromoteToMaestroChannelId'], ${{ parameters.channelId }}))
+    condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'], format('[{0}]', ${{ parameters.channelId }} ))
     variables:
       - group: DotNet-Symbol-Server-Pats
       - name: AzDOProjectName
@@ -96,7 +96,7 @@ stages:
         value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
       - name: AzDOBuildId
         value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
-    condition: or(contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.channelId }} )), eq(dependencies.setupMaestroVars.outputs['setReleaseVars.PromoteToMaestroChannelId'], ${{ parameters.channelId }}))
+    condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'], format('[{0}]', ${{ parameters.channelId }} ))
     pool:
       vmImage: 'windows-2019'
     steps:

eng/common/templates/post-build/channels/generic-public-channel.yml

@@ -25,7 +25,7 @@ stages:
   - job: publish_symbols
     displayName: Symbol Publishing
     dependsOn: setupMaestroVars
-    condition: or(contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.channelId }} )), eq(dependencies.setupMaestroVars.outputs['setReleaseVars.PromoteToMaestroChannelId'], ${{ parameters.channelId }}))
+    condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'], format('[{0}]', ${{ parameters.channelId }} ))
     variables:
       - group: DotNet-Symbol-Server-Pats
       - name: AzDOProjectName
@@ -99,7 +99,7 @@ stages:
         value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
       - name: ArtifactsCategory
         value: ${{ coalesce(variables._DotNetArtifactsCategory, '.NETCore') }}
-    condition: or(contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.channelId }} )), eq(dependencies.setupMaestroVars.outputs['setReleaseVars.PromoteToMaestroChannelId'], ${{ parameters.channelId }}))
+    condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'], format('[{0}]', ${{ parameters.channelId }} ))
     pool:
       vmImage: 'windows-2019'
     steps:

eng/common/templates/post-build/common-variables.yml

@@ -63,7 +63,7 @@ variables:
   - name: MaestroApiAccessToken
     value: $(MaestroAccessToken)
   - name: MaestroApiVersion
-    value: "2019-01-16"
+    value: "2020-02-20"
 
   - name: SourceLinkCLIVersion
     value: 3.0.0

eng/common/templates/post-build/post-build.yml

@@ -9,12 +9,14 @@ parameters:
     continueOnError: false
     params: ''
     artifactNames: ''
+    downloadArtifacts: true
 
   # These parameters let the user customize the call to sdk-task.ps1 for publishing
   # symbols & general artifacts as well as for signing validation
   symbolPublishingAdditionalParameters: ''
   artifactsPublishingAdditionalParameters: ''
   signingValidationAdditionalParameters: ''
+  useBuildManifest: false
 
   # Which stages should finish execution before post-build stages start
   validateDependsOn:
@@ -35,8 +37,9 @@ parameters:
   NETCoreExperimentalChannelId: 562
   NetEngServicesIntChannelId: 678
   NetEngServicesProdChannelId: 679
-  Net5Preview2ChannelId: 738
   Net5Preview3ChannelId: 739
+  Net5Preview4ChannelId: 856
+  Net5Preview5ChannelId: 857
   NetCoreSDK313xxChannelId: 759
   NetCoreSDK313xxInternalChannelId: 760
   
@@ -53,10 +56,8 @@ stages:
     displayName: Post-build Checks
     dependsOn: setupMaestroVars
     variables:
-      - name: InitialChannels
+      - name: TargetChannels
-        value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'] ]
+        value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'] ]
-      - name: PromoteToMaestroChannelId
-        value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.PromoteToMaestroChannelId'] ]
     pool:
       vmImage: 'windows-2019'
     steps:
@@ -64,8 +65,8 @@ stages:
         displayName: Maestro Channels Consistency
         inputs:
           filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1
-          arguments: -PromoteToChannels "$(InitialChannels)[$(PromoteToMaestroChannelId)]"
+          arguments: -PromoteToChannels "$(TargetChannels)"
-            -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview2ChannelId}},${{parameters.Net5Preview3ChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}}
+            -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview3ChannelId}},${{parameters.Net5Preview4ChannelId}},${{parameters.Net5Preview5ChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}}
 
   - job:
     displayName: NuGet Validation
@@ -113,6 +114,16 @@ stages:
     pool:
       vmImage: 'windows-2019'
     steps:
+      - ${{ if eq(parameters.useBuildManifest, true) }}:
+        - task: DownloadBuildArtifacts@0
+          displayName: Download build manifest
+          inputs:
+            buildType: specific
+            buildVersionToDownload: specific
+            project: $(AzDOProjectName)
+            pipeline: $(AzDOPipelineId)
+            buildId: $(AzDOBuildId)
+            artifactName: BuildManifests
       - task: DownloadBuildArtifacts@0
         displayName: Download Package Artifacts
         inputs:
@@ -135,11 +146,13 @@ stages:
           filePath: eng\common\enable-cross-org-publishing.ps1
           arguments: -token $(dn-bot-dnceng-artifact-feeds-rw)
 
+      # Signing validation will optionally work with the buildmanifest file which is downloaded from
+      # Azure DevOps above.
       - task: PowerShell@2
         displayName: Validate
         inputs:
           filePath: eng\common\sdk-task.ps1
-          arguments: -task SigningValidation -restore -msbuildEngine dotnet
+          arguments: -task SigningValidation -restore -msbuildEngine vs
             /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts'
             /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt'
             ${{ parameters.signingValidationAdditionalParameters }}
@@ -192,6 +205,7 @@ stages:
       additionalParameters: ${{ parameters.SDLValidationParameters.params }}
       continueOnError: ${{ parameters.SDLValidationParameters.continueOnError }}
       artifactNames: ${{ parameters.SDLValidationParameters.artifactNames }}
+      downloadArtifacts: ${{ parameters.SDLValidationParameters.downloadArtifacts }}
 
 - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
   parameters:
@@ -213,10 +227,10 @@ stages:
     dependsOn: ${{ parameters.publishDependsOn }}
     publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
     symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
-    stageName: 'Net5_Preview2_Publish'
+    stageName: 'Net5_Preview3_Publish'
-    channelName: '.NET 5 Preview 2'
+    channelName: '.NET 5 Preview 3'
-    akaMSChannelName: 'net5/preview2'
+    akaMSChannelName: 'net5/preview3'
-    channelId: ${{ parameters.Net5Preview2ChannelId }}
+    channelId: ${{ parameters.Net5Preview3ChannelId }}
     transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json'
     shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json'
     symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json'
@@ -227,10 +241,24 @@ stages:
     dependsOn: ${{ parameters.publishDependsOn }}
     publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
     symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
-    stageName: 'Net5_Preview3_Publish'
+    stageName: 'Net5_Preview4_Publish'
-    channelName: '.NET 5 Preview 3'
+    channelName: '.NET 5 Preview 4'
-    akaMSChannelName: 'net5/preview3'
+    akaMSChannelName: 'net5/preview4'
-    channelId: ${{ parameters.Net5Preview3ChannelId }}
+    channelId: ${{ parameters.Net5Preview4ChannelId }}
+    transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json'
+    shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json'
+    symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json'
+
+- template: \eng\common\templates\post-build\channels\generic-public-channel.yml
+  parameters:
+    artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+    dependsOn: ${{ parameters.publishDependsOn }}
+    publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+    symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+    stageName: 'Net5_Preview5_Publish'
+    channelName: '.NET 5 Preview 5'
+    akaMSChannelName: 'net5/preview5'
+    channelId: ${{ parameters.Net5Preview5ChannelId }}
     transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json'
     shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json'
     symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json'

eng/common/templates/post-build/setup-maestro-vars.yml

@@ -5,7 +5,9 @@ jobs:
     - template: common-variables.yml
     - name: BuildId
       value: $[ coalesce(variables.BARBuildId, 0) ]
-    - name: PromoteToChannelId
+    - name: PromoteToMaestroChannels
+      value: $[ coalesce(variables.PromoteToChannelIds, 0) ]
+    - name: PromoteToMaestroChannel 
       value: $[ coalesce(variables.PromoteToMaestroChannelId, 0) ]
   pool:
     vmImage: 'windows-2019'
@@ -14,7 +16,7 @@ jobs:
 
     - task: DownloadBuildArtifacts@0
       displayName: Download Release Configs
-      condition: eq(variables.PromoteToChannelId, 0)
+      condition: and(eq(variables.PromoteToMaestroChannels, 0), eq(variables.PromoteToMaestroChannel, 0))
       inputs:
         buildType: current
         artifactName: ReleaseConfigs
@@ -26,20 +28,16 @@ jobs:
         targetType: inline
         script: |
           try {
-            if ($Env:PromoteToChannelId -eq 0) {
+            if ($Env:PromoteToMaestroChannels -eq 0 -and $Env:PromoteToMaestroChannel -eq 0) {
               $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt
 
               $BarId = $Content | Select -Index 0
-
+              $Channels = $Content | Select -Index 1             
-              $Channels = ""
-              $Content | Select -Index 1 | ForEach-Object { $Channels += "$_ ," }
-              
               $IsStableBuild = $Content | Select -Index 2
 
               $AzureDevOpsProject = $Env:System_TeamProject
               $AzureDevOpsBuildDefinitionId = $Env:System_DefinitionId
               $AzureDevOpsBuildId = $Env:Build_BuildId
-              $PromoteToMaestroChannelId = 0
             }
             else {
               $buildApiEndpoint = "${Env:MaestroApiEndPoint}/api/builds/${Env:BARBuildId}?api-version=${Env:MaestroApiVersion}"
@@ -51,25 +49,23 @@ jobs:
               $buildInfo = try { Invoke-WebRequest -Method Get -Uri $buildApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" }
              
               $BarId = $Env:BARBuildId
-              $Channels = 'None'
+              $Channels = $Env:PromoteToMaestroChannels -split ","
-
+              $Channels = $Channels -join "]["
-              #TODO: Fix this once this issue is done: https://github.com/dotnet/arcade/issues/3834
+              $Channels = "[$Channels][$Env:PromoteToMaestroChannel]"
-              $IsStableBuild = 'False'
 
+              $IsStableBuild = $buildInfo.stable
               $AzureDevOpsProject = $buildInfo.azureDevOpsProject
               $AzureDevOpsBuildDefinitionId = $buildInfo.azureDevOpsBuildDefinitionId
               $AzureDevOpsBuildId = $buildInfo.azureDevOpsBuildId
-              $PromoteToMaestroChannelId = $Env:PromoteToMaestroChannelId
             }
 
             Write-Host "##vso[task.setvariable variable=BARBuildId;isOutput=true]$BarId"
-            Write-Host "##vso[task.setvariable variable=InitialChannels;isOutput=true]$Channels"
+            Write-Host "##vso[task.setvariable variable=TargetChannels;isOutput=true]$Channels"
             Write-Host "##vso[task.setvariable variable=IsStableBuild;isOutput=true]$IsStableBuild"
 
             Write-Host "##vso[task.setvariable variable=AzDOProjectName;isOutput=true]$AzureDevOpsProject"
             Write-Host "##vso[task.setvariable variable=AzDOPipelineId;isOutput=true]$AzureDevOpsBuildDefinitionId"
             Write-Host "##vso[task.setvariable variable=AzDOBuildId;isOutput=true]$AzureDevOpsBuildId"
-            Write-Host "##vso[task.setvariable variable=PromoteToMaestroChannelId;isOutput=true]$PromoteToMaestroChannelId"
           }
           catch {
             Write-Host $_

eng/common/tools.ps1

@@ -488,10 +488,11 @@ function GetNuGetPackageCachePath() {
   if ($env:NUGET_PACKAGES -eq $null) {
     # Use local cache on CI to ensure deterministic build,
     # use global cache in dev builds to avoid cost of downloading packages.
+    # For directory normalization, see also: https://github.com/NuGet/Home/issues/7968
     if ($useGlobalNuGetCache) {
-      $env:NUGET_PACKAGES = Join-Path $env:UserProfile '.nuget\packages'
+      $env:NUGET_PACKAGES = Join-Path $env:UserProfile '.nuget\packages\'
     } else {
-      $env:NUGET_PACKAGES = Join-Path $RepoRoot '.packages'
+      $env:NUGET_PACKAGES = Join-Path $RepoRoot '.packages\'
     }
   }
 
@@ -632,6 +633,8 @@ function MSBuild-Core() {
     }
   }
 
+  $env:ARCADE_BUILD_TOOL_COMMAND = "$($buildTool.Path) $cmdArgs"
+
   $exitCode = Exec-Process $buildTool.Path $cmdArgs
 
   if ($exitCode -ne 0) {

eng/common/tools.sh

@@ -201,7 +201,14 @@ function InstallDotNet {
 
       local runtimeSourceFeedKey=''
       if [[ -n "${7:-}" ]]; then
-        decodedFeedKey=`echo $7 | base64 --decode`
+        # The 'base64' binary on alpine uses '-d' and doesn't support '--decode'
+        # '-d'. To work around this, do a simple detection and switch the parameter
+        # accordingly.
+        decodeArg="--decode"
+        if base64 --help 2>&1 | grep -q "BusyBox"; then
+            decodeArg="-d"
+        fi
+        decodedFeedKey=`echo $7 | base64 $decodeArg`
         runtimeSourceFeedKey="--feed-credential $decodedFeedKey"
       fi
 
@@ -415,11 +422,17 @@ function MSBuild-Core {
     warnaserror_switch="/warnaserror"
   fi
 
-  "$_InitializeBuildTool" "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@" || {
+  function RunBuildTool {
-    local exit_code=$?
+    export ARCADE_BUILD_TOOL_COMMAND="$_InitializeBuildTool $@"
-    Write-PipelineTelemetryError -category 'Build'  "Build failed (exit code '$exit_code')."
+
-    ExitWithExitCode $exit_code
+    "$_InitializeBuildTool" "$@" || {
+      local exit_code=$?
+      Write-PipelineTaskError "Build failed (exit code '$exit_code')."
+      ExitWithExitCode $exit_code
+    }
   }
+
+  RunBuildTool "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@"
 }
 
 ResolvePath "${BASH_SOURCE[0]}"

global.json

@@ -8,6 +8,6 @@
     }
   },
   "msbuild-sdks": {
-    "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20201.2"
+    "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20224.2"
   }
 }