Update dependencies from https://github.com/dotnet/arcade build 20191023.3 (#5320)

- Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.19523.3

NuGet.config

@@ -5,6 +5,9 @@
   </solution>
   <packageSources>
     <clear />
+    <!--Begin: Package sources managed by Dependency Flow automation. Do not edit the sources below.-->
+    <add key="darc-pub-dotnet-core-setup-7d57652" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/darc-pub-dotnet-core-setup-7d57652f/nuget/v3/index.json" />
+    <!--End: Package sources managed by Dependency Flow automation. Do not edit the sources above.-->
     <add key="dotnet-core" value="https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json" />
     <add key="dotnet-tools" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json" />
     <add key="nuget.org" value="https://api.nuget.org/v3/index.json" />

eng/Version.Details.xml

@@ -104,9 +104,9 @@
     </Dependency>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="1.0.0-beta.19474.3">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="5.0.0-beta.19523.3">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>0e9ffd6464aff37aef2dc41dc2162d258f266e32</Sha>
+      <Sha>894bd005267af53f7d39ac988881e61193e67ae1</Sha>
     </Dependency>
   </ToolsetDependencies>
 </Dependencies>

eng/common/build.ps1

@@ -18,6 +18,7 @@ Param(
   [switch] $sign,
   [switch] $pack,
   [switch] $publish,
+  [switch] $clean,
   [switch][Alias('bl')]$binaryLog,
   [switch] $ci,
   [switch] $prepareMachine,
@@ -48,6 +49,7 @@ function Print-Usage() {
     Write-Host "  -pack                   Package build outputs into NuGet packages and Willow components"
     Write-Host "  -sign                   Sign build outputs"
     Write-Host "  -publish                Publish artifacts (e.g. symbols)"
+    Write-Host "  -clean                  Clean the solution"
     Write-Host ""
 
     Write-Host "Advanced settings:"
@@ -112,6 +114,14 @@ function Build {
     @properties
 }
 
+if ($clean) {
+  if(Test-Path $ArtifactsDir) {
+    Remove-Item -Recurse -Force $ArtifactsDir
+    Write-Host "Artifacts directory deleted."
+  }
+  exit 0
+}
+
 try {
   if ($help -or (($null -ne $properties) -and ($properties.Contains("/help") -or $properties.Contains("/?")))) {
     Print-Usage
@@ -123,14 +133,7 @@ try {
     $nodeReuse = $false
   }
 
-  # Import custom tools configuration, if present in the repo.
+  if ($restore) {
-  # Note: Import in global scope so that the script set top-level variables without qualification.
-  $configureToolsetScript = Join-Path $EngRoot "configure-toolset.ps1"
-  if (Test-Path $configureToolsetScript) {
-    . $configureToolsetScript
-  }
-
-  if (($restore) -and ($null -eq $env:DisableNativeToolsetInstalls)) {
     InitializeNativeTools
   }
 

eng/common/build.sh

@@ -26,6 +26,7 @@ usage()
   echo "  --pack                     Package build outputs into NuGet packages and Willow components"
   echo "  --sign                     Sign build outputs"
   echo "  --publish                  Publish artifacts (e.g. symbols)"
+  echo "  --clean                    Clean the solution"
   echo ""
 
   echo "Advanced settings:"
@@ -62,6 +63,7 @@ publish=false
 sign=false
 public=false
 ci=false
+clean=false
 
 warn_as_error=true
 node_reuse=true
@@ -82,6 +84,9 @@ while [[ $# > 0 ]]; do
       usage
       exit 0
       ;;
+    -clean)
+      clean=true
+      ;;
     -configuration|-c)
       configuration=$2
       shift
@@ -196,20 +201,15 @@ function Build {
   ExitWithExitCode 0
 }
 
-# Import custom tools configuration, if present in the repo.
+if [[ "$clean" == true ]]; then
-configure_toolset_script="$eng_root/configure-toolset.sh"
+  if [ -d "$artifacts_dir" ]; then
-if [[ -a "$configure_toolset_script" ]]; then
+    rm -rf $artifacts_dir
-  . "$configure_toolset_script"
+    echo "Artifacts directory deleted."
+  fi
+  exit 0
 fi
 
-# TODO: https://github.com/dotnet/arcade/issues/1468
+if [[ "$restore" == true ]]; then
-# Temporary workaround to avoid breaking change.
-# Remove once repos are updated.
-if [[ -n "${useInstalledDotNetCli:-}" ]]; then
-  use_installed_dotnet_cli="$useInstalledDotNetCli"
-fi
-
-if [[ "$restore" == true && -z ${DisableNativeToolsetInstalls:-} ]]; then
   InitializeNativeTools
 fi
 

eng/common/darc-init.ps1

@@ -1,7 +1,8 @@
 param (
     $darcVersion = $null,
     $versionEndpoint = "https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16",
-    $verbosity = "m"
+    $verbosity = "m",
+    $toolpath = $null
 )
 
 . $PSScriptRoot\tools.ps1
@@ -23,11 +24,15 @@ function InstallDarcCli ($darcVersion) {
     $darcVersion = $(Invoke-WebRequest -Uri $versionEndpoint -UseBasicParsing).Content
   }
 
-  $arcadeServicesSource = 'https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json'
+  $arcadeServicesSource = 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
 
   Write-Host "Installing Darc CLI version $darcVersion..."
   Write-Host "You may need to restart your command window if this is the first dotnet tool you have installed."
-  & "$dotnet" tool install $darcCliPackageName --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g
+  if (-not $toolpath) {
+    & "$dotnet" tool install $darcCliPackageName --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g
+  }else {
+    & "$dotnet" tool install $darcCliPackageName --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity --tool-path "$toolpath"    
+  }
 }
 
 InstallDarcCli $darcVersion

eng/common/darc-init.sh

@@ -20,6 +20,10 @@ while [[ $# > 0 ]]; do
       verbosity=$2
       shift
       ;;
+    --toolpath)
+      toolpath=$2
+      shift
+      ;;
     *)
       echo "Invalid argument: $1"
       usage
@@ -52,17 +56,27 @@ function InstallDarcCli {
   InitializeDotNetCli
   local dotnet_root=$_InitializeDotNetCli
 
-  local uninstall_command=`$dotnet_root/dotnet tool uninstall $darc_cli_package_name -g`
+  if [ -z "$toolpath" ]; then
-  local tool_list=$($dotnet_root/dotnet tool list -g)
+    local tool_list=$($dotnet_root/dotnet tool list -g)
-  if [[ $tool_list = *$darc_cli_package_name* ]]; then
+    if [[ $tool_list = *$darc_cli_package_name* ]]; then
-    echo $($dotnet_root/dotnet tool uninstall $darc_cli_package_name -g)
+      echo $($dotnet_root/dotnet tool uninstall $darc_cli_package_name -g)
+    fi
+  else
+    local tool_list=$($dotnet_root/dotnet tool list --tool-path "$toolpath")
+    if [[ $tool_list = *$darc_cli_package_name* ]]; then
+      echo $($dotnet_root/dotnet tool uninstall $darc_cli_package_name --tool-path "$toolpath")
+    fi
   fi
 
-  local arcadeServicesSource="https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json"
+  local arcadeServicesSource="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json"
 
   echo "Installing Darc CLI version $darcVersion..."
   echo "You may need to restart your command shell if this is the first dotnet tool you have installed."
-  echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g)
+  if [ -z "$toolpath" ]; then
+    echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g)
+  else
+    echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity --tool-path "$toolpath")
+  fi
 }
 
 InstallDarcCli

eng/common/dotnet-install.ps1

@@ -3,7 +3,9 @@ Param(
   [string] $verbosity = "minimal",
   [string] $architecture = "",
   [string] $version = "Latest",
-  [string] $runtime = "dotnet"
+  [string] $runtime = "dotnet",
+  [string] $RuntimeSourceFeed = "",
+  [string] $RuntimeSourceFeedKey = ""
 )
 
 . $PSScriptRoot\tools.ps1
@@ -15,7 +17,7 @@ try {
     if ($architecture -and $architecture.Trim() -eq "x86") {
         $installdir = Join-Path $installdir "x86"
     }
-   InstallDotNet $installdir $version $architecture $runtime $true
+   InstallDotNet $installdir $version $architecture $runtime $true -RuntimeSourceFeed $RuntimeSourceFeed -RuntimeSourceFeedKey $RuntimeSourceFeedKey
 } 
 catch {
   Write-Host $_

eng/common/dotnet-install.sh

@@ -14,6 +14,8 @@ scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
 version='Latest'
 architecture=''
 runtime='dotnet'
+runtimeSourceFeed=''
+runtimeSourceFeedKey=''
 while [[ $# > 0 ]]; do
   opt="$(echo "$1" | awk '{print tolower($0)}')"
   case "$opt" in
@@ -29,18 +31,55 @@ while [[ $# > 0 ]]; do
       shift
       runtime="$1"
       ;;
+    -runtimesourcefeed)
+      shift
+      runtimeSourceFeed="$1"
+      ;;
+    -runtimesourcefeedkey)
+      shift
+      runtimeSourceFeedKey="$1"
+      ;;
     *)
       echo "Invalid argument: $1"
-      usage
       exit 1
       ;;
   esac
   shift
 done
 
+# Use uname to determine what the CPU is.
+cpuname=$(uname -p)
+# Some Linux platforms report unknown for platform, but the arch for machine.
+if [[ "$cpuname" == "unknown" ]]; then
+  cpuname=$(uname -m)
+fi
+
+case $cpuname in
+  aarch64)
+    buildarch=arm64
+    ;;
+  amd64|x86_64)
+    buildarch=x64
+    ;;
+  armv7l)
+    buildarch=arm
+    ;;
+  i686)
+    buildarch=x86
+    ;;
+  *)
+    echo "Unknown CPU $cpuname detected, treating it as x64"
+    buildarch=x64
+    ;;
+esac
+
 . "$scriptroot/tools.sh"
 dotnetRoot="$repo_root/.dotnet"
-InstallDotNet $dotnetRoot $version "$architecture" $runtime true || {
+if [[ $architecture != "" ]] && [[ $architecture != $buildarch ]]; then
+  dotnetRoot="$dotnetRoot/$architecture"
+fi
+
+InstallDotNet $dotnetRoot $version "$architecture" $runtime true $runtimeSourceFeed $runtimeSourceFeedKey || {
   local exit_code=$?
   echo "dotnet-install.sh failed (exit code '$exit_code')." >&2
   ExitWithExitCode $exit_code

eng/common/init-tools-native.ps1

@@ -133,6 +133,7 @@ try {
   if (Test-Path $InstallBin) {
     Write-Host "Native tools are available from" (Convert-Path -Path $InstallBin)
     Write-Host "##vso[task.prependpath]$(Convert-Path -Path $InstallBin)"
+    return $InstallBin
   }
   else {
     Write-Error "Native tools install directory does not exist, installation failed"

eng/common/post-build/post-build-utils.ps1

@@ -8,6 +8,7 @@ Set-StrictMode -Version 2.0
 # scripts don't necessarily execute in the same agent that run the
 # build.ps1/sh script this variable isn't automatically set.
 $ci = $true
+$disableConfigureToolsetImport = "true"
 . $PSScriptRoot\..\tools.ps1
 
 function Create-MaestroApiRequestHeaders([string]$ContentType = "application/json") {

eng/common/post-build/setup-maestro-vars.ps1

@@ -1,26 +0,0 @@
-param(
-  [Parameter(Mandatory=$true)][string] $ReleaseConfigsPath            # Full path to ReleaseConfigs.txt asset
-)
-
-. $PSScriptRoot\post-build-utils.ps1
-
-try {
-  $Content = Get-Content $ReleaseConfigsPath
-  
-  $BarId = $Content | Select -Index 0
-  
-  $Channels = ""            
-  $Content | Select -Index 1 | ForEach-Object { $Channels += "$_ ," }
-  
-  $IsStableBuild = $Content | Select -Index 2
-
-  Write-PipelineSetVariable -Name 'BARBuildId' -Value $BarId
-  Write-PipelineSetVariable -Name 'InitialChannels' -Value "$Channels"
-  Write-PipelineSetVariable -Name 'IsStableBuild' -Value $IsStableBuild
-}
-catch {
-  Write-Host $_
-  Write-Host $_.Exception
-  Write-Host $_.ScriptStackTrace
-  ExitWithExitCode 1
-}

eng/common/sdl/execute-all-sdl-tools.ps1

@@ -1,30 +1,30 @@
 Param(
-  [string] $GuardianPackageName,                                                           # Required: the name of guardian CLI package (not needed if GuardianCliLocation is specified)
+  [string] $GuardianPackageName,                                                                 # Required: the name of guardian CLI package (not needed if GuardianCliLocation is specified)
-  [string] $NugetPackageDirectory,                                                         # Required: directory where NuGet packages are installed (not needed if GuardianCliLocation is specified)
+  [string] $NugetPackageDirectory,                                                               # Required: directory where NuGet packages are installed (not needed if GuardianCliLocation is specified)
-  [string] $GuardianCliLocation,                                                           # Optional: Direct location of Guardian CLI executable if GuardianPackageName & NugetPackageDirectory are not specified
+  [string] $GuardianCliLocation,                                                                 # Optional: Direct location of Guardian CLI executable if GuardianPackageName & NugetPackageDirectory are not specified
-  [string] $Repository=$env:BUILD_REPOSITORY_NAME,                                         # Required: the name of the repository (e.g. dotnet/arcade)
+  [string] $Repository=$env:BUILD_REPOSITORY_NAME,                                               # Required: the name of the repository (e.g. dotnet/arcade)
-  [string] $BranchName=$env:BUILD_SOURCEBRANCH,                                            # Optional: name of branch or version of gdn settings; defaults to master
+  [string] $BranchName=$env:BUILD_SOURCEBRANCH,                                                  # Optional: name of branch or version of gdn settings; defaults to master
-  [string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY,                                   # Required: the directory where source files are located
+  [string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY,                                         # Required: the directory where source files are located
-  [string] $ArtifactsDirectory = (Join-Path $env:BUILD_SOURCESDIRECTORY ("artifacts")),    # Required: the directory where build artifacts are located
+  [string] $ArtifactsDirectory = (Join-Path $env:BUILD_ARTIFACTSTAGINGDIRECTORY ("artifacts")),  # Required: the directory where build artifacts are located
-  [string] $AzureDevOpsAccessToken,                                                        # Required: access token for dnceng; should be provided via KeyVault
+  [string] $AzureDevOpsAccessToken,                                                              # Required: access token for dnceng; should be provided via KeyVault
-  [string[]] $SourceToolsList,                                                             # Optional: list of SDL tools to run on source code
+  [string[]] $SourceToolsList,                                                                   # Optional: list of SDL tools to run on source code
-  [string[]] $ArtifactToolsList,                                                           # Optional: list of SDL tools to run on built artifacts
+  [string[]] $ArtifactToolsList,                                                                 # Optional: list of SDL tools to run on built artifacts
-  [bool] $TsaPublish=$False,                                                               # Optional: true will publish results to TSA; only set to true after onboarding to TSA; TSA is the automated framework used to upload test results as bugs.
+  [bool] $TsaPublish=$False,                                                                     # Optional: true will publish results to TSA; only set to true after onboarding to TSA; TSA is the automated framework used to upload test results as bugs.
-  [string] $TsaBranchName=$env:BUILD_SOURCEBRANCH,                                         # Optional: required for TSA publish; defaults to $(Build.SourceBranchName); TSA is the automated framework used to upload test results as bugs.
+  [string] $TsaBranchName=$env:BUILD_SOURCEBRANCH,                                               # Optional: required for TSA publish; defaults to $(Build.SourceBranchName); TSA is the automated framework used to upload test results as bugs.
-  [string] $TsaRepositoryName=$env:BUILD_REPOSITORY_NAME,                                  # Optional: TSA repository name; will be generated automatically if not submitted; TSA is the automated framework used to upload test results as bugs.
+  [string] $TsaRepositoryName=$env:BUILD_REPOSITORY_NAME,                                        # Optional: TSA repository name; will be generated automatically if not submitted; TSA is the automated framework used to upload test results as bugs.
-  [string] $BuildNumber=$env:BUILD_BUILDNUMBER,                                            # Optional: required for TSA publish; defaults to $(Build.BuildNumber)
+  [string] $BuildNumber=$env:BUILD_BUILDNUMBER,                                                  # Optional: required for TSA publish; defaults to $(Build.BuildNumber)
-  [bool] $UpdateBaseline=$False,                                                           # Optional: if true, will update the baseline in the repository; should only be run after fixing any issues which need to be fixed
+  [bool] $UpdateBaseline=$False,                                                                 # Optional: if true, will update the baseline in the repository; should only be run after fixing any issues which need to be fixed
-  [bool] $TsaOnboard=$False,                                                               # Optional: if true, will onboard the repository to TSA; should only be run once; TSA is the automated framework used to upload test results as bugs.
+  [bool] $TsaOnboard=$False,                                                                     # Optional: if true, will onboard the repository to TSA; should only be run once; TSA is the automated framework used to upload test results as bugs.
-  [string] $TsaInstanceUrl,                                                                # Optional: only needed if TsaOnboard or TsaPublish is true; the instance-url registered with TSA; TSA is the automated framework used to upload test results as bugs.
+  [string] $TsaInstanceUrl,                                                                      # Optional: only needed if TsaOnboard or TsaPublish is true; the instance-url registered with TSA; TSA is the automated framework used to upload test results as bugs.
-  [string] $TsaCodebaseName,                                                               # Optional: only needed if TsaOnboard or TsaPublish is true; the name of the codebase registered with TSA; TSA is the automated framework used to upload test results as bugs.
+  [string] $TsaCodebaseName,                                                                     # Optional: only needed if TsaOnboard or TsaPublish is true; the name of the codebase registered with TSA; TSA is the automated framework used to upload test results as bugs.
-  [string] $TsaProjectName,                                                                # Optional: only needed if TsaOnboard or TsaPublish is true; the name of the project registered with TSA; TSA is the automated framework used to upload test results as bugs.
+  [string] $TsaProjectName,                                                                      # Optional: only needed if TsaOnboard or TsaPublish is true; the name of the project registered with TSA; TSA is the automated framework used to upload test results as bugs.
-  [string] $TsaNotificationEmail,                                                          # Optional: only needed if TsaOnboard is true; the email(s) which will receive notifications of TSA bug filings (e.g. alias@microsoft.com); TSA is the automated framework used to upload test results as bugs.
+  [string] $TsaNotificationEmail,                                                                # Optional: only needed if TsaOnboard is true; the email(s) which will receive notifications of TSA bug filings (e.g. alias@microsoft.com); TSA is the automated framework used to upload test results as bugs.
-  [string] $TsaCodebaseAdmin,                                                              # Optional: only needed if TsaOnboard is true; the aliases which are admins of the TSA codebase (e.g. DOMAIN\alias); TSA is the automated framework used to upload test results as bugs.
+  [string] $TsaCodebaseAdmin,                                                                    # Optional: only needed if TsaOnboard is true; the aliases which are admins of the TSA codebase (e.g. DOMAIN\alias); TSA is the automated framework used to upload test results as bugs.
-  [string] $TsaBugAreaPath,                                                                # Optional: only needed if TsaOnboard is true; the area path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs.
+  [string] $TsaBugAreaPath,                                                                      # Optional: only needed if TsaOnboard is true; the area path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs.
-  [string] $TsaIterationPath,                                                              # Optional: only needed if TsaOnboard is true; the iteration path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs.
+  [string] $TsaIterationPath,                                                                    # Optional: only needed if TsaOnboard is true; the iteration path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs.
-  [string] $GuardianLoggerLevel="Standard",                                                # Optional: the logger level for the Guardian CLI; options are Trace, Verbose, Standard, Warning, and Error
+  [string] $GuardianLoggerLevel="Standard",                                                      # Optional: the logger level for the Guardian CLI; options are Trace, Verbose, Standard, Warning, and Error
-  [string[]] $CrScanAdditionalRunConfigParams,                                             # Optional: Additional Params to custom build a CredScan run config in the format @("xyz:abc","sdf:1")
+  [string[]] $CrScanAdditionalRunConfigParams,                                                   # Optional: Additional Params to custom build a CredScan run config in the format @("xyz:abc","sdf:1")
-  [string[]] $PoliCheckAdditionalRunConfigParams                                           # Optional: Additional Params to custom build a Policheck run config in the format @("xyz:abc","sdf:1")
+  [string[]] $PoliCheckAdditionalRunConfigParams                                                 # Optional: Additional Params to custom build a Policheck run config in the format @("xyz:abc","sdf:1")
 )
 
 $ErrorActionPreference = "Stop"

eng/common/sdl/extract-artifact-packages.ps1

@@ -10,6 +10,7 @@ Set-StrictMode -Version 2.0
 # scripts don't necessarily execute in the same agent that run the
 # build.ps1/sh script this variable isn't automatically set.
 $ci = $true
+$disableConfigureToolsetImport = "true"
 . $PSScriptRoot\..\tools.ps1
 
 $ExtractPackage = {

eng/common/templates/job/execute-sdl.yml

@@ -24,15 +24,15 @@ jobs:
       buildType: current
       downloadType: specific files
       matchingPattern: "**"
-      downloadPath: $(Build.SourcesDirectory)\artifacts
+      downloadPath: $(Build.ArtifactStagingDirectory)\artifacts
   - powershell: eng/common/sdl/extract-artifact-packages.ps1
-      -InputPath $(Build.SourcesDirectory)\artifacts\BlobArtifacts
+      -InputPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts
-      -ExtractPath $(Build.SourcesDirectory)\artifacts\BlobArtifacts
+      -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts
     displayName: Extract Blob Artifacts
     continueOnError: ${{ parameters.sdlContinueOnError }}
   - powershell: eng/common/sdl/extract-artifact-packages.ps1
-      -InputPath $(Build.SourcesDirectory)\artifacts\PackageArtifacts
+      -InputPath $(Build.ArtifactStagingDirectory)\artifacts\PackageArtifacts
-      -ExtractPath $(Build.SourcesDirectory)\artifacts\PackageArtifacts
+      -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\PackageArtifacts
     displayName: Extract Package Artifacts
     continueOnError: ${{ parameters.sdlContinueOnError }}
   - task: NuGetToolInstaller@1

eng/common/templates/job/job.yml

@@ -1,67 +1,33 @@
+# Internal resources (telemetry, microbuild) can only be accessed from non-public projects,
+# and some (Microbuild) should only be applied to non-PR cases for internal builds.
+
 parameters:
 # Job schema parameters - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job
   cancelTimeoutInMinutes: ''
-
   condition: ''
-
-  continueOnError: false
-
   container: ''
-
+  continueOnError: false
   dependsOn: ''
-
   displayName: ''
-
-  steps: []
-
   pool: ''
-
+  steps: []
   strategy: ''
-
   timeoutInMinutes: ''
-
   variables: []
-
   workspace: ''
 
-  # Job base template specific parameters
+# Job base template specific parameters
-  # Optional: Enable installing Microbuild plugin
+  # See schema documentation - https://github.com/dotnet/arcade/blob/master/Documentation/AzureDevOps/TemplateSchema.md
-  #           if 'true', these "variables" must be specified in the variables object or as part of the queue matrix
+  artifacts: ''
-  #             _TeamName - the name of your team
-  #             _SignType - 'test' or 'real'
   enableMicrobuild: false
-
-  # Optional: Include PublishBuildArtifacts task
   enablePublishBuildArtifacts: false
-
-  # Optional: Enable publishing to the build asset registry
   enablePublishBuildAssets: false
-
-  # Optional: Prevent gather/push manifest from executing when using publishing pipelines
-  enablePublishUsingPipelines: false
-
-  # Optional: Include PublishTestResults task
   enablePublishTestResults: false
-
+  enablePublishUsingPipelines: false
-  # Optional: enable sending telemetry
-  enableTelemetry: false
-
-  # Optional: define the helix repo for telemetry (example: 'dotnet/arcade')
-  helixRepo: ''
-
-  # Optional: define the helix type for telemetry (example: 'build/product/')
-  helixType: ''
-
-  # Required: name of the job
   name: ''
-
+  preSteps: []
-  # Optional: should run as a public build even in the internal project
-  #           if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects.
   runAsPublic: false
 
-# Internal resources (telemetry, microbuild) can only be accessed from non-public projects,
-# and some (Microbuild) should only be applied to non-PR cases for internal builds.
-
 jobs:
 - job: ${{ parameters.name }}
 
@@ -93,7 +59,7 @@ jobs:
     timeoutInMinutes: ${{ parameters.timeoutInMinutes }}
 
   variables:
-  - ${{ if eq(parameters.enableTelemetry, 'true') }}:
+  - ${{ if ne(parameters.enableTelemetry, 'false') }}:
     - name: DOTNET_CLI_TELEMETRY_PROFILE
       value: '$(Build.Repository.Uri)'
   - ${{ each variable in parameters.variables }}:
@@ -125,21 +91,12 @@ jobs:
     workspace: ${{ parameters.workspace }}
 
   steps:
-  - ${{ if eq(parameters.enableTelemetry, 'true') }}:
+  - ${{ if ne(parameters.preSteps, '') }}:
-    # Telemetry tasks are built from https://github.com/dotnet/arcade-extensions
+    - ${{ each preStep in parameters.preSteps }}:
-    - task: sendStartTelemetry@0
+      - ${{ preStep }}
-      displayName: 'Send Helix Start Telemetry'
-      inputs:
-        helixRepo: ${{ parameters.helixRepo }}
-        ${{ if ne(parameters.helixType, '') }}:
-          helixType: ${{ parameters.helixType }}
-        buildConfig: $(_BuildConfig)
-        runAsPublic: ${{ parameters.runAsPublic }}
-      continueOnError: ${{ parameters.continueOnError }}
-      condition: always()
 
-  - ${{ if eq(parameters.enableMicrobuild, 'true') }}:
+  - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
-    - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+    - ${{ if eq(parameters.enableMicrobuild, 'true') }}:
       - task: MicroBuildSigningPlugin@2
         displayName: Install MicroBuild plugin
         inputs:
@@ -151,9 +108,16 @@ jobs:
         continueOnError: ${{ parameters.continueOnError }}
         condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT'))
 
-  - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
     - task: NuGetAuthenticate@0
 
+  - ${{ if or(eq(parameters.artifacts.download, 'true'), ne(parameters.artifacts.download, '')) }}:
+    - task: DownloadPipelineArtifact@2
+      inputs:
+        buildType: current
+        artifactName: ${{ coalesce(parameters.artifacts.download.name, 'Artifacts_$(Agent.OS)_$(_BuildConfig)') }}
+        targetPath: ${{ coalesce(parameters.artifacts.download.path, 'artifacts') }}
+        itemPattern: ${{ coalesce(parameters.artifacts.download.pattern, '**') }}
+
   - ${{ each step in parameters.steps }}:
     - ${{ step }}
 
@@ -166,20 +130,60 @@ jobs:
         env:
           TeamName: $(_TeamName)
 
-  - ${{ if eq(parameters.enableTelemetry, 'true') }}:
+  - ${{ if ne(parameters.artifacts.publish, '') }}:
-    # Telemetry tasks are built from https://github.com/dotnet/arcade-extensions
+    - ${{ if or(eq(parameters.artifacts.publish.artifacts, 'true'), ne(parameters.artifacts.publish.artifacts, '')) }}:
-    - task: sendEndTelemetry@0
+      - task: CopyFiles@2
-      displayName: 'Send Helix End Telemetry'
+        displayName: Gather binaries for publish to artifacts
-      continueOnError: ${{ parameters.continueOnError }}
+        inputs:
-      condition: always()
+          SourceFolder: 'artifacts/bin'
+          Contents: '**'
+          TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/bin'
+      - task: CopyFiles@2
+        displayName: Gather packages for publish to artifacts
+        inputs:
+          SourceFolder: 'artifacts/packages'
+          Contents: '**'
+          TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/packages'
+      - task: PublishBuildArtifacts@1
+        displayName: Publish pipeline artifacts
+        inputs:
+          PathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts'
+          PublishLocation: Container
+          ArtifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }}
+        continueOnError: true
+        condition: always()
+    - ${{ if or(eq(parameters.artifacts.publish.logs, 'true'), ne(parameters.artifacts.publish.logs, '')) }}:
+      - publish: artifacts/log
+        artifact: ${{ coalesce(parameters.artifacts.publish.logs.name, 'Logs_Build_$(Agent.Os)_$(_BuildConfig)') }}
+        displayName: Publish logs
+        continueOnError: true
+        condition: always()
+    - ${{ if or(eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}:
+      - ${{ if and(ne(parameters.enablePublishUsingPipelines, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:    
+        - task: CopyFiles@2
+          displayName: Gather Asset Manifests
+          inputs:
+            SourceFolder: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/AssetManifest'
+            TargetFolder: '$(Build.ArtifactStagingDirectory)/AssetManifests'
+          continueOnError: ${{ parameters.continueOnError }}
+          condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true'))
 
-  - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}:
+        - task: PublishBuildArtifacts@1
+          displayName: Push Asset Manifests
+          inputs:
+            PathtoPublish: '$(Build.ArtifactStagingDirectory)/AssetManifests'
+            PublishLocation: Container
+            ArtifactName: AssetManifests
+          continueOnError: ${{ parameters.continueOnError }}
+          condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true'))
+
+  - ${{ if ne(parameters.enablePublishBuildArtifacts, 'false') }}:
     - task: PublishBuildArtifacts@1
       displayName: Publish Logs
       inputs:
         PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)'
         PublishLocation: Container
-        ArtifactName: $(Agent.Os)_$(Agent.JobName)
+        ArtifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)' ) }}
       continueOnError: true
       condition: always()
 

eng/common/templates/jobs/jobs.yml

@@ -1,19 +1,10 @@
 parameters:
-  # Optional: 'true' if failures in job.yml job should not fail the job
+  # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md
   continueOnError: false
 
-  # Optional: Enable installing Microbuild plugin
-  #           if 'true', these "variables" must be specified in the variables object or as part of the queue matrix
-  #             _TeamName - the name of your team
-  #             _SignType - 'test' or 'real'
-  enableMicrobuild: false
-
   # Optional: Include PublishBuildArtifacts task
   enablePublishBuildArtifacts: false
 
-  # Optional: Enable publishing to the build asset registry
-  enablePublishBuildAssets: false
-
   # Optional: Enable publishing using release pipelines
   enablePublishUsingPipelines: false
   
@@ -23,19 +14,9 @@ parameters:
     # Optional: Include toolset dependencies in the generated graph files
     includeToolset: false
     
-  # Optional: Include PublishTestResults task
-  enablePublishTestResults: false
-
-  # Optional: enable sending telemetry
-  # if enabled then the 'helixRepo' parameter should also be specified
-  enableTelemetry: false
-
   # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job
   jobs: []
 
-  # Optional: define the helix repo for telemetry (example: 'dotnet/arcade')
-  helixRepo: ''
-
   # Optional: Override automatically derived dependsOn value for "publish build assets" job
   publishBuildAssetsDependsOn: ''
 
@@ -62,29 +43,30 @@ jobs:
 
       name: ${{ job.job }}
 
-- ${{ if and(eq(parameters.enablePublishBuildAssets, true), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
-  - template: ../job/publish-build-assets.yml
+  - ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}:
-    parameters:
+    - template: ../job/publish-build-assets.yml
-      continueOnError: ${{ parameters.continueOnError }}
+      parameters:
-      dependsOn:
+        continueOnError: ${{ parameters.continueOnError }}
-      - ${{ if ne(parameters.publishBuildAssetsDependsOn, '') }}:
+        dependsOn:
-        - ${{ each job in parameters.publishBuildAssetsDependsOn }}:
+        - ${{ if ne(parameters.publishBuildAssetsDependsOn, '') }}:
-          - ${{ job.job }}
+          - ${{ each job in parameters.publishBuildAssetsDependsOn }}:
-      - ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}:
+            - ${{ job.job }}
-        - ${{ each job in parameters.jobs }}:
+        - ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}:
-          - ${{ job.job }}
+          - ${{ each job in parameters.jobs }}:
-      pool:
+            - ${{ job.job }}
-        vmImage: vs2017-win2016
+        pool:
-      runAsPublic: ${{ parameters.runAsPublic }}
+          vmImage: vs2017-win2016
-      publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }}
+        runAsPublic: ${{ parameters.runAsPublic }}
-      enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }}
+        publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }}
-      
+        enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }}
-- ${{ if and(eq(parameters.graphFileGeneration.enabled, true), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+
-  - template: ../job/generate-graph-files.yml
+  - ${{ if eq(parameters.graphFileGeneration.enabled, true) }}:
-    parameters:
+    - template: ../job/generate-graph-files.yml
-      continueOnError: ${{ parameters.continueOnError }}
+      parameters:
-      includeToolset: ${{ parameters.graphFileGeneration.includeToolset }}
+        continueOnError: ${{ parameters.continueOnError }}
-      dependsOn:
+        includeToolset: ${{ parameters.graphFileGeneration.includeToolset }}
-        - Asset_Registry_Publish
+        dependsOn:
-      pool:
+          - Asset_Registry_Publish
-        vmImage: vs2017-win2016
+        pool:
+          vmImage: vs2017-win2016

eng/common/templates/post-build/channels/netcore-3-tools-validation.yml

@@ -1,10 +1,12 @@
 parameters:
   artifactsPublishingAdditionalParameters: ''
+  dependsOn:
+  - Validate
   publishInstallersAndChecksums: false
 
 stages:
 - stage: NetCore_3_Tools_Validation_Publish
-  dependsOn: validate
+  dependsOn: ${{ parameters.dependsOn }}
   variables:
     - template: ../common-variables.yml
   displayName: .NET 3 Tools - Validation Publishing
@@ -81,7 +83,6 @@ stages:
             /p:InstallersAzureAccountKey=$(dotnetcli-storage-key)
             /p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl)
             /p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key)
-            /p:PublishToAzureDevOpsNuGetFeeds=true
             /p:AzureDevOpsStaticShippingFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
             /p:AzureDevOpsStaticShippingFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
             /p:AzureDevOpsStaticTransportFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'

eng/common/templates/post-build/channels/netcore-3-tools.yml

@@ -1,11 +1,13 @@
 parameters:
-  symbolPublishingAdditionalParameters: ''
   artifactsPublishingAdditionalParameters: ''
+  dependsOn:
+  - Validate
   publishInstallersAndChecksums: false
+  symbolPublishingAdditionalParameters: ''
 
 stages:
 - stage: NetCore_3_Tools_Publish
-  dependsOn: validate
+  dependsOn: ${{ parameters.dependsOn }}
   variables:
     - template: ../common-variables.yml
   displayName: .NET 3 Tools Publishing
@@ -116,7 +118,6 @@ stages:
             /p:InstallersAzureAccountKey=$(dotnetcli-storage-key)
             /p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl)
             /p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key)
-            /p:PublishToAzureDevOpsNuGetFeeds=true
             /p:AzureDevOpsStaticShippingFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
             /p:AzureDevOpsStaticShippingFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
             /p:AzureDevOpsStaticTransportFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'

eng/common/templates/post-build/channels/netcore-dev-31.yml

@@ -1,11 +1,13 @@
 parameters:
-  symbolPublishingAdditionalParameters: ''
   artifactsPublishingAdditionalParameters: ''
+  dependsOn:
+  - Validate
   publishInstallersAndChecksums: false
+  symbolPublishingAdditionalParameters: ''
 
 stages:
 - stage: NetCore_Dev31_Publish
-  dependsOn: validate
+  dependsOn: ${{ parameters.dependsOn }}
   variables:
     - template: ../common-variables.yml
   displayName: .NET Core 3.1 Dev Publishing
@@ -116,7 +118,6 @@ stages:
             /p:InstallersAzureAccountKey=$(dotnetcli-storage-key)
             /p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl)
             /p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key)
-            /p:PublishToAzureDevOpsNuGetFeeds=true
             /p:AzureDevOpsStaticShippingFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1/nuget/v3/index.json'
             /p:AzureDevOpsStaticShippingFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
             /p:AzureDevOpsStaticTransportFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-transport/nuget/v3/index.json'

eng/common/templates/post-build/channels/netcore-dev-5.yml

@@ -1,11 +1,13 @@
 parameters:
-  symbolPublishingAdditionalParameters: ''
   artifactsPublishingAdditionalParameters: ''
+  dependsOn:
+  - Validate
   publishInstallersAndChecksums: false
+  symbolPublishingAdditionalParameters: ''
 
 stages:
 - stage: NetCore_Dev5_Publish
-  dependsOn: validate
+  dependsOn: ${{ parameters.dependsOn }}
   variables:
     - template: ../common-variables.yml
   displayName: .NET Core 5 Dev Publishing
@@ -116,7 +118,6 @@ stages:
             /p:InstallersAzureAccountKey=$(dotnetcli-storage-key)
             /p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl)
             /p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key)
-            /p:PublishToAzureDevOpsNuGetFeeds=true
             /p:AzureDevOpsStaticShippingFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json'
             /p:AzureDevOpsStaticShippingFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
             /p:AzureDevOpsStaticTransportFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json'

eng/common/templates/post-build/channels/netcore-internal-30.yml

@@ -1,10 +1,12 @@
 parameters:
-  symbolPublishingAdditionalParameters: ''
   artifactsPublishingAdditionalParameters: ''
+  dependsOn:
+  - Validate
+  symbolPublishingAdditionalParameters: ''
 
 stages:
 - stage: NetCore_30_Internal_Servicing_Publishing
-  dependsOn: validate
+  dependsOn: ${{ parameters.dependsOn }}
   variables:
     - template: ../common-variables.yml
   displayName: .NET Core 3.0 Internal Servicing Publishing
@@ -32,6 +34,18 @@ stages:
           artifactName: 'PDBArtifacts'
         continueOnError: true
 
+      # This is necessary whenever we want to publish/restore to an AzDO private feed
+      # Since sdk-task.ps1 tries to restore packages we need to do this authentication here
+      # otherwise it'll complain about accessing a private feed.
+      - task: NuGetAuthenticate@0
+        displayName: 'Authenticate to AzDO Feeds'
+
+      - task: PowerShell@2
+        displayName: Enable cross-org publishing
+        inputs:
+          filePath: eng\common\enable-cross-org-publishing.ps1
+          arguments: -token $(dn-bot-dnceng-artifact-feeds-rw)
+
       - task: PowerShell@2
         displayName: Publish
         inputs:
@@ -114,7 +128,6 @@ stages:
             /p:ChecksumsAzureAccountKey=$(InternalChecksumsBlobFeedKey)
             /p:InstallersTargetStaticFeed=$(InternalInstallersBlobFeedUrl)
             /p:InstallersAzureAccountKey=$(InternalInstallersBlobFeedKey)
-            /p:PublishToAzureDevOpsNuGetFeeds=true
             /p:AzureDevOpsStaticShippingFeed='https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3-internal/nuget/v3/index.json'
             /p:AzureDevOpsStaticShippingFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
             /p:AzureDevOpsStaticTransportFeed='https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3-internal-transport/nuget/v3/index.json'

eng/common/templates/post-build/channels/netcore-release-30.yml

@@ -1,11 +1,13 @@
 parameters:
-  symbolPublishingAdditionalParameters: ''
   artifactsPublishingAdditionalParameters: ''
+  dependsOn:
+  - Validate
   publishInstallersAndChecksums: false
+  symbolPublishingAdditionalParameters: ''
 
 stages:
 - stage: NetCore_Release30_Publish
-  dependsOn: validate
+  dependsOn: ${{ parameters.dependsOn }}
   variables:
     - template: ../common-variables.yml
   displayName: .NET Core 3.0 Release Publishing
@@ -116,7 +118,6 @@ stages:
             /p:InstallersAzureAccountKey=$(dotnetcli-storage-key)
             /p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl)
             /p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key)
-            /p:PublishToAzureDevOpsNuGetFeeds=true
             /p:AzureDevOpsStaticShippingFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3/nuget/v3/index.json'
             /p:AzureDevOpsStaticShippingFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
             /p:AzureDevOpsStaticTransportFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3-transport/nuget/v3/index.json'

eng/common/templates/post-build/channels/netcore-release-31.yml

@@ -1,11 +1,13 @@
 parameters:
-  symbolPublishingAdditionalParameters: ''
   artifactsPublishingAdditionalParameters: ''
+  dependsOn:
+  - Validate
   publishInstallersAndChecksums: false
+  symbolPublishingAdditionalParameters: ''
 
 stages:
 - stage: NetCore_Release31_Publish
-  dependsOn: validate
+  dependsOn: ${{ parameters.dependsOn }}
   variables:
     - template: ../common-variables.yml
   displayName: .NET Core 3.1 Release Publishing
@@ -116,7 +118,6 @@ stages:
             /p:InstallersAzureAccountKey=$(dotnetcli-storage-key)
             /p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl)
             /p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key)
-            /p:PublishToAzureDevOpsNuGetFeeds=true
             /p:AzureDevOpsStaticShippingFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1/nuget/v3/index.json'
             /p:AzureDevOpsStaticShippingFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
             /p:AzureDevOpsStaticTransportFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-transport/nuget/v3/index.json'

eng/common/templates/post-build/channels/netcore-tools-latest.yml

@@ -1,11 +1,13 @@
 parameters:
-  symbolPublishingAdditionalParameters: ''
   artifactsPublishingAdditionalParameters: ''
+  dependsOn:
+  - Validate
   publishInstallersAndChecksums: false
+  symbolPublishingAdditionalParameters: ''
 
 stages:
 - stage: NetCore_Tools_Latest_Publish
-  dependsOn: validate
+  dependsOn: ${{ parameters.dependsOn }}
   variables:
     - template: ../common-variables.yml
   displayName: .NET Tools - Latest Publishing
@@ -116,7 +118,6 @@ stages:
             /p:InstallersAzureAccountKey=$(dotnetcli-storage-key)
             /p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl)
             /p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key)
-            /p:PublishToAzureDevOpsNuGetFeeds=true
             /p:AzureDevOpsStaticShippingFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
             /p:AzureDevOpsStaticShippingFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
             /p:AzureDevOpsStaticTransportFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'

eng/common/templates/post-build/channels/netcore-tools-validation.yml

@@ -1,10 +1,12 @@
 parameters:
   artifactsPublishingAdditionalParameters: ''
+  dependsOn:
+  - Validate
   publishInstallersAndChecksums: false
 
 stages:
 - stage: PVR_Publish
-  dependsOn: validate
+  dependsOn: ${{ parameters.dependsOn }}
   variables:
     - template: ../common-variables.yml
   displayName: .NET Tools - Validation Publishing
@@ -81,7 +83,6 @@ stages:
             /p:InstallersAzureAccountKey=$(dotnetcli-storage-key)
             /p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl)
             /p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key)
-            /p:PublishToAzureDevOpsNuGetFeeds=true
             /p:AzureDevOpsStaticShippingFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
             /p:AzureDevOpsStaticShippingFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
             /p:AzureDevOpsStaticTransportFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'

eng/common/templates/post-build/post-build.yml

@@ -16,11 +16,14 @@ parameters:
   signingValidationAdditionalParameters: ''
 
   # Which stages should finish execution before post-build stages start
-  dependsOn: [build]
+  validateDependsOn:
+  - build
+  publishDependsOn: 
+  - Validate
 
 stages:
-- stage: validate
+- stage: Validate
-  dependsOn: ${{ parameters.dependsOn }}
+  dependsOn: ${{ parameters.validateDependsOn }}
   displayName: Validate
   jobs:
   - ${{ if eq(parameters.enableNugetValidation, 'true') }}:
@@ -45,6 +48,9 @@ stages:
   - ${{ if eq(parameters.enableSigningValidation, 'true') }}:
     - job:
       displayName: Signing Validation
+      variables:
+        - template: common-variables.yml
+        - group: AzureDevOps-Artifact-Feeds-Pats
       pool:
         vmImage: 'windows-2019'
       steps:
@@ -54,6 +60,19 @@ stages:
             buildType: current
             artifactName: PackageArtifacts
 
+        # This is necessary whenever we want to publish/restore to an AzDO private feed
+        # Since sdk-task.ps1 tries to restore packages we need to do this authentication here
+        # otherwise it'll complain about accessing a private feed.
+        - task: NuGetAuthenticate@0
+          condition: eq(variables['IsInternalBuild'], 'true')
+          displayName: 'Authenticate to AzDO Feeds'
+
+        - task: PowerShell@2
+          displayName: Enable cross-org publishing
+          inputs:
+            filePath: eng\common\enable-cross-org-publishing.ps1
+            arguments: -token $(dn-bot-dnceng-artifact-feeds-rw)
+
         - task: PowerShell@2
           displayName: Validate
           inputs:
@@ -97,51 +116,60 @@ stages:
 
 - template: \eng\common\templates\post-build\channels\netcore-dev-5.yml
   parameters:
-    symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
     artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+    dependsOn: ${{ parameters.publishDependsOn }}
     publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
-
+    
 - template: \eng\common\templates\post-build\channels\netcore-dev-31.yml
   parameters:
-    symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
     artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+    dependsOn: ${{ parameters.publishDependsOn }}
     publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+    symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
 
 - template: \eng\common\templates\post-build\channels\netcore-tools-latest.yml
   parameters:
-    symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
     artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+    dependsOn: ${{ parameters.publishDependsOn }}
     publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+    symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
 
 - template: \eng\common\templates\post-build\channels\netcore-tools-validation.yml
   parameters:
     artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+    dependsOn: ${{ parameters.publishDependsOn }}
     publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+    symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
 
 - template: \eng\common\templates\post-build\channels\netcore-3-tools-validation.yml
   parameters:
     artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+    dependsOn: ${{ parameters.publishDependsOn }}
     publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
 
 - template: \eng\common\templates\post-build\channels\netcore-3-tools.yml
   parameters:
+    dependsOn: ${{ parameters.publishDependsOn }}
     symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
     artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
     publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
 
 - template: \eng\common\templates\post-build\channels\netcore-release-30.yml
   parameters:
-    symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
     artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+    dependsOn: ${{ parameters.publishDependsOn }}
     publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+    symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
 
 - template: \eng\common\templates\post-build\channels\netcore-release-31.yml
   parameters:
-    symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
     artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+    dependsOn: ${{ parameters.publishDependsOn }}
     publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+    symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
 
 - template: \eng\common\templates\post-build\channels\netcore-internal-30.yml
   parameters:
-    symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
     artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+    dependsOn: ${{ parameters.publishDependsOn }}
+    symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}

eng/common/templates/post-build/setup-maestro-vars.yml

@@ -4,6 +4,8 @@ jobs:
   pool:
     vmImage: 'windows-2019'
   steps:
+    - checkout: none
+
     - task: DownloadBuildArtifacts@0
       displayName: Download Release Configs
       inputs:
@@ -14,5 +16,25 @@ jobs:
       name: setReleaseVars
       displayName: Set Release Configs Vars
       inputs:
-        filePath: $(Build.SourcesDirectory)/eng/common/post-build/setup-maestro-vars.ps1
+        targetType: inline
-        arguments: -ReleaseConfigsPath '$(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt'
+        script: |
+          try {
+            $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt
+
+            $BarId = $Content | Select -Index 0
+
+            $Channels = ""
+            $Content | Select -Index 1 | ForEach-Object { $Channels += "$_ ," }
+
+            $IsStableBuild = $Content | Select -Index 2
+
+            Write-Host "##vso[task.setvariable variable=BARBuildId;isOutput=true]$BarId"
+            Write-Host "##vso[task.setvariable variable=InitialChannels;isOutput=true]$Channels"
+            Write-Host "##vso[task.setvariable variable=IsStableBuild;isOutput=true]$IsStableBuild"
+          }
+          catch {
+            Write-Host $_
+            Write-Host $_.Exception
+            Write-Host $_.ScriptStackTrace
+            exit 1
+          }

eng/common/tools.ps1

@@ -49,6 +49,8 @@
 # An array of names of processes to stop on script exit if prepareMachine is true.
 $processesToStopOnExit = if (Test-Path variable:processesToStopOnExit) { $processesToStopOnExit } else { @("msbuild", "dotnet", "vbcscompiler") }
 
+$disableConfigureToolsetImport = if (Test-Path variable:disableConfigureToolsetImport) { $disableConfigureToolsetImport } else { $null }
+
 set-strictmode -version 2.0
 $ErrorActionPreference = "Stop"
 [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
@@ -154,15 +156,6 @@ function InitializeDotNetCli([bool]$install) {
   # Make Sure that our bootstrapped dotnet cli is available in future steps of the Azure Pipelines build
   Write-PipelinePrependPath -Path $dotnetRoot
 
-  # Work around issues with Azure Artifacts credential provider
-  # https://github.com/dotnet/arcade/issues/3932
-  if ($ci) {
-    $env:NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS = 20
-    $env:NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS = 20
-    Write-PipelineSetVariable -Name 'NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS' -Value '20'
-    Write-PipelineSetVariable -Name 'NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS' -Value '20'
-  }
-
   Write-PipelineSetVariable -Name 'DOTNET_MULTILEVEL_LOOKUP' -Value '0'
   Write-PipelineSetVariable -Name 'DOTNET_SKIP_FIRST_TIME_EXPERIENCE' -Value '1'
 
@@ -184,7 +177,14 @@ function InstallDotNetSdk([string] $dotnetRoot, [string] $version, [string] $arc
   InstallDotNet $dotnetRoot $version $architecture
 }
 
-function InstallDotNet([string] $dotnetRoot, [string] $version, [string] $architecture = "", [string] $runtime = "", [bool] $skipNonVersionedFiles = $false) {
+function InstallDotNet([string] $dotnetRoot, 
+  [string] $version, 
+  [string] $architecture = "", 
+  [string] $runtime = "", 
+  [bool] $skipNonVersionedFiles = $false, 
+  [string] $runtimeSourceFeed = "", 
+  [string] $runtimeSourceFeedKey = "") {
+
   $installScript = GetDotNetInstallScript $dotnetRoot
   $installParameters = @{
     Version = $version
@@ -195,10 +195,29 @@ function InstallDotNet([string] $dotnetRoot, [string] $version, [string] $archit
   if ($runtime) { $installParameters.Runtime = $runtime }
   if ($skipNonVersionedFiles) { $installParameters.SkipNonVersionedFiles = $skipNonVersionedFiles }
 
-  & $installScript @installParameters
+  try {
-  if ($lastExitCode -ne 0) {
+    & $installScript @installParameters
-    Write-PipelineTelemetryError -Category "InitializeToolset" -Message "Failed to install dotnet cli (exit code '$lastExitCode')."
+  }
-    ExitWithExitCode $lastExitCode
+  catch {
+    Write-PipelineTelemetryError -Category "InitializeToolset" -Message "Failed to install dotnet runtime '$runtime' from public location."
+
+    # Only the runtime can be installed from a custom [private] location.
+    if ($runtime -and ($runtimeSourceFeed -or $runtimeSourceFeedKey)) {
+      if ($runtimeSourceFeed) { $installParameters.AzureFeed = $runtimeSourceFeed }
+
+      if ($runtimeSourceFeedKey) {
+        $decodedBytes = [System.Convert]::FromBase64String($runtimeSourceFeedKey)
+        $decodedString = [System.Text.Encoding]::UTF8.GetString($decodedBytes)
+        $installParameters.FeedCredential = $decodedString
+      }
+
+      try {
+        & $installScript @installParameters
+      }
+      catch {
+        Write-PipelineTelemetryError -Category "InitializeToolset" -Message "Failed to install dotnet runtime '$runtime' from custom location '$runtimeSourceFeed'."
+      }
+    }
   }
 }
 
@@ -427,6 +446,9 @@ function GetSdkTaskProject([string]$taskName) {
 }
 
 function InitializeNativeTools() {
+  if ($env:DisableNativeToolsetInstalls) {
+    return
+  }
   if (Get-Member -InputObject $GlobalJson -Name "native-tools") {
     $nativeArgs= @{}
     if ($ci) {
@@ -504,6 +526,11 @@ function MSBuild() {
     # https://github.com/dotnet/arcade/issues/3932
     if ($ci -and $buildTool.Tool -eq "dotnet") {
       dotnet nuget locals http-cache -c
+
+      $env:NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS = 20
+      $env:NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS = 20
+      Write-PipelineSetVariable -Name 'NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS' -Value '20'
+      Write-PipelineSetVariable -Name 'NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS' -Value '20'
     }
 
     $toolsetBuildProject = InitializeToolset
@@ -603,3 +630,12 @@ Write-PipelineSetVariable -Name 'Artifacts.Toolset' -Value $ToolsetDir
 Write-PipelineSetVariable -Name 'Artifacts.Log' -Value $LogDir
 Write-PipelineSetVariable -Name 'TEMP' -Value $TempDir
 Write-PipelineSetVariable -Name 'TMP' -Value $TempDir
+
+# Import custom tools configuration, if present in the repo.
+# Note: Import in global scope so that the script set top-level variables without qualification.
+if (!$disableConfigureToolsetImport) {
+  $configureToolsetScript = Join-Path $EngRoot "configure-toolset.ps1"
+  if (Test-Path $configureToolsetScript) {
+      . $configureToolsetScript
+  }
+}

eng/common/tools.sh

@@ -4,6 +4,7 @@
 
 # CI mode - set to true on CI server for PR validation build or official build.
 ci=${ci:-false}
+disable_configure_toolset_import=${disable_configure_toolset_import:-null}
 
 # Set to true to use the pipelines logger which will enable Azure logging output.
 # https://github.com/Microsoft/azure-pipelines-tasks/blob/master/docs/authoring/commands.md
@@ -152,15 +153,6 @@ function InitializeDotNetCli {
   # build steps from using anything other than what we've downloaded.
   Write-PipelinePrependPath -path "$dotnet_root"
 
-  # Work around issues with Azure Artifacts credential provider
-  # https://github.com/dotnet/arcade/issues/3932
-  if [[ "$ci" == true ]]; then
-    export NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS=20
-    export NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS=20
-    Write-PipelineSetVariable -name "NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS" -value "20"
-    Write-PipelineSetVariable -name "NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS" -value "20"
-  fi
-
   Write-PipelineSetVariable -name "DOTNET_MULTILEVEL_LOOKUP" -value "0"
   Write-PipelineSetVariable -name "DOTNET_SKIP_FIRST_TIME_EXPERIENCE" -value "1"
 
@@ -200,8 +192,28 @@ function InstallDotNet {
   fi
   bash "$install_script" --version $version --install-dir "$root" $archArg $runtimeArg $skipNonVersionedFilesArg || {
     local exit_code=$?
-    Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK (exit code '$exit_code')."
+    Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK from public location (exit code '$exit_code')."
-    ExitWithExitCode $exit_code
+
+    if [[ -n "$runtimeArg" ]]; then
+      local runtimeSourceFeed=''
+      if [[ -n "${6:-}" ]]; then
+        runtimeSourceFeed="--azure-feed $6"
+      fi
+
+      local runtimeSourceFeedKey=''
+      if [[ -n "${7:-}" ]]; then
+        decodedFeedKey=`echo $7 | base64 --decode`
+        runtimeSourceFeedKey="--feed-credential $decodedFeedKey"
+      fi
+
+      if [[ -n "$runtimeSourceFeed" || -n "$runtimeSourceFeedKey" ]]; then
+        bash "$install_script" --version $version --install-dir "$root" $archArg $runtimeArg $skipNonVersionedFilesArg $runtimeSourceFeed $runtimeSourceFeedKey || {
+          local exit_code=$?
+          Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK from custom location '$runtimeSourceFeed' (exit code '$exit_code')."
+          ExitWithExitCode $exit_code
+        }
+      fi
+    fi
   }
 }
 
@@ -261,6 +273,9 @@ function GetNuGetPackageCachePath {
 }
 
 function InitializeNativeTools() {
+  if [[ -z "${DisableNativeToolsetInstalls:-}" ]]; then
+    return
+  fi
   if grep -Fq "native-tools" $global_json_file
   then
     local nativeArgs=""
@@ -341,7 +356,12 @@ function MSBuild {
     # Work around issues with Azure Artifacts credential provider
     # https://github.com/dotnet/arcade/issues/3932
     if [[ "$ci" == true ]]; then
-      dotnet nuget locals http-cache -c
+      "$_InitializeBuildTool" nuget locals http-cache -c
+
+      export NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS=20
+      export NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS=20
+      Write-PipelineSetVariable -name "NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS" -value "20"
+      Write-PipelineSetVariable -name "NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS" -value "20"
     fi
 
     local toolset_dir="${_InitializeToolset%/*}"
@@ -415,3 +435,18 @@ Write-PipelineSetVariable -name "Artifacts.Toolset" -value "$toolset_dir"
 Write-PipelineSetVariable -name "Artifacts.Log" -value "$log_dir"
 Write-PipelineSetVariable -name "Temp" -value "$temp_dir"
 Write-PipelineSetVariable -name "TMP" -value "$temp_dir"
+
+# Import custom tools configuration, if present in the repo.
+if [[ "$disable_configure_toolset_import" != null ]]; then
+  configure_toolset_script="$eng_root/configure-toolset.sh"
+  if [[ -a "$configure_toolset_script" ]]; then
+    . "$configure_toolset_script"
+  fi
+fi
+
+# TODO: https://github.com/dotnet/arcade/issues/1468
+# Temporary workaround to avoid breaking change.
+# Remove once repos are updated.
+if [[ -n "${useInstalledDotNetCli:-}" ]]; then
+  use_installed_dotnet_cli="$useInstalledDotNetCli"
+fi

global.json

@@ -3,6 +3,6 @@
     "dotnet": "5.0.100-alpha1-014915"
   },
   "msbuild-sdks": {
-    "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19474.3"
+    "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19523.3"
   }
 }