Update dependencies from https://github.com/dotnet/arcade build 20190821.1 (#4133)

- Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19421.1

eng/Version.Details.xml

@@ -65,9 +65,9 @@
     </Dependency>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="1.0.0-beta.19377.2">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="1.0.0-beta.19421.1">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>8b670142ae1b65aa4d834257b0b64d4f69f30eb8</Sha>
+      <Sha>7aa107d818fe87e627154c0331d6de5d47f39a45</Sha>
     </Dependency>
   </ToolsetDependencies>
 </Dependencies>

eng/common/init-tools-native.sh

@@ -70,8 +70,7 @@ function ReadGlobalJsonNativeTools {
   # Only extract the contents of the object.
   local native_tools_list=$(echo $native_tools_section | awk -F"[{}]" '{print $2}')
   native_tools_list=${native_tools_list//[\" ]/}
-  native_tools_list=${native_tools_list//,/$'\n'}
+  native_tools_list=$( echo "$native_tools_list" | sed 's/\s//g' | sed 's/,/\n/g' )
-  native_tools_list="$(echo -e "${native_tools_list}" | tr -d '[[:space:]]')"
 
   local old_IFS=$IFS
   while read -r line; do
@@ -108,6 +107,7 @@ else
     installer_command+=" --baseuri $base_uri"
     installer_command+=" --installpath $install_bin"
     installer_command+=" --version $tool_version"
+    echo $installer_command
 
     if [[ $force = true ]]; then
       installer_command+=" --force"

eng/common/native/install-cmake-test.sh

@@ -0,0 +1,117 @@
+#!/usr/bin/env bash
+
+source="${BASH_SOURCE[0]}"
+scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+
+. $scriptroot/common-library.sh
+
+base_uri=
+install_path=
+version=
+clean=false
+force=false
+download_retries=5
+retry_wait_time_seconds=30
+
+while (($# > 0)); do
+  lowerI="$(echo $1 | awk '{print tolower($0)}')"
+  case $lowerI in
+    --baseuri)
+      base_uri=$2
+      shift 2
+      ;;
+    --installpath)
+      install_path=$2
+      shift 2
+      ;;
+    --version)
+      version=$2
+      shift 2
+      ;;
+    --clean)
+      clean=true
+      shift 1
+      ;;
+    --force)
+      force=true
+      shift 1
+      ;;
+    --downloadretries)
+      download_retries=$2
+      shift 2
+      ;;
+    --retrywaittimeseconds)
+      retry_wait_time_seconds=$2
+      shift 2
+      ;;
+    --help)
+      echo "Common settings:"
+      echo "  --baseuri <value>        Base file directory or Url wrom which to acquire tool archives"
+      echo "  --installpath <value>    Base directory to install native tool to"
+      echo "  --clean                  Don't install the tool, just clean up the current install of the tool"
+      echo "  --force                  Force install of tools even if they previously exist"
+      echo "  --help                   Print help and exit"
+      echo ""
+      echo "Advanced settings:"
+      echo "  --downloadretries        Total number of retry attempts"
+      echo "  --retrywaittimeseconds   Wait time between retry attempts in seconds"
+      echo ""
+      exit 0
+      ;;
+  esac
+done
+
+tool_name="cmake-test"
+tool_os=$(GetCurrentOS)
+tool_folder=$(echo $tool_os | awk '{print tolower($0)}')
+tool_arch="x86_64"
+tool_name_moniker="$tool_name-$version-$tool_os-$tool_arch"
+tool_install_directory="$install_path/$tool_name/$version"
+tool_file_path="$tool_install_directory/$tool_name_moniker/bin/$tool_name"
+shim_path="$install_path/$tool_name.sh"
+uri="${base_uri}/$tool_folder/$tool_name/$tool_name_moniker.tar.gz"
+
+# Clean up tool and installers
+if [[ $clean = true ]]; then
+  echo "Cleaning $tool_install_directory"
+  if [[ -d $tool_install_directory ]]; then
+    rm -rf $tool_install_directory
+  fi
+
+  echo "Cleaning $shim_path"
+  if [[ -f $shim_path ]]; then
+    rm -rf $shim_path
+  fi
+
+  tool_temp_path=$(GetTempPathFileName $uri)
+  echo "Cleaning $tool_temp_path"
+  if [[ -f $tool_temp_path ]]; then
+    rm -rf $tool_temp_path
+  fi
+
+  exit 0
+fi
+
+# Install tool
+if [[ -f $tool_file_path ]] && [[ $force = false ]]; then
+  echo "$tool_name ($version) already exists, skipping install"
+  exit 0
+fi
+
+DownloadAndExtract $uri $tool_install_directory $force $download_retries $retry_wait_time_seconds
+
+if [[ $? != 0 ]]; then
+  echo "Installation failed" >&2
+  exit 1
+fi
+
+# Generate Shim
+# Always rewrite shims so that we are referencing the expected version
+NewScriptShim $shim_path $tool_file_path true
+
+if [[ $? != 0 ]]; then
+  echo "Shim generation failed" >&2
+  exit 1
+fi
+
+exit 0

eng/common/native/install-cmake.sh

@@ -69,7 +69,7 @@ tool_name_moniker="$tool_name-$version-$tool_os-$tool_arch"
 tool_install_directory="$install_path/$tool_name/$version"
 tool_file_path="$tool_install_directory/$tool_name_moniker/bin/$tool_name"
 shim_path="$install_path/$tool_name.sh"
-uri="${base_uri}/$tool_folder/cmake/$tool_name_moniker.tar.gz"
+uri="${base_uri}/$tool_folder/$tool_name/$tool_name_moniker.tar.gz"
 
 # Clean up tool and installers
 if [[ $clean = true ]]; then

eng/common/performance/performance-setup.ps1

@@ -2,7 +2,7 @@ Param(
     [string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY,
     [string] $CoreRootDirectory,
     [string] $Architecture="x64",
-    [string] $Framework="netcoreapp3.0",
+    [string] $Framework="netcoreapp5.0",
     [string] $CompilationMode="Tiered",
     [string] $Repository=$env:BUILD_REPOSITORY_NAME,
     [string] $Branch=$env:BUILD_SOURCEBRANCH,
@@ -33,7 +33,7 @@ if ($Framework.StartsWith("netcoreapp")) {
 }
 
 if ($Internal) {
-    $Queue = "Windows.10.Amd64.ClientRS5.Perf"
+    $Queue = "Windows.10.Amd64.19H1.Tiger.Perf"
     $PerfLabArguments = "--upload-to-perflab-container"
     $ExtraBenchmarkDotNetArguments = ""
     $Creator = ""

eng/common/performance/performance-setup.sh

@@ -3,7 +3,7 @@
 source_directory=$BUILD_SOURCESDIRECTORY
 core_root_directory=
 architecture=x64
-framework=netcoreapp3.0
+framework=netcoreapp5.0
 compilation_mode=tiered
 repository=$BUILD_REPOSITORY_NAME
 branch=$BUILD_SOURCEBRANCH
@@ -132,7 +132,7 @@ if [[ "$internal" == true ]]; then
     if [[ "$architecture" = "arm64" ]]; then
         queue=Ubuntu.1804.Arm64.Perf
     else
-        queue=Ubuntu.1804.Amd64.Perf
+        queue=Ubuntu.1804.Amd64.Tiger.Perf
     fi
 fi
 
@@ -156,21 +156,26 @@ if [[ "$use_core_run" = true ]]; then
     mv $core_root_directory $new_core_root
 fi
 
+ci=true
+
+_script_dir=$(pwd)/eng/common
+. "$_script_dir/pipeline-logging-functions.sh"
+
 # Make sure all of our variables are available for future steps
-echo "##vso[task.setvariable variable=UseCoreRun]$use_core_run"
+Write-PipelineSetVariable -name "UseCoreRun" -value "$use_core_run" -is_multi_job_variable false
-echo "##vso[task.setvariable variable=Architecture]$architecture"
+Write-PipelineSetVariable -name "Architecture" -value "$architecture" -is_multi_job_variable false
-echo "##vso[task.setvariable variable=PayloadDirectory]$payload_directory"
+Write-PipelineSetVariable -name "PayloadDirectory" -value "$payload_directory" -is_multi_job_variable false
-echo "##vso[task.setvariable variable=PerformanceDirectory]$performance_directory"
+Write-PipelineSetVariable -name "PerformanceDirectory" -value "$performance_directory" -is_multi_job_variable false
-echo "##vso[task.setvariable variable=WorkItemDirectory]$workitem_directory"
+Write-PipelineSetVariable -name "WorkItemDirectory" -value "$workitem_directory" -is_multi_job_variable false
-echo "##vso[task.setvariable variable=Queue]$queue"
+Write-PipelineSetVariable -name "Queue" -value "$queue" -is_multi_job_variable false
-echo "##vso[task.setvariable variable=SetupArguments]$setup_arguments"
+Write-PipelineSetVariable -name "SetupArguments" -value "$setup_arguments" -is_multi_job_variable false
-echo "##vso[task.setvariable variable=Python]python3"
+Write-PipelineSetVariable -name "Python" -value "$python3" -is_multi_job_variable false
-echo "##vso[task.setvariable variable=PerfLabArguments]$perflab_arguments"
+Write-PipelineSetVariable -name "PerfLabArguments" -value "$perflab_arguments" -is_multi_job_variable false
-echo "##vso[task.setvariable variable=ExtraBenchmarkDotNetArguments]$extra_benchmark_dotnet_arguments"
+Write-PipelineSetVariable -name "ExtraBenchmarkDotNetArguments" -value "$extra_benchmark_dotnet_arguments" -is_multi_job_variable false
-echo "##vso[task.setvariable variable=BDNCategories]$run_categories"
+Write-PipelineSetVariable -name "BDNCategories" -value "$run_categories" -is_multi_job_variable false
-echo "##vso[task.setvariable variable=TargetCsproj]$csproj"
+Write-PipelineSetVariable -name "TargetCsproj" -value "$csproj" -is_multi_job_variable false
-echo "##vso[task.setvariable variable=RunFromPerfRepo]$run_from_perf_repo"
+Write-PipelineSetVariable -name "RunFromPerfRepo" -value "$run_from_perf_repo" -is_multi_job_variable false
-echo "##vso[task.setvariable variable=Creator]$creator"
+Write-PipelineSetVariable -name "Creator" -value "$creator" -is_multi_job_variable false
-echo "##vso[task.setvariable variable=HelixSourcePrefix]$helix_source_prefix"
+Write-PipelineSetVariable -name "HelixSourcePrefix" -value "$helix_source_prefix" -is_multi_job_variable false
-echo "##vso[task.setvariable variable=Kind]$kind"
+Write-PipelineSetVariable -name "Kind" -value "$kind" -is_multi_job_variable false
-echo "##vso[task.setvariable variable=_BuildConfig]$architecture.$kind.$framework"
+Write-PipelineSetVariable -name "_BuildConfig" -value "$architecture.$kind.$framework" -is_multi_job_variable false

eng/common/post-build/darc-gather-drop.ps1

@@ -19,7 +19,17 @@ try {
     ExitWithExitCode $exitCode
   }
 
+  # For now, only use a dry run.
+  # Ideally we would change darc to enable a quick request that
+  # would check whether the file exists that you can download it,
+  # and that it won't conflict with other files.
+  # https://github.com/dotnet/arcade/issues/3674
+  # Right now we can't remove continue-on-error because we ocassionally will have
+  # dependencies that have no associated builds (e.g. an old dependency).
+  # We need to add an option to baseline specific dependencies away, or add them manually
+  # to the BAR.
   darc gather-drop --non-shipping `
+    --dry-run `
     --continue-on-error `
     --id $BarBuildId `
     --output-dir $DropLocation `

eng/common/post-build/dotnetsymbol-init.ps1

@@ -1,29 +0,0 @@
-param (
-  $dotnetsymbolVersion = $null
-)
-
-$ErrorActionPreference = "Stop"
-Set-StrictMode -Version 2.0
-
-. $PSScriptRoot\..\tools.ps1
-
-$verbosity = "minimal"
-
-function Installdotnetsymbol ($dotnetsymbolVersion) {
-  $dotnetsymbolPackageName = "dotnet-symbol"
-
-  $dotnetRoot = InitializeDotNetCli -install:$true
-  $dotnet = "$dotnetRoot\dotnet.exe"
-  $toolList = & "$dotnet" tool list --global
-
-  if (($toolList -like "*$dotnetsymbolPackageName*") -and ($toolList -like "*$dotnetsymbolVersion*")) {
-    Write-Host "dotnet-symbol version $dotnetsymbolVersion is already installed."
-  }
-  else {
-    Write-Host "Installing dotnet-symbol version $dotnetsymbolVersion..."
-    Write-Host "You may need to restart your command window if this is the first dotnet tool you have installed."
-    & "$dotnet" tool install $dotnetsymbolPackageName --version $dotnetsymbolVersion --verbosity $verbosity --global
-  }
-}
-
-Installdotnetsymbol $dotnetsymbolVersion

eng/common/post-build/sourcelink-cli-init.ps1

@@ -1,29 +0,0 @@
-param (
-  $sourcelinkCliVersion = $null
-)
-
-$ErrorActionPreference = "Stop"
-Set-StrictMode -Version 2.0
-
-. $PSScriptRoot\..\tools.ps1
-
-$verbosity = "minimal"
-
-function InstallSourcelinkCli ($sourcelinkCliVersion) {
-  $sourcelinkCliPackageName = "sourcelink"
-
-  $dotnetRoot = InitializeDotNetCli -install:$true
-  $dotnet = "$dotnetRoot\dotnet.exe"
-  $toolList = & "$dotnet" tool list --global
-
-  if (($toolList -like "*$sourcelinkCliPackageName*") -and ($toolList -like "*$sourcelinkCliVersion*")) {
-    Write-Host "SourceLink CLI version $sourcelinkCliVersion is already installed."
-  }
-  else {
-    Write-Host "Installing SourceLink CLI version $sourcelinkCliVersion..."
-    Write-Host "You may need to restart your command window if this is the first dotnet tool you have installed."
-    & "$dotnet" tool install $sourcelinkCliPackageName --version $sourcelinkCliVersion --verbosity $verbosity --global 
-  }
-}
-
-InstallSourcelinkCli $sourcelinkCliVersion

eng/common/post-build/sourcelink-validation.ps1

@@ -1,8 +1,8 @@
 param(
   [Parameter(Mandatory=$true)][string] $InputPath,              # Full path to directory where Symbols.NuGet packages to be checked are stored
   [Parameter(Mandatory=$true)][string] $ExtractPath,            # Full path to directory where the packages will be extracted during validation
-  [Parameter(Mandatory=$true)][string] $GHRepoName,             # GitHub name of the repo including the Org. E.g., dotnet/arcade
+  [Parameter(Mandatory=$false)][string] $GHRepoName,            # GitHub name of the repo including the Org. E.g., dotnet/arcade
-  [Parameter(Mandatory=$true)][string] $GHCommit,               # GitHub commit SHA used to build the packages
+  [Parameter(Mandatory=$false)][string] $GHCommit,              # GitHub commit SHA used to build the packages
   [Parameter(Mandatory=$true)][string] $SourcelinkCliVersion    # Version of SourceLink CLI to use
 )
 
@@ -13,6 +13,12 @@ param(
 # all files present in the repo at a specific commit point.
 $global:RepoFiles = @{}
 
+# Maximum number of jobs to run in parallel
+$MaxParallelJobs = 6
+
+# Wait time between check for system load
+$SecondsBetweenLoadChecks = 10
+
 $ValidatePackage = {
   param( 
     [string] $PackagePath                                 # Full path to a Symbols.NuGet package
@@ -22,8 +28,8 @@ $ValidatePackage = {
 
   # Ensure input file exist
   if (!(Test-Path $PackagePath)) {
-    Write-PipelineTaskError "Input file does not exist: $PackagePath"
+    Write-Host "Input file does not exist: $PackagePath"
-    ExitWithExitCode 1
+    return 1
   }
 
   # Extensions for which we'll look for SourceLink information
@@ -38,7 +44,7 @@ $ValidatePackage = {
 
   Add-Type -AssemblyName System.IO.Compression.FileSystem
 
-  [System.IO.Directory]::CreateDirectory($ExtractPath);
+  [System.IO.Directory]::CreateDirectory($ExtractPath)  | Out-Null
 
   try {
     $zip = [System.IO.Compression.ZipFile]::OpenRead($PackagePath)
@@ -138,16 +144,18 @@ $ValidatePackage = {
 
   if ($FailedFiles -eq 0) {
     Write-Host "Passed."
+    return 0
   }
   else {
-    Write-PipelineTaskError "$PackagePath has broken SourceLink links."
+    Write-Host "$PackagePath has broken SourceLink links."
+    return 1
   }
 }
 
 function ValidateSourceLinkLinks {
-  if (!($GHRepoName -Match "^[^\s\/]+/[^\s\/]+$")) {
+  if ($GHRepoName -ne "" -and !($GHRepoName -Match "^[^\s\/]+/[^\s\/]+$")) {
     if (!($GHRepoName -Match "^[^\s-]+-[^\s]+$")) {
-      Write-PipelineTaskError "GHRepoName should be in the format <org>/<repo> or <org>-<repo>"
+      Write-PipelineTaskError "GHRepoName should be in the format <org>/<repo> or <org>-<repo>. '$GHRepoName'"
       ExitWithExitCode 1
     }
     else {
@@ -155,30 +163,33 @@ function ValidateSourceLinkLinks {
     }
   }
 
-  if (!($GHCommit -Match "^[0-9a-fA-F]{40}$")) {
+  if ($GHCommit -ne "" -and !($GHCommit -Match "^[0-9a-fA-F]{40}$")) {
-    Write-PipelineTaskError "GHCommit should be a 40 chars hexadecimal string"
+    Write-PipelineTaskError "GHCommit should be a 40 chars hexadecimal string. '$GHCommit'"
     ExitWithExitCode 1
   }
 
-  $RepoTreeURL = -Join("http://api.github.com/repos/", $GHRepoName, "/git/trees/", $GHCommit, "?recursive=1")
+  if ($GHRepoName -ne "" -and $GHCommit -ne "") {
-  $CodeExtensions = @(".cs", ".vb", ".fs", ".fsi", ".fsx", ".fsscript")
+    $RepoTreeURL = -Join("http://api.github.com/repos/", $GHRepoName, "/git/trees/", $GHCommit, "?recursive=1")
+    $CodeExtensions = @(".cs", ".vb", ".fs", ".fsi", ".fsx", ".fsscript")
 
-  try {
+    try {
-    # Retrieve the list of files in the repo at that particular commit point and store them in the RepoFiles hash
+      # Retrieve the list of files in the repo at that particular commit point and store them in the RepoFiles hash
-    $Data = Invoke-WebRequest $RepoTreeURL -UseBasicParsing | ConvertFrom-Json | Select-Object -ExpandProperty tree
+      $Data = Invoke-WebRequest $RepoTreeURL -UseBasicParsing | ConvertFrom-Json | Select-Object -ExpandProperty tree
   
-    foreach ($file in $Data) {
+      foreach ($file in $Data) {
-      $Extension = [System.IO.Path]::GetExtension($file.path)
+        $Extension = [System.IO.Path]::GetExtension($file.path)
 
-      if ($CodeExtensions.Contains($Extension)) {
+        if ($CodeExtensions.Contains($Extension)) {
-        $RepoFiles[$file.path] = 1
+          $RepoFiles[$file.path] = 1
+        }
       }
     }
+    catch {
+      Write-Host "Problems downloading the list of files from the repo. Url used: $RepoTreeURL . Execution will proceed without caching."
+    }
   }
-  catch {
+  elseif ($GHRepoName -ne "" -or $GHCommit -ne "") {
-    Write-PipelineTaskError "Problems downloading the list of files from the repo. Url used: $RepoTreeURL"
+    Write-Host "For using the http caching mechanism both GHRepoName and GHCommit should be informed."
-    Write-Host $_
-    ExitWithExitCode 1
   }
   
   if (Test-Path $ExtractPath) {
@@ -186,14 +197,33 @@ function ValidateSourceLinkLinks {
   }
 
   # Process each NuGet package in parallel
-  $Jobs = @()
   Get-ChildItem "$InputPath\*.symbols.nupkg" |
     ForEach-Object {
-      $Jobs += Start-Job -ScriptBlock $ValidatePackage -ArgumentList $_.FullName
+      Start-Job -ScriptBlock $ValidatePackage -ArgumentList $_.FullName | Out-Null
+      $NumJobs = @(Get-Job -State 'Running').Count
+      
+      while ($NumJobs -ge $MaxParallelJobs) {
+        Write-Host "There are $NumJobs validation jobs running right now. Waiting $SecondsBetweenLoadChecks seconds to check again."
+        sleep $SecondsBetweenLoadChecks
+        $NumJobs = @(Get-Job -State 'Running').Count
+      }
+
+      foreach ($Job in @(Get-Job -State 'Completed')) {
+        Receive-Job -Id $Job.Id
+        Remove-Job -Id $Job.Id
+      }
     }
 
+  $ValidationFailures = 0
   foreach ($Job in $Jobs) {
-    Wait-Job -Id $Job.Id | Receive-Job
+    $jobResult = Wait-Job -Id $Job.Id | Receive-Job
+    if ($jobResult -ne "0") {
+      $ValidationFailures++
+    }
+  }
+  if ($ValidationFailures -gt 0) {
+    Write-PipelineTaskError " $ValidationFailures package(s) failed validation."
+    ExitWithExitCode 1
   }
 }
 

eng/common/sdl/execute-all-sdl-tools.ps1

@@ -86,8 +86,8 @@ if ($TsaPublish) {
     if (-not $TsaRepositoryName) {
       $TsaRepositoryName = "$($Repository)-$($BranchName)"
     }
-    Write-Host "$guardianCliLocation tsa-publish --all-tools --repository-name `"$TsaRepositoryName`" --branch-name `"$TsaBranchName`" --build-number `"$BuildNumber`" --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $SourceDirectory --logger-level $GuardianLoggerLevel"
+    Write-Host "$guardianCliLocation tsa-publish --all-tools --repository-name `"$TsaRepositoryName`" --branch-name `"$TsaBranchName`" --build-number `"$BuildNumber`" --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $ArtifactsDirectory --logger-level $GuardianLoggerLevel"
-    & $guardianCliLocation tsa-publish --all-tools --repository-name "$TsaRepositoryName" --branch-name "$TsaBranchName" --build-number "$BuildNumber" --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $ArtifactsDirectory  --logger-level $GuardianLoggerLevel
+    & $guardianCliLocation tsa-publish --all-tools --repository-name "$TsaRepositoryName" --branch-name "$TsaBranchName" --build-number "$BuildNumber" --onboard $True --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $ArtifactsDirectory  --logger-level $GuardianLoggerLevel
     if ($LASTEXITCODE -ne 0) {
       Write-Host "Guardian tsa-publish failed with exit code $LASTEXITCODE."
       exit $LASTEXITCODE

eng/common/sdl/packages.config

@@ -1,4 +1,4 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
-  <package id="Microsoft.Guardian.Cli" version="0.6.0"/>
+  <package id="Microsoft.Guardian.Cli" version="0.7.1"/>
 </packages>

eng/common/sdl/run-sdl.ps1

@@ -25,19 +25,19 @@ if ($ValidPath -eq $False)
   exit 1
 }
 
+$configParam = @("--config")
+
 foreach ($tool in $ToolsList) {
   $gdnConfigFile = Join-Path $gdnConfigPath "$tool-configure.gdnconfig"
-  $config = $False
   Write-Host $tool
   # We have to manually configure tools that run on source to look at the source directory only
   if ($tool -eq "credscan") {
-    Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" TargetDirectory : $TargetDirectory `" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams})"
+    Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" TargetDirectory : $TargetDirectory `" `" OutputType : pre `" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams})"
-    & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " TargetDirectory : $TargetDirectory " $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams})
+    & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " TargetDirectory : $TargetDirectory " "OutputType : pre" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams})
     if ($LASTEXITCODE -ne 0) {
       Write-Host "Guardian configure for $tool failed with exit code $LASTEXITCODE."
       exit $LASTEXITCODE
     }
-    $config = $True
   }
   if ($tool -eq "policheck") {
     Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" Target : $TargetDirectory `" $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams})"
@@ -46,22 +46,14 @@ foreach ($tool in $ToolsList) {
       Write-Host "Guardian configure for $tool failed with exit code $LASTEXITCODE."
       exit $LASTEXITCODE
     }
-    $config = $True
   }
 
-  Write-Host "$GuardianCliLocation run --working-directory $WorkingDirectory --tool $tool --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel --config $gdnConfigFile $config"
+  $configParam+=$gdnConfigFile
-  if ($config) {
-    & $GuardianCliLocation run --working-directory $WorkingDirectory --tool $tool --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel --config $gdnConfigFile
-    if ($LASTEXITCODE -ne 0) {
-      Write-Host "Guardian run for $tool using $gdnConfigFile failed with exit code $LASTEXITCODE."
-      exit $LASTEXITCODE
-    }
-  } else {
-    & $GuardianCliLocation run --working-directory $WorkingDirectory --tool $tool --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel
-    if ($LASTEXITCODE -ne 0) {
-      Write-Host "Guardian run for $tool failed with exit code $LASTEXITCODE."
-      exit $LASTEXITCODE
-    }
-  }
 }
 
+Write-Host "$GuardianCliLocation run --working-directory $WorkingDirectory --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel $configParam"
+& $GuardianCliLocation run --working-directory $WorkingDirectory --tool $tool --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel $configParam
+if ($LASTEXITCODE -ne 0) {
+  Write-Host "Guardian run for $ToolsList using $configParam failed with exit code $LASTEXITCODE."
+  exit $LASTEXITCODE
+}

eng/common/templates/job/execute-sdl.yml

@@ -10,6 +10,8 @@ jobs:
   displayName: Run SDL tool
   variables:
     - group: DotNet-VSTS-Bot
+  pool:
+    name: Hosted VS2017
   steps:
   - checkout: self
     clean: true
@@ -46,7 +48,7 @@ jobs:
       continueOnError: ${{ parameters.continueOnError }}
   - ${{ if eq(parameters.overrideParameters, '') }}:
     - powershell: eng/common/sdl/execute-all-sdl-tools.ps1
-        -GuardianPackageName Microsoft.Guardian.Cli.0.6.0
+        -GuardianPackageName Microsoft.Guardian.Cli.0.7.1
         -NugetPackageDirectory $(Build.SourcesDirectory)\.packages
         -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw)
         ${{ parameters.additionalParameters }}

eng/common/templates/job/job.yml

@@ -37,6 +37,9 @@ parameters:
   # Optional: Enable publishing to the build asset registry
   enablePublishBuildAssets: false
 
+  # Optional: Prevent gather/push manifest from executing when using publishing pipelines
+  enablePublishUsingPipelines: false
+
   # Optional: Include PublishTestResults task
   enablePublishTestResults: false
 
@@ -187,7 +190,7 @@ jobs:
       continueOnError: true
       condition: always()
     
-  - ${{ if and(eq(parameters.enablePublishBuildAssets, true), ne(variables['_PublishUsingPipelines'], 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+  - ${{ if and(eq(parameters.enablePublishBuildAssets, true), ne(parameters.enablePublishUsingPipelines, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
     - task: CopyFiles@2
       displayName: Gather Asset Manifests
       inputs:
@@ -195,6 +198,7 @@ jobs:
         TargetFolder: '$(Build.StagingDirectory)/AssetManifests'
       continueOnError: ${{ parameters.continueOnError }}
       condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true'))
+
     - task: PublishBuildArtifacts@1
       displayName: Push Asset Manifests
       inputs:

eng/common/templates/post-build/channels/internal-servicing.yml

@@ -1,5 +1,7 @@
 parameters:
   enableSymbolValidation: true
+  symbolPublishingAdditionalParameters: ''
+  artifactsPublishingAdditionalParameters: ''
 
 stages:
 - stage: IS_Publish
@@ -13,17 +15,23 @@ stages:
   - job:
     displayName: Symbol Publishing
     dependsOn: setupMaestroVars
-    condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.InternalServicing_30_Channel_Id)
+    condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.InternalServicing_30_Channel_Id))
     variables:
       - group: DotNet-Symbol-Server-Pats
     pool:
       vmImage: 'windows-2019'
     steps:
       - task: DownloadBuildArtifacts@0
-        displayName: Download Artifacts
+        displayName: Download Blob Artifacts
         inputs:
-          downloadType: specific files
+          artifactName: 'BlobArtifacts'
-          matchingPattern: "*Artifacts*"
+        continueOnError: true
+
+      - task: DownloadBuildArtifacts@0
+        displayName: Download PDB Artifacts
+        inputs:
+          artifactName: 'PDBArtifacts'
+        continueOnError: true
 
       - task: PowerShell@2
         displayName: Publish
@@ -34,7 +42,9 @@ stages:
             /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) 
             /p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/'
             /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
+            /p:SymbolPublishingExclusionsFile='$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt'
             /p:Configuration=Release
+            ${{ parameters.symbolPublishingAdditionalParameters }}
 
   - job: publish_assets
     displayName: Publish Assets
@@ -46,7 +56,7 @@ stages:
         value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ]
       - name: IsStableBuild
         value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ]
-    condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.InternalServicing_30_Channel_Id)
+    condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.InternalServicing_30_Channel_Id))
     pool:
       vmImage: 'windows-2019'
     steps:
@@ -92,6 +102,7 @@ stages:
             /p:BlobBasePath='$(Build.ArtifactStagingDirectory)\BlobArtifacts' 
             /p:PackageBasePath='$(Build.ArtifactStagingDirectory)\PackageArtifacts' 
             /p:Configuration=Release 
+            ${{ parameters.artifactsPublishingAdditionalParameters }}
         
       - task: NuGetCommand@2
         displayName: Publish Packages to AzDO Feed
@@ -126,7 +137,7 @@ stages:
     - job:
       displayName: Symbol Availability
       dependsOn: setupMaestroVars
-      condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.InternalServicing_30_Channel_Id)
+      condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.InternalServicing_30_Channel_Id))
       pool:
         vmImage: 'windows-2019'
       steps:

eng/common/templates/post-build/channels/netcore-dev-5.yml

@@ -0,0 +1,165 @@
+parameters:
+  enableSymbolValidation: true
+  symbolPublishingAdditionalParameters: ''
+  artifactsPublishingAdditionalParameters: ''
+  publishInstallersAndChecksums: false
+
+stages:
+- stage: NetCore_Dev5_Publish
+  dependsOn: validate
+  variables:
+    - template: ../common-variables.yml
+  displayName: .NET Core 5 Dev Channel
+  jobs:
+  - template: ../setup-maestro-vars.yml
+
+  - job:
+    displayName: Symbol Publishing
+    dependsOn: setupMaestroVars
+    condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.NetCore_5_Dev_Channel_Id))
+    variables:
+      - group: DotNet-Symbol-Server-Pats
+    pool:
+      vmImage: 'windows-2019'
+    steps:
+      - task: DownloadBuildArtifacts@0
+        displayName: Download Blob Artifacts
+        inputs:
+          artifactName: 'BlobArtifacts'
+        continueOnError: true
+
+      - task: DownloadBuildArtifacts@0
+        displayName: Download PDB Artifacts
+        inputs:
+          artifactName: 'PDBArtifacts'
+        continueOnError: true
+
+      - task: PowerShell@2
+        displayName: Publish
+        inputs:
+          filePath: eng\common\sdk-task.ps1
+          arguments: -task PublishToSymbolServers -restore -msbuildEngine dotnet
+            /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) 
+            /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) 
+            /p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/'
+            /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
+            /p:SymbolPublishingExclusionsFile='$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt'
+            /p:Configuration=Release
+            ${{ parameters.symbolPublishingAdditionalParameters }}
+
+  - job:
+    displayName: Publish Assets
+    dependsOn: setupMaestroVars
+    variables:
+      - group: DotNet-Blob-Feed
+      - group: AzureDevOps-Artifact-Feeds-Pats
+      - name: BARBuildId
+        value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ]
+      - name: IsStableBuild
+        value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ]
+    condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.NetCore_5_Dev_Channel_Id))
+    pool:
+      vmImage: 'windows-2019'
+    steps:
+      - task: DownloadBuildArtifacts@0
+        displayName: Download Package Artifacts
+        inputs:
+          buildType: current
+          artifactName: PackageArtifacts
+
+      - task: DownloadBuildArtifacts@0
+        displayName: Download Blob Artifacts
+        inputs:
+          buildType: current
+          artifactName: BlobArtifacts
+
+      - task: DownloadBuildArtifacts@0
+        displayName: Download Asset Manifests
+        inputs:
+          buildType: current
+          artifactName: AssetManifests
+
+      - task: PowerShell@2
+        displayName: Add Assets Location
+        env:
+          AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-unviersal-packages-rw)
+        inputs:
+          filePath: eng\common\sdk-task.ps1
+          arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet 
+            /p:ChannelId=$(NetCore_5_Dev_Channel_Id)
+            /p:ArtifactsCategory=$(_DotNetArtifactsCategory)
+            /p:IsStableBuild=$(IsStableBuild)
+            /p:IsInternalBuild=$(IsInternalBuild)
+            /p:RepositoryName=$(Build.Repository.Name)
+            /p:CommitSha=$(Build.SourceVersion)
+            /p:NugetPath=$(Agent.BuildDirectory)\Nuget\NuGet.exe
+            /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' 
+            /p:TargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' 
+            /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' 
+            /p:BARBuildId=$(BARBuildId) 
+            /p:MaestroApiEndpoint='$(MaestroApiEndPoint)' 
+            /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)' 
+            /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' 
+            /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' 
+            /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' 
+            /p:Configuration=Release
+            /p:InstallersTargetStaticFeed=$(InstallersBlobFeedUrl)
+            /p:InstallersAzureAccountKey=$(dotnetcli-storage-key)
+            /p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }}
+            /p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl)
+            /p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key)
+            ${{ parameters.artifactsPublishingAdditionalParameters }}
+        
+      - task: NuGetCommand@2
+        displayName: Publish Packages to AzDO Feed
+        condition: contains(variables['TargetAzDOFeed'], 'pkgs.visualstudio.com')
+        inputs:
+          command: push
+          vstsFeed: $(AzDoFeedName)
+          packagesToPush: $(Build.ArtifactStagingDirectory)\PackageArtifacts\*.nupkg
+          publishVstsFeed: $(AzDoFeedName)
+
+      - task: PowerShell@2
+        displayName: Publish Blobs to AzDO Feed
+        inputs:
+          filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-blobs-to-azdo.ps1
+          arguments: -FeedName $(AzDoFeedName) 
+            -SourceFolderCollection $(Build.ArtifactStagingDirectory)/BlobArtifacts/
+            -PersonalAccessToken $(dn-bot-dnceng-unviersal-packages-rw)
+        enabled: false
+
+
+- stage: NetCore_Dev5_PublishValidation
+  displayName: Publish Validation
+  variables:
+    - template: ../common-variables.yml  
+  jobs:
+  - template: ../setup-maestro-vars.yml
+
+  - ${{ if eq(parameters.enableSymbolValidation, 'true') }}:
+    - job:
+      displayName: Symbol Availability
+      dependsOn: setupMaestroVars
+      condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.NetCore_5_Dev_Channel_Id))
+      pool:
+        vmImage: 'windows-2019'
+      steps:
+        - task: DownloadBuildArtifacts@0
+          displayName: Download Package Artifacts
+          inputs:
+            buildType: current
+            artifactName: PackageArtifacts
+
+        - task: PowerShell@2
+          displayName: Check Symbol Availability
+          inputs:
+            filePath: $(Build.SourcesDirectory)/eng/common/post-build/symbols-validation.ps1
+            arguments: -InputPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ -ExtractPath $(Agent.BuildDirectory)/Temp/ -DotnetSymbolVersion $(SymbolToolVersion)
+
+  - template: ../darc-gather-drop.yml
+    parameters:
+      ChannelId: ${{ variables.NetCore_5_Dev_Channel_Id }}
+
+  - template: ../promote-build.yml	
+    parameters:	
+      ChannelId: ${{ variables.NetCore_5_Dev_Channel_Id }}

eng/common/templates/post-build/channels/netcore-tools-latest.yml

@@ -0,0 +1,165 @@
+parameters:
+  enableSymbolValidation: true
+  symbolPublishingAdditionalParameters: ''
+  artifactsPublishingAdditionalParameters: ''
+  publishInstallersAndChecksums: false
+
+stages:
+- stage: NetCore_Tools_Latest_Publish
+  dependsOn: validate
+  variables:
+    - template: ../common-variables.yml
+  displayName: .NET Tools - Latest
+  jobs:
+  - template: ../setup-maestro-vars.yml
+
+  - job:
+    displayName: Symbol Publishing
+    dependsOn: setupMaestroVars
+    condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.NetCore_Tools_Latest_Channel_Id))
+    variables:
+      - group: DotNet-Symbol-Server-Pats
+    pool:
+      vmImage: 'windows-2019'
+    steps:
+      - task: DownloadBuildArtifacts@0
+        displayName: Download Blob Artifacts
+        inputs:
+          artifactName: 'BlobArtifacts'
+        continueOnError: true
+
+      - task: DownloadBuildArtifacts@0
+        displayName: Download PDB Artifacts
+        inputs:
+          artifactName: 'PDBArtifacts'
+        continueOnError: true
+
+      - task: PowerShell@2
+        displayName: Publish
+        inputs:
+          filePath: eng\common\sdk-task.ps1
+          arguments: -task PublishToSymbolServers -restore -msbuildEngine dotnet
+            /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) 
+            /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) 
+            /p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/'
+            /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
+            /p:SymbolPublishingExclusionsFile='$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt'
+            /p:Configuration=Release
+            ${{ parameters.symbolPublishingAdditionalParameters }}
+
+  - job:
+    displayName: Publish Assets
+    dependsOn: setupMaestroVars
+    variables:
+      - group: DotNet-Blob-Feed
+      - group: AzureDevOps-Artifact-Feeds-Pats
+      - name: BARBuildId
+        value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ]
+      - name: IsStableBuild
+        value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ]
+    condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.NetCore_Tools_Latest_Channel_Id))
+    pool:
+      vmImage: 'windows-2019'
+    steps:
+      - task: DownloadBuildArtifacts@0
+        displayName: Download Package Artifacts
+        inputs:
+          buildType: current
+          artifactName: PackageArtifacts
+
+      - task: DownloadBuildArtifacts@0
+        displayName: Download Blob Artifacts
+        inputs:
+          buildType: current
+          artifactName: BlobArtifacts
+
+      - task: DownloadBuildArtifacts@0
+        displayName: Download Asset Manifests
+        inputs:
+          buildType: current
+          artifactName: AssetManifests
+
+      - task: PowerShell@2
+        displayName: Add Assets Location
+        env:
+          AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-unviersal-packages-rw)
+        inputs:
+          filePath: eng\common\sdk-task.ps1
+          arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet 
+            /p:ChannelId=$(NetCore_Tools_Latest_Channel_Id)
+            /p:ArtifactsCategory=$(_DotNetArtifactsCategory)
+            /p:IsStableBuild=$(IsStableBuild)
+            /p:IsInternalBuild=$(IsInternalBuild)
+            /p:RepositoryName=$(Build.Repository.Name)
+            /p:CommitSha=$(Build.SourceVersion)
+            /p:NugetPath=$(Agent.BuildDirectory)\Nuget\NuGet.exe
+            /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' 
+            /p:TargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' 
+            /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' 
+            /p:BARBuildId=$(BARBuildId) 
+            /p:MaestroApiEndpoint='$(MaestroApiEndPoint)' 
+            /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)' 
+            /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' 
+            /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' 
+            /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' 
+            /p:Configuration=Release
+            /p:InstallersTargetStaticFeed=$(InstallersBlobFeedUrl)
+            /p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }}
+            /p:InstallersAzureAccountKey=$(dotnetcli-storage-key)
+            /p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl)
+            /p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key)
+            ${{ parameters.artifactsPublishingAdditionalParameters }}
+        
+      - task: NuGetCommand@2
+        displayName: Publish Packages to AzDO Feed
+        condition: contains(variables['TargetAzDOFeed'], 'pkgs.visualstudio.com')
+        inputs:
+          command: push
+          vstsFeed: $(AzDoFeedName)
+          packagesToPush: $(Build.ArtifactStagingDirectory)\PackageArtifacts\*.nupkg
+          publishVstsFeed: $(AzDoFeedName)
+
+      - task: PowerShell@2
+        displayName: Publish Blobs to AzDO Feed
+        inputs:
+          filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-blobs-to-azdo.ps1
+          arguments: -FeedName $(AzDoFeedName) 
+            -SourceFolderCollection $(Build.ArtifactStagingDirectory)/BlobArtifacts/
+            -PersonalAccessToken $(dn-bot-dnceng-unviersal-packages-rw)
+        enabled: false
+
+
+- stage: NetCore_Tools_Latest_PublishValidation
+  displayName: Publish Validation
+  variables:
+    - template: ../common-variables.yml  
+  jobs:
+  - template: ../setup-maestro-vars.yml
+
+  - ${{ if eq(parameters.enableSymbolValidation, 'true') }}:
+    - job:
+      displayName: Symbol Availability
+      dependsOn: setupMaestroVars
+      condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.NetCore_Tools_Latest_Channel_Id))
+      pool:
+        vmImage: 'windows-2019'
+      steps:
+        - task: DownloadBuildArtifacts@0
+          displayName: Download Package Artifacts
+          inputs:
+            buildType: current
+            artifactName: PackageArtifacts
+
+        - task: PowerShell@2
+          displayName: Check Symbol Availability
+          inputs:
+            filePath: $(Build.SourcesDirectory)/eng/common/post-build/symbols-validation.ps1
+            arguments: -InputPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ -ExtractPath $(Agent.BuildDirectory)/Temp/ -DotnetSymbolVersion $(SymbolToolVersion)
+
+  - template: ../darc-gather-drop.yml
+    parameters:
+      ChannelId: ${{ variables.NetCore_Tools_Latest_Channel_Id }}
+
+  - template: ../promote-build.yml	
+    parameters:	
+      ChannelId: ${{ variables.NetCore_Tools_Latest_Channel_Id }}

eng/common/templates/post-build/channels/public-dev-release.yml

@@ -1,5 +1,8 @@
 parameters:
   enableSymbolValidation: true
+  symbolPublishingAdditionalParameters: ''
+  artifactsPublishingAdditionalParameters: ''
+  publishInstallersAndChecksums: false
 
 stages:
 - stage: Publish
@@ -13,17 +16,23 @@ stages:
   - job:
     displayName: Symbol Publishing
     dependsOn: setupMaestroVars
-    condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicDevRelease_30_Channel_Id)
+    condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.PublicDevRelease_30_Channel_Id))
     variables:
       - group: DotNet-Symbol-Server-Pats
     pool:
       vmImage: 'windows-2019'
     steps:
       - task: DownloadBuildArtifacts@0
-        displayName: Download Artifacts
+        displayName: Download Blob Artifacts
         inputs:
-          downloadType: specific files
+          artifactName: 'BlobArtifacts'
-          matchingPattern: "*Artifacts*"
+        continueOnError: true
+
+      - task: DownloadBuildArtifacts@0
+        displayName: Download PDB Artifacts
+        inputs:
+          artifactName: 'PDBArtifacts'
+        continueOnError: true
 
       - task: PowerShell@2
         displayName: Publish
@@ -34,7 +43,9 @@ stages:
             /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) 
             /p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/'
             /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
+            /p:SymbolPublishingExclusionsFile='$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt'
             /p:Configuration=Release
+            ${{ parameters.symbolPublishingAdditionalParameters }}
 
   - job:
     displayName: Publish Assets
@@ -46,7 +57,7 @@ stages:
         value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ]
       - name: IsStableBuild
         value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ]
-    condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicDevRelease_30_Channel_Id)
+    condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.PublicDevRelease_30_Channel_Id))
     pool:
       vmImage: 'windows-2019'
     steps:
@@ -92,6 +103,12 @@ stages:
             /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' 
             /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' 
             /p:Configuration=Release 
+            /p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }}
+            /p:InstallersTargetStaticFeed=$(InstallersBlobFeedUrl)
+            /p:InstallersAzureAccountKey=$(dotnetcli-storage-key)
+            /p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl)
+            /p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key)
+            ${{ parameters.artifactsPublishingAdditionalParameters }}
         
       - task: NuGetCommand@2
         displayName: Publish Packages to AzDO Feed
@@ -123,7 +140,7 @@ stages:
     - job:
       displayName: Symbol Availability
       dependsOn: setupMaestroVars
-      condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicDevRelease_30_Channel_Id)
+      condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.PublicDevRelease_30_Channel_Id))
       pool:
         vmImage: 'windows-2019'
       steps:

eng/common/templates/post-build/channels/public-release.yml

@@ -1,5 +1,7 @@
 parameters:
   enableSymbolValidation: true
+  symbolPublishingAdditionalParameters: ''
+  artifactsPublishingAdditionalParameters: ''
 
 stages:
 - stage: PubRel_Publish
@@ -13,17 +15,23 @@ stages:
   - job:
     displayName: Symbol Publishing
     dependsOn: setupMaestroVars
-    condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicRelease_30_Channel_Id)
+    condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.PublicRelease_30_Channel_Id))
     variables:
       - group: DotNet-Symbol-Server-Pats
     pool:
       vmImage: 'windows-2019'
     steps:
       - task: DownloadBuildArtifacts@0
-        displayName: Download Artifacts
+        displayName: Download Blob Artifacts
         inputs:
-          downloadType: specific files
+          artifactName: 'BlobArtifacts'
-          matchingPattern: "*Artifacts*"
+        continueOnError: true
+
+      - task: DownloadBuildArtifacts@0
+        displayName: Download PDB Artifacts
+        inputs:
+          artifactName: 'PDBArtifacts'
+        continueOnError: true
 
       - task: PowerShell@2
         displayName: Publish
@@ -34,7 +42,9 @@ stages:
             /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) 
             /p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/'
             /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
+            /p:SymbolPublishingExclusionsFile='$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt'
             /p:Configuration=Release
+            ${{ parameters.symbolPublishingAdditionalParameters }}
 
   - job: publish_assets
     displayName: Publish Assets
@@ -46,7 +56,7 @@ stages:
         value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ]
       - name: IsStableBuild
         value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ]
-    condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicRelease_30_Channel_Id)
+    condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.PublicRelease_30_Channel_Id))
     pool:
       vmImage: 'windows-2019'
     steps:
@@ -80,6 +90,7 @@ stages:
             /p:IsInternalBuild=$(IsInternalBuild)
             /p:RepositoryName=$(Build.Repository.Name)
             /p:CommitSha=$(Build.SourceVersion)
+            /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' 
             /p:AzureStorageAccountName=$(ProxyBackedFeedsAccountName)
             /p:AzureStorageAccountKey=$(dotnetfeed-storage-access-key-1)
             /p:AzureDevOpsFeedsBaseUrl=$(dotnetfeed-internal-private-feed-url)
@@ -92,6 +103,7 @@ stages:
             /p:BlobBasePath='$(Build.ArtifactStagingDirectory)\BlobArtifacts' 
             /p:PackageBasePath='$(Build.ArtifactStagingDirectory)\PackageArtifacts' 
             /p:Configuration=Release 
+            ${{ parameters.artifactsPublishingAdditionalParameters }}
         
       - task: NuGetCommand@2
         displayName: Publish Packages to AzDO Feed
@@ -126,7 +138,7 @@ stages:
     - job:
       displayName: Symbol Availability
       dependsOn: setupMaestroVars
-      condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicRelease_30_Channel_Id)
+      condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.PublicRelease_30_Channel_Id))
       pool:
         vmImage: 'windows-2019'
       steps:

eng/common/templates/post-build/channels/public-validation-release.yml

@@ -1,3 +1,7 @@
+parameters:
+  artifactsPublishingAdditionalParameters: ''
+  publishInstallersAndChecksums: false
+
 stages:
 - stage: PVR_Publish
   dependsOn: validate
@@ -17,7 +21,7 @@ stages:
         value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ]
       - name: IsStableBuild
         value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ]
-    condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicValidationRelease_30_Channel_Id)
+    condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.PublicValidationRelease_30_Channel_Id))
     pool:
       vmImage: 'windows-2019'
     steps:
@@ -62,7 +66,13 @@ stages:
             /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' 
             /p:BlobBasePath='$(Build.ArtifactStagingDirectory)\BlobArtifacts' 
             /p:PackageBasePath='$(Build.ArtifactStagingDirectory)\PackageArtifacts' 
-            /p:Configuration=Release 
+            /p:Configuration=Release
+            /p:InstallersTargetStaticFeed=$(InstallersBlobFeedUrl)
+            /p:InstallersAzureAccountKey=$(dotnetcli-storage-key)
+            /p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }}
+            /p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl)
+            /p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key)
+            ${{ parameters.artifactsPublishingAdditionalParameters }}
 
       - task: NuGetCommand@2
         displayName: Publish Packages to AzDO Feed

eng/common/templates/post-build/common-variables.yml

@@ -1,14 +1,23 @@
 variables:
   - group: Publish-Build-Assets
+  - group: DotNet-DotNetCli-Storage
 
   # .NET Core 3 Dev
   - name: PublicDevRelease_30_Channel_Id
     value: 3
 
+  # .NET Core 5 Dev
+  - name: NetCore_5_Dev_Channel_Id
+    value: 131
+
   # .NET Tools - Validation
   - name: PublicValidationRelease_30_Channel_Id
     value: 9
 
+  # .NET Tools - Latest
+  - name: NetCore_Tools_Latest_Channel_Id
+    value: 2
+
   # .NET Core 3.0 Internal Servicing
   - name: InternalServicing_30_Channel_Id
     value: 184
@@ -37,3 +46,9 @@ variables:
     value: 3.0.0
   - name: SymbolToolVersion
     value: 1.0.1
+
+  # Default locations for Installers and checksums
+  - name: ChecksumsBlobFeedUrl
+    value: https://dotnetcli.blob.core.windows.net/dotnet/index.json
+  - name: InstallersBlobFeedUrl
+    value: https://dotnetclichecksums.blob.core.windows.net/dotnet/index.json

eng/common/templates/post-build/darc-gather-drop.yml

@@ -5,7 +5,7 @@ jobs:
 - job: gatherDrop
   displayName: Gather Drop
   dependsOn: setupMaestroVars
-  condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], ${{ parameters.ChannelId }})
+  condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.ChannelId }}))
   variables:
     - name: BARBuildId
       value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ]

eng/common/templates/post-build/post-build.yml

@@ -3,10 +3,17 @@ parameters:
   enableSigningValidation: true
   enableSymbolValidation: true
   enableNugetValidation: true
+  publishInstallersAndChecksums: false
   SDLValidationParameters:
     enable: false
     params: ''
 
+  # These parameters let the user customize the call to sdk-task.ps1 for publishing
+  # symbols & general artifacts as well as for signing validation
+  symbolPublishingAdditionalParameters: ''
+  artifactsPublishingAdditionalParameters: ''
+  signingValidationAdditionalParameters: ''
+
   # Which stages should finish execution before post-build stages start
   dependsOn: [build]
 
@@ -52,7 +59,9 @@ stages:
             filePath: eng\common\sdk-task.ps1
             arguments: -task SigningValidation -restore -msbuildEngine dotnet
               /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts'
-              /p:Configuration=Release
+              /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt'
+              /p:Configuration=Release 
+              ${{ parameters.signingValidationAdditionalParameters }}
 
   - ${{ if eq(parameters.enableSourceLinkValidation, 'true') }}:
     - job:
@@ -77,18 +86,45 @@ stages:
               -GHRepoName $(Build.Repository.Name) 
               -GHCommit $(Build.SourceVersion)
               -SourcelinkCliVersion $(SourceLinkCLIVersion)
+          continueOnError: true
 
   - ${{ if eq(parameters.SDLValidationParameters.enable, 'true') }}:
     - template: /eng/common/templates/job/execute-sdl.yml
       parameters:
         additionalParameters: ${{ parameters.SDLValidationParameters.params }}
 
+- template: \eng\common\templates\post-build\channels\netcore-dev-5.yml
+  parameters:
+    enableSymbolValidation: ${{ parameters.enableSymbolValidation }}
+    symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+    artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+    publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+
 - template: \eng\common\templates\post-build\channels\public-dev-release.yml
   parameters:
     enableSymbolValidation: ${{ parameters.enableSymbolValidation }}
+    symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+    artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+    publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+
+- template: \eng\common\templates\post-build\channels\netcore-tools-latest.yml
+  parameters:
+    enableSymbolValidation: ${{ parameters.enableSymbolValidation }}
+    symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+    artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+    publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
 
 - template: \eng\common\templates\post-build\channels\public-validation-release.yml
+  parameters:
+    artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+    publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
 
 - template: \eng\common\templates\post-build\channels\public-release.yml
+  parameters:
+    symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+    artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
 
 - template: \eng\common\templates\post-build\channels\internal-servicing.yml
+  parameters:
+    symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+    artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}

eng/common/templates/post-build/promote-build.yml

@@ -5,7 +5,7 @@ jobs:
 - job:
   displayName: Promote Build
   dependsOn: setupMaestroVars
-  condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], ${{ parameters.ChannelId }})
+  condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.ChannelId }}))
   variables:
     - name: BARBuildId
       value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ]

eng/common/tools.ps1

@@ -39,6 +39,10 @@
 # installed on the machine instead of downloading one.
 [bool]$useInstalledDotNetCli = if (Test-Path variable:useInstalledDotNetCli) { $useInstalledDotNetCli } else { $true }
 
+# Enable repos to use a particular version of the on-line dotnet-install scripts.
+#    default URL: https://dot.net/v1/dotnet-install.ps1
+[string]$dotnetInstallScriptVersion = if (Test-Path variable:dotnetInstallScriptVersion) { $dotnetInstallScriptVersion } else { "v1" }
+
 # True to use global NuGet cache instead of restoring packages to repository-local directory.
 [bool]$useGlobalNuGetCache = if (Test-Path variable:useGlobalNuGetCache) { $useGlobalNuGetCache } else { !$ci }
 
@@ -159,7 +163,7 @@ function GetDotNetInstallScript([string] $dotnetRoot) {
   $installScript = Join-Path $dotnetRoot "dotnet-install.ps1"
   if (!(Test-Path $installScript)) {
     Create-Directory $dotnetRoot
-    Invoke-WebRequest "https://dot.net/v1/dotnet-install.ps1" -OutFile $installScript
+    Invoke-WebRequest "https://dot.net/$dotnetInstallScriptVersion/dotnet-install.ps1" -OutFile $installScript
   }
 
   return $installScript
@@ -518,6 +522,9 @@ function MSBuild-Core() {
   if ($warnAsError) {
     $cmdArgs += " /warnaserror /p:TreatWarningsAsErrors=true"
   }
+  else {
+    $cmdArgs += " /p:TreatWarningsAsErrors=false"
+  }
 
   foreach ($arg in $args) {
     if ($arg -ne $null -and $arg.Trim() -ne "") {

eng/common/tools.sh

@@ -45,6 +45,10 @@ warn_as_error=${warn_as_error:-true}
 # installed on the machine instead of downloading one.
 use_installed_dotnet_cli=${use_installed_dotnet_cli:-true}
 
+# Enable repos to use a particular version of the on-line dotnet-install scripts.
+#    default URL: https://dot.net/v1/dotnet-install.sh
+dotnetInstallScriptVersion=${dotnetInstallScriptVersion:-'v1'}
+
 # True to use global NuGet cache instead of restoring packages to repository-local directory.
 if [[ "$ci" == true ]]; then
   use_global_nuget_cache=${use_global_nuget_cache:-false}
@@ -195,7 +199,7 @@ function InstallDotNet {
 function GetDotNetInstallScript {
   local root=$1
   local install_script="$root/dotnet-install.sh"
-  local install_script_url="https://dot.net/v1/dotnet-install.sh"
+  local install_script_url="https://dot.net/$dotnetInstallScriptVersion/dotnet-install.sh"
 
   if [[ ! -a "$install_script" ]]; then
     mkdir -p "$root"

global.json

@@ -3,6 +3,6 @@
     "dotnet": "3.0.100-preview6-012264"
   },
   "msbuild-sdks": {
-    "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19377.2"
+    "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19421.1"
   }
 }