Updating arcade version.

eng/common/CIBuild.cmd

@@ -1,3 +1,2 @@
 @echo off
-powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0Build.ps1""" -restore -build -test -sign -pack -publish -ci %*"
+powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0Build.ps1""" -restore -build -test -sign -pack -publish -ci %*"
-exit /b %ErrorLevel%

eng/common/PublishBuildAssets.cmd

@@ -1,3 +1,3 @@
 @echo off
-powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0Build.ps1""" -msbuildEngine dotnet -restore -execute /p:PublishBuildAssets=true /p:SdkTaskProjects=PublishBuildAssets.proj %*"
+powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0Build.ps1""" -msbuildEngine dotnet -restore -execute -binaryLog /p:PublishBuildAssets=true /p:SdkTaskProjects=PublishBuildAssets.proj %*"
 exit /b %ErrorLevel%

eng/common/PublishToPackageFeed.proj

@@ -0,0 +1,37 @@
+<!--
+  This MSBuild file is intended to be used as the body of the default 
+  publishing release pipeline. The release pipeline will use this file
+  to invoke the PushToStaticFeed task that will read the build asset
+  manifest and publish the assets described in the manifest to
+  informed target feeds.
+-->
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <TargetFramework>netcoreapp2.1</TargetFramework>
+  </PropertyGroup>
+
+  <Import Project="$(MSBuildThisFileDirectory)MicrosoftDotNetBuildTasksFeedVersion.props" />
+  <Import Project="$(NuGetPackageRoot)microsoft.dotnet.build.tasks.feed\$(MicrosoftDotNetBuildTasksFeedVersion)\build\Microsoft.DotNet.Build.Tasks.Feed.targets" />
+
+  <Target Name="PublishToFeed">
+    <Error Condition="'$(TargetStaticFeed)' == ''" Text="TargetStaticFeed: Target feed for publishing assets wasn't provided." />
+    <Error Condition="'$(AccountKeyToStaticFeed)' == ''" Text="AccountKeyToStaticFeed: Account key for target feed wasn't provided." />
+    <Error Condition="'$(FullPathAssetManifest)' == ''" Text="Full path to asset manifest wasn't provided." />
+    <Error Condition="'$(FullPathBlobBasePath)' == '' AND '$(FullPathPackageBasePath)' == ''" Text="A valid full path to BlobBasePath of PackageBasePath is required." />
+
+    <PushArtifactsInManifestToFeed
+      ExpectedFeedUrl="$(TargetStaticFeed)"
+      AccountKey="$(AccountKeyToStaticFeed)"
+      Overwrite="$(OverrideAssetsWithSameName)"
+      PassIfExistingItemIdentical="$(PassIfExistingItemIdentical)"
+      MaxClients="$(MaxParallelUploads)"
+      UploadTimeoutInMinutes="$(MaxUploadTimeoutInMinutes)"
+      AssetManifestPath="$(FullPathAssetManifest)"
+      BlobAssetsBasePath="$(FullPathBlobBasePath)"
+      PackageAssetsBasePath="$(FullPathPackageBasePath)" />
+  </Target>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.DotNet.Build.Tasks.Feed" Version="$(MicrosoftDotNetBuildTasksFeedVersion)" />
+  </ItemGroup>
+</Project>

eng/common/README.md

@@ -0,0 +1,28 @@
+# Don't touch this folder
+
+                uuuuuuuuuuuuuuuuuuuu
+              u" uuuuuuuuuuuuuuuuuu "u
+            u" u$$$$$$$$$$$$$$$$$$$$u "u
+          u" u$$$$$$$$$$$$$$$$$$$$$$$$u "u
+        u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u
+      u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u
+    u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u
+    $ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $
+    $ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $
+    $ $$$" ... "$...  ...$" ... "$$$  ... "$$$ $
+    $ $$$u `"$$$$$$$  $$$  $$$$$  $$  $$$  $$$ $
+    $ $$$$$$uu "$$$$  $$$  $$$$$  $$  """ u$$$ $
+    $ $$$""$$$  $$$$  $$$u "$$$" u$$  $$$$$$$$ $
+    $ $$$$....,$$$$$..$$$$$....,$$$$..$$$$$$$$ $
+    $ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $
+    "u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u"
+      "u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u"
+        "u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u"
+          "u "$$$$$$$$$$$$$$$$$$$$$$$$" u"
+            "u "$$$$$$$$$$$$$$$$$$$$" u"
+              "u """""""""""""""""" u"
+                """"""""""""""""""""
+
+!!! Changes made in this directory are subject to being overwritten by automation !!!
+
+The files in this directory are shared by all Arcade repos and managed by automation. If you need to make changes to these files, open an issue or submit a pull request to https://github.com/dotnet/arcade first.

eng/common/build.ps1

@@ -1,15 +1,15 @@
 [CmdletBinding(PositionalBinding=$false)]
 Param(
-  [string] $configuration = "Debug",
+  [string][Alias('c')]$configuration = "Debug",
-  [string] $projects = "",
+  [string] $projects,
-  [string] $verbosity = "minimal",
+  [string][Alias('v')]$verbosity = "minimal",
   [string] $msbuildEngine = $null,
-  [bool] $warnaserror = $true,
+  [bool] $warnAsError = $true,
-  [bool] $nodereuse = $true,
+  [bool] $nodeReuse = $true,
   [switch] $execute,
-  [switch] $restore,
+  [switch][Alias('r')]$restore,
   [switch] $deployDeps,
-  [switch] $build,
+  [switch][Alias('b')]$build,
   [switch] $rebuild,
   [switch] $deploy,
   [switch] $test,
@@ -19,6 +19,7 @@ Param(
   [switch] $pack,
   [switch] $publish,
   [switch] $publishBuildAssets,
+  [switch][Alias('bl')]$binaryLog,
   [switch] $ci,
   [switch] $prepareMachine,
   [switch] $help,
@@ -29,14 +30,15 @@ Param(
 
 function Print-Usage() {
     Write-Host "Common settings:"
-    Write-Host "  -configuration <value>  Build configuration Debug, Release"
+    Write-Host "  -configuration <value>  Build configuration: 'Debug' or 'Release' (short: -c)"
-    Write-Host "  -verbosity <value>      Msbuild verbosity (q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic])"
+    Write-Host "  -verbosity <value>      Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic] (short: -v)"
+    Write-Host "  -binaryLog              Output binary log (short: -bl)"
     Write-Host "  -help                   Print help and exit"
     Write-Host ""
 
     Write-Host "Actions:"
-    Write-Host "  -restore                Restore dependencies"
+    Write-Host "  -restore                Restore dependencies (short: -r)"
-    Write-Host "  -build                  Build solution"
+    Write-Host "  -build                  Build solution (short: -b)"
     Write-Host "  -rebuild                Rebuild solution"
     Write-Host "  -deploy                 Deploy built VSIXes"
     Write-Host "  -deployDeps             Deploy dependencies (e.g. VSIXes for integration tests)"
@@ -46,7 +48,7 @@ function Print-Usage() {
     Write-Host "  -performanceTest        Run all performance tests in the solution"
     Write-Host "  -sign                   Sign build outputs"
     Write-Host "  -publish                Publish artifacts (e.g. symbols)"
-    Write-Host "  -publishBuildAssets        Push assets to BAR"
+    Write-Host "  -publishBuildAssets     Push assets to BAR"
     Write-Host ""
 
     Write-Host "Advanced settings:"
@@ -59,24 +61,35 @@ function Print-Usage() {
     Write-Host "The above arguments can be shortened as much as to be unambiguous (e.g. -co for configuration, -t for test, etc.)."
 }
 
-if ($help -or (($properties -ne $null) -and ($properties.Contains("/help") -or $properties.Contains("/?")))) {
-  Print-Usage
-  exit 0
-}
 
-try {
+function InitializeCustomToolset {
-  if ($projects -eq "") {
+  if (-not $restore) {
-    $projects = Join-Path $RepoRoot "*.sln"
+    return
   }
 
-  InitializeTools
+  $script = Join-Path $EngRoot "restore-toolset.ps1"
 
-  $BuildLog = Join-Path $LogDir "Build.binlog"
+  if (Test-Path $script) {
+    . $script
+  }
+}
 
-  MSBuild $ToolsetBuildProj `
+function Build {
-    /bl:$BuildLog `
+  $toolsetBuildProj = InitializeToolset
+  InitializeCustomToolset
+  $bl = if ($binaryLog) { "/bl:" + (Join-Path $LogDir "Build.binlog") } else { "" }
+
+  if ($projects) {
+    # Re-assign properties to a new variable because PowerShell doesn't let us append properties directly for unclear reasons.
+    # Explicitly set the type as string[] because otherwise PowerShell would make this char[] if $properties is empty.
+    [string[]] $msbuildArgs = $properties
+    $msbuildArgs += "/p:Projects=$projects"
+    $properties = $msbuildArgs
+  }
+
+  MSBuild $toolsetBuildProj `
+    $bl `
     /p:Configuration=$configuration `
-    /p:Projects=$projects `
     /p:RepoRoot=$RepoRoot `
     /p:Restore=$restore `
     /p:DeployDeps=$deployDeps `
@@ -92,13 +105,27 @@ try {
     /p:Execute=$execute `
     /p:ContinuousIntegrationBuild=$ci `
     @properties
+}
 
-  if ($lastExitCode -ne 0) {
+try {
-    Write-Host "Build Failed (exit code '$lastExitCode'). See log: $BuildLog" -ForegroundColor Red
+  if ($help -or (($properties -ne $null) -and ($properties.Contains("/help") -or $properties.Contains("/?")))) {
-    ExitWithExitCode $lastExitCode
+    Print-Usage
+    exit 0
   }
 
-  ExitWithExitCode $lastExitCode
+  if ($ci) {
+    $binaryLog = $true
+    $nodeReuse = $false
+  }
+
+  # Import custom tools configuration, if present in the repo.
+  # Note: Import in global scope so that the script set top-level variables without qualification.
+  $configureToolsetScript = Join-Path $EngRoot "configure-toolset.ps1"
+  if (Test-Path $configureToolsetScript) {
+    . $configureToolsetScript
+  }
+
+  Build
 }
 catch {
   Write-Host $_
@@ -106,3 +133,5 @@ catch {
   Write-Host $_.ScriptStackTrace
   ExitWithExitCode 1
 }
+
+ExitWithExitCode 0

eng/common/build.sh

@@ -1,5 +1,35 @@
 #!/usr/bin/env bash
 
+# Stop script if unbound variable found (use ${var:-} if intentional)
+set -u
+
+usage()
+{
+  echo "Common settings:"
+  echo "  --configuration <value>    Build configuration: 'Debug' or 'Release' (short: --c)"
+  echo "  --verbosity <value>        Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic] (short: -v)"
+  echo "  --binaryLog                Create MSBuild binary log (short: -bl)"
+  echo ""
+  echo "Actions:"
+  echo "  --restore                  Restore dependencies (short: -r)"
+  echo "  --build                    Build all projects (short: -b)"
+  echo "  --rebuild                  Rebuild all projects"
+  echo "  --test                     Run all unit tests (short: -t)"
+  echo "  --sign                     Sign build outputs"
+  echo "  --publish                  Publish artifacts (e.g. symbols)"
+  echo "  --pack                     Package build outputs into NuGet packages and Willow components"
+  echo "  --help                     Print help and exit (short: -h)"
+  echo ""
+  echo "Advanced settings:"
+  echo "  --projects <value>       Project or solution file(s) to build"
+  echo "  --ci                     Set when running on CI server"
+  echo "  --prepareMachine         Prepare machine for CI run, clean up processes after build"
+  echo "  --nodeReuse <value>      Sets nodereuse msbuild parameter ('true' or 'false')"
+  echo "  --warnAsError <value>    Sets warnaserror msbuild parameter ('true' or 'false')"
+  echo ""
+  echo "Command line arguments starting with '/p:' are passed through to MSBuild."
+}
+
 source="${BASH_SOURCE[0]}"
 
 # resolve $source until the file is no longer a symlink
@@ -12,7 +42,6 @@ while [[ -h "$source" ]]; do
 done
 scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
 
-help=false
 restore=false
 build=false
 rebuild=false
@@ -25,8 +54,9 @@ sign=false
 public=false
 ci=false
 
-warnaserror=true
+warn_as_error=true
-nodereuse=true
+node_reuse=true
+binary_log=false
 
 projects=''
 configuration='Debug'
@@ -34,138 +64,140 @@ prepare_machine=false
 verbosity='minimal'
 properties=''
 
-while (($# > 0)); do
+while [[ $# > 0 ]]; do
-  lowerI="$(echo $1 | awk '{print tolower($0)}')"
+  opt="$(echo "$1" | awk '{print tolower($0)}')"
-  case $lowerI in
+  case "$opt" in
-    --build)
+    --help|-h)
-      build=true
+      usage
-      shift 1
-      ;;
-    --ci)
-      ci=true
-      shift 1
-      ;;
-    --configuration)
-      configuration=$2
-      shift 2
-      ;;
-    --help)
-      echo "Common settings:"
-      echo "  --configuration <value>  Build configuration Debug, Release"
-      echo "  --verbosity <value>      Msbuild verbosity (q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic])"
-      echo "  --help                   Print help and exit"
-      echo ""
-      echo "Actions:"
-      echo "  --restore                Restore dependencies"
-      echo "  --build                  Build solution"
-      echo "  --rebuild                Rebuild solution"
-      echo "  --test                   Run all unit tests in the solution"
-      echo "  --sign                   Sign build outputs"
-      echo "  --publish                Publish artifacts (e.g. symbols)"
-      echo "  --pack                   Package build outputs into NuGet packages and Willow components"
-      echo ""
-      echo "Advanced settings:"
-      echo "  --solution <value>       Path to solution to build"
-      echo "  --ci                     Set when running on CI server"
-      echo "  --prepareMachine         Prepare machine for CI run"
-      echo ""
-      echo "Command line arguments not listed above are passed through to MSBuild."
       exit 0
       ;;
-    --pack)
+    --configuration|-c)
-      pack=true
+      configuration=$2
-      shift 1
+      shift
       ;;
-    --preparemachine)
+    --verbosity|-v)
-      prepare_machine=true
+      verbosity=$2
-      shift 1
+      shift
+      ;;
+    --binarylog|-bl)
+      binary_log=true
+      ;;
+    --restore|-r)
+      restore=true
+      ;;
+    --build|-b)
+      build=true
       ;;
     --rebuild)
       rebuild=true
-      shift 1
       ;;
-    --restore)
+    --pack)
-      restore=true
+      pack=true
-      shift 1
       ;;
-    --sign)
+    --test|-t)
-      sign=true
-      shift 1
-      ;;
-    --solution)
-      solution=$2
-      shift 2
-      ;;
-    --projects)
-      projects=$2
-      shift 2
-      ;;
-    --test)
       test=true
-      shift 1
       ;;
     --integrationtest)
       integration_test=true
-      shift 1
       ;;
     --performancetest)
       performance_test=true
-      shift 1
+      ;;
+    --sign)
+      sign=true
       ;;
     --publish)
       publish=true
-      shift 1
       ;;
-    --verbosity)
+    --preparemachine)
-      verbosity=$2
+      prepare_machine=true
-      shift 2
+      ;;
+    --projects)
+      projects=$2
+      shift
+      ;;
+    --ci)
+      ci=true
       ;;
     --warnaserror)
-      warnaserror=$2
+      warn_as_error=$2
-      shift 2
+      shift
       ;;
     --nodereuse)
-      nodereuse=$2
+      node_reuse=$2
-      shift 2
+      shift
       ;;
-      *)
+    /p:*)
       properties="$properties $1"
-      shift 1
+      ;;
+    *)
+      echo "Invalid argument: $1"
+      usage
+      exit 1
       ;;
   esac
+
+  shift
 done
 
+if [[ "$ci" == true ]]; then
+  binary_log=true
+  node_reuse=false
+fi
+
 . "$scriptroot/tools.sh"
 
-if [[ -z $projects ]]; then
+function InitializeCustomToolset {
-  projects="$repo_root/*.sln"
+  local script="$eng_root/restore-toolset.sh"
+
+  if [[ -a "$script" ]]; then
+    . "$script"
+  fi
+}
+
+function Build {
+  InitializeToolset
+  InitializeCustomToolset
+
+  if [[ ! -z "$projects" ]]; then
+    properties="$properties /p:Projects=$projects"
+  fi
+
+  local bl=""
+  if [[ "$binary_log" == true ]]; then
+    bl="/bl:\"$log_dir/Build.binlog\""
+  fi
+
+  MSBuild $_InitializeToolset \
+    $bl \
+    /p:Configuration=$configuration \
+    /p:RepoRoot="$repo_root" \
+    /p:Restore=$restore \
+    /p:Build=$build \
+    /p:Rebuild=$rebuild \
+    /p:Test=$test \
+    /p:Pack=$pack \
+    /p:IntegrationTest=$integration_test \
+    /p:PerformanceTest=$performance_test \
+    /p:Sign=$sign \
+    /p:Publish=$publish \
+    /p:ContinuousIntegrationBuild=$ci \
+    $properties
+
+  ExitWithExitCode 0
+}
+
+# Import custom tools configuration, if present in the repo.
+configure_toolset_script="$eng_root/configure-toolset.sh"
+if [[ -a "$configure_toolset_script" ]]; then
+  . "$configure_toolset_script"
 fi
 
-InitializeTools
+# TODO: https://github.com/dotnet/arcade/issues/1468
-
+# Temporary workaround to avoid breaking change.
-build_log="$log_dir/Build.binlog"
+# Remove once repos are updated.
-
+if [[ -n "${useInstalledDotNetCli:-}" ]]; then
-MSBuild "$toolset_build_proj" \
+  use_installed_dotnet_cli="$useInstalledDotNetCli"
-  /bl:"$build_log" \
-  /p:Configuration=$configuration \
-  /p:Projects="$projects" \
-  /p:RepoRoot="$repo_root" \
-  /p:Restore=$restore \
-  /p:Build=$build \
-  /p:Rebuild=$rebuild \
-  /p:Test=$test \
-  /p:Pack=$pack \
-  /p:IntegrationTest=$integration_test \
-  /p:PerformanceTest=$performance_test \
-  /p:Sign=$sign \
-  /p:Publish=$publish \
-  /p:ContinuousIntegrationBuild=$ci \
-  $properties
-
-lastexitcode=$?
-
-if [[ $lastexitcode != 0 ]]; then
-  echo "Build failed (exit code '$lastexitcode'). See log: $build_log"
 fi
 
-ExitWithExitCode $lastexitcode
+Build

eng/common/darc-init.ps1

@@ -3,7 +3,9 @@ $verbosity = "m"
 
 function InstallDarcCli {
   $darcCliPackageName = "microsoft.dotnet.darc"
-  $dotnet = "$env:DOTNET_INSTALL_DIR\dotnet.exe"
+
+  $dotnetRoot = InitializeDotNetCli -install:$true
+  $dotnet = "$dotnetRoot\dotnet.exe"
   $toolList = Invoke-Expression "& `"$dotnet`" tool list -g"
 
   if ($toolList -like "*$darcCliPackageName*") {
@@ -17,5 +19,4 @@ function InstallDarcCli {
   Invoke-Expression "& `"$dotnet`" tool install $darcCliPackageName --version $toolsetVersion -v $verbosity -g"
 }
 
-InitializeTools
 InstallDarcCli

eng/common/darc-init.sh

@@ -17,10 +17,14 @@ verbosity=m
 
 function InstallDarcCli {
   local darc_cli_package_name="microsoft.dotnet.darc"
-  local uninstall_command=`$DOTNET_INSTALL_DIR/dotnet tool uninstall $darc_cli_package_name -g`
+
-  local tool_list=$($DOTNET_INSTALL_DIR/dotnet tool list -g)
+  InitializeDotNetCli
+  local dotnet_root=$_InitializeDotNetCli
+
+  local uninstall_command=`$dotnet_root/dotnet tool uninstall $darc_cli_package_name -g`
+  local tool_list=$($dotnet_root/dotnet tool list -g)
   if [[ $tool_list = *$darc_cli_package_name* ]]; then
-    echo $($DOTNET_INSTALL_DIR/dotnet tool uninstall $darc_cli_package_name -g)
+    echo $($dotnet_root/dotnet tool uninstall $darc_cli_package_name -g)
   fi
 
   ReadGlobalVersion "Microsoft.DotNet.Arcade.Sdk"
@@ -28,8 +32,7 @@ function InstallDarcCli {
 
   echo "Installing Darc CLI version $toolset_version..."
   echo "You may need to restart your command shell if this is the first dotnet tool you have installed."
-  echo $($DOTNET_INSTALL_DIR/dotnet tool install $darc_cli_package_name --version $toolset_version -v $verbosity -g)
+  echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $toolset_version -v $verbosity -g)
 }
 
-InitializeTools
 InstallDarcCli

eng/common/msbuild.ps1

@@ -1,8 +1,8 @@
 [CmdletBinding(PositionalBinding=$false)]
 Param(
   [string] $verbosity = "minimal",
-  [bool] $warnaserror = $true,
+  [bool] $warnAsError = $true,
-  [bool] $nodereuse = $true,
+  [bool] $nodeReuse = $true,
   [switch] $ci,
   [switch] $prepareMachine,
   [Parameter(ValueFromRemainingArguments=$true)][String[]]$extraArgs
@@ -11,13 +11,17 @@ Param(
 . $PSScriptRoot\tools.ps1
 
 try {
-  InitializeTools
+  if ($ci) {
+    $nodeReuse = $false
+  }
+
   MSBuild @extraArgs
-  ExitWithExitCode $lastExitCode
+} 
-}
 catch {
   Write-Host $_
   Write-Host $_.Exception
   Write-Host $_.ScriptStackTrace
   ExitWithExitCode 1
 }
+
+ExitWithExitCode 0

eng/common/msbuild.sh

@@ -13,10 +13,10 @@ done
 scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
 
 verbosity='minimal'
-warnaserror=true
+warn_as_error=true
-nodereuse=true
+node_reuse=true
 prepare_machine=false
-extraargs=''
+extra_args=''
 
 while (($# > 0)); do
   lowerI="$(echo $1 | awk '{print tolower($0)}')"
@@ -26,11 +26,11 @@ while (($# > 0)); do
       shift 2
       ;;
     --warnaserror)
-      warnaserror=$2
+      warn_as_error=$2
       shift 2
       ;;
     --nodereuse)
-      nodereuse=$2
+      node_reuse=$2
       shift 2
       ;;
     --ci)
@@ -42,7 +42,7 @@ while (($# > 0)); do
       shift 1
       ;;
       *)
-      extraargs="$extraargs $1"
+      extra_args="$extra_args $1"
       shift 1
       ;;
   esac
@@ -50,6 +50,9 @@ done
 
 . "$scriptroot/tools.sh"
 
-InitializeTools
+if [[ "$ci" == true ]]; then
-MSBuild $extraargs
+  node_reuse=false
-ExitWithExitCode $?
+fi
+
+MSBuild $extra_args
+ExitWithExitCode 0

eng/common/templates/job/job.yml

@@ -95,11 +95,26 @@ jobs:
 
   variables:
   - ${{ each variable in parameters.variables }}:
+    # handle name-value variable syntax
+    # example:
+    # - name: [key]
+    #   value: [value]
     - ${{ if ne(variable.name, '') }}:
       - name: ${{ variable.name }}
         value: ${{ variable.value }}
+    
+    # handle variable groups
     - ${{ if ne(variable.group, '') }}:
       - group: ${{ variable.group }}
+
+    # handle key-value variable syntax.
+    # example:
+    # - [key]: [value]
+    - ${{ if and(eq(variable.name, ''), eq(variable.group, '')) }}:
+      - ${{ each pair in variable }}:
+        - name: ${{ pair.key }}
+          value: ${{ pair.value }}
+
   # Add additional variables
   - ${{ if and(ne(parameters.helixRepo, ''), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notIn(variables['Build.Reason'], 'PullRequest')) }}:
     - name: _HelixSource

eng/common/templates/job/publish-build-assets.yml

@@ -1,10 +1,4 @@
 parameters:
-  # Optional: dependencies of the job
-  dependsOn: ''
-
-  # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool
-  pool: {}
-
   configuration: 'Debug'
 
   # Optional: condition for the job to run
@@ -13,6 +7,15 @@ parameters:
   # Optional: 'true' if future jobs should run even if this job fails
   continueOnError: false
 
+  # Optional: dependencies of the job
+  dependsOn: ''
+
+  # Optional: Include PublishBuildArtifacts task
+  enablePublishBuildArtifacts: false
+
+  # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool
+  pool: {}
+
   # Optional: should run as a public build even in the internal project
   #           if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects.
   runAsPublic: false
@@ -49,3 +52,12 @@ jobs:
       displayName: Publish Build Assets
       condition: ${{ parameters.condition }}
       continueOnError: ${{ parameters.continueOnError }}
+    - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}:
+      - task: PublishBuildArtifacts@1
+        displayName: Publish Logs to VSTS
+        inputs:
+          PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)'
+          PublishLocation: Container
+          ArtifactName: $(Agent.Os)_PublishBuildAssets
+        continueOnError: true
+        condition: always()      

eng/common/templates/jobs/jobs.yml

@@ -20,8 +20,8 @@ parameters:
   # Optional: enable sending telemetry
   #           if 'true', these "variables" must be specified in the variables object or as part of the queue matrix
   #             _HelixBuildConfig - differentiate between Debug, Release, other
-  #             _HelixSource - Example: build/product/
+  #             _HelixType - Example: build/product/
-  #             _HelixType - Example: official/dotnet/arcade/$(Build.SourceBranch)
+  #             _HelixSource - Example: official/dotnet/arcade/$(Build.SourceBranch)
   enableTelemetry: false
 
   # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job
@@ -73,3 +73,5 @@ jobs:
       pool:
         vmImage: vs2017-win2016
       runAsPublic: ${{ parameters.runAsPublic }}
+      enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }}
+

eng/common/templates/steps/helix-publish.yml

@@ -40,7 +40,7 @@ steps:
       WorkItemCommand: ${{ parameters.WorkItemCommand }}
       CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }}
       XUnitProjects: ${{ parameters.XUnitProjects }}
-      XUnitTargetFramework: ${{ parameters.XUnitTargetFramework }}
+      XUnitRuntimeTargetFramework: ${{ parameters.XUnitTargetFramework }}
       XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }}
       IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }}
       DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }}

eng/common/templates/steps/send-to-helix.yml

@@ -0,0 +1,80 @@
+parameters:
+  HelixSource: 'pr/default'          # required -- sources must start with pr/, official/, prodcon/, or agent/
+  HelixType: 'tests/default/'        # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/'
+  HelixBuild: $(Build.BuildNumber)   # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number
+  HelixTargetQueues: ''              # required -- semicolon delimited list of Helix queues to test on; see https://helix.dot.net/api/2018-03-14/info/queues for a list of queues
+  HelixAccessToken: ''               # required -- access token to make Helix API requests; should be provided by the appropriate variable group
+  HelixPreCommands: ''               # optional -- commands to run before Helix work item execution
+  HelixPostCommands: ''              # optional -- commands to run after Helix work item execution
+  WorkItemDirectory: ''              # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects
+  WorkItemCommand: ''                # optional -- a command to execute on the payload; requires WorkItemDirectory; incompatible with XUnitProjects
+  CorrelationPayloadDirectory: ''    # optional -- a directory to zip up and send to Helix as a correlation payload
+  XUnitProjects: ''                  # optional -- semicolon delimited list of XUnitProjects to parse and send to Helix; requires XUnitRuntimeTargetFramework, XUnitPublishTargetFramework, XUnitRunnerVersion, and IncludeDotNetCli=true
+  XUnitPublishTargetFramework: ''    # optional -- framework to use to publish your xUnit projects
+  XUnitRuntimeTargetFramework: ''    # optional -- framework to use for the xUnit console runner
+  XUnitRunnerVersion: ''             # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects
+  IncludeDotNetCli: false            # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion
+  DotNetCliPackageType: ''           # optional -- either 'sdk' or 'runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json
+  DotNetCliVersion: ''               # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json
+  EnableXUnitReporter: false         # optional -- true enables XUnit result reporting to Mission Control
+  WaitForWorkItemCompletion: true    # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget."
+  IsExternal: false                  # optional -- true requires Creator and will make the Mission Control results visible to folks outside the Microsoft Org
+  Creator: ''                        # optional -- if the build is external, use this to specify who is sending the job
+  condition: succeeded()             # optional -- condition for step to execute; defaults to succeeded()
+  continueOnError: false             # optional -- determines whether to continue the build if the step errors; defaults to false
+
+steps:
+  - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 $env:BUILD_SOURCESDIRECTORY\eng\common\helixpublish.proj /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"'
+    displayName: Send job to Helix (Windows)
+    env:
+      BuildConfig: $(_BuildConfig)
+      HelixSource: ${{ parameters.HelixSource }}
+      HelixType: ${{ parameters.HelixType }}
+      HelixBuild: ${{ parameters.HelixBuild }}
+      HelixTargetQueues: ${{ parameters.HelixTargetQueues }}
+      HelixAccessToken: ${{ parameters.HelixAccessToken }}
+      HelixPreCommands: ${{ parameters.HelixPreCommands }}
+      HelixPostCommands: ${{ parameters.HelixPostCommands }}
+      WorkItemDirectory: ${{ parameters.WorkItemDirectory }}
+      WorkItemCommand: ${{ parameters.WorkItemCommand }}
+      CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }}
+      XUnitProjects: ${{ parameters.XUnitProjects }}
+      XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }}
+      XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }}
+      XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }}
+      IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }}
+      DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }}
+      DotNetCliVersion: ${{ parameters.DotNetCliVersion }}
+      EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }}
+      WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }}
+      IsExternal: ${{ parameters.IsExternal }}
+      Creator: ${{ parameters.Creator }}
+    condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT'))
+    continueOnError: ${{ parameters.continueOnError }}
+  - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/eng/common/helixpublish.proj /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog
+    displayName: Send job to Helix (Unix)
+    env:
+      BuildConfig: $(_BuildConfig)
+      HelixSource: ${{ parameters.HelixSource }}
+      HelixType: ${{ parameters.HelixType }}
+      HelixBuild: ${{ parameters.HelixBuild }}
+      HelixTargetQueues: ${{ parameters.HelixTargetQueues }}
+      HelixAccessToken: ${{ parameters.HelixAccessToken }}
+      HelixPreCommands: ${{ parameters.HelixPreCommands }}
+      HelixPostCommands: ${{ parameters.HelixPostCommands }}
+      WorkItemDirectory: ${{ parameters.WorkItemDirectory }}
+      WorkItemCommand: ${{ parameters.WorkItemCommand }}
+      CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }}
+      XUnitProjects: ${{ parameters.XUnitProjects }}
+      XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }}
+      XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }}
+      XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }}
+      IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }}
+      DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }}
+      DotNetCliVersion: ${{ parameters.DotNetCliVersion }}
+      EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }}
+      WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }}
+      IsExternal: ${{ parameters.IsExternal }}
+      Creator: ${{ parameters.Creator }}
+    condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT'))
+    continueOnError: ${{ parameters.continueOnError }}

eng/common/tools.ps1

@@ -1,14 +1,43 @@
-# Initialize variables if they aren't already defined
+# Initialize variables if they aren't already defined.
+# These may be defined as parameters of the importing script, or set after importing this script.
 
-$ci = if (Test-Path variable:ci) { $ci } else { $false }
+# CI mode - set to true on CI server for PR validation build or official build.
-$configuration = if (Test-Path variable:configuration) { $configuration } else { "Debug" }
+[bool]$ci = if (Test-Path variable:ci) { $ci } else { $false }
-$nodereuse = if (Test-Path variable:nodereuse) { $nodereuse } else { !$ci }
+
-$prepareMachine = if (Test-Path variable:prepareMachine) { $prepareMachine } else { $false }
+# Build configuration. Common values include 'Debug' and 'Release', but the repository may use other names.
-$restore = if (Test-Path variable:restore) { $restore } else { $true }
+[string]$configuration = if (Test-Path variable:configuration) { $configuration } else { "Debug" }
-$verbosity = if (Test-Path variable:verbosity) { $verbosity } else { "minimal" }
+
-$warnaserror = if (Test-Path variable:warnaserror) { $warnaserror } else { $true }
+# Set to true to output binary log from msbuild. Note that emitting binary log slows down the build.
-$msbuildEngine = if (Test-Path variable:msbuildEngine) { $msbuildEngine } else { $null }
+# Binary log must be enabled on CI.
-$useInstalledDotNetCli = if (Test-Path variable:useInstalledDotNetCli) { $useInstalledDotNetCli } else { $true }
+[bool]$binaryLog = if (Test-Path variable:binaryLog) { $binaryLog } else { $ci }
+
+# Turns on machine preparation/clean up code that changes the machine state (e.g. kills build processes).
+[bool]$prepareMachine = if (Test-Path variable:prepareMachine) { $prepareMachine } else { $false }
+
+# True to restore toolsets and dependencies.
+[bool]$restore = if (Test-Path variable:restore) { $restore } else { $true }
+
+# Adjusts msbuild verbosity level.
+[string]$verbosity = if (Test-Path variable:verbosity) { $verbosity } else { "minimal" }
+
+# Set to true to reuse msbuild nodes. Recommended to not reuse on CI.
+[bool]$nodeReuse = if (Test-Path variable:nodeReuse) { $nodeReuse } else { !$ci }
+
+# Configures warning treatment in msbuild.
+[bool]$warnAsError = if (Test-Path variable:warnAsError) { $warnAsError } else { $true }
+
+# Specifies which msbuild engine to use for build: 'vs', 'dotnet' or unspecified (determined based on presence of tools.vs in global.json).
+[string]$msbuildEngine = if (Test-Path variable:msbuildEngine) { $msbuildEngine } else { $null }
+
+# True to attempt using .NET Core already that meets requirements specified in global.json 
+# installed on the machine instead of downloading one.
+[bool]$useInstalledDotNetCli = if (Test-Path variable:useInstalledDotNetCli) { $useInstalledDotNetCli } else { $true }
+
+# True to use global NuGet cache instead of restoring packages to repository-local directory.
+[bool]$useGlobalNuGetCache = if (Test-Path variable:useGlobalNuGetCache) { $useGlobalNuGetCache } else { !$ci }
+
+# An array of names of processes to stop on script exit if prepareMachine is true.
+$processesToStopOnExit = if (Test-Path variable:processesToStopOnExit) { $processesToStopOnExit } else { @("msbuild", "dotnet", "vbcscompiler") }
 
 set-strictmode -version 2.0
 $ErrorActionPreference = "Stop"
@@ -25,13 +54,54 @@ function Unzip([string]$zipfile, [string]$outpath) {
   [System.IO.Compression.ZipFile]::ExtractToDirectory($zipfile, $outpath)
 }
 
+# This will exec a process using the console and return it's exit code.
+# This will not throw when the process fails.
+# Returns process exit code.
+function Exec-Process([string]$command, [string]$commandArgs) {
+  $startInfo = New-Object System.Diagnostics.ProcessStartInfo
+  $startInfo.FileName = $command
+  $startInfo.Arguments = $commandArgs
+  $startInfo.UseShellExecute = $false
+  $startInfo.WorkingDirectory = Get-Location
+
+  $process = New-Object System.Diagnostics.Process
+  $process.StartInfo = $startInfo
+  $process.Start() | Out-Null
+
+  $finished = $false
+  try {
+    while (-not $process.WaitForExit(100)) { 
+      # Non-blocking loop done to allow ctr-c interrupts
+    }
+
+    $finished = $true
+    return $global:LASTEXITCODE = $process.ExitCode
+  }
+  finally {
+    # If we didn't finish then an error occured or the user hit ctrl-c.  Either
+    # way kill the process
+    if (-not $finished) {
+      $process.Kill()
+    }
+  }
+}
+
 function InitializeDotNetCli([bool]$install) {
+  if (Test-Path variable:global:_DotNetInstallDir) {
+    return $global:_DotNetInstallDir
+  }
+
   # Don't resolve runtime, shared framework, or SDK from other locations to ensure build determinism
   $env:DOTNET_MULTILEVEL_LOOKUP=0
 
   # Disable first run since we do not need all ASP.NET packages restored.
   $env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1
 
+  # Disable telemetry on CI.
+  if ($ci) {
+    $env:DOTNET_CLI_TELEMETRY_OPTOUT=1
+  }
+
   # Source Build uses DotNetCoreSdkDir variable
   if ($env:DotNetCoreSdkDir -ne $null) {
     $env:DOTNET_INSTALL_DIR = $env:DotNetCoreSdkDir
@@ -39,7 +109,10 @@ function InitializeDotNetCli([bool]$install) {
 
   # Find the first path on %PATH% that contains the dotnet.exe
   if ($useInstalledDotNetCli -and ($env:DOTNET_INSTALL_DIR -eq $null)) {
-    $env:DOTNET_INSTALL_DIR = ${env:PATH}.Split(';') | where { ($_ -ne "") -and (Test-Path (Join-Path $_ "dotnet.exe")) }
+    $dotnetCmd = Get-Command "dotnet.exe" -ErrorAction SilentlyContinue
+    if ($dotnetCmd -ne $null) {
+      $env:DOTNET_INSTALL_DIR = Split-Path $dotnetCmd.Path -Parent
+    }
   }
 
   $dotnetSdkVersion = $GlobalJson.tools.dotnet
@@ -50,13 +123,8 @@ function InitializeDotNetCli([bool]$install) {
     $dotnetRoot = $env:DOTNET_INSTALL_DIR
   } else {
     $dotnetRoot = Join-Path $RepoRoot ".dotnet"
-    if ($env:ARCADE_DOTNET_DIR -ne $null)
-    {
-      $dotnetRoot = $env:ARCADE_DOTNET_DIR
-    }
-    $env:DOTNET_INSTALL_DIR = $dotnetRoot
 
-    if (-not (Test-Path(Join-Path $env:DOTNET_INSTALL_DIR "sdk\$dotnetSdkVersion"))) {
+    if (-not (Test-Path(Join-Path $dotnetRoot "sdk\$dotnetSdkVersion"))) {
       if ($install) {
         InstallDotNetSdk $dotnetRoot $dotnetSdkVersion
       } else {
@@ -64,9 +132,16 @@ function InitializeDotNetCli([bool]$install) {
         ExitWithExitCode 1
       }
     }
+
+    $env:DOTNET_INSTALL_DIR = $dotnetRoot
   }
 
-  return $dotnetRoot
+  # Add dotnet to PATH. This prevents any bare invocation of dotnet in custom
+  # build steps from using anything other than what we've downloaded.
+  # It also ensures that VS msbuild will use the downloaded sdk targets.
+  $env:PATH = "$dotnetRoot;$env:PATH"
+
+  return $global:_DotNetInstallDir = $dotnetRoot
 }
 
 function GetDotNetInstallScript([string] $dotnetRoot) {
@@ -99,8 +174,13 @@ function InstallDotNetSdk([string] $dotnetRoot, [string] $version) {
 # Returns full path to msbuild.exe.
 # Throws on failure.
 #
-function InitializeVisualStudioMSBuild {
+function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = $null) {
-  $vsMinVersionStr = if (!$GlobalJson.tools.vs.version) { $GlobalJson.tools.vs.version } else { "15.9" }
+  if (Test-Path variable:global:_MSBuildExe) {
+    return $global:_MSBuildExe
+  }
+
+  if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs }
+  $vsMinVersionStr = if ($vsRequirements.version) { $vsRequirements.version } else { "15.9" }
   $vsMinVersion = [Version]::new($vsMinVersionStr) 
 
   # Try msbuild command available in the environment.
@@ -108,7 +188,7 @@ function InitializeVisualStudioMSBuild {
     $msbuildCmd = Get-Command "msbuild.exe" -ErrorAction SilentlyContinue
     if ($msbuildCmd -ne $null) {
       if ($msbuildCmd.Version -ge $vsMinVersion) {
-        return $msbuildCmd.Path
+        return $global:_MSBuildExe = $msbuildCmd.Path
       }
 
       # Report error - the developer environment is initialized with incompatible VS version.
@@ -117,13 +197,13 @@ function InitializeVisualStudioMSBuild {
   }
 
   # Locate Visual Studio installation or download x-copy msbuild.
-  $vsInfo = LocateVisualStudio  
+  $vsInfo = LocateVisualStudio $vsRequirements
   if ($vsInfo -ne $null) {
     $vsInstallDir = $vsInfo.installationPath
     $vsMajorVersion = $vsInfo.installationVersion.Split('.')[0]
 
     InitializeVisualStudioEnvironmentVariables $vsInstallDir $vsMajorVersion
-  } else {
+  } elseif ($install) {
 
     if (Get-Member -InputObject $GlobalJson.tools -Name "xcopy-msbuild") {
       $xcopyMSBuildVersion = $GlobalJson.tools.'xcopy-msbuild'
@@ -134,10 +214,12 @@ function InitializeVisualStudioMSBuild {
     }
 
     $vsInstallDir = InstallXCopyMSBuild $xcopyMSBuildVersion
+  } else {
+    throw "Unable to find Visual Studio that has required version and components installed"
   }
 
   $msbuildVersionDir = if ([int]$vsMajorVersion -lt 16) { "$vsMajorVersion.0" } else { "Current" }
-  return Join-Path $vsInstallDir "MSBuild\$msbuildVersionDir\Bin\msbuild.exe"
+  return $global:_MSBuildExe = Join-Path $vsInstallDir "MSBuild\$msbuildVersionDir\Bin\msbuild.exe"
 }
 
 function InitializeVisualStudioEnvironmentVariables([string] $vsInstallDir, [string] $vsMajorVersion) {
@@ -179,9 +261,10 @@ function InstallXCopyMSBuild([string] $packageVersion) {
 # Returns JSON describing the located VS instance (same format as returned by vswhere), 
 # or $null if no instance meeting the requirements is found on the machine.
 #
-function LocateVisualStudio {
+function LocateVisualStudio([object]$vsRequirements = $null){
-  $vswhereVersion = Get-Member -InputObject $GlobalJson.tools -Name "vswhere"
+  if (Get-Member -InputObject $GlobalJson.tools -Name "vswhere") {
-  if ($vsWhereVersion -eq $null) {
+    $vswhereVersion = $GlobalJson.tools.vswhere
+  } else {
     $vswhereVersion = "2.5.2"
   }
 
@@ -194,16 +277,16 @@ function LocateVisualStudio {
     Invoke-WebRequest "https://github.com/Microsoft/vswhere/releases/download/$vswhereVersion/vswhere.exe" -OutFile $vswhereExe
   }
 
-  $vs = $GlobalJson.tools.vs
+  if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs }
   $args = @("-latest", "-prerelease", "-format", "json", "-requires", "Microsoft.Component.MSBuild")
   
-  if (Get-Member -InputObject $vs -Name "version") { 
+  if (Get-Member -InputObject $vsRequirements -Name "version") {
     $args += "-version"
-    $args += $vs.version
+    $args += $vsRequirements.version
   }
 
-  if (Get-Member -InputObject $vs -Name "components") { 
+  if (Get-Member -InputObject $vsRequirements -Name "components") {
-    foreach ($component in $vs.components) {
+    foreach ($component in $vsRequirements.components) {
       $args += "-requires"
       $args += $component
     }    
@@ -219,36 +302,19 @@ function LocateVisualStudio {
   return $vsInfo[0]
 }
 
-function ConfigureTools { 
+function InitializeBuildTool() {
-  # Include custom tools configuration
+  if (Test-Path variable:global:_BuildTool) {
-  $script = Join-Path $EngRoot "configure-toolset.ps1"
+    return $global:_BuildTool
-
-  if (Test-Path $script) {
-    . $script
-  }
-}
-
-function InitializeTools() {
-  ConfigureTools
-
-  $tools = $GlobalJson.tools
-
-  # Initialize dotnet cli if listed in 'tools'
-  $dotnetRoot = $null
-  if (Get-Member -InputObject $tools -Name "dotnet") {
-    $dotnetRoot = InitializeDotNetCli -install:$restore
   }
 
   if (-not $msbuildEngine) {
-    # Presence of tools.vs indicates the repo needs to build using VS msbuild on Windows.
+    $msbuildEngine = GetDefaultMSBuildEngine
-    if (Get-Member -InputObject $tools -Name "vs") {
+  }
-      $msbuildEngine = "vs"
+
-    } elseif ($dotnetRoot -ne $null) {
+  # Initialize dotnet cli if listed in 'tools'
-      $msbuildEngine = "dotnet"
+  $dotnetRoot = $null
-    } else {
+  if (Get-Member -InputObject $GlobalJson.tools -Name "dotnet") {
-      Write-Host "-msbuildEngine must be specified, or /global.json must specify 'tools.dotnet' or 'tools.vs'." -ForegroundColor Red
+    $dotnetRoot = InitializeDotNetCli -install:$restore
-      ExitWithExitCode 1
-    }
   }
 
   if ($msbuildEngine -eq "dotnet") {
@@ -257,34 +323,66 @@ function InitializeTools() {
       ExitWithExitCode 1
     }
 
-    $script:buildDriver = Join-Path $dotnetRoot "dotnet.exe"
+    $buildTool = @{ Path = Join-Path $dotnetRoot "dotnet.exe"; Command = "msbuild" }
-    $script:buildArgs = "msbuild"
   } elseif ($msbuildEngine -eq "vs") {
     try {
-      $script:buildDriver = InitializeVisualStudioMSBuild
+      $msbuildPath = InitializeVisualStudioMSBuild -install:$restore
-      $script:buildArgs = ""
+    } catch {
-    } catch { 
       Write-Host $_ -ForegroundColor Red
       ExitWithExitCode 1
     }
+
+    $buildTool = @{ Path = $msbuildPath; Command = "" }
   } else {
     Write-Host "Unexpected value of -msbuildEngine: '$msbuildEngine'." -ForegroundColor Red
     ExitWithExitCode 1
   }
 
-  InitializeToolSet $script:buildDriver $script:buildArgs
+  return $global:_BuildTool = $buildTool
-  InitializeCustomToolset
 }
 
-function InitializeToolset([string] $buildDriver, [string]$buildArgs) {
+function GetDefaultMSBuildEngine() {
+  # Presence of tools.vs indicates the repo needs to build using VS msbuild on Windows.
+  if (Get-Member -InputObject $GlobalJson.tools -Name "vs") {
+    return "vs"
+  }
+  
+  if (Get-Member -InputObject $GlobalJson.tools -Name "dotnet") {
+    return "dotnet"
+  }
+
+  Write-Host "-msbuildEngine must be specified, or /global.json must specify 'tools.dotnet' or 'tools.vs'." -ForegroundColor Red
+  ExitWithExitCode 1
+}
+
+function GetNuGetPackageCachePath() {
+  if ($env:NUGET_PACKAGES -eq $null) {
+    # Use local cache on CI to ensure deterministic build,
+    # use global cache in dev builds to avoid cost of downloading packages.
+    if ($useGlobalNuGetCache) {
+      $env:NUGET_PACKAGES = Join-Path $env:UserProfile ".nuget\packages"
+    } else {
+      $env:NUGET_PACKAGES = Join-Path $RepoRoot ".packages"
+    }
+  }
+
+  return $env:NUGET_PACKAGES
+}
+
+function InitializeToolset() {
+  if (Test-Path variable:global:_ToolsetBuildProj) {
+    return $global:_ToolsetBuildProj
+  }
+
+  $nugetCache = GetNuGetPackageCachePath
+
   $toolsetVersion = $GlobalJson.'msbuild-sdks'.'Microsoft.DotNet.Arcade.Sdk'
   $toolsetLocationFile = Join-Path $ToolsetDir "$toolsetVersion.txt"
 
   if (Test-Path $toolsetLocationFile) {
     $path = Get-Content $toolsetLocationFile -TotalCount 1
     if (Test-Path $path) {
-      $script:ToolsetBuildProj = $path
+      return $global:_ToolsetBuildProj = $path
-      return
     }
   }
 
@@ -293,35 +391,20 @@ function InitializeToolset([string] $buildDriver, [string]$buildArgs) {
     ExitWithExitCode 1
   }
 
-  $ToolsetRestoreLog = Join-Path $LogDir "ToolsetRestore.binlog"
+  $buildTool = InitializeBuildTool
+
   $proj = Join-Path $ToolsetDir "restore.proj"
-
+  $bl = if ($binaryLog) { "/bl:" + (Join-Path $LogDir "ToolsetRestore.binlog") } else { "" }
+  
   '<Project Sdk="Microsoft.DotNet.Arcade.Sdk"/>' | Set-Content $proj
-  MSBuild $proj /t:__WriteToolsetLocation /clp:None /bl:$ToolsetRestoreLog /p:__ToolsetLocationOutputFile=$toolsetLocationFile
+  MSBuild $proj $bl /t:__WriteToolsetLocation /noconsolelogger /p:__ToolsetLocationOutputFile=$toolsetLocationFile
-
+  
-  if ($lastExitCode -ne 0) {
-    Write-Host "Failed to restore toolset (exit code '$lastExitCode'). See log: $ToolsetRestoreLog" -ForegroundColor Red
-    ExitWithExitCode $lastExitCode
-  }
-
   $path = Get-Content $toolsetLocationFile -TotalCount 1
   if (!(Test-Path $path)) {
     throw "Invalid toolset path: $path"
   }
-
+  
-  $script:ToolsetBuildProj = $path
+  return $global:_ToolsetBuildProj = $path
-}
-
-function InitializeCustomToolset {
-  if (-not $restore) {
-    return
-  }
-
-  $script = Join-Path $EngRoot "restore-toolset.ps1"
-
-  if (Test-Path $script) {
-    . $script
-  }
 }
 
 function ExitWithExitCode([int] $exitCode) {
@@ -333,42 +416,86 @@ function ExitWithExitCode([int] $exitCode) {
 
 function Stop-Processes() {
   Write-Host "Killing running build processes..."
-  Get-Process -Name "msbuild" -ErrorAction SilentlyContinue | Stop-Process
+  foreach ($processName in $processesToStopOnExit) {
-  Get-Process -Name "dotnet" -ErrorAction SilentlyContinue | Stop-Process
+    Get-Process -Name $processName -ErrorAction SilentlyContinue | Stop-Process
-  Get-Process -Name "vbcscompiler" -ErrorAction SilentlyContinue | Stop-Process
+  }
 }
 
-function MsBuild() {
+#
-  $warnaserrorSwitch = if ($warnaserror) { "/warnaserror" } else { "" }
+# Executes msbuild (or 'dotnet msbuild') with arguments passed to the function.
-  & $buildDriver $buildArgs $warnaserrorSwitch /m /nologo /clp:Summary /v:$verbosity /nr:$nodereuse $args
+# The arguments are automatically quoted.
+# Terminates the script if the build fails.
+#
+function MSBuild() {
+  if ($ci) {
+    if (!$binaryLog) {
+      throw "Binary log must be enabled in CI build."
+    }
+
+    if ($nodeReuse) {
+      throw "Node reuse must be disabled in CI build."
+    }
+  }
+
+  $buildTool = InitializeBuildTool
+
+  $cmdArgs = "$($buildTool.Command) /m /nologo /clp:Summary /v:$verbosity /nr:$nodeReuse"
+
+  if ($warnAsError) { 
+    $cmdArgs += " /warnaserror /p:TreatWarningsAsErrors=true" 
+  }
+
+  foreach ($arg in $args) {
+    if ($arg -ne $null -and $arg.Trim() -ne "") {
+      $cmdArgs += " `"$arg`""
+    }
+  }
+  
+  $exitCode = Exec-Process $buildTool.Path $cmdArgs
+
+  if ($exitCode -ne 0) {
+    Write-Host "Build failed." -ForegroundColor Red
+
+    $buildLog = GetMSBuildBinaryLogCommandLineArgument $args
+    if ($buildLog -ne $null) {      
+      Write-Host "See log: $buildLog" -ForegroundColor DarkGray 
+    }
+
+    ExitWithExitCode $exitCode
+  }
+}
+
+function GetMSBuildBinaryLogCommandLineArgument($arguments) {  
+  foreach ($argument in $arguments) {
+    if ($argument -ne $null) {
+      $arg = $argument.Trim()
+      if ($arg.StartsWith("/bl:", "OrdinalIgnoreCase")) {
+        return $arg.Substring("/bl:".Length)
+      } 
+        
+      if ($arg.StartsWith("/binaryLogger:", "OrdinalIgnoreCase")) {
+        return $arg.Substring("/binaryLogger:".Length)
+      }
+    }
+  }
+
+  return $null
 }
 
 $RepoRoot = Resolve-Path (Join-Path $PSScriptRoot "..\..")
 $EngRoot = Resolve-Path (Join-Path $PSScriptRoot "..")
 $ArtifactsDir = Join-Path $RepoRoot "artifacts"
-if ($env:ARCADE_ARTIFACTS_DIR -ne $null)
-{
-  $ArtifactsDir = [System.IO.Path]::GetFullPath($env:ARCADE_ARTIFACTS_DIR) + "\"
-  $env:ArtifactsDir = $ArtifactsDir
-}
 $ToolsetDir = Join-Path $ArtifactsDir "toolset"
 $ToolsDir = Join-Path $RepoRoot ".tools"
 $LogDir = Join-Path (Join-Path $ArtifactsDir "log") $configuration
 $TempDir = Join-Path (Join-Path $ArtifactsDir "tmp") $configuration
 $GlobalJson = Get-Content -Raw -Path (Join-Path $RepoRoot "global.json") | ConvertFrom-Json
 
-if ($env:NUGET_PACKAGES -eq $null) {
-  # Use local cache on CI to ensure deterministic build,
-  # use global cache in dev builds to avoid cost of downloading packages.
-  $env:NUGET_PACKAGES = if ($ci) { Join-Path $RepoRoot ".packages" }
-                        else { Join-Path $env:UserProfile ".nuget\packages" }
-}
-
 Create-Directory $ToolsetDir
+Create-Directory $TempDir
 Create-Directory $LogDir
 
 if ($ci) {
-  Create-Directory $TempDir
   $env:TEMP = $TempDir
   $env:TMP = $TempDir
 }

eng/common/tools.sh

@@ -3,35 +3,52 @@
 # Stop script if unbound variable found (use ${var:-} if intentional)
 set -u
 
-ci=${ci:-false}
+# Initialize variables if they aren't already defined.
-configuration=${configuration:-'Debug'}
-nodereuse=${nodereuse:-true}
-prepare_machine=${prepare_machine:-false}
-restore=${restore:-true}
-verbosity=${verbosity:-'minimal'}
-warnaserror=${warnaserror:-true}
-useInstalledDotNetCli=${useInstalledDotNetCli:-true}
 
-repo_root="$scriptroot/../.."
+# CI mode - set to true on CI server for PR validation build or official build.
-eng_root="$scriptroot/.."
+ci=${ci:-false}
-artifacts_dir="$repo_root/artifacts"
+
-if [[ -n "${ARCADE_ARTIFACTS_DIR:-}" ]]; then
+# Build configuration. Common values include 'Debug' and 'Release', but the repository may use other names.
-  artifacts_dir="$ARCADE_ARTIFACTS_DIR"
+configuration=${configuration:-'Debug'}
-  export ArtifactsDir="$artifacts_dir"
+
+# Set to true to output binary log from msbuild. Note that emitting binary log slows down the build.
+# Binary log must be enabled on CI.
+binary_log=${binary_log:-$ci}
+
+# Turns on machine preparation/clean up code that changes the machine state (e.g. kills build processes).
+prepare_machine=${prepare_machine:-false}
+
+# True to restore toolsets and dependencies.
+restore=${restore:-true}
+
+# Adjusts msbuild verbosity level.
+verbosity=${verbosity:-'minimal'}
+
+# Set to true to reuse msbuild nodes. Recommended to not reuse on CI.
+if [[ "$ci" == true ]]; then
+  node_reuse=${node_reuse:-false}
+else
+  node_reuse=${node_reuse:-true}
 fi
 
-toolset_dir="$artifacts_dir/toolset"
+# Configures warning treatment in msbuild.
-log_dir="$artifacts_dir/log/$configuration"
+warn_as_error=${warn_as_error:-true}
-temp_dir="$artifacts_dir/tmp/$configuration"
 
-global_json_file="$repo_root/global.json"
+# True to attempt using .NET Core already that meets requirements specified in global.json 
-build_driver=""
+# installed on the machine instead of downloading one.
-toolset_build_proj=""
+use_installed_dotnet_cli=${use_installed_dotnet_cli:-true}
 
+# True to use global NuGet cache instead of restoring packages to repository-local directory.
+if [[ "$ci" == true ]]; then
+  use_global_nuget_cache=${use_global_nuget_cache:-false}
+else
+  use_global_nuget_cache=${use_global_nuget_cache:-true}
+fi
+
+# Resolve any symlinks in the given path.
 function ResolvePath {
   local path=$1
 
-  # resolve $path until the file is no longer a symlink
   while [[ -h $path ]]; do
     local dir="$( cd -P "$( dirname "$path" )" && pwd )"
     path="$(readlink "$path")"
@@ -62,6 +79,10 @@ function ReadGlobalVersion {
 }
 
 function InitializeDotNetCli {
+  if [[ -n "${_InitializeDotNetCli:-}" ]]; then
+    return
+  fi
+
   local install=$1
 
   # Don't resolve runtime, shared framework, or SDK from other locations to ensure build determinism
@@ -70,13 +91,22 @@ function InitializeDotNetCli {
   # Disable first run since we want to control all package sources
   export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1
 
+  # Disable telemetry on CI
+  if [[ $ci == true ]]; then
+    export DOTNET_CLI_TELEMETRY_OPTOUT=1
+  fi
+
+  # LTTNG is the logging infrastructure used by Core CLR. Need this variable set
+  # so it doesn't output warnings to the console.
+  export LTTNG_HOME="$HOME"
+
   # Source Build uses DotNetCoreSdkDir variable
   if [[ -n "${DotNetCoreSdkDir:-}" ]]; then
     export DOTNET_INSTALL_DIR="$DotNetCoreSdkDir"
   fi
 
   # Find the first path on $PATH that contains the dotnet.exe
-  if [[ "$useInstalledDotNetCli" == true && -z "${DOTNET_INSTALL_DIR:-}" ]]; then
+  if [[ "$use_installed_dotnet_cli" == true && -z "${DOTNET_INSTALL_DIR:-}" ]]; then
     local dotnet_path=`command -v dotnet`
     if [[ -n "$dotnet_path" ]]; then
       ResolvePath "$dotnet_path"
@@ -94,10 +124,6 @@ function InitializeDotNetCli {
     dotnet_root="$DOTNET_INSTALL_DIR"
   else
     dotnet_root="$repo_root/.dotnet"
-    if [[ -n "${ARCADE_DOTNET_DIR:-}" ]]; then
-      dotnet_root="$ARCADE_DOTNET_DIR"
-    fi
-
     export DOTNET_INSTALL_DIR="$dotnet_root"
 
     if [[ ! -d "$DOTNET_INSTALL_DIR/sdk/$dotnet_sdk_version" ]]; then
@@ -110,6 +136,10 @@ function InitializeDotNetCli {
     fi
   fi
 
+  # Add dotnet to PATH. This prevents any bare invocation of dotnet in custom
+  # build steps from using anything other than what we've downloaded.
+  export PATH="$dotnet_root:$PATH"
+
   # return value
   _InitializeDotNetCli="$dotnet_root"
 }
@@ -152,7 +182,37 @@ function GetDotNetInstallScript {
   _GetDotNetInstallScript="$install_script"
 }
 
+function InitializeBuildTool {
+  if [[ -n "${_InitializeBuildTool:-}" ]]; then
+    return
+  fi
+  
+  InitializeDotNetCli $restore
+
+  # return value
+  _InitializeBuildTool="$_InitializeDotNetCli/dotnet"  
+}
+
+function GetNuGetPackageCachePath {
+  if [[ -z ${NUGET_PACKAGES:-} ]]; then
+    if [[ "$use_global_nuget_cache" == true ]]; then
+      export NUGET_PACKAGES="$HOME/.nuget/packages"
+    else
+      export NUGET_PACKAGES="$repo_root/.packages"
+    fi
+  fi
+
+  # return value
+  _GetNuGetPackageCachePath=$NUGET_PACKAGES
+}
+
 function InitializeToolset {
+  if [[ -n "${_InitializeToolset:-}" ]]; then
+    return
+  fi
+
+  GetNuGetPackageCachePath
+
   ReadGlobalVersion "Microsoft.DotNet.Arcade.Sdk"
 
   local toolset_version=$_ReadGlobalVersion
@@ -161,7 +221,8 @@ function InitializeToolset {
   if [[ -a "$toolset_location_file" ]]; then
     local path=`cat "$toolset_location_file"`
     if [[ -a "$path" ]]; then
-      toolset_build_proj="$path"
+      # return value
+      _InitializeToolset="$path"
       return
     fi
   fi
@@ -175,47 +236,17 @@ function InitializeToolset {
   local proj="$toolset_dir/restore.proj"
 
   echo '<Project Sdk="Microsoft.DotNet.Arcade.Sdk"/>' > "$proj"
+  MSBuild "$proj" /t:__WriteToolsetLocation /noconsolelogger /bl:"$toolset_restore_log" /p:__ToolsetLocationOutputFile="$toolset_location_file"
 
-  MSBuild "$proj" /t:__WriteToolsetLocation /clp:None /bl:"$toolset_restore_log" /p:__ToolsetLocationOutputFile="$toolset_location_file"
+  local toolset_build_proj=`cat "$toolset_location_file"`
-  local lastexitcode=$?
-
-  if [[ $lastexitcode != 0 ]]; then
-    echo "Failed to restore toolset (exit code '$lastexitcode'). See log: $toolset_restore_log" >&2
-    ExitWithExitCode $lastexitcode
-  fi
-
-  toolset_build_proj=`cat "$toolset_location_file"`
 
   if [[ ! -a "$toolset_build_proj" ]]; then
     echo "Invalid toolset path: $toolset_build_proj" >&2
     ExitWithExitCode 3
   fi
-}
 
-function InitializeCustomToolset {
+  # return value
-  local script="$eng_root/restore-toolset.sh"
+  _InitializeToolset="$toolset_build_proj"
-
-  if [[ -a "$script" ]]; then
-    . "$script"
-  fi
-}
-
-function ConfigureTools {
-  local script="$eng_root/configure-toolset.sh"
-
-  if [[ -a "$script" ]]; then
-    . "$script"
-  fi
-}
-
-function InitializeTools {
-  ConfigureTools
-
-  InitializeDotNetCli $restore
-  build_driver="$_InitializeDotNetCli/dotnet"
-
-  InitializeToolset
-  InitializeCustomToolset
 }
 
 function ExitWithExitCode {
@@ -233,35 +264,57 @@ function StopProcesses {
 }
 
 function MSBuild {
+  if [[ "$ci" == true ]]; then
+    if [[ "$binary_log" != true ]]; then
+      echo "Binary log must be enabled in CI build." >&2
+      ExitWithExitCode 1
+    fi
+
+    if [[ "$node_reuse" == true ]]; then
+      echo "Node reuse must be disabled in CI build." >&2
+      ExitWithExitCode 1
+    fi
+  fi
+
+  InitializeBuildTool
+
   local warnaserror_switch=""
-  if [[ $warnaserror == true ]]; then
+  if [[ $warn_as_error == true ]]; then
     warnaserror_switch="/warnaserror"
   fi
 
-  "$build_driver" msbuild /m /nologo /clp:Summary /v:$verbosity /nr:$nodereuse $warnaserror_switch "$@"
+  "$_InitializeBuildTool" msbuild /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch /p:TreatWarningsAsErrors=$warn_as_error "$@"
+  lastexitcode=$?
 
-  return $?
+  if [[ $lastexitcode != 0 ]]; then
+    echo "Build failed (exit code '$lastexitcode')." >&2
+    ExitWithExitCode $lastexitcode
+  fi
 }
 
+ResolvePath "${BASH_SOURCE[0]}"
+_script_dir=`dirname "$_ResolvePath"`
+
+eng_root=`cd -P "$_script_dir/.." && pwd`
+repo_root=`cd -P "$_script_dir/../.." && pwd`
+artifacts_dir="$repo_root/artifacts"
+toolset_dir="$artifacts_dir/toolset"
+log_dir="$artifacts_dir/log/$configuration"
+temp_dir="$artifacts_dir/tmp/$configuration"
+
+global_json_file="$repo_root/global.json"
+
 # HOME may not be defined in some scenarios, but it is required by NuGet
 if [[ -z $HOME ]]; then
   export HOME="$repo_root/artifacts/.home/"
   mkdir -p "$HOME"
 fi
 
-if [[ -z ${NUGET_PACKAGES:-} ]]; then
-  if [[ $ci == true ]]; then
-    export NUGET_PACKAGES="$repo_root/.packages"
-  else
-    export NUGET_PACKAGES="$HOME/.nuget/packages"
-  fi
-fi
-
 mkdir -p "$toolset_dir"
+mkdir -p "$temp_dir"
 mkdir -p "$log_dir"
 
 if [[ $ci == true ]]; then
-  mkdir -p "$temp_dir"
   export TEMP="$temp_dir"
   export TMP="$temp_dir"
 fi

global.json

@@ -3,6 +3,6 @@
     "dotnet": "3.0.100-alpha1-009697"
   },
   "msbuild-sdks": {
-    "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.18577.9"
+    "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.18617.7"
   }
 }