diff --git a/eng/common/CIBuild.cmd b/eng/common/CIBuild.cmd index 6544b0cd5..56c2f25ac 100644 --- a/eng/common/CIBuild.cmd +++ b/eng/common/CIBuild.cmd @@ -1,3 +1,2 @@ @echo off -powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0Build.ps1""" -restore -build -test -sign -pack -publish -ci %*" -exit /b %ErrorLevel% +powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0Build.ps1""" -restore -build -test -sign -pack -publish -ci %*" \ No newline at end of file diff --git a/eng/common/PublishBuildAssets.cmd b/eng/common/PublishBuildAssets.cmd index 399ca0bd3..ac629f00e 100644 --- a/eng/common/PublishBuildAssets.cmd +++ b/eng/common/PublishBuildAssets.cmd @@ -1,3 +1,3 @@ @echo off -powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0Build.ps1""" -msbuildEngine dotnet -restore -execute /p:PublishBuildAssets=true /p:SdkTaskProjects=PublishBuildAssets.proj %*" +powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0Build.ps1""" -msbuildEngine dotnet -restore -execute -binaryLog /p:PublishBuildAssets=true /p:SdkTaskProjects=PublishBuildAssets.proj %*" exit /b %ErrorLevel% diff --git a/eng/common/PublishToPackageFeed.proj b/eng/common/PublishToPackageFeed.proj new file mode 100644 index 000000000..7dc478d98 --- /dev/null +++ b/eng/common/PublishToPackageFeed.proj @@ -0,0 +1,37 @@ + + + + netcoreapp2.1 + + + + + + + + + + + + + + + + + + diff --git a/eng/common/README.md b/eng/common/README.md new file mode 100644 index 000000000..ff49c3715 --- /dev/null +++ b/eng/common/README.md @@ -0,0 +1,28 @@ +# Don't touch this folder + + uuuuuuuuuuuuuuuuuuuu + u" uuuuuuuuuuuuuuuuuu "u + u" u$$$$$$$$$$$$$$$$$$$$u "u + u" u$$$$$$$$$$$$$$$$$$$$$$$$u "u + u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u + u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u + u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u + $ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $ + $ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $ + $ $$$" ... "$... ...$" ... "$$$ ... "$$$ $ + $ $$$u `"$$$$$$$ $$$ $$$$$ $$ $$$ $$$ $ + $ $$$$$$uu "$$$$ $$$ $$$$$ $$ """ u$$$ $ + $ $$$""$$$ $$$$ $$$u "$$$" u$$ $$$$$$$$ $ + $ $$$$....,$$$$$..$$$$$....,$$$$..$$$$$$$$ $ + $ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $ + "u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u" + "u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u" + "u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u" + "u "$$$$$$$$$$$$$$$$$$$$$$$$" u" + "u "$$$$$$$$$$$$$$$$$$$$" u" + "u """""""""""""""""" u" + """""""""""""""""""" + +!!! Changes made in this directory are subject to being overwritten by automation !!! + +The files in this directory are shared by all Arcade repos and managed by automation. If you need to make changes to these files, open an issue or submit a pull request to https://github.com/dotnet/arcade first. diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 index 76f108fd5..2f5e6052a 100644 --- a/eng/common/build.ps1 +++ b/eng/common/build.ps1 @@ -1,15 +1,15 @@ [CmdletBinding(PositionalBinding=$false)] Param( - [string] $configuration = "Debug", - [string] $projects = "", - [string] $verbosity = "minimal", + [string][Alias('c')]$configuration = "Debug", + [string] $projects, + [string][Alias('v')]$verbosity = "minimal", [string] $msbuildEngine = $null, - [bool] $warnaserror = $true, - [bool] $nodereuse = $true, + [bool] $warnAsError = $true, + [bool] $nodeReuse = $true, [switch] $execute, - [switch] $restore, + [switch][Alias('r')]$restore, [switch] $deployDeps, - [switch] $build, + [switch][Alias('b')]$build, [switch] $rebuild, [switch] $deploy, [switch] $test, @@ -19,6 +19,7 @@ Param( [switch] $pack, [switch] $publish, [switch] $publishBuildAssets, + [switch][Alias('bl')]$binaryLog, [switch] $ci, [switch] $prepareMachine, [switch] $help, @@ -29,14 +30,15 @@ Param( function Print-Usage() { Write-Host "Common settings:" - Write-Host " -configuration Build configuration Debug, Release" - Write-Host " -verbosity Msbuild verbosity (q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic])" + Write-Host " -configuration Build configuration: 'Debug' or 'Release' (short: -c)" + Write-Host " -verbosity Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic] (short: -v)" + Write-Host " -binaryLog Output binary log (short: -bl)" Write-Host " -help Print help and exit" Write-Host "" Write-Host "Actions:" - Write-Host " -restore Restore dependencies" - Write-Host " -build Build solution" + Write-Host " -restore Restore dependencies (short: -r)" + Write-Host " -build Build solution (short: -b)" Write-Host " -rebuild Rebuild solution" Write-Host " -deploy Deploy built VSIXes" Write-Host " -deployDeps Deploy dependencies (e.g. VSIXes for integration tests)" @@ -46,7 +48,7 @@ function Print-Usage() { Write-Host " -performanceTest Run all performance tests in the solution" Write-Host " -sign Sign build outputs" Write-Host " -publish Publish artifacts (e.g. symbols)" - Write-Host " -publishBuildAssets Push assets to BAR" + Write-Host " -publishBuildAssets Push assets to BAR" Write-Host "" Write-Host "Advanced settings:" @@ -59,24 +61,35 @@ function Print-Usage() { Write-Host "The above arguments can be shortened as much as to be unambiguous (e.g. -co for configuration, -t for test, etc.)." } -if ($help -or (($properties -ne $null) -and ($properties.Contains("/help") -or $properties.Contains("/?")))) { - Print-Usage - exit 0 -} -try { - if ($projects -eq "") { - $projects = Join-Path $RepoRoot "*.sln" +function InitializeCustomToolset { + if (-not $restore) { + return } - InitializeTools + $script = Join-Path $EngRoot "restore-toolset.ps1" - $BuildLog = Join-Path $LogDir "Build.binlog" + if (Test-Path $script) { + . $script + } +} - MSBuild $ToolsetBuildProj ` - /bl:$BuildLog ` +function Build { + $toolsetBuildProj = InitializeToolset + InitializeCustomToolset + $bl = if ($binaryLog) { "/bl:" + (Join-Path $LogDir "Build.binlog") } else { "" } + + if ($projects) { + # Re-assign properties to a new variable because PowerShell doesn't let us append properties directly for unclear reasons. + # Explicitly set the type as string[] because otherwise PowerShell would make this char[] if $properties is empty. + [string[]] $msbuildArgs = $properties + $msbuildArgs += "/p:Projects=$projects" + $properties = $msbuildArgs + } + + MSBuild $toolsetBuildProj ` + $bl ` /p:Configuration=$configuration ` - /p:Projects=$projects ` /p:RepoRoot=$RepoRoot ` /p:Restore=$restore ` /p:DeployDeps=$deployDeps ` @@ -92,13 +105,27 @@ try { /p:Execute=$execute ` /p:ContinuousIntegrationBuild=$ci ` @properties +} - if ($lastExitCode -ne 0) { - Write-Host "Build Failed (exit code '$lastExitCode'). See log: $BuildLog" -ForegroundColor Red - ExitWithExitCode $lastExitCode +try { + if ($help -or (($properties -ne $null) -and ($properties.Contains("/help") -or $properties.Contains("/?")))) { + Print-Usage + exit 0 } - ExitWithExitCode $lastExitCode + if ($ci) { + $binaryLog = $true + $nodeReuse = $false + } + + # Import custom tools configuration, if present in the repo. + # Note: Import in global scope so that the script set top-level variables without qualification. + $configureToolsetScript = Join-Path $EngRoot "configure-toolset.ps1" + if (Test-Path $configureToolsetScript) { + . $configureToolsetScript + } + + Build } catch { Write-Host $_ @@ -106,3 +133,5 @@ catch { Write-Host $_.ScriptStackTrace ExitWithExitCode 1 } + +ExitWithExitCode 0 diff --git a/eng/common/build.sh b/eng/common/build.sh old mode 100644 new mode 100755 index 941db3bd5..47af926d4 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -1,5 +1,35 @@ #!/usr/bin/env bash +# Stop script if unbound variable found (use ${var:-} if intentional) +set -u + +usage() +{ + echo "Common settings:" + echo " --configuration Build configuration: 'Debug' or 'Release' (short: --c)" + echo " --verbosity Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic] (short: -v)" + echo " --binaryLog Create MSBuild binary log (short: -bl)" + echo "" + echo "Actions:" + echo " --restore Restore dependencies (short: -r)" + echo " --build Build all projects (short: -b)" + echo " --rebuild Rebuild all projects" + echo " --test Run all unit tests (short: -t)" + echo " --sign Sign build outputs" + echo " --publish Publish artifacts (e.g. symbols)" + echo " --pack Package build outputs into NuGet packages and Willow components" + echo " --help Print help and exit (short: -h)" + echo "" + echo "Advanced settings:" + echo " --projects Project or solution file(s) to build" + echo " --ci Set when running on CI server" + echo " --prepareMachine Prepare machine for CI run, clean up processes after build" + echo " --nodeReuse Sets nodereuse msbuild parameter ('true' or 'false')" + echo " --warnAsError Sets warnaserror msbuild parameter ('true' or 'false')" + echo "" + echo "Command line arguments starting with '/p:' are passed through to MSBuild." +} + source="${BASH_SOURCE[0]}" # resolve $source until the file is no longer a symlink @@ -12,7 +42,6 @@ while [[ -h "$source" ]]; do done scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" -help=false restore=false build=false rebuild=false @@ -25,8 +54,9 @@ sign=false public=false ci=false -warnaserror=true -nodereuse=true +warn_as_error=true +node_reuse=true +binary_log=false projects='' configuration='Debug' @@ -34,138 +64,140 @@ prepare_machine=false verbosity='minimal' properties='' -while (($# > 0)); do - lowerI="$(echo $1 | awk '{print tolower($0)}')" - case $lowerI in - --build) - build=true - shift 1 - ;; - --ci) - ci=true - shift 1 - ;; - --configuration) - configuration=$2 - shift 2 - ;; - --help) - echo "Common settings:" - echo " --configuration Build configuration Debug, Release" - echo " --verbosity Msbuild verbosity (q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic])" - echo " --help Print help and exit" - echo "" - echo "Actions:" - echo " --restore Restore dependencies" - echo " --build Build solution" - echo " --rebuild Rebuild solution" - echo " --test Run all unit tests in the solution" - echo " --sign Sign build outputs" - echo " --publish Publish artifacts (e.g. symbols)" - echo " --pack Package build outputs into NuGet packages and Willow components" - echo "" - echo "Advanced settings:" - echo " --solution Path to solution to build" - echo " --ci Set when running on CI server" - echo " --prepareMachine Prepare machine for CI run" - echo "" - echo "Command line arguments not listed above are passed through to MSBuild." +while [[ $# > 0 ]]; do + opt="$(echo "$1" | awk '{print tolower($0)}')" + case "$opt" in + --help|-h) + usage exit 0 ;; - --pack) - pack=true - shift 1 + --configuration|-c) + configuration=$2 + shift ;; - --preparemachine) - prepare_machine=true - shift 1 + --verbosity|-v) + verbosity=$2 + shift + ;; + --binarylog|-bl) + binary_log=true + ;; + --restore|-r) + restore=true + ;; + --build|-b) + build=true ;; --rebuild) rebuild=true - shift 1 ;; - --restore) - restore=true - shift 1 + --pack) + pack=true ;; - --sign) - sign=true - shift 1 - ;; - --solution) - solution=$2 - shift 2 - ;; - --projects) - projects=$2 - shift 2 - ;; - --test) + --test|-t) test=true - shift 1 ;; --integrationtest) integration_test=true - shift 1 ;; --performancetest) performance_test=true - shift 1 + ;; + --sign) + sign=true ;; --publish) publish=true - shift 1 ;; - --verbosity) - verbosity=$2 - shift 2 + --preparemachine) + prepare_machine=true + ;; + --projects) + projects=$2 + shift + ;; + --ci) + ci=true ;; --warnaserror) - warnaserror=$2 - shift 2 + warn_as_error=$2 + shift ;; --nodereuse) - nodereuse=$2 - shift 2 + node_reuse=$2 + shift ;; - *) + /p:*) properties="$properties $1" - shift 1 + ;; + *) + echo "Invalid argument: $1" + usage + exit 1 ;; esac + + shift done +if [[ "$ci" == true ]]; then + binary_log=true + node_reuse=false +fi + . "$scriptroot/tools.sh" -if [[ -z $projects ]]; then - projects="$repo_root/*.sln" +function InitializeCustomToolset { + local script="$eng_root/restore-toolset.sh" + + if [[ -a "$script" ]]; then + . "$script" + fi +} + +function Build { + InitializeToolset + InitializeCustomToolset + + if [[ ! -z "$projects" ]]; then + properties="$properties /p:Projects=$projects" + fi + + local bl="" + if [[ "$binary_log" == true ]]; then + bl="/bl:\"$log_dir/Build.binlog\"" + fi + + MSBuild $_InitializeToolset \ + $bl \ + /p:Configuration=$configuration \ + /p:RepoRoot="$repo_root" \ + /p:Restore=$restore \ + /p:Build=$build \ + /p:Rebuild=$rebuild \ + /p:Test=$test \ + /p:Pack=$pack \ + /p:IntegrationTest=$integration_test \ + /p:PerformanceTest=$performance_test \ + /p:Sign=$sign \ + /p:Publish=$publish \ + /p:ContinuousIntegrationBuild=$ci \ + $properties + + ExitWithExitCode 0 +} + +# Import custom tools configuration, if present in the repo. +configure_toolset_script="$eng_root/configure-toolset.sh" +if [[ -a "$configure_toolset_script" ]]; then + . "$configure_toolset_script" fi -InitializeTools - -build_log="$log_dir/Build.binlog" - -MSBuild "$toolset_build_proj" \ - /bl:"$build_log" \ - /p:Configuration=$configuration \ - /p:Projects="$projects" \ - /p:RepoRoot="$repo_root" \ - /p:Restore=$restore \ - /p:Build=$build \ - /p:Rebuild=$rebuild \ - /p:Test=$test \ - /p:Pack=$pack \ - /p:IntegrationTest=$integration_test \ - /p:PerformanceTest=$performance_test \ - /p:Sign=$sign \ - /p:Publish=$publish \ - /p:ContinuousIntegrationBuild=$ci \ - $properties - -lastexitcode=$? - -if [[ $lastexitcode != 0 ]]; then - echo "Build failed (exit code '$lastexitcode'). See log: $build_log" +# TODO: https://github.com/dotnet/arcade/issues/1468 +# Temporary workaround to avoid breaking change. +# Remove once repos are updated. +if [[ -n "${useInstalledDotNetCli:-}" ]]; then + use_installed_dotnet_cli="$useInstalledDotNetCli" fi -ExitWithExitCode $lastexitcode +Build diff --git a/eng/common/cibuild.sh b/eng/common/cibuild.sh old mode 100644 new mode 100755 diff --git a/eng/common/darc-init.ps1 b/eng/common/darc-init.ps1 index af182d8f8..9ca150be8 100644 --- a/eng/common/darc-init.ps1 +++ b/eng/common/darc-init.ps1 @@ -3,7 +3,9 @@ $verbosity = "m" function InstallDarcCli { $darcCliPackageName = "microsoft.dotnet.darc" - $dotnet = "$env:DOTNET_INSTALL_DIR\dotnet.exe" + + $dotnetRoot = InitializeDotNetCli -install:$true + $dotnet = "$dotnetRoot\dotnet.exe" $toolList = Invoke-Expression "& `"$dotnet`" tool list -g" if ($toolList -like "*$darcCliPackageName*") { @@ -17,5 +19,4 @@ function InstallDarcCli { Invoke-Expression "& `"$dotnet`" tool install $darcCliPackageName --version $toolsetVersion -v $verbosity -g" } -InitializeTools InstallDarcCli diff --git a/eng/common/darc-init.sh b/eng/common/darc-init.sh old mode 100644 new mode 100755 index a0c733a15..bad07c3ae --- a/eng/common/darc-init.sh +++ b/eng/common/darc-init.sh @@ -17,10 +17,14 @@ verbosity=m function InstallDarcCli { local darc_cli_package_name="microsoft.dotnet.darc" - local uninstall_command=`$DOTNET_INSTALL_DIR/dotnet tool uninstall $darc_cli_package_name -g` - local tool_list=$($DOTNET_INSTALL_DIR/dotnet tool list -g) + + InitializeDotNetCli + local dotnet_root=$_InitializeDotNetCli + + local uninstall_command=`$dotnet_root/dotnet tool uninstall $darc_cli_package_name -g` + local tool_list=$($dotnet_root/dotnet tool list -g) if [[ $tool_list = *$darc_cli_package_name* ]]; then - echo $($DOTNET_INSTALL_DIR/dotnet tool uninstall $darc_cli_package_name -g) + echo $($dotnet_root/dotnet tool uninstall $darc_cli_package_name -g) fi ReadGlobalVersion "Microsoft.DotNet.Arcade.Sdk" @@ -28,8 +32,7 @@ function InstallDarcCli { echo "Installing Darc CLI version $toolset_version..." echo "You may need to restart your command shell if this is the first dotnet tool you have installed." - echo $($DOTNET_INSTALL_DIR/dotnet tool install $darc_cli_package_name --version $toolset_version -v $verbosity -g) + echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $toolset_version -v $verbosity -g) } -InitializeTools InstallDarcCli diff --git a/eng/common/msbuild.ps1 b/eng/common/msbuild.ps1 index 43b837f4d..b37fd3d5e 100644 --- a/eng/common/msbuild.ps1 +++ b/eng/common/msbuild.ps1 @@ -1,8 +1,8 @@ [CmdletBinding(PositionalBinding=$false)] Param( [string] $verbosity = "minimal", - [bool] $warnaserror = $true, - [bool] $nodereuse = $true, + [bool] $warnAsError = $true, + [bool] $nodeReuse = $true, [switch] $ci, [switch] $prepareMachine, [Parameter(ValueFromRemainingArguments=$true)][String[]]$extraArgs @@ -11,13 +11,17 @@ Param( . $PSScriptRoot\tools.ps1 try { - InitializeTools + if ($ci) { + $nodeReuse = $false + } + MSBuild @extraArgs - ExitWithExitCode $lastExitCode -} +} catch { Write-Host $_ Write-Host $_.Exception Write-Host $_.ScriptStackTrace ExitWithExitCode 1 } + +ExitWithExitCode 0 \ No newline at end of file diff --git a/eng/common/msbuild.sh b/eng/common/msbuild.sh old mode 100644 new mode 100755 index b1024487f..8160cd5a5 --- a/eng/common/msbuild.sh +++ b/eng/common/msbuild.sh @@ -13,10 +13,10 @@ done scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" verbosity='minimal' -warnaserror=true -nodereuse=true +warn_as_error=true +node_reuse=true prepare_machine=false -extraargs='' +extra_args='' while (($# > 0)); do lowerI="$(echo $1 | awk '{print tolower($0)}')" @@ -26,11 +26,11 @@ while (($# > 0)); do shift 2 ;; --warnaserror) - warnaserror=$2 + warn_as_error=$2 shift 2 ;; --nodereuse) - nodereuse=$2 + node_reuse=$2 shift 2 ;; --ci) @@ -42,7 +42,7 @@ while (($# > 0)); do shift 1 ;; *) - extraargs="$extraargs $1" + extra_args="$extra_args $1" shift 1 ;; esac @@ -50,6 +50,9 @@ done . "$scriptroot/tools.sh" -InitializeTools -MSBuild $extraargs -ExitWithExitCode $? +if [[ "$ci" == true ]]; then + node_reuse=false +fi + +MSBuild $extra_args +ExitWithExitCode 0 diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index 9fb858e48..5e293db35 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -95,11 +95,26 @@ jobs: variables: - ${{ each variable in parameters.variables }}: + # handle name-value variable syntax + # example: + # - name: [key] + # value: [value] - ${{ if ne(variable.name, '') }}: - name: ${{ variable.name }} value: ${{ variable.value }} + + # handle variable groups - ${{ if ne(variable.group, '') }}: - group: ${{ variable.group }} + + # handle key-value variable syntax. + # example: + # - [key]: [value] + - ${{ if and(eq(variable.name, ''), eq(variable.group, '')) }}: + - ${{ each pair in variable }}: + - name: ${{ pair.key }} + value: ${{ pair.value }} + # Add additional variables - ${{ if and(ne(parameters.helixRepo, ''), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notIn(variables['Build.Reason'], 'PullRequest')) }}: - name: _HelixSource diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml index b40016f6f..c094658fe 100644 --- a/eng/common/templates/job/publish-build-assets.yml +++ b/eng/common/templates/job/publish-build-assets.yml @@ -1,10 +1,4 @@ parameters: - # Optional: dependencies of the job - dependsOn: '' - - # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool - pool: {} - configuration: 'Debug' # Optional: condition for the job to run @@ -13,6 +7,15 @@ parameters: # Optional: 'true' if future jobs should run even if this job fails continueOnError: false + # Optional: dependencies of the job + dependsOn: '' + + # Optional: Include PublishBuildArtifacts task + enablePublishBuildArtifacts: false + + # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool + pool: {} + # Optional: should run as a public build even in the internal project # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. runAsPublic: false @@ -49,3 +52,12 @@ jobs: displayName: Publish Build Assets condition: ${{ parameters.condition }} continueOnError: ${{ parameters.continueOnError }} + - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: + - task: PublishBuildArtifacts@1 + displayName: Publish Logs to VSTS + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)' + PublishLocation: Container + ArtifactName: $(Agent.Os)_PublishBuildAssets + continueOnError: true + condition: always() diff --git a/eng/common/templates/jobs/jobs.yml b/eng/common/templates/jobs/jobs.yml index 6aa55e3af..c7226b12e 100644 --- a/eng/common/templates/jobs/jobs.yml +++ b/eng/common/templates/jobs/jobs.yml @@ -20,8 +20,8 @@ parameters: # Optional: enable sending telemetry # if 'true', these "variables" must be specified in the variables object or as part of the queue matrix # _HelixBuildConfig - differentiate between Debug, Release, other - # _HelixSource - Example: build/product/ - # _HelixType - Example: official/dotnet/arcade/$(Build.SourceBranch) + # _HelixType - Example: build/product/ + # _HelixSource - Example: official/dotnet/arcade/$(Build.SourceBranch) enableTelemetry: false # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job @@ -73,3 +73,5 @@ jobs: pool: vmImage: vs2017-win2016 runAsPublic: ${{ parameters.runAsPublic }} + enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }} + diff --git a/eng/common/templates/steps/helix-publish.yml b/eng/common/templates/steps/helix-publish.yml index 6dada380c..470ab65da 100644 --- a/eng/common/templates/steps/helix-publish.yml +++ b/eng/common/templates/steps/helix-publish.yml @@ -40,7 +40,7 @@ steps: WorkItemCommand: ${{ parameters.WorkItemCommand }} CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} XUnitProjects: ${{ parameters.XUnitProjects }} - XUnitTargetFramework: ${{ parameters.XUnitTargetFramework }} + XUnitRuntimeTargetFramework: ${{ parameters.XUnitTargetFramework }} XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} diff --git a/eng/common/templates/steps/send-to-helix.yml b/eng/common/templates/steps/send-to-helix.yml new file mode 100644 index 000000000..03f0e3866 --- /dev/null +++ b/eng/common/templates/steps/send-to-helix.yml @@ -0,0 +1,80 @@ +parameters: + HelixSource: 'pr/default' # required -- sources must start with pr/, official/, prodcon/, or agent/ + HelixType: 'tests/default/' # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/' + HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number + HelixTargetQueues: '' # required -- semicolon delimited list of Helix queues to test on; see https://helix.dot.net/api/2018-03-14/info/queues for a list of queues + HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group + HelixPreCommands: '' # optional -- commands to run before Helix work item execution + HelixPostCommands: '' # optional -- commands to run after Helix work item execution + WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects + WorkItemCommand: '' # optional -- a command to execute on the payload; requires WorkItemDirectory; incompatible with XUnitProjects + CorrelationPayloadDirectory: '' # optional -- a directory to zip up and send to Helix as a correlation payload + XUnitProjects: '' # optional -- semicolon delimited list of XUnitProjects to parse and send to Helix; requires XUnitRuntimeTargetFramework, XUnitPublishTargetFramework, XUnitRunnerVersion, and IncludeDotNetCli=true + XUnitPublishTargetFramework: '' # optional -- framework to use to publish your xUnit projects + XUnitRuntimeTargetFramework: '' # optional -- framework to use for the xUnit console runner + XUnitRunnerVersion: '' # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects + IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion + DotNetCliPackageType: '' # optional -- either 'sdk' or 'runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json + DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json + EnableXUnitReporter: false # optional -- true enables XUnit result reporting to Mission Control + WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget." + IsExternal: false # optional -- true requires Creator and will make the Mission Control results visible to folks outside the Microsoft Org + Creator: '' # optional -- if the build is external, use this to specify who is sending the job + condition: succeeded() # optional -- condition for step to execute; defaults to succeeded() + continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false + +steps: + - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 $env:BUILD_SOURCESDIRECTORY\eng\common\helixpublish.proj /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"' + displayName: Send job to Helix (Windows) + env: + BuildConfig: $(_BuildConfig) + HelixSource: ${{ parameters.HelixSource }} + HelixType: ${{ parameters.HelixType }} + HelixBuild: ${{ parameters.HelixBuild }} + HelixTargetQueues: ${{ parameters.HelixTargetQueues }} + HelixAccessToken: ${{ parameters.HelixAccessToken }} + HelixPreCommands: ${{ parameters.HelixPreCommands }} + HelixPostCommands: ${{ parameters.HelixPostCommands }} + WorkItemDirectory: ${{ parameters.WorkItemDirectory }} + WorkItemCommand: ${{ parameters.WorkItemCommand }} + CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} + XUnitProjects: ${{ parameters.XUnitProjects }} + XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} + XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} + XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} + IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} + DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} + DotNetCliVersion: ${{ parameters.DotNetCliVersion }} + EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }} + WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} + IsExternal: ${{ parameters.IsExternal }} + Creator: ${{ parameters.Creator }} + condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} + - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/eng/common/helixpublish.proj /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog + displayName: Send job to Helix (Unix) + env: + BuildConfig: $(_BuildConfig) + HelixSource: ${{ parameters.HelixSource }} + HelixType: ${{ parameters.HelixType }} + HelixBuild: ${{ parameters.HelixBuild }} + HelixTargetQueues: ${{ parameters.HelixTargetQueues }} + HelixAccessToken: ${{ parameters.HelixAccessToken }} + HelixPreCommands: ${{ parameters.HelixPreCommands }} + HelixPostCommands: ${{ parameters.HelixPostCommands }} + WorkItemDirectory: ${{ parameters.WorkItemDirectory }} + WorkItemCommand: ${{ parameters.WorkItemCommand }} + CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} + XUnitProjects: ${{ parameters.XUnitProjects }} + XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} + XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} + XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} + IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} + DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} + DotNetCliVersion: ${{ parameters.DotNetCliVersion }} + EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }} + WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} + IsExternal: ${{ parameters.IsExternal }} + Creator: ${{ parameters.Creator }} + condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index d877df6b8..a607ec608 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -1,14 +1,43 @@ -# Initialize variables if they aren't already defined +# Initialize variables if they aren't already defined. +# These may be defined as parameters of the importing script, or set after importing this script. -$ci = if (Test-Path variable:ci) { $ci } else { $false } -$configuration = if (Test-Path variable:configuration) { $configuration } else { "Debug" } -$nodereuse = if (Test-Path variable:nodereuse) { $nodereuse } else { !$ci } -$prepareMachine = if (Test-Path variable:prepareMachine) { $prepareMachine } else { $false } -$restore = if (Test-Path variable:restore) { $restore } else { $true } -$verbosity = if (Test-Path variable:verbosity) { $verbosity } else { "minimal" } -$warnaserror = if (Test-Path variable:warnaserror) { $warnaserror } else { $true } -$msbuildEngine = if (Test-Path variable:msbuildEngine) { $msbuildEngine } else { $null } -$useInstalledDotNetCli = if (Test-Path variable:useInstalledDotNetCli) { $useInstalledDotNetCli } else { $true } +# CI mode - set to true on CI server for PR validation build or official build. +[bool]$ci = if (Test-Path variable:ci) { $ci } else { $false } + +# Build configuration. Common values include 'Debug' and 'Release', but the repository may use other names. +[string]$configuration = if (Test-Path variable:configuration) { $configuration } else { "Debug" } + +# Set to true to output binary log from msbuild. Note that emitting binary log slows down the build. +# Binary log must be enabled on CI. +[bool]$binaryLog = if (Test-Path variable:binaryLog) { $binaryLog } else { $ci } + +# Turns on machine preparation/clean up code that changes the machine state (e.g. kills build processes). +[bool]$prepareMachine = if (Test-Path variable:prepareMachine) { $prepareMachine } else { $false } + +# True to restore toolsets and dependencies. +[bool]$restore = if (Test-Path variable:restore) { $restore } else { $true } + +# Adjusts msbuild verbosity level. +[string]$verbosity = if (Test-Path variable:verbosity) { $verbosity } else { "minimal" } + +# Set to true to reuse msbuild nodes. Recommended to not reuse on CI. +[bool]$nodeReuse = if (Test-Path variable:nodeReuse) { $nodeReuse } else { !$ci } + +# Configures warning treatment in msbuild. +[bool]$warnAsError = if (Test-Path variable:warnAsError) { $warnAsError } else { $true } + +# Specifies which msbuild engine to use for build: 'vs', 'dotnet' or unspecified (determined based on presence of tools.vs in global.json). +[string]$msbuildEngine = if (Test-Path variable:msbuildEngine) { $msbuildEngine } else { $null } + +# True to attempt using .NET Core already that meets requirements specified in global.json +# installed on the machine instead of downloading one. +[bool]$useInstalledDotNetCli = if (Test-Path variable:useInstalledDotNetCli) { $useInstalledDotNetCli } else { $true } + +# True to use global NuGet cache instead of restoring packages to repository-local directory. +[bool]$useGlobalNuGetCache = if (Test-Path variable:useGlobalNuGetCache) { $useGlobalNuGetCache } else { !$ci } + +# An array of names of processes to stop on script exit if prepareMachine is true. +$processesToStopOnExit = if (Test-Path variable:processesToStopOnExit) { $processesToStopOnExit } else { @("msbuild", "dotnet", "vbcscompiler") } set-strictmode -version 2.0 $ErrorActionPreference = "Stop" @@ -25,13 +54,54 @@ function Unzip([string]$zipfile, [string]$outpath) { [System.IO.Compression.ZipFile]::ExtractToDirectory($zipfile, $outpath) } +# This will exec a process using the console and return it's exit code. +# This will not throw when the process fails. +# Returns process exit code. +function Exec-Process([string]$command, [string]$commandArgs) { + $startInfo = New-Object System.Diagnostics.ProcessStartInfo + $startInfo.FileName = $command + $startInfo.Arguments = $commandArgs + $startInfo.UseShellExecute = $false + $startInfo.WorkingDirectory = Get-Location + + $process = New-Object System.Diagnostics.Process + $process.StartInfo = $startInfo + $process.Start() | Out-Null + + $finished = $false + try { + while (-not $process.WaitForExit(100)) { + # Non-blocking loop done to allow ctr-c interrupts + } + + $finished = $true + return $global:LASTEXITCODE = $process.ExitCode + } + finally { + # If we didn't finish then an error occured or the user hit ctrl-c. Either + # way kill the process + if (-not $finished) { + $process.Kill() + } + } +} + function InitializeDotNetCli([bool]$install) { + if (Test-Path variable:global:_DotNetInstallDir) { + return $global:_DotNetInstallDir + } + # Don't resolve runtime, shared framework, or SDK from other locations to ensure build determinism $env:DOTNET_MULTILEVEL_LOOKUP=0 # Disable first run since we do not need all ASP.NET packages restored. $env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 + # Disable telemetry on CI. + if ($ci) { + $env:DOTNET_CLI_TELEMETRY_OPTOUT=1 + } + # Source Build uses DotNetCoreSdkDir variable if ($env:DotNetCoreSdkDir -ne $null) { $env:DOTNET_INSTALL_DIR = $env:DotNetCoreSdkDir @@ -39,7 +109,10 @@ function InitializeDotNetCli([bool]$install) { # Find the first path on %PATH% that contains the dotnet.exe if ($useInstalledDotNetCli -and ($env:DOTNET_INSTALL_DIR -eq $null)) { - $env:DOTNET_INSTALL_DIR = ${env:PATH}.Split(';') | where { ($_ -ne "") -and (Test-Path (Join-Path $_ "dotnet.exe")) } + $dotnetCmd = Get-Command "dotnet.exe" -ErrorAction SilentlyContinue + if ($dotnetCmd -ne $null) { + $env:DOTNET_INSTALL_DIR = Split-Path $dotnetCmd.Path -Parent + } } $dotnetSdkVersion = $GlobalJson.tools.dotnet @@ -50,13 +123,8 @@ function InitializeDotNetCli([bool]$install) { $dotnetRoot = $env:DOTNET_INSTALL_DIR } else { $dotnetRoot = Join-Path $RepoRoot ".dotnet" - if ($env:ARCADE_DOTNET_DIR -ne $null) - { - $dotnetRoot = $env:ARCADE_DOTNET_DIR - } - $env:DOTNET_INSTALL_DIR = $dotnetRoot - if (-not (Test-Path(Join-Path $env:DOTNET_INSTALL_DIR "sdk\$dotnetSdkVersion"))) { + if (-not (Test-Path(Join-Path $dotnetRoot "sdk\$dotnetSdkVersion"))) { if ($install) { InstallDotNetSdk $dotnetRoot $dotnetSdkVersion } else { @@ -64,9 +132,16 @@ function InitializeDotNetCli([bool]$install) { ExitWithExitCode 1 } } + + $env:DOTNET_INSTALL_DIR = $dotnetRoot } - return $dotnetRoot + # Add dotnet to PATH. This prevents any bare invocation of dotnet in custom + # build steps from using anything other than what we've downloaded. + # It also ensures that VS msbuild will use the downloaded sdk targets. + $env:PATH = "$dotnetRoot;$env:PATH" + + return $global:_DotNetInstallDir = $dotnetRoot } function GetDotNetInstallScript([string] $dotnetRoot) { @@ -99,8 +174,13 @@ function InstallDotNetSdk([string] $dotnetRoot, [string] $version) { # Returns full path to msbuild.exe. # Throws on failure. # -function InitializeVisualStudioMSBuild { - $vsMinVersionStr = if (!$GlobalJson.tools.vs.version) { $GlobalJson.tools.vs.version } else { "15.9" } +function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = $null) { + if (Test-Path variable:global:_MSBuildExe) { + return $global:_MSBuildExe + } + + if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs } + $vsMinVersionStr = if ($vsRequirements.version) { $vsRequirements.version } else { "15.9" } $vsMinVersion = [Version]::new($vsMinVersionStr) # Try msbuild command available in the environment. @@ -108,7 +188,7 @@ function InitializeVisualStudioMSBuild { $msbuildCmd = Get-Command "msbuild.exe" -ErrorAction SilentlyContinue if ($msbuildCmd -ne $null) { if ($msbuildCmd.Version -ge $vsMinVersion) { - return $msbuildCmd.Path + return $global:_MSBuildExe = $msbuildCmd.Path } # Report error - the developer environment is initialized with incompatible VS version. @@ -117,13 +197,13 @@ function InitializeVisualStudioMSBuild { } # Locate Visual Studio installation or download x-copy msbuild. - $vsInfo = LocateVisualStudio + $vsInfo = LocateVisualStudio $vsRequirements if ($vsInfo -ne $null) { $vsInstallDir = $vsInfo.installationPath $vsMajorVersion = $vsInfo.installationVersion.Split('.')[0] InitializeVisualStudioEnvironmentVariables $vsInstallDir $vsMajorVersion - } else { + } elseif ($install) { if (Get-Member -InputObject $GlobalJson.tools -Name "xcopy-msbuild") { $xcopyMSBuildVersion = $GlobalJson.tools.'xcopy-msbuild' @@ -134,10 +214,12 @@ function InitializeVisualStudioMSBuild { } $vsInstallDir = InstallXCopyMSBuild $xcopyMSBuildVersion + } else { + throw "Unable to find Visual Studio that has required version and components installed" } $msbuildVersionDir = if ([int]$vsMajorVersion -lt 16) { "$vsMajorVersion.0" } else { "Current" } - return Join-Path $vsInstallDir "MSBuild\$msbuildVersionDir\Bin\msbuild.exe" + return $global:_MSBuildExe = Join-Path $vsInstallDir "MSBuild\$msbuildVersionDir\Bin\msbuild.exe" } function InitializeVisualStudioEnvironmentVariables([string] $vsInstallDir, [string] $vsMajorVersion) { @@ -179,9 +261,10 @@ function InstallXCopyMSBuild([string] $packageVersion) { # Returns JSON describing the located VS instance (same format as returned by vswhere), # or $null if no instance meeting the requirements is found on the machine. # -function LocateVisualStudio { - $vswhereVersion = Get-Member -InputObject $GlobalJson.tools -Name "vswhere" - if ($vsWhereVersion -eq $null) { +function LocateVisualStudio([object]$vsRequirements = $null){ + if (Get-Member -InputObject $GlobalJson.tools -Name "vswhere") { + $vswhereVersion = $GlobalJson.tools.vswhere + } else { $vswhereVersion = "2.5.2" } @@ -194,16 +277,16 @@ function LocateVisualStudio { Invoke-WebRequest "https://github.com/Microsoft/vswhere/releases/download/$vswhereVersion/vswhere.exe" -OutFile $vswhereExe } - $vs = $GlobalJson.tools.vs + if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs } $args = @("-latest", "-prerelease", "-format", "json", "-requires", "Microsoft.Component.MSBuild") - if (Get-Member -InputObject $vs -Name "version") { + if (Get-Member -InputObject $vsRequirements -Name "version") { $args += "-version" - $args += $vs.version + $args += $vsRequirements.version } - if (Get-Member -InputObject $vs -Name "components") { - foreach ($component in $vs.components) { + if (Get-Member -InputObject $vsRequirements -Name "components") { + foreach ($component in $vsRequirements.components) { $args += "-requires" $args += $component } @@ -219,36 +302,19 @@ function LocateVisualStudio { return $vsInfo[0] } -function ConfigureTools { - # Include custom tools configuration - $script = Join-Path $EngRoot "configure-toolset.ps1" - - if (Test-Path $script) { - . $script - } -} - -function InitializeTools() { - ConfigureTools - - $tools = $GlobalJson.tools - - # Initialize dotnet cli if listed in 'tools' - $dotnetRoot = $null - if (Get-Member -InputObject $tools -Name "dotnet") { - $dotnetRoot = InitializeDotNetCli -install:$restore +function InitializeBuildTool() { + if (Test-Path variable:global:_BuildTool) { + return $global:_BuildTool } if (-not $msbuildEngine) { - # Presence of tools.vs indicates the repo needs to build using VS msbuild on Windows. - if (Get-Member -InputObject $tools -Name "vs") { - $msbuildEngine = "vs" - } elseif ($dotnetRoot -ne $null) { - $msbuildEngine = "dotnet" - } else { - Write-Host "-msbuildEngine must be specified, or /global.json must specify 'tools.dotnet' or 'tools.vs'." -ForegroundColor Red - ExitWithExitCode 1 - } + $msbuildEngine = GetDefaultMSBuildEngine + } + + # Initialize dotnet cli if listed in 'tools' + $dotnetRoot = $null + if (Get-Member -InputObject $GlobalJson.tools -Name "dotnet") { + $dotnetRoot = InitializeDotNetCli -install:$restore } if ($msbuildEngine -eq "dotnet") { @@ -257,34 +323,66 @@ function InitializeTools() { ExitWithExitCode 1 } - $script:buildDriver = Join-Path $dotnetRoot "dotnet.exe" - $script:buildArgs = "msbuild" + $buildTool = @{ Path = Join-Path $dotnetRoot "dotnet.exe"; Command = "msbuild" } } elseif ($msbuildEngine -eq "vs") { try { - $script:buildDriver = InitializeVisualStudioMSBuild - $script:buildArgs = "" - } catch { + $msbuildPath = InitializeVisualStudioMSBuild -install:$restore + } catch { Write-Host $_ -ForegroundColor Red ExitWithExitCode 1 } + + $buildTool = @{ Path = $msbuildPath; Command = "" } } else { Write-Host "Unexpected value of -msbuildEngine: '$msbuildEngine'." -ForegroundColor Red ExitWithExitCode 1 } - InitializeToolSet $script:buildDriver $script:buildArgs - InitializeCustomToolset + return $global:_BuildTool = $buildTool } -function InitializeToolset([string] $buildDriver, [string]$buildArgs) { +function GetDefaultMSBuildEngine() { + # Presence of tools.vs indicates the repo needs to build using VS msbuild on Windows. + if (Get-Member -InputObject $GlobalJson.tools -Name "vs") { + return "vs" + } + + if (Get-Member -InputObject $GlobalJson.tools -Name "dotnet") { + return "dotnet" + } + + Write-Host "-msbuildEngine must be specified, or /global.json must specify 'tools.dotnet' or 'tools.vs'." -ForegroundColor Red + ExitWithExitCode 1 +} + +function GetNuGetPackageCachePath() { + if ($env:NUGET_PACKAGES -eq $null) { + # Use local cache on CI to ensure deterministic build, + # use global cache in dev builds to avoid cost of downloading packages. + if ($useGlobalNuGetCache) { + $env:NUGET_PACKAGES = Join-Path $env:UserProfile ".nuget\packages" + } else { + $env:NUGET_PACKAGES = Join-Path $RepoRoot ".packages" + } + } + + return $env:NUGET_PACKAGES +} + +function InitializeToolset() { + if (Test-Path variable:global:_ToolsetBuildProj) { + return $global:_ToolsetBuildProj + } + + $nugetCache = GetNuGetPackageCachePath + $toolsetVersion = $GlobalJson.'msbuild-sdks'.'Microsoft.DotNet.Arcade.Sdk' $toolsetLocationFile = Join-Path $ToolsetDir "$toolsetVersion.txt" if (Test-Path $toolsetLocationFile) { $path = Get-Content $toolsetLocationFile -TotalCount 1 if (Test-Path $path) { - $script:ToolsetBuildProj = $path - return + return $global:_ToolsetBuildProj = $path } } @@ -293,35 +391,20 @@ function InitializeToolset([string] $buildDriver, [string]$buildArgs) { ExitWithExitCode 1 } - $ToolsetRestoreLog = Join-Path $LogDir "ToolsetRestore.binlog" + $buildTool = InitializeBuildTool + $proj = Join-Path $ToolsetDir "restore.proj" - + $bl = if ($binaryLog) { "/bl:" + (Join-Path $LogDir "ToolsetRestore.binlog") } else { "" } + '' | Set-Content $proj - MSBuild $proj /t:__WriteToolsetLocation /clp:None /bl:$ToolsetRestoreLog /p:__ToolsetLocationOutputFile=$toolsetLocationFile - - if ($lastExitCode -ne 0) { - Write-Host "Failed to restore toolset (exit code '$lastExitCode'). See log: $ToolsetRestoreLog" -ForegroundColor Red - ExitWithExitCode $lastExitCode - } - + MSBuild $proj $bl /t:__WriteToolsetLocation /noconsolelogger /p:__ToolsetLocationOutputFile=$toolsetLocationFile + $path = Get-Content $toolsetLocationFile -TotalCount 1 if (!(Test-Path $path)) { throw "Invalid toolset path: $path" } - - $script:ToolsetBuildProj = $path -} - -function InitializeCustomToolset { - if (-not $restore) { - return - } - - $script = Join-Path $EngRoot "restore-toolset.ps1" - - if (Test-Path $script) { - . $script - } + + return $global:_ToolsetBuildProj = $path } function ExitWithExitCode([int] $exitCode) { @@ -333,42 +416,86 @@ function ExitWithExitCode([int] $exitCode) { function Stop-Processes() { Write-Host "Killing running build processes..." - Get-Process -Name "msbuild" -ErrorAction SilentlyContinue | Stop-Process - Get-Process -Name "dotnet" -ErrorAction SilentlyContinue | Stop-Process - Get-Process -Name "vbcscompiler" -ErrorAction SilentlyContinue | Stop-Process + foreach ($processName in $processesToStopOnExit) { + Get-Process -Name $processName -ErrorAction SilentlyContinue | Stop-Process + } } -function MsBuild() { - $warnaserrorSwitch = if ($warnaserror) { "/warnaserror" } else { "" } - & $buildDriver $buildArgs $warnaserrorSwitch /m /nologo /clp:Summary /v:$verbosity /nr:$nodereuse $args +# +# Executes msbuild (or 'dotnet msbuild') with arguments passed to the function. +# The arguments are automatically quoted. +# Terminates the script if the build fails. +# +function MSBuild() { + if ($ci) { + if (!$binaryLog) { + throw "Binary log must be enabled in CI build." + } + + if ($nodeReuse) { + throw "Node reuse must be disabled in CI build." + } + } + + $buildTool = InitializeBuildTool + + $cmdArgs = "$($buildTool.Command) /m /nologo /clp:Summary /v:$verbosity /nr:$nodeReuse" + + if ($warnAsError) { + $cmdArgs += " /warnaserror /p:TreatWarningsAsErrors=true" + } + + foreach ($arg in $args) { + if ($arg -ne $null -and $arg.Trim() -ne "") { + $cmdArgs += " `"$arg`"" + } + } + + $exitCode = Exec-Process $buildTool.Path $cmdArgs + + if ($exitCode -ne 0) { + Write-Host "Build failed." -ForegroundColor Red + + $buildLog = GetMSBuildBinaryLogCommandLineArgument $args + if ($buildLog -ne $null) { + Write-Host "See log: $buildLog" -ForegroundColor DarkGray + } + + ExitWithExitCode $exitCode + } +} + +function GetMSBuildBinaryLogCommandLineArgument($arguments) { + foreach ($argument in $arguments) { + if ($argument -ne $null) { + $arg = $argument.Trim() + if ($arg.StartsWith("/bl:", "OrdinalIgnoreCase")) { + return $arg.Substring("/bl:".Length) + } + + if ($arg.StartsWith("/binaryLogger:", "OrdinalIgnoreCase")) { + return $arg.Substring("/binaryLogger:".Length) + } + } + } + + return $null } $RepoRoot = Resolve-Path (Join-Path $PSScriptRoot "..\..") $EngRoot = Resolve-Path (Join-Path $PSScriptRoot "..") $ArtifactsDir = Join-Path $RepoRoot "artifacts" -if ($env:ARCADE_ARTIFACTS_DIR -ne $null) -{ - $ArtifactsDir = [System.IO.Path]::GetFullPath($env:ARCADE_ARTIFACTS_DIR) + "\" - $env:ArtifactsDir = $ArtifactsDir -} $ToolsetDir = Join-Path $ArtifactsDir "toolset" $ToolsDir = Join-Path $RepoRoot ".tools" $LogDir = Join-Path (Join-Path $ArtifactsDir "log") $configuration $TempDir = Join-Path (Join-Path $ArtifactsDir "tmp") $configuration $GlobalJson = Get-Content -Raw -Path (Join-Path $RepoRoot "global.json") | ConvertFrom-Json -if ($env:NUGET_PACKAGES -eq $null) { - # Use local cache on CI to ensure deterministic build, - # use global cache in dev builds to avoid cost of downloading packages. - $env:NUGET_PACKAGES = if ($ci) { Join-Path $RepoRoot ".packages" } - else { Join-Path $env:UserProfile ".nuget\packages" } -} - Create-Directory $ToolsetDir +Create-Directory $TempDir Create-Directory $LogDir if ($ci) { - Create-Directory $TempDir $env:TEMP = $TempDir $env:TMP = $TempDir } diff --git a/eng/common/tools.sh b/eng/common/tools.sh index 68d41c0d6..65f689775 100644 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -3,35 +3,52 @@ # Stop script if unbound variable found (use ${var:-} if intentional) set -u -ci=${ci:-false} -configuration=${configuration:-'Debug'} -nodereuse=${nodereuse:-true} -prepare_machine=${prepare_machine:-false} -restore=${restore:-true} -verbosity=${verbosity:-'minimal'} -warnaserror=${warnaserror:-true} -useInstalledDotNetCli=${useInstalledDotNetCli:-true} +# Initialize variables if they aren't already defined. -repo_root="$scriptroot/../.." -eng_root="$scriptroot/.." -artifacts_dir="$repo_root/artifacts" -if [[ -n "${ARCADE_ARTIFACTS_DIR:-}" ]]; then - artifacts_dir="$ARCADE_ARTIFACTS_DIR" - export ArtifactsDir="$artifacts_dir" +# CI mode - set to true on CI server for PR validation build or official build. +ci=${ci:-false} + +# Build configuration. Common values include 'Debug' and 'Release', but the repository may use other names. +configuration=${configuration:-'Debug'} + +# Set to true to output binary log from msbuild. Note that emitting binary log slows down the build. +# Binary log must be enabled on CI. +binary_log=${binary_log:-$ci} + +# Turns on machine preparation/clean up code that changes the machine state (e.g. kills build processes). +prepare_machine=${prepare_machine:-false} + +# True to restore toolsets and dependencies. +restore=${restore:-true} + +# Adjusts msbuild verbosity level. +verbosity=${verbosity:-'minimal'} + +# Set to true to reuse msbuild nodes. Recommended to not reuse on CI. +if [[ "$ci" == true ]]; then + node_reuse=${node_reuse:-false} +else + node_reuse=${node_reuse:-true} fi -toolset_dir="$artifacts_dir/toolset" -log_dir="$artifacts_dir/log/$configuration" -temp_dir="$artifacts_dir/tmp/$configuration" +# Configures warning treatment in msbuild. +warn_as_error=${warn_as_error:-true} -global_json_file="$repo_root/global.json" -build_driver="" -toolset_build_proj="" +# True to attempt using .NET Core already that meets requirements specified in global.json +# installed on the machine instead of downloading one. +use_installed_dotnet_cli=${use_installed_dotnet_cli:-true} +# True to use global NuGet cache instead of restoring packages to repository-local directory. +if [[ "$ci" == true ]]; then + use_global_nuget_cache=${use_global_nuget_cache:-false} +else + use_global_nuget_cache=${use_global_nuget_cache:-true} +fi + +# Resolve any symlinks in the given path. function ResolvePath { local path=$1 - # resolve $path until the file is no longer a symlink while [[ -h $path ]]; do local dir="$( cd -P "$( dirname "$path" )" && pwd )" path="$(readlink "$path")" @@ -62,6 +79,10 @@ function ReadGlobalVersion { } function InitializeDotNetCli { + if [[ -n "${_InitializeDotNetCli:-}" ]]; then + return + fi + local install=$1 # Don't resolve runtime, shared framework, or SDK from other locations to ensure build determinism @@ -70,13 +91,22 @@ function InitializeDotNetCli { # Disable first run since we want to control all package sources export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 + # Disable telemetry on CI + if [[ $ci == true ]]; then + export DOTNET_CLI_TELEMETRY_OPTOUT=1 + fi + + # LTTNG is the logging infrastructure used by Core CLR. Need this variable set + # so it doesn't output warnings to the console. + export LTTNG_HOME="$HOME" + # Source Build uses DotNetCoreSdkDir variable if [[ -n "${DotNetCoreSdkDir:-}" ]]; then export DOTNET_INSTALL_DIR="$DotNetCoreSdkDir" fi # Find the first path on $PATH that contains the dotnet.exe - if [[ "$useInstalledDotNetCli" == true && -z "${DOTNET_INSTALL_DIR:-}" ]]; then + if [[ "$use_installed_dotnet_cli" == true && -z "${DOTNET_INSTALL_DIR:-}" ]]; then local dotnet_path=`command -v dotnet` if [[ -n "$dotnet_path" ]]; then ResolvePath "$dotnet_path" @@ -94,10 +124,6 @@ function InitializeDotNetCli { dotnet_root="$DOTNET_INSTALL_DIR" else dotnet_root="$repo_root/.dotnet" - if [[ -n "${ARCADE_DOTNET_DIR:-}" ]]; then - dotnet_root="$ARCADE_DOTNET_DIR" - fi - export DOTNET_INSTALL_DIR="$dotnet_root" if [[ ! -d "$DOTNET_INSTALL_DIR/sdk/$dotnet_sdk_version" ]]; then @@ -110,6 +136,10 @@ function InitializeDotNetCli { fi fi + # Add dotnet to PATH. This prevents any bare invocation of dotnet in custom + # build steps from using anything other than what we've downloaded. + export PATH="$dotnet_root:$PATH" + # return value _InitializeDotNetCli="$dotnet_root" } @@ -152,7 +182,37 @@ function GetDotNetInstallScript { _GetDotNetInstallScript="$install_script" } +function InitializeBuildTool { + if [[ -n "${_InitializeBuildTool:-}" ]]; then + return + fi + + InitializeDotNetCli $restore + + # return value + _InitializeBuildTool="$_InitializeDotNetCli/dotnet" +} + +function GetNuGetPackageCachePath { + if [[ -z ${NUGET_PACKAGES:-} ]]; then + if [[ "$use_global_nuget_cache" == true ]]; then + export NUGET_PACKAGES="$HOME/.nuget/packages" + else + export NUGET_PACKAGES="$repo_root/.packages" + fi + fi + + # return value + _GetNuGetPackageCachePath=$NUGET_PACKAGES +} + function InitializeToolset { + if [[ -n "${_InitializeToolset:-}" ]]; then + return + fi + + GetNuGetPackageCachePath + ReadGlobalVersion "Microsoft.DotNet.Arcade.Sdk" local toolset_version=$_ReadGlobalVersion @@ -161,7 +221,8 @@ function InitializeToolset { if [[ -a "$toolset_location_file" ]]; then local path=`cat "$toolset_location_file"` if [[ -a "$path" ]]; then - toolset_build_proj="$path" + # return value + _InitializeToolset="$path" return fi fi @@ -175,47 +236,17 @@ function InitializeToolset { local proj="$toolset_dir/restore.proj" echo '' > "$proj" + MSBuild "$proj" /t:__WriteToolsetLocation /noconsolelogger /bl:"$toolset_restore_log" /p:__ToolsetLocationOutputFile="$toolset_location_file" - MSBuild "$proj" /t:__WriteToolsetLocation /clp:None /bl:"$toolset_restore_log" /p:__ToolsetLocationOutputFile="$toolset_location_file" - local lastexitcode=$? - - if [[ $lastexitcode != 0 ]]; then - echo "Failed to restore toolset (exit code '$lastexitcode'). See log: $toolset_restore_log" >&2 - ExitWithExitCode $lastexitcode - fi - - toolset_build_proj=`cat "$toolset_location_file"` + local toolset_build_proj=`cat "$toolset_location_file"` if [[ ! -a "$toolset_build_proj" ]]; then echo "Invalid toolset path: $toolset_build_proj" >&2 ExitWithExitCode 3 fi -} -function InitializeCustomToolset { - local script="$eng_root/restore-toolset.sh" - - if [[ -a "$script" ]]; then - . "$script" - fi -} - -function ConfigureTools { - local script="$eng_root/configure-toolset.sh" - - if [[ -a "$script" ]]; then - . "$script" - fi -} - -function InitializeTools { - ConfigureTools - - InitializeDotNetCli $restore - build_driver="$_InitializeDotNetCli/dotnet" - - InitializeToolset - InitializeCustomToolset + # return value + _InitializeToolset="$toolset_build_proj" } function ExitWithExitCode { @@ -233,35 +264,57 @@ function StopProcesses { } function MSBuild { + if [[ "$ci" == true ]]; then + if [[ "$binary_log" != true ]]; then + echo "Binary log must be enabled in CI build." >&2 + ExitWithExitCode 1 + fi + + if [[ "$node_reuse" == true ]]; then + echo "Node reuse must be disabled in CI build." >&2 + ExitWithExitCode 1 + fi + fi + + InitializeBuildTool + local warnaserror_switch="" - if [[ $warnaserror == true ]]; then + if [[ $warn_as_error == true ]]; then warnaserror_switch="/warnaserror" fi - "$build_driver" msbuild /m /nologo /clp:Summary /v:$verbosity /nr:$nodereuse $warnaserror_switch "$@" + "$_InitializeBuildTool" msbuild /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch /p:TreatWarningsAsErrors=$warn_as_error "$@" + lastexitcode=$? - return $? + if [[ $lastexitcode != 0 ]]; then + echo "Build failed (exit code '$lastexitcode')." >&2 + ExitWithExitCode $lastexitcode + fi } +ResolvePath "${BASH_SOURCE[0]}" +_script_dir=`dirname "$_ResolvePath"` + +eng_root=`cd -P "$_script_dir/.." && pwd` +repo_root=`cd -P "$_script_dir/../.." && pwd` +artifacts_dir="$repo_root/artifacts" +toolset_dir="$artifacts_dir/toolset" +log_dir="$artifacts_dir/log/$configuration" +temp_dir="$artifacts_dir/tmp/$configuration" + +global_json_file="$repo_root/global.json" + # HOME may not be defined in some scenarios, but it is required by NuGet if [[ -z $HOME ]]; then export HOME="$repo_root/artifacts/.home/" mkdir -p "$HOME" fi -if [[ -z ${NUGET_PACKAGES:-} ]]; then - if [[ $ci == true ]]; then - export NUGET_PACKAGES="$repo_root/.packages" - else - export NUGET_PACKAGES="$HOME/.nuget/packages" - fi -fi - mkdir -p "$toolset_dir" +mkdir -p "$temp_dir" mkdir -p "$log_dir" if [[ $ci == true ]]; then - mkdir -p "$temp_dir" export TEMP="$temp_dir" export TMP="$temp_dir" fi diff --git a/global.json b/global.json index ba936cbca..9868ffd21 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "3.0.100-alpha1-009697" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.18577.9" + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.18617.7" } } \ No newline at end of file