mirrors/dotnet-installer
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add arcade infrastructure

Browse source
This commit is contained in:
Daniel Plaisted 2018-10-21 22:07:26 -07:00
parent 3555f34527
commit 039b4dcd26
25 changed files with 2151 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
Directory.Build.props Normal file
View file

@ -0,0 +1,27 @@
<?xml version="1.0" encoding="utf-8"?>
<Project>
<PropertyGroup>
<ImportNetSdkFromRepoToolset>false</ImportNetSdkFromRepoToolset>
</PropertyGroup>
<Import Project="Sdk.props" Sdk="Microsoft.DotNet.Arcade.Sdk" />
<PropertyGroup>
<TreatWarningsAsErrors>True</TreatWarningsAsErrors>
<DebugType>embedded</DebugType>
<DebugSymbols>true</DebugSymbols>
<!--
Tools and packages produced by this repository support infrastructure and are not shipping on NuGet or via any other official channel.
-->
<IsShipping>false</IsShipping>
</PropertyGroup>
<PropertyGroup>
<!--
'NetFxTfm' is the standard desktop Target Framework Moniker which this repo's packages are targeting
ie. Place 'NetFxTfm' in the 'TargetFramework' property of a csproj like <TargetFrameworks>$(NetFxTfm);netcoreapp2.0</TargetFrameworks>
-->
<NetFxTfm>net461</NetFxTfm>
</PropertyGroup>
</Project>

4
Directory.Build.targets Normal file
View file

@ -0,0 +1,4 @@
<?xml version="1.0" encoding="utf-8"?>
<Project>
<Import Project="Sdk.targets" Sdk="Microsoft.DotNet.Arcade.Sdk" />
</Project>

3
eng/common/CIBuild.cmd Normal file
View file

@ -0,0 +1,3 @@
@echo off
powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0Build.ps1""" -restore -build -test -sign -pack -publish -ci %*"
exit /b %ErrorLevel%

3
eng/common/PublishBuildAssets.cmd Normal file
View file

@ -0,0 +1,3 @@
@echo off
powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0Build.ps1""" -restore -publishBuildAssets %*"
exit /b %ErrorLevel%

107
eng/common/build.ps1 Normal file
View file

@ -0,0 +1,107 @@
[CmdletBinding(PositionalBinding=$false)]
Param(
[string] $configuration = "Debug",
[string] $projects = "",
[string] $verbosity = "minimal",
[string] $msbuildEngine = $null,
[bool] $warnaserror = $true,
[bool] $nodereuse = $true,
[switch] $restore,
[switch] $deployDeps,
[switch] $build,
[switch] $rebuild,
[switch] $deploy,
[switch] $test,
[switch] $integrationTest,
[switch] $performanceTest,
[switch] $sign,
[switch] $pack,
[switch] $publish,
[switch] $publishBuildAssets,
[switch] $ci,
[switch] $prepareMachine,
[switch] $help,
[Parameter(ValueFromRemainingArguments=$true)][String[]]$properties
)
. $PSScriptRoot\tools.ps1
function Print-Usage() {
Write-Host "Common settings:"
Write-Host " -configuration <value> Build configuration Debug, Release"
Write-Host " -verbosity <value> Msbuild verbosity (q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic])"
Write-Host " -help Print help and exit"
Write-Host ""
Write-Host "Actions:"
Write-Host " -restore Restore dependencies"
Write-Host " -build Build solution"
Write-Host " -rebuild Rebuild solution"
Write-Host " -deploy Deploy built VSIXes"
Write-Host " -deployDeps Deploy dependencies (e.g. VSIXes for integration tests)"
Write-Host " -test Run all unit tests in the solution"
Write-Host " -pack Package build outputs into NuGet packages and Willow components"
Write-Host " -integrationTest Run all integration tests in the solution"
Write-Host " -performanceTest Run all performance tests in the solution"
Write-Host " -sign Sign build outputs"
Write-Host " -publish Publish artifacts (e.g. symbols)"
Write-Host " -publishBuildAssets Push assets to BAR"
Write-Host ""
Write-Host "Advanced settings:"
Write-Host " -projects <value> Semi-colon delimited list of sln/proj's to build. Globbing is supported (*.sln)"
Write-Host " -ci Set when running on CI server"
Write-Host " -prepareMachine Prepare machine for CI run"
Write-Host " -msbuildEngine <value> Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)."
Write-Host ""
Write-Host "Command line arguments not listed above are passed thru to msbuild."
Write-Host "The above arguments can be shortened as much as to be unambiguous (e.g. -co for configuration, -t for test, etc.)."
}
if ($help -or (($properties -ne $null) -and ($properties.Contains("/help") -or $properties.Contains("/?")))) {
Print-Usage
exit 0
}
try {
if ($projects -eq "") {
$projects = Join-Path $RepoRoot "*.sln"
}
InitializeTools
$BuildLog = Join-Path $LogDir "Build.binlog"
MSBuild $ToolsetBuildProj `
/bl:$BuildLog `
/p:Configuration=$configuration `
/p:Projects=$projects `
/p:RepoRoot=$RepoRoot `
/p:Restore=$restore `
/p:DeployDeps=$deployDeps `
/p:Build=$build `
/p:Rebuild=$rebuild `
/p:Deploy=$deploy `
/p:Test=$test `
/p:Pack=$pack `
/p:IntegrationTest=$integrationTest `
/p:PerformanceTest=$performanceTest `
/p:Sign=$sign `
/p:Publish=$publish `
/p:PublishBuildAssets=$publishBuildAssets `
/p:ContinuousIntegrationBuild=$ci `
@properties
if ($lastExitCode -ne 0) {
Write-Host "Build Failed (exit code '$lastExitCode'). See log: $BuildLog" -ForegroundColor Red
ExitWithExitCode $lastExitCode
}
ExitWithExitCode $lastExitCode
}
catch {
Write-Host $_
Write-Host $_.Exception
Write-Host $_.ScriptStackTrace
ExitWithExitCode 1
}

171
eng/common/build.sh Normal file
View file

@ -0,0 +1,171 @@
#!/usr/bin/env bash
source="${BASH_SOURCE[0]}"
# resolve $source until the file is no longer a symlink
while [[ -h "$source" ]]; do
scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
source="$(readlink "$source")"
# if $source was a relative symlink, we need to resolve it relative to the path where the
# symlink file was located
[[ $source != /* ]] && source="$scriptroot/$source"
done
scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
help=false
restore=false
build=false
rebuild=false
test=false
pack=false
publish=false
integration_test=false
performance_test=false
sign=false
public=false
ci=false
warnaserror=true
nodereuse=true
projects=''
configuration='Debug'
prepare_machine=false
verbosity='minimal'
properties=''
while (($# > 0)); do
lowerI="$(echo $1 | awk '{print tolower($0)}')"
case $lowerI in
--build)
build=true
shift 1
;;
--ci)
ci=true
shift 1
;;
--configuration)
configuration=$2
shift 2
;;
--help)
echo "Common settings:"
echo " --configuration <value> Build configuration Debug, Release"
echo " --verbosity <value> Msbuild verbosity (q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic])"
echo " --help Print help and exit"
echo ""
echo "Actions:"
echo " --restore Restore dependencies"
echo " --build Build solution"
echo " --rebuild Rebuild solution"
echo " --test Run all unit tests in the solution"
echo " --sign Sign build outputs"
echo " --publish Publish artifacts (e.g. symbols)"
echo " --pack Package build outputs into NuGet packages and Willow components"
echo ""
echo "Advanced settings:"
echo " --solution <value> Path to solution to build"
echo " --ci Set when running on CI server"
echo " --prepareMachine Prepare machine for CI run"
echo ""
echo "Command line arguments not listed above are passed through to MSBuild."
exit 0
;;
--pack)
pack=true
shift 1
;;
--preparemachine)
prepare_machine=true
shift 1
;;
--rebuild)
rebuild=true
shift 1
;;
--restore)
restore=true
shift 1
;;
--sign)
sign=true
shift 1
;;
--solution)
solution=$2
shift 2
;;
--projects)
projects=$2
shift 2
;;
--test)
test=true
shift 1
;;
--integrationtest)
integration_test=true
shift 1
;;
--performancetest)
performance_test=true
shift 1
;;
--publish)
publish=true
shift 1
;;
--verbosity)
verbosity=$2
shift 2
;;
--warnaserror)
warnaserror=$2
shift 2
;;
--nodereuse)
nodereuse=$2
shift 2
;;
*)
properties="$properties $1"
shift 1
;;
esac
done
. "$scriptroot/tools.sh"
if [[ -z $projects ]]; then
projects="$repo_root/*.sln"
fi
InitializeTools
build_log="$log_dir/Build.binlog"
MSBuild "$toolset_build_proj" \
/bl:"$build_log" \
/p:Configuration=$configuration \
/p:Projects="$projects" \
/p:RepoRoot="$repo_root" \
/p:Restore=$restore \
/p:Build=$build \
/p:Rebuild=$rebuild \
/p:Test=$test \
/p:Pack=$pack \
/p:IntegrationTest=$integration_test \
/p:PerformanceTest=$performance_test \
/p:Sign=$sign \
/p:Publish=$publish \
/p:ContinuousIntegrationBuild=$ci \
$properties
lastexitcode=$?
if [[ $lastexitcode != 0 ]]; then
echo "Build failed (exit code '$lastexitcode'). See log: $build_log"
fi
ExitWithExitCode $lastexitcode

16
eng/common/cibuild.sh Normal file
View file

@ -0,0 +1,16 @@
#!/usr/bin/env bash
source="${BASH_SOURCE[0]}"
# resolve $SOURCE until the file is no longer a symlink
while [[ -h $source ]]; do
scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
source="$(readlink "$source")"
# if $source was a relative symlink, we need to resolve it relative to the path where
# the symlink file was located
[[ $source != /* ]] && source="$scriptroot/$source"
done
scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
. "$scriptroot/build.sh" --restore --build --test --pack --publish --ci $@

21
eng/common/darc-init.ps1 Normal file
View file

@ -0,0 +1,21 @@
$verbosity = "m"
. $PSScriptRoot\tools.ps1
function InstallDarcCli {
$darcCliPackageName = "microsoft.dotnet.darc"
$dotnet = "$env:DOTNET_INSTALL_DIR\dotnet.exe"
$toolList = Invoke-Expression "$dotnet tool list -g"
if ($toolList -like "*$darcCliPackageName*") {
Invoke-Expression "$dotnet tool uninstall $darcCliPackageName -g"
}
$toolsetVersion = $GlobalJson.'msbuild-sdks'.'Microsoft.DotNet.Arcade.Sdk'
Write-Host "Installing Darc CLI version $toolsetVersion..."
Write-Host "You may need to restart your command window if this is the first dotnet tool you have installed."
Invoke-Expression "$dotnet tool install $darcCliPackageName --version $toolsetVersion -v $verbosity -g"
}
InitializeTools
InstallDarcCli

35
eng/common/darc-init.sh Normal file
View file

@ -0,0 +1,35 @@
#!/usr/bin/env bash
source="${BASH_SOURCE[0]}"
# resolve $source until the file is no longer a symlink
while [[ -h "$source" ]]; do
scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
source="$(readlink "$source")"
# if $source was a relative symlink, we need to resolve it relative to the path where the
# symlink file was located
[[ $source != /* ]] && source="$scriptroot/$source"
done
scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
verbosity=m
. "$scriptroot/tools.sh"
function InstallDarcCli {
local darc_cli_package_name="microsoft.dotnet.darc"
local uninstall_command=`$DOTNET_INSTALL_DIR/dotnet tool uninstall $darc_cli_package_name -g`
local tool_list=$($DOTNET_INSTALL_DIR/dotnet tool list -g)
if [[ $tool_list = *$darc_cli_package_name* ]]; then
echo $($DOTNET_INSTALL_DIR/dotnet tool uninstall $darc_cli_package_name -g)
fi
ReadGlobalVersion "Microsoft.DotNet.Arcade.Sdk"
local toolset_version=$_ReadGlobalVersion
echo "Installing Darc CLI version $toolset_version..."
echo "You may need to restart your command shell if this is the first dotnet tool you have installed."
echo $($DOTNET_INSTALL_DIR/dotnet tool install $darc_cli_package_name --version $toolset_version -v $verbosity -g)
}
InitializeTools
InstallDarcCli

9
eng/common/helixpublish.proj Normal file
View file

@ -0,0 +1,9 @@
<Project Sdk="Microsoft.DotNet.Helix.Sdk">
<ItemGroup>
<HelixWorkItem Include="WorkItem">
<PayloadDirectory>$(WorkItemDirectory)</PayloadDirectory>
<Command>$(WorkItemCommand)</Command>
</HelixWorkItem>
</ItemGroup>
</Project>

3
eng/common/init-tools-native.cmd Normal file
View file

@ -0,0 +1,3 @@
@echo off
powershell -NoProfile -NoLogo -ExecutionPolicy ByPass -command "& """%~dp0init-tools-native.ps1""" %*"
exit /b %ErrorLevel%

128
eng/common/init-tools-native.ps1 Normal file
View file

@ -0,0 +1,128 @@
<#
.SYNOPSIS
Entry point script for installing native tools
.DESCRIPTION
Reads $RepoRoot\global.json file to determine native assets to install
and executes installers for those tools
.PARAMETER BaseUri
Base file directory or Url from which to acquire tool archives
.PARAMETER InstallDirectory
Directory to install native toolset. This is a command-line override for the default
Install directory precedence order:
- InstallDirectory command-line override
- NETCOREENG_INSTALL_DIRECTORY environment variable
- (default) %USERPROFILE%/.netcoreeng/native
.PARAMETER Clean
Switch specifying to not install anything, but cleanup native asset folders
.PARAMETER Force
Clean and then install tools
.PARAMETER DownloadRetries
Total number of retry attempts
.PARAMETER RetryWaitTimeInSeconds
Wait time between retry attempts in seconds
.PARAMETER GlobalJsonFile
File path to global.json file
.NOTES
#>
[CmdletBinding(PositionalBinding=$false)]
Param (
[string] $BaseUri = "https://netcorenativeassets.blob.core.windows.net/resource-packages/external",
[string] $InstallDirectory,
[switch] $Clean = $False,
[switch] $Force = $False,
[int] $DownloadRetries = 5,
[int] $RetryWaitTimeInSeconds = 30,
[string] $GlobalJsonFile = "$PSScriptRoot\..\..\global.json"
)
Set-StrictMode -version 2.0
$ErrorActionPreference="Stop"
Import-Module -Name (Join-Path $PSScriptRoot "native\CommonLibrary.psm1")
try {
# Define verbose switch if undefined
$Verbose = $VerbosePreference -Eq "Continue"
$EngCommonBaseDir = Join-Path $PSScriptRoot "native\"
$NativeBaseDir = $InstallDirectory
if (!$NativeBaseDir) {
$NativeBaseDir = CommonLibrary\Get-NativeInstallDirectory
}
$Env:CommonLibrary_NativeInstallDir = $NativeBaseDir
$InstallBin = Join-Path $NativeBaseDir "bin"
$InstallerPath = Join-Path $EngCommonBaseDir "install-tool.ps1"
# Process tools list
Write-Host "Processing $GlobalJsonFile"
If (-Not (Test-Path $GlobalJsonFile)) {
Write-Host "Unable to find '$GlobalJsonFile'"
exit 0
}
$NativeTools = Get-Content($GlobalJsonFile) -Raw |
ConvertFrom-Json |
Select-Object -Expand "native-tools" -ErrorAction SilentlyContinue
if ($NativeTools) {
$NativeTools.PSObject.Properties | ForEach-Object {
$ToolName = $_.Name
$ToolVersion = $_.Value
$LocalInstallerCommand = $InstallerPath
$LocalInstallerCommand += " -ToolName $ToolName"
$LocalInstallerCommand += " -InstallPath $InstallBin"
$LocalInstallerCommand += " -BaseUri $BaseUri"
$LocalInstallerCommand += " -CommonLibraryDirectory $EngCommonBaseDir"
$LocalInstallerCommand += " -Version $ToolVersion"
if ($Verbose) {
$LocalInstallerCommand += " -Verbose"
}
if (Get-Variable 'Force' -ErrorAction 'SilentlyContinue') {
if($Force) {
$LocalInstallerCommand += " -Force"
}
}
if ($Clean) {
$LocalInstallerCommand += " -Clean"
}
Write-Verbose "Installing $ToolName version $ToolVersion"
Write-Verbose "Executing '$LocalInstallerCommand'"
Invoke-Expression "$LocalInstallerCommand"
if ($LASTEXITCODE -Ne "0") {
Write-Error "Execution failed"
exit 1
}
}
}
else {
Write-Host "No native tools defined in global.json"
exit 0
}
if ($Clean) {
exit 0
}
if (Test-Path $InstallBin) {
Write-Host "Native tools are available from" (Convert-Path -Path $InstallBin)
Write-Host "##vso[task.prependpath]$(Convert-Path -Path $InstallBin)"
}
else {
Write-Error "Native tools install directory does not exist, installation failed"
exit 1
}
exit 0
}
catch {
Write-Host $_
Write-Host $_.Exception
exit 1
}

145
eng/common/init-tools-native.sh Normal file
View file

@ -0,0 +1,145 @@
#!/usr/bin/env bash
source="${BASH_SOURCE[0]}"
scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
base_uri='https://netcorenativeassets.blob.core.windows.net/resource-packages/external'
install_directory=''
clean=false
force=false
download_retries=5
retry_wait_time_seconds=30
global_json_file="${scriptroot}/../../global.json"
declare -A native_assets
. $scriptroot/native/common-library.sh
while (($# > 0)); do
lowerI="$(echo $1 | awk '{print tolower($0)}')"
case $lowerI in
--baseuri)
base_uri=$2
shift 2
;;
--installdirectory)
install_directory=$2
shift 2
;;
--clean)
clean=true
shift 1
;;
--force)
force=true
shift 1
;;
--downloadretries)
download_retries=$2
shift 2
;;
--retrywaittimeseconds)
retry_wait_time_seconds=$2
shift 2
;;
--help)
echo "Common settings:"
echo " --installdirectory Directory to install native toolset."
echo " This is a command-line override for the default"
echo " Install directory precedence order:"
echo " - InstallDirectory command-line override"
echo " - NETCOREENG_INSTALL_DIRECTORY environment variable"
echo " - (default) %USERPROFILE%/.netcoreeng/native"
echo ""
echo " --clean Switch specifying not to install anything, but cleanup native asset folders"
echo " --force Clean and then install tools"
echo " --help Print help and exit"
echo ""
echo "Advanced settings:"
echo " --baseuri <value> Base URI for where to download native tools from"
echo " --downloadretries <value> Number of times a download should be attempted"
echo " --retrywaittimeseconds <value> Wait time between download attempts"
echo ""
exit 0
;;
esac
done
function ReadGlobalJsonNativeTools {
# Get the native-tools section from the global.json.
local native_tools_section=$(cat $global_json_file | awk '/"native-tools"/,/}/')
# Only extract the contents of the object.
local native_tools_list=$(echo $native_tools_section | awk -F"[{}]" '{print $2}')
native_tools_list=${native_tools_list//[\" ]/}
native_tools_list=${native_tools_list//,/$'\n'}
local old_IFS=$IFS
while read -r line; do
# Lines are of the form: 'tool:version'
IFS=:
while read -r key value; do
native_assets[$key]=$value
done <<< "$line"
done <<< "$native_tools_list"
IFS=$old_IFS
return 0;
}
native_base_dir=$install_directory
if [[ -z $install_directory ]]; then
native_base_dir=$(GetNativeInstallDirectory)
fi
install_bin="${native_base_dir}/bin"
ReadGlobalJsonNativeTools
if [[ ${#native_assets[@]} -eq 0 ]]; then
echo "No native tools defined in global.json"
exit 0;
else
native_installer_dir="$scriptroot/native"
for tool in "${!native_assets[@]}"
do
tool_version=${native_assets[$tool]}
installer_name="install-$tool.sh"
installer_command="$native_installer_dir/$installer_name"
installer_command+=" --baseuri $base_uri"
installer_command+=" --installpath $install_bin"
installer_command+=" --version $tool_version"
if [[ $force = true ]]; then
installer_command+=" --force"
fi
if [[ $clean = true ]]; then
installer_command+=" --clean"
fi
echo "Installing $tool version $tool_version"
echo "Executing '$installer_command'"
$installer_command
if [[ $? != 0 ]]; then
echo "Execution Failed" >&2
exit 1
fi
done
fi
if [[ ! -z $clean ]]; then
exit 0
fi
if [[ -d $install_bin ]]; then
echo "Native tools are available from $install_bin"
if [[ !-z BUILD_BUILDNUMBER ]]; then
echo "##vso[task.prependpath]$install_bin"
fi
else
echo "Native tools install directory does not exist, installation failed" >&2
exit 1
fi
exit 0

23
eng/common/msbuild.ps1 Normal file
View file

@ -0,0 +1,23 @@
[CmdletBinding(PositionalBinding=$false)]
Param(
[string] $verbosity = "minimal",
[bool] $warnaserror = $true,
[bool] $nodereuse = $true,
[switch] $ci,
[switch] $prepareMachine,
[Parameter(ValueFromRemainingArguments=$true)][String[]]$extraArgs
)
. $PSScriptRoot\tools.ps1
try {
InitializeTools
MSBuild @extraArgs
ExitWithExitCode $lastExitCode
}
catch {
Write-Host $_
Write-Host $_.Exception
Write-Host $_.ScriptStackTrace
ExitWithExitCode 1
}

55
eng/common/msbuild.sh Normal file
View file

@ -0,0 +1,55 @@
#!/usr/bin/env bash
source="${BASH_SOURCE[0]}"
# resolve $source until the file is no longer a symlink
while [[ -h "$source" ]]; do
scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
source="$(readlink "$source")"
# if $source was a relative symlink, we need to resolve it relative to the path where the
# symlink file was located
[[ $source != /* ]] && source="$scriptroot/$source"
done
scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
verbosity='minimal'
warnaserror=true
nodereuse=true
prepare_machine=false
extraargs=''
while (($# > 0)); do
lowerI="$(echo $1 | awk '{print tolower($0)}')"
case $lowerI in
--verbosity)
verbosity=$2
shift 2
;;
--warnaserror)
warnaserror=$2
shift 2
;;
--nodereuse)
nodereuse=$2
shift 2
;;
--ci)
ci=true
shift 1
;;
--preparemachine)
prepare_machine=true
shift 1
;;
*)
extraargs="$extraargs $1"
shift 1
;;
esac
done
. "$scriptroot/tools.sh"
InitializeTools
MSBuild $extraargs
ExitWithExitCode $?

358
eng/common/native/CommonLibrary.psm1 Normal file
View file

@ -0,0 +1,358 @@
<#
.SYNOPSIS
Helper module to install an archive to a directory
.DESCRIPTION
Helper module to download and extract an archive to a specified directory
.PARAMETER Uri
Uri of artifact to download
.PARAMETER InstallDirectory
Directory to extract artifact contents to
.PARAMETER Force
Force download / extraction if file or contents already exist. Default = False
.PARAMETER DownloadRetries
Total number of retry attempts. Default = 5
.PARAMETER RetryWaitTimeInSeconds
Wait time between retry attempts in seconds. Default = 30
.NOTES
Returns False if download or extraction fail, True otherwise
#>
function DownloadAndExtract {
[CmdletBinding(PositionalBinding=$false)]
Param (
[Parameter(Mandatory=$True)]
[string] $Uri,
[Parameter(Mandatory=$True)]
[string] $InstallDirectory,
[switch] $Force = $False,
[int] $DownloadRetries = 5,
[int] $RetryWaitTimeInSeconds = 30
)
# Define verbose switch if undefined
$Verbose = $VerbosePreference -Eq "Continue"
$TempToolPath = CommonLibrary\Get-TempPathFilename -Path $Uri
# Download native tool
$DownloadStatus = CommonLibrary\Get-File -Uri $Uri `
-Path $TempToolPath `
-DownloadRetries $DownloadRetries `
-RetryWaitTimeInSeconds $RetryWaitTimeInSeconds `
-Force:$Force `
-Verbose:$Verbose
if ($DownloadStatus -Eq $False) {
Write-Error "Download failed"
return $False
}
# Extract native tool
$UnzipStatus = CommonLibrary\Expand-Zip -ZipPath $TempToolPath `
-OutputDirectory $InstallDirectory `
-Force:$Force `
-Verbose:$Verbose
if ($UnzipStatus -Eq $False) {
Write-Error "Unzip failed"
return $False
}
return $True
}
<#
.SYNOPSIS
Download a file, retry on failure
.DESCRIPTION
Download specified file and retry if attempt fails
.PARAMETER Uri
Uri of file to download. If Uri is a local path, the file will be copied instead of downloaded
.PARAMETER Path
Path to download or copy uri file to
.PARAMETER Force
Overwrite existing file if present. Default = False
.PARAMETER DownloadRetries
Total number of retry attempts. Default = 5
.PARAMETER RetryWaitTimeInSeconds
Wait time between retry attempts in seconds Default = 30
#>
function Get-File {
[CmdletBinding(PositionalBinding=$false)]
Param (
[Parameter(Mandatory=$True)]
[string] $Uri,
[Parameter(Mandatory=$True)]
[string] $Path,
[int] $DownloadRetries = 5,
[int] $RetryWaitTimeInSeconds = 30,
[switch] $Force = $False
)
$Attempt = 0
if ($Force) {
if (Test-Path $Path) {
Remove-Item $Path -Force
}
}
if (Test-Path $Path) {
Write-Host "File '$Path' already exists, skipping download"
return $True
}
$DownloadDirectory = Split-Path -ErrorAction Ignore -Path "$Path" -Parent
if (-Not (Test-Path $DownloadDirectory)) {
New-Item -path $DownloadDirectory -force -itemType "Directory" | Out-Null
}
if (Test-Path -IsValid -Path $Uri) {
Write-Verbose "'$Uri' is a file path, copying file to '$Path'"
Copy-Item -Path $Uri -Destination $Path
return $?
}
else {
Write-Verbose "Downloading $Uri"
while($Attempt -Lt $DownloadRetries)
{
try {
Invoke-WebRequest -UseBasicParsing -Uri $Uri -OutFile $Path
Write-Verbose "Downloaded to '$Path'"
return $True
}
catch {
$Attempt++
if ($Attempt -Lt $DownloadRetries) {
$AttemptsLeft = $DownloadRetries - $Attempt
Write-Warning "Download failed, $AttemptsLeft attempts remaining, will retry in $RetryWaitTimeInSeconds seconds"
Start-Sleep -Seconds $RetryWaitTimeInSeconds
}
else {
Write-Error $_
Write-Error $_.Exception
}
}
}
}
return $False
}
<#
.SYNOPSIS
Generate a shim for a native tool
.DESCRIPTION
Creates a wrapper script (shim) that passes arguments forward to native tool assembly
.PARAMETER ShimName
The name of the shim
.PARAMETER ShimDirectory
The directory where shims are stored
.PARAMETER ToolFilePath
Path to file that shim forwards to
.PARAMETER Force
Replace shim if already present. Default = False
.NOTES
Returns $True if generating shim succeeds, $False otherwise
#>
function New-ScriptShim {
[CmdletBinding(PositionalBinding=$false)]
Param (
[Parameter(Mandatory=$True)]
[string] $ShimName,
[Parameter(Mandatory=$True)]
[string] $ShimDirectory,
[Parameter(Mandatory=$True)]
[string] $ToolFilePath,
[Parameter(Mandatory=$True)]
[string] $BaseUri,
[switch] $Force
)
try {
Write-Verbose "Generating '$ShimName' shim"
if (-Not (Test-Path $ToolFilePath)){
Write-Error "Specified tool file path '$ToolFilePath' does not exist"
return $False
}
# WinShimmer is a small .NET Framework program that creates .exe shims to bootstrapped programs
# Many of the checks for installed programs expect a .exe extension for Windows tools, rather
# than a .bat or .cmd file.
# Source: https://github.com/dotnet/arcade/tree/master/src/WinShimmer
if (-Not (Test-Path "$ShimDirectory\WinShimmer\winshimmer.exe")) {
$InstallStatus = DownloadAndExtract -Uri "$BaseUri/windows/winshimmer/WinShimmer.zip" `
-InstallDirectory $ShimDirectory\WinShimmer `
-Force:$Force `
-DownloadRetries 2 `
-RetryWaitTimeInSeconds 5 `
-Verbose:$Verbose
}
if ((Test-Path (Join-Path $ShimDirectory "$ShimName.exe"))) {
Write-Host "$ShimName.exe already exists; replacing..."
Remove-Item (Join-Path $ShimDirectory "$ShimName.exe")
}
Invoke-Expression "$ShimDirectory\WinShimmer\winshimmer.exe $ShimName $ToolFilePath $ShimDirectory"
return $True
}
catch {
Write-Host $_
Write-Host $_.Exception
return $False
}
}
<#
.SYNOPSIS
Returns the machine architecture of the host machine
.NOTES
Returns 'x64' on 64 bit machines
Returns 'x86' on 32 bit machines
#>
function Get-MachineArchitecture {
$ProcessorArchitecture = $Env:PROCESSOR_ARCHITECTURE
$ProcessorArchitectureW6432 = $Env:PROCESSOR_ARCHITEW6432
if($ProcessorArchitecture -Eq "X86")
{
if(($ProcessorArchitectureW6432 -Eq "") -Or
($ProcessorArchitectureW6432 -Eq "X86")) {
return "x86"
}
$ProcessorArchitecture = $ProcessorArchitectureW6432
}
if (($ProcessorArchitecture -Eq "AMD64") -Or
($ProcessorArchitecture -Eq "IA64") -Or
($ProcessorArchitecture -Eq "ARM64")) {
return "x64"
}
return "x86"
}
<#
.SYNOPSIS
Get the name of a temporary folder under the native install directory
#>
function Get-TempDirectory {
return Join-Path (Get-NativeInstallDirectory) "temp/"
}
function Get-TempPathFilename {
[CmdletBinding(PositionalBinding=$false)]
Param (
[Parameter(Mandatory=$True)]
[string] $Path
)
$TempDir = CommonLibrary\Get-TempDirectory
$TempFilename = Split-Path $Path -leaf
$TempPath = Join-Path $TempDir $TempFilename
return $TempPath
}
<#
.SYNOPSIS
Returns the base directory to use for native tool installation
.NOTES
Returns the value of the NETCOREENG_INSTALL_DIRECTORY if that environment variable
is set, or otherwise returns an install directory under the %USERPROFILE%
#>
function Get-NativeInstallDirectory {
$InstallDir = $Env:NETCOREENG_INSTALL_DIRECTORY
if (!$InstallDir) {
$InstallDir = Join-Path $Env:USERPROFILE ".netcoreeng/native/"
}
return $InstallDir
}
<#
.SYNOPSIS
Unzip an archive
.DESCRIPTION
Powershell module to unzip an archive to a specified directory
.PARAMETER ZipPath (Required)
Path to archive to unzip
.PARAMETER OutputDirectory (Required)
Output directory for archive contents
.PARAMETER Force
Overwrite output directory contents if they already exist
.NOTES
- Returns True and does not perform an extraction if output directory already exists but Overwrite is not True.
- Returns True if unzip operation is successful
- Returns False if Overwrite is True and it is unable to remove contents of OutputDirectory
- Returns False if unable to extract zip archive
#>
function Expand-Zip {
[CmdletBinding(PositionalBinding=$false)]
Param (
[Parameter(Mandatory=$True)]
[string] $ZipPath,
[Parameter(Mandatory=$True)]
[string] $OutputDirectory,
[switch] $Force
)
Write-Verbose "Extracting '$ZipPath' to '$OutputDirectory'"
try {
if ((Test-Path $OutputDirectory) -And (-Not $Force)) {
Write-Host "Directory '$OutputDirectory' already exists, skipping extract"
return $True
}
if (Test-Path $OutputDirectory) {
Write-Verbose "'Force' is 'True', but '$OutputDirectory' exists, removing directory"
Remove-Item $OutputDirectory -Force -Recurse
if ($? -Eq $False) {
Write-Error "Unable to remove '$OutputDirectory'"
return $False
}
}
if (-Not (Test-Path $OutputDirectory)) {
New-Item -path $OutputDirectory -Force -itemType "Directory" | Out-Null
}
Add-Type -assembly "system.io.compression.filesystem"
[io.compression.zipfile]::ExtractToDirectory("$ZipPath", "$OutputDirectory")
if ($? -Eq $False) {
Write-Error "Unable to extract '$ZipPath'"
return $False
}
}
catch {
Write-Host $_
Write-Host $_.Exception
return $False
}
return $True
}
export-modulemember -function DownloadAndExtract
export-modulemember -function Expand-Zip
export-modulemember -function Get-File
export-modulemember -function Get-MachineArchitecture
export-modulemember -function Get-NativeInstallDirectory
export-modulemember -function Get-TempDirectory
export-modulemember -function Get-TempPathFilename
export-modulemember -function New-ScriptShim

168
eng/common/native/common-library.sh Normal file
View file

@ -0,0 +1,168 @@
#!/usr/bin/env bash
function GetNativeInstallDirectory {
local install_dir
if [[ -z $NETCOREENG_INSTALL_DIRECTORY ]]; then
install_dir=$HOME/.netcoreeng/native/
else
install_dir=$NETCOREENG_INSTALL_DIRECTORY
fi
echo $install_dir
return 0
}
function GetTempDirectory {
echo $(GetNativeInstallDirectory)temp/
return 0
}
function ExpandZip {
local zip_path=$1
local output_directory=$2
local force=${3:-false}
echo "Extracting $zip_path to $output_directory"
if [[ -d $output_directory ]] && [[ $force = false ]]; then
echo "Directory '$output_directory' already exists, skipping extract"
return 0
fi
if [[ -d $output_directory ]]; then
echo "'Force flag enabled, but '$output_directory' exists. Removing directory"
rm -rf $output_directory
if [[ $? != 0 ]]; then
echo Unable to remove '$output_directory'>&2
return 1
fi
fi
echo "Creating directory: '$output_directory'"
mkdir -p $output_directory
echo "Extracting archive"
tar -xf $zip_path -C $output_directory
if [[ $? != 0 ]]; then
echo "Unable to extract '$zip_path'" >&2
return 1
fi
return 0
}
function GetCurrentOS {
local unameOut="$(uname -s)"
case $unameOut in
Linux*) echo "Linux";;
Darwin*) echo "MacOS";;
esac
return 0
}
function GetFile {
local uri=$1
local path=$2
local force=${3:-false}
local download_retries=${4:-5}
local retry_wait_time_seconds=${5:-30}
if [[ -f $path ]]; then
if [[ $force = false ]]; then
echo "File '$path' already exists. Skipping download"
return 0
else
rm -rf $path
fi
fi
if [[ -f $uri ]]; then
echo "'$uri' is a file path, copying file to '$path'"
cp $uri $path
return $?
fi
echo "Downloading $uri"
# Use curl if available, otherwise use wget
if command -v curl > /dev/null; then
curl "$uri" -sSL --retry $download_retries --retry-delay $retry_wait_time_seconds --create-dirs -o "$path" --fail
else
wget -q -O "$path" "$uri" --tries="$download_retries"
fi
return $?
}
function GetTempPathFileName {
local path=$1
local temp_dir=$(GetTempDirectory)
local temp_file_name=$(basename $path)
echo $temp_dir$temp_file_name
return 0
}
function DownloadAndExtract {
local uri=$1
local installDir=$2
local force=${3:-false}
local download_retries=${4:-5}
local retry_wait_time_seconds=${5:-30}
local temp_tool_path=$(GetTempPathFileName $uri)
echo "downloading to: $temp_tool_path"
# Download file
GetFile "$uri" "$temp_tool_path" $force $download_retries $retry_wait_time_seconds
if [[ $? != 0 ]]; then
echo "Failed to download '$uri' to '$temp_tool_path'." >&2
return 1
fi
# Extract File
echo "extracting from $temp_tool_path to $installDir"
ExpandZip "$temp_tool_path" "$installDir" $force $download_retries $retry_wait_time_seconds
if [[ $? != 0 ]]; then
echo "Failed to extract '$temp_tool_path' to '$installDir'." >&2
return 1
fi
return 0
}
function NewScriptShim {
local shimpath=$1
local tool_file_path=$2
local force=${3:-false}
echo "Generating '$shimpath' shim"
if [[ -f $shimpath ]]; then
if [[ $force = false ]]; then
echo "File '$shimpath' already exists." >&2
return 1
else
rm -rf $shimpath
fi
fi
if [[ ! -f $tool_file_path ]]; then
echo "Specified tool file path:'$tool_file_path' does not exist" >&2
return 1
fi
local shim_contents=$'#!/usr/bin/env bash\n'
shim_contents+="SHIMARGS="$'$1\n'
shim_contents+="$tool_file_path"$' $SHIMARGS\n'
# Write shim file
echo "$shim_contents" > $shimpath
chmod +x $shimpath
echo "Finished generating shim '$shimpath'"
return $?
}

117
eng/common/native/install-cmake.sh Normal file
View file

@ -0,0 +1,117 @@
#!/usr/bin/env bash
source="${BASH_SOURCE[0]}"
scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
. $scriptroot/common-library.sh
base_uri=
install_path=
version=
clean=false
force=false
download_retries=5
retry_wait_time_seconds=30
while (($# > 0)); do
lowerI="$(echo $1 | awk '{print tolower($0)}')"
case $lowerI in
--baseuri)
base_uri=$2
shift 2
;;
--installpath)
install_path=$2
shift 2
;;
--version)
version=$2
shift 2
;;
--clean)
clean=true
shift 1
;;
--force)
force=true
shift 1
;;
--downloadretries)
download_retries=$2
shift 2
;;
--retrywaittimeseconds)
retry_wait_time_seconds=$2
shift 2
;;
--help)
echo "Common settings:"
echo " --baseuri <value> Base file directory or Url wrom which to acquire tool archives"
echo " --installpath <value> Base directory to install native tool to"
echo " --clean Don't install the tool, just clean up the current install of the tool"
echo " --force Force install of tools even if they previously exist"
echo " --help Print help and exit"
echo ""
echo "Advanced settings:"
echo " --downloadretries Total number of retry attempts"
echo " --retrywaittimeseconds Wait time between retry attempts in seconds"
echo ""
exit 0
;;
esac
done
tool_name="cmake"
tool_os=$(GetCurrentOS)
tool_folder=$(echo $tool_os | awk '{print tolower($0)}')
tool_arch="x86_64"
tool_name_moniker="$tool_name-$version-$tool_os-$tool_arch"
tool_install_directory="$install_path/$tool_name/$version"
tool_file_path="$tool_install_directory/$tool_name_moniker/bin/$tool_name"
shim_path="$install_path/$tool_name.sh"
uri="${base_uri}/$tool_folder/cmake/$tool_name_moniker.tar.gz"
# Clean up tool and installers
if [[ $clean = true ]]; then
echo "Cleaning $tool_install_directory"
if [[ -d $tool_install_directory ]]; then
rm -rf $tool_install_directory
fi
echo "Cleaning $shim_path"
if [[ -f $shim_path ]]; then
rm -rf $shim_path
fi
tool_temp_path=$(GetTempPathFileName $uri)
echo "Cleaning $tool_temp_path"
if [[ -f $tool_temp_path ]]; then
rm -rf $tool_temp_path
fi
exit 0
fi
# Install tool
if [[ -f $tool_file_path ]] && [[ $force = false ]]; then
echo "$tool_name ($version) already exists, skipping install"
exit 0
fi
DownloadAndExtract $uri $tool_install_directory $force $download_retries $retry_wait_time_seconds
if [[ $? != 0 ]]; then
echo "Installation failed" >&2
exit 1
fi
# Generate Shim
# Always rewrite shims so that we are referencing the expected version
NewScriptShim $shim_path $tool_file_path true
if [[ $? != 0 ]]; then
echo "Shim generation failed" >&2
exit 1
fi
exit 0

130
eng/common/native/install-tool.ps1 Normal file
View file

@ -0,0 +1,130 @@
<#
.SYNOPSIS
Install native tool
.DESCRIPTION
Install cmake native tool from Azure blob storage
.PARAMETER InstallPath
Base directory to install native tool to
.PARAMETER BaseUri
Base file directory or Url from which to acquire tool archives
.PARAMETER CommonLibraryDirectory
Path to folder containing common library modules
.PARAMETER Force
Force install of tools even if they previously exist
.PARAMETER Clean
Don't install the tool, just clean up the current install of the tool
.PARAMETER DownloadRetries
Total number of retry attempts
.PARAMETER RetryWaitTimeInSeconds
Wait time between retry attempts in seconds
.NOTES
Returns 0 if install succeeds, 1 otherwise
#>
[CmdletBinding(PositionalBinding=$false)]
Param (
[Parameter(Mandatory=$True)]
[string] $ToolName,
[Parameter(Mandatory=$True)]
[string] $InstallPath,
[Parameter(Mandatory=$True)]
[string] $BaseUri,
[Parameter(Mandatory=$True)]
[string] $Version,
[string] $CommonLibraryDirectory = $PSScriptRoot,
[switch] $Force = $False,
[switch] $Clean = $False,
[int] $DownloadRetries = 5,
[int] $RetryWaitTimeInSeconds = 30
)
# Import common library modules
Import-Module -Name (Join-Path $CommonLibraryDirectory "CommonLibrary.psm1")
try {
# Define verbose switch if undefined
$Verbose = $VerbosePreference -Eq "Continue"
$Arch = CommonLibrary\Get-MachineArchitecture
$ToolOs = "win64"
if($Arch -Eq "x32") {
$ToolOs = "win32"
}
$ToolNameMoniker = "$ToolName-$Version-$ToolOs-$Arch"
$ToolInstallDirectory = Join-Path $InstallPath "$ToolName\$Version\"
$Uri = "$BaseUri/windows/$ToolName/$ToolNameMoniker.zip"
$ShimPath = Join-Path $InstallPath "$ToolName.exe"
if ($Clean) {
Write-Host "Cleaning $ToolInstallDirectory"
if (Test-Path $ToolInstallDirectory) {
Remove-Item $ToolInstallDirectory -Force -Recurse
}
Write-Host "Cleaning $ShimPath"
if (Test-Path $ShimPath) {
Remove-Item $ShimPath -Force
}
$ToolTempPath = CommonLibrary\Get-TempPathFilename -Path $Uri
Write-Host "Cleaning $ToolTempPath"
if (Test-Path $ToolTempPath) {
Remove-Item $ToolTempPath -Force
}
exit 0
}
# Install tool
if ((Test-Path $ToolInstallDirectory) -And (-Not $Force)) {
Write-Verbose "$ToolName ($Version) already exists, skipping install"
}
else {
$InstallStatus = CommonLibrary\DownloadAndExtract -Uri $Uri `
-InstallDirectory $ToolInstallDirectory `
-Force:$Force `
-DownloadRetries $DownloadRetries `
-RetryWaitTimeInSeconds $RetryWaitTimeInSeconds `
-Verbose:$Verbose
if ($InstallStatus -Eq $False) {
Write-Error "Installation failed"
exit 1
}
}
$ToolFilePath = Get-ChildItem $ToolInstallDirectory -Recurse -Filter "$ToolName.exe" | % { $_.FullName }
if (@($ToolFilePath).Length -Gt 1) {
Write-Error "There are multiple copies of $ToolName in $($ToolInstallDirectory): `n$(@($ToolFilePath | out-string))"
exit 1
} elseif (@($ToolFilePath).Length -Lt 1) {
Write-Error "$ToolName was not found in $ToolFilePath."
exit 1
}
# Generate shim
# Always rewrite shims so that we are referencing the expected version
$GenerateShimStatus = CommonLibrary\New-ScriptShim -ShimName $ToolName `
-ShimDirectory $InstallPath `
-ToolFilePath "$ToolFilePath" `
-BaseUri $BaseUri `
-Force:$Force `
-Verbose:$Verbose
if ($GenerateShimStatus -Eq $False) {
Write-Error "Generate shim failed"
return 1
}
exit 0
}
catch {
Write-Host $_
Write-Host $_.Exception
exit 1
}

33
eng/common/templates/phases/base.yml
View file

@ -41,7 +41,8 @@ phases:
queue: ${{ parameters.queue }}
${{ if ne(parameters.variables, '') }}:
variables: ${{ parameters.variables }}
variables:
${{ insert }}: ${{ parameters.variables }}
steps:
- checkout: self
@ -90,18 +91,18 @@ phases:
helixType: $(_HelixType)
- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- task: CopyFiles@2
displayName: Gather Asset Manifests
inputs:
SourceFolder: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/AssetManifest'
TargetFolder: '$(Build.StagingDirectory)/AssetManifests'
continueOnError: false
condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true'))
- task: PublishBuildArtifacts@1
displayName: Push Asset Manifests
inputs:
PathtoPublish: '$(Build.StagingDirectory)/AssetManifests'
PublishLocation: Container
ArtifactName: AssetManifests
continueOnError: false
condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true'))
- task: CopyFiles@2
displayName: Gather Asset Manifests
inputs:
SourceFolder: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/AssetManifest'
TargetFolder: '$(Build.StagingDirectory)/AssetManifests'
continueOnError: false
condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true'))
- task: PublishBuildArtifacts@1
displayName: Push Asset Manifests
inputs:
PathtoPublish: '$(Build.StagingDirectory)/AssetManifests'
PublishLocation: Container
ArtifactName: AssetManifests
continueOnError: false
condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true'))

27
eng/common/templates/phases/publish-build-assets.yml Normal file
View file

@ -0,0 +1,27 @@
parameters:
dependsOn: ''
queue: {}
phases:
- phase: Asset_Registry_Publish
displayName: Publish to Build Asset Registry
dependsOn: ${{ parameters.dependsOn }}
queue: ${{ parameters.queue }}
steps:
- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- task: DownloadBuildArtifacts@0
displayName: Download artifact
inputs:
artifactName: AssetManifests
downloadPath: '$(Build.StagingDirectory)/Download'
condition: succeeded()
- task: AzureKeyVault@1
inputs:
azureSubscription: 'DotNet-Engineering-Services_KeyVault'
KeyVaultName: EngKeyVault
SecretsFilter: 'MaestroAccessToken'
condition: succeeded()
- script: eng\common\publishbuildassets.cmd
/p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests'
/p:BuildAssetRegistryToken=$(MaestroAccessToken)
/p:MaestroApiEndpoint=https://maestro-prod.westus2.cloudapp.azure.com
displayName: Publish Build Assets

35
eng/common/templates/steps/helix-publish.yml Normal file
View file

@ -0,0 +1,35 @@
parameters:
HelixSource: 'pr/dotnet-github-anon-kaonashi-bot'
HelixType: ̓'tests/default'
HelixBuild: $(Build.BuildNumber)
HelixTargetQueues: ''
HelixAccessToken: ''
HelixPreCommands: ''
HelixPostCommands: ''
WorkItemDirectory: ''
WorkItemCommand: ''
IncludeDotNetSdk: false
EnableXUnitReporter: false
WaitForWorkItemCompletion: true
steps:
- task: DotNetCoreCLI@2
inputs:
command: custom
projects: eng/common/helixpublish.proj
custom: msbuild
arguments: '/t:test /p:Language=msbuild'
displayName: Send job to Helix
env:
HelixSource: ${{ parameters.HelixSource }}
HelixType: ${{ parameters.HelixType }}
HelixBuild: ${{ parameters.HelixBuild }}
HelixTargetQueues: ${{ parameters.HelixTargetQueues }}
HelixAccessToken: ${{ parameters.HelixAccessToken }}
HelixPreCommands: ${{ parameters.HelixPreCommands }}
HelixPostCommands: ${{ parameters.HelixPostCommands }}
IncludeDotNetSdk: ${{ parameters.IncludeDotNetSdk }}
EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }}
WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }}
WorkItemDirectory: ${{ parameters.WorkItemDirectory }}
WorkItemCommand: ${{ parameters.WorkItemCommand }}

283
eng/common/tools.ps1 Normal file
View file

@ -0,0 +1,283 @@
# Initialize variables if they aren't already defined
$ci = if (Test-Path variable:ci) { $ci } else { $false }
$configuration = if (Test-Path variable:configuration) { $configuration } else { "Debug" }
$nodereuse = if (Test-Path variable:nodereuse) { $nodereuse } else { !$ci }
$prepareMachine = if (Test-Path variable:prepareMachine) { $prepareMachine } else { $false }
$restore = if (Test-Path variable:restore) { $restore } else { $true }
$verbosity = if (Test-Path variable:verbosity) { $verbosity } else { "minimal" }
$warnaserror = if (Test-Path variable:warnaserror) { $warnaserror } else { $true }
$msbuildEngine = if (Test-Path variable:msbuildEngine) { $msbuildEngine } else { $null }
$useInstalledDotNetCli = if (Test-Path variable:useInstalledDotNetCli) { $useInstalledDotNetCli } else { $true }
set-strictmode -version 2.0
$ErrorActionPreference = "Stop"
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
function Create-Directory([string[]] $path) {
if (!(Test-Path $path)) {
New-Item -path $path -force -itemType "Directory" | Out-Null
}
}
function InitializeDotNetCli([bool]$install) {
# Don't resolve runtime, shared framework, or SDK from other locations to ensure build determinism
$env:DOTNET_MULTILEVEL_LOOKUP=0
# Disable first run since we do not need all ASP.NET packages restored.
$env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1
# Source Build uses DotNetCoreSdkDir variable
if ($env:DotNetCoreSdkDir -ne $null) {
$env:DOTNET_INSTALL_DIR = $env:DotNetCoreSdkDir
}
# Find the first path on %PATH% that contains the dotnet.exe
if ($useInstalledDotNetCli -and ($env:DOTNET_INSTALL_DIR -eq $null)) {
$env:DOTNET_INSTALL_DIR = ${env:PATH}.Split(';') | where { ($_ -ne "") -and (Test-Path (Join-Path $_ "dotnet.exe")) }
}
$dotnetSdkVersion = $GlobalJson.tools.dotnet
# Use dotnet installation specified in DOTNET_INSTALL_DIR if it contains the required SDK version,
# otherwise install the dotnet CLI and SDK to repo local .dotnet directory to avoid potential permission issues.
if (($env:DOTNET_INSTALL_DIR -ne $null) -and (Test-Path(Join-Path $env:DOTNET_INSTALL_DIR "sdk\$dotnetSdkVersion"))) {
$dotnetRoot = $env:DOTNET_INSTALL_DIR
} else {
$dotnetRoot = Join-Path $RepoRoot ".dotnet"
$env:DOTNET_INSTALL_DIR = $dotnetRoot
if (-not (Test-Path(Join-Path $env:DOTNET_INSTALL_DIR "sdk\$dotnetSdkVersion"))) {
if ($install) {
InstallDotNetSdk $dotnetRoot $dotnetSdkVersion
} else {
Write-Host "Unable to find dotnet with SDK version '$dotnetSdkVersion'" -ForegroundColor Red
ExitWithExitCode 1
}
}
}
return $dotnetRoot
}
function GetDotNetInstallScript([string] $dotnetRoot) {
$installScript = "$dotnetRoot\dotnet-install.ps1"
if (!(Test-Path $installScript)) {
Create-Directory $dotnetRoot
Invoke-WebRequest "https://dot.net/v1/dotnet-install.ps1" -OutFile $installScript
}
return $installScript
}
function InstallDotNetSdk([string] $dotnetRoot, [string] $version) {
$installScript = GetDotNetInstallScript $dotnetRoot
& $installScript -Version $version -InstallDir $dotnetRoot
if ($lastExitCode -ne 0) {
Write-Host "Failed to install dotnet cli (exit code '$lastExitCode')." -ForegroundColor Red
ExitWithExitCode $lastExitCode
}
}
function InitializeVisualStudioBuild {
$vsToolsPath = $env:VS150COMNTOOLS
if ($vsToolsPath -eq $null) {
$vsToolsPath = $env:VS160COMNTOOLS
}
if (($vsToolsPath -ne $null) -and (Test-Path $vsToolsPath)) {
$vsInstallDir = [System.IO.Path]::GetFullPath((Join-Path $vsToolsPath "..\.."))
} else {
$vsInfo = LocateVisualStudio
$vsInstallDir = $vsInfo.installationPath
$vsSdkInstallDir = Join-Path $vsInstallDir "VSSDK\"
$vsVersion = $vsInfo.installationVersion.Split('.')[0] + "0"
Set-Item "env:VS$($vsVersion)COMNTOOLS" (Join-Path $vsInstallDir "Common7\Tools\")
Set-Item "env:VSSDK$($vsVersion)Install" $vsSdkInstallDir
$env:VSSDKInstall = $vsSdkInstallDir
}
return $vsInstallDir
}
function LocateVisualStudio {
$vswhereVersion = $GlobalJson.tools.vswhere
if (!$vsWhereVersion) {
Write-Host "vswhere version must be specified in /global.json." -ForegroundColor Red
ExitWithExitCode 1
}
$toolsRoot = Join-Path $RepoRoot ".tools"
$vsWhereDir = Join-Path $toolsRoot "vswhere\$vswhereVersion"
$vsWhereExe = Join-Path $vsWhereDir "vswhere.exe"
if (!(Test-Path $vsWhereExe)) {
Create-Directory $vsWhereDir
Write-Host "Downloading vswhere"
Invoke-WebRequest "https://github.com/Microsoft/vswhere/releases/download/$vswhereVersion/vswhere.exe" -OutFile $vswhereExe
}
$vsInfo = & $vsWhereExe `
-latest `
-prerelease `
-format json `
-requires Microsoft.Component.MSBuild `
-requires Microsoft.VisualStudio.Component.VSSDK `
-requires Microsoft.VisualStudio.Component.Roslyn.Compiler | ConvertFrom-Json
if ($lastExitCode -ne 0) {
Write-Host "Failed to locate Visual Studio (exit code '$lastExitCode')." -ForegroundColor Red
ExitWithExitCode $lastExitCode
}
# use first matching instance
return $vsInfo[0]
}
function ConfigureTools {
# Include custom tools configuration
$script = Join-Path $EngRoot "configure-toolset.ps1"
if (Test-Path $script) {
. $script
}
}
function InitializeTools() {
ConfigureTools
$tools = $GlobalJson.tools
# Initialize dotnet cli if listed in 'tools'
$dotnetRoot = $null
if ((Get-Member -InputObject $tools -Name "dotnet") -ne $null) {
$dotnetRoot = InitializeDotNetCli -install:$restore
}
if (-not $msbuildEngine) {
# Presence of vswhere.version indicates the repo needs to build using VS msbuild.
if ((Get-Member -InputObject $tools -Name "vswhere") -ne $null) {
$msbuildEngine = "vs"
} elseif ($dotnetRoot -ne $null) {
$msbuildEngine = "dotnet"
} else {
Write-Host "-msbuildEngine must be specified, or /global.json must specify 'tools.dotnet' or 'tools.vswhere'." -ForegroundColor Red
ExitWithExitCode 1
}
}
if ($msbuildEngine -eq "dotnet") {
if (!$dotnetRoot) {
Write-Host "/global.json must specify 'tools.dotnet'." -ForegroundColor Red
ExitWithExitCode 1
}
$script:buildDriver = Join-Path $dotnetRoot "dotnet.exe"
$script:buildArgs = "msbuild"
} elseif ($msbuildEngine -eq "vs") {
$vsInstallDir = InitializeVisualStudioBuild
$script:buildDriver = Join-Path $vsInstallDir "MSBuild\15.0\Bin\msbuild.exe"
$script:buildArgs = ""
} else {
Write-Host "Unexpected value of -msbuildEngine: '$msbuildEngine'." -ForegroundColor Red
ExitWithExitCode 1
}
InitializeToolSet $script:buildDriver $script:buildArgs
InitializeCustomToolset
}
function InitializeToolset([string] $buildDriver, [string]$buildArgs) {
$toolsetVersion = $GlobalJson.'msbuild-sdks'.'Microsoft.DotNet.Arcade.Sdk'
$toolsetLocationFile = Join-Path $ToolsetDir "$toolsetVersion.txt"
if (Test-Path $toolsetLocationFile) {
$path = Get-Content $toolsetLocationFile -TotalCount 1
if (Test-Path $path) {
$script:ToolsetBuildProj = $path
return
}
}
if (-not $restore) {
Write-Host "Toolset version $toolsetVersion has not been restored."
ExitWithExitCode 1
}
$ToolsetRestoreLog = Join-Path $LogDir "ToolsetRestore.binlog"
$proj = Join-Path $ToolsetDir "restore.proj"
'<Project Sdk="Microsoft.DotNet.Arcade.Sdk"/>' | Set-Content $proj
MSBuild $proj /t:__WriteToolsetLocation /clp:None /bl:$ToolsetRestoreLog /p:__ToolsetLocationOutputFile=$toolsetLocationFile
if ($lastExitCode -ne 0) {
Write-Host "Failed to restore toolset (exit code '$lastExitCode'). See log: $ToolsetRestoreLog" -ForegroundColor Red
ExitWithExitCode $lastExitCode
}
$path = Get-Content $toolsetLocationFile -TotalCount 1
if (!(Test-Path $path)) {
throw "Invalid toolset path: $path"
}
$script:ToolsetBuildProj = $path
}
function InitializeCustomToolset {
if (-not $restore) {
return
}
$script = Join-Path $EngRoot "restore-toolset.ps1"
if (Test-Path $script) {
. $script
}
}
function ExitWithExitCode([int] $exitCode) {
if ($ci -and $prepareMachine) {
Stop-Processes
}
exit $exitCode
}
function Stop-Processes() {
Write-Host "Killing running build processes..."
Get-Process -Name "msbuild" -ErrorAction SilentlyContinue | Stop-Process
Get-Process -Name "dotnet" -ErrorAction SilentlyContinue | Stop-Process
Get-Process -Name "vbcscompiler" -ErrorAction SilentlyContinue | Stop-Process
}
function MsBuild() {
$warnaserrorSwitch = if ($warnaserror) { "/warnaserror" } else { "" }
& $buildDriver $buildArgs $warnaserrorSwitch /m /nologo /clp:Summary /v:$verbosity /nr:$nodereuse $args
}
$RepoRoot = Resolve-Path (Join-Path $PSScriptRoot "..\..")
$EngRoot = Resolve-Path (Join-Path $PSScriptRoot "..")
$ArtifactsDir = Join-Path $RepoRoot "artifacts"
$ToolsetDir = Join-Path $ArtifactsDir "toolset"
$LogDir = Join-Path (Join-Path $ArtifactsDir "log") $configuration
$TempDir = Join-Path (Join-Path $ArtifactsDir "tmp") $configuration
$GlobalJson = Get-Content -Raw -Path (Join-Path $RepoRoot "global.json") | ConvertFrom-Json
if ($env:NUGET_PACKAGES -eq $null) {
# Use local cache on CI to ensure deterministic build,
# use global cache in dev builds to avoid cost of downloading packages.
$env:NUGET_PACKAGES = if ($ci) { Join-Path $RepoRoot ".packages" }
else { Join-Path $env:UserProfile ".nuget\packages" }
}
Create-Directory $ToolsetDir
Create-Directory $LogDir
if ($ci) {
Create-Directory $TempDir
$env:TEMP = $TempDir
$env:TMP = $TempDir
}

258
eng/common/tools.sh Normal file
View file

@ -0,0 +1,258 @@
#!/usr/bin/env bash
# Stop script if unbound variable found (use ${var:-} if intentional)
set -u
ci=${ci:-false}
configuration=${configuration:-'Debug'}
nodereuse=${nodereuse:-true}
prepare_machine=${prepare_machine:-false}
restore=${restore:-true}
verbosity=${verbosity:-'minimal'}
warnaserror=${warnaserror:-true}
useInstalledDotNetCli=${useInstalledDotNetCli:-true}
repo_root="$scriptroot/../.."
eng_root="$scriptroot/.."
artifacts_dir="$repo_root/artifacts"
toolset_dir="$artifacts_dir/toolset"
log_dir="$artifacts_dir/log/$configuration"
temp_dir="$artifacts_dir/tmp/$configuration"
global_json_file="$repo_root/global.json"
build_driver=""
toolset_build_proj=""
function ResolvePath {
local path=$1
# resolve $path until the file is no longer a symlink
while [[ -h $path ]]; do
local dir="$( cd -P "$( dirname "$path" )" && pwd )"
path="$(readlink "$path")"
# if $path was a relative symlink, we need to resolve it relative to the path where the
# symlink file was located
[[ $path != /* ]] && path="$dir/$path"
done
# return value
_ResolvePath="$path"
}
# ReadVersionFromJson [json key]
function ReadGlobalVersion {
local key=$1
local line=`grep -m 1 "$key" "$global_json_file"`
local pattern="\"$key\" *: *\"(.*)\""
if [[ ! $line =~ $pattern ]]; then
echo "Error: Cannot find \"$key\" in $global_json_file" >&2
ExitWithExitCode 1
fi
# return value
_ReadGlobalVersion=${BASH_REMATCH[1]}
}
function InitializeDotNetCli {
local install=$1
# Don't resolve runtime, shared framework, or SDK from other locations to ensure build determinism
export DOTNET_MULTILEVEL_LOOKUP=0
# Disable first run since we want to control all package sources
export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1
# Source Build uses DotNetCoreSdkDir variable
if [[ -n "${DotNetCoreSdkDir:-}" ]]; then
export DOTNET_INSTALL_DIR="$DotNetCoreSdkDir"
fi
# Find the first path on $PATH that contains the dotnet.exe
if [[ "$useInstalledDotNetCli" == true && -z "${DOTNET_INSTALL_DIR:-}" ]]; then
local dotnet_path=`command -v dotnet`
if [[ -n "$dotnet_path" ]]; then
ResolvePath "$dotnet_path"
export DOTNET_INSTALL_DIR=`dirname "$_ResolvePath"`
fi
fi
ReadGlobalVersion "dotnet"
local dotnet_sdk_version=$_ReadGlobalVersion
local dotnet_root=""
# Use dotnet installation specified in DOTNET_INSTALL_DIR if it contains the required SDK version,
# otherwise install the dotnet CLI and SDK to repo local .dotnet directory to avoid potential permission issues.
if [[ -n "${DOTNET_INSTALL_DIR:-}" && -d "$DOTNET_INSTALL_DIR/sdk/$dotnet_sdk_version" ]]; then
dotnet_root="$DOTNET_INSTALL_DIR"
else
dotnet_root="$repo_root/.dotnet"
export DOTNET_INSTALL_DIR="$dotnet_root"
if [[ ! -d "$DOTNET_INSTALL_DIR/sdk/$dotnet_sdk_version" ]]; then
if [[ "$install" == true ]]; then
InstallDotNetSdk "$dotnet_root" "$dotnet_sdk_version"
else
echo "Unable to find dotnet with SDK version '$dotnet_sdk_version'" >&2
ExitWithExitCode 1
fi
fi
fi
# return value
_InitializeDotNetCli="$dotnet_root"
}
function InstallDotNetSdk {
local root=$1
local version=$2
GetDotNetInstallScript "$root"
local install_script=$_GetDotNetInstallScript
bash "$install_script" --version $version --install-dir "$root"
local lastexitcode=$?
if [[ $lastexitcode != 0 ]]; then
echo "Failed to install dotnet SDK (exit code '$lastexitcode')." >&2
ExitWithExitCode $lastexitcode
fi
}
function GetDotNetInstallScript {
local root=$1
local install_script="$root/dotnet-install.sh"
local install_script_url="https://dot.net/v1/dotnet-install.sh"
if [[ ! -a "$install_script" ]]; then
mkdir -p "$root"
echo "Downloading '$install_script_url'"
# Use curl if available, otherwise use wget
if command -v curl > /dev/null; then
curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script"
else
wget -q -O "$install_script" "$install_script_url"
fi
fi
# return value
_GetDotNetInstallScript="$install_script"
}
function InitializeToolset {
ReadGlobalVersion "Microsoft.DotNet.Arcade.Sdk"
local toolset_version=$_ReadGlobalVersion
local toolset_location_file="$toolset_dir/$toolset_version.txt"
if [[ -a "$toolset_location_file" ]]; then
local path=`cat "$toolset_location_file"`
if [[ -a "$path" ]]; then
toolset_build_proj="$path"
return
fi
fi
if [[ "$restore" != true ]]; then
echo "Toolset version $toolsetVersion has not been restored." >&2
ExitWithExitCode 2
fi
local toolset_restore_log="$log_dir/ToolsetRestore.binlog"
local proj="$toolset_dir/restore.proj"
echo '<Project Sdk="Microsoft.DotNet.Arcade.Sdk"/>' > "$proj"
MSBuild "$proj" /t:__WriteToolsetLocation /clp:None /bl:"$toolset_restore_log" /p:__ToolsetLocationOutputFile="$toolset_location_file"
local lastexitcode=$?
if [[ $lastexitcode != 0 ]]; then
echo "Failed to restore toolset (exit code '$lastexitcode'). See log: $toolset_restore_log" >&2
ExitWithExitCode $lastexitcode
fi
toolset_build_proj=`cat "$toolset_location_file"`
if [[ ! -a "$toolset_build_proj" ]]; then
echo "Invalid toolset path: $toolset_build_proj" >&2
ExitWithExitCode 3
fi
}
function InitializeCustomToolset {
local script="$eng_root/restore-toolset.sh"
if [[ -a "$script" ]]; then
. "$script"
fi
}
function ConfigureTools {
local script="$eng_root/configure-toolset.sh"
if [[ -a "$script" ]]; then
. "$script"
fi
}
function InitializeTools {
ConfigureTools
InitializeDotNetCli $restore
build_driver="$_InitializeDotNetCli/dotnet"
InitializeToolset
InitializeCustomToolset
}
function ExitWithExitCode {
if [[ "$ci" == true && "$prepare_machine" == true ]]; then
StopProcesses
fi
exit $1
}
function StopProcesses {
echo "Killing running build processes..."
pkill -9 "dotnet"
pkill -9 "vbcscompiler"
return 0
}
function MSBuild {
local warnaserror_switch=""
if [[ $warnaserror == true ]]; then
warnaserror_switch="/warnaserror"
fi
"$build_driver" msbuild /m /nologo /clp:Summary /v:$verbosity /nr:$nodereuse $warnaserror_switch "$@"
return $?
}
# HOME may not be defined in some scenarios, but it is required by NuGet
if [[ -z $HOME ]]; then
export HOME="$repo_root/artifacts/.home/"
mkdir -p "$HOME"
fi
if [[ -z ${NUGET_PACKAGES:-} ]]; then
if [[ $ci == true ]]; then
export NUGET_PACKAGES="$repo_root/.packages"
else
export NUGET_PACKAGES="$HOME/.nuget/packages"
fi
fi
mkdir -p "$toolset_dir"
mkdir -p "$log_dir"
if [[ $ci == true ]]; then
mkdir -p "$temp_dir"
export TEMP="$temp_dir"
export TMP="$temp_dir"
fi

8
global.json Normal file
View file

@ -0,0 +1,8 @@
{
"tools": {
"dotnet": "2.1.400-preview-009088"
},
"msbuild-sdks": {
"Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.18520.3"
}
}