mirrors/dotnet-installer
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

Update dependencies from https://github.com/dotnet/arcade build 20191121.10 (#5760)

Browse source 
- Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.19571.10
This commit is contained in:
dotnet-maestro[bot] 2019-11-22 13:41:58 +00:00 committed by GitHub
parent c28aeebc62
commit f5d3c3dd00
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
41 changed files with 919 additions and 804 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
eng/Version.Details.xml
View file

@ -104,9 +104,9 @@
</Dependency> </Dependency>
</ProductDependencies> </ProductDependencies>
<ToolsetDependencies> <ToolsetDependencies>
<Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="5.0.0-beta.19570.6"> <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="5.0.0-beta.19571.10">
<Uri>https://github.com/dotnet/arcade</Uri> <Uri>https://github.com/dotnet/arcade</Uri>
<Sha>be36a2fb1898f4eb69b29a1150a507e371618b8c</Sha> <Sha>6ade65ab9d2d63b3fcb73647fc109e2b36ffdc80</Sha>
</Dependency> </Dependency>
</ToolsetDependencies> </ToolsetDependencies>
</Dependencies> </Dependencies>

49
eng/common/CheckSymbols.ps1
View file

@ -5,11 +5,12 @@ param(
) )
Add-Type -AssemblyName System.IO.Compression.FileSystem Add-Type -AssemblyName System.IO.Compression.FileSystem
. $PSScriptRoot\pipeline-logging-functions.ps1
function FirstMatchingSymbolDescriptionOrDefault { function FirstMatchingSymbolDescriptionOrDefault {
param( param(
[string] $FullPath, # Full path to the module that has to be checked [string] $FullPath, # Full path to the module that has to be checked
[string] $TargetServerParam, # Parameter to pass to `Symbol Tool` indicating the server to lookup for symbols [string] $TargetServerParameter, # Parameter to pass to `Symbol Tool` indicating the server to lookup for symbols
[string] $SymbolsPath [string] $SymbolsPath
) )
@ -21,36 +22,36 @@ function FirstMatchingSymbolDescriptionOrDefault {
# checking and which type of file was uploaded. # checking and which type of file was uploaded.
# The file itself is returned # The file itself is returned
$SymbolPath = $SymbolsPath + "\" + $FileName $SymbolPath = $SymbolsPath + '\' + $FileName
# PDB file for the module # PDB file for the module
$PdbPath = $SymbolPath.Replace($Extension, ".pdb") $PdbPath = $SymbolPath.Replace($Extension, '.pdb')
# PDB file for R2R module (created by crossgen) # PDB file for R2R module (created by crossgen)
$NGenPdb = $SymbolPath.Replace($Extension, ".ni.pdb") $NGenPdb = $SymbolPath.Replace($Extension, '.ni.pdb')
# DBG file for a .so library # DBG file for a .so library
$SODbg = $SymbolPath.Replace($Extension, ".so.dbg") $SODbg = $SymbolPath.Replace($Extension, '.so.dbg')
# DWARF file for a .dylib # DWARF file for a .dylib
$DylibDwarf = $SymbolPath.Replace($Extension, ".dylib.dwarf") $DylibDwarf = $SymbolPath.Replace($Extension, '.dylib.dwarf')
.\dotnet-symbol.exe --symbols --modules --windows-pdbs $TargetServerParam $FullPath -o $SymbolsPath | Out-Null .\dotnet-symbol.exe --symbols --modules --windows-pdbs $TargetServerParameter $FullPath -o $SymbolsPath | Out-Null
if (Test-Path $PdbPath) { if (Test-Path $PdbPath) {
return "PDB" return 'PDB'
} }
elseif (Test-Path $NGenPdb) { elseif (Test-Path $NGenPdb) {
return "NGen PDB" return 'NGen PDB'
} }
elseif (Test-Path $SODbg) { elseif (Test-Path $SODbg) {
return "DBG for SO" return 'DBG for SO'
} }
elseif (Test-Path $DylibDwarf) { elseif (Test-Path $DylibDwarf) {
return "Dwarf for Dylib" return 'Dwarf for Dylib'
} }
elseif (Test-Path $SymbolPath) { elseif (Test-Path $SymbolPath) {
return "Module" return 'Module'
} }
else { else {
return $null return $null
@ -68,7 +69,7 @@ function CountMissingSymbols {
} }
# Extensions for which we'll look for symbols # Extensions for which we'll look for symbols
$RelevantExtensions = @(".dll", ".exe", ".so", ".dylib") $RelevantExtensions = @('.dll', '.exe', '.so', '.dylib')
# How many files are missing symbol information # How many files are missing symbol information
$MissingSymbols = 0 $MissingSymbols = 0
@ -76,7 +77,7 @@ function CountMissingSymbols {
$PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath)
$PackageGuid = New-Guid $PackageGuid = New-Guid
$ExtractPath = Join-Path -Path $ExtractPath -ChildPath $PackageGuid $ExtractPath = Join-Path -Path $ExtractPath -ChildPath $PackageGuid
$SymbolsPath = Join-Path -Path $ExtractPath -ChildPath "Symbols" $SymbolsPath = Join-Path -Path $ExtractPath -ChildPath 'Symbols'
[System.IO.Compression.ZipFile]::ExtractToDirectory($PackagePath, $ExtractPath) [System.IO.Compression.ZipFile]::ExtractToDirectory($PackagePath, $ExtractPath)
@ -86,31 +87,31 @@ function CountMissingSymbols {
Get-ChildItem -Recurse $ExtractPath | Get-ChildItem -Recurse $ExtractPath |
Where-Object {$RelevantExtensions -contains $_.Extension} | Where-Object {$RelevantExtensions -contains $_.Extension} |
ForEach-Object { ForEach-Object {
if ($_.FullName -Match "\\ref\\") { if ($_.FullName -Match '\\ref\\') {
Write-Host "`t Ignoring reference assembly file" $_.FullName Write-Host "`t Ignoring reference assembly file" $_.FullName
return return
} }
$SymbolsOnMSDL = FirstMatchingSymbolDescriptionOrDefault $_.FullName "--microsoft-symbol-server" $SymbolsPath $SymbolsOnMSDL = FirstMatchingSymbolDescriptionOrDefault -FullPath $_.FullName -TargetServerParameter '--microsoft-symbol-server' -SymbolsPath $SymbolsPath
$SymbolsOnSymWeb = FirstMatchingSymbolDescriptionOrDefault $_.FullName "--internal-server" $SymbolsPath $SymbolsOnSymWeb = FirstMatchingSymbolDescriptionOrDefault -FullPath $_.FullName -TargetServerParameter '--internal-server' -SymbolsPath $SymbolsPath
Write-Host -NoNewLine "`t Checking file" $_.FullName "... " Write-Host -NoNewLine "`t Checking file" $_.FullName "... "
if ($SymbolsOnMSDL -ne $null -and $SymbolsOnSymWeb -ne $null) { if ($SymbolsOnMSDL -ne $null -and $SymbolsOnSymWeb -ne $null) {
Write-Host "Symbols found on MSDL (" $SymbolsOnMSDL ") and SymWeb (" $SymbolsOnSymWeb ")" Write-Host "Symbols found on MSDL (${$SymbolsOnMSDL}) and SymWeb (${$SymbolsOnSymWeb})"
} }
else { else {
$MissingSymbols++ $MissingSymbols++
if ($SymbolsOnMSDL -eq $null -and $SymbolsOnSymWeb -eq $null) { if ($SymbolsOnMSDL -eq $null -and $SymbolsOnSymWeb -eq $null) {
Write-Host "No symbols found on MSDL or SymWeb!" Write-Host 'No symbols found on MSDL or SymWeb!'
} }
else { else {
if ($SymbolsOnMSDL -eq $null) { if ($SymbolsOnMSDL -eq $null) {
Write-Host "No symbols found on MSDL!" Write-Host 'No symbols found on MSDL!'
} }
else { else {
Write-Host "No symbols found on SymWeb!" Write-Host 'No symbols found on SymWeb!'
} }
} }
} }
@ -133,12 +134,12 @@ function CheckSymbolsAvailable {
# These packages from Arcade-Services include some native libraries that # These packages from Arcade-Services include some native libraries that
# our current symbol uploader can't handle. Below is a workaround until # our current symbol uploader can't handle. Below is a workaround until
# we get issue: https://github.com/dotnet/arcade/issues/2457 sorted. # we get issue: https://github.com/dotnet/arcade/issues/2457 sorted.
if ($FileName -Match "Microsoft\.DotNet\.Darc\.") { if ($FileName -Match 'Microsoft\.DotNet\.Darc\.') {
Write-Host "Ignoring Arcade-services file: $FileName" Write-Host "Ignoring Arcade-services file: $FileName"
Write-Host Write-Host
return return
} }
elseif ($FileName -Match "Microsoft\.DotNet\.Maestro\.Tasks\.") { elseif ($FileName -Match 'Microsoft\.DotNet\.Maestro\.Tasks\.') {
Write-Host "Ignoring Arcade-services file: $FileName" Write-Host "Ignoring Arcade-services file: $FileName"
Write-Host Write-Host
return return
@ -148,7 +149,7 @@ function CheckSymbolsAvailable {
$Status = CountMissingSymbols "$InputPath\$FileName" $Status = CountMissingSymbols "$InputPath\$FileName"
if ($Status -ne 0) { if ($Status -ne 0) {
Write-Error "Missing symbols for $Status modules in the package $FileName" Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Missing symbols for $Status modules in the package $FileName"
} }
Write-Host Write-Host

9
eng/common/SetupNugetSources.ps1
View file

@ -94,8 +94,9 @@ function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Password) {
} }
} }
try {
if (!(Test-Path $ConfigFile -PathType Leaf)) { if (!(Test-Path $ConfigFile -PathType Leaf)) {
Write-Host "Couldn't find the file NuGet config file: $ConfigFile" Write-PipelineTelemetryError -Category 'Build' -Message "Couldn't find the file NuGet config file: $ConfigFile"
ExitWithExitCode 1 ExitWithExitCode 1
} }
@ -125,3 +126,9 @@ AddPackageSource -Sources $sources -SourceName "dotnet3-internal" -SourceEndPoin
AddPackageSource -Sources $sources -SourceName "dotnet3-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3-internal-transport/nuget/v2" -Creds $creds -Username "dn-bot" -Password $Password AddPackageSource -Sources $sources -SourceName "dotnet3-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3-internal-transport/nuget/v2" -Creds $creds -Username "dn-bot" -Password $Password
$doc.Save($filename) $doc.Save($filename)
}
catch {
Write-Host $_.ScriptStackTrace
Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_
ExitWithExitCode 1
}

2
eng/common/SetupNugetSources.sh
View file

@ -42,7 +42,7 @@ scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
. "$scriptroot/tools.sh" . "$scriptroot/tools.sh"
if [ ! -f "$ConfigFile" ]; then if [ ! -f "$ConfigFile" ]; then
echo "Couldn't find the file NuGet config file: $ConfigFile" Write-PipelineTelemetryError -Category 'Build' -Message "Couldn't find the file NuGet config file: $ConfigFile"
ExitWithExitCode 1 ExitWithExitCode 1
fi fi

103
eng/common/SourceLinkValidation.ps1
View file

@ -6,6 +6,59 @@ param(
[Parameter(Mandatory=$true)][string] $GHCommit # GitHub commit SHA used to build the packages [Parameter(Mandatory=$true)][string] $GHCommit # GitHub commit SHA used to build the packages
) )
function ValidateSourceLinkLinks {
if (!($GHRepoName -Match "^[^\s\/]+/[^\s\/]+$")) {
Write-PipelineTelemetryError -Category "Build" -Message "GHRepoName should be in the format <org>/<repo>"
$global:LASTEXITCODE = 1
return
}
if (!($GHCommit -Match "^[0-9a-fA-F]{40}$")) {
Write-PipelineTelemetryError -Category "Build" -Message "GHCommit should be a 40 chars hexadecimal string"
$global:LASTEXITCODE = 1
return
}
$RepoTreeURL = -Join("https://api.github.com/repos/", $GHRepoName, "/git/trees/", $GHCommit, "?recursive=1")
$CodeExtensions = @(".cs", ".vb", ".fs", ".fsi", ".fsx", ".fsscript")
try {
# Retrieve the list of files in the repo at that particular commit point and store them in the RepoFiles hash
$Data = Invoke-WebRequest $RepoTreeURL | ConvertFrom-Json | Select-Object -ExpandProperty tree
foreach ($file in $Data) {
$Extension = [System.IO.Path]::GetExtension($file.path)
if ($CodeExtensions.Contains($Extension)) {
$RepoFiles[$file.path] = 1
}
}
}
catch {
Write-PipelineTelemetryError -Category "Build" -Message "Problems downloading the list of files from the repo. Url used: $RepoTreeURL"
$global:LASTEXITCODE = 1
return
}
if (Test-Path $ExtractPath) {
Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue
}
# Process each NuGet package in parallel
$Jobs = @()
Get-ChildItem "$InputPath\*.symbols.nupkg" |
ForEach-Object {
$Jobs += Start-Job -ScriptBlock $ValidatePackage -ArgumentList $_.FullName
}
foreach ($Job in $Jobs) {
Wait-Job -Id $Job.Id | Receive-Job
}
}
try {
. $PSScriptRoot\pipeline-logging-functions.ps1
# Cache/HashMap (File -> Exist flag) used to consult whether a file exist # Cache/HashMap (File -> Exist flag) used to consult whether a file exist
# in the repository at a specific commit point. This is populated by inserting # in the repository at a specific commit point. This is populated by inserting
# all files present in the repo at a specific commit point. # all files present in the repo at a specific commit point.
@ -131,54 +184,12 @@ $ValidatePackage = {
} }
} }
function ValidateSourceLinkLinks { Measure-Command { ValidateSourceLinkLinks }
if (!($GHRepoName -Match "^[^\s\/]+/[^\s\/]+$")) {
Write-Host "GHRepoName should be in the format <org>/<repo>"
$global:LASTEXITCODE = 1
return
}
if (!($GHCommit -Match "^[0-9a-fA-F]{40}$")) {
Write-Host "GHCommit should be a 40 chars hexadecimal string"
$global:LASTEXITCODE = 1
return
}
$RepoTreeURL = -Join("https://api.github.com/repos/", $GHRepoName, "/git/trees/", $GHCommit, "?recursive=1")
$CodeExtensions = @(".cs", ".vb", ".fs", ".fsi", ".fsx", ".fsscript")
try {
# Retrieve the list of files in the repo at that particular commit point and store them in the RepoFiles hash
$Data = Invoke-WebRequest $RepoTreeURL | ConvertFrom-Json | Select-Object -ExpandProperty tree
foreach ($file in $Data) {
$Extension = [System.IO.Path]::GetExtension($file.path)
if ($CodeExtensions.Contains($Extension)) {
$RepoFiles[$file.path] = 1
}
}
} }
catch { catch {
Write-Host "Problems downloading the list of files from the repo. Url used: $RepoTreeURL" Write-Host $_.ScriptStackTrace
$global:LASTEXITCODE = 1 Write-PipelineTelemetryError -Category 'SourceLink' -Message $_
return ExitWithExitCode 1
} }
if (Test-Path $ExtractPath) {
Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue
}
# Process each NuGet package in parallel
$Jobs = @()
Get-ChildItem "$InputPath\*.symbols.nupkg" |
ForEach-Object {
$Jobs += Start-Job -ScriptBlock $ValidatePackage -ArgumentList $_.FullName
}
foreach ($Job in $Jobs) {
Wait-Job -Id $Job.Id | Receive-Job
}
}
Measure-Command { ValidateSourceLinkLinks }

20
eng/common/build.ps1
View file

@ -26,8 +26,6 @@ Param(
[Parameter(ValueFromRemainingArguments=$true)][String[]]$properties [Parameter(ValueFromRemainingArguments=$true)][String[]]$properties
) )
. $PSScriptRoot\tools.ps1
function Print-Usage() { function Print-Usage() {
Write-Host "Common settings:" Write-Host "Common settings:"
Write-Host " -configuration <value> Build configuration: 'Debug' or 'Release' (short: -c)" Write-Host " -configuration <value> Build configuration: 'Debug' or 'Release' (short: -c)"
@ -69,7 +67,7 @@ function InitializeCustomToolset {
return return
} }
$script = Join-Path $EngRoot "restore-toolset.ps1" $script = Join-Path $EngRoot 'restore-toolset.ps1'
if (Test-Path $script) { if (Test-Path $script) {
. $script . $script
@ -80,8 +78,8 @@ function Build {
$toolsetBuildProj = InitializeToolset $toolsetBuildProj = InitializeToolset
InitializeCustomToolset InitializeCustomToolset
$bl = if ($binaryLog) { "/bl:" + (Join-Path $LogDir "Build.binlog") } else { "" } $bl = if ($binaryLog) { '/bl:' + (Join-Path $LogDir 'Build.binlog') } else { '' }
$platformArg = if ($platform) { "/p:Platform=$platform" } else { "" } $platformArg = if ($platform) { "/p:Platform=$platform" } else { '' }
if ($projects) { if ($projects) {
# Re-assign properties to a new variable because PowerShell doesn't let us append properties directly for unclear reasons. # Re-assign properties to a new variable because PowerShell doesn't let us append properties directly for unclear reasons.
@ -117,13 +115,19 @@ function Build {
if ($clean) { if ($clean) {
if(Test-Path $ArtifactsDir) { if(Test-Path $ArtifactsDir) {
Remove-Item -Recurse -Force $ArtifactsDir Remove-Item -Recurse -Force $ArtifactsDir
Write-Host "Artifacts directory deleted." Write-Host 'Artifacts directory deleted.'
} }
exit 0 exit 0
} }
try { try {
if ($help -or (($null -ne $properties) -and ($properties.Contains("/help") -or $properties.Contains("/?")))) { . $PSScriptRoot\tools.ps1
If ((Test-Path variable:LastExitCode) -And ($LastExitCode -ne 0)) {
Write-PipelineTelemetryError -Category 'InitializeToolset' -Message 'Eng/common/tools.ps1 returned a non-zero exit code.'
ExitWithExitCode $LastExitCode
}
if ($help -or (($null -ne $properties) -and ($properties.Contains('/help') -or $properties.Contains('/?')))) {
Print-Usage Print-Usage
exit 0 exit 0
} }
@ -141,7 +145,7 @@ try {
} }
catch { catch {
Write-Host $_.ScriptStackTrace Write-Host $_.ScriptStackTrace
Write-PipelineTelemetryError -Category "InitializeToolset" -Message $_ Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_
ExitWithExitCode 1 ExitWithExitCode 1
} }

15
eng/common/darc-init.ps1
View file

@ -1,14 +1,14 @@
param ( param (
$darcVersion = $null, $darcVersion = $null,
$versionEndpoint = "https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16", $versionEndpoint = 'https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16',
$verbosity = "m", $verbosity = 'minimal',
$toolpath = $null $toolpath = $null
) )
. $PSScriptRoot\tools.ps1 . $PSScriptRoot\tools.ps1
function InstallDarcCli ($darcVersion) { function InstallDarcCli ($darcVersion) {
$darcCliPackageName = "microsoft.dotnet.darc" $darcCliPackageName = 'microsoft.dotnet.darc'
$dotnetRoot = InitializeDotNetCli -install:$true $dotnetRoot = InitializeDotNetCli -install:$true
$dotnet = "$dotnetRoot\dotnet.exe" $dotnet = "$dotnetRoot\dotnet.exe"
@ -27,7 +27,7 @@ function InstallDarcCli ($darcVersion) {
$arcadeServicesSource = 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' $arcadeServicesSource = 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
Write-Host "Installing Darc CLI version $darcVersion..." Write-Host "Installing Darc CLI version $darcVersion..."
Write-Host "You may need to restart your command window if this is the first dotnet tool you have installed." Write-Host 'You may need to restart your command window if this is the first dotnet tool you have installed.'
if (-not $toolpath) { if (-not $toolpath) {
& "$dotnet" tool install $darcCliPackageName --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g & "$dotnet" tool install $darcCliPackageName --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g
}else { }else {
@ -35,4 +35,11 @@ function InstallDarcCli ($darcVersion) {
} }
} }
try {
InstallDarcCli $darcVersion InstallDarcCli $darcVersion
}
catch {
Write-Host $_.ScriptStackTrace
Write-PipelineTelemetryError -Category 'Darc' -Message $_
ExitWithExitCode 1
}

4
eng/common/darc-init.sh
View file

@ -2,8 +2,8 @@
source="${BASH_SOURCE[0]}" source="${BASH_SOURCE[0]}"
darcVersion='' darcVersion=''
versionEndpoint="https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16" versionEndpoint='https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16'
verbosity=m verbosity='minimal'
while [[ $# > 0 ]]; do while [[ $# > 0 ]]; do
opt="$(echo "$1" | awk '{print tolower($0)}')" opt="$(echo "$1" | awk '{print tolower($0)}')"

21
eng/common/dotnet-install.ps1 vendored
View file

@ -1,28 +1,27 @@
[CmdletBinding(PositionalBinding=$false)] [CmdletBinding(PositionalBinding=$false)]
Param( Param(
[string] $verbosity = "minimal", [string] $verbosity = 'minimal',
[string] $architecture = "", [string] $architecture = '',
[string] $version = "Latest", [string] $version = 'Latest',
[string] $runtime = "dotnet", [string] $runtime = 'dotnet',
[string] $RuntimeSourceFeed = "", [string] $RuntimeSourceFeed = '',
[string] $RuntimeSourceFeedKey = "" [string] $RuntimeSourceFeedKey = ''
) )
. $PSScriptRoot\tools.ps1 . $PSScriptRoot\tools.ps1
$dotnetRoot = Join-Path $RepoRoot ".dotnet" $dotnetRoot = Join-Path $RepoRoot '.dotnet'
$installdir = $dotnetRoot $installdir = $dotnetRoot
try { try {
if ($architecture -and $architecture.Trim() -eq "x86") { if ($architecture -and $architecture.Trim() -eq 'x86') {
$installdir = Join-Path $installdir "x86" $installdir = Join-Path $installdir 'x86'
} }
InstallDotNet $installdir $version $architecture $runtime $true -RuntimeSourceFeed $RuntimeSourceFeed -RuntimeSourceFeedKey $RuntimeSourceFeedKey InstallDotNet $installdir $version $architecture $runtime $true -RuntimeSourceFeed $RuntimeSourceFeed -RuntimeSourceFeedKey $RuntimeSourceFeedKey
} }
catch { catch {
Write-Host $_
Write-Host $_.Exception
Write-Host $_.ScriptStackTrace Write-Host $_.ScriptStackTrace
Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_
ExitWithExitCode 1 ExitWithExitCode 1
} }

7
eng/common/dotnet-install.sh vendored
View file

@ -11,6 +11,8 @@ while [[ -h "$source" ]]; do
done done
scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
. "$scriptroot/tools.sh"
version='Latest' version='Latest'
architecture='' architecture=''
runtime='dotnet' runtime='dotnet'
@ -40,7 +42,7 @@ while [[ $# > 0 ]]; do
runtimeSourceFeedKey="$1" runtimeSourceFeedKey="$1"
;; ;;
*) *)
echo "Invalid argument: $1" Write-PipelineTelemetryError -Category 'Build' -Message "Invalid argument: $1"
exit 1 exit 1
;; ;;
esac esac
@ -73,7 +75,6 @@ case $cpuname in
;; ;;
esac esac
. "$scriptroot/tools.sh"
dotnetRoot="$repo_root/.dotnet" dotnetRoot="$repo_root/.dotnet"
if [[ $architecture != "" ]] && [[ $architecture != $buildarch ]]; then if [[ $architecture != "" ]] && [[ $architecture != $buildarch ]]; then
dotnetRoot="$dotnetRoot/$architecture" dotnetRoot="$dotnetRoot/$architecture"
@ -81,7 +82,7 @@ fi
InstallDotNet $dotnetRoot $version "$architecture" $runtime true $runtimeSourceFeed $runtimeSourceFeedKey || { InstallDotNet $dotnetRoot $version "$architecture" $runtime true $runtimeSourceFeed $runtimeSourceFeedKey || {
local exit_code=$? local exit_code=$?
echo "dotnet-install.sh failed (exit code '$exit_code')." >&2 Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "dotnet-install.sh failed (exit code '$exit_code')." >&2
ExitWithExitCode $exit_code ExitWithExitCode $exit_code
} }

6
eng/common/enable-cross-org-publishing.ps1
View file

@ -2,5 +2,7 @@ param(
[string] $token [string] $token
) )
Write-Host "##vso[task.setvariable variable=VSS_NUGET_ACCESSTOKEN]$token" . $PSScriptRoot\pipeline-logging-functions.ps1
Write-Host "##vso[task.setvariable variable=VSS_NUGET_URI_PREFIXES]https://dnceng.pkgs.visualstudio.com/;https://pkgs.dev.azure.com/dnceng/;https://devdiv.pkgs.visualstudio.com/;https://pkgs.dev.azure.com/devdiv/"
Write-PipelineSetVariable -Name 'VSS_NUGET_ACCESSTOKEN' -Value $token
Write-PipelineSetVariable -Name 'VSS_NUGET_URI_PREFIXES' -Value 'https://dnceng.pkgs.visualstudio.com/;https://pkgs.dev.azure.com/dnceng/;https://devdiv.pkgs.visualstudio.com/;https://pkgs.dev.azure.com/devdiv/'

47
eng/common/generate-graph-files.ps1
View file

@ -9,33 +9,33 @@ Param(
# toolset dependencies see https://github.com/dotnet/arcade/blob/master/Documentation/Darc.md#toolset-vs-product-dependencies # toolset dependencies see https://github.com/dotnet/arcade/blob/master/Documentation/Darc.md#toolset-vs-product-dependencies
) )
$ErrorActionPreference = "Stop"
. $PSScriptRoot\tools.ps1
Import-Module -Name (Join-Path $PSScriptRoot "native\CommonLibrary.psm1")
function CheckExitCode ([string]$stage) function CheckExitCode ([string]$stage)
{ {
$exitCode = $LASTEXITCODE $exitCode = $LASTEXITCODE
if ($exitCode -ne 0) { if ($exitCode -ne 0) {
Write-Host "Something failed in stage: '$stage'. Check for errors above. Exiting now..." Write-PipelineTelemetryError -Category 'Arcade' -Message "Something failed in stage: '$stage'. Check for errors above. Exiting now..."
ExitWithExitCode $exitCode ExitWithExitCode $exitCode
} }
} }
try { try {
$ErrorActionPreference = 'Stop'
. $PSScriptRoot\tools.ps1
Import-Module -Name (Join-Path $PSScriptRoot 'native\CommonLibrary.psm1')
Push-Location $PSScriptRoot Push-Location $PSScriptRoot
Write-Host "Installing darc..." Write-Host 'Installing darc...'
. .\darc-init.ps1 -darcVersion $darcVersion . .\darc-init.ps1 -darcVersion $darcVersion
CheckExitCode "Running darc-init" CheckExitCode 'Running darc-init'
$engCommonBaseDir = Join-Path $PSScriptRoot "native\" $engCommonBaseDir = Join-Path $PSScriptRoot 'native\'
$graphvizInstallDir = CommonLibrary\Get-NativeInstallDirectory $graphvizInstallDir = CommonLibrary\Get-NativeInstallDirectory
$nativeToolBaseUri = "https://netcorenativeassets.blob.core.windows.net/resource-packages/external" $nativeToolBaseUri = 'https://netcorenativeassets.blob.core.windows.net/resource-packages/external'
$installBin = Join-Path $graphvizInstallDir "bin" $installBin = Join-Path $graphvizInstallDir 'bin'
Write-Host "Installing dot..." Write-Host 'Installing dot...'
.\native\install-tool.ps1 -ToolName graphviz -InstallPath $installBin -BaseUri $nativeToolBaseUri -CommonLibraryDirectory $engCommonBaseDir -Version $graphvizVersion -Verbose .\native\install-tool.ps1 -ToolName graphviz -InstallPath $installBin -BaseUri $nativeToolBaseUri -CommonLibraryDirectory $engCommonBaseDir -Version $graphvizVersion -Verbose
$darcExe = "$env:USERPROFILE\.dotnet\tools" $darcExe = "$env:USERPROFILE\.dotnet\tools"
@ -51,36 +51,35 @@ try {
$graphVizImageFilePath = "$outputFolder\graph.png" $graphVizImageFilePath = "$outputFolder\graph.png"
$normalGraphFilePath = "$outputFolder\graph-full.txt" $normalGraphFilePath = "$outputFolder\graph-full.txt"
$flatGraphFilePath = "$outputFolder\graph-flat.txt" $flatGraphFilePath = "$outputFolder\graph-flat.txt"
$baseOptions = @( "--github-pat", "$gitHubPat", "--azdev-pat", "$azdoPat", "--password", "$barToken" ) $baseOptions = @( '--github-pat', "$gitHubPat", '--azdev-pat', "$azdoPat", '--password', "$barToken" )
if ($includeToolset) { if ($includeToolset) {
Write-Host "Toolsets will be included in the graph..." Write-Host 'Toolsets will be included in the graph...'
$baseOptions += @( "--include-toolset" ) $baseOptions += @( '--include-toolset' )
} }
Write-Host "Generating standard dependency graph..." Write-Host 'Generating standard dependency graph...'
& "$darcExe" get-dependency-graph @baseOptions --output-file $normalGraphFilePath & "$darcExe" get-dependency-graph @baseOptions --output-file $normalGraphFilePath
CheckExitCode "Generating normal dependency graph" CheckExitCode 'Generating normal dependency graph'
Write-Host "Generating flat dependency graph and graphviz file..." Write-Host 'Generating flat dependency graph and graphviz file...'
& "$darcExe" get-dependency-graph @baseOptions --flat --coherency --graphviz $graphVizFilePath --output-file $flatGraphFilePath & "$darcExe" get-dependency-graph @baseOptions --flat --coherency --graphviz $graphVizFilePath --output-file $flatGraphFilePath
CheckExitCode "Generating flat and graphviz dependency graph" CheckExitCode 'Generating flat and graphviz dependency graph'
Write-Host "Generating graph image $graphVizFilePath" Write-Host "Generating graph image $graphVizFilePath"
$dotFilePath = Join-Path $installBin "graphviz\$graphvizVersion\release\bin\dot.exe" $dotFilePath = Join-Path $installBin "graphviz\$graphvizVersion\release\bin\dot.exe"
& "$dotFilePath" -Tpng -o"$graphVizImageFilePath" "$graphVizFilePath" & "$dotFilePath" -Tpng -o"$graphVizImageFilePath" "$graphVizFilePath"
CheckExitCode "Generating graphviz image" CheckExitCode 'Generating graphviz image'
Write-Host "'$graphVizFilePath', '$flatGraphFilePath', '$normalGraphFilePath' and '$graphVizImageFilePath' created!" Write-Host "'$graphVizFilePath', '$flatGraphFilePath', '$normalGraphFilePath' and '$graphVizImageFilePath' created!"
} }
catch { catch {
if (!$includeToolset) { if (!$includeToolset) {
Write-Host "This might be a toolset repo which includes only toolset dependencies. " -NoNewline -ForegroundColor Yellow Write-Host 'This might be a toolset repo which includes only toolset dependencies. ' -NoNewline -ForegroundColor Yellow
Write-Host "Since -includeToolset is not set there is no graph to create. Include -includeToolset and try again..." -ForegroundColor Yellow Write-Host 'Since -includeToolset is not set there is no graph to create. Include -includeToolset and try again...' -ForegroundColor Yellow
} }
Write-Host $_
Write-Host $_.Exception
Write-Host $_.ScriptStackTrace Write-Host $_.ScriptStackTrace
Write-PipelineTelemetryError -Category 'Arcade' -Message $_
ExitWithExitCode 1 ExitWithExitCode 1
} finally { } finally {
Pop-Location Pop-Location

34
eng/common/init-tools-native.ps1
View file

@ -35,7 +35,7 @@ File path to global.json file
#> #>
[CmdletBinding(PositionalBinding=$false)] [CmdletBinding(PositionalBinding=$false)]
Param ( Param (
[string] $BaseUri = "https://netcorenativeassets.blob.core.windows.net/resource-packages/external", [string] $BaseUri = 'https://netcorenativeassets.blob.core.windows.net/resource-packages/external',
[string] $InstallDirectory, [string] $InstallDirectory,
[switch] $Clean = $False, [switch] $Clean = $False,
[switch] $Force = $False, [switch] $Force = $False,
@ -45,26 +45,27 @@ Param (
) )
if (!$GlobalJsonFile) { if (!$GlobalJsonFile) {
$GlobalJsonFile = Join-Path (Get-Item $PSScriptRoot).Parent.Parent.FullName "global.json" $GlobalJsonFile = Join-Path (Get-Item $PSScriptRoot).Parent.Parent.FullName 'global.json'
} }
Set-StrictMode -version 2.0 Set-StrictMode -version 2.0
$ErrorActionPreference="Stop" $ErrorActionPreference='Stop'
Import-Module -Name (Join-Path $PSScriptRoot "native\CommonLibrary.psm1") . $PSScriptRoot\pipeline-logging-functions.ps1
Import-Module -Name (Join-Path $PSScriptRoot 'native\CommonLibrary.psm1')
try { try {
# Define verbose switch if undefined # Define verbose switch if undefined
$Verbose = $VerbosePreference -Eq "Continue" $Verbose = $VerbosePreference -Eq 'Continue'
$EngCommonBaseDir = Join-Path $PSScriptRoot "native\" $EngCommonBaseDir = Join-Path $PSScriptRoot 'native\'
$NativeBaseDir = $InstallDirectory $NativeBaseDir = $InstallDirectory
if (!$NativeBaseDir) { if (!$NativeBaseDir) {
$NativeBaseDir = CommonLibrary\Get-NativeInstallDirectory $NativeBaseDir = CommonLibrary\Get-NativeInstallDirectory
} }
$Env:CommonLibrary_NativeInstallDir = $NativeBaseDir $Env:CommonLibrary_NativeInstallDir = $NativeBaseDir
$InstallBin = Join-Path $NativeBaseDir "bin" $InstallBin = Join-Path $NativeBaseDir 'bin'
$InstallerPath = Join-Path $EngCommonBaseDir "install-tool.ps1" $InstallerPath = Join-Path $EngCommonBaseDir 'install-tool.ps1'
# Process tools list # Process tools list
Write-Host "Processing $GlobalJsonFile" Write-Host "Processing $GlobalJsonFile"
@ -74,7 +75,7 @@ try {
} }
$NativeTools = Get-Content($GlobalJsonFile) -Raw | $NativeTools = Get-Content($GlobalJsonFile) -Raw |
ConvertFrom-Json | ConvertFrom-Json |
Select-Object -Expand "native-tools" -ErrorAction SilentlyContinue Select-Object -Expand 'native-tools' -ErrorAction SilentlyContinue
if ($NativeTools) { if ($NativeTools) {
$NativeTools.PSObject.Properties | ForEach-Object { $NativeTools.PSObject.Properties | ForEach-Object {
$ToolName = $_.Name $ToolName = $_.Name
@ -112,18 +113,19 @@ try {
} }
$toolInstallationFailure = $true $toolInstallationFailure = $true
} else { } else {
Write-Error $errMsg Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message $errMsg
exit 1 exit 1
} }
} }
} }
if ((Get-Variable 'toolInstallationFailure' -ErrorAction 'SilentlyContinue') -and $toolInstallationFailure) { if ((Get-Variable 'toolInstallationFailure' -ErrorAction 'SilentlyContinue') -and $toolInstallationFailure) {
Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message 'Native tools bootstrap failed'
exit 1 exit 1
} }
} }
else { else {
Write-Host "No native tools defined in global.json" Write-Host 'No native tools defined in global.json'
exit 0 exit 0
} }
@ -131,18 +133,18 @@ try {
exit 0 exit 0
} }
if (Test-Path $InstallBin) { if (Test-Path $InstallBin) {
Write-Host "Native tools are available from" (Convert-Path -Path $InstallBin) Write-Host 'Native tools are available from ' (Convert-Path -Path $InstallBin)
Write-Host "##vso[task.prependpath]$(Convert-Path -Path $InstallBin)" Write-Host "##vso[task.prependpath]$(Convert-Path -Path $InstallBin)"
return $InstallBin return $InstallBin
} }
else { else {
Write-Error "Native tools install directory does not exist, installation failed" Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message 'Native tools install directory does not exist, installation failed'
exit 1 exit 1
} }
exit 0 exit 0
} }
catch { catch {
Write-Host $_ Write-Host $_.ScriptStackTrace
Write-Host $_.Exception Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message $_
exit 1 ExitWithExitCode 1
} }

5
eng/common/init-tools-native.sh
View file

@ -12,6 +12,7 @@ retry_wait_time_seconds=30
global_json_file="$(dirname "$(dirname "${scriptroot}")")/global.json" global_json_file="$(dirname "$(dirname "${scriptroot}")")/global.json"
declare -A native_assets declare -A native_assets
. $scriptroot/pipeline-logging-functions.sh
. $scriptroot/native/common-library.sh . $scriptroot/native/common-library.sh
while (($# > 0)); do while (($# > 0)); do
@ -120,7 +121,7 @@ else
$installer_command $installer_command
if [[ $? != 0 ]]; then if [[ $? != 0 ]]; then
echo "Execution Failed" >&2 Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed"
exit 1 exit 1
fi fi
done done
@ -134,7 +135,7 @@ if [[ -d $install_bin ]]; then
echo "Native tools are available from $install_bin" echo "Native tools are available from $install_bin"
echo "##vso[task.prependpath]$install_bin" echo "##vso[task.prependpath]$install_bin"
else else
echo "Native tools install directory does not exist, installation failed" >&2 Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Native tools install directory does not exist, installation failed"
exit 1 exit 1
fi fi

26
eng/common/internal-feed-operations.ps1
View file

@ -6,9 +6,8 @@ param(
[switch] $IsFeedPrivate [switch] $IsFeedPrivate
) )
$ErrorActionPreference = "Stop" $ErrorActionPreference = 'Stop'
Set-StrictMode -Version 2.0 Set-StrictMode -Version 2.0
. $PSScriptRoot\tools.ps1 . $PSScriptRoot\tools.ps1
# Sets VSS_NUGET_EXTERNAL_FEED_ENDPOINTS based on the "darc-int-*" feeds defined in NuGet.config. This is needed # Sets VSS_NUGET_EXTERNAL_FEED_ENDPOINTS based on the "darc-int-*" feeds defined in NuGet.config. This is needed
@ -21,7 +20,7 @@ function SetupCredProvider {
) )
# Install the Cred Provider NuGet plugin # Install the Cred Provider NuGet plugin
Write-Host "Setting up Cred Provider NuGet plugin in the agent..." Write-Host 'Setting up Cred Provider NuGet plugin in the agent...'
Write-Host "Getting 'installcredprovider.ps1' from 'https://github.com/microsoft/artifacts-credprovider'..." Write-Host "Getting 'installcredprovider.ps1' from 'https://github.com/microsoft/artifacts-credprovider'..."
$url = 'https://raw.githubusercontent.com/microsoft/artifacts-credprovider/master/helpers/installcredprovider.ps1' $url = 'https://raw.githubusercontent.com/microsoft/artifacts-credprovider/master/helpers/installcredprovider.ps1'
@ -29,18 +28,18 @@ function SetupCredProvider {
Write-Host "Writing the contents of 'installcredprovider.ps1' locally..." Write-Host "Writing the contents of 'installcredprovider.ps1' locally..."
Invoke-WebRequest $url -OutFile installcredprovider.ps1 Invoke-WebRequest $url -OutFile installcredprovider.ps1
Write-Host "Installing plugin..." Write-Host 'Installing plugin...'
.\installcredprovider.ps1 -Force .\installcredprovider.ps1 -Force
Write-Host "Deleting local copy of 'installcredprovider.ps1'..." Write-Host "Deleting local copy of 'installcredprovider.ps1'..."
Remove-Item .\installcredprovider.ps1 Remove-Item .\installcredprovider.ps1
if (-Not("$env:USERPROFILE\.nuget\plugins\netcore")) { if (-Not("$env:USERPROFILE\.nuget\plugins\netcore")) {
Write-Host "CredProvider plugin was not installed correctly!" Write-PipelineTelemetryError -Category 'Arcade' -Message 'CredProvider plugin was not installed correctly!'
ExitWithExitCode 1 ExitWithExitCode 1
} }
else { else {
Write-Host "CredProvider plugin was installed correctly!" Write-Host 'CredProvider plugin was installed correctly!'
} }
# Then, we set the 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' environment variable to restore from the stable # Then, we set the 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' environment variable to restore from the stable
@ -49,7 +48,7 @@ function SetupCredProvider {
$nugetConfigPath = "$RepoRoot\NuGet.config" $nugetConfigPath = "$RepoRoot\NuGet.config"
if (-Not (Test-Path -Path $nugetConfigPath)) { if (-Not (Test-Path -Path $nugetConfigPath)) {
Write-Host "NuGet.config file not found in repo's root!" Write-PipelineTelemetryError -Category 'Build' -Message 'NuGet.config file not found in repo root!'
ExitWithExitCode 1 ExitWithExitCode 1
} }
@ -81,7 +80,7 @@ function SetupCredProvider {
} }
else else
{ {
Write-Host "No internal endpoints found in NuGet.config" Write-Host 'No internal endpoints found in NuGet.config'
} }
} }
@ -99,7 +98,7 @@ function InstallDotNetSdkAndRestoreArcade {
& $dotnet restore $restoreProjPath & $dotnet restore $restoreProjPath
Write-Host "Arcade SDK restored!" Write-Host 'Arcade SDK restored!'
if (Test-Path -Path $restoreProjPath) { if (Test-Path -Path $restoreProjPath) {
Remove-Item $restoreProjPath Remove-Item $restoreProjPath
@ -113,21 +112,20 @@ function InstallDotNetSdkAndRestoreArcade {
try { try {
Push-Location $PSScriptRoot Push-Location $PSScriptRoot
if ($Operation -like "setup") { if ($Operation -like 'setup') {
SetupCredProvider $AuthToken SetupCredProvider $AuthToken
} }
elseif ($Operation -like "install-restore") { elseif ($Operation -like 'install-restore') {
InstallDotNetSdkAndRestoreArcade InstallDotNetSdkAndRestoreArcade
} }
else { else {
Write-Host "Unknown operation '$Operation'!" Write-PipelineTelemetryError -Category 'Arcade' -Message "Unknown operation '$Operation'!"
ExitWithExitCode 1 ExitWithExitCode 1
} }
} }
catch { catch {
Write-Host $_
Write-Host $_.Exception
Write-Host $_.ScriptStackTrace Write-Host $_.ScriptStackTrace
Write-PipelineTelemetryError -Category 'Arcade' -Message $_
ExitWithExitCode 1 ExitWithExitCode 1
} }
finally { finally {

4
eng/common/internal-feed-operations.sh
View file

@ -30,7 +30,7 @@ function SetupCredProvider {
rm installcredprovider.sh rm installcredprovider.sh
if [ ! -d "$HOME/.nuget/plugins" ]; then if [ ! -d "$HOME/.nuget/plugins" ]; then
echo "CredProvider plugin was not installed correctly!" Write-PipelineTelemetryError -category 'Build' 'CredProvider plugin was not installed correctly!'
ExitWithExitCode 1 ExitWithExitCode 1
else else
echo "CredProvider plugin was installed correctly!" echo "CredProvider plugin was installed correctly!"
@ -42,7 +42,7 @@ function SetupCredProvider {
local nugetConfigPath="$repo_root/NuGet.config" local nugetConfigPath="$repo_root/NuGet.config"
if [ ! "$nugetConfigPath" ]; then if [ ! "$nugetConfigPath" ]; then
echo "NuGet.config file not found in repo's root!" Write-PipelineTelemetryError -category 'Build' "NuGet.config file not found in repo's root!"
ExitWithExitCode 1 ExitWithExitCode 1
fi fi

5
eng/common/msbuild.ps1
View file

@ -1,6 +1,6 @@
[CmdletBinding(PositionalBinding=$false)] [CmdletBinding(PositionalBinding=$false)]
Param( Param(
[string] $verbosity = "minimal", [string] $verbosity = 'minimal',
[bool] $warnAsError = $true, [bool] $warnAsError = $true,
[bool] $nodeReuse = $true, [bool] $nodeReuse = $true,
[switch] $ci, [switch] $ci,
@ -18,9 +18,8 @@ try {
MSBuild @extraArgs MSBuild @extraArgs
} }
catch { catch {
Write-Host $_
Write-Host $_.Exception
Write-Host $_.ScriptStackTrace Write-Host $_.ScriptStackTrace
Write-PipelineTelemetryError -Category 'Build' -Message $_
ExitWithExitCode 1 ExitWithExitCode 1
} }

10
eng/common/native/common-library.sh
View file

@ -34,7 +34,7 @@ function ExpandZip {
echo "'Force flag enabled, but '$output_directory' exists. Removing directory" echo "'Force flag enabled, but '$output_directory' exists. Removing directory"
rm -rf $output_directory rm -rf $output_directory
if [[ $? != 0 ]]; then if [[ $? != 0 ]]; then
echo Unable to remove '$output_directory'>&2 Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Unable to remove '$output_directory'"
return 1 return 1
fi fi
fi fi
@ -45,7 +45,7 @@ function ExpandZip {
echo "Extracting archive" echo "Extracting archive"
tar -xf $zip_path -C $output_directory tar -xf $zip_path -C $output_directory
if [[ $? != 0 ]]; then if [[ $? != 0 ]]; then
echo "Unable to extract '$zip_path'" >&2 Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Unable to extract '$zip_path'"
return 1 return 1
fi fi
@ -117,7 +117,7 @@ function DownloadAndExtract {
# Download file # Download file
GetFile "$uri" "$temp_tool_path" $force $download_retries $retry_wait_time_seconds GetFile "$uri" "$temp_tool_path" $force $download_retries $retry_wait_time_seconds
if [[ $? != 0 ]]; then if [[ $? != 0 ]]; then
echo "Failed to download '$uri' to '$temp_tool_path'." >&2 Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Failed to download '$uri' to '$temp_tool_path'."
return 1 return 1
fi fi
@ -125,7 +125,7 @@ function DownloadAndExtract {
echo "extracting from $temp_tool_path to $installDir" echo "extracting from $temp_tool_path to $installDir"
ExpandZip "$temp_tool_path" "$installDir" $force $download_retries $retry_wait_time_seconds ExpandZip "$temp_tool_path" "$installDir" $force $download_retries $retry_wait_time_seconds
if [[ $? != 0 ]]; then if [[ $? != 0 ]]; then
echo "Failed to extract '$temp_tool_path' to '$installDir'." >&2 Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Failed to extract '$temp_tool_path' to '$installDir'."
return 1 return 1
fi fi
@ -148,7 +148,7 @@ function NewScriptShim {
fi fi
if [[ ! -f $tool_file_path ]]; then if [[ ! -f $tool_file_path ]]; then
echo "Specified tool file path:'$tool_file_path' does not exist" >&2 Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Specified tool file path:'$tool_file_path' does not exist"
return 1 return 1
fi fi

4
eng/common/native/install-cmake-test.sh
View file

@ -101,7 +101,7 @@ fi
DownloadAndExtract $uri $tool_install_directory $force $download_retries $retry_wait_time_seconds DownloadAndExtract $uri $tool_install_directory $force $download_retries $retry_wait_time_seconds
if [[ $? != 0 ]]; then if [[ $? != 0 ]]; then
echo "Installation failed" >&2 Write-PipelineTelemetryError -category 'NativeToolsBootstrap' 'Installation failed'
exit 1 exit 1
fi fi
@ -110,7 +110,7 @@ fi
NewScriptShim $shim_path $tool_file_path true NewScriptShim $shim_path $tool_file_path true
if [[ $? != 0 ]]; then if [[ $? != 0 ]]; then
echo "Shim generation failed" >&2 Write-PipelineTelemetryError -category 'NativeToolsBootstrap' 'Shim generation failed'
exit 1 exit 1
fi fi

4
eng/common/native/install-cmake.sh
View file

@ -101,7 +101,7 @@ fi
DownloadAndExtract $uri $tool_install_directory $force $download_retries $retry_wait_time_seconds DownloadAndExtract $uri $tool_install_directory $force $download_retries $retry_wait_time_seconds
if [[ $? != 0 ]]; then if [[ $? != 0 ]]; then
echo "Installation failed" >&2 Write-PipelineTelemetryError -category 'NativeToolsBootstrap' 'Installation failed'
exit 1 exit 1
fi fi
@ -110,7 +110,7 @@ fi
NewScriptShim $shim_path $tool_file_path true NewScriptShim $shim_path $tool_file_path true
if [[ $? != 0 ]]; then if [[ $? != 0 ]]; then
echo "Shim generation failed" >&2 Write-PipelineTelemetryError -category 'NativeToolsBootstrap' 'Shim generation failed'
exit 1 exit 1
fi fi

10
eng/common/native/install-tool.ps1
View file

@ -46,6 +46,8 @@ Param (
[int] $RetryWaitTimeInSeconds = 30 [int] $RetryWaitTimeInSeconds = 30
) )
. $PSScriptRoot\..\pipeline-logging-functions.ps1
# Import common library modules # Import common library modules
Import-Module -Name (Join-Path $CommonLibraryDirectory "CommonLibrary.psm1") Import-Module -Name (Join-Path $CommonLibraryDirectory "CommonLibrary.psm1")
@ -93,7 +95,7 @@ try {
-Verbose:$Verbose -Verbose:$Verbose
if ($InstallStatus -Eq $False) { if ($InstallStatus -Eq $False) {
Write-Error "Installation failed" Write-PipelineTelemetryError "Installation failed" -Category "NativeToolsetBootstrapping"
exit 1 exit 1
} }
} }
@ -117,14 +119,14 @@ try {
-Verbose:$Verbose -Verbose:$Verbose
if ($GenerateShimStatus -Eq $False) { if ($GenerateShimStatus -Eq $False) {
Write-Error "Generate shim failed" Write-PipelineTelemetryError "Generate shim failed" -Category "NativeToolsetBootstrapping"
return 1 return 1
} }
exit 0 exit 0
} }
catch { catch {
Write-Host $_ Write-Host $_.ScriptStackTrace
Write-Host $_.Exception Write-PipelineTelemetryError -Category "NativeToolsetBootstrapping" -Message $_
exit 1 exit 1
} }

24
eng/common/pipeline-logging-functions.ps1
View file

@ -12,6 +12,7 @@ $script:loggingCommandEscapeMappings = @( # TODO: WHAT ABOUT "="? WHAT ABOUT "%"
# TODO: BUG: Escape % ??? # TODO: BUG: Escape % ???
# TODO: Add test to verify don't need to escape "=". # TODO: Add test to verify don't need to escape "=".
# Specify "-Force" to force pipeline formatted output even if "$ci" is false or not set
function Write-PipelineTelemetryError { function Write-PipelineTelemetryError {
[CmdletBinding()] [CmdletBinding()]
param( param(
@ -25,17 +26,18 @@ function Write-PipelineTelemetryError {
[string]$SourcePath, [string]$SourcePath,
[string]$LineNumber, [string]$LineNumber,
[string]$ColumnNumber, [string]$ColumnNumber,
[switch]$AsOutput) [switch]$AsOutput,
[switch]$Force)
$PSBoundParameters.Remove("Category") | Out-Null $PSBoundParameters.Remove('Category') | Out-Null
$Message = "(NETCORE_ENGINEERING_TELEMETRY=$Category) $Message" $Message = "(NETCORE_ENGINEERING_TELEMETRY=$Category) $Message"
$PSBoundParameters.Remove("Message") | Out-Null $PSBoundParameters.Remove('Message') | Out-Null
$PSBoundParameters.Add("Message", $Message) $PSBoundParameters.Add('Message', $Message)
Write-PipelineTaskError @PSBoundParameters Write-PipelineTaskError @PSBoundParameters
} }
# Specify "-Force" to force pipeline formatted output even if "$ci" is false or not set
function Write-PipelineTaskError { function Write-PipelineTaskError {
[CmdletBinding()] [CmdletBinding()]
param( param(
@ -47,9 +49,11 @@ function Write-PipelineTaskError {
[string]$SourcePath, [string]$SourcePath,
[string]$LineNumber, [string]$LineNumber,
[string]$ColumnNumber, [string]$ColumnNumber,
[switch]$AsOutput) [switch]$AsOutput,
[switch]$Force
)
if(!$ci) { if(!$Force -And (-Not (Test-Path variable:ci) -Or !$ci)) {
if($Type -eq 'error') { if($Type -eq 'error') {
Write-Host $Message -ForegroundColor Red Write-Host $Message -ForegroundColor Red
return return
@ -64,6 +68,7 @@ function Write-PipelineTaskError {
Write-Host $Message Write-Host $Message
return return
} }
$PSBoundParameters.Remove('Force') | Out-Null
if(-not $PSBoundParameters.ContainsKey('Type')) { if(-not $PSBoundParameters.ContainsKey('Type')) {
$PSBoundParameters.Add('Type', 'error') $PSBoundParameters.Add('Type', 'error')
} }
@ -80,7 +85,7 @@ function Write-PipelineTaskError {
[switch]$AsOutput, [switch]$AsOutput,
[bool]$IsMultiJobVariable=$true) [bool]$IsMultiJobVariable=$true)
if($ci) { if(-Not (Test-Path variable:ci) -Or !$ci) {
Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data $Value -Properties @{ Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data $Value -Properties @{
'variable' = $Name 'variable' = $Name
'isSecret' = $Secret 'isSecret' = $Secret
@ -95,7 +100,8 @@ function Write-PipelineTaskError {
[Parameter(Mandatory=$true)] [Parameter(Mandatory=$true)]
[string]$Path, [string]$Path,
[switch]$AsOutput) [switch]$AsOutput)
if($ci) {
if(-Not (Test-Path variable:ci) -Or !$ci) {
Write-LoggingCommand -Area 'task' -Event 'prependpath' -Data $Path -AsOutput:$AsOutput Write-LoggingCommand -Area 'task' -Event 'prependpath' -Data $Path -AsOutput:$AsOutput
} }
} }

11
eng/common/pipeline-logging-functions.sh
View file

@ -2,6 +2,7 @@
function Write-PipelineTelemetryError { function Write-PipelineTelemetryError {
local telemetry_category='' local telemetry_category=''
local force=false
local function_args=() local function_args=()
local message='' local message=''
while [[ $# -gt 0 ]]; do while [[ $# -gt 0 ]]; do
@ -11,6 +12,9 @@ function Write-PipelineTelemetryError {
telemetry_category=$2 telemetry_category=$2
shift shift
;; ;;
-force|-f)
force=true
;;
-*) -*)
function_args+=("$1 $2") function_args+=("$1 $2")
shift shift
@ -22,19 +26,22 @@ function Write-PipelineTelemetryError {
shift shift
done done
if [[ "$ci" != true ]]; then if [[ $force != true ]] && [[ "$ci" != true ]]; then
echo "$message" >&2 echo "$message" >&2
return return
fi fi
message="(NETCORE_ENGINEERING_TELEMETRY=$telemetry_category) $message" message="(NETCORE_ENGINEERING_TELEMETRY=$telemetry_category) $message"
function_args+=("$message") function_args+=("$message")
if [[ $force == true ]]; then
function_args+=("-force")
fi
Write-PipelineTaskError $function_args Write-PipelineTaskError $function_args
} }
function Write-PipelineTaskError { function Write-PipelineTaskError {
if [[ "$ci" != true ]]; then if [[ $force != true ]] && [[ "$ci" != true ]]; then
echo "$@" >&2 echo "$@" >&2
return return
fi fi

13
eng/common/post-build/darc-gather-drop.ps1
View file

@ -2,20 +2,20 @@ param(
[Parameter(Mandatory=$true)][int] $BarBuildId, # ID of the build which assets should be downloaded [Parameter(Mandatory=$true)][int] $BarBuildId, # ID of the build which assets should be downloaded
[Parameter(Mandatory=$true)][string] $DropLocation, # Where the assets should be downloaded to [Parameter(Mandatory=$true)][string] $DropLocation, # Where the assets should be downloaded to
[Parameter(Mandatory=$true)][string] $MaestroApiAccessToken, # Token used to access Maestro API [Parameter(Mandatory=$true)][string] $MaestroApiAccessToken, # Token used to access Maestro API
[Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = "https://maestro-prod.westus2.cloudapp.azure.com", # Maestro API URL [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro-prod.westus2.cloudapp.azure.com', # Maestro API URL
[Parameter(Mandatory=$false)][string] $MaestroApiVersion = "2019-01-16" # Version of Maestro API to use [Parameter(Mandatory=$false)][string] $MaestroApiVersion = '2019-01-16' # Version of Maestro API to use
) )
try {
. $PSScriptRoot\post-build-utils.ps1 . $PSScriptRoot\post-build-utils.ps1
try { Write-Host 'Installing DARC ...'
Write-Host "Installing DARC ..."
. $PSScriptRoot\..\darc-init.ps1 . $PSScriptRoot\..\darc-init.ps1
$exitCode = $LASTEXITCODE $exitCode = $LASTEXITCODE
if ($exitCode -ne 0) { if ($exitCode -ne 0) {
Write-PipelineTaskError "Something failed while running 'darc-init.ps1'. Check for errors above. Exiting now..." Write-PipelineTelemetryError -Category "Darc" -Message "Something failed while running 'darc-init.ps1'. Check for errors above. Exiting now..."
ExitWithExitCode $exitCode ExitWithExitCode $exitCode
} }
@ -38,8 +38,7 @@ try {
--latest-location --latest-location
} }
catch { catch {
Write-Host $_
Write-Host $_.Exception
Write-Host $_.ScriptStackTrace Write-Host $_.ScriptStackTrace
Write-PipelineTelemetryError -Category "Darc" -Message $_
ExitWithExitCode 1 ExitWithExitCode 1
} }

9
eng/common/post-build/nuget-validation.ps1
View file

@ -6,20 +6,19 @@ param(
[Parameter(Mandatory=$true)][string] $ToolDestinationPath # Where the validation tool should be downloaded to [Parameter(Mandatory=$true)][string] $ToolDestinationPath # Where the validation tool should be downloaded to
) )
try {
. $PSScriptRoot\post-build-utils.ps1 . $PSScriptRoot\post-build-utils.ps1
try { $url = 'https://raw.githubusercontent.com/NuGet/NuGetGallery/jver-verify/src/VerifyMicrosoftPackage/verify.ps1'
$url = "https://raw.githubusercontent.com/NuGet/NuGetGallery/jver-verify/src/VerifyMicrosoftPackage/verify.ps1"
New-Item -ItemType "directory" -Path ${ToolDestinationPath} -Force New-Item -ItemType 'directory' -Path ${ToolDestinationPath} -Force
Invoke-WebRequest $url -OutFile ${ToolDestinationPath}\verify.ps1 Invoke-WebRequest $url -OutFile ${ToolDestinationPath}\verify.ps1
& ${ToolDestinationPath}\verify.ps1 ${PackagesPath}\*.nupkg & ${ToolDestinationPath}\verify.ps1 ${PackagesPath}\*.nupkg
} }
catch { catch {
Write-PipelineTaskError "NuGet package validation failed. Please check error logs."
Write-Host $_
Write-Host $_.ScriptStackTrace Write-Host $_.ScriptStackTrace
Write-PipelineTelemetryError -Category 'NuGetValidation' -Message $_
ExitWithExitCode 1 ExitWithExitCode 1
} }

32
eng/common/post-build/post-build-utils.ps1
View file

@ -1,17 +1,17 @@
# Most of the functions in this file require the variables `MaestroApiEndPoint`, # Most of the functions in this file require the variables `MaestroApiEndPoint`,
# `MaestroApiVersion` and `MaestroApiAccessToken` to be globally available. # `MaestroApiVersion` and `MaestroApiAccessToken` to be globally available.
$ErrorActionPreference = "Stop" $ErrorActionPreference = 'Stop'
Set-StrictMode -Version 2.0 Set-StrictMode -Version 2.0
# `tools.ps1` checks $ci to perform some actions. Since the post-build # `tools.ps1` checks $ci to perform some actions. Since the post-build
# scripts don't necessarily execute in the same agent that run the # scripts don't necessarily execute in the same agent that run the
# build.ps1/sh script this variable isn't automatically set. # build.ps1/sh script this variable isn't automatically set.
$ci = $true $ci = $true
$disableConfigureToolsetImport = "true" $disableConfigureToolsetImport = $true
. $PSScriptRoot\..\tools.ps1 . $PSScriptRoot\..\tools.ps1
function Create-MaestroApiRequestHeaders([string]$ContentType = "application/json") { function Create-MaestroApiRequestHeaders([string]$ContentType = 'application/json') {
Validate-MaestroVars Validate-MaestroVars
$headers = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]' $headers = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]'
@ -51,14 +51,6 @@ function Get-MaestroSubscriptions([string]$SourceRepository, [int]$ChannelId) {
return $result return $result
} }
function Trigger-Subscription([string]$SubscriptionId) {
Validate-MaestroVars
$apiHeaders = Create-MaestroApiRequestHeaders -AuthToken $MaestroApiAccessToken
$apiEndpoint = "$MaestroApiEndPoint/api/subscriptions/$SubscriptionId/trigger?api-version=$MaestroApiVersion"
Invoke-WebRequest -Uri $apiEndpoint -Headers $apiHeaders -Method Post | Out-Null
}
function Assign-BuildToChannel([int]$BuildId, [int]$ChannelId) { function Assign-BuildToChannel([int]$BuildId, [int]$ChannelId) {
Validate-MaestroVars Validate-MaestroVars
@ -67,24 +59,32 @@ function Assign-BuildToChannel([int]$BuildId, [int]$ChannelId) {
Invoke-WebRequest -Method Post -Uri $apiEndpoint -Headers $apiHeaders | Out-Null Invoke-WebRequest -Method Post -Uri $apiEndpoint -Headers $apiHeaders | Out-Null
} }
function Trigger-Subscription([string]$SubscriptionId) {
Validate-MaestroVars
$apiHeaders = Create-MaestroApiRequestHeaders -AuthToken $MaestroApiAccessToken
$apiEndpoint = "$MaestroApiEndPoint/api/subscriptions/$SubscriptionId/trigger?api-version=$MaestroApiVersion"
Invoke-WebRequest -Uri $apiEndpoint -Headers $apiHeaders -Method Post | Out-Null
}
function Validate-MaestroVars { function Validate-MaestroVars {
try { try {
Get-Variable MaestroApiEndPoint -Scope Global | Out-Null Get-Variable MaestroApiEndPoint -Scope Global | Out-Null
Get-Variable MaestroApiVersion -Scope Global | Out-Null Get-Variable MaestroApiVersion -Scope Global | Out-Null
Get-Variable MaestroApiAccessToken -Scope Global | Out-Null Get-Variable MaestroApiAccessToken -Scope Global | Out-Null
if (!($MaestroApiEndPoint -Match "^http[s]?://maestro-(int|prod).westus2.cloudapp.azure.com$")) { if (!($MaestroApiEndPoint -Match '^http[s]?://maestro-(int|prod).westus2.cloudapp.azure.com$')) {
Write-PipelineTaskError "MaestroApiEndPoint is not a valid Maestro URL. '$MaestroApiEndPoint'" Write-PipelineTelemetryError -Category 'MaestroVars' -Message "MaestroApiEndPoint is not a valid Maestro URL. '$MaestroApiEndPoint'"
ExitWithExitCode 1 ExitWithExitCode 1
} }
if (!($MaestroApiVersion -Match "^[0-9]{4}-[0-9]{2}-[0-9]{2}$")) { if (!($MaestroApiVersion -Match '^[0-9]{4}-[0-9]{2}-[0-9]{2}$')) {
Write-PipelineTaskError "MaestroApiVersion does not match a version string in the format yyyy-MM-DD. '$MaestroApiVersion'" Write-PipelineTelemetryError -Category 'MaestroVars' -Message "MaestroApiVersion does not match a version string in the format yyyy-MM-DD. '$MaestroApiVersion'"
ExitWithExitCode 1 ExitWithExitCode 1
} }
} }
catch { catch {
Write-PipelineTaskError "Error: Variables `MaestroApiEndPoint`, `MaestroApiVersion` and `MaestroApiAccessToken` are required while using this script." Write-PipelineTelemetryError -Category 'MaestroVars' -Message 'Error: Variables `MaestroApiEndPoint`, `MaestroApiVersion` and `MaestroApiAccessToken` are required while using this script.'
Write-Host $_ Write-Host $_
ExitWithExitCode 1 ExitWithExitCode 1
} }

16
eng/common/post-build/promote-build.ps1
View file

@ -2,18 +2,18 @@ param(
[Parameter(Mandatory=$true)][int] $BuildId, [Parameter(Mandatory=$true)][int] $BuildId,
[Parameter(Mandatory=$true)][int] $ChannelId, [Parameter(Mandatory=$true)][int] $ChannelId,
[Parameter(Mandatory=$true)][string] $MaestroApiAccessToken, [Parameter(Mandatory=$true)][string] $MaestroApiAccessToken,
[Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = "https://maestro-prod.westus2.cloudapp.azure.com", [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro-prod.westus2.cloudapp.azure.com',
[Parameter(Mandatory=$false)][string] $MaestroApiVersion = "2019-01-16" [Parameter(Mandatory=$false)][string] $MaestroApiVersion = '2019-01-16'
) )
try {
. $PSScriptRoot\post-build-utils.ps1 . $PSScriptRoot\post-build-utils.ps1
try {
# Check that the channel we are going to promote the build to exist # Check that the channel we are going to promote the build to exist
$channelInfo = Get-MaestroChannel -ChannelId $ChannelId $channelInfo = Get-MaestroChannel -ChannelId $ChannelId
if (!$channelInfo) { if (!$channelInfo) {
Write-Host "Channel with BAR ID $ChannelId was not found in BAR!" Write-PipelineTelemetryCategory -Category 'PromoteBuild' -Message "Channel with BAR ID $ChannelId was not found in BAR!"
ExitWithExitCode 1 ExitWithExitCode 1
} }
@ -21,7 +21,7 @@ try {
$buildInfo = Get-MaestroBuild -BuildId $BuildId $buildInfo = Get-MaestroBuild -BuildId $BuildId
if (!$buildInfo) { if (!$buildInfo) {
Write-Host "Build with BAR ID $BuildId was not found in BAR!" Write-PipelineTelemetryError -Category 'PromoteBuild' -Message "Build with BAR ID $BuildId was not found in BAR!"
ExitWithExitCode 1 ExitWithExitCode 1
} }
@ -39,10 +39,10 @@ try {
Assign-BuildToChannel -BuildId $BuildId -ChannelId $ChannelId Assign-BuildToChannel -BuildId $BuildId -ChannelId $ChannelId
Write-Host "done." Write-Host 'done.'
} }
catch { catch {
Write-Host "There was an error while trying to promote build '$BuildId' to channel '$ChannelId'"
Write-Host $_ Write-Host $_
Write-Host $_.ScriptStackTrace Write-PipelineTelemetryError -Category 'PromoteBuild' -Message "There was an error while trying to promote build '$BuildId' to channel '$ChannelId'"
ExitWithExitCode 1
} }

42
eng/common/post-build/sourcelink-validation.ps1
View file

@ -34,9 +34,9 @@ $ValidatePackage = {
# Extensions for which we'll look for SourceLink information # Extensions for which we'll look for SourceLink information
# For now we'll only care about Portable & Embedded PDBs # For now we'll only care about Portable & Embedded PDBs
$RelevantExtensions = @(".dll", ".exe", ".pdb") $RelevantExtensions = @('.dll', '.exe', '.pdb')
Write-Host -NoNewLine "Validating" ([System.IO.Path]::GetFileName($PackagePath)) "... " Write-Host -NoNewLine 'Validating ' ([System.IO.Path]::GetFileName($PackagePath)) '...'
$PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath)
$ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageId $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageId
@ -58,7 +58,7 @@ $ValidatePackage = {
$TargetFile = Join-Path -Path $ExtractPath -ChildPath $FakeName $TargetFile = Join-Path -Path $ExtractPath -ChildPath $FakeName
# We ignore resource DLLs # We ignore resource DLLs
if ($FileName.EndsWith(".resources.dll")) { if ($FileName.EndsWith('.resources.dll')) {
return return
} }
@ -96,7 +96,7 @@ $ValidatePackage = {
$Uri = $Link -as [System.URI] $Uri = $Link -as [System.URI]
# Only GitHub links are valid # Only GitHub links are valid
if ($Uri.AbsoluteURI -ne $null -and ($Uri.Host -match "github" -or $Uri.Host -match "githubusercontent")) { if ($Uri.AbsoluteURI -ne $null -and ($Uri.Host -match 'github' -or $Uri.Host -match 'githubusercontent')) {
$Status = (Invoke-WebRequest -Uri $Link -UseBasicParsing -Method HEAD -TimeoutSec 5).StatusCode $Status = (Invoke-WebRequest -Uri $Link -UseBasicParsing -Method HEAD -TimeoutSec 5).StatusCode
} }
else { else {
@ -143,19 +143,19 @@ $ValidatePackage = {
} }
if ($FailedFiles -eq 0) { if ($FailedFiles -eq 0) {
Write-Host "Passed." Write-Host 'Passed.'
return 0 return 0
} }
else { else {
Write-Host "$PackagePath has broken SourceLink links." Write-PipelineTelemetryError -Category 'SourceLink' -Message "$PackagePath has broken SourceLink links."
return 1 return 1
} }
} }
function ValidateSourceLinkLinks { function ValidateSourceLinkLinks {
if ($GHRepoName -ne "" -and !($GHRepoName -Match "^[^\s\/]+/[^\s\/]+$")) { if ($GHRepoName -ne '' -and !($GHRepoName -Match '^[^\s\/]+/[^\s\/]+$')) {
if (!($GHRepoName -Match "^[^\s-]+-[^\s]+$")) { if (!($GHRepoName -Match '^[^\s-]+-[^\s]+$')) {
Write-PipelineTaskError "GHRepoName should be in the format <org>/<repo> or <org>-<repo>. '$GHRepoName'" Write-PipelineTelemetryError -Category 'SourceLink' -Message "GHRepoName should be in the format <org>/<repo> or <org>-<repo>. '$GHRepoName'"
ExitWithExitCode 1 ExitWithExitCode 1
} }
else { else {
@ -163,14 +163,14 @@ function ValidateSourceLinkLinks {
} }
} }
if ($GHCommit -ne "" -and !($GHCommit -Match "^[0-9a-fA-F]{40}$")) { if ($GHCommit -ne '' -and !($GHCommit -Match '^[0-9a-fA-F]{40}$')) {
Write-PipelineTaskError "GHCommit should be a 40 chars hexadecimal string. '$GHCommit'" Write-PipelineTelemetryError -Category 'SourceLink' -Message "GHCommit should be a 40 chars hexadecimal string. '$GHCommit'"
ExitWithExitCode 1 ExitWithExitCode 1
} }
if ($GHRepoName -ne "" -and $GHCommit -ne "") { if ($GHRepoName -ne '' -and $GHCommit -ne '') {
$RepoTreeURL = -Join("http://api.github.com/repos/", $GHRepoName, "/git/trees/", $GHCommit, "?recursive=1") $RepoTreeURL = -Join('http://api.github.com/repos/', $GHRepoName, '/git/trees/', $GHCommit, '?recursive=1')
$CodeExtensions = @(".cs", ".vb", ".fs", ".fsi", ".fsx", ".fsscript") $CodeExtensions = @('.cs', '.vb', '.fs', '.fsi', '.fsx', '.fsscript')
try { try {
# Retrieve the list of files in the repo at that particular commit point and store them in the RepoFiles hash # Retrieve the list of files in the repo at that particular commit point and store them in the RepoFiles hash
@ -188,8 +188,8 @@ function ValidateSourceLinkLinks {
Write-Host "Problems downloading the list of files from the repo. Url used: $RepoTreeURL . Execution will proceed without caching." Write-Host "Problems downloading the list of files from the repo. Url used: $RepoTreeURL . Execution will proceed without caching."
} }
} }
elseif ($GHRepoName -ne "" -or $GHCommit -ne "") { elseif ($GHRepoName -ne '' -or $GHCommit -ne '') {
Write-Host "For using the http caching mechanism both GHRepoName and GHCommit should be informed." Write-Host 'For using the http caching mechanism both GHRepoName and GHCommit should be informed.'
} }
if (Test-Path $ExtractPath) { if (Test-Path $ExtractPath) {
@ -217,18 +217,18 @@ function ValidateSourceLinkLinks {
$ValidationFailures = 0 $ValidationFailures = 0
foreach ($Job in @(Get-Job)) { foreach ($Job in @(Get-Job)) {
$jobResult = Wait-Job -Id $Job.Id | Receive-Job $jobResult = Wait-Job -Id $Job.Id | Receive-Job
if ($jobResult -ne "0") { if ($jobResult -ne '0') {
$ValidationFailures++ $ValidationFailures++
} }
} }
if ($ValidationFailures -gt 0) { if ($ValidationFailures -gt 0) {
Write-PipelineTaskError " $ValidationFailures package(s) failed validation." Write-PipelineTelemetryError -Category 'SourceLink' -Message "$ValidationFailures package(s) failed validation."
ExitWithExitCode 1 ExitWithExitCode 1
} }
} }
function InstallSourcelinkCli { function InstallSourcelinkCli {
$sourcelinkCliPackageName = "sourcelink" $sourcelinkCliPackageName = 'sourcelink'
$dotnetRoot = InitializeDotNetCli -install:$true $dotnetRoot = InitializeDotNetCli -install:$true
$dotnet = "$dotnetRoot\dotnet.exe" $dotnet = "$dotnetRoot\dotnet.exe"
@ -239,7 +239,7 @@ function InstallSourcelinkCli {
} }
else { else {
Write-Host "Installing SourceLink CLI version $sourcelinkCliVersion..." Write-Host "Installing SourceLink CLI version $sourcelinkCliVersion..."
Write-Host "You may need to restart your command window if this is the first dotnet tool you have installed." Write-Host 'You may need to restart your command window if this is the first dotnet tool you have installed.'
& "$dotnet" tool install $sourcelinkCliPackageName --version $sourcelinkCliVersion --verbosity "minimal" --global & "$dotnet" tool install $sourcelinkCliPackageName --version $sourcelinkCliVersion --verbosity "minimal" --global
} }
} }
@ -250,8 +250,8 @@ try {
ValidateSourceLinkLinks ValidateSourceLinkLinks
} }
catch { catch {
Write-Host $_
Write-Host $_.Exception Write-Host $_.Exception
Write-Host $_.ScriptStackTrace Write-Host $_.ScriptStackTrace
Write-PipelineTelemetryError -Category 'SourceLink' -Message $_
ExitWithExitCode 1 ExitWithExitCode 1
} }

59
eng/common/post-build/symbols-validation.ps1
View file

@ -4,10 +4,6 @@ param(
[Parameter(Mandatory=$true)][string] $DotnetSymbolVersion # Version of dotnet symbol to use [Parameter(Mandatory=$true)][string] $DotnetSymbolVersion # Version of dotnet symbol to use
) )
. $PSScriptRoot\post-build-utils.ps1
Add-Type -AssemblyName System.IO.Compression.FileSystem
function FirstMatchingSymbolDescriptionOrDefault { function FirstMatchingSymbolDescriptionOrDefault {
param( param(
[string] $FullPath, # Full path to the module that has to be checked [string] $FullPath, # Full path to the module that has to be checked
@ -23,19 +19,19 @@ function FirstMatchingSymbolDescriptionOrDefault {
# checking and which type of file was uploaded. # checking and which type of file was uploaded.
# The file itself is returned # The file itself is returned
$SymbolPath = $SymbolsPath + "\" + $FileName $SymbolPath = $SymbolsPath + '\' + $FileName
# PDB file for the module # PDB file for the module
$PdbPath = $SymbolPath.Replace($Extension, ".pdb") $PdbPath = $SymbolPath.Replace($Extension, '.pdb')
# PDB file for R2R module (created by crossgen) # PDB file for R2R module (created by crossgen)
$NGenPdb = $SymbolPath.Replace($Extension, ".ni.pdb") $NGenPdb = $SymbolPath.Replace($Extension, '.ni.pdb')
# DBG file for a .so library # DBG file for a .so library
$SODbg = $SymbolPath.Replace($Extension, ".so.dbg") $SODbg = $SymbolPath.Replace($Extension, '.so.dbg')
# DWARF file for a .dylib # DWARF file for a .dylib
$DylibDwarf = $SymbolPath.Replace($Extension, ".dylib.dwarf") $DylibDwarf = $SymbolPath.Replace($Extension, '.dylib.dwarf')
$dotnetSymbolExe = "$env:USERPROFILE\.dotnet\tools" $dotnetSymbolExe = "$env:USERPROFILE\.dotnet\tools"
$dotnetSymbolExe = Resolve-Path "$dotnetSymbolExe\dotnet-symbol.exe" $dotnetSymbolExe = Resolve-Path "$dotnetSymbolExe\dotnet-symbol.exe"
@ -43,19 +39,19 @@ function FirstMatchingSymbolDescriptionOrDefault {
& $dotnetSymbolExe --symbols --modules --windows-pdbs $TargetServerParam $FullPath -o $SymbolsPath | Out-Null & $dotnetSymbolExe --symbols --modules --windows-pdbs $TargetServerParam $FullPath -o $SymbolsPath | Out-Null
if (Test-Path $PdbPath) { if (Test-Path $PdbPath) {
return "PDB" return 'PDB'
} }
elseif (Test-Path $NGenPdb) { elseif (Test-Path $NGenPdb) {
return "NGen PDB" return 'NGen PDB'
} }
elseif (Test-Path $SODbg) { elseif (Test-Path $SODbg) {
return "DBG for SO" return 'DBG for SO'
} }
elseif (Test-Path $DylibDwarf) { elseif (Test-Path $DylibDwarf) {
return "Dwarf for Dylib" return 'Dwarf for Dylib'
} }
elseif (Test-Path $SymbolPath) { elseif (Test-Path $SymbolPath) {
return "Module" return 'Module'
} }
else { else {
return $null return $null
@ -74,7 +70,7 @@ function CountMissingSymbols {
} }
# Extensions for which we'll look for symbols # Extensions for which we'll look for symbols
$RelevantExtensions = @(".dll", ".exe", ".so", ".dylib") $RelevantExtensions = @('.dll', '.exe', '.so', '.dylib')
# How many files are missing symbol information # How many files are missing symbol information
$MissingSymbols = 0 $MissingSymbols = 0
@ -82,38 +78,38 @@ function CountMissingSymbols {
$PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath)
$PackageGuid = New-Guid $PackageGuid = New-Guid
$ExtractPath = Join-Path -Path $ExtractPath -ChildPath $PackageGuid $ExtractPath = Join-Path -Path $ExtractPath -ChildPath $PackageGuid
$SymbolsPath = Join-Path -Path $ExtractPath -ChildPath "Symbols" $SymbolsPath = Join-Path -Path $ExtractPath -ChildPath 'Symbols'
[System.IO.Compression.ZipFile]::ExtractToDirectory($PackagePath, $ExtractPath) [System.IO.Compression.ZipFile]::ExtractToDirectory($PackagePath, $ExtractPath)
Get-ChildItem -Recurse $ExtractPath | Get-ChildItem -Recurse $ExtractPath |
Where-Object {$RelevantExtensions -contains $_.Extension} | Where-Object {$RelevantExtensions -contains $_.Extension} |
ForEach-Object { ForEach-Object {
if ($_.FullName -Match "\\ref\\") { if ($_.FullName -Match '\\ref\\') {
Write-Host "`t Ignoring reference assembly file " $_.FullName Write-Host "`t Ignoring reference assembly file " $_.FullName
return return
} }
$SymbolsOnMSDL = FirstMatchingSymbolDescriptionOrDefault $_.FullName "--microsoft-symbol-server" $SymbolsPath $SymbolsOnMSDL = FirstMatchingSymbolDescriptionOrDefault $_.FullName '--microsoft-symbol-server' $SymbolsPath
$SymbolsOnSymWeb = FirstMatchingSymbolDescriptionOrDefault $_.FullName "--internal-server" $SymbolsPath $SymbolsOnSymWeb = FirstMatchingSymbolDescriptionOrDefault $_.FullName '--internal-server' $SymbolsPath
Write-Host -NoNewLine "`t Checking file " $_.FullName "... " Write-Host -NoNewLine "`t Checking file " $_.FullName "... "
if ($SymbolsOnMSDL -ne $null -and $SymbolsOnSymWeb -ne $null) { if ($SymbolsOnMSDL -ne $null -and $SymbolsOnSymWeb -ne $null) {
Write-Host "Symbols found on MSDL (" $SymbolsOnMSDL ") and SymWeb (" $SymbolsOnSymWeb ")" Write-Host "Symbols found on MSDL ($SymbolsOnMSDL) and SymWeb ($SymbolsOnSymWeb)"
} }
else { else {
$MissingSymbols++ $MissingSymbols++
if ($SymbolsOnMSDL -eq $null -and $SymbolsOnSymWeb -eq $null) { if ($SymbolsOnMSDL -eq $null -and $SymbolsOnSymWeb -eq $null) {
Write-Host "No symbols found on MSDL or SymWeb!" Write-Host 'No symbols found on MSDL or SymWeb!'
} }
else { else {
if ($SymbolsOnMSDL -eq $null) { if ($SymbolsOnMSDL -eq $null) {
Write-Host "No symbols found on MSDL!" Write-Host 'No symbols found on MSDL!'
} }
else { else {
Write-Host "No symbols found on SymWeb!" Write-Host 'No symbols found on SymWeb!'
} }
} }
} }
@ -136,12 +132,12 @@ function CheckSymbolsAvailable {
# These packages from Arcade-Services include some native libraries that # These packages from Arcade-Services include some native libraries that
# our current symbol uploader can't handle. Below is a workaround until # our current symbol uploader can't handle. Below is a workaround until
# we get issue: https://github.com/dotnet/arcade/issues/2457 sorted. # we get issue: https://github.com/dotnet/arcade/issues/2457 sorted.
if ($FileName -Match "Microsoft\.DotNet\.Darc\.") { if ($FileName -Match 'Microsoft\.DotNet\.Darc\.') {
Write-Host "Ignoring Arcade-services file: $FileName" Write-Host "Ignoring Arcade-services file: $FileName"
Write-Host Write-Host
return return
} }
elseif ($FileName -Match "Microsoft\.DotNet\.Maestro\.Tasks\.") { elseif ($FileName -Match 'Microsoft\.DotNet\.Maestro\.Tasks\.') {
Write-Host "Ignoring Arcade-services file: $FileName" Write-Host "Ignoring Arcade-services file: $FileName"
Write-Host Write-Host
return return
@ -151,7 +147,7 @@ function CheckSymbolsAvailable {
$Status = CountMissingSymbols "$InputPath\$FileName" $Status = CountMissingSymbols "$InputPath\$FileName"
if ($Status -ne 0) { if ($Status -ne 0) {
Write-PipelineTaskError "Missing symbols for $Status modules in the package $FileName" Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Missing symbols for $Status modules in the package $FileName"
ExitWithExitCode $exitCode ExitWithExitCode $exitCode
} }
@ -160,7 +156,7 @@ function CheckSymbolsAvailable {
} }
function InstallDotnetSymbol { function InstallDotnetSymbol {
$dotnetSymbolPackageName = "dotnet-symbol" $dotnetSymbolPackageName = 'dotnet-symbol'
$dotnetRoot = InitializeDotNetCli -install:$true $dotnetRoot = InitializeDotNetCli -install:$true
$dotnet = "$dotnetRoot\dotnet.exe" $dotnet = "$dotnetRoot\dotnet.exe"
@ -171,19 +167,22 @@ function InstallDotnetSymbol {
} }
else { else {
Write-Host "Installing dotnet-symbol version $dotnetSymbolVersion..." Write-Host "Installing dotnet-symbol version $dotnetSymbolVersion..."
Write-Host "You may need to restart your command window if this is the first dotnet tool you have installed." Write-Host 'You may need to restart your command window if this is the first dotnet tool you have installed.'
& "$dotnet" tool install $dotnetSymbolPackageName --version $dotnetSymbolVersion --verbosity "minimal" --global & "$dotnet" tool install $dotnetSymbolPackageName --version $dotnetSymbolVersion --verbosity "minimal" --global
} }
} }
try { try {
. $PSScriptRoot\post-build-utils.ps1
Add-Type -AssemblyName System.IO.Compression.FileSystem
InstallDotnetSymbol InstallDotnetSymbol
CheckSymbolsAvailable CheckSymbolsAvailable
} }
catch { catch {
Write-Host $_
Write-Host $_.Exception
Write-Host $_.ScriptStackTrace Write-Host $_.ScriptStackTrace
Write-PipelineTelemetryError -Category 'CheckSymbols' -Message $_
ExitWithExitCode 1 ExitWithExitCode 1
} }

21
eng/common/post-build/trigger-subscriptions.ps1
View file

@ -2,10 +2,11 @@ param(
[Parameter(Mandatory=$true)][string] $SourceRepo, [Parameter(Mandatory=$true)][string] $SourceRepo,
[Parameter(Mandatory=$true)][int] $ChannelId, [Parameter(Mandatory=$true)][int] $ChannelId,
[Parameter(Mandatory=$true)][string] $MaestroApiAccessToken, [Parameter(Mandatory=$true)][string] $MaestroApiAccessToken,
[Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = "https://maestro-prod.westus2.cloudapp.azure.com", [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro-prod.westus2.cloudapp.azure.com',
[Parameter(Mandatory=$false)][string] $MaestroApiVersion = "2019-01-16" [Parameter(Mandatory=$false)][string] $MaestroApiVersion = '2019-01-16'
) )
try {
. $PSScriptRoot\post-build-utils.ps1 . $PSScriptRoot\post-build-utils.ps1
# Get all the $SourceRepo subscriptions # Get all the $SourceRepo subscriptions
@ -13,7 +14,7 @@ $normalizedSourceRepo = $SourceRepo.Replace('dnceng@', '')
$subscriptions = Get-MaestroSubscriptions -SourceRepository $normalizedSourceRepo -ChannelId $ChannelId $subscriptions = Get-MaestroSubscriptions -SourceRepository $normalizedSourceRepo -ChannelId $ChannelId
if (!$subscriptions) { if (!$subscriptions) {
Write-Host "No subscriptions found for source repo '$normalizedSourceRepo' in channel '$ChannelId'" Write-PipelineTelemetryError -Category 'TriggerSubscriptions' -Message "No subscriptions found for source repo '$normalizedSourceRepo' in channel '$ChannelId'"
ExitWithExitCode 0 ExitWithExitCode 0
} }
@ -23,7 +24,7 @@ $failedTriggeredSubscription = $false
# Get all enabled subscriptions that need dependency flow on 'everyBuild' # Get all enabled subscriptions that need dependency flow on 'everyBuild'
foreach ($subscription in $subscriptions) { foreach ($subscription in $subscriptions) {
if ($subscription.enabled -and $subscription.policy.updateFrequency -like 'everyBuild' -and $subscription.channel.id -eq $ChannelId) { if ($subscription.enabled -and $subscription.policy.updateFrequency -like 'everyBuild' -and $subscription.channel.id -eq $ChannelId) {
Write-Host "Should trigger this subscription: $subscription.id" Write-Host "Should trigger this subscription: ${$subscription.id}"
[void]$subscriptionsToTrigger.Add($subscription.id) [void]$subscriptionsToTrigger.Add($subscription.id)
} }
} }
@ -34,7 +35,7 @@ foreach ($subscriptionToTrigger in $subscriptionsToTrigger) {
Trigger-Subscription -SubscriptionId $subscriptionToTrigger Trigger-Subscription -SubscriptionId $subscriptionToTrigger
Write-Host "done." Write-Host 'done.'
} }
catch catch
{ {
@ -49,9 +50,15 @@ if ($subscriptionsToTrigger.Count -eq 0) {
Write-Host "No subscription matched source repo '$normalizedSourceRepo' and channel ID '$ChannelId'." Write-Host "No subscription matched source repo '$normalizedSourceRepo' and channel ID '$ChannelId'."
} }
elseif ($failedTriggeredSubscription) { elseif ($failedTriggeredSubscription) {
Write-Host "At least one subscription failed to be triggered..." Write-PipelineTelemetryError -Category 'TriggerSubscriptions' -Message 'At least one subscription failed to be triggered...'
ExitWithExitCode 1 ExitWithExitCode 1
} }
else { else {
Write-Host "All subscriptions were triggered successfully!" Write-Host 'All subscriptions were triggered successfully!'
}
}
catch {
Write-Host $_.ScriptStackTrace
Write-PipelineTelemetryError -Category 'TriggerSubscriptions' -Message $_
ExitWithExitCode 1
} }

19
eng/common/sdk-task.ps1
View file

@ -1,8 +1,8 @@
[CmdletBinding(PositionalBinding=$false)] [CmdletBinding(PositionalBinding=$false)]
Param( Param(
[string] $configuration = "Debug", [string] $configuration = 'Debug',
[string] $task, [string] $task,
[string] $verbosity = "minimal", [string] $verbosity = 'minimal',
[string] $msbuildEngine = $null, [string] $msbuildEngine = $null,
[switch] $restore, [switch] $restore,
[switch] $prepareMachine, [switch] $prepareMachine,
@ -32,7 +32,7 @@ function Print-Usage() {
} }
function Build([string]$target) { function Build([string]$target) {
$logSuffix = if ($target -eq "Execute") { "" } else { ".$target" } $logSuffix = if ($target -eq 'Execute') { '' } else { ".$target" }
$log = Join-Path $LogDir "$task$logSuffix.binlog" $log = Join-Path $LogDir "$task$logSuffix.binlog"
$outputPath = Join-Path $ToolsetDir "$task\\" $outputPath = Join-Path $ToolsetDir "$task\\"
@ -46,33 +46,32 @@ function Build([string]$target) {
} }
try { try {
if ($help -or (($null -ne $properties) -and ($properties.Contains("/help") -or $properties.Contains("/?")))) { if ($help -or (($null -ne $properties) -and ($properties.Contains('/help') -or $properties.Contains('/?')))) {
Print-Usage Print-Usage
exit 0 exit 0
} }
if ($task -eq "") { if ($task -eq "") {
Write-Host "Missing required parameter '-task <value>'" -ForegroundColor Red Write-PipelineTelemetryError -Category 'Build' -Message "Missing required parameter '-task <value>'" -ForegroundColor Red
Print-Usage Print-Usage
ExitWithExitCode 1 ExitWithExitCode 1
} }
$taskProject = GetSdkTaskProject $task $taskProject = GetSdkTaskProject $task
if (!(Test-Path $taskProject)) { if (!(Test-Path $taskProject)) {
Write-Host "Unknown task: $task" -ForegroundColor Red Write-PipelineTelemetryError -Category 'Build' -Message "Unknown task: $task" -ForegroundColor Red
ExitWithExitCode 1 ExitWithExitCode 1
} }
if ($restore) { if ($restore) {
Build "Restore" Build 'Restore'
} }
Build "Execute" Build 'Execute'
} }
catch { catch {
Write-Host $_
Write-Host $_.Exception
Write-Host $_.ScriptStackTrace Write-Host $_.ScriptStackTrace
Write-PipelineTelemetryError -Category 'Build' -Message $_
ExitWithExitCode 1 ExitWithExitCode 1
} }

48
eng/common/sdl/execute-all-sdl-tools.ps1
View file

@ -5,7 +5,7 @@ Param(
[string] $Repository=$env:BUILD_REPOSITORY_NAME, # Required: the name of the repository (e.g. dotnet/arcade) [string] $Repository=$env:BUILD_REPOSITORY_NAME, # Required: the name of the repository (e.g. dotnet/arcade)
[string] $BranchName=$env:BUILD_SOURCEBRANCH, # Optional: name of branch or version of gdn settings; defaults to master [string] $BranchName=$env:BUILD_SOURCEBRANCH, # Optional: name of branch or version of gdn settings; defaults to master
[string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY, # Required: the directory where source files are located [string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY, # Required: the directory where source files are located
[string] $ArtifactsDirectory = (Join-Path $env:BUILD_ARTIFACTSTAGINGDIRECTORY ("artifacts")), # Required: the directory where build artifacts are located [string] $ArtifactsDirectory = (Join-Path $env:BUILD_ARTIFACTSTAGINGDIRECTORY ('artifacts')), # Required: the directory where build artifacts are located
[string] $AzureDevOpsAccessToken, # Required: access token for dnceng; should be provided via KeyVault [string] $AzureDevOpsAccessToken, # Required: access token for dnceng; should be provided via KeyVault
[string[]] $SourceToolsList, # Optional: list of SDL tools to run on source code [string[]] $SourceToolsList, # Optional: list of SDL tools to run on source code
[string[]] $ArtifactToolsList, # Optional: list of SDL tools to run on built artifacts [string[]] $ArtifactToolsList, # Optional: list of SDL tools to run on built artifacts
@ -22,15 +22,19 @@ Param(
[string] $TsaCodebaseAdmin, # Optional: only needed if TsaOnboard is true; the aliases which are admins of the TSA codebase (e.g. DOMAIN\alias); TSA is the automated framework used to upload test results as bugs. [string] $TsaCodebaseAdmin, # Optional: only needed if TsaOnboard is true; the aliases which are admins of the TSA codebase (e.g. DOMAIN\alias); TSA is the automated framework used to upload test results as bugs.
[string] $TsaBugAreaPath, # Optional: only needed if TsaOnboard is true; the area path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs. [string] $TsaBugAreaPath, # Optional: only needed if TsaOnboard is true; the area path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs.
[string] $TsaIterationPath, # Optional: only needed if TsaOnboard is true; the iteration path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs. [string] $TsaIterationPath, # Optional: only needed if TsaOnboard is true; the iteration path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs.
[string] $GuardianLoggerLevel="Standard", # Optional: the logger level for the Guardian CLI; options are Trace, Verbose, Standard, Warning, and Error [string] $GuardianLoggerLevel='Standard', # Optional: the logger level for the Guardian CLI; options are Trace, Verbose, Standard, Warning, and Error
[string[]] $CrScanAdditionalRunConfigParams, # Optional: Additional Params to custom build a CredScan run config in the format @("xyz:abc","sdf:1") [string[]] $CrScanAdditionalRunConfigParams, # Optional: Additional Params to custom build a CredScan run config in the format @("xyz:abc","sdf:1")
[string[]] $PoliCheckAdditionalRunConfigParams # Optional: Additional Params to custom build a Policheck run config in the format @("xyz:abc","sdf:1") [string[]] $PoliCheckAdditionalRunConfigParams # Optional: Additional Params to custom build a Policheck run config in the format @("xyz:abc","sdf:1")
) )
$ErrorActionPreference = "Stop" try {
$ErrorActionPreference = 'Stop'
Set-StrictMode -Version 2.0 Set-StrictMode -Version 2.0
$disableConfigureToolsetImport = $true
$LASTEXITCODE = 0 $LASTEXITCODE = 0
. $PSScriptRoot\..\tools.ps1
#Replace repo names to the format of org/repo #Replace repo names to the format of org/repo
if (!($Repository.contains('/'))) { if (!($Repository.contains('/'))) {
$RepoName = $Repository -replace '(.*?)-(.*)', '$1/$2'; $RepoName = $Repository -replace '(.*?)-(.*)', '$1/$2';
@ -40,7 +44,7 @@ else{
} }
if ($GuardianPackageName) { if ($GuardianPackageName) {
$guardianCliLocation = Join-Path $NugetPackageDirectory (Join-Path $GuardianPackageName (Join-Path "tools" "guardian.cmd")) $guardianCliLocation = Join-Path $NugetPackageDirectory (Join-Path $GuardianPackageName (Join-Path 'tools' 'guardian.cmd'))
} else { } else {
$guardianCliLocation = $GuardianCliLocation $guardianCliLocation = $GuardianCliLocation
} }
@ -50,36 +54,36 @@ $ValidPath = Test-Path $guardianCliLocation
if ($ValidPath -eq $False) if ($ValidPath -eq $False)
{ {
Write-Host "Invalid Guardian CLI Location." Write-PipelineTelemetryError -Force -Category 'Sdl' -Message 'Invalid Guardian CLI Location.'
exit 1 ExitWithExitCode 1
} }
& $(Join-Path $PSScriptRoot "init-sdl.ps1") -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $workingDirectory -AzureDevOpsAccessToken $AzureDevOpsAccessToken -GuardianLoggerLevel $GuardianLoggerLevel & $(Join-Path $PSScriptRoot 'init-sdl.ps1') -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $workingDirectory -AzureDevOpsAccessToken $AzureDevOpsAccessToken -GuardianLoggerLevel $GuardianLoggerLevel
$gdnFolder = Join-Path $workingDirectory ".gdn" $gdnFolder = Join-Path $workingDirectory '.gdn'
if ($TsaOnboard) { if ($TsaOnboard) {
if ($TsaCodebaseName -and $TsaNotificationEmail -and $TsaCodebaseAdmin -and $TsaBugAreaPath) { if ($TsaCodebaseName -and $TsaNotificationEmail -and $TsaCodebaseAdmin -and $TsaBugAreaPath) {
Write-Host "$guardianCliLocation tsa-onboard --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel" Write-Host "$guardianCliLocation tsa-onboard --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel"
& $guardianCliLocation tsa-onboard --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel & $guardianCliLocation tsa-onboard --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel
if ($LASTEXITCODE -ne 0) { if ($LASTEXITCODE -ne 0) {
Write-Host "Guardian tsa-onboard failed with exit code $LASTEXITCODE." Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian tsa-onboard failed with exit code $LASTEXITCODE."
exit $LASTEXITCODE ExitWithExitCode $LASTEXITCODE
} }
} else { } else {
Write-Host "Could not onboard to TSA -- not all required values ($$TsaCodebaseName, $$TsaNotificationEmail, $$TsaCodebaseAdmin, $$TsaBugAreaPath) were specified." Write-PipelineTelemetryError -Force -Category 'Sdl' -Message 'Could not onboard to TSA -- not all required values ($TsaCodebaseName, $TsaNotificationEmail, $TsaCodebaseAdmin, $TsaBugAreaPath) were specified.'
exit 1 ExitWithExitCode 1
} }
} }
if ($ArtifactToolsList -and $ArtifactToolsList.Count -gt 0) { if ($ArtifactToolsList -and $ArtifactToolsList.Count -gt 0) {
& $(Join-Path $PSScriptRoot "run-sdl.ps1") -GuardianCliLocation $guardianCliLocation -WorkingDirectory $workingDirectory -TargetDirectory $ArtifactsDirectory -GdnFolder $gdnFolder -ToolsList $ArtifactToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams & $(Join-Path $PSScriptRoot 'run-sdl.ps1') -GuardianCliLocation $guardianCliLocation -WorkingDirectory $workingDirectory -TargetDirectory $ArtifactsDirectory -GdnFolder $gdnFolder -ToolsList $ArtifactToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams
} }
if ($SourceToolsList -and $SourceToolsList.Count -gt 0) { if ($SourceToolsList -and $SourceToolsList.Count -gt 0) {
& $(Join-Path $PSScriptRoot "run-sdl.ps1") -GuardianCliLocation $guardianCliLocation -WorkingDirectory $workingDirectory -TargetDirectory $SourceDirectory -GdnFolder $gdnFolder -ToolsList $SourceToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams & $(Join-Path $PSScriptRoot 'run-sdl.ps1') -GuardianCliLocation $guardianCliLocation -WorkingDirectory $workingDirectory -TargetDirectory $SourceDirectory -GdnFolder $gdnFolder -ToolsList $SourceToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams
} }
if ($UpdateBaseline) { if ($UpdateBaseline) {
& (Join-Path $PSScriptRoot "push-gdn.ps1") -Repository $RepoName -BranchName $BranchName -GdnFolder $GdnFolder -AzureDevOpsAccessToken $AzureDevOpsAccessToken -PushReason "Update baseline" & (Join-Path $PSScriptRoot 'push-gdn.ps1') -Repository $RepoName -BranchName $BranchName -GdnFolder $GdnFolder -AzureDevOpsAccessToken $AzureDevOpsAccessToken -PushReason 'Update baseline'
} }
if ($TsaPublish) { if ($TsaPublish) {
@ -90,11 +94,17 @@ if ($TsaPublish) {
Write-Host "$guardianCliLocation tsa-publish --all-tools --repository-name `"$TsaRepositoryName`" --branch-name `"$TsaBranchName`" --build-number `"$BuildNumber`" --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel" Write-Host "$guardianCliLocation tsa-publish --all-tools --repository-name `"$TsaRepositoryName`" --branch-name `"$TsaBranchName`" --build-number `"$BuildNumber`" --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel"
& $guardianCliLocation tsa-publish --all-tools --repository-name "$TsaRepositoryName" --branch-name "$TsaBranchName" --build-number "$BuildNumber" --onboard $True --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel & $guardianCliLocation tsa-publish --all-tools --repository-name "$TsaRepositoryName" --branch-name "$TsaBranchName" --build-number "$BuildNumber" --onboard $True --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel
if ($LASTEXITCODE -ne 0) { if ($LASTEXITCODE -ne 0) {
Write-Host "Guardian tsa-publish failed with exit code $LASTEXITCODE." Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian tsa-publish failed with exit code $LASTEXITCODE."
exit $LASTEXITCODE ExitWithExitCode $LASTEXITCODE
} }
} else { } else {
Write-Host "Could not publish to TSA -- not all required values ($$TsaBranchName, $$BuildNumber) were specified." Write-PipelineTelemetryError -Force -Category 'Sdl' -Message 'Could not publish to TSA -- not all required values ($TsaBranchName, $BuildNumber) were specified.'
ExitWithExitCode 1
}
}
}
catch {
Write-Host $_.ScriptStackTrace
Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_
exit 1 exit 1
} }
}

52
eng/common/sdl/extract-artifact-packages.ps1
View file

@ -3,14 +3,32 @@ param(
[Parameter(Mandatory=$true)][string] $ExtractPath # Full path to directory where the packages will be extracted [Parameter(Mandatory=$true)][string] $ExtractPath # Full path to directory where the packages will be extracted
) )
$ErrorActionPreference = "Stop" $ErrorActionPreference = 'Stop'
Set-StrictMode -Version 2.0 Set-StrictMode -Version 2.0
# `tools.ps1` checks $ci to perform some actions. Since the post-build # `tools.ps1` checks $ci to perform some actions. Since the post-build
# scripts don't necessarily execute in the same agent that run the # scripts don't necessarily execute in the same agent that run the
# build.ps1/sh script this variable isn't automatically set. # build.ps1/sh script this variable isn't automatically set.
$ci = $true $ci = $true
$disableConfigureToolsetImport = "true" $disableConfigureToolsetImport = $true
function ExtractArtifacts {
if (!(Test-Path $InputPath)) {
Write-Host "Input Path does not exist: $InputPath"
ExitWithExitCode 0
}
$Jobs = @()
Get-ChildItem "$InputPath\*.nupkg" |
ForEach-Object {
$Jobs += Start-Job -ScriptBlock $ExtractPackage -ArgumentList $_.FullName
}
foreach ($Job in $Jobs) {
Wait-Job -Id $Job.Id | Receive-Job
}
}
try {
. $PSScriptRoot\..\tools.ps1 . $PSScriptRoot\..\tools.ps1
$ExtractPackage = { $ExtractPackage = {
@ -19,12 +37,12 @@ $ExtractPackage = {
) )
if (!(Test-Path $PackagePath)) { if (!(Test-Path $PackagePath)) {
Write-PipelineTaskError "Input file does not exist: $PackagePath" Write-PipelineTelemetryError -Category 'Build' -Message "Input file does not exist: $PackagePath"
ExitWithExitCode 1 ExitWithExitCode 1
} }
$RelevantExtensions = @(".dll", ".exe", ".pdb") $RelevantExtensions = @('.dll', '.exe', '.pdb')
Write-Host -NoNewLine "Extracting" ([System.IO.Path]::GetFileName($PackagePath)) "... " Write-Host -NoNewLine 'Extracting ' ([System.IO.Path]::GetFileName($PackagePath)) '...'
$PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath)
$ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageId $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageId
@ -45,34 +63,18 @@ $ExtractPackage = {
} }
} }
catch { catch {
Write-Host $_.ScriptStackTrace
Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_
ExitWithExitCode 1
} }
finally { finally {
$zip.Dispose() $zip.Dispose()
} }
} }
function ExtractArtifacts {
if (!(Test-Path $InputPath)) {
Write-Host "Input Path does not exist: $InputPath"
ExitWithExitCode 0
}
$Jobs = @()
Get-ChildItem "$InputPath\*.nupkg" |
ForEach-Object {
$Jobs += Start-Job -ScriptBlock $ExtractPackage -ArgumentList $_.FullName
}
foreach ($Job in $Jobs) {
Wait-Job -Id $Job.Id | Receive-Job
}
}
try {
Measure-Command { ExtractArtifacts } Measure-Command { ExtractArtifacts }
} }
catch { catch {
Write-Host $_
Write-Host $_.Exception
Write-Host $_.ScriptStackTrace Write-Host $_.ScriptStackTrace
Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_
ExitWithExitCode 1 ExitWithExitCode 1
} }

36
eng/common/sdl/init-sdl.ps1
View file

@ -1,16 +1,19 @@
Param( Param(
[string] $GuardianCliLocation, [string] $GuardianCliLocation,
[string] $Repository, [string] $Repository,
[string] $BranchName="master", [string] $BranchName='master',
[string] $WorkingDirectory, [string] $WorkingDirectory,
[string] $AzureDevOpsAccessToken, [string] $AzureDevOpsAccessToken,
[string] $GuardianLoggerLevel="Standard" [string] $GuardianLoggerLevel='Standard'
) )
$ErrorActionPreference = "Stop" $ErrorActionPreference = 'Stop'
Set-StrictMode -Version 2.0 Set-StrictMode -Version 2.0
$disableConfigureToolsetImport = $true
$LASTEXITCODE = 0 $LASTEXITCODE = 0
. $PSScriptRoot\..\tools.ps1
# Don't display the console progress UI - it's a huge perf hit # Don't display the console progress UI - it's a huge perf hit
$ProgressPreference = 'SilentlyContinue' $ProgressPreference = 'SilentlyContinue'
@ -21,11 +24,10 @@ $uri = "https://dev.azure.com/dnceng/internal/_apis/git/repositories/sdl-tool-cf
$zipFile = "$WorkingDirectory/gdn.zip" $zipFile = "$WorkingDirectory/gdn.zip"
Add-Type -AssemblyName System.IO.Compression.FileSystem Add-Type -AssemblyName System.IO.Compression.FileSystem
$gdnFolder = (Join-Path $WorkingDirectory ".gdn") $gdnFolder = (Join-Path $WorkingDirectory '.gdn')
Try try {
{
# We try to download the zip; if the request fails (e.g. the file doesn't exist), we catch it and init guardian instead # We try to download the zip; if the request fails (e.g. the file doesn't exist), we catch it and init guardian instead
Write-Host "Downloading gdn folder from internal config repostiory..." Write-Host 'Downloading gdn folder from internal config repostiory...'
Invoke-WebRequest -Headers @{ "Accept"="application/zip"; "Authorization"="Basic $encodedPat" } -Uri $uri -OutFile $zipFile Invoke-WebRequest -Headers @{ "Accept"="application/zip"; "Authorization"="Basic $encodedPat" } -Uri $uri -OutFile $zipFile
if (Test-Path $gdnFolder) { if (Test-Path $gdnFolder) {
# Remove the gdn folder if it exists (it shouldn't unless there's too much caching; this is just in case) # Remove the gdn folder if it exists (it shouldn't unless there's too much caching; this is just in case)
@ -33,19 +35,29 @@ Try
} }
[System.IO.Compression.ZipFile]::ExtractToDirectory($zipFile, $WorkingDirectory) [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFile, $WorkingDirectory)
Write-Host $gdnFolder Write-Host $gdnFolder
} Catch [System.Net.WebException] { ExitWithExitCode 0
} catch [System.Net.WebException] { } # Catch and ignore webexception
try {
# if the folder does not exist, we'll do a guardian init and push it to the remote repository # if the folder does not exist, we'll do a guardian init and push it to the remote repository
Write-Host "Initializing Guardian..." Write-Host 'Initializing Guardian...'
Write-Host "$GuardianCliLocation init --working-directory $WorkingDirectory --logger-level $GuardianLoggerLevel" Write-Host "$GuardianCliLocation init --working-directory $WorkingDirectory --logger-level $GuardianLoggerLevel"
& $GuardianCliLocation init --working-directory $WorkingDirectory --logger-level $GuardianLoggerLevel & $GuardianCliLocation init --working-directory $WorkingDirectory --logger-level $GuardianLoggerLevel
if ($LASTEXITCODE -ne 0) { if ($LASTEXITCODE -ne 0) {
Write-Error "Guardian init failed with exit code $LASTEXITCODE." Write-PipelineTelemetryError -Force -Category 'Build' -Message "Guardian init failed with exit code $LASTEXITCODE."
ExitWithExitCode $LASTEXITCODE
} }
# We create the mainbaseline so it can be edited later # We create the mainbaseline so it can be edited later
Write-Host "$GuardianCliLocation baseline --working-directory $WorkingDirectory --name mainbaseline" Write-Host "$GuardianCliLocation baseline --working-directory $WorkingDirectory --name mainbaseline"
& $GuardianCliLocation baseline --working-directory $WorkingDirectory --name mainbaseline & $GuardianCliLocation baseline --working-directory $WorkingDirectory --name mainbaseline
if ($LASTEXITCODE -ne 0) { if ($LASTEXITCODE -ne 0) {
Write-Error "Guardian baseline failed with exit code $LASTEXITCODE." Write-PipelineTelemetryError -Force -Category 'Build' -Message "Guardian baseline failed with exit code $LASTEXITCODE."
ExitWithExitCode $LASTEXITCODE
} }
& $(Join-Path $PSScriptRoot "push-gdn.ps1") -Repository $Repository -BranchName $BranchName -GdnFolder $gdnFolder -AzureDevOpsAccessToken $AzureDevOpsAccessToken -PushReason "Initialize gdn folder" & $(Join-Path $PSScriptRoot 'push-gdn.ps1') -Repository $Repository -BranchName $BranchName -GdnFolder $gdnFolder -AzureDevOpsAccessToken $AzureDevOpsAccessToken -PushReason 'Initialize gdn folder'
ExitWithExitCode 0
}
catch {
Write-Host $_.ScriptStackTrace
Write-PipelineTelemetryError -Category 'Sdl' -Message $_
ExitWithExitCode 1
} }

34
eng/common/sdl/push-gdn.ps1
View file

@ -1,17 +1,21 @@
Param( Param(
[string] $Repository, [string] $Repository,
[string] $BranchName="master", [string] $BranchName='master',
[string] $GdnFolder, [string] $GdnFolder,
[string] $AzureDevOpsAccessToken, [string] $AzureDevOpsAccessToken,
[string] $PushReason [string] $PushReason
) )
$ErrorActionPreference = "Stop" $ErrorActionPreference = 'Stop'
Set-StrictMode -Version 2.0 Set-StrictMode -Version 2.0
$disableConfigureToolsetImport = $true
$LASTEXITCODE = 0 $LASTEXITCODE = 0
try {
. $PSScriptRoot\..\tools.ps1
# We create the temp directory where we'll store the sdl-config repository # We create the temp directory where we'll store the sdl-config repository
$sdlDir = Join-Path $env:TEMP "sdl" $sdlDir = Join-Path $env:TEMP 'sdl'
if (Test-Path $sdlDir) { if (Test-Path $sdlDir) {
Remove-Item -Force -Recurse $sdlDir Remove-Item -Force -Recurse $sdlDir
} }
@ -19,10 +23,11 @@ if (Test-Path $sdlDir) {
Write-Host "git clone https://dnceng:`$AzureDevOpsAccessToken@dev.azure.com/dnceng/internal/_git/sdl-tool-cfg $sdlDir" Write-Host "git clone https://dnceng:`$AzureDevOpsAccessToken@dev.azure.com/dnceng/internal/_git/sdl-tool-cfg $sdlDir"
git clone https://dnceng:$AzureDevOpsAccessToken@dev.azure.com/dnceng/internal/_git/sdl-tool-cfg $sdlDir git clone https://dnceng:$AzureDevOpsAccessToken@dev.azure.com/dnceng/internal/_git/sdl-tool-cfg $sdlDir
if ($LASTEXITCODE -ne 0) { if ($LASTEXITCODE -ne 0) {
Write-Error "Git clone failed with exit code $LASTEXITCODE." Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Git clone failed with exit code $LASTEXITCODE."
ExitWithExitCode $LASTEXITCODE
} }
# We copy the .gdn folder from our local run into the git repository so it can be committed # We copy the .gdn folder from our local run into the git repository so it can be committed
$sdlRepositoryFolder = Join-Path (Join-Path (Join-Path $sdlDir $Repository) $BranchName) ".gdn" $sdlRepositoryFolder = Join-Path (Join-Path (Join-Path $sdlDir $Repository) $BranchName) '.gdn'
if (Get-Command Robocopy) { if (Get-Command Robocopy) {
Robocopy /S $GdnFolder $sdlRepositoryFolder Robocopy /S $GdnFolder $sdlRepositoryFolder
} else { } else {
@ -31,21 +36,30 @@ if (Get-Command Robocopy) {
# cd to the sdl-config directory so we can run git there # cd to the sdl-config directory so we can run git there
Push-Location $sdlDir Push-Location $sdlDir
# git add . --> git commit --> git push # git add . --> git commit --> git push
Write-Host "git add ." Write-Host 'git add .'
git add . git add .
if ($LASTEXITCODE -ne 0) { if ($LASTEXITCODE -ne 0) {
Write-Error "Git add failed with exit code $LASTEXITCODE." Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Git add failed with exit code $LASTEXITCODE."
ExitWithExitCode $LASTEXITCODE
} }
Write-Host "git -c user.email=`"dn-bot@microsoft.com`" -c user.name=`"Dotnet Bot`" commit -m `"$PushReason for $Repository/$BranchName`"" Write-Host "git -c user.email=`"dn-bot@microsoft.com`" -c user.name=`"Dotnet Bot`" commit -m `"$PushReason for $Repository/$BranchName`""
git -c user.email="dn-bot@microsoft.com" -c user.name="Dotnet Bot" commit -m "$PushReason for $Repository/$BranchName" git -c user.email="dn-bot@microsoft.com" -c user.name="Dotnet Bot" commit -m "$PushReason for $Repository/$BranchName"
if ($LASTEXITCODE -ne 0) { if ($LASTEXITCODE -ne 0) {
Write-Error "Git commit failed with exit code $LASTEXITCODE." Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Git commit failed with exit code $LASTEXITCODE."
ExitWithExitCode $LASTEXITCODE
} }
Write-Host "git push" Write-Host 'git push'
git push git push
if ($LASTEXITCODE -ne 0) { if ($LASTEXITCODE -ne 0) {
Write-Error "Git push failed with exit code $LASTEXITCODE." Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Git push failed with exit code $LASTEXITCODE."
ExitWithExitCode $LASTEXITCODE
} }
# Return to the original directory # Return to the original directory
Pop-Location Pop-Location
}
catch {
Write-Host $_.ScriptStackTrace
Write-PipelineTelemetryError -Category 'Sdl' -Message $_
ExitWithExitCode 1
}

38
eng/common/sdl/run-sdl.ps1
View file

@ -5,46 +5,50 @@ Param(
[string] $GdnFolder, [string] $GdnFolder,
[string[]] $ToolsList, [string[]] $ToolsList,
[string] $UpdateBaseline, [string] $UpdateBaseline,
[string] $GuardianLoggerLevel="Standard", [string] $GuardianLoggerLevel='Standard',
[string[]] $CrScanAdditionalRunConfigParams, [string[]] $CrScanAdditionalRunConfigParams,
[string[]] $PoliCheckAdditionalRunConfigParams [string[]] $PoliCheckAdditionalRunConfigParams
) )
$ErrorActionPreference = "Stop" $ErrorActionPreference = 'Stop'
Set-StrictMode -Version 2.0 Set-StrictMode -Version 2.0
$disableConfigureToolsetImport = $true
$LASTEXITCODE = 0 $LASTEXITCODE = 0
try {
. $PSScriptRoot\..\tools.ps1
# We store config files in the r directory of .gdn # We store config files in the r directory of .gdn
Write-Host $ToolsList Write-Host $ToolsList
$gdnConfigPath = Join-Path $GdnFolder "r" $gdnConfigPath = Join-Path $GdnFolder 'r'
$ValidPath = Test-Path $GuardianCliLocation $ValidPath = Test-Path $GuardianCliLocation
if ($ValidPath -eq $False) if ($ValidPath -eq $False)
{ {
Write-Host "Invalid Guardian CLI Location." Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Invalid Guardian CLI Location."
exit 1 ExitWithExitCode 1
} }
$configParam = @("--config") $configParam = @('--config')
foreach ($tool in $ToolsList) { foreach ($tool in $ToolsList) {
$gdnConfigFile = Join-Path $gdnConfigPath "$tool-configure.gdnconfig" $gdnConfigFile = Join-Path $gdnConfigPath "$tool-configure.gdnconfig"
Write-Host $tool Write-Host $tool
# We have to manually configure tools that run on source to look at the source directory only # We have to manually configure tools that run on source to look at the source directory only
if ($tool -eq "credscan") { if ($tool -eq 'credscan') {
Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" TargetDirectory < $TargetDirectory `" `" OutputType < pre `" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams})" Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" TargetDirectory < $TargetDirectory `" `" OutputType < pre `" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams})"
& $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " TargetDirectory < $TargetDirectory " "OutputType < pre" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams}) & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " TargetDirectory < $TargetDirectory " "OutputType < pre" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams})
if ($LASTEXITCODE -ne 0) { if ($LASTEXITCODE -ne 0) {
Write-Host "Guardian configure for $tool failed with exit code $LASTEXITCODE." Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian configure for $tool failed with exit code $LASTEXITCODE."
exit $LASTEXITCODE ExitWithExitCode $LASTEXITCODE
} }
} }
if ($tool -eq "policheck") { if ($tool -eq 'policheck') {
Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" Target < $TargetDirectory `" $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams})" Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" Target < $TargetDirectory `" $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams})"
& $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " Target < $TargetDirectory " $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams}) & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " Target < $TargetDirectory " $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams})
if ($LASTEXITCODE -ne 0) { if ($LASTEXITCODE -ne 0) {
Write-Host "Guardian configure for $tool failed with exit code $LASTEXITCODE." Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian configure for $tool failed with exit code $LASTEXITCODE."
exit $LASTEXITCODE ExitWithExitCode $LASTEXITCODE
} }
} }
@ -54,6 +58,12 @@ foreach ($tool in $ToolsList) {
Write-Host "$GuardianCliLocation run --working-directory $WorkingDirectory --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel $configParam" Write-Host "$GuardianCliLocation run --working-directory $WorkingDirectory --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel $configParam"
& $GuardianCliLocation run --working-directory $WorkingDirectory --tool $tool --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel $configParam & $GuardianCliLocation run --working-directory $WorkingDirectory --tool $tool --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel $configParam
if ($LASTEXITCODE -ne 0) { if ($LASTEXITCODE -ne 0) {
Write-Host "Guardian run for $ToolsList using $configParam failed with exit code $LASTEXITCODE." Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian run for $ToolsList using $configParam failed with exit code $LASTEXITCODE."
exit $LASTEXITCODE ExitWithExitCode $LASTEXITCODE
}
}
catch {
Write-Host $_.ScriptStackTrace
Write-PipelineTelemetryError -Category 'Sdl' -Message $_
ExitWithExitCode 1
} }

14
eng/common/templates/post-build/common-variables.yml
View file

@ -13,19 +13,19 @@ variables:
- name: NetCore_5_Dev_Channel_Id - name: NetCore_5_Dev_Channel_Id
value: 131 value: 131
# .NET Tools - Validation # .NET Eng - Validation
- name: NetCore_Tools_Validation_Channel_Id - name: Net_Eng_Validation_Channel_Id
value: 9 value: 9
# .NET Tools - Latest # .NET Eng - Latest
- name: NetCore_Tools_Latest_Channel_Id - name: Net_Eng_Latest_Channel_Id
value: 2 value: 2
# .NET 3 Tools - Validation # .NET 3 Eng - Validation
- name: NETCore_3_Tools_Validation_Channel_Id - name: NET_3_Eng_Validation_Channel_Id
value: 390 value: 390
# .NET 3 Tools - Latest # .NET 3 Eng
- name: NetCore_3_Tools_Channel_Id - name: NetCore_3_Tools_Channel_Id
value: 344 value: 344

21
eng/common/templates/post-build/post-build.yml
View file

@ -137,8 +137,8 @@ stages:
dependsOn: ${{ parameters.publishDependsOn }} dependsOn: ${{ parameters.publishDependsOn }}
publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
stageName: 'NetCore_Tools_Latest_Publish' stageName: 'Net_Eng_Latest_Publish'
channelName: '.NET Tools - Latest' channelName: '.NET Eng - Latest'
channelId: 2 channelId: 2
transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json'
shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json'
@ -150,8 +150,8 @@ stages:
dependsOn: ${{ parameters.publishDependsOn }} dependsOn: ${{ parameters.publishDependsOn }}
publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
stageName: 'PVR_Publish' stageName: 'Net_Eng_Validation_Publish'
channelName: '.NET Tools - Validation' channelName: '.NET Eng - Validation'
channelId: 9 channelId: 9
transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json'
shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json'
@ -195,3 +195,16 @@ stages:
transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json' symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json'
- template: \eng\common\templates\post-build\channels\generic-internal-channel.yml
parameters:
artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
dependsOn: ${{ parameters.publishDependsOn }}
publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
stageName: 'NET_Internal_Tooling_Publishing'
channelName: '.NET Internal Tooling'
channelId: 551
transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-internal/nuget/v3/index.json'
shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-internal/nuget/v3/index.json'
symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-internal-symbols/nuget/v3/index.json'

144
eng/common/tools.ps1
View file

@ -5,7 +5,7 @@
[bool]$ci = if (Test-Path variable:ci) { $ci } else { $false } [bool]$ci = if (Test-Path variable:ci) { $ci } else { $false }
# Build configuration. Common values include 'Debug' and 'Release', but the repository may use other names. # Build configuration. Common values include 'Debug' and 'Release', but the repository may use other names.
[string]$configuration = if (Test-Path variable:configuration) { $configuration } else { "Debug" } [string]$configuration = if (Test-Path variable:configuration) { $configuration } else { 'Debug' }
# Set to true to output binary log from msbuild. Note that emitting binary log slows down the build. # Set to true to output binary log from msbuild. Note that emitting binary log slows down the build.
# Binary log must be enabled on CI. # Binary log must be enabled on CI.
@ -24,7 +24,7 @@
[bool]$restore = if (Test-Path variable:restore) { $restore } else { $true } [bool]$restore = if (Test-Path variable:restore) { $restore } else { $true }
# Adjusts msbuild verbosity level. # Adjusts msbuild verbosity level.
[string]$verbosity = if (Test-Path variable:verbosity) { $verbosity } else { "minimal" } [string]$verbosity = if (Test-Path variable:verbosity) { $verbosity } else { 'minimal' }
# Set to true to reuse msbuild nodes. Recommended to not reuse on CI. # Set to true to reuse msbuild nodes. Recommended to not reuse on CI.
[bool]$nodeReuse = if (Test-Path variable:nodeReuse) { $nodeReuse } else { !$ci } [bool]$nodeReuse = if (Test-Path variable:nodeReuse) { $nodeReuse } else { !$ci }
@ -41,23 +41,23 @@
# Enable repos to use a particular version of the on-line dotnet-install scripts. # Enable repos to use a particular version of the on-line dotnet-install scripts.
# default URL: https://dot.net/v1/dotnet-install.ps1 # default URL: https://dot.net/v1/dotnet-install.ps1
[string]$dotnetInstallScriptVersion = if (Test-Path variable:dotnetInstallScriptVersion) { $dotnetInstallScriptVersion } else { "v1" } [string]$dotnetInstallScriptVersion = if (Test-Path variable:dotnetInstallScriptVersion) { $dotnetInstallScriptVersion } else { 'v1' }
# True to use global NuGet cache instead of restoring packages to repository-local directory. # True to use global NuGet cache instead of restoring packages to repository-local directory.
[bool]$useGlobalNuGetCache = if (Test-Path variable:useGlobalNuGetCache) { $useGlobalNuGetCache } else { !$ci } [bool]$useGlobalNuGetCache = if (Test-Path variable:useGlobalNuGetCache) { $useGlobalNuGetCache } else { !$ci }
# An array of names of processes to stop on script exit if prepareMachine is true. # An array of names of processes to stop on script exit if prepareMachine is true.
$processesToStopOnExit = if (Test-Path variable:processesToStopOnExit) { $processesToStopOnExit } else { @("msbuild", "dotnet", "vbcscompiler") } $processesToStopOnExit = if (Test-Path variable:processesToStopOnExit) { $processesToStopOnExit } else { @('msbuild', 'dotnet', 'vbcscompiler') }
$disableConfigureToolsetImport = if (Test-Path variable:disableConfigureToolsetImport) { $disableConfigureToolsetImport } else { $null } $disableConfigureToolsetImport = if (Test-Path variable:disableConfigureToolsetImport) { $disableConfigureToolsetImport } else { $null }
set-strictmode -version 2.0 set-strictmode -version 2.0
$ErrorActionPreference = "Stop" $ErrorActionPreference = 'Stop'
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
function Create-Directory([string[]] $path) { function Create-Directory([string[]] $path) {
if (!(Test-Path $path)) { if (!(Test-Path $path)) {
New-Item -path $path -force -itemType "Directory" | Out-Null New-Item -path $path -force -itemType 'Directory' | Out-Null
} }
} }
@ -121,7 +121,7 @@ function InitializeDotNetCli([bool]$install) {
# Find the first path on %PATH% that contains the dotnet.exe # Find the first path on %PATH% that contains the dotnet.exe
if ($useInstalledDotNetCli -and (-not $globalJsonHasRuntimes) -and ($env:DOTNET_INSTALL_DIR -eq $null)) { if ($useInstalledDotNetCli -and (-not $globalJsonHasRuntimes) -and ($env:DOTNET_INSTALL_DIR -eq $null)) {
$dotnetCmd = Get-Command "dotnet.exe" -ErrorAction SilentlyContinue $dotnetCmd = Get-Command 'dotnet.exe' -ErrorAction SilentlyContinue
if ($dotnetCmd -ne $null) { if ($dotnetCmd -ne $null) {
$env:DOTNET_INSTALL_DIR = Split-Path $dotnetCmd.Path -Parent $env:DOTNET_INSTALL_DIR = Split-Path $dotnetCmd.Path -Parent
} }
@ -134,13 +134,13 @@ function InitializeDotNetCli([bool]$install) {
if ((-not $globalJsonHasRuntimes) -and ($env:DOTNET_INSTALL_DIR -ne $null) -and (Test-Path(Join-Path $env:DOTNET_INSTALL_DIR "sdk\$dotnetSdkVersion"))) { if ((-not $globalJsonHasRuntimes) -and ($env:DOTNET_INSTALL_DIR -ne $null) -and (Test-Path(Join-Path $env:DOTNET_INSTALL_DIR "sdk\$dotnetSdkVersion"))) {
$dotnetRoot = $env:DOTNET_INSTALL_DIR $dotnetRoot = $env:DOTNET_INSTALL_DIR
} else { } else {
$dotnetRoot = Join-Path $RepoRoot ".dotnet" $dotnetRoot = Join-Path $RepoRoot '.dotnet'
if (-not (Test-Path(Join-Path $dotnetRoot "sdk\$dotnetSdkVersion"))) { if (-not (Test-Path(Join-Path $dotnetRoot "sdk\$dotnetSdkVersion"))) {
if ($install) { if ($install) {
InstallDotNetSdk $dotnetRoot $dotnetSdkVersion InstallDotNetSdk $dotnetRoot $dotnetSdkVersion
} else { } else {
Write-PipelineTelemetryError -Category "InitializeToolset" -Message "Unable to find dotnet with SDK version '$dotnetSdkVersion'" Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Unable to find dotnet with SDK version '$dotnetSdkVersion'"
ExitWithExitCode 1 ExitWithExitCode 1
} }
} }
@ -163,7 +163,7 @@ function InitializeDotNetCli([bool]$install) {
} }
function GetDotNetInstallScript([string] $dotnetRoot) { function GetDotNetInstallScript([string] $dotnetRoot) {
$installScript = Join-Path $dotnetRoot "dotnet-install.ps1" $installScript = Join-Path $dotnetRoot 'dotnet-install.ps1'
if (!(Test-Path $installScript)) { if (!(Test-Path $installScript)) {
Create-Directory $dotnetRoot Create-Directory $dotnetRoot
$ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit
@ -173,17 +173,17 @@ function GetDotNetInstallScript([string] $dotnetRoot) {
return $installScript return $installScript
} }
function InstallDotNetSdk([string] $dotnetRoot, [string] $version, [string] $architecture = "") { function InstallDotNetSdk([string] $dotnetRoot, [string] $version, [string] $architecture = '') {
InstallDotNet $dotnetRoot $version $architecture InstallDotNet $dotnetRoot $version $architecture
} }
function InstallDotNet([string] $dotnetRoot, function InstallDotNet([string] $dotnetRoot,
[string] $version, [string] $version,
[string] $architecture = "", [string] $architecture = '',
[string] $runtime = "", [string] $runtime = '',
[bool] $skipNonVersionedFiles = $false, [bool] $skipNonVersionedFiles = $false,
[string] $runtimeSourceFeed = "", [string] $runtimeSourceFeed = '',
[string] $runtimeSourceFeedKey = "") { [string] $runtimeSourceFeedKey = '') {
$installScript = GetDotNetInstallScript $dotnetRoot $installScript = GetDotNetInstallScript $dotnetRoot
$installParameters = @{ $installParameters = @{
@ -199,7 +199,7 @@ function InstallDotNet([string] $dotnetRoot,
& $installScript @installParameters & $installScript @installParameters
} }
catch { catch {
Write-PipelineTelemetryError -Category "InitializeToolset" -Message "Failed to install dotnet runtime '$runtime' from public location." Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install dotnet runtime '$runtime' from public location."
# Only the runtime can be installed from a custom [private] location. # Only the runtime can be installed from a custom [private] location.
if ($runtime -and ($runtimeSourceFeed -or $runtimeSourceFeedKey)) { if ($runtime -and ($runtimeSourceFeed -or $runtimeSourceFeedKey)) {
@ -215,7 +215,7 @@ function InstallDotNet([string] $dotnetRoot,
& $installScript @installParameters & $installScript @installParameters
} }
catch { catch {
Write-PipelineTelemetryError -Category "InitializeToolset" -Message "Failed to install dotnet runtime '$runtime' from custom location '$runtimeSourceFeed'." Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install dotnet runtime '$runtime' from custom location '$runtimeSourceFeed'."
} }
} }
} }
@ -238,12 +238,12 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements =
} }
if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs } if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs }
$vsMinVersionStr = if ($vsRequirements.version) { $vsRequirements.version } else { "15.9" } $vsMinVersionStr = if ($vsRequirements.version) { $vsRequirements.version } else { '15.9' }
$vsMinVersion = [Version]::new($vsMinVersionStr) $vsMinVersion = [Version]::new($vsMinVersionStr)
# Try msbuild command available in the environment. # Try msbuild command available in the environment.
if ($env:VSINSTALLDIR -ne $null) { if ($env:VSINSTALLDIR -ne $null) {
$msbuildCmd = Get-Command "msbuild.exe" -ErrorAction SilentlyContinue $msbuildCmd = Get-Command 'msbuild.exe' -ErrorAction SilentlyContinue
if ($msbuildCmd -ne $null) { if ($msbuildCmd -ne $null) {
# Workaround for https://github.com/dotnet/roslyn/issues/35793 # Workaround for https://github.com/dotnet/roslyn/issues/35793
# Due to this issue $msbuildCmd.Version returns 0.0.0.0 for msbuild.exe 16.2+ # Due to this issue $msbuildCmd.Version returns 0.0.0.0 for msbuild.exe 16.2+
@ -267,7 +267,7 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements =
InitializeVisualStudioEnvironmentVariables $vsInstallDir $vsMajorVersion InitializeVisualStudioEnvironmentVariables $vsInstallDir $vsMajorVersion
} else { } else {
if (Get-Member -InputObject $GlobalJson.tools -Name "xcopy-msbuild") { if (Get-Member -InputObject $GlobalJson.tools -Name 'xcopy-msbuild') {
$xcopyMSBuildVersion = $GlobalJson.tools.'xcopy-msbuild' $xcopyMSBuildVersion = $GlobalJson.tools.'xcopy-msbuild'
$vsMajorVersion = $xcopyMSBuildVersion.Split('.')[0] $vsMajorVersion = $xcopyMSBuildVersion.Split('.')[0]
} else { } else {
@ -277,7 +277,7 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements =
$vsInstallDir = InitializeXCopyMSBuild $xcopyMSBuildVersion $install $vsInstallDir = InitializeXCopyMSBuild $xcopyMSBuildVersion $install
if ($vsInstallDir -eq $null) { if ($vsInstallDir -eq $null) {
throw "Unable to find Visual Studio that has required version and components installed" throw 'Unable to find Visual Studio that has required version and components installed'
} }
} }
@ -301,7 +301,7 @@ function InstallXCopyMSBuild([string]$packageVersion) {
} }
function InitializeXCopyMSBuild([string]$packageVersion, [bool]$install) { function InitializeXCopyMSBuild([string]$packageVersion, [bool]$install) {
$packageName = "RoslynTools.MSBuild" $packageName = 'RoslynTools.MSBuild'
$packageDir = Join-Path $ToolsDir "msbuild\$packageVersion" $packageDir = Join-Path $ToolsDir "msbuild\$packageVersion"
$packagePath = Join-Path $packageDir "$packageName.$packageVersion.nupkg" $packagePath = Join-Path $packageDir "$packageName.$packageVersion.nupkg"
@ -317,7 +317,7 @@ function InitializeXCopyMSBuild([string]$packageVersion, [bool]$install) {
Unzip $packagePath $packageDir Unzip $packagePath $packageDir
} }
return Join-Path $packageDir "tools" return Join-Path $packageDir 'tools'
} }
# #
@ -334,32 +334,32 @@ function InitializeXCopyMSBuild([string]$packageVersion, [bool]$install) {
# or $null if no instance meeting the requirements is found on the machine. # or $null if no instance meeting the requirements is found on the machine.
# #
function LocateVisualStudio([object]$vsRequirements = $null){ function LocateVisualStudio([object]$vsRequirements = $null){
if (Get-Member -InputObject $GlobalJson.tools -Name "vswhere") { if (Get-Member -InputObject $GlobalJson.tools -Name 'vswhere') {
$vswhereVersion = $GlobalJson.tools.vswhere $vswhereVersion = $GlobalJson.tools.vswhere
} else { } else {
$vswhereVersion = "2.5.2" $vswhereVersion = '2.5.2'
} }
$vsWhereDir = Join-Path $ToolsDir "vswhere\$vswhereVersion" $vsWhereDir = Join-Path $ToolsDir "vswhere\$vswhereVersion"
$vsWhereExe = Join-Path $vsWhereDir "vswhere.exe" $vsWhereExe = Join-Path $vsWhereDir 'vswhere.exe'
if (!(Test-Path $vsWhereExe)) { if (!(Test-Path $vsWhereExe)) {
Create-Directory $vsWhereDir Create-Directory $vsWhereDir
Write-Host "Downloading vswhere" Write-Host 'Downloading vswhere'
Invoke-WebRequest "https://github.com/Microsoft/vswhere/releases/download/$vswhereVersion/vswhere.exe" -OutFile $vswhereExe Invoke-WebRequest "https://github.com/Microsoft/vswhere/releases/download/$vswhereVersion/vswhere.exe" -OutFile $vswhereExe
} }
if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs } if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs }
$args = @("-latest", "-prerelease", "-format", "json", "-requires", "Microsoft.Component.MSBuild", "-products", "*") $args = @('-latest', '-prerelease', '-format', 'json', '-requires', 'Microsoft.Component.MSBuild', '-products', '*')
if (Get-Member -InputObject $vsRequirements -Name "version") { if (Get-Member -InputObject $vsRequirements -Name 'version') {
$args += "-version" $args += '-version'
$args += $vsRequirements.version $args += $vsRequirements.version
} }
if (Get-Member -InputObject $vsRequirements -Name "components") { if (Get-Member -InputObject $vsRequirements -Name 'components') {
foreach ($component in $vsRequirements.components) { foreach ($component in $vsRequirements.components) {
$args += "-requires" $args += '-requires'
$args += $component $args += $component
} }
} }
@ -385,27 +385,27 @@ function InitializeBuildTool() {
# Initialize dotnet cli if listed in 'tools' # Initialize dotnet cli if listed in 'tools'
$dotnetRoot = $null $dotnetRoot = $null
if (Get-Member -InputObject $GlobalJson.tools -Name "dotnet") { if (Get-Member -InputObject $GlobalJson.tools -Name 'dotnet') {
$dotnetRoot = InitializeDotNetCli -install:$restore $dotnetRoot = InitializeDotNetCli -install:$restore
} }
if ($msbuildEngine -eq "dotnet") { if ($msbuildEngine -eq 'dotnet') {
if (!$dotnetRoot) { if (!$dotnetRoot) {
Write-PipelineTelemetryError -Category "InitializeToolset" -Message "/global.json must specify 'tools.dotnet'." Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "/global.json must specify 'tools.dotnet'."
ExitWithExitCode 1 ExitWithExitCode 1
} }
$buildTool = @{ Path = Join-Path $dotnetRoot "dotnet.exe"; Command = "msbuild"; Tool = "dotnet"; Framework = "netcoreapp2.1" } $buildTool = @{ Path = Join-Path $dotnetRoot 'dotnet.exe'; Command = 'msbuild'; Tool = 'dotnet'; Framework = 'netcoreapp2.1' }
} elseif ($msbuildEngine -eq "vs") { } elseif ($msbuildEngine -eq "vs") {
try { try {
$msbuildPath = InitializeVisualStudioMSBuild -install:$restore $msbuildPath = InitializeVisualStudioMSBuild -install:$restore
} catch { } catch {
Write-PipelineTelemetryError -Category "InitializeToolset" -Message $_ Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_
ExitWithExitCode 1 ExitWithExitCode 1
} }
$buildTool = @{ Path = $msbuildPath; Command = ""; Tool = "vs"; Framework = "net472" } $buildTool = @{ Path = $msbuildPath; Command = ""; Tool = "vs"; Framework = "net472" }
} else { } else {
Write-PipelineTelemetryError -Category "InitializeToolset" -Message "Unexpected value of -msbuildEngine: '$msbuildEngine'." Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Unexpected value of -msbuildEngine: '$msbuildEngine'."
ExitWithExitCode 1 ExitWithExitCode 1
} }
@ -414,15 +414,15 @@ function InitializeBuildTool() {
function GetDefaultMSBuildEngine() { function GetDefaultMSBuildEngine() {
# Presence of tools.vs indicates the repo needs to build using VS msbuild on Windows. # Presence of tools.vs indicates the repo needs to build using VS msbuild on Windows.
if (Get-Member -InputObject $GlobalJson.tools -Name "vs") { if (Get-Member -InputObject $GlobalJson.tools -Name 'vs') {
return "vs" return 'vs'
} }
if (Get-Member -InputObject $GlobalJson.tools -Name "dotnet") { if (Get-Member -InputObject $GlobalJson.tools -Name 'dotnet') {
return "dotnet" return 'dotnet'
} }
Write-PipelineTelemetryError -Category "InitializeToolset" -Message "-msbuildEngine must be specified, or /global.json must specify 'tools.dotnet' or 'tools.vs'." Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "-msbuildEngine must be specified, or /global.json must specify 'tools.dotnet' or 'tools.vs'."
ExitWithExitCode 1 ExitWithExitCode 1
} }
@ -431,9 +431,9 @@ function GetNuGetPackageCachePath() {
# Use local cache on CI to ensure deterministic build, # Use local cache on CI to ensure deterministic build,
# use global cache in dev builds to avoid cost of downloading packages. # use global cache in dev builds to avoid cost of downloading packages.
if ($useGlobalNuGetCache) { if ($useGlobalNuGetCache) {
$env:NUGET_PACKAGES = Join-Path $env:UserProfile ".nuget\packages" $env:NUGET_PACKAGES = Join-Path $env:UserProfile '.nuget\packages'
} else { } else {
$env:NUGET_PACKAGES = Join-Path $RepoRoot ".packages" $env:NUGET_PACKAGES = Join-Path $RepoRoot '.packages'
} }
} }
@ -475,14 +475,14 @@ function InitializeToolset() {
} }
if (-not $restore) { if (-not $restore) {
Write-PipelineTelemetryError -Category "InitializeToolset" -Message "Toolset version $toolsetVersion has not been restored." Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Toolset version $toolsetVersion has not been restored."
ExitWithExitCode 1 ExitWithExitCode 1
} }
$buildTool = InitializeBuildTool $buildTool = InitializeBuildTool
$proj = Join-Path $ToolsetDir "restore.proj" $proj = Join-Path $ToolsetDir 'restore.proj'
$bl = if ($binaryLog) { "/bl:" + (Join-Path $LogDir "ToolsetRestore.binlog") } else { "" } $bl = if ($binaryLog) { '/bl:' + (Join-Path $LogDir 'ToolsetRestore.binlog') } else { '' }
'<Project Sdk="Microsoft.DotNet.Arcade.Sdk"/>' | Set-Content $proj '<Project Sdk="Microsoft.DotNet.Arcade.Sdk"/>' | Set-Content $proj
@ -504,7 +504,7 @@ function ExitWithExitCode([int] $exitCode) {
} }
function Stop-Processes() { function Stop-Processes() {
Write-Host "Killing running build processes..." Write-Host 'Killing running build processes...'
foreach ($processName in $processesToStopOnExit) { foreach ($processName in $processesToStopOnExit) {
Get-Process -Name $processName -ErrorAction SilentlyContinue | Stop-Process Get-Process -Name $processName -ErrorAction SilentlyContinue | Stop-Process
} }
@ -521,7 +521,7 @@ function MSBuild() {
# Work around issues with Azure Artifacts credential provider # Work around issues with Azure Artifacts credential provider
# https://github.com/dotnet/arcade/issues/3932 # https://github.com/dotnet/arcade/issues/3932
if ($ci -and $buildTool.Tool -eq "dotnet") { if ($ci -and $buildTool.Tool -eq 'dotnet') {
dotnet nuget locals http-cache -c dotnet nuget locals http-cache -c
$env:NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS = 20 $env:NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS = 20
@ -532,7 +532,7 @@ function MSBuild() {
$toolsetBuildProject = InitializeToolset $toolsetBuildProject = InitializeToolset
$path = Split-Path -parent $toolsetBuildProject $path = Split-Path -parent $toolsetBuildProject
$path = Join-Path $path (Join-Path $buildTool.Framework "Microsoft.DotNet.Arcade.Sdk.dll") $path = Join-Path $path (Join-Path $buildTool.Framework 'Microsoft.DotNet.Arcade.Sdk.dll')
$args += "/logger:$path" $args += "/logger:$path"
} }
@ -547,12 +547,12 @@ function MSBuild() {
function MSBuild-Core() { function MSBuild-Core() {
if ($ci) { if ($ci) {
if (!$binaryLog) { if (!$binaryLog) {
Write-PipelineTaskError -Message "Binary log must be enabled in CI build." Write-PipelineTelemetryError -Category 'Build' -Message 'Binary log must be enabled in CI build.'
ExitWithExitCode 1 ExitWithExitCode 1
} }
if ($nodeReuse) { if ($nodeReuse) {
Write-PipelineTaskError -Message "Node reuse must be disabled in CI build." Write-PipelineTelemetryError -Category 'Build' -Message 'Node reuse must be disabled in CI build.'
ExitWithExitCode 1 ExitWithExitCode 1
} }
} }
@ -562,10 +562,10 @@ function MSBuild-Core() {
$cmdArgs = "$($buildTool.Command) /m /nologo /clp:Summary /v:$verbosity /nr:$nodeReuse /p:ContinuousIntegrationBuild=$ci" $cmdArgs = "$($buildTool.Command) /m /nologo /clp:Summary /v:$verbosity /nr:$nodeReuse /p:ContinuousIntegrationBuild=$ci"
if ($warnAsError) { if ($warnAsError) {
$cmdArgs += " /warnaserror /p:TreatWarningsAsErrors=true" $cmdArgs += ' /warnaserror /p:TreatWarningsAsErrors=true'
} }
else { else {
$cmdArgs += " /p:TreatWarningsAsErrors=false" $cmdArgs += ' /p:TreatWarningsAsErrors=false'
} }
foreach ($arg in $args) { foreach ($arg in $args) {
@ -577,7 +577,7 @@ function MSBuild-Core() {
$exitCode = Exec-Process $buildTool.Path $cmdArgs $exitCode = Exec-Process $buildTool.Path $cmdArgs
if ($exitCode -ne 0) { if ($exitCode -ne 0) {
Write-PipelineTaskError -Message "Build failed." Write-PipelineTelemetryError Category 'Build' -Message 'Build failed.'
$buildLog = GetMSBuildBinaryLogCommandLineArgument $args $buildLog = GetMSBuildBinaryLogCommandLineArgument $args
if ($buildLog -ne $null) { if ($buildLog -ne $null) {
@ -592,12 +592,12 @@ function GetMSBuildBinaryLogCommandLineArgument($arguments) {
foreach ($argument in $arguments) { foreach ($argument in $arguments) {
if ($argument -ne $null) { if ($argument -ne $null) {
$arg = $argument.Trim() $arg = $argument.Trim()
if ($arg.StartsWith("/bl:", "OrdinalIgnoreCase")) { if ($arg.StartsWith('/bl:', "OrdinalIgnoreCase")) {
return $arg.Substring("/bl:".Length) return $arg.Substring('/bl:'.Length)
} }
if ($arg.StartsWith("/binaryLogger:", "OrdinalIgnoreCase")) { if ($arg.StartsWith('/binaryLogger:', 'OrdinalIgnoreCase')) {
return $arg.Substring("/binaryLogger:".Length) return $arg.Substring('/binaryLogger:'.Length)
} }
} }
} }
@ -607,14 +607,14 @@ function GetMSBuildBinaryLogCommandLineArgument($arguments) {
. $PSScriptRoot\pipeline-logging-functions.ps1 . $PSScriptRoot\pipeline-logging-functions.ps1
$RepoRoot = Resolve-Path (Join-Path $PSScriptRoot "..\..") $RepoRoot = Resolve-Path (Join-Path $PSScriptRoot '..\..')
$EngRoot = Resolve-Path (Join-Path $PSScriptRoot "..") $EngRoot = Resolve-Path (Join-Path $PSScriptRoot '..')
$ArtifactsDir = Join-Path $RepoRoot "artifacts" $ArtifactsDir = Join-Path $RepoRoot 'artifacts'
$ToolsetDir = Join-Path $ArtifactsDir "toolset" $ToolsetDir = Join-Path $ArtifactsDir 'toolset'
$ToolsDir = Join-Path $RepoRoot ".tools" $ToolsDir = Join-Path $RepoRoot '.tools'
$LogDir = Join-Path (Join-Path $ArtifactsDir "log") $configuration $LogDir = Join-Path (Join-Path $ArtifactsDir 'log') $configuration
$TempDir = Join-Path (Join-Path $ArtifactsDir "tmp") $configuration $TempDir = Join-Path (Join-Path $ArtifactsDir 'tmp') $configuration
$GlobalJson = Get-Content -Raw -Path (Join-Path $RepoRoot "global.json") | ConvertFrom-Json $GlobalJson = Get-Content -Raw -Path (Join-Path $RepoRoot 'global.json') | ConvertFrom-Json
# true if global.json contains a "runtimes" section # true if global.json contains a "runtimes" section
$globalJsonHasRuntimes = if ($GlobalJson.tools.PSObject.Properties.Name -Match 'runtimes') { $true } else { $false } $globalJsonHasRuntimes = if ($GlobalJson.tools.PSObject.Properties.Name -Match 'runtimes') { $true } else { $false }
@ -631,8 +631,14 @@ Write-PipelineSetVariable -Name 'TMP' -Value $TempDir
# Import custom tools configuration, if present in the repo. # Import custom tools configuration, if present in the repo.
# Note: Import in global scope so that the script set top-level variables without qualification. # Note: Import in global scope so that the script set top-level variables without qualification.
if (!$disableConfigureToolsetImport) { if (!$disableConfigureToolsetImport) {
$configureToolsetScript = Join-Path $EngRoot "configure-toolset.ps1" $configureToolsetScript = Join-Path $EngRoot 'configure-toolset.ps1'
if (Test-Path $configureToolsetScript) { if (Test-Path $configureToolsetScript) {
. $configureToolsetScript . $configureToolsetScript
if ((Test-Path variable:failOnConfigureToolsetError) -And $failOnConfigureToolsetError) {
if ((Test-Path variable:LastExitCode) -And ($LastExitCode -ne 0)) {
Write-PipelineTelemetryError -Category 'Build' -Message 'configure-toolset.ps1 returned a non-zero exit code'
ExitWithExitCode $LastExitCode
}
}
} }
} }

13
eng/common/tools.sh
View file

@ -4,7 +4,6 @@
# CI mode - set to true on CI server for PR validation build or official build. # CI mode - set to true on CI server for PR validation build or official build.
ci=${ci:-false} ci=${ci:-false}
disable_configure_toolset_import=${disable_configure_toolset_import:-}
# Set to true to use the pipelines logger which will enable Azure logging output. # Set to true to use the pipelines logger which will enable Azure logging output.
# https://github.com/Microsoft/azure-pipelines-tasks/blob/master/docs/authoring/commands.md # https://github.com/Microsoft/azure-pipelines-tasks/blob/master/docs/authoring/commands.md
@ -82,7 +81,7 @@ function ReadGlobalVersion {
local pattern="\"$key\" *: *\"(.*)\"" local pattern="\"$key\" *: *\"(.*)\""
if [[ ! $line =~ $pattern ]]; then if [[ ! $line =~ $pattern ]]; then
Write-PipelineTelemetryError -category 'InitializeToolset' "Error: Cannot find \"$key\" in $global_json_file" Write-PipelineTelemetryError -category 'Build' "Error: Cannot find \"$key\" in $global_json_file"
ExitWithExitCode 1 ExitWithExitCode 1
fi fi
@ -325,7 +324,7 @@ function InitializeToolset {
local toolset_build_proj=`cat "$toolset_location_file"` local toolset_build_proj=`cat "$toolset_location_file"`
if [[ ! -a "$toolset_build_proj" ]]; then if [[ ! -a "$toolset_build_proj" ]]; then
Write-PipelineTelemetryError -category 'InitializeToolset' "Invalid toolset path: $toolset_build_proj" Write-PipelineTelemetryError -category 'Build' "Invalid toolset path: $toolset_build_proj"
ExitWithExitCode 3 ExitWithExitCode 3
fi fi
@ -375,12 +374,12 @@ function MSBuild {
function MSBuild-Core { function MSBuild-Core {
if [[ "$ci" == true ]]; then if [[ "$ci" == true ]]; then
if [[ "$binary_log" != true ]]; then if [[ "$binary_log" != true ]]; then
Write-PipelineTaskError "Binary log must be enabled in CI build." Write-PipelineTelemetryError -category 'Build' "Binary log must be enabled in CI build."
ExitWithExitCode 1 ExitWithExitCode 1
fi fi
if [[ "$node_reuse" == true ]]; then if [[ "$node_reuse" == true ]]; then
Write-PipelineTaskError "Node reuse must be disabled in CI build." Write-PipelineTelemetryError -category 'Build' "Node reuse must be disabled in CI build."
ExitWithExitCode 1 ExitWithExitCode 1
fi fi
fi fi
@ -394,7 +393,7 @@ function MSBuild-Core {
"$_InitializeBuildTool" "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@" || { "$_InitializeBuildTool" "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@" || {
local exit_code=$? local exit_code=$?
Write-PipelineTaskError "Build failed (exit code '$exit_code')." Write-PipelineTelemetryError -category 'Build' "Build failed (exit code '$exit_code')."
ExitWithExitCode $exit_code ExitWithExitCode $exit_code
} }
} }
@ -437,7 +436,7 @@ Write-PipelineSetVariable -name "Temp" -value "$temp_dir"
Write-PipelineSetVariable -name "TMP" -value "$temp_dir" Write-PipelineSetVariable -name "TMP" -value "$temp_dir"
# Import custom tools configuration, if present in the repo. # Import custom tools configuration, if present in the repo.
if [[ -z "$disable_configure_toolset_import" ]]; then if [ -z "${disable_configure_toolset_import:-}" ]; then
configure_toolset_script="$eng_root/configure-toolset.sh" configure_toolset_script="$eng_root/configure-toolset.sh"
if [[ -a "$configure_toolset_script" ]]; then if [[ -a "$configure_toolset_script" ]]; then
. "$configure_toolset_script" . "$configure_toolset_script"

2
global.json
View file

@ -3,6 +3,6 @@
"dotnet": "5.0.100-alpha1-014915" "dotnet": "5.0.100-alpha1-014915"
}, },
"msbuild-sdks": { "msbuild-sdks": {
"Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19570.6" "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19571.10"
} }
} }