339 lines
11 KiB
PowerShell
339 lines
11 KiB
PowerShell
param(
|
|
[Parameter(Mandatory = $true)][string] $InputPath, # Full path to directory where NuGet packages to be checked are stored
|
|
[Parameter(Mandatory = $true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation
|
|
[Parameter(Mandatory = $true)][string] $DotnetSymbolVersion, # Version of dotnet symbol to use
|
|
[Parameter(Mandatory = $false)][switch] $CheckForWindowsPdbs, # If we should check for the existence of windows pdbs in addition to portable PDBs
|
|
[Parameter(Mandatory = $false)][switch] $ContinueOnError, # If we should keep checking symbols after an error
|
|
[Parameter(Mandatory = $false)][switch] $Clean, # Clean extracted symbols directory after checking symbols
|
|
[Parameter(Mandatory = $false)][string] $SymbolExclusionFile # Exclude the symbols in the file from publishing to symbol server
|
|
)
|
|
|
|
. $PSScriptRoot\..\tools.ps1
|
|
# Maximum number of jobs to run in parallel
|
|
$MaxParallelJobs = 16
|
|
|
|
# Max number of retries
|
|
$MaxRetry = 5
|
|
|
|
# Wait time between check for system load
|
|
$SecondsBetweenLoadChecks = 10
|
|
|
|
# Set error codes
|
|
Set-Variable -Name "ERROR_BADEXTRACT" -Option Constant -Value -1
|
|
Set-Variable -Name "ERROR_FILEDOESNOTEXIST" -Option Constant -Value -2
|
|
|
|
$WindowsPdbVerificationParam = ""
|
|
if ($CheckForWindowsPdbs) {
|
|
$WindowsPdbVerificationParam = "--windows-pdbs"
|
|
}
|
|
|
|
$ExclusionSet = New-Object System.Collections.Generic.HashSet[string];
|
|
|
|
if (!$InputPath -or !(Test-Path $InputPath)){
|
|
Write-Host "No symbols to validate."
|
|
ExitWithExitCode 0
|
|
}
|
|
|
|
#Check if the path exists
|
|
if ($SymbolExclusionFile -and (Test-Path $SymbolExclusionFile)){
|
|
[string[]]$Exclusions = Get-Content "$SymbolExclusionFile"
|
|
$Exclusions | foreach { if($_ -and $_.Trim()){$ExclusionSet.Add($_)} }
|
|
}
|
|
else{
|
|
Write-Host "Symbol Exclusion file does not exists. No symbols to exclude."
|
|
}
|
|
|
|
$CountMissingSymbols = {
|
|
param(
|
|
[string] $PackagePath, # Path to a NuGet package
|
|
[string] $WindowsPdbVerificationParam # If we should check for the existence of windows pdbs in addition to portable PDBs
|
|
)
|
|
|
|
Add-Type -AssemblyName System.IO.Compression.FileSystem
|
|
|
|
Write-Host "Validating $PackagePath "
|
|
|
|
# Ensure input file exist
|
|
if (!(Test-Path $PackagePath)) {
|
|
Write-PipelineTaskError "Input file does not exist: $PackagePath"
|
|
return [pscustomobject]@{
|
|
result = $using:ERROR_FILEDOESNOTEXIST
|
|
packagePath = $PackagePath
|
|
}
|
|
}
|
|
|
|
# Extensions for which we'll look for symbols
|
|
$RelevantExtensions = @('.dll', '.exe', '.so', '.dylib')
|
|
|
|
# How many files are missing symbol information
|
|
$MissingSymbols = 0
|
|
|
|
$PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath)
|
|
$PackageGuid = New-Guid
|
|
$ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageGuid
|
|
$SymbolsPath = Join-Path -Path $ExtractPath -ChildPath 'Symbols'
|
|
|
|
try {
|
|
[System.IO.Compression.ZipFile]::ExtractToDirectory($PackagePath, $ExtractPath)
|
|
}
|
|
catch {
|
|
Write-Host "Something went wrong extracting $PackagePath"
|
|
Write-Host $_
|
|
return [pscustomobject]@{
|
|
result = $using:ERROR_BADEXTRACT
|
|
packagePath = $PackagePath
|
|
}
|
|
}
|
|
|
|
Get-ChildItem -Recurse $ExtractPath |
|
|
Where-Object { $RelevantExtensions -contains $_.Extension } |
|
|
ForEach-Object {
|
|
$FileName = $_.FullName
|
|
if ($FileName -Match '\\ref\\') {
|
|
Write-Host "`t Ignoring reference assembly file " $FileName
|
|
return
|
|
}
|
|
|
|
$FirstMatchingSymbolDescriptionOrDefault = {
|
|
param(
|
|
[string] $FullPath, # Full path to the module that has to be checked
|
|
[string] $TargetServerParam, # Parameter to pass to `Symbol Tool` indicating the server to lookup for symbols
|
|
[string] $WindowsPdbVerificationParam, # Parameter to pass to potential check for windows-pdbs.
|
|
[string] $SymbolsPath
|
|
)
|
|
|
|
$FileName = [System.IO.Path]::GetFileName($FullPath)
|
|
$Extension = [System.IO.Path]::GetExtension($FullPath)
|
|
|
|
# Those below are potential symbol files that the `dotnet symbol` might
|
|
# return. Which one will be returned depend on the type of file we are
|
|
# checking and which type of file was uploaded.
|
|
|
|
# The file itself is returned
|
|
$SymbolPath = $SymbolsPath + '\' + $FileName
|
|
|
|
# PDB file for the module
|
|
$PdbPath = $SymbolPath.Replace($Extension, '.pdb')
|
|
|
|
# PDB file for R2R module (created by crossgen)
|
|
$NGenPdb = $SymbolPath.Replace($Extension, '.ni.pdb')
|
|
|
|
# DBG file for a .so library
|
|
$SODbg = $SymbolPath.Replace($Extension, '.so.dbg')
|
|
|
|
# DWARF file for a .dylib
|
|
$DylibDwarf = $SymbolPath.Replace($Extension, '.dylib.dwarf')
|
|
|
|
$dotnetSymbolExe = "$env:USERPROFILE\.dotnet\tools"
|
|
$dotnetSymbolExe = Resolve-Path "$dotnetSymbolExe\dotnet-symbol.exe"
|
|
|
|
$totalRetries = 0
|
|
|
|
while ($totalRetries -lt $using:MaxRetry) {
|
|
|
|
# Save the output and get diagnostic output
|
|
$output = & $dotnetSymbolExe --symbols --modules $WindowsPdbVerificationParam $TargetServerParam $FullPath -o $SymbolsPath --diagnostics | Out-String
|
|
|
|
if ((Test-Path $PdbPath) -and (Test-path $SymbolPath)) {
|
|
return 'Module and PDB for Module'
|
|
}
|
|
elseif ((Test-Path $NGenPdb) -and (Test-Path $PdbPath) -and (Test-Path $SymbolPath)) {
|
|
return 'Dll, PDB and NGen PDB'
|
|
}
|
|
elseif ((Test-Path $SODbg) -and (Test-Path $SymbolPath)) {
|
|
return 'So and DBG for SO'
|
|
}
|
|
elseif ((Test-Path $DylibDwarf) -and (Test-Path $SymbolPath)) {
|
|
return 'Dylib and Dwarf for Dylib'
|
|
}
|
|
elseif (Test-Path $SymbolPath) {
|
|
return 'Module'
|
|
}
|
|
else
|
|
{
|
|
$totalRetries++
|
|
}
|
|
}
|
|
|
|
return $null
|
|
}
|
|
|
|
$FileRelativePath = $FileName.Replace("$ExtractPath\", "")
|
|
if (($($using:ExclusionSet) -ne $null) -and ($($using:ExclusionSet).Contains($FileRelativePath) -or ($($using:ExclusionSet).Contains($FileRelativePath.Replace("\", "/"))))){
|
|
Write-Host "Skipping $FileName from symbol validation"
|
|
}
|
|
|
|
else {
|
|
$FileGuid = New-Guid
|
|
$ExpandedSymbolsPath = Join-Path -Path $SymbolsPath -ChildPath $FileGuid
|
|
|
|
$SymbolsOnMSDL = & $FirstMatchingSymbolDescriptionOrDefault `
|
|
-FullPath $FileName `
|
|
-TargetServerParam '--microsoft-symbol-server' `
|
|
-SymbolsPath "$ExpandedSymbolsPath-msdl" `
|
|
-WindowsPdbVerificationParam $WindowsPdbVerificationParam
|
|
$SymbolsOnSymWeb = & $FirstMatchingSymbolDescriptionOrDefault `
|
|
-FullPath $FileName `
|
|
-TargetServerParam '--internal-server' `
|
|
-SymbolsPath "$ExpandedSymbolsPath-symweb" `
|
|
-WindowsPdbVerificationParam $WindowsPdbVerificationParam
|
|
|
|
Write-Host -NoNewLine "`t Checking file " $FileName "... "
|
|
|
|
if ($SymbolsOnMSDL -ne $null -and $SymbolsOnSymWeb -ne $null) {
|
|
Write-Host "Symbols found on MSDL ($SymbolsOnMSDL) and SymWeb ($SymbolsOnSymWeb)"
|
|
}
|
|
else {
|
|
$MissingSymbols++
|
|
|
|
if ($SymbolsOnMSDL -eq $null -and $SymbolsOnSymWeb -eq $null) {
|
|
Write-Host 'No symbols found on MSDL or SymWeb!'
|
|
}
|
|
else {
|
|
if ($SymbolsOnMSDL -eq $null) {
|
|
Write-Host 'No symbols found on MSDL!'
|
|
}
|
|
else {
|
|
Write-Host 'No symbols found on SymWeb!'
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if ($using:Clean) {
|
|
Remove-Item $ExtractPath -Recurse -Force
|
|
}
|
|
|
|
Pop-Location
|
|
|
|
return [pscustomobject]@{
|
|
result = $MissingSymbols
|
|
packagePath = $PackagePath
|
|
}
|
|
}
|
|
|
|
function CheckJobResult(
|
|
$result,
|
|
$packagePath,
|
|
[ref]$DupedSymbols,
|
|
[ref]$TotalFailures) {
|
|
if ($result -eq $ERROR_BADEXTRACT) {
|
|
Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$packagePath has duplicated symbol files"
|
|
$DupedSymbols.Value++
|
|
}
|
|
elseif ($result -eq $ERROR_FILEDOESNOTEXIST) {
|
|
Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$packagePath does not exist"
|
|
$TotalFailures.Value++
|
|
}
|
|
elseif ($result -gt '0') {
|
|
Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Missing symbols for $result modules in the package $packagePath"
|
|
$TotalFailures.Value++
|
|
}
|
|
else {
|
|
Write-Host "All symbols verified for package $packagePath"
|
|
}
|
|
}
|
|
|
|
function CheckSymbolsAvailable {
|
|
if (Test-Path $ExtractPath) {
|
|
Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue
|
|
}
|
|
|
|
$TotalPackages = 0
|
|
$TotalFailures = 0
|
|
$DupedSymbols = 0
|
|
|
|
Get-ChildItem "$InputPath\*.nupkg" |
|
|
ForEach-Object {
|
|
$FileName = $_.Name
|
|
$FullName = $_.FullName
|
|
|
|
# These packages from Arcade-Services include some native libraries that
|
|
# our current symbol uploader can't handle. Below is a workaround until
|
|
# we get issue: https://github.com/dotnet/arcade/issues/2457 sorted.
|
|
if ($FileName -Match 'Microsoft\.DotNet\.Darc\.') {
|
|
Write-Host "Ignoring Arcade-services file: $FileName"
|
|
Write-Host
|
|
return
|
|
}
|
|
elseif ($FileName -Match 'Microsoft\.DotNet\.Maestro\.Tasks\.') {
|
|
Write-Host "Ignoring Arcade-services file: $FileName"
|
|
Write-Host
|
|
return
|
|
}
|
|
|
|
$TotalPackages++
|
|
|
|
Start-Job -ScriptBlock $CountMissingSymbols -ArgumentList @($FullName,$WindowsPdbVerificationParam) | Out-Null
|
|
|
|
$NumJobs = @(Get-Job -State 'Running').Count
|
|
|
|
while ($NumJobs -ge $MaxParallelJobs) {
|
|
Write-Host "There are $NumJobs validation jobs running right now. Waiting $SecondsBetweenLoadChecks seconds to check again."
|
|
sleep $SecondsBetweenLoadChecks
|
|
$NumJobs = @(Get-Job -State 'Running').Count
|
|
}
|
|
|
|
foreach ($Job in @(Get-Job -State 'Completed')) {
|
|
$jobResult = Wait-Job -Id $Job.Id | Receive-Job
|
|
CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$DupedSymbols) ([ref]$TotalFailures)
|
|
Remove-Job -Id $Job.Id
|
|
}
|
|
Write-Host
|
|
}
|
|
|
|
foreach ($Job in @(Get-Job)) {
|
|
$jobResult = Wait-Job -Id $Job.Id | Receive-Job
|
|
CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$DupedSymbols) ([ref]$TotalFailures)
|
|
}
|
|
|
|
if ($TotalFailures -gt 0 -or $DupedSymbols -gt 0) {
|
|
if ($TotalFailures -gt 0) {
|
|
Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Symbols missing for $TotalFailures/$TotalPackages packages"
|
|
}
|
|
|
|
if ($DupedSymbols -gt 0) {
|
|
Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$DupedSymbols/$TotalPackages packages had duplicated symbol files and could not be extracted"
|
|
}
|
|
|
|
ExitWithExitCode 1
|
|
}
|
|
else {
|
|
Write-Host "All symbols validated!"
|
|
}
|
|
}
|
|
|
|
function InstallDotnetSymbol {
|
|
$dotnetSymbolPackageName = 'dotnet-symbol'
|
|
|
|
$dotnetRoot = InitializeDotNetCli -install:$true
|
|
$dotnet = "$dotnetRoot\dotnet.exe"
|
|
$toolList = & "$dotnet" tool list --global
|
|
|
|
if (($toolList -like "*$dotnetSymbolPackageName*") -and ($toolList -like "*$dotnetSymbolVersion*")) {
|
|
Write-Host "dotnet-symbol version $dotnetSymbolVersion is already installed."
|
|
}
|
|
else {
|
|
Write-Host "Installing dotnet-symbol version $dotnetSymbolVersion..."
|
|
Write-Host 'You may need to restart your command window if this is the first dotnet tool you have installed.'
|
|
& "$dotnet" tool install $dotnetSymbolPackageName --version $dotnetSymbolVersion --verbosity "minimal" --global
|
|
}
|
|
}
|
|
|
|
try {
|
|
. $PSScriptRoot\post-build-utils.ps1
|
|
|
|
InstallDotnetSymbol
|
|
|
|
foreach ($Job in @(Get-Job)) {
|
|
Remove-Job -Id $Job.Id
|
|
}
|
|
|
|
CheckSymbolsAvailable
|
|
}
|
|
catch {
|
|
Write-Host $_.ScriptStackTrace
|
|
Write-PipelineTelemetryError -Category 'CheckSymbols' -Message $_
|
|
ExitWithExitCode 1
|
|
}
|