From ea0ac0d0d4249f1325ec6a8f5b1e54eb1e1710d7 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Fri, 12 Aug 2022 17:01:46 +0000 Subject: [PATCH] Update dependencies from https://github.com/dotnet/arcade build 20220811.3 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Build.Tasks.Installers , Microsoft.DotNet.CMake.Sdk From Version 6.0.0-beta.22314.7 -> To Version 6.0.0-beta.22411.3 --- NuGet.config | 8 - eng/Version.Details.xml | 12 +- eng/Versions.props | 2 +- eng/common/generate-sbom-prep.ps1 | 2 + eng/common/generate-sbom-prep.sh | 12 ++ eng/common/init-tools-native.ps1 | 142 ++++++++++++------ eng/common/internal/NuGet.config | 7 + eng/common/internal/Tools.csproj | 3 + eng/common/templates/job/execute-sdl.yml | 2 +- eng/common/templates/job/onelocbuild.yml | 2 +- eng/common/templates/jobs/jobs.yml | 2 +- .../templates/post-build/post-build.yml | 8 +- eng/common/tools.ps1 | 4 + global.json | 4 +- 14 files changed, 141 insertions(+), 69 deletions(-) create mode 100644 eng/common/internal/NuGet.config diff --git a/NuGet.config b/NuGet.config index 8cac51c22..9ee0fa704 100644 --- a/NuGet.config +++ b/NuGet.config @@ -7,20 +7,16 @@ - - - - @@ -41,15 +37,11 @@ - - - - diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 1fc7fed18..136c1e478 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -197,19 +197,19 @@ - + https://github.com/dotnet/arcade - fdd3a242bc813f371023adff4e4c05c0be705d2a + 778552f02f31d50ec1c3c18a872cc482d04aec75 - + https://github.com/dotnet/arcade - fdd3a242bc813f371023adff4e4c05c0be705d2a + 778552f02f31d50ec1c3c18a872cc482d04aec75 - + https://github.com/dotnet/arcade - fdd3a242bc813f371023adff4e4c05c0be705d2a + 778552f02f31d50ec1c3c18a872cc482d04aec75 https://github.com/dotnet/source-build-reference-packages diff --git a/eng/Versions.props b/eng/Versions.props index 7efbe79d5..ae1cb3e14 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -19,7 +19,7 @@ - 6.0.0-beta.22314.7 + 6.0.0-beta.22411.3 diff --git a/eng/common/generate-sbom-prep.ps1 b/eng/common/generate-sbom-prep.ps1 index a733a8885..3e5c1c74a 100644 --- a/eng/common/generate-sbom-prep.ps1 +++ b/eng/common/generate-sbom-prep.ps1 @@ -2,6 +2,8 @@ Param( [Parameter(Mandatory=$true)][string] $ManifestDirPath # Manifest directory where sbom will be placed ) +. $PSScriptRoot\pipeline-logging-functions.ps1 + Write-Host "Creating dir $ManifestDirPath" # create directory for sbom manifest to be placed if (!(Test-Path -path $ManifestDirPath)) diff --git a/eng/common/generate-sbom-prep.sh b/eng/common/generate-sbom-prep.sh index f6c774531..d5c76dc82 100644 --- a/eng/common/generate-sbom-prep.sh +++ b/eng/common/generate-sbom-prep.sh @@ -2,6 +2,18 @@ source="${BASH_SOURCE[0]}" +# resolve $SOURCE until the file is no longer a symlink +while [[ -h $source ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + + # if $source was a relative symlink, we need to resolve it relative to the path where the + # symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" +. $scriptroot/pipeline-logging-functions.sh + manifest_dir=$1 if [ ! -d "$manifest_dir" ] ; then diff --git a/eng/common/init-tools-native.ps1 b/eng/common/init-tools-native.ps1 index db830c00a..7428ef880 100644 --- a/eng/common/init-tools-native.ps1 +++ b/eng/common/init-tools-native.ps1 @@ -31,6 +31,10 @@ Wait time between retry attempts in seconds .PARAMETER GlobalJsonFile File path to global.json file +.PARAMETER PathPromotion +Optional switch to enable either promote native tools specified in the global.json to the path (in Azure Pipelines) +or break the build if a native tool is not found on the path (on a local dev machine) + .NOTES #> [CmdletBinding(PositionalBinding=$false)] @@ -41,7 +45,8 @@ Param ( [switch] $Force = $False, [int] $DownloadRetries = 5, [int] $RetryWaitTimeInSeconds = 30, - [string] $GlobalJsonFile + [string] $GlobalJsonFile, + [switch] $PathPromotion ) if (!$GlobalJsonFile) { @@ -77,53 +82,100 @@ try { ConvertFrom-Json | Select-Object -Expand 'native-tools' -ErrorAction SilentlyContinue if ($NativeTools) { - $NativeTools.PSObject.Properties | ForEach-Object { - $ToolName = $_.Name - $ToolVersion = $_.Value - $LocalInstallerArguments = @{ ToolName = "$ToolName" } - $LocalInstallerArguments += @{ InstallPath = "$InstallBin" } - $LocalInstallerArguments += @{ BaseUri = "$BaseUri" } - $LocalInstallerArguments += @{ CommonLibraryDirectory = "$EngCommonBaseDir" } - $LocalInstallerArguments += @{ Version = "$ToolVersion" } + if ($PathPromotion -eq $True) { + if ($env:SYSTEM_TEAMPROJECT) { # check to see if we're in an Azure pipelines build + $NativeTools.PSObject.Properties | ForEach-Object { + $ToolName = $_.Name + $ToolVersion = $_.Value + $InstalledTools = @{} - if ($Verbose) { - $LocalInstallerArguments += @{ Verbose = $True } + if ((Get-Command "$ToolName" -ErrorAction SilentlyContinue) -eq $null) { + if ($ToolVersion -eq "latest") { + $ToolVersion = "" + } + $ArcadeToolsDirectory = "C:\arcade-tools" + if (-not (Test-Path $ArcadeToolsDirectory)) { + Write-Error "Arcade tools directory '$ArcadeToolsDirectory' was not found; artifacts were not properly installed." + exit 1 + } + $ToolDirectory = (Get-ChildItem -Path "$ArcadeToolsDirectory" -Filter "$ToolName-$ToolVersion*" | Sort-Object -Descending)[0] + if ([string]::IsNullOrWhiteSpace($ToolDirectory)) { + Write-Error "Unable to find directory for $ToolName $ToolVersion; please make sure the tool is installed on this image." + exit 1 + } + $BinPathFile = "$($ToolDirectory.FullName)\binpath.txt" + if (-not (Test-Path -Path "$BinPathFile")) { + Write-Error "Unable to find binpath.txt in '$($ToolDirectory.FullName)' ($ToolName $ToolVersion); artifact is either installed incorrectly or is not a bootstrappable tool." + exit 1 + } + $BinPath = Get-Content "$BinPathFile" + $ToolPath = Convert-Path -Path $BinPath + Write-Host "Adding $ToolName to the path ($ToolPath)..." + Write-Host "##vso[task.prependpath]$ToolPath" + $InstalledTools += @{ $ToolName = $ToolDirectory.FullName } + } + } + return $InstalledTools + } else { + $NativeTools.PSObject.Properties | ForEach-Object { + $ToolName = $_.Name + $ToolVersion = $_.Value + + if ((Get-Command "$ToolName" -ErrorAction SilentlyContinue) -eq $null) { + Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message "$ToolName not found on path. Please install $ToolName $ToolVersion before proceeding." + } + } + exit 0 } - if (Get-Variable 'Force' -ErrorAction 'SilentlyContinue') { - if($Force) { - $LocalInstallerArguments += @{ Force = $True } + } else { + $NativeTools.PSObject.Properties | ForEach-Object { + $ToolName = $_.Name + $ToolVersion = $_.Value + $LocalInstallerArguments = @{ ToolName = "$ToolName" } + $LocalInstallerArguments += @{ InstallPath = "$InstallBin" } + $LocalInstallerArguments += @{ BaseUri = "$BaseUri" } + $LocalInstallerArguments += @{ CommonLibraryDirectory = "$EngCommonBaseDir" } + $LocalInstallerArguments += @{ Version = "$ToolVersion" } + + if ($Verbose) { + $LocalInstallerArguments += @{ Verbose = $True } + } + if (Get-Variable 'Force' -ErrorAction 'SilentlyContinue') { + if($Force) { + $LocalInstallerArguments += @{ Force = $True } + } + } + if ($Clean) { + $LocalInstallerArguments += @{ Clean = $True } + } + + Write-Verbose "Installing $ToolName version $ToolVersion" + Write-Verbose "Executing '$InstallerPath $($LocalInstallerArguments.Keys.ForEach({"-$_ '$($LocalInstallerArguments.$_)'"}) -join ' ')'" + & $InstallerPath @LocalInstallerArguments + if ($LASTEXITCODE -Ne "0") { + $errMsg = "$ToolName installation failed" + if ((Get-Variable 'DoNotAbortNativeToolsInstallationOnFailure' -ErrorAction 'SilentlyContinue') -and $DoNotAbortNativeToolsInstallationOnFailure) { + $showNativeToolsWarning = $true + if ((Get-Variable 'DoNotDisplayNativeToolsInstallationWarnings' -ErrorAction 'SilentlyContinue') -and $DoNotDisplayNativeToolsInstallationWarnings) { + $showNativeToolsWarning = $false + } + if ($showNativeToolsWarning) { + Write-Warning $errMsg + } + $toolInstallationFailure = $true + } else { + # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482 + Write-Host $errMsg + exit 1 + } } } - if ($Clean) { - $LocalInstallerArguments += @{ Clean = $True } + + if ((Get-Variable 'toolInstallationFailure' -ErrorAction 'SilentlyContinue') -and $toolInstallationFailure) { + # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482 + Write-Host 'Native tools bootstrap failed' + exit 1 } - - Write-Verbose "Installing $ToolName version $ToolVersion" - Write-Verbose "Executing '$InstallerPath $($LocalInstallerArguments.Keys.ForEach({"-$_ '$($LocalInstallerArguments.$_)'"}) -join ' ')'" - & $InstallerPath @LocalInstallerArguments - if ($LASTEXITCODE -Ne "0") { - $errMsg = "$ToolName installation failed" - if ((Get-Variable 'DoNotAbortNativeToolsInstallationOnFailure' -ErrorAction 'SilentlyContinue') -and $DoNotAbortNativeToolsInstallationOnFailure) { - $showNativeToolsWarning = $true - if ((Get-Variable 'DoNotDisplayNativeToolsInstallationWarnings' -ErrorAction 'SilentlyContinue') -and $DoNotDisplayNativeToolsInstallationWarnings) { - $showNativeToolsWarning = $false - } - if ($showNativeToolsWarning) { - Write-Warning $errMsg - } - $toolInstallationFailure = $true - } else { - # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482 - Write-Host $errMsg - exit 1 - } - } - } - - if ((Get-Variable 'toolInstallationFailure' -ErrorAction 'SilentlyContinue') -and $toolInstallationFailure) { - # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482 - Write-Host 'Native tools bootstrap failed' - exit 1 } } else { @@ -139,7 +191,7 @@ try { Write-Host "##vso[task.prependpath]$(Convert-Path -Path $InstallBin)" return $InstallBin } - else { + elseif (-not ($PathPromotion)) { Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message 'Native tools install directory does not exist, installation failed' exit 1 } @@ -149,4 +201,4 @@ catch { Write-Host $_.ScriptStackTrace Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message $_ ExitWithExitCode 1 -} +} \ No newline at end of file diff --git a/eng/common/internal/NuGet.config b/eng/common/internal/NuGet.config new file mode 100644 index 000000000..19d3d311b --- /dev/null +++ b/eng/common/internal/NuGet.config @@ -0,0 +1,7 @@ + + + + + + + diff --git a/eng/common/internal/Tools.csproj b/eng/common/internal/Tools.csproj index beb9c4648..7f5ce6d60 100644 --- a/eng/common/internal/Tools.csproj +++ b/eng/common/internal/Tools.csproj @@ -8,6 +8,9 @@ + + + diff --git a/eng/common/templates/job/execute-sdl.yml b/eng/common/templates/job/execute-sdl.yml index 24cec0424..9ff6a10a6 100644 --- a/eng/common/templates/job/execute-sdl.yml +++ b/eng/common/templates/job/execute-sdl.yml @@ -54,7 +54,7 @@ jobs: # If it's not devdiv, it's dnceng ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: name: NetCore1ESPool-Internal - demands: ImageOverride -equals Build.Server.Amd64.VS2019 + demands: ImageOverride -equals windows.vs2019.amd64 steps: - checkout: self clean: true diff --git a/eng/common/templates/job/onelocbuild.yml b/eng/common/templates/job/onelocbuild.yml index 9d1e3042d..991ae05f5 100644 --- a/eng/common/templates/job/onelocbuild.yml +++ b/eng/common/templates/job/onelocbuild.yml @@ -41,7 +41,7 @@ jobs: # If it's not devdiv, it's dnceng ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: name: NetCore1ESPool-Internal - demands: ImageOverride -equals Build.Server.Amd64.VS2019 + demands: ImageOverride -equals windows.vs2019.amd64 variables: - group: OneLocBuildVariables # Contains the CeapexPat and GithubPat diff --git a/eng/common/templates/jobs/jobs.yml b/eng/common/templates/jobs/jobs.yml index 554e71cfc..48918df74 100644 --- a/eng/common/templates/jobs/jobs.yml +++ b/eng/common/templates/jobs/jobs.yml @@ -90,7 +90,7 @@ jobs: # If it's not devdiv, it's dnceng ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: name: NetCore1ESPool-Internal - demands: ImageOverride -equals Build.Server.Amd64.VS2019 + demands: ImageOverride -equals windows.vs2019.amd64 runAsPublic: ${{ parameters.runAsPublic }} publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }} diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index 24e92a275..270607126 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -101,7 +101,7 @@ stages: # If it's not devdiv, it's dnceng ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: name: NetCore1ESPool-Internal - demands: ImageOverride -equals Build.Server.Amd64.VS2019 + demands: ImageOverride -equals windows.vs2019.amd64 steps: - template: setup-maestro-vars.yml @@ -138,7 +138,7 @@ stages: # If it's not devdiv, it's dnceng ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: name: NetCore1ESPool-Internal - demands: ImageOverride -equals Build.Server.Amd64.VS2019 + demands: ImageOverride -equals windows.vs2019.amd64 steps: - template: setup-maestro-vars.yml parameters: @@ -198,7 +198,7 @@ stages: # If it's not devdiv, it's dnceng ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: name: NetCore1ESPool-Internal - demands: ImageOverride -equals Build.Server.Amd64.VS2019 + demands: ImageOverride -equals windows.vs2019.amd64 steps: - template: setup-maestro-vars.yml parameters: @@ -255,7 +255,7 @@ stages: # If it's not devdiv, it's dnceng ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: name: NetCore1ESPool-Internal - demands: ImageOverride -equals Build.Server.Amd64.VS2019 + demands: ImageOverride -equals windows.vs2019.amd64 steps: - template: setup-maestro-vars.yml parameters: diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index f1e1cb539..81d7b0355 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -635,6 +635,10 @@ function InitializeNativeTools() { InstallDirectory = "$ToolsDir" } } + if ($env:NativeToolsOnMachine) { + Write-Host "Variable NativeToolsOnMachine detected, enabling native tool path promotion..." + $nativeArgs += @{ PathPromotion = $true } + } & "$PSScriptRoot/init-tools-native.ps1" @nativeArgs } } diff --git a/global.json b/global.json index 80e9668ff..bb5632c68 100644 --- a/global.json +++ b/global.json @@ -11,7 +11,7 @@ "cmake": "3.16.4" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.22314.7", - "Microsoft.DotNet.CMake.Sdk": "6.0.0-beta.22314.7" + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.22411.3", + "Microsoft.DotNet.CMake.Sdk": "6.0.0-beta.22411.3" } }