diff --git a/eng/Publishing.props b/eng/Publishing.props index 30a74f1ac..1a6232b31 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -60,15 +60,16 @@ DependsOnTargets="SetSdkVersionInfo" Condition=" '$(PublishSdkAssetsAndChecksumsToBlob)' == 'true' "> - + $(BlobStoragePartialRelativePath)/$(SdkVersion)/$([System.String]::Copy('%(Filename)%(Extension)').Replace('\' ,'/')) - + $(BlobStoragePartialRelativePath)/$(SdkVersion)/$([System.String]::Copy('%(Filename)%(Extension)').Replace('\' ,'/')) + NonShipping=true - + $(BlobStoragePartialRelativePath)/$(SdkVersion)/$([System.String]::Copy('%(Filename)%(Extension)').Replace('\' ,'/')) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 3d7c13578..efad05867 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -14,45 +14,45 @@ https://github.com/dotnet/core-setup df2b1489de5d4b8211c5e132d299a83523ce742a - + https://github.com/aspnet/AspNetCore - c5f85986e62dabfc0b7f2f2a45dc7c22e8ac815f + 8057775f5891275632d0b8deda9f71975481be0e - + https://github.com/aspnet/AspNetCore - c5f85986e62dabfc0b7f2f2a45dc7c22e8ac815f + 8057775f5891275632d0b8deda9f71975481be0e - + https://github.com/aspnet/AspNetCore - c5f85986e62dabfc0b7f2f2a45dc7c22e8ac815f + 8057775f5891275632d0b8deda9f71975481be0e - + https://github.com/aspnet/AspNetCore - c5f85986e62dabfc0b7f2f2a45dc7c22e8ac815f + 8057775f5891275632d0b8deda9f71975481be0e - + https://github.com/aspnet/AspNetCore - c5f85986e62dabfc0b7f2f2a45dc7c22e8ac815f + 8057775f5891275632d0b8deda9f71975481be0e - + https://github.com/aspnet/AspNetCore - c5f85986e62dabfc0b7f2f2a45dc7c22e8ac815f + 8057775f5891275632d0b8deda9f71975481be0e https://github.com/dotnet/templating 3ce7695b54e89523ff6e116742472e3c5f4d19bb - + https://github.com/dotnet/toolset - 27d7b970280da0d60b5b2fdb5e7f70ea63e8fefb + 4dd3ebe8036004016a2275e1b83af1adcc0b2a8a - + https://github.com/dotnet/sdk - 9a92095d8ad0112291573385dc8f67adfdfe9322 + e01aac6709b8cd81522fa23677a8e49d004983ae - + https://github.com/dotnet/cli - 99151dfa08364242d93c861568abaf044e134d8c + 9a1bfd48934d87d02731eac7238f193ab9350c41 @@ -65,9 +65,9 @@ - + https://github.com/dotnet/arcade - e6712584bba6e2f0e35a3704793c459ff97c09af + 89fab80685c91024c8f9e21f1c37f62580f648f8 diff --git a/eng/Versions.props b/eng/Versions.props index d1c915f37..936ff7013 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -12,7 +12,7 @@ preview - 3.0.100-preview6.19351.2 + 3.0.100-preview7.19351.7 @@ -24,16 +24,16 @@ - 3.0.0-preview6.19307.2 - 3.0.0-preview6.19307.2 - 3.0.0-preview6.19307.2 - 3.0.0-preview6.19307.2 - 3.0.0-preview6.19307.2 - 3.0.0-preview6.19307.2 + 3.0.0-preview7.19351.4 + 3.0.0-preview7.19351.4 + 3.0.0-preview7.19351.4 + 3.0.0-preview7.19351.4 + 3.0.0-preview7.19351.4 + 3.0.0-preview7.19351.4 - 3.0.100-preview6.19304.3 + 3.0.100-preview7.19351.9 2.0.0-preview7.19351.3 @@ -44,7 +44,7 @@ - 3.0.100-preview6.19303.1 + 3.0.100-preview7.19351.3 3.0.0-preview7-27901-06 @@ -73,6 +73,7 @@ $(RestoreSources); + https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json; https://dotnet.myget.org/F/msbuild/api/v3/index.json; https://dotnet.myget.org/F/nuget-build/api/v3/index.json; https://dotnet.myget.org/F/templating/api/v3/index.json; diff --git a/eng/common/PSScriptAnalyzerSettings.psd1 b/eng/common/PSScriptAnalyzerSettings.psd1 new file mode 100644 index 000000000..4c1ea7c98 --- /dev/null +++ b/eng/common/PSScriptAnalyzerSettings.psd1 @@ -0,0 +1,11 @@ +@{ + IncludeRules=@('PSAvoidUsingCmdletAliases', + 'PSAvoidUsingWMICmdlet', + 'PSAvoidUsingPositionalParameters', + 'PSAvoidUsingInvokeExpression', + 'PSUseDeclaredVarsMoreThanAssignments', + 'PSUseCmdletCorrectly', + 'PSStandardDSCFunctionsInResource', + 'PSUseIdenticalMandatoryParametersForDSC', + 'PSUseIdenticalParametersForDSC') +} \ No newline at end of file diff --git a/eng/common/PublishToPackageFeed.proj b/eng/common/PublishToPackageFeed.proj index e17f72644..a1b133372 100644 --- a/eng/common/PublishToPackageFeed.proj +++ b/eng/common/PublishToPackageFeed.proj @@ -53,6 +53,8 @@ https://dotnetfeed.blob.core.windows.net/dotnet-toolset/index.json https://dotnetfeed.blob.core.windows.net/dotnet-windowsdesktop/index.json https://dotnetfeed.blob.core.windows.net/nuget-nugetclient/index.json + https://dotnetfeed.blob.core.windows.net/aspnet-entityframework6/index.json + https://dotnetfeed.blob.core.windows.net/aspnet-blazor/index.json Build configuration: 'Debug' or 'Release' (short: -c)" + Write-Host " -platform Platform configuration: 'x86', 'x64' or any valid Platform value to pass to msbuild" Write-Host " -verbosity Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic] (short: -v)" Write-Host " -binaryLog Output binary log (short: -bl)" Write-Host " -help Print help and exit" @@ -77,6 +79,7 @@ function Build { InitializeCustomToolset $bl = if ($binaryLog) { "/bl:" + (Join-Path $LogDir "Build.binlog") } else { "" } + $platformArg = if ($platform) { "/p:Platform=$platform" } else { "" } if ($projects) { # Re-assign properties to a new variable because PowerShell doesn't let us append properties directly for unclear reasons. @@ -88,6 +91,7 @@ function Build { MSBuild $toolsetBuildProj ` $bl ` + $platformArg ` /p:Configuration=$configuration ` /p:RepoRoot=$RepoRoot ` /p:Restore=$restore ` @@ -129,9 +133,8 @@ try { Build } catch { - Write-Host $_ - Write-Host $_.Exception Write-Host $_.ScriptStackTrace + Write-PipelineTelemetryError -Category "InitializeToolset" -Message $_ ExitWithExitCode 1 } diff --git a/eng/common/build.sh b/eng/common/build.sh index ce846d888..6236fc4d3 100755 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -66,6 +66,7 @@ ci=false warn_as_error=true node_reuse=true binary_log=false +pipelines_log=false projects='' configuration='Debug' @@ -92,6 +93,9 @@ while [[ $# > 0 ]]; do -binarylog|-bl) binary_log=true ;; + -pipelineslog|-pl) + pipelines_log=true + ;; -restore|-r) restore=true ;; @@ -146,6 +150,7 @@ while [[ $# > 0 ]]; do done if [[ "$ci" == true ]]; then + pipelines_log=true binary_log=true node_reuse=false fi diff --git a/eng/common/cross/arm64/sources.list.buster b/eng/common/cross/arm64/sources.list.buster new file mode 100644 index 000000000..7194ac64a --- /dev/null +++ b/eng/common/cross/arm64/sources.list.buster @@ -0,0 +1,11 @@ +deb http://deb.debian.org/debian buster main +deb-src http://deb.debian.org/debian buster main + +deb http://deb.debian.org/debian-security/ buster/updates main +deb-src http://deb.debian.org/debian-security/ buster/updates main + +deb http://deb.debian.org/debian buster-updates main +deb-src http://deb.debian.org/debian buster-updates main + +deb http://deb.debian.org/debian buster-backports main contrib non-free +deb-src http://deb.debian.org/debian buster-backports main contrib non-free diff --git a/eng/common/cross/arm64/sources.list.stretch b/eng/common/cross/arm64/sources.list.stretch new file mode 100644 index 000000000..0e1215774 --- /dev/null +++ b/eng/common/cross/arm64/sources.list.stretch @@ -0,0 +1,12 @@ +deb http://deb.debian.org/debian stretch main +deb-src http://deb.debian.org/debian stretch main + +deb http://deb.debian.org/debian-security/ stretch/updates main +deb-src http://deb.debian.org/debian-security/ stretch/updates main + +deb http://deb.debian.org/debian stretch-updates main +deb-src http://deb.debian.org/debian stretch-updates main + +deb http://deb.debian.org/debian stretch-backports main contrib non-free +deb-src http://deb.debian.org/debian stretch-backports main contrib non-free + diff --git a/eng/common/cross/armel/tizen-fetch.sh b/eng/common/cross/armel/tizen-fetch.sh index ba16e991c..ed70e0a86 100644 --- a/eng/common/cross/armel/tizen-fetch.sh +++ b/eng/common/cross/armel/tizen-fetch.sh @@ -157,15 +157,15 @@ fetch_tizen_pkgs() Inform "Initialize arm base" fetch_tizen_pkgs_init standard base Inform "fetch common packages" -fetch_tizen_pkgs armv7l gcc glibc glibc-devel libicu libicu-devel +fetch_tizen_pkgs armv7l gcc glibc glibc-devel libicu libicu-devel libatomic fetch_tizen_pkgs noarch linux-glibc-devel Inform "fetch coreclr packages" -fetch_tizen_pkgs armv7l lldb lldb-devel libgcc libstdc++ libstdc++-devel libunwind libunwind-devel tizen-release lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu +fetch_tizen_pkgs armv7l lldb lldb-devel libgcc libstdc++ libstdc++-devel libunwind libunwind-devel lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu Inform "fetch corefx packages" fetch_tizen_pkgs armv7l libcom_err libcom_err-devel zlib zlib-devel libopenssl libopenssl-devel krb5 krb5-devel libcurl libcurl-devel Inform "Initialize standard unified" fetch_tizen_pkgs_init standard unified Inform "fetch corefx packages" -fetch_tizen_pkgs armv7l gssdp gssdp-devel +fetch_tizen_pkgs armv7l gssdp gssdp-devel tizen-release diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index 83ec39195..d7d5d7d5f 100644 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -2,7 +2,7 @@ usage() { - echo "Usage: $0 [BuildArch] [LinuxCodeName] [lldbx.y] [--skipunmount] --rootfs ]" + echo "Usage: $0 [BuildArch] [LinuxCodeName] [lldbx.y] [--skipunmount] --rootfsdir ]" echo "BuildArch can be: arm(default), armel, arm64, x86" echo "LinuxCodeName - optional, Code name for Linux, can be: trusty, xenial(default), zesty, bionic, alpine. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen." echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine" @@ -113,12 +113,12 @@ while :; do __LinuxCodeName=trusty fi ;; - xenial) # Ubunry 16.04 + xenial) # Ubuntu 16.04 if [ "$__LinuxCodeName" != "jessie" ]; then __LinuxCodeName=xenial fi ;; - zesty) # Ununtu 17.04 + zesty) # Ubuntu 17.04 if [ "$__LinuxCodeName" != "jessie" ]; then __LinuxCodeName=zesty fi @@ -132,7 +132,16 @@ while :; do __LinuxCodeName=jessie __UbuntuRepo="http://ftp.debian.org/debian/" ;; - # TBD Stretch -> Debian 9, Buster -> Debian 10 + stretch) # Debian 9 + __LinuxCodeName=stretch + __UbuntuRepo="http://ftp.debian.org/debian/" + __LLDB_Package="liblldb-6.0-dev" + ;; + buster) # Debian 10 + __LinuxCodeName=buster + __UbuntuRepo="http://ftp.debian.org/debian/" + __LLDB_Package="liblldb-6.0-dev" + ;; tizen) if [ "$__BuildArch" != "armel" ]; then echo "Tizen is available only for armel." @@ -172,7 +181,7 @@ if [ -z "$__RootfsDir" ] && [ ! -z "$ROOTFS_DIR" ]; then fi if [ -z "$__RootfsDir" ]; then - __RootfsDir="$__CrossDir/rootfs/$__BuildArch" + __RootfsDir="$__CrossDir/../../../.tools/rootfs/$__BuildArch" fi if [ -d "$__RootfsDir" ]; then @@ -194,6 +203,7 @@ if [[ "$__LinuxCodeName" == "alpine" ]]; then -X http://dl-cdn.alpinelinux.org/alpine/v$__AlpineVersion/main \ -X http://dl-cdn.alpinelinux.org/alpine/v$__AlpineVersion/community \ -X http://dl-cdn.alpinelinux.org/alpine/edge/testing \ + -X http://dl-cdn.alpinelinux.org/alpine/edge/main \ -U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \ add $__AlpinePackages rm -r $__ApkToolsDir diff --git a/eng/common/darc-init.ps1 b/eng/common/darc-init.ps1 index 823d72a37..8854d979f 100644 --- a/eng/common/darc-init.ps1 +++ b/eng/common/darc-init.ps1 @@ -1,5 +1,6 @@ param ( - $darcVersion = $null + $darcVersion = $null, + $versionEndpoint = "https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16" ) $verbosity = "m" @@ -10,23 +11,23 @@ function InstallDarcCli ($darcVersion) { $dotnetRoot = InitializeDotNetCli -install:$true $dotnet = "$dotnetRoot\dotnet.exe" - $toolList = Invoke-Expression "& `"$dotnet`" tool list -g" + $toolList = & "$dotnet" tool list -g if ($toolList -like "*$darcCliPackageName*") { - Invoke-Expression "& `"$dotnet`" tool uninstall $darcCliPackageName -g" + & "$dotnet" tool uninstall $darcCliPackageName -g } - # Until we can anonymously query the BAR API for the latest arcade-services - # build applied to the PROD channel, this is hardcoded. + # If the user didn't explicitly specify the darc version, + # query the Maestro API for the correct version of darc to install. if (-not $darcVersion) { - $darcVersion = '1.1.0-beta.19258.3' + $darcVersion = $(Invoke-WebRequest -Uri $versionEndpoint -UseBasicParsing).Content } - + $arcadeServicesSource = 'https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json' Write-Host "Installing Darc CLI version $darcVersion..." Write-Host "You may need to restart your command window if this is the first dotnet tool you have installed." - Invoke-Expression "& `"$dotnet`" tool install $darcCliPackageName --version $darcVersion --add-source '$arcadeServicesSource' -v $verbosity -g" + & "$dotnet" tool install $darcCliPackageName --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g } InstallDarcCli $darcVersion diff --git a/eng/common/darc-init.sh b/eng/common/darc-init.sh index 36d675738..abdd0bc05 100755 --- a/eng/common/darc-init.sh +++ b/eng/common/darc-init.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash source="${BASH_SOURCE[0]}" -darcVersion="1.1.0-beta.19258.3" +darcVersion='' +versionEndpoint="https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16" while [[ $# > 0 ]]; do opt="$(echo "$1" | awk '{print tolower($0)}')" @@ -10,6 +11,10 @@ while [[ $# > 0 ]]; do darcVersion=$2 shift ;; + --versionendpoint) + versionEndpoint=$2 + shift + ;; *) echo "Invalid argument: $1" usage @@ -33,6 +38,10 @@ verbosity=m . "$scriptroot/tools.sh" +if [ -z "$darcVersion" ]; then + darcVersion=$(curl -X GET "$versionEndpoint" -H "accept: text/plain") +fi + function InstallDarcCli { local darc_cli_package_name="microsoft.dotnet.darc" @@ -47,7 +56,7 @@ function InstallDarcCli { local arcadeServicesSource="https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json" - echo "Installing Darc CLI version $toolset_version..." + echo "Installing Darc CLI version $darcVersion..." echo "You may need to restart your command shell if this is the first dotnet tool you have installed." echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g) } diff --git a/eng/common/dotnet-install.ps1 b/eng/common/dotnet-install.ps1 index 5987943fd..0b629b830 100644 --- a/eng/common/dotnet-install.ps1 +++ b/eng/common/dotnet-install.ps1 @@ -8,9 +8,14 @@ Param( . $PSScriptRoot\tools.ps1 +$dotnetRoot = Join-Path $RepoRoot ".dotnet" + +$installdir = $dotnetRoot try { - $dotnetRoot = Join-Path $RepoRoot ".dotnet" - InstallDotNet $dotnetRoot $version $architecture $runtime $true + if ($architecture -and $architecture.Trim() -eq "x86") { + $installdir = Join-Path $installdir "x86" + } + InstallDotNet $installdir $version $architecture $runtime $true } catch { Write-Host $_ @@ -19,4 +24,4 @@ catch { ExitWithExitCode 1 } -ExitWithExitCode 0 \ No newline at end of file +ExitWithExitCode 0 diff --git a/eng/common/generate-graph-files.ps1 b/eng/common/generate-graph-files.ps1 index a05b84f79..b056e4c1a 100644 --- a/eng/common/generate-graph-files.ps1 +++ b/eng/common/generate-graph-files.ps1 @@ -25,7 +25,7 @@ function CheckExitCode ([string]$stage) try { Push-Location $PSScriptRoot - + Write-Host "Installing darc..." . .\darc-init.ps1 -darcVersion $darcVersion CheckExitCode "Running darc-init" @@ -40,9 +40,9 @@ try { $darcExe = "$env:USERPROFILE\.dotnet\tools" $darcExe = Resolve-Path "$darcExe\darc.exe" - + Create-Directory $outputFolder - + # Generate 3 graph descriptions: # 1. Flat with coherency information # 2. Graphviz (dot) file @@ -51,26 +51,26 @@ try { $graphVizImageFilePath = "$outputFolder\graph.png" $normalGraphFilePath = "$outputFolder\graph-full.txt" $flatGraphFilePath = "$outputFolder\graph-flat.txt" - $baseOptions = "get-dependency-graph --github-pat $gitHubPat --azdev-pat $azdoPat --password $barToken" - + $baseOptions = @( "--github-pat", "$gitHubPat", "--azdev-pat", "$azdoPat", "--password", "$barToken" ) + if ($includeToolset) { Write-Host "Toolsets will be included in the graph..." - $baseOptions += " --include-toolset" + $baseOptions += @( "--include-toolset" ) } Write-Host "Generating standard dependency graph..." - Invoke-Expression "& `"$darcExe`" $baseOptions --output-file $normalGraphFilePath" + & "$darcExe" get-dependency-graph @baseOptions --output-file $normalGraphFilePath CheckExitCode "Generating normal dependency graph" Write-Host "Generating flat dependency graph and graphviz file..." - Invoke-Expression "& `"$darcExe`" $baseOptions --flat --coherency --graphviz $graphVizFilePath --output-file $flatGraphFilePath" + & "$darcExe" get-dependency-graph @baseOptions --flat --coherency --graphviz $graphVizFilePath --output-file $flatGraphFilePath CheckExitCode "Generating flat and graphviz dependency graph" Write-Host "Generating graph image $graphVizFilePath" $dotFilePath = Join-Path $installBin "graphviz\$graphvizVersion\release\bin\dot.exe" - Invoke-Expression "& `"$dotFilePath`" -Tpng -o'$graphVizImageFilePath' `"$graphVizFilePath`"" + & "$dotFilePath" -Tpng -o"$graphVizImageFilePath" "$graphVizFilePath" CheckExitCode "Generating graphviz image" - + Write-Host "'$graphVizFilePath', '$flatGraphFilePath', '$normalGraphFilePath' and '$graphVizImageFilePath' created!" } catch { diff --git a/eng/common/init-tools-native.ps1 b/eng/common/init-tools-native.ps1 index a4306bd37..9d18645f4 100644 --- a/eng/common/init-tools-native.ps1 +++ b/eng/common/init-tools-native.ps1 @@ -79,28 +79,27 @@ try { $NativeTools.PSObject.Properties | ForEach-Object { $ToolName = $_.Name $ToolVersion = $_.Value - $LocalInstallerCommand = $InstallerPath - $LocalInstallerCommand += " -ToolName $ToolName" - $LocalInstallerCommand += " -InstallPath $InstallBin" - $LocalInstallerCommand += " -BaseUri $BaseUri" - $LocalInstallerCommand += " -CommonLibraryDirectory $EngCommonBaseDir" - $LocalInstallerCommand += " -Version $ToolVersion" + $LocalInstallerArguments = @{ ToolName = "$ToolName" } + $LocalInstallerArguments += @{ InstallPath = "$InstallBin" } + $LocalInstallerArguments += @{ BaseUri = "$BaseUri" } + $LocalInstallerArguments += @{ CommonLibraryDirectory = "$EngCommonBaseDir" } + $LocalInstallerArguments += @{ Version = "$ToolVersion" } if ($Verbose) { - $LocalInstallerCommand += " -Verbose" + $LocalInstallerArguments += @{ Verbose = $True } } if (Get-Variable 'Force' -ErrorAction 'SilentlyContinue') { if($Force) { - $LocalInstallerCommand += " -Force" + $LocalInstallerArguments += @{ Force = $True } } } if ($Clean) { - $LocalInstallerCommand += " -Clean" + $LocalInstallerArguments += @{ Clean = $True } } Write-Verbose "Installing $ToolName version $ToolVersion" - Write-Verbose "Executing '$LocalInstallerCommand'" - Invoke-Expression "$LocalInstallerCommand" + Write-Verbose "Executing '$InstallerPath $LocalInstallerArguments'" + & $InstallerPath @LocalInstallerArguments if ($LASTEXITCODE -Ne "0") { $errMsg = "$ToolName installation failed" if ((Get-Variable 'DoNotAbortNativeToolsInstallationOnFailure' -ErrorAction 'SilentlyContinue') -and $DoNotAbortNativeToolsInstallationOnFailure) { diff --git a/eng/common/internal-feed-operations.ps1 b/eng/common/internal-feed-operations.ps1 new file mode 100644 index 000000000..8b8bafd6a --- /dev/null +++ b/eng/common/internal-feed-operations.ps1 @@ -0,0 +1,135 @@ +param( + [Parameter(Mandatory=$true)][string] $Operation, + [string] $AuthToken, + [string] $CommitSha, + [string] $RepoName, + [switch] $IsFeedPrivate +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 + +. $PSScriptRoot\tools.ps1 + +# Sets VSS_NUGET_EXTERNAL_FEED_ENDPOINTS based on the "darc-int-*" feeds defined in NuGet.config. This is needed +# in build agents by CredProvider to authenticate the restore requests to internal feeds as specified in +# https://github.com/microsoft/artifacts-credprovider/blob/0f53327cd12fd893d8627d7b08a2171bf5852a41/README.md#environment-variables. This should ONLY be called from identified +# internal builds +function SetupCredProvider { + param( + [string] $AuthToken + ) + + # Install the Cred Provider NuGet plugin + Write-Host "Setting up Cred Provider NuGet plugin in the agent..." + Write-Host "Getting 'installcredprovider.ps1' from 'https://github.com/microsoft/artifacts-credprovider'..." + + $url = 'https://raw.githubusercontent.com/microsoft/artifacts-credprovider/master/helpers/installcredprovider.ps1' + + Write-Host "Writing the contents of 'installcredprovider.ps1' locally..." + Invoke-WebRequest $url -OutFile installcredprovider.ps1 + + Write-Host "Installing plugin..." + .\installcredprovider.ps1 -Force + + Write-Host "Deleting local copy of 'installcredprovider.ps1'..." + Remove-Item .\installcredprovider.ps1 + + if (-Not("$env:USERPROFILE\.nuget\plugins\netcore")) { + Write-Host "CredProvider plugin was not installed correctly!" + ExitWithExitCode 1 + } + else { + Write-Host "CredProvider plugin was installed correctly!" + } + + # Then, we set the 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' environment variable to restore from the stable + # feeds successfully + + $nugetConfigPath = "$RepoRoot\NuGet.config" + + if (-Not (Test-Path -Path $nugetConfigPath)) { + Write-Host "NuGet.config file not found in repo's root!" + ExitWithExitCode 1 + } + + $endpoints = New-Object System.Collections.ArrayList + $nugetConfigPackageSources = Select-Xml -Path $nugetConfigPath -XPath "//packageSources/add[contains(@key, 'darc-int-')]/@value" | foreach{$_.Node.Value} + + if (($nugetConfigPackageSources | Measure-Object).Count -gt 0 ) { + foreach ($stableRestoreResource in $nugetConfigPackageSources) { + $trimmedResource = ([string]$stableRestoreResource).Trim() + [void]$endpoints.Add(@{endpoint="$trimmedResource"; password="$AuthToken"}) + } + } + + if (($endpoints | Measure-Object).Count -gt 0) { + # Create the JSON object. It should look like '{"endpointCredentials": [{"endpoint":"http://example.index.json", "username":"optional", "password":"accesstoken"}]}' + $endpointCredentials = @{endpointCredentials=$endpoints} | ConvertTo-Json -Compress + + # Create the environment variables the AzDo way + Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data $endpointCredentials -Properties @{ + 'variable' = 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' + 'issecret' = 'false' + } + + # We don't want sessions cached since we will be updating the endpoints quite frequently + Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data 'False' -Properties @{ + 'variable' = 'NUGET_CREDENTIALPROVIDER_SESSIONTOKENCACHE_ENABLED' + 'issecret' = 'false' + } + } + else + { + Write-Host "No internal endpoints found in NuGet.config" + } +} + +#Workaround for https://github.com/microsoft/msbuild/issues/4430 +function InstallDotNetSdkAndRestoreArcade { + $dotnetTempDir = "$RepoRoot\dotnet" + $dotnetSdkVersion="2.1.507" # After experimentation we know this version works when restoring the SDK (compared to 3.0.*) + $dotnet = "$dotnetTempDir\dotnet.exe" + $restoreProjPath = "$PSScriptRoot\restore.proj" + + Write-Host "Installing dotnet SDK version $dotnetSdkVersion to restore Arcade SDK..." + InstallDotNetSdk "$dotnetTempDir" "$dotnetSdkVersion" + + '' | Out-File "$restoreProjPath" + + & $dotnet restore $restoreProjPath + + Write-Host "Arcade SDK restored!" + + if (Test-Path -Path $restoreProjPath) { + Remove-Item $restoreProjPath + } + + if (Test-Path -Path $dotnetTempDir) { + Remove-Item $dotnetTempDir -Recurse + } +} + +try { + Push-Location $PSScriptRoot + + if ($Operation -like "setup") { + SetupCredProvider $AuthToken + } + elseif ($Operation -like "install-restore") { + InstallDotNetSdkAndRestoreArcade + } + else { + Write-Host "Unknown operation '$Operation'!" + ExitWithExitCode 1 + } +} +catch { + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + ExitWithExitCode 1 +} +finally { + Pop-Location +} diff --git a/eng/common/internal-feed-operations.sh b/eng/common/internal-feed-operations.sh new file mode 100644 index 000000000..1ff654d2f --- /dev/null +++ b/eng/common/internal-feed-operations.sh @@ -0,0 +1,142 @@ +#!/usr/bin/env bash + +set -e + +# Sets VSS_NUGET_EXTERNAL_FEED_ENDPOINTS based on the "darc-int-*" feeds defined in NuGet.config. This is needed +# in build agents by CredProvider to authenticate the restore requests to internal feeds as specified in +# https://github.com/microsoft/artifacts-credprovider/blob/0f53327cd12fd893d8627d7b08a2171bf5852a41/README.md#environment-variables. +# This should ONLY be called from identified internal builds +function SetupCredProvider { + local authToken=$1 + + # Install the Cred Provider NuGet plugin + echo "Setting up Cred Provider NuGet plugin in the agent..."... + echo "Getting 'installcredprovider.ps1' from 'https://github.com/microsoft/artifacts-credprovider'..." + + local url="https://raw.githubusercontent.com/microsoft/artifacts-credprovider/master/helpers/installcredprovider.sh" + + echo "Writing the contents of 'installcredprovider.ps1' locally..." + local installcredproviderPath="installcredprovider.sh" + if command -v curl > /dev/null; then + curl $url > "$installcredproviderPath" + else + wget -q -O "$installcredproviderPath" "$url" + fi + + echo "Installing plugin..." + . "$installcredproviderPath" + + echo "Deleting local copy of 'installcredprovider.sh'..." + rm installcredprovider.sh + + if [ ! -d "$HOME/.nuget/plugins" ]; then + echo "CredProvider plugin was not installed correctly!" + ExitWithExitCode 1 + else + echo "CredProvider plugin was installed correctly!" + fi + + # Then, we set the 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' environment variable to restore from the stable + # feeds successfully + + local nugetConfigPath="$repo_root/NuGet.config" + + if [ ! "$nugetConfigPath" ]; then + echo "NuGet.config file not found in repo's root!" + ExitWithExitCode 1 + fi + + local endpoints='[' + local nugetConfigPackageValues=`cat "$nugetConfigPath" | grep "key=\"darc-int-"` + local pattern="value=\"(.*)\"" + + for value in $nugetConfigPackageValues + do + if [[ $value =~ $pattern ]]; then + local endpoint="${BASH_REMATCH[1]}" + endpoints+="{\"endpoint\": \"$endpoint\", \"password\": \"$authToken\"}," + fi + done + + endpoints=${endpoints%?} + endpoints+=']' + + if [ ${#endpoints} -gt 2 ]; then + # Create the JSON object. It should look like '{"endpointCredentials": [{"endpoint":"http://example.index.json", "username":"optional", "password":"accesstoken"}]}' + local endpointCredentials="{\"endpointCredentials\": "$endpoints"}" + + echo "##vso[task.setvariable variable=VSS_NUGET_EXTERNAL_FEED_ENDPOINTS]$endpointCredentials" + echo "##vso[task.setvariable variable=NUGET_CREDENTIALPROVIDER_SESSIONTOKENCACHE_ENABLED]False" + else + echo "No internal endpoints found in NuGet.config" + fi +} + +# Workaround for https://github.com/microsoft/msbuild/issues/4430 +function InstallDotNetSdkAndRestoreArcade { + local dotnetTempDir="$repo_root/dotnet" + local dotnetSdkVersion="2.1.507" # After experimentation we know this version works when restoring the SDK (compared to 3.0.*) + local restoreProjPath="$repo_root/eng/common/restore.proj" + + echo "Installing dotnet SDK version $dotnetSdkVersion to restore Arcade SDK..." + echo "" > "$restoreProjPath" + + InstallDotNetSdk "$dotnetTempDir" "$dotnetSdkVersion" + + local res=`$dotnetTempDir/dotnet restore $restoreProjPath` + echo "Arcade SDK restored!" + + # Cleanup + if [ "$restoreProjPath" ]; then + rm "$restoreProjPath" + fi + + if [ "$dotnetTempDir" ]; then + rm -r $dotnetTempDir + fi +} + +source="${BASH_SOURCE[0]}" +operation='' +authToken='' +repoName='' + +while [[ $# > 0 ]]; do + opt="$(echo "$1" | awk '{print tolower($0)}')" + case "$opt" in + --operation) + operation=$2 + shift + ;; + --authtoken) + authToken=$2 + shift + ;; + *) + echo "Invalid argument: $1" + usage + exit 1 + ;; + esac + + shift +done + +while [[ -h "$source" ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + # if $source was a relative symlink, we need to resolve it relative to the path where the + # symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + +. "$scriptroot/tools.sh" + +if [ "$operation" = "setup" ]; then + SetupCredProvider $authToken +elif [ "$operation" = "install-restore" ]; then + InstallDotNetSdkAndRestoreArcade +else + echo "Unknown operation '$operation'!" +fi diff --git a/eng/common/native/CommonLibrary.psm1 b/eng/common/native/CommonLibrary.psm1 index f286ae0cd..7a34c7e8a 100644 --- a/eng/common/native/CommonLibrary.psm1 +++ b/eng/common/native/CommonLibrary.psm1 @@ -209,7 +209,7 @@ function New-ScriptShim { Remove-Item (Join-Path $ShimDirectory "$ShimName.exe") } - Invoke-Expression "$ShimDirectory\WinShimmer\winshimmer.exe $ShimName $ToolFilePath $ShimDirectory" + & "$ShimDirectory\WinShimmer\winshimmer.exe" $ShimName $ToolFilePath $ShimDirectory return $True } catch { diff --git a/eng/common/pipeline-logging-functions.ps1 b/eng/common/pipeline-logging-functions.ps1 new file mode 100644 index 000000000..7b61376f8 --- /dev/null +++ b/eng/common/pipeline-logging-functions.ps1 @@ -0,0 +1,233 @@ +# Source for this file was taken from https://github.com/microsoft/azure-pipelines-task-lib/blob/11c9439d4af17e6475d9fe058e6b2e03914d17e6/powershell/VstsTaskSdk/LoggingCommandFunctions.ps1 and modified. + +# NOTE: You should not be calling these method directly as they are likely to change. Instead you should be calling the Write-Pipeline* functions defined in tools.ps1 + +$script:loggingCommandPrefix = '##vso[' +$script:loggingCommandEscapeMappings = @( # TODO: WHAT ABOUT "="? WHAT ABOUT "%"? + New-Object psobject -Property @{ Token = ';' ; Replacement = '%3B' } + New-Object psobject -Property @{ Token = "`r" ; Replacement = '%0D' } + New-Object psobject -Property @{ Token = "`n" ; Replacement = '%0A' } + New-Object psobject -Property @{ Token = "]" ; Replacement = '%5D' } +) +# TODO: BUG: Escape % ??? +# TODO: Add test to verify don't need to escape "=". + +function Write-PipelineTelemetryError { + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] + [string]$Category, + [Parameter(Mandatory = $true)] + [string]$Message, + [Parameter(Mandatory = $false)] + [string]$Type = 'error', + [string]$ErrCode, + [string]$SourcePath, + [string]$LineNumber, + [string]$ColumnNumber, + [switch]$AsOutput) + + $PSBoundParameters.Remove("Category") | Out-Null + + $Message = "(NETCORE_ENGINEERING_TELEMETRY=$Category) $Message" + $PSBoundParameters.Remove("Message") | Out-Null + $PSBoundParameters.Add("Message", $Message) + + Write-PipelineTaskError @PSBoundParameters +} + +function Write-PipelineTaskError { + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] + [string]$Message, + [Parameter(Mandatory = $false)] + [string]$Type = 'error', + [string]$ErrCode, + [string]$SourcePath, + [string]$LineNumber, + [string]$ColumnNumber, + [switch]$AsOutput) + + if(!$ci) { + if($Type -eq 'error') { + Write-Host $Message -ForegroundColor Red + return + } + elseif ($Type -eq 'warning') { + Write-Host $Message -ForegroundColor Yellow + return + } + } + + if(($Type -ne 'error') -and ($Type -ne 'warning')) { + Write-Host $Message + return + } + if(-not $PSBoundParameters.ContainsKey('Type')) { + $PSBoundParameters.Add('Type', 'error') + } + Write-LogIssue @PSBoundParameters + } + + function Write-PipelineSetVariable { + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] + [string]$Name, + [string]$Value, + [switch]$Secret, + [switch]$AsOutput) + + if($ci) { + Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data $Value -Properties @{ + 'variable' = $Name + 'isSecret' = $Secret + 'isOutput' = 'true' + } -AsOutput:$AsOutput + } + } + + function Write-PipelinePrependPath { + [CmdletBinding()] + param( + [Parameter(Mandatory=$true)] + [string]$Path, + [switch]$AsOutput) + if($ci) { + Write-LoggingCommand -Area 'task' -Event 'prependpath' -Data $Path -AsOutput:$AsOutput + } + } + +<######################################## +# Private functions. +########################################> +function Format-LoggingCommandData { + [CmdletBinding()] + param([string]$Value, [switch]$Reverse) + + if (!$Value) { + return '' + } + + if (!$Reverse) { + foreach ($mapping in $script:loggingCommandEscapeMappings) { + $Value = $Value.Replace($mapping.Token, $mapping.Replacement) + } + } else { + for ($i = $script:loggingCommandEscapeMappings.Length - 1 ; $i -ge 0 ; $i--) { + $mapping = $script:loggingCommandEscapeMappings[$i] + $Value = $Value.Replace($mapping.Replacement, $mapping.Token) + } + } + + return $Value +} + +function Format-LoggingCommand { + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] + [string]$Area, + [Parameter(Mandatory = $true)] + [string]$Event, + [string]$Data, + [hashtable]$Properties) + + # Append the preamble. + [System.Text.StringBuilder]$sb = New-Object -TypeName System.Text.StringBuilder + $null = $sb.Append($script:loggingCommandPrefix).Append($Area).Append('.').Append($Event) + + # Append the properties. + if ($Properties) { + $first = $true + foreach ($key in $Properties.Keys) { + [string]$value = Format-LoggingCommandData $Properties[$key] + if ($value) { + if ($first) { + $null = $sb.Append(' ') + $first = $false + } else { + $null = $sb.Append(';') + } + + $null = $sb.Append("$key=$value") + } + } + } + + # Append the tail and output the value. + $Data = Format-LoggingCommandData $Data + $sb.Append(']').Append($Data).ToString() +} + +function Write-LoggingCommand { + [CmdletBinding(DefaultParameterSetName = 'Parameters')] + param( + [Parameter(Mandatory = $true, ParameterSetName = 'Parameters')] + [string]$Area, + [Parameter(Mandatory = $true, ParameterSetName = 'Parameters')] + [string]$Event, + [Parameter(ParameterSetName = 'Parameters')] + [string]$Data, + [Parameter(ParameterSetName = 'Parameters')] + [hashtable]$Properties, + [Parameter(Mandatory = $true, ParameterSetName = 'Object')] + $Command, + [switch]$AsOutput) + + if ($PSCmdlet.ParameterSetName -eq 'Object') { + Write-LoggingCommand -Area $Command.Area -Event $Command.Event -Data $Command.Data -Properties $Command.Properties -AsOutput:$AsOutput + return + } + + $command = Format-LoggingCommand -Area $Area -Event $Event -Data $Data -Properties $Properties + if ($AsOutput) { + $command + } else { + Write-Host $command + } +} + +function Write-LogIssue { + [CmdletBinding()] + param( + [ValidateSet('warning', 'error')] + [Parameter(Mandatory = $true)] + [string]$Type, + [string]$Message, + [string]$ErrCode, + [string]$SourcePath, + [string]$LineNumber, + [string]$ColumnNumber, + [switch]$AsOutput) + + $command = Format-LoggingCommand -Area 'task' -Event 'logissue' -Data $Message -Properties @{ + 'type' = $Type + 'code' = $ErrCode + 'sourcepath' = $SourcePath + 'linenumber' = $LineNumber + 'columnnumber' = $ColumnNumber + } + if ($AsOutput) { + return $command + } + + if ($Type -eq 'error') { + $foregroundColor = $host.PrivateData.ErrorForegroundColor + $backgroundColor = $host.PrivateData.ErrorBackgroundColor + if ($foregroundColor -isnot [System.ConsoleColor] -or $backgroundColor -isnot [System.ConsoleColor]) { + $foregroundColor = [System.ConsoleColor]::Red + $backgroundColor = [System.ConsoleColor]::Black + } + } else { + $foregroundColor = $host.PrivateData.WarningForegroundColor + $backgroundColor = $host.PrivateData.WarningBackgroundColor + if ($foregroundColor -isnot [System.ConsoleColor] -or $backgroundColor -isnot [System.ConsoleColor]) { + $foregroundColor = [System.ConsoleColor]::Yellow + $backgroundColor = [System.ConsoleColor]::Black + } + } + + Write-Host $command -ForegroundColor $foregroundColor -BackgroundColor $backgroundColor +} \ No newline at end of file diff --git a/eng/common/pipeline-logging-functions.sh b/eng/common/pipeline-logging-functions.sh new file mode 100644 index 000000000..6098f9a54 --- /dev/null +++ b/eng/common/pipeline-logging-functions.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash + +function Write-PipelineTelemetryError { + local telemetry_category='' + local function_args=() + local message='' + while [[ $# -gt 0 ]]; do + opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')" + case "$opt" in + -category|-c) + telemetry_category=$2 + shift + ;; + -*) + function_args+=("$1 $2") + shift + ;; + *) + message=$* + ;; + esac + shift + done + + if [[ "$ci" != true ]]; then + echo "$message" >&2 + return + fi + + message="(NETCORE_ENGINEERING_TELEMETRY=$telemetry_category) $message" + function_args+=("$message") + + Write-PipelineTaskError $function_args +} + +function Write-PipelineTaskError { + if [[ "$ci" != true ]]; then + echo "$@" >&2 + return + fi + + message_type="error" + sourcepath='' + linenumber='' + columnnumber='' + error_code='' + + while [[ $# -gt 0 ]]; do + opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')" + case "$opt" in + -type|-t) + message_type=$2 + shift + ;; + -sourcepath|-s) + sourcepath=$2 + shift + ;; + -linenumber|-ln) + linenumber=$2 + shift + ;; + -columnnumber|-cn) + columnnumber=$2 + shift + ;; + -errcode|-e) + error_code=$2 + shift + ;; + *) + break + ;; + esac + + shift + done + + message="##vso[task.logissue" + + message="$message type=$message_type" + + if [ -n "$sourcepath" ]; then + message="$message;sourcepath=$sourcepath" + fi + + if [ -n "$linenumber" ]; then + message="$message;linenumber=$linenumber" + fi + + if [ -n "$columnnumber" ]; then + message="$message;columnnumber=$columnnumber" + fi + + if [ -n "$error_code" ]; then + message="$message;code=$error_code" + fi + + message="$message]$*" + echo "$message" +} + diff --git a/eng/common/post-build/dotnetsymbol-init.ps1 b/eng/common/post-build/dotnetsymbol-init.ps1 new file mode 100644 index 000000000..e7659b98c --- /dev/null +++ b/eng/common/post-build/dotnetsymbol-init.ps1 @@ -0,0 +1,29 @@ +param ( + $dotnetsymbolVersion = $null +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 + +. $PSScriptRoot\..\tools.ps1 + +$verbosity = "minimal" + +function Installdotnetsymbol ($dotnetsymbolVersion) { + $dotnetsymbolPackageName = "dotnet-symbol" + + $dotnetRoot = InitializeDotNetCli -install:$true + $dotnet = "$dotnetRoot\dotnet.exe" + $toolList = & "$dotnet" tool list --global + + if (($toolList -like "*$dotnetsymbolPackageName*") -and ($toolList -like "*$dotnetsymbolVersion*")) { + Write-Host "dotnet-symbol version $dotnetsymbolVersion is already installed." + } + else { + Write-Host "Installing dotnet-symbol version $dotnetsymbolVersion..." + Write-Host "You may need to restart your command window if this is the first dotnet tool you have installed." + & "$dotnet" tool install $dotnetsymbolPackageName --version $dotnetsymbolVersion --verbosity $verbosity --global + } +} + +Installdotnetsymbol $dotnetsymbolVersion diff --git a/eng/common/post-build/nuget-validation.ps1 b/eng/common/post-build/nuget-validation.ps1 new file mode 100644 index 000000000..1bdced1e3 --- /dev/null +++ b/eng/common/post-build/nuget-validation.ps1 @@ -0,0 +1,28 @@ +# This script validates NuGet package metadata information using this +# tool: https://github.com/NuGet/NuGetGallery/tree/jver-verify/src/VerifyMicrosoftPackage + +param( + [Parameter(Mandatory=$true)][string] $PackagesPath, # Path to where the packages to be validated are + [Parameter(Mandatory=$true)][string] $ToolDestinationPath # Where the validation tool should be downloaded to +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 + +. $PSScriptRoot\..\tools.ps1 + +try { + $url = "https://raw.githubusercontent.com/NuGet/NuGetGallery/jver-verify/src/VerifyMicrosoftPackage/verify.ps1" + + New-Item -ItemType "directory" -Path ${ToolDestinationPath} -Force + + Invoke-WebRequest $url -OutFile ${ToolDestinationPath}\verify.ps1 + + & ${ToolDestinationPath}\verify.ps1 ${PackagesPath}\*.nupkg +} +catch { + Write-PipelineTaskError "NuGet package validation failed. Please check error logs." + Write-Host $_ + Write-Host $_.ScriptStackTrace + ExitWithExitCode 1 +} diff --git a/eng/common/post-build/promote-build.ps1 b/eng/common/post-build/promote-build.ps1 new file mode 100644 index 000000000..84a608fa5 --- /dev/null +++ b/eng/common/post-build/promote-build.ps1 @@ -0,0 +1,53 @@ +param( + [Parameter(Mandatory=$true)][int] $BuildId, + [Parameter(Mandatory=$true)][int] $ChannelId, + [Parameter(Mandatory=$true)][string] $BarToken, + [string] $MaestroEndpoint = "https://maestro-prod.westus2.cloudapp.azure.com", + [string] $ApiVersion = "2019-01-16" +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 + +. $PSScriptRoot\..\tools.ps1 + +function Get-Headers([string]$accept, [string]$barToken) { + $headers = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]' + $headers.Add('Accept',$accept) + $headers.Add('Authorization',"Bearer $barToken") + return $headers +} + +try { + $maestroHeaders = Get-Headers 'application/json' $BarToken + + # Get info about which channels the build has already been promoted to + $getBuildApiEndpoint = "$MaestroEndpoint/api/builds/${BuildId}?api-version=$ApiVersion" + $buildInfo = Invoke-WebRequest -Method Get -Uri $getBuildApiEndpoint -Headers $maestroHeaders | ConvertFrom-Json + + if (!$buildInfo) { + Write-Host "Build with BAR ID $BuildId was not found in BAR!" + ExitWithExitCode 1 + } + + # Find whether the build is already assigned to the channel or not + if ($buildInfo.channels) { + foreach ($channel in $buildInfo.channels) { + if ($channel.Id -eq $ChannelId) { + Write-Host "The build with BAR ID $BuildId is already on channel $ChannelId!" + ExitWithExitCode 0 + } + } + } + + Write-Host "Build not present in channel $ChannelId. Promoting build ... " + + $promoteBuildApiEndpoint = "$maestroEndpoint/api/channels/${ChannelId}/builds/${BuildId}?api-version=$ApiVersion" + Invoke-WebRequest -Method Post -Uri $promoteBuildApiEndpoint -Headers $maestroHeaders + Write-Host "done." +} +catch { + Write-Host "There was an error while trying to promote build '$BuildId' to channel '$ChannelId'" + Write-Host $_ + Write-Host $_.ScriptStackTrace +} diff --git a/eng/common/post-build/sourcelink-cli-init.ps1 b/eng/common/post-build/sourcelink-cli-init.ps1 new file mode 100644 index 000000000..9eaa25b3b --- /dev/null +++ b/eng/common/post-build/sourcelink-cli-init.ps1 @@ -0,0 +1,29 @@ +param ( + $sourcelinkCliVersion = $null +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 + +. $PSScriptRoot\..\tools.ps1 + +$verbosity = "minimal" + +function InstallSourcelinkCli ($sourcelinkCliVersion) { + $sourcelinkCliPackageName = "sourcelink" + + $dotnetRoot = InitializeDotNetCli -install:$true + $dotnet = "$dotnetRoot\dotnet.exe" + $toolList = & "$dotnet" tool list --global + + if (($toolList -like "*$sourcelinkCliPackageName*") -and ($toolList -like "*$sourcelinkCliVersion*")) { + Write-Host "SourceLink CLI version $sourcelinkCliVersion is already installed." + } + else { + Write-Host "Installing SourceLink CLI version $sourcelinkCliVersion..." + Write-Host "You may need to restart your command window if this is the first dotnet tool you have installed." + & "$dotnet" tool install $sourcelinkCliPackageName --version $sourcelinkCliVersion --verbosity $verbosity --global + } +} + +InstallSourcelinkCli $sourcelinkCliVersion diff --git a/eng/common/post-build/sourcelink-validation.ps1 b/eng/common/post-build/sourcelink-validation.ps1 new file mode 100644 index 000000000..8abd684e9 --- /dev/null +++ b/eng/common/post-build/sourcelink-validation.ps1 @@ -0,0 +1,224 @@ +param( + [Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where Symbols.NuGet packages to be checked are stored + [Parameter(Mandatory=$true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation + [Parameter(Mandatory=$true)][string] $GHRepoName, # GitHub name of the repo including the Org. E.g., dotnet/arcade + [Parameter(Mandatory=$true)][string] $GHCommit, # GitHub commit SHA used to build the packages + [Parameter(Mandatory=$true)][string] $SourcelinkCliVersion # Version of SourceLink CLI to use +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 + +. $PSScriptRoot\..\tools.ps1 + +# Cache/HashMap (File -> Exist flag) used to consult whether a file exist +# in the repository at a specific commit point. This is populated by inserting +# all files present in the repo at a specific commit point. +$global:RepoFiles = @{} + +$ValidatePackage = { + param( + [string] $PackagePath # Full path to a Symbols.NuGet package + ) + + . $using:PSScriptRoot\..\tools.ps1 + + # Ensure input file exist + if (!(Test-Path $PackagePath)) { + Write-PipelineTaskError "Input file does not exist: $PackagePath" + ExitWithExitCode 1 + } + + # Extensions for which we'll look for SourceLink information + # For now we'll only care about Portable & Embedded PDBs + $RelevantExtensions = @(".dll", ".exe", ".pdb") + + Write-Host -NoNewLine "Validating" ([System.IO.Path]::GetFileName($PackagePath)) "... " + + $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) + $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageId + $FailedFiles = 0 + + Add-Type -AssemblyName System.IO.Compression.FileSystem + + [System.IO.Directory]::CreateDirectory($ExtractPath); + + try { + $zip = [System.IO.Compression.ZipFile]::OpenRead($PackagePath) + + $zip.Entries | + Where-Object {$RelevantExtensions -contains [System.IO.Path]::GetExtension($_.Name)} | + ForEach-Object { + $FileName = $_.FullName + $Extension = [System.IO.Path]::GetExtension($_.Name) + $FakeName = -Join((New-Guid), $Extension) + $TargetFile = Join-Path -Path $ExtractPath -ChildPath $FakeName + + # We ignore resource DLLs + if ($FileName.EndsWith(".resources.dll")) { + return + } + + [System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile, $true) + + $ValidateFile = { + param( + [string] $FullPath, # Full path to the module that has to be checked + [string] $RealPath, + [ref] $FailedFiles + ) + + $sourcelinkExe = "$env:USERPROFILE\.dotnet\tools" + $sourcelinkExe = Resolve-Path "$sourcelinkExe\sourcelink.exe" + $SourceLinkInfos = & $sourcelinkExe print-urls $FullPath | Out-String + + if ($LASTEXITCODE -eq 0 -and -not ([string]::IsNullOrEmpty($SourceLinkInfos))) { + $NumFailedLinks = 0 + + # We only care about Http addresses + $Matches = (Select-String '(http[s]?)(:\/\/)([^\s,]+)' -Input $SourceLinkInfos -AllMatches).Matches + + if ($Matches.Count -ne 0) { + $Matches.Value | + ForEach-Object { + $Link = $_ + $CommitUrl = "https://raw.githubusercontent.com/${using:GHRepoName}/${using:GHCommit}/" + + $FilePath = $Link.Replace($CommitUrl, "") + $Status = 200 + $Cache = $using:RepoFiles + + if ( !($Cache.ContainsKey($FilePath)) ) { + try { + $Uri = $Link -as [System.URI] + + # Only GitHub links are valid + if ($Uri.AbsoluteURI -ne $null -and ($Uri.Host -match "github" -or $Uri.Host -match "githubusercontent")) { + $Status = (Invoke-WebRequest -Uri $Link -UseBasicParsing -Method HEAD -TimeoutSec 5).StatusCode + } + else { + $Status = 0 + } + } + catch { + write-host $_ + $Status = 0 + } + } + + if ($Status -ne 200) { + if ($NumFailedLinks -eq 0) { + if ($FailedFiles.Value -eq 0) { + Write-Host + } + + Write-Host "`tFile $RealPath has broken links:" + } + + Write-Host "`t`tFailed to retrieve $Link" + + $NumFailedLinks++ + } + } + } + + if ($NumFailedLinks -ne 0) { + $FailedFiles.value++ + $global:LASTEXITCODE = 1 + } + } + } + + &$ValidateFile $TargetFile $FileName ([ref]$FailedFiles) + } + } + catch { + + } + finally { + $zip.Dispose() + } + + if ($FailedFiles -eq 0) { + Write-Host "Passed." + } + else { + Write-PipelineTaskError "$PackagePath has broken SourceLink links." + } +} + +function ValidateSourceLinkLinks { + if (!($GHRepoName -Match "^[^\s\/]+/[^\s\/]+$")) { + if (!($GHRepoName -Match "^[^\s-]+-[^\s]+$")) { + Write-PipelineTaskError "GHRepoName should be in the format / or -" + ExitWithExitCode 1 + } + else { + $GHRepoName = $GHRepoName -replace '^([^\s-]+)-([^\s]+)$', '$1/$2'; + } + } + + if (!($GHCommit -Match "^[0-9a-fA-F]{40}$")) { + Write-PipelineTaskError "GHCommit should be a 40 chars hexadecimal string" + ExitWithExitCode 1 + } + + $RepoTreeURL = -Join("http://api.github.com/repos/", $GHRepoName, "/git/trees/", $GHCommit, "?recursive=1") + $CodeExtensions = @(".cs", ".vb", ".fs", ".fsi", ".fsx", ".fsscript") + + try { + # Retrieve the list of files in the repo at that particular commit point and store them in the RepoFiles hash + $Data = Invoke-WebRequest $RepoTreeURL -UseBasicParsing | ConvertFrom-Json | Select-Object -ExpandProperty tree + + foreach ($file in $Data) { + $Extension = [System.IO.Path]::GetExtension($file.path) + + if ($CodeExtensions.Contains($Extension)) { + $RepoFiles[$file.path] = 1 + } + } + } + catch { + Write-PipelineTaskError "Problems downloading the list of files from the repo. Url used: $RepoTreeURL" + Write-Host $_ + ExitWithExitCode 1 + } + + if (Test-Path $ExtractPath) { + Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue + } + + # Process each NuGet package in parallel + $Jobs = @() + Get-ChildItem "$InputPath\*.symbols.nupkg" | + ForEach-Object { + $Jobs += Start-Job -ScriptBlock $ValidatePackage -ArgumentList $_.FullName + } + + foreach ($Job in $Jobs) { + Wait-Job -Id $Job.Id | Receive-Job + } +} + +function CheckExitCode ([string]$stage) { + $exitCode = $LASTEXITCODE + if ($exitCode -ne 0) { + Write-PipelineTaskError "Something failed while '$stage'. Check for errors above. Exiting now..." + ExitWithExitCode $exitCode + } +} + +try { + Write-Host "Installing SourceLink CLI..." + Get-Location + . $PSScriptRoot\sourcelink-cli-init.ps1 -sourcelinkCliVersion $SourcelinkCliVersion + CheckExitCode "Running sourcelink-cli-init" + + Measure-Command { ValidateSourceLinkLinks } +} +catch { + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + ExitWithExitCode 1 +} diff --git a/eng/common/post-build/symbols-validation.ps1 b/eng/common/post-build/symbols-validation.ps1 new file mode 100644 index 000000000..69456854e --- /dev/null +++ b/eng/common/post-build/symbols-validation.ps1 @@ -0,0 +1,186 @@ +param( + [Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where NuGet packages to be checked are stored + [Parameter(Mandatory=$true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation + [Parameter(Mandatory=$true)][string] $DotnetSymbolVersion # Version of dotnet symbol to use +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 + +. $PSScriptRoot\..\tools.ps1 + +Add-Type -AssemblyName System.IO.Compression.FileSystem + +function FirstMatchingSymbolDescriptionOrDefault { + param( + [string] $FullPath, # Full path to the module that has to be checked + [string] $TargetServerParam, # Parameter to pass to `Symbol Tool` indicating the server to lookup for symbols + [string] $SymbolsPath + ) + + $FileName = [System.IO.Path]::GetFileName($FullPath) + $Extension = [System.IO.Path]::GetExtension($FullPath) + + # Those below are potential symbol files that the `dotnet symbol` might + # return. Which one will be returned depend on the type of file we are + # checking and which type of file was uploaded. + + # The file itself is returned + $SymbolPath = $SymbolsPath + "\" + $FileName + + # PDB file for the module + $PdbPath = $SymbolPath.Replace($Extension, ".pdb") + + # PDB file for R2R module (created by crossgen) + $NGenPdb = $SymbolPath.Replace($Extension, ".ni.pdb") + + # DBG file for a .so library + $SODbg = $SymbolPath.Replace($Extension, ".so.dbg") + + # DWARF file for a .dylib + $DylibDwarf = $SymbolPath.Replace($Extension, ".dylib.dwarf") + + $dotnetsymbolExe = "$env:USERPROFILE\.dotnet\tools" + $dotnetsymbolExe = Resolve-Path "$dotnetsymbolExe\dotnet-symbol.exe" + + & $dotnetsymbolExe --symbols --modules --windows-pdbs $TargetServerParam $FullPath -o $SymbolsPath | Out-Null + + if (Test-Path $PdbPath) { + return "PDB" + } + elseif (Test-Path $NGenPdb) { + return "NGen PDB" + } + elseif (Test-Path $SODbg) { + return "DBG for SO" + } + elseif (Test-Path $DylibDwarf) { + return "Dwarf for Dylib" + } + elseif (Test-Path $SymbolPath) { + return "Module" + } + else { + return $null + } +} + +function CountMissingSymbols { + param( + [string] $PackagePath # Path to a NuGet package + ) + + # Ensure input file exist + if (!(Test-Path $PackagePath)) { + Write-PipelineTaskError "Input file does not exist: $PackagePath" + ExitWithExitCode 1 + } + + # Extensions for which we'll look for symbols + $RelevantExtensions = @(".dll", ".exe", ".so", ".dylib") + + # How many files are missing symbol information + $MissingSymbols = 0 + + $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) + $PackageGuid = New-Guid + $ExtractPath = Join-Path -Path $ExtractPath -ChildPath $PackageGuid + $SymbolsPath = Join-Path -Path $ExtractPath -ChildPath "Symbols" + + [System.IO.Compression.ZipFile]::ExtractToDirectory($PackagePath, $ExtractPath) + + Get-ChildItem -Recurse $ExtractPath | + Where-Object {$RelevantExtensions -contains $_.Extension} | + ForEach-Object { + if ($_.FullName -Match "\\ref\\") { + Write-Host "`t Ignoring reference assembly file" $_.FullName + return + } + + $SymbolsOnMSDL = FirstMatchingSymbolDescriptionOrDefault $_.FullName "--microsoft-symbol-server" $SymbolsPath + $SymbolsOnSymWeb = FirstMatchingSymbolDescriptionOrDefault $_.FullName "--internal-server" $SymbolsPath + + Write-Host -NoNewLine "`t Checking file" $_.FullName "... " + + if ($SymbolsOnMSDL -ne $null -and $SymbolsOnSymWeb -ne $null) { + Write-Host "Symbols found on MSDL (" $SymbolsOnMSDL ") and SymWeb (" $SymbolsOnSymWeb ")" + } + else { + $MissingSymbols++ + + if ($SymbolsOnMSDL -eq $null -and $SymbolsOnSymWeb -eq $null) { + Write-Host "No symbols found on MSDL or SymWeb!" + } + else { + if ($SymbolsOnMSDL -eq $null) { + Write-Host "No symbols found on MSDL!" + } + else { + Write-Host "No symbols found on SymWeb!" + } + } + } + } + + Pop-Location + + return $MissingSymbols +} + +function CheckSymbolsAvailable { + if (Test-Path $ExtractPath) { + Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue + } + + Get-ChildItem "$InputPath\*.nupkg" | + ForEach-Object { + $FileName = $_.Name + + # These packages from Arcade-Services include some native libraries that + # our current symbol uploader can't handle. Below is a workaround until + # we get issue: https://github.com/dotnet/arcade/issues/2457 sorted. + if ($FileName -Match "Microsoft\.DotNet\.Darc\.") { + Write-Host "Ignoring Arcade-services file: $FileName" + Write-Host + return + } + elseif ($FileName -Match "Microsoft\.DotNet\.Maestro\.Tasks\.") { + Write-Host "Ignoring Arcade-services file: $FileName" + Write-Host + return + } + + Write-Host "Validating $FileName " + $Status = CountMissingSymbols "$InputPath\$FileName" + + if ($Status -ne 0) { + Write-PipelineTaskError "Missing symbols for $Status modules in the package $FileName" + ExitWithExitCode $exitCode + } + + Write-Host + } +} + +function CheckExitCode ([string]$stage) { + $exitCode = $LASTEXITCODE + if ($exitCode -ne 0) { + Write-PipelineTaskError "Something failed while '$stage'. Check for errors above. Exiting now..." + ExitWithExitCode $exitCode + } +} + +try { + Write-Host "Installing dotnet symbol ..." + Get-Location + . $PSScriptRoot\dotnetsymbol-init.ps1 -dotnetsymbolVersion $DotnetSymbolVersion + CheckExitCode "Running dotnetsymbol-init" + + CheckSymbolsAvailable +} +catch { + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + ExitWithExitCode 1 +} diff --git a/eng/common/post-build/trigger-subscriptions.ps1 b/eng/common/post-build/trigger-subscriptions.ps1 new file mode 100644 index 000000000..db8a83945 --- /dev/null +++ b/eng/common/post-build/trigger-subscriptions.ps1 @@ -0,0 +1,69 @@ +param( + [Parameter(Mandatory=$true)][string] $SourceRepo, + [Parameter(Mandatory=$true)][int] $ChannelId, + [string] $MaestroEndpoint = "https://maestro-prod.westus2.cloudapp.azure.com", + [string] $BarToken, + [string] $ApiVersion = "2019-01-16" +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 + +. $PSScriptRoot\..\tools.ps1 + +function Get-Headers([string]$accept, [string]$barToken) { + $headers = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]' + $headers.Add('Accept',$accept) + $headers.Add('Authorization',"Bearer $barToken") + return $headers +} + +# Get all the $SourceRepo subscriptions +$normalizedSurceRepo = $SourceRepo.Replace('dnceng@', '') +$getSubscriptionsApiEndpoint = "$maestroEndpoint/api/subscriptions?sourceRepository=$normalizedSurceRepo&api-version=$apiVersion" +$headers = Get-Headers 'application/json' $barToken + +$subscriptions = Invoke-WebRequest -Uri $getSubscriptionsApiEndpoint -Headers $headers | ConvertFrom-Json + +if (!$subscriptions) { + Write-Host "No subscriptions found for source repo '$normalizedSurceRepo' in channel '$ChannelId'" + return +} + +$subscriptionsToTrigger = New-Object System.Collections.Generic.List[string] +$failedTriggeredSubscription = $false + +# Get all enabled subscriptions that need dependency flow on 'everyBuild' +foreach ($subscription in $subscriptions) { + if ($subscription.enabled -and $subscription.policy.updateFrequency -like 'everyBuild' -and $subscription.channel.id -eq $ChannelId) { + Write-Host "$subscription.id" + [void]$subscriptionsToTrigger.Add($subscription.id) + } +} + +foreach ($subscriptionToTrigger in $subscriptionsToTrigger) { + try { + $triggerSubscriptionApiEndpoint = "$maestroEndpoint/api/subscriptions/$subscriptionToTrigger/trigger?api-version=$apiVersion" + $headers = Get-Headers 'application/json' $BarToken + + Write-Host "Triggering subscription '$subscriptionToTrigger'..." + + Invoke-WebRequest -Uri $triggerSubscriptionApiEndpoint -Headers $headers -Method Post + + Write-Host "Subscription '$subscriptionToTrigger' triggered!" + } + catch + { + Write-Host "There was an error while triggering subscription '$subscriptionToTrigger'" + Write-Host $_ + Write-Host $_.ScriptStackTrace + $failedTriggeredSubscription = $true + } +} + +if ($failedTriggeredSubscription) { + Write-Host "At least one subscription failed to be triggered..." + ExitWithExitCode 1 +} + +Write-Host "All subscriptions were triggered successfully!" diff --git a/eng/common/sdl/NuGet.config b/eng/common/sdl/NuGet.config new file mode 100644 index 000000000..0c5451c11 --- /dev/null +++ b/eng/common/sdl/NuGet.config @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/eng/common/sdl/execute-all-sdl-tools.ps1 b/eng/common/sdl/execute-all-sdl-tools.ps1 new file mode 100644 index 000000000..0635f26fb --- /dev/null +++ b/eng/common/sdl/execute-all-sdl-tools.ps1 @@ -0,0 +1,97 @@ +Param( + [string] $GuardianPackageName, # Required: the name of guardian CLI package (not needed if GuardianCliLocation is specified) + [string] $NugetPackageDirectory, # Required: directory where NuGet packages are installed (not needed if GuardianCliLocation is specified) + [string] $GuardianCliLocation, # Optional: Direct location of Guardian CLI executable if GuardianPackageName & NugetPackageDirectory are not specified + [string] $Repository=$env:BUILD_REPOSITORY_NAME, # Required: the name of the repository (e.g. dotnet/arcade) + [string] $BranchName=$env:BUILD_SOURCEBRANCH, # Optional: name of branch or version of gdn settings; defaults to master + [string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY, # Required: the directory where source files are located + [string] $ArtifactsDirectory = (Join-Path $env:BUILD_SOURCESDIRECTORY ("artifacts")), # Required: the directory where build artifacts are located + [string] $AzureDevOpsAccessToken, # Required: access token for dnceng; should be provided via KeyVault + [string[]] $SourceToolsList, # Optional: list of SDL tools to run on source code + [string[]] $ArtifactToolsList, # Optional: list of SDL tools to run on built artifacts + [bool] $TsaPublish=$False, # Optional: true will publish results to TSA; only set to true after onboarding to TSA; TSA is the automated framework used to upload test results as bugs. + [string] $TsaBranchName=$env:BUILD_SOURCEBRANCH, # Optional: required for TSA publish; defaults to $(Build.SourceBranchName); TSA is the automated framework used to upload test results as bugs. + [string] $TsaRepositoryName=$env:BUILD_REPOSITORY_NAME, # Optional: TSA repository name; will be generated automatically if not submitted; TSA is the automated framework used to upload test results as bugs. + [string] $BuildNumber=$env:BUILD_BUILDNUMBER, # Optional: required for TSA publish; defaults to $(Build.BuildNumber) + [bool] $UpdateBaseline=$False, # Optional: if true, will update the baseline in the repository; should only be run after fixing any issues which need to be fixed + [bool] $TsaOnboard=$False, # Optional: if true, will onboard the repository to TSA; should only be run once; TSA is the automated framework used to upload test results as bugs. + [string] $TsaInstanceUrl, # Optional: only needed if TsaOnboard or TsaPublish is true; the instance-url registered with TSA; TSA is the automated framework used to upload test results as bugs. + [string] $TsaCodebaseName, # Optional: only needed if TsaOnboard or TsaPublish is true; the name of the codebase registered with TSA; TSA is the automated framework used to upload test results as bugs. + [string] $TsaProjectName, # Optional: only needed if TsaOnboard or TsaPublish is true; the name of the project registered with TSA; TSA is the automated framework used to upload test results as bugs. + [string] $TsaNotificationEmail, # Optional: only needed if TsaOnboard is true; the email(s) which will receive notifications of TSA bug filings (e.g. alias@microsoft.com); TSA is the automated framework used to upload test results as bugs. + [string] $TsaCodebaseAdmin, # Optional: only needed if TsaOnboard is true; the aliases which are admins of the TSA codebase (e.g. DOMAIN\alias); TSA is the automated framework used to upload test results as bugs. + [string] $TsaBugAreaPath, # Optional: only needed if TsaOnboard is true; the area path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs. + [string] $TsaIterationPath, # Optional: only needed if TsaOnboard is true; the iteration path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs. + [string] $GuardianLoggerLevel="Standard" # Optional: the logger level for the Guardian CLI; options are Trace, Verbose, Standard, Warning, and Error +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 +$LASTEXITCODE = 0 + +#Replace repo names to the format of org/repo +if (!($Repository.contains('/'))) { + $RepoName = $Repository -replace '(.*?)-(.*)', '$1/$2'; +} +else{ + $RepoName = $Repository; +} + +if ($GuardianPackageName) { + $guardianCliLocation = Join-Path $NugetPackageDirectory (Join-Path $GuardianPackageName (Join-Path "tools" "guardian.cmd")) +} else { + $guardianCliLocation = $GuardianCliLocation +} + +$ValidPath = Test-Path $guardianCliLocation + +if ($ValidPath -eq $False) +{ + Write-Host "Invalid Guardian CLI Location." + exit 1 +} + +& $(Join-Path $PSScriptRoot "init-sdl.ps1") -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $ArtifactsDirectory -AzureDevOpsAccessToken $AzureDevOpsAccessToken -GuardianLoggerLevel $GuardianLoggerLevel +$gdnFolder = Join-Path $ArtifactsDirectory ".gdn" + +if ($TsaOnboard) { + if ($TsaCodebaseName -and $TsaNotificationEmail -and $TsaCodebaseAdmin -and $TsaBugAreaPath) { + Write-Host "$guardianCliLocation tsa-onboard --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $ArtifactsDirectory --logger-level $GuardianLoggerLevel" + & $guardianCliLocation tsa-onboard --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $ArtifactsDirectory --logger-level $GuardianLoggerLevel + if ($LASTEXITCODE -ne 0) { + Write-Host "Guardian tsa-onboard failed with exit code $LASTEXITCODE." + exit $LASTEXITCODE + } + } else { + Write-Host "Could not onboard to TSA -- not all required values ($$TsaCodebaseName, $$TsaNotificationEmail, $$TsaCodebaseAdmin, $$TsaBugAreaPath) were specified." + exit 1 + } +} + +if ($ArtifactToolsList -and $ArtifactToolsList.Count -gt 0) { + & $(Join-Path $PSScriptRoot "run-sdl.ps1") -GuardianCliLocation $guardianCliLocation -WorkingDirectory $ArtifactsDirectory -TargetDirectory $ArtifactsDirectory -GdnFolder $gdnFolder -ToolsList $ArtifactToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel +} +if ($SourceToolsList -and $SourceToolsList.Count -gt 0) { + & $(Join-Path $PSScriptRoot "run-sdl.ps1") -GuardianCliLocation $guardianCliLocation -WorkingDirectory $ArtifactsDirectory -TargetDirectory $SourceDirectory -GdnFolder $gdnFolder -ToolsList $SourceToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel +} + +if ($UpdateBaseline) { + & (Join-Path $PSScriptRoot "push-gdn.ps1") -Repository $RepoName -BranchName $BranchName -GdnFolder $GdnFolder -AzureDevOpsAccessToken $AzureDevOpsAccessToken -PushReason "Update baseline" +} + +if ($TsaPublish) { + if ($TsaBranchName -and $BuildNumber) { + if (-not $TsaRepositoryName) { + $TsaRepositoryName = "$($Repository)-$($BranchName)" + } + Write-Host "$guardianCliLocation tsa-publish --all-tools --repository-name `"$TsaRepositoryName`" --branch-name `"$TsaBranchName`" --build-number `"$BuildNumber`" --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $SourceDirectory --logger-level $GuardianLoggerLevel" + & $guardianCliLocation tsa-publish --all-tools --repository-name "$TsaRepositoryName" --branch-name "$TsaBranchName" --build-number "$BuildNumber" --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $ArtifactsDirectory --logger-level $GuardianLoggerLevel + if ($LASTEXITCODE -ne 0) { + Write-Host "Guardian tsa-publish failed with exit code $LASTEXITCODE." + exit $LASTEXITCODE + } + } else { + Write-Host "Could not publish to TSA -- not all required values ($$TsaBranchName, $$BuildNumber) were specified." + exit 1 + } +} diff --git a/eng/common/sdl/extract-artifact-packages.ps1 b/eng/common/sdl/extract-artifact-packages.ps1 new file mode 100644 index 000000000..1fdbb1432 --- /dev/null +++ b/eng/common/sdl/extract-artifact-packages.ps1 @@ -0,0 +1,70 @@ +param( + [Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where artifact packages are stored + [Parameter(Mandatory=$true)][string] $ExtractPath # Full path to directory where the packages will be extracted +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 +$ExtractPackage = { + param( + [string] $PackagePath # Full path to a NuGet package + ) + + if (!(Test-Path $PackagePath)) { + Write-PipelineTaskError "Input file does not exist: $PackagePath" + ExitWithExitCode 1 + } + + $RelevantExtensions = @(".dll", ".exe", ".pdb") + Write-Host -NoNewLine "Extracting" ([System.IO.Path]::GetFileName($PackagePath)) "... " + + $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) + $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageId + + Add-Type -AssemblyName System.IO.Compression.FileSystem + + [System.IO.Directory]::CreateDirectory($ExtractPath); + + try { + $zip = [System.IO.Compression.ZipFile]::OpenRead($PackagePath) + + $zip.Entries | + Where-Object {$RelevantExtensions -contains [System.IO.Path]::GetExtension($_.Name)} | + ForEach-Object { + $TargetFile = Join-Path -Path $ExtractPath -ChildPath $_.Name + + [System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile, $true) + } + } + catch { + + } + finally { + $zip.Dispose() + } + } + function ExtractArtifacts { + if (!(Test-Path $InputPath)) { + Write-Host "Input Path does not exist: $InputPath" + ExitWithExitCode 0 + } + $Jobs = @() + Get-ChildItem "$InputPath\*.nupkg" | + ForEach-Object { + $Jobs += Start-Job -ScriptBlock $ExtractPackage -ArgumentList $_.FullName + } + + foreach ($Job in $Jobs) { + Wait-Job -Id $Job.Id | Receive-Job + } +} + +try { + Measure-Command { ExtractArtifacts } +} +catch { + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + ExitWithExitCode 1 +} diff --git a/eng/common/sdl/init-sdl.ps1 b/eng/common/sdl/init-sdl.ps1 new file mode 100644 index 000000000..26e01c067 --- /dev/null +++ b/eng/common/sdl/init-sdl.ps1 @@ -0,0 +1,48 @@ +Param( + [string] $GuardianCliLocation, + [string] $Repository, + [string] $BranchName="master", + [string] $WorkingDirectory, + [string] $AzureDevOpsAccessToken, + [string] $GuardianLoggerLevel="Standard" +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 +$LASTEXITCODE = 0 + +# Construct basic auth from AzDO access token; construct URI to the repository's gdn folder stored in that repository; construct location of zip file +$encodedPat = [Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes(":$AzureDevOpsAccessToken")) +$escapedRepository = [Uri]::EscapeDataString("/$Repository/$BranchName/.gdn") +$uri = "https://dev.azure.com/dnceng/internal/_apis/git/repositories/sdl-tool-cfg/Items?path=$escapedRepository&versionDescriptor[versionOptions]=0&`$format=zip&api-version=5.0-preview.1" +$zipFile = "$WorkingDirectory/gdn.zip" + +Add-Type -AssemblyName System.IO.Compression.FileSystem +$gdnFolder = (Join-Path $WorkingDirectory ".gdn") +Try +{ + # We try to download the zip; if the request fails (e.g. the file doesn't exist), we catch it and init guardian instead + Write-Host "Downloading gdn folder from internal config repostiory..." + Invoke-WebRequest -Headers @{ "Accept"="application/zip"; "Authorization"="Basic $encodedPat" } -Uri $uri -OutFile $zipFile + if (Test-Path $gdnFolder) { + # Remove the gdn folder if it exists (it shouldn't unless there's too much caching; this is just in case) + Remove-Item -Force -Recurse $gdnFolder + } + [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFile, $WorkingDirectory) + Write-Host $gdnFolder +} Catch [System.Net.WebException] { + # if the folder does not exist, we'll do a guardian init and push it to the remote repository + Write-Host "Initializing Guardian..." + Write-Host "$GuardianCliLocation init --working-directory $WorkingDirectory --logger-level $GuardianLoggerLevel" + & $GuardianCliLocation init --working-directory $WorkingDirectory --logger-level $GuardianLoggerLevel + if ($LASTEXITCODE -ne 0) { + Write-Error "Guardian init failed with exit code $LASTEXITCODE." + } + # We create the mainbaseline so it can be edited later + Write-Host "$GuardianCliLocation baseline --working-directory $WorkingDirectory --name mainbaseline" + & $GuardianCliLocation baseline --working-directory $WorkingDirectory --name mainbaseline + if ($LASTEXITCODE -ne 0) { + Write-Error "Guardian baseline failed with exit code $LASTEXITCODE." + } + & $(Join-Path $PSScriptRoot "push-gdn.ps1") -Repository $Repository -BranchName $BranchName -GdnFolder $gdnFolder -AzureDevOpsAccessToken $AzureDevOpsAccessToken -PushReason "Initialize gdn folder" +} \ No newline at end of file diff --git a/eng/common/sdl/packages.config b/eng/common/sdl/packages.config new file mode 100644 index 000000000..fb9b71286 --- /dev/null +++ b/eng/common/sdl/packages.config @@ -0,0 +1,4 @@ + + + + diff --git a/eng/common/sdl/push-gdn.ps1 b/eng/common/sdl/push-gdn.ps1 new file mode 100644 index 000000000..79c707d6d --- /dev/null +++ b/eng/common/sdl/push-gdn.ps1 @@ -0,0 +1,51 @@ +Param( + [string] $Repository, + [string] $BranchName="master", + [string] $GdnFolder, + [string] $AzureDevOpsAccessToken, + [string] $PushReason +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 +$LASTEXITCODE = 0 + +# We create the temp directory where we'll store the sdl-config repository +$sdlDir = Join-Path $env:TEMP "sdl" +if (Test-Path $sdlDir) { + Remove-Item -Force -Recurse $sdlDir +} + +Write-Host "git clone https://dnceng:`$AzureDevOpsAccessToken@dev.azure.com/dnceng/internal/_git/sdl-tool-cfg $sdlDir" +git clone https://dnceng:$AzureDevOpsAccessToken@dev.azure.com/dnceng/internal/_git/sdl-tool-cfg $sdlDir +if ($LASTEXITCODE -ne 0) { + Write-Error "Git clone failed with exit code $LASTEXITCODE." +} +# We copy the .gdn folder from our local run into the git repository so it can be committed +$sdlRepositoryFolder = Join-Path (Join-Path (Join-Path $sdlDir $Repository) $BranchName) ".gdn" +if (Get-Command Robocopy) { + Robocopy /S $GdnFolder $sdlRepositoryFolder +} else { + rsync -r $GdnFolder $sdlRepositoryFolder +} +# cd to the sdl-config directory so we can run git there +Push-Location $sdlDir +# git add . --> git commit --> git push +Write-Host "git add ." +git add . +if ($LASTEXITCODE -ne 0) { + Write-Error "Git add failed with exit code $LASTEXITCODE." +} +Write-Host "git -c user.email=`"dn-bot@microsoft.com`" -c user.name=`"Dotnet Bot`" commit -m `"$PushReason for $Repository/$BranchName`"" +git -c user.email="dn-bot@microsoft.com" -c user.name="Dotnet Bot" commit -m "$PushReason for $Repository/$BranchName" +if ($LASTEXITCODE -ne 0) { + Write-Error "Git commit failed with exit code $LASTEXITCODE." +} +Write-Host "git push" +git push +if ($LASTEXITCODE -ne 0) { + Write-Error "Git push failed with exit code $LASTEXITCODE." +} + +# Return to the original directory +Pop-Location \ No newline at end of file diff --git a/eng/common/sdl/run-sdl.ps1 b/eng/common/sdl/run-sdl.ps1 new file mode 100644 index 000000000..e6a86d03a --- /dev/null +++ b/eng/common/sdl/run-sdl.ps1 @@ -0,0 +1,65 @@ +Param( + [string] $GuardianCliLocation, + [string] $WorkingDirectory, + [string] $TargetDirectory, + [string] $GdnFolder, + [string[]] $ToolsList, + [string] $UpdateBaseline, + [string] $GuardianLoggerLevel="Standard" +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 +$LASTEXITCODE = 0 + +# We store config files in the r directory of .gdn +Write-Host $ToolsList +$gdnConfigPath = Join-Path $GdnFolder "r" +$ValidPath = Test-Path $GuardianCliLocation + +if ($ValidPath -eq $False) +{ + Write-Host "Invalid Guardian CLI Location." + exit 1 +} + +foreach ($tool in $ToolsList) { + $gdnConfigFile = Join-Path $gdnConfigPath "$tool-configure.gdnconfig" + $config = $False + Write-Host $tool + # We have to manually configure tools that run on source to look at the source directory only + if ($tool -eq "credscan") { + Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" TargetDirectory : $TargetDirectory `"" + & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " TargetDirectory : $TargetDirectory " + if ($LASTEXITCODE -ne 0) { + Write-Host "Guardian configure for $tool failed with exit code $LASTEXITCODE." + exit $LASTEXITCODE + } + $config = $True + } + if ($tool -eq "policheck") { + Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" Target : $TargetDirectory `"" + & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " Target : $TargetDirectory " + if ($LASTEXITCODE -ne 0) { + Write-Host "Guardian configure for $tool failed with exit code $LASTEXITCODE." + exit $LASTEXITCODE + } + $config = $True + } + + Write-Host "$GuardianCliLocation run --working-directory $WorkingDirectory --tool $tool --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel --config $gdnConfigFile $config" + if ($config) { + & $GuardianCliLocation run --working-directory $WorkingDirectory --tool $tool --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel --config $gdnConfigFile + if ($LASTEXITCODE -ne 0) { + Write-Host "Guardian run for $tool using $gdnConfigFile failed with exit code $LASTEXITCODE." + exit $LASTEXITCODE + } + } else { + & $GuardianCliLocation run --working-directory $WorkingDirectory --tool $tool --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel + if ($LASTEXITCODE -ne 0) { + Write-Host "Guardian run for $tool failed with exit code $LASTEXITCODE." + exit $LASTEXITCODE + } + } +} + diff --git a/eng/common/templates/job/execute-sdl.yml b/eng/common/templates/job/execute-sdl.yml new file mode 100644 index 000000000..5837f3d56 --- /dev/null +++ b/eng/common/templates/job/execute-sdl.yml @@ -0,0 +1,54 @@ +parameters: + overrideParameters: '' # Optional: to override values for parameters. + additionalParameters: '' # Optional: parameters that need user specific values eg: '-SourceToolsList @("abc","def") -ArtifactToolsList @("ghi","jkl")' + continueOnError: false # optional: determines whether to continue the build if the step errors; + dependsOn: '' # Optional: dependencies of the job + +jobs: +- job: Run_SDL + dependsOn: ${{ parameters.dependsOn }} + displayName: Run SDL tool + variables: + - group: DotNet-VSTS-Bot + steps: + - checkout: self + clean: true + - task: DownloadBuildArtifacts@0 + displayName: Download Build Artifacts + inputs: + buildType: current + downloadType: specific files + matchingPattern: "**" + downloadPath: $(Build.SourcesDirectory)\artifacts + - powershell: eng/common/sdl/extract-artifact-packages.ps1 + -InputPath $(Build.SourcesDirectory)\artifacts\BlobArtifacts + -ExtractPath $(Build.SourcesDirectory)\artifacts\BlobArtifacts + displayName: Extract Blob Artifacts + continueOnError: ${{ parameters.continueOnError }} + - powershell: eng/common/sdl/extract-artifact-packages.ps1 + -InputPath $(Build.SourcesDirectory)\artifacts\PackageArtifacts + -ExtractPath $(Build.SourcesDirectory)\artifacts\PackageArtifacts + displayName: Extract Package Artifacts + continueOnError: ${{ parameters.continueOnError }} + - task: NuGetToolInstaller@1 + displayName: 'Install NuGet.exe' + - task: NuGetCommand@2 + displayName: 'Install Guardian' + inputs: + restoreSolution: $(Build.SourcesDirectory)\eng\common\sdl\packages.config + feedsToUse: config + nugetConfigPath: $(Build.SourcesDirectory)\eng\common\sdl\NuGet.config + externalFeedCredentials: GuardianConnect + restoreDirectory: $(Build.SourcesDirectory)\.packages + - ${{ if ne(parameters.overrideParameters, '') }}: + - powershell: eng/common/sdl/execute-all-sdl-tools.ps1 ${{ parameters.overrideParameters }} + displayName: Execute SDL + continueOnError: ${{ parameters.continueOnError }} + - ${{ if eq(parameters.overrideParameters, '') }}: + - powershell: eng/common/sdl/execute-all-sdl-tools.ps1 + -GuardianPackageName Microsoft.Guardian.Cli.0.6.0 + -NugetPackageDirectory $(Build.SourcesDirectory)\.packages + -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw) + ${{ parameters.additionalParameters }} + displayName: Execute SDL + continueOnError: ${{ parameters.continueOnError }} diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml index 620bd3c62..9e77ef1b5 100644 --- a/eng/common/templates/job/publish-build-assets.yml +++ b/eng/common/templates/job/publish-build-assets.yml @@ -59,6 +59,20 @@ jobs: /p:Configuration=$(_BuildConfig) condition: ${{ parameters.condition }} continueOnError: ${{ parameters.continueOnError }} + - task: powershell@2 + displayName: Create ReleaseConfigs Artifact + inputs: + targetType: inline + script: | + Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(BARBuildId) + Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value "$(DefaultChannels)" + Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(IsStableBuild) + - task: PublishBuildArtifacts@1 + displayName: Publish ReleaseConfigs Artifact + inputs: + PathtoPublish: '$(Build.StagingDirectory)/ReleaseConfigs.txt' + PublishLocation: Container + ArtifactName: ReleaseConfigs - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: - task: PublishBuildArtifacts@1 displayName: Publish Logs to VSTS diff --git a/eng/common/templates/post-build/channels/internal-servicing.yml b/eng/common/templates/post-build/channels/internal-servicing.yml new file mode 100644 index 000000000..808d46b17 --- /dev/null +++ b/eng/common/templates/post-build/channels/internal-servicing.yml @@ -0,0 +1,170 @@ +parameters: + enableSymbolValidation: true + +stages: +- stage: IS_Publish + dependsOn: validate + variables: + - template: ../common-variables.yml + displayName: Internal Servicing + jobs: + - template: ../setup-maestro-vars.yml + + - job: + displayName: Symbol Publishing + dependsOn: setupMaestroVars + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.InternalServicing_30_Channel_Id) + variables: + - group: DotNet-Symbol-Server-Pats + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Artifacts + inputs: + downloadType: specific files + matchingPattern: "*Artifacts*" + + - task: PowerShell@2 + displayName: Publish + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishToSymbolServers -restore -msbuildEngine dotnet + /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) + /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) + /p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' + /p:Configuration=Release + + - job: publish_assets + displayName: Publish Assets + dependsOn: setupMaestroVars + variables: + - group: DotNet-Blob-Feed + - group: Publish-Build-Assets + - group: AzureDevOps-Artifact-Feeds-Pats + - name: BARBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] + - name: IsStableBuild + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ] + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.InternalServicing_30_Channel_Id) + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: current + artifactName: PackageArtifacts + + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: current + artifactName: BlobArtifacts + + - task: DownloadBuildArtifacts@0 + displayName: Download Asset Manifests + inputs: + buildType: current + artifactName: AssetManifests + + - task: PowerShell@2 + displayName: Add Assets Location + env: + AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-unviersal-packages-rw) + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet + /p:ChannelId=$(InternalServicing_30_Channel_Id) + /p:IsStableBuild=$(IsStableBuild) + /p:IsInternalBuild=$(IsInternalBuild) + /p:RepositoryName=$(Build.Repository.Name) + /p:CommitSha=$(Build.SourceVersion) + /p:NugetPath=$(Agent.BuildDirectory)\Nuget\NuGet.exe + /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' + /p:TargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' + /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' + /p:BARBuildId=$(BARBuildId) + /p:MaestroApiEndpoint='https://maestro-prod.westus2.cloudapp.azure.com' + /p:BuildAssetRegistryToken='$(MaestroAccessToken)' + /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)\BlobArtifacts' + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)\PackageArtifacts' + /p:Configuration=Release + + - task: NuGetCommand@2 + displayName: Publish Packages to AzDO Feed + condition: contains(variables['TargetAzDOFeed'], 'pkgs.visualstudio.com') + inputs: + command: push + vstsFeed: $(AzDoFeedName) + packagesToPush: $(Build.ArtifactStagingDirectory)\PackageArtifacts\*.nupkg + publishVstsFeed: $(AzDoFeedName) + + - task: PowerShell@2 + displayName: Publish Blobs to AzDO Feed + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-blobs-to-azdo.ps1 + arguments: -FeedName $(AzDoFeedName) + -SourceFolderCollection $(Build.ArtifactStagingDirectory)/BlobArtifacts/ + -PersonalAccessToken $(dn-bot-dnceng-unviersal-packages-rw) + enabled: false + + - template: ../trigger-subscription.yml + parameters: + ChannelId: ${{ variables.InternalServicing_30_Channel_Id }} + +- stage: IS_PublishValidation + displayName: Publish Validation + variables: + - template: ../common-variables.yml + jobs: + - template: ../setup-maestro-vars.yml + + - ${{ if eq(parameters.enableSymbolValidation, 'true') }}: + - job: + displayName: Symbol Availability + dependsOn: setupMaestroVars + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.InternalServicing_30_Channel_Id) + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: current + artifactName: PackageArtifacts + + - task: PowerShell@2 + displayName: Check Symbol Availability + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/symbols-validation.ps1 + arguments: -InputPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ -ExtractPath $(Agent.BuildDirectory)/Temp/ -DotnetSymbolVersion $(SymbolToolVersion) + + - job: + displayName: Gather Drop + dependsOn: setupMaestroVars + variables: + BARBuildId: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.InternalServicing_30_Channel_Id) + pool: + vmImage: 'windows-2019' + steps: + - task: PowerShell@2 + displayName: Setup Darc CLI + inputs: + targetType: filePath + filePath: '$(Build.SourcesDirectory)/eng/common/darc-init.ps1' + + - task: PowerShell@2 + displayName: Run Darc gather-drop + inputs: + targetType: inline + script: | + darc gather-drop --non-shipping --continue-on-error --id $(BARBuildId) --output-dir $(Agent.BuildDirectory)/Temp/Drop/ --bar-uri https://maestro-prod.westus2.cloudapp.azure.com/ --password $(MaestroAccessToken) --latest-location + enabled: false + + - template: ../promote-build.yml + parameters: + ChannelId: ${{ variables.InternalServicing_30_Channel_Id }} \ No newline at end of file diff --git a/eng/common/templates/post-build/channels/public-dev-release.yml b/eng/common/templates/post-build/channels/public-dev-release.yml new file mode 100644 index 000000000..79c6822db --- /dev/null +++ b/eng/common/templates/post-build/channels/public-dev-release.yml @@ -0,0 +1,166 @@ +parameters: + enableSymbolValidation: true + +stages: +- stage: Publish + dependsOn: validate + variables: + - template: ../common-variables.yml + displayName: Developer Channel + jobs: + - template: ../setup-maestro-vars.yml + + - job: + displayName: Symbol Publishing + dependsOn: setupMaestroVars + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicDevRelease_30_Channel_Id) + variables: + - group: DotNet-Symbol-Server-Pats + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Artifacts + inputs: + downloadType: specific files + matchingPattern: "*Artifacts*" + + - task: PowerShell@2 + displayName: Publish + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishToSymbolServers -restore -msbuildEngine dotnet + /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) + /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) + /p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' + /p:Configuration=Release + + - job: + displayName: Publish Assets + dependsOn: setupMaestroVars + variables: + - group: DotNet-Blob-Feed + - group: Publish-Build-Assets + - group: AzureDevOps-Artifact-Feeds-Pats + - name: BARBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] + - name: IsStableBuild + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ] + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicDevRelease_30_Channel_Id) + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: current + artifactName: PackageArtifacts + + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: current + artifactName: BlobArtifacts + + - task: DownloadBuildArtifacts@0 + displayName: Download Asset Manifests + inputs: + buildType: current + artifactName: AssetManifests + + - task: PowerShell@2 + displayName: Add Assets Location + env: + AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-unviersal-packages-rw) + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet + /p:ChannelId=$(PublicDevRelease_30_Channel_Id) + /p:IsStableBuild=$(IsStableBuild) + /p:IsInternalBuild=$(IsInternalBuild) + /p:RepositoryName=$(Build.Repository.Name) + /p:CommitSha=$(Build.SourceVersion) + /p:NugetPath=$(Agent.BuildDirectory)\Nuget\NuGet.exe + /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' + /p:TargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' + /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' + /p:BARBuildId=$(BARBuildId) + /p:MaestroApiEndpoint='https://maestro-prod.westus2.cloudapp.azure.com' + /p:BuildAssetRegistryToken='$(MaestroAccessToken)' + /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' + /p:Configuration=Release + + - task: NuGetCommand@2 + displayName: Publish Packages to AzDO Feed + condition: contains(variables['TargetAzDOFeed'], 'pkgs.visualstudio.com') + inputs: + command: push + vstsFeed: $(AzDoFeedName) + packagesToPush: $(Build.ArtifactStagingDirectory)\PackageArtifacts\*.nupkg + publishVstsFeed: $(AzDoFeedName) + + - task: PowerShell@2 + displayName: Publish Blobs to AzDO Feed + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-blobs-to-azdo.ps1 + arguments: -FeedName $(AzDoFeedName) + -SourceFolderCollection $(Build.ArtifactStagingDirectory)/BlobArtifacts/ + -PersonalAccessToken $(dn-bot-dnceng-unviersal-packages-rw) + enabled: false + + +- stage: PublishValidation + displayName: Publish Validation + variables: + - template: ../common-variables.yml + jobs: + - template: ../setup-maestro-vars.yml + + - ${{ if eq(parameters.enableSymbolValidation, 'true') }}: + - job: + displayName: Symbol Availability + dependsOn: setupMaestroVars + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicDevRelease_30_Channel_Id) + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: current + artifactName: PackageArtifacts + + - task: PowerShell@2 + displayName: Check Symbol Availability + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/symbols-validation.ps1 + arguments: -InputPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ -ExtractPath $(Agent.BuildDirectory)/Temp/ -DotnetSymbolVersion $(SymbolToolVersion) + + - job: + displayName: Gather Drop + dependsOn: setupMaestroVars + variables: + BARBuildId: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicDevRelease_30_Channel_Id) + pool: + vmImage: 'windows-2019' + steps: + - task: PowerShell@2 + displayName: Setup Darc CLI + inputs: + targetType: filePath + filePath: '$(Build.SourcesDirectory)/eng/common/darc-init.ps1' + + - task: PowerShell@2 + displayName: Run Darc gather-drop + inputs: + targetType: inline + script: | + darc gather-drop --non-shipping --continue-on-error --id $(BARBuildId) --output-dir $(Agent.BuildDirectory)/Temp/Drop/ --bar-uri https://maestro-prod.westus2.cloudapp.azure.com/ --password $(MaestroAccessToken) --latest-location + + - template: ../promote-build.yml + parameters: + ChannelId: ${{ variables.PublicDevRelease_30_Channel_Id }} diff --git a/eng/common/templates/post-build/channels/public-release.yml b/eng/common/templates/post-build/channels/public-release.yml new file mode 100644 index 000000000..25923020d --- /dev/null +++ b/eng/common/templates/post-build/channels/public-release.yml @@ -0,0 +1,170 @@ +parameters: + enableSymbolValidation: true + +stages: +- stage: PubRel_Publish + dependsOn: validate + variables: + - template: ../common-variables.yml + displayName: Public Release + jobs: + - template: ../setup-maestro-vars.yml + + - job: + displayName: Symbol Publishing + dependsOn: setupMaestroVars + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicRelease_30_Channel_Id) + variables: + - group: DotNet-Symbol-Server-Pats + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Artifacts + inputs: + downloadType: specific files + matchingPattern: "*Artifacts*" + + - task: PowerShell@2 + displayName: Publish + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishToSymbolServers -restore -msbuildEngine dotnet + /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) + /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) + /p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' + /p:Configuration=Release + + - job: publish_assets + displayName: Publish Assets + dependsOn: setupMaestroVars + variables: + - group: DotNet-Blob-Feed + - group: Publish-Build-Assets + - group: AzureDevOps-Artifact-Feeds-Pats + - name: BARBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] + - name: IsStableBuild + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ] + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicRelease_30_Channel_Id) + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: current + artifactName: PackageArtifacts + + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: current + artifactName: BlobArtifacts + + - task: DownloadBuildArtifacts@0 + displayName: Download Asset Manifests + inputs: + buildType: current + artifactName: AssetManifests + + - task: PowerShell@2 + displayName: Publish + env: + AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-unviersal-packages-rw) + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet + /p:ChannelId=$(PublicRelease_30_Channel_Id) + /p:IsStableBuild=$(IsStableBuild) + /p:IsInternalBuild=$(IsInternalBuild) + /p:RepositoryName=$(Build.Repository.Name) + /p:CommitSha=$(Build.SourceVersion) + /p:NugetPath=$(Agent.BuildDirectory)/Nuget/NuGet.exe + /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' + /p:TargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' + /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' + /p:BARBuildId=$(BARBuildId) + /p:MaestroApiEndpoint='https://maestro-prod.westus2.cloudapp.azure.com' + /p:BuildAssetRegistryToken='$(MaestroAccessToken)' + /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)\BlobArtifacts' + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)\PackageArtifacts' + /p:Configuration=Release + + - task: NuGetCommand@2 + displayName: Publish Packages to AzDO Feed + condition: contains(variables['TargetAzDOFeed'], 'pkgs.visualstudio.com') + inputs: + command: push + vstsFeed: $(AzDoFeedName) + packagesToPush: $(Build.ArtifactStagingDirectory)\PackageArtifacts\*.nupkg + publishVstsFeed: $(AzDoFeedName) + + - task: PowerShell@2 + displayName: Publish Blobs to AzDO Feed + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-blobs-to-azdo.ps1 + arguments: -FeedName $(AzDoFeedName) + -SourceFolderCollection $(Build.ArtifactStagingDirectory)/BlobArtifacts/ + -PersonalAccessToken $(dn-bot-dnceng-unviersal-packages-rw) + enabled: false + + - template: ../trigger-subscription.yml + parameters: + ChannelId: ${{ variables.PublicRelease_30_Channel_Id }} + +- stage: PubRel_PublishValidation + displayName: Publish Validation + variables: + - template: ../common-variables.yml + jobs: + - template: ../setup-maestro-vars.yml + + - ${{ if eq(parameters.enableSymbolValidation, 'true') }}: + - job: + displayName: Symbol Availability + dependsOn: setupMaestroVars + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicRelease_30_Channel_Id) + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: current + artifactName: PackageArtifacts + + - task: PowerShell@2 + displayName: Check Symbol Availability + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/symbols-validation.ps1 + arguments: -InputPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ -ExtractPath $(Agent.BuildDirectory)/Temp/ -DotnetSymbolVersion $(SymbolToolVersion) + + - job: + displayName: Gather Drop + dependsOn: setupMaestroVars + variables: + BARBuildId: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicRelease_30_Channel_Id) + pool: + vmImage: 'windows-2019' + steps: + - task: PowerShell@2 + displayName: Setup Darc CLI + inputs: + targetType: filePath + filePath: '$(Build.SourcesDirectory)/eng/common/darc-init.ps1' + + - task: PowerShell@2 + displayName: Run Darc gather-drop + inputs: + targetType: inline + script: | + darc gather-drop --non-shipping --continue-on-error --id $(BARBuildId) --output-dir $(Agent.BuildDirectory)/Temp/Drop/ --bar-uri https://maestro-prod.westus2.cloudapp.azure.com/ --password $(MaestroAccessToken) --latest-location + enabled: false + + - template: ../promote-build.yml + parameters: + ChannelId: ${{ variables.PublicRelease_30_Channel_Id }} diff --git a/eng/common/templates/post-build/channels/public-validation-release.yml b/eng/common/templates/post-build/channels/public-validation-release.yml new file mode 100644 index 000000000..114477d3a --- /dev/null +++ b/eng/common/templates/post-build/channels/public-validation-release.yml @@ -0,0 +1,119 @@ +stages: +- stage: PVR_Publish + dependsOn: validate + variables: + - template: ../common-variables.yml + displayName: Validation Channel + jobs: + - template: ../setup-maestro-vars.yml + + - job: + displayName: Publish Assets + dependsOn: setupMaestroVars + variables: + - group: DotNet-Blob-Feed + - group: Publish-Build-Assets + - group: AzureDevOps-Artifact-Feeds-Pats + - name: BARBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] + - name: IsStableBuild + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ] + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicValidationRelease_30_Channel_Id) + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: current + artifactName: PackageArtifacts + + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: current + artifactName: BlobArtifacts + + - task: DownloadBuildArtifacts@0 + displayName: Download Asset Manifests + inputs: + buildType: current + artifactName: AssetManifests + + - task: PowerShell@2 + displayName: Add Assets Location + env: + AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-unviersal-packages-rw) + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet + /p:ChannelId=$(PublicValidationRelease_30_Channel_Id) + /p:IsStableBuild=$(IsStableBuild) + /p:IsInternalBuild=$(IsInternalBuild) + /p:RepositoryName=$(Build.Repository.Name) + /p:CommitSha=$(Build.SourceVersion) + /p:NugetPath=$(Agent.BuildDirectory)\Nuget\NuGet.exe + /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' + /p:TargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' + /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' + /p:BARBuildId=$(BARBuildId) + /p:MaestroApiEndpoint='https://maestro-prod.westus2.cloudapp.azure.com' + /p:BuildAssetRegistryToken='$(MaestroAccessToken)' + /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)\BlobArtifacts' + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)\PackageArtifacts' + /p:Configuration=Release + + - task: NuGetCommand@2 + displayName: Publish Packages to AzDO Feed + condition: contains(variables['TargetAzDOFeed'], 'pkgs.visualstudio.com') + inputs: + command: push + vstsFeed: $(AzDoFeedName) + packagesToPush: $(Build.ArtifactStagingDirectory)\PackageArtifacts\*.nupkg + publishVstsFeed: $(AzDoFeedName) + + - task: PowerShell@2 + displayName: Publish Blobs to AzDO Feed + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-blobs-to-azdo.ps1 + arguments: -FeedName $(AzDoFeedName) + -SourceFolderCollection $(Build.ArtifactStagingDirectory)/BlobArtifacts/ + -PersonalAccessToken $(dn-bot-dnceng-unviersal-packages-rw) + enabled: false + + +- stage: PVR_PublishValidation + displayName: Publish Validation + variables: + - template: ../common-variables.yml + jobs: + - template: ../setup-maestro-vars.yml + + - job: + displayName: Gather Drop + dependsOn: setupMaestroVars + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicValidationRelease_30_Channel_Id) + variables: + - name: BARBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] + - group: Publish-Build-Assets + pool: + vmImage: 'windows-2019' + steps: + - task: PowerShell@2 + displayName: Setup Darc CLI + inputs: + targetType: filePath + filePath: '$(Build.SourcesDirectory)/eng/common/darc-init.ps1' + + - task: PowerShell@2 + displayName: Run Darc gather-drop + inputs: + targetType: inline + script: | + darc gather-drop --non-shipping --continue-on-error --id $(BARBuildId) --output-dir $(Agent.BuildDirectory)/Temp/Drop/ --bar-uri https://maestro-prod.westus2.cloudapp.azure.com --password $(MaestroAccessToken) --latest-location + + - template: ../promote-build.yml + parameters: + ChannelId: ${{ variables.PublicValidationRelease_30_Channel_Id }} diff --git a/eng/common/templates/post-build/common-variables.yml b/eng/common/templates/post-build/common-variables.yml new file mode 100644 index 000000000..828346735 --- /dev/null +++ b/eng/common/templates/post-build/common-variables.yml @@ -0,0 +1,18 @@ +variables: + # .NET Core 3 Dev + PublicDevRelease_30_Channel_Id: 3 + + # .NET Tools - Validation + PublicValidationRelease_30_Channel_Id: 9 + + # .NET Core 3.0 Internal Servicing + InternalServicing_30_Channel_Id: 184 + + # .NET Core 3.0 Release + PublicRelease_30_Channel_Id: 19 + + # Whether the build is internal or not + IsInternalBuild: ${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }} + + SourceLinkCLIVersion: 3.0.0 + SymbolToolVersion: 1.0.1 diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml new file mode 100644 index 000000000..daa799259 --- /dev/null +++ b/eng/common/templates/post-build/post-build.yml @@ -0,0 +1,91 @@ +parameters: + enableSourceLinkValidation: true + enableSigningValidation: true + enableSymbolValidation: true + enableNugetValidation: true + SDLValidationParameters: + enable: false + params: '' + +stages: +- stage: validate + dependsOn: build + displayName: Validate + jobs: + - ${{ if eq(parameters.enableNugetValidation, 'true') }}: + - job: + displayName: NuGet Validation + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: current + artifactName: PackageArtifacts + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1 + arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ + -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ + + - ${{ if eq(parameters.enableSigningValidation, 'true') }}: + - job: + displayName: Signing Validation + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: current + artifactName: PackageArtifacts + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task SigningValidation -restore -msbuildEngine dotnet + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' + /p:Configuration=Release + + - ${{ if eq(parameters.enableSourceLinkValidation, 'true') }}: + - job: + displayName: SourceLink Validation + variables: + - template: common-variables.yml + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: current + artifactName: BlobArtifacts + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 + arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ + -ExtractPath $(Agent.BuildDirectory)/Extract/ + -GHRepoName $(Build.Repository.Name) + -GHCommit $(Build.SourceVersion) + -SourcelinkCliVersion $(SourceLinkCLIVersion) + + - ${{ if eq(parameters.SDLValidationParameters.enable, 'true') }}: + - template: /eng/common/templates/job/execute-sdl.yml + parameters: + additionalParameters: ${{ parameters.SDLValidationParameters.params }} + +- template: \eng\common\templates\post-build\channels\public-dev-release.yml + parameters: + enableSymbolValidation: ${{ parameters.enableSymbolValidation }} + +- template: \eng\common\templates\post-build\channels\public-validation-release.yml + +- template: \eng\common\templates\post-build\channels\public-release.yml + +- template: \eng\common\templates\post-build\channels\internal-servicing.yml diff --git a/eng/common/templates/post-build/promote-build.yml b/eng/common/templates/post-build/promote-build.yml new file mode 100644 index 000000000..af48b0b33 --- /dev/null +++ b/eng/common/templates/post-build/promote-build.yml @@ -0,0 +1,24 @@ +parameters: + ChannelId: 0 + +jobs: +- job: + displayName: Promote Build + dependsOn: setupMaestroVars + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], ${{ parameters.ChannelId }}) + variables: + - name: BARBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] + - name: ChannelId + value: ${{ parameters.ChannelId }} + - group: Publish-Build-Assets + pool: + vmImage: 'windows-2019' + steps: + - task: PowerShell@2 + displayName: Add Build to Channel + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/promote-build.ps1 + arguments: -BuildId $(BARBuildId) + -ChannelId $(ChannelId) + -BarToken $(MaestroAccessToken) diff --git a/eng/common/templates/post-build/setup-maestro-vars.yml b/eng/common/templates/post-build/setup-maestro-vars.yml new file mode 100644 index 000000000..f6120dc1e --- /dev/null +++ b/eng/common/templates/post-build/setup-maestro-vars.yml @@ -0,0 +1,35 @@ +jobs: +- job: setupMaestroVars + displayName: Setup Maestro Vars + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Release Configs + inputs: + buildType: current + artifactName: ReleaseConfigs + + - task: PowerShell@2 + name: setReleaseVars + displayName: Set Release Configs Vars + inputs: + targetType: inline + script: | + # This is needed to make Write-PipelineSetVariable works in this context + $ci = $true + + . "$(Build.SourcesDirectory)/eng/common/tools.ps1" + + $Content = Get-Content "$(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt" + + $BarId = $Content | Select -Index 0 + + $Channels = "" + $Content | Select -Index 1 | ForEach-Object { $Channels += "$_ ," } + + $IsStableBuild = $Content | Select -Index 2 + + Write-PipelineSetVariable -Name 'BARBuildId' -Value $BarId + Write-PipelineSetVariable -Name 'InitialChannels' -Value "$Channels" + Write-PipelineSetVariable -Name 'IsStableBuild' -Value $IsStableBuild diff --git a/eng/common/templates/post-build/trigger-subscription.yml b/eng/common/templates/post-build/trigger-subscription.yml new file mode 100644 index 000000000..65259d4e6 --- /dev/null +++ b/eng/common/templates/post-build/trigger-subscription.yml @@ -0,0 +1,11 @@ +parameters: + ChannelId: 0 + +steps: +- task: PowerShell@2 + displayName: Triggering subscriptions + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/trigger-subscriptions.ps1 + arguments: -SourceRepo $(Build.Repository.Uri) + -ChannelId ${{ parameters.ChannelId }} + -BarToken $(MaestroAccessTokenInt) \ No newline at end of file diff --git a/eng/common/templates/steps/send-to-helix.yml b/eng/common/templates/steps/send-to-helix.yml index d1ce577db..05df886f5 100644 --- a/eng/common/templates/steps/send-to-helix.yml +++ b/eng/common/templates/steps/send-to-helix.yml @@ -5,6 +5,7 @@ parameters: HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number HelixTargetQueues: '' # required -- semicolon delimited list of Helix queues to test on; see https://helix.dot.net/ for a list of queues HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group + HelixConfiguration: '' # optional -- additional property attached to a job HelixPreCommands: '' # optional -- commands to run before Helix work item execution HelixPostCommands: '' # optional -- commands to run after Helix work item execution WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects @@ -35,6 +36,7 @@ steps: HelixSource: ${{ parameters.HelixSource }} HelixType: ${{ parameters.HelixType }} HelixBuild: ${{ parameters.HelixBuild }} + HelixConfiguration: ${{ parameters.HelixConfiguration }} HelixTargetQueues: ${{ parameters.HelixTargetQueues }} HelixAccessToken: ${{ parameters.HelixAccessToken }} HelixPreCommands: ${{ parameters.HelixPreCommands }} @@ -64,6 +66,7 @@ steps: HelixSource: ${{ parameters.HelixSource }} HelixType: ${{ parameters.HelixType }} HelixBuild: ${{ parameters.HelixBuild }} + HelixConfiguration: ${{ parameters.HelixConfiguration }} HelixTargetQueues: ${{ parameters.HelixTargetQueues }} HelixAccessToken: ${{ parameters.HelixAccessToken }} HelixPreCommands: ${{ parameters.HelixPreCommands }} diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 9ca177b82..60741f039 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -35,7 +35,7 @@ # Specifies which msbuild engine to use for build: 'vs', 'dotnet' or unspecified (determined based on presence of tools.vs in global.json). [string]$msbuildEngine = if (Test-Path variable:msbuildEngine) { $msbuildEngine } else { $null } -# True to attempt using .NET Core already that meets requirements specified in global.json +# True to attempt using .NET Core already that meets requirements specified in global.json # installed on the machine instead of downloading one. [bool]$useInstalledDotNetCli = if (Test-Path variable:useInstalledDotNetCli) { $useInstalledDotNetCli } else { $true } @@ -76,7 +76,7 @@ function Exec-Process([string]$command, [string]$commandArgs) { $finished = $false try { - while (-not $process.WaitForExit(100)) { + while (-not $process.WaitForExit(100)) { # Non-blocking loop done to allow ctr-c interrupts } @@ -134,7 +134,7 @@ function InitializeDotNetCli([bool]$install) { if ($install) { InstallDotNetSdk $dotnetRoot $dotnetSdkVersion } else { - Write-Host "Unable to find dotnet with SDK version '$dotnetSdkVersion'" -ForegroundColor Red + Write-PipelineTelemetryError -Category "InitializeToolset" -Message "Unable to find dotnet with SDK version '$dotnetSdkVersion'" ExitWithExitCode 1 } } @@ -147,12 +147,10 @@ function InitializeDotNetCli([bool]$install) { # It also ensures that VS msbuild will use the downloaded sdk targets. $env:PATH = "$dotnetRoot;$env:PATH" - if ($ci) { - # Make Sure that our bootstrapped dotnet cli is avaliable in future steps of the Azure Pipelines build - Write-Host "##vso[task.prependpath]$dotnetRoot" - Write-Host "##vso[task.setvariable variable=DOTNET_MULTILEVEL_LOOKUP]0" - Write-Host "##vso[task.setvariable variable=DOTNET_SKIP_FIRST_TIME_EXPERIENCE]1" - } + # Make Sure that our bootstrapped dotnet cli is avaliable in future steps of the Azure Pipelines build + Write-PipelinePrependPath -Path $dotnetRoot + Write-PipelineSetVariable -Name 'DOTNET_MULTILEVEL_LOOKUP' -Value '0' + Write-PipelineSetVariable -Name 'DOTNET_SKIP_FIRST_TIME_EXPERIENCE' -Value '1' return $global:_DotNetInstallDir = $dotnetRoot } @@ -184,13 +182,13 @@ function InstallDotNet([string] $dotnetRoot, [string] $version, [string] $archit & $installScript @installParameters if ($lastExitCode -ne 0) { - Write-Host "Failed to install dotnet cli (exit code '$lastExitCode')." -ForegroundColor Red + Write-PipelineTelemetryError -Category "InitializeToolset" -Message "Failed to install dotnet cli (exit code '$lastExitCode')." ExitWithExitCode $lastExitCode } } # -# Locates Visual Studio MSBuild installation. +# Locates Visual Studio MSBuild installation. # The preference order for MSBuild to use is as follows: # # 1. MSBuild from an active VS command prompt @@ -207,13 +205,17 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs } $vsMinVersionStr = if ($vsRequirements.version) { $vsRequirements.version } else { "15.9" } - $vsMinVersion = [Version]::new($vsMinVersionStr) + $vsMinVersion = [Version]::new($vsMinVersionStr) # Try msbuild command available in the environment. if ($env:VSINSTALLDIR -ne $null) { $msbuildCmd = Get-Command "msbuild.exe" -ErrorAction SilentlyContinue if ($msbuildCmd -ne $null) { - if ($msbuildCmd.Version -ge $vsMinVersion) { + # Workaround for https://github.com/dotnet/roslyn/issues/35793 + # Due to this issue $msbuildCmd.Version returns 0.0.0.0 for msbuild.exe 16.2+ + $msbuildVersion = [Version]::new((Get-Item $msbuildCmd.Path).VersionInfo.ProductVersion.Split(@('-', '+'))[0]) + + if ($msbuildVersion -ge $vsMinVersion) { return $global:_MSBuildExe = $msbuildCmd.Path } @@ -252,7 +254,7 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = function InitializeVisualStudioEnvironmentVariables([string] $vsInstallDir, [string] $vsMajorVersion) { $env:VSINSTALLDIR = $vsInstallDir Set-Item "env:VS$($vsMajorVersion)0COMNTOOLS" (Join-Path $vsInstallDir "Common7\Tools\") - + $vsSdkInstallDir = Join-Path $vsInstallDir "VSSDK\" if (Test-Path $vsSdkInstallDir) { Set-Item "env:VSSDK$($vsMajorVersion)0Install" $vsSdkInstallDir @@ -287,13 +289,13 @@ function InitializeXCopyMSBuild([string]$packageVersion, [bool]$install) { # Locates Visual Studio instance that meets the minimal requirements specified by tools.vs object in global.json. # # The following properties of tools.vs are recognized: -# "version": "{major}.{minor}" +# "version": "{major}.{minor}" # Two part minimal VS version, e.g. "15.9", "16.0", etc. -# "components": ["componentId1", "componentId2", ...] +# "components": ["componentId1", "componentId2", ...] # Array of ids of workload components that must be available in the VS instance. # See e.g. https://docs.microsoft.com/en-us/visualstudio/install/workload-component-id-vs-enterprise?view=vs-2017 # -# Returns JSON describing the located VS instance (same format as returned by vswhere), +# Returns JSON describing the located VS instance (same format as returned by vswhere), # or $null if no instance meeting the requirements is found on the machine. # function LocateVisualStudio([object]$vsRequirements = $null){ @@ -313,8 +315,8 @@ function LocateVisualStudio([object]$vsRequirements = $null){ } if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs } - $args = @("-latest", "-prerelease", "-format", "json", "-requires", "Microsoft.Component.MSBuild") - + $args = @("-latest", "-prerelease", "-format", "json", "-requires", "Microsoft.Component.MSBuild", "-products", "*") + if (Get-Member -InputObject $vsRequirements -Name "version") { $args += "-version" $args += $vsRequirements.version @@ -324,7 +326,7 @@ function LocateVisualStudio([object]$vsRequirements = $null){ foreach ($component in $vsRequirements.components) { $args += "-requires" $args += $component - } + } } $vsInfo =& $vsWhereExe $args | ConvertFrom-Json @@ -354,7 +356,7 @@ function InitializeBuildTool() { if ($msbuildEngine -eq "dotnet") { if (!$dotnetRoot) { - Write-Host "/global.json must specify 'tools.dotnet'." -ForegroundColor Red + Write-PipelineTelemetryError -Category "InitializeToolset" -Message "/global.json must specify 'tools.dotnet'." ExitWithExitCode 1 } @@ -363,13 +365,13 @@ function InitializeBuildTool() { try { $msbuildPath = InitializeVisualStudioMSBuild -install:$restore } catch { - Write-Host $_ -ForegroundColor Red + Write-PipelineTelemetryError -Category "InitializeToolset" -Message $_ ExitWithExitCode 1 } $buildTool = @{ Path = $msbuildPath; Command = ""; Tool = "vs"; Framework = "net472" } } else { - Write-Host "Unexpected value of -msbuildEngine: '$msbuildEngine'." -ForegroundColor Red + Write-PipelineTelemetryError -Category "InitializeToolset" -Message "Unexpected value of -msbuildEngine: '$msbuildEngine'." ExitWithExitCode 1 } @@ -381,12 +383,12 @@ function GetDefaultMSBuildEngine() { if (Get-Member -InputObject $GlobalJson.tools -Name "vs") { return "vs" } - + if (Get-Member -InputObject $GlobalJson.tools -Name "dotnet") { return "dotnet" } - Write-Host "-msbuildEngine must be specified, or /global.json must specify 'tools.dotnet' or 'tools.vs'." -ForegroundColor Red + Write-PipelineTelemetryError -Category "InitializeToolset" -Message "-msbuildEngine must be specified, or /global.json must specify 'tools.dotnet' or 'tools.vs'." ExitWithExitCode 1 } @@ -411,11 +413,13 @@ function GetSdkTaskProject([string]$taskName) { function InitializeNativeTools() { if (Get-Member -InputObject $GlobalJson -Name "native-tools") { - $nativeArgs="" + $nativeArgs= @{} if ($ci) { - $nativeArgs = "-InstallDirectory $ToolsDir" + $nativeArgs = @{ + InstallDirectory = "$ToolsDir" + } } - Invoke-Expression "& `"$PSScriptRoot/init-tools-native.ps1`" $nativeArgs" + & "$PSScriptRoot/init-tools-native.ps1" @nativeArgs } } @@ -437,7 +441,7 @@ function InitializeToolset() { } if (-not $restore) { - Write-Host "Toolset version $toolsetVersion has not been restored." -ForegroundColor Red + Write-PipelineTelemetryError -Category "InitializeToolset" -Message "Toolset version $toolsetVersion has not been restored." ExitWithExitCode 1 } @@ -497,11 +501,13 @@ function MSBuild() { function MSBuild-Core() { if ($ci) { if (!$binaryLog) { - throw "Binary log must be enabled in CI build." + Write-PipelineTaskError -Message "Binary log must be enabled in CI build." + ExitWithExitCode 1 } if ($nodeReuse) { - throw "Node reuse must be disabled in CI build." + Write-PipelineTaskError -Message "Node reuse must be disabled in CI build." + ExitWithExitCode 1 } } @@ -509,8 +515,8 @@ function MSBuild-Core() { $cmdArgs = "$($buildTool.Command) /m /nologo /clp:Summary /v:$verbosity /nr:$nodeReuse /p:ContinuousIntegrationBuild=$ci" - if ($warnAsError) { - $cmdArgs += " /warnaserror /p:TreatWarningsAsErrors=true" + if ($warnAsError) { + $cmdArgs += " /warnaserror /p:TreatWarningsAsErrors=true" } foreach ($arg in $args) { @@ -518,29 +524,29 @@ function MSBuild-Core() { $cmdArgs += " `"$arg`"" } } - + $exitCode = Exec-Process $buildTool.Path $cmdArgs if ($exitCode -ne 0) { - Write-Host "Build failed." -ForegroundColor Red + Write-PipelineTaskError -Message "Build failed." $buildLog = GetMSBuildBinaryLogCommandLineArgument $args - if ($buildLog -ne $null) { - Write-Host "See log: $buildLog" -ForegroundColor DarkGray + if ($buildLog -ne $null) { + Write-Host "See log: $buildLog" -ForegroundColor DarkGray } ExitWithExitCode $exitCode } } -function GetMSBuildBinaryLogCommandLineArgument($arguments) { +function GetMSBuildBinaryLogCommandLineArgument($arguments) { foreach ($argument in $arguments) { if ($argument -ne $null) { $arg = $argument.Trim() if ($arg.StartsWith("/bl:", "OrdinalIgnoreCase")) { return $arg.Substring("/bl:".Length) - } - + } + if ($arg.StartsWith("/binaryLogger:", "OrdinalIgnoreCase")) { return $arg.Substring("/binaryLogger:".Length) } @@ -550,6 +556,8 @@ function GetMSBuildBinaryLogCommandLineArgument($arguments) { return $null } +. $PSScriptRoot\pipeline-logging-functions.ps1 + $RepoRoot = Resolve-Path (Join-Path $PSScriptRoot "..\..") $EngRoot = Resolve-Path (Join-Path $PSScriptRoot "..") $ArtifactsDir = Join-Path $RepoRoot "artifacts" @@ -565,11 +573,8 @@ Create-Directory $ToolsetDir Create-Directory $TempDir Create-Directory $LogDir -if ($ci) { - Write-Host "##vso[task.setvariable variable=Artifacts]$ArtifactsDir" - Write-Host "##vso[task.setvariable variable=Artifacts.Toolset]$ToolsetDir" - Write-Host "##vso[task.setvariable variable=Artifacts.Log]$LogDir" - - $env:TEMP = $TempDir - $env:TMP = $TempDir -} +Write-PipelineSetVariable -Name 'Artifacts' -Value $ArtifactsDir +Write-PipelineSetVariable -Name 'Artifacts.Toolset' -Value $ToolsetDir +Write-PipelineSetVariable -Name 'Artifacts.Log' -Value $LogDir +Write-PipelineSetVariable -Name 'TEMP' -Value $TempDir +Write-PipelineSetVariable -Name 'TMP' -Value $TempDir diff --git a/eng/common/tools.sh b/eng/common/tools.sh index df3eb8bce..70d92cf85 100644 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -1,8 +1,20 @@ +#!/usr/bin/env bash + # Initialize variables if they aren't already defined. # CI mode - set to true on CI server for PR validation build or official build. ci=${ci:-false} +# Set to true to use the pipelines logger which will enable Azure logging output. +# https://github.com/Microsoft/azure-pipelines-tasks/blob/master/docs/authoring/commands.md +# This flag is meant as a temporary opt-opt for the feature while validate it across +# our consumers. It will be deleted in the future. +if [[ "$ci" == true ]]; then + pipelines_log=${pipelines_log:-true} +else + pipelines_log=${pipelines_log:-false} +fi + # Build configuration. Common values include 'Debug' and 'Release', but the repository may use other names. configuration=${configuration:-'Debug'} @@ -65,7 +77,7 @@ function ReadGlobalVersion { local pattern="\"$key\" *: *\"(.*)\"" if [[ ! $line =~ $pattern ]]; then - echo "Error: Cannot find \"$key\" in $global_json_file" >&2 + Write-PipelineTelemetryError -category 'InitializeTools' "Error: Cannot find \"$key\" in $global_json_file" ExitWithExitCode 1 fi @@ -126,7 +138,7 @@ function InitializeDotNetCli { if [[ "$install" == true ]]; then InstallDotNetSdk "$dotnet_root" "$dotnet_sdk_version" else - echo "Unable to find dotnet with SDK version '$dotnet_sdk_version'" >&2 + Write-PipelineTelemetryError -category 'InitializeToolset' "Unable to find dotnet with SDK version '$dotnet_sdk_version'" ExitWithExitCode 1 fi fi @@ -137,7 +149,7 @@ function InitializeDotNetCli { export PATH="$dotnet_root:$PATH" if [[ $ci == true ]]; then - # Make Sure that our bootstrapped dotnet cli is avaliable in future steps of the Azure Pipelines build + # Make Sure that our bootstrapped dotnet cli is available in future steps of the Azure Pipelines build echo "##vso[task.prependpath]$dotnet_root" echo "##vso[task.setvariable variable=DOTNET_MULTILEVEL_LOOKUP]0" echo "##vso[task.setvariable variable=DOTNET_SKIP_FIRST_TIME_EXPERIENCE]1" @@ -179,7 +191,7 @@ function InstallDotNet { fi bash "$install_script" --version $version --install-dir "$root" $archArg $runtimeArg $skipNonVersionedFilesArg || { local exit_code=$? - echo "Failed to install dotnet SDK (exit code '$exit_code')." >&2 + Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK (exit code '$exit_code')." ExitWithExitCode $exit_code } } @@ -216,6 +228,7 @@ function InitializeBuildTool { # return values _InitializeBuildTool="$_InitializeDotNetCli/dotnet" _InitializeBuildToolCommand="msbuild" + _InitializeBuildToolFramework="netcoreapp2.1" } function GetNuGetPackageCachePath { @@ -264,7 +277,7 @@ function InitializeToolset { fi if [[ "$restore" != true ]]; then - echo "Toolset version $toolsetVersion has not been restored." >&2 + Write-PipelineTelemetryError -category 'InitializeToolset' "Toolset version $toolset_version has not been restored." ExitWithExitCode 2 fi @@ -276,12 +289,12 @@ function InitializeToolset { fi echo '' > "$proj" - MSBuild "$proj" $bl /t:__WriteToolsetLocation /clp:ErrorsOnly\;NoSummary /p:__ToolsetLocationOutputFile="$toolset_location_file" + MSBuild-Core "$proj" $bl /t:__WriteToolsetLocation /clp:ErrorsOnly\;NoSummary /p:__ToolsetLocationOutputFile="$toolset_location_file" local toolset_build_proj=`cat "$toolset_location_file"` if [[ ! -a "$toolset_build_proj" ]]; then - echo "Invalid toolset path: $toolset_build_proj" >&2 + Write-PipelineTelemetryError -category 'InitializeToolset' "Invalid toolset path: $toolset_build_proj" ExitWithExitCode 3 fi @@ -304,14 +317,27 @@ function StopProcesses { } function MSBuild { + local args=$@ + if [[ "$pipelines_log" == true ]]; then + InitializeBuildTool + InitializeToolset + local toolset_dir="${_InitializeToolset%/*}" + local logger_path="$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.Arcade.Sdk.dll" + args=( "${args[@]}" "-logger:$logger_path" ) + fi + + MSBuild-Core ${args[@]} +} + +function MSBuild-Core { if [[ "$ci" == true ]]; then if [[ "$binary_log" != true ]]; then - echo "Binary log must be enabled in CI build." >&2 + Write-PipelineTaskError "Binary log must be enabled in CI build." ExitWithExitCode 1 fi if [[ "$node_reuse" == true ]]; then - echo "Node reuse must be disabled in CI build." >&2 + Write-PipelineTaskError "Node reuse must be disabled in CI build." ExitWithExitCode 1 fi fi @@ -325,7 +351,7 @@ function MSBuild { "$_InitializeBuildTool" "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@" || { local exit_code=$? - echo "Build failed (exit code '$exit_code')." >&2 + Write-PipelineTaskError "Build failed (exit code '$exit_code')." ExitWithExitCode $exit_code } } @@ -333,6 +359,8 @@ function MSBuild { ResolvePath "${BASH_SOURCE[0]}" _script_dir=`dirname "$_ResolvePath"` +. "$_script_dir/pipeline-logging-functions.sh" + eng_root=`cd -P "$_script_dir/.." && pwd` repo_root=`cd -P "$_script_dir/../.." && pwd` artifacts_dir="$repo_root/artifacts" @@ -362,4 +390,4 @@ mkdir -p "$log_dir" if [[ $ci == true ]]; then export TEMP="$temp_dir" export TMP="$temp_dir" -fi \ No newline at end of file +fi diff --git a/global.json b/global.json index 1e1c1ac9b..f66327607 100644 --- a/global.json +++ b/global.json @@ -1,8 +1,8 @@ { "tools": { - "dotnet": "3.0.100-preview4-011223" + "dotnet": "3.0.100-preview6-012264" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19263.3" + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19330.1" } } diff --git a/src/CopyToLatest/targets/BranchInfo.props b/src/CopyToLatest/targets/BranchInfo.props index d103b00f0..9bd214aa9 100644 --- a/src/CopyToLatest/targets/BranchInfo.props +++ b/src/CopyToLatest/targets/BranchInfo.props @@ -1,5 +1,5 @@ - release/3.0.1xx + master diff --git a/src/SdkResolver/SdkResolver.csproj b/src/SdkResolver/SdkResolver.csproj index 2cd928e14..70089c1f1 100644 --- a/src/SdkResolver/SdkResolver.csproj +++ b/src/SdkResolver/SdkResolver.csproj @@ -7,6 +7,7 @@ false false UNUSED + true diff --git a/src/core-sdk-tasks/core-sdk-tasks.csproj b/src/core-sdk-tasks/core-sdk-tasks.csproj index 7b04a4c20..d6c8968d4 100644 --- a/src/core-sdk-tasks/core-sdk-tasks.csproj +++ b/src/core-sdk-tasks/core-sdk-tasks.csproj @@ -8,8 +8,8 @@ - - + + diff --git a/src/redist/targets/DownloadPackage.csproj b/src/redist/targets/DownloadPackage.csproj index 618f9c510..41f96901e 100644 --- a/src/redist/targets/DownloadPackage.csproj +++ b/src/redist/targets/DownloadPackage.csproj @@ -1,5 +1,6 @@ - + + Library false @@ -7,13 +8,13 @@ - + - + - - - - + + + + diff --git a/src/redist/targets/GenerateBundledVersions.targets b/src/redist/targets/GenerateBundledVersions.targets index 676d3748c..0c7ef6dc2 100644 --- a/src/redist/targets/GenerateBundledVersions.targets +++ b/src/redist/targets/GenerateBundledVersions.targets @@ -75,6 +75,7 @@ win-arm; osx-x64; linux-musl-x64; + linux-musl-arm64; linux-x64; linux-arm; linux-arm64" /> @@ -187,9 +188,7 @@ Copyright (c) .NET Foundation. All rights reserved. LatestRuntimeFrameworkVersion="$(_NETCoreAppPackageVersion)" TargetingPackName="Microsoft.NETCore.App.Ref" TargetingPackVersion="$(NetCoreAppTargetingPackVersion)" - AppHostPackNamePattern="runtime.**RID**.Microsoft.NETCore.DotNetAppHost" - AppHostRuntimeIdentifiers="@(NetCoreRuntimePackRids, '%3B')" - RuntimePackNamePatterns="runtime.**RID**.Microsoft.NETCore.App%3Bruntime.**RID**.Microsoft.NETCore.DotNetHostResolver%3Bruntime.**RID**.Microsoft.NETCore.DotNetHostPolicy" + RuntimePackNamePatterns="Microsoft.NETCore.App.Runtime.**RID**" RuntimePackRuntimeIdentifiers="@(NetCoreRuntimePackRids, '%3B')" PackagesToReference="Microsoft.NETCore.App/$(NetCoreAppTargetingPackVersion)" IsTrimmable="true" @@ -209,11 +208,37 @@ Copyright (c) .NET Foundation. All rights reserved. LatestRuntimeFrameworkVersion="$(MicrosoftWindowsDesktopPackageVersion)" TargetingPackName="Microsoft.WindowsDesktop.App.Ref" TargetingPackVersion="$(WindowsDesktopTargetingPackVersion)" - RuntimePackNamePatterns="runtime.**RID**.Microsoft.WindowsDesktop.App" + RuntimePackNamePatterns="Microsoft.WindowsDesktop.App.Runtime.**RID**" RuntimePackRuntimeIdentifiers="@(WindowsDesktopRuntimePackRids, '%3B')" IsWindowsOnly="true" /> + + + + --> - $(RedistLayoutPath)sdk\$(SdkVersion)\ - https://dotnetcli.azureedge.net/dotnet/ + https://dotnetcli.azureedge.net/dotnet/ + https://dotnetcli.blob.core.windows.net/dotnet/ https://dotnetfeed.blob.core.windows.net/dotnet-toolset/ - $(HostRid) + $(HostRid) $(HostMonikerRid) @@ -358,7 +359,7 @@ - Microsoft.NETCore.DotNetAppHost + Microsoft.NETCore.App.Host.$(SharedFrameworkRid) $(IntermediateOutputPath)AppHostRestore\ $(SdkOutputDirectory)AppHostTemplate diff --git a/src/redist/targets/GenerateMSIs.targets b/src/redist/targets/GenerateMSIs.targets index e3f643963..dffddf82c 100644 --- a/src/redist/targets/GenerateMSIs.targets +++ b/src/redist/targets/GenerateMSIs.targets @@ -17,11 +17,11 @@ $(MSBuildThisFileDirectory)packaging/windows/clisdk/generatemsi.ps1 $(MSBuildThisFileDirectory)packaging/windows/clisdk/stablefileidforapphosttransform.xslt $(MSBuildThisFileDirectory)packaging/windows/clisdk/generatebundle.ps1 - $(MSBuildThisFileDirectory)packaging/windows/clisdk/generatenupkg.ps1 + $(MSBuildThisFileDirectory)packaging/windows/clisdk/generatenupkg.ps1 - $(MSBuildThisFileDirectory)packaging/windows/clisdk/VS.Redist.Common.Net.Core.SDK.$(Architecture).nuspec - $(ArtifactsNonShippingPackagesDir)VS.Redist.Common.Net.Core.SDK.$(Architecture).$(FullNugetVersion).nupkg - $(MSBuildThisFileDirectory)packaging/windows/clisdk/VS.Tools.Net.Core.SDK.$(Architecture).nuspec + $(MSBuildThisFileDirectory)packaging/windows/clisdk/VS.Redist.Common.NetCore.Toolset.nuspec + $(ArtifactsNonShippingPackagesDir)VS.Redist.Common.NetCore.Toolset.$(Architecture).$(FullNugetVersion).nupkg + $(MSBuildThisFileDirectory)packaging/windows/clisdk/VS.Tools.Net.Core.SDK.nuspec $(ArtifactsNonShippingPackagesDir)VS.Tools.Net.Core.SDK.$(Architecture).$(FullNugetVersion).nupkg $(ArtifactsDir)bin/SdkResolver/$(Configuration) @@ -35,6 +35,7 @@ $(ArtifactsShippingPackagesDir)$(ArtifactNameWithVersionSdk)$(InstallerExtension) + $(ArtifactsShippingPackagesDir)d$(Architecture)-1.cab $(ArtifactsShippingPackagesDir)$(ArtifactNameWithVersionCombinedHostHostFxrFrameworkSdk)$(BundleExtension) $(ProductBandCombinedHostHostFxrFrameworkSdkName) @@ -158,45 +159,48 @@ -InformationAction Continue " /> - + Inputs="$(SdkMSIInstallerFile); + $(ToolsetInstallerNuspecFile); + $(GenerateNupkgPowershellScript)" + Outputs="$(ToolsetInstallerNupkgFile)"> - + '$(ToolsetInstallerNuspecFile)' ^ + '$(ToolsetInstallerNupkgFile)' ^ + '$(Architecture)' ^ + '$(SdkMSICabFile)'" /> - + '$(VSToolsNupkgFile)' ^ + '$(Architecture)'" /> - - - - - - - - - - - - - - - - - - - - - 0 1 00 - preview6 + preview7 diff --git a/src/redist/targets/packaging/windows/clisdk/VS.Redist.Common.Net.Core.SDK.MSBuildExtensions.nuspec b/src/redist/targets/packaging/windows/clisdk/VS.Redist.Common.Net.Core.SDK.MSBuildExtensions.nuspec index b9dd3f55d..4d5b44e81 100644 --- a/src/redist/targets/packaging/windows/clisdk/VS.Redist.Common.Net.Core.SDK.MSBuildExtensions.nuspec +++ b/src/redist/targets/packaging/windows/clisdk/VS.Redist.Common.Net.Core.SDK.MSBuildExtensions.nuspec @@ -9,10 +9,10 @@ https://www.microsoft.com/net/dotnet_library_license.htm https://github.com/dotnet/core-sdk true - MSBuild extensions bundled with .NET Core SDK insertions to VS + MSBuild extensions bundled with .NET Core SDK for internal Visual Studio build consumption © Microsoft Corporation. All rights reserved. - + diff --git a/src/redist/targets/packaging/windows/clisdk/VS.Redist.Common.Net.Core.SDK.x86.nuspec b/src/redist/targets/packaging/windows/clisdk/VS.Redist.Common.Net.Core.SDK.x86.nuspec deleted file mode 100644 index a62f9d2a1..000000000 --- a/src/redist/targets/packaging/windows/clisdk/VS.Redist.Common.Net.Core.SDK.x86.nuspec +++ /dev/null @@ -1,18 +0,0 @@ - - - - VS.Redist.Common.Net.Core.SDK.x86 - 1.0.0 - VS.Redist.Common.Net.Core.SDK.x86 - Microsoft - Microsoft - https://www.microsoft.com/net/dotnet_library_license.htm - https://github.com/dotnet/core-sdk - true - Windows Installer (x86) for .Net Core SDK - © Microsoft Corporation. All rights reserved. - - - - - diff --git a/src/redist/targets/packaging/windows/clisdk/VS.Redist.Common.Net.Core.SDK.x64.nuspec b/src/redist/targets/packaging/windows/clisdk/VS.Redist.Common.NetCore.Toolset.nuspec similarity index 63% rename from src/redist/targets/packaging/windows/clisdk/VS.Redist.Common.Net.Core.SDK.x64.nuspec rename to src/redist/targets/packaging/windows/clisdk/VS.Redist.Common.NetCore.Toolset.nuspec index d5d6a2565..ba0555841 100644 --- a/src/redist/targets/packaging/windows/clisdk/VS.Redist.Common.Net.Core.SDK.x64.nuspec +++ b/src/redist/targets/packaging/windows/clisdk/VS.Redist.Common.NetCore.Toolset.nuspec @@ -1,18 +1,19 @@  - VS.Redist.Common.Net.Core.SDK.x64 + VS.Redist.Common.NetCore.Toolset.$ARCH$ 1.0.0 - VS.Redist.Common.Net.Core.SDK.x64 + VS.Redist.Common.NetCore.Toolset.$ARCH$ Microsoft Microsoft https://www.microsoft.com/net/dotnet_library_license.htm https://github.com/dotnet/core-sdk true - Windows Installers (x64) for .Net Core SDK + .NET Core SDK Toolset ($ARCH$) Windows Installer MSI as a .nupkg for internal Visual Studio build consumption © Microsoft Corporation. All rights reserved. - + + diff --git a/src/redist/targets/packaging/windows/clisdk/VS.Tools.Net.Core.SDK.Resolver.nuspec b/src/redist/targets/packaging/windows/clisdk/VS.Tools.Net.Core.SDK.Resolver.nuspec index b3e8a6930..bf30cb4e1 100644 --- a/src/redist/targets/packaging/windows/clisdk/VS.Tools.Net.Core.SDK.Resolver.nuspec +++ b/src/redist/targets/packaging/windows/clisdk/VS.Tools.Net.Core.SDK.Resolver.nuspec @@ -9,11 +9,11 @@ https://www.microsoft.com/net/dotnet_library_license.htm https://github.com/dotnet/core-sdk true - .Net Core SDK MSBuild resolver for internal VS build consumption + .NET Core SDK MSBuild resolver as a .nupkg for internal Visual Studio build consumption © Microsoft Corporation. All rights reserved. - + diff --git a/src/redist/targets/packaging/windows/clisdk/VS.Tools.Net.Core.SDK.x86.nuspec b/src/redist/targets/packaging/windows/clisdk/VS.Tools.Net.Core.SDK.nuspec similarity index 70% rename from src/redist/targets/packaging/windows/clisdk/VS.Tools.Net.Core.SDK.x86.nuspec rename to src/redist/targets/packaging/windows/clisdk/VS.Tools.Net.Core.SDK.nuspec index 814067d06..b986d3440 100644 --- a/src/redist/targets/packaging/windows/clisdk/VS.Tools.Net.Core.SDK.x86.nuspec +++ b/src/redist/targets/packaging/windows/clisdk/VS.Tools.Net.Core.SDK.nuspec @@ -1,19 +1,19 @@  - VS.Tools.Net.Core.SDK.x86 + VS.Tools.Net.Core.SDK.$ARCH$ 1.0.0 - VS.Tools.Net.Core.SDK.x86 + VS.Tools.Net.Core.SDK.$ARCH$ Microsoft Microsoft https://www.microsoft.com/net/dotnet_library_license.htm https://github.com/dotnet/core-sdk true - .Net Core SDK (x86) as a .nupkg for internal VS build consumption + .NET Core SDK ($ARCH$) as a .nupkg for internal Visual Studio build consumption © Microsoft Corporation. All rights reserved. - + diff --git a/src/redist/targets/packaging/windows/clisdk/generatenupkg.ps1 b/src/redist/targets/packaging/windows/clisdk/generatenupkg.ps1 index ffd020dca..db130d14e 100644 --- a/src/redist/targets/packaging/windows/clisdk/generatenupkg.ps1 +++ b/src/redist/targets/packaging/windows/clisdk/generatenupkg.ps1 @@ -2,17 +2,22 @@ # Licensed under the MIT license. See LICENSE file in the project root for full license information. param( - [Parameter(Mandatory=$true)][string]$SdkBundlePath, + [Parameter(Mandatory=$true)][string]$ContentPath, [Parameter(Mandatory=$true)][string]$NugetVersion, [Parameter(Mandatory=$true)][string]$NuspecFile, - [Parameter(Mandatory=$true)][string]$NupkgFile + [Parameter(Mandatory=$true)][string]$NupkgFile, + [Parameter(Mandatory=$false)][string]$Architecture, + [Parameter(Mandatory=$false)][string]$CabPath ) $RepoRoot = Convert-Path "$PSScriptRoot\..\..\..\..\..\.." $NuGetDir = Join-Path $RepoRoot "artifacts\Tools\nuget" $NuGetExe = Join-Path $NuGetDir "nuget.exe" $OutputDirectory = [System.IO.Path]::GetDirectoryName($NupkgFile) -$SdkBundlePath = [System.IO.Path]::GetFullPath($SdkBundlePath) +$ContentPath = [System.IO.Path]::GetFullPath($ContentPath) +if ($CabPath) { + $CabPath = [System.IO.Path]::GetFullPath($CabPath) +} if (-not (Test-Path $NuGetDir)) { New-Item -ItemType Directory -Force -Path $NuGetDir | Out-Null @@ -28,5 +33,5 @@ if (Test-Path $NupkgFile) { Remove-Item -Force $NupkgFile } -& $NuGetExe pack $NuspecFile -Version $NugetVersion -OutputDirectory $OutputDirectory -NoDefaultExcludes -NoPackageAnalysis -Properties DOTNET_BUNDLE=$SdkBundlePath +& $NuGetExe pack $NuspecFile -Version $NugetVersion -OutputDirectory $OutputDirectory -NoDefaultExcludes -NoPackageAnalysis -Properties PAYLOAD_FILES=$ContentPath`;DOTNET_CAB_FILE=$CabPath`;ARCH=$Architecture Exit $LastExitCode diff --git a/test/EndToEnd/GivenAspNetAppsResolveImplicitVersions.cs b/test/EndToEnd/GivenAspNetAppsResolveImplicitVersions.cs deleted file mode 100644 index f4443cf96..000000000 --- a/test/EndToEnd/GivenAspNetAppsResolveImplicitVersions.cs +++ /dev/null @@ -1,235 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Xml.Linq; -using FluentAssertions; -using Microsoft.DotNet.TestFramework; -using Microsoft.DotNet.Tools.Test.Utilities; -using NuGet.ProjectModel; -using NuGet.Versioning; -using Xunit; - -namespace EndToEnd -{ - public class GivenAspNetAppsResolveImplicitVersions : TestBase - { - private const string AspNetTestProject = "TestWebAppSimple"; - - [Fact(Skip ="https://github.com/dotnet/core-sdk/issues/21")] - public void PortablePublishWithLatestTFMUsesBundledAspNetCoreAppVersion() - { - var _testInstance = TestAssets.Get(AspNetTestProject) - .CreateInstance(identifier: LatestSupportedAspNetCoreAppVersion) - .WithSourceFiles(); - - string projectDirectory = _testInstance.Root.FullName; - string projectPath = Path.Combine(projectDirectory, $"{AspNetTestProject}.csproj"); - - var project = XDocument.Load(projectPath); - var ns = project.Root.Name.Namespace; - - // Update TargetFramework to the right version of .NET Core - project.Root.Element(ns + "PropertyGroup") - .Element(ns + "TargetFramework") - .Value = "netcoreapp" + LatestSupportedAspNetCoreAppVersion; - - project.Save(projectPath); - - // Get the implicit version - new RestoreCommand() - .WithWorkingDirectory(projectDirectory) - .Execute("/bl:restore.binlog") - .Should().Pass(); - - var assetsFilePath = Path.Combine(projectDirectory, "obj", "project.assets.json"); - var assetsFile = new LockFileFormat().Read(assetsFilePath); - - var restoredVersion = GetAspNetCoreAppVersion(assetsFile, portable: true); - restoredVersion.Should().NotBeNull(); - - var bundledVersionPath = Path.Combine(projectDirectory, ".BundledAspNetCoreVersion"); - var bundledVersion = File.ReadAllText(bundledVersionPath).Trim(); - - restoredVersion.ToNormalizedString().Should().BeEquivalentTo(bundledVersion, - "The bundled aspnetcore versions set in Microsoft.NETCoreSdk.BundledVersions.props should be identical to the versions generated." + - "Please update MSBuildExtensions.targets in this repo so these versions match."); - } - - [Fact(Skip = "https://github.com/dotnet/core-sdk/issues/21")] - public void StandalonePublishWithLatestTFMUsesBundledAspNetCoreAppVersion() - { - var _testInstance = TestAssets.Get(AspNetTestProject) - .CreateInstance(identifier: LatestSupportedAspNetCoreAppVersion) - .WithSourceFiles(); - - string projectDirectory = _testInstance.Root.FullName; - string projectPath = Path.Combine(projectDirectory, $"{AspNetTestProject}.csproj"); - - var project = XDocument.Load(projectPath); - var ns = project.Root.Name.Namespace; - - // Update TargetFramework to the right version of .NET Core - project.Root.Element(ns + "PropertyGroup") - .Element(ns + "TargetFramework") - .Value = "netcoreapp" + LatestSupportedAspNetCoreAppVersion; - - var rid = Microsoft.DotNet.PlatformAbstractions.RuntimeEnvironment.GetRuntimeIdentifier(); - - // Set RuntimeIdentifier to simulate standalone publish - project.Root.Element(ns + "PropertyGroup") - .Add(new XElement(ns + "RuntimeIdentifier", rid)); - - project.Save(projectPath); - - // Get the implicit version - new RestoreCommand() - .WithWorkingDirectory(projectDirectory) - .Execute() - .Should().Pass(); - - var assetsFilePath = Path.Combine(projectDirectory, "obj", "project.assets.json"); - var assetsFile = new LockFileFormat().Read(assetsFilePath); - - var restoredVersion = GetAspNetCoreAppVersion(assetsFile); - restoredVersion.Should().NotBeNull(); - - var bundledVersionPath = Path.Combine(projectDirectory, ".BundledAspNetCoreVersion"); - var bundledVersion = File.ReadAllText(bundledVersionPath).Trim(); - - restoredVersion.ToNormalizedString().Should().BeEquivalentTo(bundledVersion, - "The bundled aspnetcore versions set in Microsoft.NETCoreSdk.BundledVersions.props should be identical to the versions set in DependencyVersions.props." + - "Please update MSBuildExtensions.targets in this repo so these versions match."); - } - - [Theory] - [MemberData(nameof(SupportedAspNetCoreAppVersions))] - public void ItRollsForwardToTheLatestVersion(string minorVersion) - { - if (minorVersion == "3.0") - { - // https://github.com/dotnet/core-sdk/issues/621 - return; - } - - var testProjectCreator = new TestProjectCreator(identifier: minorVersion) - { - PackageName = TestProjectCreator.AspNetCoreAppPackageName, - MinorVersion = minorVersion , - }; - - var _testInstance = testProjectCreator.Create() - .WithSourceFiles(); - - string projectDirectory = _testInstance.Root.FullName; - - string projectPath = Path.Combine(projectDirectory, $"TestAppSimple.csproj"); - - var project = XDocument.Load(projectPath); - var ns = project.Root.Name.Namespace; - - // Update TargetFramework to the right version of .NET Core - project.Root.Element(ns + "PropertyGroup") - .Element(ns + "TargetFramework") - .Value = "netcoreapp" + minorVersion; - - var rid = Microsoft.DotNet.PlatformAbstractions.RuntimeEnvironment.GetRuntimeIdentifier(); - - // Set RuntimeIdentifier to opt in to roll-forward behavior - project.Root.Element(ns + "PropertyGroup") - .Add(new XElement(ns + "RuntimeIdentifier", rid)); - - project.Save(projectPath); - - // Get the version rolled forward to - new RestoreCommand() - .WithWorkingDirectory(projectDirectory) - .Execute() - .Should().Pass(); - - string assetsFilePath = Path.Combine(projectDirectory, "obj", "project.assets.json"); - var assetsFile = new LockFileFormat().Read(assetsFilePath); - - var rolledForwardVersion = GetAspNetCoreAppVersion(assetsFile); - rolledForwardVersion.Should().NotBeNull(); - - if (rolledForwardVersion.IsPrerelease) - { - // If this version of .NET Core is still prerelease, then: - // - Floating the patch by adding ".*" to the major.minor version won't work, but - // - There aren't any patches to roll-forward to, so we skip testing this until the version - // leaves prerelease. - return; - } - - // Float the RuntimeFrameworkVersion to get the latest version of the runtime available from feeds - Directory.Delete(Path.Combine(projectDirectory, "obj"), true); - project.Root.Element(ns + "PropertyGroup") - .Add(new XElement(ns + "RuntimeFrameworkVersion", $"{minorVersion}.*")); - project.Save(projectPath); - - new RestoreCommand() - .WithWorkingDirectory(projectDirectory) - .Execute() - .Should().Pass(); - - var floatedAssetsFile = new LockFileFormat().Read(assetsFilePath); - - var floatedVersion = GetAspNetCoreAppVersion(floatedAssetsFile); - floatedVersion.Should().NotBeNull(); - - rolledForwardVersion.ToNormalizedString().Should().BeEquivalentTo(floatedVersion.ToNormalizedString(), - "the latest patch version properties in Microsoft.NETCoreSdk.BundledVersions.props need to be updated " + - "(see MSBuildExtensions.targets in this repo)"); - } - - [Fact] - public void WeCoverLatestAspNetCoreAppRollForward() - { - var directory = TestAssets.CreateTestDirectory(); - string projectDirectory = directory.FullName; - - // Run "dotnet new web", get TargetFramework property, and make sure it's covered in SupportedAspNetCoreAppVersions - - new NewCommandShim() - .WithWorkingDirectory(projectDirectory) - .Execute("web --no-restore") - .Should().Pass(); - - string projectPath = Path.Combine(projectDirectory, Path.GetFileName(projectDirectory) + ".csproj"); - - var project = XDocument.Load(projectPath); - var ns = project.Root.Name.Namespace; - - string targetFramework = project.Root.Element(ns + "PropertyGroup") - .Element(ns + "TargetFramework") - .Value; - - SupportedAspNetCoreAppVersions.Select(v => $"netcoreapp{v[0]}") - .Should().Contain(targetFramework, $"the {nameof(SupportedAspNetCoreAppVersions)} property should include the default version " + - "of Microsoft.AspNetCore.App used by the templates created by \"dotnet new web\""); - - } - - private NuGetVersion GetAspNetCoreAppVersion(LockFile lockFile, bool portable = false) - { - return lockFile?.Targets?.SingleOrDefault(t => portable == (t.RuntimeIdentifier == null)) - ?.Libraries?.SingleOrDefault(l => - string.Compare(l.Name, "Microsoft.AspNetCore.App", StringComparison.CurrentCultureIgnoreCase) == 0) - ?.Version; - } - - public static string LatestSupportedAspNetCoreAppVersion = "3.0"; - - public static IEnumerable SupportedAspNetCoreAppVersions - { - get - { - yield return new object[] { "2.1" }; - // https://github.com/dotnet/core-sdk/issues/780 - // yield return new object[] { "2.2" }; - yield return new object[] { LatestSupportedAspNetCoreAppVersion }; - } - } - } -} diff --git a/test/EndToEnd/GivenUnixPlatform.cs b/test/EndToEnd/GivenUnixPlatform.cs index 83e377f7a..bb93b72fe 100644 --- a/test/EndToEnd/GivenUnixPlatform.cs +++ b/test/EndToEnd/GivenUnixPlatform.cs @@ -10,7 +10,7 @@ namespace EndToEnd.Tests { public class GivenUnixPlatform : TestBase { - [UnixOnlyTheory] + [UnixOnlyTheory(Skip="https://github.com/dotnet/templating/issues/1979")] [InlineData("wpf")] [InlineData("winforms")] public void ItDoesNotIncludeWindowsOnlyProjectTemplates(string template) diff --git a/test/EndToEnd/SupportedNetCoreAppVersions.cs b/test/EndToEnd/SupportedNetCoreAppVersions.cs index 593396c51..6c9adff95 100644 --- a/test/EndToEnd/SupportedNetCoreAppVersions.cs +++ b/test/EndToEnd/SupportedNetCoreAppVersions.cs @@ -20,8 +20,7 @@ namespace EndToEnd "1.1", "2.0", "2.1", - // https://github.com/dotnet/core-sdk/issues/780 - // "2.2", + "2.2", "3.0" }; }