diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 1dacc5750..31dff1c60 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -10,8 +10,11 @@ variables: value: https://dotnetcli.blob.core.windows.net/dotnet/index.json - name: _PublishChecksumsBlobFeedUrl value: https://dotnetclichecksums.blob.core.windows.net/dotnet/index.json +- name: _ArcadePublishBlobFeedUrl + value: https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - group: DotNet-DotNetCli-Storage + - group: DotNet-Blob-Feed jobs: - template: /eng/build.yml @@ -57,7 +60,9 @@ jobs: _BuildArchitecture: x64 _DOTNET_CLI_UI_LANGUAGE: '' _DropSuffix: '' - _AdditionalBuildParameters: '/p:PublishInternalAsset=true' + _AdditionalBuildParameters: '/p:PublishInternalAsset=true + /p:DotNetPublishBlobFeedKey=$(dotnetfeed-storage-access-key-1) + /p:DotNetPublishBlobFeedUrl=$(_ArcadePublishBlobFeedUrl)' Build_Release_arm: _BuildConfig: Release _BuildArchitecture: arm diff --git a/Directory.Build.props b/Directory.Build.props index b05a362ef..b65a72410 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -16,8 +16,7 @@ netcoreapp3.0 $(ArtifactsDir)symbols\$(Configuration)\Shipping - - NU5125;NU5105 + NU5125;NU5105;NU1701 diff --git a/NuGet.config b/NuGet.config index 8f21de1b1..227180760 100644 --- a/NuGet.config +++ b/NuGet.config @@ -8,6 +8,15 @@ + + + + + + + + + diff --git a/eng/Publishing.props b/eng/Publishing.props index 1a6232b31..f40cf64be 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -8,7 +8,6 @@ Sdk $(Product) assets/$(Product) - true $(DotnetPublishChecksumsBlobFeedUrl) $(DotNetPublishChecksumsBlobFeedKey) $(DotnetPublishSdkAssetsBlobFeedUrl) @@ -50,6 +49,7 @@ true + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 4811241a1..f62f5a032 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -2,72 +2,72 @@ - + https://github.com/dotnet/core-setup - 4da6ee6450e4dd43ef8c05925fec22d9ac3892f3 + 823e54187b997cb8303f44cd8a6d4fc1adefee47 - + https://github.com/dotnet/core-setup - 4da6ee6450e4dd43ef8c05925fec22d9ac3892f3 + 823e54187b997cb8303f44cd8a6d4fc1adefee47 - + https://github.com/dotnet/core-setup - 4da6ee6450e4dd43ef8c05925fec22d9ac3892f3 + 823e54187b997cb8303f44cd8a6d4fc1adefee47 - + https://github.com/aspnet/AspNetCore - d27c0100bc8b9b93eb8102b097c20b40f7de9c49 + c23b9feb19a9809a82e541427028b63ca66fa847 - + https://github.com/aspnet/AspNetCore - d27c0100bc8b9b93eb8102b097c20b40f7de9c49 + c23b9feb19a9809a82e541427028b63ca66fa847 - + https://github.com/aspnet/AspNetCore - d27c0100bc8b9b93eb8102b097c20b40f7de9c49 + c23b9feb19a9809a82e541427028b63ca66fa847 - + https://github.com/aspnet/AspNetCore - d27c0100bc8b9b93eb8102b097c20b40f7de9c49 + c23b9feb19a9809a82e541427028b63ca66fa847 - + https://github.com/aspnet/AspNetCore - d27c0100bc8b9b93eb8102b097c20b40f7de9c49 + c23b9feb19a9809a82e541427028b63ca66fa847 - + https://github.com/aspnet/AspNetCore - d27c0100bc8b9b93eb8102b097c20b40f7de9c49 + c23b9feb19a9809a82e541427028b63ca66fa847 - + https://github.com/dotnet/templating - 98b3cf626437e671f0f4d216a8b208f1ce36008c + bf2a1c940b5c11075626dba31c119e6de3137df0 - + https://github.com/dotnet/toolset - 7f41a7da0404eeb740b6d06ee74d70a7bdc25aec + b875d6b8bdddac469866ee0f080a7641ff7617d4 - + https://github.com/dotnet/sdk - 78b8210cf94a9d516de694815ce746e84ab21545 + d8cbde7b101e21dc6e1cd413fa3920197d28c107 - + https://github.com/dotnet/cli - cc245f5512f90e0ad591452c740847e971dede18 + ad7a14419de328074faf7ba287c01da22d868735 - + https://github.com/dotnet/winforms - 52be7e5282560d26926a288d75a8489cd856ec4a + 38b423c42682b9a24119a9c25b9a16ca0255d896 - + https://github.com/dotnet/wpf - 76573a9bb7ab5cd61dbf4456600631e63accf5c0 + 7b28ced98e1830b6417ebda79cf803abd6688872 - + https://github.com/dotnet/arcade - 89fab80685c91024c8f9e21f1c37f62580f648f8 + 8b670142ae1b65aa4d834257b0b64d4f69f30eb8 diff --git a/eng/Versions.props b/eng/Versions.props index 9ff53e9c5..d13ef535a 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -12,42 +12,39 @@ preview - 3.0.100-preview7.19366.11 + 3.0.100-preview8.19376.13 - 4.8.0-preview7.19352.2 + 4.8.0-preview8.19378.1 - 3.0.0-preview7.19352.4 + 3.0.0-preview8.19378.5 - 3.0.0-preview7.19365.7 - 3.0.0-preview7.19365.7 - 3.0.0-preview7.19365.7 - 3.0.0-preview7.19365.7 - 3.0.0-preview7.19365.7 - 3.0.0-preview7.19365.7 + 3.0.0-preview8.19377.1 + 3.0.0-preview8.19377.1 + 3.0.0-preview8.19377.1 + 3.0.0-preview8.19377.1 + 3.0.0-preview8.19377.1 + 3.0.0-preview8.19377.1 - 3.0.100-preview7.19366.7 + 3.0.100-preview8.19378.7 - 2.0.0-preview7.19365.3 - $(MicrosoftDotNetCommonItemTemplatesPackageVersion) - 1.0.2-beta4.19155.2 0.2.0 - 1.6.1 - 3.0.100-preview7.19362.3 + 3.0.100-preview8.19378.1 + $(MicrosoftNETSdkPackageVersion) - 3.0.0-preview7-27912-14 + 3.0.0-preview8-28378-08 $(MicrosoftNETCoreAppPackageVersion) $(MicrosoftNETCoreAppPackageVersion) $(MicrosoftNETCoreAppPackageVersion) @@ -55,13 +52,35 @@ $(MicrosoftNETCoreAppPackageVersion) $(MicrosoftNETCoreAppPackageVersion) $(MicrosoftNETCoreAppPackageVersion) - 2.1.0-preview7-27912-14 + 2.1.0-preview8-28378-08 $(MicrosoftNETCoreAppPackageVersion) $(MicrosoftAspNetCoreAppRuntimeWinX64PackageVersion) $(MicrosoftAspNetCoreAppRefPackageVersion) - 3.0.0-preview7-27912-14 + 3.0.0-preview8-28378-08 $(MicrosoftWindowsDesktopAppPackageVersion) + + + $(MicrosoftDotnetWinFormsProjectTemplatesPackageVersion) + $(MicrosoftDotNetWpfProjectTemplatesPackageVersion) + 1.6.1 + 2.0.0-preview8.19373.1 + $(MicrosoftDotNetCommonItemTemplates30PackageVersion) + 1.0.2-beta4.19354.2 + $(AspNetCoreVersion) + + 1.6.0 + 1.0.2-beta4 + $(MicrosoftDotNetCommonItemTemplates22PackageVersion) + 1.0.2-beta4.19155.2 + 2.2.6 + + 1.5.3 + 1.0.2-beta3 + $(MicrosoftDotNetCommonItemTemplates21PackageVersion) + 1.0.2-beta4-20181009-2100240 + 2.1.12 + 2.2.0-beta.19072.10 @@ -70,6 +89,10 @@ 15.8.0 1.0.0-beta2-18618-05 + + + 2.0.1-servicing-26011-01 + $(RestoreSources); diff --git a/eng/build.yml b/eng/build.yml index 92add5df4..c47ccad98 100644 --- a/eng/build.yml +++ b/eng/build.yml @@ -37,6 +37,7 @@ phases: - group: DotNet-Symbol-Server-PATs - group: DotNet-HelixApi-Access - _DotNetPublishToBlobFeed: true + - _PushToVSFeed: true - _SignType: real - _BuildArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER) /p:DotNetPublishToBlobFeed=$(_DotNetPublishToBlobFeed) @@ -98,9 +99,19 @@ phases: $(_AdditionalBuildParameters) displayName: Build + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.agentOs, 'Windows_NT')) }}: + - task: NuGetCommand@2 + displayName: Push Visual Studio NuPkgs + inputs: + command: push + packagesToPush: '$(Build.SourcesDirectory)/artifacts/packages/$(_BuildConfig)/NonShipping/VS.*.nupkg' + nuGetFeedType: external + publishFeedCredentials: 'DevDiv - VS package feed' + condition: and(succeeded(), eq(variables['_PushToVSFeed'], 'true'), eq(variables['_DotNetPublishToBlobFeed'], 'true'), or(eq(variables['_BuildArchitecture'], 'x64'), eq(variables['_BuildArchitecture'], 'x86'))) + - task: PublishTestResults@1 displayName: Publish Test Results - inputs: + inputs: testRunner: XUnit testResultsFiles: 'artifacts/TestResults/$(_BuildConfig)/*.xml' testRunTitle: '$(_AgentOSName)_$(Agent.JobName)' @@ -110,7 +121,7 @@ phases: - task: CopyFiles@2 displayName: Gather Logs - inputs: + inputs: SourceFolder: '$(Build.SourcesDirectory)/artifacts' Contents: | log/$(_BuildConfig)/**/* @@ -121,26 +132,9 @@ phases: - task: PublishBuildArtifacts@1 displayName: Publish Logs to VSTS - inputs: + inputs: PathtoPublish: '$(Build.ArtifactStagingDirectory)' ArtifactName: '$(_AgentOSName)_$(Agent.JobName)_$(Build.BuildNumber)' publishLocation: Container continueOnError: true condition: not(succeeded()) - - - ${{ if and(eq(parameters.enablePublishBuildAssets, true), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - task: CopyFiles@2 - displayName: Gather Asset Manifests - inputs: - SourceFolder: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/AssetManifest' - TargetFolder: '$(Build.StagingDirectory)/AssetManifests' - continueOnError: ${{ parameters.continueOnError }} - condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true'), eq(variables['_PublishBuildAssetManifest'], 'true')) - - task: PublishBuildArtifacts@1 - displayName: Push Asset Manifests - inputs: - PathtoPublish: '$(Build.StagingDirectory)/AssetManifests' - PublishLocation: Container - ArtifactName: AssetManifests - continueOnError: ${{ parameters.continueOnError }} - condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true'), eq(variables['_PublishBuildAssetManifest'], 'true')) \ No newline at end of file diff --git a/eng/common/SigningValidation.proj b/eng/common/SigningValidation.proj index 7045fb6fb..3d0ac80af 100644 --- a/eng/common/SigningValidation.proj +++ b/eng/common/SigningValidation.proj @@ -3,7 +3,7 @@ + + + $(WorkItemDirectory) + $(WorkItemCommand) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument) --artifacts $(ArtifactsDirectory) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)" + 4:00 + + + + + $(WorkItemDirectory) + $(WorkItemCommand) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument) --artifacts $(ArtifactsDirectory)" + 4:00 + + + \ No newline at end of file diff --git a/eng/common/performance/performance-setup.ps1 b/eng/common/performance/performance-setup.ps1 new file mode 100644 index 000000000..7e5441f79 --- /dev/null +++ b/eng/common/performance/performance-setup.ps1 @@ -0,0 +1,91 @@ +Param( + [string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY, + [string] $CoreRootDirectory, + [string] $Architecture="x64", + [string] $Framework="netcoreapp3.0", + [string] $CompilationMode="Tiered", + [string] $Repository=$env:BUILD_REPOSITORY_NAME, + [string] $Branch=$env:BUILD_SOURCEBRANCH, + [string] $CommitSha=$env:BUILD_SOURCEVERSION, + [string] $BuildNumber=$env:BUILD_BUILDNUMBER, + [string] $RunCategories="coreclr corefx", + [string] $Csproj="src\benchmarks\micro\MicroBenchmarks.csproj", + [string] $Kind="micro", + [switch] $Internal, + [string] $Configurations="CompilationMode=$CompilationMode" +) + +$RunFromPerformanceRepo = ($Repository -eq "dotnet/performance") +$UseCoreRun = ($CoreRootDirectory -ne [string]::Empty) + +$PayloadDirectory = (Join-Path $SourceDirectory "Payload") +$PerformanceDirectory = (Join-Path $PayloadDirectory "performance") +$WorkItemDirectory = (Join-Path $SourceDirectory "workitem") +$ExtraBenchmarkDotNetArguments = "--iterationCount 1 --warmupCount 0 --invocationCount 1 --unrollFactor 1 --strategy ColdStart --stopOnFirstError true" +$Creator = $env:BUILD_DEFINITIONNAME +$PerfLabArguments = "" +$HelixSourcePrefix = "pr" + +$Queue = "Windows.10.Amd64.ClientRS4.DevEx.15.8.Open" + +if ($Framework.StartsWith("netcoreapp")) { + $Queue = "Windows.10.Amd64.ClientRS4.Open" +} + +if ($Internal) { + $Queue = "Windows.10.Amd64.ClientRS5.Perf" + $PerfLabArguments = "--upload-to-perflab-container" + $ExtraBenchmarkDotNetArguments = "" + $Creator = "" + $HelixSourcePrefix = "official" +} + +$CommonSetupArguments="--frameworks $Framework --queue $Queue --build-number $BuildNumber --build-configs $Configurations" +$SetupArguments = "--repository https://github.com/$Repository --branch $Branch --get-perf-hash --commit-sha $CommitSha $CommonSetupArguments" + +if ($RunFromPerformanceRepo) { + $SetupArguments = "--perf-hash $CommitSha $CommonSetupArguments" + + robocopy $SourceDirectory $PerformanceDirectory /E /XD $PayloadDirectory $SourceDirectory\artifacts $SourceDirectory\.git +} +else { + git clone --branch master --depth 1 --quiet https://github.com/dotnet/performance $PerformanceDirectory +} + +if ($UseCoreRun) { + $NewCoreRoot = (Join-Path $PayloadDirectory "Core_Root") + Move-Item -Path $CoreRootDirectory -Destination $NewCoreRoot +} + +$DocsDir = (Join-Path $PerformanceDirectory "docs") +robocopy $DocsDir $WorkItemDirectory + +# Set variables that we will need to have in future steps +$ci = $true + +. "$PSScriptRoot\..\pipeline-logging-functions.ps1" + +# Directories +Write-PipelineSetVariable -Name 'PayloadDirectory' -Value "$PayloadDirectory" -IsMultiJobVariable $false +Write-PipelineSetVariable -Name 'PerformanceDirectory' -Value "$PerformanceDirectory" -IsMultiJobVariable $false +Write-PipelineSetVariable -Name 'WorkItemDirectory' -Value "$WorkItemDirectory" -IsMultiJobVariable $false + +# Script Arguments +Write-PipelineSetVariable -Name 'Python' -Value "py -3" -IsMultiJobVariable $false +Write-PipelineSetVariable -Name 'ExtraBenchmarkDotNetArguments' -Value "$ExtraBenchmarkDotNetArguments" -IsMultiJobVariable $false +Write-PipelineSetVariable -Name 'SetupArguments' -Value "$SetupArguments" -IsMultiJobVariable $false +Write-PipelineSetVariable -Name 'PerfLabArguments' -Value "$PerfLabArguments" -IsMultiJobVariable $false +Write-PipelineSetVariable -Name 'BDNCategories' -Value "$RunCategories" -IsMultiJobVariable $false +Write-PipelineSetVariable -Name 'TargetCsproj' -Value "$Csproj" -IsMultiJobVariable $false +Write-PipelineSetVariable -Name 'Kind' -Value "$Kind" -IsMultiJobVariable $false +Write-PipelineSetVariable -Name 'Architecture' -Value "$Architecture" -IsMultiJobVariable $false +Write-PipelineSetVariable -Name 'UseCoreRun' -Value "$UseCoreRun" -IsMultiJobVariable $false +Write-PipelineSetVariable -Name 'RunFromPerfRepo' -Value "$RunFromPerformanceRepo" -IsMultiJobVariable $false + +# Helix Arguments +Write-PipelineSetVariable -Name 'Creator' -Value "$Creator" -IsMultiJobVariable $false +Write-PipelineSetVariable -Name 'Queue' -Value "$Queue" -IsMultiJobVariable $false +Write-PipelineSetVariable -Name 'HelixSourcePrefix' -Value "$HelixSourcePrefix" -IsMultiJobVariable $false +Write-PipelineSetVariable -Name '_BuildConfig' -Value "$Architecture.$Kind.$Framework" -IsMultiJobVariable $false + +exit 0 \ No newline at end of file diff --git a/eng/common/performance/performance-setup.sh b/eng/common/performance/performance-setup.sh new file mode 100755 index 000000000..126da5f76 --- /dev/null +++ b/eng/common/performance/performance-setup.sh @@ -0,0 +1,176 @@ +#!/usr/bin/env bash + +source_directory=$BUILD_SOURCESDIRECTORY +core_root_directory= +architecture=x64 +framework=netcoreapp3.0 +compilation_mode=tiered +repository=$BUILD_REPOSITORY_NAME +branch=$BUILD_SOURCEBRANCH +commit_sha=$BUILD_SOURCEVERSION +build_number=$BUILD_BUILDNUMBER +internal=false +kind="micro" +run_categories="coreclr corefx" +csproj="src\benchmarks\micro\MicroBenchmarks.csproj" +configurations= +run_from_perf_repo=false +use_core_run=true + +while (($# > 0)); do + lowerI="$(echo $1 | awk '{print tolower($0)}')" + case $lowerI in + --sourcedirectory) + source_directory=$2 + shift 2 + ;; + --corerootdirectory) + core_root_directory=$2 + shift 2 + ;; + --architecture) + architecture=$2 + shift 2 + ;; + --framework) + framework=$2 + shift 2 + ;; + --compilationmode) + compilation_mode=$2 + shift 2 + ;; + --repository) + repository=$2 + shift 2 + ;; + --branch) + branch=$2 + shift 2 + ;; + --commitsha) + commit_sha=$2 + shift 2 + ;; + --buildnumber) + build_number=$2 + shift 2 + ;; + --kind) + kind=$2 + shift 2 + ;; + --runcategories) + run_categories=$2 + shift 2 + ;; + --csproj) + csproj=$2 + shift 2 + ;; + --internal) + internal=true + shift 1 + ;; + --configurations) + configurations=$2 + shift 2 + ;; + --help) + echo "Common settings:" + echo " --corerootdirectory Directory where Core_Root exists, if running perf testing with --corerun" + echo " --architecture Architecture of the testing being run" + echo " --configurations List of key=value pairs that will be passed to perf testing infrastructure." + echo " ex: --configurations \"CompilationMode=Tiered OptimzationLevel=PGO\"" + echo " --help Print help and exit" + echo "" + echo "Advanced settings:" + echo " --framework The framework to run, if not running in master" + echo " --compliationmode The compilation mode if not passing --configurations" + echo " --sourcedirectory The directory of the sources. Defaults to env:BUILD_SOURCESDIRECTORY" + echo " --repository The name of the repository in the / format. Defaults to env:BUILD_REPOSITORY_NAME" + echo " --branch The name of the branch. Defaults to env:BUILD_SOURCEBRANCH" + echo " --commitsha The commit sha1 to run against. Defaults to env:BUILD_SOURCEVERSION" + echo " --buildnumber The build number currently running. Defaults to env:BUILD_BUILDNUMBER" + echo " --csproj The relative path to the benchmark csproj whose tests should be run. Defaults to src\benchmarks\micro\MicroBenchmarks.csproj" + echo " --kind Related to csproj. The kind of benchmarks that should be run. Defaults to micro" + echo " --runcategories Related to csproj. Categories of benchmarks to run. Defaults to \"coreclr corefx\"" + echo " --internal If the benchmarks are running as an official job." + echo "" + exit 0 + ;; + esac +done + +if [[ "$repository" == "dotnet/performance" ]]; then + run_from_perf_repo=true +fi + +if [ -z "$configurations" ]; then + configurations="CompliationMode=$compilation_mode" +fi + +if [ -z "$core_root_directory" ]; then + use_core_run=false +fi + +payload_directory=$source_directory/Payload +performance_directory=$payload_directory/performance +workitem_directory=$source_directory/workitem +extra_benchmark_dotnet_arguments="--iterationCount 1 --warmupCount 0 --invocationCount 1 --unrollFactor 1 --strategy ColdStart --stopOnFirstError true" +perflab_arguments= +queue=Ubuntu.1804.Amd64.Open +creator=$BUILD_DEFINITIONNAME +helix_source_prefix="pr" + +if [[ "$internal" == true ]]; then + perflab_arguments="--upload-to-perflab-container" + helix_source_prefix="official" + creator= + extra_benchmark_dotnet_arguments= + + if [[ "$architecture" = "arm64" ]]; then + queue=Ubuntu.1804.Arm64.Perf + else + queue=Ubuntu.1804.Amd64.Perf + fi +fi + +common_setup_arguments="--frameworks $framework --queue $queue --build-number $build_number --build-configs $configurations" +setup_arguments="--repository https://github.com/$repository --branch $branch --get-perf-hash --commit-sha $commit_sha $common_setup_arguments" + +if [[ "$run_from_perf_repo" = true ]]; then + payload_directory= + workitem_directory=$source_directory + performance_directory=$workitem_directory + setup_arguments="--perf-hash $commit_sha $common_setup_arguments" +else + git clone --branch master --depth 1 --quiet https://github.com/dotnet/performance $performance_directory + + docs_directory=$performance_directory/docs + mv $docs_directory $workitem_directory +fi + +if [[ "$use_core_run" = true ]]; then + new_core_root=$payload_directory/Core_Root + mv $core_root_directory $new_core_root +fi + +# Make sure all of our variables are available for future steps +echo "##vso[task.setvariable variable=UseCoreRun]$use_core_run" +echo "##vso[task.setvariable variable=Architecture]$architecture" +echo "##vso[task.setvariable variable=PayloadDirectory]$payload_directory" +echo "##vso[task.setvariable variable=PerformanceDirectory]$performance_directory" +echo "##vso[task.setvariable variable=WorkItemDirectory]$workitem_directory" +echo "##vso[task.setvariable variable=Queue]$queue" +echo "##vso[task.setvariable variable=SetupArguments]$setup_arguments" +echo "##vso[task.setvariable variable=Python]python3" +echo "##vso[task.setvariable variable=PerfLabArguments]$perflab_arguments" +echo "##vso[task.setvariable variable=ExtraBenchmarkDotNetArguments]$extra_benchmark_dotnet_arguments" +echo "##vso[task.setvariable variable=BDNCategories]$run_categories" +echo "##vso[task.setvariable variable=TargetCsproj]$csproj" +echo "##vso[task.setvariable variable=RunFromPerfRepo]$run_from_perf_repo" +echo "##vso[task.setvariable variable=Creator]$creator" +echo "##vso[task.setvariable variable=HelixSourcePrefix]$helix_source_prefix" +echo "##vso[task.setvariable variable=Kind]$kind" +echo "##vso[task.setvariable variable=_BuildConfig]$architecture.$kind.$framework" \ No newline at end of file diff --git a/eng/common/pipeline-logging-functions.ps1 b/eng/common/pipeline-logging-functions.ps1 index 7b61376f8..af5f48aac 100644 --- a/eng/common/pipeline-logging-functions.ps1 +++ b/eng/common/pipeline-logging-functions.ps1 @@ -77,13 +77,14 @@ function Write-PipelineTaskError { [string]$Name, [string]$Value, [switch]$Secret, - [switch]$AsOutput) - + [switch]$AsOutput, + [bool]$IsMultiJobVariable=$true) + if($ci) { Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data $Value -Properties @{ 'variable' = $Name 'isSecret' = $Secret - 'isOutput' = 'true' + 'isOutput' = $IsMultiJobVariable } -AsOutput:$AsOutput } } diff --git a/eng/common/pipeline-logging-functions.sh b/eng/common/pipeline-logging-functions.sh old mode 100644 new mode 100755 index 6098f9a54..1c560a506 --- a/eng/common/pipeline-logging-functions.sh +++ b/eng/common/pipeline-logging-functions.sh @@ -39,11 +39,11 @@ function Write-PipelineTaskError { return fi - message_type="error" - sourcepath='' - linenumber='' - columnnumber='' - error_code='' + local message_type="error" + local sourcepath='' + local linenumber='' + local columnnumber='' + local error_code='' while [[ $# -gt 0 ]]; do opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')" @@ -76,7 +76,7 @@ function Write-PipelineTaskError { shift done - message="##vso[task.logissue" + local message="##vso[task.logissue" message="$message type=$message_type" @@ -100,3 +100,73 @@ function Write-PipelineTaskError { echo "$message" } +function Write-PipelineSetVariable { + if [[ "$ci" != true ]]; then + return + fi + + local name='' + local value='' + local secret=false + local as_output=false + local is_multi_job_variable=true + + while [[ $# -gt 0 ]]; do + opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')" + case "$opt" in + -name|-n) + name=$2 + shift + ;; + -value|-v) + value=$2 + shift + ;; + -secret|-s) + secret=true + ;; + -as_output|-a) + as_output=true + ;; + -is_multi_job_variable|-i) + is_multi_job_variable=$2 + shift + ;; + esac + shift + done + + value=${value/;/%3B} + value=${value/\\r/%0D} + value=${value/\\n/%0A} + value=${value/]/%5D} + + local message="##vso[task.setvariable variable=$name;isSecret=$secret;isOutput=$is_multi_job_variable]$value" + + if [[ "$as_output" == true ]]; then + $message + else + echo "$message" + fi +} + +function Write-PipelinePrependPath { + local prepend_path='' + + while [[ $# -gt 0 ]]; do + opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')" + case "$opt" in + -path|-p) + prepend_path=$2 + shift + ;; + esac + shift + done + + export PATH="$prepend_path:$PATH" + + if [[ "$ci" == true ]]; then + echo "##vso[task.prependpath]$prepend_path" + fi +} \ No newline at end of file diff --git a/eng/common/post-build/darc-gather-drop.ps1 b/eng/common/post-build/darc-gather-drop.ps1 new file mode 100644 index 000000000..93a0bd832 --- /dev/null +++ b/eng/common/post-build/darc-gather-drop.ps1 @@ -0,0 +1,35 @@ +param( + [Parameter(Mandatory=$true)][int] $BarBuildId, # ID of the build which assets should be downloaded + [Parameter(Mandatory=$true)][string] $DropLocation, # Where the assets should be downloaded to + [Parameter(Mandatory=$true)][string] $MaestroApiAccessToken, # Token used to access Maestro API + [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = "https://maestro-prod.westus2.cloudapp.azure.com", # Maestro API URL + [Parameter(Mandatory=$false)][string] $MaestroApiVersion = "2019-01-16" # Version of Maestro API to use +) + +. $PSScriptRoot\post-build-utils.ps1 + +try { + Write-Host "Installing DARC ..." + + . $PSScriptRoot\..\darc-init.ps1 + $exitCode = $LASTEXITCODE + + if ($exitCode -ne 0) { + Write-PipelineTaskError "Something failed while running 'darc-init.ps1'. Check for errors above. Exiting now..." + ExitWithExitCode $exitCode + } + + darc gather-drop --non-shipping ` + --continue-on-error ` + --id $BarBuildId ` + --output-dir $DropLocation ` + --bar-uri $MaestroApiEndpoint ` + --password $MaestroApiAccessToken ` + --latest-location +} +catch { + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + ExitWithExitCode 1 +} diff --git a/eng/common/post-build/nuget-validation.ps1 b/eng/common/post-build/nuget-validation.ps1 index 1bdced1e3..78ed0d540 100644 --- a/eng/common/post-build/nuget-validation.ps1 +++ b/eng/common/post-build/nuget-validation.ps1 @@ -6,10 +6,7 @@ param( [Parameter(Mandatory=$true)][string] $ToolDestinationPath # Where the validation tool should be downloaded to ) -$ErrorActionPreference = "Stop" -Set-StrictMode -Version 2.0 - -. $PSScriptRoot\..\tools.ps1 +. $PSScriptRoot\post-build-utils.ps1 try { $url = "https://raw.githubusercontent.com/NuGet/NuGetGallery/jver-verify/src/VerifyMicrosoftPackage/verify.ps1" diff --git a/eng/common/post-build/post-build-utils.ps1 b/eng/common/post-build/post-build-utils.ps1 new file mode 100644 index 000000000..551ae113f --- /dev/null +++ b/eng/common/post-build/post-build-utils.ps1 @@ -0,0 +1,90 @@ +# Most of the functions in this file require the variables `MaestroApiEndPoint`, +# `MaestroApiVersion` and `MaestroApiAccessToken` to be globally available. + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 + +# `tools.ps1` checks $ci to perform some actions. Since the post-build +# scripts don't necessarily execute in the same agent that run the +# build.ps1/sh script this variable isn't automatically set. +$ci = $true +. $PSScriptRoot\..\tools.ps1 + +function Create-MaestroApiRequestHeaders([string]$ContentType = "application/json") { + Validate-MaestroVars + + $headers = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]' + $headers.Add('Accept', $ContentType) + $headers.Add('Authorization',"Bearer $MaestroApiAccessToken") + return $headers +} + +function Get-MaestroChannel([int]$ChannelId) { + Validate-MaestroVars + + $apiHeaders = Create-MaestroApiRequestHeaders + $apiEndpoint = "$MaestroApiEndPoint/api/channels/${ChannelId}?api-version=$MaestroApiVersion" + + $result = try { Invoke-WebRequest -Method Get -Uri $apiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } + return $result +} + +function Get-MaestroBuild([int]$BuildId) { + Validate-MaestroVars + + $apiHeaders = Create-MaestroApiRequestHeaders -AuthToken $MaestroApiAccessToken + $apiEndpoint = "$MaestroApiEndPoint/api/builds/${BuildId}?api-version=$MaestroApiVersion" + + $result = try { return Invoke-WebRequest -Method Get -Uri $apiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } + return $result +} + +function Get-MaestroSubscriptions([string]$SourceRepository, [int]$ChannelId) { + Validate-MaestroVars + + $SourceRepository = [System.Web.HttpUtility]::UrlEncode($SourceRepository) + $apiHeaders = Create-MaestroApiRequestHeaders -AuthToken $MaestroApiAccessToken + $apiEndpoint = "$MaestroApiEndPoint/api/subscriptions?sourceRepository=$SourceRepository&channelId=$ChannelId&api-version=$MaestroApiVersion" + + $result = try { Invoke-WebRequest -Method Get -Uri $apiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } + return $result +} + +function Trigger-Subscription([string]$SubscriptionId) { + Validate-MaestroVars + + $apiHeaders = Create-MaestroApiRequestHeaders -AuthToken $MaestroApiAccessToken + $apiEndpoint = "$MaestroApiEndPoint/api/subscriptions/$SubscriptionId/trigger?api-version=$MaestroApiVersion" + Invoke-WebRequest -Uri $apiEndpoint -Headers $apiHeaders -Method Post | Out-Null +} + +function Assign-BuildToChannel([int]$BuildId, [int]$ChannelId) { + Validate-MaestroVars + + $apiHeaders = Create-MaestroApiRequestHeaders -AuthToken $MaestroApiAccessToken + $apiEndpoint = "$MaestroApiEndPoint/api/channels/${ChannelId}/builds/${BuildId}?api-version=$MaestroApiVersion" + Invoke-WebRequest -Method Post -Uri $apiEndpoint -Headers $apiHeaders | Out-Null +} + +function Validate-MaestroVars { + try { + Get-Variable MaestroApiEndPoint -Scope Global | Out-Null + Get-Variable MaestroApiVersion -Scope Global | Out-Null + Get-Variable MaestroApiAccessToken -Scope Global | Out-Null + + if (!($MaestroApiEndPoint -Match "^http[s]?://maestro-(int|prod).westus2.cloudapp.azure.com$")) { + Write-PipelineTaskError "MaestroApiEndPoint is not a valid Maestro URL. '$MaestroApiEndPoint'" + ExitWithExitCode 1 + } + + if (!($MaestroApiVersion -Match "^[0-9]{4}-[0-9]{2}-[0-9]{2}$")) { + Write-PipelineTaskError "MaestroApiVersion does not match a version string in the format yyyy-MM-DD. '$MaestroApiVersion'" + ExitWithExitCode 1 + } + } + catch { + Write-PipelineTaskError "Error: Variables `MaestroApiEndPoint`, `MaestroApiVersion` and `MaestroApiAccessToken` are required while using this script." + Write-Host $_ + ExitWithExitCode 1 + } +} diff --git a/eng/common/post-build/promote-build.ps1 b/eng/common/post-build/promote-build.ps1 index 84a608fa5..e5ae85f25 100644 --- a/eng/common/post-build/promote-build.ps1 +++ b/eng/common/post-build/promote-build.ps1 @@ -1,30 +1,25 @@ param( [Parameter(Mandatory=$true)][int] $BuildId, [Parameter(Mandatory=$true)][int] $ChannelId, - [Parameter(Mandatory=$true)][string] $BarToken, - [string] $MaestroEndpoint = "https://maestro-prod.westus2.cloudapp.azure.com", - [string] $ApiVersion = "2019-01-16" + [Parameter(Mandatory=$true)][string] $MaestroApiAccessToken, + [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = "https://maestro-prod.westus2.cloudapp.azure.com", + [Parameter(Mandatory=$false)][string] $MaestroApiVersion = "2019-01-16" ) -$ErrorActionPreference = "Stop" -Set-StrictMode -Version 2.0 - -. $PSScriptRoot\..\tools.ps1 - -function Get-Headers([string]$accept, [string]$barToken) { - $headers = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]' - $headers.Add('Accept',$accept) - $headers.Add('Authorization',"Bearer $barToken") - return $headers -} +. $PSScriptRoot\post-build-utils.ps1 try { - $maestroHeaders = Get-Headers 'application/json' $BarToken + # Check that the channel we are going to promote the build to exist + $channelInfo = Get-MaestroChannel -ChannelId $ChannelId + + if (!$channelInfo) { + Write-Host "Channel with BAR ID $ChannelId was not found in BAR!" + ExitWithExitCode 1 + } # Get info about which channels the build has already been promoted to - $getBuildApiEndpoint = "$MaestroEndpoint/api/builds/${BuildId}?api-version=$ApiVersion" - $buildInfo = Invoke-WebRequest -Method Get -Uri $getBuildApiEndpoint -Headers $maestroHeaders | ConvertFrom-Json - + $buildInfo = Get-MaestroBuild -BuildId $BuildId + if (!$buildInfo) { Write-Host "Build with BAR ID $BuildId was not found in BAR!" ExitWithExitCode 1 @@ -40,10 +35,10 @@ try { } } - Write-Host "Build not present in channel $ChannelId. Promoting build ... " + Write-Host "Promoting build '$BuildId' to channel '$ChannelId'." + + Assign-BuildToChannel -BuildId $BuildId -ChannelId $ChannelId - $promoteBuildApiEndpoint = "$maestroEndpoint/api/channels/${ChannelId}/builds/${BuildId}?api-version=$ApiVersion" - Invoke-WebRequest -Method Post -Uri $promoteBuildApiEndpoint -Headers $maestroHeaders Write-Host "done." } catch { diff --git a/eng/common/post-build/setup-maestro-vars.ps1 b/eng/common/post-build/setup-maestro-vars.ps1 new file mode 100644 index 000000000..d7f64dc63 --- /dev/null +++ b/eng/common/post-build/setup-maestro-vars.ps1 @@ -0,0 +1,26 @@ +param( + [Parameter(Mandatory=$true)][string] $ReleaseConfigsPath # Full path to ReleaseConfigs.txt asset +) + +. $PSScriptRoot\post-build-utils.ps1 + +try { + $Content = Get-Content $ReleaseConfigsPath + + $BarId = $Content | Select -Index 0 + + $Channels = "" + $Content | Select -Index 1 | ForEach-Object { $Channels += "$_ ," } + + $IsStableBuild = $Content | Select -Index 2 + + Write-PipelineSetVariable -Name 'BARBuildId' -Value $BarId + Write-PipelineSetVariable -Name 'InitialChannels' -Value "$Channels" + Write-PipelineSetVariable -Name 'IsStableBuild' -Value $IsStableBuild +} +catch { + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + ExitWithExitCode 1 +} diff --git a/eng/common/post-build/sourcelink-validation.ps1 b/eng/common/post-build/sourcelink-validation.ps1 index 8abd684e9..41e01ae6e 100644 --- a/eng/common/post-build/sourcelink-validation.ps1 +++ b/eng/common/post-build/sourcelink-validation.ps1 @@ -6,10 +6,7 @@ param( [Parameter(Mandatory=$true)][string] $SourcelinkCliVersion # Version of SourceLink CLI to use ) -$ErrorActionPreference = "Stop" -Set-StrictMode -Version 2.0 - -. $PSScriptRoot\..\tools.ps1 +. $PSScriptRoot\post-build-utils.ps1 # Cache/HashMap (File -> Exist flag) used to consult whether a file exist # in the repository at a specific commit point. This is populated by inserting @@ -200,21 +197,27 @@ function ValidateSourceLinkLinks { } } -function CheckExitCode ([string]$stage) { - $exitCode = $LASTEXITCODE - if ($exitCode -ne 0) { - Write-PipelineTaskError "Something failed while '$stage'. Check for errors above. Exiting now..." - ExitWithExitCode $exitCode +function InstallSourcelinkCli { + $sourcelinkCliPackageName = "sourcelink" + + $dotnetRoot = InitializeDotNetCli -install:$true + $dotnet = "$dotnetRoot\dotnet.exe" + $toolList = & "$dotnet" tool list --global + + if (($toolList -like "*$sourcelinkCliPackageName*") -and ($toolList -like "*$sourcelinkCliVersion*")) { + Write-Host "SourceLink CLI version $sourcelinkCliVersion is already installed." + } + else { + Write-Host "Installing SourceLink CLI version $sourcelinkCliVersion..." + Write-Host "You may need to restart your command window if this is the first dotnet tool you have installed." + & "$dotnet" tool install $sourcelinkCliPackageName --version $sourcelinkCliVersion --verbosity "minimal" --global } } try { - Write-Host "Installing SourceLink CLI..." - Get-Location - . $PSScriptRoot\sourcelink-cli-init.ps1 -sourcelinkCliVersion $SourcelinkCliVersion - CheckExitCode "Running sourcelink-cli-init" + InstallSourcelinkCli - Measure-Command { ValidateSourceLinkLinks } + ValidateSourceLinkLinks } catch { Write-Host $_ diff --git a/eng/common/post-build/symbols-validation.ps1 b/eng/common/post-build/symbols-validation.ps1 index 69456854e..d5ec51b15 100644 --- a/eng/common/post-build/symbols-validation.ps1 +++ b/eng/common/post-build/symbols-validation.ps1 @@ -4,10 +4,7 @@ param( [Parameter(Mandatory=$true)][string] $DotnetSymbolVersion # Version of dotnet symbol to use ) -$ErrorActionPreference = "Stop" -Set-StrictMode -Version 2.0 - -. $PSScriptRoot\..\tools.ps1 +. $PSScriptRoot\post-build-utils.ps1 Add-Type -AssemblyName System.IO.Compression.FileSystem @@ -162,19 +159,25 @@ function CheckSymbolsAvailable { } } -function CheckExitCode ([string]$stage) { - $exitCode = $LASTEXITCODE - if ($exitCode -ne 0) { - Write-PipelineTaskError "Something failed while '$stage'. Check for errors above. Exiting now..." - ExitWithExitCode $exitCode +function Installdotnetsymbol { + $dotnetsymbolPackageName = "dotnet-symbol" + + $dotnetRoot = InitializeDotNetCli -install:$true + $dotnet = "$dotnetRoot\dotnet.exe" + $toolList = & "$dotnet" tool list --global + + if (($toolList -like "*$dotnetsymbolPackageName*") -and ($toolList -like "*$dotnetsymbolVersion*")) { + Write-Host "dotnet-symbol version $dotnetsymbolVersion is already installed." + } + else { + Write-Host "Installing dotnet-symbol version $dotnetsymbolVersion..." + Write-Host "You may need to restart your command window if this is the first dotnet tool you have installed." + & "$dotnet" tool install $dotnetsymbolPackageName --version $dotnetsymbolVersion --verbosity "minimal" --global } } try { - Write-Host "Installing dotnet symbol ..." - Get-Location - . $PSScriptRoot\dotnetsymbol-init.ps1 -dotnetsymbolVersion $DotnetSymbolVersion - CheckExitCode "Running dotnetsymbol-init" + Installdotnetsymbol CheckSymbolsAvailable } diff --git a/eng/common/post-build/trigger-subscriptions.ps1 b/eng/common/post-build/trigger-subscriptions.ps1 index db8a83945..926d5b455 100644 --- a/eng/common/post-build/trigger-subscriptions.ps1 +++ b/eng/common/post-build/trigger-subscriptions.ps1 @@ -1,33 +1,20 @@ -param( +param( [Parameter(Mandatory=$true)][string] $SourceRepo, [Parameter(Mandatory=$true)][int] $ChannelId, - [string] $MaestroEndpoint = "https://maestro-prod.westus2.cloudapp.azure.com", - [string] $BarToken, - [string] $ApiVersion = "2019-01-16" + [Parameter(Mandatory=$true)][string] $MaestroApiAccessToken, + [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = "https://maestro-prod.westus2.cloudapp.azure.com", + [Parameter(Mandatory=$false)][string] $MaestroApiVersion = "2019-01-16" ) -$ErrorActionPreference = "Stop" -Set-StrictMode -Version 2.0 - -. $PSScriptRoot\..\tools.ps1 - -function Get-Headers([string]$accept, [string]$barToken) { - $headers = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]' - $headers.Add('Accept',$accept) - $headers.Add('Authorization',"Bearer $barToken") - return $headers -} +. $PSScriptRoot\post-build-utils.ps1 # Get all the $SourceRepo subscriptions -$normalizedSurceRepo = $SourceRepo.Replace('dnceng@', '') -$getSubscriptionsApiEndpoint = "$maestroEndpoint/api/subscriptions?sourceRepository=$normalizedSurceRepo&api-version=$apiVersion" -$headers = Get-Headers 'application/json' $barToken - -$subscriptions = Invoke-WebRequest -Uri $getSubscriptionsApiEndpoint -Headers $headers | ConvertFrom-Json +$normalizedSourceRepo = $SourceRepo.Replace('dnceng@', '') +$subscriptions = Get-MaestroSubscriptions -SourceRepository $normalizedSourceRepo -ChannelId $ChannelId if (!$subscriptions) { - Write-Host "No subscriptions found for source repo '$normalizedSurceRepo' in channel '$ChannelId'" - return + Write-Host "No subscriptions found for source repo '$normalizedSourceRepo' in channel '$ChannelId'" + ExitWithExitCode 0 } $subscriptionsToTrigger = New-Object System.Collections.Generic.List[string] @@ -36,21 +23,18 @@ $failedTriggeredSubscription = $false # Get all enabled subscriptions that need dependency flow on 'everyBuild' foreach ($subscription in $subscriptions) { if ($subscription.enabled -and $subscription.policy.updateFrequency -like 'everyBuild' -and $subscription.channel.id -eq $ChannelId) { - Write-Host "$subscription.id" + Write-Host "Should trigger this subscription: $subscription.id" [void]$subscriptionsToTrigger.Add($subscription.id) } } foreach ($subscriptionToTrigger in $subscriptionsToTrigger) { try { - $triggerSubscriptionApiEndpoint = "$maestroEndpoint/api/subscriptions/$subscriptionToTrigger/trigger?api-version=$apiVersion" - $headers = Get-Headers 'application/json' $BarToken - - Write-Host "Triggering subscription '$subscriptionToTrigger'..." + Write-Host "Triggering subscription '$subscriptionToTrigger'." - Invoke-WebRequest -Uri $triggerSubscriptionApiEndpoint -Headers $headers -Method Post + Trigger-Subscription -SubscriptionId $subscriptionToTrigger - Write-Host "Subscription '$subscriptionToTrigger' triggered!" + Write-Host "done." } catch { @@ -61,9 +45,13 @@ foreach ($subscriptionToTrigger in $subscriptionsToTrigger) { } } -if ($failedTriggeredSubscription) { +if ($subscriptionsToTrigger.Count -eq 0) { + Write-Host "No subscription matched source repo '$normalizedSourceRepo' and channel ID '$ChannelId'." +} +elseif ($failedTriggeredSubscription) { Write-Host "At least one subscription failed to be triggered..." ExitWithExitCode 1 } - -Write-Host "All subscriptions were triggered successfully!" +else { + Write-Host "All subscriptions were triggered successfully!" +} diff --git a/eng/common/sdl/execute-all-sdl-tools.ps1 b/eng/common/sdl/execute-all-sdl-tools.ps1 index 0635f26fb..aab7589f2 100644 --- a/eng/common/sdl/execute-all-sdl-tools.ps1 +++ b/eng/common/sdl/execute-all-sdl-tools.ps1 @@ -22,7 +22,9 @@ Param( [string] $TsaCodebaseAdmin, # Optional: only needed if TsaOnboard is true; the aliases which are admins of the TSA codebase (e.g. DOMAIN\alias); TSA is the automated framework used to upload test results as bugs. [string] $TsaBugAreaPath, # Optional: only needed if TsaOnboard is true; the area path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs. [string] $TsaIterationPath, # Optional: only needed if TsaOnboard is true; the iteration path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs. - [string] $GuardianLoggerLevel="Standard" # Optional: the logger level for the Guardian CLI; options are Trace, Verbose, Standard, Warning, and Error + [string] $GuardianLoggerLevel="Standard", # Optional: the logger level for the Guardian CLI; options are Trace, Verbose, Standard, Warning, and Error + [string[]] $CrScanAdditionalRunConfigParams, # Optional: Additional Params to custom build a CredScan run config in the format @("xyz:abc","sdf:1") + [string[]] $PoliCheckAdditionalRunConfigParams # Optional: Additional Params to custom build a Policheck run config in the format @("xyz:abc","sdf:1") ) $ErrorActionPreference = "Stop" @@ -69,10 +71,10 @@ if ($TsaOnboard) { } if ($ArtifactToolsList -and $ArtifactToolsList.Count -gt 0) { - & $(Join-Path $PSScriptRoot "run-sdl.ps1") -GuardianCliLocation $guardianCliLocation -WorkingDirectory $ArtifactsDirectory -TargetDirectory $ArtifactsDirectory -GdnFolder $gdnFolder -ToolsList $ArtifactToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel + & $(Join-Path $PSScriptRoot "run-sdl.ps1") -GuardianCliLocation $guardianCliLocation -WorkingDirectory $ArtifactsDirectory -TargetDirectory $ArtifactsDirectory -GdnFolder $gdnFolder -ToolsList $ArtifactToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams } if ($SourceToolsList -and $SourceToolsList.Count -gt 0) { - & $(Join-Path $PSScriptRoot "run-sdl.ps1") -GuardianCliLocation $guardianCliLocation -WorkingDirectory $ArtifactsDirectory -TargetDirectory $SourceDirectory -GdnFolder $gdnFolder -ToolsList $SourceToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel + & $(Join-Path $PSScriptRoot "run-sdl.ps1") -GuardianCliLocation $guardianCliLocation -WorkingDirectory $ArtifactsDirectory -TargetDirectory $SourceDirectory -GdnFolder $gdnFolder -ToolsList $SourceToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams } if ($UpdateBaseline) { diff --git a/eng/common/sdl/run-sdl.ps1 b/eng/common/sdl/run-sdl.ps1 index e6a86d03a..d7b856445 100644 --- a/eng/common/sdl/run-sdl.ps1 +++ b/eng/common/sdl/run-sdl.ps1 @@ -5,7 +5,9 @@ Param( [string] $GdnFolder, [string[]] $ToolsList, [string] $UpdateBaseline, - [string] $GuardianLoggerLevel="Standard" + [string] $GuardianLoggerLevel="Standard", + [string[]] $CrScanAdditionalRunConfigParams, + [string[]] $PoliCheckAdditionalRunConfigParams ) $ErrorActionPreference = "Stop" @@ -29,8 +31,8 @@ foreach ($tool in $ToolsList) { Write-Host $tool # We have to manually configure tools that run on source to look at the source directory only if ($tool -eq "credscan") { - Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" TargetDirectory : $TargetDirectory `"" - & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " TargetDirectory : $TargetDirectory " + Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" TargetDirectory : $TargetDirectory `" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams})" + & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " TargetDirectory : $TargetDirectory " $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams}) if ($LASTEXITCODE -ne 0) { Write-Host "Guardian configure for $tool failed with exit code $LASTEXITCODE." exit $LASTEXITCODE @@ -38,8 +40,8 @@ foreach ($tool in $ToolsList) { $config = $True } if ($tool -eq "policheck") { - Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" Target : $TargetDirectory `"" - & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " Target : $TargetDirectory " + Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" Target : $TargetDirectory `" $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams})" + & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " Target : $TargetDirectory " $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams}) if ($LASTEXITCODE -ne 0) { Write-Host "Guardian configure for $tool failed with exit code $LASTEXITCODE." exit $LASTEXITCODE diff --git a/eng/common/templates/job/performance.yml b/eng/common/templates/job/performance.yml new file mode 100644 index 000000000..ef809253d --- /dev/null +++ b/eng/common/templates/job/performance.yml @@ -0,0 +1,93 @@ +parameters: + steps: [] # optional -- any additional steps that need to happen before pulling down the performance repo and sending the performance benchmarks to helix (ie building your repo) + variables: [] # optional -- list of additional variables to send to the template + jobName: '' # required -- job name + displayName: '' # optional -- display name for the job. Will use jobName if not passed + pool: '' # required -- name of the Build pool + container: '' # required -- name of the container + extraSetupParameters: '' # optional -- extra arguments to pass to the setup script + frameworks: ['netcoreapp3.0'] # optional -- list of frameworks to run against + continueOnError: 'false' # optional -- determines whether to continue the build if the step errors + dependsOn: '' # optional -- dependencies of the job + timeoutInMinutes: 320 # optional -- timeout for the job + enableTelemetry: false # optional -- enable for telemetry + +jobs: +- template: ../jobs/jobs.yml + parameters: + dependsOn: ${{ parameters.dependsOn }} + enableTelemetry: ${{ parameters.enableTelemetry }} + enablePublishBuildArtifacts: true + continueOnError: ${{ parameters.continueOnError }} + + jobs: + - job: '${{ parameters.jobName }}' + + ${{ if ne(parameters.displayName, '') }}: + displayName: '${{ parameters.displayName }}' + ${{ if eq(parameters.displayName, '') }}: + displayName: '${{ parameters.jobName }}' + + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + + variables: + + - ${{ each variable in parameters.variables }}: + - ${{ if ne(variable.name, '') }}: + - name: ${{ variable.name }} + value: ${{ variable.value }} + - ${{ if ne(variable.group, '') }}: + - group: ${{ variable.group }} + + - IsInternal: '' + - HelixApiAccessToken: '' + - HelixPreCommand: '' + + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - ${{ if eq(variables['Agent.Os'], 'Windows_NT') }}: + - HelixPreCommand: 'set "PERFLAB_UPLOAD_TOKEN=$(PerfCommandUploadToken)"' + - IsInternal: -Internal + - ${{ if ne(variables['Agent.Os'], 'Windows_NT') }}: + - HelixPreCommand: 'export PERFLAB_UPLOAD_TOKEN="$(PerfCommandUploadTokenLinux)"' + - IsInternal: --internal + - group: DotNet-HelixApi-Access + - group: dotnet-benchview + + workspace: + clean: all + pool: + ${{ parameters.pool }} + container: ${{ parameters.container }} + strategy: + matrix: + ${{ each framework in parameters.frameworks }}: + ${{ framework }}: + _Framework: ${{ framework }} + steps: + - checkout: self + clean: true + # Run all of the steps to setup repo + - ${{ each step in parameters.steps }}: + - ${{ step }} + - powershell: $(Build.SourcesDirectory)\eng\common\performance\performance-setup.ps1 $(IsInternal) -Framework $(_Framework) ${{ parameters.extraSetupParameters }} + displayName: Performance Setup (Windows) + condition: and(succeeded(), eq(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} + - script: $(Build.SourcesDirectory)/eng/common/performance/performance-setup.sh $(IsInternal) --framework $(_Framework) ${{ parameters.extraSetupParameters }} + displayName: Performance Setup (Unix) + condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} + - script: $(Python) $(PerformanceDirectory)/scripts/ci_setup.py $(SetupArguments) + displayName: Run ci setup script + # Run perf testing in helix + - template: /eng/common/templates/steps/perf-send-to-helix.yml + parameters: + HelixSource: '$(HelixSourcePrefix)/$(Build.Repository.Name)/$(Build.SourceBranch)' # sources must start with pr/, official/, prodcon/, or agent/ + HelixType: 'test/performance/$(Kind)/$(_Framework)/$(Architecture)' + HelixAccessToken: $(HelixApiAccessToken) + HelixTargetQueues: $(Queue) + HelixPreCommands: $(HelixPreCommand) + Creator: $(Creator) + WorkItemTimeout: 4:00 # 4 hours + WorkItemDirectory: '$(WorkItemDirectory)' # WorkItemDirectory can not be empty, so we send it some docs to keep it happy + CorrelationPayloadDirectory: '$(PayloadDirectory)' # it gets checked out to a folder with shorter path than WorkItemDirectory so we can avoid file name too long exceptions \ No newline at end of file diff --git a/eng/common/templates/post-build/channels/internal-servicing.yml b/eng/common/templates/post-build/channels/internal-servicing.yml index 808d46b17..12fd2b465 100644 --- a/eng/common/templates/post-build/channels/internal-servicing.yml +++ b/eng/common/templates/post-build/channels/internal-servicing.yml @@ -41,7 +41,6 @@ stages: dependsOn: setupMaestroVars variables: - group: DotNet-Blob-Feed - - group: Publish-Build-Assets - group: AzureDevOps-Artifact-Feeds-Pats - name: BARBuildId value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] @@ -81,13 +80,14 @@ stages: /p:IsInternalBuild=$(IsInternalBuild) /p:RepositoryName=$(Build.Repository.Name) /p:CommitSha=$(Build.SourceVersion) + /p:AzureStorageAccountName=$(ProxyBackedFeedsAccountName) + /p:AzureStorageAccountKey=$(dotnetfeed-storage-access-key-1) + /p:AzureDevOpsFeedsBaseUrl=$(dotnetfeed-internal-private-feed-url) + /p:StaticInternalFeed=$(dotnetfeed-internal-nonstable-feed-url) /p:NugetPath=$(Agent.BuildDirectory)\Nuget\NuGet.exe - /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' - /p:TargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' - /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' /p:BARBuildId=$(BARBuildId) - /p:MaestroApiEndpoint='https://maestro-prod.westus2.cloudapp.azure.com' - /p:BuildAssetRegistryToken='$(MaestroAccessToken)' + /p:MaestroApiEndpoint='$(MaestroApiEndPoint)' + /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)' /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' /p:BlobBasePath='$(Build.ArtifactStagingDirectory)\BlobArtifacts' /p:PackageBasePath='$(Build.ArtifactStagingDirectory)\PackageArtifacts' @@ -142,29 +142,6 @@ stages: filePath: $(Build.SourcesDirectory)/eng/common/post-build/symbols-validation.ps1 arguments: -InputPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ -ExtractPath $(Agent.BuildDirectory)/Temp/ -DotnetSymbolVersion $(SymbolToolVersion) - - job: - displayName: Gather Drop - dependsOn: setupMaestroVars - variables: - BARBuildId: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] - condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.InternalServicing_30_Channel_Id) - pool: - vmImage: 'windows-2019' - steps: - - task: PowerShell@2 - displayName: Setup Darc CLI - inputs: - targetType: filePath - filePath: '$(Build.SourcesDirectory)/eng/common/darc-init.ps1' - - - task: PowerShell@2 - displayName: Run Darc gather-drop - inputs: - targetType: inline - script: | - darc gather-drop --non-shipping --continue-on-error --id $(BARBuildId) --output-dir $(Agent.BuildDirectory)/Temp/Drop/ --bar-uri https://maestro-prod.westus2.cloudapp.azure.com/ --password $(MaestroAccessToken) --latest-location - enabled: false - - template: ../promote-build.yml parameters: - ChannelId: ${{ variables.InternalServicing_30_Channel_Id }} \ No newline at end of file + ChannelId: ${{ variables.InternalServicing_30_Channel_Id }} diff --git a/eng/common/templates/post-build/channels/public-dev-release.yml b/eng/common/templates/post-build/channels/public-dev-release.yml index 79c6822db..b0f085b14 100644 --- a/eng/common/templates/post-build/channels/public-dev-release.yml +++ b/eng/common/templates/post-build/channels/public-dev-release.yml @@ -41,7 +41,6 @@ stages: dependsOn: setupMaestroVars variables: - group: DotNet-Blob-Feed - - group: Publish-Build-Assets - group: AzureDevOps-Artifact-Feeds-Pats - name: BARBuildId value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] @@ -77,6 +76,7 @@ stages: filePath: eng\common\sdk-task.ps1 arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet /p:ChannelId=$(PublicDevRelease_30_Channel_Id) + /p:ArtifactsCategory=$(_DotNetArtifactsCategory) /p:IsStableBuild=$(IsStableBuild) /p:IsInternalBuild=$(IsInternalBuild) /p:RepositoryName=$(Build.Repository.Name) @@ -86,8 +86,8 @@ stages: /p:TargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' /p:BARBuildId=$(BARBuildId) - /p:MaestroApiEndpoint='https://maestro-prod.westus2.cloudapp.azure.com' - /p:BuildAssetRegistryToken='$(MaestroAccessToken)' + /p:MaestroApiEndpoint='$(MaestroApiEndPoint)' + /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)' /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' @@ -139,27 +139,9 @@ stages: filePath: $(Build.SourcesDirectory)/eng/common/post-build/symbols-validation.ps1 arguments: -InputPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ -ExtractPath $(Agent.BuildDirectory)/Temp/ -DotnetSymbolVersion $(SymbolToolVersion) - - job: - displayName: Gather Drop - dependsOn: setupMaestroVars - variables: - BARBuildId: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] - condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicDevRelease_30_Channel_Id) - pool: - vmImage: 'windows-2019' - steps: - - task: PowerShell@2 - displayName: Setup Darc CLI - inputs: - targetType: filePath - filePath: '$(Build.SourcesDirectory)/eng/common/darc-init.ps1' - - - task: PowerShell@2 - displayName: Run Darc gather-drop - inputs: - targetType: inline - script: | - darc gather-drop --non-shipping --continue-on-error --id $(BARBuildId) --output-dir $(Agent.BuildDirectory)/Temp/Drop/ --bar-uri https://maestro-prod.westus2.cloudapp.azure.com/ --password $(MaestroAccessToken) --latest-location + - template: ../darc-gather-drop.yml + parameters: + ChannelId: ${{ variables.PublicDevRelease_30_Channel_Id }} - template: ../promote-build.yml parameters: diff --git a/eng/common/templates/post-build/channels/public-release.yml b/eng/common/templates/post-build/channels/public-release.yml index 25923020d..4c63fb43f 100644 --- a/eng/common/templates/post-build/channels/public-release.yml +++ b/eng/common/templates/post-build/channels/public-release.yml @@ -41,7 +41,6 @@ stages: dependsOn: setupMaestroVars variables: - group: DotNet-Blob-Feed - - group: Publish-Build-Assets - group: AzureDevOps-Artifact-Feeds-Pats - name: BARBuildId value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] @@ -81,13 +80,14 @@ stages: /p:IsInternalBuild=$(IsInternalBuild) /p:RepositoryName=$(Build.Repository.Name) /p:CommitSha=$(Build.SourceVersion) - /p:NugetPath=$(Agent.BuildDirectory)/Nuget/NuGet.exe - /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' - /p:TargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' - /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' + /p:AzureStorageAccountName=$(ProxyBackedFeedsAccountName) + /p:AzureStorageAccountKey=$(dotnetfeed-storage-access-key-1) + /p:AzureDevOpsFeedsBaseUrl=$(dotnetfeed-internal-private-feed-url) + /p:StaticInternalFeed=$(dotnetfeed-internal-nonstable-feed-url) + /p:NugetPath=$(Agent.BuildDirectory)\Nuget\NuGet.exe /p:BARBuildId=$(BARBuildId) - /p:MaestroApiEndpoint='https://maestro-prod.westus2.cloudapp.azure.com' - /p:BuildAssetRegistryToken='$(MaestroAccessToken)' + /p:MaestroApiEndpoint='$(MaestroApiEndPoint)' + /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)' /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' /p:BlobBasePath='$(Build.ArtifactStagingDirectory)\BlobArtifacts' /p:PackageBasePath='$(Build.ArtifactStagingDirectory)\PackageArtifacts' @@ -142,29 +142,6 @@ stages: filePath: $(Build.SourcesDirectory)/eng/common/post-build/symbols-validation.ps1 arguments: -InputPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ -ExtractPath $(Agent.BuildDirectory)/Temp/ -DotnetSymbolVersion $(SymbolToolVersion) - - job: - displayName: Gather Drop - dependsOn: setupMaestroVars - variables: - BARBuildId: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] - condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicRelease_30_Channel_Id) - pool: - vmImage: 'windows-2019' - steps: - - task: PowerShell@2 - displayName: Setup Darc CLI - inputs: - targetType: filePath - filePath: '$(Build.SourcesDirectory)/eng/common/darc-init.ps1' - - - task: PowerShell@2 - displayName: Run Darc gather-drop - inputs: - targetType: inline - script: | - darc gather-drop --non-shipping --continue-on-error --id $(BARBuildId) --output-dir $(Agent.BuildDirectory)/Temp/Drop/ --bar-uri https://maestro-prod.westus2.cloudapp.azure.com/ --password $(MaestroAccessToken) --latest-location - enabled: false - - template: ../promote-build.yml parameters: ChannelId: ${{ variables.PublicRelease_30_Channel_Id }} diff --git a/eng/common/templates/post-build/channels/public-validation-release.yml b/eng/common/templates/post-build/channels/public-validation-release.yml index 114477d3a..1089ac5fa 100644 --- a/eng/common/templates/post-build/channels/public-validation-release.yml +++ b/eng/common/templates/post-build/channels/public-validation-release.yml @@ -12,7 +12,6 @@ stages: dependsOn: setupMaestroVars variables: - group: DotNet-Blob-Feed - - group: Publish-Build-Assets - group: AzureDevOps-Artifact-Feeds-Pats - name: BARBuildId value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] @@ -48,6 +47,7 @@ stages: filePath: eng\common\sdk-task.ps1 arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet /p:ChannelId=$(PublicValidationRelease_30_Channel_Id) + /p:ArtifactsCategory=$(_DotNetValidationArtifactsCategory) /p:IsStableBuild=$(IsStableBuild) /p:IsInternalBuild=$(IsInternalBuild) /p:RepositoryName=$(Build.Repository.Name) @@ -57,13 +57,13 @@ stages: /p:TargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)' /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' /p:BARBuildId=$(BARBuildId) - /p:MaestroApiEndpoint='https://maestro-prod.westus2.cloudapp.azure.com' - /p:BuildAssetRegistryToken='$(MaestroAccessToken)' + /p:MaestroApiEndpoint='$(MaestroApiEndPoint)' + /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)' /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' /p:BlobBasePath='$(Build.ArtifactStagingDirectory)\BlobArtifacts' /p:PackageBasePath='$(Build.ArtifactStagingDirectory)\PackageArtifacts' /p:Configuration=Release - + - task: NuGetCommand@2 displayName: Publish Packages to AzDO Feed condition: contains(variables['TargetAzDOFeed'], 'pkgs.visualstudio.com') @@ -90,29 +90,9 @@ stages: jobs: - template: ../setup-maestro-vars.yml - - job: - displayName: Gather Drop - dependsOn: setupMaestroVars - condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicValidationRelease_30_Channel_Id) - variables: - - name: BARBuildId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] - - group: Publish-Build-Assets - pool: - vmImage: 'windows-2019' - steps: - - task: PowerShell@2 - displayName: Setup Darc CLI - inputs: - targetType: filePath - filePath: '$(Build.SourcesDirectory)/eng/common/darc-init.ps1' - - - task: PowerShell@2 - displayName: Run Darc gather-drop - inputs: - targetType: inline - script: | - darc gather-drop --non-shipping --continue-on-error --id $(BARBuildId) --output-dir $(Agent.BuildDirectory)/Temp/Drop/ --bar-uri https://maestro-prod.westus2.cloudapp.azure.com --password $(MaestroAccessToken) --latest-location + - template: ../darc-gather-drop.yml + parameters: + ChannelId: ${{ variables.PublicValidationRelease_30_Channel_Id }} - template: ../promote-build.yml parameters: diff --git a/eng/common/templates/post-build/common-variables.yml b/eng/common/templates/post-build/common-variables.yml index 828346735..bd0bc5e4d 100644 --- a/eng/common/templates/post-build/common-variables.yml +++ b/eng/common/templates/post-build/common-variables.yml @@ -1,18 +1,39 @@ variables: + - group: Publish-Build-Assets + # .NET Core 3 Dev - PublicDevRelease_30_Channel_Id: 3 + - name: PublicDevRelease_30_Channel_Id + value: 3 # .NET Tools - Validation - PublicValidationRelease_30_Channel_Id: 9 + - name: PublicValidationRelease_30_Channel_Id + value: 9 # .NET Core 3.0 Internal Servicing - InternalServicing_30_Channel_Id: 184 + - name: InternalServicing_30_Channel_Id + value: 184 # .NET Core 3.0 Release - PublicRelease_30_Channel_Id: 19 + - name: PublicRelease_30_Channel_Id + value: 19 # Whether the build is internal or not - IsInternalBuild: ${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }} + - name: IsInternalBuild + value: ${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }} - SourceLinkCLIVersion: 3.0.0 - SymbolToolVersion: 1.0.1 + # Storage account name for proxy-backed feeds + - name: ProxyBackedFeedsAccountName + value: dotnetfeed + + # Default Maestro++ API Endpoint and API Version + - name: MaestroApiEndPoint + value: "https://maestro-prod.westus2.cloudapp.azure.com" + - name: MaestroApiAccessToken + value: $(MaestroAccessToken) + - name: MaestroApiVersion + value: "2019-01-16" + + - name: SourceLinkCLIVersion + value: 3.0.0 + - name: SymbolToolVersion + value: 1.0.1 diff --git a/eng/common/templates/post-build/darc-gather-drop.yml b/eng/common/templates/post-build/darc-gather-drop.yml new file mode 100644 index 000000000..f4e3bfcf5 --- /dev/null +++ b/eng/common/templates/post-build/darc-gather-drop.yml @@ -0,0 +1,23 @@ +parameters: + ChannelId: 0 + +jobs: +- job: gatherDrop + displayName: Gather Drop + dependsOn: setupMaestroVars + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], ${{ parameters.ChannelId }}) + variables: + - name: BARBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] + pool: + vmImage: 'windows-2019' + steps: + - task: PowerShell@2 + displayName: Darc gather-drop + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/darc-gather-drop.ps1 + arguments: -BarBuildId $(BARBuildId) + -DropLocation $(Agent.BuildDirectory)/Temp/Drop/ + -MaestroApiAccessToken $(MaestroApiAccessToken) + -MaestroApiEndPoint $(MaestroApiEndPoint) + -MaestroApiVersion $(MaestroApiVersion) diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index daa799259..0872db4ed 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -7,9 +7,12 @@ parameters: enable: false params: '' + # Which stages should finish execution before post-build stages start + dependsOn: [build] + stages: - stage: validate - dependsOn: build + dependsOn: ${{ parameters.dependsOn }} displayName: Validate jobs: - ${{ if eq(parameters.enableNugetValidation, 'true') }}: diff --git a/eng/common/templates/post-build/promote-build.yml b/eng/common/templates/post-build/promote-build.yml index af48b0b33..9387c583b 100644 --- a/eng/common/templates/post-build/promote-build.yml +++ b/eng/common/templates/post-build/promote-build.yml @@ -11,7 +11,6 @@ jobs: value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] - name: ChannelId value: ${{ parameters.ChannelId }} - - group: Publish-Build-Assets pool: vmImage: 'windows-2019' steps: @@ -21,4 +20,6 @@ jobs: filePath: $(Build.SourcesDirectory)/eng/common/post-build/promote-build.ps1 arguments: -BuildId $(BARBuildId) -ChannelId $(ChannelId) - -BarToken $(MaestroAccessToken) + -MaestroApiAccessToken $(MaestroApiAccessToken) + -MaestroApiEndPoint $(MaestroApiEndPoint) + -MaestroApiVersion $(MaestroApiVersion) diff --git a/eng/common/templates/post-build/setup-maestro-vars.yml b/eng/common/templates/post-build/setup-maestro-vars.yml index f6120dc1e..56242b068 100644 --- a/eng/common/templates/post-build/setup-maestro-vars.yml +++ b/eng/common/templates/post-build/setup-maestro-vars.yml @@ -14,22 +14,5 @@ jobs: name: setReleaseVars displayName: Set Release Configs Vars inputs: - targetType: inline - script: | - # This is needed to make Write-PipelineSetVariable works in this context - $ci = $true - - . "$(Build.SourcesDirectory)/eng/common/tools.ps1" - - $Content = Get-Content "$(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt" - - $BarId = $Content | Select -Index 0 - - $Channels = "" - $Content | Select -Index 1 | ForEach-Object { $Channels += "$_ ," } - - $IsStableBuild = $Content | Select -Index 2 - - Write-PipelineSetVariable -Name 'BARBuildId' -Value $BarId - Write-PipelineSetVariable -Name 'InitialChannels' -Value "$Channels" - Write-PipelineSetVariable -Name 'IsStableBuild' -Value $IsStableBuild + filePath: $(Build.SourcesDirectory)/eng/common/post-build/setup-maestro-vars.ps1 + arguments: -ReleaseConfigsPath '$(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt' diff --git a/eng/common/templates/post-build/trigger-subscription.yml b/eng/common/templates/post-build/trigger-subscription.yml index 65259d4e6..da669030d 100644 --- a/eng/common/templates/post-build/trigger-subscription.yml +++ b/eng/common/templates/post-build/trigger-subscription.yml @@ -8,4 +8,6 @@ steps: filePath: $(Build.SourcesDirectory)/eng/common/post-build/trigger-subscriptions.ps1 arguments: -SourceRepo $(Build.Repository.Uri) -ChannelId ${{ parameters.ChannelId }} - -BarToken $(MaestroAccessTokenInt) \ No newline at end of file + -MaestroApiAccessToken $(MaestroAccessToken) + -MaestroApiEndPoint $(MaestroApiEndPoint) + -MaestroApiVersion $(MaestroApiVersion) diff --git a/eng/common/templates/steps/perf-send-to-helix.yml b/eng/common/templates/steps/perf-send-to-helix.yml new file mode 100644 index 000000000..b3ea9acf1 --- /dev/null +++ b/eng/common/templates/steps/perf-send-to-helix.yml @@ -0,0 +1,66 @@ +# Please remember to update the documentation if you make changes to these parameters! +parameters: + HelixSource: 'pr/default' # required -- sources must start with pr/, official/, prodcon/, or agent/ + HelixType: 'tests/default/' # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/' + HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number + HelixTargetQueues: '' # required -- semicolon delimited list of Helix queues to test on; see https://helix.dot.net/ for a list of queues + HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group + HelixPreCommands: '' # optional -- commands to run before Helix work item execution + HelixPostCommands: '' # optional -- commands to run after Helix work item execution + WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects + CorrelationPayloadDirectory: '' # optional -- a directory to zip up and send to Helix as a correlation payload + IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion + DotNetCliPackageType: '' # optional -- either 'sdk' or 'runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json + DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json + EnableXUnitReporter: false # optional -- true enables XUnit result reporting to Mission Control + WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget." + Creator: '' # optional -- if the build is external, use this to specify who is sending the job + DisplayNamePrefix: 'Send job to Helix' # optional -- rename the beginning of the displayName of the steps in AzDO + condition: succeeded() # optional -- condition for step to execute; defaults to succeeded() + continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false + +steps: + - powershell: $(Build.SourcesDirectory)\eng\common\msbuild.ps1 $(Build.SourcesDirectory)\eng\common\performance\perfhelixpublish.proj /restore /t:Test /bl:$(Build.SourcesDirectory)\artifacts\log\$env:BuildConfig\SendToHelix.binlog + displayName: ${{ parameters.DisplayNamePrefix }} (Windows) + env: + BuildConfig: $(_BuildConfig) + HelixSource: ${{ parameters.HelixSource }} + HelixType: ${{ parameters.HelixType }} + HelixBuild: ${{ parameters.HelixBuild }} + HelixTargetQueues: ${{ parameters.HelixTargetQueues }} + HelixAccessToken: ${{ parameters.HelixAccessToken }} + HelixPreCommands: ${{ parameters.HelixPreCommands }} + HelixPostCommands: ${{ parameters.HelixPostCommands }} + WorkItemDirectory: ${{ parameters.WorkItemDirectory }} + CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} + IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} + DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} + DotNetCliVersion: ${{ parameters.DotNetCliVersion }} + EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }} + WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} + Creator: ${{ parameters.Creator }} + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} + - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/eng/common/performance/perfhelixpublish.proj /restore /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog + displayName: ${{ parameters.DisplayNamePrefix }} (Unix) + env: + BuildConfig: $(_BuildConfig) + HelixSource: ${{ parameters.HelixSource }} + HelixType: ${{ parameters.HelixType }} + HelixBuild: ${{ parameters.HelixBuild }} + HelixTargetQueues: ${{ parameters.HelixTargetQueues }} + HelixAccessToken: ${{ parameters.HelixAccessToken }} + HelixPreCommands: ${{ parameters.HelixPreCommands }} + HelixPostCommands: ${{ parameters.HelixPostCommands }} + WorkItemDirectory: ${{ parameters.WorkItemDirectory }} + CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} + IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} + DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} + DotNetCliVersion: ${{ parameters.DotNetCliVersion }} + EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }} + WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} + Creator: ${{ parameters.Creator }} + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 60741f039..8fe2b11ad 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -84,7 +84,7 @@ function Exec-Process([string]$command, [string]$commandArgs) { return $global:LASTEXITCODE = $process.ExitCode } finally { - # If we didn't finish then an error occured or the user hit ctrl-c. Either + # If we didn't finish then an error occurred or the user hit ctrl-c. Either # way kill the process if (-not $finished) { $process.Kill() @@ -147,7 +147,7 @@ function InitializeDotNetCli([bool]$install) { # It also ensures that VS msbuild will use the downloaded sdk targets. $env:PATH = "$dotnetRoot;$env:PATH" - # Make Sure that our bootstrapped dotnet cli is avaliable in future steps of the Azure Pipelines build + # Make Sure that our bootstrapped dotnet cli is available in future steps of the Azure Pipelines build Write-PipelinePrependPath -Path $dotnetRoot Write-PipelineSetVariable -Name 'DOTNET_MULTILEVEL_LOOKUP' -Value '0' Write-PipelineSetVariable -Name 'DOTNET_SKIP_FIRST_TIME_EXPERIENCE' -Value '1' @@ -169,7 +169,7 @@ function InstallDotNetSdk([string] $dotnetRoot, [string] $version, [string] $arc InstallDotNet $dotnetRoot $version $architecture } -function InstallDotNet([string] $dotnetRoot, [string] $version, [string] $architecture = "", [string] $runtime = "", [bool] $skipNonVersionedFiles = $false) { $installScript = GetDotNetInstallScript $dotnetRoot +function InstallDotNet([string] $dotnetRoot, [string] $version, [string] $architecture = "", [string] $runtime = "", [bool] $skipNonVersionedFiles = $false) { $installScript = GetDotNetInstallScript $dotnetRoot $installParameters = @{ Version = $version diff --git a/eng/common/tools.sh b/eng/common/tools.sh old mode 100644 new mode 100755 index 70d92cf85..738bb5669 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -77,7 +77,7 @@ function ReadGlobalVersion { local pattern="\"$key\" *: *\"(.*)\"" if [[ ! $line =~ $pattern ]]; then - Write-PipelineTelemetryError -category 'InitializeTools' "Error: Cannot find \"$key\" in $global_json_file" + Write-PipelineTelemetryError -category 'InitializeToolset' "Error: Cannot find \"$key\" in $global_json_file" ExitWithExitCode 1 fi @@ -146,14 +146,10 @@ function InitializeDotNetCli { # Add dotnet to PATH. This prevents any bare invocation of dotnet in custom # build steps from using anything other than what we've downloaded. - export PATH="$dotnet_root:$PATH" + Write-PipelinePrependPath -path "$dotnet_root" - if [[ $ci == true ]]; then - # Make Sure that our bootstrapped dotnet cli is available in future steps of the Azure Pipelines build - echo "##vso[task.prependpath]$dotnet_root" - echo "##vso[task.setvariable variable=DOTNET_MULTILEVEL_LOOKUP]0" - echo "##vso[task.setvariable variable=DOTNET_SKIP_FIRST_TIME_EXPERIENCE]1" - fi + Write-PipelineSetVariable -name "DOTNET_MULTILEVEL_LOOKUP" -value "0" + Write-PipelineSetVariable -name "DOTNET_SKIP_FIRST_TIME_EXPERIENCE" -value "1" # return value _InitializeDotNetCli="$dotnet_root" @@ -249,7 +245,7 @@ function InitializeNativeTools() { then local nativeArgs="" if [[ "$ci" == true ]]; then - nativeArgs="-InstallDirectory $tools_dir" + nativeArgs="--installDirectory $tools_dir" fi "$_script_dir/init-tools-native.sh" $nativeArgs fi @@ -387,7 +383,8 @@ mkdir -p "$toolset_dir" mkdir -p "$temp_dir" mkdir -p "$log_dir" -if [[ $ci == true ]]; then - export TEMP="$temp_dir" - export TMP="$temp_dir" -fi +Write-PipelineSetVariable -name "Artifacts" -value "$artifacts_dir" +Write-PipelineSetVariable -name "Artifacts.Toolset" -value "$toolset_dir" +Write-PipelineSetVariable -name "Artifacts.Log" -value "$log_dir" +Write-PipelineSetVariable -name "Temp" -value "$temp_dir" +Write-PipelineSetVariable -name "TMP" -value "$temp_dir" diff --git a/global.json b/global.json index f66327607..6c742c4eb 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "3.0.100-preview6-012264" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19330.1" + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19377.2" } } diff --git a/src/core-sdk-tasks/GenerateMSBuildExtensionsSWR.cs b/src/core-sdk-tasks/GenerateMSBuildExtensionsSWR.cs new file mode 100644 index 000000000..03e407b2e --- /dev/null +++ b/src/core-sdk-tasks/GenerateMSBuildExtensionsSWR.cs @@ -0,0 +1,82 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using System; +using System.IO; +using System.Linq; +using System.Text; + +namespace Microsoft.DotNet.Cli.Build +{ + public class GenerateMSBuildExtensionsSWR : Task + { + [Required] + public string MSBuildExtensionsLayoutDirectory { get; set; } + + [Required] + public string OutputFile { get; set; } + + public override bool Execute() + { + StringBuilder sb = new StringBuilder(SWR_HEADER); + + AddFolder(sb, + @"MSBuildSdkResolver", + @"MSBuild\Current\Bin\SdkResolvers\Microsoft.DotNet.MSBuildSdkResolver"); + + AddFolder(sb, + @"msbuildExtensions", + @"MSBuild"); + + AddFolder(sb, + @"msbuildExtensions-ver", + @"MSBuild\Current"); + + File.WriteAllText(OutputFile, sb.ToString()); + + return true; + } + + private void AddFolder(StringBuilder sb, string relativeSourcePath, string swrInstallDir) + { + string sourceFolder = Path.Combine(MSBuildExtensionsLayoutDirectory, relativeSourcePath); + var files = Directory.GetFiles(sourceFolder) + .Where(f => !Path.GetExtension(f).Equals(".pdb", StringComparison.OrdinalIgnoreCase)) + .ToList(); + if (files.Any(f => !Path.GetFileName(f).Equals("_._"))) + { + sb.Append(@"folder ""InstallDir:\"); + sb.Append(swrInstallDir); + sb.AppendLine(@"\"""); + + foreach (var file in files) + { + sb.Append(@" file source=""!(bindpath.sources)\Redist\Common\NetCoreSDK\MSBuildExtensions\"); + sb.Append(Path.Combine(relativeSourcePath, Path.GetFileName(file))); + sb.AppendLine("\""); + } + + sb.AppendLine(); + } + + foreach (var subfolder in Directory.GetDirectories(sourceFolder)) + { + string subfolderName = Path.GetFileName(subfolder); + string newRelativeSourcePath = Path.Combine(relativeSourcePath, subfolderName); + string newSwrInstallDir = Path.Combine(swrInstallDir, subfolderName); + + AddFolder(sb, newRelativeSourcePath, newSwrInstallDir); + } + } + + readonly string SWR_HEADER = @"use vs + +package name=Microsoft.Net.Core.SDK.MSBuildExtensions + version=$(ProductsBuildVersion) + vs.package.internalRevision=$(PackageInternalRevision) + +"; + } +} diff --git a/src/redist/redist.csproj b/src/redist/redist.csproj index e48bbd806..e9f84e078 100644 --- a/src/redist/redist.csproj +++ b/src/redist/redist.csproj @@ -30,6 +30,7 @@ + diff --git a/src/redist/targets/Branding.targets b/src/redist/targets/Branding.targets index 9f81239cd..e70d058c2 100644 --- a/src/redist/targets/Branding.targets +++ b/src/redist/targets/Branding.targets @@ -12,6 +12,9 @@ Microsoft .NET Core Host FX Resolver $(MicrosoftNETCoreAppPackageVersion) Microsoft.NETCore.App $(SharedFrameworkName) + Microsoft .NET Core 3.0 Templates + Microsoft .NET Core 2.2 Templates + Microsoft .NET Core 2.1 Templates diff --git a/src/redist/targets/BuildCoreSdkTasks.targets b/src/redist/targets/BuildCoreSdkTasks.targets index ce846644b..fb6aa70bb 100644 --- a/src/redist/targets/BuildCoreSdkTasks.targets +++ b/src/redist/targets/BuildCoreSdkTasks.targets @@ -37,5 +37,6 @@ + diff --git a/src/redist/targets/BundledTemplates.targets b/src/redist/targets/BundledTemplates.targets index 28e7f1f75..caf7b6a62 100644 --- a/src/redist/targets/BundledTemplates.targets +++ b/src/redist/targets/BundledTemplates.targets @@ -1,37 +1,134 @@ + + + $(AspNetCorePackageVersionFor30Templates.IndexOf('-')) + $(AspNetCorePackageVersionFor30Templates) + $(AspNetCorePackageVersionFor30Templates.Substring(0, $(AspNetCore30VersionPreReleaseSeparator))) + $(AspNetCore30VersionMajorMinorPatchVersion)-$(VersionSuffix) + $(AspNetCore30VersionMajorMinorPatchVersion).$(GitCommitCount) + $(AspNetCore30VersionMajorMinorPatchVersion) + $(BundledTemplates30InstallPath)-$(VersionSuffix) + $([MSBuild]::Add($(AspNetCore30VersionMajorMinorPatchVersion.IndexOf('.')), 1)) + $(AspNetCore30VersionMajorMinorPatchVersion.IndexOf('.', $(Templates30VersionPatchSeparatorIndex))) + $(AspNetCore30VersionMajorMinorPatchVersion.Substring(0, $(Templates30VersionPatchSeparatorIndex))) + + $(AspNetCorePackageVersionFor22Templates.IndexOf('-')) + $(AspNetCorePackageVersionFor22Templates) + $(AspNetCorePackageVersionFor22Templates.Substring(0, $(AspNetCore22VersionPreReleaseSeparator))) + $(AspNetCore22VersionMajorMinorPatchVersion)-$(VersionSuffix) + $(AspNetCore22VersionMajorMinorPatchVersion).$(GitCommitCount) + $(AspNetCore22VersionMajorMinorPatchVersion) + $(BundledTemplates22InstallPath)-$(VersionSuffix) + $([MSBuild]::Add($(AspNetCore22VersionMajorMinorPatchVersion.IndexOf('.')), 1)) + $(AspNetCore22VersionMajorMinorPatchVersion.IndexOf('.', $(Templates22VersionPatchSeparatorIndex))) + $(AspNetCore22VersionMajorMinorPatchVersion.Substring(0, $(Templates22VersionPatchSeparatorIndex))) + + $(AspNetCorePackageVersionFor21Templates.IndexOf('-')) + $(AspNetCorePackageVersionFor21Templates) + $(AspNetCorePackageVersionFor21Templates.Substring(0, $(AspNetCore21VersionPreReleaseSeparator))) + $(AspNetCore21VersionMajorMinorPatchVersion)-$(VersionSuffix) + $(AspNetCore21VersionMajorMinorPatchVersion).$(GitCommitCount) + $(AspNetCore21VersionMajorMinorPatchVersion) + $(BundledTemplates21InstallPath)-$(VersionSuffix) + $([MSBuild]::Add($(AspNetCore21VersionMajorMinorPatchVersion.IndexOf('.')), 1)) + $(AspNetCore21VersionMajorMinorPatchVersion.IndexOf('.', $(Templates21VersionPatchSeparatorIndex))) + $(AspNetCore21VersionMajorMinorPatchVersion.Substring(0, $(Templates21VersionPatchSeparatorIndex))) + + + - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - + + + %(Identity)/%(PackageVersion)/%(Identity).%(PackageVersion).nupkg + $(NuGetPackageRoot)$([MSBuild]::ValueOrDefault('%(NupkgPathRelativeToPackageRoot)', '').ToLower()) + [%(PackageVersion)] + + + + + %(Identity)/%(PackageVersion)/%(Identity).%(PackageVersion).nupkg + $(NuGetPackageRoot)$([MSBuild]::ValueOrDefault('%(NupkgPathRelativeToPackageRoot)', '').ToLower()) + [%(PackageVersion)] + + + + + %(Identity)/%(PackageVersion)/%(Identity).%(PackageVersion).nupkg + $(NuGetPackageRoot)$([MSBuild]::ValueOrDefault('%(NupkgPathRelativeToPackageRoot)', '').ToLower()) + [%(PackageVersion)] + + - + + - + - - - %(Identity)/%(Version)/%(Identity).%(Version).nupkg - $(NuGetPackageRoot)$([MSBuild]::ValueOrDefault('%(NupkgPathRelativeToPackageRoot)', '').ToLower()) - - - - + + + + + - + + + + + + + + + + diff --git a/src/redist/targets/GenerateBundledVersions.targets b/src/redist/targets/GenerateBundledVersions.targets index cb2ac9f4e..1177dee24 100644 --- a/src/redist/targets/GenerateBundledVersions.targets +++ b/src/redist/targets/GenerateBundledVersions.targets @@ -190,7 +190,6 @@ Copyright (c) .NET Foundation. All rights reserved. TargetingPackVersion="$(NetCoreAppTargetingPackVersion)" RuntimePackNamePatterns="Microsoft.NETCore.App.Runtime.**RID**" RuntimePackRuntimeIdentifiers="@(NetCoreRuntimePackRids, '%3B')" - PackagesToReference="Microsoft.NETCore.App/$(NetCoreAppTargetingPackVersion)" IsTrimmable="true" /> diff --git a/src/redist/targets/GenerateDebs.targets b/src/redist/targets/GenerateDebs.targets index c911132fc..421b6e70a 100644 --- a/src/redist/targets/GenerateDebs.targets +++ b/src/redist/targets/GenerateDebs.targets @@ -83,10 +83,12 @@ $(RepoRoot)Documentation/manpages $(RepoRoot)test/EndToEnd/EndToEnd.Tests.csproj $(RedistLayoutPath)sdk/ + $(RedistLayoutPath)templates/ - + + @@ -227,7 +229,7 @@ DependsOnTargets="PrepareDotnetDebDirectories; PrepareDotnetDebTool; GetAspNetSharedFxInstallArgs;" - Inputs="@(CLISdkFiles)" + Inputs="@(CLISdkFiles);@(TemplatesFiles)" Outputs="$(SdkDebInstallerFile)" > @@ -248,6 +250,14 @@ SkipUnchangedFiles="False" UseHardlinksIfPossible="False" /> + + + $(BaseOutputPath)$(Configuration)\dotnet\ $(BaseOutputPath)$(Configuration)\dotnet-internal\ + $(BaseOutputPath)$(Configuration)\templates-3.0\ + $(BaseOutputPath)$(Configuration)\templates-2.2\ + $(BaseOutputPath)$(Configuration)\templates-2.1\ $(IntermediateOutputPath)downloads\ @@ -105,13 +108,6 @@ packs/%(PackageName)/%(PackageVersion) - - Microsoft.WindowsDesktop.App.Ref - $(WindowsDesktopTargetingPackVersion) - $(TargetFramework) - packs/%(PackageName)/%(PackageVersion) - - Microsoft.NETCore.App.Host.$(SharedFrameworkRid) $(NetCoreAppHostPackVersion) @@ -237,6 +233,13 @@ + + Microsoft.WindowsDesktop.App.Ref + $(WindowsDesktopTargetingPackVersion) + $(TargetFramework) + packs/%(PackageName)/%(PackageVersion) + + $(WinFormsAndWpfSharedFxRootUrl)$(MicrosoftWindowsDesktopPackageVersion) $(WinFormsAndWpfSharedFxArchiveFileName) diff --git a/src/redist/targets/GenerateMSBuildExtensions.targets b/src/redist/targets/GenerateMSBuildExtensions.targets new file mode 100644 index 000000000..2936166f7 --- /dev/null +++ b/src/redist/targets/GenerateMSBuildExtensions.targets @@ -0,0 +1,49 @@ + + + $(BaseOutputPath)$(Configuration)\MSBuildExtensionsLayout\ + + Microsoft.NET.Build.Extensions + $(NuGetPackageRoot)$(MSBuildExtensionsPackageName.ToLower())/$(MicrosoftNETBuildExtensionsPackageVersion.ToLower()) + + NETStandard.Library.NETFramework + $(NuGetPackageRoot)$(NETStandardLibraryNETFrameworkPackageName.ToLower())/$(CLI_NETStandardLibraryNETFrameworkVersion.ToLower()) + + + + + + + + + + + + + + + + + + + + + + + + + + + $(MSBuildExtensionsLayoutDirectory)/%(VSMSBuildExtensionsContent.DeploymentSubpath)%(RecursiveDir)%(Filename)%(Extension) + + + + + + \ No newline at end of file diff --git a/src/redist/targets/GenerateMSIs.targets b/src/redist/targets/GenerateMSIs.targets index dffddf82c..9fdb03b75 100644 --- a/src/redist/targets/GenerateMSIs.targets +++ b/src/redist/targets/GenerateMSIs.targets @@ -28,14 +28,17 @@ $(MSBuildThisFileDirectory)packaging/windows/clisdk/VS.Tools.Net.Core.SDK.Resolver.nuspec $(ArtifactsNonShippingPackagesDir)VS.Tools.Net.Core.SDK.Resolver.nupkg - + $(ArtifactsNonShippingPackagesDir)VS.Redist.Common.Net.Core.SDK.MSBuildExtensions.swr $(ArtifactsShippingPackagesDir)$(ArtifactNameWithVersionSdk)$(InstallerExtension) $(ArtifactsShippingPackagesDir)d$(Architecture)-1.cab + $(ArtifactsShippingPackagesDir)$(ArtifactNameTemplates)-$(BundledTemplates30Version)-$(ProductMonikerRid)$(InstallerExtension) + $(ArtifactsShippingPackagesDir)$(ArtifactNameTemplates)-$(BundledTemplates22Version)-$(ProductMonikerRid)$(InstallerExtension) + $(ArtifactsShippingPackagesDir)$(ArtifactNameTemplates)-$(BundledTemplates21Version)-$(ProductMonikerRid)$(InstallerExtension) $(ArtifactsShippingPackagesDir)$(ArtifactNameWithVersionCombinedHostHostFxrFrameworkSdk)$(BundleExtension) $(ProductBandCombinedHostHostFxrFrameworkSdkName) @@ -49,15 +52,9 @@ - - - - - - + + + + + + + + + + + + + @@ -104,8 +116,9 @@ + + + + + + + + $(MSBuildThisFileDirectory)packaging/windows/clisdk/generatetemplatesmsi.ps1 + + + + + $(Templates30LayoutPath.TrimEnd('\')) + $(Templates30MSIInstallerFile) + $(BundledTemplates30BrandName) + $(BundledTemplates30MsiVersion) + $(BundledTemplates30Version) + $(Templates30InstallerUpgradeCode) + + + $(Templates22LayoutPath.TrimEnd('\')) + $(Templates22MSIInstallerFile) + $(BundledTemplates22BrandName) + $(BundledTemplates22MsiVersion) + $(BundledTemplates22Version) + $(Templates22InstallerUpgradeCode) + + + $(Templates21LayoutPath.TrimEnd('\')) + $(Templates21MSIInstallerFile) + $(BundledTemplates21BrandName) + $(BundledTemplates21MsiVersion) + $(BundledTemplates21Version) + $(Templates21InstallerUpgradeCode) + + + + + + + + + + + + $(MSBuildThisFileDirectory)packaging/windows/clisdk/VS.Redist.Common.NetCore.Templates.nuspec + + + + + $(Templates30MSIInstallerFile) + $(ArtifactsNonShippingPackagesDir)VS.Redist.Common.NetCore.Templates.$(BundledTemplates30MajorMinorVersion).$(BundledTemplates30Version).nupkg + $(BundledTemplates30Version) + $(BundledTemplates30MajorMinorVersion) + + + $(Templates22MSIInstallerFile) + $(ArtifactsNonShippingPackagesDir)VS.Redist.Common.NetCore.Templates.$(BundledTemplates22MajorMinorVersion).$(BundledTemplates22Version).nupkg + $(BundledTemplates22Version) + $(BundledTemplates22MajorMinorVersion) + + + $(Templates21MSIInstallerFile) + $(ArtifactsNonShippingPackagesDir)VS.Redist.Common.NetCore.Templates.$(BundledTemplates21MajorMinorVersion).$(BundledTemplates21Version).nupkg + $(BundledTemplates21Version) + $(BundledTemplates21MajorMinorVersion) + + + + + '$(ArtifactsDir)' ^ + '$(RedistLayoutPath.TrimEnd('\'))' ^ + '$(FullNugetVersion)' ^ + '$(VSToolsNuspecFile)' ^ + '$(VSToolsNupkgFile)' ^ + '$(Architecture)'" /> + '$(ArtifactsDir)' ^ + '$(SdkResolverLayoutPath.TrimEnd('\'))' ^ + '$(FullNugetVersion)' ^ + '$(VSToolsResolverNuspecFile)' ^ + '$(VSToolsResolverNupkgFile)'" /> - - - + OutputFile="$(SdkMSBuildExtensionsSwrFile)"/> @@ -235,13 +349,16 @@ AcquireWix; GenerateSdkMsi; SignSdkMsi; + GenerateTemplatesMsis; + SignTemplatesMsis; GenerateSdkBundle; SignSdkBundle; GenerateToolsetNupkg; + GenerateTemplatesNupkgs; GenerateVSToolsNupkg; GenerateVSToolsResolverNupkg; GenerateSdkMSBuildExtensionsNupkg" - Condition=" '$(OS)' == 'Windows_NT' and !$(Architecture.StartsWith('arm'))" /> + Condition=" '$(OS)' == 'Windows_NT' and !$(Architecture.StartsWith('arm')) " /> diff --git a/src/redist/targets/GeneratePKG.targets b/src/redist/targets/GeneratePKG.targets index 9add88c7f..8f1cf9d65 100644 --- a/src/redist/targets/GeneratePKG.targets +++ b/src/redist/targets/GeneratePKG.targets @@ -129,6 +129,10 @@ Outputs="$(SdkPKGInstallerFile)" DependsOnTargets="GenerateLayout;SetupPkgInputsOutputs" Condition=" '$(OSName)' == 'osx' "> + + + + + + + @@ -50,6 +51,12 @@ SkipUnchangedFiles="False" UseHardlinksIfPossible="False"/> + + $(ArtifactNameCombinedHostHostFxrFrameworkSdk)-$(SdkRpmPackageVersion) $(SdkVersion) $(RedistLayoutPath)sdk/ + $(RedistLayoutPath)templates/ $(ArtifactsShippingPackagesDir)$(DistroSpecificArtifactNameWithVersionCombinedHostHostFxrFrameworkSdk)$(InstallerExtension) $(SdkRPMInstallerFile) @@ -183,6 +191,7 @@ $(RpmIntermediatesDir)/RpmLayoutDirectory/ $(RpmLayoutDirectory)package_root $(RpmLayoutPackageRoot)/sdk + $(RpmLayoutPackageRoot)/templates $(RpmLayoutDirectory)docs $(RpmLayoutDirectory)templates $(RpmLayoutDirectory)scripts diff --git a/src/redist/targets/GetRuntimeInformation.targets b/src/redist/targets/GetRuntimeInformation.targets index ff9ee1781..3b14d1203 100644 --- a/src/redist/targets/GetRuntimeInformation.targets +++ b/src/redist/targets/GetRuntimeInformation.targets @@ -42,6 +42,7 @@ $(Architecture) dotnet-sdk-internal + dotnet-templates dotnet-sdk $(ArtifactNameSdk)-$(SdkVersion)-$(ProductMonikerRid) diff --git a/src/redist/targets/Signing.targets b/src/redist/targets/Signing.targets index 0e39f28d5..4ea8c667f 100644 --- a/src/redist/targets/Signing.targets +++ b/src/redist/targets/Signing.targets @@ -171,6 +171,37 @@ + + + + + + + + + $(InternalCertificateId) + + + + + + + + diff --git a/src/redist/targets/Versions.targets b/src/redist/targets/Versions.targets index d6b16b412..273089432 100644 --- a/src/redist/targets/Versions.targets +++ b/src/redist/targets/Versions.targets @@ -4,7 +4,7 @@ 0 1 00 - preview7 + preview8 diff --git a/src/redist/targets/packaging/windows/clisdk/VS.Redist.Common.NetCore.Templates.nuspec b/src/redist/targets/packaging/windows/clisdk/VS.Redist.Common.NetCore.Templates.nuspec new file mode 100644 index 000000000..b6bcf5915 --- /dev/null +++ b/src/redist/targets/packaging/windows/clisdk/VS.Redist.Common.NetCore.Templates.nuspec @@ -0,0 +1,18 @@ + + + + VS.Redist.Common.NetCore.Templates.$ARCH$.$MAJOR_MINOR$ + 1.0.0 + VS.Redist.Common.NetCore.Templates.$ARCH$.$MAJOR_MINOR$ + Microsoft + Microsoft + https://www.microsoft.com/net/dotnet_library_license.htm + https://github.com/dotnet/core-sdk + true + .NET Core $MAJOR_MINOR$ Templates Windows Installer MSI as a .nupkg for internal Visual Studio build consumption + © Microsoft Corporation. All rights reserved. + + + + + diff --git a/src/redist/targets/packaging/windows/clisdk/VS.Redist.Common.NetCore.Toolset.nuspec b/src/redist/targets/packaging/windows/clisdk/VS.Redist.Common.NetCore.Toolset.nuspec index ba0555841..1c71d7a34 100644 --- a/src/redist/targets/packaging/windows/clisdk/VS.Redist.Common.NetCore.Toolset.nuspec +++ b/src/redist/targets/packaging/windows/clisdk/VS.Redist.Common.NetCore.Toolset.nuspec @@ -9,7 +9,7 @@ https://www.microsoft.com/net/dotnet_library_license.htm https://github.com/dotnet/core-sdk true - .NET Core SDK Toolset ($ARCH$) Windows Installer MSI as a .nupkg for internal Visual Studio build consumption + .NET Core $MAJOR_MINOR$ SDK Toolset ($ARCH$) Windows Installer MSI as a .nupkg for internal Visual Studio build consumption © Microsoft Corporation. All rights reserved. diff --git a/src/redist/targets/packaging/windows/clisdk/bundle.wxs b/src/redist/targets/packaging/windows/clisdk/bundle.wxs index 773810f95..27d2f03ba 100644 --- a/src/redist/targets/packaging/windows/clisdk/bundle.wxs +++ b/src/redist/targets/packaging/windows/clisdk/bundle.wxs @@ -142,6 +142,10 @@ + + + + diff --git a/src/redist/targets/packaging/windows/clisdk/generatebundle.ps1 b/src/redist/targets/packaging/windows/clisdk/generatebundle.ps1 index 64ad428c1..4af217d69 100644 --- a/src/redist/targets/packaging/windows/clisdk/generatebundle.ps1 +++ b/src/redist/targets/packaging/windows/clisdk/generatebundle.ps1 @@ -13,6 +13,7 @@ param( [Parameter(Mandatory=$true)][string]$NetStandardTargetingPackMSIFile, [Parameter(Mandatory=$true)][string]$AspNetTargetingPackMSIFile, [Parameter(Mandatory=$true)][string]$WindowsDesktopTargetingPackMSIFile, + [Parameter(Mandatory=$true)][string]$TemplatesMSIFile, [Parameter(Mandatory=$true)][string]$DotnetBundleOutput, [Parameter(Mandatory=$true)][string]$WixRoot, [Parameter(Mandatory=$true)][string]$ProductMoniker, @@ -53,6 +54,7 @@ function RunCandleForBundle -dNetStandardTargetingPackMsiSourcePath="$NetStandardTargetingPackMSIFile" ` -dAspNetTargetingPackMsiSourcePath="$AspNetTargetingPackMSIFile" ` -dWindowsDesktopTargetingPackMsiSourcePath="$WindowsDesktopTargetingPackMSIFile" ` + -dTemplatesMsiSourcePath="$TemplatesMSIFile" ` -dWinFormsAndWpfVersion="$WindowsDesktopVersion" ` -dAdditionalSharedFXMsiSourcePath="$AdditionalSharedFxMSIFile" ` -dAdditionalHostFXRMsiSourcePath="$AdditionalHostFxrMSIFile" ` diff --git a/src/redist/targets/packaging/windows/clisdk/generatenupkg.ps1 b/src/redist/targets/packaging/windows/clisdk/generatenupkg.ps1 index db130d14e..9fda65872 100644 --- a/src/redist/targets/packaging/windows/clisdk/generatenupkg.ps1 +++ b/src/redist/targets/packaging/windows/clisdk/generatenupkg.ps1 @@ -2,16 +2,17 @@ # Licensed under the MIT license. See LICENSE file in the project root for full license information. param( + [Parameter(Mandatory=$true)][string]$BinDir, [Parameter(Mandatory=$true)][string]$ContentPath, [Parameter(Mandatory=$true)][string]$NugetVersion, [Parameter(Mandatory=$true)][string]$NuspecFile, [Parameter(Mandatory=$true)][string]$NupkgFile, [Parameter(Mandatory=$false)][string]$Architecture, + [Parameter(Mandatory=$false)][string]$MmVersion, [Parameter(Mandatory=$false)][string]$CabPath ) -$RepoRoot = Convert-Path "$PSScriptRoot\..\..\..\..\..\.." -$NuGetDir = Join-Path $RepoRoot "artifacts\Tools\nuget" +$NuGetDir = Join-Path $BinDir "nuget" $NuGetExe = Join-Path $NuGetDir "nuget.exe" $OutputDirectory = [System.IO.Path]::GetDirectoryName($NupkgFile) $ContentPath = [System.IO.Path]::GetFullPath($ContentPath) @@ -33,5 +34,5 @@ if (Test-Path $NupkgFile) { Remove-Item -Force $NupkgFile } -& $NuGetExe pack $NuspecFile -Version $NugetVersion -OutputDirectory $OutputDirectory -NoDefaultExcludes -NoPackageAnalysis -Properties PAYLOAD_FILES=$ContentPath`;DOTNET_CAB_FILE=$CabPath`;ARCH=$Architecture +& $NuGetExe pack $NuspecFile -Version $NugetVersion -OutputDirectory $OutputDirectory -NoDefaultExcludes -NoPackageAnalysis -Properties PAYLOAD_FILES=$ContentPath`;DOTNET_CAB_FILE=$CabPath`;ARCH=$Architecture`;MAJOR_MINOR=$MmVersion Exit $LastExitCode diff --git a/src/redist/targets/packaging/windows/clisdk/generatetemplatesmsi.ps1 b/src/redist/targets/packaging/windows/clisdk/generatetemplatesmsi.ps1 new file mode 100644 index 000000000..ecb17fc2d --- /dev/null +++ b/src/redist/targets/packaging/windows/clisdk/generatetemplatesmsi.ps1 @@ -0,0 +1,149 @@ +# Copyright (c) .NET Foundation and contributors. All rights reserved. +# Licensed under the MIT license. See LICENSE file in the project root for full license information. + +param( + [Parameter(Mandatory=$true)][string]$inputDir, + [Parameter(Mandatory=$true)][string]$DotnetMSIOutput, + [Parameter(Mandatory=$true)][string]$WixRoot, + [Parameter(Mandatory=$true)][string]$ProductMoniker, + [Parameter(Mandatory=$true)][string]$DotnetMSIVersion, + [Parameter(Mandatory=$true)][string]$SDKBundleVersion, + [Parameter(Mandatory=$true)][string]$DotnetCLINugetVersion, + [Parameter(Mandatory=$true)][string]$UpgradeCode, + [Parameter(Mandatory=$true)][string]$Architecture +) + +$InstallFileswsx = ".\template-install-files.wxs" +$InstallFilesWixobj = "template-install-files.wixobj" + +function RunHeat +{ + $result = $true + pushd "$WixRoot" + + Write-Information "Running heat.." + + $heatOutput = .\heat.exe dir `"$inputDir`" -template fragment ` + -sreg -gg ` + -var var.DotnetSrc ` + -cg InstallFiles ` + -srd ` + -dr DOTNETHOME ` + -out template-install-files.wxs + + Write-Information "Heat output: $heatOutput" + + if($LastExitCode -ne 0) + { + $result = $false + Write-Information "Heat failed with exit code $LastExitCode." + } + + popd + Write-Information "RunHeat result: $result" + return $result +} + +function RunCandle +{ + $result = $true + pushd "$WixRoot" + + Write-Information "Running candle.." + + $candleOutput = .\candle.exe -nologo ` + -dDotnetSrc="$inputDir" ` + -dMicrosoftEula="$PSScriptRoot\clisdk\dummyeula.rtf" ` + -dProductMoniker="$ProductMoniker" ` + -dBuildVersion="$DotnetMSIVersion" ` + -dSDKBundleVersion="$SDKBundleVersion" ` + -dNugetVersion="$DotnetCLINugetVersion" ` + -dUpgradeCode="$UpgradeCode" ` + -arch "$Architecture" ` + -ext WixDependencyExtension.dll ` + "$PSScriptRoot\templates.wxs" ` + "$PSScriptRoot\provider.wxs" ` + $InstallFileswsx + + Write-Information "Candle output: $candleOutput" + + if($LastExitCode -ne 0) + { + $result = $false + Write-Information "Candle failed with exit code $LastExitCode." + } + + popd + return $result +} + +function RunLight +{ + $result = $true + pushd "$WixRoot" + + Write-Information "Running light.." + $CabCache = Join-Path $WixRoot "cabcache" + + $lightOutput = .\light.exe -nologo -ext WixUIExtension -ext WixDependencyExtension -ext WixUtilExtension ` + -cultures:en-us ` + templates.wixobj ` + provider.wixobj ` + $InstallFilesWixobj ` + -b "$inputDir" ` + -b "$PSScriptRoot" ` + -reusecab ` + -cc "$CabCache" ` + -out $DotnetMSIOutput + + Write-Information "Light output: $lightOutput" + + if($LastExitCode -ne 0) + { + $result = $false + Write-Information "Light failed with exit code $LastExitCode." + } + + popd + return $result +} + +if(!(Test-Path $inputDir)) +{ + throw "$inputDir not found" +} + +Write-Information "Creating templates MSI at $DotnetMSIOutput" + +if([string]::IsNullOrEmpty($WixRoot)) +{ + Exit -1 +} + +if(-Not (RunHeat)) +{ + Write-Information "Heat failed" + Exit -1 +} + +if(-Not (RunCandle)) +{ + Write-Information "Candle failed" + Exit -1 +} + +if(-Not (RunLight)) +{ + Write-Information "Light failed" + Exit -1 +} + +if(!(Test-Path $DotnetMSIOutput)) +{ + throw "Unable to create the templates MSI." + Exit -1 +} + +Write-Information "Successfully created templates MSI - $DotnetMSIOutput" + +exit $LastExitCode diff --git a/src/redist/targets/packaging/windows/clisdk/templates.wxs b/src/redist/targets/packaging/windows/clisdk/templates.wxs new file mode 100644 index 000000000..bb4b4815e --- /dev/null +++ b/src/redist/targets/packaging/windows/clisdk/templates.wxs @@ -0,0 +1,44 @@ + + + + + + + + Installed OR ALLOWMSIINSTALL + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +