From 4d44907633e3dd1642ed159f79d9ba78afb27df2 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 28 Sep 2020 16:12:55 +0000 Subject: [PATCH] Update dependencies from https://github.com/dotnet/arcade build 20200924.4 (#8738) [release/5.0.1xx] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 8 +-- eng/Versions.props | 2 +- eng/common/performance/performance-setup.sh | 2 +- eng/common/templates/job/source-build.yml | 49 ++++++++++++++ eng/common/templates/jobs/jobs.yml | 16 +++++ eng/common/templates/jobs/source-build.yml | 48 ++++++++++++++ .../templates/post-build/post-build.yml | 31 ++++----- eng/common/templates/steps/source-build.yml | 66 +++++++++++++++++++ eng/common/tools.ps1 | 4 +- eng/common/tools.sh | 2 + global.json | 2 +- 11 files changed, 207 insertions(+), 23 deletions(-) create mode 100644 eng/common/templates/job/source-build.yml create mode 100644 eng/common/templates/jobs/source-build.yml create mode 100644 eng/common/templates/steps/source-build.yml diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 7ff38de08..67a58bb8d 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -140,13 +140,13 @@ - + https://github.com/dotnet/arcade - 0365488709f58e37de6c2180e7fb243203ca0a9c + 61cde6e8fb9d5c9790867b279deb41783a780cd8 - + https://github.com/dotnet/arcade - 0365488709f58e37de6c2180e7fb243203ca0a9c + 61cde6e8fb9d5c9790867b279deb41783a780cd8 https://github.com/dotnet/source-build-reference-packages diff --git a/eng/Versions.props b/eng/Versions.props index 026111ca7..f372bcfc1 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -20,7 +20,7 @@ - 5.0.0-beta.20471.1 + 5.0.0-beta.20474.4 diff --git a/eng/common/performance/performance-setup.sh b/eng/common/performance/performance-setup.sh index 806e56c61..99d1b7bc1 100755 --- a/eng/common/performance/performance-setup.sh +++ b/eng/common/performance/performance-setup.sh @@ -201,7 +201,6 @@ if [[ "$internal" == true ]]; then fi if [[ "$mono_dotnet" != "" ]] && [[ "$monointerpreter" == "false" ]]; then - configurations="$configurations LLVM=$llvm MonoInterpreter=$monointerpreter MonoAOT=$monoaot" extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoMono" fi @@ -211,6 +210,7 @@ if [[ "$wasm_runtime_loc" != "" ]]; then fi if [[ "$mono_dotnet" != "" ]] && [[ "$monointerpreter" == "true" ]]; then + configurations="$configurations LLVM=$llvm MonoInterpreter=$monointerpreter MonoAOT=$monoaot" extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoInterpreter NoMono" fi diff --git a/eng/common/templates/job/source-build.yml b/eng/common/templates/job/source-build.yml new file mode 100644 index 000000000..9332f5ecc --- /dev/null +++ b/eng/common/templates/job/source-build.yml @@ -0,0 +1,49 @@ +parameters: + # This template adds arcade-powered source-build to CI. The template produces a server job with a + # default ID 'Source_Build_Complete' to put in a dependency list if necessary. + + # Specifies the prefix for source-build jobs added to pipeline. Use this if disambiguation needed. + jobNamePrefix: 'Source_Build' + + # Defines the platform on which to run the job. By default, a linux-x64 machine, suitable for + # managed-only repositories. This is an object with these properties: + # + # name: '' + # The name of the job. This is included in the job ID. + # targetRID: '' + # The name of the target RID to use, instead of the one auto-detected by Arcade. + # nonPortable: false + # Enables non-portable mode. This means a more specific RID (e.g. fedora.32-x64 rather than + # linux-x64), and compiling against distro-provided packages rather than portable ones. + # container: '' + # A container to use. Runs in docker. + # pool: {} + # A pool to use. Runs directly on an agent. + # buildScript: '' + # Specifies the build script to invoke to perform the build in the repo. The default + # './build.sh' should work for typical Arcade repositories, but this is customizable for + # difficult situations. + # jobProperties: {} + # A list of job properties to inject at the top level, for potential extensibility beyond + # container and pool. + platform: {} + +jobs: +- job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }} + displayName: Source-Build (${{ parameters.platform.name }}) + + ${{ each property in parameters.platform.jobProperties }}: + ${{ property.key }}: ${{ property.value }} + + ${{ if ne(parameters.platform.container, '') }}: + container: ${{ parameters.platform.container }} + ${{ if ne(parameters.platform.pool, '') }}: + pool: ${{ parameters.platform.pool }} + + workspace: + clean: all + + steps: + - template: /eng/common/templates/steps/source-build.yml + parameters: + platform: ${{ parameters.platform }} diff --git a/eng/common/templates/jobs/jobs.yml b/eng/common/templates/jobs/jobs.yml index c08225a9a..08845950f 100644 --- a/eng/common/templates/jobs/jobs.yml +++ b/eng/common/templates/jobs/jobs.yml @@ -24,6 +24,13 @@ parameters: # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. runAsPublic: false + # Optional: Enable running the source-build jobs to build repo from source + runSourceBuild: false + + # Optional: Parameters for source-build template. + # See /eng/common/templates/jobs/source-build.yml for options + sourceBuildParameters: [] + # Internal resources (telemetry, microbuild) can only be accessed from non-public projects, # and some (Microbuild) should only be applied to non-PR cases for internal builds. @@ -43,6 +50,13 @@ jobs: name: ${{ job.job }} +- ${{ if eq(parameters.runSourceBuild, true) }}: + - template: /eng/common/templates/jobs/source-build.yml + parameters: + allCompletedJobId: Source_Build_Complete + ${{ each parameter in parameters.sourceBuildParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}: - template: ../job/publish-build-assets.yml @@ -55,6 +69,8 @@ jobs: - ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}: - ${{ each job in parameters.jobs }}: - ${{ job.job }} + - ${{ if eq(parameters.runSourceBuild, true) }}: + - Source_Build_Complete pool: vmImage: vs2017-win2016 runAsPublic: ${{ parameters.runAsPublic }} diff --git a/eng/common/templates/jobs/source-build.yml b/eng/common/templates/jobs/source-build.yml new file mode 100644 index 000000000..f463011e7 --- /dev/null +++ b/eng/common/templates/jobs/source-build.yml @@ -0,0 +1,48 @@ +parameters: + # This template adds arcade-powered source-build to CI. A job is created for each platform, as + # well as an optional server job that completes when all platform jobs complete. + + # The name of the "join" job for all source-build platforms. If set to empty string, the job is + # not included. Existing repo pipelines can use this job depend on all source-build jobs + # completing without maintaining a separate list of every single job ID: just depend on this one + # server job. By default, not included. Recommended name if used: 'Source_Build_Complete'. + allCompletedJobId: '' + + # See /eng/common/templates/job/source-build.yml + jobNamePrefix: 'Source_Build' + + # If changed to true, causes this template to include the default platform for a managed-only + # repo. The exact Docker image used for this build will be provided by Arcade. This has some risk, + # but since the repo is supposed to be managed-only, the risk should be very low. + includeDefaultManagedPlatform: false + defaultManagedPlatform: + name: 'Managed' + container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-3e800f1-20190501005343' + + # Defines the platforms on which to run build jobs. One job is created for each platform, and the + # object in this array is sent to the job template as 'platform'. + platforms: [] + +jobs: + +- ${{ if ne(parameters.allCompletedJobId, '') }}: + - job: ${{ parameters.allCompletedJobId }} + displayName: Source-Build Complete + pool: server + dependsOn: + - ${{ each platform in parameters.platforms }}: + - ${{ parameters.jobNamePrefix }}_${{ platform.name }} + - ${{ if eq(parameters.includeDefaultManagedPlatform, true) }}: + - ${{ parameters.jobNamePrefix }}_${{ parameters.defaultManagedPlatform.name }} + +- ${{ each platform in parameters.platforms }}: + - template: /eng/common/templates/job/source-build.yml + parameters: + jobNamePrefix: ${{ parameters.jobNamePrefix }} + platform: ${{ platform }} + +- ${{ if eq(parameters.includeDefaultManagedPlatform, true) }}: + - template: /eng/common/templates/job/source-build.yml + parameters: + jobNamePrefix: ${{ parameters.jobNamePrefix }} + platform: ${{ parameters.defaultManagedPlatform }} diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index df06f5371..0854e4896 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -78,21 +78,22 @@ stages: BARBuildId: ${{ parameters.BARBuildId }} PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - job: - displayName: Post-build Checks - dependsOn: setupMaestroVars - variables: - - name: TargetChannels - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'] ] - pool: - vmImage: 'windows-2019' - steps: - - task: PowerShell@2 - displayName: Maestro Channels Consistency - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1 - arguments: -PromoteToChannels "$(TargetChannels)" - -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.NetDev6ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview8ChannelId}},${{parameters.Net5RC1ChannelId}},${{parameters.Net5RC2ChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}},${{parameters.VS166ChannelId}},${{parameters.VS167ChannelId}},${{parameters.VS168ChannelId}},${{parameters.VSMasterChannelId}} + - ${{ if and(le(parameters.publishingInfraVersion, 2), eq(parameters.inline, 'true')) }}: + - job: + displayName: Post-build Checks + dependsOn: setupMaestroVars + variables: + - name: TargetChannels + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'] ] + pool: + vmImage: 'windows-2019' + steps: + - task: PowerShell@2 + displayName: Maestro Channels Consistency + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1 + arguments: -PromoteToChannels "$(TargetChannels)" + -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.NetDev6ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview8ChannelId}},${{parameters.Net5RC1ChannelId}},${{parameters.Net5RC2ChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}},${{parameters.VS166ChannelId}},${{parameters.VS167ChannelId}},${{parameters.VS168ChannelId}},${{parameters.VSMasterChannelId}} - job: displayName: NuGet Validation diff --git a/eng/common/templates/steps/source-build.yml b/eng/common/templates/steps/source-build.yml new file mode 100644 index 000000000..8e336b7d1 --- /dev/null +++ b/eng/common/templates/steps/source-build.yml @@ -0,0 +1,66 @@ +parameters: + # This template adds arcade-powered source-build to CI. + + # This is a 'steps' template, and is intended for advanced scenarios where the existing build + # infra has a careful build methodology that must be followed. For example, a repo + # (dotnet/runtime) might choose to clone the GitHub repo only once and store it as a pipeline + # artifact for all subsequent jobs to use, to reduce dependence on a strong network connection to + # GitHub. Using this steps template leaves room for that infra to be included. + + # Defines the platform on which to run the steps. See 'eng/common/templates/job/source-build.yml' + # for details. The entire object is described in the 'job' template for simplicity, even though + # the usage of the properties on this object is split between the 'job' and 'steps' templates. + platform: {} + +steps: +# Build. Keep it self-contained for simple reusability. (No source-build-specific job variables.) +- script: | + set -x + df -h + + buildConfig=Release + # Check if AzDO substitutes in a build config from a variable, and use it if so. + if [ '$(_BuildConfig)' != '$''(_BuildConfig)' ]; then + buildConfig='$(_BuildConfig)' + fi + + officialBuildArgs= + if [ '${{ and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}' = 'True' ]; then + officialBuildArgs='/p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER)' + fi + + targetRidArgs= + if [ '${{ parameters.platform.targetRID }}' != '' ]; then + targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}' + fi + + ${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \ + --configuration $buildConfig \ + --restore --build --pack --publish \ + $officialBuildArgs \ + $targetRidArgs \ + /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \ + /p:ArcadeBuildFromSource=true + displayName: Build + +# Upload build logs for diagnosis. +- task: CopyFiles@2 + displayName: Prepare BuildLogs staging directory + inputs: + SourceFolder: '$(Build.SourcesDirectory)' + Contents: | + **/*.log + **/*.binlog + artifacts/source-build/self/prebuilt-report/** + TargetFolder: '$(Build.StagingDirectory)/BuildLogs' + CleanTargetFolder: true + continueOnError: true + condition: succeededOrFailed() + +- task: PublishPipelineArtifact@1 + displayName: Publish BuildLogs + inputs: + targetPath: '$(Build.StagingDirectory)/BuildLogs' + artifactName: BuildLogs_SourceBuild_${{ parameters.platform.name }}_Attempt$(System.JobAttempt) + continueOnError: true + condition: succeededOrFailed() diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 60066083b..9014e0625 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -540,13 +540,15 @@ function GetDefaultMSBuildEngine() { function GetNuGetPackageCachePath() { if ($env:NUGET_PACKAGES -eq $null) { - # Use local cache on CI to ensure deterministic build, + # Use local cache on CI to ensure deterministic build. + # Avoid using the http cache as workaround for https://github.com/NuGet/Home/issues/3116 # use global cache in dev builds to avoid cost of downloading packages. # For directory normalization, see also: https://github.com/NuGet/Home/issues/7968 if ($useGlobalNuGetCache) { $env:NUGET_PACKAGES = Join-Path $env:UserProfile '.nuget\packages\' } else { $env:NUGET_PACKAGES = Join-Path $RepoRoot '.packages\' + $env:RESTORENOCACHE = $true } } diff --git a/eng/common/tools.sh b/eng/common/tools.sh index c722a0585..b5d63cb1b 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -312,12 +312,14 @@ function InitializeBuildTool { _InitializeBuildToolFramework="netcoreapp2.1" } +# Set RestoreNoCache as a workaround for https://github.com/NuGet/Home/issues/3116 function GetNuGetPackageCachePath { if [[ -z ${NUGET_PACKAGES:-} ]]; then if [[ "$use_global_nuget_cache" == true ]]; then export NUGET_PACKAGES="$HOME/.nuget/packages" else export NUGET_PACKAGES="$repo_root/.packages" + export RESTORENOCACHE=true fi fi diff --git a/global.json b/global.json index d4a907589..c403f91c9 100644 --- a/global.json +++ b/global.json @@ -8,6 +8,6 @@ } }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20471.1" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20474.4" } }