Update dependencies from https://github.com/dotnet/arcade build 20200924.4 (#8738)

[release/5.0.1xx] Update dependencies from dotnet/arcade

eng/Version.Details.xml

@@ -140,13 +140,13 @@
     </Dependency>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="5.0.0-beta.20471.1">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="5.0.0-beta.20474.4">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>0365488709f58e37de6c2180e7fb243203ca0a9c</Sha>
+      <Sha>61cde6e8fb9d5c9790867b279deb41783a780cd8</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Build.Tasks.Installers" Version="5.0.0-beta.20471.1">
+    <Dependency Name="Microsoft.DotNet.Build.Tasks.Installers" Version="5.0.0-beta.20474.4">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>0365488709f58e37de6c2180e7fb243203ca0a9c</Sha>
+      <Sha>61cde6e8fb9d5c9790867b279deb41783a780cd8</Sha>
     </Dependency>
     <Dependency Name="Private.SourceBuild.ReferencePackages" Version="1.0.0-beta.20217.1">
       <Uri>https://github.com/dotnet/source-build-reference-packages</Uri>

eng/Versions.props

@@ -20,7 +20,7 @@
   </PropertyGroup>
   <PropertyGroup>
     <!-- Dependency from https://github.com/dotnet/arcade -->
-    <MicrosoftDotNetBuildTasksInstallersPackageVersion>5.0.0-beta.20471.1</MicrosoftDotNetBuildTasksInstallersPackageVersion>
+    <MicrosoftDotNetBuildTasksInstallersPackageVersion>5.0.0-beta.20474.4</MicrosoftDotNetBuildTasksInstallersPackageVersion>
   </PropertyGroup>
   <PropertyGroup>
     <!-- Dependency from https://github.com/dotnet/winforms -->

eng/common/performance/performance-setup.sh

@@ -201,7 +201,6 @@ if [[ "$internal" == true ]]; then
 fi
 
 if [[ "$mono_dotnet" != "" ]] && [[ "$monointerpreter" == "false" ]]; then
-    configurations="$configurations LLVM=$llvm MonoInterpreter=$monointerpreter MonoAOT=$monoaot"
     extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoMono"
 fi
 
@@ -211,6 +210,7 @@ if [[ "$wasm_runtime_loc" != "" ]]; then
 fi
 
 if [[ "$mono_dotnet" != "" ]] && [[ "$monointerpreter" == "true" ]]; then
+    configurations="$configurations LLVM=$llvm MonoInterpreter=$monointerpreter MonoAOT=$monoaot"
     extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoInterpreter NoMono"
 fi
 

eng/common/templates/job/source-build.yml

@@ -0,0 +1,49 @@
+parameters:
+  # This template adds arcade-powered source-build to CI. The template produces a server job with a
+  # default ID 'Source_Build_Complete' to put in a dependency list if necessary.
+
+  # Specifies the prefix for source-build jobs added to pipeline. Use this if disambiguation needed.
+  jobNamePrefix: 'Source_Build'
+
+  # Defines the platform on which to run the job. By default, a linux-x64 machine, suitable for
+  # managed-only repositories. This is an object with these properties:
+  #
+  # name: ''
+  #   The name of the job. This is included in the job ID.
+  # targetRID: ''
+  #   The name of the target RID to use, instead of the one auto-detected by Arcade.
+  # nonPortable: false
+  #   Enables non-portable mode. This means a more specific RID (e.g. fedora.32-x64 rather than
+  #   linux-x64), and compiling against distro-provided packages rather than portable ones.
+  # container: ''
+  #   A container to use. Runs in docker.
+  # pool: {}
+  #   A pool to use. Runs directly on an agent.
+  # buildScript: ''
+  #   Specifies the build script to invoke to perform the build in the repo. The default
+  #   './build.sh' should work for typical Arcade repositories, but this is customizable for
+  #   difficult situations.
+  # jobProperties: {}
+  #   A list of job properties to inject at the top level, for potential extensibility beyond
+  #   container and pool.
+  platform: {}
+
+jobs:
+- job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }}
+  displayName: Source-Build (${{ parameters.platform.name }})
+
+  ${{ each property in parameters.platform.jobProperties }}:
+    ${{ property.key }}: ${{ property.value }}
+
+  ${{ if ne(parameters.platform.container, '') }}:
+    container: ${{ parameters.platform.container }}
+  ${{ if ne(parameters.platform.pool, '') }}:
+    pool: ${{ parameters.platform.pool }}
+
+  workspace:
+    clean: all
+
+  steps:
+  - template: /eng/common/templates/steps/source-build.yml
+    parameters:
+      platform: ${{ parameters.platform }}

eng/common/templates/jobs/jobs.yml

@@ -24,6 +24,13 @@ parameters:
   #           if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects.
   runAsPublic: false
 
+  # Optional: Enable running the source-build jobs to build repo from source
+  runSourceBuild: false
+
+  # Optional: Parameters for source-build template.
+  #           See /eng/common/templates/jobs/source-build.yml for options
+  sourceBuildParameters: []
+
 # Internal resources (telemetry, microbuild) can only be accessed from non-public projects,
 # and some (Microbuild) should only be applied to non-PR cases for internal builds.
 
@@ -43,6 +50,13 @@ jobs:
 
       name: ${{ job.job }}
 
+- ${{ if eq(parameters.runSourceBuild, true) }}:
+  - template: /eng/common/templates/jobs/source-build.yml
+    parameters:
+      allCompletedJobId: Source_Build_Complete
+      ${{ each parameter in parameters.sourceBuildParameters }}:
+        ${{ parameter.key }}: ${{ parameter.value }}
+
 - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
   - ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}:
     - template: ../job/publish-build-assets.yml
@@ -55,6 +69,8 @@ jobs:
         - ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}:
           - ${{ each job in parameters.jobs }}:
             - ${{ job.job }}
+        - ${{ if eq(parameters.runSourceBuild, true) }}:
+          - Source_Build_Complete
         pool:
           vmImage: vs2017-win2016
         runAsPublic: ${{ parameters.runAsPublic }}

eng/common/templates/jobs/source-build.yml

@@ -0,0 +1,48 @@
+parameters:
+  # This template adds arcade-powered source-build to CI. A job is created for each platform, as
+  # well as an optional server job that completes when all platform jobs complete.
+
+  # The name of the "join" job for all source-build platforms. If set to empty string, the job is
+  # not included. Existing repo pipelines can use this job depend on all source-build jobs
+  # completing without maintaining a separate list of every single job ID: just depend on this one
+  # server job. By default, not included. Recommended name if used: 'Source_Build_Complete'.
+  allCompletedJobId: ''
+
+  # See /eng/common/templates/job/source-build.yml
+  jobNamePrefix: 'Source_Build'
+
+  # If changed to true, causes this template to include the default platform for a managed-only
+  # repo. The exact Docker image used for this build will be provided by Arcade. This has some risk,
+  # but since the repo is supposed to be managed-only, the risk should be very low.
+  includeDefaultManagedPlatform: false
+  defaultManagedPlatform:
+    name: 'Managed'
+    container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-3e800f1-20190501005343'
+
+  # Defines the platforms on which to run build jobs. One job is created for each platform, and the
+  # object in this array is sent to the job template as 'platform'.
+  platforms: []
+
+jobs:
+
+- ${{ if ne(parameters.allCompletedJobId, '') }}:
+  - job: ${{ parameters.allCompletedJobId }}
+    displayName: Source-Build Complete
+    pool: server
+    dependsOn:
+    - ${{ each platform in parameters.platforms }}:
+      - ${{ parameters.jobNamePrefix }}_${{ platform.name }}
+    - ${{ if eq(parameters.includeDefaultManagedPlatform, true) }}:
+      - ${{ parameters.jobNamePrefix }}_${{ parameters.defaultManagedPlatform.name }}
+
+- ${{ each platform in parameters.platforms }}:
+  - template: /eng/common/templates/job/source-build.yml
+    parameters:
+      jobNamePrefix: ${{ parameters.jobNamePrefix }}
+      platform: ${{ platform }}
+
+- ${{ if eq(parameters.includeDefaultManagedPlatform, true) }}:
+  - template: /eng/common/templates/job/source-build.yml
+    parameters:
+      jobNamePrefix: ${{ parameters.jobNamePrefix }}
+      platform: ${{ parameters.defaultManagedPlatform }}

eng/common/templates/post-build/post-build.yml

@@ -78,21 +78,22 @@ stages:
       BARBuildId: ${{ parameters.BARBuildId }}
       PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
 
-  - job:
-    displayName: Post-build Checks
-    dependsOn: setupMaestroVars
-    variables:
-      - name: TargetChannels
-        value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'] ]
-    pool:
-      vmImage: 'windows-2019'
-    steps:
-      - task: PowerShell@2
-        displayName: Maestro Channels Consistency
-        inputs:
-          filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1
-          arguments: -PromoteToChannels "$(TargetChannels)"
-            -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.NetDev6ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview8ChannelId}},${{parameters.Net5RC1ChannelId}},${{parameters.Net5RC2ChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}},${{parameters.VS166ChannelId}},${{parameters.VS167ChannelId}},${{parameters.VS168ChannelId}},${{parameters.VSMasterChannelId}}
+  - ${{ if and(le(parameters.publishingInfraVersion, 2), eq(parameters.inline, 'true')) }}:
+    - job:
+      displayName: Post-build Checks
+      dependsOn: setupMaestroVars
+      variables:
+        - name: TargetChannels
+          value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'] ]
+      pool:
+        vmImage: 'windows-2019'
+      steps:
+        - task: PowerShell@2
+          displayName: Maestro Channels Consistency
+          inputs:
+            filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1
+            arguments: -PromoteToChannels "$(TargetChannels)"
+              -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.NetDev6ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview8ChannelId}},${{parameters.Net5RC1ChannelId}},${{parameters.Net5RC2ChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}},${{parameters.VS166ChannelId}},${{parameters.VS167ChannelId}},${{parameters.VS168ChannelId}},${{parameters.VSMasterChannelId}}
 
   - job:
     displayName: NuGet Validation

eng/common/templates/steps/source-build.yml

@@ -0,0 +1,66 @@
+parameters:
+  # This template adds arcade-powered source-build to CI.
+
+  # This is a 'steps' template, and is intended for advanced scenarios where the existing build
+  # infra has a careful build methodology that must be followed. For example, a repo
+  # (dotnet/runtime) might choose to clone the GitHub repo only once and store it as a pipeline
+  # artifact for all subsequent jobs to use, to reduce dependence on a strong network connection to
+  # GitHub. Using this steps template leaves room for that infra to be included.
+
+  # Defines the platform on which to run the steps. See 'eng/common/templates/job/source-build.yml'
+  # for details. The entire object is described in the 'job' template for simplicity, even though
+  # the usage of the properties on this object is split between the 'job' and 'steps' templates.
+  platform: {}
+
+steps:
+# Build. Keep it self-contained for simple reusability. (No source-build-specific job variables.)
+- script: |
+    set -x
+    df -h
+
+    buildConfig=Release
+    # Check if AzDO substitutes in a build config from a variable, and use it if so.
+    if [ '$(_BuildConfig)' != '$''(_BuildConfig)' ]; then
+      buildConfig='$(_BuildConfig)'
+    fi
+
+    officialBuildArgs=
+    if [ '${{ and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}' = 'True' ]; then
+      officialBuildArgs='/p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER)'
+    fi
+
+    targetRidArgs=
+    if [ '${{ parameters.platform.targetRID }}' != '' ]; then
+      targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}'
+    fi
+
+    ${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \
+      --configuration $buildConfig \
+      --restore --build --pack --publish \
+      $officialBuildArgs \
+      $targetRidArgs \
+      /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \
+      /p:ArcadeBuildFromSource=true
+  displayName: Build
+
+# Upload build logs for diagnosis.
+- task: CopyFiles@2
+  displayName: Prepare BuildLogs staging directory
+  inputs:
+    SourceFolder: '$(Build.SourcesDirectory)'
+    Contents: |
+      **/*.log
+      **/*.binlog
+      artifacts/source-build/self/prebuilt-report/**
+    TargetFolder: '$(Build.StagingDirectory)/BuildLogs'
+    CleanTargetFolder: true
+  continueOnError: true
+  condition: succeededOrFailed()
+
+- task: PublishPipelineArtifact@1
+  displayName: Publish BuildLogs
+  inputs:
+    targetPath: '$(Build.StagingDirectory)/BuildLogs'
+    artifactName: BuildLogs_SourceBuild_${{ parameters.platform.name }}_Attempt$(System.JobAttempt)
+  continueOnError: true
+  condition: succeededOrFailed()

eng/common/tools.ps1

@@ -540,13 +540,15 @@ function GetDefaultMSBuildEngine() {
 
 function GetNuGetPackageCachePath() {
   if ($env:NUGET_PACKAGES -eq $null) {
-    # Use local cache on CI to ensure deterministic build,
+    # Use local cache on CI to ensure deterministic build. 
+    # Avoid using the http cache as workaround for https://github.com/NuGet/Home/issues/3116
     # use global cache in dev builds to avoid cost of downloading packages.
     # For directory normalization, see also: https://github.com/NuGet/Home/issues/7968
     if ($useGlobalNuGetCache) {
       $env:NUGET_PACKAGES = Join-Path $env:UserProfile '.nuget\packages\'
     } else {
       $env:NUGET_PACKAGES = Join-Path $RepoRoot '.packages\'
+      $env:RESTORENOCACHE = $true
     }
   }
 

eng/common/tools.sh

@@ -312,12 +312,14 @@ function InitializeBuildTool {
   _InitializeBuildToolFramework="netcoreapp2.1"
 }
 
+# Set RestoreNoCache as a workaround for https://github.com/NuGet/Home/issues/3116
 function GetNuGetPackageCachePath {
   if [[ -z ${NUGET_PACKAGES:-} ]]; then
     if [[ "$use_global_nuget_cache" == true ]]; then
       export NUGET_PACKAGES="$HOME/.nuget/packages"
     else
       export NUGET_PACKAGES="$repo_root/.packages"
+      export RESTORENOCACHE=true
     fi
   fi
 

global.json

@@ -8,6 +8,6 @@
     }
   },
   "msbuild-sdks": {
-    "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20471.1"
+    "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20474.4"
   }
 }