[release/7.0.3xx] Update dependencies from dotnet/arcade (#15486)

[release/7.0.3xx] Update dependencies from dotnet/arcade

eng/Version.Details.xml

@@ -188,18 +188,18 @@
     </Dependency>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="7.0.0-beta.23073.6">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="7.0.0-beta.23108.1">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>5f8519337c864af63b8692754ca0fa971bfc55d4</Sha>
+      <Sha>5eac3ace1b09862bd7421ba55795cc2a9c6a57d5</Sha>
       <SourceBuild RepoName="arcade" ManagedOnly="true" />
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.CMake.Sdk" Version="7.0.0-beta.23073.6">
+    <Dependency Name="Microsoft.DotNet.CMake.Sdk" Version="7.0.0-beta.23108.1">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>5f8519337c864af63b8692754ca0fa971bfc55d4</Sha>
+      <Sha>5eac3ace1b09862bd7421ba55795cc2a9c6a57d5</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Build.Tasks.Installers" Version="7.0.0-beta.23073.6">
+    <Dependency Name="Microsoft.DotNet.Build.Tasks.Installers" Version="7.0.0-beta.23108.1">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>5f8519337c864af63b8692754ca0fa971bfc55d4</Sha>
+      <Sha>5eac3ace1b09862bd7421ba55795cc2a9c6a57d5</Sha>
     </Dependency>
     <Dependency Name="Microsoft.SourceBuild.Intermediate.source-build-reference-packages" Version="7.0.0-alpha.1.23063.1">
       <Uri>https://github.com/dotnet/source-build-reference-packages</Uri>

eng/Versions.props

@@ -27,7 +27,7 @@
   </PropertyGroup>
   <PropertyGroup>
     <!-- Dependency from https://github.com/dotnet/arcade -->
-    <MicrosoftDotNetBuildTasksInstallersPackageVersion>7.0.0-beta.23073.6</MicrosoftDotNetBuildTasksInstallersPackageVersion>
+    <MicrosoftDotNetBuildTasksInstallersPackageVersion>7.0.0-beta.23108.1</MicrosoftDotNetBuildTasksInstallersPackageVersion>
   </PropertyGroup>
   <PropertyGroup>
     <!-- Dependency from https://github.com/dotnet/winforms -->

eng/common/generate-locproject.ps1

@@ -34,6 +34,25 @@ $jsonTemplateFiles | ForEach-Object {
 $jsonWinformsTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\\strings\.json" } # current winforms pattern
 
 $wxlFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\\.+\.wxl" -And -Not( $_.Directory.Name -Match "\d{4}" ) } # localized files live in four digit lang ID directories; this excludes them
+if (-not $wxlFiles) {
+    $wxlEnFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\\1033\\.+\.wxl" } #  pick up en files (1033 = en) specifically so we can copy them to use as the neutral xlf files
+    if ($wxlEnFiles) {
+      $wxlFiles = @()
+      $wxlEnFiles | ForEach-Object {
+        $destinationFile = "$($_.Directory.Parent.FullName)\$($_.Name)"
+        $wxlFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru
+      }
+    }
+}
+
+$macosHtmlEnFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\.lproj\\.+\.html" } # add installer HTML files
+$macosHtmlFiles = @()
+if ($macosHtmlEnFiles) {
+    $macosHtmlEnFiles | ForEach-Object {
+        $destinationFile = "$($_.Directory.Parent.FullName)\$($_.Name)"
+        $macosHtmlFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru
+    }
+}
 
 $xlfFiles = @()
 
@@ -91,6 +110,7 @@ $locJson = @{
             )
         },
         @{
+            LanguageSet = $LanguageSet
             CloneLanguageSet = "WiX_CloneLanguages"
             LssFiles = @( "wxl_loc.lss" )
             LocItems = @(
@@ -98,8 +118,7 @@ $locJson = @{
                     $outputPath = "$($_.Directory.FullName | Resolve-Path -Relative)\"
                     $continue = $true
                     foreach ($exclusion in $exclusions.Exclusions) {
-                        if ($_.FullName.Contains($exclusion))
+                        if ($_.FullName.Contains($exclusion)) {
-                        {
                             $continue = $false
                         }
                     }
@@ -110,7 +129,30 @@ $locJson = @{
                             SourceFile = $sourceFile
                             CopyOption = "LangIDOnPath"
                             OutputPath = $outputPath
-                            Languages = "cs-CZ;de-DE;es-ES;fr-FR;it-IT;ja-JP;ko-KR;pl-PL;pt-BR;ru-RU;tr-TR;zh-CN;zh-TW"
+                        }
+                    }
+                }
+            )
+        },
+        @{
+            LanguageSet = $LanguageSet
+            CloneLanguageSet = "VS_macOS_CloneLanguages"
+            LssFiles = @( ".\eng\common\loc\P22DotNetHtmlLocalization.lss" )
+            LocItems = @(
+                $macosHtmlFiles | ForEach-Object {
+                    $outputPath = "$($_.Directory.FullName | Resolve-Path -Relative)\"
+                    $continue = $true
+                    foreach ($exclusion in $exclusions.Exclusions) {
+                        if ($_.FullName.Contains($exclusion)) {
+                            $continue = $false
+                        }
+                    }
+                    $sourceFile = ($_.FullName | Resolve-Path -Relative)
+                    if ($continue) {
+                        return @{
+                            SourceFile = $sourceFile
+                            CopyOption = "LangIDOnPath"
+                            OutputPath = $outputPath
                         }
                     }
                 }

eng/common/templates/job/execute-sdl.yml

@@ -46,6 +46,7 @@ jobs:
     - template: /eng/common/templates/variables/sdl-variables.yml
     - name: GuardianVersion
       value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }}
+    - template: /eng/common/templates/variables/pool-providers.yml
   pool:
     # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com)
     ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
@@ -53,7 +54,7 @@ jobs:
       demands: Cmd
     # If it's not devdiv, it's dnceng
     ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}:
-      name: NetCore1ESPool-Svc-Internal
+      name: $(DncEngInternalBuildPool)
       demands: ImageOverride -equals windows.vs2019.amd64
   steps:
   - checkout: self

eng/common/templates/job/onelocbuild.yml

@@ -14,6 +14,7 @@ parameters:
   ReusePr: true
   UseLfLineEndings: true
   UseCheckedInLocProjectJson: false
+  SkipLocProjectJsonGeneration: false
   LanguageSet: VS_Main_Languages
   LclSource: lclFilesInRepo
   LclPackageId: ''
@@ -22,13 +23,25 @@ parameters:
   MirrorRepo: ''
   MirrorBranch: main
   condition: ''
+  JobNameSuffix: ''
 
 jobs:
-- job: OneLocBuild
+- job: OneLocBuild${{ parameters.JobNameSuffix }}
   
   dependsOn: ${{ parameters.dependsOn }}
 
-  displayName: OneLocBuild
+  displayName: OneLocBuild${{ parameters.JobNameSuffix }}
+
+  variables:
+    - group: OneLocBuildVariables # Contains the CeapexPat and GithubPat
+    - name: _GenerateLocProjectArguments
+      value: -SourcesDirectory ${{ parameters.SourcesDirectory }}
+        -LanguageSet "${{ parameters.LanguageSet }}"
+        -CreateNeutralXlfs
+    - ${{ if eq(parameters.UseCheckedInLocProjectJson, 'true') }}:
+      - name: _GenerateLocProjectArguments
+        value: ${{ variables._GenerateLocProjectArguments }} -UseCheckedInLocProjectJson
+    - template: /eng/common/templates/variables/pool-providers.yml
 
   ${{ if ne(parameters.pool, '') }}:
     pool: ${{ parameters.pool }}
@@ -40,27 +53,17 @@ jobs:
         demands: Cmd
       # If it's not devdiv, it's dnceng
       ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}:
-        name: NetCore1ESPool-Svc-Internal
+        name: $(DncEngInternalBuildPool)
         demands: ImageOverride -equals windows.vs2019.amd64
 
-  variables:
-    - group: OneLocBuildVariables # Contains the CeapexPat and GithubPat
-    - name: _GenerateLocProjectArguments
-      value: -SourcesDirectory ${{ parameters.SourcesDirectory }}
-        -LanguageSet "${{ parameters.LanguageSet }}"
-        -CreateNeutralXlfs
-    - ${{ if eq(parameters.UseCheckedInLocProjectJson, 'true') }}:
-      - name: _GenerateLocProjectArguments
-        value: ${{ variables._GenerateLocProjectArguments }} -UseCheckedInLocProjectJson
-      
-
   steps:
-    - task: Powershell@2
+    - ${{ if ne(parameters.SkipLocProjectJsonGeneration, 'true') }}:
-      inputs:
+      - task: Powershell@2
-        filePath: $(Build.SourcesDirectory)/eng/common/generate-locproject.ps1
+        inputs:
-        arguments: $(_GenerateLocProjectArguments)
+          filePath: $(Build.SourcesDirectory)/eng/common/generate-locproject.ps1
-      displayName: Generate LocProject.json
+          arguments: $(_GenerateLocProjectArguments)
-      condition: ${{ parameters.condition }}
+        displayName: Generate LocProject.json
+        condition: ${{ parameters.condition }}
 
     - task: OneLocBuild@2
       displayName: OneLocBuild

eng/common/templates/job/publish-build-assets.yml

@@ -34,15 +34,15 @@ jobs:
 - job: Asset_Registry_Publish
 
   dependsOn: ${{ parameters.dependsOn }}
+  timeoutInMinutes: 150
 
   ${{ if eq(parameters.publishAssetsImmediately, 'true') }}:
     displayName: Publish Assets
   ${{ else }}:
     displayName: Publish to Build Asset Registry
 
-  pool: ${{ parameters.pool }}
-
   variables:
+  - template: /eng/common/templates/variables/pool-providers.yml
   - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
     - group: Publish-Build-Assets
     - group: AzureDevOps-Artifact-Feeds-Pats
@@ -51,6 +51,16 @@ jobs:
     - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}:
       - template: /eng/common/templates/post-build/common-variables.yml
 
+  pool:
+    # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com)
+    ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
+      name: VSEngSS-MicroBuild2022-1ES
+      demands: Cmd
+    # If it's not devdiv, it's dnceng
+    ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}:
+      name: $(DncEngInternalBuildPool)
+      demands: ImageOverride -equals windows.vs2019.amd64
+
   steps:
   - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
     - task: DownloadBuildArtifacts@0

eng/common/templates/job/source-build.yml

@@ -44,13 +44,16 @@ jobs:
   ${{ if eq(parameters.platform.pool, '') }}:
     # The default VM host AzDO pool. This should be capable of running Docker containers: almost all
     # source-build builds run in Docker, including the default managed platform.
+    # /eng/common/templates/variables/pool-providers.yml can't be used here (some customers declare variables already), so duplicate its logic
     pool:
       ${{ if eq(variables['System.TeamProject'], 'public') }}:
-        name: NetCore-Svc-Public
+        name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')]
         demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open
+
       ${{ if eq(variables['System.TeamProject'], 'internal') }}:
-        name: NetCore1ESPool-Svc-Internal
+        name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')]
         demands: ImageOverride -equals Build.Ubuntu.1804.Amd64
+
   ${{ if ne(parameters.platform.pool, '') }}:
     pool: ${{ parameters.platform.pool }}
 

eng/common/templates/job/source-index-stage1.yml

@@ -22,16 +22,17 @@ jobs:
     value: ${{ parameters.binlogPath }}
   - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
     - group: source-dot-net stage1 variables
+  - template: /eng/common/templates/variables/pool-providers.yml
 
   ${{ if ne(parameters.pool, '') }}:
     pool: ${{ parameters.pool }}
   ${{ if eq(parameters.pool, '') }}:
     pool:
       ${{ if eq(variables['System.TeamProject'], 'public') }}:
-        name: NetCore-Svc-Public
+        name: $(DncEngPublicBuildPool)
         demands: ImageOverride -equals windows.vs2019.amd64.open
       ${{ if eq(variables['System.TeamProject'], 'internal') }}:
-        name: NetCore1ESPool-Svc-Internal
+        name: $(DncEngInternalBuildPool)
         demands: ImageOverride -equals windows.vs2019.amd64
 
   steps:

eng/common/templates/jobs/jobs.yml

@@ -88,15 +88,6 @@ jobs:
             - ${{ job.job }}
         - ${{ if eq(parameters.enableSourceBuild, true) }}:
           - Source_Build_Complete
-        pool:
-          # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com)
-          ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
-            name: VSEngSS-MicroBuild2022-1ES
-            demands: Cmd
-          # If it's not devdiv, it's dnceng
-          ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}:
-            name: NetCore1ESPool-Svc-Internal
-            demands: ImageOverride -equals windows.vs2019.amd64
 
         runAsPublic: ${{ parameters.runAsPublic }}
         publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }}

eng/common/templates/post-build/post-build.yml

@@ -95,6 +95,7 @@ stages:
     displayName: Validate Build Assets
     variables:
       - template: common-variables.yml
+      - template: /eng/common/templates/variables/pool-providers.yml
     jobs:
     - job:
       displayName: NuGet Validation
@@ -106,7 +107,7 @@ stages:
           demands: Cmd
         # If it's not devdiv, it's dnceng
         ${{ else }}:
-          name: NetCore1ESPool-Svc-Internal
+          name: $(DncEngInternalBuildPool)
           demands: ImageOverride -equals windows.vs2019.amd64
 
       steps:
@@ -143,7 +144,7 @@ stages:
           demands: Cmd
         # If it's not devdiv, it's dnceng
         ${{ else }}:
-          name: NetCore1ESPool-Svc-Internal
+          name: $(DncEngInternalBuildPool)
           demands: ImageOverride -equals windows.vs2019.amd64
       steps:
         - template: setup-maestro-vars.yml
@@ -203,7 +204,7 @@ stages:
           demands: Cmd
         # If it's not devdiv, it's dnceng
         ${{ else }}:
-          name: NetCore1ESPool-Svc-Internal
+          name: $(DncEngInternalBuildPool)
           demands: ImageOverride -equals windows.vs2019.amd64
       steps:
         - template: setup-maestro-vars.yml
@@ -251,6 +252,7 @@ stages:
     displayName: Publish using Darc
     variables:
       - template: common-variables.yml
+      - template: /eng/common/templates/variables/pool-providers.yml
     jobs:
     - job:
       displayName: Publish Using Darc
@@ -262,7 +264,7 @@ stages:
           demands: Cmd
         # If it's not devdiv, it's dnceng
         ${{ else }}:
-          name: NetCore1ESPool-Svc-Internal
+          name: $(DncEngInternalBuildPool)
           demands: ImageOverride -equals windows.vs2019.amd64
       steps:
         - template: setup-maestro-vars.yml
@@ -282,4 +284,4 @@ stages:
               -MaestroToken '$(MaestroApiAccessToken)'
               -WaitPublishingFinish true
               -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}'
-              -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}'
+              -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}'

eng/common/templates/variables/pool-providers.yml

@@ -0,0 +1,57 @@
+# Select a pool provider based off branch name. Anything with branch name containing 'release' must go into an -Svc pool,
+# otherwise it should go into the "normal" pools. This separates out the queueing and billing of released branches.
+
+# Motivation: 
+#   Once a given branch of a repository's output has been officially "shipped" once, it is then considered to be COGS
+#   (Cost of goods sold) and should be moved to a servicing pool provider. This allows both separation of queueing
+#   (allowing release builds and main PR builds to not intefere with each other) and billing (required for COGS)
+#   Additionally, the pool provider name itself may be subject to change when the .NET Core Engineering Services 
+#   team needs to move resources around and create new and potentially differently-named pools. Using this template 
+#   file from an Arcade-ified repo helps guard against both having to update one's release/* branches and renaming.
+
+# How to use: 
+#  This yaml assumes your shipped product branches use the naming convention "release/..." (which many do).
+#  If we find alternate naming conventions in broad usage these can be added to the condition below.
+#
+#  First, import the template in an arcade-ified repo to pick up the variables, e.g.:
+#
+#  variables:
+#  - template: /eng/common/templates/variables/pool-providers.yml
+#
+#  ... then anywhere specifying the pool provider use the runtime variables,
+#      $(DncEngInternalBuildPool) and $  (DncEngPublicBuildPool), e.g.:
+#
+#        pool:
+#           name: $(DncEngInternalBuildPool)
+#           demands: ImageOverride -equals windows.vs2019.amd64
+
+variables:
+  # Coalesce the target and source branches so we know when a PR targets a release branch
+  # If these variables are somehow missing, fall back to main (tends to have more capacity)
+
+  # Any new -Svc alternative pools should have variables added here to allow for splitting work
+  - name: DncEngPublicBuildPool
+    value: $[
+        replace(
+          replace(
+            eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'),
+            True,
+            'NetCore-Svc-Public'
+          ),
+          False,
+          'NetCore-Public'
+        )
+      ]
+
+  - name: DncEngInternalBuildPool
+    value: $[
+        replace(
+          replace(
+            eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'),
+            True,
+            'NetCore1ESPool-Svc-Internal'
+          ),
+          False,
+          'NetCore1ESPool-Internal'
+        )
+      ]

global.json

@@ -11,7 +11,7 @@
     "cmake": "3.16.4"
   },
   "msbuild-sdks": {
-    "Microsoft.DotNet.Arcade.Sdk": "7.0.0-beta.23073.6",
+    "Microsoft.DotNet.Arcade.Sdk": "7.0.0-beta.23108.1",
-    "Microsoft.DotNet.CMake.Sdk": "7.0.0-beta.23073.6"
+    "Microsoft.DotNet.CMake.Sdk": "7.0.0-beta.23108.1"
   }
 }