Update dependencies from https://github.com/dotnet/arcade build 20220106.6 (#12988)

[main] Update dependencies from dotnet/arcade
- Coherency Updates:
  - Microsoft.DotNet.XliffTasks: from 1.0.0-beta.21631.1 to 1.0.0-beta.22055.1 (parent: Microsoft.DotNet.Arcade.Sdk)

eng/Version.Details.xml

@@ -172,19 +172,19 @@
     </Dependency>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="7.0.0-beta.22054.5">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="7.0.0-beta.22056.6">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>85188b0c348f3f7f8206f441ec2ee2da6dbb06fe</Sha>
+      <Sha>34bc5b1611e13bd0ee6a9f38ab8524d2ee489be5</Sha>
       <SourceBuild RepoName="arcade" ManagedOnly="true" />
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.CMake.Sdk" Version="7.0.0-beta.22054.5">
+    <Dependency Name="Microsoft.DotNet.CMake.Sdk" Version="7.0.0-beta.22056.6">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>85188b0c348f3f7f8206f441ec2ee2da6dbb06fe</Sha>
+      <Sha>34bc5b1611e13bd0ee6a9f38ab8524d2ee489be5</Sha>
       <SourceBuild RepoName="arcade" ManagedOnly="true" />
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Build.Tasks.Installers" Version="7.0.0-beta.22054.5">
+    <Dependency Name="Microsoft.DotNet.Build.Tasks.Installers" Version="7.0.0-beta.22056.6">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>85188b0c348f3f7f8206f441ec2ee2da6dbb06fe</Sha>
+      <Sha>34bc5b1611e13bd0ee6a9f38ab8524d2ee489be5</Sha>
     </Dependency>
     <Dependency Name="Microsoft.SourceBuild.Intermediate.source-build-reference-packages" Version="7.0.0-alpha.1.22055.1">
       <Uri>https://github.com/dotnet/source-build-reference-packages</Uri>
@@ -196,9 +196,9 @@
       <Sha>427936db7c8a37605c4116c4fc785bd0196f34fe</Sha>
       <SourceBuild RepoName="sourcelink" ManagedOnly="true" />
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.XliffTasks" Version="1.0.0-beta.21631.1" CoherentParentDependency="Microsoft.DotNet.Arcade.Sdk">
+    <Dependency Name="Microsoft.DotNet.XliffTasks" Version="1.0.0-beta.22055.1" CoherentParentDependency="Microsoft.DotNet.Arcade.Sdk">
       <Uri>https://github.com/dotnet/xliff-tasks</Uri>
-      <Sha>228b671d9e34cedd19608457b3d76455cd4deedb</Sha>
+      <Sha>4b9b46a95bebd9f058238c50f0039590e6c58d51</Sha>
       <SourceBuild RepoName="xliff-tasks" ManagedOnly="true" />
     </Dependency>
   </ToolsetDependencies>

eng/Versions.props

@@ -26,7 +26,7 @@
   </PropertyGroup>
   <PropertyGroup>
     <!-- Dependency from https://github.com/dotnet/arcade -->
-    <MicrosoftDotNetBuildTasksInstallersPackageVersion>7.0.0-beta.22054.5</MicrosoftDotNetBuildTasksInstallersPackageVersion>
+    <MicrosoftDotNetBuildTasksInstallersPackageVersion>7.0.0-beta.22056.6</MicrosoftDotNetBuildTasksInstallersPackageVersion>
   </PropertyGroup>
   <PropertyGroup>
     <!-- Dependency from https://github.com/dotnet/winforms -->

eng/common/generate-graph-files.ps1

@@ -1,86 +0,0 @@
-Param(
-  [Parameter(Mandatory=$true)][string] $barToken,       # Token generated at https://maestro-prod.westus2.cloudapp.azure.com/Account/Tokens
-  [Parameter(Mandatory=$true)][string] $gitHubPat,      # GitHub personal access token from https://github.com/settings/tokens (no auth scopes needed)
-  [Parameter(Mandatory=$true)][string] $azdoPat,        # Azure Dev Ops tokens from https://dev.azure.com/dnceng/_details/security/tokens (code read scope needed)
-  [Parameter(Mandatory=$true)][string] $outputFolder,   # Where the graphviz.txt file will be created
-  [string] $darcVersion,                                # darc's version
-  [string] $graphvizVersion = '2.38',                   # GraphViz version
-  [switch] $includeToolset                              # Whether the graph should include toolset dependencies or not. i.e. arcade, optimization. For more about
-                                                        # toolset dependencies see https://github.com/dotnet/arcade/blob/master/Documentation/Darc.md#toolset-vs-product-dependencies
-)
-
-function CheckExitCode ([string]$stage)
-{
-  $exitCode = $LASTEXITCODE
-  if ($exitCode  -ne 0) {
-    Write-PipelineTelemetryError -Category 'Arcade' -Message "Something failed in stage: '$stage'. Check for errors above. Exiting now..."
-    ExitWithExitCode $exitCode
-  }
-}
-
-try {
-  $ErrorActionPreference = 'Stop'
-  . $PSScriptRoot\tools.ps1
-  
-  Import-Module -Name (Join-Path $PSScriptRoot 'native\CommonLibrary.psm1')
-
-  Push-Location $PSScriptRoot
-
-  Write-Host 'Installing darc...'
-  . .\darc-init.ps1 -darcVersion $darcVersion
-  CheckExitCode 'Running darc-init'
-
-  $engCommonBaseDir = Join-Path $PSScriptRoot 'native\'
-  $graphvizInstallDir = CommonLibrary\Get-NativeInstallDirectory
-  $nativeToolBaseUri = 'https://netcorenativeassets.blob.core.windows.net/resource-packages/external'
-  $installBin = Join-Path $graphvizInstallDir 'bin'
-
-  Write-Host 'Installing dot...'
-  .\native\install-tool.ps1 -ToolName graphviz -InstallPath $installBin -BaseUri $nativeToolBaseUri -CommonLibraryDirectory $engCommonBaseDir -Version $graphvizVersion -Verbose
-
-  $darcExe = "$env:USERPROFILE\.dotnet\tools"
-  $darcExe = Resolve-Path "$darcExe\darc.exe"
-
-  Create-Directory $outputFolder
-
-  # Generate 3 graph descriptions:
-  # 1. Flat with coherency information
-  # 2. Graphviz (dot) file
-  # 3. Standard dependency graph
-  $graphVizFilePath = "$outputFolder\graphviz.txt"
-  $graphVizImageFilePath = "$outputFolder\graph.png"
-  $normalGraphFilePath = "$outputFolder\graph-full.txt"
-  $flatGraphFilePath = "$outputFolder\graph-flat.txt"
-  $baseOptions = @( '--github-pat', "$gitHubPat", '--azdev-pat', "$azdoPat", '--password', "$barToken" )
-
-  if ($includeToolset) {
-    Write-Host 'Toolsets will be included in the graph...'
-    $baseOptions += @( '--include-toolset' )
-  }
-
-  Write-Host 'Generating standard dependency graph...'
-  & "$darcExe" get-dependency-graph @baseOptions --output-file $normalGraphFilePath
-  CheckExitCode 'Generating normal dependency graph'
-
-  Write-Host 'Generating flat dependency graph and graphviz file...'
-  & "$darcExe" get-dependency-graph @baseOptions --flat --coherency --graphviz $graphVizFilePath --output-file $flatGraphFilePath
-  CheckExitCode 'Generating flat and graphviz dependency graph'
-
-  Write-Host "Generating graph image $graphVizFilePath"
-  $dotFilePath = Join-Path $installBin "graphviz\$graphvizVersion\release\bin\dot.exe"
-  & "$dotFilePath" -Tpng -o"$graphVizImageFilePath" "$graphVizFilePath"
-  CheckExitCode 'Generating graphviz image'
-
-  Write-Host "'$graphVizFilePath', '$flatGraphFilePath', '$normalGraphFilePath' and '$graphVizImageFilePath' created!"
-}
-catch {
-  if (!$includeToolset) {
-    Write-Host 'This might be a toolset repo which includes only toolset dependencies. ' -NoNewline -ForegroundColor Yellow
-    Write-Host 'Since -includeToolset is not set there is no graph to create. Include -includeToolset and try again...' -ForegroundColor Yellow
-  }
-  Write-Host $_.ScriptStackTrace
-  Write-PipelineTelemetryError -Category 'Arcade' -Message $_
-  ExitWithExitCode 1
-} finally {
-  Pop-Location
-}

eng/common/sdl/configure-sdl-tool.ps1

@@ -15,7 +15,9 @@ Param(
   # Optional: Additional params to add to any tool using CredScan.
   [string[]] $CrScanAdditionalRunConfigParams,
   # Optional: Additional params to add to any tool using PoliCheck.
-  [string[]] $PoliCheckAdditionalRunConfigParams
+  [string[]] $PoliCheckAdditionalRunConfigParams,
+  # Optional: Additional params to add to any tool using CodeQL/Semmle.
+  [string[]] $CodeQLAdditionalRunConfigParams
 )
 
 $ErrorActionPreference = 'Stop'
@@ -78,6 +80,11 @@ try {
         $tool.Args += "`"Target < $TargetDirectory`""
       }
       $tool.Args += $PoliCheckAdditionalRunConfigParams
+    } elseif ($tool.Name -eq 'semmle' -or $tool.Name -eq 'codeql') {
+      if ($targetDirectory) {
+        $tool.Args += "`"SourceCodeDirectory < $TargetDirectory`""
+      }
+      $tool.Args += $CodeQLAdditionalRunConfigParams
     }
 
     # Create variable pointing to the args array directly so we can use splat syntax later.

eng/common/sdl/execute-all-sdl-tools.ps1

@@ -34,6 +34,7 @@ Param(
   [string] $GuardianLoggerLevel='Standard',                                                      # Optional: the logger level for the Guardian CLI; options are Trace, Verbose, Standard, Warning, and Error
   [string[]] $CrScanAdditionalRunConfigParams,                                                   # Optional: Additional Params to custom build a CredScan run config in the format @("xyz:abc","sdf:1")
   [string[]] $PoliCheckAdditionalRunConfigParams,                                                # Optional: Additional Params to custom build a Policheck run config in the format @("xyz:abc","sdf:1")
+  [string[]] $CodeQLAdditionalRunConfigParams,                                                   # Optional: Additional Params to custom build a Semmle/CodeQL run config in the format @("xyz < abc","sdf < 1")
   [bool] $BreakOnFailure=$False                                                                  # Optional: Fail the build if there were errors during the run
 )
 
@@ -105,7 +106,8 @@ try {
           -AzureDevOpsAccessToken $AzureDevOpsAccessToken `
           -GuardianLoggerLevel $GuardianLoggerLevel `
           -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams `
-          -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams
+          -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams `
+          -CodeQLAdditionalRunConfigParams $CodeQLAdditionalRunConfigParams
         if ($BreakOnFailure) {
           Exit-IfNZEC "Sdl"
         }

eng/common/templates/job/generate-graph-files.yml

@@ -1,48 +0,0 @@
-parameters:
-  # Optional: dependencies of the job
-  dependsOn: ''
-
-  # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool
-  pool: {}
-
-  # Optional: Include toolset dependencies in the generated graph files
-  includeToolset: false
-  
-jobs:
-- job: Generate_Graph_Files
-
-  dependsOn: ${{ parameters.dependsOn }}
-
-  displayName: Generate Graph Files
-
-  pool: ${{ parameters.pool }}
-
-  variables:
-    # Publish-Build-Assets provides: MaestroAccessToken, BotAccount-dotnet-maestro-bot-PAT
-    # DotNet-AllOrgs-Darc-Pats provides: dn-bot-devdiv-dnceng-rw-code-pat
-    - group: Publish-Build-Assets
-    - group: DotNet-AllOrgs-Darc-Pats
-    - name: _GraphArguments
-      value: -gitHubPat $(BotAccount-dotnet-maestro-bot-PAT) 
-        -azdoPat $(dn-bot-devdiv-dnceng-rw-code-pat) 
-        -barToken $(MaestroAccessToken) 
-        -outputFolder '$(Build.StagingDirectory)/GraphFiles/'
-    - ${{ if ne(parameters.includeToolset, 'false') }}:
-      - name: _GraphArguments
-        value: ${{ variables._GraphArguments }} -includeToolset
-
-  steps:
-    - task: PowerShell@2
-      displayName: Generate Graph Files
-      inputs:
-        filePath: eng\common\generate-graph-files.ps1 
-        arguments: $(_GraphArguments)
-      continueOnError: true
-    - task: PublishBuildArtifacts@1
-      displayName: Publish Graph to Artifacts
-      inputs:
-        PathtoPublish: '$(Build.StagingDirectory)/GraphFiles'
-        PublishLocation: Container
-        ArtifactName: GraphFiles
-      continueOnError: true
-      condition: always()      

eng/common/templates/jobs/codeql-build.yml

@@ -0,0 +1,31 @@
+parameters:
+  # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md
+  continueOnError: false
+  # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job
+  jobs: []
+  # Optional: if specified, restore and use this version of Guardian instead of the default.
+  overrideGuardianVersion: ''
+
+jobs:
+- template: /eng/common/templates/jobs/jobs.yml
+  parameters:
+    enableMicrobuild: false
+    enablePublishBuildArtifacts: false
+    enablePublishTestResults: false
+    enablePublishBuildAssets: false
+    enablePublishUsingPipelines: false
+    enableTelemetry: true
+
+    variables:
+      - group: Publish-Build-Assets
+      # The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in
+      # sync with the packages.config file.
+      - name: DefaultGuardianVersion
+        value: 0.109.0
+      - name: GuardianPackagesConfigFile
+        value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config
+      - name: GuardianVersion
+        value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }}
+  
+    jobs: ${{ parameters.jobs }}
+        

eng/common/templates/jobs/jobs.yml

@@ -87,13 +87,3 @@ jobs:
         runAsPublic: ${{ parameters.runAsPublic }}
         publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }}
         enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }}
-
-  - ${{ if eq(parameters.graphFileGeneration.enabled, true) }}:
-    - template: ../job/generate-graph-files.yml
-      parameters:
-        continueOnError: ${{ parameters.continueOnError }}
-        includeToolset: ${{ parameters.graphFileGeneration.includeToolset }}
-        dependsOn:
-          - Asset_Registry_Publish
-        pool:
-          vmImage: 'windows-2019'

eng/common/templates/steps/execute-codeql.yml

@@ -0,0 +1,32 @@
+parameters:
+  # Language that should be analyzed. Defaults to csharp
+  language: csharp
+  # Build Commands
+  buildCommands: ''
+  overrideParameters: ''                                       # Optional: to override values for parameters.
+  additionalParameters: ''                                     # Optional: parameters that need user specific values eg: '-SourceToolsList @("abc","def") -ArtifactToolsList @("ghi","jkl")'
+  # Optional: if specified, restore and use this version of Guardian instead of the default.
+  overrideGuardianVersion: ''
+  # Optional: if true, publish the '.gdn' folder as a pipeline artifact. This can help with in-depth
+  # diagnosis of problems with specific tool configurations.
+  publishGuardianDirectoryToPipeline: false
+  # The script to run to execute all SDL tools. Use this if you want to use a script to define SDL
+  # parameters rather than relying on YAML. It may be better to use a local script, because you can
+  # reproduce results locally without piecing together a command based on the YAML.
+  executeAllSdlToolsScript: 'eng/common/sdl/execute-all-sdl-tools.ps1'
+  # There is some sort of bug (has been reported) in Azure DevOps where if this parameter is named
+  # 'continueOnError', the parameter value is not correctly picked up.
+  # This can also be remedied by the caller (post-build.yml) if it does not use a nested parameter
+  # optional: determines whether to continue the build if the step errors;
+  sdlContinueOnError: false
+
+steps:
+- template: /eng/common/templates/steps/execute-sdl.yml
+  parameters:
+    overrideGuardianVersion: ${{ parameters.overrideGuardianVersion }}
+    executeAllSdlToolsScript: ${{ parameters.executeAllSdlToolsScript }}
+    overrideParameters: ${{ parameters.overrideParameters }}
+    additionalParameters: '${{ parameters.additionalParameters }}
+      -CodeQLAdditionalRunConfigParams @("BuildCommands < ${{ parameters.buildCommands }}", "Language < ${{ parameters.language }}")'
+    publishGuardianDirectoryToPipeline: ${{ parameters.publishGuardianDirectoryToPipeline }}
+    sdlContinueOnError: ${{ parameters.sdlContinueOnError }}

global.json

@@ -11,7 +11,7 @@
     "cmake": "3.16.4"
   },
   "msbuild-sdks": {
-    "Microsoft.DotNet.Arcade.Sdk": "7.0.0-beta.22054.5",
+    "Microsoft.DotNet.Arcade.Sdk": "7.0.0-beta.22056.6",
-    "Microsoft.DotNet.CMake.Sdk": "7.0.0-beta.22054.5"
+    "Microsoft.DotNet.CMake.Sdk": "7.0.0-beta.22056.6"
   }
 }