[master] Update dependencies from dotnet/arcade (#4126)

* Update dependencies from https://github.com/dotnet/arcade build 20190820.8 - Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19420.8 * Update dependencies from https://github.com/dotnet/arcade build 20190821.4 - Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19421.4
2019-08-22 12:34:05 +00:00 · 2019-08-22 12:34:05 +00:00 · b8dcb2ee22
commit b8dcb2ee22
parent 2865939007
10 changed files with 100 additions and 32 deletions
--- a/eng/Version.Details.xml
+++ b/eng/Version.Details.xml
@ -65,9 +65,9 @@
    </Dependency>
  </ProductDependencies>
  <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="1.0.0-beta.19419.12">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="1.0.0-beta.19421.4">
      <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>10b2260aeed5f07582bf8d8dcd4221a447b171c5</Sha>
+      <Sha>27a6926f6b1d4b36d86a236d2b05cdee3669589e</Sha>
    </Dependency>
  </ToolsetDependencies>
 </Dependencies>
--- a/eng/common/post-build/sourcelink-validation.ps1
+++ b/eng/common/post-build/sourcelink-validation.ps1
@ -1,8 +1,8 @@
 param(
  [Parameter(Mandatory=$true)][string] $InputPath,              # Full path to directory where Symbols.NuGet packages to be checked are stored
  [Parameter(Mandatory=$true)][string] $ExtractPath,            # Full path to directory where the packages will be extracted during validation
-  [Parameter(Mandatory=$true)][string] $GHRepoName,             # GitHub name of the repo including the Org. E.g., dotnet/arcade
-  [Parameter(Mandatory=$true)][string] $GHCommit,               # GitHub commit SHA used to build the packages
+  [Parameter(Mandatory=$false)][string] $GHRepoName,            # GitHub name of the repo including the Org. E.g., dotnet/arcade
+  [Parameter(Mandatory=$false)][string] $GHCommit,              # GitHub commit SHA used to build the packages
  [Parameter(Mandatory=$true)][string] $SourcelinkCliVersion    # Version of SourceLink CLI to use
 )

@ -13,6 +13,12 @@ param(
 # all files present in the repo at a specific commit point.
 $global:RepoFiles = @{}

+# Maximum number of jobs to run in parallel
+$MaxParallelJobs = 6
+
+# Wait time between check for system load
+$SecondsBetweenLoadChecks = 10
+
 $ValidatePackage = {
  param( 
    [string] $PackagePath                                 # Full path to a Symbols.NuGet package
@ -22,8 +28,8 @@ $ValidatePackage = {

  # Ensure input file exist
  if (!(Test-Path $PackagePath)) {
-    Write-PipelineTaskError "Input file does not exist: $PackagePath"
-    ExitWithExitCode 1
+    Write-Host "Input file does not exist: $PackagePath"
+    return 1
  }

  # Extensions for which we'll look for SourceLink information
@ -38,7 +44,7 @@ $ValidatePackage = {

  Add-Type -AssemblyName System.IO.Compression.FileSystem

-  [System.IO.Directory]::CreateDirectory($ExtractPath);
+  [System.IO.Directory]::CreateDirectory($ExtractPath)  | Out-Null

  try {
    $zip = [System.IO.Compression.ZipFile]::OpenRead($PackagePath)
@ -138,16 +144,18 @@ $ValidatePackage = {

  if ($FailedFiles -eq 0) {
    Write-Host "Passed."
+    return 0
  }
  else {
-    Write-PipelineTaskError "$PackagePath has broken SourceLink links."
+    Write-Host "$PackagePath has broken SourceLink links."
+    return 1
  }
 }

 function ValidateSourceLinkLinks {
-  if (!($GHRepoName -Match "^[^\s\/]+/[^\s\/]+$")) {
+  if ($GHRepoName -ne "" -and !($GHRepoName -Match "^[^\s\/]+/[^\s\/]+$")) {
    if (!($GHRepoName -Match "^[^\s-]+-[^\s]+$")) {
-      Write-PipelineTaskError "GHRepoName should be in the format <org>/<repo> or <org>-<repo>"
+      Write-PipelineTaskError "GHRepoName should be in the format <org>/<repo> or <org>-<repo>. '$GHRepoName'"
      ExitWithExitCode 1
    }
    else {
@ -155,30 +163,33 @@ function ValidateSourceLinkLinks {
    }
  }

-  if (!($GHCommit -Match "^[0-9a-fA-F]{40}$")) {
-    Write-PipelineTaskError "GHCommit should be a 40 chars hexadecimal string"
+  if ($GHCommit -ne "" -and !($GHCommit -Match "^[0-9a-fA-F]{40}$")) {
+    Write-PipelineTaskError "GHCommit should be a 40 chars hexadecimal string. '$GHCommit'"
    ExitWithExitCode 1
  }

-  $RepoTreeURL = -Join("http://api.github.com/repos/", $GHRepoName, "/git/trees/", $GHCommit, "?recursive=1")
-  $CodeExtensions = @(".cs", ".vb", ".fs", ".fsi", ".fsx", ".fsscript")
+  if ($GHRepoName -ne "" -and $GHCommit -ne "") {
+    $RepoTreeURL = -Join("http://api.github.com/repos/", $GHRepoName, "/git/trees/", $GHCommit, "?recursive=1")
+    $CodeExtensions = @(".cs", ".vb", ".fs", ".fsi", ".fsx", ".fsscript")

-  try {
-    # Retrieve the list of files in the repo at that particular commit point and store them in the RepoFiles hash
-    $Data = Invoke-WebRequest $RepoTreeURL -UseBasicParsing | ConvertFrom-Json | Select-Object -ExpandProperty tree
+    try {
+      # Retrieve the list of files in the repo at that particular commit point and store them in the RepoFiles hash
+      $Data = Invoke-WebRequest $RepoTreeURL -UseBasicParsing | ConvertFrom-Json | Select-Object -ExpandProperty tree
  
-    foreach ($file in $Data) {
-      $Extension = [System.IO.Path]::GetExtension($file.path)
+      foreach ($file in $Data) {
+        $Extension = [System.IO.Path]::GetExtension($file.path)

-      if ($CodeExtensions.Contains($Extension)) {
-        $RepoFiles[$file.path] = 1
+        if ($CodeExtensions.Contains($Extension)) {
+          $RepoFiles[$file.path] = 1
+        }
      }
    }
+    catch {
+      Write-Host "Problems downloading the list of files from the repo. Url used: $RepoTreeURL . Execution will proceed without caching."
+    }
  }
-  catch {
-    Write-PipelineTaskError "Problems downloading the list of files from the repo. Url used: $RepoTreeURL"
-    Write-Host $_
-    ExitWithExitCode 1
+  elseif ($GHRepoName -ne "" -or $GHCommit -ne "") {
+    Write-Host "For using the http caching mechanism both GHRepoName and GHCommit should be informed."
  }
  
  if (Test-Path $ExtractPath) {
@ -186,14 +197,33 @@ function ValidateSourceLinkLinks {
  }

  # Process each NuGet package in parallel
-  $Jobs = @()
  Get-ChildItem "$InputPath\*.symbols.nupkg" |
    ForEach-Object {
-      $Jobs += Start-Job -ScriptBlock $ValidatePackage -ArgumentList $_.FullName
+      Start-Job -ScriptBlock $ValidatePackage -ArgumentList $_.FullName | Out-Null
+      $NumJobs = @(Get-Job -State 'Running').Count
+      
+      while ($NumJobs -ge $MaxParallelJobs) {
+        Write-Host "There are $NumJobs validation jobs running right now. Waiting $SecondsBetweenLoadChecks seconds to check again."
+        sleep $SecondsBetweenLoadChecks
+        $NumJobs = @(Get-Job -State 'Running').Count
+      }
+
+      foreach ($Job in @(Get-Job -State 'Completed')) {
+        Receive-Job -Id $Job.Id
+        Remove-Job -Id $Job.Id
+      }
    }

+  $ValidationFailures = 0
  foreach ($Job in $Jobs) {
-    Wait-Job -Id $Job.Id | Receive-Job
+    $jobResult = Wait-Job -Id $Job.Id | Receive-Job
+    if ($jobResult -ne "0") {
+      $ValidationFailures++
+    }
+  }
+  if ($ValidationFailures -gt 0) {
+    Write-PipelineTaskError " $ValidationFailures package(s) failed validation."
+    ExitWithExitCode 1
  }
 }

--- a/eng/common/templates/post-build/channels/netcore-dev-5.yml
+++ b/eng/common/templates/post-build/channels/netcore-dev-5.yml
@ -2,6 +2,7 @@ parameters:
  enableSymbolValidation: true
  symbolPublishingAdditionalParameters: ''
  artifactsPublishingAdditionalParameters: ''
+  publishInstallersAndChecksums: false

 stages:
 - stage: NetCore_Dev5_Publish
@ -101,7 +102,12 @@ stages:
            /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' 
            /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' 
            /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' 
-            /p:Configuration=Release 
+            /p:Configuration=Release
+            /p:InstallersTargetStaticFeed=$(InstallersBlobFeedUrl)
+            /p:InstallersAzureAccountKey=$(dotnetcli-storage-key)
+            /p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }}
+            /p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl)
+            /p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key)
            ${{ parameters.artifactsPublishingAdditionalParameters }}
        
      - task: NuGetCommand@2
--- a/eng/common/templates/post-build/channels/netcore-tools-latest.yml
+++ b/eng/common/templates/post-build/channels/netcore-tools-latest.yml
@ -2,6 +2,7 @@ parameters:
  enableSymbolValidation: true
  symbolPublishingAdditionalParameters: ''
  artifactsPublishingAdditionalParameters: ''
+  publishInstallersAndChecksums: false

 stages:
 - stage: NetCore_Tools_Latest_Publish
@ -101,7 +102,12 @@ stages:
            /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' 
            /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' 
            /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' 
-            /p:Configuration=Release 
+            /p:Configuration=Release
+            /p:InstallersTargetStaticFeed=$(InstallersBlobFeedUrl)
+            /p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }}
+            /p:InstallersAzureAccountKey=$(dotnetcli-storage-key)
+            /p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl)
+            /p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key)
            ${{ parameters.artifactsPublishingAdditionalParameters }}
        
      - task: NuGetCommand@2
--- a/eng/common/templates/post-build/channels/public-dev-release.yml
+++ b/eng/common/templates/post-build/channels/public-dev-release.yml
@ -2,6 +2,7 @@ parameters:
  enableSymbolValidation: true
  symbolPublishingAdditionalParameters: ''
  artifactsPublishingAdditionalParameters: ''
+  publishInstallersAndChecksums: false

 stages:
 - stage: Publish
@ -102,6 +103,11 @@ stages:
            /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' 
            /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' 
            /p:Configuration=Release 
+            /p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }}
+            /p:InstallersTargetStaticFeed=$(InstallersBlobFeedUrl)
+            /p:InstallersAzureAccountKey=$(dotnetcli-storage-key)
+            /p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl)
+            /p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key)
            ${{ parameters.artifactsPublishingAdditionalParameters }}
        
      - task: NuGetCommand@2
--- a/eng/common/templates/post-build/channels/public-release.yml
+++ b/eng/common/templates/post-build/channels/public-release.yml
@ -90,6 +90,7 @@ stages:
            /p:IsInternalBuild=$(IsInternalBuild)
            /p:RepositoryName=$(Build.Repository.Name)
            /p:CommitSha=$(Build.SourceVersion)
+            /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' 
            /p:AzureStorageAccountName=$(ProxyBackedFeedsAccountName)
            /p:AzureStorageAccountKey=$(dotnetfeed-storage-access-key-1)
            /p:AzureDevOpsFeedsBaseUrl=$(dotnetfeed-internal-private-feed-url)
--- a/eng/common/templates/post-build/channels/public-validation-release.yml
+++ b/eng/common/templates/post-build/channels/public-validation-release.yml
@ -1,5 +1,6 @@
 parameters:
  artifactsPublishingAdditionalParameters: ''
+  publishInstallersAndChecksums: false

 stages:
 - stage: PVR_Publish
@ -65,7 +66,12 @@ stages:
            /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' 
            /p:BlobBasePath='$(Build.ArtifactStagingDirectory)\BlobArtifacts' 
            /p:PackageBasePath='$(Build.ArtifactStagingDirectory)\PackageArtifacts' 
-            /p:Configuration=Release 
+            /p:Configuration=Release
+            /p:InstallersTargetStaticFeed=$(InstallersBlobFeedUrl)
+            /p:InstallersAzureAccountKey=$(dotnetcli-storage-key)
+            /p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }}
+            /p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl)
+            /p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key)
            ${{ parameters.artifactsPublishingAdditionalParameters }}

      - task: NuGetCommand@2
--- a/eng/common/templates/post-build/common-variables.yml
+++ b/eng/common/templates/post-build/common-variables.yml
@ -1,5 +1,6 @@
 variables:
  - group: Publish-Build-Assets
+  - group: DotNet-DotNetCli-Storage

  # .NET Core 3 Dev
  - name: PublicDevRelease_30_Channel_Id
@ -45,3 +46,9 @@ variables:
    value: 3.0.0
  - name: SymbolToolVersion
    value: 1.0.1
+
+  # Default locations for Installers and checksums
+  - name: ChecksumsBlobFeedUrl
+    value: https://dotnetcli.blob.core.windows.net/dotnet/index.json
+  - name: InstallersBlobFeedUrl
+    value: https://dotnetclichecksums.blob.core.windows.net/dotnet/index.json
--- a/eng/common/templates/post-build/post-build.yml
+++ b/eng/common/templates/post-build/post-build.yml
@ -3,6 +3,7 @@ parameters:
  enableSigningValidation: true
  enableSymbolValidation: true
  enableNugetValidation: true
+  publishInstallersAndChecksums: false
  SDLValidationParameters:
    enable: false
    params: ''
@ -85,6 +86,7 @@ stages:
              -GHRepoName $(Build.Repository.Name) 
              -GHCommit $(Build.SourceVersion)
              -SourcelinkCliVersion $(SourceLinkCLIVersion)
+          continueOnError: true

  - ${{ if eq(parameters.SDLValidationParameters.enable, 'true') }}:
    - template: /eng/common/templates/job/execute-sdl.yml
@ -96,22 +98,26 @@ stages:
    enableSymbolValidation: ${{ parameters.enableSymbolValidation }}
    symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
    artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+    publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}

 - template: \eng\common\templates\post-build\channels\public-dev-release.yml
  parameters:
    enableSymbolValidation: ${{ parameters.enableSymbolValidation }}
    symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
    artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+    publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}

 - template: \eng\common\templates\post-build\channels\netcore-tools-latest.yml
  parameters:
    enableSymbolValidation: ${{ parameters.enableSymbolValidation }}
    symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
    artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+    publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}

 - template: \eng\common\templates\post-build\channels\public-validation-release.yml
  parameters:
    artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+    publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}

 - template: \eng\common\templates\post-build\channels\public-release.yml
  parameters:
--- a/global.json
+++ b/global.json
@ -3,6 +3,6 @@
    "dotnet": "3.0.100-preview6-012264"
  },
  "msbuild-sdks": {
-    "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19419.12"
+    "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19421.4"
  }
 }