Update dependencies from https://github.com/dotnet/arcade build 20190201.16 (#371)

This change updates the following dependencies
- Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19101.16

eng/Version.Details.xml

@@ -28,9 +28,9 @@
     </Dependency>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="1.0.0-beta.19081.3">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="1.0.0-beta.19101.16">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>1e859f1c17fffbe9c4fb6bbfc0fc71cd0c56563b</Sha>
+      <Sha>159c3e6c65ab7b1e89e962290538364898d02cd8</Sha>
     </Dependency>
   </ToolsetDependencies>
 </Dependencies>

eng/common/PublishToPackageFeed.proj

@@ -10,15 +10,25 @@
     <TargetFramework>netcoreapp2.1</TargetFramework>
   </PropertyGroup>
 
-  <Import Project="$(MSBuildThisFileDirectory)MicrosoftDotNetBuildTasksFeedVersion.props" />
+  <Import Project="$(MSBuildThisFileDirectory)DefaultVersions.props" Condition="Exists('$(MSBuildThisFileDirectory)DefaultVersions.props')" />
+  <Import Project="$(MSBuildThisFileDirectory)Versions.props" Condition="Exists('$(MSBuildThisFileDirectory)Versions.props')" />
+  
   <Import Project="$(NuGetPackageRoot)microsoft.dotnet.build.tasks.feed\$(MicrosoftDotNetBuildTasksFeedVersion)\build\Microsoft.DotNet.Build.Tasks.Feed.targets" />
 
   <Target Name="PublishToFeed">
     <Error Condition="'$(TargetStaticFeed)' == ''" Text="TargetStaticFeed: Target feed for publishing assets wasn't provided." />
     <Error Condition="'$(AccountKeyToStaticFeed)' == ''" Text="AccountKeyToStaticFeed: Account key for target feed wasn't provided." />
-    <Error Condition="'$(FullPathAssetManifest)' == ''" Text="Full path to asset manifest wasn't provided." />
+    <Error Condition="'$(ManifestsBasePath)' == ''" Text="Full path to asset manifests directory wasn't provided." />
-    <Error Condition="'$(FullPathBlobBasePath)' == '' AND '$(FullPathPackageBasePath)' == ''" Text="A valid full path to BlobBasePath of PackageBasePath is required." />
+    <Error Condition="'$(BlobBasePath)' == '' AND '$(PackageBasePath)' == ''" Text="A valid full path to BlobBasePath of PackageBasePath is required." />
 
+    <ItemGroup>
+      <!-- Include all manifests found in the manifest folder. -->
+      <ManifestFiles Include="$(ManifestsBasePath)*.xml" />
+    </ItemGroup>
+
+    <Error Condition="'@(ManifestFiles)' == ''" Text="No manifest file was found in the provided path: $(ManifestsBasePath)" />
+	
+    <!-- Iterate publishing assets from each manifest file. -->
     <PushArtifactsInManifestToFeed
       ExpectedFeedUrl="$(TargetStaticFeed)"
       AccountKey="$(AccountKeyToStaticFeed)"
@@ -26,9 +36,9 @@
       PassIfExistingItemIdentical="$(PassIfExistingItemIdentical)"
       MaxClients="$(MaxParallelUploads)"
       UploadTimeoutInMinutes="$(MaxUploadTimeoutInMinutes)"
-      AssetManifestPath="$(FullPathAssetManifest)"
+      AssetManifestPath="%(ManifestFiles.Identity)"
-      BlobAssetsBasePath="$(FullPathBlobBasePath)"
+      BlobAssetsBasePath="$(BlobBasePath)"
-      PackageAssetsBasePath="$(FullPathPackageBasePath)" />
+      PackageAssetsBasePath="$(PackageBasePath)" />
   </Target>
 
   <ItemGroup>

eng/common/templates/steps/telemetry-end.yml

@@ -1,3 +1,7 @@
+parameters:
+  maxRetries: 5
+  retryDelay: 10 # in seconds
+
 steps:
 - bash: |
     if [ "$AGENT_JOBSTATUS" = "Succeeded" ] || [ "$AGENT_JOBSTATUS" = "PartiallySucceeded" ]; then
@@ -7,27 +11,41 @@ steps:
     fi
     warningCount=0
 
-    # create a temporary file for curl output
+    curlStatus=1
-    res=`mktemp`
+    retryCount=0
-
+    # retry loop to harden against spotty telemetry connections
-    curlResult=`
+    # we don't retry successes and 4xx client errors
-      curl --verbose --output $res --write-out "%{http_code}"\
+    until [[ $curlStatus -eq 0 || ( $curlStatus -ge 400 && $curlStatus -le 499 ) || $retryCount -ge $MaxRetries ]]
-      -H 'Content-Type: application/json' \
+    do
-      -H "X-Helix-Job-Token: $Helix_JobToken" \
+      if [ $retryCount -gt 0 ]; then
-      -H 'Content-Length: 0' \
+        echo "Failed to send telemetry to Helix; waiting $RetryDelay seconds before retrying..."
-      -X POST -G "https://helix.dot.net/api/2018-03-14/telemetry/job/build/$Helix_WorkItemId/finish" \
+        sleep $RetryDelay
-      --data-urlencode "errorCount=$errorCount" \
-      --data-urlencode "warningCount=$warningCount"`
-    curlStatus=$?
-
-    if [ $curlStatus -eq 0 ]; then
-      if [ $curlResult -gt 299 ] || [ $curlResult -lt 200 ]; then
-        curlStatus=$curlResult
       fi
-    fi
+
+      # create a temporary file for curl output
+      res=`mktemp`
+
+      curlResult=`
+        curl --verbose --output $res --write-out "%{http_code}"\
+        -H 'Content-Type: application/json' \
+        -H "X-Helix-Job-Token: $Helix_JobToken" \
+        -H 'Content-Length: 0' \
+        -X POST -G "https://helix.dot.net/api/2018-03-14/telemetry/job/build/$Helix_WorkItemId/finish" \
+        --data-urlencode "errorCount=$errorCount" \
+        --data-urlencode "warningCount=$warningCount"`
+      curlStatus=$?
+
+      if [ $curlStatus -eq 0 ]; then
+        if [ $curlResult -gt 299 ] || [ $curlResult -lt 200 ]; then
+          curlStatus=$curlResult
+        fi
+      fi
+
+      let retryCount++
+    done
 
     if [ $curlStatus -ne 0 ]; then
-      echo "Failed to Send Build Finish information"
+      echo "Failed to Send Build Finish information after $retryCount retries"
       vstsLogOutput="vso[task.logissue type=error;sourcepath=templates/steps/telemetry-end.yml;code=1;]Failed to Send Build Finish information: $curlStatus"
       echo "##$vstsLogOutput"
       exit 1
@@ -37,6 +55,8 @@ steps:
     # defined via VSTS variables in start-job.sh
     Helix_JobToken: $(Helix_JobToken)
     Helix_WorkItemId: $(Helix_WorkItemId)
+    MaxRetries: ${{ parameters.maxRetries }}
+    RetryDelay: ${{ parameters.retryDelay }}
   condition: and(always(), ne(variables['Agent.Os'], 'Windows_NT'))
 - powershell: |
     if (($env:Agent_JobStatus -eq 'Succeeded') -or ($env:Agent_JobStatus -eq 'PartiallySucceeded')) {
@@ -46,13 +66,30 @@ steps:
     }
     $WarningCount = 0
 
-    try {
+    # Basic retry loop to harden against server flakiness
-      Invoke-RestMethod -Uri "https://helix.dot.net/api/2018-03-14/telemetry/job/build/$env:Helix_WorkItemId/finish?errorCount=$ErrorCount&warningCount=$WarningCount" -Method Post -ContentType "application/json" -Body "" `
+    $retryCount = 0
-        -Headers @{ 'X-Helix-Job-Token'=$env:Helix_JobToken }
+    while ($retryCount -lt $env:MaxRetries) {
+      try {
+        Invoke-RestMethod -Uri "https://helix.dot.net/api/2018-03-14/telemetry/job/build/$env:Helix_WorkItemId/finish?errorCount=$ErrorCount&warningCount=$WarningCount" -Method Post -ContentType "application/json" -Body "" `
+          -Headers @{ 'X-Helix-Job-Token'=$env:Helix_JobToken }
+        break
+      }
+      catch {
+        $statusCode = $_.Exception.Response.StatusCode.value__
+        if ($statusCode -ge 400 -and $statusCode -le 499) {
+          Write-Host "##vso[task.logissue]error Failed to send telemetry to Helix (status code $statusCode); not retrying (4xx client error)"
+          Write-Host "##vso[task.logissue]error ", $_.Exception.GetType().FullName, $_.Exception.Message
+          exit 1
+        }
+        Write-Host "Failed to send telemetry to Helix (status code $statusCode); waiting $env:RetryDelay seconds before retrying..."
+        $retryCount++
+        sleep $env:RetryDelay
+        continue
+      }
     }
-    catch {
+
-      Write-Error $_
+    if ($retryCount -ge $env:MaxRetries) {
-      Write-Error $_.Exception
+      Write-Host "##vso[task.logissue]error Failed to send telemetry to Helix after $retryCount retries."
       exit 1
     }
   displayName: Send Windows Build End Telemetry
@@ -60,4 +97,6 @@ steps:
     # defined via VSTS variables in start-job.ps1
     Helix_JobToken: $(Helix_JobToken)
     Helix_WorkItemId: $(Helix_WorkItemId)
+    MaxRetries: ${{ parameters.maxRetries }}
+    RetryDelay: ${{ parameters.retryDelay }}
   condition: and(always(),eq(variables['Agent.Os'], 'Windows_NT'))

eng/common/templates/steps/telemetry-start.yml

@@ -3,6 +3,8 @@ parameters:
   helixType: 'undefined_defaulted_in_telemetry.yml'
   buildConfig: ''
   runAsPublic: false
+  maxRetries: 5
+  retryDelay: 10 # in seconds
 
 steps:
 - ${{ if and(eq(parameters.runAsPublic, 'false'), not(eq(variables['System.TeamProject'], 'public'))) }}:
@@ -30,7 +32,7 @@ steps:
       }
     }
     JobListStuff
-    
+
     cat $jobInfo
 
     # create a temporary file for curl output
@@ -38,30 +40,44 @@ steps:
 
     accessTokenParameter="?access_token=$HelixApiAccessToken"
 
-    curlResult=`
+    curlStatus=1
-      cat $jobInfo |\
+    retryCount=0
-      curl --trace - --verbose --output $res --write-out "%{http_code}" \
+    # retry loop to harden against spotty telemetry connections
-      -H 'Content-Type: application/json' \
+    # we don't retry successes and 4xx client errors
-      -X POST "https://helix.dot.net/api/2018-03-14/telemetry/job$accessTokenParameter" -d @-`
+    until [[ $curlStatus -eq 0 || ( $curlStatus -ge 400 && $curlStatus -le 499 ) || $retryCount -ge $MaxRetries ]]
-    curlStatus=$?
+    do
-
+      if [ $retryCount -gt 0 ]; then
-    if [ $curlStatus -eq 0 ]; then
+        echo "Failed to send telemetry to Helix; waiting $RetryDelay seconds before retrying..."
-      if [ $curlResult -gt 299 ] || [ $curlResult -lt 200 ]; then
+        sleep $RetryDelay
-        curlStatus=$curlResult
       fi
-    fi
+
+      curlResult=`
+        cat $jobInfo |\
+        curl --trace - --verbose --output $res --write-out "%{http_code}" \
+        -H 'Content-Type: application/json' \
+        -X POST "https://helix.dot.net/api/2018-03-14/telemetry/job$accessTokenParameter" -d @-`
+      curlStatus=$?
+
+      if [ $curlStatus -eq 0 ]; then
+        if [ $curlResult -gt 299 ] || [ $curlResult -lt 200 ]; then
+          curlStatus=$curlResult
+        fi
+      fi
+
+      let retryCount++
+    done
 
     curlResult=`cat $res`
-    
+
     # validate status of curl command
     if [ $curlStatus -ne 0 ]; then
-      echo "Failed To Send Job Start information"
+      echo "Failed To Send Job Start information after $retryCount retries"
       # We have to append the ## vso prefix or vso will pick up the command when it dumps the inline script into the shell
       vstsLogOutput="vso[task.logissue type=error;sourcepath=telemetry/start-job.sh;code=1;]Failed to Send Job Start information: $curlStatus"
       echo "##$vstsLogOutput"
       exit 1
     fi
-    
+
     # Set the Helix_JobToken variable
     export Helix_JobToken=`echo $curlResult | xargs echo` # Strip Quotes
     echo "##vso[task.setvariable variable=Helix_JobToken;issecret=true;]$Helix_JobToken"
@@ -75,29 +91,44 @@ steps:
     Attempt: 1
     OperatingSystem: $(Agent.Os)
     Configuration: ${{ parameters.buildConfig }}
+    MaxRetries: ${{ parameters.maxRetries }}
+    RetryDelay: ${{ parameters.retryDelay }}
   condition: and(always(), ne(variables['Agent.Os'], 'Windows_NT'))
 - bash: |
-    res=`mktemp`
+    curlStatus=1
-    curlResult=`
+    retryCount=0
-      curl --verbose --output $res --write-out "%{http_code}"\
+    # retry loop to harden against spotty telemetry connections
-      -H 'Content-Type: application/json' \
+    # we don't retry successes and 4xx client errors
-      -H "X-Helix-Job-Token: $Helix_JobToken" \
+    until [[ $curlStatus -eq 0 || ( $curlStatus -ge 400 && $curlStatus -le 499 ) || $retryCount -ge $MaxRetries ]]
-      -H 'Content-Length: 0' \
+    do
-      -X POST -G "https://helix.dot.net/api/2018-03-14/telemetry/job/build" \
+      if [ $retryCount -gt 0 ]; then
-      --data-urlencode "buildUri=$BuildUri"`
+        echo "Failed to send telemetry to Helix; waiting $RetryDelay seconds before retrying..."
-    curlStatus=$?
+        sleep $RetryDelay
-
-    if [ $curlStatus -eq 0 ]; then
-      if [ $curlResult -gt 299 ] || [ $curlResult -lt 200 ]; then
-        curlStatus=$curlResult
       fi
-    fi
 
-    curlResult=`cat $res`
+      res=`mktemp`
+      curlResult=`
+        curl --verbose --output $res --write-out "%{http_code}"\
+        -H 'Content-Type: application/json' \
+        -H "X-Helix-Job-Token: $Helix_JobToken" \
+        -H 'Content-Length: 0' \
+        -X POST -G "https://helix.dot.net/api/2018-03-14/telemetry/job/build" \
+        --data-urlencode "buildUri=$BuildUri"`
+      curlStatus=$?
+
+      if [ $curlStatus -eq 0 ]; then
+        if [ $curlResult -gt 299 ] || [ $curlResult -lt 200 ]; then
+          curlStatus=$curlResult
+        fi
+      fi
+
+      curlResult=`cat $res`
+      let retryCount++
+    done
 
     # validate status of curl command
     if [ $curlStatus -ne 0 ]; then
-      echo "Failed to Send Build Start information"
+      echo "Failed to Send Build Start information after $retryCount retries"
       vstsLogOutput="vso[task.logissue type=error;sourcepath=telemetry/build/start.sh;code=1;]Failed to Send Build Start information: $curlStatus"
       echo "##$vstsLogOutput"
       exit 1
@@ -109,8 +140,10 @@ steps:
   env:
     BuildUri: $(System.TaskDefinitionsUri)$(System.TeamProject)/_build/index?buildId=$(Build.BuildId)&_a=summary
     Helix_JobToken: $(Helix_JobToken)
+    MaxRetries: ${{ parameters.maxRetries }}
+    RetryDelay: ${{ parameters.retryDelay }}
   condition: and(always(), ne(variables['Agent.Os'], 'Windows_NT'))
-  
+
 - powershell: |
     $jobInfo = [pscustomobject]@{
       QueueId=$env:QueueId;
@@ -120,17 +153,42 @@ steps:
       Attempt=$env:Attempt;
       Properties=[pscustomobject]@{ operatingSystem=$env:OperatingSystem; configuration=$env:Configuration };
     }
-    
+
     $jobInfoJson = $jobInfo | ConvertTo-Json
 
     if ($env:HelixApiAccessToken) {
       $accessTokenParameter="?access_token=$($env:HelixApiAccessToken)"
     }
     Write-Host "Job Info: $jobInfoJson"
-    $jobToken = Invoke-RestMethod -Uri "https://helix.dot.net/api/2018-03-14/telemetry/job$($accessTokenParameter)" -Method Post -ContentType "application/json" -Body $jobInfoJson
+
+    # Basic retry loop to harden against server flakiness
+    $retryCount = 0
+    while ($retryCount -lt $env:MaxRetries) {
+      try {
+        $jobToken = Invoke-RestMethod -Uri "https://helix.dot.net/api/2018-03-14/telemetry/job$($accessTokenParameter)" -Method Post -ContentType "application/json" -Body $jobInfoJson
+        break
+      }
+      catch {
+        $statusCode = $_.Exception.Response.StatusCode.value__
+        if ($statusCode -ge 400 -and $statusCode -le 499) {
+          Write-Host "##vso[task.logissue]error Failed to send telemetry to Helix (status code $statusCode); not retrying (4xx client error)"
+          Write-Host "##vso[task.logissue]error ", $_.Exception.GetType().FullName, $_.Exception.Message
+          exit 1
+        }
+        Write-Host "Failed to send telemetry to Helix (status code $statusCode); waiting $env:RetryDelay seconds before retrying..."
+        $retryCount++
+        sleep $env:RetryDelay
+        continue
+      }
+    }
+
+    if ($retryCount -ge $env:MaxRetries) {
+      Write-Host "##vso[task.logissue]error Failed to send telemetry to Helix after $retryCount retries."
+      exit 1
+    }
+
     $env:Helix_JobToken = $jobToken
     Write-Host "##vso[task.setvariable variable=Helix_JobToken;issecret=true;]$env:Helix_JobToken"
-  displayName: Send Windows Job Start Telemetry
   env:
     HelixApiAccessToken: $(HelixApiAccessToken)
     Source: ${{ parameters.helixSource }}
@@ -140,15 +198,43 @@ steps:
     Attempt: 1
     OperatingSystem: $(Agent.Os)
     Configuration: ${{ parameters.buildConfig }}
+    MaxRetries: ${{ parameters.maxRetries }}
+    RetryDelay: ${{ parameters.retryDelay }}
   condition: and(always(), eq(variables['Agent.Os'], 'Windows_NT'))
 - powershell: |
-    $workItemId = Invoke-RestMethod -Uri "https://helix.dot.net/api/2018-03-14/telemetry/job/build?buildUri=$([Net.WebUtility]::UrlEncode($env:BuildUri))" -Method Post -ContentType "application/json" -Body "" `
+    # Basic retry loop to harden against server flakiness
-      -Headers @{ 'X-Helix-Job-Token'=$env:Helix_JobToken }
+    $retryCount = 0
-  
+    while ($retryCount -lt $env:MaxRetries) {
+      try {
+        $workItemId = Invoke-RestMethod -Uri "https://helix.dot.net/api/2018-03-14/telemetry/job/build?buildUri=$([Net.WebUtility]::UrlEncode($env:BuildUri))" -Method Post -ContentType "application/json" -Body "" `
+          -Headers @{ 'X-Helix-Job-Token'=$env:Helix_JobToken }
+        break
+      }
+      catch {
+        $statusCode = $_.Exception.Response.StatusCode.value__
+        if ($statusCode -ge 400 -and $statusCode -le 499) {
+          Write-Host "##vso[task.logissue]error Failed to send telemetry to Helix (status code $statusCode); not retrying (4xx client error)"
+          Write-Host "##vso[task.logissue]error ", $_.Exception.GetType().FullName, $_.Exception.Message
+          exit 1
+        }
+        Write-Host "Failed to send telemetry to Helix (status code $statusCode); waiting $env:RetryDelay seconds before retrying..."
+        $retryCount++
+        sleep $env:RetryDelay
+        continue
+      }
+    }
+
+    if ($retryCount -ge $env:MaxRetries) {
+      Write-Host "##vso[task.logissue]error Failed to send telemetry to Helix after $retryCount retries."
+      exit 1
+    }
+
     $env:Helix_WorkItemId = $workItemId
     Write-Host "##vso[task.setvariable variable=Helix_WorkItemId]$env:Helix_WorkItemId"
   displayName: Send Windows Build Start Telemetry
   env:
     BuildUri: $(System.TaskDefinitionsUri)$(System.TeamProject)/_build/index?buildId=$(Build.BuildId)&_a=summary
     Helix_JobToken: $(Helix_JobToken)
+    MaxRetries: ${{ parameters.maxRetries }}
+    RetryDelay: ${{ parameters.retryDelay }}
   condition: and(always(), eq(variables['Agent.Os'], 'Windows_NT'))

global.json

@@ -3,6 +3,6 @@
     "dotnet": "3.0.100-preview-009812"
   },
   "msbuild-sdks": {
-    "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19081.3"
+    "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19101.16"
   }
 }