diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index b979b3903..7e8df41ba 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -28,9 +28,9 @@ - + https://github.com/dotnet/arcade - 1e859f1c17fffbe9c4fb6bbfc0fc71cd0c56563b + 159c3e6c65ab7b1e89e962290538364898d02cd8 diff --git a/eng/common/PublishToPackageFeed.proj b/eng/common/PublishToPackageFeed.proj index 7dc478d98..9db453894 100644 --- a/eng/common/PublishToPackageFeed.proj +++ b/eng/common/PublishToPackageFeed.proj @@ -10,15 +10,25 @@ netcoreapp2.1 - + + + - - + + + + + + + + + + + AssetManifestPath="%(ManifestFiles.Identity)" + BlobAssetsBasePath="$(BlobBasePath)" + PackageAssetsBasePath="$(PackageBasePath)" /> diff --git a/eng/common/templates/steps/telemetry-end.yml b/eng/common/templates/steps/telemetry-end.yml index 9b61481e7..fadc04ca1 100644 --- a/eng/common/templates/steps/telemetry-end.yml +++ b/eng/common/templates/steps/telemetry-end.yml @@ -1,3 +1,7 @@ +parameters: + maxRetries: 5 + retryDelay: 10 # in seconds + steps: - bash: | if [ "$AGENT_JOBSTATUS" = "Succeeded" ] || [ "$AGENT_JOBSTATUS" = "PartiallySucceeded" ]; then @@ -7,27 +11,41 @@ steps: fi warningCount=0 - # create a temporary file for curl output - res=`mktemp` - - curlResult=` - curl --verbose --output $res --write-out "%{http_code}"\ - -H 'Content-Type: application/json' \ - -H "X-Helix-Job-Token: $Helix_JobToken" \ - -H 'Content-Length: 0' \ - -X POST -G "https://helix.dot.net/api/2018-03-14/telemetry/job/build/$Helix_WorkItemId/finish" \ - --data-urlencode "errorCount=$errorCount" \ - --data-urlencode "warningCount=$warningCount"` - curlStatus=$? - - if [ $curlStatus -eq 0 ]; then - if [ $curlResult -gt 299 ] || [ $curlResult -lt 200 ]; then - curlStatus=$curlResult + curlStatus=1 + retryCount=0 + # retry loop to harden against spotty telemetry connections + # we don't retry successes and 4xx client errors + until [[ $curlStatus -eq 0 || ( $curlStatus -ge 400 && $curlStatus -le 499 ) || $retryCount -ge $MaxRetries ]] + do + if [ $retryCount -gt 0 ]; then + echo "Failed to send telemetry to Helix; waiting $RetryDelay seconds before retrying..." + sleep $RetryDelay fi - fi + + # create a temporary file for curl output + res=`mktemp` + + curlResult=` + curl --verbose --output $res --write-out "%{http_code}"\ + -H 'Content-Type: application/json' \ + -H "X-Helix-Job-Token: $Helix_JobToken" \ + -H 'Content-Length: 0' \ + -X POST -G "https://helix.dot.net/api/2018-03-14/telemetry/job/build/$Helix_WorkItemId/finish" \ + --data-urlencode "errorCount=$errorCount" \ + --data-urlencode "warningCount=$warningCount"` + curlStatus=$? + + if [ $curlStatus -eq 0 ]; then + if [ $curlResult -gt 299 ] || [ $curlResult -lt 200 ]; then + curlStatus=$curlResult + fi + fi + + let retryCount++ + done if [ $curlStatus -ne 0 ]; then - echo "Failed to Send Build Finish information" + echo "Failed to Send Build Finish information after $retryCount retries" vstsLogOutput="vso[task.logissue type=error;sourcepath=templates/steps/telemetry-end.yml;code=1;]Failed to Send Build Finish information: $curlStatus" echo "##$vstsLogOutput" exit 1 @@ -37,6 +55,8 @@ steps: # defined via VSTS variables in start-job.sh Helix_JobToken: $(Helix_JobToken) Helix_WorkItemId: $(Helix_WorkItemId) + MaxRetries: ${{ parameters.maxRetries }} + RetryDelay: ${{ parameters.retryDelay }} condition: and(always(), ne(variables['Agent.Os'], 'Windows_NT')) - powershell: | if (($env:Agent_JobStatus -eq 'Succeeded') -or ($env:Agent_JobStatus -eq 'PartiallySucceeded')) { @@ -46,13 +66,30 @@ steps: } $WarningCount = 0 - try { - Invoke-RestMethod -Uri "https://helix.dot.net/api/2018-03-14/telemetry/job/build/$env:Helix_WorkItemId/finish?errorCount=$ErrorCount&warningCount=$WarningCount" -Method Post -ContentType "application/json" -Body "" ` - -Headers @{ 'X-Helix-Job-Token'=$env:Helix_JobToken } + # Basic retry loop to harden against server flakiness + $retryCount = 0 + while ($retryCount -lt $env:MaxRetries) { + try { + Invoke-RestMethod -Uri "https://helix.dot.net/api/2018-03-14/telemetry/job/build/$env:Helix_WorkItemId/finish?errorCount=$ErrorCount&warningCount=$WarningCount" -Method Post -ContentType "application/json" -Body "" ` + -Headers @{ 'X-Helix-Job-Token'=$env:Helix_JobToken } + break + } + catch { + $statusCode = $_.Exception.Response.StatusCode.value__ + if ($statusCode -ge 400 -and $statusCode -le 499) { + Write-Host "##vso[task.logissue]error Failed to send telemetry to Helix (status code $statusCode); not retrying (4xx client error)" + Write-Host "##vso[task.logissue]error ", $_.Exception.GetType().FullName, $_.Exception.Message + exit 1 + } + Write-Host "Failed to send telemetry to Helix (status code $statusCode); waiting $env:RetryDelay seconds before retrying..." + $retryCount++ + sleep $env:RetryDelay + continue + } } - catch { - Write-Error $_ - Write-Error $_.Exception + + if ($retryCount -ge $env:MaxRetries) { + Write-Host "##vso[task.logissue]error Failed to send telemetry to Helix after $retryCount retries." exit 1 } displayName: Send Windows Build End Telemetry @@ -60,4 +97,6 @@ steps: # defined via VSTS variables in start-job.ps1 Helix_JobToken: $(Helix_JobToken) Helix_WorkItemId: $(Helix_WorkItemId) + MaxRetries: ${{ parameters.maxRetries }} + RetryDelay: ${{ parameters.retryDelay }} condition: and(always(),eq(variables['Agent.Os'], 'Windows_NT')) diff --git a/eng/common/templates/steps/telemetry-start.yml b/eng/common/templates/steps/telemetry-start.yml index 79c128c5d..f6a521be2 100644 --- a/eng/common/templates/steps/telemetry-start.yml +++ b/eng/common/templates/steps/telemetry-start.yml @@ -3,6 +3,8 @@ parameters: helixType: 'undefined_defaulted_in_telemetry.yml' buildConfig: '' runAsPublic: false + maxRetries: 5 + retryDelay: 10 # in seconds steps: - ${{ if and(eq(parameters.runAsPublic, 'false'), not(eq(variables['System.TeamProject'], 'public'))) }}: @@ -30,7 +32,7 @@ steps: } } JobListStuff - + cat $jobInfo # create a temporary file for curl output @@ -38,30 +40,44 @@ steps: accessTokenParameter="?access_token=$HelixApiAccessToken" - curlResult=` - cat $jobInfo |\ - curl --trace - --verbose --output $res --write-out "%{http_code}" \ - -H 'Content-Type: application/json' \ - -X POST "https://helix.dot.net/api/2018-03-14/telemetry/job$accessTokenParameter" -d @-` - curlStatus=$? - - if [ $curlStatus -eq 0 ]; then - if [ $curlResult -gt 299 ] || [ $curlResult -lt 200 ]; then - curlStatus=$curlResult + curlStatus=1 + retryCount=0 + # retry loop to harden against spotty telemetry connections + # we don't retry successes and 4xx client errors + until [[ $curlStatus -eq 0 || ( $curlStatus -ge 400 && $curlStatus -le 499 ) || $retryCount -ge $MaxRetries ]] + do + if [ $retryCount -gt 0 ]; then + echo "Failed to send telemetry to Helix; waiting $RetryDelay seconds before retrying..." + sleep $RetryDelay fi - fi + + curlResult=` + cat $jobInfo |\ + curl --trace - --verbose --output $res --write-out "%{http_code}" \ + -H 'Content-Type: application/json' \ + -X POST "https://helix.dot.net/api/2018-03-14/telemetry/job$accessTokenParameter" -d @-` + curlStatus=$? + + if [ $curlStatus -eq 0 ]; then + if [ $curlResult -gt 299 ] || [ $curlResult -lt 200 ]; then + curlStatus=$curlResult + fi + fi + + let retryCount++ + done curlResult=`cat $res` - + # validate status of curl command if [ $curlStatus -ne 0 ]; then - echo "Failed To Send Job Start information" + echo "Failed To Send Job Start information after $retryCount retries" # We have to append the ## vso prefix or vso will pick up the command when it dumps the inline script into the shell vstsLogOutput="vso[task.logissue type=error;sourcepath=telemetry/start-job.sh;code=1;]Failed to Send Job Start information: $curlStatus" echo "##$vstsLogOutput" exit 1 fi - + # Set the Helix_JobToken variable export Helix_JobToken=`echo $curlResult | xargs echo` # Strip Quotes echo "##vso[task.setvariable variable=Helix_JobToken;issecret=true;]$Helix_JobToken" @@ -75,29 +91,44 @@ steps: Attempt: 1 OperatingSystem: $(Agent.Os) Configuration: ${{ parameters.buildConfig }} + MaxRetries: ${{ parameters.maxRetries }} + RetryDelay: ${{ parameters.retryDelay }} condition: and(always(), ne(variables['Agent.Os'], 'Windows_NT')) - bash: | - res=`mktemp` - curlResult=` - curl --verbose --output $res --write-out "%{http_code}"\ - -H 'Content-Type: application/json' \ - -H "X-Helix-Job-Token: $Helix_JobToken" \ - -H 'Content-Length: 0' \ - -X POST -G "https://helix.dot.net/api/2018-03-14/telemetry/job/build" \ - --data-urlencode "buildUri=$BuildUri"` - curlStatus=$? - - if [ $curlStatus -eq 0 ]; then - if [ $curlResult -gt 299 ] || [ $curlResult -lt 200 ]; then - curlStatus=$curlResult + curlStatus=1 + retryCount=0 + # retry loop to harden against spotty telemetry connections + # we don't retry successes and 4xx client errors + until [[ $curlStatus -eq 0 || ( $curlStatus -ge 400 && $curlStatus -le 499 ) || $retryCount -ge $MaxRetries ]] + do + if [ $retryCount -gt 0 ]; then + echo "Failed to send telemetry to Helix; waiting $RetryDelay seconds before retrying..." + sleep $RetryDelay fi - fi - curlResult=`cat $res` + res=`mktemp` + curlResult=` + curl --verbose --output $res --write-out "%{http_code}"\ + -H 'Content-Type: application/json' \ + -H "X-Helix-Job-Token: $Helix_JobToken" \ + -H 'Content-Length: 0' \ + -X POST -G "https://helix.dot.net/api/2018-03-14/telemetry/job/build" \ + --data-urlencode "buildUri=$BuildUri"` + curlStatus=$? + + if [ $curlStatus -eq 0 ]; then + if [ $curlResult -gt 299 ] || [ $curlResult -lt 200 ]; then + curlStatus=$curlResult + fi + fi + + curlResult=`cat $res` + let retryCount++ + done # validate status of curl command if [ $curlStatus -ne 0 ]; then - echo "Failed to Send Build Start information" + echo "Failed to Send Build Start information after $retryCount retries" vstsLogOutput="vso[task.logissue type=error;sourcepath=telemetry/build/start.sh;code=1;]Failed to Send Build Start information: $curlStatus" echo "##$vstsLogOutput" exit 1 @@ -109,8 +140,10 @@ steps: env: BuildUri: $(System.TaskDefinitionsUri)$(System.TeamProject)/_build/index?buildId=$(Build.BuildId)&_a=summary Helix_JobToken: $(Helix_JobToken) + MaxRetries: ${{ parameters.maxRetries }} + RetryDelay: ${{ parameters.retryDelay }} condition: and(always(), ne(variables['Agent.Os'], 'Windows_NT')) - + - powershell: | $jobInfo = [pscustomobject]@{ QueueId=$env:QueueId; @@ -120,17 +153,42 @@ steps: Attempt=$env:Attempt; Properties=[pscustomobject]@{ operatingSystem=$env:OperatingSystem; configuration=$env:Configuration }; } - + $jobInfoJson = $jobInfo | ConvertTo-Json if ($env:HelixApiAccessToken) { $accessTokenParameter="?access_token=$($env:HelixApiAccessToken)" } Write-Host "Job Info: $jobInfoJson" - $jobToken = Invoke-RestMethod -Uri "https://helix.dot.net/api/2018-03-14/telemetry/job$($accessTokenParameter)" -Method Post -ContentType "application/json" -Body $jobInfoJson + + # Basic retry loop to harden against server flakiness + $retryCount = 0 + while ($retryCount -lt $env:MaxRetries) { + try { + $jobToken = Invoke-RestMethod -Uri "https://helix.dot.net/api/2018-03-14/telemetry/job$($accessTokenParameter)" -Method Post -ContentType "application/json" -Body $jobInfoJson + break + } + catch { + $statusCode = $_.Exception.Response.StatusCode.value__ + if ($statusCode -ge 400 -and $statusCode -le 499) { + Write-Host "##vso[task.logissue]error Failed to send telemetry to Helix (status code $statusCode); not retrying (4xx client error)" + Write-Host "##vso[task.logissue]error ", $_.Exception.GetType().FullName, $_.Exception.Message + exit 1 + } + Write-Host "Failed to send telemetry to Helix (status code $statusCode); waiting $env:RetryDelay seconds before retrying..." + $retryCount++ + sleep $env:RetryDelay + continue + } + } + + if ($retryCount -ge $env:MaxRetries) { + Write-Host "##vso[task.logissue]error Failed to send telemetry to Helix after $retryCount retries." + exit 1 + } + $env:Helix_JobToken = $jobToken Write-Host "##vso[task.setvariable variable=Helix_JobToken;issecret=true;]$env:Helix_JobToken" - displayName: Send Windows Job Start Telemetry env: HelixApiAccessToken: $(HelixApiAccessToken) Source: ${{ parameters.helixSource }} @@ -140,15 +198,43 @@ steps: Attempt: 1 OperatingSystem: $(Agent.Os) Configuration: ${{ parameters.buildConfig }} + MaxRetries: ${{ parameters.maxRetries }} + RetryDelay: ${{ parameters.retryDelay }} condition: and(always(), eq(variables['Agent.Os'], 'Windows_NT')) - powershell: | - $workItemId = Invoke-RestMethod -Uri "https://helix.dot.net/api/2018-03-14/telemetry/job/build?buildUri=$([Net.WebUtility]::UrlEncode($env:BuildUri))" -Method Post -ContentType "application/json" -Body "" ` - -Headers @{ 'X-Helix-Job-Token'=$env:Helix_JobToken } - + # Basic retry loop to harden against server flakiness + $retryCount = 0 + while ($retryCount -lt $env:MaxRetries) { + try { + $workItemId = Invoke-RestMethod -Uri "https://helix.dot.net/api/2018-03-14/telemetry/job/build?buildUri=$([Net.WebUtility]::UrlEncode($env:BuildUri))" -Method Post -ContentType "application/json" -Body "" ` + -Headers @{ 'X-Helix-Job-Token'=$env:Helix_JobToken } + break + } + catch { + $statusCode = $_.Exception.Response.StatusCode.value__ + if ($statusCode -ge 400 -and $statusCode -le 499) { + Write-Host "##vso[task.logissue]error Failed to send telemetry to Helix (status code $statusCode); not retrying (4xx client error)" + Write-Host "##vso[task.logissue]error ", $_.Exception.GetType().FullName, $_.Exception.Message + exit 1 + } + Write-Host "Failed to send telemetry to Helix (status code $statusCode); waiting $env:RetryDelay seconds before retrying..." + $retryCount++ + sleep $env:RetryDelay + continue + } + } + + if ($retryCount -ge $env:MaxRetries) { + Write-Host "##vso[task.logissue]error Failed to send telemetry to Helix after $retryCount retries." + exit 1 + } + $env:Helix_WorkItemId = $workItemId Write-Host "##vso[task.setvariable variable=Helix_WorkItemId]$env:Helix_WorkItemId" displayName: Send Windows Build Start Telemetry env: BuildUri: $(System.TaskDefinitionsUri)$(System.TeamProject)/_build/index?buildId=$(Build.BuildId)&_a=summary Helix_JobToken: $(Helix_JobToken) + MaxRetries: ${{ parameters.maxRetries }} + RetryDelay: ${{ parameters.retryDelay }} condition: and(always(), eq(variables['Agent.Os'], 'Windows_NT')) diff --git a/global.json b/global.json index a4f0a16e7..4db30d012 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "3.0.100-preview-009812" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19081.3" + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19101.16" } }