diff --git a/eng/build.yml b/eng/build.yml index 944d1e0ce..750111576 100644 --- a/eng/build.yml +++ b/eng/build.yml @@ -43,7 +43,7 @@ phases: - _SignType: test - _DOTNETCLIMSRC_READ_SAS_TOKEN: '' - - ${{ if and(eq(variables['System.TeamProject'], 'internal'), contains(variables['Build.SourceBranch'], 'internal')) }}: + - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - group: DotNet-MSRC-Storage - _DOTNETCLIMSRC_READ_SAS_TOKEN: $(dotnetclimsrc-read-sas-token) diff --git a/eng/dockerrun.sh b/eng/dockerrun.sh index a41b3c059..762e177a7 100755 --- a/eng/dockerrun.sh +++ b/eng/dockerrun.sh @@ -143,7 +143,7 @@ docker run $INTERACTIVE -t --rm --sig-proxy=true \ -e BUILD_BUILDNUMBER \ -e BUILD_SOURCEVERSION \ -e SYSTEM_TEAMPROJECT \ - -e DOTNECLIMSRC_READ_SAS_TOKEN \ + -e DOTNETCLIMSRC_READ_SAS_TOKEN \ -e AGENT_JOBNAME \ -e AGENT_OS \ -e VSS_NUGET_URI_PREFIXES \ diff --git a/src/core-sdk-tasks/DownloadFile.cs b/src/core-sdk-tasks/DownloadFile.cs index 895a6749f..40178d1eb 100644 --- a/src/core-sdk-tasks/DownloadFile.cs +++ b/src/core-sdk-tasks/DownloadFile.cs @@ -3,23 +3,39 @@ using System; using System.IO; +using System.Net; using System.Net.Http; +using System.Threading.Tasks; +using System.Collections.Generic; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; namespace Microsoft.DotNet.Cli.Build { - public class DownloadFile : Task + public class DownloadFile : Microsoft.Build.Utilities.Task { [Required] public string Uri { get; set; } + /// + /// If this field is set and the task fail to download the file from `Uri`, with a NotFound + /// status, it will try to download the file from `PrivateUri`. + /// + public string PrivateUri { get; set; } + + public int MaxRetries { get; set; } = 5; + [Required] public string DestinationPath { get; set; } public bool Overwrite { get; set; } public override bool Execute() + { + return ExecuteAsync().GetAwaiter().GetResult(); + } + + private async System.Threading.Tasks.Task ExecuteAsync() { string destinationDir = Path.GetDirectoryName(DestinationPath); if (!Directory.Exists(destinationDir)) @@ -39,31 +55,84 @@ namespace Microsoft.DotNet.Cli.Build var filePath = Uri.Substring(FileUriProtocol.Length); Log.LogMessage($"Copying '{filePath}' to '{DestinationPath}'"); File.Copy(filePath, DestinationPath); + return true; } - else + + List errorMessages = new List(); + bool? downloadStatus = await DownloadWithRetriesAsync(Uri, DestinationPath, errorMessages); + + if (downloadStatus == false && !string.IsNullOrEmpty(PrivateUri)) { - Log.LogMessage(MessageImportance.High, $"Downloading '{Uri}' to '{DestinationPath}'"); + downloadStatus = await DownloadWithRetriesAsync(PrivateUri, DestinationPath, errorMessages); + } - using (var httpClient = new HttpClient()) + if (downloadStatus != true) + { + foreach (var error in errorMessages) { - var getTask = httpClient.GetStreamAsync(Uri); - - try - { - using (var outStream = File.Create(DestinationPath)) - { - getTask.Result.CopyTo(outStream); - } - } - catch (Exception) - { - File.Delete(DestinationPath); - throw; - } + Log.LogError(error); } } - return true; + return downloadStatus == true; + } + + /// + /// Attempt to download file from `source` with retries when response error is different of FileNotFound and Success. + /// + /// URL to the file to be downloaded. + /// Local path where to put the downloaded file. + /// true: Download Succeeded. false: Download failed with 404. null: Download failed but is retriable. + private async Task DownloadWithRetriesAsync(string source, string target, List errorMessages) + { + Random rng = new Random(); + + Log.LogMessage(MessageImportance.High, $"Attempting download '{source}' to '{target}'"); + + using (var httpClient = new HttpClient()) + { + for (int retryNumber = 0; retryNumber < MaxRetries; retryNumber++) + { + try + { + var httpResponse = await httpClient.GetAsync(source); + + Log.LogMessage(MessageImportance.High, $"{source} -> {httpResponse.StatusCode}"); + + // The Azure Storage REST API returns '400 - Bad Request' in some cases + // where the resource is not found on the storage. + // https://docs.microsoft.com/en-us/rest/api/storageservices/common-rest-api-error-codes + if (httpResponse.StatusCode == HttpStatusCode.NotFound || + httpResponse.ReasonPhrase.IndexOf("The requested URI does not represent any resource on the server.", StringComparison.OrdinalIgnoreCase) == 0) + { + errorMessages.Add($"Problems downloading file from '{source}'. Does the resource exist on the storage? {httpResponse.StatusCode} : {httpResponse.ReasonPhrase}"); + return false; + } + + httpResponse.EnsureSuccessStatusCode(); + + using (var outStream = File.Create(target)) + { + await httpResponse.Content.CopyToAsync(outStream); + } + + Log.LogMessage(MessageImportance.High, $"returning true {source} -> {httpResponse.StatusCode}"); + return true; + } + catch (Exception e) + { + Log.LogMessage(MessageImportance.High, $"returning error in {source} "); + errorMessages.Add($"Problems downloading file from '{source}'. {e.Message} {e.StackTrace}"); + File.Delete(target); + } + + await System.Threading.Tasks.Task.Delay(rng.Next(1000, 10000)); + } + } + + Log.LogMessage(MessageImportance.High, $"giving up {source} "); + errorMessages.Add($"Giving up downloading the file from '{source}' after {MaxRetries} retries."); + return null; } } } diff --git a/src/redist/targets/GenerateLayout.targets b/src/redist/targets/GenerateLayout.targets index ec327aedd..1970faf41 100644 --- a/src/redist/targets/GenerateLayout.targets +++ b/src/redist/targets/GenerateLayout.targets @@ -20,21 +20,16 @@ 3.1.0 3.0.0 - + $(RedistLayoutPath)sdk\$(SdkVersion)\ - - true - $(DOTNETCLIMSRC_READ_SAS_TOKEN) - https://dotnetclimsrc.blob.core.windows.net/dotnet/ + true + https://dotnetclimsrc.blob.core.windows.net/dotnet/ + https://dotnetcli.blob.core.windows.net/dotnet/ - - https://dotnetclimsrc.blob.core.windows.net/dotnet/ https://dotnetcli.blob.core.windows.net/dotnet/ - - https://dotnetclimsrc.blob.core.windows.net/dotnet/ https://dotnetfeed.blob.core.windows.net/dotnet-toolset/ $(HostRid) @@ -112,7 +107,6 @@ $(CoreSetupRootUrl)$(CoreSetupBlobVersion) $(CombinedFrameworkHostArchiveFileName) - $(CoreSetupBlobAccessTokenParam) @@ -169,35 +163,30 @@ Condition="('$(IsDebianBaseDistro)' == 'true' OR '$(IsRPMBasedDistro)' == 'true') And '$(SkipBuildingInstallers)' != 'true' And '$(InstallerExtension)' != '' And !$(Architecture.StartsWith('arm'))"> $(CoreSetupRootUrl)$(CoreSetupBlobVersion) $(DownloadedRuntimeDepsInstallerFileName) - $(CoreSetupBlobAccessTokenParam) $(CoreSetupRootUrl)$(CoreSetupBlobVersion) $(DownloadedSharedFrameworkInstallerFileName) - $(CoreSetupBlobAccessTokenParam) $(CoreSetupRootUrl)$(CoreSetupBlobVersion) $(DownloadedSharedHostInstallerFileName) - $(CoreSetupBlobAccessTokenParam) $(CoreSetupRootUrl)$(CoreSetupBlobVersion) $(DownloadedHostFxrInstallerFileName) - $(CoreSetupBlobAccessTokenParam) $(CoreSetupRootUrl)$(NETCoreAppTargetingPackBlobVersion) $(DownloadedNetCoreAppTargetingPackInstallerFileName) - $(CoreSetupBlobAccessTokenParam) $(CoreSetupRootUrl)$(MicrosoftNETCoreAppRuntimewinx64PackageVersion) $(CoreSetupRootUrl)3.0.0 $(DownloadedNetStandardTargetingPackInstallerFileName) - $(CoreSetupBlobAccessTokenParam) $(CoreSetupRootUrl)$(CoreSetupBlobVersion) $(DownloadedNetCoreAppHostPackInstallerFileName) - $(CoreSetupBlobAccessTokenParam) $(CoreSetupRootUrl)$(CoreSetupBlobVersion) $(DownloadedAlternateNetCoreAppHostPackInstallerFileName) - $(CoreSetupBlobAccessTokenParam) $(CoreSetupRootUrl)$(CoreSetupBlobVersion) $(DownloadedArmNetCoreAppHostPackInstallerFileName) - $(CoreSetupBlobAccessTokenParam) $(CoreSetupRootUrl)$(CoreSetupBlobVersion) $(DownloadedArm64NetCoreAppHostPackInstallerFileName) - $(CoreSetupBlobAccessTokenParam) $(CoreSetupRootUrl)$(WindowsDesktopTargetingPackBlobVersion) $(DownloadedWindowsDesktopTargetingPackInstallerFileName) - $(CoreSetupBlobAccessTokenParam) $(DotnetToolsetBlobRootUrl)Toolset/$(MicrosoftDotnetToolsetInternalPackageVersion) dotnet-toolset-internal-$(MicrosoftDotnetToolsetInternalPackageVersion).zip - $(CoreSetupBlobAccessTokenParam) sdk/$(SdkVersion) @@ -255,7 +237,6 @@ $(AspNetCoreSharedFxRootUrl)$(AspNetCoreBlobVersion) $(AspNetCoreSharedFxArchiveFileName) - $(CoreSetupBlobAccessTokenParam) @@ -265,7 +246,6 @@ Condition="'$(InstallerExtension)' == '.pkg' And '$(SkipBuildingInstallers)' != 'true' And '$(InstallerExtension)' != '' And !$(Architecture.StartsWith('arm'))"> $(AspNetCoreSharedFxRootUrl)$(AspNetCoreTargetingPackBlobVersion) $(AspNetTargetingPackArchiveFileName) - $(CoreSetupBlobAccessTokenParam) @@ -273,14 +253,12 @@ Condition="'$(InstallerExtension)' != '.pkg' And '$(SkipBuildingInstallers)' != 'true' And '$(InstallerExtension)' != '' And !$(Architecture.StartsWith('arm'))"> $(AspNetCoreSharedFxRootUrl)$(AspNetCoreTargetingPackBlobVersion) $(DownloadedAspNetTargetingPackInstallerFileName) - $(CoreSetupBlobAccessTokenParam) $(AspNetCoreSharedFxRootUrl)$(AspNetCoreBlobVersion) $(DownloadedAspNetCoreSharedFxInstallerFileName) - $(CoreSetupBlobAccessTokenParam) $(AspNetCoreSharedFxRootUrl)$(AspNetCoreBlobVersion) $(AspNetCoreSharedFxBaseRuntimeVersionFileName) - $(CoreSetupBlobAccessTokenParam) @@ -322,14 +299,12 @@ $(WinFormsAndWpfSharedFxRootUrl)$(CoreSetupBlobVersion) $(WinFormsAndWpfSharedFxArchiveFileName) - $(CoreSetupBlobAccessTokenParam) $(WinFormsAndWpfSharedFxRootUrl)$(CoreSetupBlobVersion) $(DownloadedWinFormsAndWpfSharedFrameworkInstallerFileName) - $(CoreSetupBlobAccessTokenParam) @@ -345,11 +320,20 @@ true + + + %(BaseUrl) + $([System.String]::new('%(ComponentToDownload.PrivateBaseUrl)').Replace('$(CoreSetupBlobRootUrl)', '$(InternalBaseURL)')) + $([System.String]::new('%(ComponentToDownload.PrivateBaseUrl)').Replace('$(DotnetExtensionsBlobRootUrl)', '$(InternalBaseURL)')) + $([System.String]::new('%(ComponentToDownload.PrivateBaseUrl)').Replace('$(DotnetToolsetBlobRootUrl)', '$(InternalBaseURL)')) @@ -373,7 +357,7 @@ $([MSBuild]::ValueOrDefault('%(BundledLayoutPackage.PackageName)', '').ToLower()) - + %(BundledLayoutPackage.RelativeLayoutPath) %(BundledLayoutPackage.Identity) @@ -438,7 +422,7 @@ - + Microsoft.NETCore.App.Host.$(SharedFrameworkRid) $(IntermediateOutputPath)AppHostRestore\ @@ -464,7 +448,7 @@ BuildInParallel="False" Projects="@(AppHostTemplateDownloadPackageProject)"> - + AppHost$(ExeExtension) @@ -511,7 +495,7 @@ - + diff --git a/src/redist/targets/Versions.targets b/src/redist/targets/Versions.targets index a83319ba0..a46acac70 100644 --- a/src/redist/targets/Versions.targets +++ b/src/redist/targets/Versions.targets @@ -3,7 +3,7 @@ 3 1 1 - 01 + 02 servicing