diff --git a/eng/build.yml b/eng/build.yml index 944d1e0ce..750111576 100644 --- a/eng/build.yml +++ b/eng/build.yml @@ -43,7 +43,7 @@ phases: - _SignType: test - _DOTNETCLIMSRC_READ_SAS_TOKEN: '' - - ${{ if and(eq(variables['System.TeamProject'], 'internal'), contains(variables['Build.SourceBranch'], 'internal')) }}: + - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - group: DotNet-MSRC-Storage - _DOTNETCLIMSRC_READ_SAS_TOKEN: $(dotnetclimsrc-read-sas-token) diff --git a/src/core-sdk-tasks/DownloadFile.cs b/src/core-sdk-tasks/DownloadFile.cs index 895a6749f..40178d1eb 100644 --- a/src/core-sdk-tasks/DownloadFile.cs +++ b/src/core-sdk-tasks/DownloadFile.cs @@ -3,23 +3,39 @@ using System; using System.IO; +using System.Net; using System.Net.Http; +using System.Threading.Tasks; +using System.Collections.Generic; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; namespace Microsoft.DotNet.Cli.Build { - public class DownloadFile : Task + public class DownloadFile : Microsoft.Build.Utilities.Task { [Required] public string Uri { get; set; } + /// + /// If this field is set and the task fail to download the file from `Uri`, with a NotFound + /// status, it will try to download the file from `PrivateUri`. + /// + public string PrivateUri { get; set; } + + public int MaxRetries { get; set; } = 5; + [Required] public string DestinationPath { get; set; } public bool Overwrite { get; set; } public override bool Execute() + { + return ExecuteAsync().GetAwaiter().GetResult(); + } + + private async System.Threading.Tasks.Task ExecuteAsync() { string destinationDir = Path.GetDirectoryName(DestinationPath); if (!Directory.Exists(destinationDir)) @@ -39,31 +55,84 @@ namespace Microsoft.DotNet.Cli.Build var filePath = Uri.Substring(FileUriProtocol.Length); Log.LogMessage($"Copying '{filePath}' to '{DestinationPath}'"); File.Copy(filePath, DestinationPath); + return true; } - else + + List errorMessages = new List(); + bool? downloadStatus = await DownloadWithRetriesAsync(Uri, DestinationPath, errorMessages); + + if (downloadStatus == false && !string.IsNullOrEmpty(PrivateUri)) { - Log.LogMessage(MessageImportance.High, $"Downloading '{Uri}' to '{DestinationPath}'"); + downloadStatus = await DownloadWithRetriesAsync(PrivateUri, DestinationPath, errorMessages); + } - using (var httpClient = new HttpClient()) + if (downloadStatus != true) + { + foreach (var error in errorMessages) { - var getTask = httpClient.GetStreamAsync(Uri); - - try - { - using (var outStream = File.Create(DestinationPath)) - { - getTask.Result.CopyTo(outStream); - } - } - catch (Exception) - { - File.Delete(DestinationPath); - throw; - } + Log.LogError(error); } } - return true; + return downloadStatus == true; + } + + /// + /// Attempt to download file from `source` with retries when response error is different of FileNotFound and Success. + /// + /// URL to the file to be downloaded. + /// Local path where to put the downloaded file. + /// true: Download Succeeded. false: Download failed with 404. null: Download failed but is retriable. + private async Task DownloadWithRetriesAsync(string source, string target, List errorMessages) + { + Random rng = new Random(); + + Log.LogMessage(MessageImportance.High, $"Attempting download '{source}' to '{target}'"); + + using (var httpClient = new HttpClient()) + { + for (int retryNumber = 0; retryNumber < MaxRetries; retryNumber++) + { + try + { + var httpResponse = await httpClient.GetAsync(source); + + Log.LogMessage(MessageImportance.High, $"{source} -> {httpResponse.StatusCode}"); + + // The Azure Storage REST API returns '400 - Bad Request' in some cases + // where the resource is not found on the storage. + // https://docs.microsoft.com/en-us/rest/api/storageservices/common-rest-api-error-codes + if (httpResponse.StatusCode == HttpStatusCode.NotFound || + httpResponse.ReasonPhrase.IndexOf("The requested URI does not represent any resource on the server.", StringComparison.OrdinalIgnoreCase) == 0) + { + errorMessages.Add($"Problems downloading file from '{source}'. Does the resource exist on the storage? {httpResponse.StatusCode} : {httpResponse.ReasonPhrase}"); + return false; + } + + httpResponse.EnsureSuccessStatusCode(); + + using (var outStream = File.Create(target)) + { + await httpResponse.Content.CopyToAsync(outStream); + } + + Log.LogMessage(MessageImportance.High, $"returning true {source} -> {httpResponse.StatusCode}"); + return true; + } + catch (Exception e) + { + Log.LogMessage(MessageImportance.High, $"returning error in {source} "); + errorMessages.Add($"Problems downloading file from '{source}'. {e.Message} {e.StackTrace}"); + File.Delete(target); + } + + await System.Threading.Tasks.Task.Delay(rng.Next(1000, 10000)); + } + } + + Log.LogMessage(MessageImportance.High, $"giving up {source} "); + errorMessages.Add($"Giving up downloading the file from '{source}' after {MaxRetries} retries."); + return null; } } } diff --git a/src/redist/targets/GenerateLayout.targets b/src/redist/targets/GenerateLayout.targets index e595067aa..8f93245f2 100644 --- a/src/redist/targets/GenerateLayout.targets +++ b/src/redist/targets/GenerateLayout.targets @@ -19,23 +19,18 @@ 3.0.0 3.0.0 - + $(RedistLayoutPath)sdk\$(SdkVersion)\ - - true - $(DOTNETCLIMSRC_READ_SAS_TOKEN) - https://dotnetclimsrc.blob.core.windows.net/dotnet/ + true + https://dotnetclimsrc.blob.core.windows.net/dotnet/ + https://dotnetcli.blob.core.windows.net/dotnet/ - - https://dotnetclimsrc.blob.core.windows.net/dotnet/ https://dotnetcli.blob.core.windows.net/dotnet/ - - https://dotnetclimsrc.blob.core.windows.net/dotnet/ https://dotnetfeed.blob.core.windows.net/dotnet-toolset/ - + $(HostRid) $(HostMonikerRid) @@ -104,7 +99,6 @@ $(CoreSetupRootUrl)$(CoreSetupBlobVersion) $(CombinedFrameworkHostCompressedFileName) - $(CoreSetupBlobAccessTokenParam) @@ -121,7 +115,7 @@ $(NetStandardTargetingPackTargetFramework) packs/%(PackageName)/%(PackageVersion) - + Microsoft.AspNetCore.App.Ref $(MicrosoftAspNetCoreAppRefPackageVersion) @@ -147,62 +141,53 @@ Condition="('$(IsDebianBaseDistro)' == 'true' OR '$(IsRPMBasedDistro)' == 'true') And '$(SkipBuildingInstallers)' != 'true' And '$(InstallerExtension)' != '' And !$(Architecture.StartsWith('arm'))"> $(CoreSetupRootUrl)$(CoreSetupBlobVersion) $(DownloadedRuntimeDepsInstallerFileName) - $(CoreSetupBlobAccessTokenParam) - + $(CoreSetupRootUrl)$(CoreSetupBlobVersion) $(DownloadedSharedFrameworkInstallerFileName) - $(CoreSetupBlobAccessTokenParam) - + $(CoreSetupRootUrl)$(CoreSetupBlobVersion) $(DownloadedSharedHostInstallerFileName) - $(CoreSetupBlobAccessTokenParam) - + $(CoreSetupRootUrl)$(CoreSetupBlobVersion) $(DownloadedHostFxrInstallerFileName) - $(CoreSetupBlobAccessTokenParam) - + $(CoreSetupRootUrl)$(NETCoreAppTargetingPackBlobVersion) $(DownloadedNetCoreAppTargetingPackInstallerFileName) - $(CoreSetupBlobAccessTokenParam) $(CoreSetupRootUrl)$(NETStandardTargetingPackBlobVersion) $(DownloadedNetStandardTargetingPackInstallerFileName) - $(CoreSetupBlobAccessTokenParam) $(CoreSetupRootUrl)$(CoreSetupBlobVersion) $(DownloadedNetCoreAppHostPackInstallerFileName) - $(CoreSetupBlobAccessTokenParam) - + $(CoreSetupRootUrl)$(WindowsDesktopTargetingPackBlobVersion) $(DownloadedWindowsDesktopTargetingPackInstallerFileName) - $(CoreSetupBlobAccessTokenParam) $(DotnetToolsetBlobRootUrl)Toolset/$(MicrosoftDotnetToolsetInternalPackageVersion) dotnet-toolset-internal-$(MicrosoftDotnetToolsetInternalPackageVersion).zip - $(CoreSetupBlobAccessTokenParam) sdk/$(SdkVersion) @@ -211,7 +196,6 @@ $(AspNetCoreSharedFxRootUrl)$(AspNetCoreBlobVersion) $(AspNetCoreSharedFxArchiveFileName) - $(CoreSetupBlobAccessTokenParam) @@ -221,38 +205,34 @@ Condition="'$(InstallerExtension)' == '.pkg' And '$(SkipBuildingInstallers)' != 'true' And '$(InstallerExtension)' != '' And !$(Architecture.StartsWith('arm'))"> $(AspNetCoreSharedFxRootUrl)$(AspNetCoreTargetingPackBlobVersion) $(AspNetTargetingPackArchiveFileName) - $(CoreSetupBlobAccessTokenParam) - + $(AspNetCoreSharedFxRootUrl)$(AspNetCoreTargetingPackBlobVersion) $(DownloadedAspNetTargetingPackInstallerFileName) - $(CoreSetupBlobAccessTokenParam) $(AspNetCoreSharedFxRootUrl)$(AspNetCoreBlobVersion) $(DownloadedAspNetCoreSharedFxInstallerFileName) - $(CoreSetupBlobAccessTokenParam) $(AspNetCoreSharedFxRootUrl)$(AspNetCoreBlobVersion) $(AspNetCoreSharedFxBaseRuntimeVersionFileName) - $(CoreSetupBlobAccessTokenParam) - + false true false - + Microsoft.WindowsDesktop.App.Ref @@ -264,14 +244,12 @@ $(WinFormsAndWpfSharedFxRootUrl)$(CoreSetupBlobVersion) $(WinFormsAndWpfSharedFxArchiveFileName) - $(CoreSetupBlobAccessTokenParam) $(WinFormsAndWpfSharedFxRootUrl)$(CoreSetupBlobVersion) $(DownloadedWinFormsAndWpfSharedFrameworkInstallerFileName) - $(CoreSetupBlobAccessTokenParam) @@ -287,11 +265,20 @@ true + + + %(BaseUrl) + $([System.String]::new('%(ComponentToDownload.PrivateBaseUrl)').Replace('$(CoreSetupBlobRootUrl)', '$(InternalBaseURL)')) + $([System.String]::new('%(ComponentToDownload.PrivateBaseUrl)').Replace('$(DotnetExtensionsBlobRootUrl)', '$(InternalBaseURL)')) + $([System.String]::new('%(ComponentToDownload.PrivateBaseUrl)').Replace('$(DotnetToolsetBlobRootUrl)', '$(InternalBaseURL)')) - + @@ -315,7 +302,7 @@ $([MSBuild]::ValueOrDefault('%(BundledLayoutPackage.PackageName)', '').ToLower()) - + %(BundledLayoutPackage.RelativeLayoutPath) %(BundledLayoutPackage.Identity) @@ -355,7 +342,7 @@ DestinationFiles="@(BundledLayoutPackageDownloadFilesWithDestination->'%(DestinationPath)')" SkipUnchangedFiles="true" /> - + @@ -372,24 +359,24 @@ ReplacementPatterns="$(ReplacementPattern)" ReplacementStrings="$(ReplacementString)" /> - + - + - + Microsoft.NETCore.App.Host.$(SharedFrameworkRid) $(IntermediateOutputPath)AppHostRestore\ $(SdkOutputDirectory)AppHostTemplate - + - + @@ -406,7 +393,7 @@ BuildInParallel="False" Projects="@(AppHostTemplateDownloadPackageProject)"> - + AppHost$(ExeExtension) @@ -427,9 +414,9 @@ - + - + - + - + - +