diff --git a/NuGet.config b/NuGet.config index d8f4a4717..56e3557f0 100644 --- a/NuGet.config +++ b/NuGet.config @@ -12,6 +12,7 @@ + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 1b39be1d0..bbfbed12d 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -67,9 +67,9 @@ https://github.com/aspnet/AspNetCore 2bbf66ad44de54898f6cfe210bcbe079e62fbfc9 - + https://github.com/dotnet/test-templates - 6ae8a6e7dd862c9447771c108d43fb9028f22c22 + 5117d3d9d7dfd3583f0c42a639efaf092f957b7b https://github.com/dotnet/templating diff --git a/eng/build.yml b/eng/build.yml index 944d1e0ce..750111576 100644 --- a/eng/build.yml +++ b/eng/build.yml @@ -43,7 +43,7 @@ phases: - _SignType: test - _DOTNETCLIMSRC_READ_SAS_TOKEN: '' - - ${{ if and(eq(variables['System.TeamProject'], 'internal'), contains(variables['Build.SourceBranch'], 'internal')) }}: + - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - group: DotNet-MSRC-Storage - _DOTNETCLIMSRC_READ_SAS_TOKEN: $(dotnetclimsrc-read-sas-token) diff --git a/eng/common/SetupNugetSources.ps1 b/eng/common/SetupNugetSources.ps1 index a5a1e711d..c3c473eb8 100644 --- a/eng/common/SetupNugetSources.ps1 +++ b/eng/common/SetupNugetSources.ps1 @@ -83,7 +83,7 @@ function AddCredential($creds, $source, $username, $password) { $passwordElement.SetAttribute("value", $Password) } -function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Password) { +function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $Password) { $maestroPrivateSources = $Sources.SelectNodes("add[contains(@key,'darc-int')]") Write-Host "Inserting credentials for $($maestroPrivateSources.Count) Maestro's private feeds." @@ -123,19 +123,21 @@ if ($creds -eq $null) { $doc.DocumentElement.AppendChild($creds) | Out-Null } +$userName = "dn-bot" + # Insert credential nodes for Maestro's private feeds -InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Password $Password +InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -Password $Password $dotnet3Source = $sources.SelectSingleNode("add[@key='dotnet3']") if ($dotnet3Source -ne $null) { - AddPackageSource -Sources $sources -SourceName "dotnet3-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3-internal/nuget/v2" -Creds $creds -Username "dn-bot" -Password $Password - AddPackageSource -Sources $sources -SourceName "dotnet3-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3-internal-transport/nuget/v2" -Creds $creds -Username "dn-bot" -Password $Password + AddPackageSource -Sources $sources -SourceName "dotnet3-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password + AddPackageSource -Sources $sources -SourceName "dotnet3-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password } $dotnet31Source = $sources.SelectSingleNode("add[@key='dotnet3.1']") if ($dotnet31Source -ne $null) { - AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username "dn-bot" -Password $Password - AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username "dn-bot" -Password $Password + AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password + AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password } $doc.Save($filename) diff --git a/eng/common/templates/post-build/channels/generic-internal-channel.yml b/eng/common/templates/post-build/channels/generic-internal-channel.yml index 3a8755fbb..6a4ab7f53 100644 --- a/eng/common/templates/post-build/channels/generic-internal-channel.yml +++ b/eng/common/templates/post-build/channels/generic-internal-channel.yml @@ -29,6 +29,10 @@ stages: pool: vmImage: 'windows-2019' steps: + # This is necessary whenever we want to publish/restore to an AzDO private feed + - task: NuGetAuthenticate@0 + displayName: 'Authenticate to AzDO Feeds' + - task: DownloadBuildArtifacts@0 displayName: Download Blob Artifacts inputs: diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index 8a8d84f20..70a5b6f8a 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -54,6 +54,12 @@ stages: pool: vmImage: 'windows-2019' steps: + # This is necessary whenever we want to publish/restore to an AzDO private feed + # Since sdk-task.ps1 tries to restore packages we need to do this authentication here + # otherwise it'll complain about accessing a private feed. + - task: NuGetAuthenticate@0 + displayName: 'Authenticate to AzDO Feeds' + - task: DownloadBuildArtifacts@0 displayName: Download Package Artifacts inputs: diff --git a/eng/dockerrun.sh b/eng/dockerrun.sh index a41b3c059..762e177a7 100755 --- a/eng/dockerrun.sh +++ b/eng/dockerrun.sh @@ -143,7 +143,7 @@ docker run $INTERACTIVE -t --rm --sig-proxy=true \ -e BUILD_BUILDNUMBER \ -e BUILD_SOURCEVERSION \ -e SYSTEM_TEAMPROJECT \ - -e DOTNECLIMSRC_READ_SAS_TOKEN \ + -e DOTNETCLIMSRC_READ_SAS_TOKEN \ -e AGENT_JOBNAME \ -e AGENT_OS \ -e VSS_NUGET_URI_PREFIXES \ diff --git a/src/core-sdk-tasks/DownloadFile.cs b/src/core-sdk-tasks/DownloadFile.cs index 895a6749f..40178d1eb 100644 --- a/src/core-sdk-tasks/DownloadFile.cs +++ b/src/core-sdk-tasks/DownloadFile.cs @@ -3,23 +3,39 @@ using System; using System.IO; +using System.Net; using System.Net.Http; +using System.Threading.Tasks; +using System.Collections.Generic; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; namespace Microsoft.DotNet.Cli.Build { - public class DownloadFile : Task + public class DownloadFile : Microsoft.Build.Utilities.Task { [Required] public string Uri { get; set; } + /// + /// If this field is set and the task fail to download the file from `Uri`, with a NotFound + /// status, it will try to download the file from `PrivateUri`. + /// + public string PrivateUri { get; set; } + + public int MaxRetries { get; set; } = 5; + [Required] public string DestinationPath { get; set; } public bool Overwrite { get; set; } public override bool Execute() + { + return ExecuteAsync().GetAwaiter().GetResult(); + } + + private async System.Threading.Tasks.Task ExecuteAsync() { string destinationDir = Path.GetDirectoryName(DestinationPath); if (!Directory.Exists(destinationDir)) @@ -39,31 +55,84 @@ namespace Microsoft.DotNet.Cli.Build var filePath = Uri.Substring(FileUriProtocol.Length); Log.LogMessage($"Copying '{filePath}' to '{DestinationPath}'"); File.Copy(filePath, DestinationPath); + return true; } - else + + List errorMessages = new List(); + bool? downloadStatus = await DownloadWithRetriesAsync(Uri, DestinationPath, errorMessages); + + if (downloadStatus == false && !string.IsNullOrEmpty(PrivateUri)) { - Log.LogMessage(MessageImportance.High, $"Downloading '{Uri}' to '{DestinationPath}'"); + downloadStatus = await DownloadWithRetriesAsync(PrivateUri, DestinationPath, errorMessages); + } - using (var httpClient = new HttpClient()) + if (downloadStatus != true) + { + foreach (var error in errorMessages) { - var getTask = httpClient.GetStreamAsync(Uri); - - try - { - using (var outStream = File.Create(DestinationPath)) - { - getTask.Result.CopyTo(outStream); - } - } - catch (Exception) - { - File.Delete(DestinationPath); - throw; - } + Log.LogError(error); } } - return true; + return downloadStatus == true; + } + + /// + /// Attempt to download file from `source` with retries when response error is different of FileNotFound and Success. + /// + /// URL to the file to be downloaded. + /// Local path where to put the downloaded file. + /// true: Download Succeeded. false: Download failed with 404. null: Download failed but is retriable. + private async Task DownloadWithRetriesAsync(string source, string target, List errorMessages) + { + Random rng = new Random(); + + Log.LogMessage(MessageImportance.High, $"Attempting download '{source}' to '{target}'"); + + using (var httpClient = new HttpClient()) + { + for (int retryNumber = 0; retryNumber < MaxRetries; retryNumber++) + { + try + { + var httpResponse = await httpClient.GetAsync(source); + + Log.LogMessage(MessageImportance.High, $"{source} -> {httpResponse.StatusCode}"); + + // The Azure Storage REST API returns '400 - Bad Request' in some cases + // where the resource is not found on the storage. + // https://docs.microsoft.com/en-us/rest/api/storageservices/common-rest-api-error-codes + if (httpResponse.StatusCode == HttpStatusCode.NotFound || + httpResponse.ReasonPhrase.IndexOf("The requested URI does not represent any resource on the server.", StringComparison.OrdinalIgnoreCase) == 0) + { + errorMessages.Add($"Problems downloading file from '{source}'. Does the resource exist on the storage? {httpResponse.StatusCode} : {httpResponse.ReasonPhrase}"); + return false; + } + + httpResponse.EnsureSuccessStatusCode(); + + using (var outStream = File.Create(target)) + { + await httpResponse.Content.CopyToAsync(outStream); + } + + Log.LogMessage(MessageImportance.High, $"returning true {source} -> {httpResponse.StatusCode}"); + return true; + } + catch (Exception e) + { + Log.LogMessage(MessageImportance.High, $"returning error in {source} "); + errorMessages.Add($"Problems downloading file from '{source}'. {e.Message} {e.StackTrace}"); + File.Delete(target); + } + + await System.Threading.Tasks.Task.Delay(rng.Next(1000, 10000)); + } + } + + Log.LogMessage(MessageImportance.High, $"giving up {source} "); + errorMessages.Add($"Giving up downloading the file from '{source}' after {MaxRetries} retries."); + return null; } } } diff --git a/src/redist/targets/GenerateLayout.targets b/src/redist/targets/GenerateLayout.targets index b074760f2..d0e6b7bcf 100644 --- a/src/redist/targets/GenerateLayout.targets +++ b/src/redist/targets/GenerateLayout.targets @@ -21,21 +21,16 @@ $(WindowsDesktopBlobVersion) 3.0.0 - + $(RedistLayoutPath)sdk\$(SdkVersion)\ - - true - $(DOTNETCLIMSRC_READ_SAS_TOKEN) - https://dotnetclimsrc.blob.core.windows.net/dotnet/ + true + https://dotnetclimsrc.blob.core.windows.net/dotnet/ + https://dotnetcli.blob.core.windows.net/dotnet/ - - https://dotnetclimsrc.blob.core.windows.net/dotnet/ https://dotnetcli.blob.core.windows.net/dotnet/ - - https://dotnetclimsrc.blob.core.windows.net/dotnet/ https://dotnetfeed.blob.core.windows.net/dotnet-toolset/ $(HostRid) @@ -113,7 +108,6 @@ $(CoreSetupRootUrl)$(CoreSetupBlobVersion) $(CombinedFrameworkHostArchiveFileName) - $(CoreSetupBlobAccessTokenParam) @@ -170,35 +164,30 @@ Condition="('$(IsDebianBaseDistro)' == 'true' OR '$(IsRPMBasedDistro)' == 'true') And '$(SkipBuildingInstallers)' != 'true' And '$(InstallerExtension)' != '' And !$(Architecture.StartsWith('arm'))"> $(CoreSetupRootUrl)$(CoreSetupBlobVersion) $(DownloadedRuntimeDepsInstallerFileName) - $(CoreSetupBlobAccessTokenParam) $(CoreSetupRootUrl)$(CoreSetupBlobVersion) $(DownloadedSharedFrameworkInstallerFileName) - $(CoreSetupBlobAccessTokenParam) $(CoreSetupRootUrl)$(CoreSetupBlobVersion) $(DownloadedSharedHostInstallerFileName) - $(CoreSetupBlobAccessTokenParam) $(CoreSetupRootUrl)$(CoreSetupBlobVersion) $(DownloadedHostFxrInstallerFileName) - $(CoreSetupBlobAccessTokenParam) $(CoreSetupRootUrl)$(NETCoreAppTargetingPackBlobVersion) $(DownloadedNetCoreAppTargetingPackInstallerFileName) - $(CoreSetupBlobAccessTokenParam) $(CoreSetupRootUrl)$(NETCoreAppTargetingPackBlobVersion) $(CoreSetupRootUrl)3.0.0 $(DownloadedNetStandardTargetingPackInstallerFileName) - $(CoreSetupBlobAccessTokenParam) $(CoreSetupRootUrl)$(CoreSetupBlobVersion) $(DownloadedNetCoreAppHostPackInstallerFileName) - $(CoreSetupBlobAccessTokenParam) $(CoreSetupRootUrl)$(CoreSetupBlobVersion) $(DownloadedAlternateNetCoreAppHostPackInstallerFileName) - $(CoreSetupBlobAccessTokenParam) $(CoreSetupRootUrl)$(CoreSetupBlobVersion) $(DownloadedArmNetCoreAppHostPackInstallerFileName) - $(CoreSetupBlobAccessTokenParam) $(CoreSetupRootUrl)$(CoreSetupBlobVersion) $(DownloadedArm64NetCoreAppHostPackInstallerFileName) - $(CoreSetupBlobAccessTokenParam) $(WinFormsAndWpfSharedFxRootUrl)$(WindowsDesktopTargetingPackBlobVersion) $(DownloadedWindowsDesktopTargetingPackInstallerFileName) - $(CoreSetupBlobAccessTokenParam) $(DotnetToolsetBlobRootUrl)Toolset/$(MicrosoftDotnetToolsetInternalPackageVersion) dotnet-toolset-internal-$(MicrosoftDotnetToolsetInternalPackageVersion).zip - $(CoreSetupBlobAccessTokenParam) sdk/$(SdkVersion) @@ -256,7 +238,6 @@ $(AspNetCoreSharedFxRootUrl)$(AspNetCoreBlobVersion) $(AspNetCoreSharedFxArchiveFileName) - $(CoreSetupBlobAccessTokenParam) @@ -266,7 +247,6 @@ Condition="'$(InstallerExtension)' == '.pkg' And '$(SkipBuildingInstallers)' != 'true' And '$(InstallerExtension)' != '' And !$(Architecture.StartsWith('arm'))"> $(AspNetCoreSharedFxRootUrl)$(AspNetCoreTargetingPackBlobVersion) $(AspNetTargetingPackArchiveFileName) - $(CoreSetupBlobAccessTokenParam) @@ -274,35 +254,30 @@ Condition="'$(InstallerExtension)' != '.pkg' And '$(SkipBuildingInstallers)' != 'true' And '$(InstallerExtension)' != '' And !$(Architecture.StartsWith('arm'))"> $(AspNetCoreSharedFxRootUrl)$(AspNetCoreTargetingPackBlobVersion) $(DownloadedAspNetTargetingPackInstallerFileName) - $(CoreSetupBlobAccessTokenParam) $(AspNetCoreSharedFxRootUrl)$(AspNetCoreBlobVersion) $(DownloadedAspNetCoreSharedFxInstallerFileName) - $(CoreSetupBlobAccessTokenParam) $(AspNetCoreSharedFxRootUrl)$(AspNetCoreBlobVersion) $(DownloadedAspNetCoreSharedFxWixLibFileName) - $(CoreSetupBlobAccessTokenParam) $(AspNetCoreSharedFxRootUrl)$(AspNetCoreBlobVersion) $(DownloadedAspNetCoreV2ModuleInstallerFileName) - $(CoreSetupBlobAccessTokenParam) $(AspNetCoreSharedFxRootUrl)$(AspNetCoreBlobVersion) $(AspNetCoreSharedFxBaseRuntimeVersionFileName) - $(CoreSetupBlobAccessTokenParam) @@ -323,14 +298,12 @@ $(WinFormsAndWpfSharedFxRootUrl)$(WindowsDesktopTargetingPackBlobVersion) $(WinFormsAndWpfSharedFxArchiveFileName) - $(CoreSetupBlobAccessTokenParam) $(WinFormsAndWpfSharedFxRootUrl)$(WindowsDesktopBlobVersion) $(DownloadedWinFormsAndWpfSharedFrameworkInstallerFileName) - $(CoreSetupBlobAccessTokenParam) @@ -346,11 +319,20 @@ true + + + %(BaseUrl) + $([System.String]::new('%(ComponentToDownload.PrivateBaseUrl)').Replace('$(CoreSetupBlobRootUrl)', '$(InternalBaseURL)')) + $([System.String]::new('%(ComponentToDownload.PrivateBaseUrl)').Replace('$(DotnetExtensionsBlobRootUrl)', '$(InternalBaseURL)')) + $([System.String]::new('%(ComponentToDownload.PrivateBaseUrl)').Replace('$(DotnetToolsetBlobRootUrl)', '$(InternalBaseURL)')) @@ -374,7 +356,7 @@ $([MSBuild]::ValueOrDefault('%(BundledLayoutPackage.PackageName)', '').ToLower()) - + %(BundledLayoutPackage.RelativeLayoutPath) %(BundledLayoutPackage.Identity) @@ -439,7 +421,7 @@ - + Microsoft.NETCore.App.Host.$(SharedFrameworkRid) $(IntermediateOutputPath)AppHostRestore\ @@ -465,7 +447,7 @@ BuildInParallel="False" Projects="@(AppHostTemplateDownloadPackageProject)"> - + AppHost$(ExeExtension) @@ -512,7 +494,7 @@ - +