diff --git a/NuGet.config b/NuGet.config
index d8f4a4717..56e3557f0 100644
--- a/NuGet.config
+++ b/NuGet.config
@@ -12,6 +12,7 @@
+
diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml
index 1b39be1d0..bbfbed12d 100644
--- a/eng/Version.Details.xml
+++ b/eng/Version.Details.xml
@@ -67,9 +67,9 @@
https://github.com/aspnet/AspNetCore
2bbf66ad44de54898f6cfe210bcbe079e62fbfc9
-
+
https://github.com/dotnet/test-templates
- 6ae8a6e7dd862c9447771c108d43fb9028f22c22
+ 5117d3d9d7dfd3583f0c42a639efaf092f957b7b
https://github.com/dotnet/templating
diff --git a/eng/build.yml b/eng/build.yml
index 944d1e0ce..750111576 100644
--- a/eng/build.yml
+++ b/eng/build.yml
@@ -43,7 +43,7 @@ phases:
- _SignType: test
- _DOTNETCLIMSRC_READ_SAS_TOKEN: ''
- - ${{ if and(eq(variables['System.TeamProject'], 'internal'), contains(variables['Build.SourceBranch'], 'internal')) }}:
+ - ${{ if eq(variables['System.TeamProject'], 'internal') }}:
- group: DotNet-MSRC-Storage
- _DOTNETCLIMSRC_READ_SAS_TOKEN: $(dotnetclimsrc-read-sas-token)
diff --git a/eng/common/SetupNugetSources.ps1 b/eng/common/SetupNugetSources.ps1
index a5a1e711d..c3c473eb8 100644
--- a/eng/common/SetupNugetSources.ps1
+++ b/eng/common/SetupNugetSources.ps1
@@ -83,7 +83,7 @@ function AddCredential($creds, $source, $username, $password) {
$passwordElement.SetAttribute("value", $Password)
}
-function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Password) {
+function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $Password) {
$maestroPrivateSources = $Sources.SelectNodes("add[contains(@key,'darc-int')]")
Write-Host "Inserting credentials for $($maestroPrivateSources.Count) Maestro's private feeds."
@@ -123,19 +123,21 @@ if ($creds -eq $null) {
$doc.DocumentElement.AppendChild($creds) | Out-Null
}
+$userName = "dn-bot"
+
# Insert credential nodes for Maestro's private feeds
-InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Password $Password
+InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -Password $Password
$dotnet3Source = $sources.SelectSingleNode("add[@key='dotnet3']")
if ($dotnet3Source -ne $null) {
- AddPackageSource -Sources $sources -SourceName "dotnet3-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3-internal/nuget/v2" -Creds $creds -Username "dn-bot" -Password $Password
- AddPackageSource -Sources $sources -SourceName "dotnet3-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3-internal-transport/nuget/v2" -Creds $creds -Username "dn-bot" -Password $Password
+ AddPackageSource -Sources $sources -SourceName "dotnet3-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password
+ AddPackageSource -Sources $sources -SourceName "dotnet3-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password
}
$dotnet31Source = $sources.SelectSingleNode("add[@key='dotnet3.1']")
if ($dotnet31Source -ne $null) {
- AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username "dn-bot" -Password $Password
- AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username "dn-bot" -Password $Password
+ AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password
+ AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password
}
$doc.Save($filename)
diff --git a/eng/common/templates/post-build/channels/generic-internal-channel.yml b/eng/common/templates/post-build/channels/generic-internal-channel.yml
index 3a8755fbb..6a4ab7f53 100644
--- a/eng/common/templates/post-build/channels/generic-internal-channel.yml
+++ b/eng/common/templates/post-build/channels/generic-internal-channel.yml
@@ -29,6 +29,10 @@ stages:
pool:
vmImage: 'windows-2019'
steps:
+ # This is necessary whenever we want to publish/restore to an AzDO private feed
+ - task: NuGetAuthenticate@0
+ displayName: 'Authenticate to AzDO Feeds'
+
- task: DownloadBuildArtifacts@0
displayName: Download Blob Artifacts
inputs:
diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml
index 8a8d84f20..70a5b6f8a 100644
--- a/eng/common/templates/post-build/post-build.yml
+++ b/eng/common/templates/post-build/post-build.yml
@@ -54,6 +54,12 @@ stages:
pool:
vmImage: 'windows-2019'
steps:
+ # This is necessary whenever we want to publish/restore to an AzDO private feed
+ # Since sdk-task.ps1 tries to restore packages we need to do this authentication here
+ # otherwise it'll complain about accessing a private feed.
+ - task: NuGetAuthenticate@0
+ displayName: 'Authenticate to AzDO Feeds'
+
- task: DownloadBuildArtifacts@0
displayName: Download Package Artifacts
inputs:
diff --git a/eng/dockerrun.sh b/eng/dockerrun.sh
index a41b3c059..762e177a7 100755
--- a/eng/dockerrun.sh
+++ b/eng/dockerrun.sh
@@ -143,7 +143,7 @@ docker run $INTERACTIVE -t --rm --sig-proxy=true \
-e BUILD_BUILDNUMBER \
-e BUILD_SOURCEVERSION \
-e SYSTEM_TEAMPROJECT \
- -e DOTNECLIMSRC_READ_SAS_TOKEN \
+ -e DOTNETCLIMSRC_READ_SAS_TOKEN \
-e AGENT_JOBNAME \
-e AGENT_OS \
-e VSS_NUGET_URI_PREFIXES \
diff --git a/src/core-sdk-tasks/DownloadFile.cs b/src/core-sdk-tasks/DownloadFile.cs
index 895a6749f..40178d1eb 100644
--- a/src/core-sdk-tasks/DownloadFile.cs
+++ b/src/core-sdk-tasks/DownloadFile.cs
@@ -3,23 +3,39 @@
using System;
using System.IO;
+using System.Net;
using System.Net.Http;
+using System.Threading.Tasks;
+using System.Collections.Generic;
using Microsoft.Build.Framework;
using Microsoft.Build.Utilities;
namespace Microsoft.DotNet.Cli.Build
{
- public class DownloadFile : Task
+ public class DownloadFile : Microsoft.Build.Utilities.Task
{
[Required]
public string Uri { get; set; }
+ ///
+ /// If this field is set and the task fail to download the file from `Uri`, with a NotFound
+ /// status, it will try to download the file from `PrivateUri`.
+ ///
+ public string PrivateUri { get; set; }
+
+ public int MaxRetries { get; set; } = 5;
+
[Required]
public string DestinationPath { get; set; }
public bool Overwrite { get; set; }
public override bool Execute()
+ {
+ return ExecuteAsync().GetAwaiter().GetResult();
+ }
+
+ private async System.Threading.Tasks.Task ExecuteAsync()
{
string destinationDir = Path.GetDirectoryName(DestinationPath);
if (!Directory.Exists(destinationDir))
@@ -39,31 +55,84 @@ namespace Microsoft.DotNet.Cli.Build
var filePath = Uri.Substring(FileUriProtocol.Length);
Log.LogMessage($"Copying '{filePath}' to '{DestinationPath}'");
File.Copy(filePath, DestinationPath);
+ return true;
}
- else
+
+ List errorMessages = new List();
+ bool? downloadStatus = await DownloadWithRetriesAsync(Uri, DestinationPath, errorMessages);
+
+ if (downloadStatus == false && !string.IsNullOrEmpty(PrivateUri))
{
- Log.LogMessage(MessageImportance.High, $"Downloading '{Uri}' to '{DestinationPath}'");
+ downloadStatus = await DownloadWithRetriesAsync(PrivateUri, DestinationPath, errorMessages);
+ }
- using (var httpClient = new HttpClient())
+ if (downloadStatus != true)
+ {
+ foreach (var error in errorMessages)
{
- var getTask = httpClient.GetStreamAsync(Uri);
-
- try
- {
- using (var outStream = File.Create(DestinationPath))
- {
- getTask.Result.CopyTo(outStream);
- }
- }
- catch (Exception)
- {
- File.Delete(DestinationPath);
- throw;
- }
+ Log.LogError(error);
}
}
- return true;
+ return downloadStatus == true;
+ }
+
+ ///
+ /// Attempt to download file from `source` with retries when response error is different of FileNotFound and Success.
+ ///
+ /// URL to the file to be downloaded.
+ /// Local path where to put the downloaded file.
+ /// true: Download Succeeded. false: Download failed with 404. null: Download failed but is retriable.
+ private async Task DownloadWithRetriesAsync(string source, string target, List errorMessages)
+ {
+ Random rng = new Random();
+
+ Log.LogMessage(MessageImportance.High, $"Attempting download '{source}' to '{target}'");
+
+ using (var httpClient = new HttpClient())
+ {
+ for (int retryNumber = 0; retryNumber < MaxRetries; retryNumber++)
+ {
+ try
+ {
+ var httpResponse = await httpClient.GetAsync(source);
+
+ Log.LogMessage(MessageImportance.High, $"{source} -> {httpResponse.StatusCode}");
+
+ // The Azure Storage REST API returns '400 - Bad Request' in some cases
+ // where the resource is not found on the storage.
+ // https://docs.microsoft.com/en-us/rest/api/storageservices/common-rest-api-error-codes
+ if (httpResponse.StatusCode == HttpStatusCode.NotFound ||
+ httpResponse.ReasonPhrase.IndexOf("The requested URI does not represent any resource on the server.", StringComparison.OrdinalIgnoreCase) == 0)
+ {
+ errorMessages.Add($"Problems downloading file from '{source}'. Does the resource exist on the storage? {httpResponse.StatusCode} : {httpResponse.ReasonPhrase}");
+ return false;
+ }
+
+ httpResponse.EnsureSuccessStatusCode();
+
+ using (var outStream = File.Create(target))
+ {
+ await httpResponse.Content.CopyToAsync(outStream);
+ }
+
+ Log.LogMessage(MessageImportance.High, $"returning true {source} -> {httpResponse.StatusCode}");
+ return true;
+ }
+ catch (Exception e)
+ {
+ Log.LogMessage(MessageImportance.High, $"returning error in {source} ");
+ errorMessages.Add($"Problems downloading file from '{source}'. {e.Message} {e.StackTrace}");
+ File.Delete(target);
+ }
+
+ await System.Threading.Tasks.Task.Delay(rng.Next(1000, 10000));
+ }
+ }
+
+ Log.LogMessage(MessageImportance.High, $"giving up {source} ");
+ errorMessages.Add($"Giving up downloading the file from '{source}' after {MaxRetries} retries.");
+ return null;
}
}
}
diff --git a/src/redist/targets/GenerateLayout.targets b/src/redist/targets/GenerateLayout.targets
index b074760f2..d0e6b7bcf 100644
--- a/src/redist/targets/GenerateLayout.targets
+++ b/src/redist/targets/GenerateLayout.targets
@@ -21,21 +21,16 @@
$(WindowsDesktopBlobVersion)
3.0.0
-
+
$(RedistLayoutPath)sdk\$(SdkVersion)\
-
- true
- $(DOTNETCLIMSRC_READ_SAS_TOKEN)
- https://dotnetclimsrc.blob.core.windows.net/dotnet/
+ true
+ https://dotnetclimsrc.blob.core.windows.net/dotnet/
+
https://dotnetcli.blob.core.windows.net/dotnet/
-
- https://dotnetclimsrc.blob.core.windows.net/dotnet/
https://dotnetcli.blob.core.windows.net/dotnet/
-
- https://dotnetclimsrc.blob.core.windows.net/dotnet/
https://dotnetfeed.blob.core.windows.net/dotnet-toolset/
$(HostRid)
@@ -113,7 +108,6 @@
$(CoreSetupRootUrl)$(CoreSetupBlobVersion)
$(CombinedFrameworkHostArchiveFileName)
- $(CoreSetupBlobAccessTokenParam)
@@ -170,35 +164,30 @@
Condition="('$(IsDebianBaseDistro)' == 'true' OR '$(IsRPMBasedDistro)' == 'true') And '$(SkipBuildingInstallers)' != 'true' And '$(InstallerExtension)' != '' And !$(Architecture.StartsWith('arm'))">
$(CoreSetupRootUrl)$(CoreSetupBlobVersion)
$(DownloadedRuntimeDepsInstallerFileName)
- $(CoreSetupBlobAccessTokenParam)
$(CoreSetupRootUrl)$(CoreSetupBlobVersion)
$(DownloadedSharedFrameworkInstallerFileName)
- $(CoreSetupBlobAccessTokenParam)
$(CoreSetupRootUrl)$(CoreSetupBlobVersion)
$(DownloadedSharedHostInstallerFileName)
- $(CoreSetupBlobAccessTokenParam)
$(CoreSetupRootUrl)$(CoreSetupBlobVersion)
$(DownloadedHostFxrInstallerFileName)
- $(CoreSetupBlobAccessTokenParam)
$(CoreSetupRootUrl)$(NETCoreAppTargetingPackBlobVersion)
$(DownloadedNetCoreAppTargetingPackInstallerFileName)
- $(CoreSetupBlobAccessTokenParam)
$(CoreSetupRootUrl)$(NETCoreAppTargetingPackBlobVersion)
$(CoreSetupRootUrl)3.0.0
$(DownloadedNetStandardTargetingPackInstallerFileName)
- $(CoreSetupBlobAccessTokenParam)
$(CoreSetupRootUrl)$(CoreSetupBlobVersion)
$(DownloadedNetCoreAppHostPackInstallerFileName)
- $(CoreSetupBlobAccessTokenParam)
$(CoreSetupRootUrl)$(CoreSetupBlobVersion)
$(DownloadedAlternateNetCoreAppHostPackInstallerFileName)
- $(CoreSetupBlobAccessTokenParam)
$(CoreSetupRootUrl)$(CoreSetupBlobVersion)
$(DownloadedArmNetCoreAppHostPackInstallerFileName)
- $(CoreSetupBlobAccessTokenParam)
$(CoreSetupRootUrl)$(CoreSetupBlobVersion)
$(DownloadedArm64NetCoreAppHostPackInstallerFileName)
- $(CoreSetupBlobAccessTokenParam)
$(WinFormsAndWpfSharedFxRootUrl)$(WindowsDesktopTargetingPackBlobVersion)
$(DownloadedWindowsDesktopTargetingPackInstallerFileName)
- $(CoreSetupBlobAccessTokenParam)
$(DotnetToolsetBlobRootUrl)Toolset/$(MicrosoftDotnetToolsetInternalPackageVersion)
dotnet-toolset-internal-$(MicrosoftDotnetToolsetInternalPackageVersion).zip
- $(CoreSetupBlobAccessTokenParam)
sdk/$(SdkVersion)
@@ -256,7 +238,6 @@
$(AspNetCoreSharedFxRootUrl)$(AspNetCoreBlobVersion)
$(AspNetCoreSharedFxArchiveFileName)
- $(CoreSetupBlobAccessTokenParam)
@@ -266,7 +247,6 @@
Condition="'$(InstallerExtension)' == '.pkg' And '$(SkipBuildingInstallers)' != 'true' And '$(InstallerExtension)' != '' And !$(Architecture.StartsWith('arm'))">
$(AspNetCoreSharedFxRootUrl)$(AspNetCoreTargetingPackBlobVersion)
$(AspNetTargetingPackArchiveFileName)
- $(CoreSetupBlobAccessTokenParam)
@@ -274,35 +254,30 @@
Condition="'$(InstallerExtension)' != '.pkg' And '$(SkipBuildingInstallers)' != 'true' And '$(InstallerExtension)' != '' And !$(Architecture.StartsWith('arm'))">
$(AspNetCoreSharedFxRootUrl)$(AspNetCoreTargetingPackBlobVersion)
$(DownloadedAspNetTargetingPackInstallerFileName)
- $(CoreSetupBlobAccessTokenParam)
$(AspNetCoreSharedFxRootUrl)$(AspNetCoreBlobVersion)
$(DownloadedAspNetCoreSharedFxInstallerFileName)
- $(CoreSetupBlobAccessTokenParam)
$(AspNetCoreSharedFxRootUrl)$(AspNetCoreBlobVersion)
$(DownloadedAspNetCoreSharedFxWixLibFileName)
- $(CoreSetupBlobAccessTokenParam)
$(AspNetCoreSharedFxRootUrl)$(AspNetCoreBlobVersion)
$(DownloadedAspNetCoreV2ModuleInstallerFileName)
- $(CoreSetupBlobAccessTokenParam)
$(AspNetCoreSharedFxRootUrl)$(AspNetCoreBlobVersion)
$(AspNetCoreSharedFxBaseRuntimeVersionFileName)
- $(CoreSetupBlobAccessTokenParam)
@@ -323,14 +298,12 @@
$(WinFormsAndWpfSharedFxRootUrl)$(WindowsDesktopTargetingPackBlobVersion)
$(WinFormsAndWpfSharedFxArchiveFileName)
- $(CoreSetupBlobAccessTokenParam)
$(WinFormsAndWpfSharedFxRootUrl)$(WindowsDesktopBlobVersion)
$(DownloadedWinFormsAndWpfSharedFrameworkInstallerFileName)
- $(CoreSetupBlobAccessTokenParam)
@@ -346,11 +319,20 @@
true
+
+
+ %(BaseUrl)
+ $([System.String]::new('%(ComponentToDownload.PrivateBaseUrl)').Replace('$(CoreSetupBlobRootUrl)', '$(InternalBaseURL)'))
+ $([System.String]::new('%(ComponentToDownload.PrivateBaseUrl)').Replace('$(DotnetExtensionsBlobRootUrl)', '$(InternalBaseURL)'))
+ $([System.String]::new('%(ComponentToDownload.PrivateBaseUrl)').Replace('$(DotnetToolsetBlobRootUrl)', '$(InternalBaseURL)'))
@@ -374,7 +356,7 @@
$([MSBuild]::ValueOrDefault('%(BundledLayoutPackage.PackageName)', '').ToLower())
-
+
%(BundledLayoutPackage.RelativeLayoutPath)
%(BundledLayoutPackage.Identity)
@@ -439,7 +421,7 @@
-
+
Microsoft.NETCore.App.Host.$(SharedFrameworkRid)
$(IntermediateOutputPath)AppHostRestore\
@@ -465,7 +447,7 @@
BuildInParallel="False"
Projects="@(AppHostTemplateDownloadPackageProject)">
-
+
AppHost$(ExeExtension)
@@ -512,7 +494,7 @@
-
+