mirrors/dotnet-installer
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

Merge pull request #5858 from mmitche/merge-from-311xx

Browse source 
[release/3.1.2xx] Merge from 3.1.1xx
This commit is contained in:
Matt Mitchell 2019-12-11 06:51:21 -08:00 committed by GitHub
commit a8d4e70db8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 108 additions and 55 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
eng/build.yml
View file

@ -43,7 +43,7 @@ phases:
- _SignType: test
- _DOTNETCLIMSRC_READ_SAS_TOKEN: ''
- ${{ if and(eq(variables['System.TeamProject'], 'internal'), contains(variables['Build.SourceBranch'], 'internal')) }}:
- ${{ if eq(variables['System.TeamProject'], 'internal') }}:
- group: DotNet-MSRC-Storage
- _DOTNETCLIMSRC_READ_SAS_TOKEN: $(dotnetclimsrc-read-sas-token)

2
eng/dockerrun.sh
View file

@ -143,7 +143,7 @@ docker run $INTERACTIVE -t --rm --sig-proxy=true \
-e BUILD_BUILDNUMBER \
-e BUILD_SOURCEVERSION \
-e SYSTEM_TEAMPROJECT \
-e DOTNECLIMSRC_READ_SAS_TOKEN \
-e DOTNETCLIMSRC_READ_SAS_TOKEN \
-e AGENT_JOBNAME \
-e AGENT_OS \
-e VSS_NUGET_URI_PREFIXES \

93
src/core-sdk-tasks/DownloadFile.cs
View file

@ -3,23 +3,39 @@
using System;
using System.IO;
using System.Net;
using System.Net.Http;
using System.Threading.Tasks;
using System.Collections.Generic;
using Microsoft.Build.Framework;
using Microsoft.Build.Utilities;
namespace Microsoft.DotNet.Cli.Build
{
public class DownloadFile : Task
public class DownloadFile : Microsoft.Build.Utilities.Task
{
[Required]
public string Uri { get; set; }
/// <summary>
/// If this field is set and the task fail to download the file from `Uri`, with a NotFound
/// status, it will try to download the file from `PrivateUri`.
/// </summary>
public string PrivateUri { get; set; }
public int MaxRetries { get; set; } = 5;
[Required]
public string DestinationPath { get; set; }
public bool Overwrite { get; set; }
public override bool Execute()
{
return ExecuteAsync().GetAwaiter().GetResult();
}
private async System.Threading.Tasks.Task<bool> ExecuteAsync()
{
string destinationDir = Path.GetDirectoryName(DestinationPath);
if (!Directory.Exists(destinationDir))
@ -39,31 +55,84 @@ namespace Microsoft.DotNet.Cli.Build
var filePath = Uri.Substring(FileUriProtocol.Length);
Log.LogMessage($"Copying '{filePath}' to '{DestinationPath}'");
File.Copy(filePath, DestinationPath);
return true;
}
else
List<string> errorMessages = new List<string>();
bool? downloadStatus = await DownloadWithRetriesAsync(Uri, DestinationPath, errorMessages);
if (downloadStatus == false && !string.IsNullOrEmpty(PrivateUri))
{
Log.LogMessage(MessageImportance.High, $"Downloading '{Uri}' to '{DestinationPath}'");
downloadStatus = await DownloadWithRetriesAsync(PrivateUri, DestinationPath, errorMessages);
}
if (downloadStatus != true)
{
foreach (var error in errorMessages)
{
Log.LogError(error);
}
}
return downloadStatus == true;
}
/// <summary>
/// Attempt to download file from `source` with retries when response error is different of FileNotFound and Success.
/// </summary>
/// <param name="source">URL to the file to be downloaded.</param>
/// <param name="target">Local path where to put the downloaded file.</param>
/// <returns>true: Download Succeeded. false: Download failed with 404. null: Download failed but is retriable.</returns>
private async Task<bool?> DownloadWithRetriesAsync(string source, string target, List<string> errorMessages)
{
Random rng = new Random();
Log.LogMessage(MessageImportance.High, $"Attempting download '{source}' to '{target}'");
using (var httpClient = new HttpClient())
{
var getTask = httpClient.GetStreamAsync(Uri);
for (int retryNumber = 0; retryNumber < MaxRetries; retryNumber++)
{
try
{
using (var outStream = File.Create(DestinationPath))
var httpResponse = await httpClient.GetAsync(source);
Log.LogMessage(MessageImportance.High, $"{source} -> {httpResponse.StatusCode}");
// The Azure Storage REST API returns '400 - Bad Request' in some cases
// where the resource is not found on the storage.
// https://docs.microsoft.com/en-us/rest/api/storageservices/common-rest-api-error-codes
if (httpResponse.StatusCode == HttpStatusCode.NotFound ||
httpResponse.ReasonPhrase.IndexOf("The requested URI does not represent any resource on the server.", StringComparison.OrdinalIgnoreCase) == 0)
{
getTask.Result.CopyTo(outStream);
errorMessages.Add($"Problems downloading file from '{source}'. Does the resource exist on the storage? {httpResponse.StatusCode} : {httpResponse.ReasonPhrase}");
return false;
}
}
catch (Exception)
httpResponse.EnsureSuccessStatusCode();
using (var outStream = File.Create(target))
{
File.Delete(DestinationPath);
throw;
await httpResponse.Content.CopyToAsync(outStream);
}
Log.LogMessage(MessageImportance.High, $"returning true {source} -> {httpResponse.StatusCode}");
return true;
}
catch (Exception e)
{
Log.LogMessage(MessageImportance.High, $"returning error in {source} ");
errorMessages.Add($"Problems downloading file from '{source}'. {e.Message} {e.StackTrace}");
File.Delete(target);
}
await System.Threading.Tasks.Task.Delay(rng.Next(1000, 10000));
}
}
return true;
Log.LogMessage(MessageImportance.High, $"giving up {source} ");
errorMessages.Add($"Giving up downloading the file from '{source}' after {MaxRetries} retries.");
return null;
}
}
}

40
src/redist/targets/GenerateLayout.targets
View file

@ -25,16 +25,11 @@
<PropertyGroup>
<SdkOutputDirectory>$(RedistLayoutPath)sdk\$(SdkVersion)\</SdkOutputDirectory>
<InternalBuild Condition="$(SYSTEM_TEAMPROJECT) == 'internal' and $(BUILD_SOURCEBRANCH.ToLower().contains('internal'))">true</InternalBuild>
<CoreSetupBlobAccessTokenParam Condition="'$(InternalBuild)' == 'true'">$(DOTNETCLIMSRC_READ_SAS_TOKEN)</CoreSetupBlobAccessTokenParam>
<InternalBuild Condition="$(SYSTEM_TEAMPROJECT) == 'internal'">true</InternalBuild>
<InternalBaseURL Condition="$(SYSTEM_TEAMPROJECT) == 'internal'">https://dotnetclimsrc.blob.core.windows.net/dotnet/</InternalBaseURL>
<CoreSetupBlobRootUrl Condition="'$(CoreSetupBlobRootUrl)' == '' and '$(InternalBuild)' == 'true'">https://dotnetclimsrc.blob.core.windows.net/dotnet/</CoreSetupBlobRootUrl>
<CoreSetupBlobRootUrl Condition="'$(CoreSetupBlobRootUrl)' == ''">https://dotnetcli.blob.core.windows.net/dotnet/</CoreSetupBlobRootUrl>
<DotnetExtensionsBlobRootUrl Condition="'$(DotnetExtensionsBlobRootUrl)' == '' and '$(InternalBuild)' == 'true'">https://dotnetclimsrc.blob.core.windows.net/dotnet/</DotnetExtensionsBlobRootUrl>
<DotnetExtensionsBlobRootUrl Condition="'$(DotnetExtensionsBlobRootUrl)' == ''">https://dotnetcli.blob.core.windows.net/dotnet/</DotnetExtensionsBlobRootUrl>
<DotnetToolsetBlobRootUrl Condition="'$(DotnetToolsetBlobRootUrl)' == '' and '$(InternalBuild)' == 'true'">https://dotnetclimsrc.blob.core.windows.net/dotnet/</DotnetToolsetBlobRootUrl>
<DotnetToolsetBlobRootUrl Condition="'$(DotnetToolsetBlobRootUrl)' == ''">https://dotnetfeed.blob.core.windows.net/dotnet-toolset/</DotnetToolsetBlobRootUrl>
<CoreSetupRid Condition="'$(CoreSetupRid)' == ''">$(HostRid)</CoreSetupRid>
@ -112,7 +107,6 @@
<BundledLayoutComponent Include="CombinedSharedHostAndFrameworkArchive">
<BaseUrl>$(CoreSetupRootUrl)$(CoreSetupBlobVersion)</BaseUrl>
<DownloadFileName>$(CombinedFrameworkHostArchiveFileName)</DownloadFileName>
<AccessToken>$(CoreSetupBlobAccessTokenParam)</AccessToken>
<RelativeLayoutPath></RelativeLayoutPath>
</BundledLayoutComponent>
@ -169,35 +163,30 @@
Condition="('$(IsDebianBaseDistro)' == 'true' OR '$(IsRPMBasedDistro)' == 'true') And '$(SkipBuildingInstallers)' != 'true' And '$(InstallerExtension)' != '' And !$(Architecture.StartsWith('arm'))">
<BaseUrl>$(CoreSetupRootUrl)$(CoreSetupBlobVersion)</BaseUrl>
<DownloadFileName>$(DownloadedRuntimeDepsInstallerFileName)</DownloadFileName>
<AccessToken>$(CoreSetupBlobAccessTokenParam)</AccessToken>
</BundledInstallerComponent>
<BundledInstallerComponent Include="DownloadedSharedFrameworkInstallerFile"
Condition="'$(SkipBuildingInstallers)' != 'true' And '$(InstallerExtension)' != '' And !$(Architecture.StartsWith('arm'))">
<BaseUrl>$(CoreSetupRootUrl)$(CoreSetupBlobVersion)</BaseUrl>
<DownloadFileName>$(DownloadedSharedFrameworkInstallerFileName)</DownloadFileName>
<AccessToken>$(CoreSetupBlobAccessTokenParam)</AccessToken>
</BundledInstallerComponent>
<BundledInstallerComponent Include="DownloadedSharedHostInstallerFile"
Condition="'$(SkipBuildingInstallers)' != 'true' And '$(InstallerExtension)' != '' And !$(Architecture.StartsWith('arm'))">
<BaseUrl>$(CoreSetupRootUrl)$(CoreSetupBlobVersion)</BaseUrl>
<DownloadFileName>$(DownloadedSharedHostInstallerFileName)</DownloadFileName>
<AccessToken>$(CoreSetupBlobAccessTokenParam)</AccessToken>
</BundledInstallerComponent>
<BundledInstallerComponent Include="DownloadedHostFxrInstallerFile"
Condition="'$(SkipBuildingInstallers)' != 'true' And '$(InstallerExtension)' != '' And !$(Architecture.StartsWith('arm'))">
<BaseUrl>$(CoreSetupRootUrl)$(CoreSetupBlobVersion)</BaseUrl>
<DownloadFileName>$(DownloadedHostFxrInstallerFileName)</DownloadFileName>
<AccessToken>$(CoreSetupBlobAccessTokenParam)</AccessToken>
</BundledInstallerComponent>
<BundledInstallerComponent Include="DownloadedNetCoreAppTargetingPackInstallerFile"
Condition="'$(SkipBuildingInstallers)' != 'true' And '$(InstallerExtension)' != '' And !$(Architecture.StartsWith('arm'))">
<BaseUrl>$(CoreSetupRootUrl)$(NETCoreAppTargetingPackBlobVersion)</BaseUrl>
<DownloadFileName>$(DownloadedNetCoreAppTargetingPackInstallerFileName)</DownloadFileName>
<AccessToken>$(CoreSetupBlobAccessTokenParam)</AccessToken>
</BundledInstallerComponent>
<BundledInstallerComponent Include="DownloadedNetStandardTargetingPackInstallerFile"
@ -205,48 +194,41 @@
<BaseUrl>$(CoreSetupRootUrl)$(MicrosoftNETCoreAppRuntimewinx64PackageVersion)</BaseUrl>
<BaseUrl>$(CoreSetupRootUrl)3.0.0</BaseUrl>
<DownloadFileName>$(DownloadedNetStandardTargetingPackInstallerFileName)</DownloadFileName>
<AccessToken>$(CoreSetupBlobAccessTokenParam)</AccessToken>
</BundledInstallerComponent>
<BundledInstallerComponent Include="DownloadedNetCoreAppHostPackInstallerFile"
Condition="'$(SkipBuildingInstallers)' != 'true' And '$(InstallerExtension)' != '' And !$(Architecture.StartsWith('arm'))">
<BaseUrl>$(CoreSetupRootUrl)$(CoreSetupBlobVersion)</BaseUrl>
<DownloadFileName>$(DownloadedNetCoreAppHostPackInstallerFileName)</DownloadFileName>
<AccessToken>$(CoreSetupBlobAccessTokenParam)</AccessToken>
</BundledInstallerComponent>
<BundledInstallerComponent Include="DownloadedAlternateNetCoreAppHostPackInstallerFile"
Condition="'$(SkipBuildingInstallers)' != 'true' And '$(InstallerExtension)' == '.msi' And !$(Architecture.StartsWith('arm'))">
<BaseUrl>$(CoreSetupRootUrl)$(CoreSetupBlobVersion)</BaseUrl>
<DownloadFileName>$(DownloadedAlternateNetCoreAppHostPackInstallerFileName)</DownloadFileName>
<AccessToken>$(CoreSetupBlobAccessTokenParam)</AccessToken>
</BundledInstallerComponent>
<BundledInstallerComponent Include="DownloadedArmNetCoreAppHostPackInstallerFile"
Condition="'$(SkipBuildingInstallers)' != 'true' And '$(InstallerExtension)' == '.msi' And !$(Architecture.StartsWith('arm'))">
<BaseUrl>$(CoreSetupRootUrl)$(CoreSetupBlobVersion)</BaseUrl>
<DownloadFileName>$(DownloadedArmNetCoreAppHostPackInstallerFileName)</DownloadFileName>
<AccessToken>$(CoreSetupBlobAccessTokenParam)</AccessToken>
</BundledInstallerComponent>
<BundledInstallerComponent Include="DownloadedArm64NetCoreAppHostPackInstallerFile"
Condition="'$(SkipBuildingInstallers)' != 'true' And '$(InstallerExtension)' == '.msi' And !$(Architecture.StartsWith('arm'))">
<BaseUrl>$(CoreSetupRootUrl)$(CoreSetupBlobVersion)</BaseUrl>
<DownloadFileName>$(DownloadedArm64NetCoreAppHostPackInstallerFileName)</DownloadFileName>
<AccessToken>$(CoreSetupBlobAccessTokenParam)</AccessToken>
</BundledInstallerComponent>
<BundledInstallerComponent Include="DownloadedWindowsDesktopTargetingPackInstallerFile"
Condition="'$(InstallerExtension)' == '.msi' And '$(SkipBuildingInstallers)' != 'true' And !$(Architecture.StartsWith('arm'))">
<BaseUrl>$(CoreSetupRootUrl)$(WindowsDesktopTargetingPackBlobVersion)</BaseUrl>
<DownloadFileName>$(DownloadedWindowsDesktopTargetingPackInstallerFileName)</DownloadFileName>
<AccessToken>$(CoreSetupBlobAccessTokenParam)</AccessToken>
</BundledInstallerComponent>
<BundledLayoutComponent Include="ToolsetArchive">
<BaseUrl>$(DotnetToolsetBlobRootUrl)Toolset/$(MicrosoftDotnetToolsetInternalPackageVersion)</BaseUrl>
<DownloadFileName>dotnet-toolset-internal-$(MicrosoftDotnetToolsetInternalPackageVersion).zip</DownloadFileName>
<AccessToken>$(CoreSetupBlobAccessTokenParam)</AccessToken>
<RelativeLayoutPath>sdk/$(SdkVersion)</RelativeLayoutPath>
</BundledLayoutComponent>
</ItemGroup>
@ -255,7 +237,6 @@
<BundledLayoutComponent Include="DownloadedAspNetCoreSharedFxArchiveFile">
<BaseUrl>$(AspNetCoreSharedFxRootUrl)$(AspNetCoreBlobVersion)</BaseUrl>
<DownloadFileName>$(AspNetCoreSharedFxArchiveFileName)</DownloadFileName>
<AccessToken>$(CoreSetupBlobAccessTokenParam)</AccessToken>
<RelativeLayoutPath></RelativeLayoutPath>
</BundledLayoutComponent>
@ -265,7 +246,6 @@
Condition="'$(InstallerExtension)' == '.pkg' And '$(SkipBuildingInstallers)' != 'true' And '$(InstallerExtension)' != '' And !$(Architecture.StartsWith('arm'))">
<BaseUrl>$(AspNetCoreSharedFxRootUrl)$(AspNetCoreTargetingPackBlobVersion)</BaseUrl>
<DownloadFileName>$(AspNetTargetingPackArchiveFileName)</DownloadFileName>
<AccessToken>$(CoreSetupBlobAccessTokenParam)</AccessToken>
<RelativeLayoutPath></RelativeLayoutPath>
</BundledLayoutComponent>
@ -273,14 +253,12 @@
Condition="'$(InstallerExtension)' != '.pkg' And '$(SkipBuildingInstallers)' != 'true' And '$(InstallerExtension)' != '' And !$(Architecture.StartsWith('arm'))">
<BaseUrl>$(AspNetCoreSharedFxRootUrl)$(AspNetCoreTargetingPackBlobVersion)</BaseUrl>
<DownloadFileName>$(DownloadedAspNetTargetingPackInstallerFileName)</DownloadFileName>
<AccessToken>$(CoreSetupBlobAccessTokenParam)</AccessToken>
</BundledInstallerComponent>
<BundledInstallerComponent Include="DownloadedAspNetCoreSharedFxInstallerFile"
Condition="'$(InstallerExtension)' != '.pkg' And '$(SkipBuildingInstallers)' != 'true' And '$(InstallerExtension)' != '' And !$(Architecture.StartsWith('arm'))">
<BaseUrl>$(AspNetCoreSharedFxRootUrl)$(AspNetCoreBlobVersion)</BaseUrl>
<DownloadFileName>$(DownloadedAspNetCoreSharedFxInstallerFileName)</DownloadFileName>
<AccessToken>$(CoreSetupBlobAccessTokenParam)</AccessToken>
</BundledInstallerComponent>
<BundledInstallerComponent Include="DownloadedAspNetCoreSharedFxWixLibFile"
@ -301,7 +279,6 @@
Condition="!$(Architecture.StartsWith('arm'))">
<BaseUrl>$(AspNetCoreSharedFxRootUrl)$(AspNetCoreBlobVersion)</BaseUrl>
<DownloadFileName>$(AspNetCoreSharedFxBaseRuntimeVersionFileName)</DownloadFileName>
<AccessToken>$(CoreSetupBlobAccessTokenParam)</AccessToken>
</BundledInstallerComponent>
</ItemGroup>
@ -322,14 +299,12 @@
<BundledLayoutComponent Include="WinFormsAndWpfSharedFxArchiveFile">
<BaseUrl>$(WinFormsAndWpfSharedFxRootUrl)$(CoreSetupBlobVersion)</BaseUrl>
<DownloadFileName>$(WinFormsAndWpfSharedFxArchiveFileName)</DownloadFileName>
<AccessToken>$(CoreSetupBlobAccessTokenParam)</AccessToken>
</BundledLayoutComponent>
<BundledInstallerComponent Include="DownloadedWinFormsAndWpfSharedFrameworkInstallerFile"
Condition="'$(SkipBuildingInstallers)' != 'true' And '$(InstallerExtension)' != '' And !$(Architecture.StartsWith('arm'))">
<BaseUrl>$(WinFormsAndWpfSharedFxRootUrl)$(CoreSetupBlobVersion)</BaseUrl>
<DownloadFileName>$(DownloadedWinFormsAndWpfSharedFrameworkInstallerFileName)</DownloadFileName>
<AccessToken>$(CoreSetupBlobAccessTokenParam)</AccessToken>
</BundledInstallerComponent>
</ItemGroup>
@ -345,11 +320,20 @@
</BundledInstallerComponent>
<ComponentToDownload Include="@(BundledLayoutComponent);@(BundledInstallerComponent)">
<ShouldDownload Condition="!Exists('%(DownloadDestination)')">true</ShouldDownload>
<!--
Replaces the public base url with the private one.
-->
<PrivateBaseUrl>%(BaseUrl)</PrivateBaseUrl>
<PrivateBaseUrl Condition="'$(InternalBuild)' == 'true'">$([System.String]::new('%(ComponentToDownload.PrivateBaseUrl)').Replace('$(CoreSetupBlobRootUrl)', '$(InternalBaseURL)'))</PrivateBaseUrl>
<PrivateBaseUrl Condition="'$(InternalBuild)' == 'true'">$([System.String]::new('%(ComponentToDownload.PrivateBaseUrl)').Replace('$(DotnetExtensionsBlobRootUrl)', '$(InternalBaseURL)'))</PrivateBaseUrl>
<PrivateBaseUrl Condition="'$(InternalBuild)' == 'true'">$([System.String]::new('%(ComponentToDownload.PrivateBaseUrl)').Replace('$(DotnetToolsetBlobRootUrl)', '$(InternalBaseURL)'))</PrivateBaseUrl>
</ComponentToDownload>
</ItemGroup>
<DownloadFile Condition=" '@(ComponentToDownload)' != '' And '%(ComponentToDownload.ShouldDownload)' == 'true'"
Uri="%(ComponentToDownload.BaseUrl)/%(ComponentToDownload.DownloadFileName)%(ComponentToDownload.AccessToken)"
Uri="%(ComponentToDownload.BaseUrl)/%(ComponentToDownload.DownloadFileName)"
PrivateUri="%(ComponentToDownload.PrivateBaseUrl)/%(ComponentToDownload.DownloadFileName)$(DOTNETCLIMSRC_READ_SAS_TOKEN)"
DestinationPath="%(ComponentToDownload.DownloadDestination)" />
<ItemGroup>