Remove unused PublishPrebuiltReportData target and associated tasks (#17212)
This commit is contained in:
Michael Simons
GitHub
parent
39f5eb5ef8
commit
7ace008866
5 changed files with 0 additions and 1040 deletions
|
|
@ -163,31 +163,6 @@
|
|||
<Message Importance="High" Text="Packaged smoke-test prereqs in '$(SmokeTestPrereqsTarballName)'" />
|
||||
</Target>
|
||||
|
||||
<UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="UploadToAzure" />
|
||||
|
||||
<Target Name="PublishPrebuiltReportData">
|
||||
<Error Text="RelativeBlobPath must be set to a non-empty string." Condition="'$(RelativeBlobPath)' == ''" />
|
||||
<Error Text="ContainerName must be set to a non-empty string." Condition="'$(ContainerName)' == ''" />
|
||||
<Error Text="AzureAccountName must be set to a non-empty string." Condition="'$(AzureAccountName)' == ''" />
|
||||
<Error Text="AzureAccessToken must be set to a non-empty string." Condition="'$(AzureAccessToken)' == ''" />
|
||||
|
||||
<ItemGroup>
|
||||
<ItemsToPublish Include="$(PackageReportDir)*.xml" />
|
||||
<ItemsToPublish Include="$(PackageReportDir)*.csv" />
|
||||
<ItemsToPublish>
|
||||
<RelativeBlobPath>$(RelativeBlobPath)/%(Filename)%(Extension)</RelativeBlobPath>
|
||||
</ItemsToPublish>
|
||||
</ItemGroup>
|
||||
|
||||
<Message Text="Uploading files to '$(AzureAccountName)' blob storage at $(ContainerName)/$(RelativeBlobPath)" />
|
||||
|
||||
<UploadToAzure AccountName="$(AzureAccountName)"
|
||||
AccountKey="$(AzureAccessToken)"
|
||||
ContainerName="$(ContainerName)"
|
||||
Items="@(ItemsToPublish)"
|
||||
Overwrite="true" />
|
||||
</Target>
|
||||
|
||||
<Target Name="CreatePrebuiltsTarball"
|
||||
AfterTargets="Build"
|
||||
DependsOnTargets="
|
||||
|
|
|
|||
|
|
@ -1,61 +0,0 @@
|
|||
// Licensed to the .NET Foundation under one or more agreements.
|
||||
// The .NET Foundation licenses this file to you under the MIT license.
|
||||
// See the LICENSE file in the project root for more information.
|
||||
|
||||
using Microsoft.Build.Utilities;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace Microsoft.DotNet.Build.Tasks
|
||||
{
|
||||
public abstract class AzureConnectionStringBuildTask : Task
|
||||
{
|
||||
/// <summary>
|
||||
/// Azure Storage account connection string. Supersedes Account Key / Name.
|
||||
/// Will cause errors if both are set.
|
||||
/// </summary>
|
||||
public string ConnectionString { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The Azure account key used when creating the connection string.
|
||||
/// When we fully deprecate these, can just make them get; only.
|
||||
/// </summary>
|
||||
public string AccountKey { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The Azure account name used when creating the connection string.
|
||||
/// When we fully deprecate these, can just make them get; only.
|
||||
/// </summary>
|
||||
public string AccountName { get; set; }
|
||||
|
||||
public void ParseConnectionString()
|
||||
{
|
||||
if (!string.IsNullOrEmpty(ConnectionString))
|
||||
{
|
||||
if (!(string.IsNullOrEmpty(AccountKey) && string.IsNullOrEmpty(AccountName)))
|
||||
{
|
||||
Log.LogError("If the ConnectionString property is set, you must not provide AccountKey / AccountName. These values will be deprecated in the future.");
|
||||
}
|
||||
else
|
||||
{
|
||||
Regex storageConnectionStringRegex = new Regex("AccountName=(?<name>.+?);AccountKey=(?<key>.+?);");
|
||||
|
||||
MatchCollection matches = storageConnectionStringRegex.Matches(ConnectionString);
|
||||
if (matches.Count > 0)
|
||||
{
|
||||
// When we deprecate this format, we'll want to demote these to private
|
||||
AccountName = matches[0].Groups["name"].Value;
|
||||
AccountKey = matches[0].Groups["key"].Value;
|
||||
}
|
||||
else
|
||||
{
|
||||
Log.LogError("Error parsing connection string. Please review its value.");
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (string.IsNullOrEmpty(AccountKey) || string.IsNullOrEmpty(AccountName))
|
||||
{
|
||||
Log.LogError("Error, must provide either ConnectionString or AccountName with AccountKey");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,461 +0,0 @@
|
|||
// Licensed to the .NET Foundation under one or more agreements.
|
||||
// The .NET Foundation licenses this file to you under the MIT license.
|
||||
// See the LICENSE file in the project root for more information.
|
||||
|
||||
using Microsoft.Build.Framework;
|
||||
using Microsoft.Build.Utilities;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace Microsoft.DotNet.Build.Tasks
|
||||
{
|
||||
public static class AzureHelper
|
||||
{
|
||||
/// <summary>
|
||||
/// The storage api version.
|
||||
/// </summary>
|
||||
public static readonly string StorageApiVersion = "2015-04-05";
|
||||
public const string DateHeaderString = "x-ms-date";
|
||||
public const string VersionHeaderString = "x-ms-version";
|
||||
public const string AuthorizationHeaderString = "Authorization";
|
||||
public const string CacheControlString = "x-ms-blob-cache-control";
|
||||
public const string ContentTypeString = "x-ms-blob-content-type";
|
||||
|
||||
public enum SasAccessType
|
||||
{
|
||||
Read,
|
||||
Write,
|
||||
};
|
||||
|
||||
public static string AuthorizationHeader(
|
||||
string storageAccount,
|
||||
string storageKey,
|
||||
string method,
|
||||
DateTime now,
|
||||
HttpRequestMessage request,
|
||||
string ifMatch = "",
|
||||
string contentMD5 = "",
|
||||
string size = "",
|
||||
string contentType = "")
|
||||
{
|
||||
string stringToSign = string.Format(
|
||||
"{0}\n\n\n{1}\n{5}\n{6}\n\n\n{2}\n\n\n\n{3}{4}",
|
||||
method,
|
||||
(size == string.Empty) ? string.Empty : size,
|
||||
ifMatch,
|
||||
GetCanonicalizedHeaders(request),
|
||||
GetCanonicalizedResource(request.RequestUri, storageAccount),
|
||||
contentMD5,
|
||||
contentType);
|
||||
byte[] signatureBytes = Encoding.UTF8.GetBytes(stringToSign);
|
||||
string authorizationHeader;
|
||||
using (HMACSHA256 hmacsha256 = new HMACSHA256(Convert.FromBase64String(storageKey)))
|
||||
{
|
||||
authorizationHeader = "SharedKey " + storageAccount + ":"
|
||||
+ Convert.ToBase64String(hmacsha256.ComputeHash(signatureBytes));
|
||||
}
|
||||
|
||||
return authorizationHeader;
|
||||
}
|
||||
|
||||
public static string CreateContainerSasToken(
|
||||
string accountName,
|
||||
string containerName,
|
||||
string key,
|
||||
SasAccessType accessType,
|
||||
int validityTimeInDays)
|
||||
{
|
||||
string signedPermissions = string.Empty;
|
||||
switch (accessType)
|
||||
{
|
||||
case SasAccessType.Read:
|
||||
signedPermissions = "r";
|
||||
break;
|
||||
case SasAccessType.Write:
|
||||
signedPermissions = "wdl";
|
||||
break;
|
||||
default:
|
||||
throw new ArgumentOutOfRangeException(nameof(accessType), accessType, "Unrecognized value");
|
||||
}
|
||||
|
||||
string signedStart = DateTime.UtcNow.ToString("O");
|
||||
string signedExpiry = DateTime.UtcNow.AddDays(validityTimeInDays).ToString("O");
|
||||
string canonicalizedResource = "/blob/" + accountName + "/" + containerName;
|
||||
string signedIdentifier = string.Empty;
|
||||
string signedVersion = StorageApiVersion;
|
||||
|
||||
string stringToSign = ConstructServiceStringToSign(
|
||||
signedPermissions,
|
||||
signedVersion,
|
||||
signedExpiry,
|
||||
canonicalizedResource,
|
||||
signedIdentifier,
|
||||
signedStart);
|
||||
|
||||
byte[] signatureBytes = Encoding.UTF8.GetBytes(stringToSign);
|
||||
string signature;
|
||||
using (HMACSHA256 hmacSha256 = new HMACSHA256(Convert.FromBase64String(key)))
|
||||
{
|
||||
signature = Convert.ToBase64String(hmacSha256.ComputeHash(signatureBytes));
|
||||
}
|
||||
|
||||
string sasToken = string.Format(
|
||||
"?sv={0}&sr={1}&sig={2}&st={3}&se={4}&sp={5}",
|
||||
WebUtility.UrlEncode(signedVersion),
|
||||
WebUtility.UrlEncode("c"),
|
||||
WebUtility.UrlEncode(signature),
|
||||
WebUtility.UrlEncode(signedStart),
|
||||
WebUtility.UrlEncode(signedExpiry),
|
||||
WebUtility.UrlEncode(signedPermissions));
|
||||
|
||||
return sasToken;
|
||||
}
|
||||
|
||||
public static string GetCanonicalizedHeaders(HttpRequestMessage request)
|
||||
{
|
||||
StringBuilder sb = new StringBuilder();
|
||||
List<string> headerNameList = (from headerName in request.Headers
|
||||
where
|
||||
headerName.Key.ToLowerInvariant()
|
||||
.StartsWith("x-ms-", StringComparison.Ordinal)
|
||||
select headerName.Key.ToLowerInvariant()).ToList();
|
||||
headerNameList.Sort();
|
||||
foreach (string headerName in headerNameList)
|
||||
{
|
||||
StringBuilder builder = new StringBuilder(headerName);
|
||||
string separator = ":";
|
||||
foreach (string headerValue in GetHeaderValues(request.Headers, headerName))
|
||||
{
|
||||
string trimmedValue = headerValue.Replace("\r\n", string.Empty);
|
||||
builder.Append(separator);
|
||||
builder.Append(trimmedValue);
|
||||
separator = ",";
|
||||
}
|
||||
|
||||
sb.Append(builder);
|
||||
sb.Append("\n");
|
||||
}
|
||||
|
||||
return sb.ToString();
|
||||
}
|
||||
|
||||
public static string GetCanonicalizedResource(Uri address, string accountName)
|
||||
{
|
||||
StringBuilder str = new StringBuilder();
|
||||
StringBuilder builder = new StringBuilder("/");
|
||||
builder.Append(accountName);
|
||||
builder.Append(address.AbsolutePath);
|
||||
str.Append(builder);
|
||||
Dictionary<string, HashSet<string>> queryKeyValues = ExtractQueryKeyValues(address);
|
||||
Dictionary<string, HashSet<string>> dictionary = GetCommaSeparatedList(queryKeyValues);
|
||||
|
||||
foreach (KeyValuePair<string, HashSet<string>> pair in dictionary.OrderBy(p => p.Key))
|
||||
{
|
||||
StringBuilder stringBuilder = new StringBuilder(string.Empty);
|
||||
stringBuilder.Append(pair.Key + ":");
|
||||
string commaList = string.Join(",", pair.Value);
|
||||
stringBuilder.Append(commaList);
|
||||
str.Append("\n");
|
||||
str.Append(stringBuilder);
|
||||
}
|
||||
|
||||
return str.ToString();
|
||||
}
|
||||
|
||||
public static List<string> GetHeaderValues(HttpRequestHeaders headers, string headerName)
|
||||
{
|
||||
List<string> list = new List<string>();
|
||||
IEnumerable<string> values;
|
||||
headers.TryGetValues(headerName, out values);
|
||||
if (values != null)
|
||||
{
|
||||
list.Add((values.FirstOrDefault() ?? string.Empty).TrimStart(null));
|
||||
}
|
||||
|
||||
return list;
|
||||
}
|
||||
|
||||
private static bool IsWithinRetryRange(HttpStatusCode statusCode)
|
||||
{
|
||||
// Retry on http client and server error codes (4xx - 5xx) as well as redirect
|
||||
|
||||
var rawStatus = (int)statusCode;
|
||||
if (rawStatus == 302)
|
||||
return true;
|
||||
else if (rawStatus >= 400 && rawStatus <= 599)
|
||||
return true;
|
||||
else
|
||||
return false;
|
||||
}
|
||||
|
||||
public static async Task<HttpResponseMessage> RequestWithRetry(TaskLoggingHelper loggingHelper, HttpClient client,
|
||||
Func<HttpRequestMessage> createRequest, Func<HttpResponseMessage, bool> validationCallback = null, int retryCount = 5,
|
||||
int retryDelaySeconds = 5)
|
||||
{
|
||||
if (loggingHelper == null)
|
||||
throw new ArgumentNullException(nameof(loggingHelper));
|
||||
if (client == null)
|
||||
throw new ArgumentNullException(nameof(client));
|
||||
if (createRequest == null)
|
||||
throw new ArgumentNullException(nameof(createRequest));
|
||||
if (retryCount < 1)
|
||||
throw new ArgumentException(nameof(retryCount));
|
||||
if (retryDelaySeconds < 1)
|
||||
throw new ArgumentException(nameof(retryDelaySeconds));
|
||||
|
||||
int retries = 0;
|
||||
HttpResponseMessage response = null;
|
||||
|
||||
// add a bit of randomness to the retry delay
|
||||
var rng = new Random();
|
||||
|
||||
while (retries < retryCount)
|
||||
{
|
||||
if (retries > 0)
|
||||
{
|
||||
if (response != null)
|
||||
{
|
||||
response.Dispose();
|
||||
response = null;
|
||||
}
|
||||
|
||||
int delay = retryDelaySeconds * retries * rng.Next(1, 5);
|
||||
loggingHelper.LogMessage(MessageImportance.Low, "Waiting {0} seconds before retry", delay);
|
||||
await System.Threading.Tasks.Task.Delay(delay * 1000);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
using (var request = createRequest())
|
||||
response = await client.SendAsync(request);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
loggingHelper.LogWarningFromException(e, true);
|
||||
|
||||
// if this is the final iteration let the exception bubble up
|
||||
if (retries + 1 == retryCount)
|
||||
throw;
|
||||
}
|
||||
|
||||
// response can be null if we fail to send the request
|
||||
if (response != null)
|
||||
{
|
||||
if (validationCallback == null)
|
||||
{
|
||||
// check if the response code is within the range of failures
|
||||
if (!IsWithinRetryRange(response.StatusCode))
|
||||
{
|
||||
return response;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
bool isSuccess = validationCallback(response);
|
||||
if (!isSuccess)
|
||||
{
|
||||
loggingHelper.LogMessage("Validation callback returned retry for status code {0}", response.StatusCode);
|
||||
}
|
||||
else
|
||||
{
|
||||
loggingHelper.LogMessage("Validation callback returned success for status code {0}", response.StatusCode);
|
||||
return response;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
++retries;
|
||||
}
|
||||
|
||||
// retry count exceeded
|
||||
loggingHelper.LogWarning("Retry count {0} exceeded", retryCount);
|
||||
|
||||
// set some default values in case response is null
|
||||
var statusCode = "None";
|
||||
var contentStr = "Null";
|
||||
if (response != null)
|
||||
{
|
||||
statusCode = response.StatusCode.ToString();
|
||||
contentStr = await response.Content.ReadAsStringAsync();
|
||||
response.Dispose();
|
||||
}
|
||||
|
||||
throw new HttpRequestException($"Request {createRequest().RequestUri} failed with status {statusCode}. Response : {contentStr}");
|
||||
}
|
||||
|
||||
private static string ConstructServiceStringToSign(
|
||||
string signedPermissions,
|
||||
string signedVersion,
|
||||
string signedExpiry,
|
||||
string canonicalizedResource,
|
||||
string signedIdentifier,
|
||||
string signedStart,
|
||||
string signedIP = "",
|
||||
string signedProtocol = "",
|
||||
string rscc = "",
|
||||
string rscd = "",
|
||||
string rsce = "",
|
||||
string rscl = "",
|
||||
string rsct = "")
|
||||
{
|
||||
// constructing string to sign based on spec in https://msdn.microsoft.com/en-us/library/azure/dn140255.aspx
|
||||
var stringToSign = string.Join(
|
||||
"\n",
|
||||
signedPermissions,
|
||||
signedStart,
|
||||
signedExpiry,
|
||||
canonicalizedResource,
|
||||
signedIdentifier,
|
||||
signedIP,
|
||||
signedProtocol,
|
||||
signedVersion,
|
||||
rscc,
|
||||
rscd,
|
||||
rsce,
|
||||
rscl,
|
||||
rsct);
|
||||
return stringToSign;
|
||||
}
|
||||
|
||||
private static Dictionary<string, HashSet<string>> ExtractQueryKeyValues(Uri address)
|
||||
{
|
||||
Dictionary<string, HashSet<string>> values = new Dictionary<string, HashSet<string>>();
|
||||
//Decode this to allow the regex to pull out the correct groups for signing
|
||||
address = new Uri(WebUtility.UrlDecode(address.ToString()));
|
||||
Regex newreg = new Regex(@"(?:\?|&)([^=]+)=([^&]+)");
|
||||
MatchCollection matches = newreg.Matches(address.Query);
|
||||
foreach (Match match in matches)
|
||||
{
|
||||
string key, value;
|
||||
if (!string.IsNullOrEmpty(match.Groups[1].Value))
|
||||
{
|
||||
key = match.Groups[1].Value;
|
||||
value = match.Groups[2].Value;
|
||||
}
|
||||
else
|
||||
{
|
||||
key = match.Groups[3].Value;
|
||||
value = match.Groups[4].Value;
|
||||
}
|
||||
|
||||
HashSet<string> setOfValues;
|
||||
if (values.TryGetValue(key, out setOfValues))
|
||||
{
|
||||
setOfValues.Add(value);
|
||||
}
|
||||
else
|
||||
{
|
||||
HashSet<string> newSet = new HashSet<string> { value };
|
||||
values.Add(key, newSet);
|
||||
}
|
||||
}
|
||||
|
||||
return values;
|
||||
}
|
||||
|
||||
private static Dictionary<string, HashSet<string>> GetCommaSeparatedList(
|
||||
Dictionary<string, HashSet<string>> queryKeyValues)
|
||||
{
|
||||
Dictionary<string, HashSet<string>> dictionary = new Dictionary<string, HashSet<string>>();
|
||||
|
||||
foreach (string queryKeys in queryKeyValues.Keys)
|
||||
{
|
||||
HashSet<string> setOfValues;
|
||||
queryKeyValues.TryGetValue(queryKeys, out setOfValues);
|
||||
List<string> list = new List<string>();
|
||||
list.AddRange(setOfValues);
|
||||
list.Sort();
|
||||
string commaSeparatedValues = string.Join(",", list);
|
||||
string key = queryKeys.ToLowerInvariant();
|
||||
HashSet<string> setOfValues2;
|
||||
if (dictionary.TryGetValue(key, out setOfValues2))
|
||||
{
|
||||
setOfValues2.Add(commaSeparatedValues);
|
||||
}
|
||||
else
|
||||
{
|
||||
HashSet<string> newSet = new HashSet<string> { commaSeparatedValues };
|
||||
dictionary.Add(key, newSet);
|
||||
}
|
||||
}
|
||||
|
||||
return dictionary;
|
||||
}
|
||||
|
||||
public static Func<HttpRequestMessage> RequestMessage(string method, string url, string accountName, string accountKey, List<Tuple<string, string>> additionalHeaders = null, string body = null)
|
||||
{
|
||||
Func<HttpRequestMessage> requestFunc = () =>
|
||||
{
|
||||
HttpMethod httpMethod = HttpMethod.Get;
|
||||
if (method == "PUT")
|
||||
{
|
||||
httpMethod = HttpMethod.Put;
|
||||
}
|
||||
else if (method == "DELETE")
|
||||
{
|
||||
httpMethod = HttpMethod.Delete;
|
||||
}
|
||||
DateTime dateTime = DateTime.UtcNow;
|
||||
var request = new HttpRequestMessage(httpMethod, url);
|
||||
request.Headers.Add(AzureHelper.DateHeaderString, dateTime.ToString("R", CultureInfo.InvariantCulture));
|
||||
request.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion);
|
||||
if (additionalHeaders != null)
|
||||
{
|
||||
foreach (Tuple<string, string> additionalHeader in additionalHeaders)
|
||||
{
|
||||
request.Headers.Add(additionalHeader.Item1, additionalHeader.Item2);
|
||||
}
|
||||
}
|
||||
if (body != null)
|
||||
{
|
||||
request.Content = new StringContent(body);
|
||||
request.Headers.Add(AzureHelper.AuthorizationHeaderString, AzureHelper.AuthorizationHeader(
|
||||
accountName,
|
||||
accountKey,
|
||||
method,
|
||||
dateTime,
|
||||
request,
|
||||
"",
|
||||
"",
|
||||
request.Content.Headers.ContentLength.ToString(),
|
||||
request.Content.Headers.ContentType.ToString()));
|
||||
}
|
||||
else
|
||||
{
|
||||
request.Headers.Add(AzureHelper.AuthorizationHeaderString, AzureHelper.AuthorizationHeader(
|
||||
accountName,
|
||||
accountKey,
|
||||
method,
|
||||
dateTime,
|
||||
request));
|
||||
}
|
||||
return request;
|
||||
};
|
||||
return requestFunc;
|
||||
}
|
||||
|
||||
public static string GetRootRestUrl(string accountName)
|
||||
{
|
||||
return $"https://{accountName}.blob.core.windows.net";
|
||||
}
|
||||
|
||||
public static string GetContainerRestUrl(string accountName, string containerName)
|
||||
{
|
||||
return $"{GetRootRestUrl(accountName)}/{containerName}";
|
||||
}
|
||||
|
||||
public static string GetBlobRestUrl(string accountName, string containerName, string blob)
|
||||
{
|
||||
return $"{GetContainerRestUrl(accountName, containerName)}/{blob}";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,285 +0,0 @@
|
|||
// Licensed to the .NET Foundation under one or more agreements.
|
||||
// The .NET Foundation licenses this file to you under the MIT license.
|
||||
// See the LICENSE file in the project root for more information.
|
||||
|
||||
using Microsoft.Build.Framework;
|
||||
using Microsoft.Build.Utilities;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Task = System.Threading.Tasks.Task;
|
||||
|
||||
namespace Microsoft.DotNet.Build.Tasks
|
||||
{
|
||||
public class UploadClient
|
||||
{
|
||||
private TaskLoggingHelper log;
|
||||
|
||||
public UploadClient(TaskLoggingHelper loggingHelper)
|
||||
{
|
||||
log = loggingHelper;
|
||||
}
|
||||
|
||||
public string EncodeBlockIds(int numberOfBlocks, int lengthOfId)
|
||||
{
|
||||
string numberOfBlocksString = numberOfBlocks.ToString("D" + lengthOfId);
|
||||
if (Encoding.UTF8.GetByteCount(numberOfBlocksString) <= 64)
|
||||
{
|
||||
byte[] bytes = Encoding.UTF8.GetBytes(numberOfBlocksString);
|
||||
return Convert.ToBase64String(bytes);
|
||||
}
|
||||
else
|
||||
{
|
||||
throw new Exception("Task failed - Could not encode block id.");
|
||||
}
|
||||
}
|
||||
|
||||
public async Task UploadBlockBlobAsync(
|
||||
CancellationToken ct,
|
||||
string AccountName,
|
||||
string AccountKey,
|
||||
string ContainerName,
|
||||
string filePath,
|
||||
string destinationBlob,
|
||||
string contentType,
|
||||
int uploadTimeout,
|
||||
string leaseId = "")
|
||||
{
|
||||
string resourceUrl = AzureHelper.GetContainerRestUrl(AccountName, ContainerName);
|
||||
|
||||
string fileName = destinationBlob;
|
||||
fileName = fileName.Replace("\\", "/");
|
||||
string blobUploadUrl = resourceUrl + "/" + fileName;
|
||||
int size = (int)new FileInfo(filePath).Length;
|
||||
int blockSize = 4 * 1024 * 1024; //4MB max size of a block blob
|
||||
int bytesLeft = size;
|
||||
List<string> blockIds = new List<string>();
|
||||
int numberOfBlocks = (size / blockSize) + 1;
|
||||
int countForId = 0;
|
||||
using (FileStream fileStreamTofilePath = new FileStream(filePath, FileMode.Open, FileAccess.Read))
|
||||
{
|
||||
int offset = 0;
|
||||
|
||||
while (bytesLeft > 0)
|
||||
{
|
||||
int nextBytesToRead = (bytesLeft < blockSize) ? bytesLeft : blockSize;
|
||||
byte[] fileBytes = new byte[blockSize];
|
||||
int read = fileStreamTofilePath.Read(fileBytes, 0, nextBytesToRead);
|
||||
|
||||
if (nextBytesToRead != read)
|
||||
{
|
||||
throw new Exception(string.Format(
|
||||
"Number of bytes read ({0}) from file {1} isn't equal to the number of bytes expected ({2}) .",
|
||||
read, fileName, nextBytesToRead));
|
||||
}
|
||||
|
||||
string blockId = EncodeBlockIds(countForId, numberOfBlocks.ToString().Length);
|
||||
|
||||
blockIds.Add(blockId);
|
||||
string blockUploadUrl = blobUploadUrl + "?comp=block&blockid=" + WebUtility.UrlEncode(blockId);
|
||||
|
||||
using (HttpClient client = new HttpClient())
|
||||
{
|
||||
client.DefaultRequestHeaders.Clear();
|
||||
|
||||
// In random occassions the request fails if the network is slow and it takes more than 100 seconds to upload 4MB.
|
||||
client.Timeout = TimeSpan.FromMinutes(uploadTimeout);
|
||||
Func<HttpRequestMessage> createRequest = () =>
|
||||
{
|
||||
DateTime dt = DateTime.UtcNow;
|
||||
var req = new HttpRequestMessage(HttpMethod.Put, blockUploadUrl);
|
||||
req.Headers.Add(
|
||||
AzureHelper.DateHeaderString,
|
||||
dt.ToString("R", CultureInfo.InvariantCulture));
|
||||
req.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion);
|
||||
if (!string.IsNullOrWhiteSpace(leaseId))
|
||||
{
|
||||
log.LogMessage($"Sending request: {leaseId} {blockUploadUrl}");
|
||||
req.Headers.Add("x-ms-lease-id", leaseId);
|
||||
}
|
||||
req.Headers.Add(
|
||||
AzureHelper.AuthorizationHeaderString,
|
||||
AzureHelper.AuthorizationHeader(
|
||||
AccountName,
|
||||
AccountKey,
|
||||
"PUT",
|
||||
dt,
|
||||
req,
|
||||
string.Empty,
|
||||
string.Empty,
|
||||
nextBytesToRead.ToString(),
|
||||
string.Empty));
|
||||
|
||||
Stream postStream = new MemoryStream();
|
||||
postStream.Write(fileBytes, 0, nextBytesToRead);
|
||||
postStream.Seek(0, SeekOrigin.Begin);
|
||||
req.Content = new StreamContent(postStream);
|
||||
return req;
|
||||
};
|
||||
|
||||
log.LogMessage(MessageImportance.Low, "Sending request to upload part {0} of file {1}", countForId, fileName);
|
||||
|
||||
using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(log, client, createRequest))
|
||||
{
|
||||
log.LogMessage(
|
||||
MessageImportance.Low,
|
||||
"Received response to upload part {0} of file {1}: Status Code:{2} Status Desc: {3}",
|
||||
countForId,
|
||||
fileName,
|
||||
response.StatusCode,
|
||||
await response.Content.ReadAsStringAsync());
|
||||
}
|
||||
}
|
||||
|
||||
offset += read;
|
||||
bytesLeft -= nextBytesToRead;
|
||||
countForId += 1;
|
||||
}
|
||||
}
|
||||
|
||||
string blockListUploadUrl = blobUploadUrl + "?comp=blocklist";
|
||||
|
||||
using (HttpClient client = new HttpClient())
|
||||
{
|
||||
Func<HttpRequestMessage> createRequest = () =>
|
||||
{
|
||||
DateTime dt1 = DateTime.UtcNow;
|
||||
var req = new HttpRequestMessage(HttpMethod.Put, blockListUploadUrl);
|
||||
req.Headers.Add(AzureHelper.DateHeaderString, dt1.ToString("R", CultureInfo.InvariantCulture));
|
||||
req.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion);
|
||||
if (string.IsNullOrEmpty(contentType))
|
||||
{
|
||||
contentType = DetermineContentTypeBasedOnFileExtension(filePath);
|
||||
}
|
||||
if (!string.IsNullOrEmpty(contentType))
|
||||
{
|
||||
req.Headers.Add(AzureHelper.ContentTypeString, contentType);
|
||||
}
|
||||
string cacheControl = DetermineCacheControlBasedOnFileExtension(filePath);
|
||||
if (!string.IsNullOrEmpty(cacheControl))
|
||||
{
|
||||
req.Headers.Add(AzureHelper.CacheControlString, cacheControl);
|
||||
}
|
||||
|
||||
var body = new StringBuilder("<?xml version=\"1.0\" encoding=\"UTF-8\"?><BlockList>");
|
||||
foreach (object item in blockIds)
|
||||
body.AppendFormat("<Latest>{0}</Latest>", item);
|
||||
|
||||
body.Append("</BlockList>");
|
||||
byte[] bodyData = Encoding.UTF8.GetBytes(body.ToString());
|
||||
if (!string.IsNullOrWhiteSpace(leaseId))
|
||||
{
|
||||
log.LogMessage($"Sending list request: {leaseId} {blockListUploadUrl}");
|
||||
req.Headers.Add("x-ms-lease-id", leaseId);
|
||||
}
|
||||
req.Headers.Add(
|
||||
AzureHelper.AuthorizationHeaderString,
|
||||
AzureHelper.AuthorizationHeader(
|
||||
AccountName,
|
||||
AccountKey,
|
||||
"PUT",
|
||||
dt1,
|
||||
req,
|
||||
string.Empty,
|
||||
string.Empty,
|
||||
bodyData.Length.ToString(),
|
||||
string.Empty));
|
||||
|
||||
Stream postStream = new MemoryStream();
|
||||
postStream.Write(bodyData, 0, bodyData.Length);
|
||||
postStream.Seek(0, SeekOrigin.Begin);
|
||||
req.Content = new StreamContent(postStream);
|
||||
return req;
|
||||
};
|
||||
|
||||
using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(log, client, createRequest))
|
||||
{
|
||||
log.LogMessage(
|
||||
MessageImportance.Low,
|
||||
"Received response to combine block list for file {0}: Status Code:{1} Status Desc: {2}",
|
||||
fileName,
|
||||
response.StatusCode,
|
||||
await response.Content.ReadAsStringAsync());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<bool> FileEqualsExistingBlobAsync(
|
||||
string accountName,
|
||||
string accountKey,
|
||||
string containerName,
|
||||
string filePath,
|
||||
string destinationBlob,
|
||||
int uploadTimeout)
|
||||
{
|
||||
using (var client = new HttpClient
|
||||
{
|
||||
Timeout = TimeSpan.FromMinutes(uploadTimeout)
|
||||
})
|
||||
{
|
||||
log.LogMessage(
|
||||
MessageImportance.Low,
|
||||
$"Downloading blob {destinationBlob} to check if identical.");
|
||||
|
||||
string blobUrl = AzureHelper.GetBlobRestUrl(accountName, containerName, destinationBlob);
|
||||
var createRequest = AzureHelper.RequestMessage("GET", blobUrl, accountName, accountKey);
|
||||
|
||||
using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(
|
||||
log,
|
||||
client,
|
||||
createRequest))
|
||||
{
|
||||
if (!response.IsSuccessStatusCode)
|
||||
{
|
||||
throw new HttpRequestException(
|
||||
$"Failed to retrieve existing blob {destinationBlob}, " +
|
||||
$"status code {response.StatusCode}.");
|
||||
}
|
||||
|
||||
byte[] existingBytes = await response.Content.ReadAsByteArrayAsync();
|
||||
byte[] localBytes = File.ReadAllBytes(filePath);
|
||||
|
||||
bool equal = localBytes.SequenceEqual(existingBytes);
|
||||
|
||||
if (equal)
|
||||
{
|
||||
log.LogMessage(
|
||||
MessageImportance.Normal,
|
||||
"Item exists in blob storage, and is verified to be identical. " +
|
||||
$"File: '{filePath}' Blob: '{destinationBlob}'");
|
||||
}
|
||||
|
||||
return equal;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private string DetermineContentTypeBasedOnFileExtension(string filename)
|
||||
{
|
||||
if (Path.GetExtension(filename) == ".svg")
|
||||
{
|
||||
return "image/svg+xml";
|
||||
}
|
||||
else if (Path.GetExtension(filename) == ".version")
|
||||
{
|
||||
return "text/plain";
|
||||
}
|
||||
return string.Empty;
|
||||
}
|
||||
private string DetermineCacheControlBasedOnFileExtension(string filename)
|
||||
{
|
||||
if (Path.GetExtension(filename) == ".svg")
|
||||
{
|
||||
return "No-Cache";
|
||||
}
|
||||
return string.Empty;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,208 +0,0 @@
|
|||
// Licensed to the .NET Foundation under one or more agreements.
|
||||
// The .NET Foundation licenses this file to you under the MIT license.
|
||||
// See the LICENSE file in the project root for more information.
|
||||
|
||||
using Microsoft.Build.Framework;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Net.Http;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using System.Xml;
|
||||
using ThreadingTask = System.Threading.Tasks.Task;
|
||||
|
||||
namespace Microsoft.DotNet.Build.Tasks
|
||||
{
|
||||
|
||||
public class UploadToAzure : AzureConnectionStringBuildTask, ICancelableTask
|
||||
{
|
||||
private static readonly CancellationTokenSource TokenSource = new CancellationTokenSource();
|
||||
private static readonly CancellationToken CancellationToken = TokenSource.Token;
|
||||
|
||||
/// <summary>
|
||||
/// The name of the container to access. The specified name must be in the correct format, see the
|
||||
/// following page for more info. https://msdn.microsoft.com/en-us/library/azure/dd135715.aspx
|
||||
/// </summary>
|
||||
[Required]
|
||||
public string ContainerName { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// An item group of files to upload. Each item must have metadata RelativeBlobPath
|
||||
/// that specifies the path relative to ContainerName where the item will be uploaded.
|
||||
/// </summary>
|
||||
[Required]
|
||||
public ITaskItem[] Items { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Indicates if the destination blob should be overwritten if it already exists. The default if false.
|
||||
/// </summary>
|
||||
public bool Overwrite { get; set; } = false;
|
||||
|
||||
/// <summary>
|
||||
/// Enables idempotency when Overwrite is false.
|
||||
///
|
||||
/// false: (default) Attempting to upload an item that already exists fails.
|
||||
///
|
||||
/// true: When an item already exists, download the existing blob to check if it's
|
||||
/// byte-for-byte identical to the one being uploaded. If so, pass. If not, fail.
|
||||
/// </summary>
|
||||
public bool PassIfExistingItemIdentical { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Specifies the maximum number of clients to concurrently upload blobs to azure
|
||||
/// </summary>
|
||||
public int MaxClients { get; set; } = 8;
|
||||
|
||||
public int UploadTimeoutInMinutes { get; set; } = 5;
|
||||
|
||||
public void Cancel()
|
||||
{
|
||||
TokenSource.Cancel();
|
||||
}
|
||||
|
||||
public override bool Execute()
|
||||
{
|
||||
return ExecuteAsync(CancellationToken).GetAwaiter().GetResult();
|
||||
}
|
||||
|
||||
public async Task<bool> ExecuteAsync(CancellationToken ct)
|
||||
{
|
||||
ParseConnectionString();
|
||||
// If the connection string AND AccountKey & AccountName are provided, error out.
|
||||
if (Log.HasLoggedErrors)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
Log.LogMessage(
|
||||
MessageImportance.Normal,
|
||||
"Begin uploading blobs to Azure account {0} in container {1}.",
|
||||
AccountName,
|
||||
ContainerName);
|
||||
|
||||
if (Items.Length == 0)
|
||||
{
|
||||
Log.LogError("No items were provided for upload.");
|
||||
return false;
|
||||
}
|
||||
|
||||
// first check what blobs are present
|
||||
string checkListUrl = $"{AzureHelper.GetContainerRestUrl(AccountName, ContainerName)}?restype=container&comp=list";
|
||||
|
||||
HashSet<string> blobsPresent = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
try
|
||||
{
|
||||
using (HttpClient client = new HttpClient())
|
||||
{
|
||||
var createRequest = AzureHelper.RequestMessage("GET", checkListUrl, AccountName, AccountKey);
|
||||
|
||||
Log.LogMessage(MessageImportance.Low, "Sending request to check whether Container blobs exist");
|
||||
using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest))
|
||||
{
|
||||
var doc = new XmlDocument();
|
||||
doc.LoadXml(await response.Content.ReadAsStringAsync());
|
||||
|
||||
XmlNodeList nodes = doc.DocumentElement.GetElementsByTagName("Blob");
|
||||
|
||||
foreach (XmlNode node in nodes)
|
||||
{
|
||||
blobsPresent.Add(node["Name"].InnerText);
|
||||
}
|
||||
|
||||
Log.LogMessage(MessageImportance.Low, "Received response to check whether Container blobs exist");
|
||||
}
|
||||
}
|
||||
|
||||
using (var clientThrottle = new SemaphoreSlim(this.MaxClients, this.MaxClients))
|
||||
{
|
||||
await ThreadingTask.WhenAll(Items.Select(item => UploadAsync(ct, item, blobsPresent, clientThrottle)));
|
||||
}
|
||||
|
||||
Log.LogMessage(MessageImportance.Normal, "Upload to Azure is complete, a total of {0} items were uploaded.", Items.Length);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Log.LogErrorFromException(e, true);
|
||||
}
|
||||
return !Log.HasLoggedErrors;
|
||||
}
|
||||
|
||||
private async ThreadingTask UploadAsync(CancellationToken ct, ITaskItem item, HashSet<string> blobsPresent, SemaphoreSlim clientThrottle)
|
||||
{
|
||||
if (ct.IsCancellationRequested)
|
||||
{
|
||||
Log.LogError("Task UploadToAzure cancelled");
|
||||
ct.ThrowIfCancellationRequested();
|
||||
}
|
||||
|
||||
string relativeBlobPath = item.GetMetadata("RelativeBlobPath");
|
||||
if (string.IsNullOrEmpty(relativeBlobPath))
|
||||
throw new Exception(string.Format("Metadata 'RelativeBlobPath' is missing for item '{0}'.", item.ItemSpec));
|
||||
|
||||
if (!File.Exists(item.ItemSpec))
|
||||
throw new Exception(string.Format("The file '{0}' does not exist.", item.ItemSpec));
|
||||
|
||||
UploadClient uploadClient = new UploadClient(Log);
|
||||
|
||||
if (!Overwrite && blobsPresent.Contains(relativeBlobPath))
|
||||
{
|
||||
if (PassIfExistingItemIdentical &&
|
||||
await ItemEqualsExistingBlobAsync(item, relativeBlobPath, uploadClient, clientThrottle))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
throw new Exception(string.Format("The blob '{0}' already exists.", relativeBlobPath));
|
||||
}
|
||||
|
||||
string contentType = item.GetMetadata("ContentType");
|
||||
|
||||
await clientThrottle.WaitAsync();
|
||||
|
||||
try
|
||||
{
|
||||
Log.LogMessage("Uploading {0} to {1}.", item.ItemSpec, ContainerName);
|
||||
await
|
||||
uploadClient.UploadBlockBlobAsync(
|
||||
ct,
|
||||
AccountName,
|
||||
AccountKey,
|
||||
ContainerName,
|
||||
item.ItemSpec,
|
||||
relativeBlobPath,
|
||||
contentType,
|
||||
UploadTimeoutInMinutes);
|
||||
}
|
||||
finally
|
||||
{
|
||||
clientThrottle.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<bool> ItemEqualsExistingBlobAsync(
|
||||
ITaskItem item,
|
||||
string relativeBlobPath,
|
||||
UploadClient client,
|
||||
SemaphoreSlim clientThrottle)
|
||||
{
|
||||
await clientThrottle.WaitAsync();
|
||||
try
|
||||
{
|
||||
return await client.FileEqualsExistingBlobAsync(
|
||||
AccountName,
|
||||
AccountKey,
|
||||
ContainerName,
|
||||
item.ItemSpec,
|
||||
relativeBlobPath,
|
||||
UploadTimeoutInMinutes);
|
||||
}
|
||||
finally
|
||||
{
|
||||
clientThrottle.Release();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue