diff --git a/src/SourceBuild/content/build.proj b/src/SourceBuild/content/build.proj index 55e5a66c0..b2227b954 100644 --- a/src/SourceBuild/content/build.proj +++ b/src/SourceBuild/content/build.proj @@ -163,31 +163,6 @@ - - - - - - - - - - - - - $(RelativeBlobPath)/%(Filename)%(Extension) - - - - - - - - .+?);AccountKey=(?.+?);"); - - MatchCollection matches = storageConnectionStringRegex.Matches(ConnectionString); - if (matches.Count > 0) - { - // When we deprecate this format, we'll want to demote these to private - AccountName = matches[0].Groups["name"].Value; - AccountKey = matches[0].Groups["key"].Value; - } - else - { - Log.LogError("Error parsing connection string. Please review its value."); - } - } - } - else if (string.IsNullOrEmpty(AccountKey) || string.IsNullOrEmpty(AccountName)) - { - Log.LogError("Error, must provide either ConnectionString or AccountName with AccountKey"); - } - } - } -} \ No newline at end of file diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/AzureHelper.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/AzureHelper.cs deleted file mode 100644 index ac08cfb3a..000000000 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/AzureHelper.cs +++ /dev/null @@ -1,461 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; -using System; -using System.Collections.Generic; -using System.Globalization; -using System.Linq; -using System.Net; -using System.Net.Http; -using System.Net.Http.Headers; -using System.Security.Cryptography; -using System.Text; -using System.Text.RegularExpressions; -using System.Threading.Tasks; - -namespace Microsoft.DotNet.Build.Tasks -{ - public static class AzureHelper - { - /// - /// The storage api version. - /// - public static readonly string StorageApiVersion = "2015-04-05"; - public const string DateHeaderString = "x-ms-date"; - public const string VersionHeaderString = "x-ms-version"; - public const string AuthorizationHeaderString = "Authorization"; - public const string CacheControlString = "x-ms-blob-cache-control"; - public const string ContentTypeString = "x-ms-blob-content-type"; - - public enum SasAccessType - { - Read, - Write, - }; - - public static string AuthorizationHeader( - string storageAccount, - string storageKey, - string method, - DateTime now, - HttpRequestMessage request, - string ifMatch = "", - string contentMD5 = "", - string size = "", - string contentType = "") - { - string stringToSign = string.Format( - "{0}\n\n\n{1}\n{5}\n{6}\n\n\n{2}\n\n\n\n{3}{4}", - method, - (size == string.Empty) ? string.Empty : size, - ifMatch, - GetCanonicalizedHeaders(request), - GetCanonicalizedResource(request.RequestUri, storageAccount), - contentMD5, - contentType); - byte[] signatureBytes = Encoding.UTF8.GetBytes(stringToSign); - string authorizationHeader; - using (HMACSHA256 hmacsha256 = new HMACSHA256(Convert.FromBase64String(storageKey))) - { - authorizationHeader = "SharedKey " + storageAccount + ":" - + Convert.ToBase64String(hmacsha256.ComputeHash(signatureBytes)); - } - - return authorizationHeader; - } - - public static string CreateContainerSasToken( - string accountName, - string containerName, - string key, - SasAccessType accessType, - int validityTimeInDays) - { - string signedPermissions = string.Empty; - switch (accessType) - { - case SasAccessType.Read: - signedPermissions = "r"; - break; - case SasAccessType.Write: - signedPermissions = "wdl"; - break; - default: - throw new ArgumentOutOfRangeException(nameof(accessType), accessType, "Unrecognized value"); - } - - string signedStart = DateTime.UtcNow.ToString("O"); - string signedExpiry = DateTime.UtcNow.AddDays(validityTimeInDays).ToString("O"); - string canonicalizedResource = "/blob/" + accountName + "/" + containerName; - string signedIdentifier = string.Empty; - string signedVersion = StorageApiVersion; - - string stringToSign = ConstructServiceStringToSign( - signedPermissions, - signedVersion, - signedExpiry, - canonicalizedResource, - signedIdentifier, - signedStart); - - byte[] signatureBytes = Encoding.UTF8.GetBytes(stringToSign); - string signature; - using (HMACSHA256 hmacSha256 = new HMACSHA256(Convert.FromBase64String(key))) - { - signature = Convert.ToBase64String(hmacSha256.ComputeHash(signatureBytes)); - } - - string sasToken = string.Format( - "?sv={0}&sr={1}&sig={2}&st={3}&se={4}&sp={5}", - WebUtility.UrlEncode(signedVersion), - WebUtility.UrlEncode("c"), - WebUtility.UrlEncode(signature), - WebUtility.UrlEncode(signedStart), - WebUtility.UrlEncode(signedExpiry), - WebUtility.UrlEncode(signedPermissions)); - - return sasToken; - } - - public static string GetCanonicalizedHeaders(HttpRequestMessage request) - { - StringBuilder sb = new StringBuilder(); - List headerNameList = (from headerName in request.Headers - where - headerName.Key.ToLowerInvariant() - .StartsWith("x-ms-", StringComparison.Ordinal) - select headerName.Key.ToLowerInvariant()).ToList(); - headerNameList.Sort(); - foreach (string headerName in headerNameList) - { - StringBuilder builder = new StringBuilder(headerName); - string separator = ":"; - foreach (string headerValue in GetHeaderValues(request.Headers, headerName)) - { - string trimmedValue = headerValue.Replace("\r\n", string.Empty); - builder.Append(separator); - builder.Append(trimmedValue); - separator = ","; - } - - sb.Append(builder); - sb.Append("\n"); - } - - return sb.ToString(); - } - - public static string GetCanonicalizedResource(Uri address, string accountName) - { - StringBuilder str = new StringBuilder(); - StringBuilder builder = new StringBuilder("/"); - builder.Append(accountName); - builder.Append(address.AbsolutePath); - str.Append(builder); - Dictionary> queryKeyValues = ExtractQueryKeyValues(address); - Dictionary> dictionary = GetCommaSeparatedList(queryKeyValues); - - foreach (KeyValuePair> pair in dictionary.OrderBy(p => p.Key)) - { - StringBuilder stringBuilder = new StringBuilder(string.Empty); - stringBuilder.Append(pair.Key + ":"); - string commaList = string.Join(",", pair.Value); - stringBuilder.Append(commaList); - str.Append("\n"); - str.Append(stringBuilder); - } - - return str.ToString(); - } - - public static List GetHeaderValues(HttpRequestHeaders headers, string headerName) - { - List list = new List(); - IEnumerable values; - headers.TryGetValues(headerName, out values); - if (values != null) - { - list.Add((values.FirstOrDefault() ?? string.Empty).TrimStart(null)); - } - - return list; - } - - private static bool IsWithinRetryRange(HttpStatusCode statusCode) - { - // Retry on http client and server error codes (4xx - 5xx) as well as redirect - - var rawStatus = (int)statusCode; - if (rawStatus == 302) - return true; - else if (rawStatus >= 400 && rawStatus <= 599) - return true; - else - return false; - } - - public static async Task RequestWithRetry(TaskLoggingHelper loggingHelper, HttpClient client, - Func createRequest, Func validationCallback = null, int retryCount = 5, - int retryDelaySeconds = 5) - { - if (loggingHelper == null) - throw new ArgumentNullException(nameof(loggingHelper)); - if (client == null) - throw new ArgumentNullException(nameof(client)); - if (createRequest == null) - throw new ArgumentNullException(nameof(createRequest)); - if (retryCount < 1) - throw new ArgumentException(nameof(retryCount)); - if (retryDelaySeconds < 1) - throw new ArgumentException(nameof(retryDelaySeconds)); - - int retries = 0; - HttpResponseMessage response = null; - - // add a bit of randomness to the retry delay - var rng = new Random(); - - while (retries < retryCount) - { - if (retries > 0) - { - if (response != null) - { - response.Dispose(); - response = null; - } - - int delay = retryDelaySeconds * retries * rng.Next(1, 5); - loggingHelper.LogMessage(MessageImportance.Low, "Waiting {0} seconds before retry", delay); - await System.Threading.Tasks.Task.Delay(delay * 1000); - } - - try - { - using (var request = createRequest()) - response = await client.SendAsync(request); - } - catch (Exception e) - { - loggingHelper.LogWarningFromException(e, true); - - // if this is the final iteration let the exception bubble up - if (retries + 1 == retryCount) - throw; - } - - // response can be null if we fail to send the request - if (response != null) - { - if (validationCallback == null) - { - // check if the response code is within the range of failures - if (!IsWithinRetryRange(response.StatusCode)) - { - return response; - } - } - else - { - bool isSuccess = validationCallback(response); - if (!isSuccess) - { - loggingHelper.LogMessage("Validation callback returned retry for status code {0}", response.StatusCode); - } - else - { - loggingHelper.LogMessage("Validation callback returned success for status code {0}", response.StatusCode); - return response; - } - } - } - - ++retries; - } - - // retry count exceeded - loggingHelper.LogWarning("Retry count {0} exceeded", retryCount); - - // set some default values in case response is null - var statusCode = "None"; - var contentStr = "Null"; - if (response != null) - { - statusCode = response.StatusCode.ToString(); - contentStr = await response.Content.ReadAsStringAsync(); - response.Dispose(); - } - - throw new HttpRequestException($"Request {createRequest().RequestUri} failed with status {statusCode}. Response : {contentStr}"); - } - - private static string ConstructServiceStringToSign( - string signedPermissions, - string signedVersion, - string signedExpiry, - string canonicalizedResource, - string signedIdentifier, - string signedStart, - string signedIP = "", - string signedProtocol = "", - string rscc = "", - string rscd = "", - string rsce = "", - string rscl = "", - string rsct = "") - { - // constructing string to sign based on spec in https://msdn.microsoft.com/en-us/library/azure/dn140255.aspx - var stringToSign = string.Join( - "\n", - signedPermissions, - signedStart, - signedExpiry, - canonicalizedResource, - signedIdentifier, - signedIP, - signedProtocol, - signedVersion, - rscc, - rscd, - rsce, - rscl, - rsct); - return stringToSign; - } - - private static Dictionary> ExtractQueryKeyValues(Uri address) - { - Dictionary> values = new Dictionary>(); - //Decode this to allow the regex to pull out the correct groups for signing - address = new Uri(WebUtility.UrlDecode(address.ToString())); - Regex newreg = new Regex(@"(?:\?|&)([^=]+)=([^&]+)"); - MatchCollection matches = newreg.Matches(address.Query); - foreach (Match match in matches) - { - string key, value; - if (!string.IsNullOrEmpty(match.Groups[1].Value)) - { - key = match.Groups[1].Value; - value = match.Groups[2].Value; - } - else - { - key = match.Groups[3].Value; - value = match.Groups[4].Value; - } - - HashSet setOfValues; - if (values.TryGetValue(key, out setOfValues)) - { - setOfValues.Add(value); - } - else - { - HashSet newSet = new HashSet { value }; - values.Add(key, newSet); - } - } - - return values; - } - - private static Dictionary> GetCommaSeparatedList( - Dictionary> queryKeyValues) - { - Dictionary> dictionary = new Dictionary>(); - - foreach (string queryKeys in queryKeyValues.Keys) - { - HashSet setOfValues; - queryKeyValues.TryGetValue(queryKeys, out setOfValues); - List list = new List(); - list.AddRange(setOfValues); - list.Sort(); - string commaSeparatedValues = string.Join(",", list); - string key = queryKeys.ToLowerInvariant(); - HashSet setOfValues2; - if (dictionary.TryGetValue(key, out setOfValues2)) - { - setOfValues2.Add(commaSeparatedValues); - } - else - { - HashSet newSet = new HashSet { commaSeparatedValues }; - dictionary.Add(key, newSet); - } - } - - return dictionary; - } - - public static Func RequestMessage(string method, string url, string accountName, string accountKey, List> additionalHeaders = null, string body = null) - { - Func requestFunc = () => - { - HttpMethod httpMethod = HttpMethod.Get; - if (method == "PUT") - { - httpMethod = HttpMethod.Put; - } - else if (method == "DELETE") - { - httpMethod = HttpMethod.Delete; - } - DateTime dateTime = DateTime.UtcNow; - var request = new HttpRequestMessage(httpMethod, url); - request.Headers.Add(AzureHelper.DateHeaderString, dateTime.ToString("R", CultureInfo.InvariantCulture)); - request.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion); - if (additionalHeaders != null) - { - foreach (Tuple additionalHeader in additionalHeaders) - { - request.Headers.Add(additionalHeader.Item1, additionalHeader.Item2); - } - } - if (body != null) - { - request.Content = new StringContent(body); - request.Headers.Add(AzureHelper.AuthorizationHeaderString, AzureHelper.AuthorizationHeader( - accountName, - accountKey, - method, - dateTime, - request, - "", - "", - request.Content.Headers.ContentLength.ToString(), - request.Content.Headers.ContentType.ToString())); - } - else - { - request.Headers.Add(AzureHelper.AuthorizationHeaderString, AzureHelper.AuthorizationHeader( - accountName, - accountKey, - method, - dateTime, - request)); - } - return request; - }; - return requestFunc; - } - - public static string GetRootRestUrl(string accountName) - { - return $"https://{accountName}.blob.core.windows.net"; - } - - public static string GetContainerRestUrl(string accountName, string containerName) - { - return $"{GetRootRestUrl(accountName)}/{containerName}"; - } - - public static string GetBlobRestUrl(string accountName, string containerName, string blob) - { - return $"{GetContainerRestUrl(accountName, containerName)}/{blob}"; - } - } -} \ No newline at end of file diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UploadClient.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UploadClient.cs deleted file mode 100644 index c02055705..000000000 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UploadClient.cs +++ /dev/null @@ -1,285 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; -using System; -using System.Collections.Generic; -using System.Globalization; -using System.IO; -using System.Linq; -using System.Net; -using System.Net.Http; -using System.Text; -using System.Threading; -using System.Threading.Tasks; -using Task = System.Threading.Tasks.Task; - -namespace Microsoft.DotNet.Build.Tasks -{ - public class UploadClient - { - private TaskLoggingHelper log; - - public UploadClient(TaskLoggingHelper loggingHelper) - { - log = loggingHelper; - } - - public string EncodeBlockIds(int numberOfBlocks, int lengthOfId) - { - string numberOfBlocksString = numberOfBlocks.ToString("D" + lengthOfId); - if (Encoding.UTF8.GetByteCount(numberOfBlocksString) <= 64) - { - byte[] bytes = Encoding.UTF8.GetBytes(numberOfBlocksString); - return Convert.ToBase64String(bytes); - } - else - { - throw new Exception("Task failed - Could not encode block id."); - } - } - - public async Task UploadBlockBlobAsync( - CancellationToken ct, - string AccountName, - string AccountKey, - string ContainerName, - string filePath, - string destinationBlob, - string contentType, - int uploadTimeout, - string leaseId = "") - { - string resourceUrl = AzureHelper.GetContainerRestUrl(AccountName, ContainerName); - - string fileName = destinationBlob; - fileName = fileName.Replace("\\", "/"); - string blobUploadUrl = resourceUrl + "/" + fileName; - int size = (int)new FileInfo(filePath).Length; - int blockSize = 4 * 1024 * 1024; //4MB max size of a block blob - int bytesLeft = size; - List blockIds = new List(); - int numberOfBlocks = (size / blockSize) + 1; - int countForId = 0; - using (FileStream fileStreamTofilePath = new FileStream(filePath, FileMode.Open, FileAccess.Read)) - { - int offset = 0; - - while (bytesLeft > 0) - { - int nextBytesToRead = (bytesLeft < blockSize) ? bytesLeft : blockSize; - byte[] fileBytes = new byte[blockSize]; - int read = fileStreamTofilePath.Read(fileBytes, 0, nextBytesToRead); - - if (nextBytesToRead != read) - { - throw new Exception(string.Format( - "Number of bytes read ({0}) from file {1} isn't equal to the number of bytes expected ({2}) .", - read, fileName, nextBytesToRead)); - } - - string blockId = EncodeBlockIds(countForId, numberOfBlocks.ToString().Length); - - blockIds.Add(blockId); - string blockUploadUrl = blobUploadUrl + "?comp=block&blockid=" + WebUtility.UrlEncode(blockId); - - using (HttpClient client = new HttpClient()) - { - client.DefaultRequestHeaders.Clear(); - - // In random occassions the request fails if the network is slow and it takes more than 100 seconds to upload 4MB. - client.Timeout = TimeSpan.FromMinutes(uploadTimeout); - Func createRequest = () => - { - DateTime dt = DateTime.UtcNow; - var req = new HttpRequestMessage(HttpMethod.Put, blockUploadUrl); - req.Headers.Add( - AzureHelper.DateHeaderString, - dt.ToString("R", CultureInfo.InvariantCulture)); - req.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion); - if (!string.IsNullOrWhiteSpace(leaseId)) - { - log.LogMessage($"Sending request: {leaseId} {blockUploadUrl}"); - req.Headers.Add("x-ms-lease-id", leaseId); - } - req.Headers.Add( - AzureHelper.AuthorizationHeaderString, - AzureHelper.AuthorizationHeader( - AccountName, - AccountKey, - "PUT", - dt, - req, - string.Empty, - string.Empty, - nextBytesToRead.ToString(), - string.Empty)); - - Stream postStream = new MemoryStream(); - postStream.Write(fileBytes, 0, nextBytesToRead); - postStream.Seek(0, SeekOrigin.Begin); - req.Content = new StreamContent(postStream); - return req; - }; - - log.LogMessage(MessageImportance.Low, "Sending request to upload part {0} of file {1}", countForId, fileName); - - using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(log, client, createRequest)) - { - log.LogMessage( - MessageImportance.Low, - "Received response to upload part {0} of file {1}: Status Code:{2} Status Desc: {3}", - countForId, - fileName, - response.StatusCode, - await response.Content.ReadAsStringAsync()); - } - } - - offset += read; - bytesLeft -= nextBytesToRead; - countForId += 1; - } - } - - string blockListUploadUrl = blobUploadUrl + "?comp=blocklist"; - - using (HttpClient client = new HttpClient()) - { - Func createRequest = () => - { - DateTime dt1 = DateTime.UtcNow; - var req = new HttpRequestMessage(HttpMethod.Put, blockListUploadUrl); - req.Headers.Add(AzureHelper.DateHeaderString, dt1.ToString("R", CultureInfo.InvariantCulture)); - req.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion); - if (string.IsNullOrEmpty(contentType)) - { - contentType = DetermineContentTypeBasedOnFileExtension(filePath); - } - if (!string.IsNullOrEmpty(contentType)) - { - req.Headers.Add(AzureHelper.ContentTypeString, contentType); - } - string cacheControl = DetermineCacheControlBasedOnFileExtension(filePath); - if (!string.IsNullOrEmpty(cacheControl)) - { - req.Headers.Add(AzureHelper.CacheControlString, cacheControl); - } - - var body = new StringBuilder(""); - foreach (object item in blockIds) - body.AppendFormat("{0}", item); - - body.Append(""); - byte[] bodyData = Encoding.UTF8.GetBytes(body.ToString()); - if (!string.IsNullOrWhiteSpace(leaseId)) - { - log.LogMessage($"Sending list request: {leaseId} {blockListUploadUrl}"); - req.Headers.Add("x-ms-lease-id", leaseId); - } - req.Headers.Add( - AzureHelper.AuthorizationHeaderString, - AzureHelper.AuthorizationHeader( - AccountName, - AccountKey, - "PUT", - dt1, - req, - string.Empty, - string.Empty, - bodyData.Length.ToString(), - string.Empty)); - - Stream postStream = new MemoryStream(); - postStream.Write(bodyData, 0, bodyData.Length); - postStream.Seek(0, SeekOrigin.Begin); - req.Content = new StreamContent(postStream); - return req; - }; - - using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(log, client, createRequest)) - { - log.LogMessage( - MessageImportance.Low, - "Received response to combine block list for file {0}: Status Code:{1} Status Desc: {2}", - fileName, - response.StatusCode, - await response.Content.ReadAsStringAsync()); - } - } - } - - public async Task FileEqualsExistingBlobAsync( - string accountName, - string accountKey, - string containerName, - string filePath, - string destinationBlob, - int uploadTimeout) - { - using (var client = new HttpClient - { - Timeout = TimeSpan.FromMinutes(uploadTimeout) - }) - { - log.LogMessage( - MessageImportance.Low, - $"Downloading blob {destinationBlob} to check if identical."); - - string blobUrl = AzureHelper.GetBlobRestUrl(accountName, containerName, destinationBlob); - var createRequest = AzureHelper.RequestMessage("GET", blobUrl, accountName, accountKey); - - using (HttpResponseMessage response = await AzureHelper.RequestWithRetry( - log, - client, - createRequest)) - { - if (!response.IsSuccessStatusCode) - { - throw new HttpRequestException( - $"Failed to retrieve existing blob {destinationBlob}, " + - $"status code {response.StatusCode}."); - } - - byte[] existingBytes = await response.Content.ReadAsByteArrayAsync(); - byte[] localBytes = File.ReadAllBytes(filePath); - - bool equal = localBytes.SequenceEqual(existingBytes); - - if (equal) - { - log.LogMessage( - MessageImportance.Normal, - "Item exists in blob storage, and is verified to be identical. " + - $"File: '{filePath}' Blob: '{destinationBlob}'"); - } - - return equal; - } - } - } - - private string DetermineContentTypeBasedOnFileExtension(string filename) - { - if (Path.GetExtension(filename) == ".svg") - { - return "image/svg+xml"; - } - else if (Path.GetExtension(filename) == ".version") - { - return "text/plain"; - } - return string.Empty; - } - private string DetermineCacheControlBasedOnFileExtension(string filename) - { - if (Path.GetExtension(filename) == ".svg") - { - return "No-Cache"; - } - return string.Empty; - } - } -} \ No newline at end of file diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UploadToAzure.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UploadToAzure.cs deleted file mode 100644 index 5431194d6..000000000 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UploadToAzure.cs +++ /dev/null @@ -1,208 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using Microsoft.Build.Framework; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Net.Http; -using System.Threading; -using System.Threading.Tasks; -using System.Xml; -using ThreadingTask = System.Threading.Tasks.Task; - -namespace Microsoft.DotNet.Build.Tasks -{ - - public class UploadToAzure : AzureConnectionStringBuildTask, ICancelableTask - { - private static readonly CancellationTokenSource TokenSource = new CancellationTokenSource(); - private static readonly CancellationToken CancellationToken = TokenSource.Token; - - /// - /// The name of the container to access. The specified name must be in the correct format, see the - /// following page for more info. https://msdn.microsoft.com/en-us/library/azure/dd135715.aspx - /// - [Required] - public string ContainerName { get; set; } - - /// - /// An item group of files to upload. Each item must have metadata RelativeBlobPath - /// that specifies the path relative to ContainerName where the item will be uploaded. - /// - [Required] - public ITaskItem[] Items { get; set; } - - /// - /// Indicates if the destination blob should be overwritten if it already exists. The default if false. - /// - public bool Overwrite { get; set; } = false; - - /// - /// Enables idempotency when Overwrite is false. - /// - /// false: (default) Attempting to upload an item that already exists fails. - /// - /// true: When an item already exists, download the existing blob to check if it's - /// byte-for-byte identical to the one being uploaded. If so, pass. If not, fail. - /// - public bool PassIfExistingItemIdentical { get; set; } - - /// - /// Specifies the maximum number of clients to concurrently upload blobs to azure - /// - public int MaxClients { get; set; } = 8; - - public int UploadTimeoutInMinutes { get; set; } = 5; - - public void Cancel() - { - TokenSource.Cancel(); - } - - public override bool Execute() - { - return ExecuteAsync(CancellationToken).GetAwaiter().GetResult(); - } - - public async Task ExecuteAsync(CancellationToken ct) - { - ParseConnectionString(); - // If the connection string AND AccountKey & AccountName are provided, error out. - if (Log.HasLoggedErrors) - { - return false; - } - - Log.LogMessage( - MessageImportance.Normal, - "Begin uploading blobs to Azure account {0} in container {1}.", - AccountName, - ContainerName); - - if (Items.Length == 0) - { - Log.LogError("No items were provided for upload."); - return false; - } - - // first check what blobs are present - string checkListUrl = $"{AzureHelper.GetContainerRestUrl(AccountName, ContainerName)}?restype=container&comp=list"; - - HashSet blobsPresent = new HashSet(StringComparer.OrdinalIgnoreCase); - - try - { - using (HttpClient client = new HttpClient()) - { - var createRequest = AzureHelper.RequestMessage("GET", checkListUrl, AccountName, AccountKey); - - Log.LogMessage(MessageImportance.Low, "Sending request to check whether Container blobs exist"); - using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest)) - { - var doc = new XmlDocument(); - doc.LoadXml(await response.Content.ReadAsStringAsync()); - - XmlNodeList nodes = doc.DocumentElement.GetElementsByTagName("Blob"); - - foreach (XmlNode node in nodes) - { - blobsPresent.Add(node["Name"].InnerText); - } - - Log.LogMessage(MessageImportance.Low, "Received response to check whether Container blobs exist"); - } - } - - using (var clientThrottle = new SemaphoreSlim(this.MaxClients, this.MaxClients)) - { - await ThreadingTask.WhenAll(Items.Select(item => UploadAsync(ct, item, blobsPresent, clientThrottle))); - } - - Log.LogMessage(MessageImportance.Normal, "Upload to Azure is complete, a total of {0} items were uploaded.", Items.Length); - } - catch (Exception e) - { - Log.LogErrorFromException(e, true); - } - return !Log.HasLoggedErrors; - } - - private async ThreadingTask UploadAsync(CancellationToken ct, ITaskItem item, HashSet blobsPresent, SemaphoreSlim clientThrottle) - { - if (ct.IsCancellationRequested) - { - Log.LogError("Task UploadToAzure cancelled"); - ct.ThrowIfCancellationRequested(); - } - - string relativeBlobPath = item.GetMetadata("RelativeBlobPath"); - if (string.IsNullOrEmpty(relativeBlobPath)) - throw new Exception(string.Format("Metadata 'RelativeBlobPath' is missing for item '{0}'.", item.ItemSpec)); - - if (!File.Exists(item.ItemSpec)) - throw new Exception(string.Format("The file '{0}' does not exist.", item.ItemSpec)); - - UploadClient uploadClient = new UploadClient(Log); - - if (!Overwrite && blobsPresent.Contains(relativeBlobPath)) - { - if (PassIfExistingItemIdentical && - await ItemEqualsExistingBlobAsync(item, relativeBlobPath, uploadClient, clientThrottle)) - { - return; - } - - throw new Exception(string.Format("The blob '{0}' already exists.", relativeBlobPath)); - } - - string contentType = item.GetMetadata("ContentType"); - - await clientThrottle.WaitAsync(); - - try - { - Log.LogMessage("Uploading {0} to {1}.", item.ItemSpec, ContainerName); - await - uploadClient.UploadBlockBlobAsync( - ct, - AccountName, - AccountKey, - ContainerName, - item.ItemSpec, - relativeBlobPath, - contentType, - UploadTimeoutInMinutes); - } - finally - { - clientThrottle.Release(); - } - } - - private async Task ItemEqualsExistingBlobAsync( - ITaskItem item, - string relativeBlobPath, - UploadClient client, - SemaphoreSlim clientThrottle) - { - await clientThrottle.WaitAsync(); - try - { - return await client.FileEqualsExistingBlobAsync( - AccountName, - AccountKey, - ContainerName, - item.ItemSpec, - relativeBlobPath, - UploadTimeoutInMinutes); - } - finally - { - clientThrottle.Release(); - } - } - } -} \ No newline at end of file