diff --git a/build/Microsoft.DotNet.Cli.Publish.targets b/build/Microsoft.DotNet.Cli.Publish.targets
index 6d35e94f7..d791c9d3b 100644
--- a/build/Microsoft.DotNet.Cli.Publish.targets
+++ b/build/Microsoft.DotNet.Cli.Publish.targets
@@ -1,12 +1,8 @@
-
+
-
-
-
-
-
-
-
+
+
+
+
diff --git a/build/publish/PublishContent.targets b/build/publish/PublishContent.targets
new file mode 100644
index 000000000..63427baba
--- /dev/null
+++ b/build/publish/PublishContent.targets
@@ -0,0 +1,41 @@
+
+
+
+ false
+
+
+
+
+
+
+
+
+
+
+
+ $([System.String]::Copy('%(RecursiveDir)%(Filename)%(Extension)').Replace('\' ,'/'))
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/build_projects/dotnet-cli-build/AzureHelper.cs b/build_projects/dotnet-cli-build/AzureHelper.cs
new file mode 100644
index 000000000..1c3f55e7f
--- /dev/null
+++ b/build_projects/dotnet-cli-build/AzureHelper.cs
@@ -0,0 +1,401 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using Microsoft.Build.Framework;
+using Microsoft.Build.Utilities;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Net;
+using System.Net.Http;
+using System.Net.Http.Headers;
+using System.Security.Cryptography;
+using System.Text;
+using System.Text.RegularExpressions;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Microsoft.DotNet.Build.CloudTestTasks
+{
+ public static class AzureHelper
+ {
+ ///
+ /// The storage api version.
+ ///
+ public static readonly string StorageApiVersion = "2015-04-05";
+
+ public const string DateHeaderString = "x-ms-date";
+
+ public const string VersionHeaderString = "x-ms-version";
+
+ public const string AuthorizationHeaderString = "Authorization";
+
+ public enum SasAccessType
+ {
+ Read,
+ Write,
+ };
+
+ public static string AuthorizationHeader(
+ string storageAccount,
+ string storageKey,
+ string method,
+ DateTime now,
+ HttpRequestMessage request,
+ string ifMatch = "",
+ string contentMD5 = "",
+ string size = "",
+ string contentType = "")
+ {
+ string stringToSign = string.Format(
+ "{0}\n\n\n{1}\n{5}\n{6}\n\n\n{2}\n\n\n\n{3}{4}",
+ method,
+ (size == string.Empty) ? string.Empty : size,
+ ifMatch,
+ GetCanonicalizedHeaders(request),
+ GetCanonicalizedResource(request.RequestUri, storageAccount),
+ contentMD5,
+ contentType);
+
+ byte[] signatureBytes = Encoding.UTF8.GetBytes(stringToSign);
+ string authorizationHeader;
+ using (HMACSHA256 hmacsha256 = new HMACSHA256(Convert.FromBase64String(storageKey)))
+ {
+ authorizationHeader = "SharedKey " + storageAccount + ":"
+ + Convert.ToBase64String(hmacsha256.ComputeHash(signatureBytes));
+ }
+
+ return authorizationHeader;
+ }
+
+ public static string CreateContainerSasToken(
+ string accountName,
+ string containerName,
+ string key,
+ SasAccessType accessType,
+ int validityTimeInDays)
+ {
+ string signedPermissions = string.Empty;
+ switch (accessType)
+ {
+ case SasAccessType.Read:
+ signedPermissions = "r";
+ break;
+ case SasAccessType.Write:
+ signedPermissions = "wdl";
+ break;
+ default:
+ throw new ArgumentOutOfRangeException(nameof(accessType), accessType, "Unrecognized value");
+ }
+
+ string signedStart = DateTime.UtcNow.ToString("O");
+ string signedExpiry = DateTime.UtcNow.AddDays(validityTimeInDays).ToString("O");
+ string canonicalizedResource = "/blob/" + accountName + "/" + containerName;
+ string signedIdentifier = string.Empty;
+ string signedVersion = StorageApiVersion;
+
+ string stringToSign = ConstructServiceStringToSign(
+ signedPermissions,
+ signedVersion,
+ signedExpiry,
+ canonicalizedResource,
+ signedIdentifier,
+ signedStart);
+
+ byte[] signatureBytes = Encoding.UTF8.GetBytes(stringToSign);
+ string signature;
+ using (HMACSHA256 hmacSha256 = new HMACSHA256(Convert.FromBase64String(key)))
+ {
+ signature = Convert.ToBase64String(hmacSha256.ComputeHash(signatureBytes));
+ }
+
+ string sasToken = string.Format(
+ "?sv={0}&sr={1}&sig={2}&st={3}&se={4}&sp={5}",
+ WebUtility.UrlEncode(signedVersion),
+ WebUtility.UrlEncode("c"),
+ WebUtility.UrlEncode(signature),
+ WebUtility.UrlEncode(signedStart),
+ WebUtility.UrlEncode(signedExpiry),
+ WebUtility.UrlEncode(signedPermissions));
+
+ return sasToken;
+ }
+
+ public static string GetCanonicalizedHeaders(HttpRequestMessage request)
+ {
+ StringBuilder sb = new StringBuilder();
+ List headerNameList = (from headerName in request.Headers
+ where
+ headerName.Key.ToLowerInvariant()
+ .StartsWith("x-ms-", StringComparison.Ordinal)
+ select headerName.Key.ToLowerInvariant()).ToList();
+ headerNameList.Sort();
+ foreach (string headerName in headerNameList)
+ {
+ StringBuilder builder = new StringBuilder(headerName);
+ string separator = ":";
+ foreach (string headerValue in GetHeaderValues(request.Headers, headerName))
+ {
+ string trimmedValue = headerValue.Replace("\r\n", string.Empty);
+ builder.Append(separator);
+ builder.Append(trimmedValue);
+ separator = ",";
+ }
+
+ sb.Append(builder);
+ sb.Append("\n");
+ }
+
+ return sb.ToString();
+ }
+
+ public static string GetCanonicalizedResource(Uri address, string accountName)
+ {
+ StringBuilder str = new StringBuilder();
+ StringBuilder builder = new StringBuilder("/");
+ builder.Append(accountName);
+ builder.Append(address.AbsolutePath);
+ str.Append(builder);
+ Dictionary> queryKeyValues = ExtractQueryKeyValues(address);
+ Dictionary> dictionary = GetCommaSeparatedList(queryKeyValues);
+
+ foreach (KeyValuePair> pair in dictionary.OrderBy(p => p.Key))
+ {
+ StringBuilder stringBuilder = new StringBuilder(string.Empty);
+ stringBuilder.Append(pair.Key + ":");
+ string commaList = string.Join(",", pair.Value);
+ stringBuilder.Append(commaList);
+ str.Append("\n");
+ str.Append(stringBuilder);
+ }
+
+ return str.ToString();
+ }
+
+ public static List GetHeaderValues(HttpRequestHeaders headers, string headerName)
+ {
+ List list = new List();
+ IEnumerable values;
+ headers.TryGetValues(headerName, out values);
+ if (values != null)
+ {
+ list.Add((values.FirstOrDefault() ?? string.Empty).TrimStart(null));
+ }
+
+ return list;
+ }
+
+ private static bool IsWithinRetryRange(HttpStatusCode statusCode)
+ {
+ // Retry on http client and server error codes (4xx - 5xx) as well as redirect
+
+ var rawStatus = (int)statusCode;
+ if (rawStatus == 302)
+ return true;
+ else if (rawStatus >= 400 && rawStatus <= 599)
+ return true;
+ else
+ return false;
+ }
+
+ public static async Task RequestWithRetry(TaskLoggingHelper loggingHelper, HttpClient client,
+ Func createRequest, Func validationCallback = null, int retryCount = 5,
+ int retryDelaySeconds = 5)
+ {
+ if (loggingHelper == null)
+ throw new ArgumentNullException(nameof(loggingHelper));
+ if (client == null)
+ throw new ArgumentNullException(nameof(client));
+ if (createRequest == null)
+ throw new ArgumentNullException(nameof(createRequest));
+ if (retryCount < 1)
+ throw new ArgumentException(nameof(retryCount));
+ if (retryDelaySeconds < 1)
+ throw new ArgumentException(nameof(retryDelaySeconds));
+
+ int retries = 0;
+ HttpResponseMessage response = null;
+
+ // add a bit of randomness to the retry delay
+ var rng = new Random();
+
+ while (retries < retryCount)
+ {
+ if (retries > 0)
+ {
+ if (response != null)
+ {
+ response.Dispose();
+ response = null;
+ }
+
+ int delay = retryDelaySeconds * retries * rng.Next(1, 5);
+ loggingHelper.LogMessage(MessageImportance.Low, "Waiting {0} seconds before retry", delay);
+ await System.Threading.Tasks.Task.Delay(delay * 1000);
+ }
+
+ try
+ {
+ using (var request = createRequest())
+ response = await client.SendAsync(request);
+ }
+ catch (Exception e)
+ {
+ loggingHelper.LogWarningFromException(e, true);
+
+ // if this is the final iteration let the exception bubble up
+ if (retries + 1 == retryCount)
+ throw;
+ }
+
+ // response can be null if we fail to send the request
+ if (response != null)
+ {
+ if (validationCallback == null)
+ {
+ // check if the response code is within the range of failures
+ if (IsWithinRetryRange(response.StatusCode))
+ {
+ loggingHelper.LogWarning("Request failed with status code {0}", response.StatusCode);
+ }
+ else
+ {
+ loggingHelper.LogMessage(MessageImportance.Low, "Response completed with status code {0}", response.StatusCode);
+ return response;
+ }
+ }
+ else
+ {
+ bool isSuccess = validationCallback(response);
+ if (!isSuccess)
+ {
+ loggingHelper.LogMessage("Validation callback returned retry for status code {0}", response.StatusCode);
+ }
+ else
+ {
+ loggingHelper.LogMessage("Validation callback returned success for status code {0}", response.StatusCode);
+ return response;
+ }
+ }
+ }
+
+ ++retries;
+ }
+
+ // retry count exceeded
+ loggingHelper.LogWarning("Retry count {0} exceeded", retryCount);
+
+ // set some default values in case response is null
+ var statusCode = "None";
+ var contentStr = "Null";
+ if (response != null)
+ {
+ statusCode = response.StatusCode.ToString();
+ contentStr = await response.Content.ReadAsStringAsync();
+ response.Dispose();
+ }
+
+ throw new HttpRequestException(string.Format("Request failed with status {0} response {1}", statusCode, contentStr));
+ }
+
+ private static string ConstructServiceStringToSign(
+ string signedPermissions,
+ string signedVersion,
+ string signedExpiry,
+ string canonicalizedResource,
+ string signedIdentifier,
+ string signedStart,
+ string signedIP = "",
+ string signedProtocol = "",
+ string rscc = "",
+ string rscd = "",
+ string rsce = "",
+ string rscl = "",
+ string rsct = "")
+ {
+ // constructing string to sign based on spec in https://msdn.microsoft.com/en-us/library/azure/dn140255.aspx
+ var stringToSign = string.Join(
+ "\n",
+ signedPermissions,
+ signedStart,
+ signedExpiry,
+ canonicalizedResource,
+ signedIdentifier,
+ signedIP,
+ signedProtocol,
+ signedVersion,
+ rscc,
+ rscd,
+ rsce,
+ rscl,
+ rsct);
+ return stringToSign;
+ }
+
+ private static Dictionary> ExtractQueryKeyValues(Uri address)
+ {
+ Dictionary> values = new Dictionary>();
+ //Decode this to allow the regex to pull out the correct groups for signing
+ address = new Uri(WebUtility.UrlDecode(address.ToString()));
+ Regex newreg = new Regex(@"\?(\w+)\=([\w|\=]+)|\&(\w+)\=([\w|\=]+)");
+ MatchCollection matches = newreg.Matches(address.Query);
+ foreach (Match match in matches)
+ {
+ string key, value;
+ if (!string.IsNullOrEmpty(match.Groups[1].Value))
+ {
+ key = match.Groups[1].Value;
+ value = match.Groups[2].Value;
+ }
+ else
+ {
+ key = match.Groups[3].Value;
+ value = match.Groups[4].Value;
+ }
+
+ HashSet setOfValues;
+ if (values.TryGetValue(key, out setOfValues))
+ {
+ setOfValues.Add(value);
+ }
+ else
+ {
+ HashSet newSet = new HashSet { value };
+ values.Add(key, newSet);
+ }
+ }
+
+ return values;
+ }
+
+ private static Dictionary> GetCommaSeparatedList(
+ Dictionary> queryKeyValues)
+ {
+ Dictionary> dictionary = new Dictionary>();
+
+ foreach (string queryKeys in queryKeyValues.Keys)
+ {
+ HashSet setOfValues;
+ queryKeyValues.TryGetValue(queryKeys, out setOfValues);
+ List list = new List();
+ list.AddRange(setOfValues);
+ list.Sort();
+ string commaSeparatedValues = string.Join(",", list);
+ string key = queryKeys.ToLowerInvariant();
+ HashSet setOfValues2;
+ if (dictionary.TryGetValue(key, out setOfValues2))
+ {
+ setOfValues2.Add(commaSeparatedValues);
+ }
+ else
+ {
+ HashSet newSet = new HashSet { commaSeparatedValues };
+ dictionary.Add(key, newSet);
+ }
+ }
+
+ return dictionary;
+ }
+ }
+}
\ No newline at end of file
diff --git a/build_projects/dotnet-cli-build/CreateAzureContainer.cs b/build_projects/dotnet-cli-build/CreateAzureContainer.cs
new file mode 100644
index 000000000..9ad34e06c
--- /dev/null
+++ b/build_projects/dotnet-cli-build/CreateAzureContainer.cs
@@ -0,0 +1,167 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Globalization;
+using System.IO;
+using System.Net;
+using System.Threading.Tasks;
+using System.Net.Http;
+using Microsoft.Build.Framework;
+
+using Task = Microsoft.Build.Utilities.Task;
+
+namespace Microsoft.DotNet.Build.CloudTestTasks
+{
+ public sealed class CreateAzureContainer : Task
+ {
+ ///
+ /// The Azure account key used when creating the connection string.
+ ///
+ [Required]
+ public string AccountKey { get; set; }
+
+ ///
+ /// The Azure account name used when creating the connection string.
+ ///
+ [Required]
+ public string AccountName { get; set; }
+
+ ///
+ /// The name of the container to create. The specified name must be in the correct format, see the
+ /// following page for more info. https://msdn.microsoft.com/en-us/library/azure/dd135715.aspx
+ ///
+ [Required]
+ public string ContainerName { get; set; }
+
+ ///
+ /// When false, if the specified container already exists get a reference to it.
+ /// When true, if the specified container already exists the task will fail.
+ ///
+ public bool FailIfExists { get; set; }
+
+ ///
+ /// The read-only SAS token created when ReadOnlyTokenDaysValid is greater than zero.
+ ///
+ [Output]
+ public string ReadOnlyToken { get; set; }
+
+ ///
+ /// The number of days for which the read-only token should be valid.
+ ///
+ public int ReadOnlyTokenDaysValid { get; set; }
+
+ ///
+ /// The URI of the created container.
+ ///
+ [Output]
+ public string StorageUri { get; set; }
+
+ ///
+ /// The write-only SAS token create when WriteOnlyTokenDaysValid is greater than zero.
+ ///
+ [Output]
+ public string WriteOnlyToken { get; set; }
+
+ ///
+ /// The number of days for which the write-only token should be valid.
+ ///
+ public int WriteOnlyTokenDaysValid { get; set; }
+
+ public override bool Execute()
+ {
+ return ExecuteAsync().GetAwaiter().GetResult();
+ }
+
+ public async Task ExecuteAsync()
+ {
+ Log.LogMessage(
+ MessageImportance.High,
+ "Creating container named '{0}' in storage account {1}.",
+ ContainerName,
+ AccountName);
+ string url = string.Format(
+ "https://{0}.blob.core.windows.net/{1}?restype=container",
+ AccountName,
+ ContainerName);
+ StorageUri = string.Format(
+ "https://{0}.blob.core.windows.net/{1}/",
+ AccountName,
+ ContainerName);
+
+ Log.LogMessage(MessageImportance.Low, "Sending request to create Container");
+ using (HttpClient client = new HttpClient())
+ {
+ Func createRequest = () =>
+ {
+ DateTime dt = DateTime.UtcNow;
+ var req = new HttpRequestMessage(HttpMethod.Put, url);
+ req.Headers.Add(AzureHelper.DateHeaderString, dt.ToString("R", CultureInfo.InvariantCulture));
+ req.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion);
+ req.Headers.Add(AzureHelper.AuthorizationHeaderString, AzureHelper.AuthorizationHeader(
+ AccountName,
+ AccountKey,
+ "PUT",
+ dt,
+ req));
+ byte[] bytestoWrite = new byte[0];
+ int bytesToWriteLength = 0;
+
+ Stream postStream = new MemoryStream();
+ postStream.Write(bytestoWrite, 0, bytesToWriteLength);
+ req.Content = new StreamContent(postStream);
+
+ return req;
+ };
+
+ Func validate = (HttpResponseMessage response) =>
+ {
+ // the Conflict status (409) indicates that the container already exists, so
+ // if FailIfExists is set to false and we get a 409 don't fail the task.
+ return response.IsSuccessStatusCode || (!FailIfExists && response.StatusCode == HttpStatusCode.Conflict);
+ };
+
+ using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest, validate))
+ {
+ try
+ {
+ Log.LogMessage(
+ MessageImportance.Low,
+ "Received response to create Container {0}: Status Code: {1} {2}",
+ ContainerName, response.StatusCode, response.Content.ToString());
+
+ // specifying zero is valid, it means "I don't want a token"
+ if (ReadOnlyTokenDaysValid > 0)
+ {
+ ReadOnlyToken = AzureHelper.CreateContainerSasToken(
+ AccountName,
+ ContainerName,
+ AccountKey,
+ AzureHelper.SasAccessType.Read,
+ ReadOnlyTokenDaysValid);
+ }
+
+ // specifying zero is valid, it means "I don't want a token"
+ if (WriteOnlyTokenDaysValid > 0)
+ {
+ WriteOnlyToken = AzureHelper.CreateContainerSasToken(
+ AccountName,
+ ContainerName,
+ AccountKey,
+ AzureHelper.SasAccessType.Write,
+ WriteOnlyTokenDaysValid);
+ }
+ }
+ catch (Exception e)
+ {
+ Log.LogErrorFromException(e, true);
+ return false;
+ }
+ }
+ }
+
+ return true;
+ }
+ }
+}
\ No newline at end of file
diff --git a/build_projects/dotnet-cli-build/UploadClient.cs b/build_projects/dotnet-cli-build/UploadClient.cs
new file mode 100644
index 000000000..6fe51e31f
--- /dev/null
+++ b/build_projects/dotnet-cli-build/UploadClient.cs
@@ -0,0 +1,183 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using Microsoft.Build.Framework;
+using Microsoft.Build.Utilities;
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
+using System.Net;
+using System.Net.Http;
+using System.Text;
+using System.Threading;
+using Task = System.Threading.Tasks.Task;
+
+namespace Microsoft.DotNet.Build.CloudTestTasks
+{
+ public class UploadClient
+ {
+ private TaskLoggingHelper log;
+
+ public UploadClient(TaskLoggingHelper loggingHelper)
+ {
+ log = loggingHelper;
+ }
+
+ public string EncodeBlockIds(int numberOfBlocks, int lengthOfId)
+ {
+ string numberOfBlocksString = numberOfBlocks.ToString("D" + lengthOfId);
+ if (Encoding.UTF8.GetByteCount(numberOfBlocksString) <= 64)
+ {
+ byte[] bytes = Encoding.UTF8.GetBytes(numberOfBlocksString);
+ return Convert.ToBase64String(bytes);
+ }
+ else
+ {
+ throw new Exception("Task failed - Could not encode block id.");
+ }
+ }
+
+ public async Task UploadBlockBlobAsync(
+ CancellationToken ct,
+ string AccountName,
+ string AccountKey,
+ string ContainerName,
+ string filePath,
+ string destinationBlob)
+ {
+
+ string resourceUrl = string.Format("https://{0}.blob.core.windows.net/{1}", AccountName, ContainerName);
+
+ string fileName = destinationBlob;
+ fileName = fileName.Replace("\\", "/");
+ string blobUploadUrl = resourceUrl + "/" + fileName;
+ int size = (int)new FileInfo(filePath).Length;
+ int blockSize = 4 * 1024 * 1024; //4MB max size of a block blob
+ int bytesLeft = size;
+ List blockIds = new List();
+ int numberOfBlocks = (size / blockSize) + 1;
+ int countForId = 0;
+ using (FileStream fileStreamTofilePath = new FileStream(filePath, FileMode.Open))
+ {
+ int offset = 0;
+
+ while (bytesLeft > 0)
+ {
+ int nextBytesToRead = (bytesLeft < blockSize) ? bytesLeft : blockSize;
+ byte[] fileBytes = new byte[blockSize];
+ int read = fileStreamTofilePath.Read(fileBytes, 0, nextBytesToRead);
+
+ if (nextBytesToRead != read)
+ {
+ throw new Exception(string.Format(
+ "Number of bytes read ({0}) from file {1} isn't equal to the number of bytes expected ({2}) .",
+ read, fileName, nextBytesToRead));
+ }
+
+ string blockId = EncodeBlockIds(countForId, numberOfBlocks.ToString().Length);
+
+ blockIds.Add(blockId);
+ string blockUploadUrl = blobUploadUrl + "?comp=block&blockid=" + WebUtility.UrlEncode(blockId);
+
+ using (HttpClient client = new HttpClient())
+ {
+ client.DefaultRequestHeaders.Clear();
+ Func createRequest = () =>
+ {
+ DateTime dt = DateTime.UtcNow;
+ var req = new HttpRequestMessage(HttpMethod.Put, blockUploadUrl);
+ req.Headers.Add(
+ AzureHelper.DateHeaderString,
+ dt.ToString("R", CultureInfo.InvariantCulture));
+ req.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion);
+ req.Headers.Add(
+ AzureHelper.AuthorizationHeaderString,
+ AzureHelper.AuthorizationHeader(
+ AccountName,
+ AccountKey,
+ "PUT",
+ dt,
+ req,
+ string.Empty,
+ string.Empty,
+ nextBytesToRead.ToString(),
+ string.Empty));
+
+ Stream postStream = new MemoryStream();
+ postStream.Write(fileBytes, 0, nextBytesToRead);
+ postStream.Seek(0, SeekOrigin.Begin);
+ req.Content = new StreamContent(postStream);
+ return req;
+ };
+
+ log.LogMessage(MessageImportance.Low, "Sending request to upload part {0} of file {1}", countForId, fileName);
+
+ using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(log, client, createRequest))
+ {
+ log.LogMessage(
+ MessageImportance.Low,
+ "Received response to upload part {0} of file {1}: Status Code:{2} Status Desc: {3}",
+ countForId,
+ fileName,
+ response.StatusCode,
+ await response.Content.ReadAsStringAsync());
+ }
+ }
+
+ offset += read;
+ bytesLeft -= nextBytesToRead;
+ countForId += 1;
+ }
+ }
+
+ string blockListUploadUrl = blobUploadUrl + "?comp=blocklist";
+
+ using (HttpClient client = new HttpClient())
+ {
+ Func createRequest = () =>
+ {
+ DateTime dt1 = DateTime.UtcNow;
+ var req = new HttpRequestMessage(HttpMethod.Put, blockListUploadUrl);
+ req.Headers.Add(AzureHelper.DateHeaderString, dt1.ToString("R", CultureInfo.InvariantCulture));
+ req.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion);
+
+ var body = new StringBuilder("");
+ foreach (object item in blockIds)
+ body.AppendFormat("{0}", item);
+
+ body.Append("");
+ byte[] bodyData = Encoding.UTF8.GetBytes(body.ToString());
+ req.Headers.Add(
+ AzureHelper.AuthorizationHeaderString,
+ AzureHelper.AuthorizationHeader(
+ AccountName,
+ AccountKey,
+ "PUT",
+ dt1,
+ req,
+ string.Empty,
+ string.Empty,
+ bodyData.Length.ToString(),
+ ""));
+ Stream postStream = new MemoryStream();
+ postStream.Write(bodyData, 0, bodyData.Length);
+ postStream.Seek(0, SeekOrigin.Begin);
+ req.Content = new StreamContent(postStream);
+ return req;
+ };
+
+ using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(log, client, createRequest))
+ {
+ log.LogMessage(
+ MessageImportance.Low,
+ "Received response to combine block list for file {0}: Status Code:{1} Status Desc: {2}",
+ fileName,
+ response.StatusCode,
+ await response.Content.ReadAsStringAsync());
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/build_projects/dotnet-cli-build/UploadToAzure.cs b/build_projects/dotnet-cli-build/UploadToAzure.cs
new file mode 100644
index 000000000..b166a87dc
--- /dev/null
+++ b/build_projects/dotnet-cli-build/UploadToAzure.cs
@@ -0,0 +1,186 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
+using System.Net;
+using System.Threading;
+using System.Threading.Tasks;
+using System.Xml;
+using System.Linq;
+using System.Net.Http;
+
+using Microsoft.Build.Framework;
+
+using Task = Microsoft.Build.Utilities.Task;
+using ThreadingTask = System.Threading.Tasks.Task;
+
+namespace Microsoft.DotNet.Build.CloudTestTasks
+{
+ public class UploadToAzure : Task, ICancelableTask
+ {
+ private static readonly CancellationTokenSource TokenSource = new CancellationTokenSource();
+ private static readonly CancellationToken CancellationToken = TokenSource.Token;
+
+ ///
+ /// The Azure account key used when creating the connection string.
+ ///
+ [Required]
+ public string AccountKey { get; set; }
+
+ ///
+ /// The Azure account name used when creating the connection string.
+ ///
+ [Required]
+ public string AccountName { get; set; }
+
+ ///
+ /// The name of the container to access. The specified name must be in the correct format, see the
+ /// following page for more info. https://msdn.microsoft.com/en-us/library/azure/dd135715.aspx
+ ///
+ [Required]
+ public string ContainerName { get; set; }
+
+ ///
+ /// An item group of files to upload. Each item must have metadata RelativeBlobPath
+ /// that specifies the path relative to ContainerName where the item will be uploaded.
+ ///
+ [Required]
+ public ITaskItem[] Items { get; set; }
+
+ ///
+ /// Indicates if the destination blob should be overwritten if it already exists. The default if false.
+ ///
+ public bool Overwrite { get; set; } = false;
+
+ ///
+ /// Specifies the maximum number of clients to concurrently upload blobs to azure
+ ///
+ public int MaxClients { get; set; } = 8;
+
+ public void Cancel()
+ {
+ TokenSource.Cancel();
+ }
+
+ public override bool Execute()
+ {
+ return ExecuteAsync(CancellationToken).GetAwaiter().GetResult();
+ }
+
+ public async Task ExecuteAsync(CancellationToken ct)
+ {
+ Log.LogMessage(
+ MessageImportance.High,
+ "Begin uploading blobs to Azure account {0} in container {1}.",
+ AccountName,
+ ContainerName);
+
+ if (Items.Length == 0)
+ {
+ Log.LogError("No items were provided for upload.");
+ return false;
+ }
+
+ // first check what blobs are present
+ string checkListUrl = string.Format(
+ "https://{0}.blob.core.windows.net/{1}?restype=container&comp=list",
+ AccountName,
+ ContainerName);
+
+ HashSet blobsPresent = new HashSet(StringComparer.OrdinalIgnoreCase);
+
+ try
+ {
+ using (HttpClient client = new HttpClient())
+ {
+ Func createRequest = () =>
+ {
+ DateTime dt = DateTime.UtcNow;
+ var req = new HttpRequestMessage(HttpMethod.Get, checkListUrl);
+ req.Headers.Add(AzureHelper.DateHeaderString, dt.ToString("R", CultureInfo.InvariantCulture));
+ req.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion);
+ req.Headers.Add(AzureHelper.AuthorizationHeaderString, AzureHelper.AuthorizationHeader(
+ AccountName,
+ AccountKey,
+ "GET",
+ dt,
+ req));
+ return req;
+ };
+
+ Log.LogMessage(MessageImportance.Low, "Sending request to check whether Container blobs exist");
+ using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest))
+ {
+ var doc = new XmlDocument();
+ doc.LoadXml(await response.Content.ReadAsStringAsync());
+
+ XmlNodeList nodes = doc.DocumentElement.GetElementsByTagName("Blob");
+
+ foreach (XmlNode node in nodes)
+ {
+ blobsPresent.Add(node["Name"].InnerText);
+ }
+
+ Log.LogMessage(MessageImportance.Low, "Received response to check whether Container blobs exist");
+ }
+ }
+
+ using (var clientThrottle = new SemaphoreSlim(this.MaxClients, this.MaxClients))
+ {
+ await ThreadingTask.WhenAll(Items.Select(item => UploadAsync(ct, item, blobsPresent, clientThrottle)));
+ }
+
+ Log.LogMessage(MessageImportance.High, "Upload to Azure is complete, a total of {0} items were uploaded.", Items.Length);
+ return true;
+ }
+ catch (Exception e)
+ {
+ Log.LogErrorFromException(e, true);
+ return false;
+ }
+ }
+
+ private async ThreadingTask UploadAsync(CancellationToken ct, ITaskItem item, HashSet blobsPresent, SemaphoreSlim clientThrottle)
+ {
+ if (ct.IsCancellationRequested)
+ {
+ Log.LogError("Task UploadToAzure cancelled");
+ ct.ThrowIfCancellationRequested();
+ }
+
+ string relativeBlobPath = item.GetMetadata("RelativeBlobPath");
+ if (string.IsNullOrEmpty(relativeBlobPath))
+ throw new Exception(string.Format("Metadata 'RelativeBlobPath' is missing for item '{0}'.", item.ItemSpec));
+
+ if (!File.Exists(item.ItemSpec))
+ throw new Exception(string.Format("The file '{0}' does not exist.", item.ItemSpec));
+
+ if (!Overwrite && blobsPresent.Contains(relativeBlobPath))
+ throw new Exception(string.Format("The blob '{0}' already exists.", relativeBlobPath));
+
+ await clientThrottle.WaitAsync();
+
+ try
+ {
+ Log.LogMessage("Uploading {0} to {1}.", item.ItemSpec, ContainerName);
+ UploadClient uploadClient = new UploadClient(Log);
+ await
+ uploadClient.UploadBlockBlobAsync(
+ ct,
+ AccountName,
+ AccountKey,
+ ContainerName,
+ item.ItemSpec,
+ relativeBlobPath);
+ }
+ finally
+ {
+ clientThrottle.Release();
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/build_projects/dotnet-cli-build/ZipFileCreateFromDirectory.cs b/build_projects/dotnet-cli-build/ZipFileCreateFromDirectory.cs
new file mode 100644
index 000000000..8ca191167
--- /dev/null
+++ b/build_projects/dotnet-cli-build/ZipFileCreateFromDirectory.cs
@@ -0,0 +1,131 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.IO;
+using System.IO.Compression;
+using System.Text.RegularExpressions;
+using Microsoft.Build.Framework;
+using Microsoft.Build.Utilities;
+
+namespace Microsoft.DotNet.Build.Tasks
+{
+ public sealed class ZipFileCreateFromDirectory : Task
+ {
+ ///
+ /// The path to the directory to be archived.
+ ///
+ [Required]
+ public string SourceDirectory { get; set; }
+
+ ///
+ /// The path of the archive to be created.
+ ///
+ [Required]
+ public string DestinationArchive { get; set; }
+
+ ///
+ /// Indicates if the destination archive should be overwritten if it already exists.
+ ///
+ public bool OverwriteDestination { get; set; }
+
+ ///
+ /// If zipping an entire folder without exclusion patterns, whether to include the folder in the archive.
+ ///
+ public bool IncludeBaseDirectory { get; set; }
+
+ ///
+ /// An item group of regular expressions for content to exclude from the archive.
+ ///
+ public ITaskItem[] ExcludePatterns { get; set; }
+
+ public override bool Execute()
+ {
+ try
+ {
+ if (File.Exists(DestinationArchive))
+ {
+ if (OverwriteDestination == true)
+ {
+ Log.LogMessage(MessageImportance.Low, "{0} already existed, deleting before zipping...", DestinationArchive);
+ File.Delete(DestinationArchive);
+ }
+ else
+ {
+ Log.LogWarning("'{0}' already exists. Did you forget to set '{1}' to true?", DestinationArchive, nameof(OverwriteDestination));
+ }
+ }
+
+ Log.LogMessage(MessageImportance.High, "Compressing {0} into {1}...", SourceDirectory, DestinationArchive);
+ if (!Directory.Exists(Path.GetDirectoryName(DestinationArchive)))
+ Directory.CreateDirectory(Path.GetDirectoryName(DestinationArchive));
+
+ if (ExcludePatterns == null)
+ {
+ ZipFile.CreateFromDirectory(SourceDirectory, DestinationArchive, CompressionLevel.Optimal, IncludeBaseDirectory);
+ }
+ else
+ {
+ // convert to regular expressions
+ Regex[] regexes = new Regex[ExcludePatterns.Length];
+ for (int i = 0; i < ExcludePatterns.Length; ++i)
+ regexes[i] = new Regex(ExcludePatterns[i].ItemSpec, RegexOptions.IgnoreCase);
+
+ using (FileStream writer = new FileStream(DestinationArchive, FileMode.CreateNew))
+ {
+ using (ZipArchive zipFile = new ZipArchive(writer, ZipArchiveMode.Create))
+ {
+ var files = Directory.GetFiles(SourceDirectory, "*", SearchOption.AllDirectories);
+
+ foreach (var file in files)
+ {
+ // look for a match
+ bool foundMatch = false;
+ foreach (var regex in regexes)
+ {
+ if (regex.IsMatch(file))
+ {
+ foundMatch = true;
+ break;
+ }
+ }
+
+ if (foundMatch)
+ {
+ Log.LogMessage(MessageImportance.Low, "Excluding {0} from archive.", file);
+ continue;
+ }
+
+ var relativePath = MakeRelativePath(SourceDirectory, file);
+ zipFile.CreateEntryFromFile(file, relativePath, CompressionLevel.Optimal);
+ }
+ }
+ }
+ }
+ }
+ catch (Exception e)
+ {
+ // We have 2 log calls because we want a nice error message but we also want to capture the callstack in the log.
+ Log.LogError("An exception has occured while trying to compress '{0}' into '{1}'.", SourceDirectory, DestinationArchive);
+ Log.LogMessage(MessageImportance.Low, e.ToString());
+ return false;
+ }
+
+ return true;
+ }
+
+ private string MakeRelativePath(string root, string subdirectory)
+ {
+ if (!subdirectory.StartsWith(root))
+ throw new Exception(string.Format("'{0}' is not a subdirectory of '{1}'.", subdirectory, root));
+
+ // returned string should not start with a directory separator
+ int chop = root.Length;
+ if (subdirectory[chop] == Path.DirectorySeparatorChar)
+ ++chop;
+
+ return subdirectory.Substring(chop);
+ }
+ }
+}
\ No newline at end of file
diff --git a/build_projects/dotnet-cli-build/ZipFileExtractToDirectory.cs b/build_projects/dotnet-cli-build/ZipFileExtractToDirectory.cs
new file mode 100644
index 000000000..a17baf99a
--- /dev/null
+++ b/build_projects/dotnet-cli-build/ZipFileExtractToDirectory.cs
@@ -0,0 +1,65 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using Microsoft.Build.Framework;
+using Microsoft.Build.Utilities;
+using System;
+using System.IO;
+using System.IO.Compression;
+
+namespace Microsoft.DotNet.Build.Tasks
+{
+ public sealed class ZipFileExtractToDirectory : Task
+ {
+ ///
+ /// The path to the directory to be archived.
+ ///
+ [Required]
+ public string SourceArchive { get; set; }
+
+ ///
+ /// The path of the archive to be created.
+ ///
+ [Required]
+ public string DestinationDirectory { get; set; }
+
+ ///
+ /// Indicates if the destination archive should be overwritten if it already exists.
+ ///
+ public bool OverwriteDestination { get; set; }
+
+ public override bool Execute()
+ {
+ try
+ {
+ if (Directory.Exists(DestinationDirectory))
+ {
+ if (OverwriteDestination == true)
+ {
+ Log.LogMessage(MessageImportance.Low, "'{0}' already exists, trying to delete before unzipping...", DestinationDirectory);
+ Directory.Delete(DestinationDirectory, recursive: true);
+ }
+ else
+ {
+ Log.LogWarning("'{0}' already exists. Did you forget to set '{1}' to true?", DestinationDirectory, nameof(OverwriteDestination));
+ }
+ }
+
+ Log.LogMessage(MessageImportance.High, "Decompressing '{0}' into '{1}'...", SourceArchive, DestinationDirectory);
+ if (!Directory.Exists(Path.GetDirectoryName(DestinationDirectory)))
+ Directory.CreateDirectory(Path.GetDirectoryName(DestinationDirectory));
+
+ ZipFile.ExtractToDirectory(SourceArchive, DestinationDirectory);
+ }
+ catch (Exception e)
+ {
+ // We have 2 log calls because we want a nice error message but we also want to capture the callstack in the log.
+ Log.LogError("An exception has occured while trying to decompress '{0}' into '{1}'.", SourceArchive, DestinationDirectory);
+ Log.LogMessage(MessageImportance.Low, e.ToString());
+ return false;
+ }
+ return true;
+ }
+ }
+}
\ No newline at end of file