// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. using Microsoft.Build.Framework; using Microsoft.Build.Utilities; using System; using System.Collections.Generic; using System.Globalization; using System.IO; using System.Net; using System.Net.Http; using System.Text; using System.Threading; using Task = System.Threading.Tasks.Task; namespace Microsoft.DotNet.Build.CloudTestTasks { public class UploadClient { private TaskLoggingHelper log; public UploadClient(TaskLoggingHelper loggingHelper) { log = loggingHelper; } public string EncodeBlockIds(int numberOfBlocks, int lengthOfId) { string numberOfBlocksString = numberOfBlocks.ToString("D" + lengthOfId); if (Encoding.UTF8.GetByteCount(numberOfBlocksString) <= 64) { byte[] bytes = Encoding.UTF8.GetBytes(numberOfBlocksString); return Convert.ToBase64String(bytes); } else { throw new Exception("Task failed - Could not encode block id."); } } public async Task UploadBlockBlobAsync( CancellationToken ct, string AccountName, string AccountKey, string ContainerName, string filePath, string destinationBlob) { string resourceUrl = string.Format("https://{0}.blob.core.windows.net/{1}", AccountName, ContainerName); string fileName = destinationBlob; fileName = fileName.Replace("\\", "/"); string blobUploadUrl = resourceUrl + "/" + fileName; int size = (int)new FileInfo(filePath).Length; int blockSize = 4 * 1024 * 1024; //4MB max size of a block blob int bytesLeft = size; List blockIds = new List(); int numberOfBlocks = (size / blockSize) + 1; int countForId = 0; using (FileStream fileStreamTofilePath = new FileStream(filePath, FileMode.Open)) { int offset = 0; while (bytesLeft > 0) { int nextBytesToRead = (bytesLeft < blockSize) ? bytesLeft : blockSize; byte[] fileBytes = new byte[blockSize]; int read = fileStreamTofilePath.Read(fileBytes, 0, nextBytesToRead); if (nextBytesToRead != read) { throw new Exception(string.Format( "Number of bytes read ({0}) from file {1} isn't equal to the number of bytes expected ({2}) .", read, fileName, nextBytesToRead)); } string blockId = EncodeBlockIds(countForId, numberOfBlocks.ToString().Length); blockIds.Add(blockId); string blockUploadUrl = blobUploadUrl + "?comp=block&blockid=" + WebUtility.UrlEncode(blockId); using (HttpClient client = new HttpClient()) { client.DefaultRequestHeaders.Clear(); Func createRequest = () => { DateTime dt = DateTime.UtcNow; var req = new HttpRequestMessage(HttpMethod.Put, blockUploadUrl); req.Headers.Add( AzureHelper.DateHeaderString, dt.ToString("R", CultureInfo.InvariantCulture)); req.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion); req.Headers.Add( AzureHelper.AuthorizationHeaderString, AzureHelper.AuthorizationHeader( AccountName, AccountKey, "PUT", dt, req, string.Empty, string.Empty, nextBytesToRead.ToString(), string.Empty)); Stream postStream = new MemoryStream(); postStream.Write(fileBytes, 0, nextBytesToRead); postStream.Seek(0, SeekOrigin.Begin); req.Content = new StreamContent(postStream); return req; }; log.LogMessage(MessageImportance.Low, "Sending request to upload part {0} of file {1}", countForId, fileName); using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(log, client, createRequest)) { log.LogMessage( MessageImportance.Low, "Received response to upload part {0} of file {1}: Status Code:{2} Status Desc: {3}", countForId, fileName, response.StatusCode, await response.Content.ReadAsStringAsync()); } } offset += read; bytesLeft -= nextBytesToRead; countForId += 1; } } string blockListUploadUrl = blobUploadUrl + "?comp=blocklist"; using (HttpClient client = new HttpClient()) { Func createRequest = () => { DateTime dt1 = DateTime.UtcNow; var req = new HttpRequestMessage(HttpMethod.Put, blockListUploadUrl); req.Headers.Add(AzureHelper.DateHeaderString, dt1.ToString("R", CultureInfo.InvariantCulture)); req.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion); var body = new StringBuilder(""); foreach (object item in blockIds) body.AppendFormat("{0}", item); body.Append(""); byte[] bodyData = Encoding.UTF8.GetBytes(body.ToString()); req.Headers.Add( AzureHelper.AuthorizationHeaderString, AzureHelper.AuthorizationHeader( AccountName, AccountKey, "PUT", dt1, req, string.Empty, string.Empty, bodyData.Length.ToString(), "")); Stream postStream = new MemoryStream(); postStream.Write(bodyData, 0, bodyData.Length); postStream.Seek(0, SeekOrigin.Begin); req.Content = new StreamContent(postStream); return req; }; using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(log, client, createRequest)) { log.LogMessage( MessageImportance.Low, "Received response to combine block list for file {0}: Status Code:{1} Status Desc: {2}", fileName, response.StatusCode, await response.Content.ReadAsStringAsync()); } } } } }