mirrors/dotnet-installer
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions 1

Port #7460 from master to 2.0.x

Browse source 
Use Rest Api to upload to the feed

Add pulling logic to make sure it is uploaded to the feed.

Add retry logic for the whole upload process

Remove the old upload script

(cherry picked from commit 7f54ccb903)
This commit is contained in:
William Li 2017-08-09 16:57:47 -07:00 • committed by Nick Guerrera
parent 1f21ad0206
commit c834676ecb
18 changed files with 646 additions and 215 deletions
Download patch file Download diff file Expand all files Collapse all files

62
build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/AddPackageStrategy.cs Normal file
View file

@ -0,0 +1,62 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Text;
using System.Threading.Tasks;
using NuGet.Protocol;
namespace Microsoft.DotNet.Cli.Build.UploadToLinuxPackageRepository
{
internal class AddPackageStrategy : IAzurelinuxRepositoryServiceHttpStrategy
{
private readonly IdInRepositoryService _idInRepositoryService;
private readonly string _packageName;
private readonly string _packageVersion;
private readonly string _repositoryId;
public AddPackageStrategy(
IdInRepositoryService idInRepositoryService,
string packageName,
string packageVersion,
string repositoryId)
{
_idInRepositoryService = idInRepositoryService
?? throw new ArgumentNullException(nameof(idInRepositoryService));
_packageName = packageName;
_packageVersion = packageVersion;
_repositoryId = repositoryId;
}
public async Task<string> Execute(HttpClient client, Uri baseAddress)
{
var debianUploadJsonContent = new Dictionary<string, string>
{
["name"] = _packageName,
["version"] = AppendDebianRevisionNumber(_packageVersion),
["fileId"] = _idInRepositoryService.Id,
["repositoryId"] = _repositoryId
}.ToJson();
var content = new StringContent(debianUploadJsonContent,
Encoding.UTF8,
"application/json");
using (var response = await client.PostAsync(new Uri(baseAddress, "/v1/packages"), content))
{
if (!response.IsSuccessStatusCode)
throw new FailedToAddPackageToPackageRepositoryException(
$"request:{debianUploadJsonContent} response:{response.ToJson()}");
return response.Headers.GetValues("Location").Single();
}
}
private static string AppendDebianRevisionNumber(string packageVersion)
{
return packageVersion + "-1";
}
}
}

53
build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/ExponentialRetry.cs Normal file
View file

@ -0,0 +1,53 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
namespace Microsoft.DotNet.Cli.Build.UploadToLinuxPackageRepository
{
public static class ExponentialRetry
{
public static IEnumerable<TimeSpan> Intervals
{
get
{
var seconds = 5;
while (true)
{
yield return TimeSpan.FromSeconds(seconds);
seconds *= 2;
}
}
}
public static async Task ExecuteWithRetry(Func<Task<string>> action,
Func<string, bool> isSuccess,
int maxRetryCount,
Func<IEnumerable<Task>> timer,
string taskDescription = "")
{
var count = 0;
foreach (var t in timer())
{
await t;
var result = await action();
if (isSuccess(result))
return;
count++;
if (count == maxRetryCount)
throw new RetryFailedException(
$"Retry failed for {taskDescription} after {count} times with result: {result}");
}
throw new Exception("Timer should not be exhausted");
}
public static IEnumerable<Task> Timer(IEnumerable<TimeSpan> interval)
{
return interval.Select(Task.Delay);
}
}
}

24
build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/FailedToAddPackageToPackageRepositoryException.cs Normal file
View file

@ -0,0 +1,24 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.
using System;
namespace Microsoft.DotNet.Cli.Build.UploadToLinuxPackageRepository
{
public class FailedToAddPackageToPackageRepositoryException : Exception
{
public FailedToAddPackageToPackageRepositoryException(string message) : base(message)
{
}
public FailedToAddPackageToPackageRepositoryException()
{
}
public FailedToAddPackageToPackageRepositoryException(string message, Exception innerException) : base(message,
innerException)
{
}
}
}

49
build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/FileUploadStrategy.cs Normal file
View file

@ -0,0 +1,49 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.
using System;
using System.IO;
using System.Net.Http;
using System.Threading.Tasks;
using NuGet.Protocol;
namespace Microsoft.DotNet.Cli.Build.UploadToLinuxPackageRepository
{
internal class FileUploadStrategy : IAzurelinuxRepositoryServiceHttpStrategy
{
private readonly string _pathToPackageToUpload;
public FileUploadStrategy(string pathToPackageToUpload)
{
_pathToPackageToUpload = pathToPackageToUpload
?? throw new ArgumentNullException(nameof(pathToPackageToUpload));
}
public async Task<string> Execute(HttpClient client, Uri baseAddress)
{
var fileName = Path.GetFileName(_pathToPackageToUpload);
using (var content =
new MultipartFormDataContent())
{
var url = new Uri(baseAddress, "/v1/files");
content.Add(
new StreamContent(
new MemoryStream(
File.ReadAllBytes(_pathToPackageToUpload))),
"file",
fileName);
using (var message = await client.PostAsync(url, content))
{
if (!message.IsSuccessStatusCode)
{
throw new FailedToAddPackageToPackageRepositoryException(
$"{message.ToJson()} failed to post file to {url} file name:{fileName} pathToPackageToUpload:{_pathToPackageToUpload}");
}
return await message.Content.ReadAsStringAsync();
}
}
}
}
}

15
build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/IAzurelinuxRepositoryServiceHttpStrategy.cs Normal file
View file

@ -0,0 +1,15 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.
using System;
using System.Net.Http;
using System.Threading.Tasks;
namespace Microsoft.DotNet.Cli.Build.UploadToLinuxPackageRepository
{
internal interface IAzurelinuxRepositoryServiceHttpStrategy
{
Task<string> Execute(HttpClient client, Uri baseAddress);
}
}

18
build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/IdInRepositoryService.cs Normal file
View file

@ -0,0 +1,18 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.
using System;
namespace Microsoft.DotNet.Cli.Build.UploadToLinuxPackageRepository
{
internal class IdInRepositoryService
{
public IdInRepositoryService(string id)
{
Id = id ?? throw new ArgumentNullException(nameof(id));
}
public string Id { get; }
}
}

38
build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/LinuxPackageRepositoryDestiny.cs Normal file
View file

@ -0,0 +1,38 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.
using System;
namespace Microsoft.DotNet.Cli.Build.UploadToLinuxPackageRepository
{
internal class LinuxPackageRepositoryDestiny
{
private readonly string _password;
private readonly string _server;
private readonly string _username;
public LinuxPackageRepositoryDestiny(string username,
string password,
string server,
string repositoryId)
{
_username = username ?? throw new ArgumentNullException(nameof(username));
_password = password ?? throw new ArgumentNullException(nameof(password));
_server = server ?? throw new ArgumentNullException(nameof(server));
RepositoryId = repositoryId ?? throw new ArgumentNullException(nameof(repositoryId));
}
public string RepositoryId { get; }
public Uri GetBaseAddress()
{
return new Uri($"https://{_server}");
}
public string GetSimpleAuth()
{
return $"{_username}:{_password}";
}
}
}

46
build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/LinuxPackageRepositoryHttpPrepare.cs Normal file
View file

@ -0,0 +1,46 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.
using System;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text;
using System.Threading.Tasks;
namespace Microsoft.DotNet.Cli.Build.UploadToLinuxPackageRepository
{
internal class LinuxPackageRepositoryHttpPrepare
{
private readonly IAzurelinuxRepositoryServiceHttpStrategy _httpStrategy;
private readonly LinuxPackageRepositoryDestiny _linuxPackageRepositoryDestiny;
public LinuxPackageRepositoryHttpPrepare(
LinuxPackageRepositoryDestiny linuxPackageRepositoryDestiny,
IAzurelinuxRepositoryServiceHttpStrategy httpStrategy
)
{
_linuxPackageRepositoryDestiny = linuxPackageRepositoryDestiny
?? throw new ArgumentNullException(nameof(linuxPackageRepositoryDestiny));
_httpStrategy = httpStrategy ?? throw new ArgumentNullException(nameof(httpStrategy));
}
public async Task<string> RemoteCall()
{
using (var handler = new HttpClientHandler())
{
using (var client = new HttpClient(handler))
{
var authHeader =
Convert.ToBase64String(Encoding.UTF8.GetBytes((string) _linuxPackageRepositoryDestiny.GetSimpleAuth()));
client.DefaultRequestHeaders.Authorization =
new AuthenticationHeaderValue("Basic", authHeader);
client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
client.Timeout = TimeSpan.FromMinutes(10);
return await _httpStrategy.Execute(client, _linuxPackageRepositoryDestiny.GetBaseAddress());
}
}
}
}
}

34
build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/PullQueuedPackageStatus.cs Normal file
View file

@ -0,0 +1,34 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.
using System;
using System.Net.Http;
using System.Threading.Tasks;
using Newtonsoft.Json.Linq;
namespace Microsoft.DotNet.Cli.Build.UploadToLinuxPackageRepository
{
internal class PullQueuedPackageStatus : IAzurelinuxRepositoryServiceHttpStrategy
{
private readonly QueueResourceLocation _queueResourceLocation;
public PullQueuedPackageStatus(QueueResourceLocation queueResourceLocation)
{
_queueResourceLocation = queueResourceLocation
?? throw new ArgumentNullException(nameof(queueResourceLocation));
}
public async Task<string> Execute(HttpClient client, Uri baseAddress)
{
using (var response = await client.GetAsync(new Uri(baseAddress, _queueResourceLocation.Location)))
{
if (!response.IsSuccessStatusCode)
throw new FailedToAddPackageToPackageRepositoryException(
"Failed to make request to " + _queueResourceLocation.Location);
var body = await response.Content.ReadAsStringAsync();
return !body.Contains("status") ? "" : JObject.Parse(body)["status"].ToString();
}
}
}
}

18
build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/QueueResourceLocation.cs Normal file
View file

@ -0,0 +1,18 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.
using System;
namespace Microsoft.DotNet.Cli.Build.UploadToLinuxPackageRepository
{
internal class QueueResourceLocation
{
public QueueResourceLocation(string location)
{
Location = location ?? throw new ArgumentNullException(nameof(location));
}
public string Location { get; }
}
}

23
build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/RetryFailedException.cs Normal file
View file

@ -0,0 +1,23 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.
using System;
namespace Microsoft.DotNet.Cli.Build.UploadToLinuxPackageRepository
{
public class RetryFailedException : Exception
{
public RetryFailedException(string message) : base(message)
{
}
public RetryFailedException()
{
}
public RetryFailedException(string message, Exception innerException) : base(message, innerException)
{
}
}
}

108
build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/UploadToLinuxPackageRepository.cs Normal file
View file

@ -0,0 +1,108 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.
using System;
using System.Net.Http;
using System.Threading.Tasks;
using Microsoft.Build.Framework;
using Newtonsoft.Json.Linq;
using Task = Microsoft.Build.Utilities.Task;
namespace Microsoft.DotNet.Cli.Build.UploadToLinuxPackageRepository
{
public class UploadToLinuxPackageRepository : Task
{
/// <summary>
/// The Azure repository service user name.
/// </summary>
[Required]
public string Username { get; set; }
/// <summary>
/// The Azure repository service Password.
/// </summary>
[Required]
public string Password { get; set; }
/// <summary>
/// The Azure repository service URL ex: "tux-devrepo.corp.microsoft.com".
/// </summary>
[Required]
public string Server { get; set; }
[Required]
public string RepositoryId { get; set; }
[Required]
public string PathOfPackageToUpload { get; set; }
[Required]
public string PackageNameInLinuxPackageRepository { get; set; }
[Required]
public string PackageVersionInLinuxPackageRepository { get; set; }
public override bool Execute()
{
ExecuteAsyncWithRetry().GetAwaiter().GetResult();
return true;
}
private async System.Threading.Tasks.Task ExecuteAsyncWithRetry()
{
await ExponentialRetry.ExecuteWithRetry(
UploadAndAddpackageAndEnsureItIsReady,
s => s == "",
maxRetryCount: 3,
timer: () => ExponentialRetry.Timer(ExponentialRetry.Intervals),
taskDescription: $"running {nameof(UploadAndAddpackageAndEnsureItIsReady)}");
}
private async Task<string> UploadAndAddpackageAndEnsureItIsReady()
{
try
{
var linuxPackageRepositoryDestiny =
new LinuxPackageRepositoryDestiny(Username, Password, Server, RepositoryId);
var uploadResponse = await new LinuxPackageRepositoryHttpPrepare(
linuxPackageRepositoryDestiny,
new FileUploadStrategy(PathOfPackageToUpload)).RemoteCall();
var idInRepositoryService = new IdInRepositoryService(JObject.Parse(uploadResponse)["id"].ToString());
var addPackageResponse = await new LinuxPackageRepositoryHttpPrepare(
linuxPackageRepositoryDestiny,
new AddPackageStrategy(
idInRepositoryService,
PackageNameInLinuxPackageRepository,
PackageVersionInLinuxPackageRepository,
linuxPackageRepositoryDestiny.RepositoryId)).RemoteCall();
var queueResourceLocation = new QueueResourceLocation(addPackageResponse);
Func<Task<string>> pullQueuedPackageStatus = new LinuxPackageRepositoryHttpPrepare(
linuxPackageRepositoryDestiny,
new PullQueuedPackageStatus(queueResourceLocation)).RemoteCall;
await ExponentialRetry.ExecuteWithRetry(
pullQueuedPackageStatus,
s => s == "fileReady",
5,
() => ExponentialRetry.Timer(ExponentialRetry.Intervals),
$"PullQueuedPackageStatus location: {queueResourceLocation.Location}");
return "";
}
catch (FailedToAddPackageToPackageRepositoryException e)
{
return e.ToString();
}
catch (HttpRequestException e)
{
return e.ToString();
}
}
}
}