Use Rest Api to upload to the feed
Add pulling logic to make sure it is uploaded to the feed. Add retry logic for the whole upload process Remove the old upload script
This commit is contained in:
William Li
William Lee
parent
8de61cdcdf
commit
7f54ccb903
19 changed files with 647 additions and 214 deletions
|
|
@ -227,6 +227,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "tool_fsc", "src\tool_fsharp
|
|||
EndProject
|
||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.DotNet.MSBuildSdkResolver", "src\Microsoft.DotNet.MSBuildSdkResolver\Microsoft.DotNet.MSBuildSdkResolver.csproj", "{FCDFAF40-CC16-4D49-96C0-E49F195E7142}"
|
||||
EndProject
|
||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "dotnet-cli-build.Tests", "build_projects\dotnet-cli-build.Tests\dotnet-cli-build.Tests.csproj", "{84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}"
|
||||
EndProject
|
||||
Global
|
||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||
Debug|Any CPU = Debug|Any CPU
|
||||
|
|
@ -1563,6 +1565,30 @@ Global
|
|||
{FCDFAF40-CC16-4D49-96C0-E49F195E7142}.RelWithDebInfo|x64.Build.0 = Release|Any CPU
|
||||
{FCDFAF40-CC16-4D49-96C0-E49F195E7142}.RelWithDebInfo|x86.ActiveCfg = Release|Any CPU
|
||||
{FCDFAF40-CC16-4D49-96C0-E49F195E7142}.RelWithDebInfo|x86.Build.0 = Release|Any CPU
|
||||
{84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
{84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.Debug|x64.Build.0 = Debug|Any CPU
|
||||
{84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
|
||||
{84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
||||
{84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
||||
{84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
|
||||
{84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.MinSizeRel|x86.Build.0 = Debug|Any CPU
|
||||
{84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.Release|x64.Build.0 = Release|Any CPU
|
||||
{84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.Release|x86.Build.0 = Release|Any CPU
|
||||
{84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.RelWithDebInfo|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.RelWithDebInfo|Any CPU.Build.0 = Release|Any CPU
|
||||
{84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.RelWithDebInfo|x64.ActiveCfg = Release|Any CPU
|
||||
{84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.RelWithDebInfo|x64.Build.0 = Release|Any CPU
|
||||
{84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.RelWithDebInfo|x86.ActiveCfg = Release|Any CPU
|
||||
{84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.RelWithDebInfo|x86.Build.0 = Release|Any CPU
|
||||
EndGlobalSection
|
||||
GlobalSection(SolutionProperties) = preSolution
|
||||
HideSolutionNode = FALSE
|
||||
|
|
@ -1633,6 +1659,7 @@ Global
|
|||
{08A40B6A-F695-4EA9-AC8D-CF88FADEA796} = {17735A9D-BFD9-4585-A7CB-3208CA6EA8A7}
|
||||
{602976C5-2477-4B4C-AD9A-1EAFB250529A} = {ED2FE3E2-F7E7-4389-8231-B65123F2076F}
|
||||
{FCDFAF40-CC16-4D49-96C0-E49F195E7142} = {ED2FE3E2-F7E7-4389-8231-B65123F2076F}
|
||||
{84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94} = {88278B81-7649-45DC-8A6A-D3A645C5AFC3}
|
||||
EndGlobalSection
|
||||
GlobalSection(ExtensibilityGlobals) = postSolution
|
||||
SolutionGuid = {B526D2CE-EE2D-4AD4-93EF-1867D90FF1F5}
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@
|
|||
<UsingTask TaskName="TarGzFileExtractToDirectory" AssemblyFile="$(CLIBuildDll)" />
|
||||
<UsingTask TaskName="UpdateVersionsRepo" AssemblyFile="$(CLIBuildDll)"/>
|
||||
<UsingTask TaskName="UploadToAzure" AssemblyFile="$(CLIBuildDll)"/>
|
||||
<UsingTask TaskName="UploadToLinuxPackageRepository" AssemblyFile="$(CLIBuildDll)"/>
|
||||
<UsingTask TaskName="ZipFileCreateFromDirectory" AssemblyFile="$(CLIBuildDll)"/>
|
||||
<UsingTask TaskName="ZipFileExtractToDirectory" AssemblyFile="$(CLIBuildDll)"/>
|
||||
</Project>
|
||||
|
|
|
|||
|
|
@ -88,6 +88,8 @@
|
|||
<Target Name="TestsForBuildItself">
|
||||
<DotNetMSBuild Arguments="/v:diag $(RepoRoot)/test/MsBuildScript.Tests/runtests.target"
|
||||
ToolPath="$(Stage0Directory)" />
|
||||
<DotNetTest ProjectPath="$(RepoRoot)/build_projects/dotnet-cli-build.Tests/dotnet-cli-build.Tests.csproj"
|
||||
ToolPath="$(Stage0Directory)" />
|
||||
</Target>
|
||||
|
||||
<Target Name="EnsureStageSeparation">
|
||||
|
|
|
|||
|
|
@ -6,28 +6,23 @@
|
|||
|
||||
<PropertyGroup>
|
||||
<SdkDebianUploadUrl>$(DotnetBlobRootUrl)/$(Product)/$(FullNugetVersion)/$(DistroSpecificArtifactNameWithVersionCombinedHostHostFxrFrameworkSdk)-$(Architecture)$(InstallerExtension)</SdkDebianUploadUrl>
|
||||
<DebianUploadJsonFile>$(SdkDebianIntermediateDirectory)/package_upload.json</DebianUploadJsonFile>
|
||||
<DebianRevisionNumber>1</DebianRevisionNumber>
|
||||
|
||||
<DebianUploadJsonContent>
|
||||
{
|
||||
"name":"$(SdkDebianPackageName)",
|
||||
"version":"$(NugetVersion)-$(DebianRevisionNumber)",
|
||||
"repositoryId":"$(REPO_ID)",
|
||||
"sourceUrl": "$(SdkDebianUploadUrl)"
|
||||
}
|
||||
</DebianUploadJsonContent>
|
||||
</PropertyGroup>
|
||||
|
||||
<Target Name="PublishDebFilesToDebianRepo" Condition=" '$(IsDebianBaseDistro)' == 'True' AND '$(SkipPublishToDebianRepo)' != 'true' AND '$(IslinuxPortable)' != 'true' ">
|
||||
<Error Condition="'$(REPO_ID)' == ''" Text="REPO_ID must be set as an environment variable for debian publishing." />
|
||||
<Error Condition="'$(REPO_USER)' == ''" Text="REPO_USER must be set as an environment variable for debian publishing." />
|
||||
<Error Condition="'$(REPO_PASS)' == ''" Text="REPO_PASS must be set as an environment variable for debian publishing." />
|
||||
<Error Condition="'$(REPO_SERVER)' == ''" Text="REPO_SERVER must be set as an environment variable for debian publishing." />
|
||||
<Target Name="PublishDebFilesToDebianRepo"
|
||||
DependsOnTargets="SetupDebProps;"
|
||||
Condition=" '$(IsDebianBaseDistro)' == 'True' AND '$(SkipPublishToDebianRepo)' != 'true' AND '$(IslinuxPortable)' != 'true' " >
|
||||
<Error Condition="'$(REPO_ID)' == ''" Text="REPO_ID must be set as a MsBuild Property variable for debian publishing." />
|
||||
<Error Condition="'$(REPO_USER)' == ''" Text="REPO_USER must be set as a MsBuild Property variable for debian publishing." />
|
||||
<Error Condition="'$(REPO_PASS)' == ''" Text="REPO_PASS must be set as a MsBuild Property variable for debian publishing." />
|
||||
<Error Condition="'$(REPO_SERVER)' == ''" Text="REPO_SERVER must be set as a MsBuild Property variable for debian publishing." />
|
||||
|
||||
<Delete Files="$(DebianUploadJsonFile)" />
|
||||
<WriteLinesToFile File="$(DebianUploadJsonFile)" Lines="$(DebianUploadJsonContent)" />
|
||||
|
||||
<Exec Command="REPO_PASS=$(REPO_PASS) sh -c '$(RepoRoot)/scripts/publish/repoapi_client.sh -addpkg $(DebianUploadJsonFile)'" />
|
||||
<UploadToLinuxPackageRepository
|
||||
Username="$(REPO_USER)"
|
||||
Password='$(REPO_PASS)'
|
||||
Server='$(REPO_SERVER)'
|
||||
RepositoryId='$(REPO_ID)'
|
||||
PathOfPackageToUpload='$(SdkInstallerFile)'
|
||||
PackageNameInLinuxPackageRepository='$(SdkDebianPackageName)'
|
||||
PackageVersionInLinuxPackageRepository='$(NugetVersion)' />
|
||||
</Target>
|
||||
</Project>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,92 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using FluentAssertions;
|
||||
using Microsoft.DotNet.Cli.Build.UploadToLinuxPackageRepository;
|
||||
using Xunit;
|
||||
|
||||
namespace dotnet_cli_build.Tests
|
||||
{
|
||||
public class GivenActionAndRetryTimes
|
||||
{
|
||||
public static IEnumerable<Task> NoWaitTimer()
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
yield return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ExponentialRetryShouldProvideIntervalSequence()
|
||||
{
|
||||
ExponentialRetry.Intervals.First().Should().Be(TimeSpan.FromSeconds(5));
|
||||
ExponentialRetry.Intervals.Skip(1).First().Should().Be(TimeSpan.FromSeconds(10));
|
||||
ExponentialRetry.Intervals.Skip(2).First().Should().Be(TimeSpan.FromSeconds(20));
|
||||
ExponentialRetry.Intervals.Skip(3).First().Should().Be(TimeSpan.FromSeconds(40));
|
||||
ExponentialRetry.Intervals.Skip(4).First().Should().Be(TimeSpan.FromSeconds(80));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ExponentialShouldNotRetryAfterFirstSucceess()
|
||||
{
|
||||
var fakeAction = new FakeAction(0);
|
||||
ExponentialRetry.ExecuteWithRetry(
|
||||
fakeAction.Run,
|
||||
s => s == "success",
|
||||
10,
|
||||
NoWaitTimer).Wait();
|
||||
fakeAction.Count.Should().Be(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ExponentialShouldRetryUntilSuccess()
|
||||
{
|
||||
var fakeAction = new FakeAction(5);
|
||||
ExponentialRetry.ExecuteWithRetry(
|
||||
fakeAction.Run,
|
||||
s => s == "success",
|
||||
10,
|
||||
NoWaitTimer).Wait();
|
||||
fakeAction.Count.Should().Be(5);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ExponentialShouldThrowAfterMaximumAmountReached()
|
||||
{
|
||||
var fakeAction = new FakeAction(10);
|
||||
Action a = () => ExponentialRetry.ExecuteWithRetry(
|
||||
fakeAction.Run,
|
||||
s => s == "success",
|
||||
5,
|
||||
NoWaitTimer,
|
||||
"testing retry").Wait();
|
||||
a.ShouldThrow<RetryFailedException>()
|
||||
.WithMessage("Retry failed for testing retry after 5 times with result: fail");
|
||||
}
|
||||
}
|
||||
|
||||
public class FakeAction
|
||||
{
|
||||
private readonly int _successAfter;
|
||||
|
||||
public FakeAction(int successAfter)
|
||||
{
|
||||
_successAfter = successAfter;
|
||||
}
|
||||
|
||||
public int Count { get; private set; }
|
||||
|
||||
public Task<string> Run()
|
||||
{
|
||||
if (_successAfter == Count)
|
||||
{
|
||||
return Task.FromResult("success");
|
||||
}
|
||||
|
||||
Count++;
|
||||
return Task.FromResult("fail");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.props))\dir.props" />
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>$(CliTargetFramework)</TargetFramework>
|
||||
<VersionPrefix>1.0.0</VersionPrefix>
|
||||
<AssetTargetFallback>$(AssetTargetFallback);portable-net45+win8+wp8+wpa81</AssetTargetFallback>
|
||||
<IsPackable>false</IsPackable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="$(CLI_TestPlatform_Version)" />
|
||||
<PackageReference Include="xunit" Version="2.2.0" />
|
||||
<PackageReference Include="xunit.runner.visualstudio" Version="2.2.0" />
|
||||
<PackageReference Include="FluentAssertions" Version="4.18.0" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\dotnet-cli-build\dotnet-cli-build.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
|
@ -0,0 +1,62 @@
|
|||
// Licensed to the .NET Foundation under one or more agreements.
|
||||
// The .NET Foundation licenses this file to you under the MIT license.
|
||||
// See the LICENSE file in the project root for more information.
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Net.Http;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using NuGet.Protocol;
|
||||
|
||||
namespace Microsoft.DotNet.Cli.Build.UploadToLinuxPackageRepository
|
||||
{
|
||||
internal class AddPackageStrategy : IAzurelinuxRepositoryServiceHttpStrategy
|
||||
{
|
||||
private readonly IdInRepositoryService _idInRepositoryService;
|
||||
private readonly string _packageName;
|
||||
private readonly string _packageVersion;
|
||||
private readonly string _repositoryId;
|
||||
|
||||
public AddPackageStrategy(
|
||||
IdInRepositoryService idInRepositoryService,
|
||||
string packageName,
|
||||
string packageVersion,
|
||||
string repositoryId)
|
||||
{
|
||||
_idInRepositoryService = idInRepositoryService
|
||||
?? throw new ArgumentNullException(nameof(idInRepositoryService));
|
||||
_packageName = packageName;
|
||||
_packageVersion = packageVersion;
|
||||
_repositoryId = repositoryId;
|
||||
}
|
||||
|
||||
public async Task<string> Execute(HttpClient client, Uri baseAddress)
|
||||
{
|
||||
var debianUploadJsonContent = new Dictionary<string, string>
|
||||
{
|
||||
["name"] = _packageName,
|
||||
["version"] = AppendDebianRevisionNumber(_packageVersion),
|
||||
["fileId"] = _idInRepositoryService.Id,
|
||||
["repositoryId"] = _repositoryId
|
||||
}.ToJson();
|
||||
var content = new StringContent(debianUploadJsonContent,
|
||||
Encoding.UTF8,
|
||||
"application/json");
|
||||
|
||||
using (var response = await client.PostAsync(new Uri(baseAddress, "/v1/packages"), content))
|
||||
{
|
||||
if (!response.IsSuccessStatusCode)
|
||||
throw new FailedToAddPackageToPackageRepositoryException(
|
||||
$"request:{debianUploadJsonContent} response:{response.ToJson()}");
|
||||
return response.Headers.GetValues("Location").Single();
|
||||
}
|
||||
}
|
||||
|
||||
private static string AppendDebianRevisionNumber(string packageVersion)
|
||||
{
|
||||
return packageVersion + "-1";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,53 @@
|
|||
// Licensed to the .NET Foundation under one or more agreements.
|
||||
// The .NET Foundation licenses this file to you under the MIT license.
|
||||
// See the LICENSE file in the project root for more information.
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace Microsoft.DotNet.Cli.Build.UploadToLinuxPackageRepository
|
||||
{
|
||||
public static class ExponentialRetry
|
||||
{
|
||||
public static IEnumerable<TimeSpan> Intervals
|
||||
{
|
||||
get
|
||||
{
|
||||
var seconds = 5;
|
||||
while (true)
|
||||
{
|
||||
yield return TimeSpan.FromSeconds(seconds);
|
||||
seconds *= 2;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static async Task ExecuteWithRetry(Func<Task<string>> action,
|
||||
Func<string, bool> isSuccess,
|
||||
int maxRetryCount,
|
||||
Func<IEnumerable<Task>> timer,
|
||||
string taskDescription = "")
|
||||
{
|
||||
var count = 0;
|
||||
foreach (var t in timer())
|
||||
{
|
||||
await t;
|
||||
var result = await action();
|
||||
if (isSuccess(result))
|
||||
return;
|
||||
count++;
|
||||
if (count == maxRetryCount)
|
||||
throw new RetryFailedException(
|
||||
$"Retry failed for {taskDescription} after {count} times with result: {result}");
|
||||
}
|
||||
throw new Exception("Timer should not be exhausted");
|
||||
}
|
||||
|
||||
public static IEnumerable<Task> Timer(IEnumerable<TimeSpan> interval)
|
||||
{
|
||||
return interval.Select(Task.Delay);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
// Licensed to the .NET Foundation under one or more agreements.
|
||||
// The .NET Foundation licenses this file to you under the MIT license.
|
||||
// See the LICENSE file in the project root for more information.
|
||||
|
||||
using System;
|
||||
|
||||
namespace Microsoft.DotNet.Cli.Build.UploadToLinuxPackageRepository
|
||||
{
|
||||
public class FailedToAddPackageToPackageRepositoryException : Exception
|
||||
{
|
||||
public FailedToAddPackageToPackageRepositoryException(string message) : base(message)
|
||||
{
|
||||
}
|
||||
|
||||
public FailedToAddPackageToPackageRepositoryException()
|
||||
{
|
||||
}
|
||||
|
||||
public FailedToAddPackageToPackageRepositoryException(string message, Exception innerException) : base(message,
|
||||
innerException)
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,49 @@
|
|||
// Licensed to the .NET Foundation under one or more agreements.
|
||||
// The .NET Foundation licenses this file to you under the MIT license.
|
||||
// See the LICENSE file in the project root for more information.
|
||||
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Net.Http;
|
||||
using System.Threading.Tasks;
|
||||
using NuGet.Protocol;
|
||||
|
||||
namespace Microsoft.DotNet.Cli.Build.UploadToLinuxPackageRepository
|
||||
{
|
||||
internal class FileUploadStrategy : IAzurelinuxRepositoryServiceHttpStrategy
|
||||
{
|
||||
private readonly string _pathToPackageToUpload;
|
||||
|
||||
public FileUploadStrategy(string pathToPackageToUpload)
|
||||
{
|
||||
_pathToPackageToUpload = pathToPackageToUpload
|
||||
?? throw new ArgumentNullException(nameof(pathToPackageToUpload));
|
||||
}
|
||||
|
||||
public async Task<string> Execute(HttpClient client, Uri baseAddress)
|
||||
{
|
||||
var fileName = Path.GetFileName(_pathToPackageToUpload);
|
||||
|
||||
using (var content =
|
||||
new MultipartFormDataContent())
|
||||
{
|
||||
var url = new Uri(baseAddress, "/v1/files");
|
||||
content.Add(
|
||||
new StreamContent(
|
||||
new MemoryStream(
|
||||
File.ReadAllBytes(_pathToPackageToUpload))),
|
||||
"file",
|
||||
fileName);
|
||||
using (var message = await client.PostAsync(url, content))
|
||||
{
|
||||
if (!message.IsSuccessStatusCode)
|
||||
{
|
||||
throw new FailedToAddPackageToPackageRepositoryException(
|
||||
$"{message.ToJson()} failed to post file to {url} file name:{fileName} pathToPackageToUpload:{_pathToPackageToUpload}");
|
||||
}
|
||||
return await message.Content.ReadAsStringAsync();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
// Licensed to the .NET Foundation under one or more agreements.
|
||||
// The .NET Foundation licenses this file to you under the MIT license.
|
||||
// See the LICENSE file in the project root for more information.
|
||||
|
||||
using System;
|
||||
using System.Net.Http;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace Microsoft.DotNet.Cli.Build.UploadToLinuxPackageRepository
|
||||
{
|
||||
internal interface IAzurelinuxRepositoryServiceHttpStrategy
|
||||
{
|
||||
Task<string> Execute(HttpClient client, Uri baseAddress);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
// Licensed to the .NET Foundation under one or more agreements.
|
||||
// The .NET Foundation licenses this file to you under the MIT license.
|
||||
// See the LICENSE file in the project root for more information.
|
||||
|
||||
using System;
|
||||
|
||||
namespace Microsoft.DotNet.Cli.Build.UploadToLinuxPackageRepository
|
||||
{
|
||||
internal class IdInRepositoryService
|
||||
{
|
||||
public IdInRepositoryService(string id)
|
||||
{
|
||||
Id = id ?? throw new ArgumentNullException(nameof(id));
|
||||
}
|
||||
|
||||
public string Id { get; }
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
// Licensed to the .NET Foundation under one or more agreements.
|
||||
// The .NET Foundation licenses this file to you under the MIT license.
|
||||
// See the LICENSE file in the project root for more information.
|
||||
|
||||
using System;
|
||||
|
||||
namespace Microsoft.DotNet.Cli.Build.UploadToLinuxPackageRepository
|
||||
{
|
||||
internal class LinuxPackageRepositoryDestiny
|
||||
{
|
||||
private readonly string _password;
|
||||
private readonly string _server;
|
||||
private readonly string _username;
|
||||
|
||||
public LinuxPackageRepositoryDestiny(string username,
|
||||
string password,
|
||||
string server,
|
||||
string repositoryId)
|
||||
{
|
||||
_username = username ?? throw new ArgumentNullException(nameof(username));
|
||||
_password = password ?? throw new ArgumentNullException(nameof(password));
|
||||
_server = server ?? throw new ArgumentNullException(nameof(server));
|
||||
RepositoryId = repositoryId ?? throw new ArgumentNullException(nameof(repositoryId));
|
||||
}
|
||||
|
||||
public string RepositoryId { get; }
|
||||
|
||||
public Uri GetBaseAddress()
|
||||
{
|
||||
return new Uri($"https://{_server}");
|
||||
}
|
||||
|
||||
public string GetSimpleAuth()
|
||||
{
|
||||
return $"{_username}:{_password}";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
// Licensed to the .NET Foundation under one or more agreements.
|
||||
// The .NET Foundation licenses this file to you under the MIT license.
|
||||
// See the LICENSE file in the project root for more information.
|
||||
|
||||
using System;
|
||||
using System.Net.Http;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace Microsoft.DotNet.Cli.Build.UploadToLinuxPackageRepository
|
||||
{
|
||||
internal class LinuxPackageRepositoryHttpPrepare
|
||||
{
|
||||
private readonly IAzurelinuxRepositoryServiceHttpStrategy _httpStrategy;
|
||||
private readonly LinuxPackageRepositoryDestiny _linuxPackageRepositoryDestiny;
|
||||
|
||||
public LinuxPackageRepositoryHttpPrepare(
|
||||
LinuxPackageRepositoryDestiny linuxPackageRepositoryDestiny,
|
||||
IAzurelinuxRepositoryServiceHttpStrategy httpStrategy
|
||||
)
|
||||
{
|
||||
_linuxPackageRepositoryDestiny = linuxPackageRepositoryDestiny
|
||||
?? throw new ArgumentNullException(nameof(linuxPackageRepositoryDestiny));
|
||||
_httpStrategy = httpStrategy ?? throw new ArgumentNullException(nameof(httpStrategy));
|
||||
}
|
||||
|
||||
public async Task<string> RemoteCall()
|
||||
{
|
||||
using (var handler = new HttpClientHandler())
|
||||
{
|
||||
using (var client = new HttpClient(handler))
|
||||
{
|
||||
var authHeader =
|
||||
Convert.ToBase64String(Encoding.UTF8.GetBytes((string) _linuxPackageRepositoryDestiny.GetSimpleAuth()));
|
||||
client.DefaultRequestHeaders.Authorization =
|
||||
new AuthenticationHeaderValue("Basic", authHeader);
|
||||
client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
|
||||
client.Timeout = TimeSpan.FromMinutes(10);
|
||||
|
||||
return await _httpStrategy.Execute(client, _linuxPackageRepositoryDestiny.GetBaseAddress());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
// Licensed to the .NET Foundation under one or more agreements.
|
||||
// The .NET Foundation licenses this file to you under the MIT license.
|
||||
// See the LICENSE file in the project root for more information.
|
||||
|
||||
using System;
|
||||
using System.Net.Http;
|
||||
using System.Threading.Tasks;
|
||||
using Newtonsoft.Json.Linq;
|
||||
|
||||
namespace Microsoft.DotNet.Cli.Build.UploadToLinuxPackageRepository
|
||||
{
|
||||
internal class PullQueuedPackageStatus : IAzurelinuxRepositoryServiceHttpStrategy
|
||||
{
|
||||
private readonly QueueResourceLocation _queueResourceLocation;
|
||||
|
||||
public PullQueuedPackageStatus(QueueResourceLocation queueResourceLocation)
|
||||
{
|
||||
_queueResourceLocation = queueResourceLocation
|
||||
?? throw new ArgumentNullException(nameof(queueResourceLocation));
|
||||
}
|
||||
|
||||
public async Task<string> Execute(HttpClient client, Uri baseAddress)
|
||||
{
|
||||
using (var response = await client.GetAsync(new Uri(baseAddress, _queueResourceLocation.Location)))
|
||||
{
|
||||
if (!response.IsSuccessStatusCode)
|
||||
throw new FailedToAddPackageToPackageRepositoryException(
|
||||
"Failed to make request to " + _queueResourceLocation.Location);
|
||||
var body = await response.Content.ReadAsStringAsync();
|
||||
return !body.Contains("status") ? "" : JObject.Parse(body)["status"].ToString();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
// Licensed to the .NET Foundation under one or more agreements.
|
||||
// The .NET Foundation licenses this file to you under the MIT license.
|
||||
// See the LICENSE file in the project root for more information.
|
||||
|
||||
using System;
|
||||
|
||||
namespace Microsoft.DotNet.Cli.Build.UploadToLinuxPackageRepository
|
||||
{
|
||||
internal class QueueResourceLocation
|
||||
{
|
||||
public QueueResourceLocation(string location)
|
||||
{
|
||||
Location = location ?? throw new ArgumentNullException(nameof(location));
|
||||
}
|
||||
|
||||
public string Location { get; }
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
// Licensed to the .NET Foundation under one or more agreements.
|
||||
// The .NET Foundation licenses this file to you under the MIT license.
|
||||
// See the LICENSE file in the project root for more information.
|
||||
|
||||
using System;
|
||||
|
||||
namespace Microsoft.DotNet.Cli.Build.UploadToLinuxPackageRepository
|
||||
{
|
||||
public class RetryFailedException : Exception
|
||||
{
|
||||
public RetryFailedException(string message) : base(message)
|
||||
{
|
||||
}
|
||||
|
||||
public RetryFailedException()
|
||||
{
|
||||
}
|
||||
|
||||
public RetryFailedException(string message, Exception innerException) : base(message, innerException)
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,108 @@
|
|||
// Licensed to the .NET Foundation under one or more agreements.
|
||||
// The .NET Foundation licenses this file to you under the MIT license.
|
||||
// See the LICENSE file in the project root for more information.
|
||||
|
||||
using System;
|
||||
using System.Net.Http;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Build.Framework;
|
||||
using Newtonsoft.Json.Linq;
|
||||
using Task = Microsoft.Build.Utilities.Task;
|
||||
|
||||
namespace Microsoft.DotNet.Cli.Build.UploadToLinuxPackageRepository
|
||||
{
|
||||
public class UploadToLinuxPackageRepository : Task
|
||||
{
|
||||
/// <summary>
|
||||
/// The Azure repository service user name.
|
||||
/// </summary>
|
||||
[Required]
|
||||
public string Username { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The Azure repository service Password.
|
||||
/// </summary>
|
||||
[Required]
|
||||
public string Password { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The Azure repository service URL ex: "tux-devrepo.corp.microsoft.com".
|
||||
/// </summary>
|
||||
[Required]
|
||||
public string Server { get; set; }
|
||||
|
||||
[Required]
|
||||
public string RepositoryId { get; set; }
|
||||
|
||||
[Required]
|
||||
public string PathOfPackageToUpload { get; set; }
|
||||
|
||||
[Required]
|
||||
public string PackageNameInLinuxPackageRepository { get; set; }
|
||||
|
||||
|
||||
[Required]
|
||||
public string PackageVersionInLinuxPackageRepository { get; set; }
|
||||
|
||||
|
||||
public override bool Execute()
|
||||
{
|
||||
ExecuteAsyncWithRetry().GetAwaiter().GetResult();
|
||||
return true;
|
||||
}
|
||||
|
||||
private async System.Threading.Tasks.Task ExecuteAsyncWithRetry()
|
||||
{
|
||||
await ExponentialRetry.ExecuteWithRetry(
|
||||
UploadAndAddpackageAndEnsureItIsReady,
|
||||
s => s == "",
|
||||
maxRetryCount: 3,
|
||||
timer: () => ExponentialRetry.Timer(ExponentialRetry.Intervals),
|
||||
taskDescription: $"running {nameof(UploadAndAddpackageAndEnsureItIsReady)}");
|
||||
}
|
||||
|
||||
private async Task<string> UploadAndAddpackageAndEnsureItIsReady()
|
||||
{
|
||||
try
|
||||
{
|
||||
var linuxPackageRepositoryDestiny =
|
||||
new LinuxPackageRepositoryDestiny(Username, Password, Server, RepositoryId);
|
||||
var uploadResponse = await new LinuxPackageRepositoryHttpPrepare(
|
||||
linuxPackageRepositoryDestiny,
|
||||
new FileUploadStrategy(PathOfPackageToUpload)).RemoteCall();
|
||||
|
||||
var idInRepositoryService = new IdInRepositoryService(JObject.Parse(uploadResponse)["id"].ToString());
|
||||
|
||||
var addPackageResponse = await new LinuxPackageRepositoryHttpPrepare(
|
||||
linuxPackageRepositoryDestiny,
|
||||
new AddPackageStrategy(
|
||||
idInRepositoryService,
|
||||
PackageNameInLinuxPackageRepository,
|
||||
PackageVersionInLinuxPackageRepository,
|
||||
linuxPackageRepositoryDestiny.RepositoryId)).RemoteCall();
|
||||
|
||||
var queueResourceLocation = new QueueResourceLocation(addPackageResponse);
|
||||
|
||||
Func<Task<string>> pullQueuedPackageStatus = new LinuxPackageRepositoryHttpPrepare(
|
||||
linuxPackageRepositoryDestiny,
|
||||
new PullQueuedPackageStatus(queueResourceLocation)).RemoteCall;
|
||||
|
||||
await ExponentialRetry.ExecuteWithRetry(
|
||||
pullQueuedPackageStatus,
|
||||
s => s == "fileReady",
|
||||
5,
|
||||
() => ExponentialRetry.Timer(ExponentialRetry.Intervals),
|
||||
$"PullQueuedPackageStatus location: {queueResourceLocation.Location}");
|
||||
return "";
|
||||
}
|
||||
catch (FailedToAddPackageToPackageRepositoryException e)
|
||||
{
|
||||
return e.ToString();
|
||||
}
|
||||
catch (HttpRequestException e)
|
||||
{
|
||||
return e.ToString();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,194 +0,0 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Copyright (c) .NET Foundation and contributors. All rights reserved.
|
||||
# Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
||||
#
|
||||
|
||||
# This is a VERY basic script for Create/Delete operations on repos and packages
|
||||
#
|
||||
# Environment Dependencies:
|
||||
# $REPO_SERVER
|
||||
# $REPO_USER
|
||||
# $REPO_PASS
|
||||
|
||||
cmd=$1
|
||||
urls=urls.txt
|
||||
defaultPackageFile=new_package.json
|
||||
defaultRepoFile=new_repo.json
|
||||
repositoryId=$REPO_ID
|
||||
server=$REPO_SERVER
|
||||
user=$REPO_USER
|
||||
pass=$REPO_PASS
|
||||
protocol=https
|
||||
port=443
|
||||
baseurl="$protocol://$user:$pass@$server:$port"
|
||||
|
||||
echo $baseurl
|
||||
|
||||
function BailIf
|
||||
{
|
||||
if [ $1 -ne 0 ]; then
|
||||
echo "Failure occurred communicating with $server"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# List packages, using $1 as a regex to filter results
|
||||
function ListPackages
|
||||
{
|
||||
curl -k "$baseurl/v1/packages" | sed 's/{/\n{/g' | egrep "$1" | sed 's/,/,\n/g' | sed 's/^"/\t"/g'
|
||||
echo ""
|
||||
}
|
||||
|
||||
# Create a new Repo using the specified JSON file
|
||||
function AddRepo
|
||||
{
|
||||
repoFile=$1
|
||||
if [ -z $repoFile ]; then
|
||||
echo "Error: Must specify a JSON-formatted file. Reference $defaultRepoFile.template"
|
||||
exit 1
|
||||
fi
|
||||
if [ ! -f $repoFile ]; then
|
||||
echo "Error: Cannot create repo - $repoFile does not exist"
|
||||
exit 1
|
||||
fi
|
||||
packageUrl=$(grep "url" $repoFile | head -n 1 | awk '{print $2}' | tr -d ',')
|
||||
echo "Creating new repo on $server [$packageUrl]"
|
||||
curl -i -k "$baseurl/v1/repositories" --data @./$repoFile -H "Content-Type: application/json"
|
||||
BailIf $?
|
||||
echo ""
|
||||
}
|
||||
|
||||
# Upload a single package using the specified JSON file
|
||||
function AddPackage
|
||||
{
|
||||
packageFile=$1
|
||||
if [ -z $packageFile ]; then
|
||||
echo "Error: Must specify a JSON-formatted file. Reference $defaultPackageFile.template"
|
||||
exit 1
|
||||
fi
|
||||
if [ ! -f $packageFile ]; then
|
||||
echo "Error: Cannot add package - $packageFile does not exist"
|
||||
exit 1
|
||||
fi
|
||||
packageUrl=$(grep "sourceUrl" $packageFile | head -n 1 | awk '{print $2}')
|
||||
echo "Adding package to $server [$packageUrl]"
|
||||
curl -i -k "$baseurl/v1/packages" --data @$packageFile -H "Content-Type: application/json"
|
||||
BailIf $?
|
||||
echo ""
|
||||
}
|
||||
|
||||
# Upload a single package by dynamically creating a JSON file using a provided URL
|
||||
function AddPackageByUrl
|
||||
{
|
||||
# Parse URL
|
||||
url=$(echo "$1")
|
||||
if [ -z $url ]; then
|
||||
return
|
||||
fi
|
||||
escapedUrl=$(echo "$url" | sed 's/\//\\\//g')
|
||||
set -- "$1"
|
||||
oldIFS=$IFS
|
||||
IFS="/"; declare -a splitUrl=($*)
|
||||
index=${#splitUrl[@]}
|
||||
let "index -= 1"
|
||||
filename=${splitUrl[$index]}
|
||||
set -- "$filename"
|
||||
IFS="_"; declare -a splitFile=($*)
|
||||
IFS=$oldIFS
|
||||
pkgName=${splitFile[0]}
|
||||
pkgVer=${splitFile[1]}
|
||||
if [ -z $pkgName ] || [ -z $pkgVer ]; then
|
||||
echo "ERROR parsing $url"
|
||||
return
|
||||
fi
|
||||
# Create Package .json file
|
||||
cp $defaultPackageFile.template $defaultPackageFile
|
||||
sed -i "s/PACKAGENAME/$pkgName/g" $defaultPackageFile
|
||||
sed -i "s/PACKAGEVERSION/$pkgVer/g" $defaultPackageFile
|
||||
sed -i "s/PACKAGEURL/$escapedUrl/g" $defaultPackageFile
|
||||
sed -i "s/REPOSITORYID/$repositoryId/g" $defaultPackageFile
|
||||
# Test that URL is ok
|
||||
wget -q --spider "$url"
|
||||
if [[ $? -eq 0 ]]; then
|
||||
echo "Ready to upload $pkgName [$pkgVer]"
|
||||
else
|
||||
echo "ERROR testing URL $url"
|
||||
return
|
||||
fi
|
||||
# Perform Upload
|
||||
AddPackage $defaultPackageFile
|
||||
# Cleanup
|
||||
# rm $defaultPackageFile
|
||||
}
|
||||
|
||||
# Upload multiple packages by reading urls line-by-line from the specified file
|
||||
function AddPackages
|
||||
{
|
||||
urlFile=$1
|
||||
if [ -z $urlFile ]; then
|
||||
echo "Error: Must specify a flat text file containing one or more URLs"
|
||||
exit 1
|
||||
fi
|
||||
if [ ! -f $urlFile ]; then
|
||||
echo "Error: Cannot add packages. File $urlFile does not exist"
|
||||
exit 1
|
||||
fi
|
||||
for url in $(cat $urlFile); do
|
||||
AddPackageByUrl "$url"
|
||||
sleep 5
|
||||
done
|
||||
}
|
||||
|
||||
# Delete the specified repo
|
||||
function DeleteRepo
|
||||
{
|
||||
repoId=$1
|
||||
if [ -z $repoId ]; then
|
||||
echo "Error: Please specify repository ID. Run -listrepos for a list of IDs"
|
||||
exit 1
|
||||
fi
|
||||
curl -I -k -X DELETE "$baseurl/v1/repositories/$repoId"
|
||||
BailIf $?
|
||||
}
|
||||
|
||||
# Delete the specified package
|
||||
function DeletePackage
|
||||
{
|
||||
packageId=$1
|
||||
if [ -z $packageId ]; then
|
||||
echo "Error: Please specify package ID. Run -listpkgs for a list of IDs"
|
||||
exit 1
|
||||
fi
|
||||
echo Removing pkgId $packageId from repo $repositoryId
|
||||
curl -I -k -X DELETE "$baseurl/v1/packages/$packageId"
|
||||
BailIf $?
|
||||
}
|
||||
|
||||
if [[ "$1" == "-listrepos" ]]; then
|
||||
echo "Fetching repo list from $server..."
|
||||
curl -k "$baseurl/v1/repositories" | sed 's/,/,\n/g' | sed 's/^"/\t"/g'
|
||||
echo ""
|
||||
elif [[ "$1" == "-listpkgs" ]]; then
|
||||
echo "Fetching package list from $server"
|
||||
ListPackages $2
|
||||
elif [[ "$1" == "-addrepo" ]]; then
|
||||
AddRepo $2
|
||||
elif [[ "$1" == "-addpkg" ]]; then
|
||||
AddPackage $2
|
||||
elif [[ "$1" == "-addpkgs" ]]; then
|
||||
AddPackages $2
|
||||
elif [[ "$1" == "-delrepo" ]]; then
|
||||
DeleteRepo $2
|
||||
elif [[ "$1" == "-delpkg" ]]; then
|
||||
DeletePackage $2
|
||||
else
|
||||
echo "USAGE: ./repotool.sh -OPTION"
|
||||
echo "-listrepos: Gather a list of repos"
|
||||
echo "-listpkgs: Gather a list of packages"
|
||||
echo "-addrepo [FILENAME] : Create a new repo using the specified JSON file"
|
||||
echo "-addpkg [FILENAME] : Add package to repo using the specified JSON file"
|
||||
echo "-addpkgs [FILENAME] : Add packages to repo using urls contained in FILENAME"
|
||||
echo "-delrepo REPOID : Delete the specified repo by ID"
|
||||
echo "-delpkg PKGID : Delete the specified package by ID"
|
||||
fi
|
||||
Loading…
Add table
Reference in a new issue