Merge pull request #3872 from livarcocc/publish_msbuild
Publish msbuild
This commit is contained in:
Livar
GitHub
commit
3f52e83daf
7 changed files with 383 additions and 13 deletions
29
build.proj
29
build.proj
|
|
@ -18,6 +18,7 @@
|
|||
|
||||
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
|
||||
<RepoRoot>$(MSBuildThisFileDirectory)</RepoRoot>
|
||||
<BuildToolsDir>$(RepoRoot)/build_tools</BuildToolsDir>
|
||||
|
||||
<CoreSetupChannel>preview</CoreSetupChannel>
|
||||
<SharedFrameworkName>Microsoft.NETCore.App</SharedFrameworkName>
|
||||
|
|
@ -46,32 +47,36 @@
|
|||
<Message Text="Dont remove this target" />
|
||||
</Target>
|
||||
|
||||
<Target Name="BuildDotnetCliBuildFramework"
|
||||
Inputs="@(DotnetCliBuildFrameworkInputs)"
|
||||
<Target Name="BuildDotnetCliBuildFramework"
|
||||
Inputs="@(DotnetCliBuildFrameworkInputs)"
|
||||
Outputs="$(CLIBuildDll)"
|
||||
DependsOnTargets="MSBuildWorkaroundTarget">
|
||||
<PropertyGroup>
|
||||
<DotnetCliBuildDirectory>$(MSBuildThisFileDirectory)/build_projects/dotnet-cli-build</DotnetCliBuildDirectory>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<DotnetSdkDirectories Include="$([System.IO.Directory]::GetDirectories("$(Stage0Path)/sdk"))" />
|
||||
<AzureStorageToCopy Include="@(DotnetSdkDirectories)">
|
||||
<Source>$(DotnetCliBuildDirectory)/bin/Microsoft.WindowsAzure.Storage.dll</Source>
|
||||
<Destination>%(Identity)/Microsoft.WindowsAzure.Storage.dll</Destination>
|
||||
</AzureStorageToCopy>
|
||||
</ItemGroup>
|
||||
|
||||
<Exec Command="$(DotnetStage0) restore" WorkingDirectory="$(DotnetCliBuildDirectory)"/>
|
||||
|
||||
<Exec Command="$(DotnetStage0) publish -o $(DotnetCliBuildDirectory)/bin --framework netcoreapp1.0" WorkingDirectory="$(DotnetCliBuildDirectory)"/>
|
||||
<Exec Command="$(DotnetStage0) publish -o $(DotnetCliBuildDirectory)/bin --framework netcoreapp1.0" WorkingDirectory="$(DotnetCliBuildDirectory)"/>
|
||||
|
||||
<!-- This is a work around to issue https://github.com/Microsoft/msbuild/issues/658 -->
|
||||
<Copy SourceFiles="%(AzureStorageToCopy.Source)"
|
||||
DestinationFiles="%(AzureStorageToCopy.Destination)" />
|
||||
</Target>
|
||||
|
||||
<Target DependsOnTargets="$(CLITargets)" Name="BuildTheWholeCli"></Target>
|
||||
|
||||
<UsingTask TaskName="PublishTargets" AssemblyFile="$(CLIBuildDll)" />
|
||||
|
||||
<Target DependsOnTargets="BuildDotnetCliBuildFramework;GenerateVersionBadge" Name="Publish">
|
||||
<PublishTargets />
|
||||
</Target>
|
||||
|
||||
<Import Project="build/Microsoft.DotNet.Cli.Prepare.targets" />
|
||||
<Import Project="build/Microsoft.DotNet.Cli.Compile.targets" />
|
||||
<Import Project="build/Microsoft.DotNet.Cli.Package.targets" />
|
||||
<Import Project="build/Microsoft.DotNet.Cli.Test.targets" />
|
||||
|
||||
<!-- This should be in the build/Microsoft.DotNet.Cli.Publish.targets -->
|
||||
<Import Project="build/publish/Microsoft.DotNet.Cli.Badge.targets" />
|
||||
<Import Project="build/Microsoft.DotNet.Cli.Publish.targets" />
|
||||
</Project>
|
||||
|
|
|
|||
111
build/Microsoft.DotNet.Cli.Publish.targets
Normal file
111
build/Microsoft.DotNet.Cli.Publish.targets
Normal file
|
|
@ -0,0 +1,111 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<Import Project="$(BuildToolsDir)/PublishContent.targets" />
|
||||
<Import Project="$(MSBuildThisFileDirectory)/publish/Microsoft.DotNet.Cli.Badge.targets" />
|
||||
|
||||
<UsingTask TaskName="UploadToAzure" AssemblyFile="$(BuildToolsTaskDir)/Microsoft.DotNet.Build.CloudTestTasks.dll"/>
|
||||
<UsingTask TaskName="FinalizeBuild" AssemblyFile="$(CLIBuildDll)" />
|
||||
<UsingTask TaskName="SetBlobPropertiesBasedOnFileType" AssemblyFile="$(CLIBuildDll)" />
|
||||
|
||||
<!-- PUBLISH_TO_AZURE_BLOB env variable set by CI -->
|
||||
<Target Name="Publish"
|
||||
Condition=" '$(PUBLISH_TO_AZURE_BLOB)' != '' "
|
||||
DependsOnTargets="Init;
|
||||
Package;
|
||||
PublishArtifacts;
|
||||
FinishBuild" />
|
||||
|
||||
<!-- UploadToAzure target comes from Build Tools -->
|
||||
<Target Name="PublishArtifacts"
|
||||
DependsOnTargets="SetupAzureBlobInformation;
|
||||
GenerateVersionBadge;
|
||||
GatherItemsForPattern;
|
||||
UploadToAzure;
|
||||
PublishDebFilesToDebianRepo;
|
||||
PublishCliVersionBadge" />
|
||||
|
||||
<Target Name="FinishBuild">
|
||||
<FinalizeBuild AccountName="$(CloudDropAccountName)"
|
||||
AccountKey="$(CloudDropAccessToken)"
|
||||
NugetVersion="$(NugetVersion)"
|
||||
Channel="$(Channel)"
|
||||
CommitHash="$(CommitHash)"
|
||||
BranchName="$(BranchName)" />
|
||||
</Target>
|
||||
|
||||
<Target Name="SetupAzureBlobInformation">
|
||||
<PropertyGroup>
|
||||
<Product>Sdk</Product>
|
||||
<ContainerName>$(STORAGE_CONTAINER)</ContainerName>
|
||||
<CloudDropAccessToken>$(STORAGE_KEY)</CloudDropAccessToken>
|
||||
<CloudDropAccountName>$(STORAGE_ACCOUNT)</CloudDropAccountName>
|
||||
<DotnetBlobRootUrl>https://$(CloudDropAccountName).blob.core.windows.net/$(ContainerName)</DotnetBlobRootUrl>
|
||||
</PropertyGroup>
|
||||
</Target>
|
||||
|
||||
<Target Name="GatherItemsForPattern">
|
||||
<ItemGroup>
|
||||
<ForPublishing Include="@(GeneratedInstallers)" />
|
||||
<ForPublishing Include="%(GenerateArchivesInputsOutputs.Outputs)" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ForPublishing>
|
||||
<RelativeBlobPath>$(Product)/$(NugetVersion)/$([System.String]::Copy('%(Filename)%(Extension)').Replace('\' ,'/'))</RelativeBlobPath>
|
||||
</ForPublishing>
|
||||
</ItemGroup>
|
||||
</Target>
|
||||
|
||||
<Target Name="PublishDebFilesToDebianRepo" Condition=" '$(OSName)' == 'ubuntu' ">
|
||||
<Error Condition="'$(REPO_ID)' == ''" Text="REPO_ID must be set as an environment variable for debian publishing." />
|
||||
<Error Condition="'$(REPO_USER)' == ''" Text="REPO_USER must be set as an environment variable for debian publishing." />
|
||||
<Error Condition="'$(REPO_PASS)' == ''" Text="REPO_PASS must be set as an environment variable for debian publishing." />
|
||||
<Error Condition="'$(REPO_SERVER)' == ''" Text="REPO_SERVER must be set as an environment variable for debian publishing." />
|
||||
|
||||
<ItemGroup>
|
||||
<SdkInstallerFileItemGroup Include="$(SdkInstallerFile)" />
|
||||
</ItemGroup>
|
||||
<PropertyGroup>
|
||||
<SdkDebianUploadUrl>$(DotnetBlobRootUrl)/$(Product)/$(NugetVersion)/%(SdkInstallerFileItemGroup.Filename)%(SdkInstallerFileItemGroup.Extension)</SdkDebianUploadUrl>
|
||||
<DebianUploadJsonFile>$(SdkDebianIntermediateDirectory)/package_upload.json</DebianUploadJsonFile>
|
||||
<DebianRevisionNumber>1</DebianRevisionNumber>
|
||||
|
||||
<DebianUploadJsonContent>
|
||||
{
|
||||
"name":"$(SdkDebianPackageName)",
|
||||
"version":"$(NugetVersion)-$(DebianRevisionNumber)",
|
||||
"repositoryId":"$(REPO_ID)",
|
||||
"sourceUrl": "$(SdkDebianUploadUrl)"
|
||||
}
|
||||
</DebianUploadJsonContent>
|
||||
</PropertyGroup>
|
||||
|
||||
<Delete Files="$(DebianUploadJsonFile)" />
|
||||
<WriteLinesToFile File="$(DebianUploadJsonFile)" Lines="$(DebianUploadJsonContent)" />
|
||||
|
||||
<Exec Command="$(RepoRoot)/scripts/publish/repoapi_client.sh -addpkg $(DebianUploadJsonFile)" />
|
||||
</Target>
|
||||
|
||||
<Target Name="PublishCliVersionBadge">
|
||||
<ItemGroup>
|
||||
<CliVersionBadgeToUpload Include="$(VersionBadge)" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<CliVersionBadgeToUpload>
|
||||
<RelativeBlobPath>$(Product)/$(NugetVersion)/$([System.String]::Copy('%(Filename)%(Extension)').Replace('\' ,'/'))</RelativeBlobPath>
|
||||
</CliVersionBadgeToUpload>
|
||||
</ItemGroup>
|
||||
|
||||
<UploadToAzure
|
||||
AccountKey="$(CloudDropAccessToken)"
|
||||
AccountName="$(CloudDropAccountName)"
|
||||
ContainerName="$(ContainerName)"
|
||||
Items="@(CliVersionBadgeToUpload)" />
|
||||
|
||||
<SetBlobPropertiesBasedOnFileType
|
||||
AccountKey="$(CloudDropAccessToken)"
|
||||
AccountName="$(CloudDropAccountName)"
|
||||
Items="@(CliVersionBadgeToUpload)" />
|
||||
</Target>
|
||||
</Project>
|
||||
|
|
@ -35,6 +35,11 @@
|
|||
<EndToEndTestDirectory>$(RepoRoot)/test/EndToEnd</EndToEndTestDirectory>
|
||||
</PropertyGroup>
|
||||
|
||||
<!-- Consumed By Publish -->
|
||||
<ItemGroup>
|
||||
<GeneratedInstallers Include="$(SdkInstallerFile)" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<SdkDebInputFiles Include="$(SdkLayoutOutputDirectory)/**/*" />
|
||||
</ItemGroup>
|
||||
|
|
|
|||
|
|
@ -63,7 +63,7 @@
|
|||
NoBuild="True"
|
||||
Output="$(NupkgOutputDirectory)"
|
||||
ProjectPath="%(ProjectsToPack.Identity)"
|
||||
ToolPath="$(DotnetStage2)"
|
||||
ToolPath="$(Stage2Directory)"
|
||||
VersionSuffix="$(NupkgVersionSuffix)"
|
||||
Configuration="$(Configuration)" />
|
||||
</Target>
|
||||
|
|
|
|||
163
build_projects/dotnet-cli-build/FinalizeBuildTask.cs
Normal file
163
build_projects/dotnet-cli-build/FinalizeBuildTask.cs
Normal file
|
|
@ -0,0 +1,163 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Text.RegularExpressions;
|
||||
using Microsoft.Build.Framework;
|
||||
using Microsoft.Build.Utilities;
|
||||
|
||||
namespace Microsoft.DotNet.Cli.Build
|
||||
{
|
||||
public class FinalizeBuild : Task
|
||||
{
|
||||
private AzurePublisher _azurePublisher;
|
||||
|
||||
[Required]
|
||||
public string AccountName { get; set; }
|
||||
|
||||
[Required]
|
||||
public string AccountKey { get; set; }
|
||||
|
||||
[Required]
|
||||
public string NugetVersion { get; set; }
|
||||
|
||||
[Required]
|
||||
public string Channel { get; set; }
|
||||
|
||||
[Required]
|
||||
public string CommitHash { get; set; }
|
||||
|
||||
[Required]
|
||||
public string BranchName { get; set; }
|
||||
|
||||
private AzurePublisher AzurePublisherTool
|
||||
{
|
||||
get
|
||||
{
|
||||
if(_azurePublisher == null)
|
||||
{
|
||||
_azurePublisher = new AzurePublisher(AccountName, AccountKey);
|
||||
}
|
||||
|
||||
return _azurePublisher;
|
||||
}
|
||||
}
|
||||
|
||||
public override bool Execute()
|
||||
{
|
||||
DoFinalizeBuild();
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private void DoFinalizeBuild()
|
||||
{
|
||||
if (CheckIfAllBuildsHavePublished())
|
||||
{
|
||||
string targetContainer = $"{AzurePublisher.Product.Sdk}/{Channel}";
|
||||
string targetVersionFile = $"{targetContainer}/{CommitHash}";
|
||||
string semaphoreBlob = $"{targetContainer}/publishSemaphore";
|
||||
AzurePublisherTool.CreateBlobIfNotExists(semaphoreBlob);
|
||||
string leaseId = AzurePublisherTool.AcquireLeaseOnBlob(semaphoreBlob);
|
||||
|
||||
// Prevent race conditions by dropping a version hint of what version this is. If we see this file
|
||||
// and it is the same as our version then we know that a race happened where two+ builds finished
|
||||
// at the same time and someone already took care of publishing and we have no work to do.
|
||||
if (AzurePublisherTool.IsLatestSpecifiedVersion(targetVersionFile))
|
||||
{
|
||||
AzurePublisherTool.ReleaseLeaseOnBlob(semaphoreBlob, leaseId);
|
||||
return;
|
||||
}
|
||||
else
|
||||
{
|
||||
Regex versionFileRegex = new Regex(@"(?<CommitHash>[\w\d]{40})");
|
||||
|
||||
// Delete old version files
|
||||
AzurePublisherTool.ListBlobs(targetContainer)
|
||||
.Where(s => versionFileRegex.IsMatch(s))
|
||||
.ToList()
|
||||
.ForEach(f => AzurePublisherTool.TryDeleteBlob(f));
|
||||
|
||||
// Drop the version file signaling such for any race-condition builds (see above comment).
|
||||
AzurePublisherTool.DropLatestSpecifiedVersion(targetVersionFile);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
CopyBlobsToLatest(targetContainer);
|
||||
|
||||
string cliVersion = Utils.GetVersionFileContent(CommitHash, NugetVersion);
|
||||
AzurePublisherTool.PublishStringToBlob($"{targetContainer}/latest.version", cliVersion);
|
||||
|
||||
UpdateVersionsRepo();
|
||||
}
|
||||
finally
|
||||
{
|
||||
AzurePublisherTool.ReleaseLeaseOnBlob(semaphoreBlob, leaseId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private bool CheckIfAllBuildsHavePublished()
|
||||
{
|
||||
var badges = new Dictionary<string, bool>()
|
||||
{
|
||||
{ "Windows_x86", false },
|
||||
{ "Windows_x64", false },
|
||||
{ "Ubuntu_x64", false },
|
||||
{ "Ubuntu_16_04_x64", false },
|
||||
{ "RHEL_x64", false },
|
||||
{ "OSX_x64", false },
|
||||
{ "Debian_x64", false },
|
||||
{ "CentOS_x64", false },
|
||||
{ "Fedora_23_x64", false },
|
||||
{ "openSUSE_13_2_x64", false }
|
||||
};
|
||||
|
||||
var versionBadgeName = $"{Monikers.GetBadgeMoniker()}";
|
||||
if (!badges.ContainsKey(versionBadgeName))
|
||||
{
|
||||
throw new ArgumentException($"A new OS build '{versionBadgeName}' was added without adding the moniker to the {nameof(badges)} lookup");
|
||||
}
|
||||
|
||||
IEnumerable<string> blobs = AzurePublisherTool.ListBlobs(AzurePublisher.Product.Sdk, NugetVersion);
|
||||
foreach (string file in blobs)
|
||||
{
|
||||
string name = Path.GetFileName(file);
|
||||
foreach (string img in badges.Keys)
|
||||
{
|
||||
if ((name.StartsWith($"{img}")) && (name.EndsWith(".svg")))
|
||||
{
|
||||
badges[img] = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return badges.Values.All(v => v);
|
||||
}
|
||||
|
||||
private void CopyBlobsToLatest(string destinationFolder)
|
||||
{
|
||||
foreach (string blob in AzurePublisherTool.ListBlobs(AzurePublisher.Product.Sdk, NugetVersion))
|
||||
{
|
||||
string targetName = Path.GetFileName(blob)
|
||||
.Replace(NugetVersion, "latest");
|
||||
|
||||
string target = $"{destinationFolder}/{targetName}";
|
||||
AzurePublisherTool.CopyBlob(blob, target);
|
||||
}
|
||||
}
|
||||
|
||||
private void UpdateVersionsRepo()
|
||||
{
|
||||
string githubAuthToken = EnvVars.EnsureVariable("GITHUB_PASSWORD");
|
||||
string nupkgFilePath = Dirs.Packages;
|
||||
string branchName = BranchName;
|
||||
string versionsRepoPath = $"build-info/dotnet/cli/{branchName}/Latest";
|
||||
|
||||
VersionRepoUpdater repoUpdater = new VersionRepoUpdater(githubAuthToken);
|
||||
repoUpdater.UpdatePublishedVersions(nupkgFilePath, versionsRepoPath).Wait();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,61 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Text.RegularExpressions;
|
||||
using Microsoft.Build.Framework;
|
||||
using Microsoft.Build.Utilities;
|
||||
|
||||
namespace Microsoft.DotNet.Cli.Build
|
||||
{
|
||||
public class SetBlobPropertiesBasedOnFileType : Task
|
||||
{
|
||||
private AzurePublisher _azurePublisher;
|
||||
|
||||
[Required]
|
||||
public string AccountName { get; set; }
|
||||
|
||||
[Required]
|
||||
public string AccountKey { get; set; }
|
||||
|
||||
[Required]
|
||||
public ITaskItem[] Items { get; set; }
|
||||
|
||||
private AzurePublisher AzurePublisherTool
|
||||
{
|
||||
get
|
||||
{
|
||||
if(_azurePublisher == null)
|
||||
{
|
||||
_azurePublisher = new AzurePublisher(AccountName, AccountKey);
|
||||
}
|
||||
|
||||
return _azurePublisher;
|
||||
}
|
||||
}
|
||||
|
||||
public override bool Execute()
|
||||
{
|
||||
if (Items.Length == 0)
|
||||
{
|
||||
Log.LogError("No items were provided for upload.");
|
||||
return false;
|
||||
}
|
||||
|
||||
foreach(var item in Items)
|
||||
{
|
||||
string relativeBlobPath = item.GetMetadata("RelativeBlobPath");
|
||||
if (string.IsNullOrEmpty(relativeBlobPath))
|
||||
{
|
||||
throw new Exception(string.Format(
|
||||
"Metadata 'RelativeBlobPath' is missing for item '{0}'.",
|
||||
item.ItemSpec));
|
||||
}
|
||||
|
||||
AzurePublisherTool.SetBlobPropertiesBasedOnFileType(relativeBlobPath);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -7,6 +7,7 @@ using System.Net.Http;
|
|||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.WindowsAzure.Storage;
|
||||
using Microsoft.WindowsAzure.Storage.Auth;
|
||||
using Microsoft.WindowsAzure.Storage.Blob;
|
||||
|
||||
namespace Microsoft.DotNet.Cli.Build
|
||||
|
|
@ -33,9 +34,27 @@ namespace Microsoft.DotNet.Cli.Build
|
|||
_blobContainer = GetDotnetBlobContainer(_connectionString);
|
||||
}
|
||||
|
||||
public AzurePublisher(string accountName, string accountKey)
|
||||
{
|
||||
_blobContainer = GetDotnetBlobContainer(accountName, accountKey);
|
||||
}
|
||||
|
||||
private CloudBlobContainer GetDotnetBlobContainer(string connectionString)
|
||||
{
|
||||
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(connectionString);
|
||||
|
||||
return GetDotnetBlobContainer(storageAccount);
|
||||
}
|
||||
|
||||
private CloudBlobContainer GetDotnetBlobContainer(string accountName, string accountKey)
|
||||
{
|
||||
var storageCredentials = new StorageCredentials(accountName, accountKey);
|
||||
var storageAccount = new CloudStorageAccount(storageCredentials, true);
|
||||
return GetDotnetBlobContainer(storageAccount);
|
||||
}
|
||||
|
||||
private CloudBlobContainer GetDotnetBlobContainer(CloudStorageAccount storageAccount)
|
||||
{
|
||||
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
|
||||
|
||||
return blobClient.GetContainerReference(s_dotnetBlobContainerName);
|
||||
|
|
@ -73,6 +92,12 @@ namespace Microsoft.DotNet.Cli.Build
|
|||
target.StartCopyAsync(source).Wait();
|
||||
}
|
||||
|
||||
public void SetBlobPropertiesBasedOnFileType(string path)
|
||||
{
|
||||
CloudBlockBlob blob = _blobContainer.GetBlockBlobReference(path);
|
||||
SetBlobPropertiesBasedOnFileType(blob);
|
||||
}
|
||||
|
||||
private void SetBlobPropertiesBasedOnFileType(CloudBlockBlob blockBlob)
|
||||
{
|
||||
if (Path.GetExtension(blockBlob.Uri.AbsolutePath.ToLower()) == ".svg")
|
||||
|
|
|
|||
Loading…
Reference in a new issue