diff --git a/build.proj b/build.proj index bce9d2b0e..debe1b415 100644 --- a/build.proj +++ b/build.proj @@ -18,6 +18,7 @@ <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration> <RepoRoot>$(MSBuildThisFileDirectory)</RepoRoot> + <ToolsDir>$(RepoRoot)/build_tools</ToolsDir> <CoreSetupChannel>preview</CoreSetupChannel> <SharedFrameworkName>Microsoft.NETCore.App</SharedFrameworkName> @@ -46,32 +47,36 @@ <Message Text="Dont remove this target" /> </Target> - <Target Name="BuildDotnetCliBuildFramework" - Inputs="@(DotnetCliBuildFrameworkInputs)" + <Target Name="BuildDotnetCliBuildFramework" + Inputs="@(DotnetCliBuildFrameworkInputs)" Outputs="$(CLIBuildDll)" DependsOnTargets="MSBuildWorkaroundTarget"> <PropertyGroup> <DotnetCliBuildDirectory>$(MSBuildThisFileDirectory)/build_projects/dotnet-cli-build</DotnetCliBuildDirectory> </PropertyGroup> + <ItemGroup> + <DotnetSdkDirectories Include="$([System.IO.Directory]::GetDirectories("$(Stage0Path)/sdk"))" /> + <AzureStorageToCopy Include="@(DotnetSdkDirectories)"> + <Source>$(DotnetCliBuildDirectory)/bin/Microsoft.WindowsAzure.Storage.dll</Source> + <Destination>%(Identity)/Microsoft.WindowsAzure.Storage.dll</Destination> + </AzureStorageToCopy> + </ItemGroup> + <Exec Command="$(DotnetStage0) restore" WorkingDirectory="$(DotnetCliBuildDirectory)"/> - <Exec Command="$(DotnetStage0) publish -o $(DotnetCliBuildDirectory)/bin --framework netcoreapp1.0" WorkingDirectory="$(DotnetCliBuildDirectory)"/> + <Exec Command="$(DotnetStage0) publish -o $(DotnetCliBuildDirectory)/bin --framework netcoreapp1.0" WorkingDirectory="$(DotnetCliBuildDirectory)"/> + + <!-- This is a work around to issue https://github.com/Microsoft/msbuild/issues/658 --> + <Copy SourceFiles="%(AzureStorageToCopy.Source)" + DestinationFiles="%(AzureStorageToCopy.Destination)" /> </Target> <Target DependsOnTargets="$(CLITargets)" Name="BuildTheWholeCli"></Target> - <UsingTask TaskName="PublishTargets" AssemblyFile="$(CLIBuildDll)" /> - - <Target DependsOnTargets="BuildDotnetCliBuildFramework;GenerateVersionBadge" Name="Publish"> - <PublishTargets /> - </Target> - <Import Project="build/Microsoft.DotNet.Cli.Prepare.targets" /> <Import Project="build/Microsoft.DotNet.Cli.Compile.targets" /> <Import Project="build/Microsoft.DotNet.Cli.Package.targets" /> <Import Project="build/Microsoft.DotNet.Cli.Test.targets" /> - - <!-- This should be in the build/Microsoft.DotNet.Cli.Publish.targets --> - <Import Project="build/publish/Microsoft.DotNet.Cli.Badge.targets" /> + <Import Project="build/Microsoft.DotNet.Cli.Publish.targets" /> </Project> diff --git a/build/Microsoft.DotNet.Cli.Publish.targets b/build/Microsoft.DotNet.Cli.Publish.targets new file mode 100644 index 000000000..7a43b3008 --- /dev/null +++ b/build/Microsoft.DotNet.Cli.Publish.targets @@ -0,0 +1,101 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + <Import Project="$(ToolsDir)/PublishContent.targets" /> + <Import Project="$(MSBuildThisFileDirectory)/publish/Microsoft.DotNet.Cli.Badge.targets" /> + + <UsingTask TaskName="UploadToAzure" AssemblyFile="$(BuildToolsTaskDir)/Microsoft.DotNet.Build.CloudTestTasks.dll"/> + <UsingTask TaskName="FinalizeBuildTask" AssemblyFile="$(CLIBuildDll)" /> + + <!-- PUBLISH_TO_AZURE_BLOB env variable set by CI --> + <Target Name="Publish" + Condition=" '$(PUBLISH_TO_AZURE_BLOB)' != '' " + DependsOnTargets="Prepare; + Package; + PublishArtifacts; + FinalizeBuild" /> + + <Target Name="PublishArtifacts" DependsOnTargets="SetupAzureBlobInformation; + GenerateVersionBadge; + GatherItemsForPattern; + UploadToAzure; + PublishDebFilesToDebianRepo; + PublishCliVersionBadge" /> + + <Target Name="FinalizeBuild"> + <FinalizeBuildTask AccountName="$(CloudDropAccountName)" + AccountKey="$(CloudDropAccessToken)" + NugetVersion="$(NugetVersion)" + Channel="$(Channel)" + CommitHash="$(CommitHash)" + BranchName="$(BranchName)" /> + </Target> + + <Target Name="SetupAzureBlobInformation"> + <PropertyGroup> + <Product>Sdk</Product> + <ContainerName>$(STORAGE_CONTAINER)</ContainerName> + <CloudDropAccessToken>$(STORAGE_KEY)</CloudDropAccessToken> + <CloudDropAccountName>$(STORAGE_ACCOUNT)</CloudDropAccountName> + <DotnetBlobRootUrl>https://$(CloudDropAccountName).blob.core.windows.net/$(ContainerName)</DotnetBlobRootUrl> + </PropertyGroup> + </Target> + + <Target Name="GatherItemsForPattern"> + <ItemGroup> + <ForPublishing Include="@(GeneratedInstallers)" /> + <ForPublishing Include="%(GenerateArchivesInputsOutputs.Outputs)" /> + </ItemGroup> + + <ItemGroup> + <ForPublishing> + <RelativeBlobPath>$(Product)/$(NugetVersion)/$([System.String]::Copy('%(Filename)%(Extension)').Replace('\' ,'/'))</RelativeBlobPath> + </ForPublishing> + </ItemGroup> + </Target> + + <Target Name="PublishDebFilesToDebianRepo" Condition=" '$(OSName)' == 'ubuntu' "> + <Error Condition="'$(REPO_ID)' == ''" Text="REPO_ID must be set as an environment variable for debian publishing." /> + + <ItemGroup> + <SdkInstallerFileItemGroup Include="$(SdkInstallerFile)" /> + </ItemGroup> + <PropertyGroup> + <SdkDebianPackageName>dotnet-dev-$(NugetVersion)</SdkDebianPackageName> + <SdkDebianUploadUrl>$(DotnetBlobRootUrl)/$(Product)/$(NugetVersion)/%(SdkInstallerFileItemGroup.Filename)%(SdkInstallerFileItemGroup.Extension)</SdkDebianUploadUrl> + <DebianUploadJsonFile>$(PackageDirectory)/package_upload.json</DebianUploadJsonFile> + <DebianRevisionNumber>1</DebianRevisionNumber> + + <DebianUploadJsonContent> + { + "name":"$(SdkDebianPackageName)", + "version":"$(NugetVersion)-$(DebianRevisionNumber)", + "repositoryId":"$(REPO_ID)", + "sourceUrl": "$(SdkDebianUploadUrl)" + } + </DebianUploadJsonContent> + </PropertyGroup> + + <Delete Files="$(DebianUploadJsonFile)" /> + <WriteLinesToFile File="$(DebianUploadJsonFile)" Lines="$(DebianUploadJsonContent)" /> + + <Exec Command="$(RepoRoot)/scripts/publish/repoapi_client.sh -addpkg $(DebianUploadJsonFile)" /> + </Target> + + <Target Name="PublishCliVersionBadge"> + <ItemGroup> + <CliVersionBadgeToUpload Include="$(VersionBadge)" /> + </ItemGroup> + + <ItemGroup> + <CliVersionBadgeToUpload> + <RelativeBlobPath>$(Product)/$(NugetVersion)/$([System.String]::Copy('%(Filename)%(Extension)').Replace('\' ,'/'))</RelativeBlobPath> + </CliVersionBadgeToUpload> + </ItemGroup> + + <UploadToAzure + AccountKey="$(CloudDropAccessToken)" + AccountName="$(CloudDropAccountName)" + ContainerName="$(ContainerName)" + Items="@(CliVersionBadgeToUpload)" /> + </Target> +</Project> \ No newline at end of file diff --git a/build/package/Microsoft.DotNet.Cli.Installer.DEB.targets b/build/package/Microsoft.DotNet.Cli.Installer.DEB.targets index 24c4dbe4e..ee4de25ad 100644 --- a/build/package/Microsoft.DotNet.Cli.Installer.DEB.targets +++ b/build/package/Microsoft.DotNet.Cli.Installer.DEB.targets @@ -35,6 +35,11 @@ <EndToEndTestDirectory>$(RepoRoot)/test/EndToEnd</EndToEndTestDirectory> </PropertyGroup> + <!-- Consumed By Publish --> + <ItemGroup> + <GeneratedInstallers Include="$(SdkInstallerFile)" /> + </ItemGroup> + <ItemGroup> <SdkDebInputFiles Include="$(SdkLayoutOutputDirectory)/**/*" /> </ItemGroup> diff --git a/build/package/Microsoft.DotNet.Cli.Nupkg.targets b/build/package/Microsoft.DotNet.Cli.Nupkg.targets index f33c2f9ab..1e97bae08 100644 --- a/build/package/Microsoft.DotNet.Cli.Nupkg.targets +++ b/build/package/Microsoft.DotNet.Cli.Nupkg.targets @@ -63,7 +63,7 @@ NoBuild="True" Output="$(NupkgOutputDirectory)" ProjectPath="%(ProjectsToPack.Identity)" - ToolPath="$(DotnetStage2)" + ToolPath="$(Stage2Directory)" VersionSuffix="$(NupkgVersionSuffix)" Configuration="$(Configuration)" /> </Target> diff --git a/build_projects/dotnet-cli-build/FinalizeBuildTask.cs b/build_projects/dotnet-cli-build/FinalizeBuildTask.cs new file mode 100644 index 000000000..2e8d2901a --- /dev/null +++ b/build_projects/dotnet-cli-build/FinalizeBuildTask.cs @@ -0,0 +1,162 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.RegularExpressions; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +namespace Microsoft.DotNet.Cli.Build +{ + public class FinalizeBuildTask : Task + { + private AzurePublisher _azurePublisher; + + [Required] + public string AccountName { get; set; } + + [Required] + public string AccountKey { get; set; } + + [Required] + public string NugetVersion { get; set; } + + [Required] + public string Channel { get; set; } + + [Required] + public string CommitHash { get; set; } + + [Required] + public string BranchName { get; set; } + + private AzurePublisher AzurePublisherTool { + get + { + if(_azurePublisher == null) + { + _azurePublisher = new AzurePublisher(AccountName, AccountKey); + } + + return _azurePublisher; + } + } + + public override bool Execute() + { + FinalizeBuild(); + + return true; + } + + private void FinalizeBuild() + { + if (CheckIfAllBuildsHavePublished()) + { + string targetContainer = $"{AzurePublisher.Product.Sdk}/{Channel}"; + string targetVersionFile = $"{targetContainer}/{CommitHash}"; + string semaphoreBlob = $"{targetContainer}/publishSemaphore"; + AzurePublisherTool.CreateBlobIfNotExists(semaphoreBlob); + string leaseId = AzurePublisherTool.AcquireLeaseOnBlob(semaphoreBlob); + + // Prevent race conditions by dropping a version hint of what version this is. If we see this file + // and it is the same as our version then we know that a race happened where two+ builds finished + // at the same time and someone already took care of publishing and we have no work to do. + if (AzurePublisherTool.IsLatestSpecifiedVersion(targetVersionFile)) + { + AzurePublisherTool.ReleaseLeaseOnBlob(semaphoreBlob, leaseId); + return; + } + else + { + Regex versionFileRegex = new Regex(@"(?<CommitHash>[\w\d]{40})"); + + // Delete old version files + AzurePublisherTool.ListBlobs(targetContainer) + .Where(s => versionFileRegex.IsMatch(s)) + .ToList() + .ForEach(f => AzurePublisherTool.TryDeleteBlob(f)); + + // Drop the version file signaling such for any race-condition builds (see above comment). + AzurePublisherTool.DropLatestSpecifiedVersion(targetVersionFile); + } + + try + { + CopyBlobsToLatest(targetContainer); + + string cliVersion = Utils.GetVersionFileContent(CommitHash, NugetVersion); + AzurePublisherTool.PublishStringToBlob($"{targetContainer}/latest.version", cliVersion); + + UpdateVersionsRepo(); + } + finally + { + AzurePublisherTool.ReleaseLeaseOnBlob(semaphoreBlob, leaseId); + } + } + } + + private bool CheckIfAllBuildsHavePublished() + { + var badges = new Dictionary<string, bool>() + { + { "Windows_x86", false }, + { "Windows_x64", false }, + { "Ubuntu_x64", false }, + { "Ubuntu_16_04_x64", false }, + { "RHEL_x64", false }, + { "OSX_x64", false }, + { "Debian_x64", false }, + { "CentOS_x64", false }, + { "Fedora_23_x64", false }, + { "openSUSE_13_2_x64", false } + }; + + var versionBadgeName = $"{Monikers.GetBadgeMoniker()}"; + if (!badges.ContainsKey(versionBadgeName)) + { + throw new ArgumentException($"A new OS build '{versionBadgeName}' was added without adding the moniker to the {nameof(badges)} lookup"); + } + + IEnumerable<string> blobs = AzurePublisherTool.ListBlobs(AzurePublisher.Product.Sdk, NugetVersion); + foreach (string file in blobs) + { + string name = Path.GetFileName(file); + foreach (string img in badges.Keys) + { + if ((name.StartsWith($"{img}")) && (name.EndsWith(".svg"))) + { + badges[img] = true; + break; + } + } + } + + return badges.Values.All(v => v); + } + + private void CopyBlobsToLatest(string destinationFolder) + { + foreach (string blob in AzurePublisherTool.ListBlobs(AzurePublisher.Product.Sdk, NugetVersion)) + { + string targetName = Path.GetFileName(blob) + .Replace(NugetVersion, "latest"); + + string target = $"{destinationFolder}/{targetName}"; + AzurePublisherTool.CopyBlob(blob, target); + } + } + + private void UpdateVersionsRepo() + { + string githubAuthToken = EnvVars.EnsureVariable("GITHUB_PASSWORD"); + string nupkgFilePath = Dirs.Packages; + string branchName = BranchName; + string versionsRepoPath = $"build-info/dotnet/cli/{branchName}/Latest"; + + VersionRepoUpdater repoUpdater = new VersionRepoUpdater(githubAuthToken); + repoUpdater.UpdatePublishedVersions(nupkgFilePath, versionsRepoPath).Wait(); + } + } +} \ No newline at end of file diff --git a/build_projects/shared-build-targets-utils/Publishing/AzurePublisher.cs b/build_projects/shared-build-targets-utils/Publishing/AzurePublisher.cs index 35d0afde7..28e1b5bc8 100644 --- a/build_projects/shared-build-targets-utils/Publishing/AzurePublisher.cs +++ b/build_projects/shared-build-targets-utils/Publishing/AzurePublisher.cs @@ -7,6 +7,7 @@ using System.Net.Http; using System.Threading; using System.Threading.Tasks; using Microsoft.WindowsAzure.Storage; +using Microsoft.WindowsAzure.Storage.Auth; using Microsoft.WindowsAzure.Storage.Blob; namespace Microsoft.DotNet.Cli.Build @@ -33,9 +34,27 @@ namespace Microsoft.DotNet.Cli.Build _blobContainer = GetDotnetBlobContainer(_connectionString); } + public AzurePublisher(string accountName, string accountKey) + { + _blobContainer = GetDotnetBlobContainer(accountName, accountKey); + } + private CloudBlobContainer GetDotnetBlobContainer(string connectionString) { CloudStorageAccount storageAccount = CloudStorageAccount.Parse(connectionString); + + return GetDotnetBlobContainer(storageAccount); + } + + private CloudBlobContainer GetDotnetBlobContainer(string accountName, string accountKey) + { + var storageCredentials = new StorageCredentials(accountName, accountKey); + var storageAccount = new CloudStorageAccount(storageCredentials, true); + return GetDotnetBlobContainer(storageAccount); + } + + private CloudBlobContainer GetDotnetBlobContainer(CloudStorageAccount storageAccount) + { CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); return blobClient.GetContainerReference(s_dotnetBlobContainerName);