From c4d7251efbf87c35b4a1529cbdca8d3af030fc1b Mon Sep 17 00:00:00 2001 From: Jonathan Miller Date: Mon, 25 Apr 2016 13:17:46 -0700 Subject: [PATCH] Fixing publish scripts to only push to Latest when all bits are there --- scripts/dotnet-cli-build/PublishTargets.cs | 162 +++++++++++++----- .../Publishing/AzurePublisher.cs | 83 +++++++-- scripts/dotnet-cli-build/Utils/Utils.cs | 15 ++ 3 files changed, 201 insertions(+), 59 deletions(-) diff --git a/scripts/dotnet-cli-build/PublishTargets.cs b/scripts/dotnet-cli-build/PublishTargets.cs index f7eccbdeb..189f0e2d1 100644 --- a/scripts/dotnet-cli-build/PublishTargets.cs +++ b/scripts/dotnet-cli-build/PublishTargets.cs @@ -1,5 +1,7 @@ using System; +using System.Collections.Generic; using System.IO; +using System.Linq; using System.Net.Http; using System.Text; using Microsoft.DotNet.Cli.Build.Framework; @@ -24,6 +26,8 @@ namespace Microsoft.DotNet.Cli.Build private static string SharedFrameworkNugetVersion { get; set; } + private static List _latestBlobs; + [Target] public static BuildTargetResult InitPublish(BuildTargetContext c) { @@ -41,19 +45,119 @@ namespace Microsoft.DotNet.Cli.Build [Target(nameof(PrepareTargets.Init), nameof(PublishTargets.InitPublish), nameof(PublishTargets.PublishArtifacts), - nameof(PublishTargets.TriggerDockerHubBuilds))] + nameof(PublishTargets.TriggerDockerHubBuilds), + nameof(PublishTargets.FinalizeBuild))] [Environment("PUBLISH_TO_AZURE_BLOB", "1", "true")] // This is set by CI systems public static BuildTargetResult Publish(BuildTargetContext c) { return c.Success(); } + [Target] + public static BuildTargetResult FinalizeBuild(BuildTargetContext c) + { + if (CheckIfAllBuildsHavePublished()) + { + string targetContainer = $"{Channel}/Binaries/Latest/"; + string targetVersionFile = $"{targetContainer}{CliNuGetVersion}"; + string leaseId = AzurePublisherTool.AcquireLeaseOnBlob(targetContainer); + + // Prevent race conditions by dropping a version hint of what version this is. If we see this file + // and it is the same as our version then we know that a race happened where two+ builds finished + // at the same time and someone already took care of publishing and we have no work to do. + if (AzurePublisherTool.IsLatestSpecifiedVersion(targetVersionFile)) + { + AzurePublisherTool.ReleaseLeaseOnBlob(targetContainer, leaseId); + return c.Success(); + } + else + { + // This is an old drop of latest so remove all old files to ensure a clean state + AzurePublisherTool.ListBlobs($"{targetContainer}").ToList().ForEach(f => AzurePublisherTool.DeleteBlob(f)); + + // Drop the version file signaling such for any race-condition builds (see above comment). + AzurePublisherTool.DropLatestSpecifiedVersion(targetVersionFile); + } + + try + { + // Iterate over every blob in the latest version folder and copy it to the 'latest' folder + foreach (string blob in _latestBlobs) + { + string targetUrl = $"{targetContainer}{Path.GetFileName(blob)}" + .Replace(CliNuGetVersion, "latest") + .Replace(SharedFrameworkNugetVersion, "latest"); + AzurePublisherTool.CopyBlob(blob.Replace("/dotnet/", ""), targetUrl); + } + + // Generate the CLI and SDK Version text files + List versionFiles = new List() { "win.x86.version", "win.x64.version", "ubuntu.x64.version", "rhel.x64.version", "osx.x64.version", "debian.x64.version", "centos.x64.version" }; + string cliVersion = Utils.GetCliVersionFileContent(c); + string sfxVersion = Utils.GetSharedFrameworkVersionFileContent(c); + foreach (string version in versionFiles) + { + AzurePublisherTool.PublishStringToBlob($"{Channel}/dnvm/latest.{version}", cliVersion); + AzurePublisherTool.PublishStringToBlob($"{Channel}/dnvm/latest.sharedfx.{version}", sfxVersion); + } + } + finally + { + AzurePublisherTool.ReleaseLeaseOnBlob(targetContainer, leaseId); + } + } + + return c.Success(); + } + + private static bool CheckIfAllBuildsHavePublished() + { + Dictionary badges = new Dictionary() + { + { "Windows_x86", false }, + { "Windows_x64", false }, + { "Ubuntu_x64", false }, + { "RHEL_x64", false }, + { "OSX_x64", false }, + { "Debian_x64", false }, + { "CentOS_x64", false } + }; + + _latestBlobs = new List(AzurePublisherTool.ListBlobs($"{Channel}/Binaries/{CliNuGetVersion}/")); + + var config = Environment.GetEnvironmentVariable("CONFIGURATION"); + var versionBadgeName = $"{CurrentPlatform.Current}_{CurrentArchitecture.Current}"; + if (badges.ContainsKey(versionBadgeName) == false) + { + throw new ArgumentException("A new OS build was added without adding the moniker to the {nameof(badges)} lookup"); + } + + foreach (string file in _latestBlobs) + { + string name = Path.GetFileName(file); + string key = string.Empty; + + foreach (string img in badges.Keys) + { + if ((name.StartsWith($"{img}")) && (name.EndsWith(".svg"))) + { + key = img; + break; + } + } + + if (string.IsNullOrEmpty(key) == false) + { + badges[key] = true; + } + } + + return badges.Keys.All(key => badges[key]); + } + [Target( nameof(PublishTargets.PublishInstallerFilesToAzure), nameof(PublishTargets.PublishArchivesToAzure), nameof(PublishTargets.PublishDebFilesToDebianRepo), - nameof(PublishTargets.PublishLatestCliVersionTextFile), - nameof(PublishTargets.PublishLatestSharedFrameworkVersionTextFile), nameof(PublishTargets.PublishCoreHostPackages), nameof(PublishTargets.PublishCliVersionBadge))] public static BuildTargetResult PublishArtifacts(BuildTargetContext c) => c.Success(); @@ -87,11 +191,8 @@ namespace Microsoft.DotNet.Cli.Build public static BuildTargetResult PublishCliVersionBadge(BuildTargetContext c) { var versionBadge = c.BuildContext.Get("VersionBadge"); - var latestVersionBadgeBlob = $"{Channel}/Binaries/Latest/{Path.GetFileName(versionBadge)}"; var versionBadgeBlob = $"{Channel}/Binaries/{CliNuGetVersion}/{Path.GetFileName(versionBadge)}"; - AzurePublisherTool.PublishFile(versionBadgeBlob, versionBadge); - AzurePublisherTool.PublishFile(latestVersionBadgeBlob, versionBadge); return c.Success(); } @@ -120,7 +221,7 @@ namespace Microsoft.DotNet.Cli.Build installerFile = Path.ChangeExtension(installerFile, "msi"); } - AzurePublisherTool.PublishInstallerFileAndLatest(installerFile, Channel, version); + AzurePublisherTool.PublishInstallerFile(installerFile, Channel, version); return c.Success(); } @@ -132,7 +233,7 @@ namespace Microsoft.DotNet.Cli.Build var version = SharedFrameworkNugetVersion; var installerFile = c.BuildContext.Get("SharedFrameworkInstallerFile"); - AzurePublisherTool.PublishInstallerFileAndLatest(installerFile, Channel, version); + AzurePublisherTool.PublishInstallerFile(installerFile, Channel, version); return c.Success(); } @@ -144,7 +245,7 @@ namespace Microsoft.DotNet.Cli.Build var version = CliNuGetVersion; var installerFile = c.BuildContext.Get("SdkInstallerFile"); - AzurePublisherTool.PublishInstallerFileAndLatest(installerFile, Channel, version); + AzurePublisherTool.PublishInstallerFile(installerFile, Channel, version); return c.Success(); } @@ -156,7 +257,7 @@ namespace Microsoft.DotNet.Cli.Build var version = SharedFrameworkNugetVersion; var installerFile = c.BuildContext.Get("CombinedFrameworkHostInstallerFile"); - AzurePublisherTool.PublishInstallerFileAndLatest(installerFile, Channel, version); + AzurePublisherTool.PublishInstallerFile(installerFile, Channel, version); return c.Success(); } @@ -168,7 +269,7 @@ namespace Microsoft.DotNet.Cli.Build var version = CliNuGetVersion; var installerFile = c.BuildContext.Get("CombinedFrameworkSDKHostInstallerFile"); - AzurePublisherTool.PublishInstallerFileAndLatest(installerFile, Channel, version); + AzurePublisherTool.PublishInstallerFile(installerFile, Channel, version); return c.Success(); } @@ -179,7 +280,7 @@ namespace Microsoft.DotNet.Cli.Build var version = CliNuGetVersion; var archiveFile = c.BuildContext.Get("CombinedFrameworkSDKCompressedFile"); - AzurePublisherTool.PublishArchiveAndLatest(archiveFile, Channel, version); + AzurePublisherTool.PublishArchive(archiveFile, Channel, version); return c.Success(); } @@ -190,7 +291,7 @@ namespace Microsoft.DotNet.Cli.Build var version = CliNuGetVersion; var archiveFile = c.BuildContext.Get("CombinedFrameworkSDKHostCompressedFile"); - AzurePublisherTool.PublishArchiveAndLatest(archiveFile, Channel, version); + AzurePublisherTool.PublishArchive(archiveFile, Channel, version); return c.Success(); } @@ -201,7 +302,7 @@ namespace Microsoft.DotNet.Cli.Build var version = CliNuGetVersion; var archiveFile = c.BuildContext.Get("SdkSymbolsCompressedFile"); - AzurePublisherTool.PublishArchiveAndLatest(archiveFile, Channel, version); + AzurePublisherTool.PublishArchive(archiveFile, Channel, version); return c.Success(); } @@ -212,38 +313,7 @@ namespace Microsoft.DotNet.Cli.Build var version = SharedFrameworkNugetVersion; var archiveFile = c.BuildContext.Get("CombinedFrameworkHostCompressedFile"); - AzurePublisherTool.PublishArchiveAndLatest(archiveFile, Channel, version); - return c.Success(); - } - - [Target] - public static BuildTargetResult PublishLatestCliVersionTextFile(BuildTargetContext c) - { - var version = CliNuGetVersion; - - var osname = Monikers.GetOSShortName(); - var latestCliVersionBlob = $"{Channel}/dnvm/latest.{osname}.{CurrentArchitecture.Current}.version"; - var latestCliVersionFile = Path.Combine(Dirs.Stage2, "sdk", version, ".version"); - - AzurePublisherTool.PublishFile(latestCliVersionBlob, latestCliVersionFile); - return c.Success(); - } - - [Target] - public static BuildTargetResult PublishLatestSharedFrameworkVersionTextFile(BuildTargetContext c) - { - var version = SharedFrameworkNugetVersion; - - var osname = Monikers.GetOSShortName(); - var latestSharedFXVersionBlob = $"{Channel}/dnvm/latest.sharedfx.{osname}.{CurrentArchitecture.Current}.version"; - var latestSharedFXVersionFile = Path.Combine( - Dirs.Stage2, - "shared", - CompileTargets.SharedFrameworkName, - version, - ".version"); - - AzurePublisherTool.PublishFile(latestSharedFXVersionBlob, latestSharedFXVersionFile); + AzurePublisherTool.PublishArchive(archiveFile, Channel, version); return c.Success(); } diff --git a/scripts/dotnet-cli-build/Publishing/AzurePublisher.cs b/scripts/dotnet-cli-build/Publishing/AzurePublisher.cs index 0ac9834d1..575e3b22c 100644 --- a/scripts/dotnet-cli-build/Publishing/AzurePublisher.cs +++ b/scripts/dotnet-cli-build/Publishing/AzurePublisher.cs @@ -1,5 +1,7 @@ using System; +using System.Collections.Generic; using System.IO; +using System.Linq; using System.Net.Http; using System.Text; using Microsoft.DotNet.Cli.Build.Framework; @@ -21,7 +23,6 @@ namespace Microsoft.DotNet.Cli.Build public AzurePublisher() { _connectionString = Environment.GetEnvironmentVariable("CONNECTION_STRING").Trim('"'); - _blobContainer = GetDotnetBlobContainer(_connectionString); } @@ -33,26 +34,16 @@ namespace Microsoft.DotNet.Cli.Build return blobClient.GetContainerReference(s_dotnetBlobContainerName); } - public void PublishInstallerFileAndLatest(string installerFile, string channel, string version) + public void PublishInstallerFile(string installerFile, string channel, string version) { var installerFileBlob = CalculateInstallerBlob(installerFile, channel, version); - - var latestInstallerFileName = installerFile.Replace(version, "latest"); - var latestInstallerFileBlob = CalculateInstallerBlob(latestInstallerFileName, channel, "Latest"); - PublishFile(installerFileBlob, installerFile); - PublishFile(latestInstallerFileBlob, installerFile); } - public void PublishArchiveAndLatest(string archiveFile, string channel, string version) + public void PublishArchive(string archiveFile, string channel, string version) { var archiveFileBlob = CalculateArchiveBlob(archiveFile, channel, version); - - var latestArchiveFileName = archiveFile.Replace(version, "latest"); - var latestArchiveFileBlob = CalculateArchiveBlob(latestArchiveFileName, channel, "Latest"); - PublishFile(archiveFileBlob, archiveFile); - PublishFile(latestArchiveFileBlob, archiveFile); } public void PublishFile(string blob, string file) @@ -63,6 +54,27 @@ namespace Microsoft.DotNet.Cli.Build SetBlobPropertiesBasedOnFileType(blockBlob); } + public void PublishStringToBlob(string blob, string content) + { + CloudBlockBlob blockBlob = _blobContainer.GetBlockBlobReference(blob); + blockBlob.UploadTextAsync(content).Wait(); + } + + public void CopyBlob(string sourceBlob, string targetBlob) + { + CloudBlockBlob source = _blobContainer.GetBlockBlobReference(sourceBlob); + CloudBlockBlob target = _blobContainer.GetBlockBlobReference(targetBlob); + + // Create the empty blob + using (MemoryStream ms = new MemoryStream()) + { + target.UploadFromStreamAsync(ms).Wait(); + } + + // Copy actual blob data + target.StartCopyAsync(source).Wait(); + } + private void SetBlobPropertiesBasedOnFileType(CloudBlockBlob blockBlob) { if (Path.GetExtension(blockBlob.Uri.AbsolutePath.ToLower()) == ".svg") @@ -73,6 +85,51 @@ namespace Microsoft.DotNet.Cli.Build } } + public IEnumerable ListBlobs(string virtualDirectory) + { + CloudBlobDirectory blobDir = _blobContainer.GetDirectoryReference(virtualDirectory); + BlobContinuationToken continuationToken = new BlobContinuationToken(); + + var blobFiles = blobDir.ListBlobsSegmentedAsync(continuationToken).Result; + return blobFiles.Results.Select(bf => bf.Uri.PathAndQuery); + } + + public string AcquireLeaseOnBlob(string blob) + { + CloudBlockBlob cloudBlob = _blobContainer.GetBlockBlobReference(blob); + System.Threading.Tasks.Task task = cloudBlob.AcquireLeaseAsync(TimeSpan.FromMinutes(1), null); + task.Wait(); + return task.Result; + } + + public void ReleaseLeaseOnBlob(string blob, string leaseId) + { + CloudBlockBlob cloudBlob = _blobContainer.GetBlockBlobReference(blob); + AccessCondition ac = new AccessCondition() { LeaseId = leaseId }; + cloudBlob.ReleaseLeaseAsync(ac).Wait(); + } + + public bool IsLatestSpecifiedVersion(string version) + { + System.Threading.Tasks.Task task = _blobContainer.GetBlockBlobReference(version).ExistsAsync(); + task.Wait(); + return task.Result; + } + + public void DropLatestSpecifiedVersion(string version) + { + CloudBlockBlob blob = _blobContainer.GetBlockBlobReference(version); + using (MemoryStream ms = new MemoryStream()) + { + blob.UploadFromStreamAsync(ms).Wait(); + } + } + + public void DeleteBlob(string path) + { + _blobContainer.GetBlockBlobReference(path).DeleteAsync().Wait(); + } + public string CalculateInstallerUploadUrl(string installerFile, string channel, string version) { return $"{s_dotnetBlobRootUrl}{CalculateInstallerBlob(installerFile, channel, version)}"; diff --git a/scripts/dotnet-cli-build/Utils/Utils.cs b/scripts/dotnet-cli-build/Utils/Utils.cs index 799bda56f..283c4dd7b 100644 --- a/scripts/dotnet-cli-build/Utils/Utils.cs +++ b/scripts/dotnet-cli-build/Utils/Utils.cs @@ -3,6 +3,8 @@ using System.IO; using System.Runtime.InteropServices; using System.Security.Cryptography; +using Microsoft.DotNet.Cli.Build.Framework; + namespace Microsoft.DotNet.Cli.Build { public static class Utils @@ -133,5 +135,18 @@ namespace Microsoft.DotNet.Cli.Build File.Copy(file, destFile, true); } } + + public static string GetSharedFrameworkVersionFileContent(BuildTargetContext c) + { + string SharedFrameworkNugetVersion = c.BuildContext.Get("SharedFrameworkNugetVersion"); + return $@"{c.BuildContext["CommitHash"]}{Environment.NewLine}{SharedFrameworkNugetVersion}{Environment.NewLine}"; + } + + public static string GetCliVersionFileContent(BuildTargetContext c) + { + var buildVersion = c.BuildContext.Get("BuildVersion"); + var version = buildVersion.NuGetVersion; + return $@"{c.BuildContext["CommitHash"]}{Environment.NewLine}{version}{Environment.NewLine}"; + } } }