Merge pull request #2703 from sokket/publish

Fixing publish scripts to only push to Latest when all bits are there

scripts/dotnet-cli-build/PublishTargets.cs

@@ -1,5 +1,7 @@
 using System;
+using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using System.Net.Http;
 using System.Text;
 using Microsoft.DotNet.Cli.Build.Framework;
@@ -24,6 +26,8 @@ namespace Microsoft.DotNet.Cli.Build
 
         private static string SharedFrameworkNugetVersion { get; set; }
 
+        private static List<string> _latestBlobs;
+
         [Target]
         public static BuildTargetResult InitPublish(BuildTargetContext c)
         {
@@ -41,19 +45,119 @@ namespace Microsoft.DotNet.Cli.Build
         [Target(nameof(PrepareTargets.Init),
         nameof(PublishTargets.InitPublish),
         nameof(PublishTargets.PublishArtifacts),
-        nameof(PublishTargets.TriggerDockerHubBuilds))]
+        nameof(PublishTargets.TriggerDockerHubBuilds),
+        nameof(PublishTargets.FinalizeBuild))]
         [Environment("PUBLISH_TO_AZURE_BLOB", "1", "true")] // This is set by CI systems
         public static BuildTargetResult Publish(BuildTargetContext c)
         {
             return c.Success();
         }
 
+        [Target]
+        public static BuildTargetResult FinalizeBuild(BuildTargetContext c)
+        {
+            if (CheckIfAllBuildsHavePublished())
+            {
+                string targetContainer = $"{Channel}/Binaries/Latest/";
+                string targetVersionFile = $"{targetContainer}{CliNuGetVersion}";
+                string leaseId = AzurePublisherTool.AcquireLeaseOnBlob(targetContainer);
+
+                // Prevent race conditions by dropping a version hint of what version this is. If we see this file
+                // and it is the same as our version then we know that a race happened where two+ builds finished 
+                // at the same time and someone already took care of publishing and we have no work to do.
+                if (AzurePublisherTool.IsLatestSpecifiedVersion(targetVersionFile))
+                {
+                    AzurePublisherTool.ReleaseLeaseOnBlob(targetContainer, leaseId);
+                    return c.Success();
+                }
+                else
+                {
+                    // This is an old drop of latest so remove all old files to ensure a clean state
+                    AzurePublisherTool.ListBlobs($"{targetContainer}").ToList().ForEach(f => AzurePublisherTool.DeleteBlob(f));
+
+                    // Drop the version file signaling such for any race-condition builds (see above comment).
+                    AzurePublisherTool.DropLatestSpecifiedVersion(targetVersionFile);
+                }
+
+                try
+                {
+                    // Iterate over every blob in the latest version folder and copy it to the 'latest' folder
+                    foreach (string blob in _latestBlobs)
+                    {
+                        string targetUrl = $"{targetContainer}{Path.GetFileName(blob)}"
+                                        .Replace(CliNuGetVersion, "latest")
+                                        .Replace(SharedFrameworkNugetVersion, "latest");
+                        AzurePublisherTool.CopyBlob(blob.Replace("/dotnet/", ""), targetUrl);
+                    }
+
+                    // Generate the CLI and SDK Version text files
+                    List<string> versionFiles = new List<string>() { "win.x86.version", "win.x64.version", "ubuntu.x64.version", "rhel.x64.version", "osx.x64.version", "debian.x64.version", "centos.x64.version" };
+                    string cliVersion = Utils.GetCliVersionFileContent(c);
+                    string sfxVersion = Utils.GetSharedFrameworkVersionFileContent(c);
+                    foreach (string version in versionFiles)
+                    {
+                        AzurePublisherTool.PublishStringToBlob($"{Channel}/dnvm/latest.{version}", cliVersion);
+                        AzurePublisherTool.PublishStringToBlob($"{Channel}/dnvm/latest.sharedfx.{version}", sfxVersion);
+                    }
+                }
+                finally
+                {
+                    AzurePublisherTool.ReleaseLeaseOnBlob(targetContainer, leaseId);
+                }
+            }
+
+            return c.Success();
+        }
+
+        private static bool CheckIfAllBuildsHavePublished()
+        {
+             Dictionary<string, bool> badges = new Dictionary<string, bool>()
+             {
+                 { "Windows_x86", false },
+                 { "Windows_x64", false },
+                 { "Ubuntu_x64", false },
+                 { "RHEL_x64", false },
+                 { "OSX_x64", false },
+                 { "Debian_x64", false },
+                 { "CentOS_x64", false }
+             };
+
+            _latestBlobs = new List<string>(AzurePublisherTool.ListBlobs($"{Channel}/Binaries/{CliNuGetVersion}/"));
+
+            var config = Environment.GetEnvironmentVariable("CONFIGURATION");
+            var versionBadgeName = $"{CurrentPlatform.Current}_{CurrentArchitecture.Current}";
+            if (badges.ContainsKey(versionBadgeName) == false)
+            {
+                throw new ArgumentException("A new OS build was added without adding the moniker to the {nameof(badges)} lookup");
+            }
+
+            foreach (string file in _latestBlobs)
+            {
+                string name = Path.GetFileName(file);
+                string key = string.Empty;
+
+                foreach (string img in badges.Keys)
+                {
+                    if ((name.StartsWith($"{img}")) && (name.EndsWith(".svg")))
+                    {
+                        key = img;
+                        break;
+                    }
+                }
+
+                if (string.IsNullOrEmpty(key) == false)
+                {
+                    badges[key] = true;
+                }
+            }
+
+            return badges.Keys.All(key => badges[key]);
+        }
+
         [Target(
             nameof(PublishTargets.PublishInstallerFilesToAzure),
             nameof(PublishTargets.PublishArchivesToAzure),
             nameof(PublishTargets.PublishDebFilesToDebianRepo),
-            nameof(PublishTargets.PublishLatestCliVersionTextFile),
-            nameof(PublishTargets.PublishLatestSharedFrameworkVersionTextFile),
             nameof(PublishTargets.PublishCoreHostPackages),
             nameof(PublishTargets.PublishCliVersionBadge))]
         public static BuildTargetResult PublishArtifacts(BuildTargetContext c) => c.Success();
@@ -87,11 +191,8 @@ namespace Microsoft.DotNet.Cli.Build
         public static BuildTargetResult PublishCliVersionBadge(BuildTargetContext c)
         {
             var versionBadge = c.BuildContext.Get<string>("VersionBadge");
-            var latestVersionBadgeBlob = $"{Channel}/Binaries/Latest/{Path.GetFileName(versionBadge)}";
             var versionBadgeBlob = $"{Channel}/Binaries/{CliNuGetVersion}/{Path.GetFileName(versionBadge)}";
-
             AzurePublisherTool.PublishFile(versionBadgeBlob, versionBadge);
-            AzurePublisherTool.PublishFile(latestVersionBadgeBlob, versionBadge);
             return c.Success();
         }
 
@@ -120,7 +221,7 @@ namespace Microsoft.DotNet.Cli.Build
                 installerFile = Path.ChangeExtension(installerFile, "msi");
             }
 
-            AzurePublisherTool.PublishInstallerFileAndLatest(installerFile, Channel, version);
+            AzurePublisherTool.PublishInstallerFile(installerFile, Channel, version);
 
             return c.Success();
         }
@@ -132,7 +233,7 @@ namespace Microsoft.DotNet.Cli.Build
             var version = SharedFrameworkNugetVersion;
             var installerFile = c.BuildContext.Get<string>("SharedFrameworkInstallerFile");
 
-            AzurePublisherTool.PublishInstallerFileAndLatest(installerFile, Channel, version);
+            AzurePublisherTool.PublishInstallerFile(installerFile, Channel, version);
 
             return c.Success();
         }
@@ -144,7 +245,7 @@ namespace Microsoft.DotNet.Cli.Build
             var version = CliNuGetVersion;
             var installerFile = c.BuildContext.Get<string>("SdkInstallerFile");
 
-            AzurePublisherTool.PublishInstallerFileAndLatest(installerFile, Channel, version);
+            AzurePublisherTool.PublishInstallerFile(installerFile, Channel, version);
 
             return c.Success();
         }
@@ -156,7 +257,7 @@ namespace Microsoft.DotNet.Cli.Build
             var version = SharedFrameworkNugetVersion;
             var installerFile = c.BuildContext.Get<string>("CombinedFrameworkHostInstallerFile");
 
-            AzurePublisherTool.PublishInstallerFileAndLatest(installerFile, Channel, version);
+            AzurePublisherTool.PublishInstallerFile(installerFile, Channel, version);
 
             return c.Success();
         }
@@ -168,7 +269,7 @@ namespace Microsoft.DotNet.Cli.Build
             var version = CliNuGetVersion;
             var installerFile = c.BuildContext.Get<string>("CombinedFrameworkSDKHostInstallerFile");
 
-            AzurePublisherTool.PublishInstallerFileAndLatest(installerFile, Channel, version);
+            AzurePublisherTool.PublishInstallerFile(installerFile, Channel, version);
 
             return c.Success();
         }
@@ -180,7 +281,7 @@ namespace Microsoft.DotNet.Cli.Build
             var version = CliNuGetVersion;
             var archiveFile = c.BuildContext.Get<string>("CombinedFrameworkSDKCompressedFile");
 
-            AzurePublisherTool.PublishArchiveAndLatest(archiveFile, Channel, version);
+            AzurePublisherTool.PublishArchive(archiveFile, Channel, version);
 
             return c.Success();
         }
@@ -191,7 +292,7 @@ namespace Microsoft.DotNet.Cli.Build
             var version = CliNuGetVersion;
             var archiveFile = c.BuildContext.Get<string>("CombinedFrameworkSDKHostCompressedFile");
 
-            AzurePublisherTool.PublishArchiveAndLatest(archiveFile, Channel, version);
+            AzurePublisherTool.PublishArchive(archiveFile, Channel, version);
 
             return c.Success();
         }
@@ -202,7 +303,7 @@ namespace Microsoft.DotNet.Cli.Build
             var version = CliNuGetVersion;
             var archiveFile = c.BuildContext.Get<string>("SdkSymbolsCompressedFile");
 
-            AzurePublisherTool.PublishArchiveAndLatest(archiveFile, Channel, version);
+            AzurePublisherTool.PublishArchive(archiveFile, Channel, version);
 
             return c.Success();
         }
@@ -213,38 +314,7 @@ namespace Microsoft.DotNet.Cli.Build
             var version = SharedFrameworkNugetVersion;
             var archiveFile = c.BuildContext.Get<string>("CombinedFrameworkHostCompressedFile");
 
-            AzurePublisherTool.PublishArchiveAndLatest(archiveFile, Channel, version);
+            AzurePublisherTool.PublishArchive(archiveFile, Channel, version);
-            return c.Success();
-        }
-
-        [Target]
-        public static BuildTargetResult PublishLatestCliVersionTextFile(BuildTargetContext c)
-        {
-            var version = CliNuGetVersion;
-
-            var osname = Monikers.GetOSShortName();
-            var latestCliVersionBlob = $"{Channel}/dnvm/latest.{osname}.{CurrentArchitecture.Current}.version";
-            var latestCliVersionFile = Path.Combine(Dirs.Stage2, "sdk", version, ".version");
-
-            AzurePublisherTool.PublishFile(latestCliVersionBlob, latestCliVersionFile);
-            return c.Success();
-        }
-
-        [Target]
-        public static BuildTargetResult PublishLatestSharedFrameworkVersionTextFile(BuildTargetContext c)
-        {
-            var version = SharedFrameworkNugetVersion;
-
-            var osname = Monikers.GetOSShortName();
-            var latestSharedFXVersionBlob = $"{Channel}/dnvm/latest.sharedfx.{osname}.{CurrentArchitecture.Current}.version";
-            var latestSharedFXVersionFile = Path.Combine(
-                Dirs.Stage2,
-                "shared",
-                CompileTargets.SharedFrameworkName,
-                version,
-                ".version");
-
-            AzurePublisherTool.PublishFile(latestSharedFXVersionBlob, latestSharedFXVersionFile);
             return c.Success();
         }
 

scripts/dotnet-cli-build/Publishing/AzurePublisher.cs

@@ -1,5 +1,7 @@
 using System;
+using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using System.Net.Http;
 using System.Text;
 using Microsoft.DotNet.Cli.Build.Framework;
@@ -21,7 +23,6 @@ namespace Microsoft.DotNet.Cli.Build
         public AzurePublisher()
         {
             _connectionString = Environment.GetEnvironmentVariable("CONNECTION_STRING").Trim('"');
-
             _blobContainer = GetDotnetBlobContainer(_connectionString);
         }
 
@@ -33,26 +34,16 @@ namespace Microsoft.DotNet.Cli.Build
             return blobClient.GetContainerReference(s_dotnetBlobContainerName);
         }
 
-        public void PublishInstallerFileAndLatest(string installerFile, string channel, string version)
+        public void PublishInstallerFile(string installerFile, string channel, string version)
         {
             var installerFileBlob = CalculateInstallerBlob(installerFile, channel, version);
-
-            var latestInstallerFileName = installerFile.Replace(version, "latest");
-            var latestInstallerFileBlob = CalculateInstallerBlob(latestInstallerFileName, channel, "Latest");
-
             PublishFile(installerFileBlob, installerFile);
-            PublishFile(latestInstallerFileBlob, installerFile);
         }
 
-        public void PublishArchiveAndLatest(string archiveFile, string channel, string version)
+        public void PublishArchive(string archiveFile, string channel, string version)
         {
             var archiveFileBlob = CalculateArchiveBlob(archiveFile, channel, version);
-
-            var latestArchiveFileName = archiveFile.Replace(version, "latest");
-            var latestArchiveFileBlob = CalculateArchiveBlob(latestArchiveFileName, channel, "Latest");
-
             PublishFile(archiveFileBlob, archiveFile);
-            PublishFile(latestArchiveFileBlob, archiveFile);
         }
 
         public void PublishFile(string blob, string file)
@@ -63,6 +54,27 @@ namespace Microsoft.DotNet.Cli.Build
             SetBlobPropertiesBasedOnFileType(blockBlob);
         }
 
+        public void PublishStringToBlob(string blob, string content)
+        {
+            CloudBlockBlob blockBlob = _blobContainer.GetBlockBlobReference(blob);
+            blockBlob.UploadTextAsync(content).Wait();
+        }
+
+        public void CopyBlob(string sourceBlob, string targetBlob)
+        {
+            CloudBlockBlob source = _blobContainer.GetBlockBlobReference(sourceBlob);
+            CloudBlockBlob target = _blobContainer.GetBlockBlobReference(targetBlob);
+
+            // Create the empty blob
+            using (MemoryStream ms = new MemoryStream())
+            {
+                target.UploadFromStreamAsync(ms).Wait();
+            }
+
+            // Copy actual blob data
+            target.StartCopyAsync(source).Wait();
+        }
+
         private void SetBlobPropertiesBasedOnFileType(CloudBlockBlob blockBlob)
         {
             if (Path.GetExtension(blockBlob.Uri.AbsolutePath.ToLower()) == ".svg")
@@ -73,6 +85,51 @@ namespace Microsoft.DotNet.Cli.Build
             }
         }
 
+        public IEnumerable<string> ListBlobs(string virtualDirectory)
+        {
+            CloudBlobDirectory blobDir = _blobContainer.GetDirectoryReference(virtualDirectory);
+            BlobContinuationToken continuationToken = new BlobContinuationToken();
+
+            var blobFiles = blobDir.ListBlobsSegmentedAsync(continuationToken).Result;
+            return blobFiles.Results.Select(bf => bf.Uri.PathAndQuery);
+        }
+
+        public string AcquireLeaseOnBlob(string blob)
+        {
+            CloudBlockBlob cloudBlob = _blobContainer.GetBlockBlobReference(blob);
+            System.Threading.Tasks.Task<string> task = cloudBlob.AcquireLeaseAsync(TimeSpan.FromMinutes(1), null);
+            task.Wait(); 
+            return task.Result;
+        }
+
+        public void ReleaseLeaseOnBlob(string blob, string leaseId)
+        {
+            CloudBlockBlob cloudBlob = _blobContainer.GetBlockBlobReference(blob);
+            AccessCondition ac = new AccessCondition() { LeaseId = leaseId };
+            cloudBlob.ReleaseLeaseAsync(ac).Wait();
+        }
+
+        public bool IsLatestSpecifiedVersion(string version)
+        {
+            System.Threading.Tasks.Task<bool> task = _blobContainer.GetBlockBlobReference(version).ExistsAsync();
+            task.Wait();
+            return task.Result;
+        }
+
+        public void DropLatestSpecifiedVersion(string version)
+        {
+            CloudBlockBlob blob = _blobContainer.GetBlockBlobReference(version);
+            using (MemoryStream ms = new MemoryStream())
+            {
+                blob.UploadFromStreamAsync(ms).Wait();
+            }
+        }
+
+        public void DeleteBlob(string path)
+        {
+            _blobContainer.GetBlockBlobReference(path).DeleteAsync().Wait();
+        }
+
         public string CalculateInstallerUploadUrl(string installerFile, string channel, string version)
         {
             return $"{s_dotnetBlobRootUrl}{CalculateInstallerBlob(installerFile, channel, version)}";

scripts/dotnet-cli-build/Utils/Utils.cs

@@ -3,6 +3,8 @@ using System.IO;
 using System.Runtime.InteropServices;
 using System.Security.Cryptography;
 
+using Microsoft.DotNet.Cli.Build.Framework;
+
 namespace Microsoft.DotNet.Cli.Build
 {
     public static class Utils
@@ -133,5 +135,18 @@ namespace Microsoft.DotNet.Cli.Build
                 File.Copy(file, destFile, true);
             }
         }
+
+        public static string GetSharedFrameworkVersionFileContent(BuildTargetContext c)
+        {
+            string SharedFrameworkNugetVersion = c.BuildContext.Get<string>("SharedFrameworkNugetVersion");
+            return $@"{c.BuildContext["CommitHash"]}{Environment.NewLine}{SharedFrameworkNugetVersion}{Environment.NewLine}";
+        }
+
+        public static string GetCliVersionFileContent(BuildTargetContext c)
+        {
+            var buildVersion = c.BuildContext.Get<BuildVersion>("BuildVersion");
+            var version = buildVersion.NuGetVersion;
+            return $@"{c.BuildContext["CommitHash"]}{Environment.NewLine}{version}{Environment.NewLine}";
+        }
     }
 }