Fixing up build publishing to be more organized

Partly fixes #3120 by organizing the output files in blob storage according to the design in the bug

build_projects/dotnet-cli-build/PrepareTargets.cs

@@ -175,14 +175,14 @@ namespace Microsoft.DotNet.Cli.Build
 
             Mkdirp(Path.GetDirectoryName(combinedSharedHostAndFrameworkArchiveDownloadFile));
 
-            if ( ! File.Exists(combinedSharedHostAndFrameworkArchiveDownloadFile))
+            if (!File.Exists(combinedSharedHostAndFrameworkArchiveDownloadFile))
             {
                 // Needed for computing the blob path
-                var combinedSharedHostAndFrameworkArchiveBuildContextFile = 
+                var combinedSharedHostAndFrameworkArchiveBuildContextFile =
                     c.BuildContext.Get<string>("CombinedFrameworkHostCompressedFile");
 
                 AzurePublisher.DownloadFile(
-                    AzurePublisher.CalculateArchiveBlob(
+                    CalculateArchiveBlob(
                         combinedSharedHostAndFrameworkArchiveBuildContextFile,
                         sharedFrameworkChannel,
                         sharedFrameworkVersion),
@@ -225,13 +225,13 @@ namespace Microsoft.DotNet.Cli.Build
             Mkdirp(Path.GetDirectoryName(sharedHostInstallerDownloadFile));
             Mkdirp(Path.GetDirectoryName(hostFxrInstallerDownloadFile));
 
-            if ( ! File.Exists(sharedFrameworkInstallerDownloadFile))
+            if (!File.Exists(sharedFrameworkInstallerDownloadFile))
             {
                 var sharedFrameworkInstallerDestinationFile = c.BuildContext.Get<string>("SharedFrameworkInstallerFile");
                 Mkdirp(Path.GetDirectoryName(sharedFrameworkInstallerDestinationFile));
-                
+
                 AzurePublisher.DownloadFile(
-                    AzurePublisher.CalculateInstallerBlob(
+                    CalculateInstallerBlob(
                         sharedFrameworkInstallerDestinationFile,
                         sharedFrameworkChannel,
                         sharedFrameworkVersion),
@@ -239,14 +239,14 @@ namespace Microsoft.DotNet.Cli.Build
 
                 File.Copy(sharedFrameworkInstallerDownloadFile, sharedFrameworkInstallerDestinationFile, true);
             }
-            
+
-            if ( ! File.Exists(sharedHostInstallerDownloadFile))
+            if (!File.Exists(sharedHostInstallerDownloadFile))
             {
                 var sharedHostInstallerDestinationFile = c.BuildContext.Get<string>("SharedHostInstallerFile");
                 Mkdirp(Path.GetDirectoryName(sharedHostInstallerDestinationFile));
 
                 AzurePublisher.DownloadFile(
-                   AzurePublisher.CalculateInstallerBlob(
+                   CalculateInstallerBlob(
                        sharedHostInstallerDestinationFile,
                        sharedHostChannel,
                        hostVersion),
@@ -255,13 +255,13 @@ namespace Microsoft.DotNet.Cli.Build
                 File.Copy(sharedHostInstallerDownloadFile, sharedHostInstallerDestinationFile, true);
             }
 
-            if ( ! File.Exists(hostFxrInstallerDownloadFile))
+            if (!File.Exists(hostFxrInstallerDownloadFile))
             {
                 var hostFxrInstallerDestinationFile = c.BuildContext.Get<string>("HostFxrInstallerFile");
                 Mkdirp(Path.GetDirectoryName(hostFxrInstallerDestinationFile));
 
                 AzurePublisher.DownloadFile(
-                   AzurePublisher.CalculateInstallerBlob(
+                   CalculateInstallerBlob(
                        hostFxrInstallerDestinationFile,
                        hostFxrChannel,
                        hostFxrVersion),
@@ -277,7 +277,7 @@ namespace Microsoft.DotNet.Cli.Build
         public static BuildTargetResult CheckPackageCache(BuildTargetContext c)
         {
             var ciBuild = string.Equals(Environment.GetEnvironmentVariable("CI_BUILD"), "1", StringComparison.Ordinal);
-            
+
             // Always set the package cache location local to the build
             Environment.SetEnvironmentVariable("NUGET_PACKAGES", Dirs.NuGetPackages);
 
@@ -548,7 +548,18 @@ cmake is required to build the native host 'corehost'";
             {
                 c.BuildContext[contextPrefix + "InstallerFile"] = Path.Combine(Dirs.Packages, installer);
             }
+        }
 
+        // The following CalculateBlob methods are temporary until the core-setup repo up-takes the new Azure Publish layout and
+        // CLI consumes newer Shared FX versions.
+        private static string CalculateArchiveBlob(string archiveFile, string channel, string version)
+        {
+            return $"{channel}/Binaries/{version}/{Path.GetFileName(archiveFile)}";
+        }
+
+        private static string CalculateInstallerBlob(string installerFile, string channel, string version)
+        {
+            return $"{channel}/Installers/{version}/{Path.GetFileName(installerFile)}";
         }
     }
 }

build_projects/dotnet-cli-build/PublishTargets.cs

@@ -32,10 +32,11 @@ namespace Microsoft.DotNet.Cli.Build
             return c.Success();
         }
 
-        [Target(nameof(PrepareTargets.Init),
+        [Target(
-        nameof(PublishTargets.InitPublish),
+            nameof(PrepareTargets.Init),
-        nameof(PublishTargets.PublishArtifacts),
+            nameof(PublishTargets.InitPublish),
-        nameof(PublishTargets.FinalizeBuild))]
+            nameof(PublishTargets.PublishArtifacts),
+            nameof(PublishTargets.FinalizeBuild))]
         [Environment("PUBLISH_TO_AZURE_BLOB", "1", "true")] // This is set by CI systems
         public static BuildTargetResult Publish(BuildTargetContext c)
         {
@@ -47,9 +48,9 @@ namespace Microsoft.DotNet.Cli.Build
         {
             if (CheckIfAllBuildsHavePublished())
             {
-                string targetContainer = $"{Channel}/Binaries/Latest/";
+                string targetContainer = $"{AzurePublisher.Product.Sdk}/{Channel}/";
                 string targetVersionFile = $"{targetContainer}{CliNuGetVersion}";
-                string semaphoreBlob = $"{Channel}/Binaries/publishSemaphore";
+                string semaphoreBlob = $"{targetContainer}/publishSemaphore";
                 AzurePublisherTool.CreateBlobIfNotExists(semaphoreBlob);
                 string leaseId = AzurePublisherTool.AcquireLeaseOnBlob(semaphoreBlob);
 
@@ -66,8 +67,7 @@ namespace Microsoft.DotNet.Cli.Build
                     Regex versionFileRegex = new Regex(@"(?<version>\d\.\d\.\d)-(?<release>.*)?");
 
                     // Delete old version files
-                    AzurePublisherTool.ListBlobs($"{targetContainer}")
+                    AzurePublisherTool.ListBlobs(targetContainer)
-                        .Select(s => s.Replace("/dotnet/", ""))
                         .Where(s => versionFileRegex.IsMatch(s))
                         .ToList()
                         .ForEach(f => AzurePublisherTool.TryDeleteBlob(f));
@@ -78,32 +78,10 @@ namespace Microsoft.DotNet.Cli.Build
 
                 try
                 {
-                    // Copy the latest CLI bits
+                    CopyBlobsToLatest(targetContainer);
-                    CopyBlobs($"{Channel}/Binaries/{CliNuGetVersion}/", targetContainer);
-
-                    // Copy the latest installer files
-                    CopyBlobs($"{Channel}/Installers/{CliNuGetVersion}/", $"{Channel}/Installers/Latest/");
-
-                    // Generate the CLI and SDK Version text files
-                    List<string> versionFiles = new List<string>()
-                    {
-                        "win.x86.version",
-                        "win.x64.version",
-                        "ubuntu.x64.version",
-                        "ubuntu.16.04.x64.version",
-                        "rhel.x64.version",
-                        "osx.x64.version",
-                        "debian.x64.version",
-                        "centos.x64.version",
-                        "fedora.23.x64.version",
-                        "opensuse.13.2.x64.version"
-                    };
 
                     string cliVersion = Utils.GetCliVersionFileContent(c);
-                    foreach (string version in versionFiles)
+                    AzurePublisherTool.PublishStringToBlob($"{targetContainer}/latest.version", cliVersion);
-                    {
-                        AzurePublisherTool.PublishStringToBlob($"{Channel}/dnvm/latest.{version}", cliVersion);
-                    }
 
                     UpdateVersionsRepo(c);
                 }
@@ -116,64 +94,55 @@ namespace Microsoft.DotNet.Cli.Build
             return c.Success();
         }
 
-        private static void CopyBlobs(string sourceFolder, string destinationFolder)
+        private static void CopyBlobsToLatest(string destinationFolder)
         {
-            foreach (string blob in AzurePublisherTool.ListBlobs(sourceFolder))
+            foreach (string blob in AzurePublisherTool.ListBlobs(AzurePublisher.Product.Sdk, CliNuGetVersion))
             {
-                string source = blob.Replace("/dotnet/", "");
                 string targetName = Path.GetFileName(blob)
                                         .Replace(CliNuGetVersion, "latest");
 
                 string target = $"{destinationFolder}{targetName}";
-                AzurePublisherTool.CopyBlob(source, target);
+                AzurePublisherTool.CopyBlob(blob, target);
             }
         }
 
         private static bool CheckIfAllBuildsHavePublished()
         {
             Dictionary<string, bool> badges = new Dictionary<string, bool>()
-             {
+            {
-                 { "Windows_x86", false },
+                { "Windows_x86", false },
-                 { "Windows_x64", false },
+                { "Windows_x64", false },
-                 { "Ubuntu_x64", false },
+                { "Ubuntu_x64", false },
-                 { "Ubuntu_16_04_x64", false },
+                { "Ubuntu_16_04_x64", false },
-                 { "RHEL_x64", false },
+                { "RHEL_x64", false },
-                 { "OSX_x64", false },
+                { "OSX_x64", false },
-                 { "Debian_x64", false },
+                { "Debian_x64", false },
-                 { "CentOS_x64", false },
+                { "CentOS_x64", false },
-                 { "Fedora_23_x64", false },
+                { "Fedora_23_x64", false },
-                 { "openSUSE_13_2_x64", false }
+                { "openSUSE_13_2_x64", false }
-             };
+            };
-
-            List<string> blobs = new List<string>(AzurePublisherTool.ListBlobs($"{Channel}/Binaries/{CliNuGetVersion}/"));
 
             var versionBadgeName = $"{Monikers.GetBadgeMoniker()}";
-            if (badges.ContainsKey(versionBadgeName) == false)
+            if (!badges.ContainsKey(versionBadgeName))
             {
-                throw new ArgumentException("A new OS build was added without adding the moniker to the {nameof(badges)} lookup");
+                throw new ArgumentException($"A new OS build '{versionBadgeName}' was added without adding the moniker to the {nameof(badges)} lookup");
             }
 
+            IEnumerable<string> blobs = AzurePublisherTool.ListBlobs(AzurePublisher.Product.Sdk, CliNuGetVersion);
             foreach (string file in blobs)
             {
                 string name = Path.GetFileName(file);
-                string key = string.Empty;
-
                 foreach (string img in badges.Keys)
                 {
                     if ((name.StartsWith($"{img}")) && (name.EndsWith(".svg")))
                     {
-                        key = img;
+                        badges[img] = true;
                         break;
                     }
                 }
-
-                if (string.IsNullOrEmpty(key) == false)
-                {
-                    badges[key] = true;
-                }
             }
 
-            return badges.Keys.All(key => badges[key]);
+            return badges.Values.All(v => v);
         }
 
         [Target(
@@ -206,8 +175,8 @@ namespace Microsoft.DotNet.Cli.Build
         public static BuildTargetResult PublishCliVersionBadge(BuildTargetContext c)
         {
             var versionBadge = c.BuildContext.Get<string>("VersionBadge");
-            var versionBadgeBlob = $"{Channel}/Binaries/{CliNuGetVersion}/{Path.GetFileName(versionBadge)}";
+            UploadFile(versionBadge);
-            AzurePublisherTool.PublishFile(versionBadgeBlob, versionBadge);
+
             return c.Success();
         }
 
@@ -215,10 +184,8 @@ namespace Microsoft.DotNet.Cli.Build
         [BuildPlatforms(BuildPlatform.Ubuntu)]
         public static BuildTargetResult PublishSdkInstallerFileToAzure(BuildTargetContext c)
         {
-            var version = CliNuGetVersion;
             var installerFile = c.BuildContext.Get<string>("SdkInstallerFile");
-
+            UploadFile(installerFile);
-            AzurePublisherTool.PublishInstallerFile(installerFile, Channel, version);
 
             return c.Success();
         }
@@ -227,10 +194,8 @@ namespace Microsoft.DotNet.Cli.Build
         [BuildPlatforms(BuildPlatform.Windows, BuildPlatform.OSX)]
         public static BuildTargetResult PublishCombinedFrameworkSDKHostInstallerFileToAzure(BuildTargetContext c)
         {
-            var version = CliNuGetVersion;
             var installerFile = c.BuildContext.Get<string>("CombinedFrameworkSDKHostInstallerFile");
-
+            UploadFile(installerFile);
-            AzurePublisherTool.PublishInstallerFile(installerFile, Channel, version);
 
             return c.Success();
         }
@@ -239,10 +204,8 @@ namespace Microsoft.DotNet.Cli.Build
         [BuildPlatforms(BuildPlatform.Windows)]
         public static BuildTargetResult PublishCombinedFrameworkSDKArchiveToAzure(BuildTargetContext c)
         {
-            var version = CliNuGetVersion;
             var archiveFile = c.BuildContext.Get<string>("CombinedFrameworkSDKCompressedFile");
-
+            UploadFile(archiveFile);
-            AzurePublisherTool.PublishArchive(archiveFile, Channel, version);
 
             return c.Success();
         }
@@ -250,10 +213,8 @@ namespace Microsoft.DotNet.Cli.Build
         [Target]
         public static BuildTargetResult PublishCombinedHostFrameworkSdkArchiveToAzure(BuildTargetContext c)
         {
-            var version = CliNuGetVersion;
             var archiveFile = c.BuildContext.Get<string>("CombinedFrameworkSDKHostCompressedFile");
-
+            UploadFile(archiveFile);
-            AzurePublisherTool.PublishArchive(archiveFile, Channel, version);
 
             return c.Success();
         }
@@ -261,10 +222,8 @@ namespace Microsoft.DotNet.Cli.Build
         [Target]
         public static BuildTargetResult PublishSDKSymbolsArchiveToAzure(BuildTargetContext c)
         {
-            var version = CliNuGetVersion;
             var archiveFile = c.BuildContext.Get<string>("SdkSymbolsCompressedFile");
-
+            UploadFile(archiveFile);
-            AzurePublisherTool.PublishArchive(archiveFile, Channel, version);
 
             return c.Success();
         }
@@ -277,7 +236,7 @@ namespace Microsoft.DotNet.Cli.Build
 
             var packageName = CliMonikers.GetSdkDebianPackageName(c);
             var installerFile = c.BuildContext.Get<string>("SdkInstallerFile");
-            var uploadUrl = AzurePublisherTool.CalculateInstallerUploadUrl(installerFile, Channel, version);
+            var uploadUrl = AzurePublisher.CalculateUploadUrlForFile(installerFile, AzurePublisher.Product.Sdk, version);
 
             DebRepoPublisherTool.PublishDebFileToDebianRepo(
                 packageName,
@@ -297,6 +256,11 @@ namespace Microsoft.DotNet.Cli.Build
             VersionRepoUpdater repoUpdater = new VersionRepoUpdater(githubAuthToken);
             repoUpdater.UpdatePublishedVersions(nupkgFilePath, versionsRepoPath).Wait();
         }
+
+        private static string UploadFile(string file)
+        {
+            return AzurePublisherTool.UploadFile(file, AzurePublisher.Product.Sdk, CliNuGetVersion);
+        }
     }
 }
 

build_projects/shared-build-targets-utils/Publishing/AzurePublisher.cs

@@ -3,20 +3,24 @@ using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Net.Http;
-using System.Text;
+using System.Threading.Tasks;
-using Microsoft.DotNet.Cli.Build.Framework;
 using Microsoft.WindowsAzure.Storage;
 using Microsoft.WindowsAzure.Storage.Blob;
 
-using static Microsoft.DotNet.Cli.Build.Framework.BuildHelpers;
-using System.Threading.Tasks;
-
 namespace Microsoft.DotNet.Cli.Build
 {
     public class AzurePublisher
     {
-        private static readonly string s_dotnetBlobRootUrl = "https://dotnetcli.blob.core.windows.net/dotnet/";
+        public enum Product
-        private static readonly string s_dotnetBlobContainerName = "dotnet";
+        {
+            SharedFramework,
+            Host,
+            HostFxr,
+            Sdk,
+        }
+
+        private const string s_dotnetBlobRootUrl = "https://dotnetcli.blob.core.windows.net/" + s_dotnetBlobContainerName;
+        private const string s_dotnetBlobContainerName = "dotnet";
 
         private string _connectionString { get; set; }
         private CloudBlobContainer _blobContainer { get; set; }
@@ -35,33 +39,21 @@ namespace Microsoft.DotNet.Cli.Build
             return blobClient.GetContainerReference(s_dotnetBlobContainerName);
         }
 
-        public void PublishInstallerFile(string installerFile, string channel, string version)
+        public string UploadFile(string file, Product product, string version)
         {
-            var installerFileBlob = CalculateInstallerBlob(installerFile, channel, version);
+            string url = CalculateUploadUrlForFile(file, product, version);
-            PublishFile(installerFileBlob, installerFile);
+            CloudBlockBlob blob = _blobContainer.GetBlockBlobReference(url);
-        }
+            blob.UploadFromFileAsync(file, FileMode.Open).Wait();
-
+            SetBlobPropertiesBasedOnFileType(blob);
-        public void PublishArchive(string archiveFile, string channel, string version)
+            return url;
-        {
-            var archiveFileBlob = CalculateArchiveBlob(archiveFile, channel, version);
-            PublishFile(archiveFileBlob, archiveFile);
-        }
-
-        public void PublishFile(string blob, string file)
-        {
-            CloudBlockBlob blockBlob = _blobContainer.GetBlockBlobReference(blob);
-            blockBlob.UploadFromFileAsync(file, FileMode.Open).Wait();
-
-            SetBlobPropertiesBasedOnFileType(blockBlob);
         }
 
         public void PublishStringToBlob(string blob, string content)
         {
             CloudBlockBlob blockBlob = _blobContainer.GetBlockBlobReference(blob);
             blockBlob.UploadTextAsync(content).Wait();
-            
+
-            blockBlob.Properties.ContentType = "text/plain";
+            SetBlobPropertiesBasedOnFileType(blockBlob);
-            blockBlob.SetPropertiesAsync().Wait();
         }
 
         public void CopyBlob(string sourceBlob, string targetBlob)
@@ -90,24 +82,31 @@ namespace Microsoft.DotNet.Cli.Build
             else if (Path.GetExtension(blockBlob.Uri.AbsolutePath.ToLower()) == ".version")
             {
                 blockBlob.Properties.ContentType = "text/plain";
+                blockBlob.Properties.CacheControl = "no-cache";
                 blockBlob.SetPropertiesAsync().Wait();
             }
         }
 
+        public IEnumerable<string> ListBlobs(Product product, string version)
+        {
+            string virtualDirectory = $"{product}/{version}";
+            return ListBlobs(virtualDirectory);
+        }
+
         public IEnumerable<string> ListBlobs(string virtualDirectory)
         {
             CloudBlobDirectory blobDir = _blobContainer.GetDirectoryReference(virtualDirectory);
             BlobContinuationToken continuationToken = new BlobContinuationToken();
 
             var blobFiles = blobDir.ListBlobsSegmentedAsync(continuationToken).Result;
-            return blobFiles.Results.Select(bf => bf.Uri.PathAndQuery);
+            return blobFiles.Results.Select(bf => bf.Uri.PathAndQuery.Replace($"/{s_dotnetBlobContainerName}/", string.Empty));
         }
 
         public string AcquireLeaseOnBlob(string blob)
         {
             CloudBlockBlob cloudBlob = _blobContainer.GetBlockBlobReference(blob);
-            System.Threading.Tasks.Task<string> task = cloudBlob.AcquireLeaseAsync(TimeSpan.FromMinutes(1), null);
+            Task<string> task = cloudBlob.AcquireLeaseAsync(TimeSpan.FromMinutes(1), null);
-            task.Wait(); 
+            task.Wait();
             return task.Result;
         }
 
@@ -120,7 +119,7 @@ namespace Microsoft.DotNet.Cli.Build
 
         public bool IsLatestSpecifiedVersion(string version)
         {
-            System.Threading.Tasks.Task<bool> task = _blobContainer.GetBlockBlobReference(version).ExistsAsync();
+            Task<bool> task = _blobContainer.GetBlockBlobReference(version).ExistsAsync();
             task.Wait();
             return task.Result;
         }
@@ -136,7 +135,7 @@ namespace Microsoft.DotNet.Cli.Build
 
         public void CreateBlobIfNotExists(string path)
         {
-            System.Threading.Tasks.Task<bool> task = _blobContainer.GetBlockBlobReference(path).ExistsAsync();
+            Task<bool> task = _blobContainer.GetBlockBlobReference(path).ExistsAsync();
             task.Wait();
             if (!task.Result)
             {
@@ -153,40 +152,30 @@ namespace Microsoft.DotNet.Cli.Build
             try
             {
                 DeleteBlob(path);
-                
+
                 return true;
             }
             catch (Exception e)
             {
                 Console.WriteLine($"Deleting blob {path} failed with \r\n{e.Message}");
-                
+
                 return false;
             }
         }
 
-        public void DeleteBlob(string path)
+        private void DeleteBlob(string path)
         {
             _blobContainer.GetBlockBlobReference(path).DeleteAsync().Wait();
         }
 
-        public string CalculateInstallerUploadUrl(string installerFile, string channel, string version)
+        public static string CalculateUploadUrlForFile(string file, Product product, string version)
         {
-            return $"{s_dotnetBlobRootUrl}{CalculateInstallerBlob(installerFile, channel, version)}";
+            return $"{s_dotnetBlobRootUrl}/{product}/{version}/{Path.GetFileName(file)}";
-        }
-
-        public static string CalculateInstallerBlob(string installerFile, string channel, string version)
-        {
-            return $"{channel}/Installers/{version}/{Path.GetFileName(installerFile)}";
-        }
-
-        public static string CalculateArchiveBlob(string archiveFile, string channel, string version)
-        {
-            return $"{channel}/Binaries/{version}/{Path.GetFileName(archiveFile)}";
         }
 
         public static async Task DownloadFile(string blobFilePath, string localDownloadPath)
         {
-            var blobUrl = $"{s_dotnetBlobRootUrl}{blobFilePath}";
+            var blobUrl = $"{s_dotnetBlobRootUrl}/{blobFilePath}";
 
             using (var client = new HttpClient())
             {
@@ -195,7 +184,7 @@ namespace Microsoft.DotNet.Cli.Build
                 var response = sendTask.Result.EnsureSuccessStatusCode();
 
                 var httpStream = await response.Content.ReadAsStreamAsync();
-                
+
                 using (var fileStream = File.Create(localDownloadPath))
                 using (var reader = new StreamReader(httpStream))
                 {
@@ -204,23 +193,5 @@ namespace Microsoft.DotNet.Cli.Build
                 }
             }
         }
-
-        public void DownloadFilesWithExtension(string blobVirtualDirectory, string fileExtension, string localDownloadPath)
-        {
-            CloudBlobDirectory blobDir = _blobContainer.GetDirectoryReference(blobVirtualDirectory);
-            BlobContinuationToken continuationToken = new BlobContinuationToken();
-
-            var blobFiles = blobDir.ListBlobsSegmentedAsync(continuationToken).Result;
-
-            foreach (var blobFile in blobFiles.Results.OfType<CloudBlockBlob>())
-            {
-                if (Path.GetExtension(blobFile.Uri.AbsoluteUri) == fileExtension)
-                {
-                    string localBlobFile = Path.Combine(localDownloadPath, Path.GetFileName(blobFile.Uri.AbsoluteUri));
-                    Console.WriteLine($"Downloading {blobFile.Uri.AbsoluteUri} to {localBlobFile}...");
-                    blobFile.DownloadToFileAsync(localBlobFile, FileMode.Create).Wait();
-                }
-            }
-        }
     }
 }