From f5466be4a7883d8e8cc2c80bb3841d079e8a7af5 Mon Sep 17 00:00:00 2001 From: Wes Haggard Date: Fri, 8 Apr 2016 21:36:32 -0700 Subject: [PATCH] Update PullNupkgFilesFromBlob target Add support to enable polling to watch for new builds. It will try to publish the last 10 builds by default. If the build has all the necessary runtime packages and doesn't have the packages.pushed semaphore file. Also add support for forcing the publish of a given build set in the following environment variable FORCE_PUBLISH_BLOB_BUILD_VERSION. This will blindly try to pull all the nupkgs for the given build ignoring if they were already published or have all the runtimes specified. --- scripts/dotnet-cli-build/PublishTargets.cs | 135 ++++++++++++--------- 1 file changed, 79 insertions(+), 56 deletions(-) diff --git a/scripts/dotnet-cli-build/PublishTargets.cs b/scripts/dotnet-cli-build/PublishTargets.cs index 56267603c..808d9b692 100644 --- a/scripts/dotnet-cli-build/PublishTargets.cs +++ b/scripts/dotnet-cli-build/PublishTargets.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Net.Http; using System.Text; using System.Text.RegularExpressions; @@ -325,32 +326,28 @@ namespace Microsoft.DotNet.Cli.Build return c.Success(); } + private const string PackagePushedSemaphoreFileName = "packages.pushed"; + [Target(nameof(PrepareTargets.Init), nameof(InitPublish))] public static BuildTargetResult PullNupkgFilesFromBlob(BuildTargetContext c) { - var pathToPublish = Environment.GetEnvironmentVariable("BLOB_VIRTUAL_PATH_TO_PUBLISH"); - if (string.IsNullOrEmpty(pathToPublish)) - pathToPublish = FindLatestCompleteBuild(); + Directory.CreateDirectory(Dirs.PackagesNoRID); - if (string.IsNullOrEmpty(pathToPublish)) + var hostBlob = $"{Channel}/Binaries/"; + + string forcePushBuild = Environment.GetEnvironmentVariable("FORCE_PUBLISH_BLOB_BUILD_VERSION"); + + if (!string.IsNullOrEmpty(forcePushBuild)) { - Console.WriteLine("Didn't find a directory with all the necessary runtime packages!"); - return c.Failed(); + Console.WriteLine($"Forcing all nupkg packages for build version {forcePushBuild}."); + DownloadPackagesForPush(hostBlob + forcePushBuild); + return c.Success(); } - Directory.CreateDirectory(Dirs.PackagesNoRID); - AzurePublisherTool.DownloadFiles(pathToPublish, ".nupkg", Dirs.PackagesNoRID); - - return c.Success(); - } - - private static string FindLatestCompleteBuild() - { - var hostBlob = $"{Channel}/Binaries/"; + List buildVersions = new List(); Regex buildVersionRegex = new Regex(@"Binaries/(?\d+\.\d+\.\d+-[^-]+-\d{6})/$"); - List buildVersions = new List(); foreach (string file in AzurePublisherTool.ListBlobs(hostBlob)) { var match = buildVersionRegex.Match(file); @@ -360,55 +357,81 @@ namespace Microsoft.DotNet.Cli.Build } } + // Sort decending buildVersions.Sort(); buildVersions.Reverse(); - Dictionary runtimes = new Dictionary() + // Try to publish the last 10 builds + foreach (var bv in buildVersions.Take(10)) { - {"win7", false }, - {"osx.10.10", false }, - {"rhel.7", false }, - {"ubuntu.14.04", false }, + Console.WriteLine($"Checking drop version: {bv}"); - }; - - foreach (var bv in buildVersions) - { - Console.WriteLine($"Version: {bv}"); - - var buildFiles = AzurePublisherTool.ListBlobs(hostBlob + bv); - - foreach (var bf in buildFiles) + if (ShouldDownloadAndPush(hostBlob, bv)) { - string buildFile = Path.GetFileName(bf); - - foreach (var runtime in runtimes.Keys) - { - if (buildFile.StartsWith($"runtime.{runtime}")) - { - runtimes[runtime] = true; - break; - } - } - } - - bool missingRuntime = false; - foreach (var runtime in runtimes) - { - if (!runtime.Value) - { - missingRuntime = true; - Console.WriteLine($"Version {bv} is missing packages for runtime {runtime.Key}"); - } - } - - if (!missingRuntime) - { - return hostBlob + bv; + DownloadPackagesForPush(hostBlob + bv); } } - return null; + return c.Success(); + } + + private static bool ShouldDownloadAndPush(string hostBlob, string buildVersion) + { + // Set of runtime ids to look for to act as the signal that the build + // as finished each of these legs of the build. + Dictionary runtimes = new Dictionary() + { + {"win7-x64", false }, + {"win7-x86", false }, + {"osx.10.10-x64", false }, + {"rhel.7-x64", false }, + {"ubuntu.14.04-x64", false }, + }; + + var buildFiles = AzurePublisherTool.ListBlobs(hostBlob + buildVersion); + + foreach (var bf in buildFiles) + { + string buildFile = Path.GetFileName(bf); + + if (buildFile == PackagePushedSemaphoreFileName) + { + Console.WriteLine($"Found '{PackagePushedSemaphoreFileName}' for build version {buildVersion} so skipping this drop."); + // Nothing to do because the latest build is uploaded. + return false; + } + + foreach (var runtime in runtimes.Keys) + { + if (buildFile.StartsWith($"runtime.{runtime}")) + { + runtimes[runtime] = true; + break; + } + } + } + + bool missingRuntime = false; + foreach (var runtime in runtimes) + { + if (!runtime.Value) + { + missingRuntime = true; + Console.WriteLine($"Version {buildVersion} missing packages for runtime {runtime.Key}"); + } + } + + Console.WriteLine($"Build version {buildVersion} is missing some runtime packages so not pushing this drop."); + return !missingRuntime; + } + + private static void DownloadPackagesForPush(string pathToDownload) + { + AzurePublisherTool.DownloadFiles(pathToDownload, ".nupkg", Dirs.PackagesNoRID); + + string pushedSemaphore = Path.Combine(Dirs.PackagesNoRID, PackagePushedSemaphoreFileName); + File.WriteAllText(pushedSemaphore, $"Packages pushed for build {pathToDownload}"); + AzurePublisherTool.PublishFile(pathToDownload + "/" + PackagePushedSemaphoreFileName, pushedSemaphore); } } }