From d7376f84c3af79848d9b88af713af006dd13e6af Mon Sep 17 00:00:00 2001 From: Wes Haggard Date: Thu, 7 Apr 2016 17:25:37 -0700 Subject: [PATCH 1/6] Add PullNupkgFilesFromBlob target This target pulls all the nupkgs from the matching azure storage and then copies them into the artifact\$(RID)\packages folder. --- scripts/dotnet-cli-build/PublishTargets.cs | 11 +++++++++++ .../Publishing/AzurePublisher.cs | 19 +++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/scripts/dotnet-cli-build/PublishTargets.cs b/scripts/dotnet-cli-build/PublishTargets.cs index 125456e4a..7b7d24f8d 100644 --- a/scripts/dotnet-cli-build/PublishTargets.cs +++ b/scripts/dotnet-cli-build/PublishTargets.cs @@ -322,5 +322,16 @@ namespace Microsoft.DotNet.Cli.Build } return c.Success(); } + + [Target(nameof(PrepareTargets.Init), nameof(InitPublish))] + public static BuildTargetResult PullNupkgFilesFromBlob(BuildTargetContext c) + { + var hostBlob = $"{Channel}/Binaries/{CliNuGetVersion}/"; + + Directory.CreateDirectory(Dirs.Packages); + AzurePublisherTool.DownloadFiles(hostBlob, ".nupkg", Dirs.Packages); + + return c.Success(); + } } } diff --git a/scripts/dotnet-cli-build/Publishing/AzurePublisher.cs b/scripts/dotnet-cli-build/Publishing/AzurePublisher.cs index 0ac9834d1..d0e2b7c5b 100644 --- a/scripts/dotnet-cli-build/Publishing/AzurePublisher.cs +++ b/scripts/dotnet-cli-build/Publishing/AzurePublisher.cs @@ -1,5 +1,6 @@ using System; using System.IO; +using System.Linq; using System.Net.Http; using System.Text; using Microsoft.DotNet.Cli.Build.Framework; @@ -92,5 +93,23 @@ namespace Microsoft.DotNet.Cli.Build { return $"{channel}/Binaries/{version}/{Path.GetFileName(archiveFile)}"; } + + public async void DownloadFiles(string blobVirtualDirectory, string fileExtension, string downloadPath) + { + CloudBlobDirectory blobDir = _blobContainer.GetDirectoryReference(blobVirtualDirectory); + BlobContinuationToken continuationToken = new BlobContinuationToken(); + + var blobFiles = blobDir.ListBlobsSegmentedAsync(continuationToken).Result; + + foreach (var blobFile in blobFiles.Results.OfType()) + { + if (Path.GetExtension(blobFile.Uri.AbsoluteUri) == fileExtension) + { + string localBlobFile = Path.Combine(downloadPath, Path.GetFileName(blobFile.Uri.AbsoluteUri)); + Console.WriteLine($"Downloading {blobFile.Uri.AbsoluteUri} to {localBlobFile}..."); + blobFile.DownloadToFileAsync(localBlobFile, FileMode.Create).Wait(); + } + } + } } } From 0b913968b67bb4c2b4f05766b6839193bd3a0ae7 Mon Sep 17 00:00:00 2001 From: Wes Haggard Date: Fri, 8 Apr 2016 08:57:24 -0700 Subject: [PATCH 2/6] Update PullNupkgFilesFromBlob to find the latest Azure blob drop that contains all the runtime packages --- scripts/dotnet-cli-build/PublishTargets.cs | 83 ++++++++++++++++++- .../Publishing/AzurePublisher.cs | 13 ++- scripts/dotnet-cli-build/Utils/Dirs.cs | 1 + 3 files changed, 93 insertions(+), 4 deletions(-) diff --git a/scripts/dotnet-cli-build/PublishTargets.cs b/scripts/dotnet-cli-build/PublishTargets.cs index 7b7d24f8d..56267603c 100644 --- a/scripts/dotnet-cli-build/PublishTargets.cs +++ b/scripts/dotnet-cli-build/PublishTargets.cs @@ -1,7 +1,9 @@ using System; +using System.Collections.Generic; using System.IO; using System.Net.Http; using System.Text; +using System.Text.RegularExpressions; using Microsoft.DotNet.Cli.Build.Framework; using Microsoft.WindowsAzure.Storage; using Microsoft.WindowsAzure.Storage.Blob; @@ -326,12 +328,87 @@ namespace Microsoft.DotNet.Cli.Build [Target(nameof(PrepareTargets.Init), nameof(InitPublish))] public static BuildTargetResult PullNupkgFilesFromBlob(BuildTargetContext c) { - var hostBlob = $"{Channel}/Binaries/{CliNuGetVersion}/"; + var pathToPublish = Environment.GetEnvironmentVariable("BLOB_VIRTUAL_PATH_TO_PUBLISH"); + if (string.IsNullOrEmpty(pathToPublish)) + pathToPublish = FindLatestCompleteBuild(); - Directory.CreateDirectory(Dirs.Packages); - AzurePublisherTool.DownloadFiles(hostBlob, ".nupkg", Dirs.Packages); + if (string.IsNullOrEmpty(pathToPublish)) + { + Console.WriteLine("Didn't find a directory with all the necessary runtime packages!"); + return c.Failed(); + } + + Directory.CreateDirectory(Dirs.PackagesNoRID); + AzurePublisherTool.DownloadFiles(pathToPublish, ".nupkg", Dirs.PackagesNoRID); return c.Success(); } + + private static string FindLatestCompleteBuild() + { + var hostBlob = $"{Channel}/Binaries/"; + + Regex buildVersionRegex = new Regex(@"Binaries/(?\d+\.\d+\.\d+-[^-]+-\d{6})/$"); + + List buildVersions = new List(); + foreach (string file in AzurePublisherTool.ListBlobs(hostBlob)) + { + var match = buildVersionRegex.Match(file); + if (match.Success) + { + buildVersions.Add(match.Groups["version"].Value); + } + } + + buildVersions.Sort(); + buildVersions.Reverse(); + + Dictionary runtimes = new Dictionary() + { + {"win7", false }, + {"osx.10.10", false }, + {"rhel.7", false }, + {"ubuntu.14.04", false }, + + }; + + foreach (var bv in buildVersions) + { + Console.WriteLine($"Version: {bv}"); + + var buildFiles = AzurePublisherTool.ListBlobs(hostBlob + bv); + + foreach (var bf in buildFiles) + { + string buildFile = Path.GetFileName(bf); + + foreach (var runtime in runtimes.Keys) + { + if (buildFile.StartsWith($"runtime.{runtime}")) + { + runtimes[runtime] = true; + break; + } + } + } + + bool missingRuntime = false; + foreach (var runtime in runtimes) + { + if (!runtime.Value) + { + missingRuntime = true; + Console.WriteLine($"Version {bv} is missing packages for runtime {runtime.Key}"); + } + } + + if (!missingRuntime) + { + return hostBlob + bv; + } + } + + return null; + } } } diff --git a/scripts/dotnet-cli-build/Publishing/AzurePublisher.cs b/scripts/dotnet-cli-build/Publishing/AzurePublisher.cs index d0e2b7c5b..e5b6a86e9 100644 --- a/scripts/dotnet-cli-build/Publishing/AzurePublisher.cs +++ b/scripts/dotnet-cli-build/Publishing/AzurePublisher.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.IO; using System.Linq; using System.Net.Http; @@ -94,7 +95,7 @@ namespace Microsoft.DotNet.Cli.Build return $"{channel}/Binaries/{version}/{Path.GetFileName(archiveFile)}"; } - public async void DownloadFiles(string blobVirtualDirectory, string fileExtension, string downloadPath) + public void DownloadFiles(string blobVirtualDirectory, string fileExtension, string downloadPath) { CloudBlobDirectory blobDir = _blobContainer.GetDirectoryReference(blobVirtualDirectory); BlobContinuationToken continuationToken = new BlobContinuationToken(); @@ -111,5 +112,15 @@ namespace Microsoft.DotNet.Cli.Build } } } + + public IEnumerable ListBlobs(string blobVirtualDirectory) + { + CloudBlobDirectory blobDir = _blobContainer.GetDirectoryReference(blobVirtualDirectory); + BlobContinuationToken continuationToken = new BlobContinuationToken(); + + var blobFiles = blobDir.ListBlobsSegmentedAsync(continuationToken).Result; + + return blobFiles.Results.Select(bf => bf.Uri.AbsoluteUri); + } } } diff --git a/scripts/dotnet-cli-build/Utils/Dirs.cs b/scripts/dotnet-cli-build/Utils/Dirs.cs index cec5db1cb..2b6e89d19 100644 --- a/scripts/dotnet-cli-build/Utils/Dirs.cs +++ b/scripts/dotnet-cli-build/Utils/Dirs.cs @@ -14,6 +14,7 @@ namespace Microsoft.DotNet.Cli.Build PlatformServices.Default.Runtime.GetRuntimeIdentifier()); public static readonly string PackagesIntermediate = Path.Combine(Output, "packages/intermediate"); + public static readonly string PackagesNoRID = Path.Combine(RepoRoot, "artifacts", "packages"); public static readonly string Packages = Path.Combine(Output, "packages"); public static readonly string Stage1 = Path.Combine(Output, "stage1"); public static readonly string Stage1Compilation = Path.Combine(Output, "stage1compilation"); From f5466be4a7883d8e8cc2c80bb3841d079e8a7af5 Mon Sep 17 00:00:00 2001 From: Wes Haggard Date: Fri, 8 Apr 2016 21:36:32 -0700 Subject: [PATCH 3/6] Update PullNupkgFilesFromBlob target Add support to enable polling to watch for new builds. It will try to publish the last 10 builds by default. If the build has all the necessary runtime packages and doesn't have the packages.pushed semaphore file. Also add support for forcing the publish of a given build set in the following environment variable FORCE_PUBLISH_BLOB_BUILD_VERSION. This will blindly try to pull all the nupkgs for the given build ignoring if they were already published or have all the runtimes specified. --- scripts/dotnet-cli-build/PublishTargets.cs | 135 ++++++++++++--------- 1 file changed, 79 insertions(+), 56 deletions(-) diff --git a/scripts/dotnet-cli-build/PublishTargets.cs b/scripts/dotnet-cli-build/PublishTargets.cs index 56267603c..808d9b692 100644 --- a/scripts/dotnet-cli-build/PublishTargets.cs +++ b/scripts/dotnet-cli-build/PublishTargets.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Net.Http; using System.Text; using System.Text.RegularExpressions; @@ -325,32 +326,28 @@ namespace Microsoft.DotNet.Cli.Build return c.Success(); } + private const string PackagePushedSemaphoreFileName = "packages.pushed"; + [Target(nameof(PrepareTargets.Init), nameof(InitPublish))] public static BuildTargetResult PullNupkgFilesFromBlob(BuildTargetContext c) { - var pathToPublish = Environment.GetEnvironmentVariable("BLOB_VIRTUAL_PATH_TO_PUBLISH"); - if (string.IsNullOrEmpty(pathToPublish)) - pathToPublish = FindLatestCompleteBuild(); + Directory.CreateDirectory(Dirs.PackagesNoRID); - if (string.IsNullOrEmpty(pathToPublish)) + var hostBlob = $"{Channel}/Binaries/"; + + string forcePushBuild = Environment.GetEnvironmentVariable("FORCE_PUBLISH_BLOB_BUILD_VERSION"); + + if (!string.IsNullOrEmpty(forcePushBuild)) { - Console.WriteLine("Didn't find a directory with all the necessary runtime packages!"); - return c.Failed(); + Console.WriteLine($"Forcing all nupkg packages for build version {forcePushBuild}."); + DownloadPackagesForPush(hostBlob + forcePushBuild); + return c.Success(); } - Directory.CreateDirectory(Dirs.PackagesNoRID); - AzurePublisherTool.DownloadFiles(pathToPublish, ".nupkg", Dirs.PackagesNoRID); - - return c.Success(); - } - - private static string FindLatestCompleteBuild() - { - var hostBlob = $"{Channel}/Binaries/"; + List buildVersions = new List(); Regex buildVersionRegex = new Regex(@"Binaries/(?\d+\.\d+\.\d+-[^-]+-\d{6})/$"); - List buildVersions = new List(); foreach (string file in AzurePublisherTool.ListBlobs(hostBlob)) { var match = buildVersionRegex.Match(file); @@ -360,55 +357,81 @@ namespace Microsoft.DotNet.Cli.Build } } + // Sort decending buildVersions.Sort(); buildVersions.Reverse(); - Dictionary runtimes = new Dictionary() + // Try to publish the last 10 builds + foreach (var bv in buildVersions.Take(10)) { - {"win7", false }, - {"osx.10.10", false }, - {"rhel.7", false }, - {"ubuntu.14.04", false }, + Console.WriteLine($"Checking drop version: {bv}"); - }; - - foreach (var bv in buildVersions) - { - Console.WriteLine($"Version: {bv}"); - - var buildFiles = AzurePublisherTool.ListBlobs(hostBlob + bv); - - foreach (var bf in buildFiles) + if (ShouldDownloadAndPush(hostBlob, bv)) { - string buildFile = Path.GetFileName(bf); - - foreach (var runtime in runtimes.Keys) - { - if (buildFile.StartsWith($"runtime.{runtime}")) - { - runtimes[runtime] = true; - break; - } - } - } - - bool missingRuntime = false; - foreach (var runtime in runtimes) - { - if (!runtime.Value) - { - missingRuntime = true; - Console.WriteLine($"Version {bv} is missing packages for runtime {runtime.Key}"); - } - } - - if (!missingRuntime) - { - return hostBlob + bv; + DownloadPackagesForPush(hostBlob + bv); } } - return null; + return c.Success(); + } + + private static bool ShouldDownloadAndPush(string hostBlob, string buildVersion) + { + // Set of runtime ids to look for to act as the signal that the build + // as finished each of these legs of the build. + Dictionary runtimes = new Dictionary() + { + {"win7-x64", false }, + {"win7-x86", false }, + {"osx.10.10-x64", false }, + {"rhel.7-x64", false }, + {"ubuntu.14.04-x64", false }, + }; + + var buildFiles = AzurePublisherTool.ListBlobs(hostBlob + buildVersion); + + foreach (var bf in buildFiles) + { + string buildFile = Path.GetFileName(bf); + + if (buildFile == PackagePushedSemaphoreFileName) + { + Console.WriteLine($"Found '{PackagePushedSemaphoreFileName}' for build version {buildVersion} so skipping this drop."); + // Nothing to do because the latest build is uploaded. + return false; + } + + foreach (var runtime in runtimes.Keys) + { + if (buildFile.StartsWith($"runtime.{runtime}")) + { + runtimes[runtime] = true; + break; + } + } + } + + bool missingRuntime = false; + foreach (var runtime in runtimes) + { + if (!runtime.Value) + { + missingRuntime = true; + Console.WriteLine($"Version {buildVersion} missing packages for runtime {runtime.Key}"); + } + } + + Console.WriteLine($"Build version {buildVersion} is missing some runtime packages so not pushing this drop."); + return !missingRuntime; + } + + private static void DownloadPackagesForPush(string pathToDownload) + { + AzurePublisherTool.DownloadFiles(pathToDownload, ".nupkg", Dirs.PackagesNoRID); + + string pushedSemaphore = Path.Combine(Dirs.PackagesNoRID, PackagePushedSemaphoreFileName); + File.WriteAllText(pushedSemaphore, $"Packages pushed for build {pathToDownload}"); + AzurePublisherTool.PublishFile(pathToDownload + "/" + PackagePushedSemaphoreFileName, pushedSemaphore); } } } From 0b77c8730c04ae960e8681c2fc2796cc47ae8263 Mon Sep 17 00:00:00 2001 From: Wes Haggard Date: Fri, 8 Apr 2016 21:44:36 -0700 Subject: [PATCH 4/6] Fix condition on message about missing runtime packages --- scripts/dotnet-cli-build/PublishTargets.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/dotnet-cli-build/PublishTargets.cs b/scripts/dotnet-cli-build/PublishTargets.cs index 808d9b692..b6caecd38 100644 --- a/scripts/dotnet-cli-build/PublishTargets.cs +++ b/scripts/dotnet-cli-build/PublishTargets.cs @@ -421,7 +421,9 @@ namespace Microsoft.DotNet.Cli.Build } } - Console.WriteLine($"Build version {buildVersion} is missing some runtime packages so not pushing this drop."); + if (missingRuntime) + Console.WriteLine($"Build version {buildVersion} is missing some runtime packages so not pushing this drop."); + return !missingRuntime; } From 0c1db5d3e3fc6c655ae7a781e5920193762d83bc Mon Sep 17 00:00:00 2001 From: MichaelSimons Date: Fri, 15 Apr 2016 10:05:56 -0500 Subject: [PATCH 5/6] Updating package publish with Debian --- scripts/dotnet-cli-build/PublishTargets.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/dotnet-cli-build/PublishTargets.cs b/scripts/dotnet-cli-build/PublishTargets.cs index b6caecd38..5e860de0e 100644 --- a/scripts/dotnet-cli-build/PublishTargets.cs +++ b/scripts/dotnet-cli-build/PublishTargets.cs @@ -386,6 +386,7 @@ namespace Microsoft.DotNet.Cli.Build {"osx.10.10-x64", false }, {"rhel.7-x64", false }, {"ubuntu.14.04-x64", false }, + {"debian.8-x64", false }, }; var buildFiles = AzurePublisherTool.ListBlobs(hostBlob + buildVersion); From c867787675c833a4852b25d926ee38dc75dd2459 Mon Sep 17 00:00:00 2001 From: Eric Erhardt Date: Sat, 7 May 2016 14:45:23 -0500 Subject: [PATCH 6/6] Fixing the PullNupkgFilesFromBlob script to look for the corehost and metapackage nupkgs under "preview1" instead of finding the old "rc2" packages. --- scripts/dotnet-cli-build/PublishTargets.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/dotnet-cli-build/PublishTargets.cs b/scripts/dotnet-cli-build/PublishTargets.cs index 5e860de0e..92f1c62bd 100644 --- a/scripts/dotnet-cli-build/PublishTargets.cs +++ b/scripts/dotnet-cli-build/PublishTargets.cs @@ -346,7 +346,10 @@ namespace Microsoft.DotNet.Cli.Build List buildVersions = new List(); - Regex buildVersionRegex = new Regex(@"Binaries/(?\d+\.\d+\.\d+-[^-]+-\d{6})/$"); + // The corehost packages are published to the CLI version, which is "preview1-xxxxxx" right now. + // But there are -rc2-xxxxxx blobs on Azure, which breaks the sort descending. So only look for + // -p(.*)-xxxxxx for now to ignore those rc2 packages. + Regex buildVersionRegex = new Regex(@"Binaries/(?\d+\.\d+\.\d+-p[^-]+-\d{6})/$"); foreach (string file in AzurePublisherTool.ListBlobs(hostBlob)) {