diff --git a/scripts/dotnet-cli-build/PublishTargets.cs b/scripts/dotnet-cli-build/PublishTargets.cs index 93e21eee4..0831f4e81 100644 --- a/scripts/dotnet-cli-build/PublishTargets.cs +++ b/scripts/dotnet-cli-build/PublishTargets.cs @@ -4,6 +4,7 @@ using System.IO; using System.Linq; using System.Net.Http; using System.Text; +using System.Text.RegularExpressions; using Microsoft.DotNet.Cli.Build.Framework; using Microsoft.WindowsAzure.Storage; using Microsoft.WindowsAzure.Storage.Blob; @@ -436,5 +437,119 @@ namespace Microsoft.DotNet.Cli.Build } return c.Success(); } + + private const string PackagePushedSemaphoreFileName = "packages.pushed"; + + [Target(nameof(PrepareTargets.Init), nameof(InitPublish))] + public static BuildTargetResult PullNupkgFilesFromBlob(BuildTargetContext c) + { + Directory.CreateDirectory(Dirs.PackagesNoRID); + + var hostBlob = $"{Channel}/Binaries/"; + + string forcePushBuild = Environment.GetEnvironmentVariable("FORCE_PUBLISH_BLOB_BUILD_VERSION"); + + if (!string.IsNullOrEmpty(forcePushBuild)) + { + Console.WriteLine($"Forcing all nupkg packages for build version {forcePushBuild}."); + DownloadPackagesForPush(hostBlob + forcePushBuild); + return c.Success(); + } + + List buildVersions = new List(); + + // The corehost packages are published to the CLI version, which is "preview1-xxxxxx" right now. + // But there are -rc2-xxxxxx blobs on Azure, which breaks the sort descending. So only look for + // -p(.*)-xxxxxx for now to ignore those rc2 packages. + Regex buildVersionRegex = new Regex(@"Binaries/(?\d+\.\d+\.\d+-p[^-]+-\d{6})/$"); + + foreach (string file in AzurePublisherTool.ListBlobs(hostBlob)) + { + var match = buildVersionRegex.Match(file); + if (match.Success) + { + buildVersions.Add(match.Groups["version"].Value); + } + } + + // Sort decending + buildVersions.Sort(); + buildVersions.Reverse(); + + // Try to publish the last 10 builds + foreach (var bv in buildVersions.Take(10)) + { + Console.WriteLine($"Checking drop version: {bv}"); + + if (ShouldDownloadAndPush(hostBlob, bv)) + { + DownloadPackagesForPush(hostBlob + bv); + } + } + + return c.Success(); + } + + private static bool ShouldDownloadAndPush(string hostBlob, string buildVersion) + { + // Set of runtime ids to look for to act as the signal that the build + // as finished each of these legs of the build. + Dictionary runtimes = new Dictionary() + { + {"win7-x64", false }, + {"win7-x86", false }, + {"osx.10.10-x64", false }, + {"rhel.7-x64", false }, + {"ubuntu.14.04-x64", false }, + {"debian.8-x64", false }, + }; + + var buildFiles = AzurePublisherTool.ListBlobs(hostBlob + buildVersion); + + foreach (var bf in buildFiles) + { + string buildFile = Path.GetFileName(bf); + + if (buildFile == PackagePushedSemaphoreFileName) + { + Console.WriteLine($"Found '{PackagePushedSemaphoreFileName}' for build version {buildVersion} so skipping this drop."); + // Nothing to do because the latest build is uploaded. + return false; + } + + foreach (var runtime in runtimes.Keys) + { + if (buildFile.StartsWith($"runtime.{runtime}")) + { + runtimes[runtime] = true; + break; + } + } + } + + bool missingRuntime = false; + foreach (var runtime in runtimes) + { + if (!runtime.Value) + { + missingRuntime = true; + Console.WriteLine($"Version {buildVersion} missing packages for runtime {runtime.Key}"); + } + } + + if (missingRuntime) + Console.WriteLine($"Build version {buildVersion} is missing some runtime packages so not pushing this drop."); + + return !missingRuntime; + } + + private static void DownloadPackagesForPush(string pathToDownload) + { + AzurePublisherTool.DownloadFiles(pathToDownload, ".nupkg", Dirs.PackagesNoRID); + + string pushedSemaphore = Path.Combine(Dirs.PackagesNoRID, PackagePushedSemaphoreFileName); + File.WriteAllText(pushedSemaphore, $"Packages pushed for build {pathToDownload}"); + AzurePublisherTool.PublishFile(pathToDownload + "/" + PackagePushedSemaphoreFileName, pushedSemaphore); + } } } diff --git a/scripts/dotnet-cli-build/Publishing/AzurePublisher.cs b/scripts/dotnet-cli-build/Publishing/AzurePublisher.cs index c16eff43f..f9f781ca2 100644 --- a/scripts/dotnet-cli-build/Publishing/AzurePublisher.cs +++ b/scripts/dotnet-cli-build/Publishing/AzurePublisher.cs @@ -187,5 +187,33 @@ namespace Microsoft.DotNet.Cli.Build { return $"{channel}/Binaries/{version}/{Path.GetFileName(archiveFile)}"; } + + public void DownloadFiles(string blobVirtualDirectory, string fileExtension, string downloadPath) + { + CloudBlobDirectory blobDir = _blobContainer.GetDirectoryReference(blobVirtualDirectory); + BlobContinuationToken continuationToken = new BlobContinuationToken(); + + var blobFiles = blobDir.ListBlobsSegmentedAsync(continuationToken).Result; + + foreach (var blobFile in blobFiles.Results.OfType()) + { + if (Path.GetExtension(blobFile.Uri.AbsoluteUri) == fileExtension) + { + string localBlobFile = Path.Combine(downloadPath, Path.GetFileName(blobFile.Uri.AbsoluteUri)); + Console.WriteLine($"Downloading {blobFile.Uri.AbsoluteUri} to {localBlobFile}..."); + blobFile.DownloadToFileAsync(localBlobFile, FileMode.Create).Wait(); + } + } + } + + public IEnumerable ListBlobs(string blobVirtualDirectory) + { + CloudBlobDirectory blobDir = _blobContainer.GetDirectoryReference(blobVirtualDirectory); + BlobContinuationToken continuationToken = new BlobContinuationToken(); + + var blobFiles = blobDir.ListBlobsSegmentedAsync(continuationToken).Result; + + return blobFiles.Results.Select(bf => bf.Uri.AbsoluteUri); + } } } diff --git a/scripts/dotnet-cli-build/Utils/Dirs.cs b/scripts/dotnet-cli-build/Utils/Dirs.cs index f812887ef..0b17603f5 100644 --- a/scripts/dotnet-cli-build/Utils/Dirs.cs +++ b/scripts/dotnet-cli-build/Utils/Dirs.cs @@ -15,6 +15,7 @@ namespace Microsoft.DotNet.Cli.Build public static readonly string Intermediate = Path.Combine(Output, "intermediate"); public static readonly string PackagesIntermediate = Path.Combine(Output, "packages/intermediate"); + public static readonly string PackagesNoRID = Path.Combine(RepoRoot, "artifacts", "packages"); public static readonly string Packages = Path.Combine(Output, "packages"); public static readonly string Stage1 = Path.Combine(Output, "stage1"); public static readonly string Stage1Compilation = Path.Combine(Output, "stage1compilation");