diff --git a/scripts/dotnet-cli-build/CompileTargets.cs b/scripts/dotnet-cli-build/CompileTargets.cs index 7c6c80bb4..f78b0a2a8 100644 --- a/scripts/dotnet-cli-build/CompileTargets.cs +++ b/scripts/dotnet-cli-build/CompileTargets.cs @@ -7,6 +7,8 @@ using Microsoft.Extensions.PlatformAbstractions; using static Microsoft.DotNet.Cli.Build.FS; using static Microsoft.DotNet.Cli.Build.Framework.BuildHelpers; +using System.Text.RegularExpressions; +using System.Reflection.PortableExecutable; namespace Microsoft.DotNet.Cli.Build { @@ -45,6 +47,12 @@ namespace Microsoft.DotNet.Cli.Build "Microsoft.Extensions.Testing.Abstractions" }; + public const string SharedFrameworkName = "Microsoft.NETCore.App"; + + private static string CoreHostBaseName => $"corehost{Constants.ExeSuffix}"; + private static string DotnetHostFxrBaseName => $"{Constants.DynamicLibPrefix}hostfxr{Constants.DynamicLibSuffix}"; + private static string HostPolicyBaseName => $"{Constants.DynamicLibPrefix}hostpolicy{Constants.DynamicLibSuffix}"; + // Updates the stage 2 with recent changes. [Target(nameof(PrepareTargets.Init), nameof(CompileStage2))] public static BuildTargetResult UpdateBuild(BuildTargetContext c) @@ -118,9 +126,9 @@ namespace Microsoft.DotNet.Cli.Build rid); // Copy the output out - File.Copy(Path.Combine(cmakeOut, "cli", "corehost"), Path.Combine(Dirs.Corehost, "corehost"), overwrite: true); - File.Copy(Path.Combine(cmakeOut, "cli", "dll", $"{Constants.DynamicLibPrefix}hostpolicy{Constants.DynamicLibSuffix}"), Path.Combine(Dirs.Corehost, $"{Constants.DynamicLibPrefix}hostpolicy{Constants.DynamicLibSuffix}"), overwrite: true); - File.Copy(Path.Combine(cmakeOut, "cli", "fxr", $"{Constants.DynamicLibPrefix}hostfxr{Constants.DynamicLibSuffix}"), Path.Combine(Dirs.Corehost, $"{Constants.DynamicLibPrefix}hostfxr{Constants.DynamicLibSuffix}"), overwrite: true); + File.Copy(Path.Combine(cmakeOut, "cli", "corehost"), Path.Combine(Dirs.Corehost, CoreHostBaseName), overwrite: true); + File.Copy(Path.Combine(cmakeOut, "cli", "dll", DotnetHostFxrBaseName), Path.Combine(Dirs.Corehost, DotnetHostFxrBaseName), overwrite: true); + File.Copy(Path.Combine(cmakeOut, "cli", "fxr", HostPolicyBaseName), Path.Combine(Dirs.Corehost, HostPolicyBaseName), overwrite: true); } return c.Success(); @@ -131,7 +139,16 @@ namespace Microsoft.DotNet.Cli.Build { CleanBinObj(c, Path.Combine(c.BuildContext.BuildDirectory, "src")); CleanBinObj(c, Path.Combine(c.BuildContext.BuildDirectory, "test")); - return CompileStage(c, + + if (Directory.Exists(Dirs.Stage1)) + { + Utils.DeleteDirectory(Dirs.Stage1); + } + Directory.CreateDirectory(Dirs.Stage1); + + CopySharedHost(Dirs.Stage1); + PackageSharedFramework(c, Dirs.Stage1, DotNetCli.Stage0); + return CompileCliSdk(c, dotnet: DotNetCli.Stage0, outputDir: Dirs.Stage1); } @@ -143,7 +160,16 @@ namespace Microsoft.DotNet.Cli.Build CleanBinObj(c, Path.Combine(c.BuildContext.BuildDirectory, "src")); CleanBinObj(c, Path.Combine(c.BuildContext.BuildDirectory, "test")); - var result = CompileStage(c, + + if (Directory.Exists(Dirs.Stage2)) + { + Utils.DeleteDirectory(Dirs.Stage2); + } + Directory.CreateDirectory(Dirs.Stage2); + + PackageSharedFramework(c, Dirs.Stage2, DotNetCli.Stage1); + CopySharedHost(Dirs.Stage2); + var result = CompileCliSdk(c, dotnet: DotNetCli.Stage1, outputDir: Dirs.Stage2); @@ -174,15 +200,134 @@ namespace Microsoft.DotNet.Cli.Build return c.Success(); } - private static BuildTargetResult CompileStage(BuildTargetContext c, DotNetCli dotnet, string outputDir) + private static void CopySharedHost(string outputDir) { - Rmdir(outputDir); + // corehost will be renamed to dotnet at some point and then this can be removed. + File.Copy( + Path.Combine(Dirs.Corehost, CoreHostBaseName), + Path.Combine(outputDir, $"dotnet{Constants.ExeSuffix}"), true); + File.Copy( + Path.Combine(Dirs.Corehost, DotnetHostFxrBaseName), + Path.Combine(outputDir, DotnetHostFxrBaseName), true); + } + public static void PackageSharedFramework(BuildTargetContext c, string outputDir, DotNetCli dotnetCli) + { + string SharedFrameworkSourceRoot = Path.Combine(Dirs.RepoRoot, "src", "sharedframework", "framework"); + string SharedFrameworkNugetVersion = GetVersionFromProjectJson(Path.Combine(Path.Combine(Dirs.RepoRoot, "src", "sharedframework", "framework"), "project.json")); + + // We publish to a sub folder of the PublishRoot so tools like heat and zip can generate folder structures easier. + string SharedFrameworkNameAndVersionRoot = Path.Combine(outputDir, "shared", SharedFrameworkName, SharedFrameworkNugetVersion); + + if (Directory.Exists(SharedFrameworkNameAndVersionRoot)) + { + Utils.DeleteDirectory(SharedFrameworkNameAndVersionRoot); + } + + string publishFramework = "dnxcore50"; // Temporary, use "netcoreapp" when we update nuget. + string publishRuntime; + if (PlatformServices.Default.Runtime.OperatingSystemPlatform == Platform.Windows) + { + publishRuntime = $"win7-{PlatformServices.Default.Runtime.RuntimeArchitecture}"; + } + else + { + publishRuntime = PlatformServices.Default.Runtime.GetRuntimeIdentifier(); + } + + dotnetCli.Publish( + "--output", SharedFrameworkNameAndVersionRoot, + "-r", publishRuntime, + "-f", publishFramework, + SharedFrameworkSourceRoot).Execute().EnsureSuccessful(); + + // Clean up artifacts that dotnet-publish generates which we don't need + File.Delete(Path.Combine(SharedFrameworkNameAndVersionRoot, $"framework{Constants.ExeSuffix}")); + File.Delete(Path.Combine(SharedFrameworkNameAndVersionRoot, "framework.dll")); + File.Delete(Path.Combine(SharedFrameworkNameAndVersionRoot, "framework.pdb")); + File.Delete(Path.Combine(SharedFrameworkNameAndVersionRoot, "framework.runtimeconfig.json")); + + // Rename the .deps file + var destinationDeps = Path.Combine(SharedFrameworkNameAndVersionRoot, $"{SharedFrameworkName}.deps.json"); + File.Move(Path.Combine(SharedFrameworkNameAndVersionRoot, "framework.deps"), Path.Combine(SharedFrameworkNameAndVersionRoot, $"{SharedFrameworkName}.deps")); + File.Move(Path.Combine(SharedFrameworkNameAndVersionRoot, "framework.deps.json"), destinationDeps); + + // Generate RID fallback graph + string runtimeGraphGeneratorRuntime = null; + switch (PlatformServices.Default.Runtime.OperatingSystemPlatform) + { + case Platform.Windows: + runtimeGraphGeneratorRuntime = "win"; + break; + case Platform.Linux: + runtimeGraphGeneratorRuntime = "linux"; + break; + case Platform.Darwin: + runtimeGraphGeneratorRuntime = "osx"; + break; + } + if (!string.IsNullOrEmpty(runtimeGraphGeneratorRuntime)) + { + var runtimeGraphGeneratorName = "RuntimeGraphGenerator"; + var runtimeGraphGeneratorProject = Path.Combine(Dirs.RepoRoot, "tools", runtimeGraphGeneratorName); + var runtimeGraphGeneratorOutput = Path.Combine(Dirs.Output, "tools", runtimeGraphGeneratorName); + + dotnetCli.Publish( + "--output", runtimeGraphGeneratorOutput, + runtimeGraphGeneratorProject).Execute().EnsureSuccessful(); + var runtimeGraphGeneratorExe = Path.Combine(runtimeGraphGeneratorOutput, $"{runtimeGraphGeneratorName}{Constants.ExeSuffix}"); + + Cmd(runtimeGraphGeneratorExe, "--project", SharedFrameworkSourceRoot, "--deps", destinationDeps, runtimeGraphGeneratorRuntime) + .Execute(); + } + else + { + c.Error($"Could not determine rid graph generation runtime for platform {PlatformServices.Default.Runtime.OperatingSystemPlatform}"); + } + + // corehost will be renamed to dotnet at some point and then we will not need to rename it here. + File.Copy( + Path.Combine(Dirs.Corehost, CoreHostBaseName), + Path.Combine(SharedFrameworkNameAndVersionRoot, $"dotnet{Constants.ExeSuffix}")); + File.Copy( + Path.Combine(Dirs.Corehost, HostPolicyBaseName), + Path.Combine(SharedFrameworkNameAndVersionRoot, HostPolicyBaseName), true); + + if (File.Exists(Path.Combine(SharedFrameworkNameAndVersionRoot, "mscorlib.ni.dll"))) + { + // Publish already places the crossgen'd version of mscorlib into the output, so we can + // remove the IL version + File.Delete(Path.Combine(SharedFrameworkNameAndVersionRoot, "mscorlib.dll")); + } + + CrossgenSharedFx(c, SharedFrameworkNameAndVersionRoot); + } + + private static string GetVersionFromProjectJson(string pathToProjectJson) + { + Regex r = new Regex($"\"{Regex.Escape(SharedFrameworkName)}\"\\s*:\\s*\"(?'version'[^\"]*)\""); + + foreach (var line in File.ReadAllLines(pathToProjectJson)) + { + var m = r.Match(line); + + if (m.Success) + { + return m.Groups["version"].Value; + } + } + + throw new InvalidOperationException("Unable to match the version name from " + pathToProjectJson); + } + + private static BuildTargetResult CompileCliSdk(BuildTargetContext c, DotNetCli dotnet, string outputDir) + { var configuration = c.BuildContext.Get("Configuration"); - var binDir = Path.Combine(outputDir, "bin"); - var buildVesion = c.BuildContext.Get("BuildVersion"); + var buildVersion = c.BuildContext.Get("BuildVersion"); + outputDir = Path.Combine(outputDir, "sdk", buildVersion.NuGetVersion); - Mkdirp(binDir); + Rmdir(outputDir); + Mkdirp(outputDir); foreach (var project in ProjectsToPublish) { @@ -191,11 +336,11 @@ namespace Microsoft.DotNet.Cli.Build dotnet.Publish( "--native-subdirectory", "--output", - binDir, + outputDir, "--configuration", configuration, Path.Combine(c.BuildContext.BuildDirectory, "src", project)) - .Environment("DOTNET_BUILD_VERSION", buildVesion.VersionSuffix) + .Environment("DOTNET_BUILD_VERSION", buildVersion.VersionSuffix) .Execute() .EnsureSuccessful(); } @@ -203,9 +348,9 @@ namespace Microsoft.DotNet.Cli.Build FixModeFlags(outputDir); // Copy corehost - File.Copy(Path.Combine(Dirs.Corehost, $"corehost{Constants.ExeSuffix}"), Path.Combine(binDir, $"corehost{Constants.ExeSuffix}"), overwrite: true); - File.Copy(Path.Combine(Dirs.Corehost, $"{Constants.DynamicLibPrefix}hostpolicy{Constants.DynamicLibSuffix}"), Path.Combine(binDir, $"{Constants.DynamicLibPrefix}hostpolicy{Constants.DynamicLibSuffix}"), overwrite: true); - File.Copy(Path.Combine(Dirs.Corehost, $"{Constants.DynamicLibPrefix}hostfxr{Constants.DynamicLibSuffix}"), Path.Combine(binDir, $"{Constants.DynamicLibPrefix}hostfxr{Constants.DynamicLibSuffix}"), overwrite: true); + File.Copy(Path.Combine(Dirs.Corehost, $"corehost{Constants.ExeSuffix}"), Path.Combine(outputDir, $"corehost{Constants.ExeSuffix}"), overwrite: true); + File.Copy(Path.Combine(Dirs.Corehost, $"{Constants.DynamicLibPrefix}hostpolicy{Constants.DynamicLibSuffix}"), Path.Combine(outputDir, $"{Constants.DynamicLibPrefix}hostpolicy{Constants.DynamicLibSuffix}"), overwrite: true); + File.Copy(Path.Combine(Dirs.Corehost, $"{Constants.DynamicLibPrefix}hostfxr{Constants.DynamicLibSuffix}"), Path.Combine(outputDir, $"{Constants.DynamicLibPrefix}hostfxr{Constants.DynamicLibSuffix}"), overwrite: true); // Corehostify binaries foreach (var binaryToCorehostify in BinariesForCoreHost) @@ -213,9 +358,9 @@ namespace Microsoft.DotNet.Cli.Build try { // Yes, it is .exe even on Linux. This is the managed exe we're working with - File.Copy(Path.Combine(binDir, $"{binaryToCorehostify}.exe"), Path.Combine(binDir, $"{binaryToCorehostify}.dll")); - File.Delete(Path.Combine(binDir, $"{binaryToCorehostify}.exe")); - File.Copy(Path.Combine(binDir, $"corehost{Constants.ExeSuffix}"), Path.Combine(binDir, binaryToCorehostify + Constants.ExeSuffix)); + File.Copy(Path.Combine(outputDir, $"{binaryToCorehostify}.exe"), Path.Combine(outputDir, $"{binaryToCorehostify}.dll")); + File.Delete(Path.Combine(outputDir, $"{binaryToCorehostify}.exe")); + File.Copy(Path.Combine(outputDir, $"corehost{Constants.ExeSuffix}"), Path.Combine(outputDir, binaryToCorehostify + Constants.ExeSuffix)); } catch (Exception ex) { @@ -224,25 +369,25 @@ namespace Microsoft.DotNet.Cli.Build } // dotnet.exe is from stage0. But we must be using the newly built corehost in stage1 - File.Delete(Path.Combine(binDir, $"dotnet{Constants.ExeSuffix}")); - File.Copy(Path.Combine(binDir, $"corehost{Constants.ExeSuffix}"), Path.Combine(binDir, $"dotnet{Constants.ExeSuffix}")); + File.Delete(Path.Combine(outputDir, $"dotnet{Constants.ExeSuffix}")); + File.Copy(Path.Combine(outputDir, $"corehost{Constants.ExeSuffix}"), Path.Combine(outputDir, $"dotnet{Constants.ExeSuffix}")); // Crossgen Roslyn - var result = Crossgen(c, binDir); + var result = CrossgenCliSdk(c, outputDir); if (!result.Success) { return result; } // Copy AppDeps - result = CopyAppDeps(c, binDir); + result = CopyAppDeps(c, outputDir); if (!result.Success) { return result; } // Generate .version file - var version = buildVesion.SimpleVersion; + var version = buildVersion.SimpleVersion; var content = $@"{c.BuildContext["CommitHash"]}{Environment.NewLine}{version}{Environment.NewLine}"; File.WriteAllText(Path.Combine(outputDir, ".version"), content); @@ -296,12 +441,12 @@ namespace Microsoft.DotNet.Cli.Build return c.Success(); } - private static BuildTargetResult Crossgen(BuildTargetContext c, string outputDir) + private static BuildTargetResult CrossgenCliSdk(BuildTargetContext c, string outputDir) { // Check if we need to skip crossgen if (string.Equals(Environment.GetEnvironmentVariable("DOTNET_BUILD_SKIP_CROSSGEN"), "1")) { - c.Warn("Skipping crossgen because DOTNET_BUILD_SKIP_CROSSGEN is set"); + c.Warn("Skipping crossgen for Cli Sdk because DOTNET_BUILD_SKIP_CROSSGEN is set"); return c.Success(); } @@ -346,6 +491,58 @@ namespace Microsoft.DotNet.Cli.Build return c.Success(); } + public static BuildTargetResult CrossgenSharedFx(BuildTargetContext c, string pathToAssemblies) + { + // Check if we need to skip crossgen + if (!string.Equals(Environment.GetEnvironmentVariable("CROSSGEN_SHAREDFRAMEWORK"), "1")) + { + c.Warn("Skipping crossgen for SharedFx because CROSSGEN_SHAREDFRAMEWORK is not set to 1"); + return c.Success(); + } + + foreach (var file in Directory.GetFiles(pathToAssemblies)) + { + string fileName = Path.GetFileName(file); + + if (fileName == "mscorlib.dll" || fileName == "mscorlib.ni.dll" || !HasMetadata(file)) + { + continue; + } + + string tempPathName = Path.ChangeExtension(file, "readytorun"); + + // This is not always correct. The version of crossgen we need to pick up is whatever one was restored as part + // of the Microsoft.NETCore.Runtime.CoreCLR package that is part of the shared library. For now, the version hardcoded + // in CompileTargets and the one in the shared library project.json match and are updated in lock step, but long term + // we need to be able to look at the project.lock.json file and figure out what version of Microsoft.NETCore.Runtime.CoreCLR + // was used, and then select that version. + ExecSilent(Crossgen.GetCrossgenPathForVersion(CompileTargets.CoreCLRVersion), + "-readytorun", "-in", file, "-out", tempPathName, "-platform_assemblies_paths", pathToAssemblies); + + File.Delete(file); + File.Move(tempPathName, file); + } + + return c.Success(); + } + + private static bool HasMetadata(string pathToFile) + { + try + { + using (var inStream = File.OpenRead(pathToFile)) + { + using (var peReader = new PEReader(inStream)) + { + return peReader.HasMetadata; + } + } + } + catch (BadImageFormatException) { } + + return false; + } + private static List GetAssembliesToCrossGen() { return new List diff --git a/scripts/dotnet-cli-build/Utils/DotNetCli.cs b/scripts/dotnet-cli-build/Utils/DotNetCli.cs index a84ec89dc..01e618fac 100644 --- a/scripts/dotnet-cli-build/Utils/DotNetCli.cs +++ b/scripts/dotnet-cli-build/Utils/DotNetCli.cs @@ -7,11 +7,11 @@ using Microsoft.Extensions.PlatformAbstractions; namespace Microsoft.DotNet.Cli.Build { - internal class DotNetCli + public class DotNetCli { public static readonly DotNetCli Stage0 = new DotNetCli(GetStage0Path()); - public static readonly DotNetCli Stage1 = new DotNetCli(Path.Combine(Dirs.Stage1, "bin")); - public static readonly DotNetCli Stage2 = new DotNetCli(Path.Combine(Dirs.Stage2, "bin")); + public static readonly DotNetCli Stage1 = new DotNetCli(Dirs.Stage1); + public static readonly DotNetCli Stage2 = new DotNetCli(Dirs.Stage2); public string BinPath { get; }