From d6fe71d4dd1aa1d3aa8dc47f47df97364ebe0086 Mon Sep 17 00:00:00 2001 From: Sridhar Periyasamy Date: Thu, 14 Apr 2016 18:02:26 -0700 Subject: [PATCH 1/4] Enable crossgen for CLI SDK binaries. TODO: Write tests to make sure aseemblies are crossgened. --- scripts/dotnet-cli-build/CompileTargets.cs | 62 +++----------------- scripts/dotnet-cli-build/Utils/Crossgen.cs | 67 ++++++++++++++++++++-- scripts/dotnet-cli-build/Utils/PEUtils.cs | 27 +++++++++ 3 files changed, 97 insertions(+), 59 deletions(-) create mode 100644 scripts/dotnet-cli-build/Utils/PEUtils.cs diff --git a/scripts/dotnet-cli-build/CompileTargets.cs b/scripts/dotnet-cli-build/CompileTargets.cs index 60136fe78..ae3393bfa 100644 --- a/scripts/dotnet-cli-build/CompileTargets.cs +++ b/scripts/dotnet-cli-build/CompileTargets.cs @@ -37,6 +37,8 @@ namespace Microsoft.DotNet.Cli.Build public const string SharedFrameworkName = "Microsoft.NETCore.App"; + public static Crossgen CrossgenUtil = new Crossgen(CoreCLRVersion); + private static string CoreHostBaseName => $"corehost{Constants.ExeSuffix}"; private static string DotnetHostFxrBaseName => $"{Constants.DynamicLibPrefix}hostfxr{Constants.DynamicLibSuffix}"; private static string HostPolicyBaseName => $"{Constants.DynamicLibPrefix}hostpolicy{Constants.DynamicLibSuffix}"; @@ -343,7 +345,7 @@ namespace Microsoft.DotNet.Cli.Build } string SharedFrameworkSourceRoot = GenerateSharedFrameworkProject(c, SharedFrameworkTemplateSourceRoot, sharedFrameworkRid); - + dotnetCli.Restore("--verbosity", "verbose", "--disable-parallel", "--infer-runtimes", "--fallbacksource", Dirs.Corehost) .WorkingDirectory(SharedFrameworkSourceRoot) .Execute() @@ -430,7 +432,7 @@ namespace Microsoft.DotNet.Cli.Build File.Delete(Path.Combine(SharedFrameworkNameAndVersionRoot, "mscorlib.dll")); } - CrossgenSharedFx(c, SharedFrameworkNameAndVersionRoot); + CrossgenUtil.CrossgenDirectory(c, SharedFrameworkNameAndVersionRoot); // Generate .version file for sharedfx var version = SharedFrameworkNugetVersion; @@ -530,6 +532,8 @@ namespace Microsoft.DotNet.Cli.Build File.Delete(compilersDeps); File.Delete(compilersRuntimeConfig); + CrossgenUtil.CrossgenDirectory(c, outputDir); + // Generate .version file var version = buildVersion.NuGetVersion; var content = $@"{c.BuildContext["CommitHash"]}{Environment.NewLine}{version}{Environment.NewLine}"; @@ -538,41 +542,6 @@ namespace Microsoft.DotNet.Cli.Build return c.Success(); } - public static BuildTargetResult CrossgenSharedFx(BuildTargetContext c, string pathToAssemblies) - { - // Check if we need to skip crossgen - if (string.Equals(Environment.GetEnvironmentVariable("DONT_CROSSGEN_SHAREDFRAMEWORK"), "1")) - { - c.Warn("Skipping crossgen for SharedFx because DONT_CROSSGEN_SHAREDFRAMEWORK is set to 1"); - return c.Success(); - } - - foreach (var file in Directory.GetFiles(pathToAssemblies)) - { - string fileName = Path.GetFileName(file); - - if (fileName == "mscorlib.dll" || fileName == "mscorlib.ni.dll" || !HasMetadata(file)) - { - continue; - } - - string tempPathName = Path.ChangeExtension(file, "readytorun"); - - // This is not always correct. The version of crossgen we need to pick up is whatever one was restored as part - // of the Microsoft.NETCore.Runtime.CoreCLR package that is part of the shared library. For now, the version hardcoded - // in CompileTargets and the one in the shared library project.json match and are updated in lock step, but long term - // we need to be able to look at the project.lock.json file and figure out what version of Microsoft.NETCore.Runtime.CoreCLR - // was used, and then select that version. - ExecSilent(Crossgen.GetCrossgenPathForVersion(CoreCLRVersion), - "-readytorun", "-in", file, "-out", tempPathName, "-platform_assemblies_paths", pathToAssemblies); - - File.Delete(file); - File.Move(tempPathName, file); - } - - return c.Success(); - } - private static void ChangeEntryPointLibraryName(string depsFile, string newName) { JToken deps; @@ -603,7 +572,7 @@ namespace Microsoft.DotNet.Cli.Build library.Replace(new JProperty(newName + '/' + version, library.Value)); } using (var file = File.CreateText(depsFile)) - using (var writer = new JsonTextWriter(file) { Formatting = Formatting.Indented}) + using (var writer = new JsonTextWriter(file) { Formatting = Formatting.Indented }) { deps.WriteTo(writer); } @@ -615,22 +584,5 @@ namespace Microsoft.DotNet.Cli.Build File.Delete(Path.Combine(path, $"{name}.dll")); File.Delete(Path.Combine(path, $"{name}.pdb")); } - - private static bool HasMetadata(string pathToFile) - { - try - { - using (var inStream = File.OpenRead(pathToFile)) - { - using (var peReader = new PEReader(inStream)) - { - return peReader.HasMetadata; - } - } - } - catch (BadImageFormatException) { } - - return false; - } } } diff --git a/scripts/dotnet-cli-build/Utils/Crossgen.cs b/scripts/dotnet-cli-build/Utils/Crossgen.cs index 37cf836aa..f7dd9417a 100644 --- a/scripts/dotnet-cli-build/Utils/Crossgen.cs +++ b/scripts/dotnet-cli-build/Utils/Crossgen.cs @@ -1,15 +1,32 @@ +using System; +using System.Collections.Generic; using System.IO; using System.Linq; -using System; using System.Runtime.InteropServices; using Microsoft.DotNet.Cli.Build.Framework; using Microsoft.Extensions.PlatformAbstractions; +using static Microsoft.DotNet.Cli.Build.Framework.BuildHelpers; + namespace Microsoft.DotNet.Cli.Build { - internal static class Crossgen + public class Crossgen { - public static string GetCrossgenPathForVersion(string coreClrVersion) + private string _coreClrVersion; + private string _crossGenPath; + + // This is not always correct. The version of crossgen we need to pick up is whatever one was restored as part + // of the Microsoft.NETCore.Runtime.CoreCLR package that is part of the shared library. For now, the version hardcoded + // in CompileTargets and the one in the shared library project.json match and are updated in lock step, but long term + // we need to be able to look at the project.lock.json file and figure out what version of Microsoft.NETCore.Runtime.CoreCLR + // was used, and then select that version. + public Crossgen(string coreClrVersion) + { + _coreClrVersion = coreClrVersion; + _crossGenPath = GetCrossgenPathForVersion(); + } + + private string GetCrossgenPathForVersion() { string arch = PlatformServices.Default.Runtime.RuntimeArchitecture; string packageId; @@ -42,9 +59,51 @@ namespace Microsoft.DotNet.Cli.Build return Path.Combine( Dirs.NuGetPackages, packageId, - coreClrVersion, + _coreClrVersion, "tools", $"crossgen{Constants.ExeSuffix}"); } + + public void CrossgenDirectory(BuildTargetContext c, string pathToAssemblies) + { + // Check if we need to skip crossgen + if (string.Equals(Environment.GetEnvironmentVariable("DISABLE_CROSSGEN"), "1")) + { + c.Warn("Skipping crossgen for because DISABLE_CROSSGEN is set to 1"); + return; + } + + foreach (var file in Directory.GetFiles(pathToAssemblies)) + { + string fileName = Path.GetFileName(file); + + if (fileName == "mscorlib.dll" || fileName == "mscorlib.ni.dll" || !PEUtils.HasMetadata(file)) + { + continue; + } + + string tempPathName = Path.ChangeExtension(file, "readytorun"); + string sharedFxPath = c.BuildContext.Get("SharedFrameworkPath"); + + // HACK + // The input directory can be a portable FAT app (example the CLI itself). + // In that case there can be RID specific managed dependencies which are not right next to the app binary (example System.Diagnostics.TraceSource). + // We need those dependencies during crossgen. For now we just pass all subdirectories of the input directory as input to crossgen. + // The right fix - + // If the assembly has deps.json then parse the json file to get all the dependencies. Pass these dependencies as input to crossgen. + // else pass the current directory of assembly as input to crossgen. + string addtionalPaths = string.Join(";", Directory.GetDirectories(pathToAssemblies, "*", SearchOption.AllDirectories)); + + IList crossgenArgs = new List { + "-readytorun", "-in", file, "-out", tempPathName, + "-platform_assemblies_paths", $"{sharedFxPath};{pathToAssemblies};{addtionalPaths}" + }; + + ExecSilent(_crossGenPath, crossgenArgs); + + File.Delete(file); + File.Move(tempPathName, file); + } + } } } diff --git a/scripts/dotnet-cli-build/Utils/PEUtils.cs b/scripts/dotnet-cli-build/Utils/PEUtils.cs new file mode 100644 index 000000000..ad702b51a --- /dev/null +++ b/scripts/dotnet-cli-build/Utils/PEUtils.cs @@ -0,0 +1,27 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Reflection.PortableExecutable; + +namespace Microsoft.DotNet.Cli.Build +{ + public class PEUtils + { + public static bool HasMetadata(string pathToFile) + { + try + { + using (var inStream = File.OpenRead(pathToFile)) + { + using (var peReader = new PEReader(inStream)) + { + return peReader.HasMetadata; + } + } + } + catch (BadImageFormatException) { } + + return false; + } + } +} From 59f81484bb10a69469227c8d28bf348a6a1a722d Mon Sep 17 00:00:00 2001 From: Sridhar Periyasamy Date: Thu, 14 Apr 2016 18:36:33 -0700 Subject: [PATCH 2/4] Disable partial crossgen. This will crossgen all the methods but the size of aseemblies increase. --- .../BuildHelpers.cs | 25 +++++++++++++------ scripts/dotnet-cli-build/Utils/Crossgen.cs | 7 +++++- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/scripts/Microsoft.DotNet.Cli.Build.Framework/BuildHelpers.cs b/scripts/Microsoft.DotNet.Cli.Build.Framework/BuildHelpers.cs index cb7aa8ede..ee8060415 100644 --- a/scripts/Microsoft.DotNet.Cli.Build.Framework/BuildHelpers.cs +++ b/scripts/Microsoft.DotNet.Cli.Build.Framework/BuildHelpers.cs @@ -5,16 +5,17 @@ namespace Microsoft.DotNet.Cli.Build.Framework public static class BuildHelpers { public static int ExecInSilent(string workingDirectory, string command, params string[] args) => ExecInSilent(workingDirectory, command, (IEnumerable)args); - public static int ExecInSilent(string workingDirectory, string command, IEnumerable args) => ExecCore(command, args, workingDirectory, silent: true); + public static int ExecInSilent(string workingDirectory, string command, IEnumerable args) => ExecCore(command, args, workingDirectory, silent: true, env: null); public static int ExecIn(string workingDirectory, string command, params string[] args) => ExecIn(workingDirectory, command, (IEnumerable)args); - public static int ExecIn(string workingDirectory, string command, IEnumerable args) => ExecCore(command, args, workingDirectory, silent: false); + public static int ExecIn(string workingDirectory, string command, IEnumerable args) => ExecCore(command, args, workingDirectory, silent: false, env: null); public static int ExecSilent(string command, params string[] args) => ExecSilent(command, (IEnumerable)args); - public static int ExecSilent(string command, IEnumerable args) => ExecCore(command, args, workingDirectory: null, silent: true); + public static int ExecSilent(string command, IEnumerable args) => ExecSilent(command, args, env: null); + public static int ExecSilent(string command, IEnumerable args, IDictionary env) => ExecCore(command, args, workingDirectory: null, silent: true, env: null); public static int Exec(string command, params string[] args) => Exec(command, (IEnumerable)args); - public static int Exec(string command, IEnumerable args) => ExecCore(command, args, workingDirectory: null, silent: false); + public static int Exec(string command, IEnumerable args) => ExecCore(command, args, workingDirectory: null, silent: false, env: null); public static Command Cmd(string command, params string[] args) => Cmd(command, (IEnumerable)args); public static Command Cmd(string command, IEnumerable args) @@ -22,21 +23,29 @@ namespace Microsoft.DotNet.Cli.Build.Framework return Command.Create(command, args); } - internal static int ExecCore(string command, IEnumerable args, string workingDirectory, bool silent) + internal static int ExecCore(string command, IEnumerable args, string workingDirectory, bool silent, IDictionary env) { var cmd = Cmd(command, args); - if(!string.IsNullOrEmpty(workingDirectory)) + if (!string.IsNullOrEmpty(workingDirectory)) { cmd.WorkingDirectory(workingDirectory); } - if(silent) + + if (silent) { cmd.CaptureStdErr().CaptureStdOut(); } - var result = cmd.Execute(); + + if (env == null) + { + env = new Dictionary(); + } + + var result = cmd.Environment(env).Execute(); result.EnsureSuccessful(); return result.ExitCode; } + } } diff --git a/scripts/dotnet-cli-build/Utils/Crossgen.cs b/scripts/dotnet-cli-build/Utils/Crossgen.cs index f7dd9417a..3faa37e6b 100644 --- a/scripts/dotnet-cli-build/Utils/Crossgen.cs +++ b/scripts/dotnet-cli-build/Utils/Crossgen.cs @@ -99,7 +99,12 @@ namespace Microsoft.DotNet.Cli.Build "-platform_assemblies_paths", $"{sharedFxPath};{pathToAssemblies};{addtionalPaths}" }; - ExecSilent(_crossGenPath, crossgenArgs); + var env = new Dictionary() + { + // disable partial ngen + { "COMPLUS_ZapDisable", "0" } + }; + ExecSilent(_crossGenPath, crossgenArgs, env); File.Delete(file); File.Move(tempPathName, file); From afafe80084be568012b8e7ed9105ae277e9e8c0c Mon Sep 17 00:00:00 2001 From: Sridhar Periyasamy Date: Thu, 14 Apr 2016 18:54:25 -0700 Subject: [PATCH 3/4] Fix unix bug. crossgen fails on unix if there is an empty path passed as input to "-platform_assemblies_paths". --- scripts/dotnet-cli-build/Utils/Crossgen.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/dotnet-cli-build/Utils/Crossgen.cs b/scripts/dotnet-cli-build/Utils/Crossgen.cs index 3faa37e6b..9b17a0df1 100644 --- a/scripts/dotnet-cli-build/Utils/Crossgen.cs +++ b/scripts/dotnet-cli-build/Utils/Crossgen.cs @@ -92,11 +92,14 @@ namespace Microsoft.DotNet.Cli.Build // The right fix - // If the assembly has deps.json then parse the json file to get all the dependencies. Pass these dependencies as input to crossgen. // else pass the current directory of assembly as input to crossgen. - string addtionalPaths = string.Join(";", Directory.GetDirectories(pathToAssemblies, "*", SearchOption.AllDirectories)); + var addtionalPaths = Directory.GetDirectories(pathToAssemblies, "*", SearchOption.AllDirectories).ToList(); + var paths = new List() { sharedFxPath, pathToAssemblies }; + paths.AddRange(addtionalPaths); + var platformAssembliesPaths = string.Join(";", paths.Distinct()); IList crossgenArgs = new List { "-readytorun", "-in", file, "-out", tempPathName, - "-platform_assemblies_paths", $"{sharedFxPath};{pathToAssemblies};{addtionalPaths}" + "-platform_assemblies_paths", platformAssembliesPaths }; var env = new Dictionary() From 16dcd770f9600ddf3c04006d5f36929381b93f23 Mon Sep 17 00:00:00 2001 From: Sridhar Periyasamy Date: Fri, 15 Apr 2016 10:31:17 -0700 Subject: [PATCH 4/4] Address PR feedback - Use the right path separator char for unix. - Contruct some of the args to crossgen outside the loop. - Add the null check inside Command.Environment. --- .../BuildHelpers.cs | 5 --- .../Command.cs | 5 +++ scripts/dotnet-cli-build/Utils/Crossgen.cs | 38 ++++++++++--------- scripts/dotnet-cli-build/Utils/PEUtils.cs | 2 +- 4 files changed, 26 insertions(+), 24 deletions(-) diff --git a/scripts/Microsoft.DotNet.Cli.Build.Framework/BuildHelpers.cs b/scripts/Microsoft.DotNet.Cli.Build.Framework/BuildHelpers.cs index ee8060415..85b71bf4b 100644 --- a/scripts/Microsoft.DotNet.Cli.Build.Framework/BuildHelpers.cs +++ b/scripts/Microsoft.DotNet.Cli.Build.Framework/BuildHelpers.cs @@ -36,11 +36,6 @@ namespace Microsoft.DotNet.Cli.Build.Framework cmd.CaptureStdErr().CaptureStdOut(); } - if (env == null) - { - env = new Dictionary(); - } - var result = cmd.Environment(env).Execute(); result.EnsureSuccessful(); diff --git a/scripts/Microsoft.DotNet.Cli.Build.Framework/Command.cs b/scripts/Microsoft.DotNet.Cli.Build.Framework/Command.cs index 89c0e9f16..a70457016 100644 --- a/scripts/Microsoft.DotNet.Cli.Build.Framework/Command.cs +++ b/scripts/Microsoft.DotNet.Cli.Build.Framework/Command.cs @@ -136,6 +136,11 @@ namespace Microsoft.DotNet.Cli.Build.Framework public Command Environment(IDictionary env) { + if (env == null) + { + return this; + } + foreach (var item in env) { _process.StartInfo.Environment[item.Key] = item.Value; diff --git a/scripts/dotnet-cli-build/Utils/Crossgen.cs b/scripts/dotnet-cli-build/Utils/Crossgen.cs index 9b17a0df1..875443dfc 100644 --- a/scripts/dotnet-cli-build/Utils/Crossgen.cs +++ b/scripts/dotnet-cli-build/Utils/Crossgen.cs @@ -73,6 +73,26 @@ namespace Microsoft.DotNet.Cli.Build return; } + string sharedFxPath = c.BuildContext.Get("SharedFrameworkPath"); + + // HACK + // The input directory can be a portable FAT app (example the CLI itself). + // In that case there can be RID specific managed dependencies which are not right next to the app binary (example System.Diagnostics.TraceSource). + // We need those dependencies during crossgen. For now we just pass all subdirectories of the input directory as input to crossgen. + // The right fix - + // If the assembly has deps.json then parse the json file to get all the dependencies, pass these dependencies as input to crossgen. + // else pass the current directory of assembly as input to crossgen. + var addtionalPaths = Directory.GetDirectories(pathToAssemblies, "*", SearchOption.AllDirectories).ToList(); + var paths = new List() { sharedFxPath, pathToAssemblies }; + paths.AddRange(addtionalPaths); + var platformAssembliesPaths = string.Join(Path.PathSeparator.ToString(), paths.Distinct()); + + var env = new Dictionary() + { + // disable partial ngen + { "COMPLUS_ZapDisable", "0" } + }; + foreach (var file in Directory.GetFiles(pathToAssemblies)) { string fileName = Path.GetFileName(file); @@ -83,30 +103,12 @@ namespace Microsoft.DotNet.Cli.Build } string tempPathName = Path.ChangeExtension(file, "readytorun"); - string sharedFxPath = c.BuildContext.Get("SharedFrameworkPath"); - - // HACK - // The input directory can be a portable FAT app (example the CLI itself). - // In that case there can be RID specific managed dependencies which are not right next to the app binary (example System.Diagnostics.TraceSource). - // We need those dependencies during crossgen. For now we just pass all subdirectories of the input directory as input to crossgen. - // The right fix - - // If the assembly has deps.json then parse the json file to get all the dependencies. Pass these dependencies as input to crossgen. - // else pass the current directory of assembly as input to crossgen. - var addtionalPaths = Directory.GetDirectories(pathToAssemblies, "*", SearchOption.AllDirectories).ToList(); - var paths = new List() { sharedFxPath, pathToAssemblies }; - paths.AddRange(addtionalPaths); - var platformAssembliesPaths = string.Join(";", paths.Distinct()); IList crossgenArgs = new List { "-readytorun", "-in", file, "-out", tempPathName, "-platform_assemblies_paths", platformAssembliesPaths }; - var env = new Dictionary() - { - // disable partial ngen - { "COMPLUS_ZapDisable", "0" } - }; ExecSilent(_crossGenPath, crossgenArgs, env); File.Delete(file); diff --git a/scripts/dotnet-cli-build/Utils/PEUtils.cs b/scripts/dotnet-cli-build/Utils/PEUtils.cs index ad702b51a..3899ade77 100644 --- a/scripts/dotnet-cli-build/Utils/PEUtils.cs +++ b/scripts/dotnet-cli-build/Utils/PEUtils.cs @@ -5,7 +5,7 @@ using System.Reflection.PortableExecutable; namespace Microsoft.DotNet.Cli.Build { - public class PEUtils + public static class PEUtils { public static bool HasMetadata(string pathToFile) {