dotnet-installer/scripts/dotnet-cli-build/CompileTargets.cs

742 lines
34 KiB
C#
Raw Normal View History

using System;
2016-02-02 18:04:50 +00:00
using System.Collections.Generic;
using System.IO;
2016-04-02 04:52:08 +00:00
using System.Linq;
2016-02-02 18:04:50 +00:00
using System.Runtime.InteropServices;
using System.Text;
using Microsoft.DotNet.Cli.Build.Framework;
using Microsoft.DotNet.InternalAbstractions;
2016-04-02 04:52:08 +00:00
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using static Microsoft.DotNet.Cli.Build.Framework.BuildHelpers;
using static Microsoft.DotNet.Cli.Build.FS;
2016-02-02 18:04:50 +00:00
namespace Microsoft.DotNet.Cli.Build
{
public class CompileTargets
{
2016-04-27 05:41:53 +00:00
public static readonly string CoreCLRVersion = "1.0.2-rc2-24027";
public static readonly bool IsWinx86 = CurrentPlatform.IsWindows && CurrentArchitecture.Isx86;
2016-02-02 18:04:50 +00:00
public static readonly string[] BinariesForCoreHost = new[]
{
"csc"
2016-02-02 18:04:50 +00:00
};
public static readonly string[] ProjectsToPublish = new[]
{
"dotnet"
};
public static readonly string[] FilesToClean = new[]
{
"vbc.exe"
2016-02-02 18:04:50 +00:00
};
2016-04-22 02:18:05 +00:00
public static string HostPackagePlatformRid => HostPackageSupportedRids[
(RuntimeEnvironment.OperatingSystemPlatform == Platform.Windows)
? $"win7-{RuntimeEnvironment.RuntimeArchitecture}"
: RuntimeEnvironment.GetRuntimeIdentifier()];
2016-04-22 02:18:05 +00:00
public static readonly Dictionary<string, string> HostPackageSupportedRids = new Dictionary<string, string>()
{
2016-04-22 02:18:05 +00:00
// Key: Current platform RID. Value: The actual publishable (non-dummy) package name produced by the build system for this RID.
{ "win7-x64", "win7-x64" },
{ "win7-x86", "win7-x86" },
{ "osx.10.10-x64", "osx.10.10-x64" },
{ "osx.10.11-x64", "osx.10.10-x64" },
{ "ubuntu.14.04-x64", "ubuntu.14.04-x64" },
{ "centos.7-x64", "rhel.7-x64" },
{ "rhel.7-x64", "rhel.7-x64" },
{ "rhel.7.2-x64", "rhel.7-x64" },
{ "debian.8-x64", "debian.8-x64" }
};
public const string SharedFrameworkName = "Microsoft.NETCore.App";
public static Crossgen CrossgenUtil = new Crossgen(CoreCLRVersion);
2016-04-22 02:18:05 +00:00
private static string DotnetHostBaseName => $"dotnet{Constants.ExeSuffix}";
private static string DotnetHostFxrBaseName => $"{Constants.DynamicLibPrefix}hostfxr{Constants.DynamicLibSuffix}";
private static string HostPolicyBaseName => $"{Constants.DynamicLibPrefix}hostpolicy{Constants.DynamicLibSuffix}";
// Updates the stage 2 with recent changes.
[Target(nameof(PrepareTargets.Init), nameof(CompileStage2))]
public static BuildTargetResult UpdateBuild(BuildTargetContext c)
{
return c.Success();
}
2016-04-07 21:05:35 +00:00
// Moving PrepareTargets.RestorePackages after PackagePkgProjects because managed code depends on the
2016-04-07 00:45:38 +00:00
// Microsoft.NETCore.App package that is created during PackagePkgProjects.
2016-04-22 02:18:05 +00:00
[Target(nameof(PrepareTargets.Init), nameof(CompileCoreHost), nameof(PackagePkgProjects), nameof(RestoreLockedCoreHost), nameof(PrepareTargets.RestorePackages), nameof(CompileStage1), nameof(CompileStage2))]
2016-02-02 18:04:50 +00:00
public static BuildTargetResult Compile(BuildTargetContext c)
{
return c.Success();
}
// We need to generate stub host packages so we can restore our standalone test assets against the metapackage
// we built earlier in the build
// https://github.com/dotnet/cli/issues/2438
[Target]
public static BuildTargetResult GenerateStubHostPackages(BuildTargetContext c)
{
var buildVersion = c.BuildContext.Get<BuildVersion>("BuildVersion");
2016-04-22 02:18:05 +00:00
var currentRid = HostPackagePlatformRid;
PrepareDummyRuntimeNuGetPackage(DotNetCli.Stage0);
foreach (var hostPackage in buildVersion.LatestHostPackages)
{
2016-04-22 02:18:05 +00:00
foreach (var rid in HostPackageSupportedRids.Values.Distinct())
{
2016-04-20 23:50:11 +00:00
if (!rid.Equals(currentRid))
{
CreateDummyRuntimeNuGetPackage(
2016-04-20 23:50:11 +00:00
DotNetCli.Stage0,
2016-04-22 02:18:05 +00:00
hostPackage.Key,
2016-04-20 23:50:11 +00:00
rid,
2016-04-22 02:18:05 +00:00
hostPackage.Value,
Dirs.CorehostDummyPackages);
}
}
}
return c.Success();
}
2016-04-22 02:18:05 +00:00
[Target(nameof(PrepareTargets.Init))]
public static BuildTargetResult RestoreLockedCoreHost(BuildTargetContext c)
{
var buildVersion = c.BuildContext.Get<BuildVersion>("BuildVersion");
var lockedHostFxrVersion = buildVersion.LockedHostFxrVersion;
var currentRid = HostPackagePlatformRid;
string projectJson = $@"{{
""dependencies"": {{
""Microsoft.NETCore.DotNetHostResolver"" : ""{lockedHostFxrVersion}""
}},
""frameworks"": {{
""netcoreapp1.0"": {{}}
}},
""runtimes"": {{
""{currentRid}"": {{}}
}}
}}";
var tempPjDirectory = Path.Combine(Dirs.Intermediate, "lockedHostTemp");
FS.Rmdir(tempPjDirectory);
Directory.CreateDirectory(tempPjDirectory);
var tempPjFile = Path.Combine(tempPjDirectory, "project.json");
File.WriteAllText(tempPjFile, projectJson);
DotNetCli.Stage0.Restore("--verbosity", "verbose", "--infer-runtimes",
"--fallbacksource", Dirs.CorehostLocalPackages,
"--fallbacksource", Dirs.CorehostDummyPackages)
.WorkingDirectory(tempPjDirectory)
.Execute()
.EnsureSuccessful();
// Clean out before publishing locked binaries
FS.Rmdir(Dirs.CorehostLocked);
// Use specific RIDS for non-backward compatible platforms.
(CurrentPlatform.IsWindows
? DotNetCli.Stage0.Publish("--output", Dirs.CorehostLocked, "--no-build")
: DotNetCli.Stage0.Publish("--output", Dirs.CorehostLocked, "--no-build", "-r", currentRid))
.WorkingDirectory(tempPjDirectory)
.Execute()
.EnsureSuccessful();
return c.Success();
}
[Target(nameof(PrepareTargets.Init))]
2016-02-02 18:04:50 +00:00
public static BuildTargetResult CompileCoreHost(BuildTargetContext c)
{
var buildVersion = c.BuildContext.Get<BuildVersion>("BuildVersion");
2016-02-02 18:04:50 +00:00
// Generate build files
2016-04-22 02:18:05 +00:00
var cmakeOut = Path.Combine(Dirs.CorehostLatest, "cmake");
2016-02-02 18:04:50 +00:00
Rmdir(cmakeOut);
Mkdirp(cmakeOut);
var configuration = c.BuildContext.Get<string>("Configuration");
2016-02-02 18:04:50 +00:00
// Run the build
string rid = GetRuntimeId();
string corehostSrcDir = Path.Combine(c.BuildContext.BuildDirectory, "src", "corehost");
2016-04-29 00:59:12 +00:00
string commitHash = c.BuildContext.Get<string>("CommitHash");
2016-04-22 02:18:05 +00:00
2016-02-02 18:04:50 +00:00
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
// Why does Windows directly call cmake but Linux/Mac calls "build.sh" in the corehost dir?
// See the comment in "src/corehost/build.sh" for details. It doesn't work for some reason.
var visualStudio = IsWinx86 ? "Visual Studio 14 2015" : "Visual Studio 14 2015 Win64";
var archMacro = IsWinx86 ? "-DCLI_CMAKE_PLATFORM_ARCH_I386=1" : "-DCLI_CMAKE_PLATFORM_ARCH_AMD64=1";
var ridMacro = $"-DCLI_CMAKE_RUNTIME_ID:STRING={rid}";
var arch = IsWinx86 ? "x86" : "x64";
var baseSupportedRid = $"win7-{arch}";
2016-04-22 02:18:05 +00:00
var cmakeHostPolicyVer = $"-DCLI_CMAKE_HOST_POLICY_VER:STRING={buildVersion.LatestHostPolicyVersion}";
var cmakeBaseRid = $"-DCLI_CMAKE_PKG_RID:STRING={baseSupportedRid}";
2016-04-29 00:59:12 +00:00
var cmakeCommitHash = $"-DCLI_CMAKE_COMMIT_HASH:STRING={commitHash}";
2016-02-02 18:04:50 +00:00
ExecIn(cmakeOut, "cmake",
corehostSrcDir,
archMacro,
ridMacro,
cmakeHostPolicyVer,
cmakeBaseRid,
2016-04-29 00:59:12 +00:00
cmakeCommitHash,
2016-02-02 18:04:50 +00:00
"-G",
visualStudio);
2016-02-02 18:04:50 +00:00
var pf32 = RuntimeInformation.OSArchitecture == Architecture.X64 ?
Environment.GetEnvironmentVariable("ProgramFiles(x86)") :
Environment.GetEnvironmentVariable("ProgramFiles");
if (configuration.Equals("Release"))
{
// Cmake calls it "RelWithDebInfo" in the generated MSBuild
configuration = "RelWithDebInfo";
}
Exec(Path.Combine(pf32, "MSBuild", "14.0", "Bin", "MSBuild.exe"),
Path.Combine(cmakeOut, "ALL_BUILD.vcxproj"),
$"/p:Configuration={configuration}");
// Copy the output out
2016-04-22 02:18:05 +00:00
File.Copy(Path.Combine(cmakeOut, "cli", configuration, "dotnet.exe"), Path.Combine(Dirs.CorehostLatest, "dotnet.exe"), overwrite: true);
File.Copy(Path.Combine(cmakeOut, "cli", configuration, "dotnet.pdb"), Path.Combine(Dirs.CorehostLatest, "dotnet.pdb"), overwrite: true);
File.Copy(Path.Combine(cmakeOut, "cli", "dll", configuration, "hostpolicy.dll"), Path.Combine(Dirs.CorehostLatest, "hostpolicy.dll"), overwrite: true);
File.Copy(Path.Combine(cmakeOut, "cli", "dll", configuration, "hostpolicy.pdb"), Path.Combine(Dirs.CorehostLatest, "hostpolicy.pdb"), overwrite: true);
File.Copy(Path.Combine(cmakeOut, "cli", "fxr", configuration, "hostfxr.dll"), Path.Combine(Dirs.CorehostLatest, "hostfxr.dll"), overwrite: true);
File.Copy(Path.Combine(cmakeOut, "cli", "fxr", configuration, "hostfxr.pdb"), Path.Combine(Dirs.CorehostLatest, "hostfxr.pdb"), overwrite: true);
2016-02-02 18:04:50 +00:00
}
else
{
ExecIn(cmakeOut, Path.Combine(c.BuildContext.BuildDirectory, "src", "corehost", "build.sh"),
"--arch",
"x64",
"--policyver",
2016-04-22 02:18:05 +00:00
buildVersion.LatestHostPolicyVersion,
"--rid",
2016-04-29 00:59:12 +00:00
rid,
"--commithash",
commitHash);
2016-02-02 18:04:50 +00:00
// Copy the output out
2016-04-22 02:18:05 +00:00
File.Copy(Path.Combine(cmakeOut, "cli", "dotnet"), Path.Combine(Dirs.CorehostLatest, "dotnet"), overwrite: true);
File.Copy(Path.Combine(cmakeOut, "cli", "dll", HostPolicyBaseName), Path.Combine(Dirs.CorehostLatest, HostPolicyBaseName), overwrite: true);
File.Copy(Path.Combine(cmakeOut, "cli", "fxr", DotnetHostFxrBaseName), Path.Combine(Dirs.CorehostLatest, DotnetHostFxrBaseName), overwrite: true);
2016-02-02 18:04:50 +00:00
}
return c.Success();
}
2016-02-02 18:04:50 +00:00
[Target(nameof(CompileTargets.GenerateStubHostPackages))]
public static BuildTargetResult PackagePkgProjects(BuildTargetContext c)
{
var arch = IsWinx86 ? "x86" : "x64";
2016-04-22 02:18:05 +00:00
var buildVersion = c.BuildContext.Get<BuildVersion>("BuildVersion");
var version = buildVersion.NuGetVersion;
var content = $@"{c.BuildContext["CommitHash"]}{Environment.NewLine}{version}{Environment.NewLine}";
var pkgDir = Path.Combine(c.BuildContext.BuildDirectory, "pkg");
File.WriteAllText(Path.Combine(pkgDir, "version.txt"), content);
if (CurrentPlatform.IsWindows)
{
Command.Create(Path.Combine(pkgDir, "pack.cmd"))
// Workaround to arg escaping adding backslashes for arguments to .cmd scripts.
.Environment("__WorkaroundCliCoreHostBuildArch", arch)
2016-04-22 02:18:05 +00:00
.Environment("__WorkaroundCliCoreHostBinDir", Dirs.CorehostLatest)
.Environment("__WorkaroundCliCoreHostPolicyVer", buildVersion.LatestHostPolicyVersionNoSuffix)
.Environment("__WorkaroundCliCoreHostFxrVer", buildVersion.LatestHostFxrVersionNoSuffix)
.Environment("__WorkaroundCliCoreHostVer", buildVersion.LatestHostVersionNoSuffix)
.Environment("__WorkaroundCliCoreHostBuildMajor", buildVersion.LatestHostBuildMajor)
.Environment("__WorkaroundCliCoreHostVersionTag", buildVersion.LatestHostPrerelease)
.ForwardStdOut()
.ForwardStdErr()
.Execute()
.EnsureSuccessful();
}
else
{
Exec(Path.Combine(pkgDir, "pack.sh"),
"--arch",
"x64",
"--hostbindir",
2016-04-22 02:18:05 +00:00
Dirs.CorehostLatest,
"--policyver",
2016-04-22 02:18:05 +00:00
buildVersion.LatestHostPolicyVersionNoSuffix,
"--fxrver",
2016-04-22 02:18:05 +00:00
buildVersion.LatestHostFxrVersionNoSuffix,
"--hostver",
2016-04-22 02:18:05 +00:00
buildVersion.LatestHostVersionNoSuffix,
"--build",
2016-04-22 02:18:05 +00:00
buildVersion.LatestHostBuildMajor,
"--vertag",
2016-04-22 02:18:05 +00:00
buildVersion.LatestHostPrerelease);
}
foreach (var file in Directory.GetFiles(Path.Combine(pkgDir, "bin", "packages"), "*.nupkg"))
{
var fileName = Path.GetFileName(file);
2016-04-22 02:18:05 +00:00
File.Copy(file, Path.Combine(Dirs.CorehostLocalPackages, fileName), true);
2016-04-22 02:18:05 +00:00
Console.WriteLine($"Copying package {fileName} to artifacts directory {Dirs.CorehostLocalPackages}.");
}
2016-04-22 02:18:05 +00:00
foreach (var item in buildVersion.LatestHostPackages)
{
2016-04-22 02:18:05 +00:00
var fileFilter = $"runtime.{HostPackagePlatformRid}.{item.Key}.{item.Value}.nupkg";
if (Directory.GetFiles(Dirs.CorehostLocalPackages, fileFilter).Length == 0)
{
throw new BuildFailureException($"Nupkg for {fileFilter} was not created.");
}
}
2016-02-02 18:04:50 +00:00
return c.Success();
}
private static string GetRuntimeId()
{
string info = DotNetCli.Stage0.Exec("", "--info").CaptureStdOut().Execute().StdOut;
string rid = Array.Find<string>(info.Split(Environment.NewLine.ToCharArray()), (e) => e.Contains("RID:"))?.Replace("RID:", "").Trim();
// TODO: when Stage0 is updated with the new --info, remove this legacy check for --version
if (string.IsNullOrEmpty(rid))
{
string version = DotNetCli.Stage0.Exec("", "--version").CaptureStdOut().Execute().StdOut;
rid = Array.Find<string>(version.Split(Environment.NewLine.ToCharArray()), (e) => e.Contains("Runtime Id:")).Replace("Runtime Id:", "").Trim();
}
if (string.IsNullOrEmpty(rid))
{
throw new BuildFailureException("Could not find the Runtime ID from Stage0 --info or --version");
}
return rid;
}
[Target(nameof(PrepareTargets.Init))]
2016-02-02 18:04:50 +00:00
public static BuildTargetResult CompileStage1(BuildTargetContext c)
{
CleanBinObj(c, Path.Combine(c.BuildContext.BuildDirectory, "src"));
if (Directory.Exists(Dirs.Stage1))
{
Utils.DeleteDirectory(Dirs.Stage1);
}
Directory.CreateDirectory(Dirs.Stage1);
CopySharedHost(Dirs.Stage1);
PublishSharedFramework(c, Dirs.Stage1, DotNetCli.Stage0);
var result = CompileCliSdk(c,
2016-02-02 18:04:50 +00:00
dotnet: DotNetCli.Stage0,
outputDir: Dirs.Stage1);
CleanOutputDir(Path.Combine(Dirs.Stage1, "sdk"));
2016-04-05 03:13:13 +00:00
FS.CopyRecursive(Dirs.Stage1, Dirs.Stage1Symbols);
RemovePdbsFromDir(Path.Combine(Dirs.Stage1, "sdk"));
return result;
2016-02-02 18:04:50 +00:00
}
[Target(nameof(PrepareTargets.Init))]
2016-02-02 18:04:50 +00:00
public static BuildTargetResult CompileStage2(BuildTargetContext c)
{
var configuration = c.BuildContext.Get<string>("Configuration");
2016-02-02 18:04:50 +00:00
CleanBinObj(c, Path.Combine(c.BuildContext.BuildDirectory, "src"));
if (Directory.Exists(Dirs.Stage2))
{
Utils.DeleteDirectory(Dirs.Stage2);
}
Directory.CreateDirectory(Dirs.Stage2);
PublishSharedFramework(c, Dirs.Stage2, DotNetCli.Stage1);
CopySharedHost(Dirs.Stage2);
var result = CompileCliSdk(c,
2016-02-02 18:04:50 +00:00
dotnet: DotNetCli.Stage1,
outputDir: Dirs.Stage2);
2016-02-15 17:42:17 +00:00
2016-02-02 18:04:50 +00:00
if (!result.Success)
{
return result;
}
Extract dotnet-compile-fsc into a standalone command Add basic Tests for dotnet-compile-fsc Package Targets execute before TestTargets. Use Generated Nuget Packages in TestTargets. Generate Nuget packages on all platforms, and in C# Fix bug in dotnet-restore, change fsharp new template, add support for native assets in DependencyContextCsvReader copy fsc.exe to temp directory instead of package cache fix rebase error fix issue fixes fixes fix temporarily disable debian package e2e testing fixes bump fsc version update fsc version fix rebase errors WIP update fsc tool WIP, rebased and working again, need to solve issues with System.CommandLine Working state for packaged, command, fsc.exe bugging out with dlopen(, 1): no suitable image found. execute fsc like a unpublished standalone app fixup after rebase working? internet is out working cleanup More cleanup, and run the debian package tests during the Test phase of the build. update FSharp Test Projects NetStandard Library Version Update Version Suffix when packing TestPackages. This will enable packing with the right dependency versions on Windows. update dotnet-test version Undo the reordering of the build fix test package project pathsj ignore net451 build failures for test packages which we need to build on non-windows update dependency of desktop test app add dotnetcli feed to nuget config for fsharp dotnet new update deps after rebase update dependency of dotnet-compile-fsc pass args before commandPath when using muxer for tools adjust testpackage cleaning not to clean packages which are also generated as part of the product from the nuget cache. undo Pass projectJson to pack instead of using WorkingDirectory fix path separators using depsjsoncommandresolver on windows, fix building only specific frameworks for testpackages on non-windows. PR Feedback rebase overwrite fsc runtimeconfig
2016-03-12 00:41:00 +00:00
if (CurrentPlatform.IsWindows)
2016-02-02 18:04:50 +00:00
{
Extract dotnet-compile-fsc into a standalone command Add basic Tests for dotnet-compile-fsc Package Targets execute before TestTargets. Use Generated Nuget Packages in TestTargets. Generate Nuget packages on all platforms, and in C# Fix bug in dotnet-restore, change fsharp new template, add support for native assets in DependencyContextCsvReader copy fsc.exe to temp directory instead of package cache fix rebase error fix issue fixes fixes fix temporarily disable debian package e2e testing fixes bump fsc version update fsc version fix rebase errors WIP update fsc tool WIP, rebased and working again, need to solve issues with System.CommandLine Working state for packaged, command, fsc.exe bugging out with dlopen(, 1): no suitable image found. execute fsc like a unpublished standalone app fixup after rebase working? internet is out working cleanup More cleanup, and run the debian package tests during the Test phase of the build. update FSharp Test Projects NetStandard Library Version Update Version Suffix when packing TestPackages. This will enable packing with the right dependency versions on Windows. update dotnet-test version Undo the reordering of the build fix test package project pathsj ignore net451 build failures for test packages which we need to build on non-windows update dependency of desktop test app add dotnetcli feed to nuget config for fsharp dotnet new update deps after rebase update dependency of dotnet-compile-fsc pass args before commandPath when using muxer for tools adjust testpackage cleaning not to clean packages which are also generated as part of the product from the nuget cache. undo Pass projectJson to pack instead of using WorkingDirectory fix path separators using depsjsoncommandresolver on windows, fix building only specific frameworks for testpackages on non-windows. PR Feedback rebase overwrite fsc runtimeconfig
2016-03-12 00:41:00 +00:00
// build projects for nuget packages
2016-02-02 18:04:50 +00:00
var packagingOutputDir = Path.Combine(Dirs.Stage2Compilation, "forPackaging");
Mkdirp(packagingOutputDir);
Extract dotnet-compile-fsc into a standalone command Add basic Tests for dotnet-compile-fsc Package Targets execute before TestTargets. Use Generated Nuget Packages in TestTargets. Generate Nuget packages on all platforms, and in C# Fix bug in dotnet-restore, change fsharp new template, add support for native assets in DependencyContextCsvReader copy fsc.exe to temp directory instead of package cache fix rebase error fix issue fixes fixes fix temporarily disable debian package e2e testing fixes bump fsc version update fsc version fix rebase errors WIP update fsc tool WIP, rebased and working again, need to solve issues with System.CommandLine Working state for packaged, command, fsc.exe bugging out with dlopen(, 1): no suitable image found. execute fsc like a unpublished standalone app fixup after rebase working? internet is out working cleanup More cleanup, and run the debian package tests during the Test phase of the build. update FSharp Test Projects NetStandard Library Version Update Version Suffix when packing TestPackages. This will enable packing with the right dependency versions on Windows. update dotnet-test version Undo the reordering of the build fix test package project pathsj ignore net451 build failures for test packages which we need to build on non-windows update dependency of desktop test app add dotnetcli feed to nuget config for fsharp dotnet new update deps after rebase update dependency of dotnet-compile-fsc pass args before commandPath when using muxer for tools adjust testpackage cleaning not to clean packages which are also generated as part of the product from the nuget cache. undo Pass projectJson to pack instead of using WorkingDirectory fix path separators using depsjsoncommandresolver on windows, fix building only specific frameworks for testpackages on non-windows. PR Feedback rebase overwrite fsc runtimeconfig
2016-03-12 00:41:00 +00:00
foreach (var project in PackageTargets.ProjectsToPack)
2016-02-02 18:04:50 +00:00
{
// Just build them, we'll pack later
Extract dotnet-compile-fsc into a standalone command Add basic Tests for dotnet-compile-fsc Package Targets execute before TestTargets. Use Generated Nuget Packages in TestTargets. Generate Nuget packages on all platforms, and in C# Fix bug in dotnet-restore, change fsharp new template, add support for native assets in DependencyContextCsvReader copy fsc.exe to temp directory instead of package cache fix rebase error fix issue fixes fixes fix temporarily disable debian package e2e testing fixes bump fsc version update fsc version fix rebase errors WIP update fsc tool WIP, rebased and working again, need to solve issues with System.CommandLine Working state for packaged, command, fsc.exe bugging out with dlopen(, 1): no suitable image found. execute fsc like a unpublished standalone app fixup after rebase working? internet is out working cleanup More cleanup, and run the debian package tests during the Test phase of the build. update FSharp Test Projects NetStandard Library Version Update Version Suffix when packing TestPackages. This will enable packing with the right dependency versions on Windows. update dotnet-test version Undo the reordering of the build fix test package project pathsj ignore net451 build failures for test packages which we need to build on non-windows update dependency of desktop test app add dotnetcli feed to nuget config for fsharp dotnet new update deps after rebase update dependency of dotnet-compile-fsc pass args before commandPath when using muxer for tools adjust testpackage cleaning not to clean packages which are also generated as part of the product from the nuget cache. undo Pass projectJson to pack instead of using WorkingDirectory fix path separators using depsjsoncommandresolver on windows, fix building only specific frameworks for testpackages on non-windows. PR Feedback rebase overwrite fsc runtimeconfig
2016-03-12 00:41:00 +00:00
var packBuildResult = DotNetCli.Stage1.Build(
2016-02-02 18:04:50 +00:00
"--build-base-path",
packagingOutputDir,
"--configuration",
configuration,
Path.Combine(c.BuildContext.BuildDirectory, "src", project))
Extract dotnet-compile-fsc into a standalone command Add basic Tests for dotnet-compile-fsc Package Targets execute before TestTargets. Use Generated Nuget Packages in TestTargets. Generate Nuget packages on all platforms, and in C# Fix bug in dotnet-restore, change fsharp new template, add support for native assets in DependencyContextCsvReader copy fsc.exe to temp directory instead of package cache fix rebase error fix issue fixes fixes fix temporarily disable debian package e2e testing fixes bump fsc version update fsc version fix rebase errors WIP update fsc tool WIP, rebased and working again, need to solve issues with System.CommandLine Working state for packaged, command, fsc.exe bugging out with dlopen(, 1): no suitable image found. execute fsc like a unpublished standalone app fixup after rebase working? internet is out working cleanup More cleanup, and run the debian package tests during the Test phase of the build. update FSharp Test Projects NetStandard Library Version Update Version Suffix when packing TestPackages. This will enable packing with the right dependency versions on Windows. update dotnet-test version Undo the reordering of the build fix test package project pathsj ignore net451 build failures for test packages which we need to build on non-windows update dependency of desktop test app add dotnetcli feed to nuget config for fsharp dotnet new update deps after rebase update dependency of dotnet-compile-fsc pass args before commandPath when using muxer for tools adjust testpackage cleaning not to clean packages which are also generated as part of the product from the nuget cache. undo Pass projectJson to pack instead of using WorkingDirectory fix path separators using depsjsoncommandresolver on windows, fix building only specific frameworks for testpackages on non-windows. PR Feedback rebase overwrite fsc runtimeconfig
2016-03-12 00:41:00 +00:00
.Execute();
packBuildResult.EnsureSuccessful();
2016-02-02 18:04:50 +00:00
}
}
CleanOutputDir(Path.Combine(Dirs.Stage2, "sdk"));
2016-04-05 03:13:13 +00:00
FS.CopyRecursive(Dirs.Stage2, Dirs.Stage2Symbols);
RemovePdbsFromDir(Path.Combine(Dirs.Stage2, "sdk"));
2016-02-02 18:04:50 +00:00
return c.Success();
}
2016-04-22 02:18:05 +00:00
private static void PrepareDummyRuntimeNuGetPackage(DotNetCli dotnet)
2016-04-20 23:50:11 +00:00
{
var projectJson = new StringBuilder();
projectJson.Append("{");
projectJson.Append(" \"dependencies\": { \"NETStandard.Library\": \"1.5.0-rc2-24008\" },");
projectJson.Append(" \"frameworks\": { \"netcoreapp1.0\": { \"imports\": [\"netstandard1.5\", \"dnxcore50\"] } },");
2016-04-20 23:50:11 +00:00
projectJson.Append(" \"runtimes\": { \"win7-x64\": { } },");
projectJson.Append("}");
var programCs = "using System; namespace ConsoleApplication { public class Program { public static void Main(string[] args) { Console.WriteLine(\"Hello World!\"); } } }";
var tempPjDirectory = Path.Combine(Dirs.Intermediate, "dummyNuGetPackageIntermediate");
FS.Rmdir(tempPjDirectory);
Directory.CreateDirectory(tempPjDirectory);
var tempPjFile = Path.Combine(tempPjDirectory, "project.json");
var tempSourceFile = Path.Combine(tempPjDirectory, "Program.cs");
File.WriteAllText(tempPjFile, projectJson.ToString());
File.WriteAllText(tempSourceFile, programCs.ToString());
dotnet.Restore("--verbosity", "verbose", "--disable-parallel")
.WorkingDirectory(tempPjDirectory)
.Execute()
.EnsureSuccessful();
2016-04-20 23:50:11 +00:00
dotnet.Build(tempPjFile, "--runtime", "win7-x64")
.WorkingDirectory(tempPjDirectory)
.Execute()
.EnsureSuccessful();
2016-04-20 23:50:11 +00:00
}
private static void CreateDummyRuntimeNuGetPackage(DotNetCli dotnet, string basePackageId, string rid, string version, string outputDir)
{
var packageId = $"runtime.{rid}.{basePackageId}";
var projectJson = new StringBuilder();
projectJson.Append("{");
projectJson.Append($" \"version\": \"{version}\",");
projectJson.Append($" \"name\": \"{packageId}\",");
projectJson.Append(" \"dependencies\": { \"NETStandard.Library\": \"1.5.0-rc2-24008\" },");
projectJson.Append(" \"frameworks\": { \"netcoreapp1.0\": { \"imports\": [\"netstandard1.5\", \"dnxcore50\"] } },");
projectJson.Append("}");
var tempPjDirectory = Path.Combine(Dirs.Intermediate, "dummyNuGetPackageIntermediate");
var tempPjFile = Path.Combine(tempPjDirectory, "project.json");
File.WriteAllText(tempPjFile, projectJson.ToString());
dotnet.Pack(
tempPjFile, "--no-build",
"--output", outputDir)
.WorkingDirectory(tempPjDirectory)
.Execute()
.EnsureSuccessful();
}
private static void CleanOutputDir(string directory)
{
2016-04-05 03:13:13 +00:00
foreach (var file in FilesToClean)
{
FS.RmFilesInDirRecursive(directory, file);
}
}
private static void RemovePdbsFromDir(string directory)
{
FS.RmFilesInDirRecursive(directory, "*.pdb");
}
private static void CopySharedHost(string outputDir)
2016-02-02 18:04:50 +00:00
{
File.Copy(
2016-04-22 02:18:05 +00:00
Path.Combine(Dirs.CorehostLocked, DotnetHostBaseName),
Path.Combine(outputDir, DotnetHostBaseName), true);
File.Copy(
2016-04-22 02:18:05 +00:00
Path.Combine(Dirs.CorehostLocked, DotnetHostFxrBaseName),
Path.Combine(outputDir, DotnetHostFxrBaseName), true);
}
public static void PublishSharedFramework(BuildTargetContext c, string outputDir, DotNetCli dotnetCli)
{
2016-04-07 00:45:38 +00:00
string SharedFrameworkTemplateSourceRoot = Path.Combine(Dirs.RepoRoot, "src", "sharedframework", "framework");
string SharedFrameworkNugetVersion = c.BuildContext.Get<string>("SharedFrameworkNugetVersion");
2016-04-07 00:45:38 +00:00
string sharedFrameworkRid;
if (RuntimeEnvironment.OperatingSystemPlatform == Platform.Windows)
2016-04-07 00:45:38 +00:00
{
sharedFrameworkRid = $"win7-{RuntimeEnvironment.RuntimeArchitecture}";
2016-04-07 00:45:38 +00:00
}
else
{
sharedFrameworkRid = RuntimeEnvironment.GetRuntimeIdentifier();
2016-04-07 00:45:38 +00:00
}
string SharedFrameworkSourceRoot = GenerateSharedFrameworkProject(c, SharedFrameworkTemplateSourceRoot, sharedFrameworkRid);
dotnetCli.Restore(
"--verbosity", "verbose",
"--disable-parallel",
"--infer-runtimes",
2016-04-22 02:18:05 +00:00
"--fallbacksource", Dirs.CorehostLocalPackages)
2016-04-07 00:45:38 +00:00
.WorkingDirectory(SharedFrameworkSourceRoot)
.Execute()
.EnsureSuccessful();
// We publish to a sub folder of the PublishRoot so tools like heat and zip can generate folder structures easier.
string SharedFrameworkNameAndVersionRoot = Path.Combine(outputDir, "shared", SharedFrameworkName, SharedFrameworkNugetVersion);
2016-03-24 16:36:05 +00:00
c.BuildContext["SharedFrameworkPath"] = SharedFrameworkNameAndVersionRoot;
if (Directory.Exists(SharedFrameworkNameAndVersionRoot))
{
Utils.DeleteDirectory(SharedFrameworkNameAndVersionRoot);
}
dotnetCli.Publish(
"--output", SharedFrameworkNameAndVersionRoot,
2016-04-07 00:45:38 +00:00
"-r", sharedFrameworkRid,
SharedFrameworkSourceRoot).Execute().EnsureSuccessful();
// Clean up artifacts that dotnet-publish generates which we don't need
2016-03-24 16:36:05 +00:00
DeleteMainPublishOutput(SharedFrameworkNameAndVersionRoot, "framework");
File.Delete(Path.Combine(SharedFrameworkNameAndVersionRoot, "framework.runtimeconfig.json"));
// Rename the .deps file
var destinationDeps = Path.Combine(SharedFrameworkNameAndVersionRoot, $"{SharedFrameworkName}.deps.json");
File.Move(Path.Combine(SharedFrameworkNameAndVersionRoot, "framework.deps.json"), destinationDeps);
2016-04-02 04:52:08 +00:00
ChangeEntryPointLibraryName(destinationDeps, null);
// Generate RID fallback graph
string runtimeGraphGeneratorRuntime = null;
switch (RuntimeEnvironment.OperatingSystemPlatform)
{
case Platform.Windows:
runtimeGraphGeneratorRuntime = "win";
break;
case Platform.Linux:
runtimeGraphGeneratorRuntime = "linux";
break;
case Platform.Darwin:
runtimeGraphGeneratorRuntime = "osx";
break;
}
if (!string.IsNullOrEmpty(runtimeGraphGeneratorRuntime))
{
var runtimeGraphGeneratorName = "RuntimeGraphGenerator";
var runtimeGraphGeneratorProject = Path.Combine(Dirs.RepoRoot, "tools", runtimeGraphGeneratorName);
var runtimeGraphGeneratorOutput = Path.Combine(Dirs.Output, "tools", runtimeGraphGeneratorName);
dotnetCli.Publish(
"--output", runtimeGraphGeneratorOutput,
runtimeGraphGeneratorProject).Execute().EnsureSuccessful();
var runtimeGraphGeneratorExe = Path.Combine(runtimeGraphGeneratorOutput, $"{runtimeGraphGeneratorName}{Constants.ExeSuffix}");
Cmd(runtimeGraphGeneratorExe, "--project", SharedFrameworkSourceRoot, "--deps", destinationDeps, runtimeGraphGeneratorRuntime)
.Execute()
.EnsureSuccessful();
}
else
{
c.Error($"Could not determine rid graph generation runtime for platform {RuntimeEnvironment.OperatingSystemPlatform}");
}
File.Copy(
2016-04-22 02:18:05 +00:00
Path.Combine(Dirs.CorehostLocked, DotnetHostBaseName),
Path.Combine(SharedFrameworkNameAndVersionRoot, DotnetHostBaseName), true);
File.Copy(
Path.Combine(Dirs.CorehostLocked, DotnetHostBaseName),
Path.Combine(SharedFrameworkNameAndVersionRoot, $"corehost{Constants.ExeSuffix}"), true);
2016-03-24 16:36:05 +00:00
File.Copy(
2016-04-22 02:18:05 +00:00
Path.Combine(Dirs.CorehostLocked, DotnetHostFxrBaseName),
Path.Combine(SharedFrameworkNameAndVersionRoot, DotnetHostFxrBaseName), true);
// Hostpolicy should be the latest and not the locked version as it is supposed to evolve for
// the framework and has a tight coupling with coreclr's API in the framework.
File.Copy(
2016-04-22 02:18:05 +00:00
Path.Combine(Dirs.CorehostLatest, HostPolicyBaseName),
Path.Combine(SharedFrameworkNameAndVersionRoot, HostPolicyBaseName), true);
if (File.Exists(Path.Combine(SharedFrameworkNameAndVersionRoot, "mscorlib.ni.dll")))
{
// Publish already places the crossgen'd version of mscorlib into the output, so we can
// remove the IL version
File.Delete(Path.Combine(SharedFrameworkNameAndVersionRoot, "mscorlib.dll"));
}
CrossgenUtil.CrossgenDirectory(c, SharedFrameworkNameAndVersionRoot);
// Generate .version file for sharedfx
var version = SharedFrameworkNugetVersion;
var content = $@"{c.BuildContext["CommitHash"]}{Environment.NewLine}{version}{Environment.NewLine}";
File.WriteAllText(Path.Combine(SharedFrameworkNameAndVersionRoot, ".version"), content);
}
2016-02-02 18:04:50 +00:00
2016-04-07 00:45:38 +00:00
/// <summary>
/// Generates the real shared framework project that will get published.
/// </summary>
/// <param name="sharedFrameworkTemplatePath">The "sharedFramework" source template folder.</param>
private static string GenerateSharedFrameworkProject(BuildTargetContext c, string sharedFrameworkTemplatePath, string rid)
{
string sharedFrameworkProjectPath = Path.Combine(Dirs.Intermediate, "sharedFramework", "framework");
Utils.DeleteDirectory(sharedFrameworkProjectPath);
CopyRecursive(sharedFrameworkTemplatePath, sharedFrameworkProjectPath, true);
string templateFile = Path.Combine(sharedFrameworkProjectPath, "project.json.template");
JObject sharedFrameworkProject = JsonUtils.ReadProject(templateFile);
sharedFrameworkProject["dependencies"]["Microsoft.NETCore.App"] = c.BuildContext.Get<BuildVersion>("BuildVersion").NetCoreAppVersion;
((JObject)sharedFrameworkProject["runtimes"]).RemoveAll();
sharedFrameworkProject["runtimes"][rid] = new JObject();
string projectJsonPath = Path.Combine(sharedFrameworkProjectPath, "project.json");
JsonUtils.WriteProject(sharedFrameworkProject, projectJsonPath);
Rm(templateFile);
return sharedFrameworkProjectPath;
}
private static BuildTargetResult CompileCliSdk(BuildTargetContext c, DotNetCli dotnet, string outputDir)
{
var configuration = c.BuildContext.Get<string>("Configuration");
var buildVersion = c.BuildContext.Get<BuildVersion>("BuildVersion");
outputDir = Path.Combine(outputDir, "sdk", buildVersion.NuGetVersion);
2016-02-02 18:04:50 +00:00
Rmdir(outputDir);
Mkdirp(outputDir);
2016-02-02 18:04:50 +00:00
foreach (var project in ProjectsToPublish)
{
// TODO: Use the flag once we get a full build round tripped
// --version-suffix buildVesion.VersionSuffix
2016-02-02 18:04:50 +00:00
dotnet.Publish(
"--native-subdirectory",
"--output",
outputDir,
2016-02-02 18:04:50 +00:00
"--configuration",
configuration,
Path.Combine(c.BuildContext.BuildDirectory, "src", project))
.Environment("DOTNET_BUILD_VERSION", buildVersion.VersionSuffix)
2016-02-02 18:04:50 +00:00
.Execute()
.EnsureSuccessful();
}
FixModeFlags(outputDir);
2016-03-24 16:36:05 +00:00
string compilersProject = Path.Combine(Dirs.RepoRoot, "src", "compilers");
dotnet.Publish(compilersProject,
"--output",
outputDir,
"--framework",
2016-04-02 04:52:08 +00:00
"netstandard1.5")
2016-03-24 16:36:05 +00:00
.Execute()
.EnsureSuccessful();
var compilersDeps = Path.Combine(outputDir, "compilers.deps.json");
var compilersRuntimeConfig = Path.Combine(outputDir, "compilers.runtimeconfig.json");
2016-04-22 02:18:05 +00:00
File.Copy(Path.Combine(Dirs.CorehostLocked, DotnetHostBaseName), Path.Combine(outputDir, $"corehost{Constants.ExeSuffix}"), overwrite: true);
File.Copy(Path.Combine(Dirs.CorehostLocked, $"{Constants.DynamicLibPrefix}hostfxr{Constants.DynamicLibSuffix}"), Path.Combine(outputDir, $"{Constants.DynamicLibPrefix}hostfxr{Constants.DynamicLibSuffix}"), overwrite: true);
File.Copy(Path.Combine(Dirs.CorehostLatest, $"{Constants.DynamicLibPrefix}hostpolicy{Constants.DynamicLibSuffix}"), Path.Combine(outputDir, $"{Constants.DynamicLibPrefix}hostpolicy{Constants.DynamicLibSuffix}"), overwrite: true);
2016-02-02 18:04:50 +00:00
2016-03-24 16:36:05 +00:00
var binaryToCorehostifyOutDir = Path.Combine(outputDir, "runtimes", "any", "native");
2016-02-02 18:04:50 +00:00
// Corehostify binaries
2016-02-15 17:42:17 +00:00
foreach (var binaryToCorehostify in BinariesForCoreHost)
2016-02-02 18:04:50 +00:00
{
try
{
// Yes, it is .exe even on Linux. This is the managed exe we're working with
2016-03-24 16:36:05 +00:00
File.Copy(Path.Combine(binaryToCorehostifyOutDir, $"{binaryToCorehostify}.exe"), Path.Combine(outputDir, $"{binaryToCorehostify}.dll"));
File.Delete(Path.Combine(binaryToCorehostifyOutDir, $"{binaryToCorehostify}.exe"));
File.Copy(compilersDeps, Path.Combine(outputDir, binaryToCorehostify + ".deps.json"));
File.Copy(compilersRuntimeConfig, Path.Combine(outputDir, binaryToCorehostify + ".runtimeconfig.json"));
2016-04-02 04:52:08 +00:00
ChangeEntryPointLibraryName(Path.Combine(outputDir, binaryToCorehostify + ".deps.json"), binaryToCorehostify);
2016-02-02 18:04:50 +00:00
}
2016-02-15 17:42:17 +00:00
catch (Exception ex)
2016-02-02 18:04:50 +00:00
{
return c.Failed($"Failed to corehostify '{binaryToCorehostify}': {ex.ToString()}");
}
}
2016-03-24 16:36:05 +00:00
// cleanup compilers project output we don't need
DeleteMainPublishOutput(outputDir, "compilers");
File.Delete(compilersDeps);
File.Delete(compilersRuntimeConfig);
CrossgenUtil.CrossgenDirectory(c, outputDir);
2016-02-02 18:04:50 +00:00
// Generate .version file
var version = buildVersion.NuGetVersion;
2016-02-13 01:16:28 +00:00
var content = $@"{c.BuildContext["CommitHash"]}{Environment.NewLine}{version}{Environment.NewLine}";
2016-02-02 18:04:50 +00:00
File.WriteAllText(Path.Combine(outputDir, ".version"), content);
return c.Success();
}
2016-04-02 04:52:08 +00:00
private static void ChangeEntryPointLibraryName(string depsFile, string newName)
{
JToken deps;
using (var file = File.OpenText(depsFile))
using (JsonTextReader reader = new JsonTextReader(file))
{
deps = JObject.ReadFrom(reader);
}
2016-04-29 15:11:59 +00:00
string version = null;
foreach (JProperty target in deps["targets"])
2016-04-02 04:52:08 +00:00
{
2016-04-29 15:11:59 +00:00
var targetLibrary = target.Value.Children<JProperty>().FirstOrDefault();
if (targetLibrary == null)
{
continue;
}
version = targetLibrary.Name.Substring(targetLibrary.Name.IndexOf('/') + 1);
if (newName == null)
{
targetLibrary.Remove();
}
else
{
targetLibrary.Replace(new JProperty(newName + '/' + version, targetLibrary.Value));
}
2016-04-02 04:52:08 +00:00
}
2016-04-29 15:11:59 +00:00
if (version != null)
2016-04-02 04:52:08 +00:00
{
2016-04-29 15:11:59 +00:00
var library = deps["libraries"].Children<JProperty>().First();
if (newName == null)
{
library.Remove();
}
else
{
library.Replace(new JProperty(newName + '/' + version, library.Value));
}
using (var file = File.CreateText(depsFile))
using (var writer = new JsonTextWriter(file) { Formatting = Formatting.Indented })
{
deps.WriteTo(writer);
}
2016-04-02 04:52:08 +00:00
}
}
2016-03-24 16:36:05 +00:00
private static void DeleteMainPublishOutput(string path, string name)
{
File.Delete(Path.Combine(path, $"{name}{Constants.ExeSuffix}"));
File.Delete(Path.Combine(path, $"{name}.dll"));
File.Delete(Path.Combine(path, $"{name}.pdb"));
}
2016-02-02 18:04:50 +00:00
}
}