dotnet-installer/build_projects/dotnet-cli-build/CompileTargets.cs

using System;
using System.Collections.Generic;
using System.IO;
using Microsoft.Build.Utilities;
using Microsoft.DotNet.Cli.Build.Framework;
using Microsoft.DotNet.InternalAbstractions;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using static Microsoft.DotNet.Cli.Build.Framework.BuildHelpers;
using static Microsoft.DotNet.Cli.Build.FS;

namespace Microsoft.DotNet.Cli.Build
{
    public class CompileTargets : Task
    {
        public static readonly bool IsWinx86 = CurrentPlatform.IsWindows && CurrentArchitecture.Isx86;

        public static readonly string[] BinariesForCoreHost = new[]
        {
            "csc"
        };

        public static readonly string[] ProjectsToPublish = new[]
        {
            "dotnet"
        };

        public static readonly string[] FilesToClean = new[]
        {
            "vbc.exe"
        };

        public static string HostPackagePlatformRid => HostPackageSupportedRids[
                             (RuntimeEnvironment.OperatingSystemPlatform == Platform.Windows)
                             ? $"win7-{RuntimeEnvironment.RuntimeArchitecture}"
                             : RuntimeEnvironment.GetRuntimeIdentifier()];

        public static readonly Dictionary<string, string> HostPackageSupportedRids = new Dictionary<string, string>()
        {
            // Key: Current platform RID. Value: The actual publishable (non-dummy) package name produced by the build system for this RID.
            { "win7-x64", "win7-x64" },
            { "win7-x86", "win7-x86" },
            { "osx.10.10-x64", "osx.10.10-x64" },
            { "osx.10.11-x64", "osx.10.10-x64" },
            { "ubuntu.14.04-x64", "ubuntu.14.04-x64" },
            { "ubuntu.16.04-x64", "ubuntu.16.04-x64" },
            { "centos.7-x64", "rhel.7-x64" },
            { "rhel.7-x64", "rhel.7-x64" },
            { "rhel.7.2-x64", "rhel.7-x64" },
            { "debian.8-x64", "debian.8-x64" },
            { "fedora.23-x64", "fedora.23-x64" },
            { "opensuse.13.2-x64", "opensuse.13.2-x64" }
        };

        public const string SharedFrameworkName = "Microsoft.NETCore.App";

        public static Crossgen CrossgenUtil = new Crossgen(DependencyVersions.CoreCLRVersion, DependencyVersions.JitVersion);

        public override bool Execute()
        {
            BuildContext context = new BuildSetup("MSBuild").UseAllTargetsFromAssembly<CompileTargets>().CreateBuildContext();
            BuildTargetContext c = new BuildTargetContext(context, null, null);

            return Compile(c).Success;
        }

        [Target]
        public static BuildTargetResult Compile(BuildTargetContext c)
        {
            PrepareTargets.Init(c);
            CompileStage1(c);
            CompileStage2(c);

            return c.Success();
        }

        public static BuildTargetResult CompileStage1(BuildTargetContext c)
        {
            CleanBinObj(c, Path.Combine(c.BuildContext.BuildDirectory, "src"));

            if (Directory.Exists(Dirs.Stage1))
            {
                Utils.DeleteDirectory(Dirs.Stage1);
            }
            Directory.CreateDirectory(Dirs.Stage1);

            var result = CompileCliSdk(c,
                dotnet: DotNetCli.Stage0,
                rootOutputDirectory: Dirs.Stage1);

            CleanOutputDir(Path.Combine(Dirs.Stage1, "sdk"));
            FS.CopyRecursive(Dirs.Stage1, Dirs.Stage1Symbols);

            RemovePdbsFromDir(Path.Combine(Dirs.Stage1, "sdk"));

            return result;
        }

        public static BuildTargetResult CompileStage2(BuildTargetContext c)
        {
            var configuration = c.BuildContext.Get<string>("Configuration");

            CleanBinObj(c, Path.Combine(c.BuildContext.BuildDirectory, "src"));

            if (Directory.Exists(Dirs.Stage2))
            {
                Utils.DeleteDirectory(Dirs.Stage2);
            }
            Directory.CreateDirectory(Dirs.Stage2);

            var result = CompileCliSdk(c,
                dotnet: DotNetCli.Stage1,
                rootOutputDirectory: Dirs.Stage2,
                generateNugetPackagesArchive: true);

            if (!result.Success)
            {
                return result;
            }

            if (CurrentPlatform.IsWindows)
            {
                // build projects for nuget packages
                var packagingOutputDir = Path.Combine(Dirs.Stage2Compilation, "forPackaging");
                Mkdirp(packagingOutputDir);
                foreach (var project in PackageTargets.ProjectsToPack)
                {
                    // Just build them, we'll pack later
                    var packBuildResult = DotNetCli.Stage1.Build(
                        "--build-base-path",
                        packagingOutputDir,
                        "--configuration",
                        configuration,
                        Path.Combine(c.BuildContext.BuildDirectory, "src", project))
                        .Execute();

                    packBuildResult.EnsureSuccessful();
                }
            }

            CleanOutputDir(Path.Combine(Dirs.Stage2, "sdk"));
            FS.CopyRecursive(Dirs.Stage2, Dirs.Stage2Symbols);

            RemovePdbsFromDir(Path.Combine(Dirs.Stage2, "sdk"));

            return c.Success();
        }

        private static void CleanOutputDir(string directory)
        {
            foreach (var file in FilesToClean)
            {
                FS.RmFilesInDirRecursive(directory, file);
            }
        }

        private static void RemovePdbsFromDir(string directory)
        {
            FS.RmFilesInDirRecursive(directory, "*.pdb");
        }

        private static BuildTargetResult CompileCliSdk(
            BuildTargetContext c,
            DotNetCli dotnet,
            string rootOutputDirectory,
            bool generateNugetPackagesArchive = false)
        {
            var configuration = c.BuildContext.Get<string>("Configuration");
            var buildVersion = c.BuildContext.Get<BuildVersion>("BuildVersion");
            var srcDir = Path.Combine(c.BuildContext.BuildDirectory, "src");
            var sdkOutputDirectory = Path.Combine(rootOutputDirectory, "sdk", buildVersion.NuGetVersion);

            CopySharedFramework(Dirs.SharedFrameworkPublish, rootOutputDirectory);

            FS.CleanBinObj(c, srcDir);
            Rmdir(sdkOutputDirectory);
            Mkdirp(sdkOutputDirectory);

            foreach (var project in ProjectsToPublish)
            {
                dotnet.Publish(
                    "--native-subdirectory",
                    "--output", sdkOutputDirectory,
                    "--configuration", configuration,
                    "--version-suffix", buildVersion.CommitCountString,
                    Path.Combine(srcDir, project))
                    .Execute()
                    .EnsureSuccessful();
            }

            FixModeFlags(sdkOutputDirectory);

            string compilersProject = Path.Combine(Dirs.RepoRoot, "src", "compilers");
            dotnet.Publish(compilersProject,
                    "--output",
                    sdkOutputDirectory,
                    "--framework",
                    "netcoreapp1.0")
                    .Execute()
                    .EnsureSuccessful();

            var compilersDeps = Path.Combine(sdkOutputDirectory, "compilers.deps.json");
            var compilersRuntimeConfig = Path.Combine(sdkOutputDirectory, "compilers.runtimeconfig.json");

            var binaryToCorehostifyRelDir = Path.Combine("runtimes", "any", "native");
            var binaryToCorehostifyOutDir = Path.Combine(sdkOutputDirectory, binaryToCorehostifyRelDir);
            // Corehostify binaries
            foreach (var binaryToCorehostify in BinariesForCoreHost)
            {
                try
                {
                    // Yes, it is .exe even on Linux. This is the managed exe we're working with
                    File.Copy(Path.Combine(binaryToCorehostifyOutDir, $"{binaryToCorehostify}.exe"), Path.Combine(sdkOutputDirectory, $"{binaryToCorehostify}.dll"));
                    File.Move(Path.Combine(binaryToCorehostifyOutDir, $"{binaryToCorehostify}.exe"), Path.Combine(sdkOutputDirectory, $"{binaryToCorehostify}.exe"));
                    var binaryToCoreHostifyDeps = Path.Combine(sdkOutputDirectory, binaryToCorehostify + ".deps.json");

                    File.Copy(compilersDeps, Path.Combine(sdkOutputDirectory, binaryToCorehostify + ".deps.json"));
                    File.Copy(compilersRuntimeConfig, Path.Combine(sdkOutputDirectory, binaryToCorehostify + ".runtimeconfig.json"));
                    PublishMutationUtilties.ChangeEntryPointLibraryName(binaryToCoreHostifyDeps, binaryToCorehostify);
                    foreach (var binaryToRemove in new string[] { "csc", "vbc" })
                    {
                        var assetPath = Path.Combine(binaryToCorehostifyRelDir, $"{binaryToRemove}.exe").Replace(Path.DirectorySeparatorChar, '/');
                        RemoveAssetFromDepsPackages.DoRemoveAssetFromDepsPackages(
                            binaryToCoreHostifyDeps,
                            "runtimeTargets",
                            assetPath);
                        RemoveAssetFromDepsPackages.DoRemoveAssetFromDepsPackages(
                            Path.Combine(sdkOutputDirectory, "dotnet.deps.json"),
                            "runtimeTargets",
                            assetPath);
                    }
                }
                catch (Exception ex)
                {
                    return c.Failed($"Failed to corehostify '{binaryToCorehostify}': {ex.ToString()}");
                }
            }

            // cleanup compilers project output we don't need
            PublishMutationUtilties.CleanPublishOutput(
                sdkOutputDirectory, 
                "compilers", 
                deleteRuntimeConfigJson: true, 
                deleteDepsJson: true);

            // Crossgen SDK directory
            var sharedFrameworkNugetVersion = CliDependencyVersions.SharedFrameworkVersion;
            var sharedFrameworkNameVersionPath = SharedFrameworkPublisher.GetSharedFrameworkPublishPath(
                rootOutputDirectory,
                sharedFrameworkNugetVersion);

            // Copy Host to SDK Directory
            File.Copy(
                Path.Combine(sharedFrameworkNameVersionPath, HostArtifactNames.DotnetHostBaseName), 
                Path.Combine(sdkOutputDirectory, $"corehost{Constants.ExeSuffix}"),
                overwrite: true);
            File.Copy(
                Path.Combine(sharedFrameworkNameVersionPath, HostArtifactNames.DotnetHostFxrBaseName),
                Path.Combine(sdkOutputDirectory, HostArtifactNames.DotnetHostFxrBaseName), 
                overwrite: true);
            File.Copy(
                Path.Combine(sharedFrameworkNameVersionPath, HostArtifactNames.HostPolicyBaseName), 
                Path.Combine(sdkOutputDirectory, HostArtifactNames.HostPolicyBaseName), 
                overwrite: true);

            CrossgenUtil.CrossgenDirectory(
                sharedFrameworkNameVersionPath,
                sdkOutputDirectory);

            // Generate .version file
            var version = buildVersion.NuGetVersion;
            var content = $@"{c.BuildContext["CommitHash"]}{Environment.NewLine}{version}{Environment.NewLine}";
            File.WriteAllText(Path.Combine(sdkOutputDirectory, ".version"), content);

            if(generateNugetPackagesArchive)
            {
                GenerateNuGetPackagesArchive(c, dotnet, sdkOutputDirectory);
            }

            CopyMSBuildTargetsToSDKRoot(sdkOutputDirectory);

            return c.Success();
        }

        private static void CopyMSBuildTargetsToSDKRoot(string sdkOutputDirectory)
        {
            var msbuildTargetsDirectory = Path.Combine(sdkOutputDirectory, "runtimes", "any", "native");
            
            var filesToCopy = new List<string>();
            filesToCopy.AddRange(Directory.EnumerateFiles(msbuildTargetsDirectory, "*.targets", SearchOption.AllDirectories));
            filesToCopy.AddRange(Directory.EnumerateFiles(msbuildTargetsDirectory, "*.Targets", SearchOption.AllDirectories));
            filesToCopy.AddRange(Directory.EnumerateFiles(msbuildTargetsDirectory, "*.props", SearchOption.AllDirectories));
            filesToCopy.AddRange(Directory.EnumerateFiles(msbuildTargetsDirectory, "*.overridetasks", SearchOption.AllDirectories));
            filesToCopy.AddRange(Directory.EnumerateFiles(msbuildTargetsDirectory, "*.tasks", SearchOption.AllDirectories));

            foreach (var fileFullPath in filesToCopy)
            {
                var fileRelativePath = fileFullPath.Substring(msbuildTargetsDirectory.Length + 1);
                var destinationFilePath = Path.Combine(sdkOutputDirectory, fileRelativePath);

                File.Copy(fileFullPath, destinationFilePath, true);
            }
        }

        private static void GenerateNuGetPackagesArchive(
            BuildTargetContext c,
            DotNetCli dotnet,
            string sdkOutputDirectory)
        {
            var nuGetPackagesArchiveProject = Path.Combine(Dirs.Intermediate, "NuGetPackagesArchiveProject");
            var nuGetPackagesArchiveFolder = Path.Combine(Dirs.Intermediate, "nuGetPackagesArchiveFolder");

            RestoreNuGetPackagesArchive(dotnet, nuGetPackagesArchiveProject, nuGetPackagesArchiveFolder);

            CompressNuGetPackagesArchive(c, dotnet, nuGetPackagesArchiveFolder, sdkOutputDirectory);
        }

        private static void RestoreNuGetPackagesArchive(
            DotNetCli dotnet,
            string nuGetPackagesArchiveProject,
            string nuGetPackagesArchiveFolder)
        {
            Rmdir(nuGetPackagesArchiveProject);
            Mkdirp(nuGetPackagesArchiveProject);

            Rmdir(nuGetPackagesArchiveFolder);
            Mkdirp(nuGetPackagesArchiveFolder);

            dotnet.New()
                .WorkingDirectory(nuGetPackagesArchiveProject)
                .Execute()
                .EnsureSuccessful();

            dotnet.Restore("--packages", nuGetPackagesArchiveFolder)
                .WorkingDirectory(nuGetPackagesArchiveProject)
                .Execute()
                .EnsureSuccessful();
        }

        private static void CompressNuGetPackagesArchive(
            BuildTargetContext c,
            DotNetCli dotnet,
            string nuGetPackagesArchiveFolder,
            string sdkOutputDirectory)
        {
            var configuration = c.BuildContext.Get<string>("Configuration");
            var archiverExe = Path.Combine(Dirs.Output, "tools", $"Archiver{Constants.ExeSuffix}");
            var intermediateArchive = Path.Combine(Dirs.Intermediate, "nuGetPackagesArchive.lzma");
            var finalArchive = Path.Combine(sdkOutputDirectory, "nuGetPackagesArchive.lzma");

            Rm(intermediateArchive);
            Rm($"{intermediateArchive}.zip");

            c.Info("Publishing Archiver");
            dotnet.Publish("--output", Path.Combine(Dirs.Output, "tools"), "--configuration", configuration)
                .WorkingDirectory(Path.Combine(Dirs.RepoRoot, "tools", "Archiver"))
                .Execute()
                .EnsureSuccessful();

            Cmd(archiverExe,
                "-a", intermediateArchive,
                nuGetPackagesArchiveFolder)
                .Execute();

            File.Copy(intermediateArchive, finalArchive);
        }

        private static void CopySharedFramework(string sharedFrameworkPublish, string rootOutputDirectory)
        {
            CopyRecursive(sharedFrameworkPublish, rootOutputDirectory);
        }
    }
}