Add project to project graph

This commit is contained in:
Pavel Krymets 2016-04-17 20:53:49 -07:00
parent ef0c3b2cee
commit 3e5b68dc43
14 changed files with 793 additions and 743 deletions

View file

@ -18,7 +18,7 @@ using Newtonsoft.Json.Linq;
using Newtonsoft.Json;
using System.Reflection.PortableExecutable;
namespace Microsoft.Dotnet.Cli.Compiler.Common
namespace Microsoft.DotNet.Cli.Compiler.Common
{
public class Executable
{

View file

@ -16,6 +16,8 @@ namespace Microsoft.DotNet.ProjectModel
{
private string[] _runtimeFallbacks;
public ProjectContextIdentity Identity { get; }
public GlobalSettings GlobalSettings { get; }
public ProjectDescription RootProject { get; }
@ -51,6 +53,7 @@ namespace Microsoft.DotNet.ProjectModel
LibraryManager libraryManager,
LockFile lockfile)
{
Identity = new ProjectContextIdentity(rootProject?.Path, targetFramework);
GlobalSettings = globalSettings;
RootProject = rootProject;
PlatformLibrary = platformLibrary;

View file

@ -0,0 +1,36 @@
using Microsoft.Extensions.Internal;
using NuGet.Frameworks;
namespace Microsoft.DotNet.ProjectModel
{
public struct ProjectContextIdentity
{
public ProjectContextIdentity(string path, NuGetFramework targetFramework)
{
Path = path;
TargetFramework = targetFramework;
}
public string Path { get; }
public NuGetFramework TargetFramework { get; }
public bool Equals(ProjectContextIdentity other)
{
return string.Equals(Path, other.Path) && Equals(TargetFramework, other.TargetFramework);
}
public override bool Equals(object obj)
{
if (ReferenceEquals(null, obj)) return false;
return obj is ProjectContextIdentity && Equals((ProjectContextIdentity) obj);
}
public override int GetHashCode()
{
var combiner = HashCodeCombiner.Start();
combiner.Add(Path);
combiner.Add(TargetFramework);
return combiner.CombinedHash;
}
}
}

View file

@ -1,684 +0,0 @@
// Copyright (c) .NET Foundation and contributors. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Microsoft.Dotnet.Cli.Compiler.Common;
using Microsoft.DotNet.Cli.Compiler.Common;
using Microsoft.DotNet.Cli.Utils;
using Microsoft.DotNet.ProjectModel;
using Microsoft.DotNet.ProjectModel.Utilities;
using Microsoft.DotNet.Tools.Compiler;
using Microsoft.Extensions.PlatformAbstractions;
using Microsoft.DotNet.ProjectModel.Compilation;
namespace Microsoft.DotNet.Tools.Build
{
// todo: Convert CompileContext into a DAG of dependencies: if a node needs recompilation, the entire path up to root needs compilation
// Knows how to orchestrate compilation for a ProjectContext
// Collects icnremental safety checks and transitively compiles a project context
internal class CompileContext
{
public static readonly string[] KnownCompilers = { "csc", "vbc", "fsc" };
private readonly ProjectContext _rootProject;
private readonly ProjectDependenciesFacade _rootProjectDependencies;
private readonly BuilderCommandApp _args;
private readonly IncrementalPreconditions _preconditions;
public bool IsSafeForIncrementalCompilation => !_preconditions.PreconditionsDetected();
public CompileContext(ProjectContext rootProject, BuilderCommandApp args)
{
_rootProject = rootProject;
// Cleaner to clone the args and mutate the clone than have separate CompileContext fields for mutated args
// and then reasoning which ones to get from args and which ones from fields.
_args = (BuilderCommandApp)args.ShallowCopy();
// Set up dependencies
_rootProjectDependencies = new ProjectDependenciesFacade(_rootProject, _args.ConfigValue);
// gather preconditions
_preconditions = GatherIncrementalPreconditions();
}
public bool Compile(bool incremental)
{
CreateOutputDirectories();
return CompileDependencies(incremental) && CompileRootProject(incremental);
}
private bool CompileRootProject(bool incremental)
{
try
{
if (incremental && !NeedsRebuilding(_rootProject, _rootProjectDependencies))
{
return true;
}
var success = InvokeCompileOnRootProject();
PrintSummary(success);
return success;
}
finally
{
StampProjectWithSDKVersion(_rootProject);
}
}
private bool CompileDependencies(bool incremental)
{
if (_args.ShouldSkipDependencies)
{
return true;
}
foreach (var dependency in Sort(_rootProjectDependencies))
{
var dependencyProjectContext = ProjectContext.Create(dependency.Path, dependency.Framework, new[] { _rootProject.RuntimeIdentifier });
try
{
if (incremental && !NeedsRebuilding(dependencyProjectContext, new ProjectDependenciesFacade(dependencyProjectContext, _args.ConfigValue)))
{
continue;
}
if (!InvokeCompileOnDependency(dependency))
{
return false;
}
}
finally
{
StampProjectWithSDKVersion(dependencyProjectContext);
}
}
return true;
}
private bool NeedsRebuilding(ProjectContext project, ProjectDependenciesFacade dependencies)
{
if (CLIChangedSinceLastCompilation(project))
{
Reporter.Output.WriteLine($"Project {project.GetDisplayName()} will be compiled because the version or bitness of the CLI changed since the last build");
return true;
}
var compilerIO = GetCompileIO(project, dependencies);
// rebuild if empty inputs / outputs
if (!(compilerIO.Outputs.Any() && compilerIO.Inputs.Any()))
{
Reporter.Output.WriteLine($"Project {project.GetDisplayName()} will be compiled because it either has empty inputs or outputs");
return true;
}
//rebuild if missing inputs / outputs
if (AnyMissingIO(project, compilerIO.Outputs, "outputs") || AnyMissingIO(project, compilerIO.Inputs, "inputs"))
{
return true;
}
// find the output with the earliest write time
var minOutputPath = compilerIO.Outputs.First();
var minDateUtc = File.GetLastWriteTimeUtc(minOutputPath);
foreach (var outputPath in compilerIO.Outputs)
{
if (File.GetLastWriteTimeUtc(outputPath) >= minDateUtc)
{
continue;
}
minDateUtc = File.GetLastWriteTimeUtc(outputPath);
minOutputPath = outputPath;
}
// find inputs that are older than the earliest output
var newInputs = compilerIO.Inputs.FindAll(p => File.GetLastWriteTimeUtc(p) >= minDateUtc);
if (!newInputs.Any())
{
Reporter.Output.WriteLine($"Project {project.GetDisplayName()} was previously compiled. Skipping compilation.");
return false;
}
Reporter.Output.WriteLine($"Project {project.GetDisplayName()} will be compiled because some of its inputs were newer than its oldest output.");
Reporter.Verbose.WriteLine();
Reporter.Verbose.WriteLine($" Oldest output item:");
Reporter.Verbose.WriteLine($" {minDateUtc.ToLocalTime()}: {minOutputPath}");
Reporter.Verbose.WriteLine();
Reporter.Verbose.WriteLine($" Inputs newer than the oldest output item:");
foreach (var newInput in newInputs)
{
Reporter.Verbose.WriteLine($" {File.GetLastWriteTime(newInput)}: {newInput}");
}
Reporter.Verbose.WriteLine();
return true;
}
private static bool AnyMissingIO(ProjectContext project, IEnumerable<string> items, string itemsType)
{
var missingItems = items.Where(i => !File.Exists(i)).ToList();
if (!missingItems.Any())
{
return false;
}
Reporter.Verbose.WriteLine($"Project {project.GetDisplayName()} will be compiled because expected {itemsType} are missing.");
foreach (var missing in missingItems)
{
Reporter.Verbose.WriteLine($" {missing}");
}
Reporter.Verbose.WriteLine(); ;
return true;
}
private bool CLIChangedSinceLastCompilation(ProjectContext project)
{
var currentVersionFile = DotnetFiles.VersionFile;
var versionFileFromLastCompile = project.GetSDKVersionFile(_args.ConfigValue, _args.BuildBasePathValue, _args.OutputValue);
if (!File.Exists(currentVersionFile))
{
// this CLI does not have a version file; cannot tell if CLI changed
return false;
}
if (!File.Exists(versionFileFromLastCompile))
{
// this is the first compilation; cannot tell if CLI changed
return false;
}
var currentContent = ComputeCurrentVersionFileData();
var versionsAreEqual = string.Equals(currentContent, File.ReadAllText(versionFileFromLastCompile), StringComparison.OrdinalIgnoreCase);
return !versionsAreEqual;
}
private void StampProjectWithSDKVersion(ProjectContext project)
{
if (File.Exists(DotnetFiles.VersionFile))
{
var projectVersionFile = project.GetSDKVersionFile(_args.ConfigValue, _args.BuildBasePathValue, _args.OutputValue);
var parentDirectory = Path.GetDirectoryName(projectVersionFile);
if (!Directory.Exists(parentDirectory))
{
Directory.CreateDirectory(parentDirectory);
}
string content = ComputeCurrentVersionFileData();
File.WriteAllText(projectVersionFile, content);
}
else
{
Reporter.Verbose.WriteLine($"Project {project.GetDisplayName()} was not stamped with a CLI version because the version file does not exist: {DotnetFiles.VersionFile}");
}
}
private static string ComputeCurrentVersionFileData()
{
var content = File.ReadAllText(DotnetFiles.VersionFile);
content += Environment.NewLine;
content += PlatformServices.Default.Runtime.GetRuntimeIdentifier();
return content;
}
private void PrintSummary(bool success)
{
// todo: Ideally it's the builder's responsibility for adding the time elapsed. That way we avoid cross cutting display concerns between compile and build for printing time elapsed
if (success)
{
Reporter.Output.Write(" " + _preconditions.LogMessage());
Reporter.Output.WriteLine();
}
Reporter.Output.WriteLine();
}
private void CreateOutputDirectories()
{
if (!string.IsNullOrEmpty(_args.OutputValue))
{
Directory.CreateDirectory(_args.OutputValue);
}
if (!string.IsNullOrEmpty(_args.BuildBasePathValue))
{
Directory.CreateDirectory(_args.BuildBasePathValue);
}
}
private IncrementalPreconditions GatherIncrementalPreconditions()
{
var preconditions = new IncrementalPreconditions(_args.ShouldPrintIncrementalPreconditions);
if (_args.ShouldNotUseIncrementality)
{
preconditions.AddForceUnsafePrecondition();
}
var projectsToCheck = GetProjectsToCheck();
foreach (var project in projectsToCheck)
{
CollectScriptPreconditions(project, preconditions);
CollectCompilerNamePreconditions(project, preconditions);
CollectCheckPathProbingPreconditions(project, preconditions);
}
return preconditions;
}
// check the entire project tree that needs to be compiled, duplicated for each framework
private List<ProjectContext> GetProjectsToCheck()
{
if (_args.ShouldSkipDependencies)
{
return new List<ProjectContext>(1) { _rootProject };
}
// include initial root project
var contextsToCheck = new List<ProjectContext>(1 + _rootProjectDependencies.ProjectDependenciesWithSources.Count) { _rootProject };
// convert ProjectDescription to ProjectContext
var dependencyContexts = _rootProjectDependencies.ProjectDependenciesWithSources.Select
(keyValuePair => ProjectContext.Create(keyValuePair.Value.Path, keyValuePair.Value.Framework));
contextsToCheck.AddRange(dependencyContexts);
return contextsToCheck;
}
private void CollectCheckPathProbingPreconditions(ProjectContext project, IncrementalPreconditions preconditions)
{
var pathCommands = CompilerUtil.GetCommandsInvokedByCompile(project)
.Select(commandName => Command.CreateDotNet(commandName, Enumerable.Empty<string>(), project.TargetFramework))
.Where(c => c.ResolutionStrategy.Equals(CommandResolutionStrategy.Path));
foreach (var pathCommand in pathCommands)
{
preconditions.AddPathProbingPrecondition(project.ProjectName(), pathCommand.CommandName);
}
}
private void CollectCompilerNamePreconditions(ProjectContext project, IncrementalPreconditions preconditions)
{
if (project.ProjectFile != null)
{
var projectCompiler = project.ProjectFile.CompilerName;
if (!KnownCompilers.Any(knownCompiler => knownCompiler.Equals(projectCompiler, StringComparison.Ordinal)))
{
preconditions.AddUnknownCompilerPrecondition(project.ProjectName(), projectCompiler);
}
}
}
private void CollectScriptPreconditions(ProjectContext project, IncrementalPreconditions preconditions)
{
if (project.ProjectFile != null)
{
var preCompileScripts = project.ProjectFile.Scripts.GetOrEmpty(ScriptNames.PreCompile);
var postCompileScripts = project.ProjectFile.Scripts.GetOrEmpty(ScriptNames.PostCompile);
if (preCompileScripts.Any())
{
preconditions.AddPrePostScriptPrecondition(project.ProjectName(), ScriptNames.PreCompile);
}
if (postCompileScripts.Any())
{
preconditions.AddPrePostScriptPrecondition(project.ProjectName(), ScriptNames.PostCompile);
}
}
}
private bool InvokeCompileOnDependency(ProjectDescription projectDependency)
{
var args = new List<string>();
args.Add("--framework");
args.Add($"{projectDependency.Framework}");
args.Add("--configuration");
args.Add(_args.ConfigValue);
args.Add(projectDependency.Project.ProjectDirectory);
if (!string.IsNullOrWhiteSpace(_args.RuntimeValue))
{
args.Add("--runtime");
args.Add(_args.RuntimeValue);
}
if (!string.IsNullOrEmpty(_args.VersionSuffixValue))
{
args.Add("--version-suffix");
args.Add(_args.VersionSuffixValue);
}
if (!string.IsNullOrWhiteSpace(_args.BuildBasePathValue))
{
args.Add("--build-base-path");
args.Add(_args.BuildBasePathValue);
}
var compileResult = CompileCommand.Run(args.ToArray());
return compileResult == 0;
}
private bool InvokeCompileOnRootProject()
{
// todo: add methods to CompilerCommandApp to generate the arg string?
var args = new List<string>();
args.Add("--framework");
args.Add(_rootProject.TargetFramework.ToString());
args.Add("--configuration");
args.Add(_args.ConfigValue);
if (!string.IsNullOrWhiteSpace(_args.RuntimeValue))
{
args.Add("--runtime");
args.Add(_args.RuntimeValue);
}
if (!string.IsNullOrEmpty(_args.OutputValue))
{
args.Add("--output");
args.Add(_args.OutputValue);
}
if (!string.IsNullOrEmpty(_args.VersionSuffixValue))
{
args.Add("--version-suffix");
args.Add(_args.VersionSuffixValue);
}
if (!string.IsNullOrEmpty(_args.BuildBasePathValue))
{
args.Add("--build-base-path");
args.Add(_args.BuildBasePathValue);
}
//native args
if (_args.IsNativeValue)
{
args.Add("--native");
}
if (_args.IsCppModeValue)
{
args.Add("--cpp");
}
if (!string.IsNullOrWhiteSpace(_args.CppCompilerFlagsValue))
{
args.Add("--cppcompilerflags");
args.Add(_args.CppCompilerFlagsValue);
}
if (!string.IsNullOrWhiteSpace(_args.ArchValue))
{
args.Add("--arch");
args.Add(_args.ArchValue);
}
foreach (var ilcArg in _args.IlcArgsValue)
{
args.Add("--ilcarg");
args.Add(ilcArg);
}
if (!string.IsNullOrWhiteSpace(_args.IlcPathValue))
{
args.Add("--ilcpath");
args.Add(_args.IlcPathValue);
}
if (!string.IsNullOrWhiteSpace(_args.IlcSdkPathValue))
{
args.Add("--ilcsdkpath");
args.Add(_args.IlcSdkPathValue);
}
args.Add(_rootProject.ProjectDirectory);
var compileResult = CompileCommand.Run(args.ToArray());
var succeeded = compileResult == 0;
if (succeeded)
{
MakeRunnable();
}
return succeeded;
}
private void CopyCompilationOutput(OutputPaths outputPaths)
{
var dest = outputPaths.RuntimeOutputPath;
var source = outputPaths.CompilationOutputPath;
// No need to copy if dest and source are the same
if(string.Equals(dest, source, StringComparison.OrdinalIgnoreCase))
{
return;
}
foreach (var file in outputPaths.CompilationFiles.All())
{
var destFileName = file.Replace(source, dest);
var directoryName = Path.GetDirectoryName(destFileName);
if (!Directory.Exists(directoryName))
{
Directory.CreateDirectory(directoryName);
}
File.Copy(file, destFileName, true);
}
}
private void MakeRunnable()
{
var runtimeContext = _rootProject.ProjectFile.HasRuntimeOutput(_args.ConfigValue) ?
_rootProject.CreateRuntimeContext(_args.GetRuntimes()) :
_rootProject;
var outputPaths = runtimeContext.GetOutputPaths(_args.ConfigValue, _args.BuildBasePathValue, _args.OutputValue);
var libraryExporter = runtimeContext.CreateExporter(_args.ConfigValue, _args.BuildBasePathValue);
CopyCompilationOutput(outputPaths);
var executable = new Executable(runtimeContext, outputPaths, libraryExporter, _args.ConfigValue);
executable.MakeCompilationOutputRunnable();
}
private static IEnumerable<ProjectDescription> Sort(ProjectDependenciesFacade dependencies)
{
var outputs = new List<ProjectDescription>();
foreach (var pair in dependencies.Dependencies)
{
Sort(pair.Value, dependencies, outputs);
}
return outputs.Distinct(new ProjectComparer());
}
private static void Sort(LibraryExport node, ProjectDependenciesFacade dependencies, IList<ProjectDescription> outputs)
{
// Sorts projects in dependency order so that we only build them once per chain
ProjectDescription projectDependency;
foreach (var dependency in node.Library.Dependencies)
{
// Sort the children
Sort(dependencies.Dependencies[dependency.Name], dependencies, outputs);
}
// Add this node to the list if it is a project
if (dependencies.ProjectDependenciesWithSources.TryGetValue(node.Library.Identity.Name, out projectDependency))
{
// Add to the list of projects to build
outputs.Add(projectDependency);
}
}
private class ProjectComparer : IEqualityComparer<ProjectDescription>
{
public bool Equals(ProjectDescription x, ProjectDescription y) => string.Equals(x.Identity.Name, y.Identity.Name, StringComparison.Ordinal);
public int GetHashCode(ProjectDescription obj) => obj.Identity.Name.GetHashCode();
}
public struct CompilerIO
{
public readonly List<string> Inputs;
public readonly List<string> Outputs;
public CompilerIO(List<string> inputs, List<string> outputs)
{
Inputs = inputs;
Outputs = outputs;
}
}
// computes all the inputs and outputs that would be used in the compilation of a project
// ensures that all paths are files
// ensures no missing inputs
public CompilerIO GetCompileIO(ProjectContext project, ProjectDependenciesFacade dependencies)
{
var buildConfiguration = _args.ConfigValue;
var buildBasePath = _args.BuildBasePathValue;
var outputPath = _args.OutputValue;
var isRootProject = project == _rootProject;
var compilerIO = new CompilerIO(new List<string>(), new List<string>());
var calculator = project.GetOutputPaths(buildConfiguration, buildBasePath, outputPath);
var binariesOutputPath = calculator.CompilationOutputPath;
// input: project.json
compilerIO.Inputs.Add(project.ProjectFile.ProjectFilePath);
// input: lock file; find when dependencies change
AddLockFile(project, compilerIO);
// input: source files
compilerIO.Inputs.AddRange(CompilerUtil.GetCompilationSources(project));
// todo: Factor out dependency resolution between Build and Compile. Ideally Build injects the dependencies into Compile
// input: dependencies
AddDependencies(dependencies, compilerIO);
var allOutputPath = new HashSet<string>(calculator.CompilationFiles.All());
if (isRootProject && project.ProjectFile.HasRuntimeOutput(buildConfiguration))
{
var runtimeContext = project.CreateRuntimeContext(_args.GetRuntimes());
foreach (var path in runtimeContext.GetOutputPaths(buildConfiguration, buildBasePath, outputPath).RuntimeFiles.All())
{
allOutputPath.Add(path);
}
}
// output: compiler outputs
foreach (var path in allOutputPath)
{
compilerIO.Outputs.Add(path);
}
// input compilation options files
AddCompilationOptions(project, buildConfiguration, compilerIO);
// input / output: resources with culture
AddNonCultureResources(project, calculator.IntermediateOutputDirectoryPath, compilerIO);
// input / output: resources without culture
AddCultureResources(project, binariesOutputPath, compilerIO);
return compilerIO;
}
private static void AddLockFile(ProjectContext project, CompilerIO compilerIO)
{
if (project.LockFile == null)
{
var errorMessage = $"Project {project.ProjectName()} does not have a lock file.";
Reporter.Error.WriteLine(errorMessage);
throw new InvalidOperationException(errorMessage);
}
compilerIO.Inputs.Add(project.LockFile.LockFilePath);
if (project.LockFile.ExportFile != null)
{
compilerIO.Inputs.Add(project.LockFile.ExportFile.ExportFilePath);
}
}
private static void AddDependencies(ProjectDependenciesFacade dependencies, CompilerIO compilerIO)
{
// add dependency sources that need compilation
compilerIO.Inputs.AddRange(dependencies.ProjectDependenciesWithSources.Values.SelectMany(p => p.Project.Files.SourceFiles));
// non project dependencies get captured by changes in the lock file
}
private static void AddCompilationOptions(ProjectContext project, string config, CompilerIO compilerIO)
{
var compilerOptions = project.ResolveCompilationOptions(config);
// input: key file
if (compilerOptions.KeyFile != null)
{
compilerIO.Inputs.Add(compilerOptions.KeyFile);
}
}
private static void AddNonCultureResources(ProjectContext project, string intermediaryOutputPath, CompilerIO compilerIO)
{
foreach (var resourceIO in CompilerUtil.GetNonCultureResources(project.ProjectFile, intermediaryOutputPath))
{
compilerIO.Inputs.Add(resourceIO.InputFile);
if (resourceIO.OutputFile != null)
{
compilerIO.Outputs.Add(resourceIO.OutputFile);
}
}
}
private static void AddCultureResources(ProjectContext project, string outputPath, CompilerIO compilerIO)
{
foreach (var cultureResourceIO in CompilerUtil.GetCultureResources(project.ProjectFile, outputPath))
{
compilerIO.Inputs.AddRange(cultureResourceIO.InputFileToMetadata.Keys);
if (cultureResourceIO.OutputFile != null)
{
compilerIO.Outputs.Add(cultureResourceIO.OutputFile);
}
}
}
}
}

View file

@ -0,0 +1,16 @@
using System.Collections.Generic;
namespace Microsoft.DotNet.Tools.Build
{
public struct CompilerIO
{
public readonly List<string> Inputs;
public readonly List<string> Outputs;
public CompilerIO(List<string> inputs, List<string> outputs)
{
Inputs = inputs;
Outputs = outputs;
}
}
}

View file

@ -0,0 +1,156 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Microsoft.DotNet.Tools.Compiler;
using Microsoft.DotNet.ProjectModel;
using Microsoft.DotNet.Cli.Utils;
using Microsoft.DotNet.Cli.Compiler.Common;
namespace Microsoft.DotNet.Tools.Build
{
internal class CompilerIOManager
{
private readonly string _configuration;
private readonly string _outputPath;
private readonly string _buildBasePath;
private readonly IList<string> _runtimes;
public CompilerIOManager(string configuration,
string outputPath,
string buildBasePath,
IEnumerable<string> runtimes)
{
_configuration = configuration;
_outputPath = outputPath;
_buildBasePath = buildBasePath;
_runtimes = runtimes.ToList();
}
public bool AnyMissingIO(ProjectContext project, IEnumerable<string> items, string itemsType)
{
var missingItems = items.Where(i => !File.Exists(i)).ToList();
if (!missingItems.Any())
{
return false;
}
Reporter.Verbose.WriteLine($"Project {project.GetDisplayName()} will be compiled because expected {itemsType} are missing.");
foreach (var missing in missingItems)
{
Reporter.Verbose.WriteLine($" {missing}");
}
Reporter.Verbose.WriteLine(); ;
return true;
}
// computes all the inputs and outputs that would be used in the compilation of a project
// ensures that all paths are files
// ensures no missing inputs
public CompilerIO GetCompileIO(ProjectGraphNode graphNode)
{
var isRootProject = graphNode.IsRoot;
var project = graphNode.ProjectContext;
var compilerIO = new CompilerIO(new List<string>(), new List<string>());
var calculator = project.GetOutputPaths(_configuration, _buildBasePath, _outputPath);
var binariesOutputPath = calculator.CompilationOutputPath;
// input: project.json
compilerIO.Inputs.Add(project.ProjectFile.ProjectFilePath);
// input: lock file; find when dependencies change
AddLockFile(project, compilerIO);
// input: source files
compilerIO.Inputs.AddRange(CompilerUtil.GetCompilationSources(project));
var allOutputPath = new HashSet<string>(calculator.CompilationFiles.All());
if (isRootProject && project.ProjectFile.HasRuntimeOutput(_configuration))
{
var runtimeContext = project.CreateRuntimeContext(_runtimes);
foreach (var path in runtimeContext.GetOutputPaths(_configuration, _buildBasePath, _outputPath).RuntimeFiles.All())
{
allOutputPath.Add(path);
}
}
// output: compiler outputs
foreach (var path in allOutputPath)
{
compilerIO.Outputs.Add(path);
}
// input compilation options files
AddCompilationOptions(project, _configuration, compilerIO);
// input / output: resources with culture
AddNonCultureResources(project, calculator.IntermediateOutputDirectoryPath, compilerIO);
// input / output: resources without culture
AddCultureResources(project, binariesOutputPath, compilerIO);
return compilerIO;
}
private static void AddLockFile(ProjectContext project, CompilerIO compilerIO)
{
if (project.LockFile == null)
{
var errorMessage = $"Project {project.ProjectName()} does not have a lock file.";
Reporter.Error.WriteLine(errorMessage);
throw new InvalidOperationException(errorMessage);
}
compilerIO.Inputs.Add(project.LockFile.LockFilePath);
if (project.LockFile.ExportFile != null)
{
compilerIO.Inputs.Add(project.LockFile.ExportFile.ExportFilePath);
}
}
private static void AddCompilationOptions(ProjectContext project, string config, CompilerIO compilerIO)
{
var compilerOptions = project.ResolveCompilationOptions(config);
// input: key file
if (compilerOptions.KeyFile != null)
{
compilerIO.Inputs.Add(compilerOptions.KeyFile);
}
}
private static void AddNonCultureResources(ProjectContext project, string intermediaryOutputPath, CompilerIO compilerIO)
{
foreach (var resourceIO in CompilerUtil.GetNonCultureResources(project.ProjectFile, intermediaryOutputPath))
{
compilerIO.Inputs.Add(resourceIO.InputFile);
if (resourceIO.OutputFile != null)
{
compilerIO.Outputs.Add(resourceIO.OutputFile);
}
}
}
private static void AddCultureResources(ProjectContext project, string outputPath, CompilerIO compilerIO)
{
foreach (var cultureResourceIO in CompilerUtil.GetCultureResources(project.ProjectFile, outputPath))
{
compilerIO.Inputs.AddRange(cultureResourceIO.InputFileToMetadata.Keys);
if (cultureResourceIO.OutputFile != null)
{
compilerIO.Outputs.Add(cultureResourceIO.OutputFile);
}
}
}
}
}

View file

@ -0,0 +1,264 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Microsoft.DotNet.Cli;
using Microsoft.DotNet.Cli.Compiler.Common;
using Microsoft.DotNet.Cli.Utils;
using Microsoft.DotNet.ProjectModel;
using Microsoft.DotNet.Tools.Compiler;
using Microsoft.Extensions.PlatformAbstractions;
namespace Microsoft.DotNet.Tools.Build
{
class DotNetProjectBuilder : ProjectBuilder
{
private readonly BuilderCommandApp _args;
private readonly IncrementalPreconditionManager _preconditionManager;
private readonly CompilerIOManager _compilerIOManager;
private readonly ScriptRunner _scriptRunner;
private readonly DotNetCommandFactory _commandFactory;
public DotNetProjectBuilder(BuilderCommandApp args) : base(args.ShouldSkipDependencies)
{
_args = (BuilderCommandApp)args.ShallowCopy();
_preconditionManager = new IncrementalPreconditionManager(
args.ShouldPrintIncrementalPreconditions,
args.ShouldNotUseIncrementality,
args.ShouldSkipDependencies);
_compilerIOManager = new CompilerIOManager(
args.ConfigValue,
args.OutputValue,
args.BuildBasePathValue,
args.GetRuntimes()
);
_scriptRunner = new ScriptRunner();
_commandFactory = new DotNetCommandFactory();
}
private void StampProjectWithSDKVersion(ProjectContext project)
{
if (File.Exists(DotnetFiles.VersionFile))
{
var projectVersionFile = project.GetSDKVersionFile(_args.ConfigValue, _args.BuildBasePathValue, _args.OutputValue);
var parentDirectory = Path.GetDirectoryName(projectVersionFile);
if (!Directory.Exists(parentDirectory))
{
Directory.CreateDirectory(parentDirectory);
}
string content = ComputeCurrentVersionFileData();
File.WriteAllText(projectVersionFile, content);
}
else
{
Reporter.Verbose.WriteLine($"Project {project.GetDisplayName()} was not stamped with a CLI version because the version file does not exist: {DotnetFiles.VersionFile}");
}
}
private static string ComputeCurrentVersionFileData()
{
var content = File.ReadAllText(DotnetFiles.VersionFile);
content += Environment.NewLine;
content += PlatformServices.Default.Runtime.GetRuntimeIdentifier();
return content;
}
private void PrintSummary(ProjectGraphNode projectNode, bool success)
{
// todo: Ideally it's the builder's responsibility for adding the time elapsed. That way we avoid cross cutting display concerns between compile and build for printing time elapsed
if (success)
{
var preconditions = _preconditionManager.GetIncrementalPreconditions(projectNode);
Reporter.Output.Write(" " + preconditions.LogMessage());
Reporter.Output.WriteLine();
}
Reporter.Output.WriteLine();
}
private void CreateOutputDirectories()
{
if (!string.IsNullOrEmpty(_args.OutputValue))
{
Directory.CreateDirectory(_args.OutputValue);
}
if (!string.IsNullOrEmpty(_args.BuildBasePathValue))
{
Directory.CreateDirectory(_args.BuildBasePathValue);
}
}
private void CopyCompilationOutput(OutputPaths outputPaths)
{
var dest = outputPaths.RuntimeOutputPath;
var source = outputPaths.CompilationOutputPath;
// No need to copy if dest and source are the same
if (string.Equals(dest, source, StringComparison.OrdinalIgnoreCase))
{
return;
}
foreach (var file in outputPaths.CompilationFiles.All())
{
var destFileName = file.Replace(source, dest);
var directoryName = Path.GetDirectoryName(destFileName);
if (!Directory.Exists(directoryName))
{
Directory.CreateDirectory(directoryName);
}
File.Copy(file, destFileName, true);
}
}
private void MakeRunnable(ProjectGraphNode graphNode)
{
var runtimeContext = graphNode.ProjectContext.ProjectFile.HasRuntimeOutput(_args.ConfigValue) ?
graphNode.ProjectContext.CreateRuntimeContext(_args.GetRuntimes()) :
graphNode.ProjectContext;
var outputPaths = runtimeContext.GetOutputPaths(_args.ConfigValue, _args.BuildBasePathValue, _args.OutputValue);
var libraryExporter = runtimeContext.CreateExporter(_args.ConfigValue, _args.BuildBasePathValue);
CopyCompilationOutput(outputPaths);
var executable = new Executable(runtimeContext, outputPaths, libraryExporter, _args.ConfigValue);
executable.MakeCompilationOutputRunnable();
}
protected override CompilationResult RunCompile(ProjectGraphNode projectNode)
{
try
{
var managedCompiler = new ManagedCompiler(_scriptRunner, _commandFactory);
var success = managedCompiler.Compile(projectNode.ProjectContext, _args);
if (projectNode.IsRoot)
{
MakeRunnable(projectNode);
PrintSummary(projectNode, success);
}
return success ? CompilationResult.Success : CompilationResult.Failure;
}
finally
{
StampProjectWithSDKVersion(projectNode.ProjectContext);
}
}
protected override void ProjectSkiped(ProjectGraphNode projectNode)
{
StampProjectWithSDKVersion(projectNode.ProjectContext);
}
private bool CLIChangedSinceLastCompilation(ProjectContext project)
{
var currentVersionFile = DotnetFiles.VersionFile;
var versionFileFromLastCompile = project.GetSDKVersionFile(_args.ConfigValue, _args.BuildBasePathValue, _args.OutputValue);
if (!File.Exists(currentVersionFile))
{
// this CLI does not have a version file; cannot tell if CLI changed
return false;
}
if (!File.Exists(versionFileFromLastCompile))
{
// this is the first compilation; cannot tell if CLI changed
return false;
}
var currentContent = ComputeCurrentVersionFileData();
var versionsAreEqual = string.Equals(currentContent, File.ReadAllText(versionFileFromLastCompile), StringComparison.OrdinalIgnoreCase);
return !versionsAreEqual;
}
protected override bool NeedsRebuilding(ProjectGraphNode graphNode)
{
var project = graphNode.ProjectContext;
if (_args.ShouldNotUseIncrementality)
{
return true;
}
if (!_args.ShouldSkipDependencies &&
graphNode.Dependencies.Any(d => GetCompilationResult(d) != CompilationResult.IncrementalSkip))
{
Reporter.Output.WriteLine($"Project {project.GetDisplayName()} will be compiled because some of it's dependencies changed");
return true;
}
var preconditions = _preconditionManager.GetIncrementalPreconditions(graphNode);
if (preconditions.PreconditionsDetected())
{
return true;
}
if (CLIChangedSinceLastCompilation(project))
{
Reporter.Output.WriteLine($"Project {project.GetDisplayName()} will be compiled because the version or bitness of the CLI changed since the last build");
return true;
}
var compilerIO = _compilerIOManager.GetCompileIO(graphNode);
// rebuild if empty inputs / outputs
if (!(compilerIO.Outputs.Any() && compilerIO.Inputs.Any()))
{
Reporter.Output.WriteLine($"Project {project.GetDisplayName()} will be compiled because it either has empty inputs or outputs");
return true;
}
//rebuild if missing inputs / outputs
if (_compilerIOManager.AnyMissingIO(project, compilerIO.Outputs, "outputs") || _compilerIOManager.AnyMissingIO(project, compilerIO.Inputs, "inputs"))
{
return true;
}
// find the output with the earliest write time
var minOutputPath = compilerIO.Outputs.First();
var minDateUtc = File.GetLastWriteTimeUtc(minOutputPath);
foreach (var outputPath in compilerIO.Outputs)
{
if (File.GetLastWriteTimeUtc(outputPath) >= minDateUtc)
{
continue;
}
minDateUtc = File.GetLastWriteTimeUtc(outputPath);
minOutputPath = outputPath;
}
// find inputs that are older than the earliest output
var newInputs = compilerIO.Inputs.FindAll(p => File.GetLastWriteTimeUtc(p) >= minDateUtc);
if (!newInputs.Any())
{
Reporter.Output.WriteLine($"Project {project.GetDisplayName()} was previously compiled. Skipping compilation.");
return false;
}
Reporter.Output.WriteLine($"Project {project.GetDisplayName()} will be compiled because some of its inputs were newer than its oldest output.");
Reporter.Verbose.WriteLine();
Reporter.Verbose.WriteLine($" Oldest output item:");
Reporter.Verbose.WriteLine($" {minDateUtc.ToLocalTime()}: {minOutputPath}");
Reporter.Verbose.WriteLine();
Reporter.Verbose.WriteLine($" Inputs newer than the oldest output item:");
foreach (var newInput in newInputs)
{
Reporter.Verbose.WriteLine($" {File.GetLastWriteTime(newInput)}: {newInput}");
}
Reporter.Verbose.WriteLine();
return true;
}
}
}

View file

@ -0,0 +1,117 @@
using System;
using System.Collections.Generic;
using System.Linq;
using Microsoft.DotNet.Tools.Compiler;
using Microsoft.DotNet.ProjectModel;
using Microsoft.DotNet.Cli.Utils;
using Microsoft.DotNet.Cli.Compiler.Common;
using Microsoft.DotNet.ProjectModel.Utilities;
namespace Microsoft.DotNet.Tools.Build
{
class IncrementalPreconditionManager
{
private readonly bool _printPreconditions;
private readonly bool _forceNonIncremental;
private readonly bool _skipDependencies;
private Dictionary<ProjectContextIdentity, IncrementalPreconditions> _preconditions;
public IncrementalPreconditionManager(bool printPreconditions, bool forceNonIncremental, bool skipDependencies)
{
_printPreconditions = printPreconditions;
_forceNonIncremental = forceNonIncremental;
_skipDependencies = skipDependencies;
_preconditions = new Dictionary<ProjectContextIdentity, IncrementalPreconditions>();
}
public static readonly string[] KnownCompilers = { "csc", "vbc", "fsc" };
public IncrementalPreconditions GetIncrementalPreconditions(ProjectGraphNode projectNode)
{
IncrementalPreconditions preconditions;
if (_preconditions.TryGetValue(projectNode.ProjectContext.Identity, out preconditions))
{
return preconditions;
}
preconditions = new IncrementalPreconditions(_printPreconditions);
if (_forceNonIncremental)
{
preconditions.AddForceUnsafePrecondition();
}
var projectsToCheck = GetProjectsToCheck(projectNode);
foreach (var project in projectsToCheck)
{
CollectScriptPreconditions(project, preconditions);
CollectCompilerNamePreconditions(project, preconditions);
CollectCheckPathProbingPreconditions(project, preconditions);
}
_preconditions[projectNode.ProjectContext.Identity] = preconditions;
return preconditions;
}
private List<ProjectContext> GetProjectsToCheck(ProjectGraphNode projectNode)
{
if (_skipDependencies)
{
return new List<ProjectContext>(1) { projectNode.ProjectContext };
}
// include initial root project
var contextsToCheck = new List<ProjectContext>(1 + projectNode.Dependencies.Count) { projectNode.ProjectContext };
// TODO: not traversing deeper than 1 level of dependencies
contextsToCheck.AddRange(projectNode.Dependencies.Select(n => n.ProjectContext));
return contextsToCheck;
}
private void CollectCheckPathProbingPreconditions(ProjectContext project, IncrementalPreconditions preconditions)
{
var pathCommands = CompilerUtil.GetCommandsInvokedByCompile(project)
.Select(commandName => Command.CreateDotNet(commandName, Enumerable.Empty<string>(), project.TargetFramework))
.Where(c => c.ResolutionStrategy.Equals(CommandResolutionStrategy.Path));
foreach (var pathCommand in pathCommands)
{
preconditions.AddPathProbingPrecondition(project.ProjectName(), pathCommand.CommandName);
}
}
private void CollectCompilerNamePreconditions(ProjectContext project, IncrementalPreconditions preconditions)
{
if (project.ProjectFile != null)
{
var projectCompiler = project.ProjectFile.CompilerName;
if (!KnownCompilers.Any(knownCompiler => knownCompiler.Equals(projectCompiler, StringComparison.Ordinal)))
{
preconditions.AddUnknownCompilerPrecondition(project.ProjectName(), projectCompiler);
}
}
}
private void CollectScriptPreconditions(ProjectContext project, IncrementalPreconditions preconditions)
{
if (project.ProjectFile != null)
{
var preCompileScripts = project.ProjectFile.Scripts.GetOrEmpty(ScriptNames.PreCompile);
var postCompileScripts = project.ProjectFile.Scripts.GetOrEmpty(ScriptNames.PostCompile);
if (preCompileScripts.Any())
{
preconditions.AddPrePostScriptPrecondition(project.ProjectName(), ScriptNames.PreCompile);
}
if (postCompileScripts.Any())
{
preconditions.AddPrePostScriptPrecondition(project.ProjectName(), ScriptNames.PostCompile);
}
}
}
}
}

View file

@ -5,10 +5,10 @@ using System;
using System.Collections.Generic;
using System.Linq;
using System.IO;
using Microsoft.DotNet.Cli.Utils;
using System.Threading.Tasks;
using Microsoft.DotNet.ProjectModel;
using Microsoft.DotNet.Tools.Compiler;
using Microsoft.DotNet.Cli.Utils;
namespace Microsoft.DotNet.Tools.Build
{
@ -36,11 +36,10 @@ namespace Microsoft.DotNet.Tools.Build
private static bool OnExecute(List<ProjectContext> contexts, CompilerCommandApp args)
{
var compileContexts = contexts.Select(context => new CompileContext(context, (BuilderCommandApp)args)).ToList();
var incrementalSafe = compileContexts.All(c => c.IsSafeForIncrementalCompilation);
return compileContexts.All(c => c.Compile(incrementalSafe));
var graphCollector = new ProjectGraphCollector((project, target) => ProjectContext.Create(project, target));
var graph = graphCollector.Collect(contexts).ToArray();
var builder = new DotNetProjectBuilder((BuilderCommandApp) args);
return builder.Build(graph).All(r => r != CompilationResult.Failure);
}
}
}

View file

@ -0,0 +1,97 @@
using System;
using System.Collections.Generic;
using System.Linq;
using Microsoft.DotNet.ProjectModel;
using NuGet.Frameworks;
namespace Microsoft.DotNet.Tools.Build
{
internal enum CompilationResult
{
IncrementalSkip, Success, Failure
}
internal abstract class ProjectBuilder
{
private readonly bool _skipDependencies;
public ProjectBuilder(bool skipDependencies)
{
_skipDependencies = skipDependencies;
}
private Dictionary<ProjectContextIdentity, CompilationResult> _compilationResults = new Dictionary<ProjectContextIdentity, CompilationResult>();
public IEnumerable<CompilationResult> Build(IEnumerable<ProjectGraphNode> roots)
{
foreach (var projectNode in roots)
{
Console.WriteLine(projectNode.ProjectContext.Identity.TargetFramework);
yield return Build(projectNode);
}
}
protected CompilationResult? GetCompilationResult(ProjectGraphNode projectNode)
{
CompilationResult result;
if (_compilationResults.TryGetValue(projectNode.ProjectContext.Identity, out result))
{
return result;
}
return null;
}
protected virtual bool NeedsRebuilding(ProjectGraphNode projectNode)
{
return true;
}
protected virtual void ProjectSkiped(ProjectGraphNode projectNode)
{
}
protected abstract CompilationResult RunCompile(ProjectGraphNode projectNode);
private CompilationResult Build(ProjectGraphNode projectNode)
{
CompilationResult result;
if (_compilationResults.TryGetValue(projectNode.ProjectContext.Identity, out result))
{
return result;
}
result = CompileWithDependencies(projectNode);
_compilationResults[projectNode.ProjectContext.Identity] = result;
return result;
}
private CompilationResult CompileWithDependencies(ProjectGraphNode projectNode)
{
if (!_skipDependencies)
{
foreach (var dependency in projectNode.Dependencies)
{
var context = dependency.ProjectContext;
if (!context.ProjectFile.Files.SourceFiles.Any())
{
continue;
}
var result = Build(dependency);
if (result == CompilationResult.Failure)
{
return CompilationResult.Failure;
}
}
}
if (NeedsRebuilding(projectNode))
{
return RunCompile(projectNode);
}
else
{
ProjectSkiped(projectNode);
return CompilationResult.IncrementalSkip;
}
}
}
}

View file

@ -1,50 +0,0 @@
// Copyright (c) .NET Foundation and contributors. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
using System.Collections.Generic;
using System.Linq;
using Microsoft.DotNet.ProjectModel;
using Microsoft.DotNet.ProjectModel.Compilation;
namespace Microsoft.DotNet.Tools.Build
{
// facade over the dependencies of a project context
internal class ProjectDependenciesFacade
{
// projectName -> ProjectDescription
public Dictionary<string, ProjectDescription> ProjectDependenciesWithSources { get; }
public Dictionary<string, LibraryExport> Dependencies { get; }
public ProjectDependenciesFacade(ProjectContext rootProject, string configValue)
{
Dependencies = GetProjectDependencies(rootProject, configValue);
ProjectDependenciesWithSources = new Dictionary<string, ProjectDescription>();
// Build project references
foreach (var dependency in Dependencies)
{
var projectDependency = dependency.Value.Library as ProjectDescription;
if (projectDependency != null && projectDependency.Resolved && projectDependency.Project.Files.SourceFiles.Any())
{
ProjectDependenciesWithSources[projectDependency.Identity.Name] = projectDependency;
}
}
}
// todo make extension of ProjectContext?
private static Dictionary<string, LibraryExport> GetProjectDependencies(ProjectContext projectContext, string configuration)
{
// Create the library exporter
var exporter = projectContext.CreateExporter(configuration);
// Gather exports for the project
var dependencies = exporter.GetDependencies().ToList();
return dependencies.ToDictionary(d => d.Library.Identity.Name);
}
}
}

View file

@ -0,0 +1,71 @@
using System;
using System.Collections.Generic;
using NuGet.Frameworks;
using Microsoft.DotNet.ProjectModel;
using Microsoft.DotNet.ProjectModel.Graph;
using System.Linq;
using System.Threading.Tasks;
namespace Microsoft.DotNet.Tools.Build
{
public class ProjectGraphCollector
{
private readonly Func<string, NuGetFramework, ProjectContext> _projectContextFactory;
public ProjectGraphCollector(Func<string, NuGetFramework, ProjectContext> projectContextFactory)
{
_projectContextFactory = projectContextFactory;
}
public IEnumerable<ProjectGraphNode> Collect(IEnumerable<ProjectContext> contexts)
{
foreach (var context in contexts)
{
var libraries = context.LibraryManager.GetLibraries();
var lookup = libraries.ToDictionary(l => l.Identity.Name);
var root = lookup[context.ProjectFile.Name];
yield return TraverseProject((ProjectDescription)root, lookup, context);
}
}
private ProjectGraphNode TraverseProject(ProjectDescription project, IDictionary<string, LibraryDescription> lookup, ProjectContext context = null)
{
var deps = new List<ProjectGraphNode>();
foreach (var dependency in project.Dependencies)
{
var libraryDescription = lookup[dependency.Name];
if (libraryDescription.Identity.Type.Equals(LibraryType.Project))
{
deps.Add(TraverseProject((ProjectDescription)libraryDescription, lookup));
}
else
{
deps.AddRange(TraverseNonProject(libraryDescription, lookup));
}
}
var task = context != null ? Task.FromResult(context) : Task.Run(() => _projectContextFactory(project.Path, project.Framework));
return new ProjectGraphNode(task, deps, context != null);
}
private IEnumerable<ProjectGraphNode> TraverseNonProject(LibraryDescription root, IDictionary<string, LibraryDescription> lookup)
{
foreach (var dependency in root.Dependencies)
{
var libraryDescription = lookup[dependency.Name];
if (libraryDescription.Identity.Type.Equals(LibraryType.Project))
{
yield return TraverseProject((ProjectDescription)libraryDescription, lookup);
}
else
{
foreach(var node in TraverseNonProject(libraryDescription, lookup))
{
yield return node;
}
}
}
}
}
}

View file

@ -0,0 +1,26 @@
using Microsoft.DotNet.ProjectModel;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
namespace Microsoft.DotNet.Tools.Build
{
public class ProjectGraphNode
{
private readonly Task<ProjectContext> _projectContextCreator;
public ProjectGraphNode(Task<ProjectContext> projectContext, IEnumerable<ProjectGraphNode> dependencies, bool isRoot = false)
{
_projectContextCreator = projectContext;
Dependencies = dependencies.ToList();
IsRoot = isRoot;
}
public ProjectContext ProjectContext { get { return _projectContextCreator.GetAwaiter().GetResult(); } }
public IReadOnlyList<ProjectGraphNode> Dependencies { get; }
public bool IsRoot { get; }
}
}

View file

@ -8,7 +8,6 @@ using System.Linq;
using System.Text;
using Microsoft.DotNet.Cli.Utils;
using Microsoft.Dnx.Runtime.Common.CommandLine;
using Microsoft.Dotnet.Cli.Compiler.Common;
using Microsoft.DotNet.ProjectModel;
using Microsoft.DotNet.ProjectModel.Graph;
using NuGet.Frameworks;