diff --git a/TestAssets/TestProjects/TestSimpleIncrementalApp/Program2.cs b/TestAssets/TestProjects/TestSimpleIncrementalApp/Program2.cs new file mode 100644 index 000000000..6c936abc9 --- /dev/null +++ b/TestAssets/TestProjects/TestSimpleIncrementalApp/Program2.cs @@ -0,0 +1,12 @@ +using System; + +namespace ConsoleApplication +{ + public class Program2 + { + public static void Foo(string[] args) + { + Console.WriteLine("Hello World!"); + } + } +} diff --git a/src/Microsoft.DotNet.Cli.Utils/DotnetFiles.cs b/src/Microsoft.DotNet.Cli.Utils/DotnetFiles.cs index 7d760841f..a676723cc 100644 --- a/src/Microsoft.DotNet.Cli.Utils/DotnetFiles.cs +++ b/src/Microsoft.DotNet.Cli.Utils/DotnetFiles.cs @@ -1,8 +1,10 @@ // Copyright (c) .NET Foundation and contributors. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +using System; using System.IO; using System.Reflection; +using Microsoft.Extensions.PlatformAbstractions; namespace Microsoft.DotNet.Cli.Utils { @@ -12,5 +14,16 @@ namespace Microsoft.DotNet.Cli.Utils /// The CLI ships with a .version file that stores the commit information and CLI version /// public static string VersionFile => Path.GetFullPath(Path.Combine(typeof(DotnetFiles).GetTypeInfo().Assembly.Location, "..", ".version")); + + /// + /// Reads the version file and adds runtime specific information + /// + public static string ReadAndInterpretVersionFile() + { + var content = File.ReadAllText(DotnetFiles.VersionFile); + content += Environment.NewLine; + content += PlatformServices.Default.Runtime.GetRuntimeIdentifier(); + return content; + } } } diff --git a/src/Microsoft.DotNet.Cli.Utils/Reporter.cs b/src/Microsoft.DotNet.Cli.Utils/Reporter.cs index 9b5984773..4e85c672b 100644 --- a/src/Microsoft.DotNet.Cli.Utils/Reporter.cs +++ b/src/Microsoft.DotNet.Cli.Utils/Reporter.cs @@ -37,12 +37,14 @@ namespace Microsoft.DotNet.Cli.Utils { Output = new Reporter(AnsiConsole.GetOutput()); Error = new Reporter(AnsiConsole.GetError()); - Verbose = CommandContext.IsVerbose() ? + Verbose = IsVerbose ? new Reporter(AnsiConsole.GetOutput()) : NullReporter; } } + public static bool IsVerbose => CommandContext.IsVerbose(); + public void WriteLine(string message) { lock (_lock) diff --git a/src/Microsoft.DotNet.Compiler.Common/ProjectContextExtensions.cs b/src/Microsoft.DotNet.Compiler.Common/ProjectContextExtensions.cs index ec78c9ef4..d0cc3bb1e 100644 --- a/src/Microsoft.DotNet.Compiler.Common/ProjectContextExtensions.cs +++ b/src/Microsoft.DotNet.Compiler.Common/ProjectContextExtensions.cs @@ -37,6 +37,12 @@ namespace Microsoft.DotNet.Cli.Compiler.Common return Path.Combine(intermediatePath, ".SDKVersion"); } + public static string IncrementalCacheFile(this ProjectContext context, string configuration, string buildBasePath, string outputPath) + { + var intermediatePath = context.GetOutputPaths(configuration, buildBasePath, outputPath).IntermediateOutputDirectoryPath; + return Path.Combine(intermediatePath, ".IncrementalCache"); + } + // used in incremental compilation for the key file public static CommonCompilerOptions ResolveCompilationOptions(this ProjectContext context, string configuration) { diff --git a/src/Microsoft.DotNet.TestFramework/Microsoft.DotNet.TestFramework.TestInstance.cs b/src/Microsoft.DotNet.TestFramework/Microsoft.DotNet.TestFramework.TestInstance.cs index 8912c13ee..99954d074 100644 --- a/src/Microsoft.DotNet.TestFramework/Microsoft.DotNet.TestFramework.TestInstance.cs +++ b/src/Microsoft.DotNet.TestFramework/Microsoft.DotNet.TestFramework.TestInstance.cs @@ -11,6 +11,9 @@ namespace Microsoft.DotNet.TestFramework { public class TestInstance { + // made tolower because the rest of the class works with normalized tolower strings + private static readonly IEnumerable BuildArtifactBlackList = new List() {".IncrementalCache", ".SDKVersion"}.Select(s => s.ToLower()).ToArray(); + private string _testDestination; private string _testAssetRoot; @@ -105,8 +108,13 @@ namespace Microsoft.DotNet.TestFramework .Where(file => { file = file.ToLower(); - return file.Contains($"{Path.DirectorySeparatorChar}bin{Path.DirectorySeparatorChar}") + + var isArtifact = file.Contains($"{Path.DirectorySeparatorChar}bin{Path.DirectorySeparatorChar}") || file.Contains($"{Path.DirectorySeparatorChar}obj{Path.DirectorySeparatorChar}"); + + var isBlackListed = BuildArtifactBlackList.Any(b => file.Contains(b)); + + return isArtifact && !isBlackListed; }); foreach (string binFile in binFiles) diff --git a/src/dotnet/commands/dotnet-build/CompilerIO.cs b/src/dotnet/commands/dotnet-build/CompilerIO.cs index cd7fbb8f8..87db75356 100644 --- a/src/dotnet/commands/dotnet-build/CompilerIO.cs +++ b/src/dotnet/commands/dotnet-build/CompilerIO.cs @@ -2,18 +2,43 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System.Collections.Generic; +using System.Linq; namespace Microsoft.DotNet.Tools.Build { - internal struct CompilerIO + internal class CompilerIO { - public readonly List Inputs; - public readonly List Outputs; + public readonly IEnumerable Inputs; + public readonly IEnumerable Outputs; - public CompilerIO(List inputs, List outputs) + public CompilerIO(IEnumerable inputs, IEnumerable outputs) { Inputs = inputs; Outputs = outputs; } + + public DiffResult DiffInputs(CompilerIO other) + { + var myInputSet = new HashSet(Inputs); + var otherInputSet = new HashSet(other.Inputs); + + var additions = myInputSet.Except(otherInputSet); + var deletions = otherInputSet.Except(myInputSet); + + return new DiffResult(additions, deletions); + } + + internal class DiffResult + { + public IEnumerable Additions { get; private set; } + public IEnumerable Deletions { get; private set; } + + public DiffResult(IEnumerable additions, IEnumerable deletions) + { + Additions = additions; + Deletions = deletions; + } + } + } } \ No newline at end of file diff --git a/src/dotnet/commands/dotnet-build/CompilerIOManager.cs b/src/dotnet/commands/dotnet-build/CompilerIOManager.cs index 2bac2204f..ed2b52843 100644 --- a/src/dotnet/commands/dotnet-build/CompilerIOManager.cs +++ b/src/dotnet/commands/dotnet-build/CompilerIOManager.cs @@ -2,6 +2,7 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; @@ -19,6 +20,7 @@ namespace Microsoft.DotNet.Tools.Build private readonly string _buildBasePath; private readonly IList _runtimes; private readonly WorkspaceContext _workspace; + private readonly ConcurrentDictionary _cache; public CompilerIOManager(string configuration, string outputPath, @@ -31,49 +33,36 @@ namespace Microsoft.DotNet.Tools.Build _buildBasePath = buildBasePath; _runtimes = runtimes.ToList(); _workspace = workspace; + + _cache = new ConcurrentDictionary(); } - public bool AnyMissingIO(ProjectContext project, IEnumerable items, string itemsType) - { - var missingItems = items.Where(i => !File.Exists(i)).ToList(); - - if (!missingItems.Any()) - { - return false; - } - - Reporter.Verbose.WriteLine($"Project {project.GetDisplayName()} will be compiled because expected {itemsType} are missing."); - - foreach (var missing in missingItems) - { - Reporter.Verbose.WriteLine($" {missing}"); - } - - Reporter.Verbose.WriteLine(); ; - - return true; - } // computes all the inputs and outputs that would be used in the compilation of a project - // ensures that all paths are files - // ensures no missing inputs public CompilerIO GetCompileIO(ProjectGraphNode graphNode) { + return _cache.GetOrAdd(graphNode.ProjectContext.Identity, i => ComputeIO(graphNode)); + } + + private CompilerIO ComputeIO(ProjectGraphNode graphNode) + { + var inputs = new List(); + var outputs = new List(); + var isRootProject = graphNode.IsRoot; var project = graphNode.ProjectContext; - var compilerIO = new CompilerIO(new List(), new List()); var calculator = project.GetOutputPaths(_configuration, _buildBasePath, _outputPath); var binariesOutputPath = calculator.CompilationOutputPath; // input: project.json - compilerIO.Inputs.Add(project.ProjectFile.ProjectFilePath); + inputs.Add(project.ProjectFile.ProjectFilePath); // input: lock file; find when dependencies change - AddLockFile(project, compilerIO); + AddLockFile(project, inputs); // input: source files - compilerIO.Inputs.AddRange(CompilerUtil.GetCompilationSources(project)); + inputs.AddRange(CompilerUtil.GetCompilationSources(project)); var allOutputPath = new HashSet(calculator.CompilationFiles.All()); if (isRootProject && project.ProjectFile.HasRuntimeOutput(_configuration)) @@ -88,23 +77,22 @@ namespace Microsoft.DotNet.Tools.Build // output: compiler outputs foreach (var path in allOutputPath) { - compilerIO.Outputs.Add(path); + outputs.Add(path); } // input compilation options files - AddCompilationOptions(project, _configuration, compilerIO); + AddCompilationOptions(project, _configuration, inputs); // input / output: resources with culture - AddNonCultureResources(project, calculator.IntermediateOutputDirectoryPath, compilerIO); + AddNonCultureResources(project, calculator.IntermediateOutputDirectoryPath, inputs, outputs); // input / output: resources without culture - AddCultureResources(project, binariesOutputPath, compilerIO); + AddCultureResources(project, binariesOutputPath, inputs, outputs); - return compilerIO; + return new CompilerIO(inputs, outputs); } - - private static void AddLockFile(ProjectContext project, CompilerIO compilerIO) + private static void AddLockFile(ProjectContext project, List inputs) { if (project.LockFile == null) { @@ -113,48 +101,48 @@ namespace Microsoft.DotNet.Tools.Build throw new InvalidOperationException(errorMessage); } - compilerIO.Inputs.Add(project.LockFile.LockFilePath); + inputs.Add(project.LockFile.LockFilePath); if (project.LockFile.ExportFile != null) { - compilerIO.Inputs.Add(project.LockFile.ExportFile.ExportFilePath); + inputs.Add(project.LockFile.ExportFile.ExportFilePath); } } - private static void AddCompilationOptions(ProjectContext project, string config, CompilerIO compilerIO) + private static void AddCompilationOptions(ProjectContext project, string config, List inputs) { var compilerOptions = project.ResolveCompilationOptions(config); // input: key file if (compilerOptions.KeyFile != null) { - compilerIO.Inputs.Add(compilerOptions.KeyFile); + inputs.Add(compilerOptions.KeyFile); } } - private static void AddNonCultureResources(ProjectContext project, string intermediaryOutputPath, CompilerIO compilerIO) + private static void AddNonCultureResources(ProjectContext project, string intermediaryOutputPath, List inputs, IList outputs) { foreach (var resourceIO in CompilerUtil.GetNonCultureResources(project.ProjectFile, intermediaryOutputPath)) { - compilerIO.Inputs.Add(resourceIO.InputFile); + inputs.Add(resourceIO.InputFile); if (resourceIO.OutputFile != null) { - compilerIO.Outputs.Add(resourceIO.OutputFile); + outputs.Add(resourceIO.OutputFile); } } } - private static void AddCultureResources(ProjectContext project, string outputPath, CompilerIO compilerIO) + private static void AddCultureResources(ProjectContext project, string outputPath, List inputs, List outputs) { foreach (var cultureResourceIO in CompilerUtil.GetCultureResources(project.ProjectFile, outputPath)) { - compilerIO.Inputs.AddRange(cultureResourceIO.InputFileToMetadata.Keys); + inputs.AddRange(cultureResourceIO.InputFileToMetadata.Keys); if (cultureResourceIO.OutputFile != null) { - compilerIO.Outputs.Add(cultureResourceIO.OutputFile); + outputs.Add(cultureResourceIO.OutputFile); } } } diff --git a/src/dotnet/commands/dotnet-build/DotnetProjectBuilder.cs b/src/dotnet/commands/dotnet-build/DotnetProjectBuilder.cs index bcc40fcbd..2bc9e507d 100644 --- a/src/dotnet/commands/dotnet-build/DotnetProjectBuilder.cs +++ b/src/dotnet/commands/dotnet-build/DotnetProjectBuilder.cs @@ -21,14 +21,17 @@ namespace Microsoft.DotNet.Tools.Build private readonly CompilerIOManager _compilerIOManager; private readonly ScriptRunner _scriptRunner; private readonly DotNetCommandFactory _commandFactory; + private readonly IncrementalManager _incrementalManager; public DotNetProjectBuilder(BuildCommandApp args) { _args = args; + _preconditionManager = new IncrementalPreconditionManager( args.ShouldPrintIncrementalPreconditions, args.ShouldNotUseIncrementality, args.ShouldSkipDependencies); + _compilerIOManager = new CompilerIOManager( args.ConfigValue, args.OutputValue, @@ -36,7 +39,19 @@ namespace Microsoft.DotNet.Tools.Build args.GetRuntimes(), args.Workspace ); + + _incrementalManager = new IncrementalManager( + this, + _compilerIOManager, + _preconditionManager, + _args.ShouldSkipDependencies, + _args.ConfigValue, + _args.BuildBasePathValue, + _args.OutputValue + ); + _scriptRunner = new ScriptRunner(); + _commandFactory = new DotNetCommandFactory(); } @@ -52,7 +67,7 @@ namespace Microsoft.DotNet.Tools.Build Directory.CreateDirectory(parentDirectory); } - string content = ComputeCurrentVersionFileData(); + string content = DotnetFiles.ReadAndInterpretVersionFile(); File.WriteAllText(projectVersionFile, content); } @@ -62,14 +77,6 @@ namespace Microsoft.DotNet.Tools.Build } } - private static string ComputeCurrentVersionFileData() - { - var content = File.ReadAllText(DotnetFiles.VersionFile); - content += Environment.NewLine; - content += PlatformServices.Default.Runtime.GetRuntimeIdentifier(); - return content; - } - private void PrintSummary(ProjectGraphNode projectNode, bool success) { // todo: Ideally it's the builder's responsibility for adding the time elapsed. That way we avoid cross cutting display concerns between compile and build for printing time elapsed @@ -83,17 +90,6 @@ namespace Microsoft.DotNet.Tools.Build Reporter.Output.WriteLine(); } - private void CreateOutputDirectories() - { - if (!string.IsNullOrEmpty(_args.OutputValue)) - { - Directory.CreateDirectory(_args.OutputValue); - } - if (!string.IsNullOrEmpty(_args.BuildBasePathValue)) - { - Directory.CreateDirectory(_args.BuildBasePathValue); - } - } private void CopyCompilationOutput(OutputPaths outputPaths) { @@ -151,118 +147,47 @@ namespace Microsoft.DotNet.Tools.Build finally { StampProjectWithSDKVersion(projectNode.ProjectContext); + _incrementalManager.CacheIncrementalState(projectNode); } } protected override void ProjectSkiped(ProjectGraphNode projectNode) { StampProjectWithSDKVersion(projectNode.ProjectContext); - } - - private bool CLIChangedSinceLastCompilation(ProjectContext project) - { - var currentVersionFile = DotnetFiles.VersionFile; - var versionFileFromLastCompile = project.GetSDKVersionFile(_args.ConfigValue, _args.BuildBasePathValue, _args.OutputValue); - - if (!File.Exists(currentVersionFile)) - { - // this CLI does not have a version file; cannot tell if CLI changed - return false; - } - - if (!File.Exists(versionFileFromLastCompile)) - { - // this is the first compilation; cannot tell if CLI changed - return false; - } - - var currentContent = ComputeCurrentVersionFileData(); - - var versionsAreEqual = string.Equals(currentContent, File.ReadAllText(versionFileFromLastCompile), StringComparison.OrdinalIgnoreCase); - - return !versionsAreEqual; + _incrementalManager.CacheIncrementalState(projectNode); } protected override bool NeedsRebuilding(ProjectGraphNode graphNode) { - var project = graphNode.ProjectContext; - if (_args.ShouldNotUseIncrementality) - { - return true; - } - if (!_args.ShouldSkipDependencies && - graphNode.Dependencies.Any(d => GetCompilationResult(d) != CompilationResult.IncrementalSkip)) - { - Reporter.Output.WriteLine($"Project {project.GetDisplayName()} will be compiled because some of it's dependencies changed"); - return true; - } - var preconditions = _preconditionManager.GetIncrementalPreconditions(graphNode); - if (preconditions.PreconditionsDetected()) - { - return true; - } + var result = _incrementalManager.NeedsRebuilding(graphNode); - if (CLIChangedSinceLastCompilation(project)) + PrintIncrementalResult(graphNode.ProjectContext.GetDisplayName(), result); + + return result.NeedsRebuilding; + } + + private void PrintIncrementalResult(string projectName, IncrementalResult result) + { + if (result.NeedsRebuilding) { - Reporter.Output.WriteLine($"Project {project.GetDisplayName()} will be compiled because the version or bitness of the CLI changed since the last build"); - return true; + Reporter.Output.WriteLine($"Project {projectName} will be compiled because {result.Reason}"); + PrintIncrementalItems(result); } - - var compilerIO = _compilerIOManager.GetCompileIO(graphNode); - - // rebuild if empty inputs / outputs - if (!(compilerIO.Outputs.Any() && compilerIO.Inputs.Any())) + else { - Reporter.Output.WriteLine($"Project {project.GetDisplayName()} will be compiled because it either has empty inputs or outputs"); - return true; + Reporter.Output.WriteLine($"Project {projectName} was previously compiled. Skipping compilation."); } + } - //rebuild if missing inputs / outputs - if (_compilerIOManager.AnyMissingIO(project, compilerIO.Outputs, "outputs") || _compilerIOManager.AnyMissingIO(project, compilerIO.Inputs, "inputs")) + private static void PrintIncrementalItems(IncrementalResult result) + { + if (Reporter.IsVerbose) { - return true; - } - - // find the output with the earliest write time - var minOutputPath = compilerIO.Outputs.First(); - var minDateUtc = File.GetLastWriteTimeUtc(minOutputPath); - - foreach (var outputPath in compilerIO.Outputs) - { - if (File.GetLastWriteTimeUtc(outputPath) >= minDateUtc) + foreach (var item in result.Items) { - continue; + Reporter.Verbose.WriteLine($"\t{item}"); } - - minDateUtc = File.GetLastWriteTimeUtc(outputPath); - minOutputPath = outputPath; } - - // find inputs that are older than the earliest output - var newInputs = compilerIO.Inputs.FindAll(p => File.GetLastWriteTimeUtc(p) >= minDateUtc); - - if (!newInputs.Any()) - { - Reporter.Output.WriteLine($"Project {project.GetDisplayName()} was previously compiled. Skipping compilation."); - return false; - } - - Reporter.Output.WriteLine($"Project {project.GetDisplayName()} will be compiled because some of its inputs were newer than its oldest output."); - Reporter.Verbose.WriteLine(); - Reporter.Verbose.WriteLine($" Oldest output item:"); - Reporter.Verbose.WriteLine($" {minDateUtc.ToLocalTime()}: {minOutputPath}"); - Reporter.Verbose.WriteLine(); - - Reporter.Verbose.WriteLine($" Inputs newer than the oldest output item:"); - - foreach (var newInput in newInputs) - { - Reporter.Verbose.WriteLine($" {File.GetLastWriteTime(newInput)}: {newInput}"); - } - - Reporter.Verbose.WriteLine(); - - return true; } } } \ No newline at end of file diff --git a/src/dotnet/commands/dotnet-build/IncrementalCache.cs b/src/dotnet/commands/dotnet-build/IncrementalCache.cs new file mode 100644 index 000000000..af48ca082 --- /dev/null +++ b/src/dotnet/commands/dotnet-build/IncrementalCache.cs @@ -0,0 +1,92 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.IO; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; + +namespace Microsoft.DotNet.Tools.Build +{ + internal class IncrementalCache + { + private const string InputsKeyName = "inputs"; + private const string OutputsKeyNane = "outputs"; + + public CompilerIO CompilerIO { get; } + + public IncrementalCache(CompilerIO compilerIO) + { + CompilerIO = compilerIO; + } + + public void WriteToFile(string cacheFile) + { + try + { + CreatePathIfAbsent(cacheFile); + + using (var streamWriter = new StreamWriter(new FileStream(cacheFile, FileMode.Create, FileAccess.Write, FileShare.None))) + { + var rootObject = new JObject(); + rootObject[InputsKeyName] = new JArray(CompilerIO.Inputs); + rootObject[OutputsKeyNane] = new JArray(CompilerIO.Outputs); + + JsonSerializer.Create().Serialize(streamWriter, rootObject); + } + } + catch (Exception e) + { + throw new InvalidDataException($"Could not write the incremental cache file: {cacheFile}", e); + } + } + + private static void CreatePathIfAbsent(string filePath) + { + var parentDir = Path.GetDirectoryName(filePath); + + if (!Directory.Exists(parentDir)) + { + Directory.CreateDirectory(parentDir); + } + } + + public static IncrementalCache ReadFromFile(string cacheFile) + { + try + { + using (var streamReader = new StreamReader(new FileStream(cacheFile, FileMode.Open, FileAccess.Read, FileShare.Read))) + { + var jObject = JObject.Parse(streamReader.ReadToEnd()); + + if (jObject == null) + { + throw new InvalidDataException(); + } + + var inputs = ReadArray(jObject, InputsKeyName); + var outputs = ReadArray(jObject, OutputsKeyNane); + + return new IncrementalCache(new CompilerIO(inputs, outputs)); + } + } + catch (Exception e) + { + throw new InvalidDataException($"Could not read the incremental cache file: {cacheFile}", e); + } + } + + private static IEnumerable ReadArray(JObject jObject, string keyName) + { + var array = jObject.Value(keyName)?.Values(); + + if (array == null) + { + throw new InvalidDataException($"Could not read key {keyName}"); + } + + return array; + } + } +} diff --git a/src/dotnet/commands/dotnet-build/IncrementalManager.cs b/src/dotnet/commands/dotnet-build/IncrementalManager.cs new file mode 100644 index 000000000..c367079a6 --- /dev/null +++ b/src/dotnet/commands/dotnet-build/IncrementalManager.cs @@ -0,0 +1,207 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Microsoft.DotNet.Cli.Compiler.Common; +using Microsoft.DotNet.Cli.Utils; +using Microsoft.DotNet.ProjectModel; +using Microsoft.DotNet.ProjectModel.Utilities; +using Microsoft.DotNet.Tools.Compiler; +using Microsoft.Extensions.PlatformAbstractions; +using Microsoft.DotNet.ProjectModel.Compilation; +using NuGet.Protocol.Core.Types; + +namespace Microsoft.DotNet.Tools.Build +{ + internal class IncrementalManager + { + private readonly ProjectBuilder _projectBuilder; + private readonly CompilerIOManager _compilerIoManager; + private readonly IncrementalPreconditionManager _preconditionManager; + private readonly bool _shouldSkipDependencies; + private readonly string _configuration; + private readonly string _buildBasePath; + private readonly string _outputPath; + + public IncrementalManager( + ProjectBuilder projectBuilder, + CompilerIOManager compilerIOManager, + IncrementalPreconditionManager incrementalPreconditionManager, + bool shouldSkipDependencies, + string configuration, + string buildBasePath, + string outputPath) + { + _projectBuilder = projectBuilder; + _compilerIoManager = compilerIOManager; + _preconditionManager = incrementalPreconditionManager; + _shouldSkipDependencies = shouldSkipDependencies; + _configuration = configuration; + _buildBasePath = buildBasePath; + _outputPath = outputPath; + } + + public IncrementalResult NeedsRebuilding(ProjectGraphNode graphNode) + { + if (!_shouldSkipDependencies && + graphNode.Dependencies.Any(d => _projectBuilder.GetCompilationResult(d) != CompilationResult.IncrementalSkip)) + { + return new IncrementalResult("dependencies changed"); + } + + var preconditions = _preconditionManager.GetIncrementalPreconditions(graphNode); + if (preconditions.PreconditionsDetected()) + { + return new IncrementalResult($"project is not safe for incremental compilation. Use {BuildCommandApp.BuildProfileFlag} flag for more information."); + } + + var compilerIO = _compilerIoManager.GetCompileIO(graphNode); + + var result = CLIChanged(graphNode); + if (result.NeedsRebuilding) + { + return result; + } + + result = InputItemsChanged(graphNode, compilerIO); + if (result.NeedsRebuilding) + { + return result; + } + + result = TimestampsChanged(compilerIO); + if (result.NeedsRebuilding) + { + return result; + } + + return IncrementalResult.DoesNotNeedRebuild; + } + + private IncrementalResult CLIChanged(ProjectGraphNode graphNode) + { + var currentVersionFile = DotnetFiles.VersionFile; + var versionFileFromLastCompile = graphNode.ProjectContext.GetSDKVersionFile(_configuration, _buildBasePath, _outputPath); + + if (!File.Exists(currentVersionFile)) + { + // this CLI does not have a version file; cannot tell if CLI changed + return IncrementalResult.DoesNotNeedRebuild; + } + + if (!File.Exists(versionFileFromLastCompile)) + { + // this is the first compilation; cannot tell if CLI changed + return IncrementalResult.DoesNotNeedRebuild; + } + + var currentContent = DotnetFiles.ReadAndInterpretVersionFile(); + + var versionsAreEqual = string.Equals(currentContent, File.ReadAllText(versionFileFromLastCompile), StringComparison.OrdinalIgnoreCase); + + return versionsAreEqual + ? IncrementalResult.DoesNotNeedRebuild + : new IncrementalResult("the version or bitness of the CLI changed since the last build"); + } + + private IncrementalResult InputItemsChanged(ProjectGraphNode graphNode, CompilerIO compilerIO) + { + // check empty inputs / outputs + if (!compilerIO.Inputs.Any()) + { + return new IncrementalResult("the project has no inputs"); + } + + if (!compilerIO.Outputs.Any()) + { + return new IncrementalResult("the project has no outputs"); + } + + // check non existent items + var result = CheckMissingIO(compilerIO.Inputs, "inputs"); + if (result.NeedsRebuilding) + { + return result; + } + + result = CheckMissingIO(compilerIO.Outputs, "outputs"); + if (result.NeedsRebuilding) + { + return result; + } + + return CheckInputGlobChanges(graphNode, compilerIO); + } + + private IncrementalResult CheckInputGlobChanges(ProjectGraphNode graphNode, CompilerIO compilerIO) + { + // check cache against input glob pattern changes + var incrementalCacheFile = graphNode.ProjectContext.IncrementalCacheFile(_configuration, _buildBasePath, _outputPath); + + if (!File.Exists(incrementalCacheFile)) + { + // cache is not present (first compilation); can't determine if globs changed; cache will be generated after build processes project + return IncrementalResult.DoesNotNeedRebuild; + } + + var incrementalCache = IncrementalCache.ReadFromFile(incrementalCacheFile); + + var diffResult = compilerIO.DiffInputs(incrementalCache.CompilerIO); + + if (diffResult.Deletions.Any()) + { + return new IncrementalResult("Input items removed from last build", diffResult.Deletions); + } + + if (diffResult.Additions.Any()) + { + return new IncrementalResult("Input items added from last build", diffResult.Additions); + } + + return IncrementalResult.DoesNotNeedRebuild; + } + + private IncrementalResult CheckMissingIO(IEnumerable items, string itemsType) + { + var missingItems = items.Where(i => !File.Exists(i)).ToList(); + + return missingItems.Any() + ? new IncrementalResult($"expected {itemsType} are missing", missingItems) + : IncrementalResult.DoesNotNeedRebuild; + } + + private IncrementalResult TimestampsChanged(CompilerIO compilerIO) + { + // find the output with the earliest write time + var minDateUtc = DateTime.MaxValue; + + foreach (var outputPath in compilerIO.Outputs) + { + var lastWriteTimeUtc = File.GetLastWriteTimeUtc(outputPath); + + if (lastWriteTimeUtc < minDateUtc) + { + minDateUtc = lastWriteTimeUtc; + } + } + + // find inputs that are older than the earliest output + var newInputs = compilerIO.Inputs.Where(p => File.GetLastWriteTimeUtc(p) >= minDateUtc); + + return newInputs.Any() + ? new IncrementalResult("inputs were modified", newInputs) + : IncrementalResult.DoesNotNeedRebuild; + } + + public void CacheIncrementalState(ProjectGraphNode graphNode) + { + var incrementalCacheFile = graphNode.ProjectContext.IncrementalCacheFile(_configuration, _buildBasePath, _outputPath); + + var incrementalCache = new IncrementalCache(_compilerIoManager.GetCompileIO(graphNode)); + incrementalCache.WriteToFile(incrementalCacheFile); + } + } +} diff --git a/src/dotnet/commands/dotnet-build/IncrementalResult.cs b/src/dotnet/commands/dotnet-build/IncrementalResult.cs new file mode 100644 index 000000000..712de6176 --- /dev/null +++ b/src/dotnet/commands/dotnet-build/IncrementalResult.cs @@ -0,0 +1,34 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Collections.Generic; +using System.Linq; + +namespace Microsoft.DotNet.Tools.Build +{ + internal class IncrementalResult + { + public static readonly IncrementalResult DoesNotNeedRebuild = new IncrementalResult(false, "", Enumerable.Empty()); + + public bool NeedsRebuilding { get; } + public string Reason { get; } + public IEnumerable Items { get; } + + private IncrementalResult(bool needsRebuilding, string reason, IEnumerable items) + { + NeedsRebuilding = needsRebuilding; + Reason = reason; + Items = items; + } + + public IncrementalResult(string reason) + : this(true, reason, Enumerable.Empty()) + { + } + + public IncrementalResult(string reason, IEnumerable items) + : this(true, reason, items) + { + } + } +} \ No newline at end of file diff --git a/src/dotnet/commands/dotnet-build/ProjectBuilder.cs b/src/dotnet/commands/dotnet-build/ProjectBuilder.cs index a6af4a5fe..663c609b1 100644 --- a/src/dotnet/commands/dotnet-build/ProjectBuilder.cs +++ b/src/dotnet/commands/dotnet-build/ProjectBuilder.cs @@ -21,7 +21,7 @@ namespace Microsoft.DotNet.Tools.Build } } - protected CompilationResult? GetCompilationResult(ProjectGraphNode projectNode) + public CompilationResult? GetCompilationResult(ProjectGraphNode projectNode) { CompilationResult result; if (_compilationResults.TryGetValue(projectNode.ProjectContext.Identity, out result)) diff --git a/test/dotnet-build.Tests/IncrementalTests.cs b/test/dotnet-build.Tests/IncrementalTests.cs index 78778fb0a..bd3ad8b69 100644 --- a/test/dotnet-build.Tests/IncrementalTests.cs +++ b/test/dotnet-build.Tests/IncrementalTests.cs @@ -38,6 +38,7 @@ namespace Microsoft.DotNet.Tools.Builder.Tests buildResult.Should().HaveCompiledProject(MainProject, _appFrameworkFullName); buildResult = BuildProject(noIncremental: true); + buildResult.Should().HaveCompiledProject(MainProject, _appFrameworkFullName); Assert.Contains("[Forced Unsafe]", buildResult.StdOut); } @@ -85,12 +86,12 @@ namespace Microsoft.DotNet.Tools.Builder.Tests CreateTestInstance(); BuildProject().Should().HaveCompiledProject(MainProject, _appFrameworkFullName); - //change version file + // change version file var versionFile = Path.Combine(GetIntermediaryOutputPath(), ".SDKVersion"); File.Exists(versionFile).Should().BeTrue(); File.AppendAllText(versionFile, "text"); - //assert rebuilt + // assert rebuilt BuildProject().Should().HaveCompiledProject(MainProject, _appFrameworkFullName); } @@ -100,19 +101,80 @@ namespace Microsoft.DotNet.Tools.Builder.Tests CreateTestInstance(); BuildProject().Should().HaveCompiledProject(MainProject, _appFrameworkFullName); - //delete version file + // delete version file var versionFile = Path.Combine(GetIntermediaryOutputPath(), ".SDKVersion"); File.Exists(versionFile).Should().BeTrue(); File.Delete(versionFile); File.Exists(versionFile).Should().BeFalse(); - //assert build skipped due to no version file + // assert build skipped due to no version file BuildProject().Should().HaveSkippedProjectCompilation(MainProject, _appFrameworkFullName); - //the version file should have been regenerated during the build, even if compilation got skipped + // the version file should have been regenerated during the build, even if compilation got skipped File.Exists(versionFile).Should().BeTrue(); } + [Fact] + public void TestRebuildDeletedSource() + { + CreateTestInstance(); + BuildProject().Should().HaveCompiledProject(MainProject, _appFrameworkFullName); + + var sourceFile = Path.Combine(GetProjectDirectory(MainProject), "Program2.cs"); + File.Delete(sourceFile); + Assert.False(File.Exists(sourceFile)); + + // second build; should get rebuilt since we deleted a source file + BuildProject().Should().HaveCompiledProject(MainProject, _appFrameworkFullName); + + // third build; incremental cache should have been regenerated and project skipped + BuildProject().Should().HaveSkippedProjectCompilation(MainProject, _appFrameworkFullName); + } + + [Fact] + public void TestRebuildRenamedSource() + { + CreateTestInstance(); + var buildResult = BuildProject(); + buildResult.Should().HaveCompiledProject(MainProject, _appFrameworkFullName); + + var sourceFile = Path.Combine(GetProjectDirectory(MainProject), "Program2.cs"); + var destinationFile = Path.Combine(Path.GetDirectoryName(sourceFile), "ProgramNew.cs"); + File.Move(sourceFile, destinationFile); + Assert.False(File.Exists(sourceFile)); + Assert.True(File.Exists(destinationFile)); + + // second build; should get rebuilt since we renamed a source file + buildResult = BuildProject(); + buildResult.Should().HaveCompiledProject(MainProject, _appFrameworkFullName); + + // third build; incremental cache should have been regenerated and project skipped + BuildProject().Should().HaveSkippedProjectCompilation(MainProject, _appFrameworkFullName); + } + + [Fact] + public void TestRebuildDeletedSourceAfterCliChanged() + { + CreateTestInstance(); + BuildProject().Should().HaveCompiledProject(MainProject, _appFrameworkFullName); + + // change version file + var versionFile = Path.Combine(GetIntermediaryOutputPath(), ".SDKVersion"); + File.Exists(versionFile).Should().BeTrue(); + File.AppendAllText(versionFile, "text"); + + // delete a source file + var sourceFile = Path.Combine(GetProjectDirectory(MainProject), "Program2.cs"); + File.Delete(sourceFile); + Assert.False(File.Exists(sourceFile)); + + // should get rebuilt since we changed version file and deleted source file + BuildProject().Should().HaveCompiledProject(MainProject, _appFrameworkFullName); + + // third build; incremental cache should have been regenerated and project skipped + BuildProject().Should().HaveSkippedProjectCompilation(MainProject, _appFrameworkFullName); + } + [Fact] public void TestRebuildChangedLockFile() {