Revert "Add performance tests"

This commit is contained in:
Piotr Puszkiewicz 2016-04-29 16:35:27 -07:00
parent 2a49edbb6e
commit ce3eed0306
33 changed files with 4 additions and 729 deletions

View file

@ -1,11 +0,0 @@
using System;
namespace ConsoleApplication
{
public class Program
{
public static void Main()
{
}
}
}

View file

@ -1,15 +0,0 @@
{
"version": "1.0.0-*",
"compilationOptions": {
"emitEntryPoint": true
},
"dependencies": {
"Microsoft.NETCore.App": {
"type": "platform",
"version": "1.0.0-rc2-*"
},
},
"frameworks": {
"netcoreapp1.0": { }
}
}

View file

@ -1,12 +0,0 @@
using System;
namespace ConsoleApplication
{
public class Program
{
public static void Main()
{
ConsoleLibrary.P1.Program.Main();
}
}
}

View file

@ -1,18 +0,0 @@
{
"version": "1.0.0-*",
"compilationOptions": {
"emitEntryPoint": true
},
"dependencies": {
"SingleTargetP1" : {
"target": "project"
},
"Microsoft.NETCore.App": {
"type": "platform",
"version": "1.0.0-rc2-*"
},
},
"frameworks": {
"netcoreapp1.0": { }
}
}

View file

@ -1,12 +0,0 @@
using System;
namespace ConsoleLibrary.P1
{
public class Program
{
public static void Main()
{
ConsoleLibrary.P2.Program.Main();
}
}
}

View file

@ -1,15 +0,0 @@
{
"version": "1.0.0-*",
"dependencies": {
"SingleTargetP2" : {
"target": "project"
},
"Microsoft.NETCore.App": {
"type": "platform",
"version": "1.0.0-rc2-*"
},
},
"frameworks": {
"netcoreapp1.0": { }
}
}

View file

@ -1,11 +0,0 @@
using System;
namespace ConsoleLibrary.P2
{
public class Program
{
public static void Main()
{
}
}
}

View file

@ -1,12 +0,0 @@
{
"version": "1.0.0-*",
"dependencies": {
"Microsoft.NETCore.App": {
"type": "platform",
"version": "1.0.0-rc2-*"
},
},
"frameworks": {
"netcoreapp1.0": { }
}
}

View file

@ -1,11 +0,0 @@
using System;
namespace ConsoleApplication
{
public class Program
{
public static void Main()
{
}
}
}

View file

@ -1,16 +0,0 @@
{
"version": "1.0.0-*",
"compilationOptions": {
"emitEntryPoint": true
},
"dependencies": {
"Microsoft.NETCore.App": {
"type": "platform",
"version": "1.0.0-rc2-*"
},
},
"frameworks": {
"netcoreapp1.0": { },
"netstandard1.5": { }
}
}

View file

@ -1,12 +0,0 @@
using System;
namespace ConsoleApplication
{
public class Program
{
public static void Main()
{
ConsoleLibrary.P1.Program.Main();
}
}
}

View file

@ -1,19 +0,0 @@
{
"version": "1.0.0-*",
"compilationOptions": {
"emitEntryPoint": true
},
"dependencies": {
"TwoTargetP1" : {
"target": "project"
},
"Microsoft.NETCore.App": {
"type": "platform",
"version": "1.0.0-rc2-*"
},
},
"frameworks": {
"netcoreapp1.0": { },
"netstandard1.5": { }
}
}

View file

@ -1,12 +0,0 @@
using System;
namespace ConsoleLibrary.P1
{
public class Program
{
public static void Main()
{
ConsoleLibrary.P2.Program.Main();
}
}
}

View file

@ -1,16 +0,0 @@
{
"version": "1.0.0-*",
"dependencies": {
"TwoTargetP2" : {
"target": "project"
},
"Microsoft.NETCore.App": {
"type": "platform",
"version": "1.0.0-rc2-*"
},
},
"frameworks": {
"netcoreapp1.0": { },
"netstandard1.5": { }
}
}

View file

@ -1,11 +0,0 @@
using System;
namespace ConsoleLibrary.P2
{
public class Program
{
public static void Main()
{
}
}
}

View file

@ -1,13 +0,0 @@
{
"version": "1.0.0-*",
"dependencies": {
"Microsoft.NETCore.App": {
"type": "platform",
"version": "1.0.0-rc2-*"
},
},
"frameworks": {
"netcoreapp1.0": { },
"netstandard1.5": { }
}
}

View file

@ -1,12 +0,0 @@
using System;
namespace ConsoleApplication
{
public class Program
{
public static void Main()
{
ConsoleLibrary.P1.Program.Main();
}
}
}

View file

@ -1,19 +0,0 @@
{
"version": "1.0.0-*",
"compilationOptions": {
"emitEntryPoint": true
},
"dependencies": {
"TwoTargetLargeP1" : {
"target": "project"
},
"Microsoft.NETCore.App": {
"type": "platform",
"version": "1.0.0-rc2-*"
},
},
"frameworks": {
"netcoreapp1.0": { },
"netstandard1.5": { }
}
}

View file

@ -1,13 +0,0 @@
using System;
namespace ConsoleLibrary.P1
{
public class Program
{
public static void Main()
{
ConsoleLibrary.P2.Program.Main();
ConsoleLibrary.P4.Program.Main();
}
}
}

View file

@ -1,19 +0,0 @@
{
"version": "1.0.0-*",
"dependencies": {
"TwoTargetLargeP2" : {
"target": "project"
},
"TwoTargetLargeP4" : {
"target": "project"
},
"Microsoft.NETCore.App": {
"type": "platform",
"version": "1.0.0-rc2-*"
},
},
"frameworks": {
"netcoreapp1.0": { },
"netstandard1.5": { }
}
}

View file

@ -1,13 +0,0 @@
using System;
namespace ConsoleLibrary.P2
{
public class Program
{
public static void Main()
{
ConsoleLibrary.P3.Program.Main();
ConsoleLibrary.P4.Program.Main();
}
}
}

View file

@ -1,19 +0,0 @@
{
"version": "1.0.0-*",
"dependencies": {
"TwoTargetLargeP3" : {
"target": "project"
},
"TwoTargetLargeP4" : {
"target": "project"
},
"Microsoft.NETCore.App": {
"type": "platform",
"version": "1.0.0-rc2-*"
},
},
"frameworks": {
"netcoreapp1.0": { },
"netstandard1.5": { }
}
}

View file

@ -1,12 +0,0 @@
using System;
namespace ConsoleLibrary.P3
{
public class Program
{
public static void Main()
{
ConsoleLibrary.P4.Program.Main();
}
}
}

View file

@ -1,16 +0,0 @@
{
"version": "1.0.0-*",
"dependencies": {
"TwoTargetLargeP4" : {
"target": "project"
},
"Microsoft.NETCore.App": {
"type": "platform",
"version": "1.0.0-rc2-*"
},
},
"frameworks": {
"netcoreapp1.0": { },
"netstandard1.5": { }
}
}

View file

@ -1,13 +0,0 @@
using System;
namespace ConsoleLibrary.P4
{
public class Program
{
public static void Main()
{
ConsoleLibrary.P5.Program.Main();
ConsoleLibrary.P6.Program.Main();
}
}
}

View file

@ -1,19 +0,0 @@
{
"version": "1.0.0-*",
"dependencies": {
"TwoTargetLargeP5" : {
"target": "project"
},
"TwoTargetLargeP6" : {
"target": "project"
},
"Microsoft.NETCore.App": {
"type": "platform",
"version": "1.0.0-rc2-*"
},
},
"frameworks": {
"netcoreapp1.0": { },
"netstandard1.5": { }
}
}

View file

@ -1,11 +0,0 @@
using System;
namespace ConsoleLibrary.P5
{
public class Program
{
public static void Main()
{
}
}
}

View file

@ -1,13 +0,0 @@
{
"version": "1.0.0-*",
"dependencies": {
"Microsoft.NETCore.App": {
"type": "platform",
"version": "1.0.0-rc2-*"
},
},
"frameworks": {
"netcoreapp1.0": { },
"netstandard1.5": { }
}
}

View file

@ -1,11 +0,0 @@
using System;
namespace ConsoleLibrary.P6
{
public class Program
{
public static void Main()
{
}
}
}

View file

@ -1,13 +0,0 @@
{
"version": "1.0.0-*",
"dependencies": {
"Microsoft.NETCore.App": {
"type": "platform",
"version": "1.0.0-rc2-*"
},
},
"frameworks": {
"netcoreapp1.0": { },
"netstandard1.5": { }
}
}

View file

@ -36,8 +36,7 @@ namespace Microsoft.DotNet.Cli.Build
"Microsoft.DotNet.Cli.Utils.Tests", "Microsoft.DotNet.Cli.Utils.Tests",
"Microsoft.DotNet.Compiler.Common.Tests", "Microsoft.DotNet.Compiler.Common.Tests",
"Microsoft.DotNet.ProjectModel.Tests", "Microsoft.DotNet.ProjectModel.Tests",
"Microsoft.Extensions.DependencyModel.Tests", "Microsoft.Extensions.DependencyModel.Tests"
"Performance"
}; };
public static readonly string[] WindowsTestProjects = new[] public static readonly string[] WindowsTestProjects = new[]

View file

@ -29,7 +29,6 @@ namespace Microsoft.DotNet.Tools.Test.Utilities
private readonly bool _noIncremental; private readonly bool _noIncremental;
private readonly bool _noDependencies; private readonly bool _noDependencies;
private readonly string _runtime; private readonly string _runtime;
private readonly bool _verbose;
private string OutputOption private string OutputOption
{ {
@ -200,14 +199,6 @@ namespace Microsoft.DotNet.Tools.Test.Utilities
} }
} }
private string Verbose
{
get
{
return _verbose ? "--verbose" : "";
}
}
public BuildCommand( public BuildCommand(
string projectPath, string projectPath,
string output="", string output="",
@ -226,8 +217,7 @@ namespace Microsoft.DotNet.Tools.Test.Utilities
string cppCompilerFlags="", string cppCompilerFlags="",
bool buildProfile=true, bool buildProfile=true,
bool noIncremental=false, bool noIncremental=false,
bool noDependencies=false, bool noDependencies=false)
bool verbose=true)
: base("dotnet") : base("dotnet")
{ {
_projectPath = projectPath; _projectPath = projectPath;
@ -250,18 +240,17 @@ namespace Microsoft.DotNet.Tools.Test.Utilities
_buildProfile = buildProfile; _buildProfile = buildProfile;
_noIncremental = noIncremental; _noIncremental = noIncremental;
_noDependencies = noDependencies; _noDependencies = noDependencies;
_verbose = verbose;
} }
public override CommandResult Execute(string args = "") public override CommandResult Execute(string args = "")
{ {
args = $"{Verbose} build {BuildArgs()} {args}"; args = $"--verbose build {BuildArgs()} {args}";
return base.Execute(args); return base.Execute(args);
} }
public override CommandResult ExecuteWithCapturedOutput(string args = "") public override CommandResult ExecuteWithCapturedOutput(string args = "")
{ {
args = $"{Verbose} build {BuildArgs()} {args}"; args = $"--verbose build {BuildArgs()} {args}";
return base.ExecuteWithCapturedOutput(args); return base.ExecuteWithCapturedOutput(args);
} }

View file

@ -1,294 +0,0 @@
// Copyright (c) .NET Foundation and contributors. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
using System.Linq;
using Microsoft.DotNet.Tools.Test.Utilities;
using Microsoft.Xunit.Performance;
using Microsoft.DotNet.TestFramework;
using System.IO;
using System.Runtime.CompilerServices;
using System;
using System.Collections.Generic;
namespace Microsoft.DotNet.Tools.Builder.Tests
{
public class BuildPerformanceTest : TestBase
{
private static string SingleTargetApp = "SingleTargetApp";
private static string TwoTargetApp = "TwoTargetApp";
private static string[] SingleTargetGraph = new[]
{
"SingleTargetGraph/SingleTargetP0",
"SingleTargetGraph/SingleTargetP1",
"SingleTargetGraph/SingleTargetP2"
};
private static string[] TwoTargetGraph = new[]
{
"TwoTargetGraph/TwoTargetP0",
"TwoTargetGraph/TwoTargetP1",
"TwoTargetGraph/TwoTargetP2"
};
private static string[] TwoTargetGraphLarge = new[]
{
"TwoTargetGraphLarge/TwoTargetLargeP0",
"TwoTargetGraphLarge/TwoTargetLargeP1",
"TwoTargetGraphLarge/TwoTargetLargeP2",
"TwoTargetGraphLarge/TwoTargetLargeP3",
"TwoTargetGraphLarge/TwoTargetLargeP4",
"TwoTargetGraphLarge/TwoTargetLargeP5",
"TwoTargetGraphLarge/TwoTargetLargeP6"
};
[Benchmark]
public void BuildSingleProject_SingleTargetApp() => BuildSingleProject(CreateTestInstance(SingleTargetApp));
[Benchmark]
public void BuildSingleProject_TwoTargetApp() => BuildSingleProject(CreateTestInstance(TwoTargetApp));
public void BuildSingleProject(TestInstance instance)
{
foreach (var iteration in Benchmark.Iterations)
{
var buildCommand = new BuildCommand(instance.TestRoot, buildProfile: false);
using (iteration.StartMeasurement())
{
buildCommand.Execute().Should().Pass();
}
TouchSource(instance.TestRoot);
}
}
[Benchmark]
public void IncrementalSkipSingleProject_SingleTargetApp() => IncrementalSkipSingleProject(CreateTestInstance(SingleTargetApp));
[Benchmark]
public void IncrementalSkipSingleProject_TwoTargetApp() => IncrementalSkipSingleProject(CreateTestInstance(TwoTargetApp));
public void IncrementalSkipSingleProject(TestInstance instance)
{
new BuildCommand(instance.TestRoot, buildProfile: false)
.Execute().Should().Pass();
foreach (var iteration in Benchmark.Iterations)
{
var buildCommand = new BuildCommand(instance.TestRoot, buildProfile: false);
using (iteration.StartMeasurement())
{
buildCommand.Execute().Should().Pass();
}
}
}
[Benchmark]
public void BuildAllInGraph_SingleTargetGraph() => BuildAllInGraph(CreateTestInstances(SingleTargetGraph));
[Benchmark]
public void BuildAllInGraph_TwoTargetGraph() => BuildAllInGraph(CreateTestInstances(TwoTargetGraph));
[Benchmark]
public void BuildAllInGraph_TwoTargetGraphLarge() => BuildAllInGraph(CreateTestInstances(TwoTargetGraphLarge));
public void BuildAllInGraph(TestInstance[] instances)
{
var instance = instances[0];
foreach (var iteration in Benchmark.Iterations)
{
var buildCommand = new BuildCommand(instance.TestRoot, buildProfile: false);
using (iteration.StartMeasurement())
{
buildCommand.Execute().Should().Pass();
}
foreach (var i in instances)
{
TouchSource(i.TestRoot);
}
}
}
[Benchmark]
public void IncrementalSkipAllInGraph_SingleTargetGraph() =>
IncrementalSkipAllInGraph(CreateTestInstances(SingleTargetGraph));
[Benchmark]
public void IncrementalSkipAllInGraph_TwoTargetGraph() =>
IncrementalSkipAllInGraph(CreateTestInstances(TwoTargetGraph));
[Benchmark]
public void IncrementalSkipAllInGraphh_TwoTargetGraphLarge() =>
IncrementalSkipAllInGraph(CreateTestInstances(TwoTargetGraphLarge));
public void IncrementalSkipAllInGraph(TestInstance[] instances)
{
var instance = instances[0];
new BuildCommand(instance.TestRoot, buildProfile: false)
.Execute().Should().Pass();
foreach (var iteration in Benchmark.Iterations)
{
var buildCommand = new BuildCommand(instance.TestRoot, buildProfile: false);
using (iteration.StartMeasurement())
{
buildCommand.Execute().Should().Pass();
}
}
}
[Benchmark]
public void IncrementalRebuildWithRootChangedInGraph_SingleTargetGraph() =>
IncrementalRebuildWithRootChangedInGraph(CreateTestInstances(SingleTargetGraph));
[Benchmark]
public void IncrementalRebuildWithRootChangedInGraph_TwoTargetGraph() =>
IncrementalRebuildWithRootChangedInGraph(CreateTestInstances(TwoTargetGraph));
[Benchmark]
public void IncrementalRebuildWithRootChangedInGraph_TwoTargetGraphLarge() =>
IncrementalRebuildWithRootChangedInGraph(CreateTestInstances(TwoTargetGraphLarge));
public void IncrementalRebuildWithRootChangedInGraph(TestInstance[] instances)
{
var instance = instances[0];
new BuildCommand(instance.TestRoot, buildProfile: false)
.Execute().Should().Pass();
foreach (var iteration in Benchmark.Iterations)
{
var buildCommand = new BuildCommand(instance.TestRoot, buildProfile: false);
using (iteration.StartMeasurement())
{
buildCommand.Execute().Should().Pass();
}
TouchSource(instance.TestRoot);
}
}
[Benchmark]
public void IncrementalRebuildWithLastChangedInGraph_SingleTargetGraph() =>
IncrementalRebuildWithLastChangedInGraph(CreateTestInstances(SingleTargetGraph));
[Benchmark]
public void IncrementalRebuildWithLastChangedInGraph_TwoTargetGraph() =>
IncrementalRebuildWithLastChangedInGraph(CreateTestInstances(TwoTargetGraph));
[Benchmark]
public void IncrementalRebuildWithLastChangedInGraph_TwoTargetGraphLarge() =>
IncrementalRebuildWithLastChangedInGraph(CreateTestInstances(TwoTargetGraphLarge));
public void IncrementalRebuildWithLastChangedInGraph(TestInstance[] instances)
{
var instance = instances[0];
new BuildCommand(instance.TestRoot, buildProfile: false)
.Execute().Should().Pass();
foreach (var iteration in Benchmark.Iterations)
{
var buildCommand = new BuildCommand(instance.TestRoot, buildProfile: false);
using (iteration.StartMeasurement())
{
buildCommand.Execute().Should().Pass();
}
TouchSource(instances.Last().TestRoot);
}
}
[Benchmark]
public void IncrementalSkipAllNoDependenciesInGraph_SingleTargetGraph() =>
IncrementalSkipAllNoDependenciesInGraph(CreateTestInstances(SingleTargetGraph));
[Benchmark]
public void IncrementalSkipAllNoDependenciesInGraph_TwoTargetGraph() =>
IncrementalSkipAllNoDependenciesInGraph(CreateTestInstances(TwoTargetGraph));
[Benchmark]
public void IncrementalSkipAllNoDependenciesInGraph_TwoTargetGraphLarge() =>
IncrementalSkipAllNoDependenciesInGraph(CreateTestInstances(TwoTargetGraphLarge));
public void IncrementalSkipAllNoDependenciesInGraph(TestInstance[] instances)
{
var instance = instances[0];
new BuildCommand(instance.TestRoot, buildProfile: false)
.Execute().Should().Pass();
foreach (var iteration in Benchmark.Iterations)
{
var commands = new List<BuildCommand>();
foreach (var i in instances.Reverse())
{
commands.Add(new BuildCommand(i.TestRoot,
framework: DefaultFramework,
noDependencies: true,
buildProfile: false));
}
using (iteration.StartMeasurement())
{
foreach (var buildCommand in commands)
{
buildCommand.Execute().Should().Pass();
}
}
}
}
[Benchmark]
public void BuildAllNoDependenciesInGraph_SingleTargetGraph() =>
BuildAllNoDependenciesInGraph(CreateTestInstances(SingleTargetGraph));
[Benchmark]
public void BuildAllNoDependenciesInGraph_TwoTargetGraph() =>
BuildAllNoDependenciesInGraph(CreateTestInstances(TwoTargetGraph));
[Benchmark]
public void BuildAllNoDependenciesInGraph_TwoTargetGraphLarge() =>
BuildAllNoDependenciesInGraph(CreateTestInstances(TwoTargetGraphLarge));
public void BuildAllNoDependenciesInGraph(TestInstance[] instances)
{
foreach (var iteration in Benchmark.Iterations)
{
var commands = new List<BuildCommand>();
foreach (var i in instances.Reverse())
{
commands.Add(new BuildCommand(i.TestRoot,
framework: DefaultFramework,
noDependencies: true,
buildProfile: false));
}
using (iteration.StartMeasurement())
{
foreach (var buildCommand in commands)
{
buildCommand.Execute().Should().Pass();
}
}
foreach (var instance in instances)
{
TouchSource(instance.TestRoot);
}
}
}
protected void TouchSource(string project)
{
var sourceFile = Directory.GetFiles(project, "*.cs").FirstOrDefault();
if (sourceFile == null)
{
throw new InvalidOperationException($"'.cs' files not found in {project}");
}
File.SetLastWriteTime(sourceFile, DateTime.Now);
}
protected TestInstance[] CreateTestInstances(string[] testProjectNames, [CallerMemberName] string callingMethod = "")
{
return testProjectNames.Select(testProjectName =>
{
return CreateTestInstance(testProjectName, callingMethod);
}).ToArray();
}
protected TestInstance CreateTestInstance(string testProjectName, [CallerMemberName] string callingMethod = "")
{
return TestAssetsManager.CreateTestInstance(Path.Combine("PerformanceTestProjects", testProjectName), callingMethod)
.WithLockFiles();
}
}
}