Add performance tests

This commit is contained in:
Pavel Krymets 2016-04-22 17:06:55 -07:00
parent 7c504b2456
commit 366bad99ac
37 changed files with 785 additions and 3 deletions

View file

@ -0,0 +1,11 @@
using System;
namespace ConsoleApplication
{
public class Program
{
public static void Main()
{
}
}
}

View file

@ -0,0 +1,15 @@
{
"version": "1.0.0-*",
"compilationOptions": {
"emitEntryPoint": true
},
"dependencies": {
"Microsoft.NETCore.App": {
"type": "platform",
"version": "1.0.0-rc2-*"
},
},
"frameworks": {
"netcoreapp1.0": { }
}
}

View file

@ -0,0 +1,11 @@
using System;
namespace ConsoleApplication
{
public class Program
{
public static void Main()
{
}
}
}

View file

@ -0,0 +1,18 @@
{
"version": "1.0.0-*",
"compilationOptions": {
"emitEntryPoint": true
},
"dependencies": {
"SingleTargetP1" : {
"target": "project"
},
"Microsoft.NETCore.App": {
"type": "platform",
"version": "1.0.0-rc2-*"
},
},
"frameworks": {
"netcoreapp1.0": { }
}
}

View file

@ -0,0 +1,11 @@
using System;
namespace ConsoleApplication
{
public class Program
{
public static void Main()
{
}
}
}

View file

@ -0,0 +1,18 @@
{
"version": "1.0.0-*",
"compilationOptions": {
"emitEntryPoint": true
},
"dependencies": {
"SingleTargetP2" : {
"target": "project"
},
"Microsoft.NETCore.App": {
"type": "platform",
"version": "1.0.0-rc2-*"
},
},
"frameworks": {
"netcoreapp1.0": { }
}
}

View file

@ -0,0 +1,11 @@
using System;
namespace ConsoleApplication
{
public class Program
{
public static void Main()
{
}
}
}

View file

@ -0,0 +1,15 @@
{
"version": "1.0.0-*",
"compilationOptions": {
"emitEntryPoint": true
},
"dependencies": {
"Microsoft.NETCore.App": {
"type": "platform",
"version": "1.0.0-rc2-*"
},
},
"frameworks": {
"netcoreapp1.0": { }
}
}

View file

@ -0,0 +1,11 @@
using System;
namespace ConsoleApplication
{
public class Program
{
public static void Main()
{
}
}
}

View file

@ -0,0 +1,16 @@
{
"version": "1.0.0-*",
"compilationOptions": {
"emitEntryPoint": true
},
"dependencies": {
"Microsoft.NETCore.App": {
"type": "platform",
"version": "1.0.0-rc2-*"
},
},
"frameworks": {
"netcoreapp1.0": { },
"netstandard1.5": { }
}
}

View file

@ -0,0 +1,11 @@
using System;
namespace ConsoleApplication
{
public class Program
{
public static void Main()
{
}
}
}

View file

@ -0,0 +1,19 @@
{
"version": "1.0.0-*",
"compilationOptions": {
"emitEntryPoint": true
},
"dependencies": {
"TwoTargetP1" : {
"target": "project"
},
"Microsoft.NETCore.App": {
"type": "platform",
"version": "1.0.0-rc2-*"
},
},
"frameworks": {
"netcoreapp1.0": { },
"netstandard1.5": { }
}
}

View file

@ -0,0 +1,11 @@
using System;
namespace ConsoleApplication
{
public class Program
{
public static void Main()
{
}
}
}

View file

@ -0,0 +1,19 @@
{
"version": "1.0.0-*",
"compilationOptions": {
"emitEntryPoint": true
},
"dependencies": {
"TwoTargetP2" : {
"target": "project"
},
"Microsoft.NETCore.App": {
"type": "platform",
"version": "1.0.0-rc2-*"
},
},
"frameworks": {
"netcoreapp1.0": { },
"netstandard1.5": { }
}
}

View file

@ -0,0 +1,11 @@
using System;
namespace ConsoleApplication
{
public class Program
{
public static void Main()
{
}
}
}

View file

@ -0,0 +1,16 @@
{
"version": "1.0.0-*",
"compilationOptions": {
"emitEntryPoint": true
},
"dependencies": {
"Microsoft.NETCore.App": {
"type": "platform",
"version": "1.0.0-rc2-*"
},
},
"frameworks": {
"netcoreapp1.0": { },
"netstandard1.5": { }
}
}

View file

@ -0,0 +1,11 @@
using System;
namespace ConsoleApplication
{
public class Program
{
public static void Main()
{
}
}
}

View file

@ -0,0 +1,19 @@
{
"version": "1.0.0-*",
"compilationOptions": {
"emitEntryPoint": true
},
"dependencies": {
"TwoTargetLargeP1" : {
"target": "project"
},
"Microsoft.NETCore.App": {
"type": "platform",
"version": "1.0.0-rc2-*"
},
},
"frameworks": {
"netcoreapp1.0": { },
"netstandard1.5": { }
}
}

View file

@ -0,0 +1,11 @@
using System;
namespace ConsoleApplication
{
public class Program
{
public static void Main()
{
}
}
}

View file

@ -0,0 +1,22 @@
{
"version": "1.0.0-*",
"compilationOptions": {
"emitEntryPoint": true
},
"dependencies": {
"TwoTargetLargeP2" : {
"target": "project"
},
"TwoTargetLargeP4" : {
"target": "project"
},
"Microsoft.NETCore.App": {
"type": "platform",
"version": "1.0.0-rc2-*"
},
},
"frameworks": {
"netcoreapp1.0": { },
"netstandard1.5": { }
}
}

View file

@ -0,0 +1,11 @@
using System;
namespace ConsoleApplication
{
public class Program
{
public static void Main()
{
}
}
}

View file

@ -0,0 +1,22 @@
{
"version": "1.0.0-*",
"compilationOptions": {
"emitEntryPoint": true
},
"dependencies": {
"TwoTargetLargeP3" : {
"target": "project"
},
"TwoTargetLargeP4" : {
"target": "project"
},
"Microsoft.NETCore.App": {
"type": "platform",
"version": "1.0.0-rc2-*"
},
},
"frameworks": {
"netcoreapp1.0": { },
"netstandard1.5": { }
}
}

View file

@ -0,0 +1,11 @@
using System;
namespace ConsoleApplication
{
public class Program
{
public static void Main()
{
}
}
}

View file

@ -0,0 +1,19 @@
{
"version": "1.0.0-*",
"compilationOptions": {
"emitEntryPoint": true
},
"dependencies": {
"TwoTargetLargeP4" : {
"target": "project"
},
"Microsoft.NETCore.App": {
"type": "platform",
"version": "1.0.0-rc2-*"
},
},
"frameworks": {
"netcoreapp1.0": { },
"netstandard1.5": { }
}
}

View file

@ -0,0 +1,11 @@
using System;
namespace ConsoleApplication
{
public class Program
{
public static void Main()
{
}
}
}

View file

@ -0,0 +1,22 @@
{
"version": "1.0.0-*",
"compilationOptions": {
"emitEntryPoint": true
},
"dependencies": {
"TwoTargetLargeP5" : {
"target": "project"
},
"TwoTargetLargeP6" : {
"target": "project"
},
"Microsoft.NETCore.App": {
"type": "platform",
"version": "1.0.0-rc2-*"
},
},
"frameworks": {
"netcoreapp1.0": { },
"netstandard1.5": { }
}
}

View file

@ -0,0 +1,11 @@
using System;
namespace ConsoleApplication
{
public class Program
{
public static void Main()
{
}
}
}

View file

@ -0,0 +1,16 @@
{
"version": "1.0.0-*",
"compilationOptions": {
"emitEntryPoint": true
},
"dependencies": {
"Microsoft.NETCore.App": {
"type": "platform",
"version": "1.0.0-rc2-*"
},
},
"frameworks": {
"netcoreapp1.0": { },
"netstandard1.5": { }
}
}

View file

@ -0,0 +1,11 @@
using System;
namespace ConsoleApplication
{
public class Program
{
public static void Main()
{
}
}
}

View file

@ -0,0 +1,16 @@
{
"version": "1.0.0-*",
"compilationOptions": {
"emitEntryPoint": true
},
"dependencies": {
"Microsoft.NETCore.App": {
"type": "platform",
"version": "1.0.0-rc2-*"
},
},
"frameworks": {
"netcoreapp1.0": { },
"netstandard1.5": { }
}
}

View file

@ -8,6 +8,7 @@ using System.IO;
using System.Linq;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading;
using Microsoft.DotNet.Cli;
using Microsoft.DotNet.Cli.Compiler.Common;
using Microsoft.DotNet.Cli.Utils;

View file

@ -29,6 +29,7 @@ namespace Microsoft.DotNet.Tools.Test.Utilities
private readonly bool _noIncremental;
private readonly bool _noDependencies;
private readonly string _runtime;
private readonly bool _verbose;
private string OutputOption
{
@ -199,6 +200,16 @@ namespace Microsoft.DotNet.Tools.Test.Utilities
}
}
private string Verbose
{
get
{
return _verbose ?
"--verbose" :
"";
}
}
public BuildCommand(
string projectPath,
string output="",
@ -217,7 +228,8 @@ namespace Microsoft.DotNet.Tools.Test.Utilities
string cppCompilerFlags="",
bool buildProfile=true,
bool noIncremental=false,
bool noDependencies=false)
bool noDependencies=false,
bool verbose=true)
: base("dotnet")
{
_projectPath = projectPath;
@ -240,17 +252,18 @@ namespace Microsoft.DotNet.Tools.Test.Utilities
_buildProfile = buildProfile;
_noIncremental = noIncremental;
_noDependencies = noDependencies;
_verbose = verbose;
}
public override CommandResult Execute(string args = "")
{
args = $"--verbose build {BuildArgs()} {args}";
args = $"{Verbose} build {BuildArgs()} {args}";
return base.Execute(args);
}
public override CommandResult ExecuteWithCapturedOutput(string args = "")
{
args = $"--verbose build {BuildArgs()} {args}";
args = $"{Verbose} build {BuildArgs()} {args}";
return base.ExecuteWithCapturedOutput(args);
}

View file

@ -0,0 +1,182 @@
// Copyright (c) .NET Foundation and contributors. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
using System.Collections.Generic;
using System.Linq;
using Microsoft.DotNet.Tools.Test.Utilities;
using Xunit;
namespace Microsoft.DotNet.Tools.Builder.Tests
{
public class BuildPerformanceTest : PerformanceTestBase
{
public static IEnumerable<object> SingleProjects
{
get
{
yield return new [] { "TwoTargetApp"};
yield return new [] { "SingleTargetApp" };
}
}
public static IEnumerable<object> GraphProjects
{
get
{
yield return new object[]
{
"TwoTargetGraph",
new[] { "TwoTargetGraph/TwoTargetP0", "TwoTargetGraph/TwoTargetP1", "TwoTargetGraph/TwoTargetP2" }
};
yield return new object[]
{
"TwoTargetGraphLarge",
new[]
{
"TwoTargetGraphLarge/TwoTargetLargeP0",
"TwoTargetGraphLarge/TwoTargetLargeP1",
"TwoTargetGraphLarge/TwoTargetLargeP2",
"TwoTargetGraphLarge/TwoTargetLargeP3",
"TwoTargetGraphLarge/TwoTargetLargeP4",
"TwoTargetGraphLarge/TwoTargetLargeP5",
"TwoTargetGraphLarge/TwoTargetLargeP6"
}
};
yield return new object[]
{
"SingleTargetGraph",
new[] { "SingleTargetGraph/SingleTargetP0", "SingleTargetGraph/SingleTargetP1", "SingleTargetGraph/SingleTargetP2" }
};
}
}
[Theory]
[MemberData(nameof(SingleProjects))]
public void BuildSingleProject(string project)
{
var instance = CreateTestInstance(project);
Iterate(c =>
{
c.Measure(() => Build(instance.TestRoot));
RemoveBin(instance.TestRoot);
}, project);
}
[Theory]
[MemberData(nameof(SingleProjects))]
public void IncrementalSkipSingleProject(string project)
{
var instance = CreateTestInstance(project);
Build(instance.TestRoot);
Iterate(c =>
{
c.Measure(() => Build(instance.TestRoot));
}, project);
}
[Theory]
[MemberData(nameof(GraphProjects))]
public void BuildAllInGraph(string variation, string[] projects)
{
var instances = projects.Select(p => CreateTestInstance(p, variation)).ToArray();
var instance = instances[0];
Iterate(c =>
{
c.Measure(() => Build(instance.TestRoot));
foreach (var i in instances)
{
RemoveBin(i.TestRoot);
}
}, variation);
}
[Theory]
[MemberData(nameof(GraphProjects))]
public void IncrementalSkipAllInGraph(string variation, string[] projects)
{
var instances = projects.Select(p => CreateTestInstance(p, variation)).ToArray();
var instance = instances[0];
Build(instance.TestRoot);
Iterate(c =>
{
c.Measure(() => Build(instance.TestRoot));
}, variation);
}
[Theory]
[MemberData(nameof(GraphProjects))]
public void IncrementalRebuildWithRootChangedInGraph(string variation, string[] projects)
{
var instances = projects.Select(p => CreateTestInstance(p, variation)).ToArray();
var instance = instances[0];
Build(instance.TestRoot);
Iterate(c =>
{
c.Measure(() => Build(instance.TestRoot));
RemoveBin(instance.TestRoot);
}, variation);
}
[Theory]
[MemberData(nameof(GraphProjects))]
public void IncrementalRebuildWithLastChangedInGraph(string variation, string[] projects)
{
var instances = projects.Select(p => CreateTestInstance(p, variation)).ToArray();
var instance = instances[0];
Build(instance.TestRoot);
Iterate(c =>
{
c.Measure(() => Build(instance.TestRoot));
RemoveBin(instances.Last().TestRoot);
}, variation);
}
[Theory]
[MemberData(nameof(GraphProjects))]
public void IncrementalSkipAllNoDependenciesInGraph(string variation, string[] projects)
{
var instances = projects.Select(p => CreateTestInstance(p, variation)).ToArray();
var instance = instances[0];
Build(instance.TestRoot);
Iterate(c =>
{
foreach (var i in instances)
{
c.Measure(() => Run(new BuildCommand(i.TestRoot, framework: DefaultFramework, noDependencies: true, buildProfile: false)));
}
}, variation);
}
[Theory]
[MemberData(nameof(GraphProjects))]
public void BuildAllNoDependenciesInGraph(string variation, string[] projects)
{
var instances = projects.Select(p => CreateTestInstance(p, variation)).ToArray();
Iterate(c =>
{
foreach (var i in instances.Reverse())
{
c.Measure(() => Run(new BuildCommand(i.TestRoot, framework: DefaultFramework, noDependencies: true, buildProfile: false)));
}
foreach (var instance in instances)
{
RemoveBin(instance.TestRoot);
}
}, variation);
}
}
}

View file

@ -0,0 +1,63 @@
using System;
using System.Diagnostics;
using System.IO;
using System.Runtime.CompilerServices;
using Microsoft.DotNet.Tools.Test.Utilities;
using Microsoft.DotNet.TestFramework;
using Microsoft.DotNet.Cli.Utils;
namespace Microsoft.DotNet.Tools.Builder.Tests
{
public class PerformanceTestBase : TestBase
{
protected void Build(string project)
{
Run(new BuildCommand(project, buildProfile: false));
}
protected void Run(TestCommand command)
{
command.Execute().Should().Pass();
}
protected void RemoveBin(string project)
{
Directory.Delete(Path.Combine(project, "bin"), true);
}
protected TestInstance CreateTestInstance(string testProjectName, string variation = "", [CallerMemberName] string callingMethod = "")
{
return TestAssetsManager.CreateTestInstance(Path.Combine("PerformanceTestProjects", testProjectName), callingMethod + variation)
.WithLockFiles();
}
public void Iterate(Action<PerformanceIterationContext> action, string variation = "", int iterations = 3, [CallerMemberName] string callingMethod = "")
{
var fullname = callingMethod + (variation != "" ? "" + variation : "");
// Heat up iteration
var context = new PerformanceIterationContext();
action(context);
TimeSpan totalTime;
for (int i = 0; i < iterations; i++)
{
context = new PerformanceIterationContext();
action(context);
totalTime += context.Stopwatch.Elapsed;
}
Reporter.Output.WriteLine($"[RESULT] {callingMethod}-{variation} {totalTime.TotalSeconds/iterations:F} sec/iteration".Bold());
}
public class PerformanceIterationContext
{
public Stopwatch Stopwatch { get; } = new Stopwatch();
public void Measure(Action action)
{
Stopwatch.Start();
action();
Stopwatch.Stop();
}
}
}
}

View file

@ -0,0 +1,38 @@
param($versions,$file)
function cleanpath()
{
$env:PATH=($env:PATH.Split(";") | Where { !$_.Contains("dotnet") }) -Join ";"
}
foreach($ver in $versions)
{
cleanpath;
if ($ver -ne "dev")
{
Write-Host -ForegroundColor Green "Installing $ver"
$dotnetPath = "$PSScriptRoot\.dotnet\$ver"
if (!(test-path $dotnetPath))
{
& ($PSScriptRoot+"\..\..\scripts\obtain\install.ps1") preview $ver $dotnetPath;
}
$env:PATH = "$dotnetPath;"+$env:PATH
}
else
{
Write-Host -ForegroundColor Green "Using dev"
& ($PSScriptRoot+"\..\..\scripts\use-dev.ps1");
}
cmd /c "where dotnet" | Write-Host -ForegroundColor Green
dotnet --version | Write-Host -ForegroundColor Green
dotnet test `
| Where {$_.startswith("[RESULT]") } `
| %{
$_ | Write-Host -ForegroundColor Blue
if ($file)
{
$csvLine = $ver + $_.Replace(" ", ",").Replace("[RESULT]","").Replace("sec/iteration", "");
Add-Content $file $csvLine
}
}
}

View file

@ -0,0 +1,19 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="14.0.25123" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup>
<VisualStudioVersion Condition="'$(VisualStudioVersion)' == ''">14.0.25123</VisualStudioVersion>
<VSToolsPath Condition="'$(VSToolsPath)' == ''">$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)</VSToolsPath>
</PropertyGroup>
<Import Project="$(VSToolsPath)\DotNet\Microsoft.DotNet.Props" Condition="'$(VSToolsPath)' != ''" />
<PropertyGroup Label="Globals">
<ProjectGuid>0ae3d045-c119-4ed7-bd0c-287029b7264a</ProjectGuid>
<RootNamespace>dotnet-performance.Tests</RootNamespace>
<BaseIntermediateOutputPath Condition="'$(BaseIntermediateOutputPath)'=='' ">..\..\artifacts\obj\$(MSBuildProjectName)</BaseIntermediateOutputPath>
<OutputPath Condition="'$(OutputPath)'=='' ">..\..\artifacts\</OutputPath>
</PropertyGroup>
<PropertyGroup>
<SchemaVersion>2.0</SchemaVersion>
</PropertyGroup>
<Import Project="$(VSToolsPath)\DotNet\Microsoft.DotNet.targets" Condition="'$(VSToolsPath)' != ''" />
</Project>

View file

@ -0,0 +1,29 @@
{
"version": "1.0.0-*",
"dependencies": {
"Microsoft.NETCore.App": "1.0.0-rc2-*",
"System.Runtime.Serialization.Primitives": "4.1.1-rc2-24022",
"Microsoft.DotNet.Tools.Tests.Utilities": {
"target": "project"
},
"Microsoft.DotNet.Cli.Utils": {
"target": "project"
},
"Newtonsoft.Json": "7.0.1",
"xunit": "2.1.0",
"dotnet-test-xunit": "1.0.0-dev-140469-38"
},
"frameworks": {
"netcoreapp1.0": {
"imports": [
"dotnet5.4",
"portable-net451+win8"
]
}
},
"testRunner": "xunit",
"runtimes":
{
"win7-x64": {}
}
}