From 366bad99ac31c968983ea80f8a6845a63d64e228 Mon Sep 17 00:00:00 2001 From: Pavel Krymets Date: Fri, 22 Apr 2016 17:06:55 -0700 Subject: [PATCH 1/4] Add performance tests --- .../SingleTargetApp/Program.cs | 11 ++ .../SingleTargetApp/project.json | 15 ++ .../SingleTargetP0/Program.cs | 11 ++ .../SingleTargetP0/project.json | 18 ++ .../SingleTargetP1/Program.cs | 11 ++ .../SingleTargetP1/project.json | 18 ++ .../SingleTargetP2/Program.cs | 11 ++ .../SingleTargetP2/project.json | 15 ++ .../TwoTargetApp/Program.cs | 11 ++ .../TwoTargetApp/project.json | 16 ++ .../TwoTargetGraph/TwoTargetP0/Program.cs | 11 ++ .../TwoTargetGraph/TwoTargetP0/project.json | 19 ++ .../TwoTargetGraph/TwoTargetP1/Program.cs | 11 ++ .../TwoTargetGraph/TwoTargetP1/project.json | 19 ++ .../TwoTargetGraph/TwoTargetP2/Program.cs | 11 ++ .../TwoTargetGraph/TwoTargetP2/project.json | 16 ++ .../TwoTargetLargeP0/Program.cs | 11 ++ .../TwoTargetLargeP0/project.json | 19 ++ .../TwoTargetLargeP1/Program.cs | 11 ++ .../TwoTargetLargeP1/project.json | 22 +++ .../TwoTargetLargeP2/Program.cs | 11 ++ .../TwoTargetLargeP2/project.json | 22 +++ .../TwoTargetLargeP3/Program.cs | 11 ++ .../TwoTargetLargeP3/project.json | 19 ++ .../TwoTargetLargeP4/Program.cs | 11 ++ .../TwoTargetLargeP4/project.json | 22 +++ .../TwoTargetLargeP5/Program.cs | 11 ++ .../TwoTargetLargeP5/project.json | 16 ++ .../TwoTargetLargeP6/Program.cs | 11 ++ .../TwoTargetLargeP6/project.json | 16 ++ .../dotnet-compile/ManagedCompiler.cs | 1 + .../Commands/BuildCommand.cs | 19 +- .../BuildPerformanceTest.cs | 182 ++++++++++++++++++ .../PerformanceTestBase.cs | 63 ++++++ test/dotnet-performance.Tests/RunMultiCli.ps1 | 38 ++++ .../dotnet-performance.Tests.xproj | 19 ++ test/dotnet-performance.Tests/project.json | 29 +++ 37 files changed, 785 insertions(+), 3 deletions(-) create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/SingleTargetApp/Program.cs create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/SingleTargetApp/project.json create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP0/Program.cs create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP0/project.json create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP1/Program.cs create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP1/project.json create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP2/Program.cs create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP2/project.json create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/TwoTargetApp/Program.cs create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/TwoTargetApp/project.json create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP0/Program.cs create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP0/project.json create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP1/Program.cs create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP1/project.json create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP2/Program.cs create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP2/project.json create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP0/Program.cs create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP0/project.json create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP1/Program.cs create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP1/project.json create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP2/Program.cs create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP2/project.json create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP3/Program.cs create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP3/project.json create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP4/Program.cs create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP4/project.json create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP5/Program.cs create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP5/project.json create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP6/Program.cs create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP6/project.json create mode 100644 test/dotnet-performance.Tests/BuildPerformanceTest.cs create mode 100644 test/dotnet-performance.Tests/PerformanceTestBase.cs create mode 100644 test/dotnet-performance.Tests/RunMultiCli.ps1 create mode 100644 test/dotnet-performance.Tests/dotnet-performance.Tests.xproj create mode 100644 test/dotnet-performance.Tests/project.json diff --git a/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetApp/Program.cs b/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetApp/Program.cs new file mode 100644 index 000000000..9bfc3c72c --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetApp/Program.cs @@ -0,0 +1,11 @@ +using System; + +namespace ConsoleApplication +{ + public class Program + { + public static void Main() + { + } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetApp/project.json b/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetApp/project.json new file mode 100644 index 000000000..89cf3a3bf --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetApp/project.json @@ -0,0 +1,15 @@ +{ + "version": "1.0.0-*", + "compilationOptions": { + "emitEntryPoint": true + }, + "dependencies": { + "Microsoft.NETCore.App": { + "type": "platform", + "version": "1.0.0-rc2-*" + }, + }, + "frameworks": { + "netcoreapp1.0": { } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP0/Program.cs b/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP0/Program.cs new file mode 100644 index 000000000..9bfc3c72c --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP0/Program.cs @@ -0,0 +1,11 @@ +using System; + +namespace ConsoleApplication +{ + public class Program + { + public static void Main() + { + } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP0/project.json b/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP0/project.json new file mode 100644 index 000000000..c65037aca --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP0/project.json @@ -0,0 +1,18 @@ +{ + "version": "1.0.0-*", + "compilationOptions": { + "emitEntryPoint": true + }, + "dependencies": { + "SingleTargetP1" : { + "target": "project" + }, + "Microsoft.NETCore.App": { + "type": "platform", + "version": "1.0.0-rc2-*" + }, + }, + "frameworks": { + "netcoreapp1.0": { } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP1/Program.cs b/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP1/Program.cs new file mode 100644 index 000000000..9bfc3c72c --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP1/Program.cs @@ -0,0 +1,11 @@ +using System; + +namespace ConsoleApplication +{ + public class Program + { + public static void Main() + { + } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP1/project.json b/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP1/project.json new file mode 100644 index 000000000..a6c2ff8b5 --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP1/project.json @@ -0,0 +1,18 @@ +{ + "version": "1.0.0-*", + "compilationOptions": { + "emitEntryPoint": true + }, + "dependencies": { + "SingleTargetP2" : { + "target": "project" + }, + "Microsoft.NETCore.App": { + "type": "platform", + "version": "1.0.0-rc2-*" + }, + }, + "frameworks": { + "netcoreapp1.0": { } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP2/Program.cs b/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP2/Program.cs new file mode 100644 index 000000000..9bfc3c72c --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP2/Program.cs @@ -0,0 +1,11 @@ +using System; + +namespace ConsoleApplication +{ + public class Program + { + public static void Main() + { + } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP2/project.json b/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP2/project.json new file mode 100644 index 000000000..89cf3a3bf --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP2/project.json @@ -0,0 +1,15 @@ +{ + "version": "1.0.0-*", + "compilationOptions": { + "emitEntryPoint": true + }, + "dependencies": { + "Microsoft.NETCore.App": { + "type": "platform", + "version": "1.0.0-rc2-*" + }, + }, + "frameworks": { + "netcoreapp1.0": { } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetApp/Program.cs b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetApp/Program.cs new file mode 100644 index 000000000..9bfc3c72c --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetApp/Program.cs @@ -0,0 +1,11 @@ +using System; + +namespace ConsoleApplication +{ + public class Program + { + public static void Main() + { + } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetApp/project.json b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetApp/project.json new file mode 100644 index 000000000..5ea926028 --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetApp/project.json @@ -0,0 +1,16 @@ +{ + "version": "1.0.0-*", + "compilationOptions": { + "emitEntryPoint": true + }, + "dependencies": { + "Microsoft.NETCore.App": { + "type": "platform", + "version": "1.0.0-rc2-*" + }, + }, + "frameworks": { + "netcoreapp1.0": { }, + "netstandard1.5": { } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP0/Program.cs b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP0/Program.cs new file mode 100644 index 000000000..9bfc3c72c --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP0/Program.cs @@ -0,0 +1,11 @@ +using System; + +namespace ConsoleApplication +{ + public class Program + { + public static void Main() + { + } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP0/project.json b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP0/project.json new file mode 100644 index 000000000..2b03279e5 --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP0/project.json @@ -0,0 +1,19 @@ +{ + "version": "1.0.0-*", + "compilationOptions": { + "emitEntryPoint": true + }, + "dependencies": { + "TwoTargetP1" : { + "target": "project" + }, + "Microsoft.NETCore.App": { + "type": "platform", + "version": "1.0.0-rc2-*" + }, + }, + "frameworks": { + "netcoreapp1.0": { }, + "netstandard1.5": { } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP1/Program.cs b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP1/Program.cs new file mode 100644 index 000000000..9bfc3c72c --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP1/Program.cs @@ -0,0 +1,11 @@ +using System; + +namespace ConsoleApplication +{ + public class Program + { + public static void Main() + { + } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP1/project.json b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP1/project.json new file mode 100644 index 000000000..b9fbb9f8d --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP1/project.json @@ -0,0 +1,19 @@ +{ + "version": "1.0.0-*", + "compilationOptions": { + "emitEntryPoint": true + }, + "dependencies": { + "TwoTargetP2" : { + "target": "project" + }, + "Microsoft.NETCore.App": { + "type": "platform", + "version": "1.0.0-rc2-*" + }, + }, + "frameworks": { + "netcoreapp1.0": { }, + "netstandard1.5": { } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP2/Program.cs b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP2/Program.cs new file mode 100644 index 000000000..9bfc3c72c --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP2/Program.cs @@ -0,0 +1,11 @@ +using System; + +namespace ConsoleApplication +{ + public class Program + { + public static void Main() + { + } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP2/project.json b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP2/project.json new file mode 100644 index 000000000..5ea926028 --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP2/project.json @@ -0,0 +1,16 @@ +{ + "version": "1.0.0-*", + "compilationOptions": { + "emitEntryPoint": true + }, + "dependencies": { + "Microsoft.NETCore.App": { + "type": "platform", + "version": "1.0.0-rc2-*" + }, + }, + "frameworks": { + "netcoreapp1.0": { }, + "netstandard1.5": { } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP0/Program.cs b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP0/Program.cs new file mode 100644 index 000000000..9bfc3c72c --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP0/Program.cs @@ -0,0 +1,11 @@ +using System; + +namespace ConsoleApplication +{ + public class Program + { + public static void Main() + { + } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP0/project.json b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP0/project.json new file mode 100644 index 000000000..22689acf8 --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP0/project.json @@ -0,0 +1,19 @@ +{ + "version": "1.0.0-*", + "compilationOptions": { + "emitEntryPoint": true + }, + "dependencies": { + "TwoTargetLargeP1" : { + "target": "project" + }, + "Microsoft.NETCore.App": { + "type": "platform", + "version": "1.0.0-rc2-*" + }, + }, + "frameworks": { + "netcoreapp1.0": { }, + "netstandard1.5": { } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP1/Program.cs b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP1/Program.cs new file mode 100644 index 000000000..9bfc3c72c --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP1/Program.cs @@ -0,0 +1,11 @@ +using System; + +namespace ConsoleApplication +{ + public class Program + { + public static void Main() + { + } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP1/project.json b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP1/project.json new file mode 100644 index 000000000..2e36f00eb --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP1/project.json @@ -0,0 +1,22 @@ +{ + "version": "1.0.0-*", + "compilationOptions": { + "emitEntryPoint": true + }, + "dependencies": { + "TwoTargetLargeP2" : { + "target": "project" + }, + "TwoTargetLargeP4" : { + "target": "project" + }, + "Microsoft.NETCore.App": { + "type": "platform", + "version": "1.0.0-rc2-*" + }, + }, + "frameworks": { + "netcoreapp1.0": { }, + "netstandard1.5": { } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP2/Program.cs b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP2/Program.cs new file mode 100644 index 000000000..9bfc3c72c --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP2/Program.cs @@ -0,0 +1,11 @@ +using System; + +namespace ConsoleApplication +{ + public class Program + { + public static void Main() + { + } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP2/project.json b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP2/project.json new file mode 100644 index 000000000..a6fdc9d7d --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP2/project.json @@ -0,0 +1,22 @@ +{ + "version": "1.0.0-*", + "compilationOptions": { + "emitEntryPoint": true + }, + "dependencies": { + "TwoTargetLargeP3" : { + "target": "project" + }, + "TwoTargetLargeP4" : { + "target": "project" + }, + "Microsoft.NETCore.App": { + "type": "platform", + "version": "1.0.0-rc2-*" + }, + }, + "frameworks": { + "netcoreapp1.0": { }, + "netstandard1.5": { } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP3/Program.cs b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP3/Program.cs new file mode 100644 index 000000000..9bfc3c72c --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP3/Program.cs @@ -0,0 +1,11 @@ +using System; + +namespace ConsoleApplication +{ + public class Program + { + public static void Main() + { + } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP3/project.json b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP3/project.json new file mode 100644 index 000000000..821a50a5f --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP3/project.json @@ -0,0 +1,19 @@ +{ + "version": "1.0.0-*", + "compilationOptions": { + "emitEntryPoint": true + }, + "dependencies": { + "TwoTargetLargeP4" : { + "target": "project" + }, + "Microsoft.NETCore.App": { + "type": "platform", + "version": "1.0.0-rc2-*" + }, + }, + "frameworks": { + "netcoreapp1.0": { }, + "netstandard1.5": { } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP4/Program.cs b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP4/Program.cs new file mode 100644 index 000000000..9bfc3c72c --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP4/Program.cs @@ -0,0 +1,11 @@ +using System; + +namespace ConsoleApplication +{ + public class Program + { + public static void Main() + { + } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP4/project.json b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP4/project.json new file mode 100644 index 000000000..e0d6ab60c --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP4/project.json @@ -0,0 +1,22 @@ +{ + "version": "1.0.0-*", + "compilationOptions": { + "emitEntryPoint": true + }, + "dependencies": { + "TwoTargetLargeP5" : { + "target": "project" + }, + "TwoTargetLargeP6" : { + "target": "project" + }, + "Microsoft.NETCore.App": { + "type": "platform", + "version": "1.0.0-rc2-*" + }, + }, + "frameworks": { + "netcoreapp1.0": { }, + "netstandard1.5": { } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP5/Program.cs b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP5/Program.cs new file mode 100644 index 000000000..9bfc3c72c --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP5/Program.cs @@ -0,0 +1,11 @@ +using System; + +namespace ConsoleApplication +{ + public class Program + { + public static void Main() + { + } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP5/project.json b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP5/project.json new file mode 100644 index 000000000..5ea926028 --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP5/project.json @@ -0,0 +1,16 @@ +{ + "version": "1.0.0-*", + "compilationOptions": { + "emitEntryPoint": true + }, + "dependencies": { + "Microsoft.NETCore.App": { + "type": "platform", + "version": "1.0.0-rc2-*" + }, + }, + "frameworks": { + "netcoreapp1.0": { }, + "netstandard1.5": { } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP6/Program.cs b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP6/Program.cs new file mode 100644 index 000000000..9bfc3c72c --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP6/Program.cs @@ -0,0 +1,11 @@ +using System; + +namespace ConsoleApplication +{ + public class Program + { + public static void Main() + { + } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP6/project.json b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP6/project.json new file mode 100644 index 000000000..5ea926028 --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP6/project.json @@ -0,0 +1,16 @@ +{ + "version": "1.0.0-*", + "compilationOptions": { + "emitEntryPoint": true + }, + "dependencies": { + "Microsoft.NETCore.App": { + "type": "platform", + "version": "1.0.0-rc2-*" + }, + }, + "frameworks": { + "netcoreapp1.0": { }, + "netstandard1.5": { } + } +} diff --git a/src/dotnet/commands/dotnet-compile/ManagedCompiler.cs b/src/dotnet/commands/dotnet-compile/ManagedCompiler.cs index 70a2d4f9d..f32266e18 100644 --- a/src/dotnet/commands/dotnet-compile/ManagedCompiler.cs +++ b/src/dotnet/commands/dotnet-compile/ManagedCompiler.cs @@ -8,6 +8,7 @@ using System.IO; using System.Linq; using System.Runtime.InteropServices; using System.Text; +using System.Threading; using Microsoft.DotNet.Cli; using Microsoft.DotNet.Cli.Compiler.Common; using Microsoft.DotNet.Cli.Utils; diff --git a/test/Microsoft.DotNet.Tools.Tests.Utilities/Commands/BuildCommand.cs b/test/Microsoft.DotNet.Tools.Tests.Utilities/Commands/BuildCommand.cs index b427bd0de..0ebd04b90 100644 --- a/test/Microsoft.DotNet.Tools.Tests.Utilities/Commands/BuildCommand.cs +++ b/test/Microsoft.DotNet.Tools.Tests.Utilities/Commands/BuildCommand.cs @@ -29,6 +29,7 @@ namespace Microsoft.DotNet.Tools.Test.Utilities private readonly bool _noIncremental; private readonly bool _noDependencies; private readonly string _runtime; + private readonly bool _verbose; private string OutputOption { @@ -199,6 +200,16 @@ namespace Microsoft.DotNet.Tools.Test.Utilities } } + private string Verbose + { + get + { + return _verbose ? + "--verbose" : + ""; + } + } + public BuildCommand( string projectPath, string output="", @@ -217,7 +228,8 @@ namespace Microsoft.DotNet.Tools.Test.Utilities string cppCompilerFlags="", bool buildProfile=true, bool noIncremental=false, - bool noDependencies=false) + bool noDependencies=false, + bool verbose=true) : base("dotnet") { _projectPath = projectPath; @@ -240,17 +252,18 @@ namespace Microsoft.DotNet.Tools.Test.Utilities _buildProfile = buildProfile; _noIncremental = noIncremental; _noDependencies = noDependencies; + _verbose = verbose; } public override CommandResult Execute(string args = "") { - args = $"--verbose build {BuildArgs()} {args}"; + args = $"{Verbose} build {BuildArgs()} {args}"; return base.Execute(args); } public override CommandResult ExecuteWithCapturedOutput(string args = "") { - args = $"--verbose build {BuildArgs()} {args}"; + args = $"{Verbose} build {BuildArgs()} {args}"; return base.ExecuteWithCapturedOutput(args); } diff --git a/test/dotnet-performance.Tests/BuildPerformanceTest.cs b/test/dotnet-performance.Tests/BuildPerformanceTest.cs new file mode 100644 index 000000000..3953e9916 --- /dev/null +++ b/test/dotnet-performance.Tests/BuildPerformanceTest.cs @@ -0,0 +1,182 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Collections.Generic; +using System.Linq; +using Microsoft.DotNet.Tools.Test.Utilities; +using Xunit; + +namespace Microsoft.DotNet.Tools.Builder.Tests +{ + public class BuildPerformanceTest : PerformanceTestBase + { + public static IEnumerable SingleProjects + { + get + { + yield return new [] { "TwoTargetApp"}; + yield return new [] { "SingleTargetApp" }; + } + } + + public static IEnumerable GraphProjects + { + get + { + yield return new object[] + { + "TwoTargetGraph", + new[] { "TwoTargetGraph/TwoTargetP0", "TwoTargetGraph/TwoTargetP1", "TwoTargetGraph/TwoTargetP2" } + }; + yield return new object[] + { + "TwoTargetGraphLarge", + new[] + { + "TwoTargetGraphLarge/TwoTargetLargeP0", + "TwoTargetGraphLarge/TwoTargetLargeP1", + "TwoTargetGraphLarge/TwoTargetLargeP2", + "TwoTargetGraphLarge/TwoTargetLargeP3", + "TwoTargetGraphLarge/TwoTargetLargeP4", + "TwoTargetGraphLarge/TwoTargetLargeP5", + "TwoTargetGraphLarge/TwoTargetLargeP6" + } + }; + yield return new object[] + { + "SingleTargetGraph", + new[] { "SingleTargetGraph/SingleTargetP0", "SingleTargetGraph/SingleTargetP1", "SingleTargetGraph/SingleTargetP2" } + }; + } + } + + [Theory] + [MemberData(nameof(SingleProjects))] + public void BuildSingleProject(string project) + { + var instance = CreateTestInstance(project); + + Iterate(c => + { + c.Measure(() => Build(instance.TestRoot)); + RemoveBin(instance.TestRoot); + }, project); + } + + [Theory] + [MemberData(nameof(SingleProjects))] + public void IncrementalSkipSingleProject(string project) + { + var instance = CreateTestInstance(project); + Build(instance.TestRoot); + + Iterate(c => + { + c.Measure(() => Build(instance.TestRoot)); + }, project); + } + + [Theory] + [MemberData(nameof(GraphProjects))] + public void BuildAllInGraph(string variation, string[] projects) + { + var instances = projects.Select(p => CreateTestInstance(p, variation)).ToArray(); + var instance = instances[0]; + + Iterate(c => + { + c.Measure(() => Build(instance.TestRoot)); + foreach (var i in instances) + { + RemoveBin(i.TestRoot); + } + }, variation); + } + + [Theory] + [MemberData(nameof(GraphProjects))] + public void IncrementalSkipAllInGraph(string variation, string[] projects) + { + var instances = projects.Select(p => CreateTestInstance(p, variation)).ToArray(); + var instance = instances[0]; + + Build(instance.TestRoot); + + Iterate(c => + { + c.Measure(() => Build(instance.TestRoot)); + }, variation); + } + + [Theory] + [MemberData(nameof(GraphProjects))] + public void IncrementalRebuildWithRootChangedInGraph(string variation, string[] projects) + { + var instances = projects.Select(p => CreateTestInstance(p, variation)).ToArray(); + var instance = instances[0]; + + Build(instance.TestRoot); + + Iterate(c => + { + c.Measure(() => Build(instance.TestRoot)); + RemoveBin(instance.TestRoot); + }, variation); + } + + [Theory] + [MemberData(nameof(GraphProjects))] + public void IncrementalRebuildWithLastChangedInGraph(string variation, string[] projects) + { + var instances = projects.Select(p => CreateTestInstance(p, variation)).ToArray(); + var instance = instances[0]; + + Build(instance.TestRoot); + + Iterate(c => + { + c.Measure(() => Build(instance.TestRoot)); + RemoveBin(instances.Last().TestRoot); + }, variation); + } + + + [Theory] + [MemberData(nameof(GraphProjects))] + public void IncrementalSkipAllNoDependenciesInGraph(string variation, string[] projects) + { + var instances = projects.Select(p => CreateTestInstance(p, variation)).ToArray(); + var instance = instances[0]; + + Build(instance.TestRoot); + + Iterate(c => + { + foreach (var i in instances) + { + c.Measure(() => Run(new BuildCommand(i.TestRoot, framework: DefaultFramework, noDependencies: true, buildProfile: false))); + } + }, variation); + } + + [Theory] + [MemberData(nameof(GraphProjects))] + public void BuildAllNoDependenciesInGraph(string variation, string[] projects) + { + var instances = projects.Select(p => CreateTestInstance(p, variation)).ToArray(); + + Iterate(c => + { + foreach (var i in instances.Reverse()) + { + c.Measure(() => Run(new BuildCommand(i.TestRoot, framework: DefaultFramework, noDependencies: true, buildProfile: false))); + } + foreach (var instance in instances) + { + RemoveBin(instance.TestRoot); + } + + }, variation); + } + } +} diff --git a/test/dotnet-performance.Tests/PerformanceTestBase.cs b/test/dotnet-performance.Tests/PerformanceTestBase.cs new file mode 100644 index 000000000..e35606328 --- /dev/null +++ b/test/dotnet-performance.Tests/PerformanceTestBase.cs @@ -0,0 +1,63 @@ +using System; +using System.Diagnostics; +using System.IO; +using System.Runtime.CompilerServices; +using Microsoft.DotNet.Tools.Test.Utilities; +using Microsoft.DotNet.TestFramework; +using Microsoft.DotNet.Cli.Utils; + +namespace Microsoft.DotNet.Tools.Builder.Tests +{ + public class PerformanceTestBase : TestBase + { + protected void Build(string project) + { + Run(new BuildCommand(project, buildProfile: false)); + } + + protected void Run(TestCommand command) + { + command.Execute().Should().Pass(); + } + + protected void RemoveBin(string project) + { + Directory.Delete(Path.Combine(project, "bin"), true); + } + + protected TestInstance CreateTestInstance(string testProjectName, string variation = "", [CallerMemberName] string callingMethod = "") + { + return TestAssetsManager.CreateTestInstance(Path.Combine("PerformanceTestProjects", testProjectName), callingMethod + variation) + .WithLockFiles(); + } + + public void Iterate(Action action, string variation = "", int iterations = 3, [CallerMemberName] string callingMethod = "") + { + var fullname = callingMethod + (variation != "" ? "" + variation : ""); + // Heat up iteration + var context = new PerformanceIterationContext(); + action(context); + + TimeSpan totalTime; + for (int i = 0; i < iterations; i++) + { + context = new PerformanceIterationContext(); + action(context); + totalTime += context.Stopwatch.Elapsed; + } + Reporter.Output.WriteLine($"[RESULT] {callingMethod}-{variation} {totalTime.TotalSeconds/iterations:F} sec/iteration".Bold()); + } + + public class PerformanceIterationContext + { + public Stopwatch Stopwatch { get; } = new Stopwatch(); + + public void Measure(Action action) + { + Stopwatch.Start(); + action(); + Stopwatch.Stop(); + } + } + } +} \ No newline at end of file diff --git a/test/dotnet-performance.Tests/RunMultiCli.ps1 b/test/dotnet-performance.Tests/RunMultiCli.ps1 new file mode 100644 index 000000000..2caa37e25 --- /dev/null +++ b/test/dotnet-performance.Tests/RunMultiCli.ps1 @@ -0,0 +1,38 @@ +param($versions,$file) + +function cleanpath() +{ + $env:PATH=($env:PATH.Split(";") | Where { !$_.Contains("dotnet") }) -Join ";" +} +foreach($ver in $versions) +{ + cleanpath; + if ($ver -ne "dev") + { + Write-Host -ForegroundColor Green "Installing $ver" + $dotnetPath = "$PSScriptRoot\.dotnet\$ver" + if (!(test-path $dotnetPath)) + { + & ($PSScriptRoot+"\..\..\scripts\obtain\install.ps1") preview $ver $dotnetPath; + } + $env:PATH = "$dotnetPath;"+$env:PATH + } + else + { + Write-Host -ForegroundColor Green "Using dev" + & ($PSScriptRoot+"\..\..\scripts\use-dev.ps1"); + } + cmd /c "where dotnet" | Write-Host -ForegroundColor Green + dotnet --version | Write-Host -ForegroundColor Green + + dotnet test ` + | Where {$_.startswith("[RESULT]") } ` + | %{ + $_ | Write-Host -ForegroundColor Blue + if ($file) + { + $csvLine = $ver + $_.Replace(" ", ",").Replace("[RESULT]","").Replace("sec/iteration", ""); + Add-Content $file $csvLine + } + } +} \ No newline at end of file diff --git a/test/dotnet-performance.Tests/dotnet-performance.Tests.xproj b/test/dotnet-performance.Tests/dotnet-performance.Tests.xproj new file mode 100644 index 000000000..4a6c22bc9 --- /dev/null +++ b/test/dotnet-performance.Tests/dotnet-performance.Tests.xproj @@ -0,0 +1,19 @@ + + + + 14.0.25123 + $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion) + + + + 0ae3d045-c119-4ed7-bd0c-287029b7264a + dotnet-performance.Tests + ..\..\artifacts\obj\$(MSBuildProjectName) + ..\..\artifacts\ + + + + 2.0 + + + \ No newline at end of file diff --git a/test/dotnet-performance.Tests/project.json b/test/dotnet-performance.Tests/project.json new file mode 100644 index 000000000..ff0e453f4 --- /dev/null +++ b/test/dotnet-performance.Tests/project.json @@ -0,0 +1,29 @@ +{ + "version": "1.0.0-*", + "dependencies": { + "Microsoft.NETCore.App": "1.0.0-rc2-*", + "System.Runtime.Serialization.Primitives": "4.1.1-rc2-24022", + "Microsoft.DotNet.Tools.Tests.Utilities": { + "target": "project" + }, + "Microsoft.DotNet.Cli.Utils": { + "target": "project" + }, + "Newtonsoft.Json": "7.0.1", + "xunit": "2.1.0", + "dotnet-test-xunit": "1.0.0-dev-140469-38" + }, + "frameworks": { + "netcoreapp1.0": { + "imports": [ + "dotnet5.4", + "portable-net451+win8" + ] + } + }, + "testRunner": "xunit", + "runtimes": + { + "win7-x64": {} + } +} From 0401935d3164429cc51aab314bb7ea7410617b5a Mon Sep 17 00:00:00 2001 From: Pavel Krymets Date: Tue, 26 Apr 2016 09:10:28 -0700 Subject: [PATCH 2/4] Use performance test --- .../BuildPerformanceTest.cs | 319 +++++++++++------- .../PerformanceTestBase.cs | 43 +-- test/dotnet-performance.Tests/nuget.config | 6 + test/dotnet-performance.Tests/project.json | 3 +- 4 files changed, 211 insertions(+), 160 deletions(-) create mode 100644 test/dotnet-performance.Tests/nuget.config diff --git a/test/dotnet-performance.Tests/BuildPerformanceTest.cs b/test/dotnet-performance.Tests/BuildPerformanceTest.cs index 3953e9916..c8b140d48 100644 --- a/test/dotnet-performance.Tests/BuildPerformanceTest.cs +++ b/test/dotnet-performance.Tests/BuildPerformanceTest.cs @@ -1,182 +1,249 @@ // Copyright (c) .NET Foundation and contributors. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using System.Collections.Generic; using System.Linq; using Microsoft.DotNet.Tools.Test.Utilities; -using Xunit; +using Microsoft.Xunit.Performance; +using Microsoft.DotNet.TestFramework; namespace Microsoft.DotNet.Tools.Builder.Tests { public class BuildPerformanceTest : PerformanceTestBase { - public static IEnumerable SingleProjects - { - get - { - yield return new [] { "TwoTargetApp"}; - yield return new [] { "SingleTargetApp" }; - } - } + private static string SingleTargetApp = "SingleTargetApp"; + private static string TwoTargetApp = "TwoTargetApp"; - public static IEnumerable GraphProjects + private static string[] SingleTargetGraph = new[] { - get - { - yield return new object[] - { - "TwoTargetGraph", - new[] { "TwoTargetGraph/TwoTargetP0", "TwoTargetGraph/TwoTargetP1", "TwoTargetGraph/TwoTargetP2" } - }; - yield return new object[] - { - "TwoTargetGraphLarge", - new[] - { - "TwoTargetGraphLarge/TwoTargetLargeP0", - "TwoTargetGraphLarge/TwoTargetLargeP1", - "TwoTargetGraphLarge/TwoTargetLargeP2", - "TwoTargetGraphLarge/TwoTargetLargeP3", - "TwoTargetGraphLarge/TwoTargetLargeP4", - "TwoTargetGraphLarge/TwoTargetLargeP5", - "TwoTargetGraphLarge/TwoTargetLargeP6" - } - }; - yield return new object[] - { - "SingleTargetGraph", - new[] { "SingleTargetGraph/SingleTargetP0", "SingleTargetGraph/SingleTargetP1", "SingleTargetGraph/SingleTargetP2" } - }; - } - } + "SingleTargetGraph/SingleTargetP0", + "SingleTargetGraph/SingleTargetP1", + "SingleTargetGraph/SingleTargetP2" + }; - [Theory] - [MemberData(nameof(SingleProjects))] - public void BuildSingleProject(string project) + private static string[] TwoTargetGraph = new[] { - var instance = CreateTestInstance(project); + "TwoTargetGraph/TwoTargetP0", + "TwoTargetGraph/TwoTargetP1", + "TwoTargetGraph/TwoTargetP2" + }; - Iterate(c => + private static string[] TwoTargetGraphLarge = new[] + { + "TwoTargetGraphLarge/TwoTargetLargeP0", + "TwoTargetGraphLarge/TwoTargetLargeP1", + "TwoTargetGraphLarge/TwoTargetLargeP2", + "TwoTargetGraphLarge/TwoTargetLargeP3", + "TwoTargetGraphLarge/TwoTargetLargeP4", + "TwoTargetGraphLarge/TwoTargetLargeP5", + "TwoTargetGraphLarge/TwoTargetLargeP6" + }; + + [Benchmark] + public void BuildSingleProject_SingleTargetApp() => BuildSingleProject(CreateTestInstance(SingleTargetApp)); + [Benchmark] + public void BuildSingleProject_TwoTargetApp() => BuildSingleProject(CreateTestInstance(TwoTargetApp)); + + public void BuildSingleProject(TestInstance instance) + { + foreach (var iteration in Benchmark.Iterations) { - c.Measure(() => Build(instance.TestRoot)); + using (iteration.StartMeasurement()) + { + Build(instance.TestRoot); + } RemoveBin(instance.TestRoot); - }, project); + } } - [Theory] - [MemberData(nameof(SingleProjects))] - public void IncrementalSkipSingleProject(string project) + [Benchmark] + public void IncrementalSkipSingleProject_SingleTargetApp() => IncrementalSkipSingleProject(CreateTestInstance(SingleTargetApp)); + [Benchmark] + public void IncrementalSkipSingleProject_TwoTargetApp() => IncrementalSkipSingleProject(CreateTestInstance(TwoTargetApp)); + + public void IncrementalSkipSingleProject(TestInstance instance) { - var instance = CreateTestInstance(project); Build(instance.TestRoot); - Iterate(c => + foreach (var iteration in Benchmark.Iterations) { - c.Measure(() => Build(instance.TestRoot)); - }, project); + using (iteration.StartMeasurement()) + { + Build(instance.TestRoot); + } + } } - [Theory] - [MemberData(nameof(GraphProjects))] - public void BuildAllInGraph(string variation, string[] projects) + [Benchmark] + public void BuildAllInGraph_SingleTargetGraph() => BuildAllInGraph(CreateTestInstances(SingleTargetGraph)); + [Benchmark] + public void BuildAllInGraph_TwoTargetGraph() => BuildAllInGraph(CreateTestInstances(TwoTargetGraph)); + [Benchmark] + public void BuildAllInGraph_TwoTargetGraphLarge() => BuildAllInGraph(CreateTestInstances(TwoTargetGraphLarge)); + + public void BuildAllInGraph(TestInstance[] instances) { - var instances = projects.Select(p => CreateTestInstance(p, variation)).ToArray(); var instance = instances[0]; - Iterate(c => + foreach (var iteration in Benchmark.Iterations) { - c.Measure(() => Build(instance.TestRoot)); + using (iteration.StartMeasurement()) + { + Build(instance.TestRoot); + } foreach (var i in instances) { RemoveBin(i.TestRoot); } - }, variation); + } } - [Theory] - [MemberData(nameof(GraphProjects))] - public void IncrementalSkipAllInGraph(string variation, string[] projects) + [Benchmark] + public void IncrementalSkipAllInGraph_SingleTargetGraph() => + IncrementalSkipAllInGraph(CreateTestInstances(SingleTargetGraph)); + + [Benchmark] + public void IncrementalSkipAllInGraph_TwoTargetGraph() => + IncrementalSkipAllInGraph(CreateTestInstances(TwoTargetGraph)); + + [Benchmark] + public void IncrementalSkipAllInGraphh_TwoTargetGraphLarge() => + IncrementalSkipAllInGraph(CreateTestInstances(TwoTargetGraphLarge)); + + public void IncrementalSkipAllInGraph(TestInstance[] instances) { - var instances = projects.Select(p => CreateTestInstance(p, variation)).ToArray(); var instance = instances[0]; Build(instance.TestRoot); - Iterate(c => + foreach (var iteration in Benchmark.Iterations) { - c.Measure(() => Build(instance.TestRoot)); - }, variation); - } - - [Theory] - [MemberData(nameof(GraphProjects))] - public void IncrementalRebuildWithRootChangedInGraph(string variation, string[] projects) - { - var instances = projects.Select(p => CreateTestInstance(p, variation)).ToArray(); - var instance = instances[0]; - - Build(instance.TestRoot); - - Iterate(c => - { - c.Measure(() => Build(instance.TestRoot)); - RemoveBin(instance.TestRoot); - }, variation); - } - - [Theory] - [MemberData(nameof(GraphProjects))] - public void IncrementalRebuildWithLastChangedInGraph(string variation, string[] projects) - { - var instances = projects.Select(p => CreateTestInstance(p, variation)).ToArray(); - var instance = instances[0]; - - Build(instance.TestRoot); - - Iterate(c => - { - c.Measure(() => Build(instance.TestRoot)); - RemoveBin(instances.Last().TestRoot); - }, variation); - } - - - [Theory] - [MemberData(nameof(GraphProjects))] - public void IncrementalSkipAllNoDependenciesInGraph(string variation, string[] projects) - { - var instances = projects.Select(p => CreateTestInstance(p, variation)).ToArray(); - var instance = instances[0]; - - Build(instance.TestRoot); - - Iterate(c => - { - foreach (var i in instances) + using (iteration.StartMeasurement()) { - c.Measure(() => Run(new BuildCommand(i.TestRoot, framework: DefaultFramework, noDependencies: true, buildProfile: false))); + Build(instance.TestRoot); } - }, variation); + } } - [Theory] - [MemberData(nameof(GraphProjects))] - public void BuildAllNoDependenciesInGraph(string variation, string[] projects) - { - var instances = projects.Select(p => CreateTestInstance(p, variation)).ToArray(); + [Benchmark] + public void IncrementalRebuildWithRootChangedInGraph_SingleTargetGraph() => + IncrementalRebuildWithRootChangedInGraph(CreateTestInstances(SingleTargetGraph)); - Iterate(c => + [Benchmark] + public void IncrementalRebuildWithRootChangedInGraph_TwoTargetGraph() => + IncrementalRebuildWithRootChangedInGraph(CreateTestInstances(TwoTargetGraph)); + + [Benchmark] + public void IncrementalRebuildWithRootChangedInGraph_TwoTargetGraphLarge() => + IncrementalRebuildWithRootChangedInGraph(CreateTestInstances(TwoTargetGraphLarge)); + + public void IncrementalRebuildWithRootChangedInGraph(TestInstance[] instances) + { + var instance = instances[0]; + + Build(instance.TestRoot); + + foreach (var iteration in Benchmark.Iterations) { - foreach (var i in instances.Reverse()) + using (iteration.StartMeasurement()) { - c.Measure(() => Run(new BuildCommand(i.TestRoot, framework: DefaultFramework, noDependencies: true, buildProfile: false))); + Build(instance.TestRoot); + } + RemoveBin(instance.TestRoot); + } + } + + [Benchmark] + public void IncrementalRebuildWithLastChangedInGraph_SingleTargetGraph() => + IncrementalRebuildWithLastChangedInGraph(CreateTestInstances(SingleTargetGraph)); + + [Benchmark] + public void IncrementalRebuildWithLastChangedInGraph_TwoTargetGraph() => + IncrementalRebuildWithLastChangedInGraph(CreateTestInstances(TwoTargetGraph)); + + [Benchmark] + public void IncrementalRebuildWithLastChangedInGraph_TwoTargetGraphLarge() => + IncrementalRebuildWithLastChangedInGraph(CreateTestInstances(TwoTargetGraphLarge)); + + public void IncrementalRebuildWithLastChangedInGraph(TestInstance[] instances) + { + var instance = instances[0]; + + Build(instance.TestRoot); + + foreach (var iteration in Benchmark.Iterations) + { + using (iteration.StartMeasurement()) + { + Build(instance.TestRoot); + } + RemoveBin(instances.Last().TestRoot); + } + } + + + [Benchmark] + public void IncrementalSkipAllNoDependenciesInGraph_SingleTargetGraph() => + IncrementalSkipAllNoDependenciesInGraph(CreateTestInstances(SingleTargetGraph)); + + [Benchmark] + public void IncrementalSkipAllNoDependenciesInGraph_TwoTargetGraph() => + IncrementalSkipAllNoDependenciesInGraph(CreateTestInstances(TwoTargetGraph)); + + [Benchmark] + public void IncrementalSkipAllNoDependenciesInGraph_TwoTargetGraphLarge() => + IncrementalSkipAllNoDependenciesInGraph(CreateTestInstances(TwoTargetGraphLarge)); + + public void IncrementalSkipAllNoDependenciesInGraph(TestInstance[] instances) + { + var instance = instances[0]; + + Build(instance.TestRoot); + + foreach (var iteration in Benchmark.Iterations) + { + using (iteration.StartMeasurement()) + { + foreach (var i in instances) + { + Run(new BuildCommand(i.TestRoot, + framework: DefaultFramework, + noDependencies: true, + buildProfile: false)); + } + } + } + } + [Benchmark] + public void BuildAllNoDependenciesInGraphh_SingleTargetGraph() => + BuildAllNoDependenciesInGraph(CreateTestInstances(SingleTargetGraph)); + + [Benchmark] + public void BuildAllNoDependenciesInGraph_TwoTargetGraph() => + BuildAllNoDependenciesInGraph(CreateTestInstances(TwoTargetGraph)); + + [Benchmark] + public void BuildAllNoDependenciesInGraph_TwoTargetGraphLarge() => + BuildAllNoDependenciesInGraph(CreateTestInstances(TwoTargetGraphLarge)); + + public void BuildAllNoDependenciesInGraph(TestInstance[] instances) + { + foreach (var iteration in Benchmark.Iterations) + { + using (iteration.StartMeasurement()) + { + foreach (var i in instances.Reverse()) + { + Run(new BuildCommand(i.TestRoot, + framework: DefaultFramework, + noDependencies: true, + buildProfile: false)); + } } foreach (var instance in instances) { RemoveBin(instance.TestRoot); } - - }, variation); + } } } } diff --git a/test/dotnet-performance.Tests/PerformanceTestBase.cs b/test/dotnet-performance.Tests/PerformanceTestBase.cs index e35606328..8ae23e8d7 100644 --- a/test/dotnet-performance.Tests/PerformanceTestBase.cs +++ b/test/dotnet-performance.Tests/PerformanceTestBase.cs @@ -1,10 +1,8 @@ -using System; -using System.Diagnostics; -using System.IO; +using System.IO; +using System.Linq; using System.Runtime.CompilerServices; using Microsoft.DotNet.Tools.Test.Utilities; using Microsoft.DotNet.TestFramework; -using Microsoft.DotNet.Cli.Utils; namespace Microsoft.DotNet.Tools.Builder.Tests { @@ -25,39 +23,18 @@ namespace Microsoft.DotNet.Tools.Builder.Tests Directory.Delete(Path.Combine(project, "bin"), true); } - protected TestInstance CreateTestInstance(string testProjectName, string variation = "", [CallerMemberName] string callingMethod = "") + protected TestInstance[] CreateTestInstances(string[] testProjectNames, [CallerMemberName] string callingMethod = "") { - return TestAssetsManager.CreateTestInstance(Path.Combine("PerformanceTestProjects", testProjectName), callingMethod + variation) - .WithLockFiles(); + return testProjectNames.Select(testProjectName => + { + return CreateTestInstance(testProjectName, callingMethod); + }).ToArray(); } - public void Iterate(Action action, string variation = "", int iterations = 3, [CallerMemberName] string callingMethod = "") + protected TestInstance CreateTestInstance(string testProjectName, [CallerMemberName] string callingMethod = "") { - var fullname = callingMethod + (variation != "" ? "" + variation : ""); - // Heat up iteration - var context = new PerformanceIterationContext(); - action(context); - - TimeSpan totalTime; - for (int i = 0; i < iterations; i++) - { - context = new PerformanceIterationContext(); - action(context); - totalTime += context.Stopwatch.Elapsed; - } - Reporter.Output.WriteLine($"[RESULT] {callingMethod}-{variation} {totalTime.TotalSeconds/iterations:F} sec/iteration".Bold()); - } - - public class PerformanceIterationContext - { - public Stopwatch Stopwatch { get; } = new Stopwatch(); - - public void Measure(Action action) - { - Stopwatch.Start(); - action(); - Stopwatch.Stop(); - } + return TestAssetsManager.CreateTestInstance(Path.Combine("PerformanceTestProjects", testProjectName), callingMethod) + .WithLockFiles(); } } } \ No newline at end of file diff --git a/test/dotnet-performance.Tests/nuget.config b/test/dotnet-performance.Tests/nuget.config new file mode 100644 index 000000000..faeee1d91 --- /dev/null +++ b/test/dotnet-performance.Tests/nuget.config @@ -0,0 +1,6 @@ + + + + + + diff --git a/test/dotnet-performance.Tests/project.json b/test/dotnet-performance.Tests/project.json index ff0e453f4..87c1e90ae 100644 --- a/test/dotnet-performance.Tests/project.json +++ b/test/dotnet-performance.Tests/project.json @@ -11,7 +11,8 @@ }, "Newtonsoft.Json": "7.0.1", "xunit": "2.1.0", - "dotnet-test-xunit": "1.0.0-dev-140469-38" + "dotnet-test-xunit": "1.0.0-dev-140469-38", + "Microsoft.DotNet.xunit.performance": "1.0.0-alpha-build0028" }, "frameworks": { "netcoreapp1.0": { From cdb75966eba580970f52316ec78582e958fb6565 Mon Sep 17 00:00:00 2001 From: Pavel Krymets Date: Tue, 26 Apr 2016 09:12:39 -0700 Subject: [PATCH 3/4] Remove test script --- .../Commands/BuildCommand.cs | 4 +- test/dotnet-performance.Tests/RunMultiCli.ps1 | 38 ------------------- test/dotnet-performance.Tests/nuget.config | 6 --- 3 files changed, 1 insertion(+), 47 deletions(-) delete mode 100644 test/dotnet-performance.Tests/RunMultiCli.ps1 delete mode 100644 test/dotnet-performance.Tests/nuget.config diff --git a/test/Microsoft.DotNet.Tools.Tests.Utilities/Commands/BuildCommand.cs b/test/Microsoft.DotNet.Tools.Tests.Utilities/Commands/BuildCommand.cs index 0ebd04b90..351724f4a 100644 --- a/test/Microsoft.DotNet.Tools.Tests.Utilities/Commands/BuildCommand.cs +++ b/test/Microsoft.DotNet.Tools.Tests.Utilities/Commands/BuildCommand.cs @@ -204,9 +204,7 @@ namespace Microsoft.DotNet.Tools.Test.Utilities { get { - return _verbose ? - "--verbose" : - ""; + return _verbose ? "--verbose" : ""; } } diff --git a/test/dotnet-performance.Tests/RunMultiCli.ps1 b/test/dotnet-performance.Tests/RunMultiCli.ps1 deleted file mode 100644 index 2caa37e25..000000000 --- a/test/dotnet-performance.Tests/RunMultiCli.ps1 +++ /dev/null @@ -1,38 +0,0 @@ -param($versions,$file) - -function cleanpath() -{ - $env:PATH=($env:PATH.Split(";") | Where { !$_.Contains("dotnet") }) -Join ";" -} -foreach($ver in $versions) -{ - cleanpath; - if ($ver -ne "dev") - { - Write-Host -ForegroundColor Green "Installing $ver" - $dotnetPath = "$PSScriptRoot\.dotnet\$ver" - if (!(test-path $dotnetPath)) - { - & ($PSScriptRoot+"\..\..\scripts\obtain\install.ps1") preview $ver $dotnetPath; - } - $env:PATH = "$dotnetPath;"+$env:PATH - } - else - { - Write-Host -ForegroundColor Green "Using dev" - & ($PSScriptRoot+"\..\..\scripts\use-dev.ps1"); - } - cmd /c "where dotnet" | Write-Host -ForegroundColor Green - dotnet --version | Write-Host -ForegroundColor Green - - dotnet test ` - | Where {$_.startswith("[RESULT]") } ` - | %{ - $_ | Write-Host -ForegroundColor Blue - if ($file) - { - $csvLine = $ver + $_.Replace(" ", ",").Replace("[RESULT]","").Replace("sec/iteration", ""); - Add-Content $file $csvLine - } - } -} \ No newline at end of file diff --git a/test/dotnet-performance.Tests/nuget.config b/test/dotnet-performance.Tests/nuget.config deleted file mode 100644 index faeee1d91..000000000 --- a/test/dotnet-performance.Tests/nuget.config +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - From 1104e568952ec30ce060ed2e71750218c1899d55 Mon Sep 17 00:00:00 2001 From: Pavel Krymets Date: Wed, 27 Apr 2016 08:44:12 -0700 Subject: [PATCH 4/4] Rebase --- .../SingleTargetP0/Program.cs | 1 + .../SingleTargetP1/Program.cs | 3 +- .../SingleTargetP1/project.json | 3 - .../SingleTargetP2/Program.cs | 2 +- .../SingleTargetP2/project.json | 3 - .../TwoTargetGraph/TwoTargetP0/Program.cs | 1 + .../TwoTargetGraph/TwoTargetP1/Program.cs | 3 +- .../TwoTargetGraph/TwoTargetP1/project.json | 3 - .../TwoTargetGraph/TwoTargetP2/Program.cs | 2 +- .../TwoTargetGraph/TwoTargetP2/project.json | 3 - .../TwoTargetLargeP0/Program.cs | 1 + .../TwoTargetLargeP1/Program.cs | 4 +- .../TwoTargetLargeP1/project.json | 3 - .../TwoTargetLargeP2/Program.cs | 4 +- .../TwoTargetLargeP2/project.json | 3 - .../TwoTargetLargeP3/Program.cs | 3 +- .../TwoTargetLargeP3/project.json | 3 - .../TwoTargetLargeP4/Program.cs | 4 +- .../TwoTargetLargeP4/project.json | 3 - .../TwoTargetLargeP5/Program.cs | 2 +- .../TwoTargetLargeP5/project.json | 3 - .../TwoTargetLargeP6/Program.cs | 2 +- .../TwoTargetLargeP6/project.json | 3 - scripts/dotnet-cli-build/TestTargets.cs | 3 +- .../dotnet-compile/ManagedCompiler.cs | 1 - .../BuildPerformanceTest.cs | 109 +++++++++++++----- .../PerformanceTestBase.cs | 40 ------- .../dotnet-performance.Tests.xproj | 19 --- test/dotnet-performance.Tests/project.json | 30 ----- 29 files changed, 101 insertions(+), 163 deletions(-) rename test/{dotnet-performance.Tests => Performance}/BuildPerformanceTest.cs (66%) delete mode 100644 test/dotnet-performance.Tests/PerformanceTestBase.cs delete mode 100644 test/dotnet-performance.Tests/dotnet-performance.Tests.xproj delete mode 100644 test/dotnet-performance.Tests/project.json diff --git a/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP0/Program.cs b/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP0/Program.cs index 9bfc3c72c..ca05ade7a 100644 --- a/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP0/Program.cs +++ b/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP0/Program.cs @@ -6,6 +6,7 @@ namespace ConsoleApplication { public static void Main() { + ConsoleLibrary.P1.Program.Main(); } } } diff --git a/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP1/Program.cs b/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP1/Program.cs index 9bfc3c72c..ea50de9e8 100644 --- a/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP1/Program.cs +++ b/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP1/Program.cs @@ -1,11 +1,12 @@ using System; -namespace ConsoleApplication +namespace ConsoleLibrary.P1 { public class Program { public static void Main() { + ConsoleLibrary.P2.Program.Main(); } } } diff --git a/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP1/project.json b/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP1/project.json index a6c2ff8b5..0f4fee5ad 100644 --- a/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP1/project.json +++ b/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP1/project.json @@ -1,8 +1,5 @@ { "version": "1.0.0-*", - "compilationOptions": { - "emitEntryPoint": true - }, "dependencies": { "SingleTargetP2" : { "target": "project" diff --git a/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP2/Program.cs b/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP2/Program.cs index 9bfc3c72c..0ae001811 100644 --- a/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP2/Program.cs +++ b/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP2/Program.cs @@ -1,6 +1,6 @@ using System; -namespace ConsoleApplication +namespace ConsoleLibrary.P2 { public class Program { diff --git a/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP2/project.json b/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP2/project.json index 89cf3a3bf..71ae27297 100644 --- a/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP2/project.json +++ b/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP2/project.json @@ -1,8 +1,5 @@ { "version": "1.0.0-*", - "compilationOptions": { - "emitEntryPoint": true - }, "dependencies": { "Microsoft.NETCore.App": { "type": "platform", diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP0/Program.cs b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP0/Program.cs index 9bfc3c72c..ca05ade7a 100644 --- a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP0/Program.cs +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP0/Program.cs @@ -6,6 +6,7 @@ namespace ConsoleApplication { public static void Main() { + ConsoleLibrary.P1.Program.Main(); } } } diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP1/Program.cs b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP1/Program.cs index 9bfc3c72c..ea50de9e8 100644 --- a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP1/Program.cs +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP1/Program.cs @@ -1,11 +1,12 @@ using System; -namespace ConsoleApplication +namespace ConsoleLibrary.P1 { public class Program { public static void Main() { + ConsoleLibrary.P2.Program.Main(); } } } diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP1/project.json b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP1/project.json index b9fbb9f8d..3e279184d 100644 --- a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP1/project.json +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP1/project.json @@ -1,8 +1,5 @@ { "version": "1.0.0-*", - "compilationOptions": { - "emitEntryPoint": true - }, "dependencies": { "TwoTargetP2" : { "target": "project" diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP2/Program.cs b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP2/Program.cs index 9bfc3c72c..0ae001811 100644 --- a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP2/Program.cs +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP2/Program.cs @@ -1,6 +1,6 @@ using System; -namespace ConsoleApplication +namespace ConsoleLibrary.P2 { public class Program { diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP2/project.json b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP2/project.json index 5ea926028..a1f979cc9 100644 --- a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP2/project.json +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP2/project.json @@ -1,8 +1,5 @@ { "version": "1.0.0-*", - "compilationOptions": { - "emitEntryPoint": true - }, "dependencies": { "Microsoft.NETCore.App": { "type": "platform", diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP0/Program.cs b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP0/Program.cs index 9bfc3c72c..ca05ade7a 100644 --- a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP0/Program.cs +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP0/Program.cs @@ -6,6 +6,7 @@ namespace ConsoleApplication { public static void Main() { + ConsoleLibrary.P1.Program.Main(); } } } diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP1/Program.cs b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP1/Program.cs index 9bfc3c72c..7b1284ad8 100644 --- a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP1/Program.cs +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP1/Program.cs @@ -1,11 +1,13 @@ using System; -namespace ConsoleApplication +namespace ConsoleLibrary.P1 { public class Program { public static void Main() { + ConsoleLibrary.P2.Program.Main(); + ConsoleLibrary.P4.Program.Main(); } } } diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP1/project.json b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP1/project.json index 2e36f00eb..5b080a6f1 100644 --- a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP1/project.json +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP1/project.json @@ -1,8 +1,5 @@ { "version": "1.0.0-*", - "compilationOptions": { - "emitEntryPoint": true - }, "dependencies": { "TwoTargetLargeP2" : { "target": "project" diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP2/Program.cs b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP2/Program.cs index 9bfc3c72c..121dbbfb7 100644 --- a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP2/Program.cs +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP2/Program.cs @@ -1,11 +1,13 @@ using System; -namespace ConsoleApplication +namespace ConsoleLibrary.P2 { public class Program { public static void Main() { + ConsoleLibrary.P3.Program.Main(); + ConsoleLibrary.P4.Program.Main(); } } } diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP2/project.json b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP2/project.json index a6fdc9d7d..8b7d44b36 100644 --- a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP2/project.json +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP2/project.json @@ -1,8 +1,5 @@ { "version": "1.0.0-*", - "compilationOptions": { - "emitEntryPoint": true - }, "dependencies": { "TwoTargetLargeP3" : { "target": "project" diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP3/Program.cs b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP3/Program.cs index 9bfc3c72c..2e779be39 100644 --- a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP3/Program.cs +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP3/Program.cs @@ -1,11 +1,12 @@ using System; -namespace ConsoleApplication +namespace ConsoleLibrary.P3 { public class Program { public static void Main() { + ConsoleLibrary.P4.Program.Main(); } } } diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP3/project.json b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP3/project.json index 821a50a5f..e75856ac1 100644 --- a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP3/project.json +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP3/project.json @@ -1,8 +1,5 @@ { "version": "1.0.0-*", - "compilationOptions": { - "emitEntryPoint": true - }, "dependencies": { "TwoTargetLargeP4" : { "target": "project" diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP4/Program.cs b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP4/Program.cs index 9bfc3c72c..71a2f4d72 100644 --- a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP4/Program.cs +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP4/Program.cs @@ -1,11 +1,13 @@ using System; -namespace ConsoleApplication +namespace ConsoleLibrary.P4 { public class Program { public static void Main() { + ConsoleLibrary.P5.Program.Main(); + ConsoleLibrary.P6.Program.Main(); } } } diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP4/project.json b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP4/project.json index e0d6ab60c..ea7cad01d 100644 --- a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP4/project.json +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP4/project.json @@ -1,8 +1,5 @@ { "version": "1.0.0-*", - "compilationOptions": { - "emitEntryPoint": true - }, "dependencies": { "TwoTargetLargeP5" : { "target": "project" diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP5/Program.cs b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP5/Program.cs index 9bfc3c72c..9fc92636d 100644 --- a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP5/Program.cs +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP5/Program.cs @@ -1,6 +1,6 @@ using System; -namespace ConsoleApplication +namespace ConsoleLibrary.P5 { public class Program { diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP5/project.json b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP5/project.json index 5ea926028..a1f979cc9 100644 --- a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP5/project.json +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP5/project.json @@ -1,8 +1,5 @@ { "version": "1.0.0-*", - "compilationOptions": { - "emitEntryPoint": true - }, "dependencies": { "Microsoft.NETCore.App": { "type": "platform", diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP6/Program.cs b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP6/Program.cs index 9bfc3c72c..ae467ffd4 100644 --- a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP6/Program.cs +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP6/Program.cs @@ -1,6 +1,6 @@ using System; -namespace ConsoleApplication +namespace ConsoleLibrary.P6 { public class Program { diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP6/project.json b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP6/project.json index 5ea926028..a1f979cc9 100644 --- a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP6/project.json +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP6/project.json @@ -1,8 +1,5 @@ { "version": "1.0.0-*", - "compilationOptions": { - "emitEntryPoint": true - }, "dependencies": { "Microsoft.NETCore.App": { "type": "platform", diff --git a/scripts/dotnet-cli-build/TestTargets.cs b/scripts/dotnet-cli-build/TestTargets.cs index a8e4eb6e0..f1b39d736 100644 --- a/scripts/dotnet-cli-build/TestTargets.cs +++ b/scripts/dotnet-cli-build/TestTargets.cs @@ -36,7 +36,8 @@ namespace Microsoft.DotNet.Cli.Build "Microsoft.DotNet.Cli.Utils.Tests", "Microsoft.DotNet.Compiler.Common.Tests", "Microsoft.DotNet.ProjectModel.Tests", - "Microsoft.Extensions.DependencyModel.Tests" + "Microsoft.Extensions.DependencyModel.Tests", + "Performance" }; public static readonly string[] WindowsTestProjects = new[] diff --git a/src/dotnet/commands/dotnet-compile/ManagedCompiler.cs b/src/dotnet/commands/dotnet-compile/ManagedCompiler.cs index f32266e18..70a2d4f9d 100644 --- a/src/dotnet/commands/dotnet-compile/ManagedCompiler.cs +++ b/src/dotnet/commands/dotnet-compile/ManagedCompiler.cs @@ -8,7 +8,6 @@ using System.IO; using System.Linq; using System.Runtime.InteropServices; using System.Text; -using System.Threading; using Microsoft.DotNet.Cli; using Microsoft.DotNet.Cli.Compiler.Common; using Microsoft.DotNet.Cli.Utils; diff --git a/test/dotnet-performance.Tests/BuildPerformanceTest.cs b/test/Performance/BuildPerformanceTest.cs similarity index 66% rename from test/dotnet-performance.Tests/BuildPerformanceTest.cs rename to test/Performance/BuildPerformanceTest.cs index c8b140d48..6b7e28a6b 100644 --- a/test/dotnet-performance.Tests/BuildPerformanceTest.cs +++ b/test/Performance/BuildPerformanceTest.cs @@ -5,10 +5,14 @@ using System.Linq; using Microsoft.DotNet.Tools.Test.Utilities; using Microsoft.Xunit.Performance; using Microsoft.DotNet.TestFramework; +using System.IO; +using System.Runtime.CompilerServices; +using System; +using System.Collections.Generic; namespace Microsoft.DotNet.Tools.Builder.Tests { - public class BuildPerformanceTest : PerformanceTestBase + public class BuildPerformanceTest : TestBase { private static string SingleTargetApp = "SingleTargetApp"; private static string TwoTargetApp = "TwoTargetApp"; @@ -47,11 +51,12 @@ namespace Microsoft.DotNet.Tools.Builder.Tests { foreach (var iteration in Benchmark.Iterations) { + var buildCommand = new BuildCommand(instance.TestRoot, buildProfile: false); using (iteration.StartMeasurement()) { - Build(instance.TestRoot); + buildCommand.Execute().Should().Pass(); } - RemoveBin(instance.TestRoot); + TouchSource(instance.TestRoot); } } @@ -62,13 +67,15 @@ namespace Microsoft.DotNet.Tools.Builder.Tests public void IncrementalSkipSingleProject(TestInstance instance) { - Build(instance.TestRoot); + new BuildCommand(instance.TestRoot, buildProfile: false) + .Execute().Should().Pass(); foreach (var iteration in Benchmark.Iterations) { + var buildCommand = new BuildCommand(instance.TestRoot, buildProfile: false); using (iteration.StartMeasurement()) { - Build(instance.TestRoot); + buildCommand.Execute().Should().Pass(); } } } @@ -86,13 +93,14 @@ namespace Microsoft.DotNet.Tools.Builder.Tests foreach (var iteration in Benchmark.Iterations) { + var buildCommand = new BuildCommand(instance.TestRoot, buildProfile: false); using (iteration.StartMeasurement()) { - Build(instance.TestRoot); + buildCommand.Execute().Should().Pass(); } foreach (var i in instances) { - RemoveBin(i.TestRoot); + TouchSource(i.TestRoot); } } } @@ -112,14 +120,15 @@ namespace Microsoft.DotNet.Tools.Builder.Tests public void IncrementalSkipAllInGraph(TestInstance[] instances) { var instance = instances[0]; - - Build(instance.TestRoot); + new BuildCommand(instance.TestRoot, buildProfile: false) + .Execute().Should().Pass(); foreach (var iteration in Benchmark.Iterations) { + var buildCommand = new BuildCommand(instance.TestRoot, buildProfile: false); using (iteration.StartMeasurement()) { - Build(instance.TestRoot); + buildCommand.Execute().Should().Pass(); } } } @@ -139,16 +148,17 @@ namespace Microsoft.DotNet.Tools.Builder.Tests public void IncrementalRebuildWithRootChangedInGraph(TestInstance[] instances) { var instance = instances[0]; - - Build(instance.TestRoot); + new BuildCommand(instance.TestRoot, buildProfile: false) + .Execute().Should().Pass(); foreach (var iteration in Benchmark.Iterations) { + var buildCommand = new BuildCommand(instance.TestRoot, buildProfile: false); using (iteration.StartMeasurement()) { - Build(instance.TestRoot); + buildCommand.Execute().Should().Pass(); } - RemoveBin(instance.TestRoot); + TouchSource(instance.TestRoot); } } @@ -167,16 +177,17 @@ namespace Microsoft.DotNet.Tools.Builder.Tests public void IncrementalRebuildWithLastChangedInGraph(TestInstance[] instances) { var instance = instances[0]; - - Build(instance.TestRoot); + new BuildCommand(instance.TestRoot, buildProfile: false) + .Execute().Should().Pass(); foreach (var iteration in Benchmark.Iterations) { + var buildCommand = new BuildCommand(instance.TestRoot, buildProfile: false); using (iteration.StartMeasurement()) { - Build(instance.TestRoot); + buildCommand.Execute().Should().Pass(); } - RemoveBin(instances.Last().TestRoot); + TouchSource(instances.Last().TestRoot); } } @@ -196,25 +207,30 @@ namespace Microsoft.DotNet.Tools.Builder.Tests public void IncrementalSkipAllNoDependenciesInGraph(TestInstance[] instances) { var instance = instances[0]; - - Build(instance.TestRoot); + new BuildCommand(instance.TestRoot, buildProfile: false) + .Execute().Should().Pass(); foreach (var iteration in Benchmark.Iterations) { + var commands = new List(); + foreach (var i in instances.Reverse()) + { + commands.Add(new BuildCommand(i.TestRoot, + framework: DefaultFramework, + noDependencies: true, + buildProfile: false)); + } using (iteration.StartMeasurement()) { - foreach (var i in instances) + foreach (var buildCommand in commands) { - Run(new BuildCommand(i.TestRoot, - framework: DefaultFramework, - noDependencies: true, - buildProfile: false)); + buildCommand.Execute().Should().Pass(); } } } } [Benchmark] - public void BuildAllNoDependenciesInGraphh_SingleTargetGraph() => + public void BuildAllNoDependenciesInGraph_SingleTargetGraph() => BuildAllNoDependenciesInGraph(CreateTestInstances(SingleTargetGraph)); [Benchmark] @@ -229,21 +245,50 @@ namespace Microsoft.DotNet.Tools.Builder.Tests { foreach (var iteration in Benchmark.Iterations) { + var commands = new List(); + foreach (var i in instances.Reverse()) + { + commands.Add(new BuildCommand(i.TestRoot, + framework: DefaultFramework, + noDependencies: true, + buildProfile: false)); + } using (iteration.StartMeasurement()) { - foreach (var i in instances.Reverse()) + foreach (var buildCommand in commands) { - Run(new BuildCommand(i.TestRoot, - framework: DefaultFramework, - noDependencies: true, - buildProfile: false)); + buildCommand.Execute().Should().Pass(); } } foreach (var instance in instances) { - RemoveBin(instance.TestRoot); + TouchSource(instance.TestRoot); } } } + + protected void TouchSource(string project) + { + var sourceFile = Directory.GetFiles(project, "*.cs").FirstOrDefault(); + if (sourceFile == null) + { + throw new InvalidOperationException($"'.cs' files not found in {project}"); + } + File.SetLastWriteTime(sourceFile, DateTime.Now); + } + + protected TestInstance[] CreateTestInstances(string[] testProjectNames, [CallerMemberName] string callingMethod = "") + { + return testProjectNames.Select(testProjectName => + { + return CreateTestInstance(testProjectName, callingMethod); + }).ToArray(); + } + + protected TestInstance CreateTestInstance(string testProjectName, [CallerMemberName] string callingMethod = "") + { + return TestAssetsManager.CreateTestInstance(Path.Combine("PerformanceTestProjects", testProjectName), callingMethod) + .WithLockFiles(); + } } } diff --git a/test/dotnet-performance.Tests/PerformanceTestBase.cs b/test/dotnet-performance.Tests/PerformanceTestBase.cs deleted file mode 100644 index 8ae23e8d7..000000000 --- a/test/dotnet-performance.Tests/PerformanceTestBase.cs +++ /dev/null @@ -1,40 +0,0 @@ -using System.IO; -using System.Linq; -using System.Runtime.CompilerServices; -using Microsoft.DotNet.Tools.Test.Utilities; -using Microsoft.DotNet.TestFramework; - -namespace Microsoft.DotNet.Tools.Builder.Tests -{ - public class PerformanceTestBase : TestBase - { - protected void Build(string project) - { - Run(new BuildCommand(project, buildProfile: false)); - } - - protected void Run(TestCommand command) - { - command.Execute().Should().Pass(); - } - - protected void RemoveBin(string project) - { - Directory.Delete(Path.Combine(project, "bin"), true); - } - - protected TestInstance[] CreateTestInstances(string[] testProjectNames, [CallerMemberName] string callingMethod = "") - { - return testProjectNames.Select(testProjectName => - { - return CreateTestInstance(testProjectName, callingMethod); - }).ToArray(); - } - - protected TestInstance CreateTestInstance(string testProjectName, [CallerMemberName] string callingMethod = "") - { - return TestAssetsManager.CreateTestInstance(Path.Combine("PerformanceTestProjects", testProjectName), callingMethod) - .WithLockFiles(); - } - } -} \ No newline at end of file diff --git a/test/dotnet-performance.Tests/dotnet-performance.Tests.xproj b/test/dotnet-performance.Tests/dotnet-performance.Tests.xproj deleted file mode 100644 index 4a6c22bc9..000000000 --- a/test/dotnet-performance.Tests/dotnet-performance.Tests.xproj +++ /dev/null @@ -1,19 +0,0 @@ - - - - 14.0.25123 - $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion) - - - - 0ae3d045-c119-4ed7-bd0c-287029b7264a - dotnet-performance.Tests - ..\..\artifacts\obj\$(MSBuildProjectName) - ..\..\artifacts\ - - - - 2.0 - - - \ No newline at end of file diff --git a/test/dotnet-performance.Tests/project.json b/test/dotnet-performance.Tests/project.json deleted file mode 100644 index 87c1e90ae..000000000 --- a/test/dotnet-performance.Tests/project.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "version": "1.0.0-*", - "dependencies": { - "Microsoft.NETCore.App": "1.0.0-rc2-*", - "System.Runtime.Serialization.Primitives": "4.1.1-rc2-24022", - "Microsoft.DotNet.Tools.Tests.Utilities": { - "target": "project" - }, - "Microsoft.DotNet.Cli.Utils": { - "target": "project" - }, - "Newtonsoft.Json": "7.0.1", - "xunit": "2.1.0", - "dotnet-test-xunit": "1.0.0-dev-140469-38", - "Microsoft.DotNet.xunit.performance": "1.0.0-alpha-build0028" - }, - "frameworks": { - "netcoreapp1.0": { - "imports": [ - "dotnet5.4", - "portable-net451+win8" - ] - } - }, - "testRunner": "xunit", - "runtimes": - { - "win7-x64": {} - } -}