From 366bad99ac31c968983ea80f8a6845a63d64e228 Mon Sep 17 00:00:00 2001 From: Pavel Krymets Date: Fri, 22 Apr 2016 17:06:55 -0700 Subject: [PATCH] Add performance tests --- .../SingleTargetApp/Program.cs | 11 ++ .../SingleTargetApp/project.json | 15 ++ .../SingleTargetP0/Program.cs | 11 ++ .../SingleTargetP0/project.json | 18 ++ .../SingleTargetP1/Program.cs | 11 ++ .../SingleTargetP1/project.json | 18 ++ .../SingleTargetP2/Program.cs | 11 ++ .../SingleTargetP2/project.json | 15 ++ .../TwoTargetApp/Program.cs | 11 ++ .../TwoTargetApp/project.json | 16 ++ .../TwoTargetGraph/TwoTargetP0/Program.cs | 11 ++ .../TwoTargetGraph/TwoTargetP0/project.json | 19 ++ .../TwoTargetGraph/TwoTargetP1/Program.cs | 11 ++ .../TwoTargetGraph/TwoTargetP1/project.json | 19 ++ .../TwoTargetGraph/TwoTargetP2/Program.cs | 11 ++ .../TwoTargetGraph/TwoTargetP2/project.json | 16 ++ .../TwoTargetLargeP0/Program.cs | 11 ++ .../TwoTargetLargeP0/project.json | 19 ++ .../TwoTargetLargeP1/Program.cs | 11 ++ .../TwoTargetLargeP1/project.json | 22 +++ .../TwoTargetLargeP2/Program.cs | 11 ++ .../TwoTargetLargeP2/project.json | 22 +++ .../TwoTargetLargeP3/Program.cs | 11 ++ .../TwoTargetLargeP3/project.json | 19 ++ .../TwoTargetLargeP4/Program.cs | 11 ++ .../TwoTargetLargeP4/project.json | 22 +++ .../TwoTargetLargeP5/Program.cs | 11 ++ .../TwoTargetLargeP5/project.json | 16 ++ .../TwoTargetLargeP6/Program.cs | 11 ++ .../TwoTargetLargeP6/project.json | 16 ++ .../dotnet-compile/ManagedCompiler.cs | 1 + .../Commands/BuildCommand.cs | 19 +- .../BuildPerformanceTest.cs | 182 ++++++++++++++++++ .../PerformanceTestBase.cs | 63 ++++++ test/dotnet-performance.Tests/RunMultiCli.ps1 | 38 ++++ .../dotnet-performance.Tests.xproj | 19 ++ test/dotnet-performance.Tests/project.json | 29 +++ 37 files changed, 785 insertions(+), 3 deletions(-) create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/SingleTargetApp/Program.cs create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/SingleTargetApp/project.json create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP0/Program.cs create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP0/project.json create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP1/Program.cs create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP1/project.json create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP2/Program.cs create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP2/project.json create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/TwoTargetApp/Program.cs create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/TwoTargetApp/project.json create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP0/Program.cs create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP0/project.json create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP1/Program.cs create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP1/project.json create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP2/Program.cs create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP2/project.json create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP0/Program.cs create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP0/project.json create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP1/Program.cs create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP1/project.json create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP2/Program.cs create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP2/project.json create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP3/Program.cs create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP3/project.json create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP4/Program.cs create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP4/project.json create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP5/Program.cs create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP5/project.json create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP6/Program.cs create mode 100644 TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP6/project.json create mode 100644 test/dotnet-performance.Tests/BuildPerformanceTest.cs create mode 100644 test/dotnet-performance.Tests/PerformanceTestBase.cs create mode 100644 test/dotnet-performance.Tests/RunMultiCli.ps1 create mode 100644 test/dotnet-performance.Tests/dotnet-performance.Tests.xproj create mode 100644 test/dotnet-performance.Tests/project.json diff --git a/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetApp/Program.cs b/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetApp/Program.cs new file mode 100644 index 000000000..9bfc3c72c --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetApp/Program.cs @@ -0,0 +1,11 @@ +using System; + +namespace ConsoleApplication +{ + public class Program + { + public static void Main() + { + } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetApp/project.json b/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetApp/project.json new file mode 100644 index 000000000..89cf3a3bf --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetApp/project.json @@ -0,0 +1,15 @@ +{ + "version": "1.0.0-*", + "compilationOptions": { + "emitEntryPoint": true + }, + "dependencies": { + "Microsoft.NETCore.App": { + "type": "platform", + "version": "1.0.0-rc2-*" + }, + }, + "frameworks": { + "netcoreapp1.0": { } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP0/Program.cs b/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP0/Program.cs new file mode 100644 index 000000000..9bfc3c72c --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP0/Program.cs @@ -0,0 +1,11 @@ +using System; + +namespace ConsoleApplication +{ + public class Program + { + public static void Main() + { + } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP0/project.json b/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP0/project.json new file mode 100644 index 000000000..c65037aca --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP0/project.json @@ -0,0 +1,18 @@ +{ + "version": "1.0.0-*", + "compilationOptions": { + "emitEntryPoint": true + }, + "dependencies": { + "SingleTargetP1" : { + "target": "project" + }, + "Microsoft.NETCore.App": { + "type": "platform", + "version": "1.0.0-rc2-*" + }, + }, + "frameworks": { + "netcoreapp1.0": { } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP1/Program.cs b/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP1/Program.cs new file mode 100644 index 000000000..9bfc3c72c --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP1/Program.cs @@ -0,0 +1,11 @@ +using System; + +namespace ConsoleApplication +{ + public class Program + { + public static void Main() + { + } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP1/project.json b/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP1/project.json new file mode 100644 index 000000000..a6c2ff8b5 --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP1/project.json @@ -0,0 +1,18 @@ +{ + "version": "1.0.0-*", + "compilationOptions": { + "emitEntryPoint": true + }, + "dependencies": { + "SingleTargetP2" : { + "target": "project" + }, + "Microsoft.NETCore.App": { + "type": "platform", + "version": "1.0.0-rc2-*" + }, + }, + "frameworks": { + "netcoreapp1.0": { } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP2/Program.cs b/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP2/Program.cs new file mode 100644 index 000000000..9bfc3c72c --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP2/Program.cs @@ -0,0 +1,11 @@ +using System; + +namespace ConsoleApplication +{ + public class Program + { + public static void Main() + { + } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP2/project.json b/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP2/project.json new file mode 100644 index 000000000..89cf3a3bf --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/SingleTargetGraph/SingleTargetP2/project.json @@ -0,0 +1,15 @@ +{ + "version": "1.0.0-*", + "compilationOptions": { + "emitEntryPoint": true + }, + "dependencies": { + "Microsoft.NETCore.App": { + "type": "platform", + "version": "1.0.0-rc2-*" + }, + }, + "frameworks": { + "netcoreapp1.0": { } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetApp/Program.cs b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetApp/Program.cs new file mode 100644 index 000000000..9bfc3c72c --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetApp/Program.cs @@ -0,0 +1,11 @@ +using System; + +namespace ConsoleApplication +{ + public class Program + { + public static void Main() + { + } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetApp/project.json b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetApp/project.json new file mode 100644 index 000000000..5ea926028 --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetApp/project.json @@ -0,0 +1,16 @@ +{ + "version": "1.0.0-*", + "compilationOptions": { + "emitEntryPoint": true + }, + "dependencies": { + "Microsoft.NETCore.App": { + "type": "platform", + "version": "1.0.0-rc2-*" + }, + }, + "frameworks": { + "netcoreapp1.0": { }, + "netstandard1.5": { } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP0/Program.cs b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP0/Program.cs new file mode 100644 index 000000000..9bfc3c72c --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP0/Program.cs @@ -0,0 +1,11 @@ +using System; + +namespace ConsoleApplication +{ + public class Program + { + public static void Main() + { + } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP0/project.json b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP0/project.json new file mode 100644 index 000000000..2b03279e5 --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP0/project.json @@ -0,0 +1,19 @@ +{ + "version": "1.0.0-*", + "compilationOptions": { + "emitEntryPoint": true + }, + "dependencies": { + "TwoTargetP1" : { + "target": "project" + }, + "Microsoft.NETCore.App": { + "type": "platform", + "version": "1.0.0-rc2-*" + }, + }, + "frameworks": { + "netcoreapp1.0": { }, + "netstandard1.5": { } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP1/Program.cs b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP1/Program.cs new file mode 100644 index 000000000..9bfc3c72c --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP1/Program.cs @@ -0,0 +1,11 @@ +using System; + +namespace ConsoleApplication +{ + public class Program + { + public static void Main() + { + } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP1/project.json b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP1/project.json new file mode 100644 index 000000000..b9fbb9f8d --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP1/project.json @@ -0,0 +1,19 @@ +{ + "version": "1.0.0-*", + "compilationOptions": { + "emitEntryPoint": true + }, + "dependencies": { + "TwoTargetP2" : { + "target": "project" + }, + "Microsoft.NETCore.App": { + "type": "platform", + "version": "1.0.0-rc2-*" + }, + }, + "frameworks": { + "netcoreapp1.0": { }, + "netstandard1.5": { } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP2/Program.cs b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP2/Program.cs new file mode 100644 index 000000000..9bfc3c72c --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP2/Program.cs @@ -0,0 +1,11 @@ +using System; + +namespace ConsoleApplication +{ + public class Program + { + public static void Main() + { + } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP2/project.json b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP2/project.json new file mode 100644 index 000000000..5ea926028 --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraph/TwoTargetP2/project.json @@ -0,0 +1,16 @@ +{ + "version": "1.0.0-*", + "compilationOptions": { + "emitEntryPoint": true + }, + "dependencies": { + "Microsoft.NETCore.App": { + "type": "platform", + "version": "1.0.0-rc2-*" + }, + }, + "frameworks": { + "netcoreapp1.0": { }, + "netstandard1.5": { } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP0/Program.cs b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP0/Program.cs new file mode 100644 index 000000000..9bfc3c72c --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP0/Program.cs @@ -0,0 +1,11 @@ +using System; + +namespace ConsoleApplication +{ + public class Program + { + public static void Main() + { + } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP0/project.json b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP0/project.json new file mode 100644 index 000000000..22689acf8 --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP0/project.json @@ -0,0 +1,19 @@ +{ + "version": "1.0.0-*", + "compilationOptions": { + "emitEntryPoint": true + }, + "dependencies": { + "TwoTargetLargeP1" : { + "target": "project" + }, + "Microsoft.NETCore.App": { + "type": "platform", + "version": "1.0.0-rc2-*" + }, + }, + "frameworks": { + "netcoreapp1.0": { }, + "netstandard1.5": { } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP1/Program.cs b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP1/Program.cs new file mode 100644 index 000000000..9bfc3c72c --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP1/Program.cs @@ -0,0 +1,11 @@ +using System; + +namespace ConsoleApplication +{ + public class Program + { + public static void Main() + { + } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP1/project.json b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP1/project.json new file mode 100644 index 000000000..2e36f00eb --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP1/project.json @@ -0,0 +1,22 @@ +{ + "version": "1.0.0-*", + "compilationOptions": { + "emitEntryPoint": true + }, + "dependencies": { + "TwoTargetLargeP2" : { + "target": "project" + }, + "TwoTargetLargeP4" : { + "target": "project" + }, + "Microsoft.NETCore.App": { + "type": "platform", + "version": "1.0.0-rc2-*" + }, + }, + "frameworks": { + "netcoreapp1.0": { }, + "netstandard1.5": { } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP2/Program.cs b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP2/Program.cs new file mode 100644 index 000000000..9bfc3c72c --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP2/Program.cs @@ -0,0 +1,11 @@ +using System; + +namespace ConsoleApplication +{ + public class Program + { + public static void Main() + { + } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP2/project.json b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP2/project.json new file mode 100644 index 000000000..a6fdc9d7d --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP2/project.json @@ -0,0 +1,22 @@ +{ + "version": "1.0.0-*", + "compilationOptions": { + "emitEntryPoint": true + }, + "dependencies": { + "TwoTargetLargeP3" : { + "target": "project" + }, + "TwoTargetLargeP4" : { + "target": "project" + }, + "Microsoft.NETCore.App": { + "type": "platform", + "version": "1.0.0-rc2-*" + }, + }, + "frameworks": { + "netcoreapp1.0": { }, + "netstandard1.5": { } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP3/Program.cs b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP3/Program.cs new file mode 100644 index 000000000..9bfc3c72c --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP3/Program.cs @@ -0,0 +1,11 @@ +using System; + +namespace ConsoleApplication +{ + public class Program + { + public static void Main() + { + } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP3/project.json b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP3/project.json new file mode 100644 index 000000000..821a50a5f --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP3/project.json @@ -0,0 +1,19 @@ +{ + "version": "1.0.0-*", + "compilationOptions": { + "emitEntryPoint": true + }, + "dependencies": { + "TwoTargetLargeP4" : { + "target": "project" + }, + "Microsoft.NETCore.App": { + "type": "platform", + "version": "1.0.0-rc2-*" + }, + }, + "frameworks": { + "netcoreapp1.0": { }, + "netstandard1.5": { } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP4/Program.cs b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP4/Program.cs new file mode 100644 index 000000000..9bfc3c72c --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP4/Program.cs @@ -0,0 +1,11 @@ +using System; + +namespace ConsoleApplication +{ + public class Program + { + public static void Main() + { + } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP4/project.json b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP4/project.json new file mode 100644 index 000000000..e0d6ab60c --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP4/project.json @@ -0,0 +1,22 @@ +{ + "version": "1.0.0-*", + "compilationOptions": { + "emitEntryPoint": true + }, + "dependencies": { + "TwoTargetLargeP5" : { + "target": "project" + }, + "TwoTargetLargeP6" : { + "target": "project" + }, + "Microsoft.NETCore.App": { + "type": "platform", + "version": "1.0.0-rc2-*" + }, + }, + "frameworks": { + "netcoreapp1.0": { }, + "netstandard1.5": { } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP5/Program.cs b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP5/Program.cs new file mode 100644 index 000000000..9bfc3c72c --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP5/Program.cs @@ -0,0 +1,11 @@ +using System; + +namespace ConsoleApplication +{ + public class Program + { + public static void Main() + { + } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP5/project.json b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP5/project.json new file mode 100644 index 000000000..5ea926028 --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP5/project.json @@ -0,0 +1,16 @@ +{ + "version": "1.0.0-*", + "compilationOptions": { + "emitEntryPoint": true + }, + "dependencies": { + "Microsoft.NETCore.App": { + "type": "platform", + "version": "1.0.0-rc2-*" + }, + }, + "frameworks": { + "netcoreapp1.0": { }, + "netstandard1.5": { } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP6/Program.cs b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP6/Program.cs new file mode 100644 index 000000000..9bfc3c72c --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP6/Program.cs @@ -0,0 +1,11 @@ +using System; + +namespace ConsoleApplication +{ + public class Program + { + public static void Main() + { + } + } +} diff --git a/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP6/project.json b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP6/project.json new file mode 100644 index 000000000..5ea926028 --- /dev/null +++ b/TestAssets/TestProjects/PerformanceTestProjects/TwoTargetGraphLarge/TwoTargetLargeP6/project.json @@ -0,0 +1,16 @@ +{ + "version": "1.0.0-*", + "compilationOptions": { + "emitEntryPoint": true + }, + "dependencies": { + "Microsoft.NETCore.App": { + "type": "platform", + "version": "1.0.0-rc2-*" + }, + }, + "frameworks": { + "netcoreapp1.0": { }, + "netstandard1.5": { } + } +} diff --git a/src/dotnet/commands/dotnet-compile/ManagedCompiler.cs b/src/dotnet/commands/dotnet-compile/ManagedCompiler.cs index 70a2d4f9d..f32266e18 100644 --- a/src/dotnet/commands/dotnet-compile/ManagedCompiler.cs +++ b/src/dotnet/commands/dotnet-compile/ManagedCompiler.cs @@ -8,6 +8,7 @@ using System.IO; using System.Linq; using System.Runtime.InteropServices; using System.Text; +using System.Threading; using Microsoft.DotNet.Cli; using Microsoft.DotNet.Cli.Compiler.Common; using Microsoft.DotNet.Cli.Utils; diff --git a/test/Microsoft.DotNet.Tools.Tests.Utilities/Commands/BuildCommand.cs b/test/Microsoft.DotNet.Tools.Tests.Utilities/Commands/BuildCommand.cs index b427bd0de..0ebd04b90 100644 --- a/test/Microsoft.DotNet.Tools.Tests.Utilities/Commands/BuildCommand.cs +++ b/test/Microsoft.DotNet.Tools.Tests.Utilities/Commands/BuildCommand.cs @@ -29,6 +29,7 @@ namespace Microsoft.DotNet.Tools.Test.Utilities private readonly bool _noIncremental; private readonly bool _noDependencies; private readonly string _runtime; + private readonly bool _verbose; private string OutputOption { @@ -199,6 +200,16 @@ namespace Microsoft.DotNet.Tools.Test.Utilities } } + private string Verbose + { + get + { + return _verbose ? + "--verbose" : + ""; + } + } + public BuildCommand( string projectPath, string output="", @@ -217,7 +228,8 @@ namespace Microsoft.DotNet.Tools.Test.Utilities string cppCompilerFlags="", bool buildProfile=true, bool noIncremental=false, - bool noDependencies=false) + bool noDependencies=false, + bool verbose=true) : base("dotnet") { _projectPath = projectPath; @@ -240,17 +252,18 @@ namespace Microsoft.DotNet.Tools.Test.Utilities _buildProfile = buildProfile; _noIncremental = noIncremental; _noDependencies = noDependencies; + _verbose = verbose; } public override CommandResult Execute(string args = "") { - args = $"--verbose build {BuildArgs()} {args}"; + args = $"{Verbose} build {BuildArgs()} {args}"; return base.Execute(args); } public override CommandResult ExecuteWithCapturedOutput(string args = "") { - args = $"--verbose build {BuildArgs()} {args}"; + args = $"{Verbose} build {BuildArgs()} {args}"; return base.ExecuteWithCapturedOutput(args); } diff --git a/test/dotnet-performance.Tests/BuildPerformanceTest.cs b/test/dotnet-performance.Tests/BuildPerformanceTest.cs new file mode 100644 index 000000000..3953e9916 --- /dev/null +++ b/test/dotnet-performance.Tests/BuildPerformanceTest.cs @@ -0,0 +1,182 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Collections.Generic; +using System.Linq; +using Microsoft.DotNet.Tools.Test.Utilities; +using Xunit; + +namespace Microsoft.DotNet.Tools.Builder.Tests +{ + public class BuildPerformanceTest : PerformanceTestBase + { + public static IEnumerable SingleProjects + { + get + { + yield return new [] { "TwoTargetApp"}; + yield return new [] { "SingleTargetApp" }; + } + } + + public static IEnumerable GraphProjects + { + get + { + yield return new object[] + { + "TwoTargetGraph", + new[] { "TwoTargetGraph/TwoTargetP0", "TwoTargetGraph/TwoTargetP1", "TwoTargetGraph/TwoTargetP2" } + }; + yield return new object[] + { + "TwoTargetGraphLarge", + new[] + { + "TwoTargetGraphLarge/TwoTargetLargeP0", + "TwoTargetGraphLarge/TwoTargetLargeP1", + "TwoTargetGraphLarge/TwoTargetLargeP2", + "TwoTargetGraphLarge/TwoTargetLargeP3", + "TwoTargetGraphLarge/TwoTargetLargeP4", + "TwoTargetGraphLarge/TwoTargetLargeP5", + "TwoTargetGraphLarge/TwoTargetLargeP6" + } + }; + yield return new object[] + { + "SingleTargetGraph", + new[] { "SingleTargetGraph/SingleTargetP0", "SingleTargetGraph/SingleTargetP1", "SingleTargetGraph/SingleTargetP2" } + }; + } + } + + [Theory] + [MemberData(nameof(SingleProjects))] + public void BuildSingleProject(string project) + { + var instance = CreateTestInstance(project); + + Iterate(c => + { + c.Measure(() => Build(instance.TestRoot)); + RemoveBin(instance.TestRoot); + }, project); + } + + [Theory] + [MemberData(nameof(SingleProjects))] + public void IncrementalSkipSingleProject(string project) + { + var instance = CreateTestInstance(project); + Build(instance.TestRoot); + + Iterate(c => + { + c.Measure(() => Build(instance.TestRoot)); + }, project); + } + + [Theory] + [MemberData(nameof(GraphProjects))] + public void BuildAllInGraph(string variation, string[] projects) + { + var instances = projects.Select(p => CreateTestInstance(p, variation)).ToArray(); + var instance = instances[0]; + + Iterate(c => + { + c.Measure(() => Build(instance.TestRoot)); + foreach (var i in instances) + { + RemoveBin(i.TestRoot); + } + }, variation); + } + + [Theory] + [MemberData(nameof(GraphProjects))] + public void IncrementalSkipAllInGraph(string variation, string[] projects) + { + var instances = projects.Select(p => CreateTestInstance(p, variation)).ToArray(); + var instance = instances[0]; + + Build(instance.TestRoot); + + Iterate(c => + { + c.Measure(() => Build(instance.TestRoot)); + }, variation); + } + + [Theory] + [MemberData(nameof(GraphProjects))] + public void IncrementalRebuildWithRootChangedInGraph(string variation, string[] projects) + { + var instances = projects.Select(p => CreateTestInstance(p, variation)).ToArray(); + var instance = instances[0]; + + Build(instance.TestRoot); + + Iterate(c => + { + c.Measure(() => Build(instance.TestRoot)); + RemoveBin(instance.TestRoot); + }, variation); + } + + [Theory] + [MemberData(nameof(GraphProjects))] + public void IncrementalRebuildWithLastChangedInGraph(string variation, string[] projects) + { + var instances = projects.Select(p => CreateTestInstance(p, variation)).ToArray(); + var instance = instances[0]; + + Build(instance.TestRoot); + + Iterate(c => + { + c.Measure(() => Build(instance.TestRoot)); + RemoveBin(instances.Last().TestRoot); + }, variation); + } + + + [Theory] + [MemberData(nameof(GraphProjects))] + public void IncrementalSkipAllNoDependenciesInGraph(string variation, string[] projects) + { + var instances = projects.Select(p => CreateTestInstance(p, variation)).ToArray(); + var instance = instances[0]; + + Build(instance.TestRoot); + + Iterate(c => + { + foreach (var i in instances) + { + c.Measure(() => Run(new BuildCommand(i.TestRoot, framework: DefaultFramework, noDependencies: true, buildProfile: false))); + } + }, variation); + } + + [Theory] + [MemberData(nameof(GraphProjects))] + public void BuildAllNoDependenciesInGraph(string variation, string[] projects) + { + var instances = projects.Select(p => CreateTestInstance(p, variation)).ToArray(); + + Iterate(c => + { + foreach (var i in instances.Reverse()) + { + c.Measure(() => Run(new BuildCommand(i.TestRoot, framework: DefaultFramework, noDependencies: true, buildProfile: false))); + } + foreach (var instance in instances) + { + RemoveBin(instance.TestRoot); + } + + }, variation); + } + } +} diff --git a/test/dotnet-performance.Tests/PerformanceTestBase.cs b/test/dotnet-performance.Tests/PerformanceTestBase.cs new file mode 100644 index 000000000..e35606328 --- /dev/null +++ b/test/dotnet-performance.Tests/PerformanceTestBase.cs @@ -0,0 +1,63 @@ +using System; +using System.Diagnostics; +using System.IO; +using System.Runtime.CompilerServices; +using Microsoft.DotNet.Tools.Test.Utilities; +using Microsoft.DotNet.TestFramework; +using Microsoft.DotNet.Cli.Utils; + +namespace Microsoft.DotNet.Tools.Builder.Tests +{ + public class PerformanceTestBase : TestBase + { + protected void Build(string project) + { + Run(new BuildCommand(project, buildProfile: false)); + } + + protected void Run(TestCommand command) + { + command.Execute().Should().Pass(); + } + + protected void RemoveBin(string project) + { + Directory.Delete(Path.Combine(project, "bin"), true); + } + + protected TestInstance CreateTestInstance(string testProjectName, string variation = "", [CallerMemberName] string callingMethod = "") + { + return TestAssetsManager.CreateTestInstance(Path.Combine("PerformanceTestProjects", testProjectName), callingMethod + variation) + .WithLockFiles(); + } + + public void Iterate(Action action, string variation = "", int iterations = 3, [CallerMemberName] string callingMethod = "") + { + var fullname = callingMethod + (variation != "" ? "" + variation : ""); + // Heat up iteration + var context = new PerformanceIterationContext(); + action(context); + + TimeSpan totalTime; + for (int i = 0; i < iterations; i++) + { + context = new PerformanceIterationContext(); + action(context); + totalTime += context.Stopwatch.Elapsed; + } + Reporter.Output.WriteLine($"[RESULT] {callingMethod}-{variation} {totalTime.TotalSeconds/iterations:F} sec/iteration".Bold()); + } + + public class PerformanceIterationContext + { + public Stopwatch Stopwatch { get; } = new Stopwatch(); + + public void Measure(Action action) + { + Stopwatch.Start(); + action(); + Stopwatch.Stop(); + } + } + } +} \ No newline at end of file diff --git a/test/dotnet-performance.Tests/RunMultiCli.ps1 b/test/dotnet-performance.Tests/RunMultiCli.ps1 new file mode 100644 index 000000000..2caa37e25 --- /dev/null +++ b/test/dotnet-performance.Tests/RunMultiCli.ps1 @@ -0,0 +1,38 @@ +param($versions,$file) + +function cleanpath() +{ + $env:PATH=($env:PATH.Split(";") | Where { !$_.Contains("dotnet") }) -Join ";" +} +foreach($ver in $versions) +{ + cleanpath; + if ($ver -ne "dev") + { + Write-Host -ForegroundColor Green "Installing $ver" + $dotnetPath = "$PSScriptRoot\.dotnet\$ver" + if (!(test-path $dotnetPath)) + { + & ($PSScriptRoot+"\..\..\scripts\obtain\install.ps1") preview $ver $dotnetPath; + } + $env:PATH = "$dotnetPath;"+$env:PATH + } + else + { + Write-Host -ForegroundColor Green "Using dev" + & ($PSScriptRoot+"\..\..\scripts\use-dev.ps1"); + } + cmd /c "where dotnet" | Write-Host -ForegroundColor Green + dotnet --version | Write-Host -ForegroundColor Green + + dotnet test ` + | Where {$_.startswith("[RESULT]") } ` + | %{ + $_ | Write-Host -ForegroundColor Blue + if ($file) + { + $csvLine = $ver + $_.Replace(" ", ",").Replace("[RESULT]","").Replace("sec/iteration", ""); + Add-Content $file $csvLine + } + } +} \ No newline at end of file diff --git a/test/dotnet-performance.Tests/dotnet-performance.Tests.xproj b/test/dotnet-performance.Tests/dotnet-performance.Tests.xproj new file mode 100644 index 000000000..4a6c22bc9 --- /dev/null +++ b/test/dotnet-performance.Tests/dotnet-performance.Tests.xproj @@ -0,0 +1,19 @@ + + + + 14.0.25123 + $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion) + + + + 0ae3d045-c119-4ed7-bd0c-287029b7264a + dotnet-performance.Tests + ..\..\artifacts\obj\$(MSBuildProjectName) + ..\..\artifacts\ + + + + 2.0 + + + \ No newline at end of file diff --git a/test/dotnet-performance.Tests/project.json b/test/dotnet-performance.Tests/project.json new file mode 100644 index 000000000..ff0e453f4 --- /dev/null +++ b/test/dotnet-performance.Tests/project.json @@ -0,0 +1,29 @@ +{ + "version": "1.0.0-*", + "dependencies": { + "Microsoft.NETCore.App": "1.0.0-rc2-*", + "System.Runtime.Serialization.Primitives": "4.1.1-rc2-24022", + "Microsoft.DotNet.Tools.Tests.Utilities": { + "target": "project" + }, + "Microsoft.DotNet.Cli.Utils": { + "target": "project" + }, + "Newtonsoft.Json": "7.0.1", + "xunit": "2.1.0", + "dotnet-test-xunit": "1.0.0-dev-140469-38" + }, + "frameworks": { + "netcoreapp1.0": { + "imports": [ + "dotnet5.4", + "portable-net451+win8" + ] + } + }, + "testRunner": "xunit", + "runtimes": + { + "win7-x64": {} + } +}