Use performance test

test/dotnet-performance.Tests/BuildPerformanceTest.cs

@@ -1,182 +1,249 @@
 // Copyright (c) .NET Foundation and contributors. All rights reserved.
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
-using System.Collections.Generic;
 using System.Linq;
 using Microsoft.DotNet.Tools.Test.Utilities;
-using Xunit;
+using Microsoft.Xunit.Performance;
+using Microsoft.DotNet.TestFramework;
 
 namespace Microsoft.DotNet.Tools.Builder.Tests
 {
     public class BuildPerformanceTest : PerformanceTestBase
     {
-        public static IEnumerable<object> SingleProjects
+        private static string SingleTargetApp = "SingleTargetApp";
-        {
+        private static string TwoTargetApp = "TwoTargetApp";
-            get
-            {
-                yield return new [] { "TwoTargetApp"};
-                yield return new [] { "SingleTargetApp" };
-            }
-        }
 
-        public static IEnumerable<object> GraphProjects
+        private static string[] SingleTargetGraph = new[]
         {
-            get
+            "SingleTargetGraph/SingleTargetP0",
-            {
+            "SingleTargetGraph/SingleTargetP1",
-                yield return new object[]
+            "SingleTargetGraph/SingleTargetP2"
-                {
+        };
-                    "TwoTargetGraph",
-                    new[] { "TwoTargetGraph/TwoTargetP0", "TwoTargetGraph/TwoTargetP1", "TwoTargetGraph/TwoTargetP2" }
-                };
-                yield return new object[]
-                {
-                    "TwoTargetGraphLarge",
-                    new[]
-                    {
-                        "TwoTargetGraphLarge/TwoTargetLargeP0",
-                        "TwoTargetGraphLarge/TwoTargetLargeP1",
-                        "TwoTargetGraphLarge/TwoTargetLargeP2",
-                        "TwoTargetGraphLarge/TwoTargetLargeP3",
-                        "TwoTargetGraphLarge/TwoTargetLargeP4",
-                        "TwoTargetGraphLarge/TwoTargetLargeP5",
-                        "TwoTargetGraphLarge/TwoTargetLargeP6"
-                    }
-                };
-                yield return new object[]
-                {
-                    "SingleTargetGraph",
-                    new[] { "SingleTargetGraph/SingleTargetP0", "SingleTargetGraph/SingleTargetP1", "SingleTargetGraph/SingleTargetP2" }
-                };
-            }
-        }
 
-        [Theory]
+        private static string[] TwoTargetGraph = new[]
-        [MemberData(nameof(SingleProjects))]
-        public void BuildSingleProject(string project)
         {
-            var instance = CreateTestInstance(project);
+            "TwoTargetGraph/TwoTargetP0",
+            "TwoTargetGraph/TwoTargetP1",
+            "TwoTargetGraph/TwoTargetP2"
+        };
 
-            Iterate(c =>
+        private static string[] TwoTargetGraphLarge = new[]
+        {
+            "TwoTargetGraphLarge/TwoTargetLargeP0",
+            "TwoTargetGraphLarge/TwoTargetLargeP1",
+            "TwoTargetGraphLarge/TwoTargetLargeP2",
+            "TwoTargetGraphLarge/TwoTargetLargeP3",
+            "TwoTargetGraphLarge/TwoTargetLargeP4",
+            "TwoTargetGraphLarge/TwoTargetLargeP5",
+            "TwoTargetGraphLarge/TwoTargetLargeP6"
+        };
+
+        [Benchmark]
+        public void BuildSingleProject_SingleTargetApp() => BuildSingleProject(CreateTestInstance(SingleTargetApp));
+        [Benchmark]
+        public void BuildSingleProject_TwoTargetApp() => BuildSingleProject(CreateTestInstance(TwoTargetApp));
+
+        public void BuildSingleProject(TestInstance instance)
+        {
+            foreach (var iteration in Benchmark.Iterations)
             {
-                c.Measure(() => Build(instance.TestRoot));
+                using (iteration.StartMeasurement())
+                {
+                    Build(instance.TestRoot);
+                }
                 RemoveBin(instance.TestRoot);
-            }, project);
+            }
         }
 
-        [Theory]
+        [Benchmark]
-        [MemberData(nameof(SingleProjects))]
+        public void IncrementalSkipSingleProject_SingleTargetApp() => IncrementalSkipSingleProject(CreateTestInstance(SingleTargetApp));
-        public void IncrementalSkipSingleProject(string project)
+        [Benchmark]
+        public void IncrementalSkipSingleProject_TwoTargetApp() => IncrementalSkipSingleProject(CreateTestInstance(TwoTargetApp));
+
+        public void IncrementalSkipSingleProject(TestInstance instance)
         {
-            var instance = CreateTestInstance(project);
             Build(instance.TestRoot);
 
-            Iterate(c =>
+            foreach (var iteration in Benchmark.Iterations)
             {
-                c.Measure(() => Build(instance.TestRoot));
+                using (iteration.StartMeasurement())
-            }, project);
+                {
+                    Build(instance.TestRoot);
+                }
+            }
         }
 
-        [Theory]
+        [Benchmark]
-        [MemberData(nameof(GraphProjects))]
+        public void BuildAllInGraph_SingleTargetGraph() => BuildAllInGraph(CreateTestInstances(SingleTargetGraph));
-        public void BuildAllInGraph(string variation, string[] projects)
+        [Benchmark]
+        public void BuildAllInGraph_TwoTargetGraph() => BuildAllInGraph(CreateTestInstances(TwoTargetGraph));
+        [Benchmark]
+        public void BuildAllInGraph_TwoTargetGraphLarge() => BuildAllInGraph(CreateTestInstances(TwoTargetGraphLarge));
+
+        public void BuildAllInGraph(TestInstance[] instances)
         {
-            var instances = projects.Select(p => CreateTestInstance(p, variation)).ToArray();
             var instance = instances[0];
 
-            Iterate(c =>
+            foreach (var iteration in Benchmark.Iterations)
             {
-                c.Measure(() => Build(instance.TestRoot));
+                using (iteration.StartMeasurement())
+                {
+                    Build(instance.TestRoot);
+                }
                 foreach (var i in instances)
                 {
                     RemoveBin(i.TestRoot);
                 }
-            }, variation);
+            }
         }
 
-        [Theory]
+        [Benchmark]
-        [MemberData(nameof(GraphProjects))]
+        public void IncrementalSkipAllInGraph_SingleTargetGraph() =>
-        public void IncrementalSkipAllInGraph(string variation, string[] projects)
+            IncrementalSkipAllInGraph(CreateTestInstances(SingleTargetGraph));
+
+        [Benchmark]
+        public void IncrementalSkipAllInGraph_TwoTargetGraph() =>
+            IncrementalSkipAllInGraph(CreateTestInstances(TwoTargetGraph));
+
+        [Benchmark]
+        public void IncrementalSkipAllInGraphh_TwoTargetGraphLarge() =>
+            IncrementalSkipAllInGraph(CreateTestInstances(TwoTargetGraphLarge));
+
+        public void IncrementalSkipAllInGraph(TestInstance[] instances)
         {
-            var instances = projects.Select(p => CreateTestInstance(p, variation)).ToArray();
             var instance = instances[0];
 
             Build(instance.TestRoot);
 
-            Iterate(c =>
+            foreach (var iteration in Benchmark.Iterations)
             {
-                c.Measure(() => Build(instance.TestRoot));
+                using (iteration.StartMeasurement())
-            }, variation);
-        }
-
-        [Theory]
-        [MemberData(nameof(GraphProjects))]
-        public void IncrementalRebuildWithRootChangedInGraph(string variation, string[] projects)
-        {
-            var instances = projects.Select(p => CreateTestInstance(p, variation)).ToArray();
-            var instance = instances[0];
-
-            Build(instance.TestRoot);
-
-            Iterate(c =>
-            {
-                c.Measure(() => Build(instance.TestRoot));
-                RemoveBin(instance.TestRoot);
-            }, variation);
-        }
-
-        [Theory]
-        [MemberData(nameof(GraphProjects))]
-        public void IncrementalRebuildWithLastChangedInGraph(string variation, string[] projects)
-        {
-            var instances = projects.Select(p => CreateTestInstance(p, variation)).ToArray();
-            var instance = instances[0];
-
-            Build(instance.TestRoot);
-
-            Iterate(c =>
-            {
-                c.Measure(() => Build(instance.TestRoot));
-                RemoveBin(instances.Last().TestRoot);
-            }, variation);
-        }
-
-
-        [Theory]
-        [MemberData(nameof(GraphProjects))]
-        public void IncrementalSkipAllNoDependenciesInGraph(string variation, string[] projects)
-        {
-            var instances = projects.Select(p => CreateTestInstance(p, variation)).ToArray();
-            var instance = instances[0];
-
-            Build(instance.TestRoot);
-
-            Iterate(c =>
-            {
-                foreach (var i in instances)
                 {
-                    c.Measure(() => Run(new BuildCommand(i.TestRoot, framework: DefaultFramework, noDependencies: true, buildProfile: false)));
+                    Build(instance.TestRoot);
                 }
-            }, variation);
+            }
         }
 
-        [Theory]
+        [Benchmark]
-        [MemberData(nameof(GraphProjects))]
+        public void IncrementalRebuildWithRootChangedInGraph_SingleTargetGraph() =>
-        public void BuildAllNoDependenciesInGraph(string variation, string[] projects)
+            IncrementalRebuildWithRootChangedInGraph(CreateTestInstances(SingleTargetGraph));
-        {
-            var instances = projects.Select(p => CreateTestInstance(p, variation)).ToArray();
 
-            Iterate(c =>
+        [Benchmark]
+        public void IncrementalRebuildWithRootChangedInGraph_TwoTargetGraph() =>
+            IncrementalRebuildWithRootChangedInGraph(CreateTestInstances(TwoTargetGraph));
+
+        [Benchmark]
+        public void IncrementalRebuildWithRootChangedInGraph_TwoTargetGraphLarge() =>
+            IncrementalRebuildWithRootChangedInGraph(CreateTestInstances(TwoTargetGraphLarge));
+
+        public void IncrementalRebuildWithRootChangedInGraph(TestInstance[] instances)
+        {
+            var instance = instances[0];
+
+            Build(instance.TestRoot);
+
+            foreach (var iteration in Benchmark.Iterations)
             {
-                foreach (var i in instances.Reverse())
+                using (iteration.StartMeasurement())
                 {
-                    c.Measure(() => Run(new BuildCommand(i.TestRoot, framework: DefaultFramework, noDependencies: true, buildProfile: false)));
+                    Build(instance.TestRoot);
+                }
+                RemoveBin(instance.TestRoot);
+            }
+        }
+
+        [Benchmark]
+        public void IncrementalRebuildWithLastChangedInGraph_SingleTargetGraph() =>
+            IncrementalRebuildWithLastChangedInGraph(CreateTestInstances(SingleTargetGraph));
+
+        [Benchmark]
+        public void IncrementalRebuildWithLastChangedInGraph_TwoTargetGraph() =>
+            IncrementalRebuildWithLastChangedInGraph(CreateTestInstances(TwoTargetGraph));
+
+        [Benchmark]
+        public void IncrementalRebuildWithLastChangedInGraph_TwoTargetGraphLarge() =>
+            IncrementalRebuildWithLastChangedInGraph(CreateTestInstances(TwoTargetGraphLarge));
+
+        public void IncrementalRebuildWithLastChangedInGraph(TestInstance[] instances)
+        {
+            var instance = instances[0];
+
+            Build(instance.TestRoot);
+
+            foreach (var iteration in Benchmark.Iterations)
+            {
+                using (iteration.StartMeasurement())
+                {
+                    Build(instance.TestRoot);
+                }
+                RemoveBin(instances.Last().TestRoot);
+            }
+        }
+
+
+        [Benchmark]
+        public void IncrementalSkipAllNoDependenciesInGraph_SingleTargetGraph() =>
+           IncrementalSkipAllNoDependenciesInGraph(CreateTestInstances(SingleTargetGraph));
+
+        [Benchmark]
+        public void IncrementalSkipAllNoDependenciesInGraph_TwoTargetGraph() =>
+            IncrementalSkipAllNoDependenciesInGraph(CreateTestInstances(TwoTargetGraph));
+
+        [Benchmark]
+        public void IncrementalSkipAllNoDependenciesInGraph_TwoTargetGraphLarge() =>
+            IncrementalSkipAllNoDependenciesInGraph(CreateTestInstances(TwoTargetGraphLarge));
+
+        public void IncrementalSkipAllNoDependenciesInGraph(TestInstance[] instances)
+        {
+            var instance = instances[0];
+
+            Build(instance.TestRoot);
+
+            foreach (var iteration in Benchmark.Iterations)
+            {
+                using (iteration.StartMeasurement())
+                {
+                    foreach (var i in instances)
+                    {
+                        Run(new BuildCommand(i.TestRoot,
+                            framework: DefaultFramework,
+                            noDependencies: true,
+                            buildProfile: false));
+                    }
+                }
+            }
+        }
+        [Benchmark]
+        public void BuildAllNoDependenciesInGraphh_SingleTargetGraph() =>
+          BuildAllNoDependenciesInGraph(CreateTestInstances(SingleTargetGraph));
+
+        [Benchmark]
+        public void BuildAllNoDependenciesInGraph_TwoTargetGraph() =>
+            BuildAllNoDependenciesInGraph(CreateTestInstances(TwoTargetGraph));
+
+        [Benchmark]
+        public void BuildAllNoDependenciesInGraph_TwoTargetGraphLarge() =>
+            BuildAllNoDependenciesInGraph(CreateTestInstances(TwoTargetGraphLarge));
+
+        public void BuildAllNoDependenciesInGraph(TestInstance[] instances)
+        {
+            foreach (var iteration in Benchmark.Iterations)
+            {
+                using (iteration.StartMeasurement())
+                {
+                    foreach (var i in instances.Reverse())
+                    {
+                        Run(new BuildCommand(i.TestRoot,
+                            framework: DefaultFramework,
+                            noDependencies: true,
+                            buildProfile: false));
+                    }
                 }
                 foreach (var instance in instances)
                 {
                     RemoveBin(instance.TestRoot);
                 }
-
+            }
-            }, variation);
         }
     }
 }

test/dotnet-performance.Tests/PerformanceTestBase.cs

@@ -1,10 +1,8 @@
-using System;
+using System.IO;
-using System.Diagnostics;
+using System.Linq;
-using System.IO;
 using System.Runtime.CompilerServices;
 using Microsoft.DotNet.Tools.Test.Utilities;
 using Microsoft.DotNet.TestFramework;
-using Microsoft.DotNet.Cli.Utils;
 
 namespace Microsoft.DotNet.Tools.Builder.Tests
 {
@@ -25,39 +23,18 @@ namespace Microsoft.DotNet.Tools.Builder.Tests
             Directory.Delete(Path.Combine(project, "bin"), true);
         }
 
-        protected TestInstance CreateTestInstance(string testProjectName, string variation = "", [CallerMemberName] string callingMethod = "")
+        protected TestInstance[] CreateTestInstances(string[] testProjectNames, [CallerMemberName] string callingMethod = "")
         {
-            return TestAssetsManager.CreateTestInstance(Path.Combine("PerformanceTestProjects", testProjectName), callingMethod + variation)
+            return testProjectNames.Select(testProjectName =>
-                .WithLockFiles();
+            {
+                return CreateTestInstance(testProjectName, callingMethod);
+            }).ToArray();
         }
 
-        public void Iterate(Action<PerformanceIterationContext> action, string variation = "", int iterations = 3, [CallerMemberName] string callingMethod = "")
+        protected TestInstance CreateTestInstance(string testProjectName, [CallerMemberName] string callingMethod = "")
         {
-            var fullname = callingMethod + (variation != "" ? "" + variation : "");
+                return TestAssetsManager.CreateTestInstance(Path.Combine("PerformanceTestProjects", testProjectName), callingMethod)
-            // Heat up iteration
+                     .WithLockFiles();
-            var context = new PerformanceIterationContext();
-            action(context);
-
-            TimeSpan totalTime;
-            for (int i = 0; i < iterations; i++)
-            {
-                context = new PerformanceIterationContext();
-                action(context);
-                totalTime += context.Stopwatch.Elapsed;
-            }
-            Reporter.Output.WriteLine($"[RESULT] {callingMethod}-{variation} {totalTime.TotalSeconds/iterations:F} sec/iteration".Bold());
-        }
-
-        public class PerformanceIterationContext
-        {
-            public Stopwatch Stopwatch { get; } = new Stopwatch();
-
-            public void Measure(Action action)
-            {
-                Stopwatch.Start();
-                action();
-                Stopwatch.Stop();
-            }
         }
     }
 }

test/dotnet-performance.Tests/nuget.config

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+  <packageSources>
+    <add key="dotnet-buildtools" value="https://myget.org/F/dotnet-buildtools/api/v3/index.json" />
+  </packageSources>
+</configuration>

test/dotnet-performance.Tests/project.json

@@ -11,7 +11,8 @@
     },
     "Newtonsoft.Json": "7.0.1",
     "xunit": "2.1.0",
-    "dotnet-test-xunit": "1.0.0-dev-140469-38"
+    "dotnet-test-xunit": "1.0.0-dev-140469-38",
+    "Microsoft.DotNet.xunit.performance": "1.0.0-alpha-build0028"
   },
   "frameworks": {
     "netcoreapp1.0": {