Build-Time Hello World Performance Test (#2681)

Hello world build performance benchmark.

Microsoft.DotNet.Cli.sln

@@ -103,6 +103,8 @@ Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "dotnet-dependency-tool-invo
 EndProject
 Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "Microsoft.DotNet.Compiler.Common.Tests", "test\Microsoft.DotNet.Compiler.Common.Tests\Microsoft.DotNet.Compiler.Common.Tests.xproj", "{44E7D1AC-DCF1-4A18-9C22-F09E6BB302B5}"
 EndProject
+Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "Performance", "test\Performance\Performance.xproj", "{6A3095FF-A7C5-4300-85A9-C025C384401D}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -723,6 +725,22 @@ Global
 		{44E7D1AC-DCF1-4A18-9C22-F09E6BB302B5}.RelWithDebInfo|Any CPU.Build.0 = Release|Any CPU
 		{44E7D1AC-DCF1-4A18-9C22-F09E6BB302B5}.RelWithDebInfo|x64.ActiveCfg = Release|Any CPU
 		{44E7D1AC-DCF1-4A18-9C22-F09E6BB302B5}.RelWithDebInfo|x64.Build.0 = Release|Any CPU
+		{6A3095FF-A7C5-4300-85A9-C025C384401D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{6A3095FF-A7C5-4300-85A9-C025C384401D}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{6A3095FF-A7C5-4300-85A9-C025C384401D}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{6A3095FF-A7C5-4300-85A9-C025C384401D}.Debug|x64.Build.0 = Debug|Any CPU
+		{6A3095FF-A7C5-4300-85A9-C025C384401D}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
+		{6A3095FF-A7C5-4300-85A9-C025C384401D}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
+		{6A3095FF-A7C5-4300-85A9-C025C384401D}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{6A3095FF-A7C5-4300-85A9-C025C384401D}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{6A3095FF-A7C5-4300-85A9-C025C384401D}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{6A3095FF-A7C5-4300-85A9-C025C384401D}.Release|Any CPU.Build.0 = Release|Any CPU
+		{6A3095FF-A7C5-4300-85A9-C025C384401D}.Release|x64.ActiveCfg = Release|Any CPU
+		{6A3095FF-A7C5-4300-85A9-C025C384401D}.Release|x64.Build.0 = Release|Any CPU
+		{6A3095FF-A7C5-4300-85A9-C025C384401D}.RelWithDebInfo|Any CPU.ActiveCfg = Release|Any CPU
+		{6A3095FF-A7C5-4300-85A9-C025C384401D}.RelWithDebInfo|Any CPU.Build.0 = Release|Any CPU
+		{6A3095FF-A7C5-4300-85A9-C025C384401D}.RelWithDebInfo|x64.ActiveCfg = Release|Any CPU
+		{6A3095FF-A7C5-4300-85A9-C025C384401D}.RelWithDebInfo|x64.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -771,5 +789,6 @@ Global
 		{1AB5B24B-B317-4142-A5D1-A6E84F15BA34} = {ADA7052B-884B-4776-8B8D-D04191D0AA70}
 		{C26A48BB-193F-450C-AB09-4D3324C78188} = {1AB5B24B-B317-4142-A5D1-A6E84F15BA34}
 		{44E7D1AC-DCF1-4A18-9C22-F09E6BB302B5} = {17735A9D-BFD9-4585-A7CB-3208CA6EA8A7}
+		{6A3095FF-A7C5-4300-85A9-C025C384401D} = {17735A9D-BFD9-4585-A7CB-3208CA6EA8A7}
 	EndGlobalSection
 EndGlobal

test/EndToEnd/EndToEndTest.cs

@@ -211,7 +211,9 @@ namespace Microsoft.DotNet.Tests.EndToEnd
             catch(Exception) {}
 
             Directory.CreateDirectory(RestoredTestProjectDirectory);
-            WriteNuGetConfig(RestoredTestProjectDirectory);
+
+            // Todo: this is a hack until corefx is on nuget.org remove this After RC 2 Release
+            NuGetConfig.Write(RestoredTestProjectDirectory);
 
             var currentDirectory = Directory.GetCurrentDirectory();
             Directory.SetCurrentDirectory(RestoredTestProjectDirectory);
@@ -248,24 +250,6 @@ namespace Microsoft.DotNet.Tests.EndToEnd
             return isSupported;
         }
 
-        // Todo: this is a hack until corefx is on nuget.org remove this After RC 2 Release
-        private static void WriteNuGetConfig(string directory)
-        {
-            var contents = @"<?xml version=""1.0"" encoding=""utf-8""?>
-<configuration>
-<packageSources>
-<!--To inherit the global NuGet package sources remove the <clear/> line below -->
-<clear />
-<add key=""dotnet-core"" value=""https://dotnet.myget.org/F/dotnet-core/api/v3/index.json"" />
-<add key=""api.nuget.org"" value=""https://api.nuget.org/v3/index.json"" />
-</packageSources>
-</configuration>";
-
-            var path = Path.Combine(directory, "NuGet.config");
-
-            File.WriteAllText(path, contents);
-        }
-
         private static DateTime GetLastWriteTimeUtcOfDirectoryFiles(string outputDirectory)
         {
             return Directory.EnumerateFiles(outputDirectory).Max(f => File.GetLastWriteTimeUtc(f));

test/Microsoft.DotNet.Tools.Tests.Utilities/NuGetConfig.cs

@@ -0,0 +1,27 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System.IO;
+
+namespace Microsoft.DotNet.Tools.Test.Utilities
+{
+    public static class NuGetConfig
+    {
+        public static void Write(string directory)
+        {
+            var contents = @"<?xml version=""1.0"" encoding=""utf-8""?>
+<configuration>
+<packageSources>
+<!--To inherit the global NuGet package sources remove the <clear/> line below -->
+<clear />
+<add key=""dotnet-core"" value=""https://dotnet.myget.org/F/dotnet-core/api/v3/index.json"" />
+<add key=""api.nuget.org"" value=""https://api.nuget.org/v3/index.json"" />
+</packageSources>
+</configuration>";
+
+            var path = Path.Combine(directory, "NuGet.config");
+
+            File.WriteAllText(path, contents);
+        }
+    }
+}

test/Performance/HelloWorld.cs

@@ -0,0 +1,92 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
+using System.IO;
+using System.Linq;
+using System.Runtime.InteropServices;
+using Microsoft.DotNet.Tools.Test.Utilities;
+using Microsoft.Extensions.PlatformAbstractions;
+using Microsoft.Xunit.Performance;
+using Xunit;
+using System.Diagnostics;
+
+namespace Microsoft.DotNet.Tests.Performance
+{
+    public class HelloWorld : TestBase
+    {
+        private static readonly string s_testdirName = "helloworldtestroot";
+        private static readonly string s_outputdirName = "test space/bin";
+
+        private static string AssetsRoot { get; set; }
+        private static string RestoredTestProjectDirectory { get; set; }
+
+        private string TestDirectory { get; set; }
+        private string TestProject { get; set; }
+        private string OutputDirectory { get; set; }
+
+        static HelloWorld()
+        {
+            HelloWorld.SetupStaticTestProject();
+        }
+
+        public HelloWorld()
+        {
+        }
+
+        [Benchmark]
+        public void MeasureDotNetBuild()
+        {
+            foreach (var iter in Benchmark.Iterations)
+            {
+                // Setup a new instance of the test project.
+                TestInstanceSetup();
+
+                // Setup the build command.
+                var buildCommand = new BuildCommand(TestProject, output: OutputDirectory, framework: DefaultFramework);
+                using (iter.StartMeasurement())
+                {
+                    // Execute the build command.
+                    buildCommand.Execute();
+                }
+            }
+        }
+
+        private void TestInstanceSetup()
+        {
+            var root = Temp.CreateDirectory();
+
+            var testInstanceDir = root.CopyDirectory(RestoredTestProjectDirectory);
+
+            TestDirectory = testInstanceDir.Path;
+            TestProject = Path.Combine(TestDirectory, "project.json");
+            OutputDirectory = Path.Combine(TestDirectory, s_outputdirName);
+        }
+
+        private static void SetupStaticTestProject()
+        {
+            AssetsRoot = Path.Combine(AppContext.BaseDirectory, "bin");
+            RestoredTestProjectDirectory = Path.Combine(AssetsRoot, s_testdirName);
+
+            // Ignore Delete Failure
+            try
+            {
+                Directory.Delete(RestoredTestProjectDirectory, true);
+            }
+            catch (Exception) { }
+
+            Directory.CreateDirectory(RestoredTestProjectDirectory);
+
+            // Todo: this is a hack until corefx is on nuget.org remove this After RC 2 Release
+            NuGetConfig.Write(RestoredTestProjectDirectory);
+
+            var newCommand = new NewCommand();
+            newCommand.WorkingDirectory = RestoredTestProjectDirectory;
+            newCommand.Execute().Should().Pass();
+
+            var restoreCommand = new RestoreCommand();
+            restoreCommand.WorkingDirectory = RestoredTestProjectDirectory;
+            restoreCommand.Execute("--quiet").Should().Pass();
+        }
+    }
+}

test/Performance/Performance.xproj

@@ -0,0 +1,21 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="14.0.24720" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <VisualStudioVersion Condition="'$(VisualStudioVersion)' == ''">14.0.24720</VisualStudioVersion>
+    <VSToolsPath Condition="'$(VSToolsPath)' == ''">$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)</VSToolsPath>
+  </PropertyGroup>
+  <Import Project="$(VSToolsPath)\DNX\Microsoft.DNX.Props" Condition="'$(VSToolsPath)' != ''" />
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>6A3095FF-A7C5-4300-85A9-C025C384401D</ProjectGuid>
+    <RootNamespace>Microsoft.DotNet.Tests.Performance</RootNamespace>
+    <BaseIntermediateOutputPath Condition="'$(BaseIntermediateOutputPath)'=='' ">..\..\artifacts\obj\$(MSBuildProjectName)</BaseIntermediateOutputPath>
+    <OutputPath Condition="'$(OutputPath)'=='' ">..\..\artifacts\bin</OutputPath>
+  </PropertyGroup>
+  <PropertyGroup>
+    <SchemaVersion>2.0</SchemaVersion>
+  </PropertyGroup>
+  <ItemGroup>
+    <Service Include="{82a7f48d-3b50-4b1e-b82e-3ada8210c358}" />
+  </ItemGroup>
+  <Import Project="$(VSToolsPath)\DNX\Microsoft.DNX.targets" Condition="'$(VSToolsPath)' != ''" />
+</Project>

test/Performance/project.json

@@ -0,0 +1,32 @@
+{
+  "version": "1.0.0-*",
+  "dependencies": {
+    "Microsoft.NETCore.App": {
+      "type": "platform",
+      "version": "1.0.0-rc2-*"
+    },
+    "System.Runtime.Serialization.Primitives": "4.1.1-rc2-24022",
+    "Microsoft.DotNet.ProjectModel": {
+      "target": "project"
+    },
+    "Microsoft.DotNet.Cli.Utils": {
+      "target": "project"
+    },
+    "Microsoft.DotNet.Tools.Tests.Utilities": {
+      "target": "project"
+    },
+    "xunit": "2.1.0",
+    "xunit.netcore.extensions": "1.0.0-prerelease-00206",
+    "dotnet-test-xunit": "1.0.0-rc2-162081-13",
+    "Microsoft.DotNet.xunit.performance": "1.0.0-alpha-build0028"
+  },
+  "frameworks": {
+    "netcoreapp1.0": {
+      "imports": [
+        "dotnet5.4",
+        "portable-net451+win8"
+      ]
+    }
+  },
+  "testRunner": "xunit"
+}