Merge pull request #1853 from livarcocc/more_test_params

Added remaining params from https://github.com/dotnet/cli/issues/1376 and build before tests

Microsoft.DotNet.Cli.sln

@@ -81,6 +81,8 @@ Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "dotnet-test.Tests", "test\d
 EndProject
 Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "update-dependencies", "scripts\update-dependencies\update-dependencies.xproj", "{A28BD8AC-DF15-4F58-8299-98A9AE2B8726}"
 EndProject
+Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "Microsoft.DotNet.Cli.Utils.Tests", "test\Microsoft.DotNet.Cli.Utils.Tests\Microsoft.DotNet.Cli.Utils.Tests.xproj", "{09C52F96-EFDD-4448-95EC-6D362DD60BAA}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -573,6 +575,22 @@ Global
 		{A28BD8AC-DF15-4F58-8299-98A9AE2B8726}.RelWithDebInfo|Any CPU.Build.0 = Release|Any CPU
 		{A28BD8AC-DF15-4F58-8299-98A9AE2B8726}.RelWithDebInfo|x64.ActiveCfg = Release|Any CPU
 		{A28BD8AC-DF15-4F58-8299-98A9AE2B8726}.RelWithDebInfo|x64.Build.0 = Release|Any CPU
+		{09C52F96-EFDD-4448-95EC-6D362DD60BAA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{09C52F96-EFDD-4448-95EC-6D362DD60BAA}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{09C52F96-EFDD-4448-95EC-6D362DD60BAA}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{09C52F96-EFDD-4448-95EC-6D362DD60BAA}.Debug|x64.Build.0 = Debug|Any CPU
+		{09C52F96-EFDD-4448-95EC-6D362DD60BAA}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
+		{09C52F96-EFDD-4448-95EC-6D362DD60BAA}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
+		{09C52F96-EFDD-4448-95EC-6D362DD60BAA}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{09C52F96-EFDD-4448-95EC-6D362DD60BAA}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{09C52F96-EFDD-4448-95EC-6D362DD60BAA}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{09C52F96-EFDD-4448-95EC-6D362DD60BAA}.Release|Any CPU.Build.0 = Release|Any CPU
+		{09C52F96-EFDD-4448-95EC-6D362DD60BAA}.Release|x64.ActiveCfg = Release|Any CPU
+		{09C52F96-EFDD-4448-95EC-6D362DD60BAA}.Release|x64.Build.0 = Release|Any CPU
+		{09C52F96-EFDD-4448-95EC-6D362DD60BAA}.RelWithDebInfo|Any CPU.ActiveCfg = Release|Any CPU
+		{09C52F96-EFDD-4448-95EC-6D362DD60BAA}.RelWithDebInfo|Any CPU.Build.0 = Release|Any CPU
+		{09C52F96-EFDD-4448-95EC-6D362DD60BAA}.RelWithDebInfo|x64.ActiveCfg = Release|Any CPU
+		{09C52F96-EFDD-4448-95EC-6D362DD60BAA}.RelWithDebInfo|x64.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -610,5 +628,6 @@ Global
 		{857274AC-E741-4266-A7FD-14DEE0C1CC96} = {17735A9D-BFD9-4585-A7CB-3208CA6EA8A7}
 		{60C33D0A-A5D8-4AB0-9956-1F804654DF05} = {17735A9D-BFD9-4585-A7CB-3208CA6EA8A7}
 		{A28BD8AC-DF15-4F58-8299-98A9AE2B8726} = {88278B81-7649-45DC-8A6A-D3A645C5AFC3}
+		{09C52F96-EFDD-4448-95EC-6D362DD60BAA} = {17735A9D-BFD9-4585-A7CB-3208CA6EA8A7}
 	EndGlobalSection
 EndGlobal

TestAssets/TestProjects/ProjectWithTests/GivenThatIWantSomeFakeTests.cs

@@ -14,9 +14,9 @@ namespace FakeTests
         }
         
         [Fact]
-        public void It_fails()
+        public void It_also_succeeds()
         {
-            Assert.True(false);
+            Assert.True(true);
         }
     }
 }

src/Microsoft.DotNet.Cli.Utils/CommandResolution/CommandResolverArguments.cs

@@ -22,5 +22,7 @@ namespace Microsoft.DotNet.Cli.Utils
         public string Configuration { get; set; }
 
         public IEnumerable<string> InferredExtensions { get; set; }
+
+        public string BuildBasePath { get; set; }
     }
 }

src/Microsoft.DotNet.Cli.Utils/CommandResolution/ProjectDependenciesCommandResolver.cs

@@ -39,7 +39,7 @@ namespace Microsoft.DotNet.Cli.Utils
 
         public CommandSpec Resolve(CommandResolverArguments commandResolverArguments)
         {
-            if (commandResolverArguments.Framework == null 
+            if (commandResolverArguments.Framework == null
                 || commandResolverArguments.ProjectDirectory == null
                 || commandResolverArguments.Configuration == null
                 || commandResolverArguments.CommandName == null)
@@ -53,7 +53,8 @@ namespace Microsoft.DotNet.Cli.Utils
                     commandResolverArguments.Configuration,
                     commandResolverArguments.CommandName,
                     commandResolverArguments.CommandArguments.OrEmptyIfNull(),
-                    commandResolverArguments.OutputPath);
+                    commandResolverArguments.OutputPath,
+                    commandResolverArguments.BuildBasePath);
         }
 
         private CommandSpec ResolveFromProjectDependencies(
@@ -62,23 +63,25 @@ namespace Microsoft.DotNet.Cli.Utils
             string configuration,
             string commandName,
             IEnumerable<string> commandArguments,
-            string outputPath)
+            string outputPath,
+            string buildBasePath)
         {
             var allowedExtensions = GetAllowedCommandExtensionsFromEnvironment(_environment);
 
             var projectContext = GetProjectContextFromDirectory(
-                projectDirectory, 
+                projectDirectory,
                 framework);
 
             if (projectContext == null)
-            { 
+            {
                 return null;
             }
 
-            var depsFilePath = projectContext.GetOutputPaths(configuration, outputPath: outputPath).RuntimeFiles.DepsJson;
+            var depsFilePath =
+                projectContext.GetOutputPaths(configuration, buildBasePath, outputPath).RuntimeFiles.DepsJson;
 
             var dependencyLibraries = GetAllDependencyLibraries(projectContext);
-             
+
             return ResolveFromDependencyLibraries(
                 dependencyLibraries,
                 depsFilePath,

src/Microsoft.DotNet.Cli.Utils/ProjectDependenciesCommandFactory.cs

@@ -11,17 +11,20 @@ namespace Microsoft.DotNet.Cli.Utils
         private readonly NuGetFramework _nugetFramework;
         private readonly string _configuration;
         private readonly string _outputPath;
+        private readonly string _buildBasePath;
         private readonly string _projectDirectory;
 
         public ProjectDependenciesCommandFactory(
-            NuGetFramework nugetFramework, 
+            NuGetFramework nugetFramework,
-            string configuration, 
+            string configuration,
             string outputPath,
+            string buildBasePath,
             string projectDirectory)
         {
             _nugetFramework = nugetFramework;
             _configuration = configuration;
             _outputPath = outputPath;
+            _buildBasePath = buildBasePath;
             _projectDirectory = projectDirectory;
         }
 
@@ -42,11 +45,12 @@ namespace Microsoft.DotNet.Cli.Utils
             }
 
             var commandSpec = FindProjectDependencyCommands(
-                commandName, 
+                commandName,
-                args, 
+                args,
-                configuration, 
+                configuration,
-                framework, 
+                framework,
                 _outputPath,
+                _buildBasePath,
                 _projectDirectory);
 
             return Command.Create(commandSpec);
@@ -58,6 +62,7 @@ namespace Microsoft.DotNet.Cli.Utils
             string configuration,
             NuGetFramework framework,
             string outputPath,
+            string buildBasePath,
             string projectDirectory)
         {
             var commandResolverArguments = new CommandResolverArguments
@@ -67,6 +72,7 @@ namespace Microsoft.DotNet.Cli.Utils
                 Framework = framework,
                 Configuration = configuration,
                 OutputPath = outputPath,
+                BuildBasePath = buildBasePath,
                 ProjectDirectory = projectDirectory
             };
 

src/dotnet/commands/dotnet-test/AssemblyUnderTest.cs

@@ -0,0 +1,41 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System.IO;
+using Microsoft.DotNet.ProjectModel;
+
+namespace Microsoft.DotNet.Tools.Test
+{
+    public class AssemblyUnderTest
+    {
+        private readonly ProjectContext _projectContext;
+        private readonly DotnetTestParams _dotentTestParams;
+
+        public AssemblyUnderTest(ProjectContext projectContext, DotnetTestParams dotentTestParams)
+        {
+            _projectContext = projectContext;
+            _dotentTestParams = dotentTestParams;
+        }
+
+        public string Path
+        {
+            get
+            {
+                var outputPaths = _projectContext.GetOutputPaths(
+                    _dotentTestParams.Config,
+                    _dotentTestParams.BuildBasePath,
+                    _dotentTestParams.Output);
+
+                var assemblyUnderTest = outputPaths.CompilationFiles.Assembly;
+
+                if (!string.IsNullOrEmpty(_dotentTestParams.Output) ||
+                    !string.IsNullOrEmpty(_dotentTestParams.BuildBasePath))
+                {
+                    assemblyUnderTest = outputPaths.RuntimeFiles.Assembly;
+                }
+
+                return assemblyUnderTest;
+            }
+        }
+    }
+}

src/dotnet/commands/dotnet-test/BaseDotnetTestRunner.cs

@@ -0,0 +1,68 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System.Collections.Generic;
+using Microsoft.DotNet.ProjectModel;
+
+namespace Microsoft.DotNet.Tools.Test
+{
+    public abstract class BaseDotnetTestRunner : IDotnetTestRunner
+    {
+        public int RunTests(ProjectContext projectContext, DotnetTestParams dotnetTestParams)
+        {
+            var result = BuildTestProject(dotnetTestParams);
+
+            return result == 0 ? DoRunTests(projectContext, dotnetTestParams) : result;
+        }
+
+        internal abstract int DoRunTests(ProjectContext projectContext, DotnetTestParams dotnetTestParams);
+
+        private int BuildTestProject(DotnetTestParams dotnetTestParams)
+        {
+            if (dotnetTestParams.NoBuild)
+            {
+                return 0;
+            }
+
+            return DoBuildTestProject(dotnetTestParams);
+        }
+
+        private int DoBuildTestProject(DotnetTestParams dotnetTestParams)
+        {
+            var strings = new List<string>
+            {
+                $"--configuration",
+                dotnetTestParams.Config,
+                $"{dotnetTestParams.ProjectPath}"
+            };
+
+            if (dotnetTestParams.Framework != null)
+            {
+                strings.Add("--framework");
+                strings.Add($"{dotnetTestParams.Framework}");
+            }
+
+            if (!string.IsNullOrEmpty(dotnetTestParams.BuildBasePath))
+            {
+                strings.Add("--build-base-path");
+                strings.Add(dotnetTestParams.BuildBasePath);
+            }
+
+            if (!string.IsNullOrEmpty(dotnetTestParams.Output))
+            {
+                strings.Add("--output");
+                strings.Add(dotnetTestParams.Output);
+            }
+
+            if (!string.IsNullOrEmpty(dotnetTestParams.Runtime))
+            {
+                strings.Add("--runtime");
+                strings.Add(dotnetTestParams.Runtime);
+            }
+
+            var result = Build.BuildCommand.Run(strings.ToArray());
+
+            return result;
+        }
+    }
+}

src/dotnet/commands/dotnet-test/ConsoleTestRunner.cs

@@ -0,0 +1,50 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System.Collections.Generic;
+using Microsoft.DotNet.Cli.Utils;
+using Microsoft.DotNet.ProjectModel;
+
+namespace Microsoft.DotNet.Tools.Test
+{
+    public class ConsoleTestRunner : BaseDotnetTestRunner
+    {
+        internal override int DoRunTests(ProjectContext projectContext, DotnetTestParams dotnetTestParams)
+        {
+            var commandFactory =
+                new ProjectDependenciesCommandFactory(
+                    projectContext.TargetFramework,
+                    dotnetTestParams.Config,
+                    dotnetTestParams.Output,
+                    dotnetTestParams.BuildBasePath,
+                    projectContext.ProjectDirectory);
+
+            return commandFactory.Create(
+                    GetCommandName(projectContext.ProjectFile.TestRunner),
+                    GetCommandArgs(projectContext, dotnetTestParams),
+                    projectContext.TargetFramework,
+                    dotnetTestParams.Config)
+                .ForwardStdErr()
+                .ForwardStdOut()
+                .Execute()
+                .ExitCode;
+        }
+
+        private IEnumerable<string> GetCommandArgs(ProjectContext projectContext, DotnetTestParams dotnetTestParams)
+        {
+            var commandArgs = new List<string>
+            {
+                new AssemblyUnderTest(projectContext, dotnetTestParams).Path
+            };
+
+            commandArgs.AddRange(dotnetTestParams.RemainingArguments);
+
+            return commandArgs;
+        }
+
+        private static string GetCommandName(string testRunner)
+        {
+            return $"dotnet-test-{testRunner}";
+        }
+    }
+}

src/dotnet/commands/dotnet-test/DesignTimeRunner.cs

@@ -0,0 +1,69 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
+using Microsoft.DotNet.Cli.Utils;
+using Microsoft.DotNet.ProjectModel;
+
+namespace Microsoft.DotNet.Tools.Test
+{
+    public class DesignTimeRunner : BaseDotnetTestRunner
+    {
+        internal override int DoRunTests(ProjectContext projectContext, DotnetTestParams dotnetTestParams)
+        {
+            Console.WriteLine("Listening on port {0}", dotnetTestParams.Port.Value);
+
+            HandleDesignTimeMessages(projectContext, dotnetTestParams);
+
+            return 0;
+        }
+
+        private static void HandleDesignTimeMessages(
+            ProjectContext projectContext,
+            DotnetTestParams dotnetTestParams)
+        {
+            var reportingChannelFactory = new ReportingChannelFactory();
+            var adapterChannel = reportingChannelFactory.CreateAdapterChannel(dotnetTestParams.Port.Value);
+
+            try
+            {
+                var pathToAssemblyUnderTest = new AssemblyUnderTest(projectContext, dotnetTestParams).Path;
+                var messages = new TestMessagesCollection();
+                using (var dotnetTest = new DotnetTest(messages, pathToAssemblyUnderTest))
+                {
+                    var commandFactory =
+                        new ProjectDependenciesCommandFactory(
+                            projectContext.TargetFramework,
+                            dotnetTestParams.Config,
+                            dotnetTestParams.Output,
+                            dotnetTestParams.BuildBasePath,
+                            projectContext.ProjectDirectory);
+
+                    var testRunnerFactory =
+                        new TestRunnerFactory(GetCommandName(projectContext.ProjectFile.TestRunner), commandFactory);
+
+                    dotnetTest
+                        .AddNonSpecificMessageHandlers(messages, adapterChannel)
+                        .AddTestDiscoveryMessageHandlers(adapterChannel, reportingChannelFactory, testRunnerFactory)
+                        .AddTestRunMessageHandlers(adapterChannel, reportingChannelFactory, testRunnerFactory)
+                        .AddTestRunnnersMessageHandlers(adapterChannel, reportingChannelFactory);
+
+                    dotnetTest.StartListeningTo(adapterChannel);
+
+                    adapterChannel.Connect();
+
+                    dotnetTest.StartHandlingMessages();
+                }
+            }
+            catch (Exception ex)
+            {
+                adapterChannel.SendError(ex);
+            }
+        }
+
+        private static string GetCommandName(string testRunner)
+        {
+            return $"test-{testRunner}";
+        }
+    }
+}

src/dotnet/commands/dotnet-test/DotnetTestParams.cs

@@ -0,0 +1,156 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using Microsoft.Dnx.Runtime.Common.CommandLine;
+using Microsoft.DotNet.Cli.Utils;
+using NuGet.Frameworks;
+using static System.Int32;
+
+namespace Microsoft.DotNet.Tools.Test
+{
+    public class DotnetTestParams
+    {
+        private readonly CommandLineApplication _app;
+
+        private CommandOption _outputOption;
+        private CommandOption _buildBasePath;
+        private CommandOption _frameworkOption;
+        private CommandOption _runtimeOption;
+        private CommandOption _configurationOption;
+        private CommandOption _portOption;
+        private CommandOption _parentProcessIdOption;
+        private CommandArgument _projectPath;
+        private CommandOption _noBuildOption;
+
+        public int? Port { get; set; }
+
+        public int? ParentProcessId { get; set; }
+
+        public string Runtime { get; set; }
+
+        public string Config { get; set; }
+
+        public string BuildBasePath { get; set; }
+
+        public string Output { get; set; }
+
+        public string ProjectPath { get; set; }
+
+        public NuGetFramework Framework { get; set; }
+
+        public List<string> RemainingArguments { get; set; }
+
+        public bool NoBuild { get; set; }
+
+        public DotnetTestParams()
+        {
+            _app = new CommandLineApplication(false)
+            {
+                Name = "dotnet test",
+                FullName = ".NET Test Driver",
+                Description = "Test Driver for the .NET Platform"
+            };
+
+            AddDotnetTestParameters();
+        }
+
+        public void Parse(string[] args)
+        {
+            _app.OnExecute(() =>
+            {
+                // Locate the project and get the name and full path
+                ProjectPath = _projectPath.Value;
+                if (string.IsNullOrEmpty(ProjectPath))
+                {
+                    ProjectPath = Directory.GetCurrentDirectory();
+                }
+
+                if (_parentProcessIdOption.HasValue())
+                {
+                    int processId;
+
+                    if (!TryParse(_parentProcessIdOption.Value(), out processId))
+                    {
+                        throw new InvalidOperationException(
+                            $"Invalid process id '{_parentProcessIdOption.Value()}'. Process id must be an integer.");
+                    }
+
+                    ParentProcessId = processId;
+                }
+
+                if (_portOption.HasValue())
+                {
+                    int port;
+
+                    if (!TryParse(_portOption.Value(), out port))
+                    {
+                        throw new InvalidOperationException($"{_portOption.Value()} is not a valid port number.");
+                    }
+
+                    Port = port;
+                }
+
+                if (_frameworkOption.HasValue())
+                {
+                    Framework = NuGetFramework.Parse(_frameworkOption.Value());
+                }
+
+                Output = _outputOption.Value();
+                BuildBasePath = _buildBasePath.Value();
+                Config = _configurationOption.Value() ?? Constants.DefaultConfiguration;
+                Runtime = _runtimeOption.Value();
+                NoBuild = _noBuildOption.HasValue();
+
+                RemainingArguments = _app.RemainingArguments;
+
+                return 0;
+            });
+
+            _app.Execute(args);
+        }
+
+        private void AddDotnetTestParameters()
+        {
+            _app.HelpOption("-?|-h|--help");
+
+            _parentProcessIdOption = _app.Option(
+                "--parentProcessId",
+                "Used by IDEs to specify their process ID. Test will exit if the parent process does.",
+                CommandOptionType.SingleValue);
+            _portOption = _app.Option(
+                "--port",
+                "Used by IDEs to specify a port number to listen for a connection.",
+                CommandOptionType.SingleValue);
+            _configurationOption = _app.Option(
+                "-c|--configuration <CONFIGURATION>",
+                "Configuration under which to build",
+                CommandOptionType.SingleValue);
+            _outputOption = _app.Option(
+                "-o|--output <OUTPUT_DIR>",
+                "Directory in which to find the binaries to be run",
+                CommandOptionType.SingleValue);
+            _buildBasePath = _app.Option(
+                "-b|--build-base-path <OUTPUT_DIR>",
+                "Directory in which to find temporary outputs",
+                CommandOptionType.SingleValue);
+            _frameworkOption = _app.Option(
+                "-f|--framework <FRAMEWORK>",
+                "Looks for test binaries for a specific framework",
+                CommandOptionType.SingleValue);
+            _runtimeOption = _app.Option(
+                "-r|--runtime <RUNTIME_IDENTIFIER>",
+                "Look for test binaries for a for the specified runtime",
+                CommandOptionType.SingleValue);
+            _noBuildOption =
+                _app.Option("--no-build", "Do not build project before testing", CommandOptionType.NoValue);
+            _projectPath = _app.Argument(
+                "<PROJECT>",
+                "The project to test, defaults to the current directory. Can be a path to a project.json or a project directory.");            
+        }
+    }
+}

src/dotnet/commands/dotnet-test/IDotnetTestRunner.cs

@@ -0,0 +1,13 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using Microsoft.DotNet.ProjectModel;
+using Microsoft.DotNet.Cli.Utils;
+
+namespace Microsoft.DotNet.Tools.Test
+{
+    public interface IDotnetTestRunner
+    {
+        int RunTests(ProjectContext projectContext, DotnetTestParams dotnetTestParams);
+    }
+}

src/dotnet/commands/dotnet-test/Program.cs

@@ -18,187 +18,44 @@ namespace Microsoft.DotNet.Tools.Test
         {
             DebugHelper.HandleDebugSwitch(ref args);
 
-            var app = new CommandLineApplication(false)
+            var dotnetTestParams = new DotnetTestParams();
-            {
-                Name = "dotnet test",
-                FullName = ".NET Test Driver",
-                Description = "Test Driver for the .NET Platform"
-            };
 
-            app.HelpOption("-?|-h|--help");
+            dotnetTestParams.Parse(args);
-
-            var parentProcessIdOption = app.Option("--parentProcessId", "Used by IDEs to specify their process ID. Test will exit if the parent process does.", CommandOptionType.SingleValue);
-            var portOption = app.Option("--port", "Used by IDEs to specify a port number to listen for a connection.", CommandOptionType.SingleValue);
-            var configurationOption = app.Option("-c|--configuration <CONFIGURATION>", "Configuration under which to build", CommandOptionType.SingleValue);
-            var output = app.Option("-o|--output <OUTPUT_DIR>", "Directory in which to find the binaries to be run", CommandOptionType.SingleValue);
-            var projectPath = app.Argument("<PROJECT>", "The project to test, defaults to the current directory. Can be a path to a project.json or a project directory.");
-
-            app.OnExecute(() =>
-            {
-                try
-                {
-                    // Register for parent process's exit event
-                    if (parentProcessIdOption.HasValue())
-                    {
-                        int processId;
-
-                        if (!Int32.TryParse(parentProcessIdOption.Value(), out processId))
-                        {
-                            throw new InvalidOperationException($"Invalid process id '{parentProcessIdOption.Value()}'. Process id must be an integer.");
-                        }
-
-                        RegisterForParentProcessExit(processId);
-                    }
-
-                    var projectContexts = CreateProjectContexts(projectPath.Value);
-
-                    var projectContext = projectContexts.First();
-
-                    var testRunner = projectContext.ProjectFile.TestRunner;
-
-                    var configuration = configurationOption.Value() ?? Constants.DefaultConfiguration;
-
-                    var outputPath = output.Value();
-
-                    if (portOption.HasValue())
-                    {
-                        int port;
-
-                        if (!Int32.TryParse(portOption.Value(), out port))
-                        {
-                            throw new InvalidOperationException($"{portOption.Value()} is not a valid port number.");
-                        }
-
-                        return RunDesignTime(port, projectContext, testRunner, configuration, outputPath);
-                    }
-                    else
-                    {
-                        return RunConsole(projectContext, app, testRunner, configuration, outputPath);
-                    }
-                }
-                catch (InvalidOperationException ex)
-                {
-                    TestHostTracing.Source.TraceEvent(TraceEventType.Error, 0, ex.ToString());
-                    return -1;
-                }
-                catch (Exception ex)
-                {
-                    TestHostTracing.Source.TraceEvent(TraceEventType.Error, 0, ex.ToString());
-                    return -2;
-                }
-
-            });
-
-            return app.Execute(args);
-        }
-
-        private static int RunConsole(
-            ProjectContext projectContext,
-            CommandLineApplication app,
-            string testRunner,
-            string configuration,
-            string outputPath)
-        {
-            var commandArgs = new List<string> { GetAssemblyUnderTest(projectContext, configuration, outputPath) };
-            commandArgs.AddRange(app.RemainingArguments);
-
-            var commandFactory = 
-                new ProjectDependenciesCommandFactory(
-                    projectContext.TargetFramework, 
-                    configuration, 
-                    outputPath,
-                    projectContext.ProjectDirectory);
-            
-
-            return commandFactory.Create(
-                    $"dotnet-{GetCommandName(testRunner)}",
-                    commandArgs,
-                    projectContext.TargetFramework,
-                    configuration)
-                .ForwardStdErr()
-                .ForwardStdOut()
-                .Execute()
-                .ExitCode;
-        }
-
-        private static string GetAssemblyUnderTest(ProjectContext projectContext, string configuration, string outputPath)
-        {
-            var assemblyUnderTest =
-                projectContext.GetOutputPaths(configuration, outputPath: outputPath).CompilationFiles.Assembly;
-
-            if (!string.IsNullOrEmpty(outputPath))
-            {
-                assemblyUnderTest =
-                    projectContext.GetOutputPaths(configuration, outputPath: outputPath).RuntimeFiles.Assembly;
-            }
-
-            return assemblyUnderTest;
-        }
-
-        private static int RunDesignTime(
-            int port,
-            ProjectContext
-            projectContext,
-            string testRunner,
-            string configuration,
-            string outputPath)
-        {
-            Console.WriteLine("Listening on port {0}", port);
-
-            HandleDesignTimeMessages(projectContext, testRunner, port, configuration, outputPath);
-
-            return 0;
-        }
-
-        private static void HandleDesignTimeMessages(
-            ProjectContext projectContext,
-            string testRunner,
-            int port,
-            string configuration,
-            string outputPath)
-        {
-            var reportingChannelFactory = new ReportingChannelFactory();
-            var adapterChannel = reportingChannelFactory.CreateAdapterChannel(port);
 
             try
             {
-                var assemblyUnderTest = GetAssemblyUnderTest(projectContext, configuration, outputPath);
+                // Register for parent process's exit event
-                var messages = new TestMessagesCollection();
+                if (dotnetTestParams.ParentProcessId.HasValue)
-                using (var dotnetTest = new DotnetTest(messages, assemblyUnderTest))
                 {
-                    var commandFactory = 
+                    RegisterForParentProcessExit(dotnetTestParams.ParentProcessId.Value);
-                        new ProjectDependenciesCommandFactory(
-                            projectContext.TargetFramework, 
-                            configuration, 
-                            outputPath,
-                            projectContext.ProjectDirectory);
-                        
-                    var testRunnerFactory = new TestRunnerFactory(GetCommandName(testRunner), commandFactory);
-
-                    dotnetTest
-                        .AddNonSpecificMessageHandlers(messages, adapterChannel)
-                        .AddTestDiscoveryMessageHandlers(adapterChannel, reportingChannelFactory, testRunnerFactory)
-                        .AddTestRunMessageHandlers(adapterChannel, reportingChannelFactory, testRunnerFactory)
-                        .AddTestRunnnersMessageHandlers(adapterChannel, reportingChannelFactory);
-
-                    dotnetTest.StartListeningTo(adapterChannel);
-
-                    adapterChannel.Connect();
-
-                    dotnetTest.StartHandlingMessages();
                 }
+
+                var projectContexts = CreateProjectContexts(dotnetTestParams.ProjectPath);
+
+                var projectContext = projectContexts.First();
+
+                var testRunner = projectContext.ProjectFile.TestRunner;
+
+                IDotnetTestRunner dotnetTestRunner = new ConsoleTestRunner();
+                if (dotnetTestParams.Port.HasValue)
+                {
+                    dotnetTestRunner = new DesignTimeRunner();
+                }
+
+                return dotnetTestRunner.RunTests(projectContext, dotnetTestParams);
+            }
+            catch (InvalidOperationException ex)
+            {
+                TestHostTracing.Source.TraceEvent(TraceEventType.Error, 0, ex.ToString());
+                return -1;
             }
             catch (Exception ex)
             {
-                adapterChannel.SendError(ex);
+                TestHostTracing.Source.TraceEvent(TraceEventType.Error, 0, ex.ToString());
+                return -2;
             }
         }
 
-        private static string GetCommandName(string testRunner)
-        {
-            return $"test-{testRunner}";
-        }
-
         private static void RegisterForParentProcessExit(int id)
         {
             var parentProcess = Process.GetProcesses().FirstOrDefault(p => p.Id == id);

test/Microsoft.DotNet.Cli.Utils.Tests/GivenAProjectDependencyCommandResolver.cs

@@ -185,6 +185,64 @@ namespace Microsoft.DotNet.Cli.Utils.Tests
             result.Args.Should().Contain("--depsfile");
         }
 
+        [Fact]
+        public void It_sets_depsfile_based_on_output_path_when_returning_a_commandspec()
+        {
+            var projectDependenciesCommandResolver = SetupProjectDependenciesCommandResolver();
+
+            var commandResolverArguments = new CommandResolverArguments
+            {
+                CommandName = "dotnet-hello",
+                CommandArguments = null,
+                ProjectDirectory = s_liveProjectDirectory,
+                Configuration = "Debug",
+                Framework = FrameworkConstants.CommonFrameworks.NetStandardApp15,
+                OutputPath = AppContext.BaseDirectory
+            };
+
+            var projectContext = ProjectContext.Create(
+                s_liveProjectDirectory,
+                FrameworkConstants.CommonFrameworks.NetStandardApp15,
+                PlatformServices.Default.Runtime.GetAllCandidateRuntimeIdentifiers());
+
+            var depsFilePath =
+                projectContext.GetOutputPaths("Debug", outputPath: AppContext.BaseDirectory).RuntimeFiles.DepsJson;
+
+            var result = projectDependenciesCommandResolver.Resolve(commandResolverArguments);
+
+            result.Should().NotBeNull();
+            result.Args.Should().Contain($"--depsfile {depsFilePath}");
+        }
+
+        [Fact]
+        public void It_sets_depsfile_based_on_build_base_path_when_returning_a_commandspec()
+        {
+            var projectDependenciesCommandResolver = SetupProjectDependenciesCommandResolver();
+
+            var commandResolverArguments = new CommandResolverArguments
+            {
+                CommandName = "dotnet-hello",
+                CommandArguments = null,
+                ProjectDirectory = s_liveProjectDirectory,
+                Configuration = "Debug",
+                Framework = FrameworkConstants.CommonFrameworks.NetStandardApp15,
+                BuildBasePath = AppContext.BaseDirectory
+            };
+
+            var projectContext = ProjectContext.Create(
+                s_liveProjectDirectory,
+                FrameworkConstants.CommonFrameworks.NetStandardApp15,
+                PlatformServices.Default.Runtime.GetAllCandidateRuntimeIdentifiers());
+
+            var depsFilePath =
+                projectContext.GetOutputPaths("Debug", AppContext.BaseDirectory).RuntimeFiles.DepsJson;
+
+            var result = projectDependenciesCommandResolver.Resolve(commandResolverArguments);
+
+            result.Should().NotBeNull();
+            result.Args.Should().Contain($"--depsfile {depsFilePath}");
+        }
+
         [Fact]
         public void It_returns_a_CommandSpec_with_CommandName_in_Args_when_returning_a_CommandSpec_and_CommandArguments_are_null()
         {

test/Microsoft.DotNet.Cli.Utils.Tests/Microsoft.DotNet.Cli.Utils.Tests.xproj

@@ -14,5 +14,8 @@
   <PropertyGroup>
     <SchemaVersion>2.0</SchemaVersion>
   </PropertyGroup>
+  <ItemGroup>
+    <Service Include="{82a7f48d-3b50-4b1e-b82e-3ada8210c358}" />
+  </ItemGroup>
   <Import Project="$(VSToolsPath)\DNX\Microsoft.DNX.targets" Condition="'$(VSToolsPath)' != ''" />
 </Project>

test/dotnet-test.Tests/GivenThatWeWantToRunTestsInTheConsole.cs

@@ -0,0 +1,87 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
+using System.IO;
+using Microsoft.DotNet.ProjectModel;
+using Microsoft.Extensions.PlatformAbstractions;
+using Xunit;
+using Microsoft.DotNet.Tools.Test.Utilities;
+using System.Linq;
+
+namespace Microsoft.Dotnet.Tools.Test.Tests
+{
+    public class GivenThatWeWantToRunTestsInTheConsole : TestBase
+    {
+        private string _projectFilePath;
+        private string _defaultOutputPath;
+
+        public GivenThatWeWantToRunTestsInTheConsole()
+        {
+            var testInstance =
+                TestAssetsManager.CreateTestInstance("ProjectWithTests", identifier: "ConsoleTests").WithLockFiles();
+
+            _projectFilePath = Path.Combine(testInstance.TestRoot, "project.json");
+            var contexts = ProjectContext.CreateContextForEachFramework(
+                _projectFilePath,
+                null,
+                PlatformServices.Default.Runtime.GetAllCandidateRuntimeIdentifiers());
+
+            var runtime = contexts.FirstOrDefault(c => !string.IsNullOrEmpty(c.RuntimeIdentifier))?.RuntimeIdentifier;
+            _defaultOutputPath = Path.Combine(testInstance.TestRoot, "bin", "Debug", DefaultFramework, runtime);
+        }
+
+        //ISSUE https://github.com/dotnet/cli/issues/1935
+        [WindowsOnlyFact]
+        public void It_returns_a_failure_when_it_fails_to_run_the_tests()
+        {
+            var testCommand = new DotnetTestCommand();
+            var result = testCommand.Execute(
+                $"{_projectFilePath} -o {Path.Combine(AppContext.BaseDirectory, "nonExistingFolder")} --no-build");
+            result.Should().Fail();
+        }
+
+        [Fact]
+        public void It_builds_the_project_before_running()
+        {
+            var testCommand = new DotnetTestCommand();
+            var result = testCommand.Execute($"{_projectFilePath}");
+            result.Should().Pass();
+        }
+
+        [Fact]
+        public void It_builds_the_project_using_the_output_passed()
+        {
+            var testCommand = new DotnetTestCommand();
+            var result = testCommand.Execute($"{_projectFilePath} -o {AppContext.BaseDirectory} -f netstandardapp1.5");
+            result.Should().Pass();
+        }
+
+        [Fact]
+        public void It_builds_the_project_using_the_build_base_path_passed()
+        {
+            var buildBasePath = GetNotSoLongBuildBasePath();
+            var testCommand = new DotnetTestCommand();
+            var result = testCommand.Execute($"{_projectFilePath} -b {buildBasePath}");
+            result.Should().Pass();
+        }
+
+        [Fact]
+        public void It_skips_build_when_the_no_build_flag_is_passed()
+        {
+            var buildCommand = new BuildCommand(_projectFilePath);
+            var result = buildCommand.Execute();
+            result.Should().Pass();
+
+            var testCommand = new DotnetTestCommand();
+            result = testCommand.Execute($"{_projectFilePath} -o {_defaultOutputPath} --no-build");
+            result.Should().Pass();
+        }
+
+        private string GetNotSoLongBuildBasePath()
+        {
+            return Path.GetFullPath(
+                Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "..", "buildBasePathTest"));
+        }
+    }
+}

test/dotnet-test.Tests/GivenThatWeWantToUseDotnetTestE2EInDesignTime.cs

@@ -41,7 +41,7 @@ namespace Microsoft.Dotnet.Tools.Test.Tests
                 adapter.Listen();
 
                 var testCommand = new DotnetTestCommand();
-                var result = testCommand.Execute($"{_projectFilePath} -o {_outputPath} --port {adapter.Port}");
+                var result = testCommand.Execute($"{_projectFilePath} -o {_outputPath} --port {adapter.Port} --no-build");
                 result.Should().Pass();
 
                 adapter.Messages["TestSession.Connected"].Count.Should().Be(1);
@@ -58,7 +58,7 @@ namespace Microsoft.Dotnet.Tools.Test.Tests
                 adapter.Listen();
 
                 var testCommand = new DotnetTestCommand();
-                var result = testCommand.Execute($"{_projectFilePath} -o {_outputPath} --port {adapter.Port}");
+                var result = testCommand.Execute($"{_projectFilePath} -o {_outputPath} --port {adapter.Port} --no-build");
                 result.Should().Pass();
 
                 adapter.Messages["TestSession.Connected"].Count.Should().Be(1);

test/dotnet-test.Tests/dotnet-test.Tests.xproj

@@ -14,5 +14,8 @@
   <PropertyGroup>
     <SchemaVersion>2.0</SchemaVersion>
   </PropertyGroup>
+  <ItemGroup>
+    <Service Include="{82a7f48d-3b50-4b1e-b82e-3ada8210c358}" />
+  </ItemGroup>
   <Import Project="$(VSToolsPath)\DNX\Microsoft.DNX.targets" Condition="'$(VSToolsPath)' != ''" />
 </Project>

test/dotnet-test.UnitTests/GivenThatWeWantToParseArgumentsForDotnetTest.cs

@@ -0,0 +1,196 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
+using System.IO;
+using FluentAssertions;
+using Microsoft.DotNet.Tools.Test;
+using Xunit;
+
+namespace Microsoft.Dotnet.Tools.Test.Tests
+{
+    public class GivenThatWeWantToParseArgumentsForDotnetTest
+    {
+        private const string ProjectJson = "project.json";
+        private const string Framework = "netstandardapp1.5";
+        private const string Output = "some output";
+        private const string BuildBasePath = "some build base path";
+        private const string Config = "some config";
+        private const string Runtime = "some runtime";
+        private const int ParentProcessId = 1010;
+        private const int Port = 2314;
+
+        private DotnetTestParams _dotnetTestFullParams;
+        private DotnetTestParams _emptyDotnetTestParams;
+
+        public GivenThatWeWantToParseArgumentsForDotnetTest()
+        {
+            _dotnetTestFullParams = new DotnetTestParams();
+            _emptyDotnetTestParams = new DotnetTestParams();
+
+            _dotnetTestFullParams.Parse(new[]
+            {
+                ProjectJson,
+                "--parentProcessId", ParentProcessId.ToString(),
+                "--port", Port.ToString(),
+                "--framework", Framework,
+                "--output", Output,
+                "--build-base-path", BuildBasePath,
+                "--configuration", Config,
+                "--runtime", Runtime,
+                "--no-build",
+                "--additional-parameters", "additional-parameter-value"
+            });
+
+            _emptyDotnetTestParams.Parse(new string[] { });
+        }
+
+        [Fact]
+        public void It_sets_the_project_path_current_folder_if_one_is_not_passed_in()
+        {
+            _emptyDotnetTestParams.ProjectPath.Should().Be(Directory.GetCurrentDirectory());
+        }
+
+        [Fact]
+        public void It_sets_the_project_path_to_the_passed_value()
+        {
+            _dotnetTestFullParams.ProjectPath.Should().Be(ProjectJson);
+        }
+
+        [Fact]
+        public void It_throws_InvalidOperationException_if_an_invalid_parent_process_id_is_passed_to_it()
+        {
+            var dotnetTestParams = new DotnetTestParams();
+            const string invalidParentProcessId = "daddy";
+            Action action = () => dotnetTestParams.Parse(new [] { "--parentProcessId", invalidParentProcessId });
+
+            action
+                .ShouldThrow<InvalidOperationException>()
+                .WithMessage($"Invalid process id '{invalidParentProcessId}'. Process id must be an integer.");
+        }
+
+        [Fact]
+        public void It_converts_the_parent_process_id_to_int_when_a_valid_one_is_passed()
+        {
+            _dotnetTestFullParams.ParentProcessId.Should().Be(ParentProcessId);
+        }
+
+        [Fact]
+        public void It_does_not_set_parent_process_id_when_one_is_not_passed()
+        {
+            _emptyDotnetTestParams.ParentProcessId.Should().NotHaveValue();
+        }
+
+        [Fact]
+        public void It_throws_InvalidOperationException_if_an_invalid_port_is_passed_to_it()
+        {
+            var dotnetTestParams = new DotnetTestParams();
+            const string invalidPort = "door";
+            Action action = () => dotnetTestParams.Parse(new[] { "--port", invalidPort });
+
+            action
+                .ShouldThrow<InvalidOperationException>()
+                .WithMessage($"{invalidPort} is not a valid port number.");
+        }
+
+        [Fact]
+        public void It_converts_the_port_to_int_when_a_valid_one_is_passed()
+        {
+            _dotnetTestFullParams.Port.Should().Be(Port);
+        }
+
+        [Fact]
+        public void It_does_not_set_port_when_one_is_not_passed()
+        {
+            _emptyDotnetTestParams.Port.Should().NotHaveValue();
+        }
+
+        [Fact]
+        public void It_converts_the_framework_to_NugetFramework()
+        {
+            _dotnetTestFullParams.Framework.DotNetFrameworkName.Should().Be(".NETStandardApp,Version=v1.5");
+        }
+
+        [Fact]
+        public void It_does_not_set_framework_when_one_is_not_passed()
+        {
+            _emptyDotnetTestParams.Framework.Should().BeNull();
+        }
+
+        [Fact]
+        public void It_sets_the_framework_to_unsupported_when_an_invalid_framework_is_passed_in()
+        {
+            var dotnetTestParams = new DotnetTestParams();
+            dotnetTestParams.Parse(new[] { "--framework", "farm work" });
+
+            dotnetTestParams.Framework.DotNetFrameworkName.Should().Be("Unsupported,Version=v0.0");
+        }
+
+        [Fact]
+        public void It_sets_Output_when_one_is_passed_in()
+        {
+            _dotnetTestFullParams.Output.Should().Be(Output);
+        }
+
+        [Fact]
+        public void It_leaves_Output_null_when_one_is_not_passed_in()
+        {
+            _emptyDotnetTestParams.Output.Should().BeNull();
+        }
+
+        [Fact]
+        public void It_sets_BuildBasePath_when_one_is_passed_in()
+        {
+            _dotnetTestFullParams.BuildBasePath.Should().Be(BuildBasePath);
+        }
+
+        [Fact]
+        public void It_leaves_BuildBasePath_null_when_one_is_not_passed_in()
+        {
+            _emptyDotnetTestParams.BuildBasePath.Should().BeNull();
+        }
+
+        [Fact]
+        public void It_sets_Config_to_passed_in_value()
+        {
+            _dotnetTestFullParams.Config.Should().Be(Config);
+        }
+
+        [Fact]
+        public void It_sets_Config_to_Debug_when_one_is_not_passed_in()
+        {
+            _emptyDotnetTestParams.Config.Should().Be("Debug");
+        }
+
+        [Fact]
+        public void It_sets_Runtime_when_one_is_passed_in()
+        {
+            _dotnetTestFullParams.Runtime.Should().Be(Runtime);
+        }
+
+        [Fact]
+        public void It_leaves_Runtime_null_when_one_is_not_passed_in()
+        {
+            _emptyDotnetTestParams.Runtime.Should().BeNull();
+        }
+
+        [Fact]
+        public void It_sets_any_remaining_params_to_RemainingArguments()
+        {
+            _dotnetTestFullParams.RemainingArguments.ShouldBeEquivalentTo(
+                new [] { "--additional-parameters", "additional-parameter-value" });
+        }
+
+        [Fact]
+        public void It_sets_no_build_to_true_when_it_is_passed()
+        {
+            _dotnetTestFullParams.NoBuild.Should().BeTrue();
+        }
+
+        [Fact]
+        public void It_sets_no_build_to_false_when_it_is_not_passed_in()
+        {
+            _emptyDotnetTestParams.NoBuild.Should().BeFalse();
+        }
+    }
+}