Added build before running the tests and the no-build option to opt out of it.

Added new tests to ProjectDependencyCommandResolver and new E2E tests for dotnet test.

Microsoft.DotNet.Cli.sln

@@ -81,6 +81,8 @@ Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "dotnet-test.Tests", "test\d
 EndProject
 Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "update-dependencies", "scripts\update-dependencies\update-dependencies.xproj", "{A28BD8AC-DF15-4F58-8299-98A9AE2B8726}"
 EndProject
+Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "Microsoft.DotNet.Cli.Utils.Tests", "test\Microsoft.DotNet.Cli.Utils.Tests\Microsoft.DotNet.Cli.Utils.Tests.xproj", "{09C52F96-EFDD-4448-95EC-6D362DD60BAA}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -573,6 +575,22 @@ Global
 		{A28BD8AC-DF15-4F58-8299-98A9AE2B8726}.RelWithDebInfo|Any CPU.Build.0 = Release|Any CPU
 		{A28BD8AC-DF15-4F58-8299-98A9AE2B8726}.RelWithDebInfo|x64.ActiveCfg = Release|Any CPU
 		{A28BD8AC-DF15-4F58-8299-98A9AE2B8726}.RelWithDebInfo|x64.Build.0 = Release|Any CPU
+		{09C52F96-EFDD-4448-95EC-6D362DD60BAA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{09C52F96-EFDD-4448-95EC-6D362DD60BAA}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{09C52F96-EFDD-4448-95EC-6D362DD60BAA}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{09C52F96-EFDD-4448-95EC-6D362DD60BAA}.Debug|x64.Build.0 = Debug|Any CPU
+		{09C52F96-EFDD-4448-95EC-6D362DD60BAA}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
+		{09C52F96-EFDD-4448-95EC-6D362DD60BAA}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
+		{09C52F96-EFDD-4448-95EC-6D362DD60BAA}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
+		{09C52F96-EFDD-4448-95EC-6D362DD60BAA}.MinSizeRel|x64.Build.0 = Debug|Any CPU
+		{09C52F96-EFDD-4448-95EC-6D362DD60BAA}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{09C52F96-EFDD-4448-95EC-6D362DD60BAA}.Release|Any CPU.Build.0 = Release|Any CPU
+		{09C52F96-EFDD-4448-95EC-6D362DD60BAA}.Release|x64.ActiveCfg = Release|Any CPU
+		{09C52F96-EFDD-4448-95EC-6D362DD60BAA}.Release|x64.Build.0 = Release|Any CPU
+		{09C52F96-EFDD-4448-95EC-6D362DD60BAA}.RelWithDebInfo|Any CPU.ActiveCfg = Release|Any CPU
+		{09C52F96-EFDD-4448-95EC-6D362DD60BAA}.RelWithDebInfo|Any CPU.Build.0 = Release|Any CPU
+		{09C52F96-EFDD-4448-95EC-6D362DD60BAA}.RelWithDebInfo|x64.ActiveCfg = Release|Any CPU
+		{09C52F96-EFDD-4448-95EC-6D362DD60BAA}.RelWithDebInfo|x64.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -610,5 +628,6 @@ Global
 		{857274AC-E741-4266-A7FD-14DEE0C1CC96} = {17735A9D-BFD9-4585-A7CB-3208CA6EA8A7}
 		{60C33D0A-A5D8-4AB0-9956-1F804654DF05} = {17735A9D-BFD9-4585-A7CB-3208CA6EA8A7}
 		{A28BD8AC-DF15-4F58-8299-98A9AE2B8726} = {88278B81-7649-45DC-8A6A-D3A645C5AFC3}
+		{09C52F96-EFDD-4448-95EC-6D362DD60BAA} = {17735A9D-BFD9-4585-A7CB-3208CA6EA8A7}
 	EndGlobalSection
 EndGlobal

TestAssets/TestProjects/ProjectWithTests/GivenThatIWantSomeFakeTests.cs

@@ -14,9 +14,9 @@ namespace FakeTests
         }
         
         [Fact]
-        public void It_fails()
+        public void It_also_succeeds()
         {
-            Assert.True(false);
+            Assert.True(true);
         }
     }
 }

src/Microsoft.DotNet.Cli.Utils/CommandResolution/ProjectDependenciesCommandResolver.cs

@@ -78,7 +78,7 @@ namespace Microsoft.DotNet.Cli.Utils
             }
 
             var depsFilePath =
-                projectContext.GetOutputPaths(configuration, buildBasePath, outputPath).RuntimeFiles.Deps;
+                projectContext.GetOutputPaths(configuration, buildBasePath, outputPath).RuntimeFiles.DepsJson;
 
             var dependencyLibraries = GetAllDependencyLibraries(projectContext);
 

src/dotnet/commands/dotnet-test/AssemblyUnderTest.cs

@@ -21,15 +21,17 @@ namespace Microsoft.DotNet.Tools.Test
         {
             get
             {
-                var assemblyUnderTest = _projectContext.GetOutputPaths(
+                var outputPaths = _projectContext.GetOutputPaths(
                     _dotentTestParams.Config,
-                    outputPath: _dotentTestParams.Output).CompilationFiles.Assembly;
+                    _dotentTestParams.BuildBasePath,
+                    _dotentTestParams.Output);
 
-                if (!string.IsNullOrEmpty(_dotentTestParams.Output))
+                var assemblyUnderTest = outputPaths.CompilationFiles.Assembly;
+
+                if (!string.IsNullOrEmpty(_dotentTestParams.Output) ||
+                    !string.IsNullOrEmpty(_dotentTestParams.BuildBasePath))
                 {
-                    assemblyUnderTest = _projectContext.GetOutputPaths(
+                    assemblyUnderTest = outputPaths.RuntimeFiles.Assembly;
-                        _dotentTestParams.Config,
-                        outputPath: _dotentTestParams.Output).RuntimeFiles.Assembly;
                 }
 
                 return assemblyUnderTest;

src/dotnet/commands/dotnet-test/BaseDotnetTestRunner.cs

@@ -0,0 +1,68 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System.Collections.Generic;
+using Microsoft.DotNet.ProjectModel;
+
+namespace Microsoft.DotNet.Tools.Test
+{
+    public abstract class BaseDotnetTestRunner : IDotnetTestRunner
+    {
+        public int RunTests(ProjectContext projectContext, DotnetTestParams dotnetTestParams)
+        {
+            var result = BuildTestProject(dotnetTestParams);
+
+            return result == 0 ? DoRunTests(projectContext, dotnetTestParams) : result;
+        }
+
+        internal abstract int DoRunTests(ProjectContext projectContext, DotnetTestParams dotnetTestParams);
+
+        private int BuildTestProject(DotnetTestParams dotnetTestParams)
+        {
+            if (dotnetTestParams.NoBuild)
+            {
+                return 0;
+            }
+
+            return DoBuildTestProject(dotnetTestParams);
+        }
+
+        private int DoBuildTestProject(DotnetTestParams dotnetTestParams)
+        {
+            var strings = new List<string>
+            {
+                $"--configuration",
+                dotnetTestParams.Config,
+                $"{dotnetTestParams.ProjectPath}"
+            };
+
+            if (dotnetTestParams.Framework != null)
+            {
+                strings.Add("--framework");
+                strings.Add($"{dotnetTestParams.Framework}");
+            }
+
+            if (!string.IsNullOrEmpty(dotnetTestParams.BuildBasePath))
+            {
+                strings.Add("--build-base-path");
+                strings.Add(dotnetTestParams.BuildBasePath);
+            }
+
+            if (!string.IsNullOrEmpty(dotnetTestParams.Output))
+            {
+                strings.Add("--output");
+                strings.Add(dotnetTestParams.Output);
+            }
+
+            if (!string.IsNullOrEmpty(dotnetTestParams.Runtime))
+            {
+                strings.Add("--runtime");
+                strings.Add(dotnetTestParams.Runtime);
+            }
+
+            var result = Build.BuildCommand.Run(strings.ToArray());
+
+            return result;
+        }
+    }
+}

src/dotnet/commands/dotnet-test/ConsoleTestRunner.cs

@@ -7,9 +7,9 @@ using Microsoft.DotNet.ProjectModel;
 
 namespace Microsoft.DotNet.Tools.Test
 {
-    public class ConsoleTestRunner : IDotnetTestRunner
+    public class ConsoleTestRunner : BaseDotnetTestRunner
     {
-        public int RunTests(ProjectContext projectContext, DotnetTestParams dotnetTestParams)
+        internal override int DoRunTests(ProjectContext projectContext, DotnetTestParams dotnetTestParams)
         {
             var commandFactory =
                 new ProjectDependenciesCommandFactory(

src/dotnet/commands/dotnet-test/DesignTimeRunner.cs

@@ -7,9 +7,9 @@ using Microsoft.DotNet.ProjectModel;
 
 namespace Microsoft.DotNet.Tools.Test
 {
-    public class DesignTimeRunner : IDotnetTestRunner
+    public class DesignTimeRunner : BaseDotnetTestRunner
     {
-        public int RunTests(ProjectContext projectContext, DotnetTestParams dotnetTestParams)
+        internal override int DoRunTests(ProjectContext projectContext, DotnetTestParams dotnetTestParams)
         {
             Console.WriteLine("Listening on port {0}", dotnetTestParams.Port.Value);
 
@@ -50,7 +50,7 @@ namespace Microsoft.DotNet.Tools.Test
 
                     dotnetTest.StartListeningTo(adapterChannel);
 
-                    adapterChannel.Accept();
+                    adapterChannel.Connect();
 
                     dotnetTest.StartHandlingMessages();
                 }

src/dotnet/commands/dotnet-test/DotnetTestParams.cs

@@ -25,6 +25,7 @@ namespace Microsoft.DotNet.Tools.Test
         private CommandOption _portOption;
         private CommandOption _parentProcessIdOption;
         private CommandArgument _projectPath;
+        private CommandOption _noBuildOption;
 
         public int? Port { get; set; }
 
@@ -44,6 +45,8 @@ namespace Microsoft.DotNet.Tools.Test
 
         public List<string> RemainingArguments { get; set; }
 
+        public bool NoBuild { get; set; }
+
         public DotnetTestParams()
         {
             _app = new CommandLineApplication(false)
@@ -101,6 +104,7 @@ namespace Microsoft.DotNet.Tools.Test
                 BuildBasePath = _buildBasePath.Value();
                 Config = _configurationOption.Value() ?? Constants.DefaultConfiguration;
                 Runtime = _runtimeOption.Value();
+                NoBuild = _noBuildOption.HasValue();
 
                 RemainingArguments = _app.RemainingArguments;
 
@@ -142,6 +146,8 @@ namespace Microsoft.DotNet.Tools.Test
                 "-r|--runtime <RUNTIME_IDENTIFIER>",
                 "Look for test binaries for a for the specified runtime",
                 CommandOptionType.SingleValue);
+            _noBuildOption =
+                _app.Option("--no-build", "Do not build project before testing", CommandOptionType.NoValue);
             _projectPath = _app.Argument(
                 "<PROJECT>",
                 "The project to test, defaults to the current directory. Can be a path to a project.json or a project directory.");            

test/Microsoft.DotNet.Cli.Utils.Tests/GivenAProjectDependencyCommandResolver.cs

@@ -185,6 +185,64 @@ namespace Microsoft.DotNet.Cli.Utils.Tests
             result.Args.Should().Contain("--depsfile");
         }
 
+        [Fact]
+        public void It_sets_depsfile_based_on_output_path_when_returning_a_commandspec()
+        {
+            var projectDependenciesCommandResolver = SetupProjectDependenciesCommandResolver();
+
+            var commandResolverArguments = new CommandResolverArguments
+            {
+                CommandName = "dotnet-hello",
+                CommandArguments = null,
+                ProjectDirectory = s_liveProjectDirectory,
+                Configuration = "Debug",
+                Framework = FrameworkConstants.CommonFrameworks.NetStandardApp15,
+                OutputPath = AppContext.BaseDirectory
+            };
+
+            var projectContext = ProjectContext.Create(
+                s_liveProjectDirectory,
+                FrameworkConstants.CommonFrameworks.NetStandardApp15,
+                PlatformServices.Default.Runtime.GetAllCandidateRuntimeIdentifiers());
+
+            var depsFilePath =
+                projectContext.GetOutputPaths("Debug", outputPath: AppContext.BaseDirectory).RuntimeFiles.DepsJson;
+
+            var result = projectDependenciesCommandResolver.Resolve(commandResolverArguments);
+
+            result.Should().NotBeNull();
+            result.Args.Should().Contain($"--depsfile {depsFilePath}");
+        }
+
+        [Fact]
+        public void It_sets_depsfile_based_on_build_base_path_when_returning_a_commandspec()
+        {
+            var projectDependenciesCommandResolver = SetupProjectDependenciesCommandResolver();
+
+            var commandResolverArguments = new CommandResolverArguments
+            {
+                CommandName = "dotnet-hello",
+                CommandArguments = null,
+                ProjectDirectory = s_liveProjectDirectory,
+                Configuration = "Debug",
+                Framework = FrameworkConstants.CommonFrameworks.NetStandardApp15,
+                BuildBasePath = AppContext.BaseDirectory
+            };
+
+            var projectContext = ProjectContext.Create(
+                s_liveProjectDirectory,
+                FrameworkConstants.CommonFrameworks.NetStandardApp15,
+                PlatformServices.Default.Runtime.GetAllCandidateRuntimeIdentifiers());
+
+            var depsFilePath =
+                projectContext.GetOutputPaths("Debug", AppContext.BaseDirectory).RuntimeFiles.DepsJson;
+
+            var result = projectDependenciesCommandResolver.Resolve(commandResolverArguments);
+
+            result.Should().NotBeNull();
+            result.Args.Should().Contain($"--depsfile {depsFilePath}");
+        }
+
         [Fact]
         public void It_returns_a_CommandSpec_with_CommandName_in_Args_when_returning_a_CommandSpec_and_CommandArguments_are_null()
         {

test/Microsoft.DotNet.Cli.Utils.Tests/Microsoft.DotNet.Cli.Utils.Tests.xproj

@@ -14,5 +14,8 @@
   <PropertyGroup>
     <SchemaVersion>2.0</SchemaVersion>
   </PropertyGroup>
+  <ItemGroup>
+    <Service Include="{82a7f48d-3b50-4b1e-b82e-3ada8210c358}" />
+  </ItemGroup>
   <Import Project="$(VSToolsPath)\DNX\Microsoft.DNX.targets" Condition="'$(VSToolsPath)' != ''" />
 </Project>

test/dotnet-test.Tests/GivenThatWeWantToRunTestsInTheConsole.cs

@@ -0,0 +1,87 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
+using System.IO;
+using Microsoft.DotNet.ProjectModel;
+using Microsoft.Extensions.PlatformAbstractions;
+using Xunit;
+using Microsoft.DotNet.Tools.Test.Utilities;
+using System.Linq;
+
+namespace Microsoft.Dotnet.Tools.Test.Tests
+{
+    public class GivenThatWeWantToRunTestsInTheConsole : TestBase
+    {
+        private string _projectFilePath;
+        private string _defaultOutputPath;
+
+        public GivenThatWeWantToRunTestsInTheConsole()
+        {
+            var testInstance =
+                TestAssetsManager.CreateTestInstance("ProjectWithTests", identifier: "ConsoleTests").WithLockFiles();
+
+            _projectFilePath = Path.Combine(testInstance.TestRoot, "project.json");
+            var contexts = ProjectContext.CreateContextForEachFramework(
+                _projectFilePath,
+                null,
+                PlatformServices.Default.Runtime.GetAllCandidateRuntimeIdentifiers());
+
+            var runtime = contexts.FirstOrDefault(c => !string.IsNullOrEmpty(c.RuntimeIdentifier))?.RuntimeIdentifier;
+            _defaultOutputPath = Path.Combine(testInstance.TestRoot, "bin", "Debug", DefaultFramework, runtime);
+        }
+
+        //ISSUE https://github.com/dotnet/cli/issues/1935
+        [WindowsOnlyFact]
+        public void It_returns_a_failure_when_it_fails_to_run_the_tests()
+        {
+            var testCommand = new DotnetTestCommand();
+            var result = testCommand.Execute(
+                $"{_projectFilePath} -o {Path.Combine(AppContext.BaseDirectory, "nonExistingFolder")} --no-build");
+            result.Should().Fail();
+        }
+
+        [Fact]
+        public void It_builds_the_project_before_running()
+        {
+            var testCommand = new DotnetTestCommand();
+            var result = testCommand.Execute($"{_projectFilePath}");
+            result.Should().Pass();
+        }
+
+        [Fact]
+        public void It_builds_the_project_using_the_output_passed()
+        {
+            var testCommand = new DotnetTestCommand();
+            var result = testCommand.Execute($"{_projectFilePath} -o {AppContext.BaseDirectory} -f netstandardapp1.5");
+            result.Should().Pass();
+        }
+
+        [Fact]
+        public void It_builds_the_project_using_the_build_base_path_passed()
+        {
+            var buildBasePath = GetNotSoLongBuildBasePath();
+            var testCommand = new DotnetTestCommand();
+            var result = testCommand.Execute($"{_projectFilePath} -b {buildBasePath}");
+            result.Should().Pass();
+        }
+
+        [Fact]
+        public void It_skips_build_when_the_no_build_flag_is_passed()
+        {
+            var buildCommand = new BuildCommand(_projectFilePath);
+            var result = buildCommand.Execute();
+            result.Should().Pass();
+
+            var testCommand = new DotnetTestCommand();
+            result = testCommand.Execute($"{_projectFilePath} -o {_defaultOutputPath} --no-build");
+            result.Should().Pass();
+        }
+
+        private string GetNotSoLongBuildBasePath()
+        {
+            return Path.GetFullPath(
+                Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "..", "buildBasePathTest"));
+        }
+    }
+}

test/dotnet-test.Tests/GivenThatWeWantToUseDotnetTestE2EInDesignTime.cs

@@ -41,7 +41,7 @@ namespace Microsoft.Dotnet.Tools.Test.Tests
                 adapter.Listen();
 
                 var testCommand = new DotnetTestCommand();
-                var result = testCommand.Execute($"{_projectFilePath} -o {_outputPath} --port {adapter.Port}");
+                var result = testCommand.Execute($"{_projectFilePath} -o {_outputPath} --port {adapter.Port} --no-build");
                 result.Should().Pass();
 
                 adapter.Messages["TestSession.Connected"].Count.Should().Be(1);
@@ -58,7 +58,7 @@ namespace Microsoft.Dotnet.Tools.Test.Tests
                 adapter.Listen();
 
                 var testCommand = new DotnetTestCommand();
-                var result = testCommand.Execute($"{_projectFilePath} -o {_outputPath} --port {adapter.Port}");
+                var result = testCommand.Execute($"{_projectFilePath} -o {_outputPath} --port {adapter.Port} --no-build");
                 result.Should().Pass();
 
                 adapter.Messages["TestSession.Connected"].Count.Should().Be(1);

test/dotnet-test.Tests/dotnet-test.Tests.xproj

@@ -14,5 +14,8 @@
   <PropertyGroup>
     <SchemaVersion>2.0</SchemaVersion>
   </PropertyGroup>
+  <ItemGroup>
+    <Service Include="{82a7f48d-3b50-4b1e-b82e-3ada8210c358}" />
+  </ItemGroup>
   <Import Project="$(VSToolsPath)\DNX\Microsoft.DNX.targets" Condition="'$(VSToolsPath)' != ''" />
 </Project>

test/dotnet-test.UnitTests/GivenThatWeWantToParseArgumentsForDotnetTest.cs

@@ -38,6 +38,7 @@ namespace Microsoft.Dotnet.Tools.Test.Tests
                 "--build-base-path", BuildBasePath,
                 "--configuration", Config,
                 "--runtime", Runtime,
+                "--no-build",
                 "--additional-parameters", "additional-parameter-value"
             });
 
@@ -179,5 +180,17 @@ namespace Microsoft.Dotnet.Tools.Test.Tests
             _dotnetTestFullParams.RemainingArguments.ShouldBeEquivalentTo(
                 new [] { "--additional-parameters", "additional-parameter-value" });
         }
+
+        [Fact]
+        public void It_sets_no_build_to_true_when_it_is_passed()
+        {
+            _dotnetTestFullParams.NoBuild.Should().BeTrue();
+        }
+
+        [Fact]
+        public void It_sets_no_build_to_false_when_it_is_not_passed_in()
+        {
+            _emptyDotnetTestParams.NoBuild.Should().BeFalse();
+        }
     }
 }