Modify dotnet-test to run testRunner for all tfms

Fixes #2506

TestAssets/TestProjects/ProjectsWithTests/MultipleFrameworkProject/GivenThatIWantSomeFakeTests.cs

@@ -0,0 +1,55 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
+using Xunit;
+
+namespace FakeTests
+{
+    public class GivenThatIWantSomeFakeTests
+    {
+#if NET451
+        [Fact]
+        public void NET451_succeeds()
+        {
+            Assert.True(true);
+        }
+
+        [Fact(Skip="Skipped for NET451")]
+        public void SkippedTest()
+        {
+
+        }
+#else
+        [Fact]
+        public void NETCOREAPP_succeeds()
+        {
+            Assert.True(true);
+        }
+
+        [Fact(Skip="Skipped for NETCOREAPP1.0")]
+        public void SkippedTest()
+        {
+
+        }
+#endif
+
+        [Fact]
+        public void Common_succeeds()
+        {
+            Assert.True(true);
+        }
+
+        [Fact]
+        public void Fails_IfEnvironmentVariableIsSet()
+        {
+            var shouldFail = string.IsNullOrEmpty(Environment.GetEnvironmentVariable("DOTNET_TEST_SHOULD_FAIL"));
+
+#if NET451
+            Assert.True(shouldFail, "Failing in NET451");
+#else
+            Assert.True(shouldFail, "Failing in NETCOREAPP1.0");
+#endif
+        }
+    }
+}

TestAssets/TestProjects/ProjectsWithTests/MultipleFrameworkProject/project.json

@@ -0,0 +1,27 @@
+{
+  "version": "1.0.0-*",
+  "dependencies": {
+    "dotnet-test-xunit": "1.0.0-rc2-162081-13",
+    "Microsoft.NETCore.Platforms": "1.0.1-rc2-*",
+    "xunit": "2.1.0"
+  },
+  "frameworks": {
+    "netcoreapp1.0": {
+      "imports": [
+        "dotnet",
+        "portable-dnxcore50+net45+win8+wp8+wpa81",
+        "portable-net45+win8"
+      ],
+      "dependencies": {
+        "Microsoft.NETCore.App": {
+          "type": "platform",
+          "version": "1.0.0-rc2-*"
+        },
+        "System.Linq.Expressions": "4.0.11-rc2-24022",
+        "System.Runtime.Serialization.Primitives": "4.1.1-rc2-24022"
+      }
+    },
+    "net451": { }
+  },
+  "testRunner": "xunit"
+}

TestAssets/TestProjects/ProjectsWithTests/NetCoreAppOnlyProject/GivenThatIWantSomeFakeTests.cs

@@ -12,7 +12,7 @@ namespace FakeTests
         {
             Assert.True(true);
         }
-        
+
         [Fact]
         public void It_also_succeeds()
         {

TestAssets/TestProjects/ProjectsWithTests/NetCoreAppOnlyProject/project.json

@@ -8,7 +8,7 @@
     "System.Linq.Expressions": "4.0.11-rc2-24022",
     "System.Runtime.Serialization.Primitives": "4.1.1-rc2-24022",
     "xunit": "2.1.0",
-    "dotnet-test-xunit": "1.0.0-dev-140469-38"
+    "dotnet-test-xunit": "1.0.0-rc2-162081-13"
   },
   "frameworks": {
     "netcoreapp1.0": {

scripts/dotnet-cli-build/TestTargets.cs

@@ -74,8 +74,8 @@ namespace Microsoft.DotNet.Cli.Build
             CleanNuGetTempCache();
 
             var dotnet = DotNetCli.Stage2;
-            dotnet.Restore("--verbosity", "verbose", 
+            dotnet.Restore("--verbosity", "verbose",
-                "--infer-runtimes", 
+                "--infer-runtimes",
                 "--fallbacksource", Dirs.Corehost,
                 "--fallbacksource", Dirs.CorehostDummyPackages)
                 .WorkingDirectory(Path.Combine(c.BuildContext.BuildDirectory, "TestAssets", "TestPackages"))

src/dotnet/commands/dotnet-test/BaseDotnetTestRunner.cs

@@ -10,24 +10,24 @@ namespace Microsoft.DotNet.Tools.Test
     {
         public int RunTests(ProjectContext projectContext, DotnetTestParams dotnetTestParams)
         {
-            var result = BuildTestProject(dotnetTestParams);
+            var result = BuildTestProject(projectContext, dotnetTestParams);
 
             return result == 0 ? DoRunTests(projectContext, dotnetTestParams) : result;
         }
 
         internal abstract int DoRunTests(ProjectContext projectContext, DotnetTestParams dotnetTestParams);
 
-        private int BuildTestProject(DotnetTestParams dotnetTestParams)
+        private int BuildTestProject(ProjectContext projectContext, DotnetTestParams dotnetTestParams)
         {
             if (dotnetTestParams.NoBuild)
             {
                 return 0;
             }
 
-            return DoBuildTestProject(dotnetTestParams);
+            return DoBuildTestProject(projectContext, dotnetTestParams);
         }
 
-        private int DoBuildTestProject(DotnetTestParams dotnetTestParams)
+        private int DoBuildTestProject(ProjectContext projectContext, DotnetTestParams dotnetTestParams)
         {
             var strings = new List<string>
             {
@@ -36,11 +36,10 @@ namespace Microsoft.DotNet.Tools.Test
                 $"{dotnetTestParams.ProjectPath}"
             };
 
-            if (dotnetTestParams.Framework != null)
+            // Build the test specifically for the target framework \ rid of the ProjectContext. This avoids building the project
-            {
+            // for tfms that the user did not request.
-                strings.Add("--framework");
+            strings.Add("--framework");
-                strings.Add($"{dotnetTestParams.Framework}");
+            strings.Add(projectContext.TargetFramework.ToString());
-            }
 
             if (!string.IsNullOrEmpty(dotnetTestParams.BuildBasePath))
             {
@@ -54,10 +53,10 @@ namespace Microsoft.DotNet.Tools.Test
                 strings.Add(dotnetTestParams.Output);
             }
 
-            if (!string.IsNullOrEmpty(dotnetTestParams.Runtime))
+            if (!string.IsNullOrEmpty(projectContext.RuntimeIdentifier))
             {
                 strings.Add("--runtime");
-                strings.Add(dotnetTestParams.Runtime);
+                strings.Add(projectContext.RuntimeIdentifier);
             }
 
             var result = Build.BuildCommand.Run(strings.ToArray());

src/dotnet/commands/dotnet-test/Program.cs

@@ -2,7 +2,6 @@
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
 using System;
-using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
 using System.Linq;
@@ -42,18 +41,52 @@ namespace Microsoft.DotNet.Tools.Test
                     RegisterForParentProcessExit(dotnetTestParams.ParentProcessId.Value);
                 }
 
+                var projectPath = GetProjectPath(dotnetTestParams.ProjectPath);
+                var runtimeIdentifiers = !string.IsNullOrEmpty(dotnetTestParams.Runtime) ?
+                    new[] { dotnetTestParams.Runtime } :
+                    PlatformServices.Default.Runtime.GetAllCandidateRuntimeIdentifiers();
+                var exitCode = 0;
+
                 // Create a workspace
                 var workspace = WorkspaceContext.Create(ProjectReaderSettings.ReadFromEnvironment(), designTime: false);
 
-                var projectContexts = CreateProjectContexts(workspace, dotnetTestParams.ProjectPath, dotnetTestParams.Runtime);
+                if (dotnetTestParams.Framework != null)
+                {
+                    var projectContext = ProjectContext.Create(projectPath, dotnetTestParams.Framework, runtimeIdentifiers);
+                    exitCode = RunTest(projectContext, dotnetTestParams);
+                }
+                else
+                {
+                    var summary = new Summary();
+                    var projectContexts = workspace
+                        .GetProjectContextCollection(projectPath).FrameworkOnlyContexts
+                        .Select(c => workspace.GetRuntimeContext(c, runtimeIdentifiers))
+                        .ToList();
 
-                var projectContext = projectContexts.First();
+                    // Execute for all TFMs the project targets.
+                    foreach (var projectContext in projectContexts)
+                    {
+                        var result = RunTest(projectContext, dotnetTestParams);
+                        if (result == 0)
+                        {
+                            summary.Passed++;
+                        }
+                        else
+                        {
+                            summary.Failed++;
+                            if (exitCode == 0)
+                            {
+                                // If tests fail in more than one TFM, we'll have it use the result of the first one
+                                // as the exit code.
+                                exitCode = result;
+                            }
+                        }
+                    }
 
-                var testRunner = projectContext.ProjectFile.TestRunner;
+                    summary.Print();
+                }
 
-                IDotnetTestRunner dotnetTestRunner = _dotnetTestRunnerFactory.Create(dotnetTestParams.Port);
+                return exitCode;
-
-                return dotnetTestRunner.RunTests(projectContext, dotnetTestParams);
             }
             catch (InvalidOperationException ex)
             {
@@ -101,7 +134,14 @@ namespace Microsoft.DotNet.Tools.Test
             }
         }
 
-        private static IEnumerable<ProjectContext> CreateProjectContexts(WorkspaceContext workspace, string projectPath, string runtime)
+        private int RunTest(ProjectContext projectContext, DotnetTestParams dotnetTestParams)
+        {
+            var testRunner = projectContext.ProjectFile.TestRunner;
+            var dotnetTestRunner = _dotnetTestRunnerFactory.Create(dotnetTestParams.Port);
+            return dotnetTestRunner.RunTests(projectContext, dotnetTestParams);
+        }
+
+        private static string GetProjectPath(string projectPath)
         {
             projectPath = projectPath ?? Directory.GetCurrentDirectory();
 
@@ -115,12 +155,29 @@ namespace Microsoft.DotNet.Tools.Test
                 throw new InvalidOperationException($"{projectPath} does not exist.");
             }
 
-            var runtimeIdentifiers = !string.IsNullOrEmpty(runtime) ?
+            return projectPath;
-                new[] { runtime } :
+        }
-                PlatformServices.Default.Runtime.GetAllCandidateRuntimeIdentifiers();
 
-            var contexts = workspace.GetProjectContextCollection(projectPath).FrameworkOnlyContexts;
+        private class Summary
-            return contexts.Select(c => workspace.GetRuntimeContext(c, runtimeIdentifiers));
+        {
+            public int Passed { get; set; }
+
+            public int Failed { get; set; }
+
+            public int Total => Passed + Failed;
+
+            public void Print()
+            {
+                var summaryMessage = $"SUMMARY: Total: {Total} targets, Passed: {Passed}, Failed: {Failed}.";
+                if (Failed > 0)
+                {
+                    Reporter.Error.WriteLine(summaryMessage.Red());
+                }
+                else
+                {
+                    Reporter.Output.WriteLine(summaryMessage);
+                }
+            }
         }
     }
 }

test/ArgumentForwardingTests/project.json

@@ -19,7 +19,7 @@
       "target": "project"
     },
     "xunit": "2.1.0",
-    "dotnet-test-xunit": "1.0.0-dev-140469-38"
+    "dotnet-test-xunit": "1.0.0-rc2-162081-13"
   },
   "frameworks": {
     "netcoreapp1.0": {

test/EndToEnd/project.json

@@ -20,7 +20,7 @@
     },
     "xunit": "2.1.0",
     "xunit.netcore.extensions": "1.0.0-prerelease-00206",
-    "dotnet-test-xunit": "1.0.0-dev-140469-38"
+    "dotnet-test-xunit": "1.0.0-rc2-162081-13"
   },
   "frameworks": {
     "netcoreapp1.0": {

test/Kestrel.Tests/project.json

@@ -13,7 +13,7 @@
       "target": "project"
     },
     "xunit": "2.1.0",
-    "dotnet-test-xunit": "1.0.0-dev-140469-38"
+    "dotnet-test-xunit": "1.0.0-rc2-162081-13"
   },
   "frameworks": {
     "netcoreapp1.0": {

test/Microsoft.DotNet.Cli.Utils.Tests/project.json

@@ -25,7 +25,7 @@
     },
     "moq.netcore": "4.4.0-beta8",
     "xunit": "2.1.0",
-    "dotnet-test-xunit": "1.0.0-dev-140469-38"
+    "dotnet-test-xunit": "1.0.0-rc2-162081-13"
   },
   "frameworks": {
     "netcoreapp1.0": {

test/Microsoft.DotNet.Compiler.Common.Tests/project.json

@@ -16,7 +16,7 @@
       "target": "project"
     },
     "xunit": "2.1.0",
-    "dotnet-test-xunit": "1.0.0-dev-140469-38"
+    "dotnet-test-xunit": "1.0.0-rc2-162081-13"
   },
   "frameworks": {
     "netcoreapp1.0": {

test/Microsoft.DotNet.ProjectModel.Tests/project.json

@@ -16,7 +16,7 @@
       "target": "project"
     },
     "xunit": "2.1.0",
-    "dotnet-test-xunit": "1.0.0-dev-140469-38"
+    "dotnet-test-xunit": "1.0.0-rc2-162081-13"
   },
   "frameworks": {
     "netcoreapp1.0": {

test/Microsoft.DotNet.Tools.Tests.Utilities/project.json

@@ -14,7 +14,7 @@
     "System.Net.NetworkInformation": "4.1.0-rc2-24022",
     "FluentAssertions": "4.0.0",
     "xunit": "2.1.0",
-    "dotnet-test-xunit": "1.0.0-dev-140469-38",
+    "dotnet-test-xunit": "1.0.0-rc2-162081-13",
     "Microsoft.DotNet.TestFramework": "1.0.0-*",
     "Microsoft.DotNet.Cli.Utils": "1.0.0-*",
     "Microsoft.DotNet.ProjectModel": "1.0.0-*",

test/Microsoft.Extensions.DependencyModel.Tests/project.json

@@ -19,7 +19,7 @@
     "FluentAssertions": "4.0.0",
     "moq.netcore": "4.4.0-beta8",
     "xunit": "2.1.0",
-    "dotnet-test-xunit": "1.0.0-dev-140469-38"
+    "dotnet-test-xunit": "1.0.0-rc2-162081-13"
   },
   "frameworks": {
     "netcoreapp1.0": {

test/ScriptExecutorTests/project.json

@@ -15,7 +15,7 @@
       "target": "project"
     },
     "xunit": "2.1.0",
-    "dotnet-test-xunit": "1.0.0-dev-140469-38"
+    "dotnet-test-xunit": "1.0.0-rc2-162081-13"
   },
   "frameworks": {
     "netcoreapp1.0": {

test/TestingAbstractions/Microsoft.Extensions.Testing.Abstractions.Tests/project.json

@@ -19,7 +19,7 @@
       "target": "project"
     },
     "xunit": "2.1.0",
-    "dotnet-test-xunit": "1.0.0-dev-140469-38",
+    "dotnet-test-xunit": "1.0.0-rc2-162081-13",
     "FluentAssertions": "4.2.2"
   },
   "frameworks": {

test/TestingAbstractions/Microsoft.Extensions.Testing.Abstractions.UnitTests/project.json

@@ -14,7 +14,7 @@
       "target": "project"
     },
     "xunit": "2.1.0",
-    "dotnet-test-xunit": "1.0.0-dev-140469-38",
+    "dotnet-test-xunit": "1.0.0-rc2-162081-13",
     "FluentAssertions": "4.2.2",
     "moq.netcore": "4.4.0-beta8"
   },

test/dotnet-build.Tests/project.json

@@ -14,7 +14,7 @@
     },
     "Newtonsoft.Json": "7.0.1",
     "xunit": "2.1.0",
-    "dotnet-test-xunit": "1.0.0-dev-140469-38"
+    "dotnet-test-xunit": "1.0.0-rc2-162081-13"
   },
   "frameworks": {
     "netcoreapp1.0": {

test/dotnet-compile-fsc.Tests/project.json

@@ -12,7 +12,7 @@
       "target": "project"
     },
     "xunit": "2.1.0",
-    "dotnet-test-xunit": "1.0.0-dev-140469-38"
+    "dotnet-test-xunit": "1.0.0-rc2-162081-13"
   },
   "frameworks": {
     "netcoreapp1.0": {

test/dotnet-compile.Tests/project.json

@@ -13,7 +13,7 @@
       "target": "project"
     },
     "xunit": "2.1.0",
-    "dotnet-test-xunit": "1.0.0-dev-140469-38"
+    "dotnet-test-xunit": "1.0.0-rc2-162081-13"
   },
   "frameworks": {
     "netcoreapp1.0": {

test/dotnet-compile.UnitTests/project.json

@@ -25,7 +25,7 @@
       "target": "project"
     },
     "xunit": "2.1.0",
-    "dotnet-test-xunit": "1.0.0-dev-140469-38",
+    "dotnet-test-xunit": "1.0.0-rc2-162081-13",
     "moq.netcore": "4.4.0-beta8",
     "FluentAssertions": "4.2.2"
   },

test/dotnet-pack.Tests/project.json

@@ -14,7 +14,7 @@
       "target": "project"
     },
     "xunit": "2.1.0",
-    "dotnet-test-xunit": "1.0.0-dev-140469-38"
+    "dotnet-test-xunit": "1.0.0-rc2-162081-13"
   },
   "frameworks": {
     "netcoreapp1.0": {

test/dotnet-projectmodel-server.Tests/project.json

@@ -18,7 +18,7 @@
       "target": "project"
     },
     "xunit": "2.1.0",
-    "dotnet-test-xunit": "1.0.0-dev-140469-38",
+    "dotnet-test-xunit": "1.0.0-rc2-162081-13",
     "System.Net.NameResolution": "4.0.0-rc2-24022"
   },
   "frameworks": {

test/dotnet-publish.Tests/project.json

@@ -12,7 +12,7 @@
     },
     "xunit": "2.1.0",
     "xunit.netcore.extensions": "1.0.0-prerelease-00206",
-    "dotnet-test-xunit": "1.0.0-dev-140469-38",
+    "dotnet-test-xunit": "1.0.0-rc2-162081-13",
     "System.Runtime.InteropServices.RuntimeInformation": "4.0.0-rc2-24022"
   },
   "frameworks": {

test/dotnet-resgen.Tests/project.json

@@ -14,7 +14,7 @@
     },
     "xunit": "2.1.0",
     "xunit.netcore.extensions": "1.0.0-prerelease-00206",
-    "dotnet-test-xunit": "1.0.0-dev-140469-38"
+    "dotnet-test-xunit": "1.0.0-rc2-162081-13"
   },
   "frameworks": {
     "netcoreapp1.0": {

test/dotnet-run.Tests/project.json

@@ -13,7 +13,7 @@
       "target": "project"
     },
     "xunit": "2.1.0",
-    "dotnet-test-xunit": "1.0.0-dev-140469-38"
+    "dotnet-test-xunit": "1.0.0-rc2-162081-13"
   },
   "frameworks": {
     "netcoreapp1.0": {

test/dotnet-test.Tests/AssemblyInfo.cs

@@ -0,0 +1,4 @@
+using Xunit;
+
+// Don't let the tests execute concurrently
+[assembly: CollectionBehavior(CollectionBehavior.CollectionPerAssembly)]

test/dotnet-test.Tests/GivenThatWeWantToRunTestsForMultipleTFMsInTheConsole.cs

@@ -0,0 +1,155 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
+using System.IO;
+using System.Linq;
+using FluentAssertions;
+using Microsoft.DotNet.ProjectModel;
+using Microsoft.DotNet.TestFramework;
+using Microsoft.DotNet.Tools.Test.Utilities;
+using Microsoft.Extensions.PlatformAbstractions;
+using Xunit;
+
+namespace Microsoft.Dotnet.Tools.Test.Tests
+{
+    public class GivenThatWeWantToRunTestsForMultipleTFMsInTheConsole : TestBase
+    {
+        private readonly string _projectFilePath;
+        private readonly string _defaultNetCoreAppOutputPath;
+        private readonly string _defaultNet451OutputPath;
+
+        public GivenThatWeWantToRunTestsForMultipleTFMsInTheConsole()
+        {
+            var testInstance =
+                TestAssetsManager.CreateTestInstance(Path.Combine("ProjectsWithTests", "MultipleFrameworkProject"), identifier: "ConsoleTests");
+
+            _projectFilePath = Path.Combine(testInstance.TestRoot, "project.json");
+            var contexts = ProjectContext.CreateContextForEachFramework(
+                _projectFilePath,
+                null,
+                PlatformServices.Default.Runtime.GetAllCandidateRuntimeIdentifiers());
+
+            // Restore the project again in the destination to resolve projects
+            // Since the lock file has project relative paths in it, those will be broken
+            // unless we re-restore
+            new RestoreCommand() { WorkingDirectory = testInstance.TestRoot }.Execute().Should().Pass();
+
+            _defaultNetCoreAppOutputPath = Path.Combine(testInstance.TestRoot, "bin", "Debug", "netcoreapp1.0");
+            _defaultNet451OutputPath = Path.Combine(testInstance.TestRoot, "bin", "Debug", "net451", PlatformServices.Default.Runtime.GetAllCandidateRuntimeIdentifiers().First());
+        }
+
+        [WindowsOnlyFact]
+        public void It_builds_and_runs_tests_for_all_frameworks()
+        {
+            var testCommand = new DotnetTestCommand();
+            var result = testCommand
+                .ExecuteWithCapturedOutput($"test {_projectFilePath}");
+            result.Should().Pass();
+            result.StdOut.Should().Contain("Skipped for NET451");
+            result.StdOut.Should().Contain("Skipped for NETCOREAPP1.0");
+        }
+
+        [WindowsOnlyFact]
+        public void It_builds_and_runs_tests_for_net451()
+        {
+            var testCommand = new DotnetTestCommand();
+            var result = testCommand
+                .ExecuteWithCapturedOutput($"test {_projectFilePath} -f net451");
+            result.Should().Pass();
+            result.StdOut.Should().Contain($"Skipped for NET451");
+            result.StdOut.Should().NotContain($"Skipped for NETCOREAPP1.0");
+        }
+
+        [Fact]
+        public void It_builds_and_runs_tests_for_netcoreapp10()
+        {
+            var testCommand = new DotnetTestCommand();
+            var result = testCommand
+                .ExecuteWithCapturedOutput($"test {_projectFilePath} -f netcoreapp1.0");
+            result.Should().Pass();
+            result.StdOut.Should().Contain($"Skipped for NETCOREAPP1.0");
+            result.StdOut.Should().NotContain($"Skipped for NET451");
+        }
+
+        [Fact]
+        public void It_builds_the_project_using_the_output_passed()
+        {
+            var testCommand = new DotnetTestCommand();
+            var result = testCommand.Execute(
+                $"{_projectFilePath} -o {Path.Combine(AppContext.BaseDirectory, "output")} -f netcoreapp1.0");
+            result.Should().Pass();
+        }
+
+        [WindowsOnlyFact]
+        public void It_builds_the_project_using_the_build_base_path_passed()
+        {
+            var buildBasePath = GetNotSoLongBuildBasePath();
+            var testCommand = new DotnetTestCommand();
+            var result = testCommand.Execute($"{_projectFilePath} -b {buildBasePath}");
+            result.Should().Pass();
+        }
+
+        [Fact]
+        public void It_skips_build_when_the_no_build_flag_is_passed_for_netcoreapp10()
+        {
+            var buildCommand = new BuildCommand(_projectFilePath);
+            var result = buildCommand.Execute($"-f netcoreapp1.0 -o {_defaultNetCoreAppOutputPath}");
+            result.Should().Pass();
+
+            var testCommand = new DotnetTestCommand();
+            result = testCommand.Execute($"{_projectFilePath} -f netcoreapp10 -o {_defaultNetCoreAppOutputPath} --no-build");
+            result.Should().Pass();
+        }
+
+        [WindowsOnlyFact]
+        public void It_skips_build_when_the_no_build_flag_is_passed_for_net451()
+        {
+            var rid = PlatformServices.Default.Runtime.GetAllCandidateRuntimeIdentifiers().First();
+            var buildCommand = new BuildCommand(_projectFilePath);
+            var result = buildCommand.Execute($"-f net451 -r {rid} -o {_defaultNet451OutputPath}");
+            result.Should().Pass();
+
+            var testCommand = new DotnetTestCommand();
+            result = testCommand.Execute($"{_projectFilePath} -f net451 -r {rid} -o {_defaultNet451OutputPath} --no-build");
+            result.Should().Pass();
+        }
+
+        [Fact]
+        public void It_prints_error_when_no_framework_matched()
+        {
+            var nonExistentFramework = "doesnotexisttfm99.99";
+            var testCommand = new DotnetTestCommand();
+            var result = testCommand
+                .ExecuteWithCapturedOutput($"test {_projectFilePath} -f {nonExistentFramework}");
+
+            result.Should().Fail();
+            result.StdErr.Should().Contain($"does not support framework");
+        }
+
+        [WindowsOnlyFact]
+        public void It_runs_tests_for_all_tfms_if_they_fail()
+        {
+            var testCommand = new DotnetTestCommand
+            {
+                Environment =
+                {
+                    { "DOTNET_TEST_SHOULD_FAIL", "1" }
+                }
+            };
+
+            var result = testCommand
+                .ExecuteWithCapturedOutput($"test {_projectFilePath}");
+
+            result.Should().Fail();
+            result.StdOut.Should().Contain("Failing in NET451");
+            result.StdOut.Should().Contain("Failing in NETCOREAPP1.0");
+        }
+
+        private string GetNotSoLongBuildBasePath()
+        {
+            return Path.GetFullPath(
+                Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "..", "buildBasePathTest"));
+        }
+    }
+}

test/dotnet-test.Tests/GivenThatWeWantToRunTestsInTheConsole.cs

@@ -20,7 +20,7 @@ namespace Microsoft.Dotnet.Tools.Test.Tests
         public GivenThatWeWantToRunTestsInTheConsole()
         {
             var testInstance =
-                TestAssetsManager.CreateTestInstance("ProjectWithTests", identifier: "ConsoleTests");
+                TestAssetsManager.CreateTestInstance(Path.Combine("ProjectsWithTests", "NetCoreAppOnlyProject"), identifier: "ConsoleTests");
 
             _projectFilePath = Path.Combine(testInstance.TestRoot, "project.json");
             var contexts = ProjectContext.CreateContextForEachFramework(

test/dotnet-test.Tests/GivenThatWeWantToUseDotnetTestE2EInDesignTime.cs

@@ -19,7 +19,7 @@ namespace Microsoft.Dotnet.Tools.Test.Tests
 
         public GivenThatWeWantToUseDotnetTestE2EInDesignTime()
         {
-            var testInstance = TestAssetsManager.CreateTestInstance("ProjectWithTests");
+            var testInstance = TestAssetsManager.CreateTestInstance(Path.Combine("ProjectsWithTests", "NetCoreAppOnlyProject"));
 
             _projectFilePath = Path.Combine(testInstance.TestRoot, "project.json");
             var contexts = ProjectContext.CreateContextForEachFramework(

test/dotnet-test.Tests/GivenThatWeWantToUseDotnetTestE2EInDesignTimeForMultipleTFms.cs

@@ -0,0 +1,125 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System.IO;
+using System.Linq;
+using Microsoft.DotNet.ProjectModel;
+using Microsoft.DotNet.TestFramework;
+using Microsoft.DotNet.Tools.Test.Utilities;
+using Microsoft.Extensions.PlatformAbstractions;
+using FluentAssertions;
+using Xunit;
+
+namespace Microsoft.Dotnet.Tools.Test.Tests
+{
+    public class GivenThatWeWantToUseDotnetTestE2EInDesignTimeForMultipleTFms : TestBase
+    {
+        private readonly string _projectFilePath;
+        private readonly string _netCoreAppOutputPath;
+        private readonly string _net451OutputPath;
+
+        public GivenThatWeWantToUseDotnetTestE2EInDesignTimeForMultipleTFms()
+        {
+            var testInstance = TestAssetsManager.CreateTestInstance(Path.Combine("ProjectsWithTests", "MultipleFrameworkProject"));
+
+            _projectFilePath = Path.Combine(testInstance.TestRoot, "project.json");
+            var contexts = ProjectContext.CreateContextForEachFramework(
+                _projectFilePath,
+                null,
+                PlatformServices.Default.Runtime.GetAllCandidateRuntimeIdentifiers());
+
+            // Restore the project again in the destination to resolve projects
+            // Since the lock file has project relative paths in it, those will be broken
+            // unless we re-restore
+            new RestoreCommand() { WorkingDirectory = testInstance.TestRoot }.Execute().Should().Pass();
+
+            _netCoreAppOutputPath = Path.Combine(testInstance.TestRoot, "bin", "Debug", "netcoreapp1.0");
+            var buildCommand = new BuildCommand(_projectFilePath);
+            var result = buildCommand.Execute($"-f netcoreapp1.0 -o {_netCoreAppOutputPath}");
+
+            result.Should().Pass();
+
+            if (PlatformServices.Default.Runtime.OperatingSystemPlatform == Platform.Windows)
+            {
+                var rid = PlatformServices.Default.Runtime.GetAllCandidateRuntimeIdentifiers().First();
+                _net451OutputPath = Path.Combine(testInstance.TestRoot, "bin", "Debug", "net451", rid);
+                result = buildCommand.Execute($"-f net451 -r {rid} -o {_net451OutputPath}");
+                result.Should().Pass();
+            }
+        }
+
+        [WindowsOnlyFact]
+        public void It_discovers_tests_for_the_ProjectWithTestsWithNetCoreApp()
+        {
+            using (var adapter = new Adapter("TestDiscovery.Start"))
+            {
+                adapter.Listen();
+
+                var testCommand = new DotnetTestCommand();
+                var result = testCommand.Execute($"{_projectFilePath} -f netcoreapp1.0 -o {_netCoreAppOutputPath} --port {adapter.Port} --no-build");
+                result.Should().Pass();
+
+                adapter.Messages["TestSession.Connected"].Count.Should().Be(1);
+                adapter.Messages["TestDiscovery.TestFound"].Count.Should().Be(4);
+                adapter.Messages["TestDiscovery.Completed"].Count.Should().Be(1);
+            }
+        }
+
+        [WindowsOnlyFact]
+        public void It_discovers_tests_for_the_ProjectWithTestsWithNet451()
+        {
+            using (var adapter = new Adapter("TestDiscovery.Start"))
+            {
+                adapter.Listen();
+                var rid = PlatformServices.Default.Runtime.GetAllCandidateRuntimeIdentifiers().First();
+
+                var testCommand = new DotnetTestCommand();
+                var result = testCommand.Execute($"{_projectFilePath} -f net451 -r {rid} -o {_net451OutputPath} --port {adapter.Port} --no-build");
+                result.Should().Pass();
+
+                adapter.Messages["TestSession.Connected"].Count.Should().Be(1);
+                adapter.Messages["TestDiscovery.TestFound"].Count.Should().Be(4);
+                adapter.Messages["TestDiscovery.Completed"].Count.Should().Be(1);
+            }
+        }
+
+        [Fact]
+        public void It_runs_tests_for_netcoreapp10()
+        {
+            using (var adapter = new Adapter("TestExecution.GetTestRunnerProcessStartInfo"))
+            {
+                adapter.Listen();
+
+                var testCommand = new DotnetTestCommand();
+                var result = testCommand.Execute($"{_projectFilePath} -f netcoreapp1.0 -o {_netCoreAppOutputPath} --port {adapter.Port} --no-build");
+                result.Should().Pass();
+
+                adapter.Messages["TestSession.Connected"].Count.Should().Be(1);
+                adapter.Messages["TestExecution.TestRunnerProcessStartInfo"].Count.Should().Be(1);
+                adapter.Messages["TestExecution.TestStarted"].Count.Should().Be(4);
+                adapter.Messages["TestExecution.TestResult"].Count.Should().Be(4);
+                adapter.Messages["TestExecution.Completed"].Count.Should().Be(1);
+            }
+        }
+
+        [WindowsOnlyFact]
+        public void It_runs_tests_for_net451()
+        {
+            using (var adapter = new Adapter("TestExecution.GetTestRunnerProcessStartInfo"))
+            {
+                adapter.Listen();
+
+                var testCommand = new DotnetTestCommand();
+                var rid = PlatformServices.Default.Runtime.GetAllCandidateRuntimeIdentifiers().First();
+                var result = testCommand.Execute($"{_projectFilePath} -f net451 -r {rid} -o {_net451OutputPath} --port {adapter.Port} --no-build");
+                result.Should().Pass();
+
+                adapter.Messages["TestSession.Connected"].Count.Should().Be(1);
+                adapter.Messages["TestExecution.TestRunnerProcessStartInfo"].Count.Should().Be(1);
+                adapter.Messages["TestExecution.TestStarted"].Count.Should().Be(4);
+                adapter.Messages["TestExecution.TestResult"].Count.Should().Be(4);
+                adapter.Messages["TestExecution.Completed"].Count.Should().Be(1);
+            }
+        }
+    }
+}

test/dotnet-test.Tests/project.json

@@ -19,7 +19,7 @@
     "System.Net.Sockets": "4.1.0-rc2-24022",
     "System.Runtime.Serialization.Primitives": "4.1.1-rc2-24022",
     "xunit": "2.1.0",
-    "dotnet-test-xunit": "1.0.0-dev-140469-38"
+    "dotnet-test-xunit": "1.0.0-rc2-162081-13"
   },
   "frameworks": {
     "netcoreapp1.0": {
@@ -30,7 +30,7 @@
     }
   },
   "content": [
-    "../../TestAssets/TestProjects/ProjectWithTests/**/*",
+    "../../TestAssets/TestProjects/ProjectsWithTests/**/*",
     "../../TestAssets/TestProjects/global.json"
   ],
   "testRunner": "xunit"

test/dotnet-test.UnitTests/GivenATestCommand.cs

@@ -17,7 +17,8 @@ namespace Microsoft.Dotnet.Tools.Test.Tests
                 AppContext.BaseDirectory,
                 "TestAssets",
                 "TestProjects",
-                "ProjectWithTests",
+                "ProjectsWithTests",
+                "NetCoreAppOnlyProject",
                 "project.json");
 
         private TestCommand _testCommand;
@@ -49,7 +50,7 @@ namespace Microsoft.Dotnet.Tools.Test.Tests
         [Fact]
         public void It_creates_a_runner_if_the_args_do_no_include_help()
         {
-            var result = _testCommand.DoRun(new[] { ProjectJsonPath });
+            var result = _testCommand.DoRun(new[] { ProjectJsonPath, "-f", "netcoreapp1.0" });
 
             result.Should().Be(0);
             _dotnetTestRunnerFactoryMock.Verify(d => d.Create(It.IsAny<int?>()), Times.Once);
@@ -58,7 +59,7 @@ namespace Microsoft.Dotnet.Tools.Test.Tests
         [Fact]
         public void It_runs_the_tests_through_the_DotnetTestRunner()
         {
-            var result = _testCommand.DoRun(new[] { ProjectJsonPath });
+            var result = _testCommand.DoRun(new[] { ProjectJsonPath, "-f", "netcoreapp1.0" });
 
             _dotnetTestRunnerMock.Verify(
                 d => d.RunTests(It.IsAny<ProjectContext>(), It.IsAny<DotnetTestParams>()),

test/dotnet-test.UnitTests/project.json

@@ -14,7 +14,7 @@
       "exclude": "Compile"
     },
     "xunit": "2.1.0",
-    "dotnet-test-xunit": "1.0.0-dev-140469-38",
+    "dotnet-test-xunit": "1.0.0-rc2-162081-13",
     "moq.netcore": "4.4.0-beta8",
     "FluentAssertions": "4.2.2"
   },
@@ -28,8 +28,7 @@
     }
   },
   "content": [
-    "../../TestAssets/TestProjects/ProjectWithTests/project.json",
+    "../../TestAssets/TestProjects/ProjectsWithTests/NetCoreAppOnlyProject/project.json"
-    "../../TestAssets/TestProjects/ProjectWithTests/project.lock.json"
   ],
   "testRunner": "xunit"
 }

test/dotnet.Tests/project.json

@@ -17,7 +17,7 @@
       "type": "build"
     },
     "xunit": "2.1.0",
-    "dotnet-test-xunit": "1.0.0-dev-140469-38"
+    "dotnet-test-xunit": "1.0.0-rc2-162081-13"
   },
   "frameworks": {
     "netcoreapp1.0": {