From 4e496c35236435c3eb683d4bdd02e20014069c12 Mon Sep 17 00:00:00 2001 From: Pranav K Date: Thu, 14 Apr 2016 15:59:11 -0700 Subject: [PATCH] Modify dotnet-test to run testRunner for all tfms Fixes #2506 --- .../GivenThatIWantSomeFakeTests.cs | 55 +++++++ .../MultipleFrameworkProject/project.json | 27 +++ .../GivenThatIWantSomeFakeTests.cs | 2 +- .../NetCoreAppOnlyProject}/project.json | 2 +- scripts/dotnet-cli-build/TestTargets.cs | 4 +- .../dotnet-test/BaseDotnetTestRunner.cs | 21 ++- src/dotnet/commands/dotnet-test/Program.cs | 83 ++++++++-- test/ArgumentForwardingTests/project.json | 2 +- test/EndToEnd/project.json | 2 +- test/Kestrel.Tests/project.json | 2 +- .../project.json | 2 +- .../project.json | 2 +- .../project.json | 2 +- .../project.json | 2 +- .../project.json | 2 +- test/ScriptExecutorTests/project.json | 2 +- .../project.json | 2 +- .../project.json | 2 +- test/dotnet-build.Tests/project.json | 2 +- test/dotnet-compile-fsc.Tests/project.json | 2 +- test/dotnet-compile.Tests/project.json | 2 +- test/dotnet-compile.UnitTests/project.json | 2 +- test/dotnet-pack.Tests/project.json | 2 +- .../project.json | 2 +- test/dotnet-publish.Tests/project.json | 2 +- test/dotnet-resgen.Tests/project.json | 2 +- test/dotnet-run.Tests/project.json | 2 +- test/dotnet-test.Tests/AssemblyInfo.cs | 4 + ...ntToRunTestsForMultipleTFMsInTheConsole.cs | 155 ++++++++++++++++++ .../GivenThatWeWantToRunTestsInTheConsole.cs | 2 +- ...hatWeWantToUseDotnetTestE2EInDesignTime.cs | 2 +- ...otnetTestE2EInDesignTimeForMultipleTFms.cs | 125 ++++++++++++++ test/dotnet-test.Tests/project.json | 4 +- .../GivenATestCommand.cs | 7 +- test/dotnet-test.UnitTests/project.json | 5 +- test/dotnet.Tests/project.json | 2 +- 36 files changed, 481 insertions(+), 59 deletions(-) create mode 100644 TestAssets/TestProjects/ProjectsWithTests/MultipleFrameworkProject/GivenThatIWantSomeFakeTests.cs create mode 100644 TestAssets/TestProjects/ProjectsWithTests/MultipleFrameworkProject/project.json rename TestAssets/TestProjects/{ProjectWithTests => ProjectsWithTests/NetCoreAppOnlyProject}/GivenThatIWantSomeFakeTests.cs (98%) rename TestAssets/TestProjects/{ProjectWithTests => ProjectsWithTests/NetCoreAppOnlyProject}/project.json (90%) create mode 100644 test/dotnet-test.Tests/AssemblyInfo.cs create mode 100644 test/dotnet-test.Tests/GivenThatWeWantToRunTestsForMultipleTFMsInTheConsole.cs create mode 100644 test/dotnet-test.Tests/GivenThatWeWantToUseDotnetTestE2EInDesignTimeForMultipleTFms.cs diff --git a/TestAssets/TestProjects/ProjectsWithTests/MultipleFrameworkProject/GivenThatIWantSomeFakeTests.cs b/TestAssets/TestProjects/ProjectsWithTests/MultipleFrameworkProject/GivenThatIWantSomeFakeTests.cs new file mode 100644 index 000000000..6f2f5ebc8 --- /dev/null +++ b/TestAssets/TestProjects/ProjectsWithTests/MultipleFrameworkProject/GivenThatIWantSomeFakeTests.cs @@ -0,0 +1,55 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using Xunit; + +namespace FakeTests +{ + public class GivenThatIWantSomeFakeTests + { +#if NET451 + [Fact] + public void NET451_succeeds() + { + Assert.True(true); + } + + [Fact(Skip="Skipped for NET451")] + public void SkippedTest() + { + + } +#else + [Fact] + public void NETCOREAPP_succeeds() + { + Assert.True(true); + } + + [Fact(Skip="Skipped for NETCOREAPP1.0")] + public void SkippedTest() + { + + } +#endif + + [Fact] + public void Common_succeeds() + { + Assert.True(true); + } + + [Fact] + public void Fails_IfEnvironmentVariableIsSet() + { + var shouldFail = string.IsNullOrEmpty(Environment.GetEnvironmentVariable("DOTNET_TEST_SHOULD_FAIL")); + +#if NET451 + Assert.True(shouldFail, "Failing in NET451"); +#else + Assert.True(shouldFail, "Failing in NETCOREAPP1.0"); +#endif + } + } +} \ No newline at end of file diff --git a/TestAssets/TestProjects/ProjectsWithTests/MultipleFrameworkProject/project.json b/TestAssets/TestProjects/ProjectsWithTests/MultipleFrameworkProject/project.json new file mode 100644 index 000000000..146f96a86 --- /dev/null +++ b/TestAssets/TestProjects/ProjectsWithTests/MultipleFrameworkProject/project.json @@ -0,0 +1,27 @@ +{ + "version": "1.0.0-*", + "dependencies": { + "dotnet-test-xunit": "1.0.0-rc2-162081-13", + "Microsoft.NETCore.Platforms": "1.0.1-rc2-*", + "xunit": "2.1.0" + }, + "frameworks": { + "netcoreapp1.0": { + "imports": [ + "dotnet", + "portable-dnxcore50+net45+win8+wp8+wpa81", + "portable-net45+win8" + ], + "dependencies": { + "Microsoft.NETCore.App": { + "type": "platform", + "version": "1.0.0-rc2-*" + }, + "System.Linq.Expressions": "4.0.11-rc2-24022", + "System.Runtime.Serialization.Primitives": "4.1.1-rc2-24022" + } + }, + "net451": { } + }, + "testRunner": "xunit" +} diff --git a/TestAssets/TestProjects/ProjectWithTests/GivenThatIWantSomeFakeTests.cs b/TestAssets/TestProjects/ProjectsWithTests/NetCoreAppOnlyProject/GivenThatIWantSomeFakeTests.cs similarity index 98% rename from TestAssets/TestProjects/ProjectWithTests/GivenThatIWantSomeFakeTests.cs rename to TestAssets/TestProjects/ProjectsWithTests/NetCoreAppOnlyProject/GivenThatIWantSomeFakeTests.cs index 4876b4b0c..89b256e3f 100644 --- a/TestAssets/TestProjects/ProjectWithTests/GivenThatIWantSomeFakeTests.cs +++ b/TestAssets/TestProjects/ProjectsWithTests/NetCoreAppOnlyProject/GivenThatIWantSomeFakeTests.cs @@ -12,7 +12,7 @@ namespace FakeTests { Assert.True(true); } - + [Fact] public void It_also_succeeds() { diff --git a/TestAssets/TestProjects/ProjectWithTests/project.json b/TestAssets/TestProjects/ProjectsWithTests/NetCoreAppOnlyProject/project.json similarity index 90% rename from TestAssets/TestProjects/ProjectWithTests/project.json rename to TestAssets/TestProjects/ProjectsWithTests/NetCoreAppOnlyProject/project.json index acb9f131b..aa510b2ec 100644 --- a/TestAssets/TestProjects/ProjectWithTests/project.json +++ b/TestAssets/TestProjects/ProjectsWithTests/NetCoreAppOnlyProject/project.json @@ -8,7 +8,7 @@ "System.Linq.Expressions": "4.0.11-rc2-24022", "System.Runtime.Serialization.Primitives": "4.1.1-rc2-24022", "xunit": "2.1.0", - "dotnet-test-xunit": "1.0.0-dev-140469-38" + "dotnet-test-xunit": "1.0.0-rc2-162081-13" }, "frameworks": { "netcoreapp1.0": { diff --git a/scripts/dotnet-cli-build/TestTargets.cs b/scripts/dotnet-cli-build/TestTargets.cs index 0284ed9a6..3295fa083 100644 --- a/scripts/dotnet-cli-build/TestTargets.cs +++ b/scripts/dotnet-cli-build/TestTargets.cs @@ -74,8 +74,8 @@ namespace Microsoft.DotNet.Cli.Build CleanNuGetTempCache(); var dotnet = DotNetCli.Stage2; - dotnet.Restore("--verbosity", "verbose", - "--infer-runtimes", + dotnet.Restore("--verbosity", "verbose", + "--infer-runtimes", "--fallbacksource", Dirs.Corehost, "--fallbacksource", Dirs.CorehostDummyPackages) .WorkingDirectory(Path.Combine(c.BuildContext.BuildDirectory, "TestAssets", "TestPackages")) diff --git a/src/dotnet/commands/dotnet-test/BaseDotnetTestRunner.cs b/src/dotnet/commands/dotnet-test/BaseDotnetTestRunner.cs index 3d6731966..be013d38e 100644 --- a/src/dotnet/commands/dotnet-test/BaseDotnetTestRunner.cs +++ b/src/dotnet/commands/dotnet-test/BaseDotnetTestRunner.cs @@ -10,24 +10,24 @@ namespace Microsoft.DotNet.Tools.Test { public int RunTests(ProjectContext projectContext, DotnetTestParams dotnetTestParams) { - var result = BuildTestProject(dotnetTestParams); + var result = BuildTestProject(projectContext, dotnetTestParams); return result == 0 ? DoRunTests(projectContext, dotnetTestParams) : result; } internal abstract int DoRunTests(ProjectContext projectContext, DotnetTestParams dotnetTestParams); - private int BuildTestProject(DotnetTestParams dotnetTestParams) + private int BuildTestProject(ProjectContext projectContext, DotnetTestParams dotnetTestParams) { if (dotnetTestParams.NoBuild) { return 0; } - return DoBuildTestProject(dotnetTestParams); + return DoBuildTestProject(projectContext, dotnetTestParams); } - private int DoBuildTestProject(DotnetTestParams dotnetTestParams) + private int DoBuildTestProject(ProjectContext projectContext, DotnetTestParams dotnetTestParams) { var strings = new List { @@ -36,11 +36,10 @@ namespace Microsoft.DotNet.Tools.Test $"{dotnetTestParams.ProjectPath}" }; - if (dotnetTestParams.Framework != null) - { - strings.Add("--framework"); - strings.Add($"{dotnetTestParams.Framework}"); - } + // Build the test specifically for the target framework \ rid of the ProjectContext. This avoids building the project + // for tfms that the user did not request. + strings.Add("--framework"); + strings.Add(projectContext.TargetFramework.ToString()); if (!string.IsNullOrEmpty(dotnetTestParams.BuildBasePath)) { @@ -54,10 +53,10 @@ namespace Microsoft.DotNet.Tools.Test strings.Add(dotnetTestParams.Output); } - if (!string.IsNullOrEmpty(dotnetTestParams.Runtime)) + if (!string.IsNullOrEmpty(projectContext.RuntimeIdentifier)) { strings.Add("--runtime"); - strings.Add(dotnetTestParams.Runtime); + strings.Add(projectContext.RuntimeIdentifier); } var result = Build.BuildCommand.Run(strings.ToArray()); diff --git a/src/dotnet/commands/dotnet-test/Program.cs b/src/dotnet/commands/dotnet-test/Program.cs index b1abfd5d9..f89d17a33 100644 --- a/src/dotnet/commands/dotnet-test/Program.cs +++ b/src/dotnet/commands/dotnet-test/Program.cs @@ -2,7 +2,6 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System; -using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Linq; @@ -42,18 +41,52 @@ namespace Microsoft.DotNet.Tools.Test RegisterForParentProcessExit(dotnetTestParams.ParentProcessId.Value); } + var projectPath = GetProjectPath(dotnetTestParams.ProjectPath); + var runtimeIdentifiers = !string.IsNullOrEmpty(dotnetTestParams.Runtime) ? + new[] { dotnetTestParams.Runtime } : + PlatformServices.Default.Runtime.GetAllCandidateRuntimeIdentifiers(); + var exitCode = 0; + // Create a workspace var workspace = WorkspaceContext.Create(ProjectReaderSettings.ReadFromEnvironment(), designTime: false); - var projectContexts = CreateProjectContexts(workspace, dotnetTestParams.ProjectPath, dotnetTestParams.Runtime); + if (dotnetTestParams.Framework != null) + { + var projectContext = ProjectContext.Create(projectPath, dotnetTestParams.Framework, runtimeIdentifiers); + exitCode = RunTest(projectContext, dotnetTestParams); + } + else + { + var summary = new Summary(); + var projectContexts = workspace + .GetProjectContextCollection(projectPath).FrameworkOnlyContexts + .Select(c => workspace.GetRuntimeContext(c, runtimeIdentifiers)) + .ToList(); - var projectContext = projectContexts.First(); + // Execute for all TFMs the project targets. + foreach (var projectContext in projectContexts) + { + var result = RunTest(projectContext, dotnetTestParams); + if (result == 0) + { + summary.Passed++; + } + else + { + summary.Failed++; + if (exitCode == 0) + { + // If tests fail in more than one TFM, we'll have it use the result of the first one + // as the exit code. + exitCode = result; + } + } + } - var testRunner = projectContext.ProjectFile.TestRunner; + summary.Print(); + } - IDotnetTestRunner dotnetTestRunner = _dotnetTestRunnerFactory.Create(dotnetTestParams.Port); - - return dotnetTestRunner.RunTests(projectContext, dotnetTestParams); + return exitCode; } catch (InvalidOperationException ex) { @@ -101,7 +134,14 @@ namespace Microsoft.DotNet.Tools.Test } } - private static IEnumerable CreateProjectContexts(WorkspaceContext workspace, string projectPath, string runtime) + private int RunTest(ProjectContext projectContext, DotnetTestParams dotnetTestParams) + { + var testRunner = projectContext.ProjectFile.TestRunner; + var dotnetTestRunner = _dotnetTestRunnerFactory.Create(dotnetTestParams.Port); + return dotnetTestRunner.RunTests(projectContext, dotnetTestParams); + } + + private static string GetProjectPath(string projectPath) { projectPath = projectPath ?? Directory.GetCurrentDirectory(); @@ -115,12 +155,29 @@ namespace Microsoft.DotNet.Tools.Test throw new InvalidOperationException($"{projectPath} does not exist."); } - var runtimeIdentifiers = !string.IsNullOrEmpty(runtime) ? - new[] { runtime } : - PlatformServices.Default.Runtime.GetAllCandidateRuntimeIdentifiers(); + return projectPath; + } - var contexts = workspace.GetProjectContextCollection(projectPath).FrameworkOnlyContexts; - return contexts.Select(c => workspace.GetRuntimeContext(c, runtimeIdentifiers)); + private class Summary + { + public int Passed { get; set; } + + public int Failed { get; set; } + + public int Total => Passed + Failed; + + public void Print() + { + var summaryMessage = $"SUMMARY: Total: {Total} targets, Passed: {Passed}, Failed: {Failed}."; + if (Failed > 0) + { + Reporter.Error.WriteLine(summaryMessage.Red()); + } + else + { + Reporter.Output.WriteLine(summaryMessage); + } + } } } } \ No newline at end of file diff --git a/test/ArgumentForwardingTests/project.json b/test/ArgumentForwardingTests/project.json index d72a493a4..6e4b3633d 100644 --- a/test/ArgumentForwardingTests/project.json +++ b/test/ArgumentForwardingTests/project.json @@ -19,7 +19,7 @@ "target": "project" }, "xunit": "2.1.0", - "dotnet-test-xunit": "1.0.0-dev-140469-38" + "dotnet-test-xunit": "1.0.0-rc2-162081-13" }, "frameworks": { "netcoreapp1.0": { diff --git a/test/EndToEnd/project.json b/test/EndToEnd/project.json index 63347e685..83628cb89 100644 --- a/test/EndToEnd/project.json +++ b/test/EndToEnd/project.json @@ -20,7 +20,7 @@ }, "xunit": "2.1.0", "xunit.netcore.extensions": "1.0.0-prerelease-00206", - "dotnet-test-xunit": "1.0.0-dev-140469-38" + "dotnet-test-xunit": "1.0.0-rc2-162081-13" }, "frameworks": { "netcoreapp1.0": { diff --git a/test/Kestrel.Tests/project.json b/test/Kestrel.Tests/project.json index 7589fda99..8bd237865 100644 --- a/test/Kestrel.Tests/project.json +++ b/test/Kestrel.Tests/project.json @@ -13,7 +13,7 @@ "target": "project" }, "xunit": "2.1.0", - "dotnet-test-xunit": "1.0.0-dev-140469-38" + "dotnet-test-xunit": "1.0.0-rc2-162081-13" }, "frameworks": { "netcoreapp1.0": { diff --git a/test/Microsoft.DotNet.Cli.Utils.Tests/project.json b/test/Microsoft.DotNet.Cli.Utils.Tests/project.json index 390449e86..e6e8c8df0 100644 --- a/test/Microsoft.DotNet.Cli.Utils.Tests/project.json +++ b/test/Microsoft.DotNet.Cli.Utils.Tests/project.json @@ -25,7 +25,7 @@ }, "moq.netcore": "4.4.0-beta8", "xunit": "2.1.0", - "dotnet-test-xunit": "1.0.0-dev-140469-38" + "dotnet-test-xunit": "1.0.0-rc2-162081-13" }, "frameworks": { "netcoreapp1.0": { diff --git a/test/Microsoft.DotNet.Compiler.Common.Tests/project.json b/test/Microsoft.DotNet.Compiler.Common.Tests/project.json index 196872125..767b1fe88 100644 --- a/test/Microsoft.DotNet.Compiler.Common.Tests/project.json +++ b/test/Microsoft.DotNet.Compiler.Common.Tests/project.json @@ -16,7 +16,7 @@ "target": "project" }, "xunit": "2.1.0", - "dotnet-test-xunit": "1.0.0-dev-140469-38" + "dotnet-test-xunit": "1.0.0-rc2-162081-13" }, "frameworks": { "netcoreapp1.0": { diff --git a/test/Microsoft.DotNet.ProjectModel.Tests/project.json b/test/Microsoft.DotNet.ProjectModel.Tests/project.json index 985a3b34c..06823f231 100644 --- a/test/Microsoft.DotNet.ProjectModel.Tests/project.json +++ b/test/Microsoft.DotNet.ProjectModel.Tests/project.json @@ -16,7 +16,7 @@ "target": "project" }, "xunit": "2.1.0", - "dotnet-test-xunit": "1.0.0-dev-140469-38" + "dotnet-test-xunit": "1.0.0-rc2-162081-13" }, "frameworks": { "netcoreapp1.0": { diff --git a/test/Microsoft.DotNet.Tools.Tests.Utilities/project.json b/test/Microsoft.DotNet.Tools.Tests.Utilities/project.json index a251d4a7e..c36f2e0b9 100644 --- a/test/Microsoft.DotNet.Tools.Tests.Utilities/project.json +++ b/test/Microsoft.DotNet.Tools.Tests.Utilities/project.json @@ -14,7 +14,7 @@ "System.Net.NetworkInformation": "4.1.0-rc2-24022", "FluentAssertions": "4.0.0", "xunit": "2.1.0", - "dotnet-test-xunit": "1.0.0-dev-140469-38", + "dotnet-test-xunit": "1.0.0-rc2-162081-13", "Microsoft.DotNet.TestFramework": "1.0.0-*", "Microsoft.DotNet.Cli.Utils": "1.0.0-*", "Microsoft.DotNet.ProjectModel": "1.0.0-*", diff --git a/test/Microsoft.Extensions.DependencyModel.Tests/project.json b/test/Microsoft.Extensions.DependencyModel.Tests/project.json index 6348df11f..3d17a4710 100644 --- a/test/Microsoft.Extensions.DependencyModel.Tests/project.json +++ b/test/Microsoft.Extensions.DependencyModel.Tests/project.json @@ -19,7 +19,7 @@ "FluentAssertions": "4.0.0", "moq.netcore": "4.4.0-beta8", "xunit": "2.1.0", - "dotnet-test-xunit": "1.0.0-dev-140469-38" + "dotnet-test-xunit": "1.0.0-rc2-162081-13" }, "frameworks": { "netcoreapp1.0": { diff --git a/test/ScriptExecutorTests/project.json b/test/ScriptExecutorTests/project.json index b77eb08c9..fb4382bfb 100644 --- a/test/ScriptExecutorTests/project.json +++ b/test/ScriptExecutorTests/project.json @@ -15,7 +15,7 @@ "target": "project" }, "xunit": "2.1.0", - "dotnet-test-xunit": "1.0.0-dev-140469-38" + "dotnet-test-xunit": "1.0.0-rc2-162081-13" }, "frameworks": { "netcoreapp1.0": { diff --git a/test/TestingAbstractions/Microsoft.Extensions.Testing.Abstractions.Tests/project.json b/test/TestingAbstractions/Microsoft.Extensions.Testing.Abstractions.Tests/project.json index 7f1d95c0c..5045424d7 100644 --- a/test/TestingAbstractions/Microsoft.Extensions.Testing.Abstractions.Tests/project.json +++ b/test/TestingAbstractions/Microsoft.Extensions.Testing.Abstractions.Tests/project.json @@ -19,7 +19,7 @@ "target": "project" }, "xunit": "2.1.0", - "dotnet-test-xunit": "1.0.0-dev-140469-38", + "dotnet-test-xunit": "1.0.0-rc2-162081-13", "FluentAssertions": "4.2.2" }, "frameworks": { diff --git a/test/TestingAbstractions/Microsoft.Extensions.Testing.Abstractions.UnitTests/project.json b/test/TestingAbstractions/Microsoft.Extensions.Testing.Abstractions.UnitTests/project.json index 323b3e0b5..1246e2099 100644 --- a/test/TestingAbstractions/Microsoft.Extensions.Testing.Abstractions.UnitTests/project.json +++ b/test/TestingAbstractions/Microsoft.Extensions.Testing.Abstractions.UnitTests/project.json @@ -14,7 +14,7 @@ "target": "project" }, "xunit": "2.1.0", - "dotnet-test-xunit": "1.0.0-dev-140469-38", + "dotnet-test-xunit": "1.0.0-rc2-162081-13", "FluentAssertions": "4.2.2", "moq.netcore": "4.4.0-beta8" }, diff --git a/test/dotnet-build.Tests/project.json b/test/dotnet-build.Tests/project.json index 4c4de6bcb..c04a81d34 100644 --- a/test/dotnet-build.Tests/project.json +++ b/test/dotnet-build.Tests/project.json @@ -14,7 +14,7 @@ }, "Newtonsoft.Json": "7.0.1", "xunit": "2.1.0", - "dotnet-test-xunit": "1.0.0-dev-140469-38" + "dotnet-test-xunit": "1.0.0-rc2-162081-13" }, "frameworks": { "netcoreapp1.0": { diff --git a/test/dotnet-compile-fsc.Tests/project.json b/test/dotnet-compile-fsc.Tests/project.json index ce841dbf0..f68ee9b50 100644 --- a/test/dotnet-compile-fsc.Tests/project.json +++ b/test/dotnet-compile-fsc.Tests/project.json @@ -12,7 +12,7 @@ "target": "project" }, "xunit": "2.1.0", - "dotnet-test-xunit": "1.0.0-dev-140469-38" + "dotnet-test-xunit": "1.0.0-rc2-162081-13" }, "frameworks": { "netcoreapp1.0": { diff --git a/test/dotnet-compile.Tests/project.json b/test/dotnet-compile.Tests/project.json index 67a079d50..71819b612 100644 --- a/test/dotnet-compile.Tests/project.json +++ b/test/dotnet-compile.Tests/project.json @@ -13,7 +13,7 @@ "target": "project" }, "xunit": "2.1.0", - "dotnet-test-xunit": "1.0.0-dev-140469-38" + "dotnet-test-xunit": "1.0.0-rc2-162081-13" }, "frameworks": { "netcoreapp1.0": { diff --git a/test/dotnet-compile.UnitTests/project.json b/test/dotnet-compile.UnitTests/project.json index b69e61fad..57027663e 100644 --- a/test/dotnet-compile.UnitTests/project.json +++ b/test/dotnet-compile.UnitTests/project.json @@ -25,7 +25,7 @@ "target": "project" }, "xunit": "2.1.0", - "dotnet-test-xunit": "1.0.0-dev-140469-38", + "dotnet-test-xunit": "1.0.0-rc2-162081-13", "moq.netcore": "4.4.0-beta8", "FluentAssertions": "4.2.2" }, diff --git a/test/dotnet-pack.Tests/project.json b/test/dotnet-pack.Tests/project.json index 3a5e568d4..c45b5f203 100644 --- a/test/dotnet-pack.Tests/project.json +++ b/test/dotnet-pack.Tests/project.json @@ -14,7 +14,7 @@ "target": "project" }, "xunit": "2.1.0", - "dotnet-test-xunit": "1.0.0-dev-140469-38" + "dotnet-test-xunit": "1.0.0-rc2-162081-13" }, "frameworks": { "netcoreapp1.0": { diff --git a/test/dotnet-projectmodel-server.Tests/project.json b/test/dotnet-projectmodel-server.Tests/project.json index aa8fbdf6c..349228b7a 100644 --- a/test/dotnet-projectmodel-server.Tests/project.json +++ b/test/dotnet-projectmodel-server.Tests/project.json @@ -18,7 +18,7 @@ "target": "project" }, "xunit": "2.1.0", - "dotnet-test-xunit": "1.0.0-dev-140469-38", + "dotnet-test-xunit": "1.0.0-rc2-162081-13", "System.Net.NameResolution": "4.0.0-rc2-24022" }, "frameworks": { diff --git a/test/dotnet-publish.Tests/project.json b/test/dotnet-publish.Tests/project.json index 05a99439b..52b388eae 100644 --- a/test/dotnet-publish.Tests/project.json +++ b/test/dotnet-publish.Tests/project.json @@ -12,7 +12,7 @@ }, "xunit": "2.1.0", "xunit.netcore.extensions": "1.0.0-prerelease-00206", - "dotnet-test-xunit": "1.0.0-dev-140469-38", + "dotnet-test-xunit": "1.0.0-rc2-162081-13", "System.Runtime.InteropServices.RuntimeInformation": "4.0.0-rc2-24022" }, "frameworks": { diff --git a/test/dotnet-resgen.Tests/project.json b/test/dotnet-resgen.Tests/project.json index 147cbc1b6..f222777ff 100644 --- a/test/dotnet-resgen.Tests/project.json +++ b/test/dotnet-resgen.Tests/project.json @@ -14,7 +14,7 @@ }, "xunit": "2.1.0", "xunit.netcore.extensions": "1.0.0-prerelease-00206", - "dotnet-test-xunit": "1.0.0-dev-140469-38" + "dotnet-test-xunit": "1.0.0-rc2-162081-13" }, "frameworks": { "netcoreapp1.0": { diff --git a/test/dotnet-run.Tests/project.json b/test/dotnet-run.Tests/project.json index 7589fda99..8bd237865 100644 --- a/test/dotnet-run.Tests/project.json +++ b/test/dotnet-run.Tests/project.json @@ -13,7 +13,7 @@ "target": "project" }, "xunit": "2.1.0", - "dotnet-test-xunit": "1.0.0-dev-140469-38" + "dotnet-test-xunit": "1.0.0-rc2-162081-13" }, "frameworks": { "netcoreapp1.0": { diff --git a/test/dotnet-test.Tests/AssemblyInfo.cs b/test/dotnet-test.Tests/AssemblyInfo.cs new file mode 100644 index 000000000..a35a2b073 --- /dev/null +++ b/test/dotnet-test.Tests/AssemblyInfo.cs @@ -0,0 +1,4 @@ +using Xunit; + +// Don't let the tests execute concurrently +[assembly: CollectionBehavior(CollectionBehavior.CollectionPerAssembly)] \ No newline at end of file diff --git a/test/dotnet-test.Tests/GivenThatWeWantToRunTestsForMultipleTFMsInTheConsole.cs b/test/dotnet-test.Tests/GivenThatWeWantToRunTestsForMultipleTFMsInTheConsole.cs new file mode 100644 index 000000000..882f08bb1 --- /dev/null +++ b/test/dotnet-test.Tests/GivenThatWeWantToRunTestsForMultipleTFMsInTheConsole.cs @@ -0,0 +1,155 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.IO; +using System.Linq; +using FluentAssertions; +using Microsoft.DotNet.ProjectModel; +using Microsoft.DotNet.TestFramework; +using Microsoft.DotNet.Tools.Test.Utilities; +using Microsoft.Extensions.PlatformAbstractions; +using Xunit; + +namespace Microsoft.Dotnet.Tools.Test.Tests +{ + public class GivenThatWeWantToRunTestsForMultipleTFMsInTheConsole : TestBase + { + private readonly string _projectFilePath; + private readonly string _defaultNetCoreAppOutputPath; + private readonly string _defaultNet451OutputPath; + + public GivenThatWeWantToRunTestsForMultipleTFMsInTheConsole() + { + var testInstance = + TestAssetsManager.CreateTestInstance(Path.Combine("ProjectsWithTests", "MultipleFrameworkProject"), identifier: "ConsoleTests"); + + _projectFilePath = Path.Combine(testInstance.TestRoot, "project.json"); + var contexts = ProjectContext.CreateContextForEachFramework( + _projectFilePath, + null, + PlatformServices.Default.Runtime.GetAllCandidateRuntimeIdentifiers()); + + // Restore the project again in the destination to resolve projects + // Since the lock file has project relative paths in it, those will be broken + // unless we re-restore + new RestoreCommand() { WorkingDirectory = testInstance.TestRoot }.Execute().Should().Pass(); + + _defaultNetCoreAppOutputPath = Path.Combine(testInstance.TestRoot, "bin", "Debug", "netcoreapp1.0"); + _defaultNet451OutputPath = Path.Combine(testInstance.TestRoot, "bin", "Debug", "net451", PlatformServices.Default.Runtime.GetAllCandidateRuntimeIdentifiers().First()); + } + + [WindowsOnlyFact] + public void It_builds_and_runs_tests_for_all_frameworks() + { + var testCommand = new DotnetTestCommand(); + var result = testCommand + .ExecuteWithCapturedOutput($"test {_projectFilePath}"); + result.Should().Pass(); + result.StdOut.Should().Contain("Skipped for NET451"); + result.StdOut.Should().Contain("Skipped for NETCOREAPP1.0"); + } + + [WindowsOnlyFact] + public void It_builds_and_runs_tests_for_net451() + { + var testCommand = new DotnetTestCommand(); + var result = testCommand + .ExecuteWithCapturedOutput($"test {_projectFilePath} -f net451"); + result.Should().Pass(); + result.StdOut.Should().Contain($"Skipped for NET451"); + result.StdOut.Should().NotContain($"Skipped for NETCOREAPP1.0"); + } + + [Fact] + public void It_builds_and_runs_tests_for_netcoreapp10() + { + var testCommand = new DotnetTestCommand(); + var result = testCommand + .ExecuteWithCapturedOutput($"test {_projectFilePath} -f netcoreapp1.0"); + result.Should().Pass(); + result.StdOut.Should().Contain($"Skipped for NETCOREAPP1.0"); + result.StdOut.Should().NotContain($"Skipped for NET451"); + } + + [Fact] + public void It_builds_the_project_using_the_output_passed() + { + var testCommand = new DotnetTestCommand(); + var result = testCommand.Execute( + $"{_projectFilePath} -o {Path.Combine(AppContext.BaseDirectory, "output")} -f netcoreapp1.0"); + result.Should().Pass(); + } + + [WindowsOnlyFact] + public void It_builds_the_project_using_the_build_base_path_passed() + { + var buildBasePath = GetNotSoLongBuildBasePath(); + var testCommand = new DotnetTestCommand(); + var result = testCommand.Execute($"{_projectFilePath} -b {buildBasePath}"); + result.Should().Pass(); + } + + [Fact] + public void It_skips_build_when_the_no_build_flag_is_passed_for_netcoreapp10() + { + var buildCommand = new BuildCommand(_projectFilePath); + var result = buildCommand.Execute($"-f netcoreapp1.0 -o {_defaultNetCoreAppOutputPath}"); + result.Should().Pass(); + + var testCommand = new DotnetTestCommand(); + result = testCommand.Execute($"{_projectFilePath} -f netcoreapp10 -o {_defaultNetCoreAppOutputPath} --no-build"); + result.Should().Pass(); + } + + [WindowsOnlyFact] + public void It_skips_build_when_the_no_build_flag_is_passed_for_net451() + { + var rid = PlatformServices.Default.Runtime.GetAllCandidateRuntimeIdentifiers().First(); + var buildCommand = new BuildCommand(_projectFilePath); + var result = buildCommand.Execute($"-f net451 -r {rid} -o {_defaultNet451OutputPath}"); + result.Should().Pass(); + + var testCommand = new DotnetTestCommand(); + result = testCommand.Execute($"{_projectFilePath} -f net451 -r {rid} -o {_defaultNet451OutputPath} --no-build"); + result.Should().Pass(); + } + + [Fact] + public void It_prints_error_when_no_framework_matched() + { + var nonExistentFramework = "doesnotexisttfm99.99"; + var testCommand = new DotnetTestCommand(); + var result = testCommand + .ExecuteWithCapturedOutput($"test {_projectFilePath} -f {nonExistentFramework}"); + + result.Should().Fail(); + result.StdErr.Should().Contain($"does not support framework"); + } + + [WindowsOnlyFact] + public void It_runs_tests_for_all_tfms_if_they_fail() + { + var testCommand = new DotnetTestCommand + { + Environment = + { + { "DOTNET_TEST_SHOULD_FAIL", "1" } + } + }; + + var result = testCommand + .ExecuteWithCapturedOutput($"test {_projectFilePath}"); + + result.Should().Fail(); + result.StdOut.Should().Contain("Failing in NET451"); + result.StdOut.Should().Contain("Failing in NETCOREAPP1.0"); + } + + private string GetNotSoLongBuildBasePath() + { + return Path.GetFullPath( + Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "..", "buildBasePathTest")); + } + } +} diff --git a/test/dotnet-test.Tests/GivenThatWeWantToRunTestsInTheConsole.cs b/test/dotnet-test.Tests/GivenThatWeWantToRunTestsInTheConsole.cs index df2de3e94..e8c949dcc 100644 --- a/test/dotnet-test.Tests/GivenThatWeWantToRunTestsInTheConsole.cs +++ b/test/dotnet-test.Tests/GivenThatWeWantToRunTestsInTheConsole.cs @@ -20,7 +20,7 @@ namespace Microsoft.Dotnet.Tools.Test.Tests public GivenThatWeWantToRunTestsInTheConsole() { var testInstance = - TestAssetsManager.CreateTestInstance("ProjectWithTests", identifier: "ConsoleTests"); + TestAssetsManager.CreateTestInstance(Path.Combine("ProjectsWithTests", "NetCoreAppOnlyProject"), identifier: "ConsoleTests"); _projectFilePath = Path.Combine(testInstance.TestRoot, "project.json"); var contexts = ProjectContext.CreateContextForEachFramework( diff --git a/test/dotnet-test.Tests/GivenThatWeWantToUseDotnetTestE2EInDesignTime.cs b/test/dotnet-test.Tests/GivenThatWeWantToUseDotnetTestE2EInDesignTime.cs index 97024bf30..dcbe2aca0 100644 --- a/test/dotnet-test.Tests/GivenThatWeWantToUseDotnetTestE2EInDesignTime.cs +++ b/test/dotnet-test.Tests/GivenThatWeWantToUseDotnetTestE2EInDesignTime.cs @@ -19,7 +19,7 @@ namespace Microsoft.Dotnet.Tools.Test.Tests public GivenThatWeWantToUseDotnetTestE2EInDesignTime() { - var testInstance = TestAssetsManager.CreateTestInstance("ProjectWithTests"); + var testInstance = TestAssetsManager.CreateTestInstance(Path.Combine("ProjectsWithTests", "NetCoreAppOnlyProject")); _projectFilePath = Path.Combine(testInstance.TestRoot, "project.json"); var contexts = ProjectContext.CreateContextForEachFramework( diff --git a/test/dotnet-test.Tests/GivenThatWeWantToUseDotnetTestE2EInDesignTimeForMultipleTFms.cs b/test/dotnet-test.Tests/GivenThatWeWantToUseDotnetTestE2EInDesignTimeForMultipleTFms.cs new file mode 100644 index 000000000..84635d901 --- /dev/null +++ b/test/dotnet-test.Tests/GivenThatWeWantToUseDotnetTestE2EInDesignTimeForMultipleTFms.cs @@ -0,0 +1,125 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.IO; +using System.Linq; +using Microsoft.DotNet.ProjectModel; +using Microsoft.DotNet.TestFramework; +using Microsoft.DotNet.Tools.Test.Utilities; +using Microsoft.Extensions.PlatformAbstractions; +using FluentAssertions; +using Xunit; + +namespace Microsoft.Dotnet.Tools.Test.Tests +{ + public class GivenThatWeWantToUseDotnetTestE2EInDesignTimeForMultipleTFms : TestBase + { + private readonly string _projectFilePath; + private readonly string _netCoreAppOutputPath; + private readonly string _net451OutputPath; + + public GivenThatWeWantToUseDotnetTestE2EInDesignTimeForMultipleTFms() + { + var testInstance = TestAssetsManager.CreateTestInstance(Path.Combine("ProjectsWithTests", "MultipleFrameworkProject")); + + _projectFilePath = Path.Combine(testInstance.TestRoot, "project.json"); + var contexts = ProjectContext.CreateContextForEachFramework( + _projectFilePath, + null, + PlatformServices.Default.Runtime.GetAllCandidateRuntimeIdentifiers()); + + // Restore the project again in the destination to resolve projects + // Since the lock file has project relative paths in it, those will be broken + // unless we re-restore + new RestoreCommand() { WorkingDirectory = testInstance.TestRoot }.Execute().Should().Pass(); + + _netCoreAppOutputPath = Path.Combine(testInstance.TestRoot, "bin", "Debug", "netcoreapp1.0"); + var buildCommand = new BuildCommand(_projectFilePath); + var result = buildCommand.Execute($"-f netcoreapp1.0 -o {_netCoreAppOutputPath}"); + + result.Should().Pass(); + + if (PlatformServices.Default.Runtime.OperatingSystemPlatform == Platform.Windows) + { + var rid = PlatformServices.Default.Runtime.GetAllCandidateRuntimeIdentifiers().First(); + _net451OutputPath = Path.Combine(testInstance.TestRoot, "bin", "Debug", "net451", rid); + result = buildCommand.Execute($"-f net451 -r {rid} -o {_net451OutputPath}"); + result.Should().Pass(); + } + } + + [WindowsOnlyFact] + public void It_discovers_tests_for_the_ProjectWithTestsWithNetCoreApp() + { + using (var adapter = new Adapter("TestDiscovery.Start")) + { + adapter.Listen(); + + var testCommand = new DotnetTestCommand(); + var result = testCommand.Execute($"{_projectFilePath} -f netcoreapp1.0 -o {_netCoreAppOutputPath} --port {adapter.Port} --no-build"); + result.Should().Pass(); + + adapter.Messages["TestSession.Connected"].Count.Should().Be(1); + adapter.Messages["TestDiscovery.TestFound"].Count.Should().Be(4); + adapter.Messages["TestDiscovery.Completed"].Count.Should().Be(1); + } + } + + [WindowsOnlyFact] + public void It_discovers_tests_for_the_ProjectWithTestsWithNet451() + { + using (var adapter = new Adapter("TestDiscovery.Start")) + { + adapter.Listen(); + var rid = PlatformServices.Default.Runtime.GetAllCandidateRuntimeIdentifiers().First(); + + var testCommand = new DotnetTestCommand(); + var result = testCommand.Execute($"{_projectFilePath} -f net451 -r {rid} -o {_net451OutputPath} --port {adapter.Port} --no-build"); + result.Should().Pass(); + + adapter.Messages["TestSession.Connected"].Count.Should().Be(1); + adapter.Messages["TestDiscovery.TestFound"].Count.Should().Be(4); + adapter.Messages["TestDiscovery.Completed"].Count.Should().Be(1); + } + } + + [Fact] + public void It_runs_tests_for_netcoreapp10() + { + using (var adapter = new Adapter("TestExecution.GetTestRunnerProcessStartInfo")) + { + adapter.Listen(); + + var testCommand = new DotnetTestCommand(); + var result = testCommand.Execute($"{_projectFilePath} -f netcoreapp1.0 -o {_netCoreAppOutputPath} --port {adapter.Port} --no-build"); + result.Should().Pass(); + + adapter.Messages["TestSession.Connected"].Count.Should().Be(1); + adapter.Messages["TestExecution.TestRunnerProcessStartInfo"].Count.Should().Be(1); + adapter.Messages["TestExecution.TestStarted"].Count.Should().Be(4); + adapter.Messages["TestExecution.TestResult"].Count.Should().Be(4); + adapter.Messages["TestExecution.Completed"].Count.Should().Be(1); + } + } + + [WindowsOnlyFact] + public void It_runs_tests_for_net451() + { + using (var adapter = new Adapter("TestExecution.GetTestRunnerProcessStartInfo")) + { + adapter.Listen(); + + var testCommand = new DotnetTestCommand(); + var rid = PlatformServices.Default.Runtime.GetAllCandidateRuntimeIdentifiers().First(); + var result = testCommand.Execute($"{_projectFilePath} -f net451 -r {rid} -o {_net451OutputPath} --port {adapter.Port} --no-build"); + result.Should().Pass(); + + adapter.Messages["TestSession.Connected"].Count.Should().Be(1); + adapter.Messages["TestExecution.TestRunnerProcessStartInfo"].Count.Should().Be(1); + adapter.Messages["TestExecution.TestStarted"].Count.Should().Be(4); + adapter.Messages["TestExecution.TestResult"].Count.Should().Be(4); + adapter.Messages["TestExecution.Completed"].Count.Should().Be(1); + } + } + } +} diff --git a/test/dotnet-test.Tests/project.json b/test/dotnet-test.Tests/project.json index a7977b9cf..21695fca7 100644 --- a/test/dotnet-test.Tests/project.json +++ b/test/dotnet-test.Tests/project.json @@ -19,7 +19,7 @@ "System.Net.Sockets": "4.1.0-rc2-24022", "System.Runtime.Serialization.Primitives": "4.1.1-rc2-24022", "xunit": "2.1.0", - "dotnet-test-xunit": "1.0.0-dev-140469-38" + "dotnet-test-xunit": "1.0.0-rc2-162081-13" }, "frameworks": { "netcoreapp1.0": { @@ -30,7 +30,7 @@ } }, "content": [ - "../../TestAssets/TestProjects/ProjectWithTests/**/*", + "../../TestAssets/TestProjects/ProjectsWithTests/**/*", "../../TestAssets/TestProjects/global.json" ], "testRunner": "xunit" diff --git a/test/dotnet-test.UnitTests/GivenATestCommand.cs b/test/dotnet-test.UnitTests/GivenATestCommand.cs index 2d0e5c405..846920de2 100644 --- a/test/dotnet-test.UnitTests/GivenATestCommand.cs +++ b/test/dotnet-test.UnitTests/GivenATestCommand.cs @@ -17,7 +17,8 @@ namespace Microsoft.Dotnet.Tools.Test.Tests AppContext.BaseDirectory, "TestAssets", "TestProjects", - "ProjectWithTests", + "ProjectsWithTests", + "NetCoreAppOnlyProject", "project.json"); private TestCommand _testCommand; @@ -49,7 +50,7 @@ namespace Microsoft.Dotnet.Tools.Test.Tests [Fact] public void It_creates_a_runner_if_the_args_do_no_include_help() { - var result = _testCommand.DoRun(new[] { ProjectJsonPath }); + var result = _testCommand.DoRun(new[] { ProjectJsonPath, "-f", "netcoreapp1.0" }); result.Should().Be(0); _dotnetTestRunnerFactoryMock.Verify(d => d.Create(It.IsAny()), Times.Once); @@ -58,7 +59,7 @@ namespace Microsoft.Dotnet.Tools.Test.Tests [Fact] public void It_runs_the_tests_through_the_DotnetTestRunner() { - var result = _testCommand.DoRun(new[] { ProjectJsonPath }); + var result = _testCommand.DoRun(new[] { ProjectJsonPath, "-f", "netcoreapp1.0" }); _dotnetTestRunnerMock.Verify( d => d.RunTests(It.IsAny(), It.IsAny()), diff --git a/test/dotnet-test.UnitTests/project.json b/test/dotnet-test.UnitTests/project.json index c85586274..97ed66b5d 100644 --- a/test/dotnet-test.UnitTests/project.json +++ b/test/dotnet-test.UnitTests/project.json @@ -14,7 +14,7 @@ "exclude": "Compile" }, "xunit": "2.1.0", - "dotnet-test-xunit": "1.0.0-dev-140469-38", + "dotnet-test-xunit": "1.0.0-rc2-162081-13", "moq.netcore": "4.4.0-beta8", "FluentAssertions": "4.2.2" }, @@ -28,8 +28,7 @@ } }, "content": [ - "../../TestAssets/TestProjects/ProjectWithTests/project.json", - "../../TestAssets/TestProjects/ProjectWithTests/project.lock.json" + "../../TestAssets/TestProjects/ProjectsWithTests/NetCoreAppOnlyProject/project.json" ], "testRunner": "xunit" } diff --git a/test/dotnet.Tests/project.json b/test/dotnet.Tests/project.json index 719473e4c..e9eb5bef4 100644 --- a/test/dotnet.Tests/project.json +++ b/test/dotnet.Tests/project.json @@ -17,7 +17,7 @@ "type": "build" }, "xunit": "2.1.0", - "dotnet-test-xunit": "1.0.0-dev-140469-38" + "dotnet-test-xunit": "1.0.0-rc2-162081-13" }, "frameworks": { "netcoreapp1.0": {