From f3a74f116211208ea6f9e3d607164b9d30e0eadf Mon Sep 17 00:00:00 2001 From: Arun Mahapatra Date: Fri, 16 Dec 2016 15:39:52 +0530 Subject: [PATCH] Return non-zero exit code for test failure in multitargeted test project. Treat the errors from vstest task as errors (instead of warnings) before continuing further. Add asserts for exit code in tests for dotnet-test and dotnet-vstest. --- .../Microsoft.TestPlatform.CrossTargeting.targets | 2 +- ...netTestBuildsAndRunsTestFromCsprojForMultipleTFM.cs | 2 ++ .../GivenDotnetTestBuildsAndRunsTestfromCsproj.cs | 4 +++- test/dotnet-vstest.Tests/VSTestTests.cs | 10 +++++----- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/resources/MSBuildImports/15.0/Microsoft.Common.CrossTargeting.targets/ImportAfter/Microsoft.TestPlatform.CrossTargeting.targets b/resources/MSBuildImports/15.0/Microsoft.Common.CrossTargeting.targets/ImportAfter/Microsoft.TestPlatform.CrossTargeting.targets index 16ce339ae..0b4916558 100644 --- a/resources/MSBuildImports/15.0/Microsoft.Common.CrossTargeting.targets/ImportAfter/Microsoft.TestPlatform.CrossTargeting.targets +++ b/resources/MSBuildImports/15.0/Microsoft.Common.CrossTargeting.targets/ImportAfter/Microsoft.TestPlatform.CrossTargeting.targets @@ -41,7 +41,7 @@ Copyright (c) .NET Foundation. All rights reserved. Condition="'$(TargetFrameworks)' != '' " Targets="$(InnerVSTestTargets)" Properties="TargetFramework=%(_TargetFramework.Identity);VSTestNoBuild=true" - ContinueOnError="true"> + ContinueOnError="ErrorAndContinue"> diff --git a/test/dotnet-test.Tests/GivenDotnetTestBuildsAndRunsTestFromCsprojForMultipleTFM.cs b/test/dotnet-test.Tests/GivenDotnetTestBuildsAndRunsTestFromCsprojForMultipleTFM.cs index ac1eeaddb..29ba2d679 100644 --- a/test/dotnet-test.Tests/GivenDotnetTestBuildsAndRunsTestFromCsprojForMultipleTFM.cs +++ b/test/dotnet-test.Tests/GivenDotnetTestBuildsAndRunsTestFromCsprojForMultipleTFM.cs @@ -39,6 +39,7 @@ namespace Microsoft.DotNet.Cli.Test.Tests .And.Contain("Passed TestNamespace.VSTestTests.VSTestPassTestDesktop", "because .NET 4.6 tests will pass") .And.Contain("Total tests: 3. Passed: 1. Failed: 2. Skipped: 0.", "because netcoreapp1.0 tests will fail") .And.Contain("Failed TestNamespace.VSTestTests.VSTestFailTestNetCoreApp", "because netcoreapp1.0 tests will fail"); + result.ExitCode.Should().Be(1); } [WindowsOnlyFact] @@ -70,6 +71,7 @@ namespace Microsoft.DotNet.Cli.Test.Tests // for target framework netcoreapp1.0 result.StdOut.Should().Contain("Total tests: 3. Passed: 1. Failed: 2. Skipped: 0."); result.StdOut.Should().Contain("Failed TestNamespace.VSTestXunitTests.VSTestXunitFailTestNetCoreApp"); + result.ExitCode.Should().Be(1); } } } diff --git a/test/dotnet-test.Tests/GivenDotnetTestBuildsAndRunsTestfromCsproj.cs b/test/dotnet-test.Tests/GivenDotnetTestBuildsAndRunsTestfromCsproj.cs index f468fa02f..ddb3f6b52 100644 --- a/test/dotnet-test.Tests/GivenDotnetTestBuildsAndRunsTestfromCsproj.cs +++ b/test/dotnet-test.Tests/GivenDotnetTestBuildsAndRunsTestfromCsproj.cs @@ -38,6 +38,7 @@ namespace Microsoft.DotNet.Cli.Test.Tests result.StdOut.Should().Contain("Total tests: 2. Passed: 1. Failed: 1. Skipped: 0."); result.StdOut.Should().Contain("Passed TestNamespace.VSTestTests.VSTestPassTest"); result.StdOut.Should().Contain("Failed TestNamespace.VSTestTests.VSTestFailTest"); + result.ExitCode.Should().Be(1); } [Fact] @@ -65,6 +66,7 @@ namespace Microsoft.DotNet.Cli.Test.Tests result.StdOut.Should().Contain("Total tests: 2. Passed: 1. Failed: 1. Skipped: 0."); result.StdOut.Should().Contain("Passed TestNamespace.VSTestXunitTests.VSTestXunitPassTest"); result.StdOut.Should().Contain("Failed TestNamespace.VSTestXunitTests.VSTestXunitFailTest"); + result.ExitCode.Should().Be(1); } [Fact] @@ -138,4 +140,4 @@ namespace Microsoft.DotNet.Cli.Test.Tests } } } -} \ No newline at end of file +} diff --git a/test/dotnet-vstest.Tests/VSTestTests.cs b/test/dotnet-vstest.Tests/VSTestTests.cs index 0c938737d..8a9a28eb3 100644 --- a/test/dotnet-vstest.Tests/VSTestTests.cs +++ b/test/dotnet-vstest.Tests/VSTestTests.cs @@ -32,12 +32,12 @@ namespace Microsoft.DotNet.Cli.VSTest.Tests var argsForVstest = $"\"{outputDll.FullName}\""; // Call vstest - new VSTestCommand() - .ExecuteWithCapturedOutput(argsForVstest) - .StdOut + var result = new VSTestCommand().ExecuteWithCapturedOutput(argsForVstest); + result.StdOut .Should().Contain("Total tests: 2. Passed: 1. Failed: 1. Skipped: 0.") - .And.Contain("Passed TestNamespace.VSTestTests.VSTestPassTest") - .And.Contain("Failed TestNamespace.VSTestTests.VSTestFailTest"); + .And.Contain("Passed TestNamespace.VSTestTests.VSTestPassTest") + .And.Contain("Failed TestNamespace.VSTestTests.VSTestFailTest"); + result.ExitCode.Should().Be(1); } } }