From a88a82734f90545c00f2ccb59d93b8aa5b0853c6 Mon Sep 17 00:00:00 2001 From: Satya Madala Date: Tue, 17 Jul 2018 22:24:10 +0530 Subject: [PATCH] Add tests for Code coverage with dotnet test (#9355) Add tests for dotnet test --collect "Code Coverage" scenario. Fixes Microsoft/vstest#981. --- .../TestProjects/VSTestCore/VSTestCore.csproj | 4 +- .../VSTestDesktopAndNetCore.csproj | 4 +- .../TestProjects/XunitCore/XunitCore.csproj | 4 +- .../VSTestXunitDesktopAndNetCore.csproj | 4 +- build/DependencyVersions.props | 2 + build/NugetConfigFile.targets | 1 + .../CollectCodeCoverage.runsettings | 25 +++++ ...ildsAndRunsTestFromCsprojForMultipleTFM.cs | 30 ++++++ ...enDotnetTestBuildsAndRunsTestfromCsproj.cs | 95 +++++++++++++++++++ .../dotnet-test.Tests.csproj | 5 + 10 files changed, 166 insertions(+), 8 deletions(-) create mode 100644 test/dotnet-test.Tests/CollectCodeCoverage.runsettings diff --git a/TestAssets/TestProjects/VSTestCore/VSTestCore.csproj b/TestAssets/TestProjects/VSTestCore/VSTestCore.csproj index 26592bc1d..95f0c2918 100644 --- a/TestAssets/TestProjects/VSTestCore/VSTestCore.csproj +++ b/TestAssets/TestProjects/VSTestCore/VSTestCore.csproj @@ -7,8 +7,8 @@ - - + + \ No newline at end of file diff --git a/TestAssets/TestProjects/VSTestMulti/VSTestDesktopAndNetCore.csproj b/TestAssets/TestProjects/VSTestMulti/VSTestDesktopAndNetCore.csproj index 2b2cc7769..c657cca43 100644 --- a/TestAssets/TestProjects/VSTestMulti/VSTestDesktopAndNetCore.csproj +++ b/TestAssets/TestProjects/VSTestMulti/VSTestDesktopAndNetCore.csproj @@ -10,8 +10,8 @@ - - + + \ No newline at end of file diff --git a/TestAssets/TestProjects/XunitCore/XunitCore.csproj b/TestAssets/TestProjects/XunitCore/XunitCore.csproj index 84a3da759..e93fc550d 100644 --- a/TestAssets/TestProjects/XunitCore/XunitCore.csproj +++ b/TestAssets/TestProjects/XunitCore/XunitCore.csproj @@ -8,8 +8,8 @@ - - + + diff --git a/TestAssets/TestProjects/XunitMulti/VSTestXunitDesktopAndNetCore.csproj b/TestAssets/TestProjects/XunitMulti/VSTestXunitDesktopAndNetCore.csproj index 4534df62b..7269a3418 100644 --- a/TestAssets/TestProjects/XunitMulti/VSTestXunitDesktopAndNetCore.csproj +++ b/TestAssets/TestProjects/XunitMulti/VSTestXunitDesktopAndNetCore.csproj @@ -17,7 +17,7 @@ - - + + diff --git a/build/DependencyVersions.props b/build/DependencyVersions.props index 1be51ceae..fcc9ffe16 100644 --- a/build/DependencyVersions.props +++ b/build/DependencyVersions.props @@ -58,6 +58,8 @@ $(MicrosoftNETTestSdkPackageVersion) 0.2.0-beta-000042 0.2.0 + 1.3.2 + 2.3.1 1.5.1 diff --git a/build/NugetConfigFile.targets b/build/NugetConfigFile.targets index d7fd14404..535665c79 100644 --- a/build/NugetConfigFile.targets +++ b/build/NugetConfigFile.targets @@ -18,6 +18,7 @@ diff --git a/test/dotnet-test.Tests/CollectCodeCoverage.runsettings b/test/dotnet-test.Tests/CollectCodeCoverage.runsettings new file mode 100644 index 000000000..1119d1909 --- /dev/null +++ b/test/dotnet-test.Tests/CollectCodeCoverage.runsettings @@ -0,0 +1,25 @@ + + + + + + + + + + .*Test.dll + + + + + True + True + True + False + + + + + + + diff --git a/test/dotnet-test.Tests/GivenDotnetTestBuildsAndRunsTestFromCsprojForMultipleTFM.cs b/test/dotnet-test.Tests/GivenDotnetTestBuildsAndRunsTestFromCsprojForMultipleTFM.cs index 801879573..c64cdfc4e 100644 --- a/test/dotnet-test.Tests/GivenDotnetTestBuildsAndRunsTestFromCsprojForMultipleTFM.cs +++ b/test/dotnet-test.Tests/GivenDotnetTestBuildsAndRunsTestFromCsprojForMultipleTFM.cs @@ -84,6 +84,36 @@ namespace Microsoft.DotNet.Cli.Test.Tests result.ExitCode.Should().Be(1); } + [WindowsOnlyFact] + public void ItCreatesTwoCoverageFilesForMultiTargetedProject() + { + // Copy XunitMulti project in output directory of project dotnet-test.Tests + string testAppName = "XunitMulti"; + var testInstance = TestAssets.Get(testAppName) + .CreateInstance("3") + .WithSourceFiles(); + + var testProjectDirectory = testInstance.Root.FullName; + + string resultsDirectory = Path.Combine(testProjectDirectory, "RD"); + + // Delete resultsDirectory if it exist + if (Directory.Exists(resultsDirectory)) + { + Directory.Delete(resultsDirectory, true); + } + + // Call test + CommandResult result = new DotnetTestCommand() + .WithWorkingDirectory(testProjectDirectory) + .ExecuteWithCapturedOutput($"{TestBase.ConsoleLoggerOutputNormal} --collect \"Code Coverage\" --results-directory {resultsDirectory}"); + + // Verify + DirectoryInfo d = new DirectoryInfo(resultsDirectory); + FileInfo[] coverageFileInfos = d.GetFiles("*.coverage", SearchOption.AllDirectories); + Assert.Equal(2, coverageFileInfos.Length); + } + [Fact] public void ItCanTestAMultiTFMProjectWithImplicitRestore() { diff --git a/test/dotnet-test.Tests/GivenDotnetTestBuildsAndRunsTestfromCsproj.cs b/test/dotnet-test.Tests/GivenDotnetTestBuildsAndRunsTestfromCsproj.cs index 1a8dddf88..7121959fe 100644 --- a/test/dotnet-test.Tests/GivenDotnetTestBuildsAndRunsTestfromCsproj.cs +++ b/test/dotnet-test.Tests/GivenDotnetTestBuildsAndRunsTestfromCsproj.cs @@ -9,6 +9,7 @@ using Microsoft.DotNet.Cli.Utils; using System.IO; using System; using System.Runtime.CompilerServices; +using System.Xml; namespace Microsoft.DotNet.Cli.Test.Tests { @@ -311,6 +312,100 @@ namespace Microsoft.DotNet.Cli.Test.Tests result.ExitCode.Should().Be(1); } + + [WindowsOnlyFact] + public void ItCreatesCoverageFileWhenCodeCoverageEnabledByRunsettings() + { + var testProjectDirectory = this.CopyAndRestoreVSTestDotNetCoreTestApp("11"); + + string resultsDirectory = Path.Combine(testProjectDirectory, "RD"); + + // Delete resultsDirectory if it exist + if (Directory.Exists(resultsDirectory)) + { + Directory.Delete(resultsDirectory, true); + } + + var settingsPath =Path.Combine(AppContext.BaseDirectory, "CollectCodeCoverage.runsettings"); + + // Call test + CommandResult result = new DotnetTestCommand() + .WithWorkingDirectory(testProjectDirectory) + .ExecuteWithCapturedOutput( + "--settings " + settingsPath + + " --results-directory " + resultsDirectory); + + // Verify test results + if (!DotnetUnderTest.IsLocalized()) + { + result.StdOut.Should().Contain("Total tests: 2. Passed: 1. Failed: 1. Skipped: 0."); + } + + // Verify coverage file. + DirectoryInfo d = new DirectoryInfo(resultsDirectory); + FileInfo[] coverageFileInfos = d.GetFiles("*.coverage", SearchOption.AllDirectories); + Assert.Equal(1, coverageFileInfos.Length); + + result.ExitCode.Should().Be(1); + } + + [WindowsOnlyFact] + public void ItCreatesCoverageFileInResultsDirectory() + { + var testProjectDirectory = this.CopyAndRestoreVSTestDotNetCoreTestApp("12"); + + string resultsDirectory = Path.Combine(testProjectDirectory, "RD"); + + // Delete resultsDirectory if it exist + if (Directory.Exists(resultsDirectory)) + { + Directory.Delete(resultsDirectory, true); + } + + // Call test + CommandResult result = new DotnetTestCommand() + .WithWorkingDirectory(testProjectDirectory) + .ExecuteWithCapturedOutput( + "--collect \"Code Coverage\" " + + "--results-directory " + resultsDirectory); + + // Verify test results + if (!DotnetUnderTest.IsLocalized()) + { + result.StdOut.Should().Contain("Total tests: 2. Passed: 1. Failed: 1. Skipped: 0."); + } + + // Verify coverage file. + DirectoryInfo d = new DirectoryInfo(resultsDirectory); + FileInfo[] coverageFileInfos = d.GetFiles("*.coverage", SearchOption.AllDirectories); + Assert.Equal(1, coverageFileInfos.Length); + + result.ExitCode.Should().Be(1); + } + + [UnixOnlyFact] + public void ItShouldShowWarningMessageOnCollectCodeCoverage() + { + var testProjectDirectory = this.CopyAndRestoreVSTestDotNetCoreTestApp("13"); + + // Call test + CommandResult result = new DotnetTestCommand() + .WithWorkingDirectory(testProjectDirectory) + .ExecuteWithCapturedOutput( + "--collect \"Code Coverage\" " + + "--filter VSTestPassTest"); + + // Verify test results + if (!DotnetUnderTest.IsLocalized()) + { + result.StdOut.Should().Contain("No code coverage data available. Code coverage is currently supported only on Windows."); + result.StdOut.Should().Contain("Total tests: 1. Passed: 1. Failed: 0. Skipped: 0."); + result.StdOut.Should().Contain("Test Run Successful."); + } + + result.ExitCode.Should().Be(0); + } + private string CopyAndRestoreVSTestDotNetCoreTestApp([CallerMemberName] string callingMethod = "") { // Copy VSTestCore project in output directory of project dotnet-vstest.Tests diff --git a/test/dotnet-test.Tests/dotnet-test.Tests.csproj b/test/dotnet-test.Tests/dotnet-test.Tests.csproj index fc0a3a72e..c465f7227 100644 --- a/test/dotnet-test.Tests/dotnet-test.Tests.csproj +++ b/test/dotnet-test.Tests/dotnet-test.Tests.csproj @@ -16,4 +16,9 @@ + + + Always + +