From 215108835e503905d95c5a146abfa34bda601924 Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Fri, 18 Feb 2022 06:51:43 -0800 Subject: [PATCH] [release/6.0.1xx] Add source-build tests for dotnet format (#13207) * add dotnet format tests * some updates from code review * move helper methods out of DotNetFormatTests class * rearrange smoke test assets * update various smoke-test paths and remove root repo check from legacy smoke test script * fix smoke-test-prereqs tar command path * fix naming conventions and remove extra formatting test * more refactoring from pr comments * refactor BaselineHelper comparison methods * simplify comparison methods * clarify FormatTest expected result naming * remove unnecessary empty props and targets files --- src/SourceBuild/tarball/content/build.proj | 6 +-- .../BaselineHelper.cs | 30 ++++++++---- .../DotNetFormatTests.cs | 47 +++++++++++++++++++ .../DotNetHelper.cs | 15 +++++- ...osoft.DotNet.SourceBuild.SmokeTests.csproj | 13 +++-- .../SdkContentTests.cs | 2 +- .../SmokeTests.cs | 2 +- .../XmlDocTests.cs | 2 +- .../assets/FormatTestExpectedFormatted.cs | 11 +++++ .../assets/FormatTestUnformatted.cs | 11 +++++ .../{ => assets}/baselines/MissingXmlDoc.txt | 0 .../{ => assets}/baselines/MsftToSbSdk.diff | 0 .../smoke-tests/local.NuGet.Config | 0 .../smoke-tests/online.NuGet.Config | 0 .../{ => assets}/smoke-tests/smoke-test.sh | 8 +--- 15 files changed, 116 insertions(+), 31 deletions(-) create mode 100644 src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/DotNetFormatTests.cs create mode 100644 src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/FormatTestExpectedFormatted.cs create mode 100644 src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/FormatTestUnformatted.cs rename src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/{ => assets}/baselines/MissingXmlDoc.txt (100%) rename src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/{ => assets}/baselines/MsftToSbSdk.diff (100%) rename src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/{ => assets}/smoke-tests/local.NuGet.Config (100%) rename src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/{ => assets}/smoke-tests/online.NuGet.Config (100%) rename src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/{ => assets}/smoke-tests/smoke-test.sh (98%) diff --git a/src/SourceBuild/tarball/content/build.proj b/src/SourceBuild/tarball/content/build.proj index f2bdcbe28..83b0ae6c8 100644 --- a/src/SourceBuild/tarball/content/build.proj +++ b/src/SourceBuild/tarball/content/build.proj @@ -21,10 +21,6 @@ - - - - @@ -128,7 +124,7 @@ + WorkingDirectory="./test/Microsoft.DotNet.SourceBuild.SmokeTests/bin/$(Configuration)/net6.0/assets/smoke-tests/prereq-packages/"/> diff --git a/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/BaselineHelper.cs b/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/BaselineHelper.cs index 3fc18b357..b2c0908b9 100644 --- a/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/BaselineHelper.cs +++ b/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/BaselineHelper.cs @@ -14,7 +14,7 @@ namespace Microsoft.DotNet.SourceBuild.SmokeTests { internal class BaselineHelper { - public static void Compare(string baselineFileName, IOrderedEnumerable actualEntries) + public static void CompareEntries(string baselineFileName, IOrderedEnumerable actualEntries) { IEnumerable baseline = File.ReadAllLines(GetBaselineFilePath(baselineFileName)); string[] missingEntries = actualEntries.Except(baseline).ToArray(); @@ -34,20 +34,28 @@ namespace Microsoft.DotNet.SourceBuild.SmokeTests Assert.Null(message); } - public static void Compare(string baselineFileName, string actual, ITestOutputHelper outputHelper) + public static void CompareContents(string baselineFileName, string actualContents, ITestOutputHelper outputHelper) { string baselineFilePath = GetBaselineFilePath(baselineFileName); - string baseline = File.ReadAllText(baselineFilePath); + + string actualFilePath = Path.Combine(Environment.CurrentDirectory, $"{baselineFileName}"); + File.WriteAllText(actualFilePath, actualContents); + + CompareFiles(baselineFilePath, actualFilePath, outputHelper); + } + + public static void CompareFiles(string baselineFilePath, string actualFilePath, ITestOutputHelper outputHelper) + { + string baselineFileText = File.ReadAllText(baselineFilePath); + string actualFileText = File.ReadAllText(actualFilePath); string? message = null; - if (baseline != actual) - { - string actualBaselineFilePath = Path.Combine(Environment.CurrentDirectory, $"{baselineFileName}"); - File.WriteAllText(actualBaselineFilePath, actual); + if (baselineFileText != actualFileText) + { // Retrieve a diff in order to provide a UX which calls out the diffs. - string diff = DiffFiles(baselineFilePath, actualBaselineFilePath, outputHelper); - message = $"{Environment.NewLine}Baseline '{baselineFilePath}' does not match actual '{actualBaselineFilePath}`. {Environment.NewLine}" + string diff = DiffFiles(baselineFilePath, actualFilePath, outputHelper); + message = $"{Environment.NewLine}Baseline '{baselineFilePath}' does not match actual '{actualFilePath}`. {Environment.NewLine}" + $"{diff}{Environment.NewLine}"; } @@ -63,6 +71,8 @@ namespace Microsoft.DotNet.SourceBuild.SmokeTests return diffResult.StdOut; } - private static string GetBaselineFilePath(string baselineFileName) => Path.Combine(Directory.GetCurrentDirectory(), "baselines", baselineFileName); + public static string GetAssetsDirectory() => Path.Combine(Directory.GetCurrentDirectory(), "assets"); + + private static string GetBaselineFilePath(string baselineFileName) => Path.Combine(GetAssetsDirectory(), "baselines", baselineFileName); } } diff --git a/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/DotNetFormatTests.cs b/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/DotNetFormatTests.cs new file mode 100644 index 000000000..b50a7b1d0 --- /dev/null +++ b/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/DotNetFormatTests.cs @@ -0,0 +1,47 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.IO; +using Xunit; +using Xunit.Abstractions; + +namespace Microsoft.DotNet.SourceBuild.SmokeTests; + +public class DotNetFormatTests +{ + private const string UnformattedFileName = "FormatTestUnformatted.cs"; + private const string ExpectedFormattedFileName = "FormatTestExpectedFormatted.cs"; + + private ITestOutputHelper OutputHelper { get; } + private DotNetHelper DotNetHelper { get; } + + public DotNetFormatTests(ITestOutputHelper outputHelper) + { + OutputHelper = outputHelper; + DotNetHelper = new DotNetHelper(outputHelper); + } + + /// + /// Format an unformatted project and verify that the output matches the pre-computed solution. + /// + [Fact] + public void FormatProject() + { + string assetsDirectory = BaselineHelper.GetAssetsDirectory(); + + string unformattedCsFilePath = Path.Combine(assetsDirectory, UnformattedFileName); + string expectedFormattedCsFilePath = Path.Combine(assetsDirectory, ExpectedFormattedFileName); + + string projectDirectory = DotNetHelper.ExecuteNew("console", "C#", nameof(FormatProject), OutputHelper); + + string projectFilePath = Path.Combine(projectDirectory, nameof(FormatProject) + ".csproj"); + string formattedCsFilePath = Path.Combine(projectDirectory, UnformattedFileName); + + File.Copy(unformattedCsFilePath, formattedCsFilePath); + + DotNetHelper.ExecuteCmd($"format {projectFilePath}", OutputHelper); + + BaselineHelper.CompareFiles(expectedFormattedCsFilePath, formattedCsFilePath, OutputHelper); + } +} diff --git a/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/DotNetHelper.cs b/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/DotNetHelper.cs index d3385c37e..c0ba6fac7 100644 --- a/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/DotNetHelper.cs +++ b/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/DotNetHelper.cs @@ -13,6 +13,7 @@ namespace Microsoft.DotNet.SourceBuild.SmokeTests; internal class DotNetHelper { private static readonly object s_lockObj = new object(); + private static readonly string s_timestamp = DateTime.Now.ToString("yyyyMMddHHmmssffff"); public string DotNetPath { get; } @@ -35,10 +36,22 @@ internal class DotNetHelper DotNetPath = Path.Combine(Config.DotNetDirectory, "dotnet"); } - public void ExecuteDotNetCmd(string args, ITestOutputHelper outputHelper) + public void ExecuteCmd(string args, ITestOutputHelper outputHelper) { (Process Process, string StdOut, string StdErr) executeResult = ExecuteHelper.ExecuteProcess(DotNetPath, args, outputHelper); Assert.Equal(0, executeResult.Process.ExitCode); } + + /// + /// Create a new .NET project and return the path to the created project folder. + /// + public string ExecuteNew(string projectType, string language, string name, ITestOutputHelper outputHelper) + { + string outputDirectory = Path.Combine(Directory.GetCurrentDirectory(), $"projects-{s_timestamp}", name); + + ExecuteCmd($"new {projectType} --language \"{language}\" --name {name} --output {outputDirectory}", outputHelper); + + return outputDirectory; + } } diff --git a/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Microsoft.DotNet.SourceBuild.SmokeTests.csproj b/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Microsoft.DotNet.SourceBuild.SmokeTests.csproj index 0846de14b..be458761b 100644 --- a/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Microsoft.DotNet.SourceBuild.SmokeTests.csproj +++ b/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Microsoft.DotNet.SourceBuild.SmokeTests.csproj @@ -18,19 +18,22 @@ - - Always - - + Always + + + + + + - + diff --git a/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SdkContentTests.cs b/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SdkContentTests.cs index 774d76c33..7feae87f3 100644 --- a/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SdkContentTests.cs +++ b/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SdkContentTests.cs @@ -53,7 +53,7 @@ public class SdkContentTests diff = RemoveVersionedPaths(diff); diff = RemoveDiffMarkers(diff); diff = RemoveRids(diff); - BaselineHelper.Compare("MsftToSbSdk.diff", diff, OutputHelper); + BaselineHelper.CompareContents("MsftToSbSdk.diff", diff, OutputHelper); } private void WriteTarballFileList(string tarballPath, string outputFileName) diff --git a/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SmokeTests.cs b/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SmokeTests.cs index 6ebd64e7d..166883a58 100644 --- a/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SmokeTests.cs +++ b/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SmokeTests.cs @@ -38,7 +38,7 @@ public class SmokeTests smokeTestArgs += " --excludeOnlineTests"; } - (Process Process, string StdOut, string StdErr) executeResult = ExecuteHelper.ExecuteProcess("./smoke-tests/smoke-test.sh", smokeTestArgs, OutputHelper); + (Process Process, string StdOut, string StdErr) executeResult = ExecuteHelper.ExecuteProcess("./assets/smoke-tests/smoke-test.sh", smokeTestArgs, OutputHelper); Assert.Equal(0, executeResult.Process.ExitCode); } diff --git a/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/XmlDocTests.cs b/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/XmlDocTests.cs index 01168ef4e..46088361f 100644 --- a/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/XmlDocTests.cs +++ b/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/XmlDocTests.cs @@ -48,6 +48,6 @@ public class XmlDocTests } } - BaselineHelper.Compare("MissingXmlDoc.txt", missingXmlDoc.OrderBy(entry => entry)); + BaselineHelper.CompareEntries("MissingXmlDoc.txt", missingXmlDoc.OrderBy(entry => entry)); } } diff --git a/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/FormatTestExpectedFormatted.cs b/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/FormatTestExpectedFormatted.cs new file mode 100644 index 000000000..81d4a3a79 --- /dev/null +++ b/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/FormatTestExpectedFormatted.cs @@ -0,0 +1,11 @@ +namespace FormatTest +{ + internal class Test + { + public void Run() + { + string test = "Hello World"; + Console.WriteLine(test); + } + } +} diff --git a/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/FormatTestUnformatted.cs b/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/FormatTestUnformatted.cs new file mode 100644 index 000000000..41dacdf1c --- /dev/null +++ b/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/FormatTestUnformatted.cs @@ -0,0 +1,11 @@ +namespace FormatTest +{ + internal class Test + { + public void Run() + { + string test = "Hello World" ; + Console .WriteLine( test ); + } + } +} diff --git a/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/baselines/MissingXmlDoc.txt b/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/baselines/MissingXmlDoc.txt similarity index 100% rename from src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/baselines/MissingXmlDoc.txt rename to src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/baselines/MissingXmlDoc.txt diff --git a/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/baselines/MsftToSbSdk.diff b/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/baselines/MsftToSbSdk.diff similarity index 100% rename from src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/baselines/MsftToSbSdk.diff rename to src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/baselines/MsftToSbSdk.diff diff --git a/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/smoke-tests/local.NuGet.Config b/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/smoke-tests/local.NuGet.Config similarity index 100% rename from src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/smoke-tests/local.NuGet.Config rename to src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/smoke-tests/local.NuGet.Config diff --git a/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/smoke-tests/online.NuGet.Config b/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/smoke-tests/online.NuGet.Config similarity index 100% rename from src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/smoke-tests/online.NuGet.Config rename to src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/smoke-tests/online.NuGet.Config diff --git a/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/smoke-tests/smoke-test.sh b/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/smoke-tests/smoke-test.sh similarity index 98% rename from src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/smoke-tests/smoke-test.sh rename to src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/smoke-tests/smoke-test.sh index e7edfed15..797086396 100755 --- a/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/smoke-tests/smoke-test.sh +++ b/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/smoke-tests/smoke-test.sh @@ -7,8 +7,7 @@ VERSION_PREFIX=6.0 # See https://github.com/dotnet/source-build/issues/579, this version # needs to be compatible with the runtime produced from source-build DEV_CERTS_VERSION_DEFAULT=6.0.0-preview.6.21355.2 -ARTIFACTS_DIR="$SCRIPT_ROOT/../../../../../../artifacts/" -__ROOT_REPO=$(sed 's/\r$//' "${ARTIFACTS_DIR}obj/rootrepo.txt") # remove CR if mounted repo on Windows drive +ARTIFACTS_DIR="$SCRIPT_ROOT/../../../../../../../artifacts/" executingUserHome=${HOME:-} export DOTNET_CLI_TELEMETRY_OPTOUT=1 @@ -456,11 +455,6 @@ function copyRestoredPackages() { echo "RID to test: ${targetRid?not specified. Use ./build.sh --run-smoke-test to detect RID, or specify manually.}" -if [ "$__ROOT_REPO" != "known-good" ]; then - echo "Skipping smoke-tests since cli was not built"; - exit -fi - # Clean up and create directory if [ -e "$testingDir" ]; then rm -rf "$testingDir"