dotnet test and publish failing badly when the project isn't restored.

Fixing this by checking for diagnostic errors before continuing.

Fix #2692
Fix #2942

src/Microsoft.DotNet.Cli.Utils/Extensions/CollectionsExtensions.cs

@@ -1,4 +1,6 @@
-using System;
+// Copyright (c) .NET Foundation and contributors. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
 using System.Collections.Generic;
 using System.Linq;
 

src/Microsoft.DotNet.Cli.Utils/Extensions/ProjectContextCollectionExtensions.cs

@@ -0,0 +1,51 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using Microsoft.DotNet.ProjectModel;
+
+namespace Microsoft.DotNet.Cli.Utils
+{
+    public static class ProjectContextCollectionExtensions
+    {
+        public static ProjectContextCollection EnsureValid(this ProjectContextCollection contextCollection, string projectFilePath)
+        {
+            IEnumerable<DiagnosticMessage> errors;
+
+            if (contextCollection == null)
+            {
+                errors = new[]
+                {
+                    new DiagnosticMessage(
+                        ErrorCodes.DOTNET1017,
+                        $"Project file does not exist '{ProjectPathHelper.NormalizeProjectFilePath(projectFilePath)}'.",
+                        projectFilePath,
+                        DiagnosticMessageSeverity.Error)
+                };
+            }
+            else
+            {
+                errors = contextCollection
+                            .ProjectDiagnostics
+                            .Where(d => d.Severity == DiagnosticMessageSeverity.Error);
+            }
+
+            if (errors.Any())
+            {
+                StringBuilder errorMessage = new StringBuilder($"The current project is not valid because of the following errors:{Environment.NewLine}");
+
+                foreach (DiagnosticMessage message in errors)
+                {
+                    errorMessage.AppendLine(message.FormattedMessage);
+                }
+
+                throw new GracefulException(errorMessage.ToString());
+            }
+            
+            return contextCollection;
+        }
+    }
+}

src/Microsoft.DotNet.ProjectModel/DesignTimeWorkspace.cs

@@ -19,7 +19,7 @@ namespace Microsoft.DotNet.ProjectModel
 
         public void AddProject(string path)
         {
-            var projectPath = NormalizeProjectPath(path);
+            var projectPath = ProjectPathHelper.NormalizeProjectDirectoryPath(path);
 
             if (projectPath != null)
             {

src/Microsoft.DotNet.ProjectModel/ErrorCodes.DotNet.cs

@@ -22,5 +22,8 @@ namespace Microsoft.DotNet.ProjectModel
 
         // The '{0}' option in the root is deprecated. Use it in '{1}' instead.
         public static readonly string DOTNET1016 = nameof(DOTNET1016);
+
+        // Project file does not exist '{0}'.
+        public static readonly string DOTNET1017 = nameof(DOTNET1017);
     }
 }

src/Microsoft.DotNet.ProjectModel/ProjectContext.cs

@@ -42,6 +42,8 @@ namespace Microsoft.DotNet.ProjectModel
 
         public LibraryManager LibraryManager { get; }
 
+        public List<DiagnosticMessage> Diagnostics { get; }
+
         internal ProjectContext(
             GlobalSettings globalSettings,
             ProjectDescription rootProject,
@@ -51,7 +53,8 @@ namespace Microsoft.DotNet.ProjectModel
             string runtimeIdentifier,
             string packagesDirectory,
             LibraryManager libraryManager,
-            LockFile lockfile)
+            LockFile lockfile,
+            List<DiagnosticMessage> diagnostics)
         {
             Identity = new ProjectContextIdentity(rootProject?.Path, targetFramework);
             GlobalSettings = globalSettings;
@@ -63,6 +66,7 @@ namespace Microsoft.DotNet.ProjectModel
             LibraryManager = libraryManager;
             LockFile = lockfile;
             IsPortable = isPortable;
+            Diagnostics = diagnostics;
         }
 
         public LibraryExporter CreateExporter(string configuration, string buildBasePath = null)

src/Microsoft.DotNet.ProjectModel/ProjectContextBuilder.cs

@@ -355,11 +355,6 @@ namespace Microsoft.DotNet.ProjectModel
                 }
             }
 
-            if (Project != null)
-            {
-                diagnostics.AddRange(Project.Diagnostics);
-            }
-
             // Create a library manager
             var libraryManager = new LibraryManager(libraries.Values.ToList(), diagnostics, Project?.ProjectFilePath);
 
@@ -372,7 +367,8 @@ namespace Microsoft.DotNet.ProjectModel
                 runtime,
                 PackagesDirectory,
                 libraryManager,
-                LockFile);
+                LockFile,
+                diagnostics);
         }
 
         private void ReadLockFile(ICollection<DiagnosticMessage> diagnostics)

src/Microsoft.DotNet.ProjectModel/ProjectPathHelper.cs

@@ -6,7 +6,7 @@ using System.IO;
 
 namespace Microsoft.DotNet.ProjectModel
 {
-    internal static class ProjectPathHelper
+    public static class ProjectPathHelper
     {
         public static string NormalizeProjectDirectoryPath(string path)
         {

src/Microsoft.DotNet.ProjectModel/Workspace.cs

@@ -55,7 +55,7 @@ namespace Microsoft.DotNet.ProjectModel
 
         public ProjectContextCollection GetProjectContextCollection(string projectPath)
         {
-            var normalizedPath = NormalizeProjectPath(projectPath);
+            var normalizedPath = ProjectPathHelper.NormalizeProjectDirectoryPath(projectPath);
             if (normalizedPath == null)
             {
                 return null;
@@ -71,7 +71,7 @@ namespace Microsoft.DotNet.ProjectModel
 
         private LockFile GetLockFile(string projectDirectory)
         {
-            var normalizedPath = NormalizeProjectPath(projectDirectory);
+            var normalizedPath = ProjectPathHelper.NormalizeProjectDirectoryPath(projectDirectory);
             if (normalizedPath == null)
             {
                 return null;
@@ -86,7 +86,7 @@ namespace Microsoft.DotNet.ProjectModel
 
         private FileModelEntry<Project> GetProjectCore(string projectDirectory)
         {
-            var normalizedPath = NormalizeProjectPath(projectDirectory);
+            var normalizedPath = ProjectPathHelper.NormalizeProjectDirectoryPath(projectDirectory);
             if (normalizedPath == null)
             {
                 return null;
@@ -98,23 +98,6 @@ namespace Microsoft.DotNet.ProjectModel
                 (key, oldEntry) => AddProjectEntry(key, oldEntry));
         }
 
-
-        protected static string NormalizeProjectPath(string path)
-        {
-            if (File.Exists(path) &&
-                string.Equals(Path.GetFileName(path), Project.FileName, StringComparison.OrdinalIgnoreCase))
-            {
-                return Path.GetDirectoryName(Path.GetFullPath(path));
-            }
-            else if (Directory.Exists(path) &&
-                     File.Exists(Path.Combine(path, Project.FileName)))
-            {
-                return Path.GetFullPath(path);
-            }
-
-            return null;
-        }
-
         private FileModelEntry<Project> AddProjectEntry(string projectDirectory, FileModelEntry<Project> currentEntry)
         {
             if (currentEntry == null)
@@ -227,6 +210,8 @@ namespace Microsoft.DotNet.ProjectModel
                 }
 
                 currentEntry.ProjectDiagnostics.AddRange(projectEntry.Diagnostics);
+                currentEntry.ProjectDiagnostics.AddRange(
+                    currentEntry.ProjectContexts.SelectMany(c => c.Diagnostics));
             }
 
             return currentEntry;

src/dotnet/Program.cs

@@ -61,7 +61,7 @@ namespace Microsoft.DotNet.Cli
             }
             catch (GracefulException e)
             {
-                Console.WriteLine(CommandContext.IsVerbose() ? e.ToString().Red().Bold() : e.Message.Red().Bold());
+                Reporter.Error.WriteLine(CommandContext.IsVerbose() ? e.ToString().Red().Bold() : e.Message.Red().Bold());
 
                 return 1;
             }

src/dotnet/commands/dotnet-publish/PublishCommand.cs

@@ -416,7 +416,9 @@ namespace Microsoft.DotNet.Tools.Publish
                 throw new InvalidProjectException($"'{projectPath}' does not contain a project.json file");
             }
 
-            var contexts = Workspace.GetProjectContextCollection(projectPath).FrameworkOnlyContexts;
+            var contexts = Workspace.GetProjectContextCollection(projectPath)
+                .EnsureValid(projectPath)
+                .FrameworkOnlyContexts;
 
             contexts = framework == null ?
                 contexts :

src/dotnet/commands/dotnet-run/RunCommand.cs

@@ -68,7 +68,9 @@ namespace Microsoft.DotNet.Tools.Run
                 Configuration = Constants.DefaultConfiguration;
             }
 
-            var frameworkContexts = _workspace.GetProjectContextCollection(Project).FrameworkOnlyContexts;
+            var frameworkContexts = _workspace.GetProjectContextCollection(Project)
+                .EnsureValid(Project)
+                .FrameworkOnlyContexts;
 
             var rids = RuntimeEnvironmentRidExtensions.GetAllCandidateRuntimeIdentifiers();
 

src/dotnet/commands/dotnet-test/Program.cs

@@ -65,8 +65,9 @@ namespace Microsoft.DotNet.Tools.Test
                 else
                 {
                     var summary = new Summary();
-                    var projectContexts = workspace
-                        .GetProjectContextCollection(projectPath).FrameworkOnlyContexts
+                    var projectContexts = workspace.GetProjectContextCollection(projectPath)
+                        .EnsureValid(projectPath)
+                        .FrameworkOnlyContexts
                         .Select(c => workspace.GetRuntimeContext(c, runtimeIdentifiers))
                         .ToList();
 
@@ -100,7 +101,7 @@ namespace Microsoft.DotNet.Tools.Test
                 TestHostTracing.Source.TraceEvent(TraceEventType.Error, 0, ex.ToString());
                 return -1;
             }
-            catch (Exception ex)
+            catch (Exception ex) when (!(ex is GracefulException))
             {
                 TestHostTracing.Source.TraceEvent(TraceEventType.Error, 0, ex.ToString());
                 return -2;

test/Microsoft.DotNet.Tools.Tests.Utilities/Commands/DotnetTestCommand.cs

@@ -16,5 +16,11 @@ namespace Microsoft.DotNet.Tools.Test.Utilities
             args = $"test {args}";
             return base.Execute(args);
         }
+        
+        public override CommandResult ExecuteWithCapturedOutput(string args = "")
+        {
+            args = $"test {args}";
+            return base.ExecuteWithCapturedOutput(args);
+        }
     }
 }

test/dotnet-publish.Tests/PublishTests.cs

@@ -372,5 +372,21 @@ namespace Microsoft.DotNet.Tools.Publish.Tests
                 .Should()
                 .Pass();
         }
+
+        [Fact]
+        public void PublishFailsCorrectlyWithUnrestoredProject()
+        {
+            // NOTE: we don't say "WithLockFiles", so the project is "unrestored"
+            TestInstance instance = TestAssetsManager.CreateTestInstance("TestAppSimple");
+
+            new PublishCommand(instance.TestRoot)
+                .ExecuteWithCapturedOutput()
+                .Should()
+                .Fail()
+                .And
+                .HaveStdErrContaining("NU1009")
+                .And
+                .HaveStdErrContaining("dotnet restore");
+        }
     }
 }

test/dotnet-run.Tests/RunTests.cs

@@ -155,6 +155,35 @@ namespace Microsoft.DotNet.Tools.Run.Tests
                         "arg: [two]"));
         }
 
+        [Fact]
+        public void ItHandlesUnrestoredProjectFileCorrectly()
+        {
+            // NOTE: we don't say "WithLockFiles", so the project is "unrestored"
+            TestInstance instance = TestAssetsManager.CreateTestInstance("TestAppSimple");
+
+            new RunCommand(instance.TestRoot)
+                .ExecuteWithCapturedOutput()
+                .Should()
+                .Fail()
+                .And
+                .HaveStdErrContaining("NU1009")
+                .And
+                .HaveStdErrContaining("dotnet restore");
+        }
+
+        [Fact]
+        public void ItHandlesUnknownProjectFileCorrectly()
+        {
+            new RunCommand("bad path")
+                .ExecuteWithCapturedOutput()
+                .Should()
+                .Fail()
+                .And
+                .HaveStdErrContaining("DOTNET1017")
+                .And
+                .HaveStdErrContaining("bad path");
+        }
+
         private static string JoinWithNewlines(params string[] values)
         {
             return string.Join(Environment.NewLine, values);

test/dotnet-test.Tests/GivenThatWeWantToRunTests.cs

@@ -0,0 +1,29 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System.IO;
+using Microsoft.DotNet.TestFramework;
+using Microsoft.DotNet.Tools.Test.Utilities;
+using Xunit;
+
+namespace Microsoft.Dotnet.Tools.Test.Tests
+{
+    public class GivenThatWeWantToRunTests : TestBase
+    {
+        [Fact]
+        public void It_fails_correctly_with_an_unrestored_project()
+        {
+            // NOTE: we don't say "WithLockFiles", so the project is "unrestored"
+            var instance = TestAssetsManager.CreateTestInstance(Path.Combine("ProjectsWithTests", "NetCoreAppOnlyProject"));
+
+            new DotnetTestCommand()
+                .ExecuteWithCapturedOutput(instance.TestRoot)
+                .Should()
+                .Fail()
+                .And
+                .HaveStdErrContaining("NU1009")
+                .And
+                .HaveStdErrContaining("dotnet restore");
+        }
+    }
+}

test/dotnet-test.Tests/GivenThatWeWantToRunTestsForMultipleTFMsInTheConsole.cs

@@ -44,7 +44,7 @@ namespace Microsoft.Dotnet.Tools.Test.Tests
         {
             var testCommand = new DotnetTestCommand();
             var result = testCommand
-                .ExecuteWithCapturedOutput($"test {_projectFilePath}");
+                .ExecuteWithCapturedOutput($"{_projectFilePath}");
             result.Should().Pass();
             result.StdOut.Should().Contain("Skipped for NET451");
             result.StdOut.Should().Contain("Skipped for NETCOREAPP1.0");
@@ -55,7 +55,7 @@ namespace Microsoft.Dotnet.Tools.Test.Tests
         {
             var testCommand = new DotnetTestCommand();
             var result = testCommand
-                .ExecuteWithCapturedOutput($"test {_projectFilePath} -f net451");
+                .ExecuteWithCapturedOutput($"{_projectFilePath} -f net451");
             result.Should().Pass();
             result.StdOut.Should().Contain($"Skipped for NET451");
             result.StdOut.Should().NotContain($"Skipped for NETCOREAPP1.0");
@@ -66,7 +66,7 @@ namespace Microsoft.Dotnet.Tools.Test.Tests
         {
             var testCommand = new DotnetTestCommand();
             var result = testCommand
-                .ExecuteWithCapturedOutput($"test {_projectFilePath} -f netcoreapp1.0");
+                .ExecuteWithCapturedOutput($"{_projectFilePath} -f netcoreapp1.0");
             result.Should().Pass();
             result.StdOut.Should().Contain($"Skipped for NETCOREAPP1.0");
             result.StdOut.Should().NotContain($"Skipped for NET451");
@@ -121,7 +121,7 @@ namespace Microsoft.Dotnet.Tools.Test.Tests
             var nonExistentFramework = "doesnotexisttfm99.99";
             var testCommand = new DotnetTestCommand();
             var result = testCommand
-                .ExecuteWithCapturedOutput($"test {_projectFilePath} -f {nonExistentFramework}");
+                .ExecuteWithCapturedOutput($"{_projectFilePath} -f {nonExistentFramework}");
 
             result.Should().Fail();
             result.StdErr.Should().Contain($"does not support framework");
@@ -139,7 +139,7 @@ namespace Microsoft.Dotnet.Tools.Test.Tests
             };
 
             var result = testCommand
-                .ExecuteWithCapturedOutput($"test {_projectFilePath}");
+                .ExecuteWithCapturedOutput($"{_projectFilePath}");
 
             result.Should().Fail();
             result.StdOut.Should().Contain("Failing in NET451");

test/dotnet.Tests/PackagedCommandTests.cs

@@ -188,7 +188,7 @@ namespace Microsoft.DotNet.Tests
             {
                 CommandResult result = new HelloCommand().ExecuteWithCapturedOutput();
 
-                result.StdOut.Should().Contain("No executable found matching command");
+                result.StdErr.Should().Contain("No executable found matching command");
                 result.Should().Fail();
             }
             finally