Forward output unbuffered

src/Microsoft.DotNet.Cli.Utils/Command.cs

@@ -8,7 +8,8 @@ using System.IO;
 using System.Linq;
 using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
-using Microsoft.DotNet.Tools.Common;
+using System.Text;
+using System.Threading;
 using Microsoft.DotNet.ProjectModel;
 using NuGet.Frameworks;
 
@@ -16,16 +17,9 @@ namespace Microsoft.DotNet.Cli.Utils
 {
     public class Command
     {
-        private Process _process;
+        private readonly Process _process;
-
+        private readonly StreamForwarder _stdOut;
-        private StringWriter _stdOutCapture;
+        private readonly StreamForwarder _stdErr;
-        private StringWriter _stdErrCapture;
-
-        private Action<string> _stdOutForward;
-        private Action<string> _stdErrForward;
-
-        private Action<string> _stdOutHandler;
-        private Action<string> _stdErrHandler;
 
         private bool _running = false;
 
@@ -44,6 +38,9 @@ namespace Microsoft.DotNet.Cli.Utils
             {
                 StartInfo = psi
             };
+
+            _stdOut = new StreamForwarder();
+            _stdErr = new StreamForwarder();
         }
 
         public static Command Create(string executable, IEnumerable<string> args, NuGetFramework framework = null)
@@ -184,16 +181,6 @@ namespace Microsoft.DotNet.Cli.Utils
             ThrowIfRunning();
             _running = true;
 
-            _process.OutputDataReceived += (sender, args) =>
-            {
-                ProcessData(args.Data, _stdOutCapture, _stdOutForward, _stdOutHandler);
-            };
-
-            _process.ErrorDataReceived += (sender, args) =>
-            {
-                ProcessData(args.Data, _stdErrCapture, _stdErrForward, _stdErrHandler);
-            };
-
             _process.EnableRaisingEvents = true;
 
 #if DEBUG
@@ -204,10 +191,12 @@ namespace Microsoft.DotNet.Cli.Utils
 
             Reporter.Verbose.WriteLine($"Process ID: {_process.Id}");
 
-            _process.BeginOutputReadLine();
+            var threadOut = _stdOut.BeginRead(_process.StandardOutput);
-            _process.BeginErrorReadLine();
+            var threadErr = _stdErr.BeginRead(_process.StandardError);
 
             _process.WaitForExit();
+            threadOut.Join();
+            threadErr.Join();
 
             var exitCode = _process.ExitCode;
 
@@ -225,8 +214,8 @@ namespace Microsoft.DotNet.Cli.Utils
 
             return new CommandResult(
                 exitCode,
-                _stdOutCapture?.GetStringBuilder()?.ToString(),
+                _stdOut.GetCapturedOutput(),
-                _stdErrCapture?.GetStringBuilder()?.ToString());
+                _stdErr.GetCapturedOutput());
         }
 
         public Command WorkingDirectory(string projectDirectory)
@@ -244,14 +233,14 @@ namespace Microsoft.DotNet.Cli.Utils
         public Command CaptureStdOut()
         {
             ThrowIfRunning();
-            _stdOutCapture = new StringWriter();
+            _stdOut.Capture();
             return this;
         }
 
         public Command CaptureStdErr()
         {
             ThrowIfRunning();
-            _stdErrCapture = new StringWriter();
+            _stdErr.Capture();
             return this;
         }
 
@@ -262,11 +251,11 @@ namespace Microsoft.DotNet.Cli.Utils
             {
                 if (to == null)
                 {
-                    _stdOutForward = Reporter.Output.WriteLine;
+                    _stdOut.ForwardTo(write: Reporter.Output.Write, writeLine: Reporter.Output.WriteLine);
                 }
                 else
                 {
-                    _stdOutForward = to.WriteLine;
+                    _stdOut.ForwardTo(write: to.Write, writeLine: to.WriteLine);
                 }
             }
             return this;
@@ -279,11 +268,11 @@ namespace Microsoft.DotNet.Cli.Utils
             {
                 if (to == null)
                 {
-                    _stdErrForward = Reporter.Error.WriteLine;
+                    _stdErr.ForwardTo(write: Reporter.Error.Write, writeLine: Reporter.Error.WriteLine);
                 }
                 else
                 {
-                    _stdErrForward = to.WriteLine;
+                    _stdErr.ForwardTo(write: to.Write, writeLine: to.WriteLine);
                 }
             }
             return this;
@@ -292,22 +281,14 @@ namespace Microsoft.DotNet.Cli.Utils
         public Command OnOutputLine(Action<string> handler)
         {
             ThrowIfRunning();
-            if (_stdOutHandler != null)
+            _stdOut.ForwardTo(write: null, writeLine: handler);
-            {
-                throw new InvalidOperationException("Already handling stdout!");
-            }
-            _stdOutHandler = handler;
             return this;
         }
 
         public Command OnErrorLine(Action<string> handler)
         {
             ThrowIfRunning();
-            if (_stdErrHandler != null)
+            _stdErr.ForwardTo(write: null, writeLine: handler);
-            {
-                throw new InvalidOperationException("Already handling stderr!");
-            }
-            _stdErrHandler = handler;
             return this;
         }
 
@@ -328,27 +309,147 @@ namespace Microsoft.DotNet.Cli.Utils
                 throw new InvalidOperationException($"Unable to invoke {memberName} after the command has been run");
             }
         }
+    }
 
-        private void ProcessData(string data, StringWriter capture, Action<string> forward, Action<string> handler)
+    internal sealed class StreamForwarder
     {
-            if (data == null)
+        private const int DefaultBufferSize = 256;
+
+        private readonly int _bufferSize;
+        private StringBuilder _builder;
+        private StringWriter _capture;
+        private Action<string> _write;
+        private Action<string> _writeLine;
+
+        internal StreamForwarder(int bufferSize = DefaultBufferSize)
+        {
+            _bufferSize = bufferSize;
+        }
+
+        internal void Capture()
+        {
+            if (_capture != null)
+            {
+                throw new InvalidOperationException("Already capturing stream!");
+            }
+            _capture = new StringWriter();
+        }
+
+        internal string GetCapturedOutput()
+        {
+            return _capture?.GetStringBuilder()?.ToString();
+        }
+
+        internal void ForwardTo(Action<string> write, Action<string> writeLine)
+        {
+            if (writeLine == null)
+            {
+                throw new ArgumentNullException(nameof(writeLine));
+            }
+            if (_writeLine != null)
+            {
+                throw new InvalidOperationException("Already handling stream!");
+            }
+            _write = write;
+            _writeLine = writeLine;
+        }
+
+        internal Thread BeginRead(TextReader reader)
+        {
+            var thread = new Thread(() => Read(reader)) { IsBackground = true };
+            thread.Start();
+            return thread;
+        }
+
+        internal void Read(TextReader reader)
+        {
+            _builder = new StringBuilder();
+            var buffer = new char[_bufferSize];
+            int n;
+            while ((n = reader.Read(buffer, 0, _bufferSize)) > 0)
+            {
+                _builder.Append(buffer, 0, n);
+                WriteBlocks();
+            }
+            WriteRemainder();
+        }
+
+        private void WriteBlocks()
+        {
+            int n = _builder.Length;
+            if (n == 0)
             {
                 return;
             }
 
-            if (capture != null)
+            int offset = 0;
+            bool sawReturn = false;
+            for (int i = 0; i < n; i++)
             {
-                capture.WriteLine(data);
+                char c = _builder[i];
+                switch (c)
+                {
+                    case '\r':
+                        sawReturn = true;
+                        continue;
+                    case '\n':
+                        WriteLine(_builder.ToString(offset, i - offset - (sawReturn ? 1 : 0)));
+                        offset = i + 1;
+                        break;
+                }
+                sawReturn = false;
             }
 
-            if (forward != null)
+            // If the buffer contains no line breaks and _write is
+            // supported, send the buffer content.
+            if (!sawReturn &&
+                (offset == 0) &&
+                ((_write != null) || (_writeLine == null)))
             {
-                forward(data);
+                WriteRemainder();
+            }
+            else
+            {
+                _builder.Remove(0, offset);
+            }
         }
 
-            if (handler != null)
+        private void WriteRemainder()
         {
-                handler(data);
+            if (_builder.Length == 0)
+            {
+                return;
+            }
+            Write(_builder.ToString());
+            _builder.Clear();
+        }
+
+        private void WriteLine(string str)
+        {
+            if (_capture != null)
+            {
+                _capture.WriteLine(str);
+            }
+            // If _write is supported, so is _writeLine.
+            if (_writeLine != null)
+            {
+                _writeLine(str);
+            }
+        }
+
+        private void Write(string str)
+        {
+            if (_capture != null)
+            {
+                _capture.Write(str);
+            }
+            if (_write != null)
+            {
+                _write(str);
+            }
+            else if (_writeLine != null)
+            {
+                _writeLine(str);
             }
         }
     }

src/Microsoft.DotNet.Cli.Utils/Reporter.cs

@@ -2,9 +2,7 @@
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
 using System;
-using System.IO;
 using System.Runtime.InteropServices;
-using System.Text;
 
 namespace Microsoft.DotNet.Cli.Utils
 {
@@ -14,7 +12,7 @@ namespace Microsoft.DotNet.Cli.Utils
         private static readonly Reporter Null = new Reporter(console: null);
         private static object _lock = new object();
 
-        private AnsiConsole _console;
+        private readonly AnsiConsole _console;
 
         private Reporter(AnsiConsole console)
         {
@@ -52,5 +50,13 @@ namespace Microsoft.DotNet.Cli.Utils
                 _console?.Writer?.WriteLine();
             }
         }
+
+        public void Write(string message)
+        {
+            lock (_lock)
+            {
+                _console?.Writer?.Write(message);
+            }
+        }
     }
 }

src/Microsoft.DotNet.Cli/Program.cs

@@ -27,6 +27,7 @@ Common Commands:
   compile       Compiles a .NET project
   publish       Publishes a .NET project for deployment (including the runtime)
   run           Compiles and immediately executes a .NET project
+  repl          Launch an interactive session (read, eval, print, loop)
   pack          Creates a NuGet package";
 
         public static int Main(string[] args)

src/Microsoft.DotNet.Tools.Compiler/Program.cs

@@ -73,7 +73,7 @@ namespace Microsoft.DotNet.Tools.Compiler
                     ProjectContext.CreateContextForEachFramework(path);
                 foreach (var context in contexts)
                 {
-                    success &= Compile(context, configValue, outputValue, intermediateOutput.Value(), buildProjectReferences, noHost.HasValue());
+                    success &= Compile(context, configValue, outputValue, intermediateValue, buildProjectReferences, noHost.HasValue());
                     if (isNative && success)
                     {
                         success &= CompileNative(context, configValue, outputValue, buildProjectReferences, intermediateValue, archValue, ilcArgsValue, ilcPathValue, ilcSdkPathValue, isCppMode);

test/E2E/E2ETest.cs

@@ -2,8 +2,10 @@
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
 using System;
+using System.Collections.Generic;
 using System.IO;
 using System.Runtime.InteropServices;
+using System.Text;
 using Xunit;
 using Microsoft.DotNet.Cli.Utils;
 using Microsoft.DotNet.ProjectModel;
@@ -164,3 +166,101 @@ namespace ConsoleApplication
         }
     }
 }
+
+public class StreamForwarderTests
+{
+    [Fact]
+    public void Unbuffered()
+    {
+        Forward(4, true, "");
+        Forward(4, true, "123", "123");
+        Forward(4, true, "1234", "1234");
+        Forward(3, true, "123456789", "123", "456", "789");
+        Forward(4, true, "\r\n", "\n");
+        Forward(4, true, "\r\n34", "\n", "34");
+        Forward(4, true, "1\r\n4", "1\n", "4");
+        Forward(4, true, "12\r\n", "12\n");
+        Forward(4, true, "123\r\n", "123\n");
+        Forward(4, true, "1234\r\n", "1234", "\n");
+        Forward(3, true, "\r\n3456\r\n9", "\n", "3456", "\n", "9");
+        Forward(4, true, "\n", "\n");
+        Forward(4, true, "\n234", "\n", "234");
+        Forward(4, true, "1\n34", "1\n", "34");
+        Forward(4, true, "12\n4", "12\n", "4");
+        Forward(4, true, "123\n", "123\n");
+        Forward(4, true, "1234\n", "1234", "\n");
+        Forward(3, true, "\n23456\n89", "\n", "23456", "\n", "89");
+    }
+
+    [Fact]
+    public void LineBuffered()
+    {
+        Forward(4, false, "");
+        Forward(4, false, "123", "123\n");
+        Forward(4, false, "1234", "1234\n");
+        Forward(3, false, "123456789", "123456789\n");
+        Forward(4, false, "\r\n", "\n");
+        Forward(4, false, "\r\n34", "\n", "34\n");
+        Forward(4, false, "1\r\n4", "1\n", "4\n");
+        Forward(4, false, "12\r\n", "12\n");
+        Forward(4, false, "123\r\n", "123\n");
+        Forward(4, false, "1234\r\n", "1234\n");
+        Forward(3, false, "\r\n3456\r\n9", "\n", "3456\n", "9\n");
+        Forward(4, false, "\n", "\n");
+        Forward(4, false, "\n234", "\n", "234\n");
+        Forward(4, false, "1\n34", "1\n", "34\n");
+        Forward(4, false, "12\n4", "12\n", "4\n");
+        Forward(4, false, "123\n", "123\n");
+        Forward(4, false, "1234\n", "1234\n");
+        Forward(3, false, "\n23456\n89", "\n", "23456\n", "89\n");
+    }
+
+    private static void Forward(int bufferSize, bool unbuffered, string str, params string[] expectedWrites)
+    {
+        var expectedCaptured = str.Replace("\r", "").Replace("\n", Environment.NewLine);
+
+        // No forwarding.
+        Forward(bufferSize, ForwardOptions.None, str, null, new string[0]);
+
+        // Capture only.
+        Forward(bufferSize, ForwardOptions.Capture, str, expectedCaptured, new string[0]);
+
+        var writeOptions = unbuffered ?
+            ForwardOptions.Write | ForwardOptions.WriteLine :
+            ForwardOptions.WriteLine;
+
+        // Forward.
+        Forward(bufferSize, writeOptions, str, null, expectedWrites);
+
+        // Forward and capture.
+        Forward(bufferSize, writeOptions | ForwardOptions.Capture, str, expectedCaptured, expectedWrites);
+    }
+
+    private enum ForwardOptions
+    {
+        None = 0x0,
+        Capture = 0x1,
+        Write = 0x02,
+        WriteLine = 0x04,
+    }
+
+    private static void Forward(int bufferSize, ForwardOptions options, string str, string expectedCaptured, string[] expectedWrites)
+    {
+        var forwarder = new StreamForwarder(bufferSize);
+        var writes = new List<string>();
+        if ((options & ForwardOptions.WriteLine) != 0)
+        {
+            forwarder.ForwardTo(
+                write: (options & ForwardOptions.Write) == 0 ? (Action<string>)null : writes.Add,
+                writeLine: s => writes.Add(s + "\n"));
+        }
+        if ((options & ForwardOptions.Capture) != 0)
+        {
+            forwarder.Capture();
+        }
+        forwarder.Read(new StringReader(str));
+        Assert.Equal(expectedWrites, writes);
+        var captured = forwarder.GetCapturedOutput();
+        Assert.Equal(expectedCaptured, captured);
+    }
+}