Argument Forwarding Test Infrastructure

scripts/test/argument-forwarding-tests.ps1

@@ -0,0 +1,37 @@
+#
+# Copyright (c) .NET Foundation and contributors. All rights reserved.
+# Licensed under the MIT license. See LICENSE file in the project root for full license information.
+#
+
+. "$PSScriptRoot\..\common\_common.ps1"
+
+$TestPackagesPath = "$RepoRoot\tests\packages"
+
+$ArgTestRoot = "$RepoRoot\test\ArgumentForwardingTests"
+$ArgTestBinRoot = "$RepoRoot\artifacts\tests\arg-forwarding"
+
+dotnet publish --framework "dnxcore50" --runtime "$Rid" --output "$ArgTestBinRoot" --configuration "$Configuration" "$ArgTestRoot\Reflector"
+if (!$?) {
+    Write-Host Command failed: dotnet publish --framework "dnxcore50" --runtime "$Rid" --output "$ArgTestBinRoot" --configuration "$Configuration" "$ArgTestRoot\Reflector"
+    Exit 1
+}
+
+dotnet publish --framework "dnxcore50" --runtime "$Rid" --output "$ArgTestBinRoot" --configuration "$Configuration" "$ArgTestRoot\ArgumentForwardingTests"
+if (!$?) {
+    Write-Host Command failed: dotnet publish --framework "dnxcore50" --runtime "$Rid" --output "$ArgTestBinRoot" --configuration "$Configuration" "$ArgTestRoot\ArgumentForwardingTests"
+    Exit 1
+}
+
+cp "$ArgTestRoot\Reflector\reflector_cmd.cmd" "$ArgTestBinRoot"
+
+pushd $ArgTestBinRoot
+
+& ".\corerun" "xunit.console.netcore.exe" "ArgumentForwardingTests.dll" -xml "$_-testResults.xml" -notrait category=failing
+$exitCode = $LastExitCode
+
+popd
+
+# No need to output here, we'll get test results
+if ($exitCode -ne 0) {
+    Exit 1
+}

scripts/test/argument-forwarding-tests.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) .NET Foundation and contributors. All rights reserved.
+# Licensed under the MIT license. See LICENSE file in the project root for full license information.
+#
+
+set -e
+
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
+  DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+  SOURCE="$(readlink "$SOURCE")"
+  [[ "$SOURCE" != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+done
+
+DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+
+source "$DIR/../common/_common.sh"
+
+ArgTestRoot = "$REPOROOT/test/ArgumentForwardingTests"
+ArgTestBinRoot="$REPOROOT/artifacts/tests/arg-forwarding"
+
+dotnet publish --framework "dnxcore50" --runtime "$RID" --output "$ArgTestBinRoot" --configuration "$CONFIGURATION" "$ArgTestRoot/Reflector"
+dotnet publish --framework "dnxcore50" --runtime "$RID" --output "$ArgTestBinRoot" --configuration "$CONFIGURATION" "$ArgTestRoot/ArgumentForwardingTests"
+
+cp "$ArgTestRoot/Reflector/reflector_cmd.cmd" "$ArgTestBinRoot"
+
+pushd $TestBinRoot
+./corerun "xunit.console.netcore.exe" "ArgumentForwardingTests.dll" -xml "ArgumentForwardingTests-testResults.xml" -notrait category=failing
+popd

scripts/test/runtests.ps1

@@ -17,6 +17,11 @@ $TestProjects = @(
     "Microsoft.DotNet.Tools.Builder.Tests"
 )
 
+$TestScripts = @(
+    "package-command-test.ps1",
+    "argument-forwarding-tests.ps1"
+)
+
 # Publish each test project
 $TestProjects | ForEach-Object {
     dotnet publish --framework "dnxcore50" --runtime "$Rid" --output "$TestBinRoot" --configuration "$Configuration" "$RepoRoot\test\$_"
@@ -64,11 +69,14 @@ $TestProjects | ForEach-Object {
 
 popd
 
-& $RepoRoot\scripts\test\package-command-test.ps1
+$TestScripts | ForEach-Object {
-$exitCode = $LastExitCode
+    & "$RepoRoot\scripts\test\$_"
-if ($exitCode -ne 0) {
+    $exitCode = $LastExitCode
+    if ($exitCode -ne 0) {
+        $failingTests += "$_"
+    }
+
     $failCount += 1
-    $failingTests += "package-command-test"
 }
 
 if ($failCount -ne 0) {

scripts/test/runtests.sh

@@ -27,6 +27,11 @@ TestProjects=( \
     Microsoft.DotNet.Tools.Builder.Tests \
 )
 
+TestScripts=( \
+    "package-command-test.sh" \
+    "argument-forwarding-tests.sh" \
+)
+
 for project in ${TestProjects[@]}
 do
     dotnet publish --framework "dnxcore50" --output "$TestBinRoot" --configuration "$CONFIGURATION" "$REPOROOT/test/$project"
@@ -58,11 +63,15 @@ do
     fi
 done
 
-"$REPOROOT/scripts/test/package-command-test.sh"
+for script in ${TestScripts[@]}
-if [ $? -ne 0 ]; then
+do
+    "$REPOROOT/scripts/test/$script"
+    exitCode=$?
     failCount+=1
-    failedTests+=("package-command-test.sh")
+    if [ $exitCode -ne 0 ]; then
-fi
+        failedTests+=($script)
+    fi
+done
 
 for test in ${failedTests[@]}
 do

test/ArgumentForwardingTests/ArgumentForwardingTests/ArgumentForwardingTests.cs

@@ -0,0 +1,257 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Runtime.InteropServices;
+using System.Text;
+using Xunit;
+using Microsoft.DotNet.Cli.Utils;
+using Microsoft.DotNet.ProjectModel;
+using Microsoft.DotNet.Tools.Test.Utilities;
+using Microsoft.Extensions.PlatformAbstractions;
+using System.Diagnostics;
+using FluentAssertions;
+
+namespace Microsoft.DotNet.Tests.ArgumentForwarding
+{
+    public class ArgumentForwardingTests : TestBase
+    {
+        private static readonly string s_reflectorExeName = "reflector" + Constants.ExeSuffix;
+        private static readonly string s_reflectorCmdName = "reflector_cmd";
+
+        private string ReflectorPath { get; set; }
+        private string ReflectorCmdPath { get; set; }
+
+        public static void Main()
+        {
+            Console.WriteLine("Dummy Entrypoint.");
+        }
+       
+        public ArgumentForwardingTests()
+        {
+            // This test has a dependency on an argument reflector
+            // Make sure it's been binplaced properly
+            FindAndEnsureReflectorPresent();
+        }
+
+        private void FindAndEnsureReflectorPresent()
+        {
+            ReflectorPath = Path.Combine(AppContext.BaseDirectory, s_reflectorExeName);
+            ReflectorCmdPath = Path.Combine(AppContext.BaseDirectory, s_reflectorCmdName);
+            File.Exists(ReflectorPath).Should().BeTrue();
+        }
+
+        /// <summary>
+        /// Tests argument forwarding in Command.Create
+        /// This is a critical scenario for the driver.
+        /// </summary>
+        /// <param name="testUserArgument"></param>
+        [Theory]
+        [InlineData(@"""abc"" d e")]
+        [InlineData(@"""abc""      d e")]
+        [InlineData("\"abc\"\t\td\te")]
+        [InlineData(@"a\\b d""e f""g h")]
+        [InlineData(@"\ \\ \\\")]
+        [InlineData(@"a\""b c d")]
+        [InlineData(@"a\\""b c d")]
+        [InlineData(@"a\\\""b c d")]
+        [InlineData(@"a\\\\""b c d")]
+        [InlineData(@"a\\\\""b c d")]
+        [InlineData(@"a\\\\""b c"" d e")]
+        [InlineData(@"a""b c""d e""f g""h i""j k""l")]
+        [InlineData(@"a b c""def")]
+        [InlineData(@"""\a\"" \\""\\\ b c")]
+        [InlineData(@"a\""b \\ cd ""\e f\"" \\""\\\")]
+        public void TestArgumentForwarding(string testUserArgument)
+        {
+            // Get Baseline Argument Evaluation via Reflector
+            var rawEvaluatedArgument = RawEvaluateArgumentString(testUserArgument);
+
+            // Escape and Re-Evaluate the rawEvaluatedArgument
+            var escapedEvaluatedRawArgument = EscapeAndEvaluateArgumentString(rawEvaluatedArgument);
+
+            rawEvaluatedArgument.Length.Should().Be(escapedEvaluatedRawArgument.Length);
+
+            for (int i=0; i<rawEvaluatedArgument.Length; ++i)
+            {
+                var rawArg = rawEvaluatedArgument[i];
+                var escapedArg = escapedEvaluatedRawArgument[i];
+
+                rawArg.Should().Be(escapedArg);
+            }
+        }
+
+        /// <summary>
+        /// Tests argument forwarding in Command.Create to a cmd file
+        /// This is a critical scenario for the driver.
+        /// </summary>
+        /// <param name="testUserArgument"></param>
+        [Theory]
+        [InlineData(@"""abc"" d e")]
+        [InlineData(@"""abc""      d e")]
+        [InlineData("\"abc\"\t\td\te")]
+        [InlineData(@"a\\b d""e f""g h")]
+        [InlineData(@"\ \\ \\\")]
+        [InlineData(@"a\""b c d")]
+        [InlineData(@"a\\""b c d")]
+        [InlineData(@"a\\\""b c d")]
+        [InlineData(@"a\\\\""b c d")]
+        [InlineData(@"a\\\\""b c d")]
+        [InlineData(@"a\\\\""b c"" d e")]
+        [InlineData(@"a""b c""d e""f g""h i""j k""l")]
+        [InlineData(@"a b c""def")]
+        [InlineData(@"""\a\"" \\""\\\ b c")]
+        [InlineData(@"a\""b \\ cd ""\e f\"" \\""\\\")]
+        public void TestArgumentForwardingCmd(string testUserArgument)
+        {
+            // Get Baseline Argument Evaluation via Reflector
+            // This does not need to be different for cmd because
+            // it only establishes what the string[] args should be
+            var rawEvaluatedArgument = RawEvaluateArgumentString(testUserArgument);
+
+            // Escape and Re-Evaluate the rawEvaluatedArgument
+            var escapedEvaluatedRawArgument = EscapeAndEvaluateArgumentStringCmd(rawEvaluatedArgument);
+
+            try
+            {
+                rawEvaluatedArgument.Length.Should().Be(escapedEvaluatedRawArgument.Length);
+            }
+            catch(Exception e)
+            {
+                Console.WriteLine("Argument Lists differ in length.");
+
+                var expected = string.Join(",", rawEvaluatedArgument);
+                var actual = string.Join(",", escapedEvaluatedRawArgument);
+                Console.WriteLine($"Expected: {expected}");
+                Console.WriteLine($"Actual: {actual}");
+
+                throw e;
+            }
+
+            for (int i = 0; i < rawEvaluatedArgument.Length; ++i)
+            {
+                var rawArg = rawEvaluatedArgument[i];
+                var escapedArg = escapedEvaluatedRawArgument[i];
+                
+                try
+                {
+                    rawArg.Should().Be(escapedArg);
+                }
+                catch(Exception e)
+                {
+                    Console.WriteLine($"Expected: {rawArg}");
+                    Console.WriteLine($"Actual: {escapedArg}");
+                    throw e;
+                }
+            }
+        }
+
+        /// <summary>
+        /// EscapeAndEvaluateArgumentString returns a representation of string[] args
+        /// when rawEvaluatedArgument is passed as an argument to a process using
+        /// Command.Create(). Ideally this should escape the argument such that 
+        /// the output is == rawEvaluatedArgument.
+        /// </summary>
+        /// <param name="rawEvaluatedArgument">A string[] representing string[] args as already evaluated by a process</param>
+        /// <returns></returns>
+        private string[] EscapeAndEvaluateArgumentString(string[] rawEvaluatedArgument)
+        {
+            var commandResult = Command.Create(ReflectorPath, rawEvaluatedArgument)
+                .CaptureStdErr()
+                .CaptureStdOut()
+                .Execute();
+
+            commandResult.ExitCode.Should().Be(0);
+
+            return ParseReflectorOutput(commandResult.StdOut);
+        }
+
+        /// <summary>
+        /// EscapeAndEvaluateArgumentString returns a representation of string[] args
+        /// when rawEvaluatedArgument is passed as an argument to a process using
+        /// Command.Create(). Ideally this should escape the argument such that 
+        /// the output is == rawEvaluatedArgument.
+        /// </summary>
+        /// <param name="rawEvaluatedArgument">A string[] representing string[] args as already evaluated by a process</param>
+        /// <returns></returns>
+        private string[] EscapeAndEvaluateArgumentStringCmd(string[] rawEvaluatedArgument)
+        {
+            var cmd = Command.Create(s_reflectorCmdName, rawEvaluatedArgument);
+            var commandResult = cmd
+                .CaptureStdErr()
+                .CaptureStdOut()
+                .Execute();
+
+            Console.WriteLine(commandResult.StdOut);
+            Console.WriteLine(commandResult.StdErr);
+
+            commandResult.ExitCode.Should().Be(0);
+
+            return ParseReflectorCmdOutput(commandResult.StdOut);
+        }
+
+        /// <summary>
+        /// Parse the output of the reflector into a string array.
+        /// Reflector output is simply string[] args written to 
+        /// one string separated by commas.
+        /// </summary>
+        /// <param name="reflectorOutput"></param>
+        /// <returns></returns>
+        private string[] ParseReflectorOutput(string reflectorOutput)
+        {
+            return reflectorOutput.Split(',');
+        }
+
+        /// <summary>
+        /// Parse the output of the reflector into a string array.
+        /// Reflector output is simply string[] args written to 
+        /// one string separated by commas.
+        /// </summary>
+        /// <param name="reflectorOutput"></param>
+        /// <returns></returns>
+        private string[] ParseReflectorCmdOutput(string reflectorOutput)
+        {
+            var args = reflectorOutput.Split(new string[] { "," }, StringSplitOptions.None);
+            args[args.Length-1] = args[args.Length-1].TrimEnd('\r', '\n');
+
+            // To properly pass args to cmd, quotes inside a parameter are doubled
+            // Count them as a single quote for our comparison.
+            for (int i=0; i < args.Length; ++i)
+            {
+                args[i] = args[i].Replace(@"""""", @"""");
+            }
+            return args;
+        }
+
+        /// <summary>
+        /// RawEvaluateArgumentString returns a representation of string[] args
+        /// when testUserArgument is provided (unmodified) as arguments to a c#
+        /// process.
+        /// </summary>
+        /// <param name="testUserArgument">A test argument representing what a "user" would provide to a process</param>
+        /// <returns>A string[] representing string[] args with the provided testUserArgument</returns>
+        private string[] RawEvaluateArgumentString(string testUserArgument)
+        {
+            var proc = new Process
+            {
+                StartInfo = new ProcessStartInfo
+                {
+                    FileName = s_reflectorExeName,
+                    Arguments = testUserArgument,
+                    UseShellExecute = false,
+                    RedirectStandardOutput = true,
+                    CreateNoWindow = true
+                }
+            };
+
+            proc.Start();
+            var stdOut = proc.StandardOutput.ReadToEnd();
+
+            Assert.Equal(0, proc.ExitCode);
+
+            return ParseReflectorOutput(stdOut);
+        }
+    }
+}

test/ArgumentForwardingTests/ArgumentForwardingTests/ArgumentForwardingTests.xproj

@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="14.0.24720" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <VisualStudioVersion Condition="'$(VisualStudioVersion)' == ''">14.0.24720</VisualStudioVersion>
+    <VSToolsPath Condition="'$(VSToolsPath)' == ''">$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)</VSToolsPath>
+  </PropertyGroup>
+  <Import Project="$(VSToolsPath)\DNX\Microsoft.DNX.Props" Condition="'$(VSToolsPath)' != ''" />
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>6973e08d-11ec-49dc-82ef-d5effec7c6e9</ProjectGuid>
+    <RootNamespace>Microsoft.DotNet.Tests.ArgumentForwarding</RootNamespace>
+    <BaseIntermediateOutputPath Condition="'$(BaseIntermediateOutputPath)'=='' ">..\..\artifacts\obj\$(MSBuildProjectName)</BaseIntermediateOutputPath>
+    <OutputPath Condition="'$(OutputPath)'=='' ">..\..\artifacts\bin\$(MSBuildProjectName)\</OutputPath>
+  </PropertyGroup>
+  <PropertyGroup>
+    <SchemaVersion>2.0</SchemaVersion>
+  </PropertyGroup>
+  <Import Project="$(VSToolsPath)\DNX\Microsoft.DNX.targets" Condition="'$(VSToolsPath)' != ''" />
+</Project>

test/ArgumentForwardingTests/ArgumentForwardingTests/project.json

@@ -0,0 +1,24 @@
+{
+    "version": "1.0.0-*",
+    "compilationOptions": {
+        "emitEntryPoint": true
+    },
+
+    "dependencies": {
+        "NETStandard.Library" : "1.0.0-rc2-23706",
+        "Microsoft.NETCore.TestHost": "1.0.0-rc2-23706",
+
+        "xunit": "2.1.0",
+        "xunit.console.netcore": "1.0.2-prerelease-00101",
+        "xunit.netcore.extensions": "1.0.0-prerelease-*",
+        "xunit.runner.utility": "2.1.0",
+
+        "Microsoft.DotNet.ProjectModel": { "target": "project" },
+        "Microsoft.DotNet.Cli.Utils": { "target": "project" },
+        "Microsoft.DotNet.Tools.Tests.Utilities": { "target": "project" }
+    },
+
+    "frameworks": {
+        "dnxcore50": { }
+    }
+}

test/ArgumentForwardingTests/Reflector/Reflector.cs

@@ -0,0 +1,38 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Runtime.InteropServices;
+using System.Text;
+using Xunit;
+using Microsoft.DotNet.Cli.Utils;
+using Microsoft.DotNet.ProjectModel;
+using Microsoft.DotNet.Tools.Test.Utilities;
+using Microsoft.Extensions.PlatformAbstractions;
+using System.Diagnostics;
+using FluentAssertions;
+
+namespace Microsoft.DotNet.Tests.ArgumentForwarding
+{
+    public class Program
+    {
+        public static void Main(string[] args)
+        {
+            bool first=true;
+            foreach (var arg in args)
+            {
+                if (first)
+                {
+                    first=false;
+                }
+                else
+                {
+                    Console.Write(",");
+                }
+                Console.Write(arg);
+            }
+        }
+    }
+}

test/ArgumentForwardingTests/Reflector/Reflector.xproj

@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="14.0.24720" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <VisualStudioVersion Condition="'$(VisualStudioVersion)' == ''">14.0.24720</VisualStudioVersion>
+    <VSToolsPath Condition="'$(VSToolsPath)' == ''">$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)</VSToolsPath>
+  </PropertyGroup>
+  <Import Project="$(VSToolsPath)\DNX\Microsoft.DNX.Props" Condition="'$(VSToolsPath)' != ''" />
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>6f2e6f25-b43b-495d-9ca5-7f207d1dd604</ProjectGuid>
+    <RootNamespace>Microsoft.DotNet.Tests.ArgumentForwarding</RootNamespace>
+    <BaseIntermediateOutputPath Condition="'$(BaseIntermediateOutputPath)'=='' ">..\..\artifacts\obj\$(MSBuildProjectName)</BaseIntermediateOutputPath>
+    <OutputPath Condition="'$(OutputPath)'=='' ">..\..\artifacts\bin\$(MSBuildProjectName)\</OutputPath>
+  </PropertyGroup>
+  <PropertyGroup>
+    <SchemaVersion>2.0</SchemaVersion>
+  </PropertyGroup>
+  <Import Project="$(VSToolsPath)\DNX\Microsoft.DNX.targets" Condition="'$(VSToolsPath)' != ''" />
+</Project>

test/ArgumentForwardingTests/Reflector/project.json

@@ -0,0 +1,24 @@
+{
+    "version": "1.0.0-*",
+    "compilationOptions": {
+        "emitEntryPoint": true
+    },
+
+    "dependencies": {
+        "NETStandard.Library" : "1.0.0-rc2-23706",
+        "Microsoft.NETCore.TestHost": "1.0.0-rc2-23706",
+
+        "xunit": "2.1.0",
+        "xunit.console.netcore": "1.0.2-prerelease-00101",
+        "xunit.netcore.extensions": "1.0.0-prerelease-*",
+        "xunit.runner.utility": "2.1.0",
+
+        "Microsoft.DotNet.ProjectModel": { "target": "project" },
+        "Microsoft.DotNet.Cli.Utils": { "target": "project" },
+        "Microsoft.DotNet.Tools.Tests.Utilities": { "target": "project" }
+    },
+
+    "frameworks": {
+        "dnxcore50": { }
+    }
+}

test/ArgumentForwardingTests/Reflector/reflector_cmd.cmd

@@ -0,0 +1,19 @@
+@echo off
+set ALL_ARGS=
+set a=%1
+if not defined a goto :doneSetArgs
+
+set ALL_ARGS=%~1%
+shift
+
+:setArgs
+set a=%1
+if not defined a goto :doneSetArgs
+
+set ALL_ARGS=%ALL_ARGS%,%~1%
+shift
+goto setArgs
+
+:doneSetArgs
+echo %ALL_ARGS%
+goto :EOF