Merge pull request #1117 from dotnet/brthor/refactor_test_scripts

Refactor test scripts

.gitignore

@@ -269,4 +269,3 @@ _Pvt_Extensions
 # Exceptions
 # Build Scripts
 !scripts/build/
-test/PackagedCommands/Consumers/*/project.json

NuGet.Config

@@ -10,7 +10,7 @@
     <add key="api.nuget.org" value="https://api.nuget.org/v3/index.json" />
     <add key="dotnet-corefxlab" value="https://www.myget.org/F/dotnet-corefxlab/api/v3/index.json" />
     <add key="corefxlab" value="https://www.myget.org/F/netcore-package-prototyping/api/v3/index.json" />
-    <add key="corert" value="https://www.myget.org/F/dotnet/auth/3e4f1dbe-f43a-45a8-b029-3ad4d25605ac/api/v2" />
+    <add key="corert" value="https://www.myget.org/F/dotnet/api/v3/index.json" />
     <add key="dotnet-buildtools" value="https://www.myget.org/F/dotnet-buildtools/api/v3/index.json" />
     <add key="fsharp-daily" value="https://www.myget.org/F/fsharp-daily/api/v3/index.json" />
   </packageSources>

scripts/build/build.ps1

@@ -14,18 +14,6 @@ $ErrorActionPreference="Stop"
 
 . "$RepoRoot\scripts\build\generate-version.ps1"
 
-if ($env:CI_BUILD -eq "1") {
-    $env:NUGET_PACKAGES = (Join-Path $RepoRoot "artifacts\home\.nuget\packages")
-} else {
-    $env:NUGET_PACKAGES = (Join-Path $env:USERPROFILE ".nuget\packages")
-}
-
-$env:DOTNET_PACKAGES = $env:NUGET_PACKAGES
-$env:DNX_PACKAGES = $env:NUGET_PACKAGES
-if(!(Test-Path $env:NUGET_PACKAGES)) {
-    mkdir $env:NUGET_PACKAGES | Out-Null
-}
-
 header "Building dotnet tools version $($env:DOTNET_CLI_VERSION) - $Configuration"
 header "Checking Pre-Reqs"
 
@@ -51,8 +39,8 @@ _ "$RepoRoot\scripts\compile\compile.ps1" @("$Configuration")
 header "Setting Stage2 as PATH and DOTNET_TOOLS"
 setPathAndHome "$Stage2Dir"
 
-header "Running Tests"
+header "Testing"
-_ "$RepoRoot\scripts\test\runtests.ps1"
+_ "$RepoRoot\scripts\test\test.ps1"
 
 header "Validating Dependencies"
 _ "$RepoRoot\scripts\test\validate-dependencies.ps1"

scripts/build/build.sh

@@ -19,11 +19,6 @@ DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
 source "$DIR/../common/_common.sh"
 source "$REPOROOT/scripts/build/generate-version.sh"
 
-## Temporarily redirect to the NuGet package installation location
-export NUGET_PACKAGES=~/.nuget/packages
-export DOTNET_PACKAGES=$NUGET_PACKAGES
-export DNX_PACKAGES=$NUGET_PACKAGES
-
 header "Building dotnet tools version $DOTNET_CLI_VERSION - $CONFIGURATION"
 header "Checking Pre-Reqs"
 
@@ -55,8 +50,8 @@ $REPOROOT/scripts/compile/compile.sh
 header "Setting Stage2 as PATH, DOTNET_HOME, and DOTNET_TOOLS"
 export DOTNET_HOME=$STAGE2_DIR && export DOTNET_TOOLS=$STAGE2DIR && export PATH=$STAGE2_DIR/bin:$PATH 
 
-header "Running Tests"
+header "Testing"
-$REPOROOT/scripts/test/runtests.sh
+$REPOROOT/scripts/test/test.sh
 
 header "Validating Dependencies"
 $REPOROOT/scripts/test/validate-dependencies.sh

scripts/build/build_appdeps.sh

@@ -25,7 +25,7 @@ popd
 
 ## App Deps ##
 pushd $APPDEPS_PROJECT_DIR
-dotnet restore --packages $APPDEPS_PROJECT_DIR/packages
+dotnet restore --runtime $RID --packages $APPDEPS_PROJECT_DIR/packages $DISABLE_PARALLEL
 APPDEP_SDK=$APPDEPS_PROJECT_DIR/packages/toolchain*/*/
 popd
 

scripts/build/restore-packages.ps1

@@ -8,12 +8,6 @@
 # Restore packages
 # NOTE(anurse): I had to remove --quiet, because NuGet3 is too quiet when that's provided :(
 header "Restoring packages"
-dotnet restore "$RepoRoot\src" --runtime "$Rid"
-dotnet restore "$RepoRoot\test" --runtime "$Rid"
-dotnet restore "$RepoRoot\tools" --runtime "$Rid"
-
-$oldErrorAction=$ErrorActionPreference
-$ErrorActionPreference="SilentlyContinue"
-dotnet restore "$RepoRoot\testapp" --runtime "$Rid" 2>&1 | Out-Null
-$ErrorActionPreference=$oldErrorAction
 
+& dotnet restore "$RepoRoot\src" --runtime "$Rid"
+& dotnet restore "$RepoRoot\tools" --runtime "$Rid"

scripts/build/restore-packages.sh

@@ -18,9 +18,5 @@ source "$DIR/../common/_common.sh"
 
 header "Restoring packages"
 
-dotnet restore "$REPOROOT/src" --runtime $RID
+dotnet restore "$REPOROOT/src" --runtime "$RID" $DISABLE_PARALLEL
-dotnet restore "$REPOROOT/test" --runtime $RID
+dotnet restore "$REPOROOT/tools" --runtime "$RID" $DISABLE_PARALLEL
-dotnet restore "$REPOROOT/tools" --runtime $RID
-set +e
-dotnet restore "$REPOROOT/testapp" --runtime $RID >/dev/null 2>&1
-set -e

scripts/common/_common.ps1

@@ -9,21 +9,24 @@ $Rid = "win7-x64"
 $Tfm = "dnxcore50"
 $RepoRoot = Resolve-Path "$PSScriptRoot\..\.."
 $OutputDir = "$RepoRoot\artifacts\$Rid"
-$DnxDir = "$OutputDir\dnx"
-$DnxRoot = "$DnxDir\bin"
 $Stage1Dir = "$OutputDir\stage1"
 $Stage1CompilationDir = "$OutputDir\stage1compilation"
 $Stage2Dir = "$OutputDir\stage2"
 $Stage2CompilationDir = "$OutputDir\stage2compilation"
 $HostDir = "$OutputDir\corehost"
 $PackageDir = "$RepoRoot\artifacts\packages\dnvm"
+$TestBinRoot = "$RepoRoot\artifacts\tests"
+$TestPackageDir = "$TestBinRoot\packages"
+
 $env:ReleaseSuffix = "beta"
 $env:Channel = "$env:ReleaseSuffix"
-$env:TEST_ROOT = "$OutputDir\tests"
-$env:TEST_ARTIFACTS = "$env:TEST_ROOT\artifacts"
 
 # Set reasonable defaults for unset variables
 setEnvIfDefault "DOTNET_INSTALL_DIR"  "$(Convert-Path "$PSScriptRoot\..")\.dotnet_stage0\win7-x64"
 setEnvIfDefault "DOTNET_CLI_VERSION" "0.1.0.0"
 setPathAndHomeIfDefault "$Stage2Dir"
 setVarIfDefault "Configuration" "Debug"
+
+# Common Files which depend on above properties
+. $PSScriptRoot\_nuget.ps1
+. $PSScriptRoot\_configuration.ps1

scripts/common/_common.sh

@@ -18,13 +18,14 @@ source "$COMMONDIR/_rid.sh"
 export TFM=dnxcore50
 export REPOROOT=$(cd $COMMONDIR/../.. && pwd)
 export OUTPUT_ROOT=$REPOROOT/artifacts/$RID
-export DNX_DIR=$OUTPUT_ROOT/dnx
-export DNX_ROOT=$DNX_DIR/bin
 export STAGE1_DIR=$OUTPUT_ROOT/stage1
 export STAGE1_COMPILATION_DIR=$OUTPUT_ROOT/stage1compilation
 export STAGE2_DIR=$OUTPUT_ROOT/stage2
 export STAGE2_COMPILATION_DIR=$OUTPUT_ROOT/stage2compilation
 export HOST_DIR=$OUTPUT_ROOT/corehost
+export TEST_BIN_ROOT="$REPOROOT/artifacts/tests"
+export TEST_PACKAGE_DIR="$TEST_BIN_ROOT/packages"
+
 export RELEASE_SUFFIX=beta
 export CHANNEL=$RELEASE_SUFFIX
 
@@ -33,5 +34,13 @@ export CHANNEL=$RELEASE_SUFFIX
 [ -z "$DOTNET_HOME" ] && export DOTNET_HOME=$STAGE2_DIR && export PATH=$STAGE2_DIR/bin:$PATH
 [ -z "$CONFIGURATION" ] && export CONFIGURATION=Debug
 
+# Common Files which depend on above properties
+source "$COMMONDIR/_nuget.sh"
+source "$COMMONDIR/_configuration.sh"
+
+#TODO this is a workaround for a nuget bug on ubuntu. Remove
+export DISABLE_PARALLEL=""
+[[ "$RID" =~ "ubuntu" ]] && export DISABLE_PARALLEL="--disable-parallel"
+
 unset COMMONSOURCE
 unset COMMONDIR

scripts/common/_configuration.ps1

@@ -0,0 +1,24 @@
+#
+# Copyright (c) .NET Foundation and contributors. All rights reserved.
+# Licensed under the MIT license. See LICENSE file in the project root for full license information.
+#
+
+function loadTestProjectList()
+{
+    return Import-CSV "$RepoRoot\scripts\configuration\testProjects.csv" -Header "ProjectName"
+}
+
+function loadTestScriptList()
+{
+    return Import-CSV "$RepoRoot\scripts\configuration\testScripts.csv" -Header "ProjectName"
+}
+
+function loadTestPackageList()
+{
+    return Import-CSV "$RepoRoot\scripts\configuration\testPackageProjects.csv" -Header "ProjectName"
+}
+
+function loadBuildProjectList()
+{
+    return Import-CSV "$RepoRoot\scripts\configuration\buildProjects.csv" -Header "ProjectName"
+}

scripts/common/_configuration.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) .NET Foundation and contributors. All rights reserved.
+# Licensed under the MIT license. See LICENSE file in the project root for full license information.
+#
+
+loadTestProjectList()
+{
+    echo $(cat "$REPOROOT/scripts/configuration/testProjects.csv")
+}
+
+loadTestScriptList()
+{
+    echo $(cat "$REPOROOT/scripts/configuration/testScripts.csv")
+}
+
+loadTestPackageList()
+{
+    echo $(cat "$REPOROOT/scripts/configuration/testPackageProjects.csv")
+}
+
+loadBuildProjectList()
+{
+    echo $(cat "$REPOROOT/scripts/configuration/buildProjects.csv")
+}

scripts/common/_nuget.ps1

@@ -0,0 +1,16 @@
+#
+# Copyright (c) .NET Foundation and contributors. All rights reserved.
+# Licensed under the MIT license. See LICENSE file in the project root for full license information.
+#
+
+if ($env:CI_BUILD -eq "1") {
+    $env:NUGET_PACKAGES = (Join-Path $RepoRoot "artifacts\home\.nuget\packages")
+} else {
+    $env:NUGET_PACKAGES = (Join-Path $env:USERPROFILE ".nuget\packages")
+}
+
+$env:DOTNET_PACKAGES = $env:NUGET_PACKAGES
+$env:DNX_PACKAGES = $env:NUGET_PACKAGES
+if(!(Test-Path $env:NUGET_PACKAGES)) {
+    mkdir $env:NUGET_PACKAGES | Out-Null
+}

scripts/common/_nuget.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) .NET Foundation and contributors. All rights reserved.
+# Licensed under the MIT license. See LICENSE file in the project root for full license information.
+#
+
+## Temporarily redirect to the NuGet package installation location
+export NUGET_PACKAGES=~/.nuget/packages
+export DOTNET_PACKAGES=$NUGET_PACKAGES
+export DNX_PACKAGES=$NUGET_PACKAGES

scripts/compile/compile-stage.ps1

@@ -7,35 +7,14 @@ param(
     [Parameter(Mandatory=$true)][string]$Tfm,
     [Parameter(Mandatory=$true)][string]$Rid,
     [Parameter(Mandatory=$true)][string]$Configuration,
-    [Parameter(Mandatory=$true)][string]$OutputDir,
+    [Parameter(Mandatory=$true)][string]$StageOutputDir,
     [Parameter(Mandatory=$true)][string]$RepoRoot,
     [Parameter(Mandatory=$true)][string]$HostDir,
     [Parameter(Mandatory=$true)][string]$CompilationOutputDir)
 
 . $REPOROOT\scripts\package\projectsToPack.ps1
 
-$Projects = @(
+$Projects = loadBuildProjectList
-    "dotnet",
-    "dotnet-build",
-    "dotnet-compile",
-    "dotnet-compile-csc",
-    "dotnet-compile-fsc",
-    "dotnet-compile-native",
-    "dotnet-new",
-    "dotnet-pack",
-    "dotnet-projectmodel-server",
-    "dotnet-publish",
-    "dotnet-restore",
-    "dotnet-repl",
-    "dotnet-repl-csi",
-    "dotnet-resgen",
-    "dotnet-run",
-    "dotnet-test",
-    "Microsoft.DotNet.Cli.Utils",
-    "Microsoft.DotNet.ProjectModel.Loader",
-    "Microsoft.DotNet.ProjectModel.Workspaces",
-    "Microsoft.Extensions.Testing.Abstractions"
-)
 
 $BinariesForCoreHost = @(
     "csi"
@@ -51,13 +30,13 @@ $FilesToClean = @(
     "Microsoft.DotNet.Runtime.pdb"
 )
 
-$RuntimeOutputDir = "$OutputDir\runtime\coreclr"
+$RuntimeOutputDir = "$StageOutputDir\runtime\coreclr"
 
 # Publish each project
 $Projects | ForEach-Object {
-    dotnet publish --native-subdirectory --framework "$Tfm" --runtime "$Rid" --output "$OutputDir\bin" --configuration "$Configuration" "$RepoRoot\src\$_"
+    dotnet publish --native-subdirectory --framework "$Tfm" --runtime "$Rid" --output "$StageOutputDir\bin" --configuration "$Configuration" "$RepoRoot\src\$($_.ProjectName)"
     if (!$?) {
-        Write-Host Command failed: dotnet publish --native-subdirectory --framework "$Tfm" --runtime "$Rid" --output "$OutputDir\bin" --configuration "$Configuration" "$RepoRoot\src\$_"
+        Write-Host Command failed: dotnet publish --native-subdirectory --framework "$Tfm" --runtime "$Rid" --output "$StageOutputDir\bin" --configuration "$Configuration" "$RepoRoot\src\$($_.ProjectName)"
         exit 1
     }
 }
@@ -87,7 +66,7 @@ $FilesToClean | ForEach-Object {
 }
 
 # Copy the runtime app-local for the tools
-cp -rec "$RuntimeOutputDir\*" "$OutputDir\bin" -ErrorVariable capturedErrors -ErrorAction SilentlyContinue
+cp -rec "$RuntimeOutputDir\*" "$StageOutputDir\bin" -ErrorVariable capturedErrors -ErrorAction SilentlyContinue
 $capturedErrors | foreach-object {
     if ($_ -notmatch "already exists") {
         write-error $_
@@ -96,22 +75,22 @@ $capturedErrors | foreach-object {
 }
 
 # Deploy the CLR host to the output
-cp "$HostDir\corehost.exe" "$OutputDir\bin"
+cp "$HostDir\corehost.exe" "$StageOutputDir\bin"
-cp "$HostDir\hostpolicy.dll" "$OutputDir\bin"
+cp "$HostDir\hostpolicy.dll" "$StageOutputDir\bin"
 
 # corehostify externally-provided binaries (csc, vbc, etc.)
 $BinariesForCoreHost | ForEach-Object {
-    mv $OutputDir\bin\$_.exe $OutputDir\bin\$_.dll -Force
+    mv $StageOutputDir\bin\$_.exe $StageOutputDir\bin\$_.dll -Force
-    cp $OutputDir\bin\corehost.exe $OutputDir\bin\$_.exe -Force
+    cp $StageOutputDir\bin\corehost.exe $StageOutputDir\bin\$_.exe -Force
 }
 
 # Crossgen Roslyn
-#if (-not (Test-Path "$OutputDir\bin\csc.ni.exe")) {
+#if (-not (Test-Path "$StageOutputDir\bin\csc.ni.exe")) {
     #header "Crossgening Roslyn compiler ..."
-    #_cmd "$RepoRoot\scripts\crossgen\crossgen_roslyn.cmd ""$OutputDir"""
+    #_cmd "$RepoRoot\scripts\crossgen\crossgen_roslyn.cmd ""$StageOutputDir"""
 #}
 
 # Copy in AppDeps
 header "Acquiring Native App Dependencies"
-_ "$RepoRoot\scripts\build\build_appdeps.ps1" @("$RepoRoot", "$OutputDir")
+_ "$RepoRoot\scripts\build\build_appdeps.ps1" @("$RepoRoot", "$StageOutputDir")
 

scripts/compile/compile-stage.sh

@@ -22,24 +22,7 @@ source "$DIR/../common/_common.sh"
 [ ! -z "$OUTPUT_DIR" ] || die "Missing required environment variable OUTPUT_DIR"
 [ ! -z "$HOST_DIR" ] || die "Missing required environment variable HOST_DIR"
 
-PROJECTS=( \
+PROJECTS=$(loadBuildProjectList)
-    dotnet \
-    dotnet-build \
-    dotnet-compile \
-    dotnet-compile-csc \
-    dotnet-compile-fsc \
-    dotnet-compile-native \
-    dotnet-new \
-    dotnet-pack \
-    dotnet-projectmodel-server \
-    dotnet-publish \
-    dotnet-repl \
-    dotnet-repl-csi \
-    dotnet-restore \
-    dotnet-resgen \
-    dotnet-run \
-    dotnet-test \
-)
 
 BINARIES_FOR_COREHOST=( \
     csi \
@@ -57,7 +40,7 @@ FILES_TO_CLEAN=( \
 
 RUNTIME_OUTPUT_DIR="$OUTPUT_DIR/runtime/coreclr"
 
-for project in ${PROJECTS[@]}
+for project in $PROJECTS
 do
     echo dotnet publish --native-subdirectory --framework "$TFM" --output "$OUTPUT_DIR/bin" --configuration "$CONFIGURATION" "$REPOROOT/src/$project"
     dotnet publish --native-subdirectory --framework "$TFM" --output "$OUTPUT_DIR/bin" --configuration "$CONFIGURATION" "$REPOROOT/src/$project"

scripts/configuration/buildProjects.csv

@@ -0,0 +1,20 @@
+dotnet
+dotnet-build
+dotnet-compile
+dotnet-compile-csc
+dotnet-compile-fsc
+dotnet-compile-native
+dotnet-new
+dotnet-pack
+dotnet-projectmodel-server
+dotnet-publish
+dotnet-restore
+dotnet-repl
+dotnet-repl-csi
+dotnet-resgen
+dotnet-run
+dotnet-test
+Microsoft.DotNet.Cli.Utils
+Microsoft.DotNet.ProjectModel.Loader
+Microsoft.DotNet.ProjectModel.Workspaces
+Microsoft.Extensions.Testing.Abstractions

scripts/configuration/testPackageProjects.csv

@@ -0,0 +1,2 @@
+dotnet-hello/v1/dotnet-hello
+dotnet-hello/v2/dotnet-hello

scripts/configuration/testProjects.csv

@@ -0,0 +1,5 @@
+E2E
+StreamForwarderTests
+dotnet-publish.Tests
+dotnet-compile.Tests
+dotnet-build.Tests

scripts/configuration/testScripts.csv

@@ -0,0 +1,2 @@
+argument-forwarding-tests
+package-command-test

scripts/test/build-tests.ps1

@@ -0,0 +1,15 @@
+#
+# Copyright (c) .NET Foundation and contributors. All rights reserved.
+# Licensed under the MIT license. See LICENSE file in the project root for full license information.
+#
+
+. "$PSScriptRoot\..\common\_common.ps1"
+
+# Publish each test project
+loadTestProjectList | foreach {
+    dotnet publish --framework "dnxcore50" --runtime "$Rid" --output "$TestBinRoot" --configuration "$Configuration" "$RepoRoot\test\$($_.ProjectName)"
+    if (!$?) {
+        Write-Host Command failed: dotnet publish --framework "dnxcore50" --runtime "$Rid" --output "$TestBinRoot" --configuration "$Configuration" "$RepoRoot\test\$($_.ProjectName)"
+        exit 1
+    }
+}

scripts/test/build-tests.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) .NET Foundation and contributors. All rights reserved.
+# Licensed under the MIT license. See LICENSE file in the project root for full license information.
+#
+
+set -e
+
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
+  DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+  SOURCE="$(readlink "$SOURCE")"
+  [[ "$SOURCE" != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+done
+
+DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+
+source "$DIR/../common/_common.sh"
+
+PROJECTS=$(loadTestProjectList)
+
+for project in $PROJECTS
+do
+    dotnet publish --framework "dnxcore50" --runtime "$RID" --output "$TEST_BIN_ROOT" --configuration "$CONFIGURATION" "$REPOROOT/test/$project"
+done
+

scripts/test/package-command-test.ps1

@@ -5,40 +5,6 @@
 
 . "$PSScriptRoot\..\common\_common.ps1"
 
-$TestPackagesPath = "$RepoRoot\artifacts\tests\package-command-test\packages"
-
-if((Test-Path $TestPackagesPath) -eq 0)
-{
-    mkdir $TestPackagesPath;
-}
-
-"v1", "v2" |
-foreach {
-    dotnet pack "$RepoRoot\test\PackagedCommands\Commands\dotnet-hello\$_\dotnet-hello"
-    cp "$RepoRoot\test\PackagedCommands\Commands\dotnet-hello\$_\dotnet-hello\bin\Debug\*.nupkg" -Destination $TestPackagesPath
-    if (!$?) {
-        error "Command failed: dotnet pack"
-        Exit 1
-    }
-}
-
-# workaround for dotnet-restore from the root failing for these tests since their dependencies aren't built yet
-dir "$RepoRoot\test\PackagedCommands\Consumers" | where {$_.PsIsContainer} |
-foreach {
-    pushd "$RepoRoot\test\PackagedCommands\Consumers\$_"
-    copy project.json.template project.json
-    popd
-}
-
-#restore test projects
-pushd "$RepoRoot\test\PackagedCommands\Consumers"
-dotnet restore -s "$TestPackagesPath"
-if (!$?) {
-    error "Command failed: dotnet restore"
-    Exit 1
-}
-popd
-
 #compile apps
 dir "$RepoRoot\test\PackagedCommands\Consumers" | where {$_.PsIsContainer} | where {$_.Name.Contains("Direct")} |
 foreach {
@@ -55,7 +21,6 @@ foreach {
     $outputArray = dotnet hello | Out-String
     $output = [string]::Join('\n', $outputArray).Trim("`r", "`n")
     
-    del "project.json"
     if ($output -ne "hello") {
         error "Test Failed: $testName\dotnet hello"
         error "             printed $output"

scripts/test/package-command-test.sh

@@ -17,28 +17,6 @@ DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
 
 source "$DIR/../common/_common.sh"
 
-TestPackagesPath="$REPOROOT/artifacts/tests/package-command-test/packages" 
-
-mkdir -p "$TestPackagesPath"
-
-dotnet pack "$REPOROOT/test/PackagedCommands/Commands/dotnet-hello/v1/dotnet-hello"
-cp "$REPOROOT/test/PackagedCommands/Commands/dotnet-hello/v1/dotnet-hello/bin/Debug/"*.nupkg "$TestPackagesPath"
-dotnet pack "$REPOROOT/test/PackagedCommands/Commands/dotnet-hello/v2/dotnet-hello"
-cp "$REPOROOT/test/PackagedCommands/Commands/dotnet-hello/v2/dotnet-hello/bin/Debug/"*.nupkg "$TestPackagesPath"
-
-# enable restore for test projects
-for test in $(ls -l "$REPOROOT/test/PackagedCommands/Consumers" | grep ^d | awk '{print $9}' | grep "AppWith") 
-do
-    pushd "$REPOROOT/test/PackagedCommands/Consumers/$test"
-    cp "project.json.template" "project.json"
-    popd
-done
-
-# restore test projects
-pushd "$REPOROOT/test/PackagedCommands/Consumers"
-dotnet restore -s "$TestPackagesPath"
-popd
-
 #compile tests with direct dependencies
 for test in $(ls -l "$REPOROOT/test/PackagedCommands/Consumers" | grep ^d | awk '{print $9}' | grep "Direct") 
 do
@@ -56,7 +34,6 @@ do
 
     output=$(dotnet hello)
 
-    rm "project.json"
 
     if [ "$output" == "Hello" ] ;
     then

scripts/test/restore-tests.ps1

@@ -0,0 +1,11 @@
+#
+# Copyright (c) .NET Foundation and contributors. All rights reserved.
+# Licensed under the MIT license. See LICENSE file in the project root for full license information.
+#
+
+. $PSScriptRoot\..\common\_common.ps1
+
+info "Restoring Test Projects"
+
+# Restore packages
+& dotnet restore "$RepoRoot\test" -f "$TestPackageDir\Debug"

scripts/test/restore-tests.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) .NET Foundation and contributors. All rights reserved.
+# Licensed under the MIT license. See LICENSE file in the project root for full license information.
+#
+
+set -e
+
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
+  DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+  SOURCE="$(readlink "$SOURCE")"
+  [[ "$SOURCE" != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+done
+DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+
+source "$DIR/../common/_common.sh"
+
+header "Restoring Test Packages"
+
+dotnet restore "$REPOROOT/test" --runtime $RID -f "$TEST_PACKAGE_DIR/Debug" $DISABLE_PARALLEL

scripts/test/run-tests.ps1

@@ -5,35 +5,13 @@
 
 . "$PSScriptRoot\..\common\_common.ps1"
 
-$failCount = 0
-
 $TestBinRoot = "$RepoRoot\artifacts\tests"
 
-$TestProjects = @(
+$TestProjects = loadTestProjectList
-    "E2E",
+$TestScripts = loadTestScriptList
-    "StreamForwarderTests",
-    "dotnet-publish.Tests",
-    "dotnet-compile.Tests",
-    "dotnet-build.Tests"
-)
 
-$TestScripts = @(
+$failCount = 0
-    "package-command-test.ps1",
+$failingTests = @()
-    "argument-forwarding-tests.ps1"
-)
-
-# Publish each test project
-$TestProjects | ForEach-Object {
-    dotnet publish --framework "dnxcore50" --runtime "$Rid" --output "$TestBinRoot" --configuration "$Configuration" "$RepoRoot\test\$_"
-    if (!$?) {
-        Write-Host Command failed: dotnet publish --framework "dnxcore50" --runtime "$Rid" --output "$TestBinRoot" --configuration "$Configuration" "$RepoRoot\test\$_"
-        exit 1
-    }
-}
-
-if (Test-Path $TestBinRoot\$Configuration\dnxcore50) {
-    cp $TestBinRoot\$Configuration\dnxcore50\* $TestBinRoot -force -recurse
-}
 
 ## Temporary Workaround for Native Compilation
 ## Need x64 Native Tools Dev Prompt Env Vars
@@ -47,21 +25,17 @@ foreach {
 }
 popd
 
-# copy TestProjects folder which is used by the test cases
+# copy TestProjects to $TestBinRoot
 mkdir -Force "$TestBinRoot\TestProjects"
 cp -rec -Force "$RepoRoot\test\TestProjects\*" "$TestBinRoot\TestProjects"
 
-$failCount = 0
-$failingTests = @()
-
 pushd "$TestBinRoot"
-
 # Run each test project
-$TestProjects | ForEach-Object {
+$TestProjects | foreach {
-    & ".\corerun" "xunit.console.netcore.exe" "$_.dll" -xml "$_-testResults.xml" -notrait category=failing
+    & ".\corerun" "xunit.console.netcore.exe" "$($_.ProjectName).dll" -xml "$($_.ProjectName)-testResults.xml" -notrait category=failing
     $exitCode = $LastExitCode
     if ($exitCode -ne 0) {
-        $failingTests += "$_"
+        $failingTests += "$($_.ProjectName)"
     }
 
     $failCount += $exitCode
@@ -69,18 +43,20 @@ $TestProjects | ForEach-Object {
 
 popd
 
-$TestScripts | ForEach-Object {
+$TestScripts | foreach {
-    & "$RepoRoot\scripts\test\$_"
+    $scriptName = "$($_.ProjectName).ps1"
+
+    & "$RepoRoot\scripts\test\$scriptName"
     $exitCode = $LastExitCode
     if ($exitCode -ne 0) {
-        $failingTests += "$_"
+        $failingTests += "$scriptName"
         $failCount += 1
     }
 }
 
 if ($failCount -ne 0) {
     Write-Host -ForegroundColor Red "The following tests failed."
-    $failingTests | ForEach-Object {
+    $failingTests | foreach {
         Write-Host -ForegroundColor Red "$_.dll failed. Logs in '$TestBinRoot\$_-testResults.xml'"
     }
 } else {

scripts/test/run-tests.sh

@@ -12,48 +12,24 @@ while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symli
   SOURCE="$(readlink "$SOURCE")"
   [[ "$SOURCE" != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
 done
-
 DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
 
 source "$DIR/../common/_common.sh"
 
-TestBinRoot="$REPOROOT/artifacts/tests"
+TestProjects=$(loadTestProjectList)
-
+TestScripts=$(loadTestScriptList)
-TestProjects=( \
-    E2E \
-    StreamForwarderTests \
-    dotnet-publish.Tests \
-    dotnet-compile.Tests \
-    dotnet-build.Tests \
-)
-
-TestScripts=( \
-    "package-command-test.sh" \
-    "argument-forwarding-tests.sh" \
-)
-
-for project in ${TestProjects[@]}
-do
-    dotnet publish --framework "dnxcore50" --output "$TestBinRoot" --configuration "$CONFIGURATION" "$REPOROOT/test/$project"
-done
-
-if [ -d "$TestBinRoot/$CONFIGURATION/dnxcore50" ]
-then
-    cp -R -f $TestBinRoot/$CONFIGURATION/dnxcore50/* $TestBinRoot
-fi
-
-# copy TestProjects folder which is used by the test cases
-mkdir -p "$TestBinRoot/TestProjects"
-cp -a $REPOROOT/test/TestProjects/* $TestBinRoot/TestProjects
-
-
-pushd "$TestBinRoot"
-set +e
 
 failedTests=()
 failCount=0
 
-for project in ${TestProjects[@]}
+# Copy TestProjects to $TEST_BIN_ROOT
+mkdir -p "$TEST_BIN_ROOT/TestProjects"
+cp -a $REPOROOT/test/TestProjects/* $TEST_BIN_ROOT/TestProjects
+
+pushd "$TEST_BIN_ROOT"
+set +e
+
+for project in $TestProjects
 do
     ./corerun "xunit.console.netcore.exe" "$project.dll" -xml "${project}-testResults.xml" -notrait category=failing
     exitCode=$?
@@ -63,22 +39,25 @@ do
     fi
 done
 
-for script in ${TestScripts[@]}
+popd
+
+for script in $TestScripts
 do
-    "$REPOROOT/scripts/test/$script"
+    scriptName=${script}.sh
+
+    "$REPOROOT/scripts/test/${scriptName}"
     exitCode=$?
     if [ $exitCode -ne 0 ]; then
-        failedTests+=($script)
+        failedTests+=("$scriptName")
         failCount+=1
     fi
 done
 
 for test in ${failedTests[@]}
 do
-    error "$test failed. Logs in '$TestBinRoot/${test}-testResults.xml'"
+    error "$test failed. Logs in '$TEST_BIN_ROOT/${test}-testResults.xml'"
 done
 
-popd
 set -e
 
 exit $failCount

scripts/test/setup/build-test-prerequisites.ps1

@@ -0,0 +1,17 @@
+#
+# Copyright (c) .NET Foundation and contributors. All rights reserved.
+# Licensed under the MIT license. See LICENSE file in the project root for full license information.
+#
+
+. "$PSScriptRoot\..\..\common\_common.ps1"
+
+mkdir -Force $TestPackageDir
+
+loadTestPackageList | foreach {
+    dotnet pack "$RepoRoot\test\TestPackages\$($_.ProjectName)" --output "$TestPackageDir"
+    
+    if (!$?) {
+        error "Command failed: dotnet pack"
+        Exit 1
+    }
+}

scripts/test/setup/build-test-prerequisites.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) .NET Foundation and contributors. All rights reserved.
+# Licensed under the MIT license. See LICENSE file in the project root for full license information.
+#
+
+set -e
+
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
+  DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+  SOURCE="$(readlink "$SOURCE")"
+  [[ "$SOURCE" != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+done
+
+DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+
+source "$DIR/../../common/_common.sh"
+
+mkdir -p "$TEST_PACKAGE_DIR"
+
+PROJECTS=$(loadTestPackageList)
+
+for project in $PROJECTS
+do
+    dotnet pack "$REPOROOT/test/TestPackages/$project" --output "$TEST_PACKAGE_DIR"
+done

scripts/test/setup/restore-test-prerequisites.ps1

@@ -0,0 +1,14 @@
+#
+# Copyright (c) .NET Foundation and contributors. All rights reserved.
+# Licensed under the MIT license. See LICENSE file in the project root for full license information.
+#
+
+". $PSScriptRoot\..\..\common\_common.ps1"
+
+header "Restoring packages"
+& dotnet restore "$RepoRoot\test\TestPackages" --quiet --runtime "$Rid"
+
+$oldErrorAction=$ErrorActionPreference
+$ErrorActionPreference="SilentlyContinue"
+& dotnet restore "$RepoRoot\testapp" "$Rid" 2>&1 | Out-Null
+$ErrorActionPreference=$oldErrorAction

scripts/test/setup/restore-test-prerequisites.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) .NET Foundation and contributors. All rights reserved.
+# Licensed under the MIT license. See LICENSE file in the project root for full license information.
+#
+
+set -e
+
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
+  DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+  SOURCE="$(readlink "$SOURCE")"
+  [[ "$SOURCE" != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+done
+DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+
+source "$DIR/../../common/_common.sh"
+
+header "Restoring packages"
+dotnet restore "$REPOROOT/test/TestPackages" --runtime $RID --quiet $DISABLE_PARALLEL
+
+set +e
+dotnet restore "$REPOROOT/testapp" --runtime $RID  $DISABLE_PARALLEL >/dev/null 2>&1
+set -e

scripts/test/setup/setup-tests.ps1

@@ -0,0 +1,12 @@
+#
+# Copyright (c) .NET Foundation and contributors. All rights reserved.
+# Licensed under the MIT license. See LICENSE file in the project root for full license information.
+#
+
+. "$PSScriptRoot\..\..\common\_common.ps1"
+
+header "Test Setup: Restoring Prerequisites"
+_ "$RepoRoot\scripts\test\setup\restore-test-prerequisites.ps1"
+
+header "Test Setup: Building Prerequisites"
+_ "$RepoRoot\scripts\test\setup\build-test-prerequisites.ps1"

scripts/test/setup/setup-tests.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) .NET Foundation and contributors. All rights reserved.
+# Licensed under the MIT license. See LICENSE file in the project root for full license information.
+#
+
+set -e
+
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
+  DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+  SOURCE="$(readlink "$SOURCE")"
+  [[ "$SOURCE" != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+done
+DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+
+source "$DIR/../../common/_common.sh"
+
+header "Test Setup: Restoring Prerequisites"
+"$REPOROOT/scripts/test/setup/restore-test-prerequisites.sh"
+
+header "Test Setup: Building Prerequisites"
+"$REPOROOT/scripts/test/setup/build-test-prerequisites.sh"

scripts/test/test.ps1

@@ -0,0 +1,18 @@
+#
+# Copyright (c) .NET Foundation and contributors. All rights reserved.
+# Licensed under the MIT license. See LICENSE file in the project root for full license information.
+#
+
+. "$PSScriptRoot\..\common\_common.ps1"
+
+header "Setting up Tests"
+_ "$RepoRoot\scripts\test\setup\setup-tests.ps1"
+
+header "Restoring test projects"
+_ "$RepoRoot\scripts\test\restore-tests.ps1"
+
+header "Building test projects"
+_ "$RepoRoot\scripts\test\build-tests.ps1"
+
+header "Running Tests"
+_ "$RepoRoot\scripts\test\run-tests.ps1"

scripts/test/test.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) .NET Foundation and contributors. All rights reserved.
+# Licensed under the MIT license. See LICENSE file in the project root for full license information.
+#
+
+set -e
+
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
+  DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+  SOURCE="$(readlink "$SOURCE")"
+  [[ "$SOURCE" != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+done
+
+DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+
+source "$DIR/../common/_common.sh"
+
+header "Setting up Tests"
+"$REPOROOT/scripts/test/setup/setup-tests.sh"
+
+header "Restoring test projects"
+"$REPOROOT/scripts/test/restore-tests.sh"
+
+header "Building test projects"
+"$REPOROOT/scripts/test/build-tests.sh"
+
+header "Running Tests"
+"$REPOROOT/scripts/test/run-tests.sh"

scripts/test/validate-dependencies.sh

@@ -14,7 +14,7 @@ while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symli
 done
 DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
 
-source "$DIR/../common/_common.sh"
+. "$DIR/../common/_common.sh"
 
 # Run Validation for Project.json dependencies
 dotnet publish "$REPOROOT/tools/MultiProjectValidator" -o "$STAGE2_DIR/../tools" -c "$CONFIGURATION"

src/dotnet-restore/Program.cs

@@ -42,8 +42,9 @@ namespace Microsoft.DotNet.Tools.Restore
             if (!args.Any(s => s.Equals("--runtime", StringComparison.OrdinalIgnoreCase)))
             {
                 args = Enumerable.Concat(
-                    PlatformServices.Default.Runtime.GetOverrideRestoreRuntimeIdentifiers().SelectMany(r => new [] { "--runtime", r }),
+                    args,
-                    args).ToArray();
+                    PlatformServices.Default.Runtime.GetOverrideRestoreRuntimeIdentifiers().SelectMany(r => new [] { "--runtime", r })
+                    ).ToArray();
             }
 
             app.OnExecute(() =>

test/TestPackages/dotnet-hello/v1/dotnet-hello/dotnet-hello.xproj

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="14.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <VisualStudioVersion Condition="'$(VisualStudioVersion)' == ''">14.0</VisualStudioVersion>
+    <VSToolsPath Condition="'$(VSToolsPath)' == ''">$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)</VSToolsPath>
+  </PropertyGroup>
+  <Import Project="$(VSToolsPath)\DNX\Microsoft.DNX.Props" Condition="'$(VSToolsPath)' != ''" />
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>b8055234-9a66-4ba0-8b4c-d5e431494fe3</ProjectGuid>
+    <RootNamespace>dotnet-hello</RootNamespace>
+    <BaseIntermediateOutputPath Condition="'$(BaseIntermediateOutputPath)'=='' ">..\..\..\..\..\artifacts\obj\$(MSBuildProjectName)</BaseIntermediateOutputPath>
+    <OutputPath Condition="'$(OutputPath)'=='' ">..\..\..\..\..\artifacts\bin\$(MSBuildProjectName)\</OutputPath>
+  </PropertyGroup>
+
+  <PropertyGroup>
+    <SchemaVersion>2.0</SchemaVersion>
+  </PropertyGroup>
+  <Import Project="$(VSToolsPath)\DNX\Microsoft.DNX.targets" Condition="'$(VSToolsPath)' != ''" />
+</Project>

test/TestPackages/dotnet-hello/v2/dotnet-hello/dotnet-hello.xproj

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="14.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <VisualStudioVersion Condition="'$(VisualStudioVersion)' == ''">14.0</VisualStudioVersion>
+    <VSToolsPath Condition="'$(VSToolsPath)' == ''">$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)</VSToolsPath>
+  </PropertyGroup>
+  <Import Project="$(VSToolsPath)\DNX\Microsoft.DNX.Props" Condition="'$(VSToolsPath)' != ''" />
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>282c5014-d0cd-4dde-af4e-531e4a2e7bcd</ProjectGuid>
+    <RootNamespace>dotnet-hello</RootNamespace>
+    <BaseIntermediateOutputPath Condition="'$(BaseIntermediateOutputPath)'=='' ">..\..\..\..\..\artifacts\obj\$(MSBuildProjectName)</BaseIntermediateOutputPath>
+    <OutputPath Condition="'$(OutputPath)'=='' ">..\..\..\..\..\artifacts\bin\$(MSBuildProjectName)\</OutputPath>
+  </PropertyGroup>
+
+  <PropertyGroup>
+    <SchemaVersion>2.0</SchemaVersion>
+  </PropertyGroup>
+  <Import Project="$(VSToolsPath)\DNX\Microsoft.DNX.targets" Condition="'$(VSToolsPath)' != ''" />
+</Project>

test/dotnet-publish.Tests/Microsoft.DotNet.Tools.Publish.Tests.cs

@@ -149,6 +149,7 @@ namespace Microsoft.DotNet.Tools.Publish.Tests
             publishCommand.GetOutputDirectory().Should().HaveFile("TestLibraryLesser.dll");
             publishCommand.GetOutputDirectory().Should().HaveFile("TestLibraryLesser.pdb");
             publishCommand.GetOutputDirectory().Should().HaveFile("TestLibraryLesser.dll.config");
+            publishCommand.GetOutputDirectory().Should().NotHaveFile("TestLibraryLesser.deps"); 
 
             // dependencies should also be copied
             publishCommand.GetOutputDirectory().Should().HaveFile("Newtonsoft.Json.dll");
@@ -160,6 +161,8 @@ namespace Microsoft.DotNet.Tools.Publish.Tests
             publishCommand.GetOutputDirectory().Should().HaveFile("TestLibraryLesser.dll");
             publishCommand.GetOutputDirectory().Should().HaveFile("TestLibraryLesser.pdb");
             publishCommand.GetOutputDirectory().Should().NotHaveFile("TestLibraryLesser.dll.config");
+            publishCommand.GetOutputDirectory().Should().HaveFile("TestLibraryLesser.deps"); 
+
             // dependencies should also be copied
             publishCommand.GetOutputDirectory().Should().HaveFile("Newtonsoft.Json.dll");
         }