Refactor smoke-test to capture prereqs (#13145)

src/SourceBuild/Arcade/eng/common/templates/job/source-build-run-tarball-build.yml

@@ -69,11 +69,24 @@ jobs:
           _ExcludeOmniSharpTests: false
           _RunOnline: false
   timeoutInMinutes: 300
+  variables:
+  - ${{ if ne(variables['System.TeamProject'], 'public') }}:
+    - group: AzureDevOps-Artifact-Feeds-Pats
   workspace:
     clean: all
 
   steps:
-  - checkout: none
+  - checkout: self
+    clean: true
+
+  - ${{ if ne(variables['System.TeamProject'], 'public') }}:
+    - task: Bash@3
+      displayName: Setup Private Feeds Credentials
+      inputs:
+        filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh
+        arguments: $(Build.SourcesDirectory)/NuGet.config $Token
+      env:
+        Token: $(dn-bot-dnceng-artifact-feeds-rw)
 
   - template: /src/SourceBuild/Arcade/eng/common/templates/steps/source-build-build-tarball.yml
     parameters:
@@ -110,11 +123,24 @@ jobs:
           _Container: ${{ parameters.fedora33Container }}
           _RunOnline: false
     timeoutInMinutes: 180
+    variables:
+    - ${{ if ne(variables['System.TeamProject'], 'public') }}:
+      - group: AzureDevOps-Artifact-Feeds-Pats
     workspace:
       clean: all
 
     steps:
-    - checkout: none
+    - checkout: self
+      clean: true
+
+    - ${{ if ne(variables['System.TeamProject'], 'public') }}:
+      - task: Bash@3
+        displayName: Setup Private Feeds Credentials
+        inputs:
+          filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh
+          arguments: $(Build.SourcesDirectory)/NuGet.config $Token
+        env:
+          Token: $(dn-bot-dnceng-artifact-feeds-rw)
 
     - download: current
       artifact: $(_PreviousSourceBuildArtifact)

src/SourceBuild/Arcade/eng/common/templates/steps/source-build-build-tarball.yml

@@ -66,7 +66,11 @@ steps:
   - script: |
       set -x
 
-      docker run --rm -v ${{ parameters.tarballDir }}:/tarball -w /tarball -e excludeOmniSharpTests=${{ parameters.excludeOmniSharpTests}} ${{ parameters.container }} ./build.sh --run-smoke-test ${{ parameters.additionalBuildArgs }}
+      # Use installer repo's NuGet.config during online testing to utilize internal feeds
+      rm -f ${{ parameters.tarballDir }}/test/Microsoft.DotNet.SourceBuild.SmokeTests/smoke-tests/online.NuGet.Config
+      cp NuGet.config ${{ parameters.tarballDir }}/test/Microsoft.DotNet.SourceBuild.SmokeTests/smoke-tests/online.NuGet.Config
+
+      docker run --rm -v ${{ parameters.tarballDir }}:/tarball -w /tarball -e EXCLUDE_OMNISHARP_TESTS=${{ parameters.excludeOmniSharpTests}} ${{ parameters.container }} ./build.sh --run-smoke-test ${{ parameters.additionalBuildArgs }}
     displayName: Run Tests
 
   # Don't use CopyFiles@2 as it encounters permissions issues because it indexes all files in the source directory graph.
@@ -82,7 +86,7 @@ steps:
       find artifacts/prebuilt-report/ -exec cp {} --parents -t ${targetFolder} \;
       find src/ -type f -name "*.binlog" -exec cp {} --parents -t ${targetFolder} \;
       find src/ -type f -name "*.log" -exec cp {} --parents -t ${targetFolder} \;
-      find test/*/*/*/*/*/testing-smoke/logs -exec cp {} --parents -t ${targetFolder} \;
+      find test/*/*/*/*/*/testing-smoke*/logs -exec cp {} --parents -t ${targetFolder} \;
     displayName: Prepare BuildLogs staging directory
     continueOnError: true
     condition: succeededOrFailed()

src/SourceBuild/tarball/content/build.proj

@@ -99,13 +99,26 @@
       <SdkTarballPath>%(SdkTarballItem.Identity)</SdkTarballPath>
     </PropertyGroup>
 
-    <Exec Command="$(DotnetToolCommand) test ./test/Microsoft.DotNet.SourceBuild.SmokeTests --logger:trx"
+    <Exec Command="$(DotnetToolCommand) test ./test/Microsoft.DotNet.SourceBuild.SmokeTests --logger:trx -c $(Configuration)"
           EnvironmentVariables="
             DOTNET_TARBALL_PATH=$(SdkTarballPath);
             TARGET_RID=$(TargetRid);
             " />
   </Target>
 
+  <Target Name="CreateSmokeTestPrereqs"
+          AfterTargets="RunSmokeTest"
+          Condition="'$(SkipSmokeTestPrereqsTarballCreation)' != 'true'" >
+    <PropertyGroup>
+      <SmokeTestPrereqsTarballName>$(OutputPath)dotnet-smoke-test-prereqs.$(installerOutputPackageVersion).tar.gz</SmokeTestPrereqsTarballName>
+    </PropertyGroup>
+
+    <Exec Command="tar --numeric-owner -czf $(SmokeTestPrereqsTarballName) ."
+          WorkingDirectory="./test/Microsoft.DotNet.SourceBuild.SmokeTests/bin/$(Configuration)/net6.0/smoke-tests/prereq-packages/"/>
+
+    <Message Importance="High" Text="Packaged smoke-test prereqs to $(SmokeTestPrereqsTarballName)" />
+  </Target>
+
   <UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="UploadToAzure" />
 
   <Target Name="PublishPrebuiltReportData">

src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Config.cs

@@ -11,5 +11,9 @@ internal static class Config
     public static string DotNetDirectory { get; } = Environment.GetEnvironmentVariable("DOTNET_DIR") ?? "./.dotnet";
     public static string DotNetTarballPath { get; } = Environment.GetEnvironmentVariable(DotNetTarballPathEnv) ?? string.Empty;
     public const string DotNetTarballPathEnv = "DOTNET_TARBALL_PATH";
+    public static bool ExcludeOmniSharpTests { get; } =
+        bool.TryParse(Environment.GetEnvironmentVariable("EXCLUDE_OMNISHARP_TESTS"), out bool excludeOmniSharpTests) ? excludeOmniSharpTests : false;
+    public static bool ExcludeOnlineTests { get; } =
+        bool.TryParse(Environment.GetEnvironmentVariable("EXCLUDE_ONLINE_TESTS"), out bool excludeOnlineTests) ? excludeOnlineTests : false;
     public static string TargetRid { get; } = Environment.GetEnvironmentVariable("TARGET_RID") ?? string.Empty;
 }

src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/ExecuteHelper.cs

@@ -3,6 +3,7 @@
 // See the LICENSE file in the project root for more information.
 
 using System.Diagnostics;
+using System.Linq;
 using System.Text;
 using Xunit.Abstractions;
 
@@ -27,6 +28,13 @@ internal static class ExecuteHelper
             }
         };
 
+        // The `dotnet test` execution context sets a number of dotnet related ENVs that cause issues when executing
+        // dotnet commands.  Clear these to avoid side effects.
+        foreach (string key in process.StartInfo.Environment.Keys.Where(key => key != "HOME").ToList())
+        {
+            process.StartInfo.Environment.Remove(key);
+        }
+
         StringBuilder stdOutput = new();
         process.OutputDataReceived += new DataReceivedEventHandler((sender, e) => stdOutput.AppendLine(e.Data));
 

src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Microsoft.DotNet.SourceBuild.SmokeTests.csproj

@@ -19,10 +19,18 @@
 
   <ItemGroup>
     <Content Include="baselines/*">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </Content>
     <Content Include="smoke-tests/*">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </Content>
   </ItemGroup>
+
+  <ItemGroup>
+    <SmokeTestPrereqs Include="..\..\packages\smoke-test-prereqs\*" />
+  </ItemGroup>
+
+  <Target Name="CopySmokeTestPrereqs" BeforeTargets="Build">
+    <Copy SourceFiles="@(SmokeTestPrereqs)" DestinationFolder="$(OutputPath)\smoke-tests\prereq-packages" />
+  </Target>
 </Project>

src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/README.md

@@ -0,0 +1,9 @@
+# Source Build Smoke Tests
+
+* Run these tests via `build.sh --run-smoke-test`
+* Various configuration settings are stored in `Config.cs`
+
+## Prereq Packages
+Some prerelease scenarios, usually security updates, require non-source-built packages which are not publicly available.
+Place these packages in the tarball's `packages/smoke-test-prereqs`. When prereq packages are required, the
+`EXCLUDE_ONLINE_TESTS=true` environment variable should be set when running tests via `build.sh --run-smoke-test`.

src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SmokeTests.cs

@@ -24,11 +24,19 @@ public class SmokeTests
     [Fact]
     public void SmokeTestsScript()
     {
-        string smokeTestArgs = $"--dotnetDir {Directory.GetParent(DotNetHelper.DotNetPath)} --minimal --projectOutput --archiveRestoredPackages --targetRid {Config.TargetRid}";
+        string smokeTestArgs = $"--dotnetDir {Directory.GetParent(DotNetHelper.DotNetPath)} --projectOutput --archiveRestoredPackages --targetRid {Config.TargetRid}";
         if (Config.TargetRid.Contains("osx"))
         {
             smokeTestArgs += " --excludeWebHttpsTests";
         }
+        if (Config.ExcludeOmniSharpTests)
+        {
+            smokeTestArgs += " --excludeOmniSharpTests";
+        }
+        if (Config.ExcludeOnlineTests)
+        {
+            smokeTestArgs += " --excludeOnlineTests";
+        }
 
         (Process Process, string StdOut, string StdErr) executeResult = ExecuteHelper.ExecuteProcess("./smoke-tests/smoke-test.sh", smokeTestArgs, OutputHelper);
 

src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/smoke-tests/local.NuGet.Config

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+  <packageSources>
+    <clear />
+    <add key="source-built-packages" value="SOURCE_BUILT_PACKAGES" />
+    <add key="smoke-test-prereqs" value="SMOKE_TEST_PACKAGE_FEED" />
+  </packageSources>
+</configuration>

src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/smoke-tests/online.NuGet.Config

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+  <packageSources>
+    <clear />
+    <add key="dotnet-public" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/index.json" />
+    <add key="dotnet6" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet6/nuget/v3/index.json" />
+    <add key="dotnet6-transport" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet6-transport/nuget/v3/index.json" />
+  </packageSources>
+</configuration>

src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/smoke-tests/smoke-test.sh

@@ -55,7 +55,7 @@ excludeLocalTests=false
 excludeOnlineTests=false
 excludeOmniSharpTests=${excludeOmniSharpTests:-false}
 devCertsVersion="$DEV_CERTS_VERSION_DEFAULT"
-testingDir="$SCRIPT_ROOT/testing-smoke"
+testingDir="$SCRIPT_ROOT/testing-smoke-$(date +"%m%d%H%M%S")"
 cliDir="$testingDir/builtCli"
 logsDir="$testingDir/logs"
 logFile="$logsDir/smoke-test.log"
@@ -63,8 +63,8 @@ omnisharpLogFile="$logsDir/omnisharp.log"
 restoredPackagesDir="$testingDir/packages"
 testingHome="$testingDir/home"
 archiveRestoredPackages=false
-archivedPackagesDir="$testingDir/smoke-test-packages"
+smokeTestPrebuilts="$SCRIPT_ROOT/prereq-packages"
-smokeTestPrebuilts="$SCRIPT_ROOT/packages/smoke-test-packages"
+nonSbSmokeTestPrebuilts="$SCRIPT_ROOT/non-source-built-prereq-packages"
 runningOnline=false
 runningHttps=false
 
@@ -85,10 +85,7 @@ function usage() {
     echo "  --excludeOnlineTests           exclude test that use online sources for nuget packages"
     echo "  --excludeOmniSharpTests        don't run the OmniSharp tests"
     echo "  --devCertsVersion <version>    use dotnet-dev-certs <version> instead of default $DEV_CERTS_VERSION_DEFAULT"
-    echo "  --prodConBlobFeedUrl <url>     override the prodcon blob feed specified in ProdConFeed.txt, removing it if empty"
+    echo "  --archiveRestoredPackages      capture all restored packages to $smokeTestPrebuilts"
-    echo "  --archiveRestoredPackages      capture all restored packages to $archivedPackagesDir"
-    echo "environment:"
-    echo "  prodConBlobFeedUrl    override the prodcon blob feed specified in ProdConFeed.txt, removing it if empty"
     echo ""
 }
 
@@ -149,10 +146,6 @@ while :; do
             shift
             devCertsVersion="$1"
             ;;
-        --prodconblobfeedurl)
-            shift
-            prodConBlobFeedUrl="$1"
-            ;;
         --archiverestoredpackages)
             archiveRestoredPackages=true
             ;;
@@ -166,8 +159,6 @@ while :; do
     shift
 done
 
-prodConBlobFeedUrl="${prodConBlobFeedUrl-}"
-
 function doCommand() {
     lang=$1
     proj=$2
@@ -323,16 +314,19 @@ function runAllTests() {
         doCommand C# console new restore build run multi-rid-publish
         doCommand C# classlib new restore build multi-rid-publish
         doCommand C# xunit new restore test
+        doCommand C# nunit new restore test
         doCommand C# mstest new restore test
 
         doCommand VB console new restore build run multi-rid-publish
         doCommand VB classlib new restore build multi-rid-publish
         doCommand VB xunit new restore test
+        doCommand VB nunit new restore test
         doCommand VB mstest new restore test
 
         doCommand F# console new restore build run multi-rid-publish
         doCommand F# classlib new restore build multi-rid-publish
         doCommand F# xunit new restore test
+        doCommand F# nunit new restore test
         doCommand F# mstest new restore test
     fi
 
@@ -377,8 +371,7 @@ function runOmniSharpTests() {
     tar xf "../omnisharp-linux-x64.tar.gz"
     popd
 
-    # 'blazorwasm' requires prereqs (non-source-built packages) - re-enable with https://github.com/dotnet/source-build/issues/2550
+    for project in blazorwasm blazorserver classlib console mstest mvc nunit web webapp webapi worker xunit ; do
-    for project in blazorserver classlib console mstest mvc nunit web webapp webapi worker xunit ; do
 
         mkdir hello-$project
         pushd hello-$project
@@ -428,18 +421,10 @@ function resetCaches() {
     fi
 }
 
-function setupProdConFeed() {
-    if [ "$prodConBlobFeedUrl" ]; then
-        sed -i.bakProdCon "s|PRODUCT_CONTRUCTION_PACKAGES|$prodConBlobFeedUrl|g" "$testingDir/NuGet.Config"
-    else
-        sed -i.bakProdCon "/PRODUCT_CONTRUCTION_PACKAGES/d" "$testingDir/NuGet.Config"
-    fi
-}
-
 function setupSmokeTestFeed() {
     # Setup smoke-test-packages if they exist
-    if [ -e "$smokeTestPrebuilts" ]; then
+    if [ -e "$nonSbSmokeTestPrebuilts" ]; then
-        sed -i.bakSmokeTestFeed "s|SMOKE_TEST_PACKAGE_FEED|$smokeTestPrebuilts|g" "$testingDir/NuGet.Config"
+        sed -i.bakSmokeTestFeed "s|SMOKE_TEST_PACKAGE_FEED|$nonSbSmokeTestPrebuilts|g" "$testingDir/NuGet.Config"
     else
         sed -i.bakSmokeTestFeed "/SMOKE_TEST_PACKAGE_FEED/d" "$testingDir/NuGet.Config"
     fi
@@ -447,8 +432,21 @@ function setupSmokeTestFeed() {
 
 function copyRestoredPackages() {
     if [ "$archiveRestoredPackages" == "true" ]; then
-        mkdir -p "$archivedPackagesDir"
+        rm -rf "$smokeTestPrebuilts"
-        cp -rf "$restoredPackagesDir"/* "$archivedPackagesDir"
+        rm -rf "$nonSbSmokeTestPrebuilts"
+        mkdir -p "$smokeTestPrebuilts"
+        mkdir -p "$nonSbSmokeTestPrebuilts"
+        find "$restoredPackagesDir" -iname "*.nupkg" -exec mv {} "$smokeTestPrebuilts" \;
+
+        smokeTestPackages=$(find "$smokeTestPrebuilts" -iname "*.nupkg" -type f -printf "%f\n" | tr '[A-Z]' '[a-z]' | sort)
+        sourceBuiltPackages=$(find "$SOURCE_BUILT_PKGS_PATH" -iname "*.nupkg" -type f -printf "%f\n" | tr '[A-Z]' '[a-z]' | sort)
+
+        echo "Removing smoke-test prereq packages that are source built:"
+        comm -23 <(printf "$smokeTestPackages") <(printf "$sourceBuiltPackages") | while read line
+        do
+            echo "$line"
+            cp "$smokeTestPrebuilts/$line" "$nonSbSmokeTestPrebuilts"
+        done 
     fi
 }
 
@@ -494,34 +492,13 @@ SOURCE_BUILT_PKGS_PATH="${ARTIFACTS_DIR}obj/$buildArch/$configuration/blob-feed/
 export DOTNET_ROOT="$dotnetDir"
 export PATH="$dotnetDir:$PATH"
 
-# Run all tests, local restore sources first, online restore sources second
+# Run all tests, online restore sources first, local restore sources second
-if [ "$excludeLocalTests" == "false" ]; then
-    resetCaches
-    runningOnline=false
-    # Setup NuGet.Config with local restore source
-    if [ -e "$SCRIPT_ROOT/smoke-testNuGet.Config" ]; then
-        cp "$SCRIPT_ROOT/smoke-testNuGet.Config" "$testingDir/NuGet.Config"
-        sed -i.bak "s|SOURCE_BUILT_PACKAGES|$SOURCE_BUILT_PKGS_PATH|g" "$testingDir/NuGet.Config"
-        setupProdConFeed
-        setupSmokeTestFeed
-        echo "$testingDir/NuGet.Config Contents:"
-        cat "$testingDir/NuGet.Config"
-    fi
-    echo "RUN ALL TESTS - LOCAL RESTORE SOURCE"
-    runAllTests
-    copyRestoredPackages
-    echo "LOCAL RESTORE SOURCE - ALL TESTS PASSED!"
-fi
-
 if [ "$excludeOnlineTests" == "false" ]; then
     resetCaches
     runningOnline=true
     # Setup NuGet.Config to use online restore sources
-    if [ -e "$SCRIPT_ROOT/smoke-testNuGet.Config" ]; then
+    if [ -e "$SCRIPT_ROOT/online.NuGet.Config" ]; then
-        cp "$SCRIPT_ROOT/smoke-testNuGet.Config" "$testingDir/NuGet.Config"
+        cp "$SCRIPT_ROOT/online.NuGet.Config" "$testingDir/NuGet.Config"
-        sed -i.bak "/SOURCE_BUILT_PACKAGES/d" "$testingDir/NuGet.Config"
-        setupProdConFeed
-        setupSmokeTestFeed
         echo "$testingDir/NuGet.Config Contents:"
         cat "$testingDir/NuGet.Config"
     fi
@@ -531,6 +508,22 @@ if [ "$excludeOnlineTests" == "false" ]; then
     echo "ONLINE RESTORE SOURCE - ALL TESTS PASSED!"
 fi
 
+if [ "$excludeLocalTests" == "false" ]; then
+    resetCaches
+    runningOnline=false
+    # Setup NuGet.Config with local restore source
+    if [ -e "$SCRIPT_ROOT/local.NuGet.Config" ]; then
+        cp "$SCRIPT_ROOT/local.NuGet.Config" "$testingDir/NuGet.Config"
+        sed -i.bak "s|SOURCE_BUILT_PACKAGES|$SOURCE_BUILT_PKGS_PATH|g" "$testingDir/NuGet.Config"
+        setupSmokeTestFeed
+        echo "$testingDir/NuGet.Config Contents:"
+        cat "$testingDir/NuGet.Config"
+    fi
+    echo "RUN ALL TESTS - LOCAL RESTORE SOURCE"
+    runAllTests
+    echo "LOCAL RESTORE SOURCE - ALL TESTS PASSED!"
+fi
+
 if [ "$excludeOmniSharpTests" == "false" ]; then
     runOmniSharpTests
 fi

src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/smoke-tests/smoke-testNuGet.Config

@@ -1,21 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<configuration>
-  <packageSources>
-    <clear />
-    <!--  Begin: Package sources from dotnet-roslyn-analyzers -->
-    <add key="darc-pub-dotnet-roslyn-analyzers-ce71b27" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/darc-pub-dotnet-roslyn-analyzers-ce71b27b/nuget/v3/index.json" />
-    <!--  End: Package sources from dotnet-roslyn-analyzers -->
-    <!--  Begin: Package sources from dotnet-templating -->
-    <add key="darc-pub-dotnet-templating-8470ff3-4" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/darc-pub-dotnet-templating-8470ff31-4/nuget/v3/index.json" />
-    <add key="darc-pub-dotnet-templating-8470ff3-3" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/darc-pub-dotnet-templating-8470ff31-3/nuget/v3/index.json" />
-    <add key="darc-pub-dotnet-templating-8470ff3-2" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/darc-pub-dotnet-templating-8470ff31-2/nuget/v3/index.json" />
-    <add key="darc-pub-dotnet-templating-8470ff3-1" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/darc-pub-dotnet-templating-8470ff31-1/nuget/v3/index.json" />
-    <add key="darc-pub-dotnet-templating-8470ff3" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/darc-pub-dotnet-templating-8470ff31/nuget/v3/index.json" />
-    <!--  End: Package sources from dotnet-templating -->
-    <!--End: Package sources managed by Dependency Flow automation. Do not edit the sources above.-->
-    <add key="source-built-packages" value="SOURCE_BUILT_PACKAGES" />
-    <add key="smoke-test-prereqs" value="SMOKE_TEST_PACKAGE_FEED" />
-    <add key="dotnet-public" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public%40Local/nuget/v3/index.json" />
-    <add key="dotnet6" value="https://dnceng.pkgs.visualstudio.com/public/_packaging/dotnet6/nuget/v3/index.json" />
-  </packageSources>
-</configuration>