Make SmokeTests use repo infrastructure (#19290)

eng/pipelines/templates/jobs/vmr-build.yml

@@ -241,6 +241,11 @@ jobs:
         call $(sourcesPath)\build.cmd -ci -cleanWhileBuilding -prepareMachine /p:TargetOS=${{ parameters.targetOS }} /p:TargetArchitecture=${{ parameters.targetArchitecture }} ${{ parameters.extraProperties }}
       displayName: Build
 
+    - ${{ if eq(parameters.runTests, 'True') }}:
+      - script: |
+          call $(sourcesPath)\build.cmd -ci -prepareMachine -test ${{ parameters.extraProperties }}
+        displayName: Run Tests
+
   - ${{ else }}:
     - ${{ if eq(parameters.buildSourceOnly, 'true') }}:
       - script: |
@@ -356,14 +361,13 @@ jobs:
         set -ex
 
         dockerVolumeArgs="-v $(sourcesPath):/vmr"
-        dockerEnvArgs="-e SMOKE_TESTS_EXCLUDE_OMNISHARP=${{ parameters.excludeOmniSharpTests }} -e SMOKE_TESTS_WARN_SDK_CONTENT_DIFFS=true -e SMOKE_TESTS_RUNNING_IN_CI=true"
         poisonArg=''
 
         if [[ '${{ parameters.enablePoison }}' == 'True' ]]; then
           poisonArg='--poison'
         fi
 
-        docker run --rm $dockerVolumeArgs -w /vmr $dockerEnvArgs ${{ parameters.container }} ./build.sh --source-only $poisonArg --test $(additionalBuildArgs) /p:SmokeTestConsoleVerbosity=detailed
+        docker run --rm $dockerVolumeArgs -w /vmr ${{ parameters.container }} ./build.sh /bl:artifacts/log/Release/Test.binlog --source-only --test $poisonArg $(additionalBuildArgs) /p:SmokeTestsWarnOnSdkContentDiffs=true /p:SmokeTestsExcludeOmniSharpTests=${{ parameters.excludeOmniSharpTests }}
       displayName: Run Tests
 
   - ${{ if eq(parameters.targetOS, 'windows') }}:
@@ -383,14 +387,14 @@ jobs:
 
         cd "$(sourcesPath)"
 
-        CopyWithRelativeFolders "artifacts/"                 $targetFolder "*.binlog"
-        CopyWithRelativeFolders "artifacts/"                 $targetFolder "*.log"
-        CopyWithRelativeFolders "artifacts/"                 $targetFolder "*.diff"
+        CopyWithRelativeFolders "artifacts/log/"             $targetFolder "*.binlog"
+        CopyWithRelativeFolders "artifacts/log/"             $targetFolder "*.log"
+        CopyWithRelativeFolders "artifacts/TestResults/"     $targetFolder "*.binlog"
+        CopyWithRelativeFolders "artifacts/TestResults/"     $targetFolder "*.diff"
+        CopyWithRelativeFolders "artifacts/TestResults/"     $targetFolder "Updated*.txt"
+        CopyWithRelativeFolders "artifacts/TestResults/"     $targetFolder "*.trx"
         CopyWithRelativeFolders "src/"                       $targetFolder "*.binlog"
         CopyWithRelativeFolders "src/"                       $targetFolder "*.log"
-        CopyWithRelativeFolders "test/"                      $targetFolder "*.binlog"
-        CopyWithRelativeFolders "test/"                      $targetFolder "Updated*.diff"
-        CopyWithRelativeFolders "test/"                      $targetFolder "Updated*.txt"
 
         # check if we have assets to publish
         if (Test-Path "artifacts/assets/Release/*") {
@@ -410,18 +414,20 @@ jobs:
         mkdir -p ${targetFolder}
 
         cd "$(sourcesPath)"
-        find artifacts/ -type f -name "*.binlog" -exec rsync -R {} -t ${targetFolder} \;
-        find artifacts/ -type f -name "*.log" -exec rsync -R {} -t ${targetFolder} \;
-        find artifacts/ -type f -name "*.diff" -exec rsync -R {} -t ${targetFolder} \;
+        find artifacts/log/ -type f -name "*.binlog" -exec rsync -R {} -t ${targetFolder} \;
+        find artifacts/log/ -type f -name "*.log" -exec rsync -R {} -t ${targetFolder} \;
+        find artifacts/TestResults/ -type f -name "*.binlog" -exec rsync -R {} -t ${targetFolder} \;
+        find artifacts/TestResults/ -type f -name "*.diff" -exec rsync -R {} -t ${targetFolder} \;
+        find artifacts/TestResults/ -type f -name "Updated*.txt" -exec rsync -R {} -t ${targetFolder} \;
+        find artifacts/TestResults/ -type f -name "*.trx" -exec rsync -R {} -t ${targetFolder} \;
+
         if [[ "${{ parameters.buildSourceOnly }}" == "True" ]]; then
           find artifacts/prebuilt-report/ -exec rsync -R {} -t ${targetFolder} \;
           find artifacts/log/binary-report/ -exec rsync -R {} -t ${targetFolder} \;
         fi
+
         find src/ -type f -name "*.binlog" -exec rsync -R {} -t ${targetFolder} \;
         find src/ -type f -name "*.log" -exec rsync -R {} -t ${targetFolder} \;
-        find test/ -type f -name "*.binlog" -exec rsync -R {} -t ${targetFolder} \;
-        find test/ -type f -name "Updated*.diff" -exec rsync -R {} -t ${targetFolder} \;
-        find test/ -type f -name "Updated*.txt" -exec rsync -R {} -t ${targetFolder} \;
 
         # check if we have assets to publish
         if [ -n "$(ls -A 'artifacts/assets/Release/')" ]; then
@@ -446,11 +452,11 @@ jobs:
       continueOnError: true
       inputs:
         testRunner: vSTest
-        testResultsFiles: 'test/**/*.trx'
+        testResultsFiles: 'artifacts/TestResults/Release/*.trx'
         searchFolder: $(sourcesPath)
         mergeTestResults: true
         publishRunAttachments: true
-        testRunTitle: SourceBuild_SmokeTests_$(Agent.JobName)
+        testRunTitle: Tests_$(Agent.JobName)
 
   - task: CopyFiles@2
     inputs:
@@ -460,6 +466,7 @@ jobs:
         assets/**
       TargetFolder: $(Build.ArtifactStagingDirectory)/publishing
     displayName: Copy artifacts to Artifact Staging Directory
+    condition: succeededOrFailed()
 
   # When building from source, the Private.SourceBuilt.Artifacts archive already contains the nuget packages
   - ${{ if ne(parameters.buildSourceOnly, 'true') }}:
@@ -468,6 +475,7 @@ jobs:
         SourceFolder: $(sourcesPath)/artifacts/packages
         TargetFolder: $(Build.ArtifactStagingDirectory)/publishing/packages
       displayName: Copy packages to Artifact Staging Directory
+      condition: succeededOrFailed()
 
   - ${{ if or(ne(variables['System.TeamProject'], 'internal'), eq(variables['Build.Reason'], 'PullRequest')) }}:
     - publish: $(Build.ArtifactStagingDirectory)/publishing

src/SourceBuild/content/.gitignore

@@ -3,7 +3,4 @@
 /artifacts
 /prereqs/packages
 /src/nuget-client/NuGet.config
-/test/Microsoft.DotNet.SourceBuild.SmokeTests/bin
-/test/Microsoft.DotNet.SourceBuild.SmokeTests/obj
-/test/Microsoft.DotNet.SourceBuild.SmokeTests/TestResults
 *.binlog

src/SourceBuild/content/Directory.Build.props

@@ -73,15 +73,18 @@
     <TargetRid Condition="'$(ShortStack)' == 'true' and '$(TargetOS)' == 'windows'">win-$(TargetArchitecture)</TargetRid>
   </PropertyGroup>
 
+  <!-- Set NuGetPackageRoot before Arcade SDK sets it. -->
+  <PropertyGroup>
+    <!-- Set RestorePackagesPath so that we don't accidentally pull some packages from the global location. -->
+    <RestorePackagesPath Condition="'$(RestorePackagesPath)' == ''">$([MSBuild]::NormalizeDirectory('$(MSBuildThisFileDirectory)', '.packages'))</RestorePackagesPath>
+    <NuGetPackageRoot Condition="'$(NuGetPackageRoot)' == ''">$(RestorePackagesPath)</NuGetPackageRoot>
+  </PropertyGroup>
+
   <Import Project="Sdk.props" Sdk="Microsoft.DotNet.Arcade.Sdk" Condition="'$(SkipArcadeSdkImport)' != 'true'" />
 
   <!-- Init basic Arcade props, if the project importing this file doesn't use Arcade.
        Keep in sync with props/targets in the Arcade.Sdk. -->
   <PropertyGroup Condition="'$(SkipArcadeSdkImport)' == 'true'">
-    <!-- RepoLayout.props -->
-    <NuGetPackageRoot Condition="'$(NuGetPackageRoot)' != ''">$([MSBuild]::NormalizeDirectory('$(NuGetPackageRoot)'))</NuGetPackageRoot>
-    <NuGetPackageRoot Condition="'$(NuGetPackageRoot)' == '' and '$(NUGET_PACKAGES)' != ''">$([MSBuild]::NormalizeDirectory('$(NUGET_PACKAGES)'))</NuGetPackageRoot>
-
     <RepoRoot Condition="'$(RepoRoot)' == ''">$([MSBuild]::NormalizeDirectory('$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildProjectDirectory), 'global.json'))'))</RepoRoot>
 
     <!-- Respect environment variable for the .NET install directory if set; otherwise, use the repo default location -->
@@ -212,6 +215,7 @@
     <PoisonMarkerFile>.prebuilt.xml</PoisonMarkerFile>
     <PoisonReportDataFile>$(PackageReportDir)poison-catalog.xml</PoisonReportDataFile>
     <PoisonedReportFile>$(PackageReportDir)poisoned.txt</PoisonedReportFile>
+    <PoisonUsageReportFile>$(PackageReportDir)poison-usage.xml</PoisonUsageReportFile>
   </PropertyGroup>
 
 </Project>

src/SourceBuild/content/Directory.Build.targets

@@ -6,22 +6,24 @@
     <PropertyGroup>
       <SdkFilenamePrefix>dotnet-sdk-</SdkFilenamePrefix>
     </PropertyGroup>
+  
     <ItemGroup>
       <SdkTarballItem Include="$(ArtifactsAssetsDir)Sdk/**/$(SdkFilenamePrefix)*$(ArchiveExtension)" />
     </ItemGroup>
 
     <!--
       Extract SDK version from SDK tarball filename.
+      Keep in sync with dotnet-sdk's archive location and filename.
 
       Example:
-      dotnet-sdk-9.0.100-alpha.1.24057.1-fedora.38-x64.tar.gz
-      dotnet-sdk-<SdkVersion>-<TargetRid><ArchiveExtension>
+      artifacts\assets\<config>\Sdk\9.0.100-alpha.1.24057.1\dotnet-sdk-9.0.100-alpha.1.24057.1-fedora.38-x64.tar.gz
+      artifacts\assets\<config>\Sdk\<SdkVersion>\dotnet-sdk-<SdkVersion>-<TargetRid><ArchiveExtension>
     -->
     <PropertyGroup>
       <SdkFilename>%(SdkTarballItem.Filename)%(SdkTarballItem.Extension)</SdkFilename>
-      <SdkTarballPath>%(SdkTarballItem.Identity)</SdkTarballPath>
+      <SdkTarballPath Condition="'$(SdkTarballPath)' == ''">%(SdkTarballItem.Identity)</SdkTarballPath>
       <SourceBuiltSdkVersion>$(SdkFilename.Replace('$(SdkFilenamePrefix)','').Replace('-$(TargetRid)$(ArchiveExtension)',''))</SourceBuiltSdkVersion>
     </PropertyGroup>
   </Target>
 
-</Project>
+</Project>

src/SourceBuild/content/Directory.Packages.props

@@ -16,6 +16,10 @@
     <PackageVersion Include="Microsoft.Extensions.FileSystemGlobbing" Version="$(MicrosoftExtensionsFileSystemGlobbingVersion)" />
     <PackageVersion Include="Microsoft.Extensions.Logging.Console" Version="$(MicrosoftExtensionsLoggingConsoleVersion)" />
     <PackageVersion Include="Microsoft.Extensions.Logging" Version="$(MicrosoftExtensionsLoggingVersion)" />
+    <!-- External dependencies -->
+    <PackageVersion Include="Newtonsoft.Json" Version="$(NewtonsoftJsonVersion)" />
+    <PackageVersion Include="xunit.extensibility.core" Version="$(XUnitVersion)" />
+    <PackageVersion Include="xunit.extensibility.execution" Version="$(XUnitVersion)" />
   </ItemGroup>
 
 </Project>

src/SourceBuild/content/build.proj

@@ -1,15 +1,11 @@
-<Project>
-  <Import Project="Sdk.props" Sdk="Microsoft.NET.Sdk" />
+<Project Sdk="Microsoft.Build.Traversal">
 
-  <PropertyGroup>
-    <!-- Fake, to satisfy the SDK. -->
-    <TargetFramework>netstandard2.0</TargetFramework>
-    <DisableImplicitFrameworkReferences>true</DisableImplicitFrameworkReferences>
-  </PropertyGroup>
+  <ItemGroup>
+    <ProjectReference Include="$(ToolsDir)init-build.proj" BuildInParallel="false" />
+    <ProjectReference Include="$(RepoProjectsDir)$(RootRepo).proj" BuildInParallel="false" />
+  </ItemGroup>
 
-  <Import Project="Sdk.targets" Sdk="Microsoft.NET.Sdk" />
-
-  <Target Name="Build">
+  <Target Name="PrintInfo" BeforeTargets="Build">
     <PropertyGroup>
       <BuildModeInfoText Condition="'$(DotNetBuildSourceOnly)' == 'true'">source-build</BuildModeInfoText>
       <BuildModeInfoText Condition="'$(DotNetBuildSourceOnly)' != 'true'">non-source-build</BuildModeInfoText>
@@ -17,18 +13,12 @@
 
     <Message Text="Build Mode: $(BuildModeInfoText)" Importance="high" />
     <Message Text="Build Environment: $(TargetArchitecture) $(Configuration) $(TargetOS) $(TargetRid)" Importance="high" />
-
-    <MSBuild Projects="$(ToolsDir)init-build.proj;
-                       $(RepoProjectsDir)$(RootRepo).proj"
-             Targets="Build"
-             BuildInParallel="false"
-             StopOnFirstFailure="true" />
   </Target>
 
   <!-- Create a merge manifest from the individual repository manifest files. -->
-  <UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.MergeAssetManifests" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" />
+  <UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.MergeAssetManifests" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" TaskFactory="TaskHostFactory" />
   <Target Name="MergeAssetManifests" AfterTargets="Build">
-    <PropertyGroup>      
+    <PropertyGroup>
       <MergedAssetManifestOutputPath>$(ArtifactsDir)VerticalManifest.xml</MergedAssetManifestOutputPath>
     </PropertyGroup>
 
@@ -48,8 +38,8 @@
   <!-- Intentionally below the import to appear at the end. -->
   <Target Name="LogBuildOutputFolders"
           AfterTargets="Build">
-   <Message Importance="high" Text="Shipping packages are located in '$(ArtifactsShippingPackagesDir)'." />
-   <Message Importance="high" Text="Shipping assets are located in '$(ArtifactsAssetsDir)'." />
+    <Message Importance="high" Text="Shipping packages are located in '$(ArtifactsShippingPackagesDir)'." />
+    <Message Importance="high" Text="Shipping assets are located in '$(ArtifactsAssetsDir)'." />
   </Target>
 
 </Project>

src/SourceBuild/content/build.sh

@@ -18,7 +18,7 @@ usage()
   echo "Actions:"
   echo "  --clean                         Clean the solution"
   echo "  --help                          Print help and exit (short: -h)"
-  echo "  --test                          Run smoke tests (short: -t)"
+  echo "  --test                          Run tests (short: -t)"
   echo ""
 
   echo "Source-only settings:"
@@ -33,7 +33,7 @@ usage()
   echo ""
 
   echo "Advanced settings:"
-  echo "  --build-tests                   Build repository tests. May not be supported with --source-only"
+  echo "  --build-repo-tests              Build repository tests. May not be supported with --source-only"
   echo "  --ci                            Set when running on CI server"
   echo "  --clean-while-building          Cleans each repo after building (reduces disk space usage, short: -cwb)"
   echo "  --excludeCIBinarylog            Don't output binary log (short: -nobl)"
@@ -56,10 +56,7 @@ while [[ -h "$source" ]]; do
 done
 scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
 
-# Set the NUGET_PACKAGES dir so that we don't accidentally pull some packages from the global location,
-# They should be pulled from the local feeds.
 packagesRestoredDir="$scriptroot/.packages/"
-export NUGET_PACKAGES=$packagesRestoredDir/
 
 # Common settings
 binary_log=false
@@ -112,8 +109,6 @@ while [[ $# > 0 ]]; do
       exit 0
       ;;
     -test|-t)
-      export NUGET_PACKAGES=$NUGET_PACKAGES/smoke-tests
-      properties="$properties /t:RunSmokeTest"
       test=true
       ;;
 
@@ -180,7 +175,6 @@ while [[ $# > 0 ]]; do
     -use-mono-runtime)
       properties="$properties /p:SourceBuildUseMonoRuntime=true"
       ;;
-
     *)
       properties="$properties $1"
       ;;
@@ -195,19 +189,37 @@ if [[ "$ci" == true ]]; then
   fi
 fi
 
+# Never use the global nuget cache folder
+use_global_nuget_cache=false
+
 . "$scriptroot/eng/common/tools.sh"
 
+project="$scriptroot/build.proj"
+targets="/t:Build"
+
+# This repo uses the VSTest integration instead of the Arcade Test target
+if [[ "$test" == true ]]; then
+  project="$scriptroot/test/tests.proj"
+  targets="$targets;VSTest"
+fi
+
 function Build {
   if [[ "$sourceOnly" != "true" ]]; then
 
     InitializeToolset
 
+    # Manually unset NUGET_PACKAGES as InitializeToolset sets it unconditionally.
+    # The env var shouldn't be set so that the RestorePackagesPath msbuild property is respected.
+    unset NUGET_PACKAGES
+
     local bl=""
     if [[ "$binary_log" == true ]]; then
       bl="/bl:\"$log_dir/Build.binlog\""
     fi
 
-    MSBuild "$scriptroot/build.proj" \
+    MSBuild --restore \
+      $project \
+      $targets \
       $bl \
       /p:Configuration=$configuration \
       $properties
@@ -220,22 +232,22 @@ function Build {
       properties="$properties /p:ContinuousIntegrationBuild=true"
     fi
 
-    "$CLI_ROOT/dotnet" build-server shutdown
-
-    if [ "$test" == "true" ]; then
-      "$CLI_ROOT/dotnet" msbuild "$scriptroot/build.proj" -bl:"$scriptroot/artifacts/log/$configuration/BuildTests.binlog" -flp:"LogFile=$scriptroot/artifacts/log/$configuration/BuildTests.log" -clp:v=m $properties
-    else
+    if [ "$test" != "true" ]; then
+      "$CLI_ROOT/dotnet" build-server shutdown
       "$CLI_ROOT/dotnet" msbuild "$scriptroot/eng/tools/init-build.proj" -bl:"$scriptroot/artifacts/log/$configuration/BuildMSBuildSdkResolver.binlog" -flp:LogFile="$scriptroot/artifacts/log/$configuration/BuildMSBuildSdkResolver.log" /t:ExtractToolsetPackages,BuildMSBuildSdkResolver $properties
-
       # kill off the MSBuild server so that on future invocations we pick up our custom SDK Resolver
       "$CLI_ROOT/dotnet" build-server shutdown
-
-      # Point MSBuild to the custom SDK resolvers folder, so it will pick up our custom SDK Resolver
-      export MSBUILDADDITIONALSDKRESOLVERSFOLDER="$scriptroot/artifacts/toolset/VSSdkResolvers/"
-
-      "$CLI_ROOT/dotnet" msbuild "$scriptroot/build.proj" -bl:"$scriptroot/artifacts/log/$configuration/Build.binlog" -flp:"LogFile=$scriptroot/artifacts/log/$configuration/Build.log" $properties
     fi
 
+    # Point MSBuild to the custom SDK resolvers folder, so it will pick up our custom SDK Resolver
+    export MSBUILDADDITIONALSDKRESOLVERSFOLDER="$scriptroot/artifacts/toolset/VSSdkResolvers/"
+
+    local bl=""
+    if [[ "$binary_log" == true ]]; then
+      bl="/bl:\"$log_dir/Build.binlog\""
+    fi
+
+    "$CLI_ROOT/dotnet" msbuild --restore "$project" $bl $targets $properties
   fi
 }
 

src/SourceBuild/content/eng/Versions.props

@@ -25,13 +25,15 @@
     -->
     <PrivateSourceBuiltSdkVersion>9.0.100-preview.4.24179.1</PrivateSourceBuiltSdkVersion>
     <PrivateSourceBuiltArtifactsVersion>9.0.100-preview.4.24179.1</PrivateSourceBuiltArtifactsVersion>
-    <!-- msbuild -->
-    <MicrosoftBuildVersion>17.8.3</MicrosoftBuildVersion>
-    <!-- runtime -->
-    <MicrosoftExtensionsFileSystemGlobbingVersion>9.0.0-preview.2.24128.5</MicrosoftExtensionsFileSystemGlobbingVersion>
-    <MicrosoftExtensionsLoggingConsoleVersion>9.0.0-preview.2.24128.5</MicrosoftExtensionsLoggingConsoleVersion>
-    <MicrosoftExtensionsLoggingVersion>9.0.0-preview.2.24128.5</MicrosoftExtensionsLoggingVersion>
-    <!-- command-line-api -->
+    <!-- command-line-api dependencies -->
     <SystemCommandLineVersion>2.0.0-beta4.24126.1</SystemCommandLineVersion>
+    <!-- msbuild dependencies -->
+    <MicrosoftBuildVersion>17.8.3</MicrosoftBuildVersion>
+    <!-- runtime dependencies -->
+    <MicrosoftExtensionsFileSystemGlobbingVersion>8.0.0</MicrosoftExtensionsFileSystemGlobbingVersion>
+    <MicrosoftExtensionsLoggingConsoleVersion>8.0.0</MicrosoftExtensionsLoggingConsoleVersion>
+    <MicrosoftExtensionsLoggingVersion>8.0.0</MicrosoftExtensionsLoggingVersion>
+    <!-- external dependencies -->
+    <NewtonsoftJsonVersion>13.0.3</NewtonsoftJsonVersion>
   </PropertyGroup>
 </Project>

src/SourceBuild/content/eng/build.ps1

@@ -8,9 +8,10 @@ Param(
   # Actions
   [switch]$clean,
   [switch][Alias('h')]$help,
+  [switch][Alias('t')]$test,
 
   # Advanced settings
-  [switch]$buildTests,
+  [switch]$buildRepoTests,
   [switch]$ci,
   [switch][Alias('cwb')]$cleanWhileBuilding,
   [switch][Alias('nobl')]$excludeCIBinarylog,
@@ -28,10 +29,11 @@ function Get-Usage() {
   Write-Host "Actions:"
   Write-Host "  -clean                  Clean the solution"
   Write-Host "  -help                   Print help and exit (short: -h)"
+  Write-Host "  -test                   Run tests (repo tests omitted by default) (short: -t)"
   Write-Host ""
 
   Write-Host "Advanced settings:"
-  Write-Host "  -build-tests            Build repository tests"
+  Write-Host "  -buildRepoTests         Build repository tests"
   Write-Host "  -ci                     Set when running on CI server"
   Write-Host "  -cleanWhileBuilding     Cleans each repo after building (reduces disk space usage, short: -cwb)"
   Write-Host "  -excludeCIBinarylog     Don't output binary log (short: -nobl)"
@@ -39,31 +41,49 @@ function Get-Usage() {
   Write-Host ""
 }
 
-. $PSScriptRoot\common\tools.ps1
+$useGlobalNuGetCache=$false
 
-# Set the NUGET_PACKAGES dir so that we don't accidentally pull some packages from the global location,
-# They should be pulled from the local feeds.
-$env:NUGET_PACKAGES="$RepoRoot\.packages\"
+. $PSScriptRoot\common\tools.ps1
 
 if ($help) {
   Get-Usage
   exit 0
 }
 
+$project = Join-Path $RepoRoot "build.proj"
+$arguments = @()
+$targets = "/t:Build"
+
+# This repo uses the VSTest integration instead of the Arcade Test target
+if ($test) {
+  $project = Join-Path (Join-Path $RepoRoot "test") "tests.proj"
+  $targets += ";VSTest"
+}
+
+if ($buildRepoTests) {
+  $arguments += "/p:DotNetBuildTests=true"
+}
+
+if ($cleanWhileBuilding) {
+  $arguments += "/p:CleanWhileBuilding=true"
+}
+
 function Build {
   InitializeToolset
 
-  $bl = if ($binaryLog) { '/bl:' + (Join-Path $LogDir 'Build.binlog') } else { '' }
-  $cwb = if ($cleanWhileBuilding) { '/p:CleanWhileBuilding=true' } else { '' }
-  $btst = if ($buildTests) { '/p:DotNetBuildTests=true' } else { '' }
-  $buildProj = Join-Path $RepoRoot 'build.proj'
+  # Manually unset NUGET_PACKAGES as InitializeToolset sets it unconditionally.
+  # The env var shouldn't be set so that the RestorePackagesPath msbuild property is respected.
+  $env:NUGET_PACKAGES=''
 
-  MSBuild $buildProj `
+  $bl = if ($binaryLog) { '/bl:' + (Join-Path $LogDir 'Build.binlog') } else { '' }
+
+  MSBuild -restore `
+    $project `
     $bl `
+    $targets `
     /p:Configuration=$configuration `
-    $cwb `
-    $btst `
-    @properties
+    @properties `
+    @arguments
 }
 
 try {

src/SourceBuild/content/eng/build.sourcebuild.targets

@@ -1,13 +1,5 @@
 <Project>
 
-  <PropertyGroup>
-    <SmokeTestsDir>$([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'test', 'Microsoft.DotNet.SourceBuild.SmokeTests'))</SmokeTestsDir>
-  </PropertyGroup>
-
-  <PropertyGroup Condition="'$(EnablePoison)' == 'true'">
-    <PoisonUsageReportFile>$(PackageReportDir)poison-usage.xml</PoisonUsageReportFile>
-  </PropertyGroup>
-
   <!-- After building, generate a prebuilt usage report. -->
   <Target Name="ReportPrebuiltUsage"
           AfterTargets="Build"
@@ -46,7 +38,7 @@
   </Target>
 
   <!-- After building, create the sdk symbols tarball. -->
-  <UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.CreateSdkSymbolsLayout" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" />
+  <UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.CreateSdkSymbolsLayout" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" TaskFactory="TaskHostFactory" />
   <Target Name="CreateSdkSymbolsTarball"
           AfterTargets="Build"
           DependsOnTargets="RepackageSymbols"
@@ -85,7 +77,7 @@
              Targets="ReportPrebuiltUsage" />
   </Target>
 
-  <UsingTask TaskName="Microsoft.DotNet.SourceBuild.Tasks.LeakDetection.CheckForPoison" AssemblyFile="$(MicrosoftDotNetSourceBuildTasksLeakDetectionAssembly)" Condition="'$(EnablePoison)' == 'true'" />
+  <UsingTask TaskName="Microsoft.DotNet.SourceBuild.Tasks.LeakDetection.CheckForPoison" AssemblyFile="$(MicrosoftDotNetSourceBuildTasksLeakDetectionAssembly)" TaskFactory="TaskHostFactory" Condition="'$(EnablePoison)' == 'true'" />
   <Target Name="ReportPoisonUsage"
           AfterTargets="Build"
           Condition="'$(EnablePoison)' == 'true'"
@@ -117,7 +109,7 @@
     </Touch>
   </Target>
 
-  <UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.UsageReport.WriteUsageBurndownData" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" />
+  <UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.UsageReport.WriteUsageBurndownData" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" TaskFactory="TaskHostFactory" />
   <Target Name="GeneratePrebuiltBurndownData"
           Inputs="$(MSBuildProjectFullPath)"
           Outputs="$(BaseIntermediateOutputPath)GeneratePrebuiltBurndownData.complete" >
@@ -135,70 +127,6 @@
     </Touch>
   </Target>
 
-  <Target Name="RunSmokeTest">
-    <ItemGroup>
-      <SdkTarballItem Include="$(ArtifactsAssetsDir)dotnet-sdk*$(ArchiveExtension)" />
-      <SourceBuiltArtifactsItem Include="$(ArtifactsAssetsDir)$(SourceBuiltArtifactsTarballName).*$(ArchiveExtension)" />
-    </ItemGroup>
-
-    <PropertyGroup>
-      <CustomTestEnvVars Condition="'$(EnablePoison)' == 'true'">SMOKE_TESTS_POISON_REPORT_PATH=$(PoisonUsageReportFile);</CustomTestEnvVars>
-      <SdkTarballPath>%(SdkTarballItem.Identity)</SdkTarballPath>
-      <SourceBuiltArtifactsPath>%(SourceBuiltArtifactsItem.Identity)</SourceBuiltArtifactsPath>
-      <SmokeTestConsoleVerbosity Condition="'$(SmokeTestConsoleVerbosity)' == ''">normal</SmokeTestConsoleVerbosity>
-    </PropertyGroup>
-
-    <!-- Multiple loggers are specified so that results are captured in trx and pipelines can fail with AzDO pipeline warnings -->
-    <!-- Workaround https://github.com/dotnet/source-build/issues/4003 by disabling VSTestUseMSBuildOutput -->
-    <Exec Command="$(DotnetTool) test $(SmokeTestsDir) --logger:trx --logger:'console;verbosity=$(SmokeTestConsoleVerbosity)' -c $(Configuration) -p:VSTestUseMSBuildOutput=false"
-          IgnoreStandardErrorWarningFormat="true"
-          EnvironmentVariables="
-            SMOKE_TESTS_SDK_TARBALL_PATH=$(SdkTarballPath);
-            SMOKE_TESTS_SOURCEBUILT_ARTIFACTS_PATH=$(SourceBuiltArtifactsPath);
-            SMOKE_TESTS_TARGET_RID=$(TargetRid);
-            SMOKE_TESTS_PORTABLE_RID=$(PortableRid);
-            SMOKE_TESTS_CUSTOM_PACKAGES_PATH=$(CustomSourceBuiltPackagesPath);
-            $(CustomTestEnvVars)" />
-  </Target>
-
-  <Target Name="CreateSmokeTestPrereqsTarball"
-          AfterTargets="RunSmokeTest"
-          Condition="'$(SkipSmokeTestPrereqsTarballCreation)' != 'true'"
-          DependsOnTargets="
-            CheckIfCreateSmokeTestPrereqsExistToPack;
-            CreateSmokeTestPrereqsTarballIfPrereqsExist"/>
-
-  <Target Name="CheckIfCreateSmokeTestPrereqsExistToPack">
-    <PropertyGroup>
-      <SmokeTestsArtifactsDir>$(SmokeTestsDir)bin/$(Configuration)/$(NetCurrent)/</SmokeTestsArtifactsDir>
-      <SmokeTestsPackagesDir>$(SmokeTestsArtifactsDir)packages/</SmokeTestsPackagesDir>
-    </PropertyGroup>
-
-    <ItemGroup>
-      <SmokeTestsPrereqs Include="$(SmokeTestsPackagesDir)**/*.nupkg" />
-    </ItemGroup>
-
-    <Message Text="Found @(SmokeTestsPrereqs->Count()) prereqs in '$(SmokeTestsPackagesDir)'." Importance="High" />
-  </Target>
-
-  <Target Name="CreateSmokeTestPrereqsTarballIfPrereqsExist"
-          DependsOnTargets="DetermineSourceBuiltSdkVersion"
-          Condition="'@(SmokeTestsPrereqs->Count())' != '0'">
-    <PropertyGroup>
-      <SmokeTestPrereqsTarball>$(ArtifactsAssetsDir)dotnet-smoke-test-prereqs.$(SourceBuiltSdkVersion).$(TargetRid)$(ArchiveExtension)</SmokeTestPrereqsTarball>
-      <SmokeTestsPrereqPackagesDir>$(SmokeTestsArtifactsDir)prereq-packages/</SmokeTestsPrereqPackagesDir>
-    </PropertyGroup>
-
-    <Copy SourceFiles="@(SmokeTestsPrereqs)"
-          DestinationFolder="$(SmokeTestsPrereqPackagesDir)" />
-
-    <MakeDir Directories="$([System.IO.Path]::GetDirectoryName('$(SmokeTestPrereqsTarball)'))" />
-    <Exec Command="tar --numeric-owner -czf $(SmokeTestPrereqsTarball) ."
-          WorkingDirectory="$(SmokeTestsPrereqPackagesDir)"/>
-
-    <Message Importance="High" Text="Packaged smoke-test prereqs in '$(SmokeTestPrereqsTarball)'" />
-  </Target>
-
   <Target Name="CreatePrebuiltsTarball"
           AfterTargets="Build"
           DependsOnTargets="

src/SourceBuild/content/eng/pipelines/templates/jobs/sdk-diff-tests.yml

@@ -123,15 +123,13 @@ jobs:
       -bl:$(Build.SourcesDirectory)/artifacts/log/Debug/BuildTests_$(date +"%m%d%H%M%S").binlog 
       -flp:LogFile=$(Build.SourcesDirectory)/artifacts/logs/BuildTests_$(date +"%m%d%H%M%S").log
       -clp:v=m
-      -e SMOKE_TESTS_MSFT_SDK_TARBALL_PATH=$(MsftSdkTarballPath)
-      -e SMOKE_TESTS_SDK_TARBALL_PATH=$(SdkTarballPath)
-      -e SMOKE_TESTS_SOURCEBUILT_ARTIFACTS_PATH=$(SourceBuiltArtifactsPath)
-      -e SMOKE_TESTS_WARN_SDK_CONTENT_DIFFS=false
-      -e SMOKE_TESTS_RUNNING_IN_CI=true
-      -e SMOKE_TESTS_TARGET_RID=${{ parameters.targetRid }}
-      -e SMOKE_TESTS_PORTABLE_RID=$(Platform)-${{ parameters.architecture }}
-      -e SMOKE_TESTS_CUSTOM_PACKAGES_PATH=
-      -e SMOKE_TESTS_INCLUDE_ARTIFACTSSIZE=${{ parameters.includeArtifactsSize }}
+      /p:MsftSdkTarballPath=$(MsftSdkTarballPath)
+      /p:SdkTarballPath=$(SdkTarballPath)
+      /p:SourceBuiltArtifactsPath=$(SourceBuiltArtifactsPath)
+      /p:SmokeTestsWarnOnSdkContentDiffs=false
+      /p:SmokeTestsIncludeArtifactsSizeTests=${{ parameters.includeArtifactsSize }}
+      /p:TargetRid=${{ parameters.targetRid }}
+      /p:PortableRid=$(Platform)-${{ parameters.architecture }}
     displayName: Run Tests
     workingDirectory: $(Build.SourcesDirectory)
 
@@ -140,12 +138,12 @@ jobs:
       targetFolder=$(Build.StagingDirectory)/BuildLogs/
       mkdir -p ${targetFolder}
       cd "$(Build.SourcesDirectory)"
-      find artifacts/ -type f -name "BuildTests*.binlog" -exec cp {} --parents -t ${targetFolder} \;
-      find artifacts/ -type f -name "BuildTests*.log" -exec cp {} --parents -t ${targetFolder} \;
-      find artifacts/ -type f -name "Build.binlog" -exec cp {} --parents -t ${targetFolder} \;
-      find test/ -type f -name "*.binlog" -exec cp {} --parents -t ${targetFolder} \;
-      find test/ -type f -name "Updated*.diff" -exec cp {} --parents -t ${targetFolder} \;
-      find test/ -type f -name "Updated*.txt" -exec cp {} --parents -t ${targetFolder} \;
+      find artifacts/log/ -type f -name "BuildTests*.binlog" -exec cp {} --parents -t ${targetFolder} \;
+      find artifacts/log/ -type f -name "BuildTests*.log" -exec cp {} --parents -t ${targetFolder} \;
+      find artifacts/TestResults/ -type f -name "*.binlog" -exec cp {} --parents -t ${targetFolder} \;
+      find artifacts/TestResults/ -type f -name "*.log" -exec cp {} --parents -t ${targetFolder} \;
+      find artifacts/TestResults/ -type f -name "*.diff" -exec cp {} --parents -t ${targetFolder} \;
+      find artifacts/TestResults/ -type f -name "Updated*.txt" -exec cp {} --parents -t ${targetFolder} \;
     displayName: Prepare BuildLogs staging directory
     continueOnError: true
     condition: succeededOrFailed()

src/SourceBuild/content/eng/pipelines/vmr-license-scan.yml

@@ -93,11 +93,10 @@ jobs:
       -bl:$(Build.SourcesDirectory)/artifacts/log/Debug/BuildTests_$(date +"%m%d%H%M%S").binlog 
       -flp:LogFile=$(Build.SourcesDirectory)/artifacts/logs/BuildTests_$(date +"%m%d%H%M%S").log
       -clp:v=m
-      -e SMOKE_TESTS_LICENSE_SCAN_PATH=$(repoPath)
-      -e SMOKE_TESTS_RUNNING_IN_CI=true
-      -e SMOKE_TESTS_WARN_LICENSE_SCAN_DIFFS=false
-      -e SMOKE_TESTS_TARGET_RID=linux-x64
-      -e SMOKE_TESTS_PORTABLE_RID=linux-x64
+      /p:SmokeTestsLicenseScanPath=$(repoPath)
+      /p:SmokeTestsWarnOnLicenseScanDiffs=false
+      /p:TargetRid=linux-x64
+      /p:PortableRid=linux-x64
     displayName: Run Tests
     workingDirectory: $(Build.SourcesDirectory)
 
@@ -106,18 +105,19 @@ jobs:
       targetFolder=$(Build.StagingDirectory)/BuildLogs/
       mkdir -p ${targetFolder}
       cd "$(Build.SourcesDirectory)"
-      find artifacts/ -type f -name "BuildTests*.binlog" -exec cp {} --parents -t ${targetFolder} \;
-      find artifacts/ -type f -name "BuildTests*.log" -exec cp {} --parents -t ${targetFolder} \;
+      find artifacts/log/ -type f -name "BuildTests*.binlog" -exec cp {} --parents -t ${targetFolder} \;
+      find artifacts/log/ -type f -name "BuildTests*.log" -exec cp {} --parents -t ${targetFolder} \;
+      find artifacts/TestResults/ -type f -name "*.binlog" -exec cp {} --parents -t ${targetFolder} \;
+      find artifacts/TestResults/ -type f -name "*.log" -exec cp {} --parents -t ${targetFolder} \;
       echo "Updated:"
-      find test/ -type f -name "UpdatedLicenseExclusions*.txt"
-      find test/ -type f -name "UpdatedLicenseExclusions*.txt" -exec cp {} --parents -t ${targetFolder} \;
-      find test/ -type f -name "Updated*.json"
-      find test/ -type f -name "Updated*.json" -exec cp {} --parents -t ${targetFolder} \;
+      find artifacts/TestResults/ -type f -name "UpdatedLicenseExclusions*.txt"
+      find artifacts/TestResults/ -type f -name "UpdatedLicenseExclusions*.txt" -exec cp {} --parents -t ${targetFolder} \;
+      find artifacts/TestResults/ -type f -name "Updated*.json"
+      find artifacts/TestResults/ -type f -name "Updated*.json" -exec cp {} --parents -t ${targetFolder} \;
       echo "Results:"
-      find test/ -type f -name "scancode-results*.json"
-      find test/ -type f -name "scancode-results*.json" -exec cp {} --parents -t ${targetFolder} \;
+      find artifacts/TestResults/ -type f -name "scancode-results*.json" -exec cp {} --parents -t ${targetFolder} \;
       echo "All:"
-      ls -R test/
+      ls -R artifacts/TestResults/
       echo "BuildLogs:"
       ls -R ${targetFolder}
     displayName: Prepare BuildLogs staging directory

src/SourceBuild/content/eng/tools/init-build.proj

@@ -15,7 +15,6 @@
       UnpackTarballs;
       BuildUnifiedBuildTasks;
       BuildMSBuildSdkResolver;
-      RestoreUnifiedBuildValidationTests;
       BuildLeakDetection;
       ExtractToolsetPackages;
       GenerateRootFs;
@@ -113,13 +112,6 @@
     </Touch>
   </Target>
 
-  <Target Name="RestoreUnifiedBuildValidationTests"
-          Condition="'$(ShortStack)' != 'true' and '$(PortableBuild)' == 'true' and '$(PgoInstrument)' != 'true'" >
-    <MSBuild Projects="../../test/Microsoft.DotNet.UnifiedBuild.Tests/Microsoft.DotNet.UnifiedBuild.Tests.csproj"
-             Targets="Restore"
-             Properties="MSBuildRestoreSessionId=$([System.Guid]::NewGuid())" />
-  </Target>
-
   <Target Name="GenerateRootFs"
           Condition="'$(BuildOS)' != 'windows' and '$(CrossBuild)' == 'true' and '$(ROOTFS_DIR)' == ''">
     <PropertyGroup>
@@ -155,7 +147,7 @@
     </Touch>
   </Target>
 
-  <UsingTask TaskName="Microsoft.DotNet.SourceBuild.Tasks.LeakDetection.MarkAndCatalogPackages" AssemblyFile="$(MicrosoftDotNetSourceBuildTasksLeakDetectionAssembly)" Condition="'$(EnablePoison)' == 'true'" />
+  <UsingTask TaskName="Microsoft.DotNet.SourceBuild.Tasks.LeakDetection.MarkAndCatalogPackages" AssemblyFile="$(MicrosoftDotNetSourceBuildTasksLeakDetectionAssembly)" TaskFactory="TaskHostFactory" Condition="'$(EnablePoison)' == 'true'" />
   <Target Name="PoisonPrebuiltPackages"
           Condition="'$(EnablePoison)' == 'true'"
           Inputs="$(MSBuildProjectFullPath)"

src/SourceBuild/content/repo-projects/Directory.Build.targets

@@ -99,9 +99,9 @@
     </Copy>
   </Target>
 
-  <UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.AddSourceToNuGetConfig" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" />
-  <UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.RemoveInternetSourcesFromNuGetConfig" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" />
-  <UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.UpdateNuGetConfigPackageSourcesMappings" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" />
+  <UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.AddSourceToNuGetConfig" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" TaskFactory="TaskHostFactory" />
+  <UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.RemoveInternetSourcesFromNuGetConfig" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" TaskFactory="TaskHostFactory" />
+  <UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.UpdateNuGetConfigPackageSourcesMappings" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" TaskFactory="TaskHostFactory" />
   <Target Name="UpdateNuGetConfig"
           DependsOnTargets="CopyNuGetConfig;GetRepositoryReferenceInfo"
           Condition="'$(NuGetConfigFile)' != ''"
@@ -216,7 +216,7 @@
 
   <!-- Update the SDK version in the repo's global.json file.
        This guarantees that all repositories build with the VMR's SDK version. -->
-  <UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.UpdateJson" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" />
+  <UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.UpdateJson" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" TaskFactory="TaskHostFactory" />
   <Target Name="UpdateGlobalJsonVersions"
           Condition="'$(GlobalJsonFile)' != ''"
           Inputs="$(MSBuildProjectFullPath);$(MSBuildThisFileFullPath)"
@@ -256,7 +256,7 @@
 
   <!-- Before a repository builds, set up the version property files that override the repo's defaults.
        There are 3 files generated -->
-  <UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.WritePackageVersionsProps" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" />
+  <UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.WritePackageVersionsProps" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" TaskFactory="TaskHostFactory" />
   <Target Name="CreateBuildInputProps"
           DependsOnTargets="GetRepositoryReferenceInfo"
           Inputs="$(MSBuildProjectFullPath)"
@@ -587,7 +587,7 @@
     ExtractToolPackage;
     CleanupRepo" />
 
-  <UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.UsageReport.WritePackageUsageData" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" />
+  <UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.UsageReport.WritePackageUsageData" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" TaskFactory="TaskHostFactory" />
   <Target Name="WritePrebuiltUsageData"
           Inputs="$(MSBuildProjectFullPath)"
           Outputs="$(BaseIntermediateOutputPath)WritePrebuiltUsageData.complete">
@@ -598,7 +598,8 @@
     <Copy SourceFiles="@(PackageVersionPropsSnapshotFiles)" DestinationFolder="$(PackageReportDir)snapshots/" />
 
     <ItemGroup>
-      <RestoredPackageFile Include="$(NuGetPackageRoot)**/*.nupkg" />
+      <RestoredPackageFile Include="$(NuGetPackageRoot)**/*.nupkg"
+                           Exclude="$(NuGetPackageRoot)tests/**/*.nupkg" />
 
       <!-- Only contains packages when building. -->
       <TarballPrebuiltPackageFile Include="$(PrebuiltPackagesPath)*.nupkg" />
@@ -662,8 +663,8 @@
     </Touch>
   </Target>
 
-  <UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.UsageReport.ValidateUsageAgainstBaseline" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" />
-  <UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.UsageReport.WriteUsageReports" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" />
+  <UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.UsageReport.ValidateUsageAgainstBaseline" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" TaskFactory="TaskHostFactory" />
+  <UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.UsageReport.WriteUsageReports" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" TaskFactory="TaskHostFactory" />
   <Target Name="ReportPrebuiltUsage"
           Inputs="$(MSBuildProjectFullPath)"
           Outputs="$(BaseIntermediateOutputPath)ReportPrebuiltUsage.complete">

src/SourceBuild/content/repo-projects/installer.proj

@@ -141,19 +141,10 @@
   <Target Name="CopySdkArchive"
           AfterTargets="Build"
           DependsOnTargets="DetermineSourceBuiltSdkVersion">
-
-    <Copy SourceFiles="@(SdkTarballItem)"
+    <Copy SourceFiles="$(SdkTarballPath)"
           DestinationFolder="$(ArtifactsAssetsDir)"
           SkipUnchangedFiles="true"
           UseHardlinksIfPossible="true" />
   </Target>
 
-  <Target Name="RunUnifiedBuildValidation"
-          AfterTargets="Build"
-          Condition="'$(PortableBuild)' == 'true' and '$(PgoInstrument)' != 'true' and '$(DotNetBuildSourceOnly)' != 'true'">
-    <MSBuild Projects="../test/Microsoft.DotNet.UnifiedBuild.Tests/Microsoft.DotNet.UnifiedBuild.Tests.csproj"
-             Targets="VSTest"
-             ContinueOnError="true" />
-  </Target>
-
 </Project>

src/SourceBuild/content/repo-projects/nuget-client.proj

@@ -15,6 +15,7 @@
   </ItemGroup>
 
   <ItemGroup>
+    <EnvironmentVariables Include="NUGET_PACKAGES=$(NuGetPackageRoot)" />
     <EnvironmentVariables Include="MS_PFX_PATH=$(NuGetKeyFilePath)" />
     <EnvironmentVariables Include="NUGET_PFX_PATH=$(NuGetKeyFilePath)" />
   </ItemGroup>

src/SourceBuild/content/test/Directory.Build.props

@@ -0,0 +1,15 @@
+<Project>
+
+  <!-- When building source-only, use a custom package cache for tests to make prebuilt detection work. -->
+  <PropertyGroup Condition="'$(DotNetBuildSourceOnly)' == 'true'">
+    <RestorePackagesPath>$([MSBuild]::NormalizeDirectory('$(RestorePackagesPath)', 'tests'))</RestorePackagesPath>
+  </PropertyGroup>
+
+  <Import Project="..\Directory.Build.props" />
+
+  <PropertyGroup>
+    <Nullable>enable</Nullable>
+    <VSTestResultsDirectory>$(ArtifactsTestResultsDir)</VSTestResultsDirectory>
+  </PropertyGroup>
+
+</Project>

src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/ArtifactsSizeTest.cs

@@ -11,6 +11,7 @@ using System.Text.RegularExpressions;
 using System.Formats.Tar;
 using System.Text;
 using System.Threading.Tasks;
+using TestUtilities;
 using Xunit;
 using Xunit.Abstractions;
 
@@ -42,12 +43,11 @@ public class ArtifactsSizeTest : SdkTests
         }
     }
 
-    [SkippableFact(Config.IncludeArtifactsSizeEnv, skipOnFalseEnv: true)]
+    [ConditionalFact(typeof(Config), nameof(Config.IncludeArtifactsSizeTests))]
     public void CompareArtifactsToBaseline()
     {
-        Utilities.ValidateNotNullOrWhiteSpace(Config.SourceBuiltArtifactsPath, Config.SourceBuiltArtifactsPathEnv);
-        Utilities.ValidateNotNullOrWhiteSpace(Config.SdkTarballPath, Config.SdkTarballPathEnv);
-        Utilities.ValidateNotNullOrWhiteSpace(Config.TargetRid, Config.TargetRidEnv);
+        Assert.False(string.IsNullOrWhiteSpace(Config.SourceBuiltArtifactsPath));
+        Assert.False(string.IsNullOrWhiteSpace(Config.SdkTarballPath));
 
         var tarEntries = ProcessSdkAndArtifactsTarballs();
         ScanForDifferences(tarEntries);
@@ -63,8 +63,8 @@ public class ArtifactsSizeTest : SdkTests
         string tempTarballDir = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
         Directory.CreateDirectory(tempTarballDir);
 
-        Utilities.ExtractTarball(Config.SdkTarballPath, tempTarballDir, OutputHelper);
-        Utilities.ExtractTarball(Config.SourceBuiltArtifactsPath, tempTarballDir, OutputHelper);
+        Utilities.ExtractTarball(Config.SdkTarballPath!, tempTarballDir, OutputHelper);
+        Utilities.ExtractTarball(Config.SourceBuiltArtifactsPath!, tempTarballDir, OutputHelper);
 
         Dictionary<string, long> tarEntries = Directory.EnumerateFiles(tempTarballDir, "*", SearchOption.AllDirectories)
             .Where(filePath => !filePath.Contains("SourceBuildReferencePackages"))
@@ -201,7 +201,7 @@ public class ArtifactsSizeTest : SdkTests
     {
         try
         {
-            string actualFilePath = Path.Combine(LogsDirectory, $"UpdatedArtifactsSizes_{Config.TargetRid}.txt");
+            string actualFilePath = Path.Combine(Config.LogsDirectory, $"UpdatedArtifactsSizes_{Config.TargetRid}.txt");
             File.WriteAllLines(
                 actualFilePath,
                 Baseline

src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/BaselineHelper.cs

@@ -43,7 +43,7 @@ namespace Microsoft.DotNet.SourceBuild.SmokeTests
 
         public static void CompareBaselineContents(string baselineFileName, string actualContents, ITestOutputHelper outputHelper, bool warnOnDiffs = false, string baselineSubDir = "")
         {
-            string actualFilePath = Path.Combine(TestBase.LogsDirectory, $"Updated{baselineFileName}");
+            string actualFilePath = Path.Combine(Config.LogsDirectory, $"Updated{baselineFileName}");
             File.WriteAllText(actualFilePath, actualContents);
 
             CompareFiles(GetBaselineFilePath(baselineFileName, baselineSubDir), actualFilePath, outputHelper, warnOnDiffs);

src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Config.cs

@@ -4,48 +4,36 @@
 
 using System;
 using System.IO;
+using System.Runtime.Versioning;
+
+[assembly:UnsupportedOSPlatform("windows")]
 
 namespace Microsoft.DotNet.SourceBuild.SmokeTests;
 
 internal static class Config
 {
-    public const string DotNetDirectoryEnv = "SMOKE_TESTS_DOTNET_DIR";
-    public const string ExcludeOmniSharpEnv = "SMOKE_TESTS_EXCLUDE_OMNISHARP";
-    public const string IncludeArtifactsSizeEnv = "SMOKE_TESTS_INCLUDE_ARTIFACTSSIZE";
-    public const string MsftSdkTarballPathEnv = "SMOKE_TESTS_MSFT_SDK_TARBALL_PATH";
-    public const string PoisonReportPathEnv = "SMOKE_TESTS_POISON_REPORT_PATH";
-    public const string PortableRidEnv = "SMOKE_TESTS_PORTABLE_RID";
-    public const string PrereqsPathEnv = "SMOKE_TESTS_PREREQS_PATH";
-    public const string CustomPackagesPathEnv = "SMOKE_TESTS_CUSTOM_PACKAGES_PATH";
-    public const string SdkTarballPathEnv = "SMOKE_TESTS_SDK_TARBALL_PATH";
-    public const string SourceBuiltArtifactsPathEnv = "SMOKE_TESTS_SOURCEBUILT_ARTIFACTS_PATH";
-    public const string TargetRidEnv = "SMOKE_TESTS_TARGET_RID";
-    public const string WarnSdkContentDiffsEnv = "SMOKE_TESTS_WARN_SDK_CONTENT_DIFFS";
-    public const string WarnLicenseScanDiffsEnv = "SMOKE_TESTS_WARN_LICENSE_SCAN_DIFFS";
-    public const string RunningInCIEnv = "SMOKE_TESTS_RUNNING_IN_CI";
-    public const string LicenseScanPathEnv = "SMOKE_TESTS_LICENSE_SCAN_PATH";
+    const string ConfigSwitchPrefix = "Microsoft.DotNet.SourceBuild.SmokeTests.";
 
-    public static string DotNetDirectory { get; } =
-        Environment.GetEnvironmentVariable(DotNetDirectoryEnv) ?? Path.Combine(Directory.GetCurrentDirectory(), ".dotnet");
-    public static string? MsftSdkTarballPath { get; } = Environment.GetEnvironmentVariable(MsftSdkTarballPathEnv);
-    public static string? PoisonReportPath { get; } = Environment.GetEnvironmentVariable(PoisonReportPathEnv);
-    public static string PortableRid { get; } = Environment.GetEnvironmentVariable(PortableRidEnv) ??
-        throw new InvalidOperationException($"'{Config.PortableRidEnv}' must be specified");
-    public static string? PrereqsPath { get; } = Environment.GetEnvironmentVariable(PrereqsPathEnv);
-    public static string? CustomPackagesPath { get; } = Environment.GetEnvironmentVariable(CustomPackagesPathEnv);
-    public static string? SdkTarballPath { get; } = Environment.GetEnvironmentVariable(SdkTarballPathEnv);
-    public static string? SourceBuiltArtifactsPath { get; } = Environment.GetEnvironmentVariable(SourceBuiltArtifactsPathEnv);
-    public static string TargetRid { get; } = Environment.GetEnvironmentVariable(TargetRidEnv) ??
-        throw new InvalidOperationException($"'{Config.TargetRidEnv}' must be specified");
-    public static string TargetArchitecture { get; } = TargetRid.Split('-')[1];
-    public static bool WarnOnSdkContentDiffs { get; } =
-        bool.TryParse(Environment.GetEnvironmentVariable(WarnSdkContentDiffsEnv), out bool warnOnSdkContentDiffs) && warnOnSdkContentDiffs;
-    public static bool WarnOnLicenseScanDiffs { get; } =
-        bool.TryParse(Environment.GetEnvironmentVariable(WarnLicenseScanDiffsEnv), out bool warnOnLicenseScanDiffs) && warnOnLicenseScanDiffs;
+    public static string DotNetDirectory => (string)AppContext.GetData(ConfigSwitchPrefix + nameof(DotNetDirectory))! ?? throw new InvalidOperationException("DotNetDirectory must be specified");
+    public static string PortableRid => (string)AppContext.GetData(ConfigSwitchPrefix + nameof(PortableRid))! ?? throw new InvalidOperationException("Portable RID must be specified");
+    public static string TargetRid => (string)AppContext.GetData(ConfigSwitchPrefix + nameof(TargetRid))! ?? throw new InvalidOperationException("Target RID must be specified");
+    public static string LogsDirectory => (string)AppContext.GetData(ConfigSwitchPrefix + nameof(LogsDirectory))! ?? throw new InvalidOperationException("Logs directory must be specified");
+
+    public static string? CustomPackagesPath => (string)AppContext.GetData(ConfigSwitchPrefix + nameof(CustomPackagesPath))!;
+    public static bool ExcludeOmniSharpTests => bool.TryParse((string)AppContext.GetData(ConfigSwitchPrefix + nameof(ExcludeOmniSharpTests))!, out bool excludeOmniSharpTests) && excludeOmniSharpTests;
+    public static bool IncludeArtifactsSizeTests => bool.TryParse((string)AppContext.GetData(ConfigSwitchPrefix + nameof(IncludeArtifactsSizeTests))!, out bool includeArtifactsSizeTests) && includeArtifactsSizeTests;
+    public static string? LicenseScanPath => (string)AppContext.GetData(ConfigSwitchPrefix + nameof(LicenseScanPath))!;
+    public static string? MsftSdkTarballPath => (string)AppContext.GetData(ConfigSwitchPrefix + nameof(MsftSdkTarballPath))!;
+    public static string? PoisonReportPath => (string)AppContext.GetData(ConfigSwitchPrefix + nameof(PoisonReportPath))!;
+    public static string? PrereqsPath => (string)AppContext.GetData(ConfigSwitchPrefix + nameof(PrereqsPath))!;
+    public static string? SdkTarballPath => (string)AppContext.GetData(ConfigSwitchPrefix + nameof(SdkTarballPath))!;
+    public static string? SourceBuiltArtifactsPath => (string)AppContext.GetData(ConfigSwitchPrefix + nameof(SourceBuiltArtifactsPath))!;
+    public static bool WarnOnLicenseScanDiffs => bool.TryParse((string)AppContext.GetData(ConfigSwitchPrefix + nameof(WarnOnLicenseScanDiffs))!, out bool warnOnLicenseScanDiffs) && warnOnLicenseScanDiffs;
+    public static bool WarnOnSdkContentDiffs => bool.TryParse((string)AppContext.GetData(ConfigSwitchPrefix + nameof(WarnOnSdkContentDiffs))!, out bool warnOnSdkContentDiffs) && warnOnSdkContentDiffs;
 
     // Indicates whether the tests are being run in the context of a CI pipeline
-    public static bool RunningInCI { get; } =
-        bool.TryParse(Environment.GetEnvironmentVariable(RunningInCIEnv), out bool runningInCI) && runningInCI;
-    
-    public static string? LicenseScanPath { get; } = Environment.GetEnvironmentVariable(LicenseScanPathEnv);
+    public static bool RunningInCI => !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("DOTNET_CI")) ||
+        !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("AGENT_OS"));
+
+    public static string TargetArchitecture => TargetRid.Split('-')[1];
 }

src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Directory.Build.props

@@ -1,10 +1,10 @@
 <Project>
 
-  <!-- This is an empty Directory.Build.props file to prevent projects which reside
-       under this directory to use any of the repository local settings. -->
   <PropertyGroup>
-    <ImportDirectoryPackagesProps>false</ImportDirectoryPackagesProps>
-    <ImportDirectoryBuildTargets>false</ImportDirectoryBuildTargets>
+    <!-- Set manually as the project name doesn't follow the Arcade conventions. -->
+    <IsUnitTestProject>true</IsUnitTestProject>
   </PropertyGroup>
 
+  <Import Project="..\Directory.Build.props" />
+
 </Project>

src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/DotNetHelper.cs

@@ -72,7 +72,7 @@ internal class DotNetHelper
             if (!Directory.Exists(Config.PrereqsPath))
             {
                 throw new InvalidOperationException(
-                    $"Prereqs path '{Config.PrereqsPath}' specified in {Config.PrereqsPathEnv} does not exist.");
+                    $"Prereqs path '{Config.PrereqsPath}' specified via /p:SmokeTestsPrereqsPath='...' does not exist.");
             }
 
             string nugetConfig = File.ReadAllText(nugetConfigPath);
@@ -130,6 +130,10 @@ internal class DotNetHelper
         process.StartInfo.EnvironmentVariables["DOTNET_ROOT"] = Config.DotNetDirectory;
         process.StartInfo.EnvironmentVariables["NUGET_PACKAGES"] = PackagesDirectory;
         process.StartInfo.EnvironmentVariables["PATH"] = $"{Config.DotNetDirectory}:{Environment.GetEnvironmentVariable("PATH")}";
+        // Don't use the repo infrastructure
+        process.StartInfo.EnvironmentVariables["ImportDirectoryBuildProps"] = "false";
+        process.StartInfo.EnvironmentVariables["ImportDirectoryBuildTargets"] = "false";
+        process.StartInfo.EnvironmentVariables["ImportDirectoryPackagesProps"] = "false";
     }
 
     public void ExecuteBuild(string projectName) =>
@@ -253,7 +257,7 @@ internal class DotNetHelper
             fileName += $"-{differentiator}";
         }
 
-        return $"/bl:{Path.Combine(TestBase.LogsDirectory, $"{fileName}.binlog")}";
+        return $"/bl:{Path.Combine(Config.LogsDirectory, $"{fileName}.binlog")}";
     }
 
     private static bool DetermineIsMonoRuntime(string dotnetRoot)

src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/ExclusionsHelper.cs

@@ -68,7 +68,7 @@ internal class ExclusionsHelper
         string updatedFileName = updatedFileTag is null
             ? $"Updated{_exclusionsFileName}"
             : $"Updated{Path.GetFileNameWithoutExtension(_exclusionsFileName)}.{updatedFileTag}{Path.GetExtension(_exclusionsFileName)}";
-        string actualFilePath = Path.Combine(TestBase.LogsDirectory, updatedFileName);
+        string actualFilePath = Path.Combine(Config.LogsDirectory, updatedFileName);
         File.WriteAllLines(actualFilePath, newLines!);
     }
 

src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/LicenseScanTests.cs

@@ -9,6 +9,7 @@ using System.Linq;
 using System.Text.Json;
 using System.Text.Json.Serialization;
 using System.Text.RegularExpressions;
+using TestUtilities;
 using Xunit;
 using Xunit.Abstractions;
 
@@ -111,8 +112,8 @@ public class LicenseScanTests : TestBase
     };
 
     private readonly string _targetRepo;
-
     private readonly string _relativeRepoPath;
+    public static bool IncludeLicenseScanTests => !string.IsNullOrWhiteSpace(Config.LicenseScanPath);
 
     public LicenseScanTests(ITestOutputHelper outputHelper) : base(outputHelper)
     {
@@ -124,7 +125,7 @@ public class LicenseScanTests : TestBase
         _relativeRepoPath = relativeRepoPathMatch.Value;
     }
 
-    [SkippableFact(Config.LicenseScanPathEnv, skipOnNullOrWhiteSpaceEnv: true)]
+    [ConditionalFact(typeof(LicenseScanTests), nameof(IncludeLicenseScanTests))]
     public void ScanForLicenses()
     {
         Assert.NotNull(Config.LicenseScanPath);
@@ -132,7 +133,7 @@ public class LicenseScanTests : TestBase
         // Indicates how long until a timeout occurs for scanning a given file
         const int FileScanTimeoutSeconds = 240;
 
-        string scancodeResultsPath = Path.Combine(LogsDirectory, "scancode-results.json");
+        string scancodeResultsPath = Path.Combine(Config.LogsDirectory, "scancode-results.json");
 
         // Scancode Doc: https://scancode-toolkit.readthedocs.io/en/latest/index.html
         string ignoreOptions = string.Join(" ", s_ignoredFilePatterns.Select(pattern => $"--ignore {pattern}"));
@@ -156,7 +157,7 @@ public class LicenseScanTests : TestBase
         string baselineName = $"Licenses.{_targetRepo}.json";
 
         string baselinePath = BaselineHelper.GetBaselineFilePath(baselineName, BaselineSubDir);
-        string expectedFilePath = Path.Combine(LogsDirectory, baselineName);
+        string expectedFilePath = Path.Combine(Config.LogsDirectory, baselineName);
         if (File.Exists(baselinePath))
         {
             File.Copy(baselinePath, expectedFilePath, overwrite: true);
@@ -169,7 +170,7 @@ public class LicenseScanTests : TestBase
             File.WriteAllText(expectedFilePath, defaultResultsJson);
         }
 
-        string actualFilePath = Path.Combine(TestBase.LogsDirectory, $"Updated{baselineName}");
+        string actualFilePath = Path.Combine(Config.LogsDirectory, $"Updated{baselineName}");
         File.WriteAllText(actualFilePath, json);
 
         BaselineHelper.CompareFiles(expectedFilePath, actualFilePath, OutputHelper, Config.WarnOnLicenseScanDiffs);

src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Microsoft.DotNet.SourceBuild.SmokeTests.csproj

@@ -1,20 +1,24 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
+  <!-- This project is built and run as part of the VMR and by sdk-diff-tests.yml and vmr-license-scan.yml. -->
+
   <PropertyGroup>
-    <TargetFramework>net9.0</TargetFramework>
-    <Nullable>enable</Nullable>
-    <IsPackable>false</IsPackable>
+    <TargetFramework>$(NetCurrent)</TargetFramework>
     <DefaultExcludesInProjectFolder>$(DefaultExcludesInProjectFolder);assets/**/*</DefaultExcludesInProjectFolder>
+    <SmokeTestConsoleVerbosity Condition="'$(SmokeTestConsoleVerbosity)' == ''">normal</SmokeTestConsoleVerbosity>
+    <VSTestLogger>console%3bverbosity=$(SmokeTestConsoleVerbosity);trx%3bLogFileName=$(MSBuildProjectName).trx</VSTestLogger>
+    <!-- Multiple loggers are specified so that results are captured in trx and pipelines can fail with AzDO pipeline warnings
+         Workaround https://github.com/dotnet/source-build/issues/4003 by disabling VSTestUseMSBuildOutput -->
+    <VSTestUseMSBuildOutput>false</VSTestUseMSBuildOutput>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Extensions.FileSystemGlobbing" Version="7.0.0" />
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.5.0" />
-    <PackageReference Include="xunit" Version="2.5.1" />
-    <PackageReference Include="xunit.runner.visualstudio" Version="2.5.1" />
+    <ProjectReference Include="..\TestUtilities\TestUtilities.csproj" />
+  </ItemGroup>
 
-    <!-- Pin transitive package versions -->
-    <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Extensions.FileSystemGlobbing" />
+    <PackageReference Include="Newtonsoft.Json" />
   </ItemGroup>
 
   <ItemGroup>
@@ -22,10 +26,68 @@
              CopyToOutputDirectory="Always" />
   </ItemGroup>
 
-  <ItemGroup>
-    <Content Include="..\..\packages\smoke-test-prereqs\*"
-             CopyToOutputDirectory="Always"
-             Link="assets\smoke-tests\prereq-packages\%(Filename)%(Extension)" />
-  </ItemGroup>
+  <Target Name="SetRuntimeConfigOptions"
+          DependsOnTargets="DetermineSourceBuiltSdkVersion"
+          BeforeTargets="_GenerateRuntimeConfigurationFilesInputCache">
+    <ItemGroup Condition="'$(SourceBuiltArtifactsPath)' == ''">
+      <SourceBuiltArtifactsItem Include="$(ArtifactsAssetsDir)$(SourceBuiltArtifactsTarballName).*$(ArchiveExtension)" />
+    </ItemGroup>
+
+    <PropertyGroup>
+      <SourceBuiltArtifactsPath Condition="'$(SourceBuiltArtifactsPath)' == ''">%(SourceBuiltArtifactsItem.Identity)</SourceBuiltArtifactsPath>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <!-- Required values -->
+      <RuntimeHostConfigurationOption Include="$(MSBuildProjectName).LogsDirectory">
+        <Value>$(ArtifactsTestResultsDir)</Value>
+      </RuntimeHostConfigurationOption>
+      <RuntimeHostConfigurationOption Include="$(MSBuildProjectName).DotNetDirectory">
+        <Value>$(TargetDir)extracted-sdk</Value>
+      </RuntimeHostConfigurationOption>
+      <RuntimeHostConfigurationOption Include="$(MSBuildProjectName).TargetRid">
+        <Value>$(TargetRid)</Value>
+      </RuntimeHostConfigurationOption>
+      <RuntimeHostConfigurationOption Include="$(MSBuildProjectName).PortableRid">
+        <Value>$(PortableRid)</Value>
+      </RuntimeHostConfigurationOption>
+
+      <!-- Optional values -->
+      <RuntimeHostConfigurationOption Include="$(MSBuildProjectName).CustomPackagesPath">
+        <Value>$(SmokeTestsCustomSourceBuiltPackagesPath)</Value>
+      </RuntimeHostConfigurationOption>
+      <RuntimeHostConfigurationOption Include="$(MSBuildProjectName).ExcludeOmniSharpTests">
+        <Value>$(SmokeTestsExcludeOmniSharpTests)</Value>
+      </RuntimeHostConfigurationOption>
+      <RuntimeHostConfigurationOption Include="$(MSBuildProjectName).IncludeArtifactsSizeTests">
+        <Value>$(SmokeTestsIncludeArtifactsSizeTests)</Value>
+      </RuntimeHostConfigurationOption>
+      <RuntimeHostConfigurationOption Include="$(MSBuildProjectName).LicenseScanPath">
+        <Value>$(SmokeTestsLicenseScanPath)</Value>
+      </RuntimeHostConfigurationOption>
+      <RuntimeHostConfigurationOption Include="$(MSBuildProjectName).MsftSdkTarballPath">
+        <Value>$(MsftSdkTarballPath)</Value>
+      </RuntimeHostConfigurationOption>      
+      <RuntimeHostConfigurationOption Include="$(MSBuildProjectName).PoisonReportPath"
+                                      Condition="'$(EnablePoison)' == 'true'">
+        <Value>$(PoisonUsageReportFile)</Value>
+      </RuntimeHostConfigurationOption>
+      <RuntimeHostConfigurationOption Include="$(MSBuildProjectName).PrereqsPath">
+        <Value>$(SmokeTestsPrereqsPath)</Value>
+      </RuntimeHostConfigurationOption>
+      <RuntimeHostConfigurationOption Include="$(MSBuildProjectName).SdkTarballPath">
+        <Value>$(SdkTarballPath)</Value>
+      </RuntimeHostConfigurationOption>
+      <RuntimeHostConfigurationOption Include="$(MSBuildProjectName).SourceBuiltArtifactsPath">
+        <Value>$(SourceBuiltArtifactsPath)</Value>
+      </RuntimeHostConfigurationOption>
+      <RuntimeHostConfigurationOption Include="$(MSBuildProjectName).WarnOnLicenseScanDiffs">
+        <Value>$(SmokeTestsWarnOnLicenseScanDiffs)</Value>
+      </RuntimeHostConfigurationOption>
+      <RuntimeHostConfigurationOption Include="$(MSBuildProjectName).WarnOnSdkContentDiffs">
+        <Value>$(SmokeTestsWarnOnSdkContentDiffs)</Value>
+      </RuntimeHostConfigurationOption>
+    </ItemGroup>
+  </Target>
 
 </Project>

src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Microsoft.DotNet.SourceBuild.SmokeTests.sln

@@ -1,25 +0,0 @@
-
-Microsoft Visual Studio Solution File, Format Version 12.00
-# Visual Studio Version 17
-VisualStudioVersion = 17.0.32112.339
-MinimumVisualStudioVersion = 10.0.40219.1
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.DotNet.SourceBuild.SmokeTests", "Microsoft.DotNet.SourceBuild.SmokeTests.csproj", "{FEB5A0B5-460B-432A-BED8-243557188AFF}"
-EndProject
-Global
-	GlobalSection(SolutionConfigurationPlatforms) = preSolution
-		Debug|Any CPU = Debug|Any CPU
-		Release|Any CPU = Release|Any CPU
-	EndGlobalSection
-	GlobalSection(ProjectConfigurationPlatforms) = postSolution
-		{FEB5A0B5-460B-432A-BED8-243557188AFF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{FEB5A0B5-460B-432A-BED8-243557188AFF}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{FEB5A0B5-460B-432A-BED8-243557188AFF}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{FEB5A0B5-460B-432A-BED8-243557188AFF}.Release|Any CPU.Build.0 = Release|Any CPU
-	EndGlobalSection
-	GlobalSection(SolutionProperties) = preSolution
-		HideSolutionNode = FALSE
-	EndGlobalSection
-	GlobalSection(ExtensibilityGlobals) = postSolution
-		SolutionGuid = {75929D76-EEB5-4793-8335-DF623CC72B56}
-	EndGlobalSection
-EndGlobal

src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/OmniSharpTests.cs

@@ -8,6 +8,7 @@ using System.IO;
 using System.Net.Http;
 using System.Security.AccessControl;
 using System.Threading.Tasks;
+using TestUtilities;
 using Xunit;
 using Xunit.Abstractions;
 
@@ -23,9 +24,11 @@ public class OmniSharpTests : SdkTests
 
     private string OmniSharpDirectory { get; } = Path.Combine(Directory.GetCurrentDirectory(), nameof(OmniSharpTests));
 
+    public static bool IncludeOmniSharpTests => !Config.ExcludeOmniSharpTests && Config.TargetArchitecture != "ppc64le" && Config.TargetArchitecture != "s390x";
+
     public OmniSharpTests(ITestOutputHelper outputHelper) : base(outputHelper) { }
 
-    [SkippableTheory(Config.ExcludeOmniSharpEnv, skipOnTrueEnv: true, skipArchitectures: new[] { "ppc64le", "s390x" })]
+    [ConditionalTheoryAttribute(typeof(OmniSharpTests), nameof(IncludeOmniSharpTests))]
     [InlineData(DotNetTemplate.BlazorWasm)]
     [InlineData(DotNetTemplate.ClassLib)]
     [InlineData(DotNetTemplate.Console)]
@@ -37,7 +40,7 @@ public class OmniSharpTests : SdkTests
     [InlineData(DotNetTemplate.WebApi)]
     [InlineData(DotNetTemplate.Worker)]
     [InlineData(DotNetTemplate.XUnit)]
-    public async void VerifyScenario(DotNetTemplate template)
+    public async Task VerifyScenario(DotNetTemplate template)
     {
         await InitializeOmniSharp();
 
@@ -69,7 +72,7 @@ public class OmniSharpTests : SdkTests
 
             Directory.CreateDirectory(OmniSharpDirectory);
             Utilities.ExtractTarball(omniSharpTarballFile, OmniSharpDirectory, OutputHelper);
-            
+
             // Ensure the run script is executable (see https://github.com/OmniSharp/omnisharp-roslyn/issues/2547)
             File.SetUnixFileMode($"{OmniSharpDirectory}/run", UnixFileMode.UserRead | UnixFileMode.UserExecute);
         }

src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/PoisonTests.cs

@@ -5,15 +5,19 @@
 using System;
 using System.IO;
 using System.Text.RegularExpressions;
+using TestUtilities;
+using Xunit;
 using Xunit.Abstractions;
 
 namespace Microsoft.DotNet.SourceBuild.SmokeTests
 {
     public class PoisonTests : SdkTests
     {
+        public static bool IncludePoisonTests => !string.IsNullOrWhiteSpace(Config.PoisonReportPath);
+
         public PoisonTests(ITestOutputHelper outputHelper) : base(outputHelper) { }
         
-        [SkippableFact(Config.PoisonReportPathEnv, skipOnNullOrWhiteSpaceEnv: true)]
+        [ConditionalFact(typeof(PoisonTests), nameof(IncludePoisonTests))]
         public void VerifyUsage()
         {
             if (!File.Exists(Config.PoisonReportPath))

src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/README.md

@@ -1,7 +1,23 @@
 # Source Build Smoke Tests
 
-* Run these tests via `build.sh --run-smoke-test`
-* Various configuration settings are stored in `Config.cs`
+* Run these tests via `build.sh --test`
+
+The following properties are automatically available during test execution but can be overwritten:
+- PoisonUsageReportFile
+- SdkTarballPath
+- SourceBuiltArtifactsPath
+
+Optional msbuild properties:
+- MsftSdkTarballPath
+- SmokeTestsCustomSourceBuiltPackagesPath
+- SmokeTestsExcludeOmniSharpTests
+- SmokeTestsIncludeArtifactsSizeTests
+- SmokeTestsLicenseScanPath
+- SmokeTestsPrereqsPath
+- SmokeTestsWarnOnLicenseScanDiffs
+- SmokeTestsWarnOnSdkContentDiffs
+
+Make sure to rebuild the test project when changing one of those values.
 
 ## Dependencies
 
@@ -15,5 +31,5 @@ The following programs are used by some tests:
 ## Prereq Packages
 
 Some prerelease scenarios, usually security updates, require non-source-built packages which are not publicly available.
-Specify the directory where these packages can be found via the `SMOKE_TESTS_PREREQS_PATH` environment variable when running tests via `build.sh --run-smoke-test` e.g.
-`SMOKE_TESTS_PREREQS_PATH=prereqs/packages/smoke-test-prereqs`.
+Specify the directory where these packages can be found via the `SmokeTestsPrereqsPath` msbuild property when running tests via `build.sh ---test` e.g.
+`/p:SmokeTestsPrereqsPath=prereqs/packages/smoke-test-prereqs`.

src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SdkContentTests.cs

@@ -11,6 +11,7 @@ using System.Linq;
 using System.Reflection;
 using System.Text.RegularExpressions;
 using Microsoft.Extensions.FileSystemGlobbing;
+using TestUtilities;
 using Xunit;
 using Xunit.Abstractions;
 
@@ -21,6 +22,7 @@ public class SdkContentTests : SdkTests
 {
     private const string MsftSdkType = "msft";
     private const string SourceBuildSdkType = "sb";
+    public static bool IncludeSdkContentTests => !string.IsNullOrWhiteSpace(Config.MsftSdkTarballPath) && !string.IsNullOrWhiteSpace(Config.SdkTarballPath);
 
     public SdkContentTests(ITestOutputHelper outputHelper) : base(outputHelper) { }
 
@@ -31,7 +33,7 @@ public class SdkContentTests : SdkTests
     /// This makes the baseline durable between releases.  This does mean however, entries
     /// in the baseline may appear identical if the diff is version specific.
     /// </Summary>
-    [SkippableFact(new[] { Config.MsftSdkTarballPathEnv, Config.SdkTarballPathEnv }, skipOnNullOrWhiteSpaceEnv: true)]
+    [ConditionalFact(typeof(SdkContentTests), nameof(IncludeSdkContentTests))]
     public void CompareMsftToSbFileList()
     {
         const string msftFileListingFileName = "msftSdkFiles.txt";
@@ -48,7 +50,7 @@ public class SdkContentTests : SdkTests
         exclusionsHelper.GenerateNewBaselineFile("FileList");
     }
 
-    [SkippableFact(new[] { Config.MsftSdkTarballPathEnv, Config.SdkTarballPathEnv }, skipOnNullOrWhiteSpaceEnv: true)]
+    [ConditionalFact(typeof(SdkContentTests), nameof(IncludeSdkContentTests))]
     public void CompareMsftToSbAssemblyVersions()
     {
         Assert.NotNull(Config.MsftSdkTarballPath);

src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SkippableFactAttribute.cs

@@ -1,53 +0,0 @@
-// Licensed to the .NET Foundation under one or more agreements.
-// The .NET Foundation licenses this file to you under the MIT license.
-// See the LICENSE file in the project root for more information.
-
-using System;
-using System.Linq;
-using Xunit;
-
-namespace Microsoft.DotNet.SourceBuild.SmokeTests;
-
-/// <summary>
-/// A Fact that will be skipped based on the specified environment variable's value.
-/// </summary>
-internal class SkippableFactAttribute : FactAttribute
-{
-    public SkippableFactAttribute(string envName, bool skipOnNullOrWhiteSpaceEnv = false, bool skipOnTrueEnv = false, bool skipOnFalseEnv = false, string[] skipArchitectures = null) =>
-        EvaluateSkips(skipOnNullOrWhiteSpaceEnv, skipOnTrueEnv, skipOnFalseEnv, skipArchitectures, (skip) => Skip = skip, envName);
-
-    public SkippableFactAttribute(string[] envNames, bool skipOnNullOrWhiteSpaceEnv = false, bool skipOnTrueEnv = false, bool skipOnFalseEnv = false, string[] skipArchitectures = null) =>
-        EvaluateSkips(skipOnNullOrWhiteSpaceEnv, skipOnTrueEnv, skipOnFalseEnv, skipArchitectures, (skip) => Skip = skip, envNames);
-
-    public static void EvaluateSkips(bool skipOnNullOrWhiteSpaceEnv, bool skipOnTrueEnv, bool skipOnFalseEnv, string[] skipArchitectures, Action<string> setSkip, params string[] envNames)
-    {
-        foreach (string envName in envNames)
-        {
-            string? envValue = Environment.GetEnvironmentVariable(envName);
-
-            if (skipOnNullOrWhiteSpaceEnv && string.IsNullOrWhiteSpace(envValue))
-            {
-                setSkip($"Skipping because `{envName}` is null or whitespace");
-                break;
-            }
-            else if (skipOnTrueEnv && bool.TryParse(envValue, out bool boolValue) && boolValue)
-            {
-                setSkip($"Skipping because `{envName}` is set to True");
-                break;
-            }
-            else if (skipOnFalseEnv && (!bool.TryParse(envValue, out boolValue) || !boolValue))
-            {
-                setSkip($"Skipping because `{envName}` is set to False or an invalid value");
-                break;
-            }
-        }
-
-        if (skipArchitectures != null) {
-            string? arch = Config.TargetArchitecture;
-            if (skipArchitectures.Contains(arch))
-            {
-                setSkip($"Skipping because arch is `{arch}`");
-            }
-        }
-    }
-}

src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SkippableTheoryAttribute.cs

@@ -1,19 +0,0 @@
-// Licensed to the .NET Foundation under one or more agreements.
-// The .NET Foundation licenses this file to you under the MIT license.
-// See the LICENSE file in the project root for more information.
-
-using Xunit;
-
-namespace Microsoft.DotNet.SourceBuild.SmokeTests;
-
-/// <summary>
-/// A Theory that will be skipped based on the specified environment variable's value.
-/// </summary>
-internal class SkippableTheoryAttribute : TheoryAttribute
-{
-    public SkippableTheoryAttribute(string envName, bool skipOnNullOrWhiteSpaceEnv = false, bool skipOnTrueEnv = false, bool skipOnFalseEnv = false, string[] skipArchitectures = null) =>
-        SkippableFactAttribute.EvaluateSkips(skipOnNullOrWhiteSpaceEnv, skipOnTrueEnv, skipOnFalseEnv, skipArchitectures, (skip) => Skip = skip, envName);
-
-    public SkippableTheoryAttribute(string[] envNames, bool skipOnNullOrWhiteSpaceEnv = false, bool skipOnTrueEnv = false, bool skipOnFalseEnv = false, string[] skipArchitectures = null) =>
-        SkippableFactAttribute.EvaluateSkips(skipOnNullOrWhiteSpaceEnv, skipOnTrueEnv, skipOnFalseEnv, skipArchitectures, (skip) => Skip = skip, envNames);
-}

src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SourceBuiltArtifactsTests.cs

@@ -8,6 +8,7 @@ using System.IO;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
+using TestUtilities;
 using Xunit;
 using Xunit.Abstractions;
 
@@ -15,12 +16,15 @@ namespace Microsoft.DotNet.SourceBuild.SmokeTests;
 
 public class SourceBuiltArtifactsTests : SdkTests
 {
+    public static bool IncludeSourceBuiltArtifactsTests => !string.IsNullOrWhiteSpace(Config.SourceBuiltArtifactsPath);
+    
     public SourceBuiltArtifactsTests(ITestOutputHelper outputHelper) : base(outputHelper) { }
 
-    [SkippableFact(Config.SourceBuiltArtifactsPathEnv, skipOnNullOrWhiteSpaceEnv: true)]
+    [ConditionalFact(typeof(SourceBuiltArtifactsTests), nameof(IncludeSourceBuiltArtifactsTests))]
     public void VerifyVersionFile()
     {
         Assert.NotNull(Config.SourceBuiltArtifactsPath);
+
         string outputDir = Path.Combine(Directory.GetCurrentDirectory(), "sourcebuilt-artifacts");
         Directory.CreateDirectory(outputDir);
         try

src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SourcelinkTests.cs

@@ -48,7 +48,7 @@ public class SourcelinkTests : SdkTests
             // We are validating dotnet-symbols-all-*.tar.gz which contains all source-built symbols, including
             // SDK-specific symbols that are also packaged in dotnet-symbols-sdk-*.tar.gz.
             Utilities.ExtractTarball(
-                Utilities.GetFile(Path.GetDirectoryName(Config.SourceBuiltArtifactsPath), "dotnet-symbols-all-*.tar.gz"),
+                Utilities.GetFile(Path.GetDirectoryName(Config.SourceBuiltArtifactsPath)!, "dotnet-symbols-all-*.tar.gz"),
                 symbolsRoot,
                 OutputHelper);
 

src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/TestBase.cs

@@ -9,17 +9,15 @@ namespace Microsoft.DotNet.SourceBuild.SmokeTests;
 
 public abstract class TestBase
 {
-    public static string LogsDirectory { get; } = Path.Combine(Directory.GetCurrentDirectory(), "logs");
-
     public ITestOutputHelper OutputHelper { get; }
 
     public TestBase(ITestOutputHelper outputHelper)
     {
         OutputHelper = outputHelper;
 
-        if (!Directory.Exists(LogsDirectory))
+        if (!Directory.Exists(Config.LogsDirectory))
         {
-            Directory.CreateDirectory(LogsDirectory);
+            Directory.CreateDirectory(Config.LogsDirectory);
         }
     }
 }

src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Utilities.cs

@@ -35,19 +35,21 @@ public static class Utilities
         using GZipStream decompressorStream = new(fileStream, CompressionMode.Decompress);
         using TarReader reader = new(decompressorStream);
 
-        TarEntry entry;
+        TarEntry? entry;
         while ((entry = reader.GetNextEntry()) is not null)
         {
             if (matcher.Match(entry.Name).HasMatches)
             {
                 string outputPath = Path.Join(outputDir, entry.Name);
-                Directory.CreateDirectory(Path.GetDirectoryName(outputPath));
+                Directory.CreateDirectory(Path.GetDirectoryName(outputPath)!);
 
                 using FileStream outputFileStream = File.Create(outputPath);
-                entry.DataStream.CopyTo(outputFileStream);
-                break;
+                entry.DataStream!.CopyTo(outputFileStream);
+                return;
             }
         }
+
+        throw new FileNotFoundException($"Could not find {targetFilePath} in {tarballPath}.");
     }
 
     public static IEnumerable<string> GetTarballContentNames(string tarballPath)
@@ -56,7 +58,7 @@ public static class Utilities
         using GZipStream decompressorStream = new(fileStream, CompressionMode.Decompress);
         using TarReader reader = new(decompressorStream);
 
-        TarEntry entry;
+        TarEntry? entry;
         while ((entry = reader.GetNextEntry()) is not null)
         {
             yield return entry.Name;
@@ -71,7 +73,7 @@ public static class Utilities
         foreach (ZipArchiveEntry entry in zip.Entries)
         {
             string outputPath = Path.Combine(outputDir, entry.FullName);
-            Directory.CreateDirectory(Path.GetDirectoryName(outputPath));
+            Directory.CreateDirectory(Path.GetDirectoryName(outputPath)!);
             entry.ExtractToFile(outputPath);
         }
     }
@@ -129,14 +131,6 @@ public static class Utilities
         outputHelper.WriteLine("##vso[task.complete result=SucceededWithIssues;]");
     }
 
-    public static void ValidateNotNullOrWhiteSpace(string? variable, string variableName)
-    {
-        if (string.IsNullOrWhiteSpace(variable))
-        {
-            throw new ArgumentException($"{variableName} is null, empty, or whitespace.");
-        }
-    }
-
     public static string GetFile(string path, string pattern)
     {
         string[] files = Directory.GetFiles(path, pattern, SearchOption.AllDirectories);

src/SourceBuild/content/test/Microsoft.DotNet.UnifiedBuild.Tests/BaselineHelper.cs

@@ -41,9 +41,9 @@ namespace Microsoft.DotNet.SourceBuild.SmokeTests
             Assert.Null(message);
         }
 
-        public static void CompareBaselineContents(string baselineFileName, string actualContents, ITestOutputHelper outputHelper, bool warnOnDiffs = false, string baselineSubDir = "")
+        public static void CompareBaselineContents(string baselineFileName, string actualContents, string logsDirectory, ITestOutputHelper outputHelper, bool warnOnDiffs = false, string baselineSubDir = "")
         {
-            string actualFilePath = Path.Combine(TestBase.LogsDirectory, $"Updated{baselineFileName}");
+            string actualFilePath = Path.Combine(logsDirectory, $"Updated{baselineFileName}");
             if (!actualContents.EndsWith(Environment.NewLine))
                 actualContents += Environment.NewLine;
             File.WriteAllText(actualFilePath, actualContents);

src/SourceBuild/content/test/Microsoft.DotNet.UnifiedBuild.Tests/Config.cs

@@ -15,6 +15,7 @@ using Xunit.Sdk;
 namespace Microsoft.DotNet.SourceBuild.SmokeTests;
 public class Config : IDisposable
 {
+    public string LogsDirectory { get; }
     public string MsftSdkArchivePath { get; }
     public string UbBuildVersion { get; }
     public string PortableRid { get; }
@@ -31,6 +32,7 @@ public class Config : IDisposable
     {
         _sink = sink;
         string? noDiagnosticMessages = (string?)AppContext.GetData(NoDiagnosticMessagesSwitch);
+        LogsDirectory = (string)(AppContext.GetData(LogsDirectorySwitch) ?? throw new InvalidOperationException("Logs directory must be specified"));
         NoDiagnosticMessages = string.IsNullOrEmpty(noDiagnosticMessages) ? false : bool.Parse(noDiagnosticMessages);
         UbBuildVersion = (string)(AppContext.GetData(BuildVersionSwitch) ?? throw new InvalidOperationException("Unified Build version must be specified"));
         TargetRid = (string)(AppContext.GetData(TargetRidSwitch) ?? throw new InvalidOperationException("Target RID must be specified"));
@@ -48,6 +50,7 @@ public class Config : IDisposable
         }
         LogMessage($$"""
             Test config values:
+            {{nameof(LogsDirectory)}}='{{LogsDirectory}}'
             {{nameof(UbBuildVersion)}}='{{UbBuildVersion}}'
             {{nameof(TargetRid)}}='{{TargetRid}}'
             {{nameof(PortableRid)}}='{{PortableRid}}'
@@ -60,6 +63,7 @@ public class Config : IDisposable
     }
 
     const string ConfigSwitchPrefix = "Microsoft.DotNet.UnifiedBuild.Tests.";
+    const string LogsDirectorySwitch = ConfigSwitchPrefix + nameof(LogsDirectory);
     const string BuildVersionSwitch = ConfigSwitchPrefix + nameof(UbBuildVersion);
     const string TargetRidSwitch = ConfigSwitchPrefix + nameof(TargetRid);
     const string PortableRidSwitch = ConfigSwitchPrefix + nameof(PortableRid);

src/SourceBuild/content/test/Microsoft.DotNet.UnifiedBuild.Tests/Microsoft.DotNet.UnifiedBuild.Tests.csproj

@@ -2,13 +2,9 @@
 
   <PropertyGroup>
     <TargetFramework>$(NetCurrent)</TargetFramework>
-    <Nullable>enable</Nullable>
     <DefaultExcludesInProjectFolder>$(DefaultExcludesInProjectFolder);assets/**/*</DefaultExcludesInProjectFolder>
     <UBTestsWarnOnSdkContentDiffs>true</UBTestsWarnOnSdkContentDiffs>
-    <VSTestLogger>console%3bverbosity=diagnostic;trx%3bverbosity=diagnostic%3bLogFileName=Microsoft.DotNet.UnifiedBuild.Tests.xml</VSTestLogger>
-    <VSTestResultsDirectory>$(ArtifactsTestResultsDir)</VSTestResultsDirectory>
-    <VsTestUseMSBuildOutput>true</VsTestUseMSBuildOutput>
-    <VSTestCLIRunSettings>$(VSTestCLIRunSettings);RunConfiguration.DotNetHostPath=$(DotnetTool)</VSTestCLIRunSettings>
+    <VSTestLogger>console%3bverbosity=diagnostic;trx%3bverbosity=diagnostic%3bLogFileName=$(MSBuildProjectName).trx</VSTestLogger>
   </PropertyGroup>
 
   <ItemGroup>
@@ -24,26 +20,29 @@
           DependsOnTargets="DetermineSourceBuiltSdkVersion"
           BeforeTargets="_GenerateRuntimeConfigurationFilesInputCache">
     <ItemGroup>
-      <RuntimeHostConfigurationOption Include="Microsoft.DotNet.UnifiedBuild.Tests.UbSdkArchivePath">
+      <RuntimeHostConfigurationOption Include="$(MSBuildProjectName).LogsDirectory">
+        <Value>$(ArtifactsTestResultsDir)</Value>
+      </RuntimeHostConfigurationOption>
+      <RuntimeHostConfigurationOption Include="$(MSBuildProjectName).UbSdkArchivePath">
         <Value>$(SdkTarballPath)</Value>
       </RuntimeHostConfigurationOption>
-      <RuntimeHostConfigurationOption Include="Microsoft.DotNet.UnifiedBuild.Tests.TargetRid">
+      <RuntimeHostConfigurationOption Include="$(MSBuildProjectName).TargetRid">
         <Value>$(TargetRid)</Value>
       </RuntimeHostConfigurationOption>
-      <RuntimeHostConfigurationOption Include="Microsoft.DotNet.UnifiedBuild.Tests.PortableRid">
+      <RuntimeHostConfigurationOption Include="$(MSBuildProjectName).PortableRid">
         <Value>$(PortableRid)</Value>
       </RuntimeHostConfigurationOption>
-      <RuntimeHostConfigurationOption Include="Microsoft.DotNet.UnifiedBuild.Tests.UbBuildVersion">
+      <RuntimeHostConfigurationOption Include="$(MSBuildProjectName).UbBuildVersion">
         <Value>$(SourceBuiltSdkVersion)</Value>
       </RuntimeHostConfigurationOption>
-      <RuntimeHostConfigurationOption Include="Microsoft.DotNet.UnifiedBuild.Tests.WarnOnSdkContentDiffs">
+      <RuntimeHostConfigurationOption Include="$(MSBuildProjectName).WarnOnSdkContentDiffs">
         <Value>$(UBTestsWarnOnSdkContentDiffs)</Value>
       </RuntimeHostConfigurationOption>
       <!-- Set for local testing to avoid downloading a new SDK each test -->
-      <RuntimeHostConfigurationOption Include="Microsoft.DotNet.UnifiedBuild.Tests.MsftSdkArchivePath">
+      <RuntimeHostConfigurationOption Include="$(MSBuildProjectName).MsftSdkArchivePath">
         <Value>$(UBTestsMsftSdkArchivePath)</Value>
       </RuntimeHostConfigurationOption>
-      <RuntimeHostConfigurationOption Include="Microsoft.DotNet.UnifiedBuild.Tests.NoDiagnosticMessages">
+      <RuntimeHostConfigurationOption Include="$(MSBuildProjectName).NoDiagnosticMessages">
         <Value>$(VSTestUseMSBuildOutput)</Value>
       </RuntimeHostConfigurationOption>
     </ItemGroup>

src/SourceBuild/content/test/Microsoft.DotNet.UnifiedBuild.Tests/SdkContentTests.cs

@@ -44,7 +44,7 @@ public class SdkContentTests : TestBase
 
         string diff = BaselineHelper.DiffFiles(msftFileListingFileName, ubFileListingFileName, OutputHelper);
         diff = RemoveDiffMarkers(diff);
-        BaselineHelper.CompareBaselineContents(Exclusions.GetBaselineFileDiffFileName(), diff, OutputHelper, Config.WarnOnSdkContentDiffs);
+        BaselineHelper.CompareBaselineContents(Exclusions.GetBaselineFileDiffFileName(), diff, Config.LogsDirectory, OutputHelper, Config.WarnOnSdkContentDiffs);
     }
 
     [Fact]
@@ -78,7 +78,7 @@ public class SdkContentTests : TestBase
 
             string diff = BaselineHelper.DiffFiles(MsftVersionsFileName, UbVersionsFileName, OutputHelper);
             diff = RemoveDiffMarkers(diff);
-            BaselineHelper.CompareBaselineContents($"MsftToUbSdkAssemblyVersions-{Config.TargetRid}.diff", diff, OutputHelper, Config.WarnOnSdkContentDiffs);
+            BaselineHelper.CompareBaselineContents($"MsftToUbSdkAssemblyVersions-{Config.TargetRid}.diff", diff, Config.LogsDirectory, OutputHelper, Config.WarnOnSdkContentDiffs);
         }
         finally
         {

src/SourceBuild/content/test/Microsoft.DotNet.UnifiedBuild.Tests/TestBase.cs

@@ -10,8 +10,7 @@ namespace Microsoft.DotNet.SourceBuild.SmokeTests;
 
 public abstract class TestBase : IClassFixture<Config>
 {
-    protected Config Config;
-    public static string LogsDirectory { get; } = Path.Combine(Directory.GetCurrentDirectory(), "logs");
+    protected readonly Config Config;
 
     public ITestOutputHelper OutputHelper { get; }
 
@@ -19,9 +18,10 @@ public abstract class TestBase : IClassFixture<Config>
     {
         OutputHelper = outputHelper;
         Config = config;
-        if (!Directory.Exists(LogsDirectory))
+
+        if (!Directory.Exists(Config.LogsDirectory))
         {
-            Directory.CreateDirectory(LogsDirectory);
+            Directory.CreateDirectory(Config.LogsDirectory);
         }
     }
 }

src/SourceBuild/content/test/TestUtilities/ConditionalDiscovererException.cs

@@ -0,0 +1,12 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+
+namespace TestUtilities;
+
+internal class ConditionalDiscovererException : Exception
+{
+    public ConditionalDiscovererException(string message) : base(message) { }
+}

src/SourceBuild/content/test/TestUtilities/ConditionalFactAttribute.cs

@@ -0,0 +1,31 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Xunit;
+using Xunit.Abstractions;
+using Xunit.Sdk;
+
+namespace TestUtilities;
+
+[XunitTestCaseDiscoverer("TestUtilities.ConditionalFactDiscoverer", "TestUtilities")]
+[AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
+public sealed class ConditionalFactAttribute : FactAttribute
+{
+    public Type? CalleeType { get; }
+    public string[] ConditionMemberNames { get; }
+
+    public ConditionalFactAttribute(Type calleeType, params string[] conditionMemberNames)
+    {
+        CalleeType = calleeType;
+        ConditionMemberNames = conditionMemberNames;
+    }
+
+    public ConditionalFactAttribute(params string[] conditionMemberNames)
+    {
+        ConditionMemberNames = conditionMemberNames;
+    }
+}

src/SourceBuild/content/test/TestUtilities/ConditionalFactDiscoverer.cs

@@ -0,0 +1,28 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Xunit.Abstractions;
+using Xunit.Sdk;
+
+namespace TestUtilities;
+
+public class ConditionalFactDiscoverer : FactDiscoverer
+{
+    public ConditionalFactDiscoverer(IMessageSink diagnosticMessageSink) : base(diagnosticMessageSink) { }
+
+    protected override IXunitTestCase CreateTestCase(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute)
+    {
+        if (ConditionalTestDiscoverer.TryEvaluateSkipConditions(discoveryOptions, DiagnosticMessageSink, testMethod, factAttribute.GetConstructorArguments().ToArray(), out string skipReason, out ExecutionErrorTestCase errorTestCase))
+        {
+            return skipReason != null
+                ? (IXunitTestCase) new SkippedTestCase(skipReason, DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod)
+                : base.CreateTestCase(discoveryOptions, testMethod, factAttribute);
+        }
+
+        return errorTestCase;
+    }
+}

src/SourceBuild/content/test/TestUtilities/ConditionalTestDiscoverer.cs

@@ -0,0 +1,181 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+#nullable disable
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reflection;
+using Xunit.Abstractions;
+using Xunit.Sdk;
+
+namespace TestUtilities;
+
+// Internal helper class for code common to conditional test discovery through
+// [ConditionalFact] and [ConditionalTheory]
+internal static class ConditionalTestDiscoverer
+{
+    // This helper method evaluates the given condition member names for a given set of test cases.
+    // If any condition member evaluates to 'false', the test cases are marked to be skipped.
+    // The skip reason is the collection of all the condition members that evaluated to 'false'.
+    internal static string EvaluateSkipConditions(ITestMethod testMethod, object[] conditionArguments)
+    {
+        Type calleeType = null;
+        string[] conditionMemberNames = null;
+
+        if (CheckInputToSkipExecution(conditionArguments, ref calleeType, ref conditionMemberNames, testMethod)) return null;
+
+        MethodInfo testMethodInfo = testMethod.Method.ToRuntimeMethod();
+        Type testMethodDeclaringType = testMethodInfo.DeclaringType;
+        List<string> falseConditions = new List<string>(conditionMemberNames.Count());
+
+        foreach (string entry in conditionMemberNames)
+        {
+            string conditionMemberName = entry;
+
+            // Null condition member names are silently tolerated
+            if (string.IsNullOrWhiteSpace(conditionMemberName))
+            {
+                continue;
+            }
+
+            Type declaringType;
+
+            if (calleeType != null)
+            {
+                declaringType = calleeType;
+            }
+            else
+            {
+                declaringType = testMethodDeclaringType;
+
+                string[] symbols = conditionMemberName.Split('.');
+                if (symbols.Length == 2)
+                {
+                    conditionMemberName = symbols[1];
+                    ITypeInfo type = testMethod.TestClass.Class.Assembly.GetTypes(false).Where(t => t.Name.Contains(symbols[0])).FirstOrDefault();
+                    if (type != null)
+                    {
+                        declaringType = type.ToRuntimeType();
+                    }
+                }
+            }
+
+            Func<bool> conditionFunc;
+            if ((conditionFunc = LookupConditionalMember(declaringType, conditionMemberName)) == null)
+            {
+                throw new ConditionalDiscovererException(GetFailedLookupString(conditionMemberName, declaringType));
+            }
+
+            // In the case of multiple conditions, collect the results of all
+            // of them to produce a summary skip reason.
+            try
+            {
+                if (!conditionFunc())
+                {
+                    falseConditions.Add(conditionMemberName);
+                }
+            }
+            catch (Exception exc)
+            {
+                falseConditions.Add($"{conditionMemberName} ({exc.GetType().Name})");
+            }
+        }
+
+        // Compose a summary of all conditions that returned false.
+        if (falseConditions.Count > 0)
+        {
+            return string.Format("Condition(s) not met: \"{0}\"", string.Join("\", \"", falseConditions));
+        }
+
+        // No conditions returned false (including the absence of any conditions).
+        return null;
+    }
+
+    internal static bool TryEvaluateSkipConditions(ITestFrameworkDiscoveryOptions discoveryOptions, IMessageSink diagnosticMessageSink, ITestMethod testMethod, object[] conditionArguments, out string skipReason, out ExecutionErrorTestCase errorTestCase)
+    {
+        skipReason = null;
+        errorTestCase = null;
+        try
+        {
+            skipReason = EvaluateSkipConditions(testMethod, conditionArguments);
+            return true;
+        }
+        catch (ConditionalDiscovererException e)
+        {
+            errorTestCase = new ExecutionErrorTestCase(
+                diagnosticMessageSink,
+                discoveryOptions.MethodDisplayOrDefault(),
+                discoveryOptions.MethodDisplayOptionsOrDefault(),
+                testMethod,
+                e.Message);
+            return false;
+        }
+    }
+
+    internal static string GetFailedLookupString(string name, Type type)
+    {
+        return
+            $"An appropriate member '{name}' could not be found. " +
+            $"The conditional method needs to be a static method, property, or field on the type {type} or any ancestor, " +
+            "of any visibility, accepting zero arguments, and having a return type of Boolean.";
+    }
+    
+    internal static Func<bool> LookupConditionalMember(Type t, string name)
+    {
+        if (t == null || name == null)
+            return null;
+
+        TypeInfo ti = t.GetTypeInfo();
+
+        MethodInfo mi = ti.GetDeclaredMethod(name);
+        if (mi != null && mi.IsStatic && mi.GetParameters().Length == 0 && mi.ReturnType == typeof(bool))
+            return () => (bool)mi.Invoke(null, null);
+
+        PropertyInfo pi = ti.GetDeclaredProperty(name);
+        if (pi != null && pi.PropertyType == typeof(bool) && pi.GetMethod != null && pi.GetMethod.IsStatic && pi.GetMethod.GetParameters().Length == 0)
+            return () => (bool)pi.GetValue(null);
+
+        FieldInfo fi = ti.GetDeclaredField(name);
+        if (fi != null && fi.FieldType == typeof(bool) && fi.IsStatic)
+            return () => (bool)fi.GetValue(null);
+
+        return LookupConditionalMember(ti.BaseType, name);
+    }
+
+    internal static bool CheckInputToSkipExecution(object[] conditionArguments, ref Type calleeType, ref string[] conditionMemberNames, ITestMethod testMethod = null)
+    {
+        // A null or empty list of conditionArguments is treated as "no conditions".
+        // and the test cases will be executed.
+        // Example: [ConditionalClass()]
+        if (conditionArguments == null || conditionArguments.Length == 0) return true;
+
+        calleeType = conditionArguments[0] as Type;
+        if (calleeType != null)
+        {
+            if (conditionArguments.Length < 2)
+            {
+                // [ConditionalFact(typeof(x))] no provided methods.
+                return true;
+            }
+
+            // [ConditionalFact(typeof(x), "MethodName")]
+            conditionMemberNames = conditionArguments[1] as string[];
+        }
+        else
+        {
+            // For [ConditionalClass], unable to get the Type info. All test cases will be executed.
+            if (testMethod == null) return true;
+
+            // [ConditionalFact("MethodName")]
+            conditionMemberNames = conditionArguments[0] as string[];
+        }
+
+        // [ConditionalFact((string[]) null)]
+        if (conditionMemberNames == null || conditionMemberNames.Count() == 0) return true;
+
+        return false;
+    }
+}

src/SourceBuild/content/test/TestUtilities/ConditionalTheoryAttribute.cs

@@ -0,0 +1,31 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Xunit;
+using Xunit.Abstractions;
+using Xunit.Sdk;
+
+namespace TestUtilities;
+
+[XunitTestCaseDiscoverer("TestUtilities.ConditionalTheoryDiscoverer", "TestUtilities")]
+[AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
+public sealed class ConditionalTheoryAttribute : TheoryAttribute
+{
+    public Type? CalleeType { get; }
+    public string[] ConditionMemberNames { get; }
+
+    public ConditionalTheoryAttribute(Type calleeType, params string[] conditionMemberNames)
+    {
+        CalleeType = calleeType;
+        ConditionMemberNames = conditionMemberNames;
+    }
+
+    public ConditionalTheoryAttribute(params string[] conditionMemberNames)
+    {
+        ConditionMemberNames = conditionMemberNames;
+    }
+}

src/SourceBuild/content/test/TestUtilities/ConditionalTheoryDiscoverer.cs

@@ -0,0 +1,56 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Xunit.Abstractions;
+using Xunit.Sdk;
+
+namespace TestUtilities;
+
+public class ConditionalTheoryDiscoverer : TheoryDiscoverer
+{
+    private readonly Dictionary<IMethodInfo, string> _conditionCache = new();
+
+    public ConditionalTheoryDiscoverer(IMessageSink diagnosticMessageSink) : base(diagnosticMessageSink) { }
+
+    protected override IEnumerable<IXunitTestCase> CreateTestCasesForTheory(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute)
+    {
+        if (ConditionalTestDiscoverer.TryEvaluateSkipConditions(discoveryOptions, DiagnosticMessageSink, testMethod, theoryAttribute.GetConstructorArguments().ToArray(), out string skipReason, out ExecutionErrorTestCase errorTestCase))
+        {
+            return skipReason != null ?
+                new[] { new SkippedTestCase(skipReason, DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod) } :
+                base.CreateTestCasesForTheory(discoveryOptions, testMethod, theoryAttribute);
+        }
+
+        return new IXunitTestCase[] { errorTestCase };
+    }
+
+    protected override IEnumerable<IXunitTestCase> CreateTestCasesForDataRow(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute, object[] dataRow)
+    {
+        IMethodInfo methodInfo = testMethod.Method;
+        List<IXunitTestCase> skippedTestCase = new();
+
+        if (!_conditionCache.TryGetValue(methodInfo, out string? skipReason))
+        {
+            if (!ConditionalTestDiscoverer.TryEvaluateSkipConditions(discoveryOptions, DiagnosticMessageSink, testMethod, theoryAttribute.GetConstructorArguments().ToArray(), out skipReason, out ExecutionErrorTestCase errorTestCase))
+            {
+                return new IXunitTestCase[] { errorTestCase };
+            }
+
+            _conditionCache.Add(methodInfo, skipReason!);
+
+            if (skipReason != null)
+            {
+                // If this is the first time we evalute the condition we return a SkippedTestCase to avoid printing a skip for every inline-data.
+                skippedTestCase.Add(new SkippedTestCase(skipReason, DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod));
+            }
+        }
+
+        return skipReason != null ?
+            (IEnumerable<IXunitTestCase>)skippedTestCase :
+            base.CreateTestCasesForDataRow(discoveryOptions, testMethod, theoryAttribute, dataRow);
+    }
+}

src/SourceBuild/content/test/TestUtilities/SkippedTestCase.cs

@@ -0,0 +1,50 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+using Xunit.Abstractions;
+using Xunit.Sdk;
+
+namespace TestUtilities;
+
+/// <summary>Wraps another test case that should be skipped.</summary>
+internal sealed class SkippedTestCase : XunitTestCase
+{
+    private string? _skipReason;
+
+    [Obsolete("Called by the de-serializer; should only be called by deriving classes for de-serialization purposes")]
+    public SkippedTestCase() : base()
+    {
+    }
+
+    public SkippedTestCase(
+        string skipReason,
+        IMessageSink diagnosticMessageSink,
+        TestMethodDisplay defaultMethodDisplay,
+        TestMethodDisplayOptions defaultMethodDisplayOptions,
+        ITestMethod testMethod,
+        object[]? testMethodArguments = null)
+        : base(diagnosticMessageSink, defaultMethodDisplay, defaultMethodDisplayOptions, testMethod, testMethodArguments)
+    {
+        _skipReason = skipReason;
+    }
+
+    protected override string GetSkipReason(IAttributeInfo factAttribute)
+        => _skipReason ?? base.GetSkipReason(factAttribute);
+
+    public override void Deserialize(IXunitSerializationInfo data)
+    {
+        base.Deserialize(data);
+        _skipReason = data.GetValue<string>(nameof(_skipReason));
+    }
+
+    public override void Serialize(IXunitSerializationInfo data)
+    {
+        base.Serialize(data);
+        data.AddValue(nameof(_skipReason), _skipReason);
+    }
+}

src/SourceBuild/content/test/TestUtilities/TestUtilities.csproj

@@ -0,0 +1,13 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>$(NetCurrent)</TargetFramework>
+    <IsTestUtilityProject>true</IsTestUtilityProject>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="xunit.extensibility.core" />
+    <PackageReference Include="xunit.extensibility.execution" />
+  </ItemGroup>
+
+</Project>

src/SourceBuild/content/test/tests.proj

@@ -0,0 +1,10 @@
+<Project Sdk="Microsoft.Build.Traversal">
+
+  <ItemGroup>
+    <ProjectReference Include="Microsoft.DotNet.SourceBuild.SmokeTests\Microsoft.DotNet.SourceBuild.SmokeTests.csproj"
+                      Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
+    <ProjectReference Include="Microsoft.DotNet.UnifiedBuild.Tests\Microsoft.DotNet.UnifiedBuild.Tests.csproj"
+                      Condition="'$(ShortStack)' != 'true' and '$(PortableBuild)' == 'true' and '$(PgoInstrument)' != 'true'" />
+  </ItemGroup>
+
+</Project>