diff --git a/src/SourceBuild/Arcade/eng/common/templates/job/source-build-run-tarball-build.yml b/src/SourceBuild/Arcade/eng/common/templates/job/source-build-run-tarball-build.yml index 28648866d..556f3fbe4 100644 --- a/src/SourceBuild/Arcade/eng/common/templates/job/source-build-run-tarball-build.yml +++ b/src/SourceBuild/Arcade/eng/common/templates/job/source-build-run-tarball-build.yml @@ -69,11 +69,24 @@ jobs: _ExcludeOmniSharpTests: false _RunOnline: false timeoutInMinutes: 300 + variables: + - ${{ if ne(variables['System.TeamProject'], 'public') }}: + - group: AzureDevOps-Artifact-Feeds-Pats workspace: clean: all steps: - - checkout: none + - checkout: self + clean: true + + - ${{ if ne(variables['System.TeamProject'], 'public') }}: + - task: Bash@3 + displayName: Setup Private Feeds Credentials + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh + arguments: $(Build.SourcesDirectory)/NuGet.config $Token + env: + Token: $(dn-bot-dnceng-artifact-feeds-rw) - template: /src/SourceBuild/Arcade/eng/common/templates/steps/source-build-build-tarball.yml parameters: @@ -110,11 +123,24 @@ jobs: _Container: ${{ parameters.fedora33Container }} _RunOnline: false timeoutInMinutes: 180 + variables: + - ${{ if ne(variables['System.TeamProject'], 'public') }}: + - group: AzureDevOps-Artifact-Feeds-Pats workspace: clean: all steps: - - checkout: none + - checkout: self + clean: true + + - ${{ if ne(variables['System.TeamProject'], 'public') }}: + - task: Bash@3 + displayName: Setup Private Feeds Credentials + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh + arguments: $(Build.SourcesDirectory)/NuGet.config $Token + env: + Token: $(dn-bot-dnceng-artifact-feeds-rw) - download: current artifact: $(_PreviousSourceBuildArtifact) diff --git a/src/SourceBuild/Arcade/eng/common/templates/steps/source-build-build-tarball.yml b/src/SourceBuild/Arcade/eng/common/templates/steps/source-build-build-tarball.yml index 7556b961d..cfb965134 100644 --- a/src/SourceBuild/Arcade/eng/common/templates/steps/source-build-build-tarball.yml +++ b/src/SourceBuild/Arcade/eng/common/templates/steps/source-build-build-tarball.yml @@ -66,7 +66,11 @@ steps: - script: | set -x - docker run --rm -v ${{ parameters.tarballDir }}:/tarball -w /tarball -e excludeOmniSharpTests=${{ parameters.excludeOmniSharpTests}} ${{ parameters.container }} ./build.sh --run-smoke-test ${{ parameters.additionalBuildArgs }} + # Use installer repo's NuGet.config during online testing to utilize internal feeds + rm -f ${{ parameters.tarballDir }}/test/Microsoft.DotNet.SourceBuild.SmokeTests/smoke-tests/online.NuGet.Config + cp NuGet.config ${{ parameters.tarballDir }}/test/Microsoft.DotNet.SourceBuild.SmokeTests/smoke-tests/online.NuGet.Config + + docker run --rm -v ${{ parameters.tarballDir }}:/tarball -w /tarball -e EXCLUDE_OMNISHARP_TESTS=${{ parameters.excludeOmniSharpTests}} ${{ parameters.container }} ./build.sh --run-smoke-test ${{ parameters.additionalBuildArgs }} displayName: Run Tests # Don't use CopyFiles@2 as it encounters permissions issues because it indexes all files in the source directory graph. @@ -82,7 +86,7 @@ steps: find artifacts/prebuilt-report/ -exec cp {} --parents -t ${targetFolder} \; find src/ -type f -name "*.binlog" -exec cp {} --parents -t ${targetFolder} \; find src/ -type f -name "*.log" -exec cp {} --parents -t ${targetFolder} \; - find test/*/*/*/*/*/testing-smoke/logs -exec cp {} --parents -t ${targetFolder} \; + find test/*/*/*/*/*/testing-smoke*/logs -exec cp {} --parents -t ${targetFolder} \; displayName: Prepare BuildLogs staging directory continueOnError: true condition: succeededOrFailed() diff --git a/src/SourceBuild/tarball/content/build.proj b/src/SourceBuild/tarball/content/build.proj index b8bd6061e..9c59468f6 100644 --- a/src/SourceBuild/tarball/content/build.proj +++ b/src/SourceBuild/tarball/content/build.proj @@ -99,13 +99,26 @@ %(SdkTarballItem.Identity) - + + + $(OutputPath)dotnet-smoke-test-prereqs.$(installerOutputPackageVersion).tar.gz + + + + + + + diff --git a/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Config.cs b/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Config.cs index 3dfc38f69..be0e6b411 100644 --- a/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Config.cs +++ b/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Config.cs @@ -11,5 +11,9 @@ internal static class Config public static string DotNetDirectory { get; } = Environment.GetEnvironmentVariable("DOTNET_DIR") ?? "./.dotnet"; public static string DotNetTarballPath { get; } = Environment.GetEnvironmentVariable(DotNetTarballPathEnv) ?? string.Empty; public const string DotNetTarballPathEnv = "DOTNET_TARBALL_PATH"; + public static bool ExcludeOmniSharpTests { get; } = + bool.TryParse(Environment.GetEnvironmentVariable("EXCLUDE_OMNISHARP_TESTS"), out bool excludeOmniSharpTests) ? excludeOmniSharpTests : false; + public static bool ExcludeOnlineTests { get; } = + bool.TryParse(Environment.GetEnvironmentVariable("EXCLUDE_ONLINE_TESTS"), out bool excludeOnlineTests) ? excludeOnlineTests : false; public static string TargetRid { get; } = Environment.GetEnvironmentVariable("TARGET_RID") ?? string.Empty; } diff --git a/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/ExecuteHelper.cs b/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/ExecuteHelper.cs index 35c6dc7a7..2c38e43ca 100644 --- a/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/ExecuteHelper.cs +++ b/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/ExecuteHelper.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information. using System.Diagnostics; +using System.Linq; using System.Text; using Xunit.Abstractions; @@ -27,6 +28,13 @@ internal static class ExecuteHelper } }; + // The `dotnet test` execution context sets a number of dotnet related ENVs that cause issues when executing + // dotnet commands. Clear these to avoid side effects. + foreach (string key in process.StartInfo.Environment.Keys.Where(key => key != "HOME").ToList()) + { + process.StartInfo.Environment.Remove(key); + } + StringBuilder stdOutput = new(); process.OutputDataReceived += new DataReceivedEventHandler((sender, e) => stdOutput.AppendLine(e.Data)); diff --git a/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Microsoft.DotNet.SourceBuild.SmokeTests.csproj b/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Microsoft.DotNet.SourceBuild.SmokeTests.csproj index 54d87df9d..0846de14b 100644 --- a/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Microsoft.DotNet.SourceBuild.SmokeTests.csproj +++ b/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Microsoft.DotNet.SourceBuild.SmokeTests.csproj @@ -19,10 +19,18 @@ - PreserveNewest + Always - PreserveNewest + Always + + + + + + + + diff --git a/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/README.md b/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/README.md new file mode 100644 index 000000000..91317e44e --- /dev/null +++ b/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/README.md @@ -0,0 +1,9 @@ +# Source Build Smoke Tests + +* Run these tests via `build.sh --run-smoke-test` +* Various configuration settings are stored in `Config.cs` + +## Prereq Packages +Some prerelease scenarios, usually security updates, require non-source-built packages which are not publicly available. +Place these packages in the tarball's `packages/smoke-test-prereqs`. When prereq packages are required, the +`EXCLUDE_ONLINE_TESTS=true` environment variable should be set when running tests via `build.sh --run-smoke-test`. diff --git a/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SmokeTests.cs b/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SmokeTests.cs index 869e97276..6ebd64e7d 100644 --- a/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SmokeTests.cs +++ b/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SmokeTests.cs @@ -24,11 +24,19 @@ public class SmokeTests [Fact] public void SmokeTestsScript() { - string smokeTestArgs = $"--dotnetDir {Directory.GetParent(DotNetHelper.DotNetPath)} --minimal --projectOutput --archiveRestoredPackages --targetRid {Config.TargetRid}"; + string smokeTestArgs = $"--dotnetDir {Directory.GetParent(DotNetHelper.DotNetPath)} --projectOutput --archiveRestoredPackages --targetRid {Config.TargetRid}"; if (Config.TargetRid.Contains("osx")) { smokeTestArgs += " --excludeWebHttpsTests"; } + if (Config.ExcludeOmniSharpTests) + { + smokeTestArgs += " --excludeOmniSharpTests"; + } + if (Config.ExcludeOnlineTests) + { + smokeTestArgs += " --excludeOnlineTests"; + } (Process Process, string StdOut, string StdErr) executeResult = ExecuteHelper.ExecuteProcess("./smoke-tests/smoke-test.sh", smokeTestArgs, OutputHelper); diff --git a/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/smoke-tests/local.NuGet.Config b/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/smoke-tests/local.NuGet.Config new file mode 100644 index 000000000..7e093b8fb --- /dev/null +++ b/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/smoke-tests/local.NuGet.Config @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/smoke-tests/online.NuGet.Config b/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/smoke-tests/online.NuGet.Config new file mode 100644 index 000000000..59db1a3f1 --- /dev/null +++ b/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/smoke-tests/online.NuGet.Config @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/smoke-tests/smoke-test.sh b/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/smoke-tests/smoke-test.sh index ccd15fd27..1762122d8 100755 --- a/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/smoke-tests/smoke-test.sh +++ b/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/smoke-tests/smoke-test.sh @@ -55,7 +55,7 @@ excludeLocalTests=false excludeOnlineTests=false excludeOmniSharpTests=${excludeOmniSharpTests:-false} devCertsVersion="$DEV_CERTS_VERSION_DEFAULT" -testingDir="$SCRIPT_ROOT/testing-smoke" +testingDir="$SCRIPT_ROOT/testing-smoke-$(date +"%m%d%H%M%S")" cliDir="$testingDir/builtCli" logsDir="$testingDir/logs" logFile="$logsDir/smoke-test.log" @@ -63,8 +63,8 @@ omnisharpLogFile="$logsDir/omnisharp.log" restoredPackagesDir="$testingDir/packages" testingHome="$testingDir/home" archiveRestoredPackages=false -archivedPackagesDir="$testingDir/smoke-test-packages" -smokeTestPrebuilts="$SCRIPT_ROOT/packages/smoke-test-packages" +smokeTestPrebuilts="$SCRIPT_ROOT/prereq-packages" +nonSbSmokeTestPrebuilts="$SCRIPT_ROOT/non-source-built-prereq-packages" runningOnline=false runningHttps=false @@ -85,10 +85,7 @@ function usage() { echo " --excludeOnlineTests exclude test that use online sources for nuget packages" echo " --excludeOmniSharpTests don't run the OmniSharp tests" echo " --devCertsVersion use dotnet-dev-certs instead of default $DEV_CERTS_VERSION_DEFAULT" - echo " --prodConBlobFeedUrl override the prodcon blob feed specified in ProdConFeed.txt, removing it if empty" - echo " --archiveRestoredPackages capture all restored packages to $archivedPackagesDir" - echo "environment:" - echo " prodConBlobFeedUrl override the prodcon blob feed specified in ProdConFeed.txt, removing it if empty" + echo " --archiveRestoredPackages capture all restored packages to $smokeTestPrebuilts" echo "" } @@ -149,10 +146,6 @@ while :; do shift devCertsVersion="$1" ;; - --prodconblobfeedurl) - shift - prodConBlobFeedUrl="$1" - ;; --archiverestoredpackages) archiveRestoredPackages=true ;; @@ -166,8 +159,6 @@ while :; do shift done -prodConBlobFeedUrl="${prodConBlobFeedUrl-}" - function doCommand() { lang=$1 proj=$2 @@ -323,16 +314,19 @@ function runAllTests() { doCommand C# console new restore build run multi-rid-publish doCommand C# classlib new restore build multi-rid-publish doCommand C# xunit new restore test + doCommand C# nunit new restore test doCommand C# mstest new restore test doCommand VB console new restore build run multi-rid-publish doCommand VB classlib new restore build multi-rid-publish doCommand VB xunit new restore test + doCommand VB nunit new restore test doCommand VB mstest new restore test doCommand F# console new restore build run multi-rid-publish doCommand F# classlib new restore build multi-rid-publish doCommand F# xunit new restore test + doCommand F# nunit new restore test doCommand F# mstest new restore test fi @@ -377,8 +371,7 @@ function runOmniSharpTests() { tar xf "../omnisharp-linux-x64.tar.gz" popd - # 'blazorwasm' requires prereqs (non-source-built packages) - re-enable with https://github.com/dotnet/source-build/issues/2550 - for project in blazorserver classlib console mstest mvc nunit web webapp webapi worker xunit ; do + for project in blazorwasm blazorserver classlib console mstest mvc nunit web webapp webapi worker xunit ; do mkdir hello-$project pushd hello-$project @@ -428,18 +421,10 @@ function resetCaches() { fi } -function setupProdConFeed() { - if [ "$prodConBlobFeedUrl" ]; then - sed -i.bakProdCon "s|PRODUCT_CONTRUCTION_PACKAGES|$prodConBlobFeedUrl|g" "$testingDir/NuGet.Config" - else - sed -i.bakProdCon "/PRODUCT_CONTRUCTION_PACKAGES/d" "$testingDir/NuGet.Config" - fi -} - function setupSmokeTestFeed() { # Setup smoke-test-packages if they exist - if [ -e "$smokeTestPrebuilts" ]; then - sed -i.bakSmokeTestFeed "s|SMOKE_TEST_PACKAGE_FEED|$smokeTestPrebuilts|g" "$testingDir/NuGet.Config" + if [ -e "$nonSbSmokeTestPrebuilts" ]; then + sed -i.bakSmokeTestFeed "s|SMOKE_TEST_PACKAGE_FEED|$nonSbSmokeTestPrebuilts|g" "$testingDir/NuGet.Config" else sed -i.bakSmokeTestFeed "/SMOKE_TEST_PACKAGE_FEED/d" "$testingDir/NuGet.Config" fi @@ -447,8 +432,21 @@ function setupSmokeTestFeed() { function copyRestoredPackages() { if [ "$archiveRestoredPackages" == "true" ]; then - mkdir -p "$archivedPackagesDir" - cp -rf "$restoredPackagesDir"/* "$archivedPackagesDir" + rm -rf "$smokeTestPrebuilts" + rm -rf "$nonSbSmokeTestPrebuilts" + mkdir -p "$smokeTestPrebuilts" + mkdir -p "$nonSbSmokeTestPrebuilts" + find "$restoredPackagesDir" -iname "*.nupkg" -exec mv {} "$smokeTestPrebuilts" \; + + smokeTestPackages=$(find "$smokeTestPrebuilts" -iname "*.nupkg" -type f -printf "%f\n" | tr '[A-Z]' '[a-z]' | sort) + sourceBuiltPackages=$(find "$SOURCE_BUILT_PKGS_PATH" -iname "*.nupkg" -type f -printf "%f\n" | tr '[A-Z]' '[a-z]' | sort) + + echo "Removing smoke-test prereq packages that are source built:" + comm -23 <(printf "$smokeTestPackages") <(printf "$sourceBuiltPackages") | while read line + do + echo "$line" + cp "$smokeTestPrebuilts/$line" "$nonSbSmokeTestPrebuilts" + done fi } @@ -494,34 +492,13 @@ SOURCE_BUILT_PKGS_PATH="${ARTIFACTS_DIR}obj/$buildArch/$configuration/blob-feed/ export DOTNET_ROOT="$dotnetDir" export PATH="$dotnetDir:$PATH" -# Run all tests, local restore sources first, online restore sources second -if [ "$excludeLocalTests" == "false" ]; then - resetCaches - runningOnline=false - # Setup NuGet.Config with local restore source - if [ -e "$SCRIPT_ROOT/smoke-testNuGet.Config" ]; then - cp "$SCRIPT_ROOT/smoke-testNuGet.Config" "$testingDir/NuGet.Config" - sed -i.bak "s|SOURCE_BUILT_PACKAGES|$SOURCE_BUILT_PKGS_PATH|g" "$testingDir/NuGet.Config" - setupProdConFeed - setupSmokeTestFeed - echo "$testingDir/NuGet.Config Contents:" - cat "$testingDir/NuGet.Config" - fi - echo "RUN ALL TESTS - LOCAL RESTORE SOURCE" - runAllTests - copyRestoredPackages - echo "LOCAL RESTORE SOURCE - ALL TESTS PASSED!" -fi - +# Run all tests, online restore sources first, local restore sources second if [ "$excludeOnlineTests" == "false" ]; then resetCaches runningOnline=true # Setup NuGet.Config to use online restore sources - if [ -e "$SCRIPT_ROOT/smoke-testNuGet.Config" ]; then - cp "$SCRIPT_ROOT/smoke-testNuGet.Config" "$testingDir/NuGet.Config" - sed -i.bak "/SOURCE_BUILT_PACKAGES/d" "$testingDir/NuGet.Config" - setupProdConFeed - setupSmokeTestFeed + if [ -e "$SCRIPT_ROOT/online.NuGet.Config" ]; then + cp "$SCRIPT_ROOT/online.NuGet.Config" "$testingDir/NuGet.Config" echo "$testingDir/NuGet.Config Contents:" cat "$testingDir/NuGet.Config" fi @@ -531,6 +508,22 @@ if [ "$excludeOnlineTests" == "false" ]; then echo "ONLINE RESTORE SOURCE - ALL TESTS PASSED!" fi +if [ "$excludeLocalTests" == "false" ]; then + resetCaches + runningOnline=false + # Setup NuGet.Config with local restore source + if [ -e "$SCRIPT_ROOT/local.NuGet.Config" ]; then + cp "$SCRIPT_ROOT/local.NuGet.Config" "$testingDir/NuGet.Config" + sed -i.bak "s|SOURCE_BUILT_PACKAGES|$SOURCE_BUILT_PKGS_PATH|g" "$testingDir/NuGet.Config" + setupSmokeTestFeed + echo "$testingDir/NuGet.Config Contents:" + cat "$testingDir/NuGet.Config" + fi + echo "RUN ALL TESTS - LOCAL RESTORE SOURCE" + runAllTests + echo "LOCAL RESTORE SOURCE - ALL TESTS PASSED!" +fi + if [ "$excludeOmniSharpTests" == "false" ]; then runOmniSharpTests fi diff --git a/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/smoke-tests/smoke-testNuGet.Config b/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/smoke-tests/smoke-testNuGet.Config deleted file mode 100644 index 79dbf806a..000000000 --- a/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/smoke-tests/smoke-testNuGet.Config +++ /dev/null @@ -1,21 +0,0 @@ - - - - - - - - - - - - - - - - - - - - -