Merge branch 'release/6.0.4xx' into internal-merge-6.0.4xx-2022-10-11-1011

.vsts-ci.yml

@@ -375,10 +375,11 @@ stages:
 
   - template: /src/SourceBuild/Arcade/eng/common/templates/job/source-build-create-tarball.yml
 
-  - template: /src/SourceBuild/Arcade/eng/common/templates/job/source-build-run-tarball-build.yml
+  # For .NET 6.0, source-build only supports 6.0.1xx.  Disable tarball build validation in all other branches.
-    parameters:
+  # - ${{ if in(variables['Build.Reason'], 'PullRequest') }}:
-      dependsOn: Source_Build_Create_Tarball
+  #   - template: /src/SourceBuild/Arcade/eng/common/templates/job/source-build-run-tarball-build.yml
-      condition: eq(dependencies.Source_Build_Create_Tarball.outputs['Tarball_Build_Check._includeTarballBuild'], 'true')
+  #     parameters:
+  #       dependsOn: Source_Build_Create_Tarball
 
   # https://github.com/dotnet/core-sdk/issues/248
   # - template: /eng/build.yml

NuGet.config

@@ -7,10 +7,8 @@
     <clear />
     <!--Begin: Package sources managed by Dependency Flow automation. Do not edit the sources below.-->
     <!--  Begin: Package sources from dotnet-aspnetcore -->
-    <add key="darc-int-dotnet-aspnetcore-32e8c8c" value="https://pkgs.dev.azure.com/dnceng/internal/_packaging/darc-int-dotnet-aspnetcore-32e8c8ca/nuget/v3/index.json" />
     <!--  End: Package sources from dotnet-aspnetcore -->
     <!--  Begin: Package sources from dotnet-emsdk -->
-    <add key="darc-pub-dotnet-emsdk-3f6c45a" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/darc-pub-dotnet-emsdk-3f6c45a2/nuget/v3/index.json" />
     <!--  End: Package sources from dotnet-emsdk -->
     <!--  Begin: Package sources from DotNet-msbuild-Trusted -->
     <add key="darc-int-DotNet-msbuild-Trusted-5618488" value="https://pkgs.dev.azure.com/dnceng/internal/_packaging/darc-int-DotNet-msbuild-Trusted-56184888/nuget/v3/index.json" />
@@ -54,7 +52,6 @@
     <add key="darc-int-DotNet-msbuild-Trusted-5618488" value="true" />
     <!--  End: Package sources from DotNet-msbuild-Trusted -->
     <!--  Begin: Package sources from dotnet-aspnetcore -->
-    <add key="darc-int-dotnet-aspnetcore-32e8c8c" value="true" />
     <!--  End: Package sources from dotnet-aspnetcore -->
     <!--  Begin: Package sources from dotnet-runtime -->
     <add key="darc-int-dotnet-runtime-5a400c2" value="true" />

eng/Version.Details.xml

@@ -197,19 +197,19 @@
     </Dependency>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="6.0.0-beta.22463.7">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="6.0.0-beta.22512.3">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>2fbe602d12256bf82de53791ddcbb964c1541b4e</Sha>
+      <Sha>bb1e72113a7eaf8bebda940beba8cf8bee1b453f</Sha>
       <SourceBuild RepoName="arcade" ManagedOnly="true" />
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.CMake.Sdk" Version="6.0.0-beta.22463.7">
+    <Dependency Name="Microsoft.DotNet.CMake.Sdk" Version="6.0.0-beta.22512.3">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>2fbe602d12256bf82de53791ddcbb964c1541b4e</Sha>
+      <Sha>bb1e72113a7eaf8bebda940beba8cf8bee1b453f</Sha>
       <SourceBuild RepoName="arcade" ManagedOnly="true" />
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Build.Tasks.Installers" Version="6.0.0-beta.22463.7">
+    <Dependency Name="Microsoft.DotNet.Build.Tasks.Installers" Version="6.0.0-beta.22512.3">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>2fbe602d12256bf82de53791ddcbb964c1541b4e</Sha>
+      <Sha>bb1e72113a7eaf8bebda940beba8cf8bee1b453f</Sha>
     </Dependency>
     <Dependency Name="Microsoft.SourceBuild.Intermediate.source-build-reference-packages" Version="6.0.0-servicing.22419.2">
       <Uri>https://github.com/dotnet/source-build-reference-packages</Uri>

eng/Versions.props

@@ -19,7 +19,7 @@
   </PropertyGroup>
   <PropertyGroup>
     <!-- Dependency from https://github.com/dotnet/arcade -->
-    <MicrosoftDotNetBuildTasksInstallersPackageVersion>6.0.0-beta.22463.7</MicrosoftDotNetBuildTasksInstallersPackageVersion>
+    <MicrosoftDotNetBuildTasksInstallersPackageVersion>6.0.0-beta.22512.3</MicrosoftDotNetBuildTasksInstallersPackageVersion>
   </PropertyGroup>
   <PropertyGroup>
     <!-- Dependency from https://github.com/dotnet/winforms -->
@@ -131,7 +131,7 @@
     <MicrosoftDotNetCommonItemTemplates31PackageVersion>3.1.15</MicrosoftDotNetCommonItemTemplates31PackageVersion>
     <MicrosoftDotNetCommonProjectTemplates31PackageVersion>$(MicrosoftDotNetCommonItemTemplates31PackageVersion)</MicrosoftDotNetCommonProjectTemplates31PackageVersion>
     <MicrosoftDotNetTestProjectTemplates31PackageVersion>$(MicrosoftDotNetTestProjectTemplates50PackageVersion)</MicrosoftDotNetTestProjectTemplates31PackageVersion>
-    <AspNetCorePackageVersionFor31Templates>3.1.29</AspNetCorePackageVersionFor31Templates>
+    <AspNetCorePackageVersionFor31Templates>3.1.30</AspNetCorePackageVersionFor31Templates>
     <MicrosoftAspNetCoreComponentsWebAssemblyTemplatesPackageVersion>3.2.1</MicrosoftAspNetCoreComponentsWebAssemblyTemplatesPackageVersion>
     <!-- 3.0 Template versions -->
     <MicrosoftDotnetWinFormsProjectTemplates30PackageVersion>4.8.0-rc2.19462.10</MicrosoftDotnetWinFormsProjectTemplates30PackageVersion>

eng/common/build.ps1

@@ -26,6 +26,7 @@ Param(
   [string] $runtimeSourceFeed = '',
   [string] $runtimeSourceFeedKey = '',
   [switch] $excludePrereleaseVS,
+  [switch] $nativeToolsOnMachine,
   [switch] $help,
   [Parameter(ValueFromRemainingArguments=$true)][String[]]$properties
 )
@@ -67,6 +68,7 @@ function Print-Usage() {
   Write-Host "  -warnAsError <value>    Sets warnaserror msbuild parameter ('true' or 'false')"
   Write-Host "  -msbuildEngine <value>  Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)."
   Write-Host "  -excludePrereleaseVS    Set to exclude build engines in prerelease versions of Visual Studio"
+  Write-Host "  -nativeToolsOnMachine   Sets the native tools on machine environment variable (indicating that the script should use native tools on machine)"
   Write-Host ""
 
   Write-Host "Command line arguments not listed above are passed thru to msbuild."
@@ -146,6 +148,9 @@ try {
     $nodeReuse = $false
   }
 
+  if ($nativeToolsOnMachine) {
+    $env:NativeToolsOnMachine = $true
+  }
   if ($restore) {
     InitializeNativeTools
   }

eng/common/init-tools-native.ps1

@@ -112,6 +112,7 @@ try {
             $ToolPath = Convert-Path -Path $BinPath
             Write-Host "Adding $ToolName to the path ($ToolPath)..."
             Write-Host "##vso[task.prependpath]$ToolPath"
+            $env:PATH = "$ToolPath;$env:PATH"
             $InstalledTools += @{ $ToolName = $ToolDirectory.FullName }
           }
         }

global.json

@@ -1,6 +1,6 @@
 {
   "tools": {
-    "dotnet": "6.0.109",
+    "dotnet": "6.0.110",
     "runtimes": {
       "dotnet": [
         "$(VSRedistCommonNetCoreSharedFrameworkx6460PackageVersion)"
@@ -11,7 +11,7 @@
     "cmake": "3.16.4"
   },
   "msbuild-sdks": {
-    "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.22463.7",
+    "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.22512.3",
-    "Microsoft.DotNet.CMake.Sdk": "6.0.0-beta.22463.7"
+    "Microsoft.DotNet.CMake.Sdk": "6.0.0-beta.22512.3"
   }
 }

src/SourceBuild/Arcade/eng/common/templates/job/source-build-create-tarball.yml

@@ -74,18 +74,3 @@ jobs:
     displayName: Publish BuildLogs
     continueOnError: true
     condition: succeededOrFailed()
-
-  - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}:
-    # Check if the tarball should be built now so the result can be used in the build tarball job condition.
-    # This prevents allocation of additional agents if the tarball build legs should be skipped.
-    # Only build the tarball if the PR touches source-build source.
-    - script: |
-        # Temporarily disable tarball PR validation until source-build is supported in this channel.
-        echo "##vso[task.setvariable variable=_includeTarballBuild;isoutput=true]false"
-
-        # if curl "https://api.github.com/repos/dotnet/installer/pulls/$(System.PullRequest.PullRequestNumber)/files" | grep '"filename": "src/SourceBuild/*'
-        # then
-        #   echo "##vso[task.setvariable variable=_includeTarballBuild;isoutput=true]true"
-        # fi
-      displayName: Tarball Build Check
-      name: Tarball_Build_Check

src/SourceBuild/Arcade/eng/common/templates/job/source-build-run-tarball-build.yml

@@ -1,9 +1,6 @@
 # Builds a source-build tarball
 
 parameters:
-  # Custom condition to apply to the job
-  condition: true
-
   # Dependent jobs that must be completed before this job will run
   dependsOn:
 
@@ -32,7 +29,6 @@ jobs:
 - template: /src/SourceBuild/Arcade/eng/common/templates/job/source-build-build-tarball.yml
   parameters:
     architecture: x64
-    condition: ${{ parameters.condition }}
     dependsOn: ${{ parameters.dependsOn }}
     ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}:
       excludeSdkContentTests: true
@@ -86,7 +82,6 @@ jobs:
   - template: /src/SourceBuild/Arcade/eng/common/templates/job/source-build-build-tarball.yml
     parameters:
       architecture: arm64
-      condition: ${{ parameters.condition }}
       dependsOn: ${{ parameters.dependsOn }}
       ${{ if in(variables['Build.Reason'], 'PullRequest') }}:
         excludeSdkContentTests: true
@@ -108,7 +103,7 @@ jobs:
       architecture: x64
       # Always attempt to run the bootstrap leg (e.g. even when stage 1 tests fail) in order to get a complete accessment of the build status.
       # The bootstrap build will shortcut if the stage 1 build failed.
-      condition: and(${{ parameters.condition }}, succeededOrFailed())
+      condition: succeededOrFailed()
       dependsOn: Build_Tarball_x64
       excludeSdkContentTests: true
       installerBuildResourceId: ${{ parameters.installerBuildResourceId }}

src/SourceBuild/tarball/content/build.proj

@@ -107,6 +107,7 @@
             SMOKE_TESTS_SDK_TARBALL_PATH=$(SdkTarballPath);
             SMOKE_TESTS_TARGET_RID=$(TargetRid);
             SMOKE_TESTS_PORTABLE_RID=$(PortableRid);
+            SMOKE_TESTS_CUSTOM_PACKAGES_PATH=$(CustomSourceBuiltPackagesPath);
             $(CustomTestEnvVars)" />
   </Target>
 

src/SourceBuild/tarball/content/build.sh

@@ -19,8 +19,9 @@ SCRIPT_ROOT="$(cd -P "$( dirname "$0" )" && pwd)"
 
 MSBUILD_ARGUMENTS=("/flp:v=detailed")
 CUSTOM_REF_PACKAGES_DIR=''
-CUSTOM_PREVIOUSLY_BUILT_PACKAGES_DIR=''
+CUSTOM_PACKAGES_DIR=''
 alternateTarget=false
+runningSmokeTests=false
 CUSTOM_SDK_DIR=''
 
 while :; do
@@ -41,15 +42,15 @@ while :; do
             ;;
         --run-smoke-test)
             alternateTarget=true
+            runningSmokeTests=true
             MSBUILD_ARGUMENTS+=( "/t:RunSmokeTest" )
             ;;
         --with-packages)
-            CUSTOM_PREVIOUSLY_BUILT_PACKAGES_DIR="$(cd -P "$2" && pwd)"
+            CUSTOM_PACKAGES_DIR="$(cd -P "$2" && pwd)"
-            if [ ! -d "$CUSTOM_PREVIOUSLY_BUILT_PACKAGES_DIR" ]; then
+            if [ ! -d "$CUSTOM_PACKAGES_DIR" ]; then
-                echo "Custom prviously built packages directory '$CUSTOM_PREVIOUSLY_BUILT_PACKAGES_DIR' does not exist"
+                echo "Custom prviously built packages directory '$CUSTOM_PACKAGES_DIR' does not exist"
                 exit 1
             fi
-            MSBUILD_ARGUMENTS+=( "/p:CustomPrebuiltSourceBuiltPackagesPath=$CUSTOM_PREVIOUSLY_BUILT_PACKAGES_DIR" )
             shift
             ;;
         --with-sdk)
@@ -82,13 +83,21 @@ while :; do
     shift
 done
 
+if [ "$CUSTOM_PACKAGES_DIR" != "" ]; then
+  if [ "$runningSmokeTests" == "true" ]; then
+    MSBUILD_ARGUMENTS+=( "/p:CustomSourceBuiltPackagesPath=$CUSTOM_PACKAGES_DIR" )
+  else
+    MSBUILD_ARGUMENTS+=( "/p:CustomPrebuiltSourceBuiltPackagesPath=$CUSTOM_PACKAGES_DIR" )
+  fi
+fi
+
 if [ -f "$SCRIPT_ROOT/packages/archive/archiveArtifacts.txt" ]; then
   ARCHIVE_ERROR=0
   if [ ! -d "$SCRIPT_ROOT/.dotnet" ] && [ "$CUSTOM_SDK_DIR" == "" ]; then
     echo "ERROR: SDK not found at $SCRIPT_ROOT/.dotnet"
     ARCHIVE_ERROR=1
   fi
-  if [ ! -f $SCRIPT_ROOT/packages/archive/Private.SourceBuilt.Artifacts*.tar.gz ] && [ "$CUSTOM_PREVIOUSLY_BUILT_PACKAGES_DIR" == "" ]; then
+  if [ ! -f $SCRIPT_ROOT/packages/archive/Private.SourceBuilt.Artifacts*.tar.gz ] && [ "$CUSTOM_PACKAGES_DIR" == "" ]; then
     echo "ERROR: Private.SourceBuilt.Artifacts artifact not found at $SCRIPT_ROOT/packages/archive/ - Either run prep.sh or pass --with-packages parameter"
     ARCHIVE_ERROR=1
   fi
@@ -120,7 +129,9 @@ fi
 packageVersionsPath=''
 restoredPackagesDir="$SCRIPT_ROOT/packages/restored"
 
-if [ -d "$SCRIPT_ROOT/packages/archive" ]; then
+if [[ "$CUSTOM_PACKAGES_DIR" != "" && -f "$CUSTOM_PACKAGES_DIR/PackageVersions.props" ]]; then
+  packageVersionsPath="$CUSTOM_PACKAGES_DIR/PackageVersions.props"
+elif [ -d "$SCRIPT_ROOT/packages/archive" ]; then
   sourceBuiltArchive=`find $SCRIPT_ROOT/packages/archive -maxdepth 1 -name 'Private.SourceBuilt.Artifacts*.tar.gz'`
   if [ -f "$SCRIPT_ROOT/packages/previously-source-built/PackageVersions.props" ]; then
     packageVersionsPath=$SCRIPT_ROOT/packages/previously-source-built/PackageVersions.props
@@ -128,14 +139,12 @@ if [ -d "$SCRIPT_ROOT/packages/archive" ]; then
     tar -xzf "$sourceBuiltArchive" -C /tmp PackageVersions.props
     packageVersionsPath=/tmp/PackageVersions.props
   fi
-elif [ -f "$CUSTOM_PREVIOUSLY_BUILT_PACKAGES_DIR/PackageVersions.props" ]; then
-  packageVersionsPath="$CUSTOM_PREVIOUSLY_BUILT_PACKAGES_DIR/PackageVersions.props"
 fi
 
 if [ ! -f "$packageVersionsPath" ]; then
   echo "Cannot find PackagesVersions.props.  Debugging info:"
   echo "  Attempted archive path: $SCRIPT_ROOT/packages/archive"
-  echo "  Attempted custom PVP path: $CUSTOM_PREVIOUSLY_BUILT_PACKAGES_DIR/PackageVersions.props"
+  echo "  Attempted custom PVP path: $CUSTOM_PACKAGES_DIR/PackageVersions.props"
   exit 1
 fi
 

src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Config.cs

@@ -15,6 +15,7 @@ internal static class Config
     public const string PoisonReportPathEnv = "SMOKE_TESTS_POISON_REPORT_PATH";
     public const string PortableRidEnv = "SMOKE_TESTS_PORTABLE_RID";
     public const string PrereqsPathEnv = "SMOKE_TESTS_PREREQS_PATH";
+    public const string CustomPackagesPathEnv = "SMOKE_TESTS_CUSTOM_PACKAGES_PATH";
     public const string SdkTarballPathEnv = "SMOKE_TESTS_SDK_TARBALL_PATH";
     public const string TargetRidEnv = "SMOKE_TESTS_TARGET_RID";
     public const string WarnPoisonDiffsEnv = "SMOKE_TESTS_WARN_POISON_DIFFS";
@@ -27,6 +28,7 @@ internal static class Config
     public static string PortableRid { get; } = Environment.GetEnvironmentVariable(PortableRidEnv) ??
         throw new InvalidOperationException($"'{Config.PortableRidEnv}' must be specified");
     public static string? PrereqsPath { get; } = Environment.GetEnvironmentVariable(PrereqsPathEnv);
+    public static string? CustomPackagesPath { get; } = Environment.GetEnvironmentVariable(CustomPackagesPathEnv);
     public static string? SdkTarballPath { get; } = Environment.GetEnvironmentVariable(SdkTarballPathEnv);
     public static string TargetRid { get; } = Environment.GetEnvironmentVariable(TargetRidEnv) ??
         throw new InvalidOperationException($"'{Config.TargetRidEnv}' must be specified");

src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/DotNetHelper.cs

@@ -5,6 +5,7 @@
 using System;
 using System.Diagnostics;
 using System.IO;
+using System.Linq;
 using Xunit.Abstractions;
 
 namespace Microsoft.DotNet.SourceBuild.SmokeTests;
@@ -66,6 +67,8 @@ internal class DotNetHelper
 
         if (useLocalPackages)
         {
+            // When using local packages this feed is always required.  It contains packages that are
+            // not produced by source-build but are required by the various project templates.
             if (!Directory.Exists(Config.PrereqsPath))
             {
                 throw new InvalidOperationException(
@@ -74,11 +77,26 @@ internal class DotNetHelper
 
             string nugetConfig = File.ReadAllText(nugetConfigPath);
             nugetConfig = nugetConfig.Replace("SMOKE_TEST_PACKAGE_FEED", Config.PrereqsPath);
+
+            // This package feed is optional.  You can use an additional feed of source-built packages to run the
+            // smoke-tests as offline as possible.
+            if (Config.CustomPackagesPath != null)
+            {
+                if (!Directory.Exists(Config.CustomPackagesPath))
+                {
+                    throw new ArgumentException($"Specified --with-packages {Config.CustomPackagesPath} does not exist.");
+                }
+                nugetConfig = nugetConfig.Replace("CUSTOM_PACKAGE_FEED", Config.CustomPackagesPath);
+            }
+            else
+            {
+                nugetConfig = string.Join(Environment.NewLine, nugetConfig.Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries).Where(s => !s.Contains("CUSTOM_PACKAGE_FEED")).ToArray());
+            }
             File.WriteAllText(nugetConfigPath, nugetConfig);
         }
     }
 
-    public void ExecuteCmd(string args, string? workingDirectory = null, Action<Process>? additionalProcessConfigCallback = null, int expectedExitCode = 0, int millisecondTimeout = -1)
+    public void ExecuteCmd(string args, string? workingDirectory = null, Action<Process>? additionalProcessConfigCallback = null, int? expectedExitCode = 0, int millisecondTimeout = -1)
     {
         (Process Process, string StdOut, string StdErr) executeResult = ExecuteHelper.ExecuteProcess(
             DotNetPath,
@@ -87,7 +105,9 @@ internal class DotNetHelper
             configure: (process) => configureProcess(process, workingDirectory),
             millisecondTimeout: millisecondTimeout);
         
-        ExecuteHelper.ValidateExitCode(executeResult, expectedExitCode);
+        if (expectedExitCode != null) {
+            ExecuteHelper.ValidateExitCode(executeResult, (int) expectedExitCode);
+        }
 
         void configureProcess(Process process, string? workingDirectory)
         {

src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/DotNetWatchTests.cs

@@ -19,12 +19,11 @@ public class DotNetWatchTests : SmokeTests
         string projectDirectory = DotNetHelper.ExecuteNew(DotNetTemplate.Console.GetName(), nameof(DotNetWatchTests));
         bool outputChanged = false;
 
-        // We expect an exit code of 143 (128 + 15, i.e. SIGTERM) because we are killing the process manually
         DotNetHelper.ExecuteCmd(
             "watch run",
             workingDirectory: projectDirectory,
             additionalProcessConfigCallback: processConfigCallback,
-            expectedExitCode: 143,
+            expectedExitCode: null, // The exit code does not reflect whether or not dotnet watch is working properly
             millisecondTimeout: 30000);
 
         Assert.True(outputChanged);
@@ -52,7 +51,7 @@ public class DotNetWatchTests : SmokeTests
                 {
                     outputChanged = true;
                     OutputHelper.WriteLine("Successfully re-ran program after code change.");
-                    ExecuteHelper.ExecuteProcessValidateExitCode("kill", $"-s TERM {process.Id}", OutputHelper);
+                    process.Kill(true);
                 }
             });
         }

src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/local.NuGet.Config

@@ -3,5 +3,6 @@
   <packageSources>
     <clear />
     <add key="smoke-test-prereqs" value="SMOKE_TEST_PACKAGE_FEED" />
+    <add key="custom-packages" value="CUSTOM_PACKAGE_FEED" />
   </packageSources>
 </configuration>