Enable building "non-official" VMR builds (#19491)

eng/pipelines/templates/jobs/vmr-build.yml

@@ -36,6 +36,11 @@ parameters:
   type: boolean
   default: false
 
+- name: useDevVersions
+  displayName: True when build output uses dev/CI versioning instead of official build versioning
+  type: boolean
+  default: false
+
 #### SOURCE-ONLY parameters ####
 
 # Instead of building the VMR directly, exports the sources into a tarball and builds from that
@@ -226,8 +231,13 @@ jobs:
 
   - ${{ if eq(parameters.targetOS, 'windows') }}:
     - script: |
-        call $(sourcesPath)\build.cmd -ci -cleanWhileBuilding -prepareMachine /p:TargetOS=${{ parameters.targetOS }} /p:TargetArchitecture=${{ parameters.targetArchitecture }} ${{ parameters.extraProperties }}
+        call $(sourcesPath)\build.cmd -ci -cleanWhileBuilding -prepareMachine %devArgument% /p:TargetOS=${{ parameters.targetOS }} /p:TargetArchitecture=${{ parameters.targetArchitecture }} ${{ parameters.extraProperties }}
       displayName: Build
+      env:
+        ${{ if eq(parameters.useDevVersions, 'True') }}:
+          devArgument: -dev
+        ${{ else }}:
+          devArgument: ''
 
     - ${{ if eq(parameters.runTests, 'True') }}:
       - script: |
@@ -295,6 +305,10 @@ jobs:
           customBuildArgs="$customBuildArgs --use-mono-runtime"
         fi
 
+        if [[ '${{ parameters.useDevVersions }}' == 'True' ]]; then
+          customBuildArgs="$customBuildArgs --dev"
+        fi
+
         if [[ -n "${{ parameters.crossRootFs }}" ]]; then
           customEnvVars="$customEnvVars ROOTFS_DIR=${{ parameters.crossRootFs}}"
           if [[ '${{ parameters.targetArchitecture }}' != 'wasm' ]]; then

eng/pipelines/templates/stages/vmr-build.yml

@@ -301,6 +301,18 @@ stages:
         vmrBranch: ${{ parameters.vmrBranch }}
     jobs:
 
+    - template: ../jobs/vmr-build.yml
+      parameters:
+        buildName: Ubuntu2204_DevVersions
+        isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }}
+        vmrBranch: ${{ variables.VmrBranch }}
+        architecture: x64
+        pool: ${{ parameters.pool_Linux }}
+        container: ${{ variables.ubuntu2204Container }}
+        targetOS: linux
+        targetArchitecture: x64
+        useDevVersions: true # Use dev versions for CI validation of the experience. If we decide to ship assets from this leg, then we should remove this option.
+
     - template: ../jobs/vmr-build.yml
       parameters:
         buildName: Ubuntu2204

src/SourceBuild/content/build.sh

@@ -32,8 +32,12 @@ usage()
   echo "  --with-sdk <DIR>                Use the SDK in the specified directory for bootstrapping"
   echo ""
 
+  echo "Non-source-only settings:"
+  echo "  --build-repo-tests              Build repository tests"
+  echo "  --dev                           Use -dev or -ci versioning instead of .NET official build versions"
+
+
   echo "Advanced settings:"
-  echo "  --build-repo-tests              Build repository tests. May not be supported with --source-only"
   echo "  --ci                            Set when running on CI server"
   echo "  --clean-while-building          Cleans each repo after building (reduces disk space usage, short: -cwb)"
   echo "  --excludeCIBinarylog            Don't output binary log (short: -nobl)"
@@ -82,6 +86,7 @@ packagesPreviouslySourceBuiltDir="${packagesDir}previously-source-built/"
 ci=false
 exclude_ci_binary_log=false
 prepare_machine=false
+use_dev_versioning=false
 
 properties=''
 while [[ $# > 0 ]]; do
@@ -175,6 +180,9 @@ while [[ $# > 0 ]]; do
     -use-mono-runtime)
       properties="$properties /p:SourceBuildUseMonoRuntime=true"
       ;;
+    -dev)
+      use_dev_versioning=true
+      ;;
     *)
       properties="$properties $1"
       ;;
@@ -189,6 +197,10 @@ if [[ "$ci" == true ]]; then
   fi
 fi
 
+if [[ "$use_dev_versioning" == true && "$sourceOnly" != true ]]; then
+  properties="$properties /p:UseOfficialBuildVersioning=false"
+fi
+
 # Never use the global nuget cache folder
 use_global_nuget_cache=false
 

src/SourceBuild/content/eng/build.ps1

@@ -16,6 +16,7 @@ Param(
   [switch][Alias('cwb')]$cleanWhileBuilding,
   [switch][Alias('nobl')]$excludeCIBinarylog,
   [switch] $prepareMachine,
+  [switch] $dev,
   [Parameter(ValueFromRemainingArguments=$true)][String[]]$properties
 )
 
@@ -38,6 +39,7 @@ function Get-Usage() {
   Write-Host "  -cleanWhileBuilding     Cleans each repo after building (reduces disk space usage, short: -cwb)"
   Write-Host "  -excludeCIBinarylog     Don't output binary log (short: -nobl)"
   Write-Host "  -prepareMachine         Prepare machine for CI run, clean up processes after build"
+  Write-Host "  -dev                    Use -dev or -ci versioning instead of .NET official build versions"
   Write-Host ""
 }
 
@@ -68,6 +70,10 @@ if ($cleanWhileBuilding) {
   $arguments += "/p:CleanWhileBuilding=true"
 }
 
+if ($dev) {
+  $arguments += "/p:UseOfficialBuildVersioning=false"
+}
+
 function Build {
   InitializeToolset
 

src/SourceBuild/content/repo-projects/Directory.Build.props

@@ -68,7 +68,8 @@
     <BuildActions>$(BuildActions) $(FlagParameterPrefix)pack</BuildActions>
     <BuildActions>$(BuildActions) $(FlagParameterPrefix)publish</BuildActions>
 
-    <BuildArgs>$(FlagParameterPrefix)ci</BuildArgs>
+    <!-- TODO: Remove the DotNetBuildSourceOnly condition when the default build behavior changes to dev -->
+    <BuildArgs Condition="'$(ContinuousIntegrationBuild)' == 'true' or '$(DotNetBuildSourceOnly)' == 'true'">$(FlagParameterPrefix)ci</BuildArgs>
     <BuildArgs>$(BuildArgs) $(FlagParameterPrefix)configuration $(Configuration)</BuildArgs>
     <BuildArgs>$(BuildArgs) -bl</BuildArgs>
     <BuildArgs>$(BuildArgs) /p:DotNetBuildRepo=true</BuildArgs>
@@ -115,27 +116,30 @@
     <!-- TODO: Remove when all repos use a consistent set of eng/common files: https://github.com/dotnet/source-build/issues/3710. -->
     <EnvironmentVariables Include="_OverrideArcadeInitializeBuildToolFramework=$(NetCurrent)" />
 
-    <EnvironmentVariables Include="DotNetUseShippingVersions=true" />
-    <EnvironmentVariables Include="PreReleaseVersionLabel=$(PreReleaseVersionLabel)" />
-    <EnvironmentVariables Include="PackageVersionStamp=$(PreReleaseVersionLabel)" />
-
     <!-- We pass '-ci', but also apply ci mode via env var for edge cases. (E.g. misbehaving inner builds.) -->
-    <EnvironmentVariables Include="ContinuousIntegrationBuild=true" />
+    <!-- TODO: Remove the DotNetBuildSourceOnly condition when the default build behavior changes to dev -->
+    <EnvironmentVariables Condition="'$(ContinuousIntegrationBuild)' == 'true' or '$(DotNetBuildSourceOnly)' == 'true'" Include="ContinuousIntegrationBuild=true" />
 
     <!-- Turn off node reuse for source build because repos use conflicting versions
          of compilers which cause assembly load errors.
          See https://github.com/dotnet/source-build/issues/541 -->
     <EnvironmentVariables Include="MSBUILDDISABLENODEREUSE=1" />
 
+  </ItemGroup>
+
   <!--
     Apply official build versioning to match Microsoft build. These are based on build date, so
     need to be parsed from Maestro++ auto-update and passed through.
   -->
+  <ItemGroup Condition="'$(UseOfficialBuildVersioning)' != 'false'">
     <EnvironmentVariables Include="OfficialBuildId=$(OfficialBuildId)" />
     <EnvironmentVariables Include="BUILD_BUILDNUMBER=$(OfficialBuildId)" />
 
+    <EnvironmentVariables Include="DotNetUseShippingVersions=true" />
+    <EnvironmentVariables Include="PreReleaseVersionLabel=$(PreReleaseVersionLabel)" />
+    <EnvironmentVariables Include="PackageVersionStamp=$(PreReleaseVersionLabel)" />
+
     <!-- Give build access to commit info without necessarily requiring git queries. -->
-    <EnvironmentVariables Include="GitCommitCount=$(GitCommitCount)" />
     <EnvironmentVariables Include="GitCommitHash=$(GitCommitHash)" Condition="'$(GitCommitHash)' != ''" />
     <EnvironmentVariables Include="GitInfoCommitHash=$(GitCommitHash)" Condition="'$(GitCommitHash)' != ''" />
     <EnvironmentVariables Include="SourceRevisionId=$(GitCommitHash)" Condition="'$(GitCommitHash)' != ''" />
@@ -143,7 +147,9 @@
     <EnvironmentVariables Include="COMMIT_SHA=$(GitCommitHash)" Condition="'$(GitCommitHash)' != ''" />
     <EnvironmentVariables Include="GIT_COMMIT=$(GitCommitHash)" Condition="'$(GitCommitHash)' != ''" />
     <EnvironmentVariables Include="RepositoryType=Git" />
+  </ItemGroup>
 
+  <ItemGroup>
     <EnvironmentVariables Include="DeterministicSourcePaths=true" Condition="'$(DeterministicBuildOptOut)' != 'true'" />
     <EnvironmentVariables Include="DeterministicSourcePaths=false" Condition="'$(DeterministicBuildOptOut)' == 'true'" />
 

src/SourceBuild/content/repo-projects/deployment-tools.proj

@@ -9,9 +9,11 @@
     <RepositoryReference Include="source-build-reference-packages" />
   </ItemGroup>
 
-  <ItemGroup>
+  <ItemGroup Condition="'$(UseOfficialBuildVersioning)' != 'false'">
     <EnvironmentVariables Include="LatestCommit=$(GitCommitHash)" />
+  </ItemGroup>
 
+  <ItemGroup>
     <!-- https://github.com/dotnet/source-build/issues/4115. -->
     <EnvironmentVariables Include="PublishWindowsPdb=false" />
   </ItemGroup>

src/SourceBuild/content/repo-projects/roslyn.proj

@@ -12,7 +12,7 @@
     <BuildScript>$(ProjectDirectory)build$(ShellExtension)</BuildScript>
 
     <!-- roslyn by default builds with desktop msbuild (xcopy-msbuild) -->
-    <BuildArgs Condition="'$(BuildOS)' == 'windows'">$(BuildArgs) $(FlagParameterPrefix)officialBuildId $(OfficialBuildId)</BuildArgs>
+    <BuildArgs Condition="'$(BuildOS)' == 'windows' and '$(UseOfficialBuildVersioning)' != 'false'">$(BuildArgs) $(FlagParameterPrefix)officialBuildId $(OfficialBuildId)</BuildArgs>
     <BuildArgs Condition="'$(BuildOS)' == 'windows'">$(BuildArgs) $(FlagParameterPrefix)officialSkipTests true</BuildArgs>
     <BuildArgs Condition="'$(BuildOS)' == 'windows'">$(BuildArgs) $(FlagParameterPrefix)officialSkipApplyOptimizationData true</BuildArgs>
     <BuildArgs Condition="'$(BuildOS)' == 'windows'">$(BuildArgs) $(FlagParameterPrefix)officialSourceBranchName placeholder</BuildArgs>

src/SourceBuild/patches/aspnetcore/0001-Condition-RIDs-on-which-target-RIDs-are-available-wh.patch

@@ -0,0 +1,49 @@
+From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
+From: Jeremy Koritzinsky <jekoritz@microsoft.com>
+Date: Tue, 16 Apr 2024 15:56:07 -0700
+Subject: [PATCH] Condition RIDs on which target RIDs are available when doing
+ a VMR build
+
+Backport: https://github.com/dotnet/aspnetcore/pull/55172
+---
+ src/Servers/IIS/IIS/test/testassets/TestTasks/TestTasks.csproj   | 1 +
+ src/Servers/IIS/build/testsite.props                             | 1 +
+ .../ServerComparison.TestSites/ServerComparison.TestSites.csproj | 1 +
+ 3 files changed, 3 insertions(+)
+
+diff --git a/src/Servers/IIS/IIS/test/testassets/TestTasks/TestTasks.csproj b/src/Servers/IIS/IIS/test/testassets/TestTasks/TestTasks.csproj
+index f6157b9730..0e4cfec2ec 100644
+--- a/src/Servers/IIS/IIS/test/testassets/TestTasks/TestTasks.csproj
++++ b/src/Servers/IIS/IIS/test/testassets/TestTasks/TestTasks.csproj
+@@ -4,6 +4,7 @@
+     <OutputType>Exe</OutputType>
+     <TargetFramework>$(DefaultNetCoreTargetFramework)</TargetFramework>
+     <RuntimeIdentifiers>win-x64;linux-x64;osx-x64</RuntimeIdentifiers>
++    <RuntimeIdentifiers Condition="'$(DotNetBuild)' == 'true'">$(TargetRuntimeIdentifier)</RuntimeIdentifiers>
+ 
+     <!--
+       This is used as a package by ASP.NET benchmarking infrastructure. It is meant for internal use only. See also
+diff --git a/src/Servers/IIS/build/testsite.props b/src/Servers/IIS/build/testsite.props
+index 8226200d1a..429d233d8e 100644
+--- a/src/Servers/IIS/build/testsite.props
++++ b/src/Servers/IIS/build/testsite.props
+@@ -1,6 +1,7 @@
+ <Project>
+   <PropertyGroup>
+     <RuntimeIdentifiers>$(RuntimeIdentifiers);win-x64;win-x86</RuntimeIdentifiers>
++    <RuntimeIdentifiers Condition="'$(DotNetBuild)' == 'true'">$(TargetRuntimeIdentifier)</RuntimeIdentifiers>
+     <Platforms>x64;x86</Platforms>
+     <IISExpressAppHostConfig>$(MSBuildThisFileDirectory)applicationhost.config</IISExpressAppHostConfig>
+     <IISAppHostConfig>$(MSBuildThisFileDirectory)applicationhost.iis.config</IISAppHostConfig>
+diff --git a/src/Servers/testassets/ServerComparison.TestSites/ServerComparison.TestSites.csproj b/src/Servers/testassets/ServerComparison.TestSites/ServerComparison.TestSites.csproj
+index a5a54f6c9d..2b3690af74 100644
+--- a/src/Servers/testassets/ServerComparison.TestSites/ServerComparison.TestSites.csproj
++++ b/src/Servers/testassets/ServerComparison.TestSites/ServerComparison.TestSites.csproj
+@@ -5,6 +5,7 @@
+   <PropertyGroup>
+     <TargetFramework>$(DefaultNetCoreTargetFramework)</TargetFramework>
+     <RuntimeIdentifiers>win-x86;win-x64;linux-x64;osx-x64</RuntimeIdentifiers>
++    <RuntimeIdentifiers Condition="'$(DotNetBuild)' == 'true'">$(TargetRuntimeIdentifier)</RuntimeIdentifiers>
+     <InProcessTestSite>true</InProcessTestSite>
+   </PropertyGroup>
+ 

src/redist/redist.csproj

@@ -14,8 +14,12 @@
   <ItemGroup>
     <FrameworkReference Update="Microsoft.NETCore.App" TargetingPackVersion="$(MicrosoftNETCoreAppRefPackageVersion)" RuntimeFrameworkVersion="$(MicrosoftNETCoreAppRuntimePackageVersion)" />
 
-    <ProjectReference Include="projects\SdkResolver.csproj" ReferenceOutputAssembly="false" Condition="'$(DotNetBuildSourceOnly)' != 'true'" />
+    <!--
-    <ProjectReference Include="projects\VSTemplateLocator.csproj" ReferenceOutputAssembly="false" Condition="'$(DotNetBuildSourceOnly)' != 'true'" />
+      These two projects reference assets from multiple architectures, so they can't be built until we have a join point job that can pull assets from multiple legs.
+      https://github.com/dotnet/source-build/issues/4336
+    -->
+    <ProjectReference Include="projects\SdkResolver.csproj" ReferenceOutputAssembly="false" Condition="'$(DotNetBuild)' != 'true'" />
+    <ProjectReference Include="projects\VSTemplateLocator.csproj" ReferenceOutputAssembly="false" Condition="'$(DotNetBuild)' != 'true'" />
   </ItemGroup>
 
   <ItemGroup>

src/redist/targets/GenerateMSIs.targets

@@ -62,28 +62,14 @@
       <LayoutDirectory Condition=" '$(OSName)' != 'ubuntu' AND '$(OSName)' != 'debian' ">$(IntermediateOutputPath)layouts</LayoutDirectory>
       <SdkLayoutOutputDirectory>$(LayoutDirectory)$(ArtifactNameSdk)</SdkLayoutOutputDirectory>
       <MSBuildExtensionsOutputDirectory>$(LayoutDirectory)MSBuildExtensions</MSBuildExtensionsOutputDirectory>-->
-    </PropertyGroup>
-
-    <Error Condition=" '$(OfficialBuild)' == 'true' AND '$(_PatchNumber)' == '' "
-           Text="_PatchNumber should not be empty in an official build. Check if there were changes in Arcade." />
-
-    <Exec Command="git rev-list --count HEAD"
-          ConsoleToMSBuild="true"
-          Condition=" '$(GitCommitCount)' == '' AND '$(_PatchNumber)' == '' ">
-      <Output TaskParameter="ConsoleOutput" PropertyName="GitCommitCount" />
-    </Exec>
-
-    <PropertyGroup>
-      <GitCommitCount>$(GitCommitCount.PadLeft(6,'0'))</GitCommitCount>
 
       <!-- This number comes from arcade and combines the date based build number id and the revision (incremental number per day) -->
       <CombinedBuildNumberAndRevision>$(_PatchNumber)</CombinedBuildNumberAndRevision>
-      <!-- Fallback to commit count when patch number is not set. This happens only during CI. -->
+      <!-- Fallback to 0 when patch number is not set. This happens only during CI. -->
-      <CombinedBuildNumberAndRevision Condition=" '$(CombinedBuildNumberAndRevision)' == '' ">$(GitCommitCount)</CombinedBuildNumberAndRevision>
+      <CombinedBuildNumberAndRevision Condition=" '$(CombinedBuildNumberAndRevision)' == '' ">000000</CombinedBuildNumberAndRevision>
       
       <!-- This number comes from arcade and combines the date based build number id and the revision (incremental number per day) -->    
       <SDKBundleVersion>$(FileVersion)</SDKBundleVersion>
-      <!-- Fallback to commit count when patch number is not set. This happens only during CI. -->
       <SDKBundleVersion Condition=" '$(SDKBundleVersion)' == '' ">$(VersionPrefix).$(CombinedBuildNumberAndRevision)</SDKBundleVersion>
     </PropertyGroup>
   </Target>
@@ -472,9 +458,13 @@
                       '$(Architecture)'" />
   </Target>
 
+  <!--
+    This project references assets from multiple architectures, so it can't be built until we have a join point job that can pull assets from multiple legs.
+    https://github.com/dotnet/source-build/issues/4336
+  -->
   <Target Name="GenerateVSToolsResolverNupkg"
           DependsOnTargets="GenerateLayout;MsiTargetsSetupInputOutputs"
-          Condition=" '$(OS)' == 'Windows_NT' and '$(Architecture)' == 'x86' "
+          Condition=" '$(OS)' == 'Windows_NT' and '$(Architecture)' == 'x86' And '$(DotNetBuild)' != 'true'"
           Inputs="$(SdkResolverLayoutPath)/**/*;
                   $(VSToolsResolverNuspecFile);
                   $(GenerateNupkgPowershellScript)"
@@ -508,9 +498,13 @@
                       '$(SdkMSBuildExtensionsNupkgFile)'" />
   </Target>
 
+  <!--
+    This project references assets from multiple architectures, so it can't be built until we have a join point job that can pull assets from multiple legs.
+    https://github.com/dotnet/source-build/issues/4336
+  -->
   <Target Name="GenerateVSTemplateLocatorNupkg"
           DependsOnTargets="GenerateLayout;MsiTargetsSetupInputOutputs"
-          Condition=" '$(OS)' == 'Windows_NT' And '$(Architecture)' == 'x64' "
+          Condition=" '$(OS)' == 'Windows_NT' And '$(Architecture)' == 'x64' And '$(DotNetBuild)' != 'true'"
           Inputs="$(MSBuildExtensionsLayoutDirectory)/**/*;
                     $(VSTemplateLocatorNuspecFile);
                     $(GenerateNupkgPowershellScript)"