From d78dbaca86868da29b5bf02bdcf95359cc7cb5db Mon Sep 17 00:00:00 2001 From: Jo Shields Date: Tue, 30 Jan 2024 17:35:07 -0500 Subject: [PATCH 01/12] Support for short stack builds. Tested with android-x64 and android-amd64 on Linux x64, and iossimulator-x64 on macOS ARM64. Depends on https://github.com/dotnet/runtime/pull/97725 --- src/SourceBuild/content/Directory.Build.props | 14 +++++++++++++- src/SourceBuild/content/repo-projects/dotnet.proj | 4 ++++ src/SourceBuild/content/repo-projects/runtime.proj | 4 ++-- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/src/SourceBuild/content/Directory.Build.props b/src/SourceBuild/content/Directory.Build.props index 2f80065a8..c63965543 100644 --- a/src/SourceBuild/content/Directory.Build.props +++ b/src/SourceBuild/content/Directory.Build.props @@ -6,6 +6,18 @@ true + + <_OverrideTargetOS>$(OverrideTargetRid.Substring(0, $(OverrideTargetRid.LastIndexOf('-')))) + true + true + true + true + true + true + true + true + + linux osx @@ -33,7 +45,7 @@ s390x ppc64le - $(OverrideTargetRid.Substring($(OverrideTargetRid.LastIndexOf('-'))).TrimStart('-')) + $(OverrideTargetRid.Substring($(OverrideTargetRid.LastIndexOf('-'))).TrimStart('-')) x64 $(TargetArchitecture) diff --git a/src/SourceBuild/content/repo-projects/dotnet.proj b/src/SourceBuild/content/repo-projects/dotnet.proj index 0de74d679..5de28398c 100644 --- a/src/SourceBuild/content/repo-projects/dotnet.proj +++ b/src/SourceBuild/content/repo-projects/dotnet.proj @@ -24,6 +24,10 @@ + + + + diff --git a/src/SourceBuild/content/repo-projects/runtime.proj b/src/SourceBuild/content/repo-projects/runtime.proj index 50754cee4..9409f1729 100644 --- a/src/SourceBuild/content/repo-projects/runtime.proj +++ b/src/SourceBuild/content/repo-projects/runtime.proj @@ -20,8 +20,8 @@ $(BuildArgs) $(FlagParameterPrefix)arch $(OverrideTargetArch) $(BuildArgs) $(FlagParameterPrefix)os $(OverrideTargetOS) $(BuildArgs) /p:TargetRid=$(OverrideTargetRid) - $(BuildArgs) /p:RuntimeOS=$(RuntimeOS) - $(BuildArgs) /p:BaseOS=$(BaseOS) + $(BuildArgs) /p:RuntimeOS=$(RuntimeOS) + $(BuildArgs) /p:BaseOS=$(BaseOS) From 69f1c5855affa88821a7092bbaf3add2a8e73e77 Mon Sep 17 00:00:00 2001 From: Jo Shields Date: Fri, 2 Feb 2024 13:35:24 -0500 Subject: [PATCH 02/12] Refreshed for the glourious TargetOS/TargetArchitecture future --- src/SourceBuild/content/Directory.Build.props | 24 +++++++++---------- .../content/repo-projects/runtime.proj | 6 ++--- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/SourceBuild/content/Directory.Build.props b/src/SourceBuild/content/Directory.Build.props index 7e508644e..144e4a2bf 100644 --- a/src/SourceBuild/content/Directory.Build.props +++ b/src/SourceBuild/content/Directory.Build.props @@ -6,18 +6,6 @@ true - - <_OverrideTargetOS>$(OverrideTargetRid.Substring(0, $(OverrideTargetRid.LastIndexOf('-')))) - true - true - true - true - true - true - true - true - - linux osx @@ -31,6 +19,18 @@ $(BuildOS) + + + true + true + true + true + true + true + true + true + + $([System.Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture.ToString().ToLowerInvariant) $(BuildArchitecture) diff --git a/src/SourceBuild/content/repo-projects/runtime.proj b/src/SourceBuild/content/repo-projects/runtime.proj index 8f2c8d9c8..2f9e3c503 100644 --- a/src/SourceBuild/content/repo-projects/runtime.proj +++ b/src/SourceBuild/content/repo-projects/runtime.proj @@ -14,9 +14,9 @@ $(BuildArgs) $(FlagParameterPrefix)arch $(TargetArchitecture) $(BuildArgs) $(FlagParameterPrefix)os $(TargetOS) - $(BuildArgs) /p:TargetRid=$(TargetRid) - $(BuildArgs) /p:BaseOS=$(BaseOS) + $(BuildArgs) /p:TargetRid=$(TargetRid) + $(BuildArgs) /p:RuntimeOS=$(RuntimeOS) + $(BuildArgs) /p:BaseOS=$(BaseOS) From e06a805d2c6a5f962881eb986ebb7f8f6d8e3b74 Mon Sep 17 00:00:00 2001 From: Jo Shields Date: Fri, 2 Feb 2024 15:38:16 -0500 Subject: [PATCH 03/12] Fix build locally, push to test on Mac --- src/SourceBuild/content/Directory.Build.props | 1 + src/SourceBuild/content/repo-projects/runtime.proj | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/SourceBuild/content/Directory.Build.props b/src/SourceBuild/content/Directory.Build.props index 144e4a2bf..1fc83c61f 100644 --- a/src/SourceBuild/content/Directory.Build.props +++ b/src/SourceBuild/content/Directory.Build.props @@ -61,6 +61,7 @@ win-$(TargetArchitecture) $(PortableRid) + $(TargetOS)-$(TargetArchitecture) diff --git a/src/SourceBuild/content/repo-projects/runtime.proj b/src/SourceBuild/content/repo-projects/runtime.proj index 2f9e3c503..a588ecb5b 100644 --- a/src/SourceBuild/content/repo-projects/runtime.proj +++ b/src/SourceBuild/content/repo-projects/runtime.proj @@ -14,7 +14,7 @@ $(BuildArgs) $(FlagParameterPrefix)arch $(TargetArchitecture) $(BuildArgs) $(FlagParameterPrefix)os $(TargetOS) - $(BuildArgs) /p:TargetRid=$(TargetRid) + $(BuildArgs) /p:TargetRid=$(TargetRid) $(BuildArgs) /p:RuntimeOS=$(RuntimeOS) $(BuildArgs) /p:BaseOS=$(BaseOS) From 50289771fd65f2f367e5b0238451c4ba65cad8f1 Mon Sep 17 00:00:00 2001 From: Jo Shields Date: Fri, 2 Feb 2024 16:33:05 -0500 Subject: [PATCH 04/12] change rootrepo --- src/SourceBuild/content/Directory.Build.props | 2 ++ src/SourceBuild/content/repo-projects/dotnet.proj | 4 ---- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/SourceBuild/content/Directory.Build.props b/src/SourceBuild/content/Directory.Build.props index 1fc83c61f..3b9b645bd 100644 --- a/src/SourceBuild/content/Directory.Build.props +++ b/src/SourceBuild/content/Directory.Build.props @@ -29,6 +29,8 @@ true true true + + runtime diff --git a/src/SourceBuild/content/repo-projects/dotnet.proj b/src/SourceBuild/content/repo-projects/dotnet.proj index 5de28398c..0de74d679 100644 --- a/src/SourceBuild/content/repo-projects/dotnet.proj +++ b/src/SourceBuild/content/repo-projects/dotnet.proj @@ -24,10 +24,6 @@ - - - - From efb2c9e1cb5f84eb8bf2ca852d5bcaf131883408 Mon Sep 17 00:00:00 2001 From: Jo Shields Date: Wed, 7 Feb 2024 10:35:49 -0500 Subject: [PATCH 05/12] Temporarily enable full build so we can test CI --- eng/pipelines/vmr-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/vmr-build.yml b/eng/pipelines/vmr-build.yml index 15cc40fc4..9ae5ca6c3 100644 --- a/eng/pipelines/vmr-build.yml +++ b/eng/pipelines/vmr-build.yml @@ -58,7 +58,7 @@ stages: ${{ if eq(variables.isSourceOnlyBuild, 'true') }}: scope: ultralite ${{ else }}: - scope: lite + scope: full # In case the VMR Build stage is temporarily disabled, the VMR synchronization step is run to validate # that the PR can be merged and later synchronized into the VMR without problems. From 33400c93941010c36f3f1f91f0ceebad8797add2 Mon Sep 17 00:00:00 2001 From: Jo Shields Date: Wed, 7 Feb 2024 10:38:35 -0500 Subject: [PATCH 06/12] Add Android x64 to build, let's see what happens --- eng/pipelines/templates/stages/vmr-build.yml | 13 +++++++++++++ eng/pipelines/templates/variables/vmr-stage.yml | 2 ++ 2 files changed, 15 insertions(+) diff --git a/eng/pipelines/templates/stages/vmr-build.yml b/eng/pipelines/templates/stages/vmr-build.yml index 6b8bc1fca..88e848f5f 100644 --- a/eng/pipelines/templates/stages/vmr-build.yml +++ b/eng/pipelines/templates/stages/vmr-build.yml @@ -360,6 +360,19 @@ stages: targetOS: osx targetArchitecture: x64 + - template: ../jobs/vmr-build.yml + parameters: + buildName: Android_Shortstack + isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} + vmrBranch: ${{ variables.VmrBranch }} + architecture: x64 + pool: + name: ${{ variables.defaultPoolName }} + demands: ${{ variables.defaultPoolDemandsLinux }} + container: ${{ variables.androidCrossContainer }} + targetOS: android + targetArchitecture: x64 + # Cross builds are currently failing # - template: ../jobs/vmr-build.yml diff --git a/eng/pipelines/templates/variables/vmr-stage.yml b/eng/pipelines/templates/variables/vmr-stage.yml index 7363922bc..9566547c8 100644 --- a/eng/pipelines/templates/variables/vmr-stage.yml +++ b/eng/pipelines/templates/variables/vmr-stage.yml @@ -26,6 +26,8 @@ variables: value: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-amd64 - name: marinerArm64CrossContainer value: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm64 +- name: androidCrossContainer + value: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-android-amd64 - ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: defaultPoolName From 1400ef02770622226700af1688a9bd15cb1d5eb0 Mon Sep 17 00:00:00 2001 From: Jo Shields Date: Wed, 7 Feb 2024 11:02:58 -0500 Subject: [PATCH 07/12] Add patch since runtime sync is out of date & missing needed patch --- .../runtime/0002-short-stack-support.patch | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 src/SourceBuild/patches/runtime/0002-short-stack-support.patch diff --git a/src/SourceBuild/patches/runtime/0002-short-stack-support.patch b/src/SourceBuild/patches/runtime/0002-short-stack-support.patch new file mode 100644 index 000000000..5dc1b46e9 --- /dev/null +++ b/src/SourceBuild/patches/runtime/0002-short-stack-support.patch @@ -0,0 +1,41 @@ +--- ../dotnet/src/runtime/eng/SourceBuild.props 2024-02-07 11:01:33.807337902 -0500 ++++ eng/SourceBuild.props 2024-02-05 16:48:58.219933758 -0500 +@@ -15,6 +15,7 @@ + + <_targetRidPlatformIndex>$(TargetRid.LastIndexOf('-')) + $(TargetRid.Substring($(_targetRidPlatformIndex)).TrimStart('-')) ++ $(TargetRid.Substring(0, $(_targetRidPlatformIndex))) + + <_hostRidPlatformIndex>$(_hostRid.LastIndexOf('-')) + <_hostArch>$(_hostRid.Substring($(_hostRidPlatformIndex)).TrimStart('-')) +@@ -22,6 +23,17 @@ + minimal + + ++ ++ true ++ true ++ true ++ true ++ true ++ true ++ true ++ true ++ ++ + + +@@ -29,9 +41,10 @@ + This allows to build the repository using './build.sh /p:DotNetBuildFromSource=true'. + Properties that control flags from source-build, and the expected output for source-build should be added to this file. --> + $(InnerBuildArgs) $(FlagParameterPrefix)arch $(TargetArch) +- $(InnerBuildArgs) $(FlagParameterPrefix)cross ++ $(InnerBuildArgs) $(FlagParameterPrefix)os $(TargetOS) ++ $(InnerBuildArgs) $(FlagParameterPrefix)cross + $(InnerBuildArgs) $(FlagParameterPrefix)configuration $(Configuration) +- $(InnerBuildArgs) $(FlagParameterPrefix)allconfigurations ++ $(InnerBuildArgs) $(FlagParameterPrefix)allconfigurations + $(InnerBuildArgs) $(FlagParameterPrefix)verbosity $(LogVerbosity) + $(InnerBuildArgs) $(FlagParameterPrefix)nodereuse $(ArcadeFalseBoolBuildArg) + $(InnerBuildArgs) $(FlagParameterPrefix)warnAsError $(ArcadeFalseBoolBuildArg) From 280a9e0e41e57860aa9f6d28dfaebeae15b07326 Mon Sep 17 00:00:00 2001 From: Jo Shields Date: Wed, 7 Feb 2024 11:09:53 -0500 Subject: [PATCH 08/12] fix paths in patch --- .../patches/runtime/0002-short-stack-support.patch | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/SourceBuild/patches/runtime/0002-short-stack-support.patch b/src/SourceBuild/patches/runtime/0002-short-stack-support.patch index 5dc1b46e9..57bc73f01 100644 --- a/src/SourceBuild/patches/runtime/0002-short-stack-support.patch +++ b/src/SourceBuild/patches/runtime/0002-short-stack-support.patch @@ -1,5 +1,5 @@ ---- ../dotnet/src/runtime/eng/SourceBuild.props 2024-02-07 11:01:33.807337902 -0500 -+++ eng/SourceBuild.props 2024-02-05 16:48:58.219933758 -0500 +--- a/eng/SourceBuild.props 2024-02-07 11:01:33.807337902 -0500 ++++ b/eng/SourceBuild.props 2024-02-05 16:48:58.219933758 -0500 @@ -15,6 +15,7 @@ <_targetRidPlatformIndex>$(TargetRid.LastIndexOf('-')) From d83ebb22129874921e015a349d8e979c8e95bcca Mon Sep 17 00:00:00 2001 From: Jo Shields Date: Wed, 7 Feb 2024 11:38:23 -0500 Subject: [PATCH 09/12] Revert "Temporarily enable full build so we can test CI" This reverts commit efb2c9e1cb5f84eb8bf2ca852d5bcaf131883408. --- eng/pipelines/vmr-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/vmr-build.yml b/eng/pipelines/vmr-build.yml index 9ae5ca6c3..15cc40fc4 100644 --- a/eng/pipelines/vmr-build.yml +++ b/eng/pipelines/vmr-build.yml @@ -58,7 +58,7 @@ stages: ${{ if eq(variables.isSourceOnlyBuild, 'true') }}: scope: ultralite ${{ else }}: - scope: full + scope: lite # In case the VMR Build stage is temporarily disabled, the VMR synchronization step is run to validate # that the PR can be merged and later synchronized into the VMR without problems. From 3df779e4f6649610140827bb3ed7d104310e4d3e Mon Sep 17 00:00:00 2001 From: Jo Shields Date: Wed, 7 Feb 2024 13:07:38 -0500 Subject: [PATCH 10/12] If we want to do a short stack Windows build, tweak the RID --- src/SourceBuild/content/Directory.Build.props | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/SourceBuild/content/Directory.Build.props b/src/SourceBuild/content/Directory.Build.props index 3b9b645bd..7f57c6388 100644 --- a/src/SourceBuild/content/Directory.Build.props +++ b/src/SourceBuild/content/Directory.Build.props @@ -63,7 +63,8 @@ win-$(TargetArchitecture) $(PortableRid) - $(TargetOS)-$(TargetArchitecture) + $(TargetOS)-$(TargetArchitecture) + win-$(TargetArchitecture) From b426ce60f44b048f6d69e4fe08fd09fefd13ae6d Mon Sep 17 00:00:00 2001 From: Jo Shields Date: Wed, 7 Feb 2024 13:45:35 -0500 Subject: [PATCH 11/12] Annotate patch --- .../patches/runtime/0002-short-stack-support.patch | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/SourceBuild/patches/runtime/0002-short-stack-support.patch b/src/SourceBuild/patches/runtime/0002-short-stack-support.patch index 57bc73f01..783d86b3c 100644 --- a/src/SourceBuild/patches/runtime/0002-short-stack-support.patch +++ b/src/SourceBuild/patches/runtime/0002-short-stack-support.patch @@ -1,3 +1,10 @@ +From 6e36330872998c791a2c0d31b688e1bdece2451f Mon Sep 17 00:00:00 2001 +From: Jo Shields +Date: Fri, 2 Feb 2024 06:56:20 -0500 +Subject: [PATCH] Source built short stack support (#97725) + +Backport: https://github.com/dotnet/runtime/pull/97725 + --- a/eng/SourceBuild.props 2024-02-07 11:01:33.807337902 -0500 +++ b/eng/SourceBuild.props 2024-02-05 16:48:58.219933758 -0500 @@ -15,6 +15,7 @@ From 64e1ffe1baac9592f27a16636f01064ddc60a2bd Mon Sep 17 00:00:00 2001 From: Jo Shields Date: Wed, 7 Feb 2024 14:47:33 -0500 Subject: [PATCH 12/12] That error message was thoroughly unhelpful --- eng/pipelines/templates/stages/vmr-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/stages/vmr-build.yml b/eng/pipelines/templates/stages/vmr-build.yml index f315bcbea..03844d0e6 100644 --- a/eng/pipelines/templates/stages/vmr-build.yml +++ b/eng/pipelines/templates/stages/vmr-build.yml @@ -350,7 +350,7 @@ stages: - template: ../jobs/vmr-build.yml parameters: - buildName: Android_Shortstack + buildName: Android_Shortstack isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }} vmrBranch: ${{ variables.VmrBranch }} architecture: x64