From c5d2414c0faf01b0de9bf53589fe143d10bdf620 Mon Sep 17 00:00:00 2001 From: Michael Simons Date: Fri, 1 Oct 2021 14:36:06 -0500 Subject: [PATCH 1/6] Update source-build prebuilts (#12225) --- eng/Versions.props | 2 +- src/SourceBuild/tarball/content/eng/Versions.props | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/Versions.props b/eng/Versions.props index bb83161a0..aad09a72b 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -161,7 +161,7 @@ 15.7.179 15.7.179 0.1.0-6.0.100-bootstrap.19 - 0.1.0-6.0.100-23 + 0.1.0-6.0.100-24 diff --git a/src/SourceBuild/tarball/content/eng/Versions.props b/src/SourceBuild/tarball/content/eng/Versions.props index b27697389..d99821dbd 100644 --- a/src/SourceBuild/tarball/content/eng/Versions.props +++ b/src/SourceBuild/tarball/content/eng/Versions.props @@ -23,6 +23,6 @@ 0.1.0-6.0.100-bootstrap.19 0.1.0-6.0.100- - 23 + 24 From 6bcf16be37ba2e7cbed857f84c9d62b8b5407a2d Mon Sep 17 00:00:00 2001 From: Michael Simons Date: Fri, 1 Oct 2021 16:57:11 -0500 Subject: [PATCH 2/6] [ArPow] Add runtime patch to produce symbol archives (#12220) --- .../0002-Produce-symbols-archives.patch | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 src/SourceBuild/tarball/patches/runtime/0002-Produce-symbols-archives.patch diff --git a/src/SourceBuild/tarball/patches/runtime/0002-Produce-symbols-archives.patch b/src/SourceBuild/tarball/patches/runtime/0002-Produce-symbols-archives.patch new file mode 100644 index 000000000..4eb878225 --- /dev/null +++ b/src/SourceBuild/tarball/patches/runtime/0002-Produce-symbols-archives.patch @@ -0,0 +1,42 @@ +From 7577b8cf676f7d38363052e2bc8da931af1cc279 Mon Sep 17 00:00:00 2001 +From: Jeremy Koritzinsky +Date: Thu, 30 Sep 2021 16:04:21 -0700 +Subject: [PATCH] Produce symbols archives. + +Contributes to #49026 + +https://github.com/dotnet/runtime/pull/59843 handles adding the patch to the runtime repo. +--- + .../Microsoft.NETCore.App.Runtime.Composite.sfxproj | 2 +- + .../Microsoft.NETCore.App/Microsoft.NETCore.App.Runtime.sfxproj | 2 +- + 2 files changed, 2 insertions(+), 2 deletions(-) + +diff --git a/src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Runtime.Composite.sfxproj b/src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Runtime.Composite.sfxproj +index a3f364815d6..5fe3ba42252 100644 +--- a/src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Runtime.Composite.sfxproj ++++ b/src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Runtime.Composite.sfxproj +@@ -7,7 +7,7 @@ + dotnet-runtime-internal-composite + $(SharedFrameworkName).Composite + $(SharedFrameworkName).Composite.PGO +- true ++ true + dotnet-runtime-composite-symbols + true + true +diff --git a/src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Runtime.sfxproj b/src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Runtime.sfxproj +index 4fa48f4e0c9..1d43304d72b 100644 +--- a/src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Runtime.sfxproj ++++ b/src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Runtime.sfxproj +@@ -8,7 +8,7 @@ + dotnet-runtime + dotnet-runtime-internal + $(SharedFrameworkName).PGO +- true ++ true + dotnet-runtime-symbols + NetCore.SharedFramework + true +-- +2.29.2 + From b05fc9c997f410e04887413c26c8a2e4bdde5e32 Mon Sep 17 00:00:00 2001 From: Michael Simons Date: Fri, 1 Oct 2021 18:13:54 -0500 Subject: [PATCH 3/6] [ArPow] Add runtime patch to enable doc generation (#12221) --- eng/Versions.props | 2 +- .../tarball/content/eng/Versions.props | 2 +- src/SourceBuild/tarball/content/smoke-test.sh | 132 ++---------------- ...nable-doc-generation-in-source-build.patch | 25 ++++ 4 files changed, 40 insertions(+), 121 deletions(-) create mode 100644 src/SourceBuild/tarball/patches/runtime/0001-Enable-doc-generation-in-source-build.patch diff --git a/eng/Versions.props b/eng/Versions.props index aad09a72b..dabf241ce 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -161,7 +161,7 @@ 15.7.179 15.7.179 0.1.0-6.0.100-bootstrap.19 - 0.1.0-6.0.100-24 + 0.1.0-6.0.100-25 diff --git a/src/SourceBuild/tarball/content/eng/Versions.props b/src/SourceBuild/tarball/content/eng/Versions.props index d99821dbd..e5a4dd8e0 100644 --- a/src/SourceBuild/tarball/content/eng/Versions.props +++ b/src/SourceBuild/tarball/content/eng/Versions.props @@ -23,6 +23,6 @@ 0.1.0-6.0.100-bootstrap.19 0.1.0-6.0.100- - 24 + 25 diff --git a/src/SourceBuild/tarball/content/smoke-test.sh b/src/SourceBuild/tarball/content/smoke-test.sh index ea2299dde..0617f3f6e 100755 --- a/src/SourceBuild/tarball/content/smoke-test.sh +++ b/src/SourceBuild/tarball/content/smoke-test.sh @@ -466,8 +466,8 @@ function runXmlDocTests() { ) netcoreappIgnoreList=( + mscorlib.xml Microsoft.VisualBasic.xml - netstandard.xml System.AppContext.xml System.Buffers.xml System.ComponentModel.DataAnnotations.xml @@ -482,12 +482,15 @@ function runXmlDocTests() { System.Globalization.Calendars.xml System.Globalization.Extensions.xml System.Globalization.xml + System.IO.Compression.Brotli.xml System.IO.Compression.FileSystem.xml + System.IO.FileSystem.xml System.IO.FileSystem.Primitives.xml System.IO.UnmanagedMemoryStream.xml System.IO.xml System.Net.xml System.Numerics.xml + System.Numerics.Vectors.xml System.Reflection.Extensions.xml System.Reflection.xml System.Resources.Reader.xml @@ -501,6 +504,8 @@ function runXmlDocTests() { System.ServiceModel.Web.xml System.ServiceProcess.xml System.Text.Encoding.xml + System.Text.Json.SourceGeneration.resources.xml + System.Text.Json.SourceGeneration.xml System.Threading.Tasks.Extensions.xml System.Threading.Tasks.xml System.Threading.Timer.xml @@ -513,124 +518,8 @@ function runXmlDocTests() { System.Xml.Serialization.xml System.Xml.xml System.Xml.XmlDocument.xml - Microsoft.CSharp.xml - # Added temporarily due to https://github.com/dotnet/source-build/issues/2404 - Microsoft.VisualBasic.Core.xml - Microsoft.Win32.Primitives.xml - Microsoft.Win32.Registry.xml - System.Collections.Concurrent.xml - System.Collections.Immutable.xml - System.Collections.NonGeneric.xml - System.Collections.Specialized.xml - System.Collections.xml - System.ComponentModel.Annotations.xml - System.ComponentModel.EventBasedAsync.xml - System.ComponentModel.Primitives.xml - System.ComponentModel.TypeConverter.xml - System.ComponentModel.xml - System.Console.xml - System.Data.Common.xml - System.Diagnostics.Contracts.xml - System.Diagnostics.DiagnosticSource.xml - System.Diagnostics.FileVersionInfo.xml - System.Diagnostics.Process.xml - System.Diagnostics.StackTrace.xml - System.Diagnostics.TextWriterTraceListener.xml - System.Diagnostics.TraceSource.xml - System.Diagnostics.Tracing.xml - System.Drawing.Primitives.xml - System.Formats.Asn1.xml - System.IO.Compression.Brotli.xml - System.IO.Compression.ZipFile.xml - System.IO.Compression.xml - System.IO.FileSystem.AccessControl.xml - System.IO.FileSystem.DriveInfo.xml - System.IO.FileSystem.Watcher.xml - System.IO.FileSystem.xml - System.IO.IsolatedStorage.xml - System.IO.MemoryMappedFiles.xml - System.IO.Pipes.AccessControl.xml - System.IO.Pipes.xml - System.Linq.Expressions.xml - System.Linq.Parallel.xml - System.Linq.Queryable.xml - System.Linq.xml - System.Memory.xml - System.Net.Http.Json.xml - System.Net.Http.xml - System.Net.HttpListener.xml - System.Net.Mail.xml - System.Net.NameResolution.xml - System.Net.NetworkInformation.xml - System.Net.Ping.xml - System.Net.Primitives.xml - System.Net.Requests.xml - System.Net.Security.xml - System.Net.ServicePoint.xml - System.Net.Sockets.xml - System.Net.WebClient.xml - System.Net.WebHeaderCollection.xml - System.Net.WebProxy.xml - System.Net.WebSockets.Client.xml - System.Net.WebSockets.xml - System.Numerics.Vectors.xml - System.ObjectModel.xml - System.Reflection.DispatchProxy.xml - System.Reflection.Emit.ILGeneration.xml - System.Reflection.Emit.Lightweight.xml - System.Reflection.Emit.xml - System.Reflection.Metadata.xml - System.Reflection.Primitives.xml - System.Reflection.TypeExtensions.xml - System.Resources.Writer.xml - System.Runtime.CompilerServices.Unsafe.xml - System.Runtime.CompilerServices.VisualC.xml - System.Runtime.InteropServices.RuntimeInformation.xml - System.Runtime.InteropServices.xml - System.Runtime.Intrinsics.xml - System.Runtime.Loader.xml - System.Runtime.Numerics.xml - System.Runtime.Serialization.Formatters.xml - System.Runtime.Serialization.Json.xml - System.Runtime.Serialization.Primitives.xml - System.Runtime.Serialization.Xml.xml - System.Runtime.xml - System.Security.AccessControl.xml - System.Security.Claims.xml - System.Security.Cryptography.Algorithms.xml - System.Security.Cryptography.Cng.xml - System.Security.Cryptography.Csp.xml - System.Security.Cryptography.Encoding.xml - System.Security.Cryptography.OpenSsl.xml - System.Security.Cryptography.Primitives.xml - System.Security.Cryptography.X509Certificates.xml - System.Security.Principal.Windows.xml - System.Text.Encoding.CodePages.xml - System.Text.Encoding.Extensions.xml - System.Text.Encodings.Web.xml - System.Text.Json.xml - System.Text.Json.SourceGeneration.xml - System.Text.Json.SourceGeneration.resources.xml - System.Text.RegularExpressions.xml - System.Threading.Channels.xml - System.Threading.Overlapped.xml - System.Threading.Tasks.Dataflow.xml - System.Threading.Tasks.Parallel.xml - System.Threading.Thread.xml - System.Threading.ThreadPool.xml - System.Threading.xml - System.Transactions.Local.xml - System.Web.HttpUtility.xml - System.Xml.ReaderWriter.xml - System.Xml.XDocument.xml - System.Xml.XPath.XDocument.xml - System.Xml.XPath.xml - System.Xml.XmlSerializer.xml - WindowsBase.xml - mscorlib.xml ) - # Added temporarily due to https://github.com/dotnet/source-build/issues/2404 aspnetcoreappIgnoreList=( Microsoft.AspNetCore.App.Analyzers.xml Microsoft.AspNetCore.App.CodeFixes.xml @@ -660,7 +549,6 @@ function runXmlDocTests() { fi done fi - # Added temporarily due to https://github.com/dotnet/source-build/issues/2404 if [[ "$xmlDocFile" == *"/packs/Microsoft.AspNetCore.App.Ref"* ]]; then xmlFileBasename=$(basename "$xmlDocFile") for ignoreItem in "${aspnetcoreappIgnoreList[@]}"; do @@ -674,10 +562,16 @@ function runXmlDocTests() { error=1 echo "error: missing $xmlDocFile" fi + + if [[ $skip == 1 ]] && [[ -f "$xmlDocFile" ]]; then + error=1 + echo "error: Ignored xml doc was found: $xmlDocFile" + fi + done < <(find "$targetingPacksDir" -name '*.dll' -print0) if [[ $error != 0 ]]; then - echo "error: Missing xml documents" + echo "error: Missing or unexpected xml documents" exit 1 else echo "All expected xml docs are present" diff --git a/src/SourceBuild/tarball/patches/runtime/0001-Enable-doc-generation-in-source-build.patch b/src/SourceBuild/tarball/patches/runtime/0001-Enable-doc-generation-in-source-build.patch new file mode 100644 index 000000000..47c00310d --- /dev/null +++ b/src/SourceBuild/tarball/patches/runtime/0001-Enable-doc-generation-in-source-build.patch @@ -0,0 +1,25 @@ +From 4ca12d74308dceface792dad959bd4a7eda0b1b4 Mon Sep 17 00:00:00 2001 +From: MichaelSimons +Date: Thu, 30 Sep 2021 22:37:24 +0000 +Subject: [PATCH] Enable doc generation in source-build + +--- + Build.proj | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/Build.proj b/Build.proj +index bff94b97a7a..5f47e869fb6 100644 +--- a/Build.proj ++++ b/Build.proj +@@ -10,7 +10,7 @@ + + + +- ++ + + + Date: Fri, 1 Oct 2021 20:37:05 -0500 Subject: [PATCH 4/6] [ArPow] Add option to bootstrap previously source-built packages (#12222) * Add option to bootstrap previously source-built packages This option modifies the previously source-built archive to include portable packages that are downloaded from a microsoft official feed rather than source-built. * Remove working directory * Add newlines * Add additional comment about feeds * Update source-build prebuilts * Remove original archive rather than backup Co-authored-by: MichaelSimons --- src/SourceBuild/tarball/content/prep.sh | 38 +- .../buildBootstrapPreviouslySB.csproj | 61 +++ .../scripts/bootstrap/buildbootstrapcli.sh | 411 ------------------ 3 files changed, 98 insertions(+), 412 deletions(-) create mode 100644 src/SourceBuild/tarball/content/scripts/bootstrap/buildBootstrapPreviouslySB.csproj delete mode 100755 src/SourceBuild/tarball/content/scripts/bootstrap/buildbootstrapcli.sh diff --git a/src/SourceBuild/tarball/content/prep.sh b/src/SourceBuild/tarball/content/prep.sh index 794a108d2..9e749fc2b 100755 --- a/src/SourceBuild/tarball/content/prep.sh +++ b/src/SourceBuild/tarball/content/prep.sh @@ -5,13 +5,18 @@ IFS=$'\n\t' SCRIPT_ROOT="$(cd -P "$( dirname "$0" )" && pwd)" usage() { - echo "usage: $0" + echo "usage: $0 [options]" echo "" echo " Prepares a tarball to be built by downloading Private.SourceBuilt.Artifacts.*.tar.gz and" echo " installing the version of dotnet referenced in global.json" + echo "options:" + echo " --bootstrap Build a bootstrap version of previously source-built packages archive." + echo " This modifies the downloaded version, replacing portable packages" + echo " with official ms-built packages restored from package feeds." echo "" } +buildBootstrap=false positional_args=() while :; do if [ $# -le 0 ]; then @@ -23,6 +28,9 @@ while :; do usage exit 0 ;; + --bootstrap) + buildBootstrap=true + ;; *) positional_args+=("$1") ;; @@ -87,3 +95,31 @@ if [ "$installDotnet" == "true" ]; then echo " Installing dotnet..." (source ./eng/common/tools.sh && InitializeDotNetCli true) fi + +# Build bootstrap, if specified +if [ "$buildBootstrap" == "true" ]; then + DOTNET_SDK_PATH="$SCRIPT_ROOT/.dotnet" + + # Create working directory for running bootstrap project + workingDir=$(mktemp -d) + echo " Building bootstrap previously source-built in $workingDir" + + # Copy bootstrap project to working dir + cp $SCRIPT_ROOT/scripts/bootstrap/buildBootstrapPreviouslySB.csproj $workingDir + + # Copy NuGet.config from the installer repo to have the right feeds + cp $SCRIPT_ROOT/src/installer.*/NuGet.config $workingDir + + # Get PackageVersions.props from existing prev-sb archive + echo " Retrieving PackageVersions.props from existing archive" + sourceBuiltArchive=`find $SCRIPT_ROOT/packages/archive -maxdepth 1 -name 'Private.SourceBuilt.Artifacts*.tar.gz'` + if [ -f "$sourceBuiltArchive" ]; then + tar -xzf "$sourceBuiltArchive" -C $workingDir PackageVersions.props + fi + + # Run restore on project to initiate download of bootstrap packages + $DOTNET_SDK_PATH/dotnet restore $workingDir/buildBootstrapPreviouslySB.csproj /bl /p:ArchiveDir="$SCRIPT_ROOT/packages/archive/" + + # Remove working directory + rm -rf $workingDir +fi diff --git a/src/SourceBuild/tarball/content/scripts/bootstrap/buildBootstrapPreviouslySB.csproj b/src/SourceBuild/tarball/content/scripts/bootstrap/buildBootstrapPreviouslySB.csproj new file mode 100644 index 000000000..090db5fbf --- /dev/null +++ b/src/SourceBuild/tarball/content/scripts/bootstrap/buildBootstrapPreviouslySB.csproj @@ -0,0 +1,61 @@ + + + + + + + net6.0 + $(MSBuildProjectDirectory)/artifacts/ + $(MSBuildProjectDirectory)/artifacts/restoredPkgs/ + $(MSBuildProjectDirectory)/artifacts/unpacked/ + $(ArchiveDir)Private.SourceBuilt.Artifacts.Bootstrap.tar.gz + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/scripts/bootstrap/buildbootstrapcli.sh b/src/SourceBuild/tarball/content/scripts/bootstrap/buildbootstrapcli.sh deleted file mode 100755 index b27b5476e..000000000 --- a/src/SourceBuild/tarball/content/scripts/bootstrap/buildbootstrapcli.sh +++ /dev/null @@ -1,411 +0,0 @@ -#!/usr/bin/env bash -set -e -set -u -set -o pipefail - -usage() -{ - echo "Builds a bootstrap CLI from sources" - echo "Usage: $0 [BuildType] -rid -seedcli [-os ] [-clang ] [-corelib ]" - echo "" - echo "Options:" - echo " BuildType Type of build (-debug, -release), default: -release" - echo " -clang Override of the version of clang compiler to use" - echo " -corelib Path to System.Private.CoreLib.dll, default: use the System.Private.CoreLib.dll from the seed CLI" - echo " -os Operating system (used for corefx build), default: Linux" - echo " -rid Runtime identifier including the architecture part (e.g. rhel.6-x64)" - echo " -seedcli Seed CLI used to generate the target CLI" - echo " -outputpath Optional output directory to contain the generated cli and cloned repos, default: " -} - -disable_pax_mprotect() -{ - if [[ $(command -v paxctl) ]]; then - paxctl -c -m $1 - fi -} - -get_max_version() -{ - local maxversionhi=0 - local maxversionmid=0 - local maxversionlo=0 - local maxversiontag - local versionrest - local versionhi - local versionmid - local versionlo - local versiontag - local foundmax - - for d in $1/*; do - - if [[ -d $d ]]; then - versionrest=$(basename $d) - versionhi=${versionrest%%.*} - versionrest=${versionrest#*.} - versionmid=${versionrest%%.*} - versionrest=${versionrest#*.} - versionlo=${versionrest%%-*} - versiontag=${versionrest#*-} - if [[ $versiontag == $versionrest ]]; then - versiontag="" - fi - - foundmax=0 - - if [[ $versionhi -gt $maxversionhi ]]; then - foundmax=1 - elif [[ $versionhi -eq $maxversionhi ]]; then - if [[ $versionmid -gt $maxversionmid ]]; then - foundmax=1 - elif [[ $versionmid -eq $maxversionmid ]]; then - if [[ $versionlo -gt $maxversionlo ]]; then - foundmax=1 - elif [[ $versionlo -eq $maxversionlo ]]; then - # tags are used to mark pre-release versions, so a version without a tag - # is newer than a version with one. - if [[ "$versiontag" == "" || $versiontag > $maxversiontag ]]; then - foundmax=1 - fi - fi - fi - fi - - if [[ $foundmax != 0 ]]; then - maxversionhi=$versionhi - maxversionmid=$versionmid - maxversionlo=$versionlo - maxversiontag=$versiontag - fi - fi - done - - echo $maxversionhi.$maxversionmid.$maxversionlo${maxversiontag:+-$maxversiontag} -} - -getrealpath() -{ - if command -v realpath > /dev/null; then - realpath $1 - else - readlink -e $1 - fi -} - -__build_os=Linux -__runtime_id= -__corelib= -__configuration=release -__clangversion= -__outputpath= - -while [[ "${1:-}" != "" ]]; do - lowerI="$(echo $1 | awk '{print tolower($0)}')" - case $lowerI in - -h|--help) - usage - exit 1 - ;; - -rid) - shift - __runtime_id=$1 - ;; - -os) - shift - __build_os=$1 - ;; - -debug) - __configuration=debug - ;; - -release) - __configuration=release - ;; - -corelib) - shift - __corelib=`getrealpath $1` - ;; - -seedcli) - shift - __seedclipath=`getrealpath $1` - ;; - -clang) - shift - __clangversion=clang$1 - ;; - -outputpath) - shift - __outputpath=$1 - ;; - *) - echo "Unknown argument to build.sh $1"; exit 1 - esac - shift -done - - -if [[ -z "$__runtime_id" ]]; then - echo "Missing the required -rid argument" - exit 2 -fi - -if [[ -z "$__seedclipath" ]]; then - echo "Missing the required -seedcli argument" - exit 3 -fi - -__build_arch=${__runtime_id#*-} - -if [[ -z "$__outputpath" ]]; then - __outputpath=$__runtime_id/dotnetcli -fi - -if [[ -d "$__outputpath" ]]; then - /bin/rm -r $__outputpath -fi - -mkdir -p $__runtime_id -mkdir -p $__outputpath - -__outputpath=`getrealpath $__outputpath` - -cd $__runtime_id - -cp -r $__seedclipath/* $__outputpath - -__frameworkversion="2.0.0" -__sdkversion="2.0.0" -__fxrversion="2.0.0" - -echo "**** DETECTING VERSIONS IN SEED CLI ****" - -__frameworkversion=`get_max_version $__seedclipath/shared/Microsoft.NETCore.App` -__sdkversion=`get_max_version $__seedclipath/sdk` -__fxrversion=`get_max_version $__seedclipath/host/fxr` - -echo "Framework version: $__frameworkversion" -echo "SDK version: $__sdkversion" -echo "FXR version: $__fxrversion" - -__frameworkpath=$__outputpath/shared/Microsoft.NETCore.App/$__frameworkversion - -echo "**** DETECTING GIT COMMIT HASHES ****" - -# Extract the git commit hashes representig the state of the three repos that -# the seed cli package was built from -__coreclrhash=`strings $__seedclipath/shared/Microsoft.NETCore.App/$__frameworkversion/libcoreclr.so | grep "@(#)" | grep -o "[a-f0-9]\{40\}"` -__corefxhash=`strings $__seedclipath/shared/Microsoft.NETCore.App/$__frameworkversion/System.Native.so | grep "@(#)" | grep -o "[a-f0-9]\{40\}"` -__coresetuphash=`strings $__seedclipath/dotnet | grep "@(#)" | grep -o "[a-f0-9]\{40\}"` -if [[ "$__coresetuphash" == "" ]]; then - __coresetuphash=`strings $__seedclipath/dotnet | grep -o "[a-f0-9]\{40\}"` -fi - -echo "coreclr hash: $__coreclrhash" -echo "corefx hash: $__corefxhash" -echo "core-setup hash: $__coresetuphash" - -# Clone the three repos if they were not cloned yet. If the folders already -# exist, leave them alone. This allows patching the cloned sources as needed - -if [[ ! -d coreclr ]]; then - echo "**** CLONING CORECLR REPOSITORY ****" - git clone https://github.com/dotnet/coreclr.git - cd coreclr - git checkout $__coreclrhash - cd .. -fi - -if [[ ! -d corefx ]]; then - echo "**** CLONING COREFX REPOSITORY ****" - git clone https://github.com/dotnet/corefx.git - cd corefx - git checkout $__corefxhash - cd .. -fi - -if [[ ! -d core-setup ]]; then - echo "**** CLONING CORE-SETUP REPOSITORY ****" - git clone https://github.com/dotnet/core-setup.git - cd core-setup - git checkout $__coresetuphash - cd .. -fi - -echo "**** BUILDING CORE-SETUP NATIVE COMPONENTS ****" -cd core-setup -src/corehost/build.sh --configuration $__configuration --arch "$__build_arch" --hostver "2.0.0" --apphostver "2.0.0" --fxrver "2.0.0" --policyver "2.0.0" --commithash `git rev-parse HEAD` -cd .. - -echo "**** BUILDING CORECLR NATIVE COMPONENTS ****" -cd coreclr -./build.sh $__configuration $__build_arch $__clangversion -skipgenerateversion -skipmanaged -skipmscorlib -skiprestore -skiprestoreoptdata -skipnuget -nopgooptimize 2>&1 | tee coreclr.log -export __coreclrbin=$(cat coreclr.log | sed -n -e 's/^.*Product binaries are available at //p') -cd .. -echo "CoreCLR binaries will be copied from $__coreclrbin" - -echo "**** BUILDING COREFX NATIVE COMPONENTS ****" -corefx/src/Native/build-native.sh $__build_arch $__configuration $__clangversion $__build_os 2>&1 | tee corefx.log -export __corefxbin=$(cat corefx.log | sed -n -e 's/^.*Build files have been written to: //p') -echo "CoreFX binaries will be copied from $__corefxbin" - -echo "**** Copying new binaries to dotnetcli/ ****" - -# First copy the coreclr repo binaries -cp $__coreclrbin/*so $__frameworkpath -cp $__coreclrbin/corerun $__frameworkpath -cp $__coreclrbin/crossgen $__frameworkpath - -# Mark the coreclr executables as allowed to create executable memory mappings -disable_pax_mprotect $__frameworkpath/corerun -disable_pax_mprotect $__frameworkpath/crossgen - -# Now copy the core-setup repo binaries - -if [[ $__fxrversion == 2* ]]; then - cp core-setup/cli/exe/dotnet/dotnet $__outputpath - cp core-setup/cli/exe/dotnet/dotnet $__frameworkpath/corehost - - cp core-setup/cli/dll/libhostpolicy.so $__frameworkpath - cp core-setup/cli/dll/libhostpolicy.so $__outputpath/sdk/$__sdkversion - - cp core-setup/cli/fxr/libhostfxr.so $__frameworkpath - cp core-setup/cli/fxr/libhostfxr.so $__outputpath/host/fxr/$__fxrversion - cp core-setup/cli/fxr/libhostfxr.so $__outputpath/sdk/$__sdkversion -else - cp core-setup/bin/$__runtime_id.$__configuration/corehost/dotnet $__outputpath - cp core-setup/bin/$__runtime_id.$__configuration/corehost/dotnet $__frameworkpath/corehost - - cp core-setup/bin/$__runtime_id.$__configuration/corehost/libhostpolicy.so $__frameworkpath - cp core-setup/bin/$__runtime_id.$__configuration/corehost/libhostpolicy.so $__outputpath/sdk/$__sdkversion - - cp core-setup/bin/$__runtime_id.$__configuration/corehost/libhostfxr.so $__frameworkpath - cp core-setup/bin/$__runtime_id.$__configuration/corehost/libhostfxr.so $__outputpath/host/fxr/$__fxrversion - cp core-setup/bin/$__runtime_id.$__configuration/corehost/libhostfxr.so $__outputpath/sdk/$__sdkversion -fi - -# Mark the core-setup executables as allowed to create executable memory mappings -disable_pax_mprotect $__outputpath/dotnet -disable_pax_mprotect $__frameworkpath/corehost - -# Finally copy the corefx repo binaries -cp $__corefxbin/**/System.* $__frameworkpath - -# Copy System.Private.CoreLib.dll override from somewhere if requested -if [[ "$__corelib" != "" ]]; then - cp "$__corelib" $__frameworkpath -fi - -# Add the new RID to Microsoft.NETCore.App.deps.json -# Replace the linux-x64 RID in the target, runtimeTarget and runtimes by the new RID -# and add the new RID to the list of runtimes. -echo "**** Adding new rid to Microsoft.NETCore.App.deps.json ****" - -#TODO: add parameter with the parent RID sequence - -sed \ - -e 's/runtime\.linux-x64/runtime.'$__runtime_id'/g' \ - -e 's/runtimes\/linux-x64/runtimes\/'$__runtime_id'/g' \ - -e 's/Version=v\([0-9].[0-9]\)\/linux-x64/Version=v\1\/'$__runtime_id'/g' \ -$__seedclipath/shared/Microsoft.NETCore.App/$__frameworkversion/Microsoft.NETCore.App.deps.json \ ->$__frameworkpath/Microsoft.NETCore.App.deps.json - -# add the new RID to the list of runtimes iff it does not already exist (sed inplace) -__os_dependencies= -if [[ $__build_os == "Linux" ]]; then - __os_dependencies='"linux", "linux-'$__build_arch'", ' -fi -grep -q "\"$__runtime_id\":" $__frameworkpath/Microsoft.NETCore.App.deps.json || \ -sed -i \ - -e 's/"runtimes": {/&\n "'$__runtime_id'": [\n '"$__os_dependencies"'"unix", "unix-'$__build_arch'", "any", "base"\n ],/g' \ -$__frameworkpath/Microsoft.NETCore.App.deps.json - -__crossgentimeout=120 - -function crossgenone(){ - echo $2/crossgen /MissingDependenciesOK /Platform_Assemblies_Paths $2:$3 /in $1 /out $1.ni >$1.log 2>&1 - timeout $__crossgentimeout $2/crossgen /MissingDependenciesOK /Platform_Assemblies_Paths $2:$3 /in $1 /out $1.ni >>$1.log 2>&1 - exitCode=$? - if [ "$exitCode" == "0" ] - then - rm $1.log - mv $1.ni $1 - elif grep -q -e 'The module was expected to contain an assembly manifest' \ - -e 'An attempt was made to load a program with an incorrect format.' \ - -e 'File is PE32' $1.log - then - rm $1.log - echo "$1" >> crossgenskipped - else - echo "$1" >> crossgenretry - fi -} - -# Run an assembly through ildasm ilasm roundtrip to remove x64 crossgen -function uncrossgenone(){ - echo >> $1.log 2>&1 - echo mv $1 $1.x64 >> $1.log 2>&1 - echo $2/ildasm -raweh -out=$1.il $1.x64 "&& \\" >> $1.log 2>&1 - echo $2/ilasm -output=$1 -QUIET -NOLOGO -DEBUG -OPTIMIZE $1.il >> $1.log 2>&1 - - mv $1 $1.x64 - $2/ildasm -raweh -out=$1.il $1.x64 && \ - $2/ilasm -output=$1 -DLL -QUIET -NOLOGO -DEBUG -OPTIMIZE $1.il - exitCode=$? - if [ "$exitCode" == "0" ] - then - rm $1.x64 - rm $1.il - else - echo "$1" >> uncrossgenfails - fi -} - -# if $__build_arch is not x64 then any dll which was crossgened for x64 must be recrossgened for $__build_arch -if [[ "$__build_arch" != "x64" ]] -then - echo "**** Beginning crossgen for $__build_arch target ****" - export -f crossgenone - export __crossgentimeout - - rm -f crossgenretry crossgendlls crossgenskipped uncrossgenfails - - # Assumes System.Private.CoreLib was already crossgened - find $__outputpath -type f -name \*.dll -or -name \*.exe | grep -v System.Private.CoreLib > crossgendlls - - cat crossgendlls | xargs -P 0 -n 1 -I {} bash -c 'crossgenone "$@"' _ {} "$__frameworkpath" "$__outputpath/sdk/$__sdkversion" - - echo - echo "**** Crossgen skipped for non-managed assembly files:" - echo - - touch crossgenskipped - cat crossgenskipped - - echo - echo "**** Crossgen failed for the following dlls:" - echo - - touch crossgenretry - cat crossgenretry - - echo - echo "**** Beginning uncrossgen for failed dlls ****" - echo - export -f uncrossgenone - - rm -f $__coreclrbin/System.Private.CoreLib.dll - ln -s $__corelib $__coreclrbin/System.Private.CoreLib.dll - - cat crossgenretry | xargs -P 0 -n 1 -I {} bash -c 'uncrossgenone "$@"' _ {} "$__coreclrbin" - - rm -f $__coreclrbin/System.Private.CoreLib.dll - - echo - echo "**** Uncrossgen failed for the following dlls:" - echo - touch uncrossgenfails - cat uncrossgenfails -fi - -echo "**** Bootstrap CLI was successfully built ****" - From d19c0ef387bfb36b31229d36d443143ebbaa6154 Mon Sep 17 00:00:00 2001 From: Chris Rummel Date: Mon, 4 Oct 2021 15:55:41 -0500 Subject: [PATCH 5/6] [SourceBuild] Add omnisharp fix patch (#12227) * Add omnisharp fix path * Address review comments. --- ...-Fix-Omnisharp-add-needed-extra-TFMs.patch | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 src/SourceBuild/tarball/content/patches/runtime/0001-Fix-Omnisharp-add-needed-extra-TFMs.patch diff --git a/src/SourceBuild/tarball/content/patches/runtime/0001-Fix-Omnisharp-add-needed-extra-TFMs.patch b/src/SourceBuild/tarball/content/patches/runtime/0001-Fix-Omnisharp-add-needed-extra-TFMs.patch new file mode 100644 index 000000000..192ed3519 --- /dev/null +++ b/src/SourceBuild/tarball/content/patches/runtime/0001-Fix-Omnisharp-add-needed-extra-TFMs.patch @@ -0,0 +1,61 @@ +From 93de82c03ff71f897599db2824b959d020b877bd Mon Sep 17 00:00:00 2001 +From: Chris Rummel +Date: Thu, 30 Sep 2021 16:08:28 -0500 +Subject: [PATCH 1/2] Fix Omnisharp: Add needed extra TFMs. + +Omnisharp requires the net46 TFMs of this files explictly and we don't build them by default. This patch adds net46 to the relevant projects. The original issue for this was https://github.com/OmniSharp/omnisharp-vscode/issues/4610 and the long-term fix of switching to net>4 TFM is tracked in https://github.com/OmniSharp/omnisharp-vscode/issues/4360. + +--- + src/libraries/Directory.Build.props | 2 +- + .../ref/System.Resources.Extensions.csproj | 4 ++-- + .../src/System.Resources.Extensions.csproj | 2 +- + 3 files changed, 4 insertions(+), 4 deletions(-) + +diff --git a/src/libraries/Directory.Build.props b/src/libraries/Directory.Build.props +index 4d39fad397e..649c669517e 100644 +--- a/src/libraries/Directory.Build.props ++++ b/src/libraries/Directory.Build.props +@@ -27,7 +27,7 @@ + + + +- net461;net462;net47;net471;net472 ++ net46;net461;net462;net47;net471;net472 + + + +diff --git a/src/libraries/System.Resources.Extensions/ref/System.Resources.Extensions.csproj b/src/libraries/System.Resources.Extensions/ref/System.Resources.Extensions.csproj +index 99037ae459c..848be41690c 100644 +--- a/src/libraries/System.Resources.Extensions/ref/System.Resources.Extensions.csproj ++++ b/src/libraries/System.Resources.Extensions/ref/System.Resources.Extensions.csproj +@@ -1,6 +1,6 @@ + + +- $(NetCoreAppCurrent);netstandard2.0;net461 ++ $(NetCoreAppCurrent);netstandard2.0;net46;net461 + enable + + +@@ -11,4 +11,4 @@ + + + +- +\ No newline at end of file ++ +diff --git a/src/libraries/System.Resources.Extensions/src/System.Resources.Extensions.csproj b/src/libraries/System.Resources.Extensions/src/System.Resources.Extensions.csproj +index adacc9b977e..38d4c145352 100644 +--- a/src/libraries/System.Resources.Extensions/src/System.Resources.Extensions.csproj ++++ b/src/libraries/System.Resources.Extensions/src/System.Resources.Extensions.csproj +@@ -1,7 +1,7 @@ + + + true +- $(NetCoreAppCurrent);netstandard2.0;net461 ++ $(NetCoreAppCurrent);netstandard2.0;net46;net461 + $(DefineConstants);RESOURCES_EXTENSIONS + enable + +-- +2.17.1 + From a1510581577e0a9efcd8ad1ada2dac7b1c837a6f Mon Sep 17 00:00:00 2001 From: Jacques Eloff Date: Tue, 5 Oct 2021 00:15:42 -0700 Subject: [PATCH 6/6] Remove duplicate telemetry (#12243) --- src/redist/targets/packaging/windows/clisdk/dotnet.wxs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/redist/targets/packaging/windows/clisdk/dotnet.wxs b/src/redist/targets/packaging/windows/clisdk/dotnet.wxs index 902a2505f..7d7c21393 100644 --- a/src/redist/targets/packaging/windows/clisdk/dotnet.wxs +++ b/src/redist/targets/packaging/windows/clisdk/dotnet.wxs @@ -44,8 +44,8 @@ Impersonate="no"/> - - + NOT Installed + NOT Installed