diff --git a/src/SourceBuild/tarball/content/prep.sh b/src/SourceBuild/tarball/content/prep.sh index 794a108d2..9e749fc2b 100755 --- a/src/SourceBuild/tarball/content/prep.sh +++ b/src/SourceBuild/tarball/content/prep.sh @@ -5,13 +5,18 @@ IFS=$'\n\t' SCRIPT_ROOT="$(cd -P "$( dirname "$0" )" && pwd)" usage() { - echo "usage: $0" + echo "usage: $0 [options]" echo "" echo " Prepares a tarball to be built by downloading Private.SourceBuilt.Artifacts.*.tar.gz and" echo " installing the version of dotnet referenced in global.json" + echo "options:" + echo " --bootstrap Build a bootstrap version of previously source-built packages archive." + echo " This modifies the downloaded version, replacing portable packages" + echo " with official ms-built packages restored from package feeds." echo "" } +buildBootstrap=false positional_args=() while :; do if [ $# -le 0 ]; then @@ -23,6 +28,9 @@ while :; do usage exit 0 ;; + --bootstrap) + buildBootstrap=true + ;; *) positional_args+=("$1") ;; @@ -87,3 +95,31 @@ if [ "$installDotnet" == "true" ]; then echo " Installing dotnet..." (source ./eng/common/tools.sh && InitializeDotNetCli true) fi + +# Build bootstrap, if specified +if [ "$buildBootstrap" == "true" ]; then + DOTNET_SDK_PATH="$SCRIPT_ROOT/.dotnet" + + # Create working directory for running bootstrap project + workingDir=$(mktemp -d) + echo " Building bootstrap previously source-built in $workingDir" + + # Copy bootstrap project to working dir + cp $SCRIPT_ROOT/scripts/bootstrap/buildBootstrapPreviouslySB.csproj $workingDir + + # Copy NuGet.config from the installer repo to have the right feeds + cp $SCRIPT_ROOT/src/installer.*/NuGet.config $workingDir + + # Get PackageVersions.props from existing prev-sb archive + echo " Retrieving PackageVersions.props from existing archive" + sourceBuiltArchive=`find $SCRIPT_ROOT/packages/archive -maxdepth 1 -name 'Private.SourceBuilt.Artifacts*.tar.gz'` + if [ -f "$sourceBuiltArchive" ]; then + tar -xzf "$sourceBuiltArchive" -C $workingDir PackageVersions.props + fi + + # Run restore on project to initiate download of bootstrap packages + $DOTNET_SDK_PATH/dotnet restore $workingDir/buildBootstrapPreviouslySB.csproj /bl /p:ArchiveDir="$SCRIPT_ROOT/packages/archive/" + + # Remove working directory + rm -rf $workingDir +fi diff --git a/src/SourceBuild/tarball/content/scripts/bootstrap/buildBootstrapPreviouslySB.csproj b/src/SourceBuild/tarball/content/scripts/bootstrap/buildBootstrapPreviouslySB.csproj new file mode 100644 index 000000000..090db5fbf --- /dev/null +++ b/src/SourceBuild/tarball/content/scripts/bootstrap/buildBootstrapPreviouslySB.csproj @@ -0,0 +1,61 @@ + + + + + + + net6.0 + $(MSBuildProjectDirectory)/artifacts/ + $(MSBuildProjectDirectory)/artifacts/restoredPkgs/ + $(MSBuildProjectDirectory)/artifacts/unpacked/ + $(ArchiveDir)Private.SourceBuilt.Artifacts.Bootstrap.tar.gz + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/scripts/bootstrap/buildbootstrapcli.sh b/src/SourceBuild/tarball/content/scripts/bootstrap/buildbootstrapcli.sh deleted file mode 100755 index b27b5476e..000000000 --- a/src/SourceBuild/tarball/content/scripts/bootstrap/buildbootstrapcli.sh +++ /dev/null @@ -1,411 +0,0 @@ -#!/usr/bin/env bash -set -e -set -u -set -o pipefail - -usage() -{ - echo "Builds a bootstrap CLI from sources" - echo "Usage: $0 [BuildType] -rid -seedcli [-os ] [-clang ] [-corelib ]" - echo "" - echo "Options:" - echo " BuildType Type of build (-debug, -release), default: -release" - echo " -clang Override of the version of clang compiler to use" - echo " -corelib Path to System.Private.CoreLib.dll, default: use the System.Private.CoreLib.dll from the seed CLI" - echo " -os Operating system (used for corefx build), default: Linux" - echo " -rid Runtime identifier including the architecture part (e.g. rhel.6-x64)" - echo " -seedcli Seed CLI used to generate the target CLI" - echo " -outputpath Optional output directory to contain the generated cli and cloned repos, default: " -} - -disable_pax_mprotect() -{ - if [[ $(command -v paxctl) ]]; then - paxctl -c -m $1 - fi -} - -get_max_version() -{ - local maxversionhi=0 - local maxversionmid=0 - local maxversionlo=0 - local maxversiontag - local versionrest - local versionhi - local versionmid - local versionlo - local versiontag - local foundmax - - for d in $1/*; do - - if [[ -d $d ]]; then - versionrest=$(basename $d) - versionhi=${versionrest%%.*} - versionrest=${versionrest#*.} - versionmid=${versionrest%%.*} - versionrest=${versionrest#*.} - versionlo=${versionrest%%-*} - versiontag=${versionrest#*-} - if [[ $versiontag == $versionrest ]]; then - versiontag="" - fi - - foundmax=0 - - if [[ $versionhi -gt $maxversionhi ]]; then - foundmax=1 - elif [[ $versionhi -eq $maxversionhi ]]; then - if [[ $versionmid -gt $maxversionmid ]]; then - foundmax=1 - elif [[ $versionmid -eq $maxversionmid ]]; then - if [[ $versionlo -gt $maxversionlo ]]; then - foundmax=1 - elif [[ $versionlo -eq $maxversionlo ]]; then - # tags are used to mark pre-release versions, so a version without a tag - # is newer than a version with one. - if [[ "$versiontag" == "" || $versiontag > $maxversiontag ]]; then - foundmax=1 - fi - fi - fi - fi - - if [[ $foundmax != 0 ]]; then - maxversionhi=$versionhi - maxversionmid=$versionmid - maxversionlo=$versionlo - maxversiontag=$versiontag - fi - fi - done - - echo $maxversionhi.$maxversionmid.$maxversionlo${maxversiontag:+-$maxversiontag} -} - -getrealpath() -{ - if command -v realpath > /dev/null; then - realpath $1 - else - readlink -e $1 - fi -} - -__build_os=Linux -__runtime_id= -__corelib= -__configuration=release -__clangversion= -__outputpath= - -while [[ "${1:-}" != "" ]]; do - lowerI="$(echo $1 | awk '{print tolower($0)}')" - case $lowerI in - -h|--help) - usage - exit 1 - ;; - -rid) - shift - __runtime_id=$1 - ;; - -os) - shift - __build_os=$1 - ;; - -debug) - __configuration=debug - ;; - -release) - __configuration=release - ;; - -corelib) - shift - __corelib=`getrealpath $1` - ;; - -seedcli) - shift - __seedclipath=`getrealpath $1` - ;; - -clang) - shift - __clangversion=clang$1 - ;; - -outputpath) - shift - __outputpath=$1 - ;; - *) - echo "Unknown argument to build.sh $1"; exit 1 - esac - shift -done - - -if [[ -z "$__runtime_id" ]]; then - echo "Missing the required -rid argument" - exit 2 -fi - -if [[ -z "$__seedclipath" ]]; then - echo "Missing the required -seedcli argument" - exit 3 -fi - -__build_arch=${__runtime_id#*-} - -if [[ -z "$__outputpath" ]]; then - __outputpath=$__runtime_id/dotnetcli -fi - -if [[ -d "$__outputpath" ]]; then - /bin/rm -r $__outputpath -fi - -mkdir -p $__runtime_id -mkdir -p $__outputpath - -__outputpath=`getrealpath $__outputpath` - -cd $__runtime_id - -cp -r $__seedclipath/* $__outputpath - -__frameworkversion="2.0.0" -__sdkversion="2.0.0" -__fxrversion="2.0.0" - -echo "**** DETECTING VERSIONS IN SEED CLI ****" - -__frameworkversion=`get_max_version $__seedclipath/shared/Microsoft.NETCore.App` -__sdkversion=`get_max_version $__seedclipath/sdk` -__fxrversion=`get_max_version $__seedclipath/host/fxr` - -echo "Framework version: $__frameworkversion" -echo "SDK version: $__sdkversion" -echo "FXR version: $__fxrversion" - -__frameworkpath=$__outputpath/shared/Microsoft.NETCore.App/$__frameworkversion - -echo "**** DETECTING GIT COMMIT HASHES ****" - -# Extract the git commit hashes representig the state of the three repos that -# the seed cli package was built from -__coreclrhash=`strings $__seedclipath/shared/Microsoft.NETCore.App/$__frameworkversion/libcoreclr.so | grep "@(#)" | grep -o "[a-f0-9]\{40\}"` -__corefxhash=`strings $__seedclipath/shared/Microsoft.NETCore.App/$__frameworkversion/System.Native.so | grep "@(#)" | grep -o "[a-f0-9]\{40\}"` -__coresetuphash=`strings $__seedclipath/dotnet | grep "@(#)" | grep -o "[a-f0-9]\{40\}"` -if [[ "$__coresetuphash" == "" ]]; then - __coresetuphash=`strings $__seedclipath/dotnet | grep -o "[a-f0-9]\{40\}"` -fi - -echo "coreclr hash: $__coreclrhash" -echo "corefx hash: $__corefxhash" -echo "core-setup hash: $__coresetuphash" - -# Clone the three repos if they were not cloned yet. If the folders already -# exist, leave them alone. This allows patching the cloned sources as needed - -if [[ ! -d coreclr ]]; then - echo "**** CLONING CORECLR REPOSITORY ****" - git clone https://github.com/dotnet/coreclr.git - cd coreclr - git checkout $__coreclrhash - cd .. -fi - -if [[ ! -d corefx ]]; then - echo "**** CLONING COREFX REPOSITORY ****" - git clone https://github.com/dotnet/corefx.git - cd corefx - git checkout $__corefxhash - cd .. -fi - -if [[ ! -d core-setup ]]; then - echo "**** CLONING CORE-SETUP REPOSITORY ****" - git clone https://github.com/dotnet/core-setup.git - cd core-setup - git checkout $__coresetuphash - cd .. -fi - -echo "**** BUILDING CORE-SETUP NATIVE COMPONENTS ****" -cd core-setup -src/corehost/build.sh --configuration $__configuration --arch "$__build_arch" --hostver "2.0.0" --apphostver "2.0.0" --fxrver "2.0.0" --policyver "2.0.0" --commithash `git rev-parse HEAD` -cd .. - -echo "**** BUILDING CORECLR NATIVE COMPONENTS ****" -cd coreclr -./build.sh $__configuration $__build_arch $__clangversion -skipgenerateversion -skipmanaged -skipmscorlib -skiprestore -skiprestoreoptdata -skipnuget -nopgooptimize 2>&1 | tee coreclr.log -export __coreclrbin=$(cat coreclr.log | sed -n -e 's/^.*Product binaries are available at //p') -cd .. -echo "CoreCLR binaries will be copied from $__coreclrbin" - -echo "**** BUILDING COREFX NATIVE COMPONENTS ****" -corefx/src/Native/build-native.sh $__build_arch $__configuration $__clangversion $__build_os 2>&1 | tee corefx.log -export __corefxbin=$(cat corefx.log | sed -n -e 's/^.*Build files have been written to: //p') -echo "CoreFX binaries will be copied from $__corefxbin" - -echo "**** Copying new binaries to dotnetcli/ ****" - -# First copy the coreclr repo binaries -cp $__coreclrbin/*so $__frameworkpath -cp $__coreclrbin/corerun $__frameworkpath -cp $__coreclrbin/crossgen $__frameworkpath - -# Mark the coreclr executables as allowed to create executable memory mappings -disable_pax_mprotect $__frameworkpath/corerun -disable_pax_mprotect $__frameworkpath/crossgen - -# Now copy the core-setup repo binaries - -if [[ $__fxrversion == 2* ]]; then - cp core-setup/cli/exe/dotnet/dotnet $__outputpath - cp core-setup/cli/exe/dotnet/dotnet $__frameworkpath/corehost - - cp core-setup/cli/dll/libhostpolicy.so $__frameworkpath - cp core-setup/cli/dll/libhostpolicy.so $__outputpath/sdk/$__sdkversion - - cp core-setup/cli/fxr/libhostfxr.so $__frameworkpath - cp core-setup/cli/fxr/libhostfxr.so $__outputpath/host/fxr/$__fxrversion - cp core-setup/cli/fxr/libhostfxr.so $__outputpath/sdk/$__sdkversion -else - cp core-setup/bin/$__runtime_id.$__configuration/corehost/dotnet $__outputpath - cp core-setup/bin/$__runtime_id.$__configuration/corehost/dotnet $__frameworkpath/corehost - - cp core-setup/bin/$__runtime_id.$__configuration/corehost/libhostpolicy.so $__frameworkpath - cp core-setup/bin/$__runtime_id.$__configuration/corehost/libhostpolicy.so $__outputpath/sdk/$__sdkversion - - cp core-setup/bin/$__runtime_id.$__configuration/corehost/libhostfxr.so $__frameworkpath - cp core-setup/bin/$__runtime_id.$__configuration/corehost/libhostfxr.so $__outputpath/host/fxr/$__fxrversion - cp core-setup/bin/$__runtime_id.$__configuration/corehost/libhostfxr.so $__outputpath/sdk/$__sdkversion -fi - -# Mark the core-setup executables as allowed to create executable memory mappings -disable_pax_mprotect $__outputpath/dotnet -disable_pax_mprotect $__frameworkpath/corehost - -# Finally copy the corefx repo binaries -cp $__corefxbin/**/System.* $__frameworkpath - -# Copy System.Private.CoreLib.dll override from somewhere if requested -if [[ "$__corelib" != "" ]]; then - cp "$__corelib" $__frameworkpath -fi - -# Add the new RID to Microsoft.NETCore.App.deps.json -# Replace the linux-x64 RID in the target, runtimeTarget and runtimes by the new RID -# and add the new RID to the list of runtimes. -echo "**** Adding new rid to Microsoft.NETCore.App.deps.json ****" - -#TODO: add parameter with the parent RID sequence - -sed \ - -e 's/runtime\.linux-x64/runtime.'$__runtime_id'/g' \ - -e 's/runtimes\/linux-x64/runtimes\/'$__runtime_id'/g' \ - -e 's/Version=v\([0-9].[0-9]\)\/linux-x64/Version=v\1\/'$__runtime_id'/g' \ -$__seedclipath/shared/Microsoft.NETCore.App/$__frameworkversion/Microsoft.NETCore.App.deps.json \ ->$__frameworkpath/Microsoft.NETCore.App.deps.json - -# add the new RID to the list of runtimes iff it does not already exist (sed inplace) -__os_dependencies= -if [[ $__build_os == "Linux" ]]; then - __os_dependencies='"linux", "linux-'$__build_arch'", ' -fi -grep -q "\"$__runtime_id\":" $__frameworkpath/Microsoft.NETCore.App.deps.json || \ -sed -i \ - -e 's/"runtimes": {/&\n "'$__runtime_id'": [\n '"$__os_dependencies"'"unix", "unix-'$__build_arch'", "any", "base"\n ],/g' \ -$__frameworkpath/Microsoft.NETCore.App.deps.json - -__crossgentimeout=120 - -function crossgenone(){ - echo $2/crossgen /MissingDependenciesOK /Platform_Assemblies_Paths $2:$3 /in $1 /out $1.ni >$1.log 2>&1 - timeout $__crossgentimeout $2/crossgen /MissingDependenciesOK /Platform_Assemblies_Paths $2:$3 /in $1 /out $1.ni >>$1.log 2>&1 - exitCode=$? - if [ "$exitCode" == "0" ] - then - rm $1.log - mv $1.ni $1 - elif grep -q -e 'The module was expected to contain an assembly manifest' \ - -e 'An attempt was made to load a program with an incorrect format.' \ - -e 'File is PE32' $1.log - then - rm $1.log - echo "$1" >> crossgenskipped - else - echo "$1" >> crossgenretry - fi -} - -# Run an assembly through ildasm ilasm roundtrip to remove x64 crossgen -function uncrossgenone(){ - echo >> $1.log 2>&1 - echo mv $1 $1.x64 >> $1.log 2>&1 - echo $2/ildasm -raweh -out=$1.il $1.x64 "&& \\" >> $1.log 2>&1 - echo $2/ilasm -output=$1 -QUIET -NOLOGO -DEBUG -OPTIMIZE $1.il >> $1.log 2>&1 - - mv $1 $1.x64 - $2/ildasm -raweh -out=$1.il $1.x64 && \ - $2/ilasm -output=$1 -DLL -QUIET -NOLOGO -DEBUG -OPTIMIZE $1.il - exitCode=$? - if [ "$exitCode" == "0" ] - then - rm $1.x64 - rm $1.il - else - echo "$1" >> uncrossgenfails - fi -} - -# if $__build_arch is not x64 then any dll which was crossgened for x64 must be recrossgened for $__build_arch -if [[ "$__build_arch" != "x64" ]] -then - echo "**** Beginning crossgen for $__build_arch target ****" - export -f crossgenone - export __crossgentimeout - - rm -f crossgenretry crossgendlls crossgenskipped uncrossgenfails - - # Assumes System.Private.CoreLib was already crossgened - find $__outputpath -type f -name \*.dll -or -name \*.exe | grep -v System.Private.CoreLib > crossgendlls - - cat crossgendlls | xargs -P 0 -n 1 -I {} bash -c 'crossgenone "$@"' _ {} "$__frameworkpath" "$__outputpath/sdk/$__sdkversion" - - echo - echo "**** Crossgen skipped for non-managed assembly files:" - echo - - touch crossgenskipped - cat crossgenskipped - - echo - echo "**** Crossgen failed for the following dlls:" - echo - - touch crossgenretry - cat crossgenretry - - echo - echo "**** Beginning uncrossgen for failed dlls ****" - echo - export -f uncrossgenone - - rm -f $__coreclrbin/System.Private.CoreLib.dll - ln -s $__corelib $__coreclrbin/System.Private.CoreLib.dll - - cat crossgenretry | xargs -P 0 -n 1 -I {} bash -c 'uncrossgenone "$@"' _ {} "$__coreclrbin" - - rm -f $__coreclrbin/System.Private.CoreLib.dll - - echo - echo "**** Uncrossgen failed for the following dlls:" - echo - touch uncrossgenfails - cat uncrossgenfails -fi - -echo "**** Bootstrap CLI was successfully built ****" -