From 9bdfb5afaca68641207220a1efcd1cf9b26a31f5 Mon Sep 17 00:00:00 2001 From: Daniel Plaisted Date: Wed, 7 Nov 2018 17:21:16 -0800 Subject: [PATCH] Build script fixes for CI --- build.cmd | 8 +- build.sh | 53 ++++++++++-- old/build.sh | 54 ------------- old/run-build.ps1 | 118 --------------------------- old/run-build.sh | 202 ---------------------------------------------- run-build.ps1 | 28 +++++++ run-build.sh | 69 ++++++++++++++++ 7 files changed, 151 insertions(+), 381 deletions(-) delete mode 100644 old/build.sh delete mode 100644 old/run-build.ps1 delete mode 100644 old/run-build.sh create mode 100644 run-build.ps1 create mode 100644 run-build.sh diff --git a/build.cmd b/build.cmd index 6a89aa523..ce681438b 100644 --- a/build.cmd +++ b/build.cmd @@ -1,3 +1,7 @@ @echo off -powershell -NoLogo -NoProfile -ExecutionPolicy ByPass -Command "& """%~dp0eng\common\build.ps1""" -build -restore %*" -exit /b %ErrorLevel% + +REM Copyright (c) .NET Foundation and contributors. All rights reserved. +REM Licensed under the MIT license. See LICENSE file in the project root for full license information. + +powershell -ExecutionPolicy Bypass -NoProfile -NoLogo -Command "& \"%~dp0run-build.ps1\" %*; exit $LastExitCode;" +if %errorlevel% neq 0 exit /b %errorlevel% diff --git a/build.sh b/build.sh index 2d2a01b1e..99a136aa7 100755 --- a/build.sh +++ b/build.sh @@ -1,11 +1,54 @@ -#!/bin/bash +#!/usr/bin/env bash +# +# Copyright (c) .NET Foundation and contributors. All rights reserved. +# Licensed under the MIT license. See LICENSE file in the project root for full license information. +# + +# Set OFFLINE environment variable to build offline + +set -e SOURCE="${BASH_SOURCE[0]}" while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink - ScriptRoot="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$ScriptRoot/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located + [[ "$SOURCE" != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located done -ScriptRoot="$( cd -P "$( dirname "$SOURCE" )" && pwd )" +DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" -. "$ScriptRoot/eng/common/build.sh" --build --restore "$@" +# Some things depend on HOME and it may not be set. We should fix those things, but until then, we just patch a value in +if [ -z "$HOME" ]; then + export HOME=$DIR/artifacts/home + + [ ! -d "$HOME" ] || rm -Rf $HOME + mkdir -p $HOME +fi + +args= + +while [[ $# > 0 ]]; do + lowerI="$(echo $1 | awk '{print tolower($0)}')" + case $lowerI in + --docker) + export BUILD_IN_DOCKER=1 + export DOCKER_IMAGENAME=$2 + shift + ;; + *) + args="$args $1" + ;; + esac + shift +done + +dockerbuild() +{ + BUILD_COMMAND=/opt/code/run-build.sh $DIR/scripts/dockerrun.sh --non-interactive "$@" +} + +# Check if we need to build in docker +if [ ! -z "$BUILD_IN_DOCKER" ]; then + dockerbuild $args +else + $DIR/run-build.sh $args +fi diff --git a/old/build.sh b/old/build.sh deleted file mode 100644 index 99a136aa7..000000000 --- a/old/build.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) .NET Foundation and contributors. All rights reserved. -# Licensed under the MIT license. See LICENSE file in the project root for full license information. -# - -# Set OFFLINE environment variable to build offline - -set -e - -SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink - DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ "$SOURCE" != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located -done -DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" - -# Some things depend on HOME and it may not be set. We should fix those things, but until then, we just patch a value in -if [ -z "$HOME" ]; then - export HOME=$DIR/artifacts/home - - [ ! -d "$HOME" ] || rm -Rf $HOME - mkdir -p $HOME -fi - -args= - -while [[ $# > 0 ]]; do - lowerI="$(echo $1 | awk '{print tolower($0)}')" - case $lowerI in - --docker) - export BUILD_IN_DOCKER=1 - export DOCKER_IMAGENAME=$2 - shift - ;; - *) - args="$args $1" - ;; - esac - shift -done - -dockerbuild() -{ - BUILD_COMMAND=/opt/code/run-build.sh $DIR/scripts/dockerrun.sh --non-interactive "$@" -} - -# Check if we need to build in docker -if [ ! -z "$BUILD_IN_DOCKER" ]; then - dockerbuild $args -else - $DIR/run-build.sh $args -fi diff --git a/old/run-build.ps1 b/old/run-build.ps1 deleted file mode 100644 index e0db7acb9..000000000 --- a/old/run-build.ps1 +++ /dev/null @@ -1,118 +0,0 @@ -# -# Copyright (c) .NET Foundation and contributors. All rights reserved. -# Licensed under the MIT license. See LICENSE file in the project root for full license information. -# - -param( - [string]$Configuration="Debug", - [string]$Architecture="x64", - # This is here just to eat away this parameter because CI still passes this in. - [string]$Targets="Default", - [switch]$NoPackage, - [switch]$NoBuild, - [switch]$Help, - [Parameter(Position=0, ValueFromRemainingArguments=$true)] - $ExtraParameters) - -if($Help) -{ - Write-Output "Usage: .\run-build.ps1 [-Configuration ] [-Architecture ] [-NoPackage] [-NoBuild] [-Help]" - Write-Output "" - Write-Output "Options:" - Write-Output " -Configuration Build the specified Configuration (Debug or Release, default: Debug)" - Write-Output " -Architecture Build the specified architecture (x64, x86, arm, or arm64 , default: x64)" - Write-Output " -NoPackage Skip packaging targets" - Write-Output " -NoBuild Skip building the product" - Write-Output " -Help Display this help message" - exit 0 -} - -# The first 'pass' call to "dotnet msbuild build.proj" has a hard-coded "WriteDynamicPropsToStaticPropsFiles" target -# therefore, this call should not have other targets defined. Remove all targets passed in as 'extra parameters'. -if ($ExtraParameters) -{ - $ExtraParametersNoTargets = $ExtraParameters.GetRange(0,$ExtraParameters.Count) - foreach ($param in $ExtraParameters) - { - if(($param.StartsWith("/t:", [StringComparison]::OrdinalIgnoreCase)) -or ($param.StartsWith("/target:", [StringComparison]::OrdinalIgnoreCase))) - { - $ExtraParametersNoTargets.Remove("$param") | Out-Null - } - } -} - -$env:CONFIGURATION = $Configuration; -$RepoRoot = "$PSScriptRoot" -if(!$env:NUGET_PACKAGES){ - $env:NUGET_PACKAGES = "$RepoRoot\.nuget\packages" -} - -if($NoPackage) -{ - $env:DOTNET_BUILD_SKIP_PACKAGING=1 -} -else -{ - $env:DOTNET_BUILD_SKIP_PACKAGING=0 -} - -# Use a repo-local install directory for stage0 (but not the artifacts directory because that gets cleaned a lot -if (!$env:DOTNET_INSTALL_DIR) -{ - $env:DOTNET_INSTALL_DIR="$RepoRoot\.dotnet_stage0\$Architecture" -} - -if (!(Test-Path $env:DOTNET_INSTALL_DIR)) -{ - mkdir $env:DOTNET_INSTALL_DIR | Out-Null -} - -# Disable first run since we want to control all package sources -$env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 - -# Don't resolve shared frameworks from user or global locations -$env:DOTNET_MULTILEVEL_LOOKUP=0 - -# Turn off MSBuild Node re-use -$env:MSBUILDDISABLENODEREUSE=1 - -# Workaround for the sockets issue when restoring with many nuget feeds. -$env:DOTNET_SYSTEM_NET_HTTP_USESOCKETSHTTPHANDLER=0 - -# Enable vs test console logging -$env:VSTEST_BUILD_TRACE=1 -$env:VSTEST_TRACE_BUILD=1 - -# install a stage0 -[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 -$dotnetInstallPath = Join-Path $env:DOTNET_INSTALL_DIR "dotnet-install.ps1" -Invoke-WebRequest -Uri "https://dot.net/v1/dotnet-install.ps1" -OutFile "$dotnetInstallPath" - -$InstallArchitecture = $Architecture -if($Architecture.StartsWith("arm", [StringComparison]::OrdinalIgnoreCase)) -{ - $InstallArchitecture = "x64" -} - -Write-Output "$dotnetInstallPath -version ""3.0.100-preview1-009020"" -InstallDir $env:DOTNET_INSTALL_DIR -Architecture ""$InstallArchitecture""" -Invoke-Expression "$dotnetInstallPath -version ""3.0.100-preview1-009020"" -InstallDir $env:DOTNET_INSTALL_DIR -Architecture ""$InstallArchitecture""" - -if ($LastExitCode -ne 0) -{ - Copy-Item -Recurse -Force $env:DOTNET_TOOL_DIR $env:DOTNET_INSTALL_DIR -} - -# Put the stage0 on the path -$env:PATH = "$env:DOTNET_INSTALL_DIR;$env:PATH" - -if ($NoBuild) -{ - Write-Output "Not building due to --nobuild" - Write-Output "Command that would be run: 'dotnet msbuild build.proj /m /p:Architecture=$Architecture $ExtraParameters'" -} -else -{ - dotnet msbuild build.proj /bl:msbuild.generatepropsfile.binlog /p:Architecture=$Architecture /p:GeneratePropsFile=true /t:WriteDynamicPropsToStaticPropsFiles $ExtraParametersNoTargets - dotnet msbuild build.proj /bl:msbuild.mainbuild.binlog /m /v:normal /fl /flp:v=diag /bl /p:Architecture=$Architecture $ExtraParameters - if($LASTEXITCODE -ne 0) { throw "Failed to build" } -} diff --git a/old/run-build.sh b/old/run-build.sh deleted file mode 100644 index b58ec14e9..000000000 --- a/old/run-build.sh +++ /dev/null @@ -1,202 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) .NET Foundation and contributors. All rights reserved. -# Licensed under the MIT license. See LICENSE file in the project root for full license information. -# - -set -e - -machine_has() { - hash "$1" > /dev/null 2>&1 - return $? -} - -check_min_reqs() { - if ! machine_has "curl"; then - echo "run-build: Error: curl is required to download dotnet. Install curl to proceed." >&2 - return 1 - fi - return 0 -} - -# args: -# remote_path - $1 -# [out_path] - $2 - stdout if not provided -download() { - eval $invocation - - local remote_path=$1 - local out_path=${2:-} - - local failed=false - if [ -z "$out_path" ]; then - curl --retry 10 -sSL --create-dirs $remote_path || failed=true - else - curl --retry 10 -sSL --create-dirs -o $out_path $remote_path || failed=true - fi - - if [ "$failed" = true ]; then - echo "run-build: Error: Download failed" >&2 - return 1 - fi -} - -SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink - DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ "$SOURCE" != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located -done -DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" -REPOROOT="$DIR" -CONFIGURATION="debug" -ARCHITECTURE="x64" -STAGE0_SOURCE_DIR= - -source "$REPOROOT/scripts/common/_prettyprint.sh" - -BUILD=1 - -LINUX_PORTABLE_INSTALL_ARGS= -CUSTOM_BUILD_ARGS= - -# Set nuget package cache under the repo -[ -z $NUGET_PACKAGES ] && export NUGET_PACKAGES="$REPOROOT/.nuget/packages" - -args=( ) - -while [[ $# > 0 ]]; do - lowerI="$(echo $1 | awk '{print tolower($0)}')" - case $lowerI in - -c|--configuration) - CONFIGURATION=$2 - shift - ;; - --nopackage) - export DOTNET_BUILD_SKIP_PACKAGING=1 - ;; - --skip-prereqs) - # Allow CI to disable prereqs check since the CI has the pre-reqs but not ldconfig it seems - export DOTNET_INSTALL_SKIP_PREREQS=1 - ;; - --nobuild) - BUILD=0 - ;; - -a|--architecture) - ARCHITECTURE=$2 - shift - ;; - --runtime-id) - CUSTOM_BUILD_ARGS="/p:Rid=\"$2\"" - shift - ;; - # This is here just to eat away this parameter because CI still passes this in. - --targets) - shift - ;; - --linux-portable) - LINUX_PORTABLE_INSTALL_ARGS="--runtime-id linux-x64" - CUSTOM_BUILD_ARGS="/p:OSName=\"linux\" /p:IslinuxPortable=\"true\"" - ;; - --stage0) - STAGE0_SOURCE_DIR=$2 - shift - ;; - --help) - echo "Usage: $0 [--configuration ] [--architecture ] [--skip-prereqs] [--nopackage] [--nobuild ] [--docker ] [--stage0 ] [--help]" - echo "" - echo "Options:" - echo " --configuration Build the specified Configuration (Debug or Release, default: Debug)" - echo " --architecture Build the specified architecture (x64, arm or arm64 , default: x64)" - echo " --skip-prereqs Skip checks for pre-reqs in dotnet_install" - echo " --nopackage Skip packaging targets" - echo " --nobuild Skip building, showing the command that would be used to build" - echo " --docker Build in Docker using the Dockerfile located in scripts/docker/IMAGENAME" - echo " --stage0 Set the stage0 source directory. The default is to download it from Azure." - echo " --help Display this help message" - exit 0 - ;; - *) - args=$@ - break - ;; - esac - - shift -done - -export CONFIGURATION=$CONFIGURATION - -# The first 'pass' call to "dotnet msbuild build.proj" has a hard-coded "WriteDynamicPropsToStaticPropsFiles" target -# therefore, this call should not have other targets defined. Remove all targets passed in as 'extra parameters'. -argsnotargets=( ) -for arg in ${args[@]} -do - arglower="$(echo $arg | awk '{print tolower($0)}')" - if [[ $arglower != '/t:'* ]] && [[ $arglower != '/target:'* ]]; then - argsnotargets+=($arg) - fi -done - -# Create an install directory for the stage 0 CLI -[ -z "$DOTNET_INSTALL_DIR" ] && export DOTNET_INSTALL_DIR=$REPOROOT/.dotnet_stage0/$ARCHITECTURE -[ -d "$DOTNET_INSTALL_DIR" ] || mkdir -p $DOTNET_INSTALL_DIR - -# Disable first run since we want to control all package sources -export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 - -# Enable verbose VS Test Console logging -export VSTEST_BUILD_TRACE=1 -export VSTEST_TRACE_BUILD=1 - - -# Don't resolve shared frameworks from user or global locations -export DOTNET_MULTILEVEL_LOOKUP=0 - -# Turn off MSBuild Node re-use -export MSBUILDDISABLENODEREUSE=1 - -# Workaround for the sockets issue when restoring with many nuget feeds. -export DOTNET_SYSTEM_NET_HTTP_USESOCKETSHTTPHANDLER=0 - -# Install a stage 0 -INSTALL_ARCHITECTURE=$ARCHITECTURE -archlower="$(echo $ARCHITECTURE | awk '{print tolower($0)}')" -if [[ $archlower == 'arm'* ]]; then - INSTALL_ARCHITECTURE="x64" -fi - -if [ "$STAGE0_SOURCE_DIR" == "" ]; then - (set -x ; curl -sSL "https://dot.net/v1/dotnet-install.sh" | bash /dev/stdin --version "3.0.100-preview1-009020" --install-dir "$DOTNET_INSTALL_DIR" --architecture "$INSTALL_ARCHITECTURE" $LINUX_PORTABLE_INSTALL_ARGS) -else - echo "Copying bootstrap cli from $STAGE0_SOURCE_DIR" - cp -r $STAGE0_SOURCE_DIR/* "$DOTNET_INSTALL_DIR" -fi - -EXIT_CODE=$? -if [ $EXIT_CODE != 0 ]; then - echo "run-build: Error: installing stage0 with exit code $EXIT_CODE." >&2 - exit $EXIT_CODE -fi - -# Put stage 0 on the PATH (for this shell only) -PATH="$DOTNET_INSTALL_DIR:$PATH" - -# Increases the file descriptors limit for this bash. It prevents an issue we were hitting during restore -FILE_DESCRIPTOR_LIMIT=$( ulimit -n ) -if [ $FILE_DESCRIPTOR_LIMIT -lt 1024 ] -then - echo "Increasing file description limit to 1024" - ulimit -n 1024 -fi - -# Disable first run since we want to control all package sources -export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 - -if [ $BUILD -eq 1 ]; then - dotnet msbuild build.proj /bl:msbuild.generatepropsfile.binlog /p:Architecture=$ARCHITECTURE $CUSTOM_BUILD_ARGS /p:GeneratePropsFile=true /t:WriteDynamicPropsToStaticPropsFiles ${argsnotargets[@]} - dotnet msbuild build.proj /bl:msbuild.mainbuild.binlog /m /v:normal /fl /flp:v=diag /bl /p:Architecture=$ARCHITECTURE $CUSTOM_BUILD_ARGS $args -else - echo "Not building due to --nobuild" - echo "Command that would be run is: 'dotnet msbuild build.proj /m /p:Architecture=$ARCHITECTURE $CUSTOM_BUILD_ARGS $args'" -fi diff --git a/run-build.ps1 b/run-build.ps1 new file mode 100644 index 000000000..766d7937e --- /dev/null +++ b/run-build.ps1 @@ -0,0 +1,28 @@ +# +# Copyright (c) .NET Foundation and contributors. All rights reserved. +# Licensed under the MIT license. See LICENSE file in the project root for full license information. +# + +param( + [string]$Configuration="Debug", + [string]$Architecture="x64", + [Parameter(ValueFromRemainingArguments=$true)][String[]]$ExtraParameters +) + +#$env:CONFIGURATION = $Configuration; +$RepoRoot = "$PSScriptRoot" +# if(!$env:NUGET_PACKAGES){ +# $env:NUGET_PACKAGES = "$RepoRoot\.nuget\packages" +# } + +# $InstallArchitecture = $Architecture +# if($Architecture.StartsWith("arm", [StringComparison]::OrdinalIgnoreCase)) +# { +# $InstallArchitecture = "x64" +# } + +$ArchitectureParam="/p:Architecture=$Architecture" +$ConfigurationParam="-configuration $Configuration" + +Invoke-Expression "$RepoRoot\eng\common\build.ps1 -restore -build $ConfigurationParam $ArchitectureParam $ExtraParameters" +if($LASTEXITCODE -ne 0) { throw "Failed to build" } \ No newline at end of file diff --git a/run-build.sh b/run-build.sh new file mode 100644 index 000000000..fc876e0ae --- /dev/null +++ b/run-build.sh @@ -0,0 +1,69 @@ +#!/bin/bash +# +# Copyright (c) .NET Foundation and contributors. All rights reserved. +# Licensed under the MIT license. See LICENSE file in the project root for full license information. +# + +set -e + +SOURCE="${BASH_SOURCE[0]}" +while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink + DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + SOURCE="$(readlink "$SOURCE")" + [[ "$SOURCE" != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located +done +DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" +REPOROOT="$DIR" + +ARCHITECTURE="x64" + +source "$REPOROOT/scripts/common/_prettyprint.sh" + +LINUX_PORTABLE_INSTALL_ARGS= +CUSTOM_BUILD_ARGS= + +# Set nuget package cache under the repo +[ -z $NUGET_PACKAGES ] && export NUGET_PACKAGES="$REPOROOT/.nuget/packages" + +args=( ) + +while [[ $# > 0 ]]; do + lowerI="$(echo $1 | awk '{print tolower($0)}')" + case $lowerI in + -c|--configuration) + export CONFIGURATION=$2 + args+=("--configuration") + args+=("$2") + shift + ;; + -a|--architecture) + ARCHITECTURE="$2" + args+=("/p:Architecture=$ARCHITECTURE") + shift + ;; + --runtime-id) + args+=("/p:Rid=\"$2\"") + shift + ;; + --linux-portable) + args+=("/p:Rid=linux-x64 /p:OSName=\"linux\" /p:IslinuxPortable=\"true\"") + ;; + --help) + echo "Usage: $0 [--configuration ] [--architecture ] [--docker ] [--help]" + echo "" + echo "Options:" + echo " --configuration Build the specified Configuration (Debug or Release, default: Debug)" + echo " --architecture Build the specified architecture (x64, arm or arm64 , default: x64)" + echo " --docker Build in Docker using the Dockerfile located in scripts/docker/IMAGENAME" + echo " --help Display this help message" + exit 0 + ;; + *) + args+=("$1") + ;; + esac + + shift +done + +. "$REPOROOT/eng/common/build.sh" --build --restore "${args[@]}"