Clean up build scripts

Decompose into self-contained granular components
Provide reasonable defaults for cross cutting concerns, allowing for independent execution of steps
Start unifying Windows/Bash architecture

fix Bash CI scripts

dockerbuild.sh _common.sh path

Add missing restore-packages.sh

Copy/paste issues

Quote $SOURCE
fix .gitignore

PR Feedback

Merge in @SridarMS's work to avoid redownloading DNX

enabling build of dotnet-build

merge in @SridharMS's CentOS changes

Enable building FSC

enable restoring specific subdirectories

Fix dnx version check

Add missed dependency

Fix pathing to tests

Match Linux build version to Windows, fixing linux tests as a side effect.

workaround for coreclr#2215

fix pathing issue

disable building in docker

BUILD_IN_DOCKER was set, somehow...

fix headers
This commit is contained in:
Piotr Puszkiewicz 2015-12-28 11:47:21 -08:00
parent d1a257bff7
commit 4b217db9c0
66 changed files with 1163 additions and 242 deletions

View file

@ -0,0 +1,36 @@
#
# Copyright (c) .NET Foundation and contributors. All rights reserved.
# Licensed under the MIT license. See LICENSE file in the project root for full license information.
#
. $PSScriptRoot\..\common\_common.ps1
$DnxPackage = "dnx-coreclr-win-x64.1.0.0-rc1-update1.nupkg"
$DnxVersion = "1.0.0-rc1-16231"
$doInstall = $true
# check if the required dnx version is already downloaded
if ((Test-Path "$DnxRoot\dnx.exe")) {
$dnxOut = & "$DnxRoot\dnx.exe" --version
if ($dnxOut -Match $DnxVersion) {
Write-Host "Dnx version - $DnxVersion already downloaded."
$doInstall = $false
}
}
if ($doInstall)
{
# Download dnx to copy to stage2
Remove-Item -Recurse -Force -ErrorAction Ignore $DnxDir
mkdir -Force "$DnxDir" | Out-Null
Write-Host "Downloading Dnx version - $DnxVersion."
$DnxUrl="https://api.nuget.org/packages/$DnxPackage"
Invoke-WebRequest -UseBasicParsing "$DnxUrl" -OutFile "$DnxDir\dnx.zip"
Add-Type -Assembly System.IO.Compression.FileSystem | Out-Null
[System.IO.Compression.ZipFile]::ExtractToDirectory("$DnxDir\dnx.zip", "$DnxDir")
}

71
scripts/obtain/install-dnx.sh Executable file
View file

@ -0,0 +1,71 @@
#!/usr/bin/env bash
#
# Copyright (c) .NET Foundation and contributors. All rights reserved.
# Licensed under the MIT license. See LICENSE file in the project root for full license information.
#
set -e
SOURCE="${BASH_SOURCE[0]}"
while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
SOURCE="$(readlink "$SOURCE")"
[[ "$SOURCE" != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
done
DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
source "$DIR/../common/_common.sh"
say() {
printf "%b\n" "dotnet_install_dnx: $1"
}
doInstall=true
DNX_FEED="https://api.nuget.org/packages"
DNX_PACKAGE_VERSION="1.0.0-rc1-update1"
DNX_VERSION="1.0.0-rc1-16231"
if [ "$OSNAME" == "osx" ]; then
DNX_FLAVOR="dnx-coreclr-darwin-x64"
elif [ "$OSNAME" == "ubuntu" ]; then
DNX_FLAVOR="dnx-coreclr-linux-x64"
elif [ "$OSNAME" == "centos" ]; then
# No support dnx on redhat yet.
# using patched dnx
DNX_FEED="https://dotnetcli.blob.core.windows.net/dotnet/redhat_dnx"
DNX_PACKAGE_VERSION="1.0.0-rc2-15000"
DNX_VERSION="1.0.0-rc2-15000"
DNX_FLAVOR="dnx-coreclr-redhat-x64"
else
error "unknown OS: $OSNAME" 1>&2
exit 1
fi
DNX_URL="$DNX_FEED/$DNX_FLAVOR.$DNX_PACKAGE_VERSION.nupkg"
say "Preparing to install DNX to $DNX_DIR"
say "Requested Version: $DNX_VERSION"
if [ -e "$DNX_ROOT/dnx" ] ; then
dnxOut=`$DNX_ROOT/dnx --version | grep '^ Version: ' | awk '{ print $2; }'`
say "Local Version: $dnxOut"
if [ $dnxOut = $DNX_VERSION ] ; then
say "You already have the requested version."
doInstall=false
fi
else
say "Local Version: Not Installed"
fi
if [ $doInstall = true ] ; then
rm -rf $DNX_DIR
mkdir -p $DNX_DIR
curl -o $DNX_DIR/dnx.zip $DNX_URL --silent
unzip -qq $DNX_DIR/dnx.zip -d $DNX_DIR
chmod a+x $DNX_ROOT/dnu $DNX_ROOT/dnx
fi

View file

@ -0,0 +1,23 @@
#
# Copyright (c) .NET Foundation and contributors. All rights reserved.
# Licensed under the MIT license. See LICENSE file in the project root for full license information.
#
. $PSScriptRoot\..\common\_common.ps1
# Use a repo-local install directory (but not the artifacts directory because that gets cleaned a lot
if (!(Test-Path $env:DOTNET_INSTALL_DIR))
{
mkdir $env:DOTNET_INSTALL_DIR | Out-Null
}
# Install a stage 0
header "Installing dotnet stage 0"
_ "$RepoRoot\scripts\obtain\install.ps1"
# Put stage0 on the PATH
$env:PATH = "$env:DOTNET_INSTALL_DIR\cli\bin;$env:PATH"
# Download dnx to copy to stage2
header "Downloading DNX $DnxVersion"
_ "$RepoRoot\scripts\obtain\install-dnx.ps1"

31
scripts/obtain/install-tools.sh Executable file
View file

@ -0,0 +1,31 @@
#!/usr/bin/env bash
#
# Copyright (c) .NET Foundation and contributors. All rights reserved.
# Licensed under the MIT license. See LICENSE file in the project root for full license information.
#
set -e
SOURCE="${BASH_SOURCE[0]}"
while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
SOURCE="$(readlink "$SOURCE")"
[[ "$SOURCE" != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
done
DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
source "$DIR/../common/_common.sh"
# Use a repo-local install directory (but not the artifacts directory because that gets cleaned a lot
[ -d $DOTNET_INSTALL_DIR ] || mkdir -p $DOTNET_INSTALL_DIR
# Ensure the latest stage0 is installed
header "Installing dotnet stage 0"
$REPOROOT/scripts/obtain/install.sh
# Put the stage0 on the PATH
export PATH=$REPOROOT/artifacts/$RID/stage0/bin:$PATH
# Download DNX to copy into stage2
header "Downloading DNX $DNX_VERSION"
$REPOROOT/scripts/obtain/install-dnx.sh

View file

@ -0,0 +1,91 @@
#
# Copyright (c) .NET Foundation and contributors. All rights reserved.
# Licensed under the MIT license. See LICENSE file in the project root for full license information.
#
$ErrorActionPreference="Stop"
$ProgressPreference="SilentlyContinue"
$Feed="https://dotnetcli.blob.core.windows.net/dotnet"
$Channel="dev"
$DotNetFileName="dotnet-win-x64.latest.zip"
$DotNetUrl="$Feed/$Channel/Binaries/Latest"
function say($str)
{
Write-Host "dotnet_install: $str"
}
$InstallDir = $env:DOTNET_INSTALL_DIR
if (!$InstallDir) {
$InstallDir = "$env:LocalAppData\Microsoft\dotnet"
}
say "Preparing to install .NET Tools to $InstallDir"
# Check if we need to bother
$LocalFile = "$InstallDir\cli\.version"
if (Test-Path $LocalFile)
{
$LocalData = @(cat $LocalFile)
$LocalHash = $LocalData[0].Trim()
$LocalVersion = $LocalData[1].Trim()
if ($LocalVersion -and $LocalHash)
{
$RemoteResponse = Invoke-WebRequest -UseBasicParsing "$Feed/$Channel/dnvm/latest.win.version"
$RemoteData = @([Text.Encoding]::UTF8.GetString($RemoteResponse.Content).Split([char[]]@(), [StringSplitOptions]::RemoveEmptyEntries));
$RemoteHash = $RemoteData[0].Trim()
$RemoteVersion = $RemoteData[1].Trim()
if (!$RemoteVersion -or !$RemoteHash) {
throw "Invalid response from feed"
}
say "Latest version: $RemoteVersion"
say "Local Version: $LocalVersion"
if($LocalHash -eq $RemoteHash)
{
say "You already have the latest version"
exit 0
}
}
}
# Set up the install location
if (!(Test-Path $InstallDir)) {
mkdir $InstallDir | Out-Null
}
# De-powershell the path before passing to .NET APIs
$InstallDir = Convert-Path $InstallDir
say "Downloading $DotNetFileName from $DotNetUrl"
$resp = Invoke-WebRequest -UseBasicParsing "$DotNetUrl/$DotNetFileName" -OutFile "$InstallDir\$DotNetFileName"
say "Extracting zip"
# Create the destination
if (Test-Path "$InstallDir\cli_new") {
del -rec -for "$InstallDir\cli_new"
}
mkdir "$InstallDir\cli_new" | Out-Null
Add-Type -Assembly System.IO.Compression.FileSystem | Out-Null
[System.IO.Compression.ZipFile]::ExtractToDirectory("$InstallDir\$DotNetFileName", "$InstallDir\cli_new")
# Replace the old installation (if any)
if (Test-Path "$InstallDir\cli") {
del -rec -for "$InstallDir\cli"
}
mv "$InstallDir\cli_new" "$InstallDir\cli"
# Clean the zip
if (Test-Path "$InstallDir\$DotNetFileName") {
del -for "$InstallDir\$DotNetFileName"
}
say "The .NET Tools have been installed to $InstallDir\cli!"
# New layout
say "Add '$InstallDir\cli\bin' to your PATH to use dotnet"

267
scripts/obtain/install.sh Executable file
View file

@ -0,0 +1,267 @@
#!/usr/bin/env sh
#
# Copyright (c) .NET Foundation and contributors. All rights reserved.
# Licensed under the MIT license. See LICENSE file in the project root for full license information.
#
# Note: This script should be compatible with the dash shell used in Ubuntu. So avoid bashisms! See https://wiki.ubuntu.com/DashAsBinSh for more info
# This is a herestring Everything from the line AFTER the "read" until the line containing ONLY "EOF" is part of the string
# The quotes around "EOF" ensure that $ is not interpreted as a variable expansion. The indentation of the herestring must be
# kept exactly consistent
LINK_SCRIPT_CONTENT=$(cat <<-"EOF"
#!/usr/bin/env bash
SOURCE="${BASH_SOURCE[0]}"
while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
SOURCE="$(readlink "$SOURCE")"
[[ "$SOURCE" != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
done
DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
if [ -z "$DOTNET_TOOLS" ]; then
# We should be in $PREFIX/bin, so just get the directory above us.
PREFIX="$( cd -P "$DIR/.." && pwd)"
# The tools are in $PREFIX/share/dotnet/cli by default
DOTNET_TOOLS_DEFAULT=$PREFIX/share/dotnet/cli
# Check the default location
if [ -d "$DOTNET_TOOLS_DEFAULT" ]; then
export DOTNET_TOOLS=$DOTNET_TOOLS_DEFAULT
else
echo "error: the .NET tools installation appears to be corrupt!" 1>&2
echo "error: specifically, the tools could not be located in the $DOTNET_TOOLS_DEFAULT directory" 1>&2
exit 1
fi
fi
MY_NAME=$(basename ${BASH_SOURCE[0]})
MY_TARGET=$DOTNET_TOOLS/bin/$MY_NAME
if [ ! -e "$MY_TARGET" ]; then
echo "error: the tool $MY_TARGET cannot be found" 1>&2
exit 1
fi
if [ ! -x "$MY_TARGET" ]; then
echo "error: the tool $MY_TARGET is not executable" 1>&2
exit 1
fi
exec "$MY_TARGET" "$@"
EOF
)
#set default prefix (PREFIX is a fairly standard env-var, but we also want to allow the use the specific "DOTNET_INSTALL_DIR" one)
if [ ! -z "$DOTNET_INSTALL_DIR" ]; then
PREFIX=$DOTNET_INSTALL_DIR
elif [ -z "$PREFIX" ]; then
PREFIX=/usr/local
fi
#setup some colors to use. These need to work in fairly limited shells, like the Ubuntu Docker container where there are only 8 colors.
#See if stdout is a terminal
if [ -t 1 ]; then
# see if it supports colors
ncolors=$(tput colors)
if [ -n "$ncolors" ] && [ $ncolors -ge 8 ]; then
bold="$(tput bold)"
normal="$(tput sgr0)"
black="$(tput setaf 0)"
red="$(tput setaf 1)"
green="$(tput setaf 2)"
yellow="$(tput setaf 3)"
blue="$(tput setaf 4)"
magenta="$(tput setaf 5)"
cyan="$(tput setaf 6)"
white="$(tput setaf 7)"
fi
fi
#Standardise OS name to what is put into filenames of the tarballs.
current_os()
{
local uname=$(uname)
if [ "$uname" = "Darwin" ]; then
echo "osx"
else
# Detect Distro
if [ "$(cat /etc/*-release | grep -cim1 ubuntu)" -eq 1 ]; then
echo "ubuntu"
elif [ "$(cat /etc/*-release | grep -cim1 centos)" -eq 1 ]; then
echo "centos"
fi
fi
}
machine_has() {
type "$1" > /dev/null 2>&1
return $?
}
#Not 100% sure at the moment that these checks are enough. We might need to take version into account or do something
#more complicated. This seemed like a good beginning though as it should catch the default "clean" machine case and give
#people an appropriate hint.
check_pre_reqs() {
local os=$(current_os)
local _failing=false;
if [ "$DOTNET_INSTALL_SKIP_PREREQS" = "1" ]; then
return 0
fi
if [ "$(uname)" = "Linux" ]; then
[ -z "$(ldconfig -p | grep libunwind)" ] && say_err "Unable to locate libunwind. Install libunwind to continue" && _failing=true
[ -z "$(ldconfig -p | grep libssl)" ] && say_err "Unable to locate libssl. Install libssl to continue" && _failing=true
[ -z "$(ldconfig -p | grep libcurl)" ] && say_err "Unable to locate libcurl. Install libcurl to continue" && _failing=true
[ -z "$(ldconfig -p | grep libicu)" ] && say_err "Unable to locate libicu. Install libicu to continue" && _failing=true
[ -z "$(ldconfig -p | grep gettext)" ] && say_err "Unable to locate gettext. Install gettext to continue" && _failing=true
fi
if [ "$_failing" = true ]; then
return 1
fi
}
say_err() {
printf "%b\n" "${red}dotnet_install: Error: $1${normal}" >&2
}
say() {
printf "%b\n" "dotnet_install: $1"
}
make_link() {
local target_name=$1
local dest=$PREFIX/bin/$target_name
say "Linking $dest -> $PREFIX/share/dotnet/cli/$target_name"
if [ -e $dest ]; then
rm $dest
fi
[ -d "$PREFIX/bin" ] || mkdir -p $PREFIX/bin
echo "$LINK_SCRIPT_CONTENT" > $dest
# Make mode: rwxr-xr-x
chmod 755 $dest
}
install_dotnet()
{
if ! machine_has "curl"; then
printf "%b\n" "${red}curl is required to download dotnet. Install curl to proceed. ${normal}" >&2
return 1
fi
say "Preparing to install .NET Tools to $PREFIX"
if [ -e "$PREFIX/share/dotnet/cli/dotnet" ] && [ ! -w "$PREFIX/share/dotnet/cli/dotnet" ]; then
say_err "dotnet cli is already installed and not writeable. Use 'curl -sSL <url> | sudo sh' to force install."
say_err "If you have previously installed the cli using a package manager or installer then that is why it is write protected, and you need to run sudo to install the new version."
say_err "Alternatively, removing the '$PREFIX/share/dotnet' directory completely before running the script will also resolve the issue."
return 1
fi
if ! check_pre_reqs; then
say_err "Ending install due to missing pre-reqs"
return 1;
fi
local os=$(current_os)
local installLocation="$PREFIX/share/dotnet"
local dotnet_url="https://dotnetcli.blob.core.windows.net/dotnet/dev/Binaries/Latest"
local dotnet_filename="dotnet-$os-x64.latest.tar.gz"
if [ "$RELINK" = "0" ]; then
if [ "$FORCE" = "0" ]; then
# Check if we need to bother
local remoteData="$(curl -s https://dotnetcli.blob.core.windows.net/dotnet/dev/dnvm/latest.$os.version)"
[ $? != 0 ] && say_err "Unable to determine latest version." && return 1
local remoteVersion=$(IFS="\n" && echo $remoteData | tail -n 1)
local remoteHash=$(IFS="\n" && echo $remoteData | head -n 1)
local localVersion=$(tail -n 1 "$installLocation/cli/.version" 2>/dev/null)
[ -z $localVersion ] && localVersion='<none>'
local localHash=$(head -n 1 "$installLocation/cli/.version" 2>/dev/null)
say "Latest Version: $remoteVersion"
say "Local Version: $localVersion"
[ "$remoteHash" = "$localHash" ] && say "${green}You already have the latest version.${normal}" && return 0
fi
#This should noop if the directory already exists.
mkdir -p $installLocation
say "Downloading $dotnet_filename from $dotnet_url"
#Download file and check status code, error and return if we cannot download a cli tar.
local httpResult=$(curl -L -D - "$dotnet_url/$dotnet_filename" -o "$installLocation/$dotnet_filename" -# | grep "^HTTP/1.1" | head -n 1 | sed "s/HTTP.1.1 \([0-9]*\).*/\1/")
[ $httpResult -ne "302" ] && [ $httpResult -ne "200" ] && echo "${Red}HTTP Error $httpResult fetching the dotnet cli from $dotnet_url ${RCol}" && return 1
say "Extracting tarball"
#Any of these could fail for various reasons so we will check each one and end the script there if it fails.
rm -rf "$installLocation/cli_new"
mkdir "$installLocation/cli_new"
[ $? != 0 ] && say_err "failed to clean and create temporary cli directory to extract into" && return 1
tar -xzf "$installLocation/$dotnet_filename" -C "$installLocation/cli_new"
[ $? != 0 ] && say_err "failed to extract tar" && return 1
say "Moving new CLI into install location and symlinking"
rm -rf "$installLocation/cli"
[ $? != 0 ] && say_err "Failed to clean current dotnet install" && return 1
mv "$installLocation/cli_new" "$installLocation/cli"
elif [ ! -e "$installLocation/cli" ]; then
say_err "${red}cannot relink dotnet, it is not installed in $PREFIX!"
return 1
fi
for f in $(find "$installLocation/cli" -regex ".*/dotnet[a-z\-]*$")
do
local baseFile=$(basename $f)
make_link $baseFile
done
if [ -e "$installLocation/$dotnet_filename" ]; then
say "Cleaning $dotnet_filename"
if ! rm "$installLocation/$dotnet_filename"; then
say_err "Failed to delete tar after extracting."
return 1
fi
fi
}
FORCE=0
RELINK=0
while [ $# -ne 0 ]
do
if [ $1 = "-f" ] || [ $1 = "--force" ]; then
FORCE=1
elif [ $1 = "-r" ] || [ $1 = "--relink" ]; then
RELINK=1
elif [ $1 = "-?" ] || [ $1 = "-h" ] || [ $1 = "--help" ]; then
echo ".NET Tools Installer"
echo ""
echo "Usage:"
echo " $0 [-f]"
echo " $0 -r"
echo " $0 -h"
echo ""
echo "Options:"
echo " -f Force reinstallation even if you have the most recent version installed"
echo " -r Don't re-download, just recreate the links in $PREFIX/bin"
echo " -h Show this help message"
echo ""
echo "The PREFIX environment variable can be used to affect the root installation directory"
exit 0
fi
shift
done
install_dotnet