From 2074d1bbf5e69d45d11f5c4a64b49ac2a4f34537 Mon Sep 17 00:00:00 2001 From: Bryan Thornbury Date: Wed, 21 Oct 2015 23:41:38 -0700 Subject: [PATCH] build and fetch results --- debian_config.json | 4 +- package_tool/setup/build_setup.sh | 7 ++- package_tool/setup/test_setup.sh | 2 +- scripts/bootstrap.sh | 6 ++ scripts/docker/Dockerfile | 13 ++-- scripts/dockerbuild.sh | 20 +++++- scripts/package-debian.sh | 13 ++-- scripts/windows_dockerbuild.sh | 100 ++++++++++++++++++++++++++++++ 8 files changed, 148 insertions(+), 17 deletions(-) create mode 100644 scripts/windows_dockerbuild.sh diff --git a/debian_config.json b/debian_config.json index 55a8c1d14..bd8f0bd1e 100644 --- a/debian_config.json +++ b/debian_config.json @@ -28,7 +28,7 @@ }, "debian_dependencies":{ - "libssl-dev" : {} + "libssl-dev" : {} }, "symlinks": { @@ -36,7 +36,7 @@ "dotnet-compile" : "usr/bin/dotnet-compile", "dotnet-compile-csc" : "usr/bin/dotnet-compile-csc", "dotnet-publish" : "usr/bin/dotnet-publish", - "dotnet-restore" : "usr/bin/dotnet-restore", + "dotnet-restore" : "usr/bin/dotnet-restore", "resgen" : "usr/bin/resgen" } } diff --git a/package_tool/setup/build_setup.sh b/package_tool/setup/build_setup.sh index 6d70859f6..739b98b5d 100755 --- a/package_tool/setup/build_setup.sh +++ b/package_tool/setup/build_setup.sh @@ -1,6 +1,11 @@ install_dependencies(){ + # Add LLdb 3.6 package source + echo "deb http://llvm.org/apt/trusty/ llvm-toolchain-trusty-3.6 main" | tee /etc/apt/sources.list.d/llvm.list + wget -O - http://llvm.org/apt/llvm-snapshot.gpg.key | apt-key add - + + #Install Deps apt-get update - apt-get install -y debhelper build-essential devscripts git + apt-get install -y debhelper build-essential devscripts git liblttng-ust-dev lldb-3.6-dev } setup(){ diff --git a/package_tool/setup/test_setup.sh b/package_tool/setup/test_setup.sh index cdc3b0f7b..4cd2ccc7c 100755 --- a/package_tool/setup/test_setup.sh +++ b/package_tool/setup/test_setup.sh @@ -1,6 +1,6 @@ install_dependencies(){ apt-get update - apt-get install -y debhelper build-essential devscripts git + apt-get install -y debhelper build-essential devscripts git liblttng-ust-dev clang-3.6 } install_bats(){ diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh index 1a2a9fc38..bf21bc8b2 100755 --- a/scripts/bootstrap.sh +++ b/scripts/bootstrap.sh @@ -88,6 +88,9 @@ dotnet publish --framework dnxcore50 --runtime $RID --output "$STAGE1_DIR" "$REP # Add stage1 to the path and use it to build stage2 export PATH=$STAGE1_DIR:$PATH +# Make corerun explicitly executable +chmod a+x $STAGE1_DIR/corerun + echo "Building stage2 dotnet using stage1 ..." dotnet publish --framework dnxcore50 --runtime $RID --output "$STAGE2_DIR" "$REPOROOT/src/Microsoft.DotNet.Cli" rc=$?; if [[ $rc != 0 ]]; then exit $rc; fi @@ -99,3 +102,6 @@ dotnet publish --framework dnxcore50 --runtime $RID --output "$STAGE2_DIR" "$REP rc=$?; if [[ $rc != 0 ]]; then exit $rc; fi dotnet publish --framework dnxcore50 --runtime $RID --output "$STAGE2_DIR" "$REPOROOT/src/Microsoft.DotNet.Tools.Resgen" rc=$?; if [[ $rc != 0 ]]; then exit $rc; fi + +# Make Stage 2 Folder Accessible +chmod -R a+r $STAGE2_DIR \ No newline at end of file diff --git a/scripts/docker/Dockerfile b/scripts/docker/Dockerfile index f8912388c..ce020e821 100644 --- a/scripts/docker/Dockerfile +++ b/scripts/docker/Dockerfile @@ -1,11 +1,14 @@ # Dockerfile that creates a container suitable to build dotnet-cli FROM microsoft/aspnet:1.0.0-beta8-coreclr -# Temporary: Install Mono, we use MCS to bootstrap -RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF 2>&1 && \ - echo "deb http://download.mono-project.com/repo/debian nightly main" > /etc/apt/sources.list.d/mono-nightly.list && \ - apt-get -qqy update && \ - apt-get -qqy install mono-complete +RUN apt-get update +RUN apt-get install -y wget + +# Install Package Build Prereqs +RUN echo "deb http://llvm.org/apt/trusty/ llvm-toolchain-trusty-3.6 main" | tee /etc/apt/sources.list.d/llvm.list +RUN wget -O - http://llvm.org/apt/llvm-snapshot.gpg.key | apt-key add - +RUN apt-get update +RUN apt-get install -y debhelper build-essential devscripts git liblttng-ust-dev lldb-3.6-dev # Set working directory WORKDIR /opt/code diff --git a/scripts/dockerbuild.sh b/scripts/dockerbuild.sh index 8392ffbe9..90f593592 100755 --- a/scripts/dockerbuild.sh +++ b/scripts/dockerbuild.sh @@ -10,10 +10,28 @@ DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" cd $DIR/.. +# Add an option to override the Docker Host +HOST_CODE_DIR=$(pwd) +if [[ "$1" != "" ]]; then + HOST_CODE_DIR=$1 +fi + +# Add an option to override the Script to Run +BUILD_SCRIPT=/opt/code/build.sh +if [[ "$2" != "" ]]; then + BUILD_SCRIPT=$2 +fi + [ -z "$DOTNET_BUILD_CONTAINER_TAG" ] && DOTNET_BUILD_CONTAINER_TAG="dotnetcli-build" +[ -z "$DOTNET_BUILD_CONTAINER_NAME" ] && DOTNET_BUILD_CONTAINER_NAME="dotnetcli-build-container" # Build the docker container (will be fast if it is already built) docker build -t $DOTNET_BUILD_CONTAINER_TAG scripts/docker/ # Run the build in the container -docker run -it --rm -v $(pwd):/opt/code -e DOTNET_BUILD_VERSION=$DOTNET_BUILD_VERSION $DOTNET_BUILD_CONTAINER_TAG /opt/code/build.sh +docker rm -f $DOTNET_BUILD_CONTAINER_NAME +docker run \ + -v $HOST_CODE_DIR:/opt/code \ + --name $DOTNET_BUILD_CONTAINER_NAME \ + -e DOTNET_BUILD_VERSION=$DOTNET_BUILD_VERSION \ + $DOTNET_BUILD_CONTAINER_TAG $BUILD_SCRIPT diff --git a/scripts/package-debian.sh b/scripts/package-debian.sh index f424ef6f2..595b27a3f 100644 --- a/scripts/package-debian.sh +++ b/scripts/package-debian.sh @@ -19,13 +19,12 @@ if [ "$UNAME" != "Linux" ]; then exit 1 fi +REPO_ROOT=$(readlink -f $DIR/..) -REPO_ROOT=$DIR - -OUTPUT_DIR="$REPO_ROOT/bin" -PACKAGE_LAYOUT_DIR="$OUTPUT_DIR/package_layout" -PACKAGE_OUTPUT_DIR="$OUTPUT_DIR/installers" -REPO_BINARIES_DIR="$REPO_ROOT/bin/$UNAME" +OUTPUT_DIR="$REPO_ROOT/artifacts" +PACKAGE_LAYOUT_DIR="$OUTPUT_DIR/deb_intermediate" +PACKAGE_OUTPUT_DIR="$OUTPUT_DIR/packages/debian" +REPO_BINARIES_DIR="$REPO_ROOT/artifacts/ubuntu.14.04-x64/stage2" execute(){ create_empty_debian_layout @@ -54,7 +53,7 @@ copy_files_to_debian_layout(){ create_debian_package(){ mkdir -p $PACKAGE_OUTPUT_DIR - $DIR/package_tool/package_tool $PACKAGE_LAYOUT_DIR $PACKAGE_OUTPUT_DIR + $REPO_ROOT/package_tool/package_tool $PACKAGE_LAYOUT_DIR $PACKAGE_OUTPUT_DIR } execute diff --git a/scripts/windows_dockerbuild.sh b/scripts/windows_dockerbuild.sh new file mode 100644 index 000000000..c3c573424 --- /dev/null +++ b/scripts/windows_dockerbuild.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +# +# Prerequisites: +# Git Bash (http://www.git-scm.com/downloads) +# Docker Toolbox (https://www.docker.com/docker-toolbox) +# Ensure Hyper-V is disabled! + +SOURCE="${BASH_SOURCE[0]}" +while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink + DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located +done +DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + + +_convert_path(){ + local path=$1 + path=$( echo "$path" | sed -r 's/[\/]+/\\/g') + path=${path#\\} + path=//$path + + echo $path +} + +# Bypass Msys path conversion +REPO_ROOT=$(readlink -f $DIR/..) +REPO_ROOT=$(_convert_path $REPO_ROOT) + +VM_NAME="dotnet" +VM_CODE_DIR="/home/docker/code" + +RESULTS_DIR="$REPO_ROOT/artifacts" + +execute(){ + check_prereqs + + echo "Setting up VM..." + create_or_start_vm #>> /dev/null 2>&1 + + echo "Copying code from Host to VM" + eval $(docker-machine env --shell bash $VM_NAME) + copy_code_to_vm + + echo "Running Build in Docker Container" + run_build + + echo "Copying Results from VM to Hosts..." + copy_results_from_vm +} + +check_prereqs(){ + if ! which docker; then + echo "Error: Install docker toolbox (https://www.docker.com/docker-toolbox)" + exit 1 + fi + + if ! which docker-machine; then + echo "Error: Install docker toolbox (https://www.docker.com/docker-toolbox)" + exit 1 + fi + + #TODO: check virtualbox? +} + +create_or_start_vm(){ + + if [[ $(docker-machine ls | grep $VM_NAME) == "" ]]; then + docker-machine create -d virtualbox $VM_NAME + else + docker-machine start $VM_NAME + fi + +} + +copy_code_to_vm(){ + echo "Copying Code to VM..." + docker-machine ssh $VM_NAME "sudo rm -rf $VM_CODE_DIR" + docker-machine scp -r $REPO_ROOT $VM_NAME:$VM_CODE_DIR >> /dev/null 2>&1 +} + + +run_build(){ + local host_code=$(_convert_path $VM_CODE_DIR) + $DIR/dockerbuild.sh $host_code //opt\\code\\build.sh +} + +# This will duplicate the entire repo + any side effects from +# the operations in the docker container +copy_results_from_vm(){ + T_RESULTS_DIR=$( echo "$RESULTS_DIR" | sed -r 's/[\\]+/\//g') + T_RESULTS_DIR=${T_RESULTS_DIR#/} + + mkdir $T_RESULTS_DIR + + docker-machine scp -r $VM_NAME:$VM_CODE_DIR/artifacts $REPO_ROOT >> /dev/null 2>&1 +} + +execute +