From 5155aa61d7f230e054871702a1fc08fde20f96a7 Mon Sep 17 00:00:00 2001 From: Sridhar Periyasamy Date: Thu, 7 Jan 2016 18:50:00 -0800 Subject: [PATCH 1/4] Enable docker support for centos builds - Added an option '--buildindocker ' to build.sh - Fixed bug which caused packaging to be skipped by default. - Fixed bug which caused tarballs to be generated twice. - Fixed bug to propagate build params(like debug, nopackage) to docker build. --- build.sh | 18 ++++++---- scripts/build/build.sh | 10 ------ scripts/build/process-args.sh | 22 +++++++----- scripts/ci_build.sh | 18 +++++----- scripts/common/_common.sh | 1 - scripts/compile/compile-corehost.sh | 1 + scripts/docker/centos/Dockerfile | 49 ++++++++++++++++++++++++++ scripts/docker/dockerbuild.sh | 8 +++-- scripts/docker/{ => ubuntu}/Dockerfile | 0 scripts/package/package.sh | 29 +++------------ 10 files changed, 94 insertions(+), 62 deletions(-) create mode 100644 scripts/docker/centos/Dockerfile rename scripts/docker/{ => ubuntu}/Dockerfile (100%) diff --git a/build.sh b/build.sh index 6d83126c7..16aa85ce3 100755 --- a/build.sh +++ b/build.sh @@ -23,15 +23,19 @@ source "$REPOROOT/scripts/build/process-args.sh" # once that is fixed, we should remove the NOPACKAGE flag and do the full build either in # or out of docker. if [ ! -z "$BUILD_IN_DOCKER" ]; then - export BUILD_COMMAND="/opt/code/scripts/build/build.sh NOPACKAGE" + export BUILD_COMMAND=". /opt/code/scripts/build/process-args.sh $@ ; . /opt/code/scripts/build/build.sh" $REPOROOT/scripts/docker/dockerbuild.sh else - $REPOROOT/scripts/build/build.sh NOPACKAGE + $REPOROOT/scripts/build/build.sh fi -if [ ! -z "$PACKAGE_IN_DOCKER" ]; then - export BUILD_COMMAND="/opt/code/scripts/package/package-native.sh" - $REPOROOT/scripts/docker/dockerbuild.sh +if [ ! -z "$NOPACKAGE" ]; then + header "Skipping packaging" else - $REPOROOT/scripts/package/package.sh -fi \ No newline at end of file + if [ ! -z "$PACKAGE_IN_DOCKER" ]; then + export BUILD_COMMAND="$REPOROOT/scripts/package/package.sh" + $REPOROOT/scripts/docker/dockerbuild.sh + else + $REPOROOT/scripts/package/package.sh + fi +fi diff --git a/scripts/build/build.sh b/scripts/build/build.sh index e87abc6a8..128f80f56 100755 --- a/scripts/build/build.sh +++ b/scripts/build/build.sh @@ -43,13 +43,3 @@ DOTNET_HOME=$STAGE2_DIR DOTNET_TOOLS=$STAGE2_DIR $REPOROOT/scripts/test/runtests header "Validating Dependencies" $REPOROOT/scripts/test/validate-dependencies.sh - -header "Generating tarball" -$REPOROOT/scripts/package/package.sh - -if [ ! -z "$NOPACKAGE" ]; then - header "Generating Native Installer" - $REPOROOT/scripts/package/package-native.sh -else - header "Skipping packaging" -fi \ No newline at end of file diff --git a/scripts/build/process-args.sh b/scripts/build/process-args.sh index 3be6d2d7a..52aabae8b 100755 --- a/scripts/build/process-args.sh +++ b/scripts/build/process-args.sh @@ -4,25 +4,31 @@ # Licensed under the MIT license. See LICENSE file in the project root for full license information. # -for i in "$@" +params=("$@") + +for i in "${!params[@]}" do - lowerI="$(echo $i | awk '{print tolower($0)}')" + lowerI="$(echo ${params[$i]} | awk '{print tolower($0)}')" case $lowerI in - release) + "release" | "-release") export CONFIGURATION=Release ;; - debug) + "debug" | "-debug") export CONFIGURATION=Debug ;; - offline) + "offline" | "-offline") export OFFLINE=true ;; - nopackage) + "nopackage" | "-nopackage") export NOPACKAGE=true ;; - nocache) + "nocache" | "-nocache") export NOCACHE=--No-Cache ;; + "--buildindocker") + export BUILD_IN_DOCKER=true + export DOCKER_OS=${params[i+1]} + ;; *) esac -done \ No newline at end of file +done diff --git a/scripts/ci_build.sh b/scripts/ci_build.sh index d8a249c09..86eb958a3 100755 --- a/scripts/ci_build.sh +++ b/scripts/ci_build.sh @@ -28,18 +28,18 @@ if [ -z "$HOME" ]; then mkdir -p $HOME fi +# Set Docker Container name to be unique +container_name="" + +#Jenkins +[ ! -z "$BUILD_TAG" ] && container_name="$BUILD_TAG" +#VSO +[ ! -z "$BUILD_BUILDID" ] && container_name="$BUILD_BUILDID" + +export DOTNET_BUILD_CONTAINER_NAME="$container_name" if [[ "$OSNAME" == "ubuntu" ]]; then - # Set Docker Container name to be unique - container_name="" - - #Jenkins - [ ! -z "$BUILD_TAG" ] && container_name="$BUILD_TAG" - #VSO - [ ! -z "$BUILD_BUILDID" ] && container_name="$BUILD_BUILDID" - - export DOTNET_BUILD_CONTAINER_NAME="$container_name" export PACKAGE_IN_DOCKER="true" export NOCACHE="--no-cache" unset BUILD_IN_DOCKER diff --git a/scripts/common/_common.sh b/scripts/common/_common.sh index ad3e28eb1..ec3007283 100644 --- a/scripts/common/_common.sh +++ b/scripts/common/_common.sh @@ -11,7 +11,6 @@ while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symli done COMMONDIR="$( cd -P "$( dirname "$COMMONSOURCE" )" && pwd )" -source "$COMMONDIR/_clang.sh" source "$COMMONDIR/_prettyprint.sh" source "$COMMONDIR/_rid.sh" diff --git a/scripts/compile/compile-corehost.sh b/scripts/compile/compile-corehost.sh index 91926c579..195f5ee58 100755 --- a/scripts/compile/compile-corehost.sh +++ b/scripts/compile/compile-corehost.sh @@ -13,6 +13,7 @@ done DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" source "$DIR/../common/_common.sh" +source "$DIR/../common/_clang.sh" header "Building corehost" diff --git a/scripts/docker/centos/Dockerfile b/scripts/docker/centos/Dockerfile new file mode 100644 index 000000000..c44a40a41 --- /dev/null +++ b/scripts/docker/centos/Dockerfile @@ -0,0 +1,49 @@ +# +# Copyright (c) .NET Foundation and contributors. All rights reserved. +# Licensed under the MIT license. See LICENSE file in the project root for full license information. +# + +# Dockerfile that creates a container suitable to build dotnet-cli +FROM centos:7.1.1503 + +RUN yum -q -y install deltarpm +RUN yum -q -y install epel-release +# RUN yum -y update + +# This could become a "microsoft/coreclr" image, since it just installs the dependencies for CoreCLR (and stdlib) +# Install CoreCLR and CoreFx dependencies +RUN yum -q -y install unzip libunwind gettext libcurl-devel openssl-devel zlib libicu-devel + +# RUN apt-get update && \ +# apt-get -qqy install unzip curl libicu-dev libunwind8 gettext libssl-dev libcurl3-gnutls zlib1g liblttng-ust-dev lldb-3.6-dev lldb-3.6 + +# Install Build Prereqs +RUN yum -q -y install tar git cmake clang make + +# Use clang as c++ compiler +RUN update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++ 100 +RUN update-alternatives --set c++ /usr/bin/clang++ + +# Install azure cli. We need this to publish artifacts. +RUN yum -y install nodejs && \ + yum -y install npm && \ + npm install -g azure-cli + + +RUN yum -q -y install sudo + +# Setup User to match Host User, and give superuser permissions +ARG USER_ID=0 +RUN useradd -m code_executor -u ${USER_ID} -g root +RUN echo 'code_executor ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers + +# With the User Change, we need to change permssions on these directories +RUN chmod -R a+rwx /usr/local +RUN chmod -R a+rwx /home +RUN chmod -R 755 /usr/bin/sudo + +# Set user to the one we just created +USER ${USER_ID} + +# Set working directory +WORKDIR /opt/code diff --git a/scripts/docker/dockerbuild.sh b/scripts/docker/dockerbuild.sh index bcee32e24..b753d0610 100755 --- a/scripts/docker/dockerbuild.sh +++ b/scripts/docker/dockerbuild.sh @@ -21,11 +21,12 @@ cd $REPOROOT [ -z "$DOTNET_BUILD_CONTAINER_TAG" ] && DOTNET_BUILD_CONTAINER_TAG="dotnetcli-build" [ -z "$DOTNET_BUILD_CONTAINER_NAME" ] && DOTNET_BUILD_CONTAINER_NAME="dotnetcli-build-container" [ -z "$DOCKER_HOST_SHARE_DIR" ] && DOCKER_HOST_SHARE_DIR=$(pwd) -[ -z "$BUILD_COMMAND" ] && BUILD_COMMAND="/opt/code/build.sh" +[ -z "$DOCKER_OS" ] && DOCKER_OS=$OSNAME +[ -z "$BUILD_COMMAND" ] && BUILD_COMMAND="/opt/code/scripts/build/build.sh" # Build the docker container (will be fast if it is already built) header "Building Docker Container" -docker build --build-arg USER_ID=$(id -u) -t $DOTNET_BUILD_CONTAINER_TAG scripts/docker/ +docker build --build-arg USER_ID=$(id -u) -t $DOTNET_BUILD_CONTAINER_TAG scripts/docker/$DOCKER_OS # Run the build in the container header "Launching build in Docker Container" @@ -43,4 +44,5 @@ docker run -t --rm --sig-proxy=true \ -e REPO_USER \ -e REPO_PASS \ -e REPO_SERVER \ - $DOTNET_BUILD_CONTAINER_TAG $BUILD_COMMAND $1 + $DOTNET_BUILD_CONTAINER_TAG \ + bash -c "${BUILD_COMMAND}" diff --git a/scripts/docker/Dockerfile b/scripts/docker/ubuntu/Dockerfile similarity index 100% rename from scripts/docker/Dockerfile rename to scripts/docker/ubuntu/Dockerfile diff --git a/scripts/package/package.sh b/scripts/package/package.sh index 40d9f9074..385cdbd9a 100755 --- a/scripts/package/package.sh +++ b/scripts/package/package.sh @@ -16,30 +16,11 @@ source "$DIR/../common/_common.sh" if [ -z "$DOTNET_BUILD_VERSION" ]; then TIMESTAMP=$(date "+%Y%m%d%H%M%S") - DOTNET_BUILD_VERSION=0.0.1-alpha-t$TIMESTAMP + DOTNET_BUILD_VERSION=0.0.1-dev-t$TIMESTAMP fi -STAGE2_DIR=$REPOROOT/artifacts/$RID/stage2 +header "Generating tarball" +$DIR/package-dnvm.sh -if [ ! -d "$STAGE2_DIR" ]; then - error "missing stage2 output in $STAGE2_DIR" 1>&2 - exit -fi - -PACKAGE_DIR=$REPOROOT/artifacts/packages/dnvm -[ -d "$PACKAGE_DIR" ] || mkdir -p $PACKAGE_DIR - -PACKAGE_SHORT_NAME=dotnet-${OSNAME}-x64.${DOTNET_BUILD_VERSION} -PACKAGE_NAME=$PACKAGE_DIR/${PACKAGE_SHORT_NAME}.tar.gz - -cd $STAGE2_DIR - -header "Packaging $PACKAGE_SHORT_NAME" - -# Tar up the stage2 artifacts -# We need both "*" and ".version" to ensure we pick up that file -tar -czf $PACKAGE_NAME * .version - -info "Packaged stage2 to $PACKAGE_NAME" - -$REPOROOT/scripts/publish/publish.sh $PACKAGE_NAME +header "Generating Native Installer" +$DIR/package-native.sh From d73f15e4337d73c7eff54b609f84e7aa13a1b21c Mon Sep 17 00:00:00 2001 From: Sridhar Periyasamy Date: Thu, 7 Jan 2016 19:24:34 -0800 Subject: [PATCH 2/4] Fix the correct path to package.sh when packaging inside docker. --- build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sh b/build.sh index 16aa85ce3..36895885b 100755 --- a/build.sh +++ b/build.sh @@ -33,7 +33,7 @@ if [ ! -z "$NOPACKAGE" ]; then header "Skipping packaging" else if [ ! -z "$PACKAGE_IN_DOCKER" ]; then - export BUILD_COMMAND="$REPOROOT/scripts/package/package.sh" + export BUILD_COMMAND="/opt/code/scripts/package/package.sh" $REPOROOT/scripts/docker/dockerbuild.sh else $REPOROOT/scripts/package/package.sh From 6128373f0bf79d238db9a1f773c7a6dbd7d3c52a Mon Sep 17 00:00:00 2001 From: Sridhar Periyasamy Date: Fri, 8 Jan 2016 10:45:20 -0800 Subject: [PATCH 3/4] Separate building and packaging scripts for Windows Add 'nopackage' option similar to unix scripts --- scripts/build/build.ps1 | 17 ++++++------- scripts/package/package-zip.ps1 | 44 +++++++++++++++++++++++++++++++++ scripts/package/package.ps1 | 42 ++++++------------------------- 3 files changed, 59 insertions(+), 44 deletions(-) create mode 100644 scripts/package/package-zip.ps1 diff --git a/scripts/build/build.ps1 b/scripts/build/build.ps1 index bded1fa21..1a7558137 100644 --- a/scripts/build/build.ps1 +++ b/scripts/build/build.ps1 @@ -6,7 +6,8 @@ param( [string]$Configuration="Debug", [switch]$Offline, - [switch]$NoCache) + [switch]$NoCache, + [switch]$NoPackage) $ErrorActionPreference="Stop" @@ -39,11 +40,9 @@ _ "$RepoRoot\scripts\test\runtests.ps1" header "Validating Dependencies" _ "$RepoRoot\scripts\test\validate-dependencies.ps1" -header "Generating zip package" -_ "$RepoRoot\scripts\package\package.ps1" - -header "Generating dotnet MSI" -_ "$RepoRoot\packaging\windows\generatemsi.ps1" @("$Stage2Dir") - -header "Generating NuGet packages" -_ "$RepoRoot\packaging\nuget\package.ps1" @("$Stage2Dir\bin", "$VersionSuffix") \ No newline at end of file +if ($NoPackage){ + info "Skipping Packaging" +} +else { + _ "$RepoRoot\scripts\package\package.ps1" +} diff --git a/scripts/package/package-zip.ps1 b/scripts/package/package-zip.ps1 new file mode 100644 index 000000000..8381915c4 --- /dev/null +++ b/scripts/package/package-zip.ps1 @@ -0,0 +1,44 @@ +# +# Copyright (c) .NET Foundation and contributors. All rights reserved. +# Licensed under the MIT license. See LICENSE file in the project root for full license information. +# + +. "$PSScriptRoot\..\common\_common.ps1" + +if(!(Test-Path $PackageDir)) { + mkdir $PackageDir | Out-Null +} + +if(![string]::IsNullOrEmpty($env:DOTNET_BUILD_VERSION)) { + $PackageVersion = $env:DOTNET_BUILD_VERSION +} else { + $Timestamp = [DateTime]::Now.ToString("yyyyMMddHHmmss") + $PackageVersion = "0.0.1-dev-t$Timestamp" +} + +# Stamp the output with the commit metadata and version number +$Commit = git rev-parse HEAD + +$VersionContent = @" +$Commit +$PackageVersion +"@ + +$VersionContent | Out-File -Encoding UTF8 "$Stage2Dir\.version" + +$PackageName = Join-Path $PackageDir "dotnet-win-x64.$PackageVersion.zip" + +if (Test-Path $PackageName) +{ + del $PackageName +} + +Add-Type -Assembly System.IO.Compression.FileSystem +[System.IO.Compression.ZipFile]::CreateFromDirectory($Stage2Dir, $PackageName, "Optimal", $false) + +Write-Host "Packaged stage2 to $PackageName" + +$PublishScript = Join-Path $PSScriptRoot "..\publish\publish.ps1" +& $PublishScript -file $PackageName + +exit $LastExitCode diff --git a/scripts/package/package.ps1 b/scripts/package/package.ps1 index 0436afa14..c84588a3a 100644 --- a/scripts/package/package.ps1 +++ b/scripts/package/package.ps1 @@ -4,41 +4,13 @@ # . "$PSScriptRoot\..\common\_common.ps1" +. "$RepoRoot\scripts\build\generate-version.ps1" -if(!(Test-Path $PackageDir)) { - mkdir $PackageDir | Out-Null -} +header "Generating zip package" +_ "$RepoRoot\scripts\package\package-zip.ps1" -if(![string]::IsNullOrEmpty($env:DOTNET_BUILD_VERSION)) { - $PackageVersion = $env:DOTNET_BUILD_VERSION -} else { - $Timestamp = [DateTime]::Now.ToString("yyyyMMddHHmmss") - $PackageVersion = "0.0.1-alpha-t$Timestamp" -} +header "Generating dotnet MSI" +_ "$RepoRoot\packaging\windows\generatemsi.ps1" @("$Stage2Dir") -# Stamp the output with the commit metadata and version number -$Commit = git rev-parse HEAD - -$VersionContent = @" -$Commit -$PackageVersion -"@ - -$VersionContent | Out-File -Encoding UTF8 "$Stage2Dir\.version" - -$PackageName = Join-Path $PackageDir "dotnet-win-x64.$PackageVersion.zip" - -if (Test-Path $PackageName) -{ - del $PackageName -} - -Add-Type -Assembly System.IO.Compression.FileSystem -[System.IO.Compression.ZipFile]::CreateFromDirectory($Stage2Dir, $PackageName, "Optimal", $false) - -Write-Host "Packaged stage2 to $PackageName" - -$PublishScript = Join-Path $PSScriptRoot "..\publish\publish.ps1" -& $PublishScript -file $PackageName - -exit $LastExitCode +header "Generating NuGet packages" +_ "$RepoRoot\packaging\nuget\package.ps1" @("$Stage2Dir\bin", "$VersionSuffix") From 08f960b03d5d9c1be41d3208608a279fb3691c89 Mon Sep 17 00:00:00 2001 From: Sridhar Periyasamy Date: Fri, 8 Jan 2016 11:28:38 -0800 Subject: [PATCH 4/4] Address PR feedback. --- scripts/build/process-args.sh | 18 +++++++++++------- scripts/package/package.sh | 1 + 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/scripts/build/process-args.sh b/scripts/build/process-args.sh index 52aabae8b..f344d1b70 100755 --- a/scripts/build/process-args.sh +++ b/scripts/build/process-args.sh @@ -10,24 +10,28 @@ for i in "${!params[@]}" do lowerI="$(echo ${params[$i]} | awk '{print tolower($0)}')" case $lowerI in - "release" | "-release") + "release" | "--release") export CONFIGURATION=Release ;; - "debug" | "-debug") + "debug" | "--debug") export CONFIGURATION=Debug ;; - "offline" | "-offline") + "offline" | "--offline") export OFFLINE=true ;; - "nopackage" | "-nopackage") + "nopackage" | "--nopackage") export NOPACKAGE=true ;; - "nocache" | "-nocache") + "nocache" | "--nocache") export NOCACHE=--No-Cache ;; - "--buildindocker") + "--buildindocker-ubuntu") export BUILD_IN_DOCKER=true - export DOCKER_OS=${params[i+1]} + export DOCKER_OS=ubuntu + ;; + "--buildindocker-centos") + export BUILD_IN_DOCKER=true + export DOCKER_OS=centos ;; *) esac diff --git a/scripts/package/package.sh b/scripts/package/package.sh index 385cdbd9a..969a5831f 100755 --- a/scripts/package/package.sh +++ b/scripts/package/package.sh @@ -13,6 +13,7 @@ done DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" source "$DIR/../common/_common.sh" +source "$REPOROOT/scripts/build/generate-version.sh" if [ -z "$DOTNET_BUILD_VERSION" ]; then TIMESTAMP=$(date "+%Y%m%d%H%M%S")