From 1db27b7ae32e662e342cbfd1bef8e5b2b59d8ac4 Mon Sep 17 00:00:00 2001 From: Sridhar Periyasamy Date: Fri, 18 Dec 2015 11:32:20 -0800 Subject: [PATCH] Enable building dotnet-CLI for centos. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I had to patch up a redhat dnx package which supports NETStandard.Library package. It is currently uploaded to dotnet-cli blob storage. This hack will no longer be required when we move to xplat nuget to do 'dotnet restore'. Apart from this there are three issues that are tracked for centos. - compile-native not yet supported - https://github.com/dotnet/cli/issues/453 - dnu restore crashes intermittently on centos. I need to investigate this a little bit more and file issues on dnx or coreclr. This will make our CI builds very flaky. - Dotnet restore does not restore native shims when using “centos.7-x64” - https://github.com/dotnet/corefx/issues/5066 --- netci.groovy | 5 +- scripts/_common.sh | 7 +-- scripts/compile.sh | 61 ++++++++++++++----- scripts/test/smoke-test.sh | 2 +- .../RuntimeIdentifier.cs | 43 ++++++++++++- 5 files changed, 93 insertions(+), 25 deletions(-) diff --git a/netci.groovy b/netci.groovy index 4dba3547e..d58fe924f 100644 --- a/netci.groovy +++ b/netci.groovy @@ -7,11 +7,12 @@ import jobs.generation.Utilities; def project = GithubProject -def osList = ['Ubuntu', 'OSX', 'Windows_NT'] +def osList = ['Ubuntu', 'OSX', 'Windows_NT', 'CentOS7.1'] def machineLabelMap = ['Ubuntu':'ubuntu-doc', 'OSX':'mac', - 'Windows_NT':'windows'] + 'Windows_NT':'windows', + 'CentOS7.1' : 'centos-71'] def static getBuildJobName(def configuration, def os) { return configuration.toLowerCase() + '_' + os.toLowerCase() diff --git a/scripts/_common.sh b/scripts/_common.sh index ff2a2bc24..1f34df645 100644 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -77,16 +77,14 @@ if [ -z "$RID" ]; then if [ "$UNAME" == "Darwin" ]; then export OSNAME=osx export RID=osx.10.10-x64 - export DNX_FLAVOR="dnx-coreclr-darwin-x64" elif [ "$UNAME" == "Linux" ]; then # Detect Distro if [ "$(cat /etc/*-release | grep -cim1 ubuntu)" -eq 1 ]; then export OSNAME=ubuntu export RID=ubuntu.14.04-x64 - export DNX_FLAVOR="dnx-coreclr-linux-x64" elif [ "$(cat /etc/*-release | grep -cim1 centos)" -eq 1 ]; then export OSNAME=centos - export RID=centos.7-x64 + export RID=centos.7.1-x64 else error "unknown Linux Distro" 1>&2 fi @@ -96,11 +94,8 @@ if [ -z "$RID" ]; then fi fi -export DNX_VERSION="1.0.0-rc1-update1" - export REPOROOT=$(cd $COMMONDIR/.. && pwd) export OUTPUT_ROOT=$REPOROOT/artifacts/$RID -export DNX_DIR=$OUTPUT_ROOT/dnx export STAGE1_DIR=$OUTPUT_ROOT/stage1 export STAGE2_DIR=$OUTPUT_ROOT/stage2 export HOST_DIR=$OUTPUT_ROOT/corehost diff --git a/scripts/compile.sh b/scripts/compile.sh index 1c2f0f88b..cc9d376b1 100755 --- a/scripts/compile.sh +++ b/scripts/compile.sh @@ -16,6 +16,37 @@ DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" source "$DIR/_common.sh" +getDnx() +{ + DNX_FEED="https://api.nuget.org/packages" + DNX_VERSION="1.0.0-rc1-update1" + DNX_DIR=$OUTPUT_ROOT/dnx + + if [ "$OSNAME" == "osx" ]; then + DNX_FLAVOR="dnx-coreclr-darwin-x64" + elif [ "$OSNAME" == "ubuntu" ]; then + DNX_FLAVOR="dnx-coreclr-linux-x64" + elif [ "$OSNAME" == "centos" ]; then + # No support dnx on redhat yet. + # using patched dnx + DNX_FEED="https://dotnetcli.blob.core.windows.net/dotnet/redhat_dnx" + DNX_VERSION="1.0.0-rc2-15000" + DNX_FLAVOR="dnx-coreclr-redhat-x64" + else + error "unknown OS: $OSNAME" 1>&2 + exit 1 + fi + + header "Downloading DNX $DNX_VERSION" + DNX_URL="$DNX_FEED/$DNX_FLAVOR.$DNX_VERSION.nupkg" + DNX_ROOT="$DNX_DIR/bin" + rm -rf $DNX_DIR + mkdir -p $DNX_DIR + curl -o $DNX_DIR/dnx.zip $DNX_URL --silent + unzip -qq $DNX_DIR/dnx.zip -d $DNX_DIR + chmod a+x $DNX_ROOT/dnu $DNX_ROOT/dnx +} + if ! type -p cmake >/dev/null; then error "cmake is required to build the native host 'corehost'" error "OS X w/Homebrew: 'brew install cmake'" @@ -26,14 +57,8 @@ fi [ -z "$CONFIGURATION" ] && export CONFIGURATION=Debug # Download DNX to copy into stage2 -header "Downloading DNX $DNX_VERSION" -DNX_URL="https://api.nuget.org/packages/$DNX_FLAVOR.$DNX_VERSION.nupkg" -DNX_ROOT="$DNX_DIR/bin" -rm -rf $DNX_DIR -mkdir -p $DNX_DIR -curl -o $DNX_DIR/dnx.zip $DNX_URL --silent -unzip -qq $DNX_DIR/dnx.zip -d $DNX_DIR -chmod a+x $DNX_ROOT/dnu $DNX_ROOT/dnx +getDnx + # Ensure the latest stage0 is installed $DIR/install.sh @@ -103,9 +128,13 @@ cp -R $DNX_ROOT $STAGE2_DIR/bin/dnx cp $DIR/dotnet-restore.sh $STAGE2_DIR/bin/dotnet-restore chmod a+x $STAGE2_DIR/bin/dotnet-restore -# Copy in AppDeps -header "Acquiring Native App Dependencies" -DOTNET_HOME=$STAGE2_DIR DOTNET_TOOLS=$STAGE2_DIR $REPOROOT/scripts/build/build_appdeps.sh "$STAGE2_DIR/bin" +# No compile native support in centos yet +# https://github.com/dotnet/cli/issues/453 +if [ "$OSNAME" != "centos" ]; then + # Copy in AppDeps + header "Acquiring Native App Dependencies" + DOTNET_HOME=$STAGE2_DIR DOTNET_TOOLS=$STAGE2_DIR $REPOROOT/scripts/build/build_appdeps.sh "$STAGE2_DIR/bin" +fi # Stamp the output with the commit metadata COMMIT_ID=$(git rev-parse HEAD) @@ -115,9 +144,13 @@ echo $COMMIT_ID > $STAGE2_DIR/.commit header "Testing stage2 ..." DOTNET_HOME=$STAGE2_DIR DOTNET_TOOLS=$STAGE2_DIR $DIR/test/smoke-test.sh -# E2E test on the output -header "Testing stage2 End to End ..." -DOTNET_HOME=$STAGE2_DIR DOTNET_TOOLS=$STAGE2_DIR $DIR/test/e2e-test.sh +# Skipping E2E tests for centos +# tracked by issue - https://github.com/dotnet/corefx/issues/5066 +if [ "$OSNAME" != "centos" ]; then + # E2E test on the output + header "Testing stage2 End to End ..." + DOTNET_HOME=$STAGE2_DIR DOTNET_TOOLS=$STAGE2_DIR $DIR/test/e2e-test.sh +fi # Run Validation for Project.json dependencies dotnet publish "$REPOROOT/tools/MultiProjectValidator" -o "$STAGE2_DIR/../tools" diff --git a/scripts/test/smoke-test.sh b/scripts/test/smoke-test.sh index 91f35b9d5..8625475aa 100755 --- a/scripts/test/smoke-test.sh +++ b/scripts/test/smoke-test.sh @@ -18,7 +18,7 @@ REPOROOT="$( cd -P "$DIR/../.." && pwd )" source "$DIR/../_common.sh" rm "$REPOROOT/test/TestApp/project.lock.json" -dotnet restore "$REPOROOT/test/TestApp" --runtime "osx.10.10-x64" --runtime "ubuntu.14.04-x64" --runtime "win7-x64" +dotnet restore "$REPOROOT/test/TestApp" --runtime "$RID" dotnet compile "$REPOROOT/test/TestApp" --output "$REPOROOT/artifacts/$RID/smoketest" # set -e will abort if the exit code of this is non-zero diff --git a/src/Microsoft.DotNet.ProjectModel/RuntimeIdentifier.cs b/src/Microsoft.DotNet.ProjectModel/RuntimeIdentifier.cs index 7e0d0f200..43e250083 100644 --- a/src/Microsoft.DotNet.ProjectModel/RuntimeIdentifier.cs +++ b/src/Microsoft.DotNet.ProjectModel/RuntimeIdentifier.cs @@ -1,6 +1,8 @@ // Copyright (c) .NET Foundation and contributors. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +using System; +using System.IO; using System.Runtime.InteropServices; namespace Microsoft.DotNet.ProjectModel @@ -18,13 +20,50 @@ namespace Microsoft.DotNet.ProjectModel } else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) { - return "ubuntu.14.04-x64"; + if(IsCentOS()) + { + return "centos.7.1-x64"; + } + else if(IsUbuntu()) + { + return "ubuntu.14.04-x64"; + } + else + { + // unknown distro. Lets fail fast + throw new InvalidOperationException("Current linux distro is not supported."); + } } else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) { return "osx.10.10-x64"; } - return "unknown"; + + throw new InvalidOperationException("Current operating system is not supported."); + } + + private static bool IsCentOS() + { + return IsLinuxDistro("centos"); + } + + private static bool IsUbuntu() + { + return IsLinuxDistro("ubuntu"); + } + + private static bool IsLinuxDistro(string distro) + { + // HACK - A file which can be found in most linux distros + // Did not test in non-en distros + const string OSIDFILE = "/etc/os-release"; + + if(!File.Exists(OSIDFILE)) + { + return false; + } + + return File.ReadAllText(OSIDFILE).ToLower().Contains(distro); } } }