From 221ab0534645df7208cf936e20c7764b81e9fbee Mon Sep 17 00:00:00 2001 From: William Lee Date: Thu, 14 Sep 2017 13:33:46 -0700 Subject: [PATCH] Use one build definition to produce all Linux distro specific packages (#7615) By using `./build.sh /t:AllLiuxDistrosNativeInstaller` Make a sandbox folder after compile step and copy compiled artifact to it. Since the content will be Linux generic, use package step with Docker in different distros. After finishing all the distro specific packaging, upload all of them at once Publish to debian only support all distro Improve perf of AllLiuxDistrosNativeInstaller, by copying more cache in to sandbox There will be a retry if package command failed --- Directory.Build.props | 4 +- build.proj | 1 + build/Branding.props | 3 +- ...ishAllLinuxDistrosNativeInstallers.targets | 28 +++++++ build/InitRepo.props | 3 +- build/LinuxDistrosNativeInstaller.props | 28 +++++++ build/Package.targets | 9 ++- build/Prepare.targets | 11 ++- build/Publish.targets | 1 - build/SandBoxAndPackage.proj | 79 +++++++++++++++++++ build/SandBoxAndPackageInParallel.proj | 17 ++++ build/VersionBadge.props | 5 +- build/package/CopySandBoxPackageOut.targets | 9 +++ build/publish/PublishDebian.targets | 13 +-- scripts/obtain/dotnet-install.sh | 4 +- 15 files changed, 194 insertions(+), 21 deletions(-) create mode 100644 build/BuildAndPublishAllLinuxDistrosNativeInstallers.targets create mode 100644 build/LinuxDistrosNativeInstaller.props create mode 100644 build/SandBoxAndPackage.proj create mode 100644 build/SandBoxAndPackageInParallel.proj create mode 100644 build/package/CopySandBoxPackageOut.targets diff --git a/Directory.Build.props b/Directory.Build.props index bd5ba3fa1..cc9b1b47d 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -21,7 +21,9 @@ tools\TestAssetsDependencies\TestAssetsDependencies.csproj $(NUGET_PACKAGES) $(RepoRoot)/.nuget/packages - $(RepoRoot)/build_projects/dotnet-cli-build/bin/dotnet-cli-build.dll + build_projects/dotnet-cli-build/bin + dotnet-cli-build.dll + $(RepoRoot)/$(RelativeCLIBuildBinaries)/$(RelativeCLIBuildDllName) true true diff --git a/build.proj b/build.proj index 837cf150e..c7b973a7b 100644 --- a/build.proj +++ b/build.proj @@ -50,6 +50,7 @@ + diff --git a/build/Branding.props b/build/Branding.props index c972e2c1a..e2cc90d14 100644 --- a/build/Branding.props +++ b/build/Branding.props @@ -31,6 +31,7 @@ $(ArtifactNameSdkDebug)-$(SdkVersion)-$(ProductMonikerRid) $(ArtifactNameCombinedHostHostFxrFrameworkSdk)-$(SdkVersion)-$(ProductMonikerRid) $(ArtifactNameSdkLanguagePack)-$(SdkVersion)-$(ProductMonikerRid) - $(ArtifactNameCombinedHostHostFxrFrameworkSdk)-$(SdkVersion)-$(HostMonikerRid) + $(ArtifactNameCombinedHostHostFxrFrameworkSdk)-$(SdkVersion)- + $(DistroSpecificArtifactNameWithVersionCombinedHostHostFxrFrameworkSdkWithoutHostMonikerRid)$(HostMonikerRid) diff --git a/build/BuildAndPublishAllLinuxDistrosNativeInstallers.targets b/build/BuildAndPublishAllLinuxDistrosNativeInstallers.targets new file mode 100644 index 000000000..8eb8faccc --- /dev/null +++ b/build/BuildAndPublishAllLinuxDistrosNativeInstallers.targets @@ -0,0 +1,28 @@ + + + + + + + true + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/build/InitRepo.props b/build/InitRepo.props index f917b88c2..d5794af9b 100644 --- a/build/InitRepo.props +++ b/build/InitRepo.props @@ -1,6 +1,7 @@ - $(RepoRoot)/bin/obj + bin/obj + $(RepoRoot)/$(RelativeGeneratedPropsDir) $(GeneratedPropsDir)/GitCommitInfo.props $(GeneratedPropsDir)/HostInfo.props $(GeneratedPropsDir)/BuildInfo.props diff --git a/build/LinuxDistrosNativeInstaller.props b/build/LinuxDistrosNativeInstaller.props new file mode 100644 index 000000000..d10dde9bc --- /dev/null +++ b/build/LinuxDistrosNativeInstaller.props @@ -0,0 +1,28 @@ + + + + ubuntu.16.04 + deb + $(ubuntu1604x64RepositoryId) + ubuntu.16.04-x64 + + + ubuntu.14.04 + deb + $(ubuntu1404x64RepositoryId) + ubuntu-x64 + + + debian + deb + $(debian8x64RepositoryId) + debian-x64 + + + rhel + rpm + $(rhel7x64RepositoryId) + rhel-x64 + + + \ No newline at end of file diff --git a/build/Package.targets b/build/Package.targets index 8896bdb1f..eef514f56 100644 --- a/build/Package.targets +++ b/build/Package.targets @@ -7,10 +7,11 @@ + + DependsOnTargets="Prepare;Layout;GeneratePkgs;GenerateDebs;GenerateMsis;GenerateRpms" /> - + + + + \ No newline at end of file diff --git a/build/Prepare.targets b/build/Prepare.targets index 3b373ecfc..2783c8214 100644 --- a/build/Prepare.targets +++ b/build/Prepare.targets @@ -13,6 +13,14 @@ CheckPrereqs;"> + + + @@ -31,7 +39,7 @@ %(_DownloadAndExtractItem.OverwriteDestination) True - + @@ -133,3 +141,4 @@ + diff --git a/build/Publish.targets b/build/Publish.targets index 2fb513838..6bcab65ca 100644 --- a/build/Publish.targets +++ b/build/Publish.targets @@ -11,7 +11,6 @@ diff --git a/build/SandBoxAndPackage.proj b/build/SandBoxAndPackage.proj new file mode 100644 index 000000000..26b2b4b0f --- /dev/null +++ b/build/SandBoxAndPackage.proj @@ -0,0 +1,79 @@ + + + + + + bin/WorkingCopy + $(SandBoxFolderName)/RelativeSandBoxPackageOutputFolder + $(RepoRoot)/$(SandBoxFolderName)/$(LinuxDistrosNeedNativeInstaller) + .nuget + bin/2 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $(SandBoxRepoRoot)/build.sh + $(CommandToInvokeBuildScriptInDockerToPackageInSandBox) --configuration $(BuildConfiguration) + $(CommandToInvokeBuildScriptInDockerToPackageInSandBox) --docker $(DockerFolder) + $(CommandToInvokeBuildScriptInDockerToPackageInSandBox) --skip-prereqs + $(CommandToInvokeBuildScriptInDockerToPackageInSandBox) /target:GenerateInstallersAndCopyOutOfSandBox + $(CommandToInvokeBuildScriptInDockerToPackageInSandBox) /p:RelativeSandBoxPackageOutputFolder=$(RelativeSandBoxPackageOutputFolder) + 0 + 2>&1 + + + + + + + + true + + + + + + + + + + + + \ No newline at end of file diff --git a/build/SandBoxAndPackageInParallel.proj b/build/SandBoxAndPackageInParallel.proj new file mode 100644 index 000000000..5ec1930f5 --- /dev/null +++ b/build/SandBoxAndPackageInParallel.proj @@ -0,0 +1,17 @@ + + + + + + + + + diff --git a/build/VersionBadge.props b/build/VersionBadge.props index 3f84bd670..a317dafa2 100644 --- a/build/VersionBadge.props +++ b/build/VersionBadge.props @@ -1,11 +1,8 @@ $(OSName)_$(Architecture) - debian_8_x64 - rhel_7_x64 - ubuntu_14_04_x64 - ubuntu_16_04_x64 linux_x64 + all_linux_distros_native_installer $(BaseOutputDirectory)/$(VersionBadgeMoniker)_$(Configuration)_version_badge.svg $(BaseOutputDirectory)/$(VersionBadgeMoniker)_$(Configuration)_coherent_badge.svg diff --git a/build/package/CopySandBoxPackageOut.targets b/build/package/CopySandBoxPackageOut.targets new file mode 100644 index 000000000..a7bf6b524 --- /dev/null +++ b/build/package/CopySandBoxPackageOut.targets @@ -0,0 +1,9 @@ + + + + + + + diff --git a/build/publish/PublishDebian.targets b/build/publish/PublishDebian.targets index 6dd777180..801230fb0 100644 --- a/build/publish/PublishDebian.targets +++ b/build/publish/PublishDebian.targets @@ -4,24 +4,19 @@ dotnet-sdk-$(SdkVersion) - - $(DotnetBlobRootUrl)/$(Product)/$(FullNugetVersion)/$(DistroSpecificArtifactNameWithVersionCombinedHostHostFxrFrameworkSdk)-$(Architecture)$(InstallerExtension) - - - + Condition=" '$(IsBuildingAndPublishingAllLinuxDistrosNativeInstallers)' == 'true' AND '$(SkipPublishToDebianRepo)' != 'true' AND '$(IslinuxPortable)' != 'true' " > - diff --git a/scripts/obtain/dotnet-install.sh b/scripts/obtain/dotnet-install.sh index 77b7781c2..ae8f84be8 100755 --- a/scripts/obtain/dotnet-install.sh +++ b/scripts/obtain/dotnet-install.sh @@ -656,7 +656,9 @@ install_dotnet() { say_verbose "Zip path: $zip_path" say "Downloading link: $download_link" - download "$download_link" $zip_path || download_failed=true + # Failures are expected in the non-legacy case. Do not output to stderr, since + # the output stderr are considered an error + download "$download_link" $zip_path 2>&1 || download_failed=true # if the download fails, download the legacy_download_link if [ "$download_failed" = true ] && [ "$valid_legacy_download_link" = true ]; then