mirrors/dotnet-installer
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

Use one build definition to produce all Linux distro specific packages (#7615)

Browse source 
By using `./build.sh /t:AllLiuxDistrosNativeInstaller`

Make a sandbox folder after compile step and copy compiled artifact to it. Since the content will be Linux generic, use package step with Docker in different distros.
After finishing all the distro specific packaging, upload all of them at once

Publish to debian only support all distro

Improve perf of AllLiuxDistrosNativeInstaller, by copying more cache in to sandbox

There will be a retry if package command failed
This commit is contained in:
William Lee 2017-09-14 13:33:46 -07:00 committed by GitHub
parent fac7481b09
commit 221ab05346
15 changed files with 194 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
Directory.Build.props
View file

@ -21,7 +21,9 @@ tools\TestAssetsDependencies\TestAssetsDependencies.csproj
<NuGetPackagesDir>$(NUGET_PACKAGES)</NuGetPackagesDir>
<NuGetPackagesDir Condition=" '$(NuGetPackagesDir)' == '' ">$(RepoRoot)/.nuget/packages</NuGetPackagesDir>
<CLIBuildDll>$(RepoRoot)/build_projects/dotnet-cli-build/bin/dotnet-cli-build.dll</CLIBuildDll>
<RelativeCLIBuildBinaries>build_projects/dotnet-cli-build/bin</RelativeCLIBuildBinaries>
<RelativeCLIBuildDllName>dotnet-cli-build.dll</RelativeCLIBuildDllName>
<CLIBuildDll>$(RepoRoot)/$(RelativeCLIBuildBinaries)/$(RelativeCLIBuildDllName)</CLIBuildDll>
<DisableImplicitFrameworkReferences>true</DisableImplicitFrameworkReferences>
<DisableImplicitPackageTargetFallback>true</DisableImplicitPackageTargetFallback>

1
build.proj
View file

@ -50,6 +50,7 @@
<Import Project="build/BuildInfo.targets" />
<Import Project="build/Prepare.targets" />
<Import Project="build/Compile.targets" />
<Import Project="build/BuildAndPublishAllLinuxDistrosNativeInstallers.targets" />
<Import Project="build/Package.targets" />
<Import Project="build/Test.targets" />
<Import Project="build/Publish.targets" />

3
build/Branding.props
View file

@ -31,6 +31,7 @@
<ArtifactNameWithVersionSdkDebug>$(ArtifactNameSdkDebug)-$(SdkVersion)-$(ProductMonikerRid)</ArtifactNameWithVersionSdkDebug>
<ArtifactNameWithVersionCombinedHostHostFxrFrameworkSdk>$(ArtifactNameCombinedHostHostFxrFrameworkSdk)-$(SdkVersion)-$(ProductMonikerRid)</ArtifactNameWithVersionCombinedHostHostFxrFrameworkSdk>
<ArtifactNameWithVersionSdkLanguagePack>$(ArtifactNameSdkLanguagePack)-$(SdkVersion)-$(ProductMonikerRid)</ArtifactNameWithVersionSdkLanguagePack>
<DistroSpecificArtifactNameWithVersionCombinedHostHostFxrFrameworkSdk>$(ArtifactNameCombinedHostHostFxrFrameworkSdk)-$(SdkVersion)-$(HostMonikerRid)</DistroSpecificArtifactNameWithVersionCombinedHostHostFxrFrameworkSdk>
<DistroSpecificArtifactNameWithVersionCombinedHostHostFxrFrameworkSdkWithoutHostMonikerRid>$(ArtifactNameCombinedHostHostFxrFrameworkSdk)-$(SdkVersion)-</DistroSpecificArtifactNameWithVersionCombinedHostHostFxrFrameworkSdkWithoutHostMonikerRid>
<DistroSpecificArtifactNameWithVersionCombinedHostHostFxrFrameworkSdk>$(DistroSpecificArtifactNameWithVersionCombinedHostHostFxrFrameworkSdkWithoutHostMonikerRid)$(HostMonikerRid)</DistroSpecificArtifactNameWithVersionCombinedHostHostFxrFrameworkSdk>
</PropertyGroup>
</Project>

28
build/BuildAndPublishAllLinuxDistrosNativeInstallers.targets Normal file
View file

@ -0,0 +1,28 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="14.0" DefaultTargets="Layout" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="$(MSBuildThisFileDirectory)/LinuxDistrosNativeInstaller.props" />
<Target Name="BuildAndPublishAllLinuxDistrosNativeInstallers" DependsOnTargets="Prepare;Compile;RunAllSandBoxAndPackage;MultiPublish">
<PropertyGroup>
<IsBuildingAndPublishingAllLinuxDistrosNativeInstallers>true</IsBuildingAndPublishingAllLinuxDistrosNativeInstallers>
</PropertyGroup>
</Target>
<Target Name="RunAllSandBoxAndPackage" >
<MSBuild Projects="$(MSBuildThisFileDirectory)/SandBoxAndPackageInParallel.proj"
Targets="RunInSandBoxAndPackage"
Properties="RepoRoot=$(RepoRoot);
BuildConfiguration=$(BuildConfiguration);
InstallerOutputDirectory=$(InstallerOutputDirectory);
" />
</Target>
<Target Name="MultiPublish" DependsOnTargets="AddAllGeneratedInstaller;Publish">
</Target>
<Target Name="AddAllGeneratedInstaller">
<ItemGroup>
<GeneratedInstallers Include="$(InstallerOutputDirectory)/$(DistroSpecificArtifactNameWithVersionCombinedHostHostFxrFrameworkSdkWithoutHostMonikerRid)%(LinuxDistrosNeedNativeInstaller.DistroNameInInstaller).%(LinuxDistrosNeedNativeInstaller.InstallerExtension)" />
</ItemGroup>
</Target>
</Project>

3
build/InitRepo.props
View file

@ -1,6 +1,7 @@
<Project ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup>
<GeneratedPropsDir>$(RepoRoot)/bin/obj</GeneratedPropsDir>
<RelativeGeneratedPropsDir>bin/obj</RelativeGeneratedPropsDir>
<GeneratedPropsDir>$(RepoRoot)/$(RelativeGeneratedPropsDir)</GeneratedPropsDir>
<GitCommitInfoProps>$(GeneratedPropsDir)/GitCommitInfo.props</GitCommitInfoProps>
<HostInfoProps>$(GeneratedPropsDir)/HostInfo.props</HostInfoProps>
<BuildInfoProps>$(GeneratedPropsDir)/BuildInfo.props</BuildInfoProps>

28
build/LinuxDistrosNativeInstaller.props Normal file
View file

@ -0,0 +1,28 @@
<Project>
<ItemGroup>
<LinuxDistrosNeedNativeInstaller Include="ubuntu.16.04-x64">
<DockerFolder>ubuntu.16.04</DockerFolder>
<InstallerExtension>deb</InstallerExtension>
<RepositoryId>$(ubuntu1604x64RepositoryId)</RepositoryId>
<DistroNameInInstaller>ubuntu.16.04-x64</DistroNameInInstaller>
</LinuxDistrosNeedNativeInstaller>
<LinuxDistrosNeedNativeInstaller Include="ubuntu.14.04-x64">
<DockerFolder>ubuntu.14.04</DockerFolder>
<InstallerExtension>deb</InstallerExtension>
<RepositoryId>$(ubuntu1404x64RepositoryId)</RepositoryId>
<DistroNameInInstaller>ubuntu-x64</DistroNameInInstaller>
</LinuxDistrosNeedNativeInstaller>
<LinuxDistrosNeedNativeInstaller Include="debian.8-x64">
<DockerFolder>debian</DockerFolder>
<InstallerExtension>deb</InstallerExtension>
<RepositoryId>$(debian8x64RepositoryId)</RepositoryId>
<DistroNameInInstaller>debian-x64</DistroNameInInstaller>
</LinuxDistrosNeedNativeInstaller>
<LinuxDistrosNeedNativeInstaller Include="rhel.7-x64">
<DockerFolder>rhel</DockerFolder>
<InstallerExtension>rpm</InstallerExtension>
<RepositoryId>$(rhel7x64RepositoryId)</RepositoryId>
<DistroNameInInstaller>rhel-x64</DistroNameInInstaller>
</LinuxDistrosNeedNativeInstaller>
</ItemGroup>
</Project>

9
build/Package.targets
View file

@ -7,10 +7,11 @@
<Import Project="$(MSBuildThisFileDirectory)/package/Installer.MSI.targets" />
<Import Project="$(MSBuildThisFileDirectory)/package/Installer.PKG.targets" />
<Import Project="$(MSBuildThisFileDirectory)/package/Installer.RPM.targets" />
<Import Project="$(MSBuildThisFileDirectory)/package/CopySandBoxPackageOut.targets" />
<Target Name="GenerateInstallers"
Condition=" '$(SkipBuildingInstallers)' != 'true' "
DependsOnTargets="Init;Layout;GeneratePkgs;GenerateDebs;GenerateMsis;GenerateRpms" />
DependsOnTargets="Prepare;Layout;GeneratePkgs;GenerateDebs;GenerateMsis;GenerateRpms" />
<Target Name="Package"
DependsOnTargets="BuildDotnetCliBuildFramework;
@ -19,4 +20,8 @@
GenerateNugetPackages;
GenerateArchives;
GenerateInstallers" />
</Project>
<Target Name="GenerateInstallersAndCopyOutOfSandBox"
DependsOnTargets="SandBoxPrepare;Layout;GenerateDebs;GenerateRpms;CopySandBoxPackageOut" />
</Project>

11
build/Prepare.targets
View file

@ -13,6 +13,14 @@
CheckPrereqs;">
</Target>
<Target Name="SandBoxPrepare"
DependsOnTargets="PrintBuildInfo;
CheckPrereqs;
DownloadHostAndSharedFxArtifacts;
IsolateArtifactsFromDirectoryBuildFiles;
CreatePropsForNextStage">
</Target>
<Target Name="PrintBuildInfo">
<Message Text="Host info - Rid: $(HostRid), OSName: $(HostOSName)" Importance="High" />
<Message Text="Build info - Rid: $(Rid), Architecture: $(Architecture), OSName: $(OSName)" Importance="High" />
@ -31,7 +39,7 @@
<OverwriteExtractionDestination>%(_DownloadAndExtractItem.OverwriteDestination)</OverwriteExtractionDestination>
<OverwriteExtractionDestination Condition="'$(OverwriteExtractionDestination)' == ''">True</OverwriteExtractionDestination>
</PropertyGroup>
<DownloadFile Condition=" '@(_DownloadAndExtractItem)' != '' "
Uri="%(_DownloadAndExtractItem.Url)"
DestinationPath="%(_DownloadAndExtractItem.DownloadFileName)" />
@ -133,3 +141,4 @@
</Target>
</Project>

1
build/Publish.targets
View file

@ -11,7 +11,6 @@
<Target Name="Publish"
Condition=" '$(PUBLISH_TO_AZURE_BLOB)' != '' "
DependsOnTargets="Init;
Package;
EvaluateRuntimeCoherence;
PublishArtifacts;
FinishBuild" />

79
build/SandBoxAndPackage.proj Normal file
View file

@ -0,0 +1,79 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="14.0" DefaultTargets="Layout" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="$(RepoRoot)/Directory.Build.props" />
<PropertyGroup>
<SandBoxFolderName>bin/WorkingCopy</SandBoxFolderName>
<RelativeSandBoxPackageOutputFolder>$(SandBoxFolderName)/RelativeSandBoxPackageOutputFolder</RelativeSandBoxPackageOutputFolder>
<SandBoxRepoRoot>$(RepoRoot)/$(SandBoxFolderName)/$(LinuxDistrosNeedNativeInstaller)</SandBoxRepoRoot>
<NugetCacheFolderName>.nuget</NugetCacheFolderName>
<RelativeCompileOutputPath>bin/2</RelativeCompileOutputPath>
</PropertyGroup>
<ItemGroup>
<MirrorOfRepoRootForTools Include="$(RepoRoot)/**/*.*" Exclude="$(RepoRoot)/$(SandBoxFolderName)/**/*.*;$(RepoRoot)/nuget/**/*.*;$(RepoRoot)/bin/**/*.*" />
<ArtifactsToPackage Include="$(RepoRoot)/$(RelativeCompileOutputPath)/$(Rid)/**/*.*"/>
<LocalDirectoryBuildpropsToPreventFurtherLookUpForIt Include="$(RepoRoot)/bin/Directory.Build.props"/>
<ToolsCacheToSpeedUp Include="$(RepoRoot)/$(RelativeCLIBuildBinaries)/**/*.*"/>
<NugetCacheToSpeedUp Include="$(RepoRoot)/$(NugetCacheFolderName)/**/*.*" />
</ItemGroup>
<Target Name="SandBoxAndPackage" DependsOnTargets="PackageWithDocker;CopyPackageResult"/>
<Target Name="PrepareSandBox" >
<Copy
SourceFiles="@(MirrorOfRepoRootForTools)"
DestinationFiles="@(MirrorOfRepoRootForTools -> '$(SandBoxRepoRoot)/%(RecursiveDir)%(Filename)%(Extension)')"/>
<Exec Command="sh -c 'cd $(SandBoxRepoRoot) &amp;&amp; git clean -fxd'"/>
<Copy
SourceFiles="@(ArtifactsToPackage)"
DestinationFiles="@(ArtifactsToPackage -> '$(SandBoxRepoRoot)/$(RelativeCompileOutputPath)/$(Rid)/%(RecursiveDir)%(Filename)%(Extension)')"/>
<Copy
SourceFiles="@(LocalDirectoryBuildpropsToPreventFurtherLookUpForIt)"
DestinationFiles="@(LocalDirectoryBuildpropsToPreventFurtherLookUpForIt -> '$(SandBoxRepoRoot)/$(RelativeCompileOutputPath)/$(Rid)/%(RecursiveDir)%(Filename)%(Extension)')"/>
<Copy
SourceFiles="@(ToolsCacheToSpeedUp)"
DestinationFiles="@(ToolsCacheToSpeedUp -> '$(SandBoxRepoRoot)/$(RelativeCLIBuildBinaries)/%(RecursiveDir)%(Filename)%(Extension)')"/>
<Copy
SourceFiles="@(NugetCacheToSpeedUp)"
DestinationFiles="@(NugetCacheToSpeedUp -> '$(SandBoxRepoRoot)/$(NugetCacheFolderName)/%(RecursiveDir)%(Filename)%(Extension)')"/>
</Target>
<Target Name="PackageWithDocker" DependsOnTargets="PrepareSandBox">
<PropertyGroup>
<CommandToInvokeBuildScriptInDockerToPackageInSandBox>$(SandBoxRepoRoot)/build.sh</CommandToInvokeBuildScriptInDockerToPackageInSandBox>
<CommandToInvokeBuildScriptInDockerToPackageInSandBox>$(CommandToInvokeBuildScriptInDockerToPackageInSandBox) --configuration $(BuildConfiguration)</CommandToInvokeBuildScriptInDockerToPackageInSandBox>
<CommandToInvokeBuildScriptInDockerToPackageInSandBox>$(CommandToInvokeBuildScriptInDockerToPackageInSandBox) --docker $(DockerFolder)</CommandToInvokeBuildScriptInDockerToPackageInSandBox>
<CommandToInvokeBuildScriptInDockerToPackageInSandBox>$(CommandToInvokeBuildScriptInDockerToPackageInSandBox) --skip-prereqs</CommandToInvokeBuildScriptInDockerToPackageInSandBox>
<CommandToInvokeBuildScriptInDockerToPackageInSandBox>$(CommandToInvokeBuildScriptInDockerToPackageInSandBox) /target:GenerateInstallersAndCopyOutOfSandBox</CommandToInvokeBuildScriptInDockerToPackageInSandBox>
<CommandToInvokeBuildScriptInDockerToPackageInSandBox>$(CommandToInvokeBuildScriptInDockerToPackageInSandBox) /p:RelativeSandBoxPackageOutputFolder=$(RelativeSandBoxPackageOutputFolder)</CommandToInvokeBuildScriptInDockerToPackageInSandBox>
<CommandExitCodeFirstTime>0</CommandExitCodeFirstTime>
<PipeStderrToStdoutToCatchFirstFailure>2&gt;&amp;1</PipeStderrToStdoutToCatchFirstFailure>
</PropertyGroup>
<Exec Command="$(CommandToInvokeBuildScriptInDockerToPackageInSandBox) $(PipeStderrToStdoutToCatchFirstFailure)" ContinueOnError="WarnAndContinue">
<Output TaskParameter="ExitCode" PropertyName="CommandExitCodeFirstTime"/>
</Exec>
<PropertyGroup>
<ShouldRetryDueToFirstTimeFailed Condition=" '$(CommandExitCodeFirstTime)' != '0' ">true</ShouldRetryDueToFirstTimeFailed>
</PropertyGroup>
<Exec Condition=" '$(ShouldRetryDueToFirstTimeFailed)' == 'true' " Command="$(CommandToInvokeBuildScriptInDockerToPackageInSandBox)" />
</Target >
<Target Name="CopyPackageResult">
<ItemGroup>
<SandboxPackageResultFiles Include="$(SandBoxRepoRoot)/$(RelativeSandBoxPackageOutputFolder)/**/*.*"/>
</ItemGroup>
<Copy
SourceFiles="@(SandboxPackageResultFiles)"
DestinationFolder="$(InstallerOutputDirectory)/%(RecursiveDir)"/>
</Target >
</Project>

17
build/SandBoxAndPackageInParallel.proj Normal file
View file

@ -0,0 +1,17 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="14.0" DefaultTargets="Layout" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="$(MSBuildThisFileDirectory)/LinuxDistrosNativeInstaller.props" />
<Target Name="RunInSandBoxAndPackage" >
<MSBuild Projects="$(MSBuildThisFileDirectory)/SandBoxAndPackage.proj"
Targets="SandBoxAndPackage"
Properties="RepoRoot=$(RepoRoot);
LinuxDistrosNeedNativeInstaller=%(LinuxDistrosNeedNativeInstaller.Identity);
DockerFolder=%(LinuxDistrosNeedNativeInstaller.DockerFolder);
BuildConfiguration=$(BuildConfiguration);
InstallerOutputDirectory=$(InstallerOutputDirectory);
BuildInParallel='true'
" />
</Target >
</Project>

5
build/VersionBadge.props
View file

@ -1,11 +1,8 @@
<Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup>
<VersionBadgeMoniker>$(OSName)_$(Architecture)</VersionBadgeMoniker>
<VersionBadgeMoniker Condition=" '$(HostRid)' == 'debian.8-x64' ">debian_8_x64</VersionBadgeMoniker>
<VersionBadgeMoniker Condition=" '$(HostRid)' == 'rhel.7-x64' ">rhel_7_x64</VersionBadgeMoniker>
<VersionBadgeMoniker Condition=" '$(HostRid)' == 'ubuntu.14.04-x64' ">ubuntu_14_04_x64</VersionBadgeMoniker>
<VersionBadgeMoniker Condition=" '$(HostRid)' == 'ubuntu.16.04-x64' ">ubuntu_16_04_x64</VersionBadgeMoniker>
<VersionBadgeMoniker Condition=" '$(IslinuxPortable)' == 'true' ">linux_x64</VersionBadgeMoniker>
<VersionBadgeMoniker Condition=" '$(IsBuildingAndPublishingAllLinuxDistrosNativeInstallers)' == 'true' ">all_linux_distros_native_installer</VersionBadgeMoniker>
<VersionBadge>$(BaseOutputDirectory)/$(VersionBadgeMoniker)_$(Configuration)_version_badge.svg</VersionBadge>
<CoherentBadge>$(BaseOutputDirectory)/$(VersionBadgeMoniker)_$(Configuration)_coherent_badge.svg</CoherentBadge>

9
build/package/CopySandBoxPackageOut.targets Normal file
View file

@ -0,0 +1,9 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Target Name="CopySandBoxPackageOut">
<Copy
SourceFiles="@(GeneratedInstallers)"
DestinationFolder="$(RepoRoot)/$(RelativeSandBoxPackageOutputFolder)"/>
</Target>
</Project>

13
build/publish/PublishDebian.targets
View file

@ -4,24 +4,19 @@
<SdkDebianPackageName>dotnet-sdk-$(SdkVersion)</SdkDebianPackageName>
</PropertyGroup>
<PropertyGroup>
<SdkDebianUploadUrl>$(DotnetBlobRootUrl)/$(Product)/$(FullNugetVersion)/$(DistroSpecificArtifactNameWithVersionCombinedHostHostFxrFrameworkSdk)-$(Architecture)$(InstallerExtension)</SdkDebianUploadUrl>
</PropertyGroup>
<Target Name="PublishDebFilesToDebianRepo"
DependsOnTargets="SetupDebProps;"
Condition=" '$(IsDebianBaseDistro)' == 'True' AND '$(SkipPublishToDebianRepo)' != 'true' AND '$(IslinuxPortable)' != 'true' " >
<Error Condition="'$(REPO_ID)' == ''" Text="REPO_ID must be set as a MsBuild Property variable for debian publishing." />
Condition=" '$(IsBuildingAndPublishingAllLinuxDistrosNativeInstallers)' == 'true' AND '$(SkipPublishToDebianRepo)' != 'true' AND '$(IslinuxPortable)' != 'true' " >
<Error Condition="'$(REPO_USER)' == ''" Text="REPO_USER must be set as a MsBuild Property variable for debian publishing." />
<Error Condition="'$(REPO_PASS)' == ''" Text="REPO_PASS must be set as a MsBuild Property variable for debian publishing." />
<Error Condition="'$(REPO_SERVER)' == ''" Text="REPO_SERVER must be set as a MsBuild Property variable for debian publishing." />
<UploadToLinuxPackageRepository
<UploadToLinuxPackageRepository Condition=" '%(LinuxDistrosNeedNativeInstaller.RepositoryId)' != '' "
Username="$(REPO_USER)"
Password='$(REPO_PASS)'
Server='$(REPO_SERVER)'
RepositoryId='$(REPO_ID)'
PathOfPackageToUpload='$(SdkInstallerFile)'
RepositoryId='%(LinuxDistrosNeedNativeInstaller.RepositoryId)'
PathOfPackageToUpload='$(InstallerOutputDirectory)/$(DistroSpecificArtifactNameWithVersionCombinedHostHostFxrFrameworkSdkWithoutHostMonikerRid)%(LinuxDistrosNeedNativeInstaller.DistroNameInInstaller).%(LinuxDistrosNeedNativeInstaller.InstallerExtension)'
PackageNameInLinuxPackageRepository='$(SdkDebianPackageName)'
PackageVersionInLinuxPackageRepository='$(NugetVersion)' />
</Target>

4
scripts/obtain/dotnet-install.sh vendored
View file

@ -656,7 +656,9 @@ install_dotnet() {
say_verbose "Zip path: $zip_path"
say "Downloading link: $download_link"
download "$download_link" $zip_path || download_failed=true
# Failures are expected in the non-legacy case. Do not output to stderr, since
# the output stderr are considered an error
download "$download_link" $zip_path 2>&1 || download_failed=true
# if the download fails, download the legacy_download_link
if [ "$download_failed" = true ] && [ "$valid_legacy_download_link" = true ]; then