mirrors/dotnet-installer
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

[ArPow] Add offline tarball build infra and CI (#11403)

Browse source 
* Add offline build to CI

* Add sourcelink patches to unblock offline aspnetcore build
This commit is contained in:
Michael Simons 2021-08-06 12:50:48 -05:00 • committed by GitHub
parent 96475b1d6a
commit 5268979bef
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 203 additions and 112 deletions
Download patch file Download diff file Expand all files Collapse all files

5
.vsts-ci.yml
View file

@ -344,6 +344,11 @@ stages:
- template: /src/SourceBuild/Arcade/eng/common/templates/job/source-build-create-tarball.yml
- template: /src/SourceBuild/Arcade/eng/common/templates/job/source-build-run-tarball-build.yml
parameters:
dependsOn: Source_Build_Create_Tarball
condition: eq(dependencies.Source_Build_Create_Tarball.outputs['Tarball_Build_Check._includeTarballBuild'], 'true')
# https://github.com/dotnet/core-sdk/issues/248
# - template: /eng/build.yml
# parameters:

2
eng/Versions.props
View file

@ -159,7 +159,7 @@
<MicrosoftBuildFrameworkVersion>15.7.179</MicrosoftBuildFrameworkVersion>
<MicrosoftBuildUtilitiesCoreVersion>15.7.179</MicrosoftBuildUtilitiesCoreVersion>
<PrivateSourceBuiltArtifactsPackageVersion>0.1.0-6.0.100-bootstrap.11</PrivateSourceBuiltArtifactsPackageVersion>
<PrivateSourceBuiltPrebuiltsPackageVersion>0.1.0-6.0.100-1</PrivateSourceBuiltPrebuiltsPackageVersion>
<PrivateSourceBuiltPrebuiltsPackageVersion>0.1.0-6.0.100-2</PrivateSourceBuiltPrebuiltsPackageVersion>
</PropertyGroup>
<!-- Workload manifest package versions -->
<PropertyGroup>

32
eng/source-build-tarball-build.yml
View file

@ -9,32 +9,6 @@ resources:
stages:
- stage: build
jobs:
- job: build
displayName: Build
pool:
vmImage: ubuntu-20.04
container: mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-32-20210222183556-031e7d2
workspace:
clean: all
variables:
_TarballDir: $(Build.StagingDirectory)/tarball
steps:
- checkout: none
- download: installer-build-resource
patterns: 'BlobArtifacts/dotnet-sdk-source*.tar.gz'
- script: |
set -x
mkdir -p "$(_TarballDir)"
tarballFilePath="$(PIPELINE.WORKSPACE)/installer-build-resource/BlobArtifacts/dotnet-sdk-source*.tar.gz"
eval tar -ozxf "$tarballFilePath" -C "$(_TarballDir)"
df -h
displayName: Extract Tarball
- template: /src/SourceBuild/Arcade/eng/common/templates/steps/source-build-run-tarball-build.yml
parameters:
sourceFolder: $(_TarballDir)
- template: /src/SourceBuild/Arcade/eng/common/templates/job/source-build-run-tarball-build.yml
parameters:
tarballResourceId: installer-build-resource

31
src/SourceBuild/Arcade/eng/common/templates/job/source-build-create-tarball.yml
View file

@ -2,10 +2,12 @@
jobs:
- job: Source_Build_Create_Tarball
container: mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-33-20210222183538-031e7d2
displayName: Source-Build Create Tarball
pool:
vmImage: ubuntu-20.04
container: mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-32-20210222183556-031e7d2
variables:
_BuildConfig: Release
workspace:
clean: all
@ -14,40 +16,31 @@ jobs:
set -x
df -h
buildConfig=Release
# Check if AzDO substitutes in a build config from a variable, and use it if so.
if [ '$(_BuildConfig)' != '$''(_BuildConfig)' ]; then
buildConfig='$(_BuildConfig)'
fi
officialBuildArgs=
if [ '${{ and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}' = 'True' ]; then
officialBuildArgs='/p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER)'
officialBuildArgs='/p:OfficialBuildId=$(BUILD.BUILDNUMBER)'
fi
./build.sh \
--ci \
--configuration $buildConfig \
--configuration $(_BuildConfig) \
--publish \
-bl \
$officialBuildArgs \
/p:DotNetPublishUsingPipelines=true \
/p:ArcadeBuildTarball=true
displayName: Create Tarball
- template: /src/SourceBuild/Arcade/eng/common/templates/steps/source-build-publish-logs.yml
parameters:
artifactQualifier: TarballCreation
- ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}:
# Only build the tarball if the PR touches source-build source
# Check if the tarball should be built now so the result can be used in the build tarball job condition.
# This prevents allocation of additional agents if the tarball build legs should be skipped.
# Only build the tarball if the PR touches source-build source.
- script: |
if curl "https://api.github.com/repos/dotnet/installer/pulls/$(System.PullRequest.PullRequestNumber)/files" | grep '"filename": "src/SourceBuild/*'
then
echo "##vso[task.setvariable variable=_includeTarballBuild]true"
echo "##vso[task.setvariable variable=_includeTarballBuild;isoutput=true]true"
fi
displayName: Include Tarball Build Check
- template: /src/SourceBuild/Arcade/eng/common/templates/steps/source-build-run-tarball-build.yml
parameters:
sourceFolder: $(Build.SourcesDirectory)/artifacts/tarball
condition: eq(variables['_includeTarballBuild'], 'true')
displayName: Tarball Build Check
name: Tarball_Build_Check

94
src/SourceBuild/Arcade/eng/common/templates/job/source-build-run-tarball-build.yml Normal file
View file

@ -0,0 +1,94 @@
# Builds a source-build tarball
parameters:
# Custom condition to apply to the job
condition: true
# Dependent jobs that must be completed before this job will run
dependsOn:
# The resource id of the tarball to download and build
tarballResourceId: current
jobs:
- job: Source_Build_Run_Tarball_Build
condition: ${{ parameters.condition }}
displayName: Source-Build Build Tarball
dependsOn: ${{ parameters.dependsOn }}
pool:
vmImage: ubuntu-20.04
strategy:
matrix:
Fedora33-Online:
_runOnline: true
_Container: mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-33-20210222183538-031e7d2
Fedora33-Offline:
_runOnline: false
_Container: mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-33-20210222183538-031e7d2
timeoutInMinutes: 120
variables:
_TarballDir: $(Build.StagingDirectory)/tarball
workspace:
clean: all
steps:
- checkout: none
- download: ${{ parameters.tarballResourceId }}
artifact: BlobArtifacts
patterns: '**/dotnet-sdk-source*.tar.gz'
displayName: Download Tarball
- script: |
set -x
resourceIdPathSegment=
if [ '${{ parameters.tarballResourceId }}' != 'current' ]; then
resourceIdPathSegment='${{ parameters.tarballResourceId }}/'
fi
mkdir -p "$(_TarballDir)"
tarballFilePath="$(PIPELINE.WORKSPACE)/${resourceIdPathSegment}BlobArtifacts/dotnet-sdk-source*.tar.gz"
eval tar -ozxf "$tarballFilePath" -C "$(_TarballDir)"
displayName: Extract Tarball
- script: |
set -x
docker run --rm -v $(_TarballDir):/tarball -w /tarball $(_Container) ./prep.sh
displayName: Prep Tarball
- script: |
set -x
df -h
networkArgs=
customBuildArgs=
if [ '$(_runOnline)' = 'true' ]; then
customBuildArgs='--online'
else
networkArgs='--network none'
fi
docker run --rm -v $(_TarballDir):/tarball -w /tarball ${networkArgs} $(_Container) ./build.sh ${customBuildArgs} -- /p:CleanWhileBuilding=true
displayName: Build Tarball
- task: CopyFiles@2
displayName: Gather Source Build Artifacts
inputs:
SourceFolder: $(_TarballDir)
Contents: |
artifacts/**/Private.SourceBuilt.Artifacts*.tar.gz
artifacts/prebuilt-report/Private.SourceBuilt.Prebuilts.*.tar.gz
TargetFolder: '$(Build.StagingDirectory)/artifacts'
CleanTargetFolder: true
- task: PublishPipelineArtifact@1
displayName: Publish Source Build Artifacts
inputs:
targetPath: '$(Build.StagingDirectory)/artifacts'
artifactName: $(Agent.JobName)_Artifacts_Attempt$(System.JobAttempt)
- template: /src/SourceBuild/Arcade/eng/common/templates/steps/source-build-publish-logs.yml
parameters:
sourceFolder: $(_TarballDir)

12
src/SourceBuild/Arcade/eng/common/templates/steps/source-build-publish-logs.yml
View file

@ -4,12 +4,6 @@ parameters:
# Folder to scan for source-build logs from
sourceFolder: $(Build.SourcesDirectory)
# The unique qualifier to append to the artifactName
artifactQualifier:
# Custom condition to apply to each step
condition: true
steps:
- task: CopyFiles@2
displayName: Prepare BuildLogs staging directory
@ -22,12 +16,12 @@ steps:
TargetFolder: '$(Build.StagingDirectory)/BuildLogs'
CleanTargetFolder: true
continueOnError: true
condition: and(${{ parameters.condition }}, succeededOrFailed())
condition: succeededOrFailed()
- task: PublishPipelineArtifact@1
displayName: Publish BuildLogs
inputs:
targetPath: '$(Build.StagingDirectory)/BuildLogs'
artifactName: BuildLogs_SourceBuild_${{ parameters.artifactQualifier }}_Attempt$(System.JobAttempt)
artifactName: $(Agent.JobName)_BuildLogs_Attempt$(System.JobAttempt)
continueOnError: true
condition: and(${{ parameters.condition }}, succeededOrFailed())
condition: succeededOrFailed()

51
src/SourceBuild/Arcade/eng/common/templates/steps/source-build-run-tarball-build.yml
View file

@ -1,51 +0,0 @@
# Builds the specified source-build tarball
parameters:
# Folder containing the tarball source to build
sourceFolder: $(Build.SourcesDirectory)
# Custom condition to apply to each step
condition: true
steps:
- script: |
set -x
./prep.sh
workingDirectory: ${{ parameters.sourceFolder }}
displayName: Prep Tarball
condition: and(${{ parameters.condition }}, succeeded())
- script: |
set -x
./build.sh \
--online \
-- /p:CleanWhileBuilding=true
workingDirectory: ${{ parameters.sourceFolder }}
displayName: Build Tarball
condition: and(${{ parameters.condition }}, succeeded())
- task: CopyFiles@2
displayName: Gather Source Build Artifacts
inputs:
SourceFolder: '${{ parameters.sourceFolder }}'
Contents: |
artifacts/**/Private.SourceBuilt.Artifacts*.tar.gz
artifacts/prebuilt-report/Private.SourceBuilt.Prebuilts.*.tar.gz
TargetFolder: '$(Build.StagingDirectory)/artifacts'
CleanTargetFolder: true
condition: and(${{ parameters.condition }}, succeeded())
- task: PublishPipelineArtifact@1
displayName: Publish Source Build Artifacts
inputs:
targetPath: '$(Build.StagingDirectory)/artifacts'
artifactName: Artifacts_SourceBuild_TarballBuild_Attempt$(System.JobAttempt)
condition: and(${{ parameters.condition }}, succeeded())
- template: /src/SourceBuild/Arcade/eng/common/templates/steps/source-build-publish-logs.yml
parameters:
sourceFolder: ${{ parameters.sourceFolder }}
artifactQualifier: TarballBuild
condition: and(${{ parameters.condition }}, succeededOrFailed())

2
src/SourceBuild/tarball/content/eng/Versions.props
View file

@ -23,6 +23,6 @@
<PropertyGroup>
<PrivateSourceBuiltArtifactsPackageVersion>0.1.0-6.0.100-bootstrap.11</PrivateSourceBuiltArtifactsPackageVersion>
<PrivateSourceBuiltPrebuiltsPackageVersionPrefix>0.1.0-6.0.100-</PrivateSourceBuiltPrebuiltsPackageVersionPrefix>
<PrivateSourceBuiltPrebuiltsPackageVersionSuffix>1</PrivateSourceBuiltPrebuiltsPackageVersionSuffix>
<PrivateSourceBuiltPrebuiltsPackageVersionSuffix>2</PrivateSourceBuiltPrebuiltsPackageVersionSuffix>
</PropertyGroup>
</Project>

46
src/SourceBuild/tarball/content/patches/aspnetcore/0004-Revert-sourcelink-submodule-workaround.patch Normal file
View file

@ -0,0 +1,46 @@
From 9b529c2f911e9a7ad25586d0d4a8db8bc5a7b2e5 Mon Sep 17 00:00:00 2001
From: Michael Simons <msimons@microsoft.com>
Date: Thu, 5 Aug 2021 22:44:48 +0000
Subject: [PATCH] Revert sourcelink submodule workaround
aspnetcore will fail the offline tarball build because of the sourcelink workaround being removed here.
To fix this, a sourcelink patch was added to address the issue.
---
eng/SourceBuild.props | 14 --------------
1 file changed, 14 deletions(-)
diff --git a/eng/SourceBuild.props b/eng/SourceBuild.props
index 3dbfe2ce7e..403b91f976 100644
--- a/eng/SourceBuild.props
+++ b/eng/SourceBuild.props
@@ -2,7 +2,6 @@
<PropertyGroup>
<GitHubRepositoryName>aspnetcore</GitHubRepositoryName>
<SourceBuildManagedOnly>true</SourceBuildManagedOnly>
- <CloneSubmodulesToInnerSourceBuildRepo>false</CloneSubmodulesToInnerSourceBuildRepo>
</PropertyGroup>
<Target Name="PrepareGlobalJsonForSourceBuild"
@@ -13,19 +12,6 @@
WorkingDirectory="$(InnerSourceBuildRepoRoot)" />
</Target>
- <!--
- Init submodules - temporarary workaround for https://github.com/dotnet/sourcelink/pull/653
- -->
- <Target Name="InitSubmodules"
- DependsOnTargets="PrepareInnerSourceBuildRepoRoot"
- BeforeTargets="RunInnerSourceBuildCommand">
-
- <Exec
- Command="git submodule update --init --recursive"
- WorkingDirectory="$(InnerSourceBuildRepoRoot)"
- EnvironmentVariables="@(InnerBuildEnv)" />
- </Target>
-
<!--
Remove inner source .globalconfig file as both the inner and outer config files get loaded and cause a conflict.
Leaving the inner will cause all conflicting settings to be ignored.
--
2.29.2

36
src/SourceBuild/tarball/content/patches/sourcelink/0003-Allow-using-.git-directory-instead-of-gitdir-redirec.patch Normal file
View file

@ -0,0 +1,36 @@
From cfb26098630fe3cd9648ebcbbe9ac59278b63242 Mon Sep 17 00:00:00 2001
From: Michael Simons <msimons@microsoft.com>
Date: Thu, 5 Aug 2021 19:51:19 +0000
Subject: [PATCH] Allow using .git directory instead of gitdir redirect in
submodules
---
.../GitDataReader/GitRepository.cs | 12 ++++++++++++
1 file changed, 12 insertions(+)
diff --git a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitRepository.cs b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitRepository.cs
index 9f1648f..c079197 100644
--- a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitRepository.cs
+++ b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitRepository.cs
@@ -209,6 +209,18 @@ public static GitRepository OpenRepository(GitRepositoryLocation location, GitEn
/// <returns>Null if the HEAD tip reference can't be resolved.</returns>
internal string? ReadSubmoduleHeadCommitSha(string submoduleWorkingDirectoryFullPath)
{
+ // Submodules don't usually have their own .git directories but this is still legal.
+ // This can occur with older versions of Git or other tools, or when a user clones one
+ // repo into another's source tree (but it was not yet registered as a submodule).
+ // See https://git-scm.com/docs/gitsubmodules#_forms for more details.
+ // Handle this case first since the other case throws.
+ var dotGitPath = Path.Combine(submoduleWorkingDirectoryFullPath, GitDirName);
+ if (IsGitDirectory(dotGitPath, out var directSubmoduleGitDirectory))
+ {
+ var submoduleGitDirResolver = new GitReferenceResolver(directSubmoduleGitDirectory, directSubmoduleGitDirectory);
+ return submoduleGitDirResolver.ResolveHeadReference();
+ }
+
var gitDirectory = ReadDotGitFile(Path.Combine(submoduleWorkingDirectoryFullPath, GitDirName));
if (!IsGitDirectory(gitDirectory, out var commonDirectory))
{
--
2.29.2

4
src/SourceBuild/tarball/content/prep.sh
View file

@ -70,13 +70,13 @@ fi
while read -r line; do
if [[ $line == *"Private.SourceBuilt.Artifacts"* ]]; then
if [ "$downloadArtifacts" == "true" ]; then
echo " Downloading source-built artifacts..."
echo " Downloading source-built artifacts from $line..."
(cd $SCRIPT_ROOT/packages/archive/ && curl -O $line)
fi
fi
if [[ $line == *"Private.SourceBuilt.Prebuilts"* ]]; then
if [ "$downloadPrebuilts" == "true" ]; then
echo " Downloading source-built prebuilts..."
echo " Downloading source-built prebuilts from $line..."
(cd $SCRIPT_ROOT/packages/archive/ && curl -O $line)
fi
fi