Create tarball file and add tarball creation CI (#10972)

.vsts-ci.yml

@@ -335,11 +335,9 @@ stages:
             # Never run tests on arm64
             _TestArg: ''
 
-  - template: /eng/common/templates/job/source-build.yml
-    parameters:
-      platform:
-        name: 'Managed'
-        container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-3e800f1-20190501005343'
+  - template: /eng/common/templates/jobs/source-build.yml
+
+  - template: /src/SourceBuild/Arcade/eng/common/templates/job/source-build-tarball-creation.yml
 
   # https://github.com/dotnet/core-sdk/issues/248
   # - template: /eng/build.yml
@@ -361,6 +359,8 @@ stages:
           - Windows_NT
           - Linux
           - Darwin
+          - Source_Build_Managed
+          - Source_Build_Tarball_Creation
         publishUsingPipelines: true
         pool:
           vmImage: vs2017-win2016

src/SourceBuild/Arcade/eng/common/templates/job/source-build-tarball-creation.yml

@@ -0,0 +1,53 @@
+jobs:
+  - job: Source_Build_Tarball_Creation
+    displayName: Source-Build Tarball Creation
+    pool:
+      vmImage: ubuntu-20.04
+    container: mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-32-20210222183556-031e7d2
+    workspace:
+      clean: all
+
+    steps:
+    - script: |
+        set -x
+        df -h
+
+        buildConfig=Release
+        # Check if AzDO substitutes in a build config from a variable, and use it if so.
+        if [ '$(_BuildConfig)' != '$''(_BuildConfig)' ]; then
+          buildConfig='$(_BuildConfig)'
+        fi
+
+        officialBuildArgs=
+        if [ '${{ and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}' = 'True' ]; then
+          officialBuildArgs='/p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER)'
+        fi
+
+        ./build.sh \
+          --ci \
+          --configuration $buildConfig \
+          --publish \
+          -bl \
+          $officialBuildArgs \
+          /p:ArcadeBuildTarball=true
+      displayName: Create Tarball
+
+    - task: CopyFiles@2
+      displayName: Prepare BuildLogs staging directory
+      inputs:
+        SourceFolder: '$(Build.SourcesDirectory)'
+        Contents: |
+          **/*.log
+          **/*.binlog
+        TargetFolder: '$(Build.StagingDirectory)/BuildLogs'
+        CleanTargetFolder: true
+      continueOnError: true
+      condition: succeededOrFailed()
+
+    - task: PublishPipelineArtifact@1
+      displayName: Publish BuildLogs
+      inputs:
+        targetPath: '$(Build.StagingDirectory)/BuildLogs'
+        artifactName: BuildLogs_SourceBuild_TarballCreation_Attempt$(System.JobAttempt)
+      continueOnError: true
+      condition: succeededOrFailed()

src/SourceBuild/Arcade/tools/SourceBuildArcadeTarball.targets

@@ -6,16 +6,6 @@
   <UsingTask TaskName="Microsoft.DotNet.SourceBuild.Tasks.Tarball_ReadSourceBuildIntermediateNupkgDependencies" AssemblyFile="$(SourceBuildTasksAssembly)" />
   <UsingTask TaskName="Microsoft.DotNet.SourceBuild.Tasks.Tarball_WriteSourceRepoProperties" AssemblyFile="$(SourceBuildTasksAssembly)" />
 
-  <Target Name="CreateSourceTarball"
-    DependsOnTargets="
-      CreateTarballDir;
-      SetupSelfGithubInfo;
-      CloneRepoAndDependentsRecursive;
-      CopyTarballContent;
-      ">
-      <Message Text="Tarball successfully created: $(TarballRootDir)" Importance="High" />
-    </Target>
-
   <PropertyGroup>
     <TarballDir>$(RepoRoot)artifacts/tarball/</TarballDir>
     <TarballRootDir>$([MSBuild]::EnsureTrailingSlash('$(TarballDir)'))</TarballRootDir>
@@ -24,6 +14,26 @@
     <CloneVerbosity>quiet</CloneVerbosity> <!-- Support quiet and full -->
   </PropertyGroup>
 
+  <Target Name="CreateSourceTarball"
+    DependsOnTargets="
+      CreateTarballDir;
+      SetupSelfGithubInfo;
+      CloneRepoAndDependentsRecursive;
+      CopyTarballContent;
+      ">
+
+    <PropertyGroup>
+      <TarballFileDir>$([System.IO.Path]::GetDirectoryName('$(TarballFilePath)'))</TarballFileDir>
+    </PropertyGroup>
+
+    <Error Text="TarballFilePath not specified." Condition=" '$(TarballFilePath)' == '' " />
+
+    <MakeDir Directories="$(TarballFileDir)" Condition=" '$(TarballFileDir)' != '' " />
+    <Exec Command="tar --numeric-owner -zcf $(TarballFilePath) -C $(TarballDir) ." />
+
+    <Message Text="Tarball '$(TarballFilePath)' was successfully created from '$(TarballDir)'" Importance="High" />
+  </Target>
+
   <Target Name="CreateTarballDir" >
     <RemoveDir Directories='$(TarballRootDir)' Condition=" EXISTS('$(TarballRootDir)') " />
     <Error Text="Tarball directory not specified.  Pass /p:TarballDir=[new dir name] to specify a tarball directory." Condition=" '$(TarballDir)' == '' " />
@@ -61,6 +71,7 @@
         <Sha>@(RootRepoCommitSha)</Sha>
         <Uri>@(RootRepoUri)</Uri>
         <SourceBuildRepoName>$(GitHubRepositoryName)</SourceBuildRepoName>
+        <IsRootRepo>true</IsRootRepo>
       </SourceBuildRepos>
     </ItemGroup>
 
@@ -72,6 +83,7 @@
       <SourceBuildRepoName>%(SourceBuildRepos.SourceBuildRepoName)</SourceBuildRepoName>
       <RepoUri>%(SourceBuildRepos.Uri)</RepoUri>
       <RepoSha>%(SourceBuildRepos.Sha)</RepoSha>
+      <IsRootRepo>%(SourceBuildRepos.IsRootRepo)</IsRootRepo>
     </PropertyGroup>
 
     <!-- Remove root repo from SourceBuildRepos so it doesn't try to clone twice -->
@@ -82,7 +94,7 @@
 
   <Target Name="CloneRepoAndDependentsRecursive" 
     DependsOnTargets="GetSourceBuildIntermediateNupkgNameConvention">
-    
+
     <Message Text="--> Start Cloning Repo $(SourceBuildRepoName)" Importance="High" />
     <PropertyGroup>
       <SourceDir>$(SourceBuildRepoName).$(RepoSha)/</SourceDir>
@@ -92,16 +104,24 @@
       <CloneParam Condition=" '$(CloneVerbosity)' == 'quiet' ">-q</CloneParam>
     </PropertyGroup>
 
-    <Exec 
-      Command="git clone $(CloneParam) $(RepoUri) $(SourceDir)" 
-      WorkingDirectory="$(TarballSourceDir)" />
+    <!-- Perform a local clone of the root repo, it's faster and doesn't require the RepoSha to be a tree. -->
+    <Exec
+      Command="git clone ./.git $(TarballRepoSourceDir)"
+      WorkingDirectory="$(RepoRoot)"
+      Condition="$(IsRootRepo) == 'true'" />
 
-    <Exec 
-      Command="git checkout $(CloneParam) $(RepoSha)" 
-      WorkingDirectory="$(TarballRepoSourceDir)" />
+    <Exec
+      Command="git clone $(CloneParam) $(RepoUri) $(SourceDir)"
+      WorkingDirectory="$(TarballSourceDir)"
+      Condition="$(IsRootRepo) != 'true'" />
 
-    <Exec 
-      Command="git submodule update --init --recursive" 
+    <Exec
+      Command="git checkout $(CloneParam) $(RepoSha)"
+      WorkingDirectory="$(TarballRepoSourceDir)"
+      Condition="$(IsRootRepo) != 'true'" />
+
+    <Exec
+      Command="git submodule update --init --recursive"
       WorkingDirectory="$(TarballRepoSourceDir)" />
 
     <Message Text="--> Done Cloning Repo $(SourceBuildRepoName)" Importance="High" />

src/SourceBuild/tarball/BuildSourceBuildTarball.proj

@@ -1,6 +1,10 @@
 <Project>
   <Import Project="../Arcade/tools/SourceBuildArcadeTarball.targets" />
 
+  <PropertyGroup>
+    <TarballFilePath>$(ArtifactsShippingPackagesDir)dotnet-sdk-source-$(Version).tar.gz</TarballFilePath>
+  </PropertyGroup>
+
   <Target Name="Build" DependsOnTargets="CreateSourceTarball" />
 
-</Project>
+</Project>