dotnet-installer/src/SourceBuild/Arcade/tools/SourceBuildArcadeTarball.targets

426 lines
18 KiB
XML

<!-- Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this file to you under the MIT license. -->
<Project Sdk="Microsoft.DotNet.Arcade.Sdk">
<Import Project="./BuildTasks.props" />
<Import Project="../tools/SourceBuild/SourceBuildArcadeTools.targets" Sdk="Microsoft.DotNet.Arcade.Sdk" />
<Import Project="$(RepoRoot)/src/redist/targets/Versions.targets" />
<Import Project="$(RepoRoot)/src/redist/targets/SetBuildDefaults.targets" />
<Import Project="$(RepoRoot)/src/redist/targets/FileExtensions.targets" />
<Import Project="$(RepoRoot)/src/redist/targets/GetRuntimeInformation.targets" />
<Import Project="$(RepoRoot)/src/redist/targets/GenerateLayout.targets" />
<UsingTask TaskName="Microsoft.DotNet.SourceBuild.Tasks.Tarball_ReadSourceBuildIntermediateNupkgDependencies" AssemblyFile="$(SourceBuildTasksAssembly)" />
<UsingTask TaskName="Microsoft.DotNet.SourceBuild.Tasks.Tarball_WriteSourceRepoProperties" AssemblyFile="$(SourceBuildTasksAssembly)" />
<PropertyGroup>
<ArtifactsDir>$(RepoRoot)artifacts/</ArtifactsDir>
<TarballDir>$(ArtifactsDir)tarball/</TarballDir>
<TarballRootDir>$([MSBuild]::EnsureTrailingSlash('$(TarballDir)'))</TarballRootDir>
<TarballSourceDir>$(TarballRootDir)src/</TarballSourceDir>
<TarballGitInfoDir>$(TarballRootDir)git-info/</TarballGitInfoDir>
<CloneVerbosity>quiet</CloneVerbosity> <!-- Support quiet and full -->
<ConvertInternalRepos Condition="'$(ConvertInternalRepos)' == '' and '$(VSS_NUGET_EXTERNAL_FEED_ENDPOINTS)' == '' and '$(AzDoPat)' == ''">true</ConvertInternalRepos>
</PropertyGroup>
<Target Name="CreateSourceTarball"
DependsOnTargets="
GenerateFullNuGetVersion;
GenerateVersionFile;
CreateTarballDir;
SetupSelfGithubInfo;
CloneRepoAndDependentsRecursive;
CleanClonedSource;
RestoreTextOnlyPackages;
CopyTextOnlyPackages;
CopyTarballContent;
">
<PropertyGroup>
<TarballFileDir>$([System.IO.Path]::GetDirectoryName('$(TarballFilePath)'))</TarballFileDir>
</PropertyGroup>
<Error Text="TarballFilePath not specified." Condition=" '$(TarballFilePath)' == '' " />
<MakeDir Directories="$(TarballFileDir)" Condition=" '$(TarballFileDir)' != '' " />
<Exec Command="tar --numeric-owner -zcf $(TarballFilePath) -C $(TarballDir) ." />
<Message Text="Tarball '$(TarballFilePath)' was successfully created from '$(TarballDir)'" Importance="High" />
</Target>
<Target Name="CreateTarballDir" >
<RemoveDir Directories='$(TarballRootDir)' Condition=" EXISTS('$(TarballRootDir)') " />
<Error Text="Tarball directory not specified. Pass /p:TarballDir=[new dir name] to specify a tarball directory." Condition=" '$(TarballDir)' == '' " />
<Error Text="Tarball directory already exists: $(TarballRootDir)" Condition=" '$(TarballRootDir)' != '' AND EXISTS('$(TarballRootDir)') " />
<MakeDir Directories="$(TarballRootDir)" Condition=" '$(TarballRootDir)' != '' " />
<MakeDir Directories="$(TarballSourceDir)" Condition=" '$(TarballSourceDir)' != '' " />
<MakeDir Directories="$(TarballGitInfoDir)" Condition=" '$(TarballGitInfoDir)' != '' " />
</Target>
<Target Name="SetupSelfGithubInfo" Condition=" '$(IsRootRepo)' != 'true' ">
<Exec
Command="cd $(RepoRoot);git config --get remote.origin.url"
ConsoleToMSBuild="true"
WorkingDirectory="$(RepoRoot)">
<Output TaskParameter="ConsoleOutput" ItemName="RootRepoUri" />
</Exec>
<Exec
Command="cd $(RepoRoot);git rev-parse HEAD"
ConsoleToMSBuild="true"
WorkingDirectory="$(RepoRoot)">
<Output TaskParameter="ConsoleOutput" ItemName="RootRepoCommitSha" />
</Exec>
<!-- Get commit count for installer repo only -->
<Exec
Command="cd $(RepoRoot);git rev-list --count HEAD"
ConsoleToMSBuild="true"
WorkingDirectory="$(RepoRoot)">
<Output TaskParameter="ConsoleOutput" ItemName="RootRepoCommitCount" />
</Exec>
<!-- This is hardcoding version for the root repo (installer), since there
isn't a Version.Details.xml file to read it from.
See https://github.com/dotnet/source-build/issues/2250 -->
<ItemGroup>
<SourceBuildRepos Include="$(GitHubRepositoryName)">
<Name>$(GitHubRepositoryName)</Name>
<Version>$(VersionPrefix)</Version>
<ExactVersion>$(VersionPrefix)</ExactVersion>
<Sha>@(RootRepoCommitSha)</Sha>
<Uri>@(RootRepoUri)</Uri>
<GitCommitCount>@(RootRepoCommitCount)</GitCommitCount>
<SourceBuildRepoName>$(GitHubRepositoryName)</SourceBuildRepoName>
<IsRootRepo>true</IsRootRepo>
</SourceBuildRepos>
</ItemGroup>
<Tarball_WriteSourceRepoProperties
SourceBuildMetadataDir="$(TarballGitInfoDir)"
Dependencies="@(SourceBuildRepos)" />
<PropertyGroup>
<SourceBuildRepoName>%(SourceBuildRepos.SourceBuildRepoName)</SourceBuildRepoName>
<RepoUri>%(SourceBuildRepos.Uri)</RepoUri>
<RepoSha>%(SourceBuildRepos.Sha)</RepoSha>
<IsRootRepo>%(SourceBuildRepos.IsRootRepo)</IsRootRepo>
</PropertyGroup>
<!-- Remove root repo from SourceBuildRepos so it doesn't try to clone twice -->
<ItemGroup>
<SourceBuildRepos Remove="@(SourceBuildRepos)" />
</ItemGroup>
</Target>
<Target Name="CloneRepoAndDependentsRecursive"
DependsOnTargets="GetSourceBuildIntermediateNupkgNameConvention">
<Message Text="--> Start Cloning Repo $(SourceBuildRepoName)" Importance="High" />
<PropertyGroup>
<SourceDir>$(SourceBuildRepoName)/</SourceDir>
<TarballRepoSourceDir>$(TarballSourceDir)$(SourceDir)</TarballRepoSourceDir>
<TarballRepoSourceEngDir>$(TarballSourceDir)$(SourceDir)eng/</TarballRepoSourceEngDir>
<TarballVersionDetailsFile>$(TarballRepoSourceEngDir)Version.Details.xml</TarballVersionDetailsFile>
<CloneParam Condition=" '$(CloneVerbosity)' == 'quiet' ">-q</CloneParam>
<OriginalRepoUri>$(RepoUri)</OriginalRepoUri>
<RepoUri Condition=" '$(AzDoPat)' != '' ">$(RepoUri.Replace('https://dev.azure.com', 'https://dn-bot:$(AzDoPat)@dev.azure.com'))</RepoUri>
</PropertyGroup>
<!-- Perform a local clone of the root repo, it's faster and doesn't require the RepoSha to be a tree. -->
<Exec
Command="git clone ./.git $(TarballRepoSourceDir)"
WorkingDirectory="$(RepoRoot)"
Condition="$(IsRootRepo) == 'true'" />
<!-- Because the root repo was locally cloned, the remote.origin.url needs to be updated from the local source to the real origin -->
<Exec
Command="git config --file $(TarballRepoSourceDir)/.git/config --unset remote.origin.url"
WorkingDirectory="$(RepoRoot)"
Condition="$(IsRootRepo) == 'true'" />
<Exec
Command="git config --file $(TarballRepoSourceDir)/.git/config --add remote.origin.url `git config --get remote.origin.url`"
WorkingDirectory="$(RepoRoot)"
Condition="$(IsRootRepo) == 'true'" />
<!-- Needed in cases where source-build is pinned to a version of the root repo. -->
<Exec
Command="git reset --hard $(RepoSha)"
WorkingDirectory="$(TarballRepoSourceDir)"
Condition="$(IsRootRepo) == 'true'" />
<Exec
Command="git init $(TarballRepoSourceDir)"
WorkingDirectory="$(RepoRoot)"
Condition="$(IsRootRepo) != 'true'" />
<Exec
Command="git remote add origin $(RepoUri)"
WorkingDirectory="$(TarballRepoSourceDir)"
Condition="$(IsRootRepo) != 'true'" />
<!-- Fetching a sha requires git 2.5.0 or newer -->
<Exec
Command="git fetch --depth 1 origin $(RepoSha)"
WorkingDirectory="$(TarballRepoSourceDir)"
Condition="$(IsRootRepo) != 'true'" />
<Exec
Command="git reset --hard FETCH_HEAD"
WorkingDirectory="$(TarballRepoSourceDir)"
Condition="$(IsRootRepo) != 'true'" />
<Exec
Command="git submodule update --init --recursive --depth 1"
WorkingDirectory="$(TarballRepoSourceDir)" />
<Exec
Command="git config --file $(TarballRepoSourceDir)/.git/config --unset remote.origin.url"
WorkingDirectory="$(RepoRoot)"/>
<Exec
Command="git config --file $(TarballRepoSourceDir)/.git/config --add remote.origin.url $(OriginalRepoUri)"
WorkingDirectory="$(RepoRoot)"/>
<!-- Remove the git objects folder to free up tarball space -->
<Exec
Command="rm -rf objects"
WorkingDirectory="$(TarballRepoSourceDir).git"
Condition="$(PreserveTarballGitFolders) != 'true'" />
<Message Text="--> Done Cloning Repo $(SourceBuildRepoName)" Importance="High" />
<!-- Apply source-build patches -->
<ItemGroup>
<SourceBuildPatchFile Include="$(RepoRoot)src/SourceBuild/tarball/patches/$(SourceBuildRepoName)/*.patch" />
</ItemGroup>
<Message
Text="--> Applying $(SourceBuildRepoName) Patches"
Importance="High"
Condition="'@(SourceBuildPatchFile)' != ''" />
<Exec
Command="git --work-tree=$(TarballRepoSourceDir) apply --ignore-whitespace --whitespace=nowarn &quot;%(SourceBuildPatchFile.FullPath)&quot;"
WorkingDirectory="$(TarballRepoSourceDir)"
Condition="'@(SourceBuildPatchFile)' != ''" />
<Message
Text="--> Done Applying $(SourceBuildRepoName) Patches"
Importance="High"
Condition="'@(SourceBuildPatchFile)' != ''" />
<Tarball_ReadSourceBuildIntermediateNupkgDependencies
VersionDetailsXmlFile="$([MSBuild]::NormalizePath($(TarballVersionDetailsFile)))"
SourceBuildIntermediateNupkgPrefix="$(SourceBuildIntermediateNupkgPrefix)"
SourceBuildIntermediateNupkgRid="$(SourceBuildIntermediateNupkgRid)"
ConvertInternalRepos="$(ConvertInternalRepos)">
<Output TaskParameter="Dependencies" ItemName="SourceBuildRepos" />
</Tarball_ReadSourceBuildIntermediateNupkgDependencies>
<!-- Remove repo if it has already be cloned at any sha. This results in
The commit sha for the cloned repo in the tarball being the one that
was first encountered. -->
<ItemGroup>
<SourceBuildRepos Remove="@(SourceBuildRepos)" Condition=" EXISTS('$(TarballGitInfoDir)%(SourceBuildRepoName).props')" />
</ItemGroup>
<Tarball_WriteSourceRepoProperties
SourceBuildMetadataDir="$(TarballGitInfoDir)"
Dependencies="@(SourceBuildRepos)" />
<Message Text="--> Dependencies for $(TarballVersionDetailsFile):" Importance="High" />
<Message Text="--> [@(SourceBuildRepos)]" Importance="High" />
<MSBuild Projects="$(MSBuildProjectFile)"
Condition=" '@(SourceBuildRepos)' != '' "
Targets="CloneRepoAndDependentsRecursive"
Properties="SourceBuildRepoName=%(SourceBuildRepos.SourceBuildRepoName);RepoSha=%(SourceBuildRepos.Sha);RepoUri=%(SourceBuildRepos.Uri)" />
</Target>
<Target Name="CleanClonedSource">
<ItemGroup>
<TarballSrcBinaryExtension Include="
.dll;
.Dll;
.exe;
.pdb;
.mdb;
.zip;
.nupkg" />
<TarballSrcBinaryToRemove Include="$(TarballSourceDir)**\*%(TarballSrcBinaryExtension.Identity)" />
<!-- Runtime coreclr and installer parts have empty placeholder PDB files. Keep them. -->
<TarballSrcBinaryToRemove Remove="
$(TarballSourceDir)runtime*\src\coreclr\.nuget\**\*%(TarballSrcBinaryExtension.Identity);
$(TarballSourceDir)runtime*\src\installer\pkg\**\*%(TarballSrcBinaryExtension.Identity)" />
</ItemGroup>
<Delete Files="@(TarballSrcBinaryToRemove)" />
<!-- We have some not-strictly-required files that are under non-open source licenses.
See https://github.com/dotnet/source-build/issues/2359.
Remove them. -->
<ItemGroup>
<TarballSrcNonOpenSourceFiles Include="$(TarballSourceDir)**\humanizer\samples\**\*.js" />
<TarballSrcNonOpenSourceFiles Include="$(TarballSourceDir)**\*nuget-client.*\**\EndToEnd\**\jquery-validation-unobtrusive\*.js" />
<TarballSrcNonOpenSourceFiles Include="$(TarballSourceDir)**\*nuget-client.*\**\EndToEnd\**\jquery-validation-unobtrusive\.bower.json" />
<TarballSrcNonOpenSourceFiles Include="$(TarballSourceDir)**\*aspnetcore.*\**\samples\**\jquery-validation-unobtrusive\.bower.json" />
<TarballSrcNonOpenSourceFiles Include="$(TarballSourceDir)**\*aspnetcore.*\**\samples\**\jquery-validation-unobtrusive\*.js" />
</ItemGroup>
<Message Importance="High" Text="Deleting files with questionable licenses: @(TarballSrcNonOpenSourceFiles, ' ')" />
<Delete Files="@(TarballSrcNonOpenSourceFiles)" />
</Target>
<Target Name="RestoreTextOnlyPackages">
<PropertyGroup>
<TextOnlyPackageRestoreDir>$(ArtifactsDir)text-only-packages/</TextOnlyPackageRestoreDir>
<TextOnlyPackageTarballDir>$(TarballRootDir)packages/text-only/</TextOnlyPackageTarballDir>
</PropertyGroup>
<Message Text="Restoring text only packages..." Importance="High" />
<MSBuild Projects="../Arcade/tools/TextOnlyPackages.csproj"
Properties="TargetPackagesPath=$(TextOnlyPackageRestoreDir)"
Targets="Restore" />
</Target>
<Target Name="CopyTarballContent">
<ItemGroup>
<TarballContent Include="$(RepoRoot)src/SourceBuild/tarball/content/**/*" />
<EngCommonContent Include="$(RepoRoot)eng/common/**/*" />
</ItemGroup>
<Copy
SourceFiles="$(TarballGitInfoDir)runtime.props"
DestinationFiles="$(TarballGitInfoDir)runtime-portable.props" />
<Copy
SourceFiles="@(TarballContent)"
DestinationFiles="@(TarballContent->'$(TarballRootDir)%(RecursiveDir)%(Filename)%(Extension)')" />
<Copy
SourceFiles="@(EngCommonContent)"
DestinationFiles="@(EngCommonContent->'$(TarballRootDir)eng/common/%(RecursiveDir)%(Filename)%(Extension)')" />
<PropertyGroup>
<ExternalTarballsDir>$(TarballRootDir)packages/archive/</ExternalTarballsDir>
<SourceBuiltArtifactsTarballName>Private.SourceBuilt.Artifacts</SourceBuiltArtifactsTarballName>
<SourceBuiltPrebuiltsTarballName>Private.SourceBuilt.Prebuilts</SourceBuiltPrebuiltsTarballName>
<SourceBuiltArtifactsTarballUrl>https://dotnetcli.azureedge.net/source-built-artifacts/assets/</SourceBuiltArtifactsTarballUrl>
<ArchiveArtifactsTextFileName>archiveArtifacts.txt</ArchiveArtifactsTextFileName>
<ArchiveArtifactsTextFile>$(ExternalTarballsDir)$(ArchiveArtifactsTextFileName)</ArchiveArtifactsTextFile>
</PropertyGroup>
<MakeDir Directories="$(ExternalTarballsDir)" />
<ItemGroup>
<ArtifactUrls Include="$(SourceBuiltArtifactsTarballUrl)$(SourceBuiltArtifactsTarballName).$(PrivateSourceBuiltArtifactsPackageVersion).tar.gz" />
<ArtifactUrls
Include="$(SourceBuiltArtifactsTarballUrl)$(SourceBuiltPrebuiltsTarballName).$(PrivateSourceBuiltPrebuiltsPackageVersion).tar.gz"
Condition="'$(PrivateSourceBuiltPrebuiltsPackageVersion)' != ''" />
</ItemGroup>
<WriteLinesToFile
File="$(ArchiveArtifactsTextFile)"
Lines="@(ArtifactUrls)"
Overwrite="true" />
<MakeDir Directories="$(TarballRootDir)packages/prebuilt" />
</Target>
<!--
Read directories in which text-only packages have been downloaded and copy content to tarball.
-->
<Target Name="CopyTextOnlyPackages" DependsOnTargets="RestoreTextOnlyPackages" >
<ReadLinesFromFile File="$(TextOnlyPackageRestoreDir)/TextOnlyPackageDirectories.txt" Condition="Exists('$(TextOnlyPackageRestoreDir)/TextOnlyPackageDirectories.txt')">
<Output TaskParameter="Lines" ItemName="TextOnlyPackageDirectories" />
</ReadLinesFromFile>
<MSBuild Projects="$(MSBuildProjectFile)"
Targets="CopySingleTextOnlyDirectory"
Properties="TextOnlyDirectory=%(TextOnlyPackageDirectories.Identity);TextOnlyPackageTarballDir=$(TextOnlyPackageTarballDir)" />
</Target>
<Target Name="CopySingleTextOnlyDirectory">
<PropertyGroup>
<DirectoryName>$([System.IO.Path]::GetFileName('$(TextOnlyDirectory)'))</DirectoryName>
</PropertyGroup>
<ItemGroup>
<TextOnlyPackageContent
Include="$(TextOnlyDirectory)/**/*"
Exclude="
$(TextOnlyDirectory)/**/.nupkg.metadata;
$(TextOnlyDirectory)/**/.signature.p7s;
$(TextOnlyDirectory)/**/*.nupkg;
$(TextOnlyDirectory)/**/*.nupkg.sha512;
" />
</ItemGroup>
<ItemGroup>
<!-- ttf, woff, woff2, eot are permissible font related content -->
<AllowedTextOnlyExtensions Include="
.-;
.bowerrc;
.config;
.cs;
.cshtml;
.csproj;
.css;
.db;
.editorconfig;
.env;
.env.development;
.eot;
.fs;
.fsproj;
.gitignore;
.gitkeep;
.html;
.ico;
.js;
.json;
.map;
.md;
.nuspec;
.otf;
.png;
.props;
.proto;
.razor;
.sln;
.svg;
.targets;
.ts;
.ttf;
.tsx;
.txt;
.vb;
.vbproj;
.woff;
.woff2;
browserslist;
browserslistrc;
LICENSE;" />
<UnsupportedTextOnlyPackageContent Include="@(TextOnlyPackageContent)" />
<UnsupportedTextOnlyPackageContent Remove="$(TextOnlyDirectory)/**/*$([System.String]::Copy('%(AllowedTextOnlyExtensions.Identity)').ToLowerInvariant())" />
<UnsupportedTextOnlyPackageContent Remove="$(TextOnlyDirectory)/**/*$([System.String]::Copy('%(AllowedTextOnlyExtensions.Identity)').ToUpperInvariant())" />
</ItemGroup>
<Error Text="Unsupported content found in text-only packages: @(UnsupportedTextOnlyPackageContent)" Condition=" '@(UnsupportedTextOnlyPackageContent)' != '' " />
<Copy
SourceFiles="@(TextOnlyPackageContent)"
DestinationFiles="@(TextOnlyPackageContent->'$(TextOnlyPackageTarballDir)$(DirectoryName)/%(RecursiveDir)%(Filename)%(Extension)')" />
</Target>
</Project>