Remove source-build tarball creation infrastructure (#15114)

* Remove source-build tarball creation infrastructure

* Remove tarball logic to build text-only packages
This commit is contained in:
Michael Simons 2022-12-09 15:27:13 -06:00 committed by GitHub
parent 2824d95e89
commit 144a5f4328
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
16 changed files with 0 additions and 972 deletions

View file

@ -1,23 +0,0 @@
// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
// https://github.com/microsoft/vscode-dev-containers/tree/v0.238.0/containers/dotnet
{
"name": "Source-Build w/ Built Tarball",
"image": "mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-36",
// A completely built .NET source-tarball is >64 GB
"hostRequirements": {
"storage": "128gb"
},
// Configure tool-specific properties.
"customizations": {
// Configure properties specific to VS Code.
"vscode": {
// Add the IDs of extensions you want installed when the container is created.
"extensions": [
"ms-dotnettools.csharp",
"eamodio.gitlens"
]
}
},
// Use 'onCreateCommand' to run pre-build commands inside the codespace.
"onCreateCommand": "${containerWorkspaceFolder}/.devcontainer/source-build-scripts/buildTarball.sh"
}

View file

@ -1,12 +0,0 @@
#!/usr/bin/env bash
./.devcontainer/source-build-scripts/createTarball.sh
cd $(realpath ..)/dotnet-source/
./prep.sh
# GitHub Codespaces automatically sets RepositoryName, which conflicts with source-build scripts.
unset RepositoryName
./build.sh --online --clean-while-building || true

View file

@ -1,7 +0,0 @@
#!/usr/bin/env bash
# Always return exit code 0 so that we can use the codespaces prebuild to diagnose build errors.
./build.sh /p:ArcadeBuildTarball=true /p:TarballDir=$(realpath ..)/dotnet-source/ /p:PreserveTarballGitFolders=true || true
# Save the commit hash of the currently built repo, so developers know which version was built
git rev-parse HEAD > ./artifacts/prebuild.sha

View file

@ -1,20 +0,0 @@
// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
// https://github.com/microsoft/vscode-dev-containers/tree/v0.238.0/containers/dotnet
{
"name": "Source-Build w/ Tarball",
"image": "mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-36",
// A completely built .NET source tarball is >64 GB
"hostRequirements": {
"storage": "128gb"
},
"customizations": {
"vscode": {
"extensions": [
"ms-dotnettools.csharp",
"eamodio.gitlens"
]
}
},
// Use 'onCreateCommand' to run pre-build commands inside the codespace.
"onCreateCommand": "${containerWorkspaceFolder}/.devcontainer/scripts/createTarball.sh"
}

View file

@ -1,5 +0,0 @@
// Use this devcontainer if you don't need the full context of a source-build tarball
{
"name": "Source-Build",
"image": "mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-36"
}

View file

@ -1,13 +1,5 @@
<Project>
<Choose>
<When Condition=" '$(ArcadeBuildTarball)' == 'true' ">
<!-- Source build / tarball generation -->
<ItemGroup>
<ProjectToBuild Include="$(RepoRoot)src/SourceBuild/Arcade/src/SourceBuild.Tasks.csproj" BuildInParallel="false" />
<ProjectToBuild Include="$(RepoRoot)src/SourceBuild/tarball/BuildSourceBuildTarball.proj" BuildInParallel="false" />
</ItemGroup>
</When>
<When Condition=" '$(InitializeVMR)' == 'true' ">
<!-- VMR bootstrap -->
<ItemGroup>

View file

@ -1 +0,0 @@
The source and targets in the `src/SourceBuild/Arcade` directory are intended to move into the Arcade repo at some point. They are added here for ease of development while developing the tarball generation process. See https://github.com/dotnet/source-build/issues/2295

View file

@ -1,17 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFrameworks>net7.0</TargetFrameworks>
<EnableDefaultCompileItems>false</EnableDefaultCompileItems>
<RunAnalyzers>false</RunAnalyzers>
<Nullable>disable</Nullable>
</PropertyGroup>
<ItemGroup>
<Compile Include="*.cs" />
<PackageReference Include="Microsoft.Build.Framework" Version="$(MicrosoftBuildFrameworkVersion)" PrivateAssets="All" ExcludeAssets="Runtime" />
<PackageReference Include="Microsoft.Build.Utilities.Core" Version="$(MicrosoftBuildUtilitiesCoreVersion)" Publish="false" ExcludeAssets="runtime" />
<PackageReference Include="NuGet.Packaging" Version="5.8.0" />
</ItemGroup>
</Project>

View file

@ -1,180 +0,0 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
using Microsoft.Build.Framework;
using Microsoft.Build.Utilities;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Xml.Linq;
namespace Microsoft.DotNet.SourceBuild.Tasks
{
/// <summary>
/// Reads entries in a Version.Details.xml file to find intermediate nupkg dependencies. For
/// each dependency with a "SourceBuild" element, adds an item to the "Dependencies" output.
/// </summary>
public class Tarball_ReadSourceBuildIntermediateNupkgDependencies : Task
{
[Required]
public string VersionDetailsXmlFile { get; set; }
[Required]
public string SourceBuildIntermediateNupkgPrefix { get; set; }
/// <summary>
/// Convert any internal repo references to the public GitHub repos.
/// </summary>
public bool ConvertInternalRepos { get; set; }
/// <summary>
/// The intermediate nupkg RID to use if any RID-specific intermediate nupkgs are required.
/// If this parameter isn't specified, RID-specific intermediate nupkgs can't be used and
/// this task fails.
/// </summary>
public string SourceBuildIntermediateNupkgRid { get; set; }
/// <summary>
/// %(Identity): NuGet package ID.
/// %(Name): The Name of the dependency from Version.Details.xml.
/// %(ExactVersion): NuGet package version. This can be used to look up the restored package
/// contents in a package cache.
/// %(Version): NuGet package version, wrapped in "[version]" syntax for exact match.
/// %(Uri): The URI for the repo.
/// %(Sha): The commit Sha for the dependency.
/// %(SourceBuildRepoName): The repo name to use in source-build.
/// </summary>
[Output]
public ITaskItem[] Dependencies { get; set; }
public override bool Execute()
{
XElement root = XElement.Load(VersionDetailsXmlFile, LoadOptions.PreserveWhitespace);
XName CreateQualifiedName(string plainName)
{
return root.GetDefaultNamespace().GetName(plainName);
}
Dependencies = root
.Elements()
.Elements(CreateQualifiedName("Dependency"))
.Select(d =>
{
XElement sourceBuildElement = d.Element(CreateQualifiedName("SourceBuild"));
if (sourceBuildElement == null)
{
// Workaround for https://github.com/dotnet/source-build/issues/2481
sourceBuildElement = d.Element(CreateQualifiedName("SourceBuildTarball"));
if (sourceBuildElement == null)
{
// Ignore element: doesn't represent a source-build dependency.
return null;
}
}
string repoName = sourceBuildElement.Attribute("RepoName")?.Value;
if (string.IsNullOrEmpty(repoName))
{
Log.LogError($"Dependency SourceBuild RepoName null or empty in '{VersionDetailsXmlFile}' element {d}");
return null;
}
string dependencyName = d.Attribute("Name")?.Value ?? string.Empty;
if (string.IsNullOrEmpty(dependencyName))
{
// Log name missing as FYI, but this is not an error case for source-build.
Log.LogMessage($"Dependency Name null or empty in '{VersionDetailsXmlFile}' element {d}");
}
string dependencyVersion = d.Attribute("Version")?.Value;
string uri = d.Element(CreateQualifiedName("Uri"))?.Value;
if (ConvertInternalRepos)
{
uri = ConvertInternalRepo(uri);
}
string sha = d.Element(CreateQualifiedName("Sha"))?.Value;
string sourceBuildRepoName = sourceBuildElement.Attribute("RepoName")?.Value;
if (string.IsNullOrEmpty(dependencyVersion))
{
// We need a version to bring down an intermediate nupkg. Fail.
Log.LogError($"Dependency Version null or empty in '{VersionDetailsXmlFile}' element {d}");
return null;
}
string identity = SourceBuildIntermediateNupkgPrefix + repoName;
bool.TryParse(
sourceBuildElement.Attribute("ManagedOnly")?.Value,
out bool managedOnly);
// If RID-specific, add the RID to the end of the identity.
if (!managedOnly)
{
if (string.IsNullOrEmpty(SourceBuildIntermediateNupkgRid))
{
Log.LogError(
$"Parameter {nameof(SourceBuildIntermediateNupkgRid)} was " +
"not specified, indicating this project depends only on managed " +
"inputs. However, source-build element is not ManagedOnly: " +
sourceBuildElement);
return null;
}
identity += "." + SourceBuildIntermediateNupkgRid;
}
return new TaskItem(
identity,
new Dictionary<string, string>
{
["Name"] = dependencyName,
["Version"] = $"[{dependencyVersion}]",
["ExactVersion"] = dependencyVersion,
["Uri"] = uri,
["Sha"] = sha,
["SourceBuildRepoName"] = sourceBuildRepoName
});
})
.Where(d => d != null)
.ToArray();
return !Log.HasLoggedErrors;
}
private string ConvertInternalRepo(string uri)
{
if (uri.StartsWith("https://dev.azure.com", StringComparison.OrdinalIgnoreCase))
{
string[] repoParts = uri.Substring(uri.LastIndexOf('/')).Split('-', 2);
if (repoParts.Length != 2)
{
Log.LogError($"Repo '{uri}' does not end with the expected <GH organization>-<GH repo> format");
return null;
}
string org = repoParts[0];
string repo = repoParts[1];
// The internal Nuget.Client repo has suffix which needs to be accounted for.
const string trustedSuffix = "-Trusted";
if (uri.EndsWith(trustedSuffix, StringComparison.OrdinalIgnoreCase))
{
repo = repo.Substring(0, repo.Length - trustedSuffix.Length);
}
uri = $"https://github.com/{org}/{repo}";
}
return uri;
}
}
}

View file

@ -1,194 +0,0 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.
using Microsoft.Build.Framework;
using Microsoft.Build.Tasks;
using Microsoft.Build.Utilities;
using NuGet.Versioning;
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Xml.Serialization;
namespace Microsoft.DotNet.SourceBuild.Tasks
{
/// <summary>
/// Writes a props file to the given directory for each dependency specified
/// plus adds or updates an existing props file with all dependencies. The
/// intention is for the props file to be included by a source-build build
/// to get metadata about each dependent repo.
/// </summary>
public class Tarball_WriteSourceRepoProperties : Task
{
/// <summary>
/// The directory to write the props files to.
/// </summary>
[Required]
public string SourceBuildMetadataDir { get; set; }
/// <summary>
/// Dependencies to include in the props files.
///
/// %(Identity): NuGet package ID.
/// %(Name): The Name of the dependency from Version.Details.xml.
/// %(ExactVersion): NuGet package version. This can be used to look up the restored package
/// contents in a package cache.
/// %(Version): NuGet package version, wrapped in "[version]" syntax for exact match.
/// %(Uri): The URI for the repo.
/// %(Sha): The commit Sha for the dependency.
/// %(SourceBuildRepoName): The repo name to use in source-build.
/// </summary>
/// <value></value>
[Required]
public ITaskItem[] Dependencies { get; set; }
public override bool Execute()
{
var allRepoProps = new Dictionary<string, string>();
foreach (var dependency in Dependencies.Select(dep =>
new {
Name = dep.GetMetadata("Name"),
SourceBuildRepoName = dep.GetMetadata("SourceBuildRepoName"),
Version = dep.GetMetadata("ExactVersion"),
Sha = dep.GetMetadata("Sha"),
Uri = dep.GetMetadata("Uri"),
GitCommitCount = dep.GetMetadata("GitCommitCount")
}))
{
string repoName = dependency.SourceBuildRepoName;
string safeRepoName = repoName.Replace("-", "").Replace(".", "");
string propsPath = Path.Combine(SourceBuildMetadataDir, $"{repoName.Replace(".", "-")}.props");
DerivedVersion derivedVersion = GetVersionInfo(safeRepoName, dependency.Version, "0");
var repoProps = new Dictionary<string, string>
{
["GitCommitHash"] = dependency.Sha,
["OfficialBuildId"] = derivedVersion.OfficialBuildId,
["OutputPackageVersion"] = dependency.Version,
["PreReleaseVersionLabel"] = derivedVersion.PreReleaseVersionLabel,
["IsStable"] = string.IsNullOrWhiteSpace(derivedVersion.PreReleaseVersionLabel) ? "true" : "false",
};
if (!string.IsNullOrEmpty(dependency.GitCommitCount))
{
repoProps.Add("GitCommitCount", dependency.GitCommitCount);
}
WritePropsFile(propsPath, repoProps);
allRepoProps[$"{safeRepoName}GitCommitHash"] = dependency.Sha;
allRepoProps[$"{safeRepoName}OutputPackageVersion"] = dependency.Version;
}
string allRepoPropsPath = Path.Combine(SourceBuildMetadataDir, "AllRepoVersions.props");
Log.LogMessage(MessageImportance.Normal, $"[{DateTimeOffset.Now}] Writing all repo versions to {allRepoPropsPath}");
UpdatePropsFile(allRepoPropsPath, allRepoProps);
return !Log.HasLoggedErrors;
}
/// <summary>
/// Reverse a version in the Arcade style (https://github.com/dotnet/arcade/blob/fb92b14d8cd07cf44f8f7eefa8ac58d7ffd05f3f/src/Microsoft.DotNet.Arcade.Sdk/tools/Version.BeforeCommonTargets.targets#L18)
/// back to an OfficialBuildId + ReleaseLabel which we can then supply to get the same resulting version number.
/// </summary>
/// <param name="repoName">The source build name of the repo to get the version info for.</param>
/// <param name="version">The complete version, e.g. 1.0.0-beta1-19720.5</param>
/// <param name="commitCount">The current commit count of the repo. This is used for some repos that do not use the standard versioning scheme.</param>
/// <returns></returns>
private static DerivedVersion GetVersionInfo(string repoName, string version, string commitCount)
{
var nugetVersion = new NuGetVersion(version);
if (!string.IsNullOrWhiteSpace(nugetVersion.Release))
{
var releaseParts = nugetVersion.Release.Split('-', '.');
if (repoName.Contains("nuget"))
{
// NuGet does this - arbitrary build IDs
return new DerivedVersion { OfficialBuildId = DateTime.Now.ToString("yyyyMMdd.1"), PreReleaseVersionLabel = releaseParts[0] };
}
else if (releaseParts.Length == 3)
{
// VSTest uses full dates for the first part of their preview build numbers
if (repoName.Contains("vstest"))
{
return new DerivedVersion { OfficialBuildId = $"{releaseParts[1]}.{releaseParts[2]}", PreReleaseVersionLabel = releaseParts[0] };
}
else if (int.TryParse(releaseParts[1], out int datePart) && int.TryParse(releaseParts[2], out int buildPart))
{
if (datePart > 1 && datePart < 8 && buildPart > 1000 && buildPart < 10000)
{
return new DerivedVersion { OfficialBuildId = releaseParts[2], PreReleaseVersionLabel = $"{releaseParts[0]}.{releaseParts[1]}" };
}
else
{
return new DerivedVersion { OfficialBuildId = $"20{((datePart / 1000))}{((datePart % 1000) / 50):D2}{(datePart % 50):D2}.{buildPart}", PreReleaseVersionLabel = releaseParts[0] };
}
}
}
else if (releaseParts.Length == 4)
{
// new preview version style, e.g. 5.0.0-preview.7.20365.12
if (int.TryParse(releaseParts[2], out int datePart) && int.TryParse(releaseParts[3], out int buildPart))
{
return new DerivedVersion { OfficialBuildId = $"20{((datePart / 1000))}{((datePart % 1000) / 50):D2}{(datePart % 50):D2}.{buildPart}", PreReleaseVersionLabel = $"{releaseParts[0]}.{releaseParts[1]}" };
}
}
}
else
{
// finalized version number (x.y.z) - probably not our code
// Application Insights, Newtonsoft.Json do this
return new DerivedVersion { OfficialBuildId = DateTime.Now.ToString("yyyyMMdd.1"), PreReleaseVersionLabel = string.Empty };
}
throw new FormatException($"Can't derive a build ID from version {version} (commit count {commitCount}, release {string.Join(";", nugetVersion.Release.Split('-', '.'))})");
}
private static void UpdatePropsFile(string filePath, Dictionary<string, string> properties)
{
if (!File.Exists(filePath))
{
WritePropsFile(filePath, properties);
}
else
{
var content = new StringBuilder();
foreach (var line in File.ReadAllLines(filePath))
{
content.AppendLine(line);
if (line.Contains("<PropertyGroup>"))
{
foreach (var propName in properties.Keys.OrderBy(k => k))
{
content.AppendLine($" <{propName}>{properties[propName]}</{propName}>");
}
}
}
File.WriteAllText(filePath, content.ToString());
}
}
private static void WritePropsFile(string filePath, Dictionary<string, string> properties)
{
var content = new StringBuilder();
content.AppendLine("<?xml version=\"1.0\" encoding=\"utf-8\"?>");
content.AppendLine("<Project>");
content.AppendLine(" <PropertyGroup>");
foreach (var propName in properties.Keys.OrderBy(k => k))
{
content.AppendLine($" <{propName}>{properties[propName]}</{propName}>");
}
content.AppendLine(" </PropertyGroup>");
content.AppendLine("</Project>");
File.WriteAllText(filePath, content.ToString());
}
private class DerivedVersion
{
internal string OfficialBuildId { get; set; }
internal string PreReleaseVersionLabel { get; set; }
}
}
}

View file

@ -1,7 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<!-- Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this file to you under the MIT license. -->
<Project>
<PropertyGroup>
<SourceBuildTasksAssembly>$(RepoRoot)\artifacts\bin\SourceBuild.Tasks\$(Configuration)\net7.0\SourceBuild.Tasks.dll</SourceBuildTasksAssembly>
</PropertyGroup>
</Project>

View file

@ -1,415 +0,0 @@
<!-- Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this file to you under the MIT license. -->
<Project Sdk="Microsoft.DotNet.Arcade.Sdk">
<Import Project="./BuildTasks.props" />
<Import Project="../tools/SourceBuild/SourceBuildArcadeTools.targets" Sdk="Microsoft.DotNet.Arcade.Sdk" />
<Import Project="$(RepoRoot)/src/redist/targets/Versions.targets" />
<Import Project="$(RepoRoot)/src/redist/targets/SetBuildDefaults.targets" />
<Import Project="$(RepoRoot)/src/redist/targets/FileExtensions.targets" />
<Import Project="$(RepoRoot)/src/redist/targets/GetRuntimeInformation.targets" />
<Import Project="$(RepoRoot)/src/redist/targets/GenerateLayout.targets" />
<UsingTask TaskName="Microsoft.DotNet.SourceBuild.Tasks.Tarball_ReadSourceBuildIntermediateNupkgDependencies" AssemblyFile="$(SourceBuildTasksAssembly)" />
<UsingTask TaskName="Microsoft.DotNet.SourceBuild.Tasks.Tarball_WriteSourceRepoProperties" AssemblyFile="$(SourceBuildTasksAssembly)" />
<PropertyGroup>
<ArtifactsDir>$(RepoRoot)artifacts/</ArtifactsDir>
<TarballDir Condition=" '$(TarballDir)' == '' ">$(ArtifactsDir)tarball/</TarballDir>
<TarballRootDir>$([MSBuild]::EnsureTrailingSlash('$(TarballDir)'))</TarballRootDir>
<TarballSourceDir>$(TarballRootDir)src/</TarballSourceDir>
<TarballGitInfoDir>$(TarballRootDir)git-info/</TarballGitInfoDir>
<CloneVerbosity>quiet</CloneVerbosity> <!-- Support quiet and full -->
<ConvertInternalRepos Condition="'$(ConvertInternalRepos)' == '' and '$(VSS_NUGET_EXTERNAL_FEED_ENDPOINTS)' == '' and '$(AzDoPat)' == ''">true</ConvertInternalRepos>
</PropertyGroup>
<Target Name="CreateSourceTarball"
DependsOnTargets="
GenerateFullNuGetVersion;
GenerateVersionFile;
CreateTarballDir;
SetupSelfGithubInfo;
CloneRepoAndDependentsRecursive;
CleanClonedSource;
RestoreTextOnlyPackages;
CopyTextOnlyPackages;
CopyTarballContent;
">
<PropertyGroup>
<TarballFileDir>$([System.IO.Path]::GetDirectoryName('$(TarballFilePath)'))</TarballFileDir>
</PropertyGroup>
<Error Text="TarballFilePath not specified." Condition=" '$(TarballFilePath)' == '' " />
<MakeDir Directories="$(TarballFileDir)" Condition=" '$(TarballFileDir)' != '' " />
<Exec Command="tar --numeric-owner -zcf $(TarballFilePath) -C $(TarballDir) ." />
<Message Text="Tarball '$(TarballFilePath)' was successfully created from '$(TarballDir)'" Importance="High" />
</Target>
<Target Name="CreateTarballDir" >
<RemoveDir Directories='$(TarballRootDir)' Condition=" EXISTS('$(TarballRootDir)') " />
<Error Text="Tarball directory not specified. Pass /p:TarballDir=[new dir name] to specify a tarball directory." Condition=" '$(TarballDir)' == '' " />
<Error Text="Tarball directory already exists: $(TarballRootDir)" Condition=" '$(TarballRootDir)' != '' AND EXISTS('$(TarballRootDir)') " />
<MakeDir Directories="$(TarballRootDir)" Condition=" '$(TarballRootDir)' != '' " />
<MakeDir Directories="$(TarballSourceDir)" Condition=" '$(TarballSourceDir)' != '' " />
<MakeDir Directories="$(TarballGitInfoDir)" Condition=" '$(TarballGitInfoDir)' != '' " />
</Target>
<Target Name="SetupSelfGithubInfo" Condition=" '$(IsRootRepo)' != 'true' ">
<Exec
Command="cd $(RepoRoot);git config --get remote.origin.url"
ConsoleToMSBuild="true"
WorkingDirectory="$(RepoRoot)">
<Output TaskParameter="ConsoleOutput" ItemName="RootRepoUri" />
</Exec>
<Exec
Command="cd $(RepoRoot);git rev-parse HEAD"
ConsoleToMSBuild="true"
WorkingDirectory="$(RepoRoot)">
<Output TaskParameter="ConsoleOutput" ItemName="RootRepoCommitSha" />
</Exec>
<!-- Get commit count for installer repo only -->
<Exec
Command="cd $(RepoRoot);git rev-list --count HEAD"
ConsoleToMSBuild="true"
WorkingDirectory="$(RepoRoot)">
<Output TaskParameter="ConsoleOutput" ItemName="RootRepoCommitCount" />
</Exec>
<!-- This is hardcoding version for the root repo (installer), since there
isn't a Version.Details.xml file to read it from.
See https://github.com/dotnet/source-build/issues/2250 -->
<ItemGroup>
<SourceBuildRepos Include="$(GitHubRepositoryName)">
<Name>$(GitHubRepositoryName)</Name>
<Version>$(VersionPrefix)</Version>
<ExactVersion>$(VersionPrefix)</ExactVersion>
<Sha>@(RootRepoCommitSha)</Sha>
<Uri>@(RootRepoUri)</Uri>
<GitCommitCount>@(RootRepoCommitCount)</GitCommitCount>
<SourceBuildRepoName>$(GitHubRepositoryName)</SourceBuildRepoName>
<IsRootRepo>true</IsRootRepo>
</SourceBuildRepos>
</ItemGroup>
<Tarball_WriteSourceRepoProperties
SourceBuildMetadataDir="$(TarballGitInfoDir)"
Dependencies="@(SourceBuildRepos)" />
<PropertyGroup>
<SourceBuildRepoName>%(SourceBuildRepos.SourceBuildRepoName)</SourceBuildRepoName>
<RepoUri>%(SourceBuildRepos.Uri)</RepoUri>
<RepoSha>%(SourceBuildRepos.Sha)</RepoSha>
<IsRootRepo>%(SourceBuildRepos.IsRootRepo)</IsRootRepo>
</PropertyGroup>
<!-- Remove root repo from SourceBuildRepos so it doesn't try to clone twice -->
<ItemGroup>
<SourceBuildRepos Remove="@(SourceBuildRepos)" />
</ItemGroup>
</Target>
<Target Name="CloneRepoAndDependentsRecursive"
DependsOnTargets="GetSourceBuildIntermediateNupkgNameConvention">
<Message Text="--> Start Cloning Repo $(SourceBuildRepoName)" Importance="High" />
<PropertyGroup>
<SourceDir>$(SourceBuildRepoName)/</SourceDir>
<TarballRepoSourceDir>$(TarballSourceDir)$(SourceDir)</TarballRepoSourceDir>
<TarballRepoSourceEngDir>$(TarballSourceDir)$(SourceDir)eng/</TarballRepoSourceEngDir>
<TarballVersionDetailsFile>$(TarballRepoSourceEngDir)Version.Details.xml</TarballVersionDetailsFile>
<CloneParam Condition=" '$(CloneVerbosity)' == 'quiet' ">-q</CloneParam>
<OriginalRepoUri>$(RepoUri)</OriginalRepoUri>
<RepoUri Condition=" '$(AzDoPat)' != '' ">$(RepoUri.Replace('https://dev.azure.com', 'https://dn-bot:$(AzDoPat)@dev.azure.com'))</RepoUri>
</PropertyGroup>
<!-- Perform a local clone of the root repo, it's faster and doesn't require the RepoSha to be a tree. -->
<Exec
Command="git clone ./.git $(TarballRepoSourceDir)"
WorkingDirectory="$(RepoRoot)"
Condition="$(IsRootRepo) == 'true'" />
<!-- Because the root repo was locally cloned, the remote.origin.url needs to be updated from the local source to the real origin -->
<Exec
Command="git config --file $(TarballRepoSourceDir)/.git/config --unset remote.origin.url"
WorkingDirectory="$(RepoRoot)"
Condition="$(IsRootRepo) == 'true'" />
<Exec
Command="git config --file $(TarballRepoSourceDir)/.git/config --add remote.origin.url `git config --get remote.origin.url`"
WorkingDirectory="$(RepoRoot)"
Condition="$(IsRootRepo) == 'true'" />
<!-- Needed in cases where source-build is pinned to a version of the root repo. -->
<Exec
Command="git reset --hard $(RepoSha)"
WorkingDirectory="$(TarballRepoSourceDir)"
Condition="$(IsRootRepo) == 'true'" />
<Exec
Command="git init $(TarballRepoSourceDir)"
WorkingDirectory="$(RepoRoot)"
Condition="$(IsRootRepo) != 'true'" />
<Exec
Command="git remote add origin $(RepoUri)"
WorkingDirectory="$(TarballRepoSourceDir)"
Condition="$(IsRootRepo) != 'true'" />
<!-- Fetching a sha requires git 2.5.0 or newer -->
<Exec
Command="git fetch --depth 1 origin $(RepoSha)"
WorkingDirectory="$(TarballRepoSourceDir)"
Condition="$(IsRootRepo) != 'true'" />
<Exec
Command="git reset --hard FETCH_HEAD"
WorkingDirectory="$(TarballRepoSourceDir)"
Condition="$(IsRootRepo) != 'true'" />
<Exec
Command="git submodule update --init --recursive --depth 1"
WorkingDirectory="$(TarballRepoSourceDir)" />
<Exec
Command="git submodule foreach 'rm -rf %24%28git rev-parse --git-dir%29/objects ||:'"
WorkingDirectory="$(TarballRepoSourceDir)"
Condition="$(PreserveTarballGitFolders) != 'true'" />
<Exec
Command="git config --file $(TarballRepoSourceDir)/.git/config --unset remote.origin.url"
WorkingDirectory="$(RepoRoot)"/>
<Exec
Command="git config --file $(TarballRepoSourceDir)/.git/config --add remote.origin.url $(OriginalRepoUri)"
WorkingDirectory="$(RepoRoot)"/>
<!-- Remove the git objects folder to free up tarball space -->
<Exec
Command="rm -rf objects"
WorkingDirectory="$(TarballRepoSourceDir).git"
Condition="$(PreserveTarballGitFolders) != 'true'" />
<Message Text="--> Done Cloning Repo $(SourceBuildRepoName)" Importance="High" />
<!-- Apply source-build patches -->
<ItemGroup>
<SourceBuildPatchFile Include="$(RepoRoot)src/SourceBuild/tarball/patches/$(SourceBuildRepoName)/*.patch" />
</ItemGroup>
<Message
Text="--> Applying $(SourceBuildRepoName) Patches"
Importance="High"
Condition="'@(SourceBuildPatchFile)' != ''" />
<Exec
Command="git --work-tree=$(TarballRepoSourceDir) apply --ignore-whitespace --whitespace=nowarn &quot;%(SourceBuildPatchFile.FullPath)&quot;"
WorkingDirectory="$(TarballRepoSourceDir)"
Condition="'@(SourceBuildPatchFile)' != ''" />
<Message
Text="--> Done Applying $(SourceBuildRepoName) Patches"
Importance="High"
Condition="'@(SourceBuildPatchFile)' != ''" />
<Tarball_ReadSourceBuildIntermediateNupkgDependencies
VersionDetailsXmlFile="$([MSBuild]::NormalizePath($(TarballVersionDetailsFile)))"
SourceBuildIntermediateNupkgPrefix="$(SourceBuildIntermediateNupkgPrefix)"
SourceBuildIntermediateNupkgRid="$(SourceBuildIntermediateNupkgRid)"
ConvertInternalRepos="$(ConvertInternalRepos)">
<Output TaskParameter="Dependencies" ItemName="SourceBuildRepos" />
</Tarball_ReadSourceBuildIntermediateNupkgDependencies>
<!-- Remove repo if it has already be cloned at any sha. This results in
The commit sha for the cloned repo in the tarball being the one that
was first encountered. -->
<ItemGroup>
<SourceBuildRepos Remove="@(SourceBuildRepos)" Condition=" EXISTS('$(TarballGitInfoDir)%(SourceBuildRepoName).props')" />
</ItemGroup>
<Tarball_WriteSourceRepoProperties
SourceBuildMetadataDir="$(TarballGitInfoDir)"
Dependencies="@(SourceBuildRepos)" />
<Message Text="--> Dependencies for $(TarballVersionDetailsFile):" Importance="High" Condition=" '@(SourceBuildRepos)' != '' " />
<Message Text="--> %(SourceBuildRepos.SourceBuildRepoName) / %(SourceBuildRepos.Sha)" Importance="High" Condition=" '@(SourceBuildRepos)' != '' " />
<MSBuild Projects="$(MSBuildProjectFile)"
Condition=" '@(SourceBuildRepos)' != '' "
Targets="CloneRepoAndDependentsRecursive"
Properties="SourceBuildRepoName=%(SourceBuildRepos.SourceBuildRepoName);RepoSha=%(SourceBuildRepos.Sha);RepoUri=%(SourceBuildRepos.Uri)" />
</Target>
<Target Name="CleanClonedSource">
<ItemGroup>
<TarballSrcBinaryExtension Include="
.dll;
.Dll;
.exe;
.pdb;
.mdb;
.zip;
.nupkg" />
<TarballSrcBinaryToRemove Include="$(TarballSourceDir)**\*%(TarballSrcBinaryExtension.Identity)" />
<!-- Runtime coreclr and installer parts have empty placeholder PDB files. Keep them. -->
<TarballSrcBinaryToRemove Remove="
$(TarballSourceDir)runtime*\src\coreclr\.nuget\**\*%(TarballSrcBinaryExtension.Identity);
$(TarballSourceDir)runtime*\src\installer\pkg\**\*%(TarballSrcBinaryExtension.Identity)" />
</ItemGroup>
<Message Importance="Normal" Text="Deleting checked in binary files: @(TarballSrcBinaryToRemove, ' ')" />
<Delete Files="@(TarballSrcBinaryToRemove)" />
</Target>
<Target Name="RestoreTextOnlyPackages">
<PropertyGroup>
<TextOnlyPackageRestoreDir>$(ArtifactsDir)text-only-packages/</TextOnlyPackageRestoreDir>
<TextOnlyPackageTarballDir>$(TarballRootDir)packages/text-only/</TextOnlyPackageTarballDir>
</PropertyGroup>
<Message Text="Restoring text only packages..." Importance="High" />
<MSBuild Projects="$(RepoRoot)src/SourceBuild/Arcade/tools/TextOnlyPackages.csproj"
Properties="TargetPackagesPath=$(TextOnlyPackageRestoreDir)"
Targets="Restore" />
</Target>
<Target Name="CopyTarballContent">
<ItemGroup>
<TarballContent Include="$(RepoRoot)src/SourceBuild/tarball/content/**/*" />
<EngCommonContent Include="$(RepoRoot)eng/common/**/*" />
</ItemGroup>
<Copy
SourceFiles="@(TarballContent)"
DestinationFiles="@(TarballContent->'$(TarballRootDir)%(RecursiveDir)%(Filename)%(Extension)')" />
<Copy
SourceFiles="@(EngCommonContent)"
DestinationFiles="@(EngCommonContent->'$(TarballRootDir)eng/common/%(RecursiveDir)%(Filename)%(Extension)')" />
<PropertyGroup>
<ExternalTarballsDir>$(TarballRootDir)packages/archive/</ExternalTarballsDir>
<SourceBuiltArtifactsTarballName>Private.SourceBuilt.Artifacts</SourceBuiltArtifactsTarballName>
<SourceBuiltPrebuiltsTarballName>Private.SourceBuilt.Prebuilts</SourceBuiltPrebuiltsTarballName>
<SourceBuiltArtifactsTarballUrl>https://dotnetcli.azureedge.net/source-built-artifacts/assets/</SourceBuiltArtifactsTarballUrl>
<ArchiveArtifactsTextFileName>archiveArtifacts.txt</ArchiveArtifactsTextFileName>
<ArchiveArtifactsTextFile>$(ExternalTarballsDir)$(ArchiveArtifactsTextFileName)</ArchiveArtifactsTextFile>
</PropertyGroup>
<MakeDir Directories="$(ExternalTarballsDir)" />
<ItemGroup>
<ArtifactUrls Include="$(SourceBuiltArtifactsTarballUrl)$(SourceBuiltArtifactsTarballName).$(PrivateSourceBuiltArtifactsPackageVersion).tar.gz" />
<ArtifactUrls
Include="$(SourceBuiltArtifactsTarballUrl)$(SourceBuiltPrebuiltsTarballName).$(PrivateSourceBuiltPrebuiltsPackageVersion).tar.gz"
Condition="'$(PrivateSourceBuiltPrebuiltsPackageVersion)' != ''" />
</ItemGroup>
<WriteLinesToFile
File="$(ArchiveArtifactsTextFile)"
Lines="@(ArtifactUrls)"
Overwrite="true" />
<MakeDir Directories="$(TarballRootDir)packages/prebuilt" />
<!-- make sure this directory is persisted through source control, NuGet depends on it existing -->
<Exec Command="touch $(TarballRootDir)packages/prebuilt/_" />
</Target>
<!--
Read directories in which text-only packages have been downloaded and copy content to tarball.
-->
<Target Name="CopyTextOnlyPackages" DependsOnTargets="RestoreTextOnlyPackages" >
<ReadLinesFromFile File="$(TextOnlyPackageRestoreDir)/TextOnlyPackageDirectories.txt" Condition="Exists('$(TextOnlyPackageRestoreDir)/TextOnlyPackageDirectories.txt')">
<Output TaskParameter="Lines" ItemName="TextOnlyPackageDirectories" />
</ReadLinesFromFile>
<MSBuild Projects="$(MSBuildProjectFile)"
Targets="CopySingleTextOnlyDirectory"
Properties="TextOnlyDirectory=%(TextOnlyPackageDirectories.Identity);TextOnlyPackageTarballDir=$(TextOnlyPackageTarballDir)" />
</Target>
<Target Name="CopySingleTextOnlyDirectory">
<PropertyGroup>
<DirectoryName>$([System.IO.Path]::GetFileName('$(TextOnlyDirectory)'))</DirectoryName>
</PropertyGroup>
<ItemGroup>
<TextOnlyPackageContent
Include="$(TextOnlyDirectory)/**/*"
Exclude="
$(TextOnlyDirectory)/**/.nupkg.metadata;
$(TextOnlyDirectory)/**/.signature.p7s;
$(TextOnlyDirectory)/**/*.nupkg;
$(TextOnlyDirectory)/**/*.nupkg.sha512;
" />
</ItemGroup>
<ItemGroup>
<!-- ttf, woff, woff2, eot are permissible font related content -->
<AllowedTextOnlyExtensions Include="
.-;
.bowerrc;
.config;
.cs;
.cshtml;
.csproj;
.css;
.db;
.editorconfig;
.env;
.env.development;
.eot;
.fs;
.fsproj;
.gitignore;
.gitkeep;
.html;
.ico;
.js;
.json;
.map;
.md;
.nuspec;
.otf;
.png;
.props;
.proto;
.razor;
.sln;
.svg;
.targets;
.ts;
.ttf;
.tsx;
.txt;
.vb;
.vbproj;
.woff;
.woff2;
browserslist;
browserslistrc;
LICENSE;" />
<UnsupportedTextOnlyPackageContent Include="@(TextOnlyPackageContent)" />
<UnsupportedTextOnlyPackageContent Remove="$(TextOnlyDirectory)/**/*$([System.String]::Copy('%(AllowedTextOnlyExtensions.Identity)').ToLowerInvariant())" />
<UnsupportedTextOnlyPackageContent Remove="$(TextOnlyDirectory)/**/*$([System.String]::Copy('%(AllowedTextOnlyExtensions.Identity)').ToUpperInvariant())" />
</ItemGroup>
<Error Text="Unsupported content found in text-only packages: @(UnsupportedTextOnlyPackageContent)" Condition=" '@(UnsupportedTextOnlyPackageContent)' != '' " />
<Copy
SourceFiles="@(TextOnlyPackageContent)"
DestinationFiles="@(TextOnlyPackageContent->'$(TextOnlyPackageTarballDir)$(DirectoryName)/%(RecursiveDir)%(Filename)%(Extension)')" />
</Target>
</Project>

View file

@ -1,45 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFrameworks>net7.0</TargetFrameworks>
<RestorePackagesPath>$(TargetPackagesPath)</RestorePackagesPath>
</PropertyGroup>
<!--
The following text-only packages are ones that are consumed by the installer repo, whose
versions are dynamic. These are maintained here so the dynamic version numbers can
come from the source (eng/Versions.props). All other more static text-only packages
should be added to source-build-reference-packages.
-->
<ItemGroup>
<PackageDownload Include="Microsoft.NET.Sdk.Android.Manifest-$(MauiFeatureBand)" Version="[$(XamarinAndroidWorkloadManifestVersion)]" />
<PackageDownload Include="Microsoft.NET.Sdk.iOS.Manifest-$(MauiFeatureBand)" Version="[$(XamarinIOSWorkloadManifestVersion)]" />
<PackageDownload Include="Microsoft.NET.Sdk.MacCatalyst.Manifest-$(MauiFeatureBand)" Version="[$(XamarinMacCatalystWorkloadManifestVersion)]" />
<PackageDownload Include="Microsoft.NET.Sdk.macOS.Manifest-$(MauiFeatureBand)" Version="[$(XamarinMacOSWorkloadManifestVersion)]" />
<PackageDownload Include="Microsoft.NET.Sdk.Maui.Manifest-$(MauiFeatureBand)" Version="[$(MauiWorkloadManifestVersion)]" />
<PackageDownload Include="Microsoft.NET.Sdk.tvOS.Manifest-$(MauiFeatureBand)" Version="[$(XamarinTvOSWorkloadManifestVersion)]" />
<PackageDownload Include="Microsoft.NET.Workload.Mono.ToolChain.Manifest-8.0.100" Version="[$(MonoWorkloadManifestVersion)]" />
<PackageDownload Include="Microsoft.NET.Workload.Emscripten.net6.Manifest-8.0.100" Version="[$(EmscriptenWorkloadManifestVersion)]" />
<PackageDownload Include="Microsoft.NET.Workload.Emscripten.net7.Manifest-8.0.100" Version="[$(EmscriptenWorkloadManifestVersion)]" />
<PackageDownload Include="Microsoft.NET.Workload.Mono.ToolChain.net6.Manifest-8.0.100" Version="[$(MonoWorkloadManifestVersion)]" />
<PackageDownload Include="Microsoft.NET.Workload.Mono.ToolChain.net7.Manifest-8.0.100" Version="[$(MonoWorkloadManifestVersion)]" />
</ItemGroup>
<!--
Write out directories in which text-only packages are downloaded to be consumed by the "CopyTextOnlyPackages"
target.
-->
<Target Name="WriteTextOnlyDirectoryFile" BeforeTargets="Restore">
<ItemGroup>
<TextOnlyPackagesDirectories Include="$(TargetPackagesPath)$([System.String]::copy('%(PackageDownload.Identity)').ToLower())" />
</ItemGroup>
<PropertyGroup>
<TextOnlyPackageDirectoriesContent>@(TextOnlyPackagesDirectories)</TextOnlyPackageDirectoriesContent>
</PropertyGroup>
<WriteLinesToFile
Lines="$(TextOnlyPackageDirectoriesContent)"
File="$(TargetPackagesPath)/TextOnlyPackageDirectories.txt"
Overwrite="true" />
</Target>
</Project>

View file

@ -1,10 +0,0 @@
<Project>
<Import Project="../Arcade/tools/SourceBuildArcadeTarball.targets" />
<PropertyGroup>
<TarballFilePath>$(ArtifactsShippingPackagesDir)dotnet-sdk-source-$(Version).tar.gz</TarballFilePath>
</PropertyGroup>
<Target Name="Build" DependsOnTargets="CreateSourceTarball" />
</Project>

View file

@ -142,7 +142,6 @@
<ConflictingPackageReportDir>$(BaseOutputPath)conflict-report/</ConflictingPackageReportDir>
<PrebuiltBurndownDataFile>$(PackageReportDir)PrebuiltBurndownData.csv</PrebuiltBurndownDataFile>
<ExternalTarballsDir>$(ProjectDir)packages/archive/</ExternalTarballsDir>
<TextOnlyPackageBaseDir>$(ProjectDir)packages/text-only/</TextOnlyPackageBaseDir>
<ReferencePackagesDir>$(ProjectDir)packages/reference/</ReferencePackagesDir>
<SourceBuiltArtifactsTarballName>Private.SourceBuilt.Artifacts</SourceBuiltArtifactsTarballName>
<SourceBuiltPrebuiltsTarballName>Private.SourceBuilt.Prebuilts</SourceBuiltPrebuiltsTarballName>

View file

@ -27,7 +27,6 @@
BuildLeakDetection;
ExtractToolPackage;
GenerateRootFs;
BuildTextOnlyPackages;
PoisonPrebuiltPackages" />
</Target>
@ -154,32 +153,6 @@
<Exec Condition="'$(Platform)' == 'armel'" Command="$(ArmEnvironmentVariables) $(ProjectDir)cross/armel/tizen-build-rootfs.sh" />
</Target>
<Target Name="BuildTextOnlyPackages"
Inputs="$(MSBuildProjectFullPath)"
Outputs="$(CompletedSemaphorePath)BuildTextOnlyPackages.complete">
<Message Importance="High" Text="[$([System.DateTime]::Now.ToString('HH:mm:ss.ff'))] Packing text-only packages." />
<ItemGroup>
<TextOnlyPackageNuspecFile Include="$(TextOnlyPackageBaseDir)/**/*.nuspec" />
</ItemGroup>
<Message Importance="High" Text="Found text only package nuspec file: %(TextOnlyPackageNuspecFile.Identity)" />
<ReplaceTextInFile
InputFile="%(TextOnlyPackageNuspecFile.Identity)"
OldText="&lt;/package&gt;"
NewText="&lt;files&gt;&lt;file src=&quot;.\**\*&quot;/&gt;&lt;/files&gt;&lt;/package&gt;" />
<NugetPack
Nuspecs="%(TextOnlyPackageNuspecFile.Identity)"
OutputDirectory="$(SourceBuiltPackagesPath)"
ExcludeEmptyDirectories="false"
CreateSymbolPackage="false" />
<Message Importance="High" Text="[$([System.DateTime]::Now.ToString('HH:mm:ss.ff'))] Done packing text-only packages." />
<WriteLinesToFile File="$(CompletedSemaphorePath)BuildTextOnlyPackages.complete" Overwrite="true" />
</Target>
<Target Name="ExtractToolPackage"
DependsOnTargets="UnpackTarballs;BuildXPlatTasks"
Inputs="$(MSBuildProjectFullPath)"