diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json deleted file mode 100644 index 30953897c..000000000 --- a/.devcontainer/devcontainer.json +++ /dev/null @@ -1,23 +0,0 @@ -// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at: -// https://github.com/microsoft/vscode-dev-containers/tree/v0.238.0/containers/dotnet -{ - "name": "Source-Build w/ Built Tarball", - "image": "mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-36", - // A completely built .NET source-tarball is >64 GB - "hostRequirements": { - "storage": "128gb" - }, - // Configure tool-specific properties. - "customizations": { - // Configure properties specific to VS Code. - "vscode": { - // Add the IDs of extensions you want installed when the container is created. - "extensions": [ - "ms-dotnettools.csharp", - "eamodio.gitlens" - ] - } - }, - // Use 'onCreateCommand' to run pre-build commands inside the codespace. - "onCreateCommand": "${containerWorkspaceFolder}/.devcontainer/source-build-scripts/buildTarball.sh" -} \ No newline at end of file diff --git a/.devcontainer/source-build-scripts/buildTarball.sh b/.devcontainer/source-build-scripts/buildTarball.sh deleted file mode 100755 index d1b714e77..000000000 --- a/.devcontainer/source-build-scripts/buildTarball.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -./.devcontainer/source-build-scripts/createTarball.sh - -cd $(realpath ..)/dotnet-source/ - -./prep.sh - -# GitHub Codespaces automatically sets RepositoryName, which conflicts with source-build scripts. -unset RepositoryName - -./build.sh --online --clean-while-building || true diff --git a/.devcontainer/source-build-scripts/createTarball.sh b/.devcontainer/source-build-scripts/createTarball.sh deleted file mode 100755 index c66753b16..000000000 --- a/.devcontainer/source-build-scripts/createTarball.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash - -# Always return exit code 0 so that we can use the codespaces prebuild to diagnose build errors. -./build.sh /p:ArcadeBuildTarball=true /p:TarballDir=$(realpath ..)/dotnet-source/ /p:PreserveTarballGitFolders=true || true - -# Save the commit hash of the currently built repo, so developers know which version was built -git rev-parse HEAD > ./artifacts/prebuild.sha \ No newline at end of file diff --git a/.devcontainer/source-build-tarball/devcontainer.json b/.devcontainer/source-build-tarball/devcontainer.json deleted file mode 100644 index 4fa2238bf..000000000 --- a/.devcontainer/source-build-tarball/devcontainer.json +++ /dev/null @@ -1,20 +0,0 @@ -// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at: -// https://github.com/microsoft/vscode-dev-containers/tree/v0.238.0/containers/dotnet -{ - "name": "Source-Build w/ Tarball", - "image": "mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-36", - // A completely built .NET source tarball is >64 GB - "hostRequirements": { - "storage": "128gb" - }, - "customizations": { - "vscode": { - "extensions": [ - "ms-dotnettools.csharp", - "eamodio.gitlens" - ] - } - }, - // Use 'onCreateCommand' to run pre-build commands inside the codespace. - "onCreateCommand": "${containerWorkspaceFolder}/.devcontainer/scripts/createTarball.sh" -} \ No newline at end of file diff --git a/.devcontainer/source-build/devcontainer.json b/.devcontainer/source-build/devcontainer.json deleted file mode 100644 index e38bc27f0..000000000 --- a/.devcontainer/source-build/devcontainer.json +++ /dev/null @@ -1,5 +0,0 @@ -// Use this devcontainer if you don't need the full context of a source-build tarball -{ - "name": "Source-Build", - "image": "mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-36" -} \ No newline at end of file diff --git a/eng/Build.props b/eng/Build.props index ed33d0bbf..ff9b72504 100644 --- a/eng/Build.props +++ b/eng/Build.props @@ -1,13 +1,5 @@ - - - - - - - - diff --git a/src/SourceBuild/Arcade/README.md b/src/SourceBuild/Arcade/README.md deleted file mode 100644 index ad15a2f14..000000000 --- a/src/SourceBuild/Arcade/README.md +++ /dev/null @@ -1 +0,0 @@ -The source and targets in the `src/SourceBuild/Arcade` directory are intended to move into the Arcade repo at some point. They are added here for ease of development while developing the tarball generation process. See https://github.com/dotnet/source-build/issues/2295 diff --git a/src/SourceBuild/Arcade/src/SourceBuild.Tasks.csproj b/src/SourceBuild/Arcade/src/SourceBuild.Tasks.csproj deleted file mode 100644 index 2842d265d..000000000 --- a/src/SourceBuild/Arcade/src/SourceBuild.Tasks.csproj +++ /dev/null @@ -1,17 +0,0 @@ - - - net7.0 - false - false - disable - - - - - - - - - - - diff --git a/src/SourceBuild/Arcade/src/Tarball_ReadSourceBuildIntermediateNupkgDependencies.cs b/src/SourceBuild/Arcade/src/Tarball_ReadSourceBuildIntermediateNupkgDependencies.cs deleted file mode 100644 index 9b6365dfc..000000000 --- a/src/SourceBuild/Arcade/src/Tarball_ReadSourceBuildIntermediateNupkgDependencies.cs +++ /dev/null @@ -1,180 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Xml.Linq; - -namespace Microsoft.DotNet.SourceBuild.Tasks -{ - /// - /// Reads entries in a Version.Details.xml file to find intermediate nupkg dependencies. For - /// each dependency with a "SourceBuild" element, adds an item to the "Dependencies" output. - /// - public class Tarball_ReadSourceBuildIntermediateNupkgDependencies : Task - { - [Required] - public string VersionDetailsXmlFile { get; set; } - - [Required] - public string SourceBuildIntermediateNupkgPrefix { get; set; } - - /// - /// Convert any internal repo references to the public GitHub repos. - /// - public bool ConvertInternalRepos { get; set; } - - /// - /// The intermediate nupkg RID to use if any RID-specific intermediate nupkgs are required. - /// If this parameter isn't specified, RID-specific intermediate nupkgs can't be used and - /// this task fails. - /// - public string SourceBuildIntermediateNupkgRid { get; set; } - - /// - /// %(Identity): NuGet package ID. - /// %(Name): The Name of the dependency from Version.Details.xml. - /// %(ExactVersion): NuGet package version. This can be used to look up the restored package - /// contents in a package cache. - /// %(Version): NuGet package version, wrapped in "[version]" syntax for exact match. - /// %(Uri): The URI for the repo. - /// %(Sha): The commit Sha for the dependency. - /// %(SourceBuildRepoName): The repo name to use in source-build. - /// - [Output] - public ITaskItem[] Dependencies { get; set; } - - public override bool Execute() - { - XElement root = XElement.Load(VersionDetailsXmlFile, LoadOptions.PreserveWhitespace); - - XName CreateQualifiedName(string plainName) - { - return root.GetDefaultNamespace().GetName(plainName); - } - - Dependencies = root - .Elements() - .Elements(CreateQualifiedName("Dependency")) - .Select(d => - { - XElement sourceBuildElement = d.Element(CreateQualifiedName("SourceBuild")); - - if (sourceBuildElement == null) - { - // Workaround for https://github.com/dotnet/source-build/issues/2481 - sourceBuildElement = d.Element(CreateQualifiedName("SourceBuildTarball")); - - if (sourceBuildElement == null) - { - // Ignore element: doesn't represent a source-build dependency. - return null; - } - } - - string repoName = sourceBuildElement.Attribute("RepoName")?.Value; - - if (string.IsNullOrEmpty(repoName)) - { - Log.LogError($"Dependency SourceBuild RepoName null or empty in '{VersionDetailsXmlFile}' element {d}"); - return null; - } - - string dependencyName = d.Attribute("Name")?.Value ?? string.Empty; - - if (string.IsNullOrEmpty(dependencyName)) - { - // Log name missing as FYI, but this is not an error case for source-build. - Log.LogMessage($"Dependency Name null or empty in '{VersionDetailsXmlFile}' element {d}"); - } - - string dependencyVersion = d.Attribute("Version")?.Value; - - string uri = d.Element(CreateQualifiedName("Uri"))?.Value; - if (ConvertInternalRepos) - { - uri = ConvertInternalRepo(uri); - } - - string sha = d.Element(CreateQualifiedName("Sha"))?.Value; - string sourceBuildRepoName = sourceBuildElement.Attribute("RepoName")?.Value; - - if (string.IsNullOrEmpty(dependencyVersion)) - { - // We need a version to bring down an intermediate nupkg. Fail. - Log.LogError($"Dependency Version null or empty in '{VersionDetailsXmlFile}' element {d}"); - return null; - } - - string identity = SourceBuildIntermediateNupkgPrefix + repoName; - - bool.TryParse( - sourceBuildElement.Attribute("ManagedOnly")?.Value, - out bool managedOnly); - - // If RID-specific, add the RID to the end of the identity. - if (!managedOnly) - { - if (string.IsNullOrEmpty(SourceBuildIntermediateNupkgRid)) - { - Log.LogError( - $"Parameter {nameof(SourceBuildIntermediateNupkgRid)} was " + - "not specified, indicating this project depends only on managed " + - "inputs. However, source-build element is not ManagedOnly: " + - sourceBuildElement); - return null; - } - - identity += "." + SourceBuildIntermediateNupkgRid; - } - - return new TaskItem( - identity, - new Dictionary - { - ["Name"] = dependencyName, - ["Version"] = $"[{dependencyVersion}]", - ["ExactVersion"] = dependencyVersion, - ["Uri"] = uri, - ["Sha"] = sha, - ["SourceBuildRepoName"] = sourceBuildRepoName - }); - }) - .Where(d => d != null) - .ToArray(); - - return !Log.HasLoggedErrors; - } - - private string ConvertInternalRepo(string uri) - { - if (uri.StartsWith("https://dev.azure.com", StringComparison.OrdinalIgnoreCase)) - { - string[] repoParts = uri.Substring(uri.LastIndexOf('/')).Split('-', 2); - - if (repoParts.Length != 2) - { - Log.LogError($"Repo '{uri}' does not end with the expected - format"); - return null; - } - - string org = repoParts[0]; - string repo = repoParts[1]; - - // The internal Nuget.Client repo has suffix which needs to be accounted for. - const string trustedSuffix = "-Trusted"; - if (uri.EndsWith(trustedSuffix, StringComparison.OrdinalIgnoreCase)) - { - repo = repo.Substring(0, repo.Length - trustedSuffix.Length); - } - - uri = $"https://github.com/{org}/{repo}"; - } - - return uri; - } - } -} diff --git a/src/SourceBuild/Arcade/src/Tarball_WriteSourceRepoProperties.cs b/src/SourceBuild/Arcade/src/Tarball_WriteSourceRepoProperties.cs deleted file mode 100644 index b177b6f01..000000000 --- a/src/SourceBuild/Arcade/src/Tarball_WriteSourceRepoProperties.cs +++ /dev/null @@ -1,194 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using Microsoft.Build.Framework; -using Microsoft.Build.Tasks; -using Microsoft.Build.Utilities; -using NuGet.Versioning; -using System; -using System.Collections.Generic; -using System.Globalization; -using System.IO; -using System.Linq; -using System.Text; -using System.Text.RegularExpressions; -using System.Xml.Serialization; - -namespace Microsoft.DotNet.SourceBuild.Tasks -{ - /// - /// Writes a props file to the given directory for each dependency specified - /// plus adds or updates an existing props file with all dependencies. The - /// intention is for the props file to be included by a source-build build - /// to get metadata about each dependent repo. - /// - public class Tarball_WriteSourceRepoProperties : Task - { - /// - /// The directory to write the props files to. - /// - [Required] - public string SourceBuildMetadataDir { get; set; } - - /// - /// Dependencies to include in the props files. - /// - /// %(Identity): NuGet package ID. - /// %(Name): The Name of the dependency from Version.Details.xml. - /// %(ExactVersion): NuGet package version. This can be used to look up the restored package - /// contents in a package cache. - /// %(Version): NuGet package version, wrapped in "[version]" syntax for exact match. - /// %(Uri): The URI for the repo. - /// %(Sha): The commit Sha for the dependency. - /// %(SourceBuildRepoName): The repo name to use in source-build. - /// - /// - [Required] - public ITaskItem[] Dependencies { get; set; } - - public override bool Execute() - { - var allRepoProps = new Dictionary(); - - foreach (var dependency in Dependencies.Select(dep => - new { - Name = dep.GetMetadata("Name"), - SourceBuildRepoName = dep.GetMetadata("SourceBuildRepoName"), - Version = dep.GetMetadata("ExactVersion"), - Sha = dep.GetMetadata("Sha"), - Uri = dep.GetMetadata("Uri"), - GitCommitCount = dep.GetMetadata("GitCommitCount") - })) - { - string repoName = dependency.SourceBuildRepoName; - string safeRepoName = repoName.Replace("-", "").Replace(".", ""); - string propsPath = Path.Combine(SourceBuildMetadataDir, $"{repoName.Replace(".", "-")}.props"); - DerivedVersion derivedVersion = GetVersionInfo(safeRepoName, dependency.Version, "0"); - var repoProps = new Dictionary - { - ["GitCommitHash"] = dependency.Sha, - ["OfficialBuildId"] = derivedVersion.OfficialBuildId, - ["OutputPackageVersion"] = dependency.Version, - ["PreReleaseVersionLabel"] = derivedVersion.PreReleaseVersionLabel, - ["IsStable"] = string.IsNullOrWhiteSpace(derivedVersion.PreReleaseVersionLabel) ? "true" : "false", - }; - if (!string.IsNullOrEmpty(dependency.GitCommitCount)) - { - repoProps.Add("GitCommitCount", dependency.GitCommitCount); - } - WritePropsFile(propsPath, repoProps); - allRepoProps[$"{safeRepoName}GitCommitHash"] = dependency.Sha; - allRepoProps[$"{safeRepoName}OutputPackageVersion"] = dependency.Version; - } - string allRepoPropsPath = Path.Combine(SourceBuildMetadataDir, "AllRepoVersions.props"); - Log.LogMessage(MessageImportance.Normal, $"[{DateTimeOffset.Now}] Writing all repo versions to {allRepoPropsPath}"); - UpdatePropsFile(allRepoPropsPath, allRepoProps); - - return !Log.HasLoggedErrors; - } - - /// - /// Reverse a version in the Arcade style (https://github.com/dotnet/arcade/blob/fb92b14d8cd07cf44f8f7eefa8ac58d7ffd05f3f/src/Microsoft.DotNet.Arcade.Sdk/tools/Version.BeforeCommonTargets.targets#L18) - /// back to an OfficialBuildId + ReleaseLabel which we can then supply to get the same resulting version number. - /// - /// The source build name of the repo to get the version info for. - /// The complete version, e.g. 1.0.0-beta1-19720.5 - /// The current commit count of the repo. This is used for some repos that do not use the standard versioning scheme. - /// - private static DerivedVersion GetVersionInfo(string repoName, string version, string commitCount) - { - var nugetVersion = new NuGetVersion(version); - - if (!string.IsNullOrWhiteSpace(nugetVersion.Release)) - { - var releaseParts = nugetVersion.Release.Split('-', '.'); - if (repoName.Contains("nuget")) - { - // NuGet does this - arbitrary build IDs - return new DerivedVersion { OfficialBuildId = DateTime.Now.ToString("yyyyMMdd.1"), PreReleaseVersionLabel = releaseParts[0] }; - } - else if (releaseParts.Length == 3) - { - // VSTest uses full dates for the first part of their preview build numbers - if (repoName.Contains("vstest")) - { - return new DerivedVersion { OfficialBuildId = $"{releaseParts[1]}.{releaseParts[2]}", PreReleaseVersionLabel = releaseParts[0] }; - } - else if (int.TryParse(releaseParts[1], out int datePart) && int.TryParse(releaseParts[2], out int buildPart)) - { - if (datePart > 1 && datePart < 8 && buildPart > 1000 && buildPart < 10000) - { - return new DerivedVersion { OfficialBuildId = releaseParts[2], PreReleaseVersionLabel = $"{releaseParts[0]}.{releaseParts[1]}" }; - } - else - { - return new DerivedVersion { OfficialBuildId = $"20{((datePart / 1000))}{((datePart % 1000) / 50):D2}{(datePart % 50):D2}.{buildPart}", PreReleaseVersionLabel = releaseParts[0] }; - } - } - } - else if (releaseParts.Length == 4) - { - // new preview version style, e.g. 5.0.0-preview.7.20365.12 - if (int.TryParse(releaseParts[2], out int datePart) && int.TryParse(releaseParts[3], out int buildPart)) - { - return new DerivedVersion { OfficialBuildId = $"20{((datePart / 1000))}{((datePart % 1000) / 50):D2}{(datePart % 50):D2}.{buildPart}", PreReleaseVersionLabel = $"{releaseParts[0]}.{releaseParts[1]}" }; - } - } - } - else - { - // finalized version number (x.y.z) - probably not our code - // Application Insights, Newtonsoft.Json do this - return new DerivedVersion { OfficialBuildId = DateTime.Now.ToString("yyyyMMdd.1"), PreReleaseVersionLabel = string.Empty }; - } - - throw new FormatException($"Can't derive a build ID from version {version} (commit count {commitCount}, release {string.Join(";", nugetVersion.Release.Split('-', '.'))})"); - } - - private static void UpdatePropsFile(string filePath, Dictionary properties) - { - if (!File.Exists(filePath)) - { - WritePropsFile(filePath, properties); - } - else - { - var content = new StringBuilder(); - foreach (var line in File.ReadAllLines(filePath)) - { - content.AppendLine(line); - if (line.Contains("")) - { - foreach (var propName in properties.Keys.OrderBy(k => k)) - { - content.AppendLine($" <{propName}>{properties[propName]}"); - } - } - } - File.WriteAllText(filePath, content.ToString()); - } - } - - private static void WritePropsFile(string filePath, Dictionary properties) - { - var content = new StringBuilder(); - content.AppendLine(""); - content.AppendLine(""); - content.AppendLine(" "); - foreach (var propName in properties.Keys.OrderBy(k => k)) - { - content.AppendLine($" <{propName}>{properties[propName]}"); - } - content.AppendLine(" "); - content.AppendLine(""); - File.WriteAllText(filePath, content.ToString()); - } - - private class DerivedVersion - { - internal string OfficialBuildId { get; set; } - internal string PreReleaseVersionLabel { get; set; } - } - } -} diff --git a/src/SourceBuild/Arcade/tools/BuildTasks.props b/src/SourceBuild/Arcade/tools/BuildTasks.props deleted file mode 100644 index 50f49046b..000000000 --- a/src/SourceBuild/Arcade/tools/BuildTasks.props +++ /dev/null @@ -1,7 +0,0 @@ - - - - - $(RepoRoot)\artifacts\bin\SourceBuild.Tasks\$(Configuration)\net7.0\SourceBuild.Tasks.dll - - \ No newline at end of file diff --git a/src/SourceBuild/Arcade/tools/SourceBuildArcadeTarball.targets b/src/SourceBuild/Arcade/tools/SourceBuildArcadeTarball.targets deleted file mode 100644 index 1f7c8aed3..000000000 --- a/src/SourceBuild/Arcade/tools/SourceBuildArcadeTarball.targets +++ /dev/null @@ -1,415 +0,0 @@ - - - - - - - - - - - - - - - - $(RepoRoot)artifacts/ - $(ArtifactsDir)tarball/ - $([MSBuild]::EnsureTrailingSlash('$(TarballDir)')) - $(TarballRootDir)src/ - $(TarballRootDir)git-info/ - quiet - true - - - - - - $([System.IO.Path]::GetDirectoryName('$(TarballFilePath)')) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - $(GitHubRepositoryName) - $(VersionPrefix) - $(VersionPrefix) - @(RootRepoCommitSha) - @(RootRepoUri) - @(RootRepoCommitCount) - $(GitHubRepositoryName) - true - - - - - - - %(SourceBuildRepos.SourceBuildRepoName) - %(SourceBuildRepos.Uri) - %(SourceBuildRepos.Sha) - %(SourceBuildRepos.IsRootRepo) - - - - - - - - - - - - - $(SourceBuildRepoName)/ - $(TarballSourceDir)$(SourceDir) - $(TarballSourceDir)$(SourceDir)eng/ - $(TarballRepoSourceEngDir)Version.Details.xml - -q - $(RepoUri) - $(RepoUri.Replace('https://dev.azure.com', 'https://dn-bot:$(AzDoPat)@dev.azure.com')) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - $(ArtifactsDir)text-only-packages/ - $(TarballRootDir)packages/text-only/ - - - - - - - - - - - - - - - - - - - $(TarballRootDir)packages/archive/ - Private.SourceBuilt.Artifacts - Private.SourceBuilt.Prebuilts - https://dotnetcli.azureedge.net/source-built-artifacts/assets/ - archiveArtifacts.txt - $(ExternalTarballsDir)$(ArchiveArtifactsTextFileName) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - $([System.IO.Path]::GetFileName('$(TextOnlyDirectory)')) - - - - - - - - - - - - - - - - - - - - - - diff --git a/src/SourceBuild/Arcade/tools/TextOnlyPackages.csproj b/src/SourceBuild/Arcade/tools/TextOnlyPackages.csproj deleted file mode 100644 index 3de474133..000000000 --- a/src/SourceBuild/Arcade/tools/TextOnlyPackages.csproj +++ /dev/null @@ -1,45 +0,0 @@ - - - net7.0 - $(TargetPackagesPath) - - - - - - - - - - - - - - - - - - - - - - - - - @(TextOnlyPackagesDirectories) - - - - - diff --git a/src/SourceBuild/tarball/BuildSourceBuildTarball.proj b/src/SourceBuild/tarball/BuildSourceBuildTarball.proj deleted file mode 100644 index 7b5becb77..000000000 --- a/src/SourceBuild/tarball/BuildSourceBuildTarball.proj +++ /dev/null @@ -1,10 +0,0 @@ - - - - - $(ArtifactsShippingPackagesDir)dotnet-sdk-source-$(Version).tar.gz - - - - - diff --git a/src/SourceBuild/tarball/content/Directory.Build.props b/src/SourceBuild/tarball/content/Directory.Build.props index 8880b5d05..5229c0ebb 100644 --- a/src/SourceBuild/tarball/content/Directory.Build.props +++ b/src/SourceBuild/tarball/content/Directory.Build.props @@ -142,7 +142,6 @@ $(BaseOutputPath)conflict-report/ $(PackageReportDir)PrebuiltBurndownData.csv $(ProjectDir)packages/archive/ - $(ProjectDir)packages/text-only/ $(ProjectDir)packages/reference/ Private.SourceBuilt.Artifacts Private.SourceBuilt.Prebuilts diff --git a/src/SourceBuild/tarball/content/tools-local/init-build.proj b/src/SourceBuild/tarball/content/tools-local/init-build.proj index 0405ad8f1..9af3e2741 100644 --- a/src/SourceBuild/tarball/content/tools-local/init-build.proj +++ b/src/SourceBuild/tarball/content/tools-local/init-build.proj @@ -27,7 +27,6 @@ BuildLeakDetection; ExtractToolPackage; GenerateRootFs; - BuildTextOnlyPackages; PoisonPrebuiltPackages" /> @@ -154,32 +153,6 @@ - - - - - - - - - - - - - - - - -